drivers/block/osdblk.c

   1
   2 /*
   3    osdblk.c -- Export a single SCSI OSD object as a Linux block device
   4
   5
   6    Copyright 2009 Red Hat, Inc.
   7
   8    This program is free software; you can redistribute it and/or modify
   9    it under the terms of the GNU General Public License as published by
  10    the Free Software Foundation.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; see the file COPYING.  If not, write to
  19    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
  20
  21
  22    Instructions for use
  23    --------------------
  24
  25    1) Map a Linux block device to an existing OSD object.
  26
  27       In this example, we will use partition id 1234, object id 5678,
  28       OSD device /dev/osd1.
  29
  30       $ echo "1234 5678 /dev/osd1" > /sys/class/osdblk/add
  31
  32
  33    2) List all active blkdev<->object mappings.
  34
  35       In this example, we have performed step #1 twice, creating two blkdevs,
  36       mapped to two separate OSD objects.
  37
  38       $ cat /sys/class/osdblk/list
  39       0 174 1234 5678 /dev/osd1
  40       1 179 1994 897123 /dev/osd0
  41
  42       The columns, in order, are:
  43       - blkdev unique id
  44       - blkdev assigned major
  45       - OSD object partition id
  46       - OSD object id
  47       - OSD device
  48
  49
  50    3) Remove an active blkdev<->object mapping.
  51
  52       In this example, we remove the mapping with blkdev unique id 1.
  53
  54       $ echo 1 > /sys/class/osdblk/remove
  55
  56
  57    NOTE:  The actual creation and deletion of OSD objects is outside the scope
  58    of this driver.
  59
  60  */
  61
  62 #include <linux/kernel.h>
  63 #include <linux/device.h>
  64 #include <linux/module.h>
  65 #include <linux/fs.h>
  66 #include <linux/slab.h>
  67 #include <scsi/osd_initiator.h>
  68 #include <scsi/osd_attributes.h>
  69 #include <scsi/osd_sec.h>
  70 #include <scsi/scsi_device.h>
  71
  72 #define DRV_NAME "osdblk"
  73 #define PFX DRV_NAME ": "
  74
  75 /* #define _OSDBLK_DEBUG */
  76 #ifdef _OSDBLK_DEBUG
  77 #define OSDBLK_DEBUG(fmt, a...) \
  78         printk(KERN_NOTICE "osdblk @%s:%d: " fmt, __func__, __LINE__, ##a)
  79 #else
  80 #define OSDBLK_DEBUG(fmt, a...) \
  81         do { if (0) printk(fmt, ##a); } while (0)
  82 #endif
  83
  84 MODULE_AUTHOR("Jeff Garzik <jeff@garzik.org>");
  85 MODULE_DESCRIPTION("block device inside an OSD object osdblk.ko");
  86 MODULE_LICENSE("GPL");
  87
  88 struct osdblk_device;
  89
  90 enum {
  91         OSDBLK_MINORS_PER_MAJOR = 256,          /* max minors per blkdev */
  92         OSDBLK_MAX_REQ          = 32,           /* max parallel requests */
  93         OSDBLK_OP_TIMEOUT       = 4 * 60,       /* sync OSD req timeout */
  94 };
  95
  96 struct osdblk_request {
  97         struct request          *rq;            /* blk layer request */
  98         struct bio              *bio;           /* cloned bio */
  99         struct osdblk_device    *osdev;         /* associated blkdev */
 100 };
 101
 102 struct osdblk_device {
 103         int                     id;             /* blkdev unique id */
 104
 105         int                     major;          /* blkdev assigned major */
 106         struct gendisk          *disk;          /* blkdev's gendisk and rq */
 107         struct request_queue    *q;
 108
 109         struct osd_dev          *osd;           /* associated OSD */
 110
 111         char                    name[32];       /* blkdev name, e.g. osdblk34 */
 112
 113         spinlock_t              lock;           /* queue lock */
 114
 115         struct osd_obj_id       obj;            /* OSD partition, obj id */
 116         uint8_t                 obj_cred[OSD_CAP_LEN]; /* OSD cred */
 117
 118         struct osdblk_request   req[OSDBLK_MAX_REQ]; /* request table */
 119
 120         struct list_head        node;
 121
 122         char                    osd_path[0];    /* OSD device path */
 123 };
 124
 125 static struct class *class_osdblk;              /* /sys/class/osdblk */
 126 static DEFINE_MUTEX(ctl_mutex); /* Serialize open/close/setup/teardown */
 127 static LIST_HEAD(osdblkdev_list);
 128
 129 static const struct block_device_operations osdblk_bd_ops = {
 130         .owner          = THIS_MODULE,
 131 };
 132
 133 static const struct osd_attr g_attr_logical_length = ATTR_DEF(
 134         OSD_APAGE_OBJECT_INFORMATION, OSD_ATTR_OI_LOGICAL_LENGTH, 8);
 135
 136 static void osdblk_make_credential(u8 cred_a[OSD_CAP_LEN],
 137                                    const struct osd_obj_id *obj)
 138 {
 139         osd_sec_init_nosec_doall_caps(cred_a, obj, false, true);
 140 }
 141
 142 /* copied from exofs; move to libosd? */
 143 /*
 144  * Perform a synchronous OSD operation.  copied from exofs; move to libosd?
 145  */
 146 static int osd_sync_op(struct osd_request *or, int timeout, uint8_t *credential)
 147 {
 148         int ret;
 149
 150         or->timeout = timeout;
 151         ret = osd_finalize_request(or, 0, credential, NULL);
 152         if (ret)
 153                 return ret;
 154
 155         ret = osd_execute_request(or);
 156
 157         /* osd_req_decode_sense(or, ret); */
 158         return ret;
 159 }
 160
 161 /*
 162  * Perform an asynchronous OSD operation.  copied from exofs; move to libosd?
 163  */
 164 static int osd_async_op(struct osd_request *or, osd_req_done_fn *async_done,
 165                    void *caller_context, u8 *cred)
 166 {
 167         int ret;
 168
 169         ret = osd_finalize_request(or, 0, cred, NULL);
 170         if (ret)
 171                 return ret;
 172
 173         ret = osd_execute_request_async(or, async_done, caller_context);
 174
 175         return ret;
 176 }
 177
 178 /* copied from exofs; move to libosd? */
 179 static int extract_attr_from_req(struct osd_request *or, struct osd_attr *attr)
 180 {
 181         struct osd_attr cur_attr = {.attr_page = 0}; /* start with zeros */
 182         void *iter = NULL;
 183         int nelem;
 184
 185         do {
 186                 nelem = 1;
 187                 osd_req_decode_get_attr_list(or, &cur_attr, &nelem, &iter);
 188                 if ((cur_attr.attr_page == attr->attr_page) &&
 189                     (cur_attr.attr_id == attr->attr_id)) {
 190                         attr->len = cur_attr.len;
 191                         attr->val_ptr = cur_attr.val_ptr;
 192                         return 0;
 193                 }
 194         } while (iter);
 195
 196         return -EIO;
 197 }
 198
 199 static int osdblk_get_obj_size(struct osdblk_device *osdev, u64 *size_out)
 200 {
 201         struct osd_request *or;
 202         struct osd_attr attr;
 203         int ret;
 204
 205         /* start request */
 206         or = osd_start_request(osdev->osd, GFP_KERNEL);
 207         if (!or)
 208                 return -ENOMEM;
 209
 210         /* create a get-attributes(length) request */
 211         osd_req_get_attributes(or, &osdev->obj);
 212
 213         osd_req_add_get_attr_list(or, &g_attr_logical_length, 1);
 214
 215         /* execute op synchronously */
 216         ret = osd_sync_op(or, OSDBLK_OP_TIMEOUT, osdev->obj_cred);
 217         if (ret)
 218                 goto out;
 219
 220         /* extract length from returned attribute info */
 221         attr = g_attr_logical_length;
 222         ret = extract_attr_from_req(or, &attr);
 223         if (ret)
 224                 goto out;
 225
 226         *size_out = get_unaligned_be64(attr.val_ptr);
 227
 228 out:
 229         osd_end_request(or);
 230         return ret;
 231
 232 }
 233
 234 static void osdblk_osd_complete(struct osd_request *or, void *private)
 235 {
 236         struct osdblk_request *orq = private;
 237         struct osd_sense_info osi;
 238         int ret = osd_req_decode_sense(or, &osi);
 239
 240         if (ret) {
 241                 ret = -EIO;
 242                 OSDBLK_DEBUG("osdblk_osd_complete with err=%d\n", ret);
 243         }
 244
 245         /* complete OSD request */
 246         osd_end_request(or);
 247
 248         /* complete request passed to osdblk by block layer */
 249         __blk_end_request_all(orq->rq, ret);
 250 }
 251
 252 static void bio_chain_put(struct bio *chain)
 253 {
 254         struct bio *tmp;
 255
 256         while (chain) {
 257                 tmp = chain;
 258                 chain = chain->bi_next;
 259
 260                 bio_put(tmp);
 261         }
 262 }
 263
 264 static struct bio *bio_chain_clone(struct bio *old_chain, gfp_t gfpmask)
 265 {
 266         struct bio *tmp, *new_chain = NULL, *tail = NULL;
 267
 268         while (old_chain) {
 269                 tmp = bio_clone_kmalloc(old_chain, gfpmask);
 270                 if (!tmp)
 271                         goto err_out;
 272
 273                 tmp->bi_bdev = NULL;
 274                 gfpmask &= ~__GFP_DIRECT_RECLAIM;
 275                 tmp->bi_next = NULL;
 276
 277                 if (!new_chain)
 278                         new_chain = tail = tmp;
 279                 else {
 280                         tail->bi_next = tmp;
 281                         tail = tmp;
 282                 }
 283
 284                 old_chain = old_chain->bi_next;
 285         }
 286
 287         return new_chain;
 288
 289 err_out:
 290         OSDBLK_DEBUG("bio_chain_clone with err\n");
 291         bio_chain_put(new_chain);
 292         return NULL;
 293 }
 294
 295 static void osdblk_rq_fn(struct request_queue *q)
 296 {
 297         struct osdblk_device *osdev = q->queuedata;
 298
 299         while (1) {
 300                 struct request *rq;
 301                 struct osdblk_request *orq;
 302                 struct osd_request *or;
 303                 struct bio *bio;
 304                 bool do_write, do_flush;
 305
 306                 /* peek at request from block layer */
 307                 rq = blk_fetch_request(q);
 308                 if (!rq)
 309                         break;
 310
 311                 /* deduce our operation (read, write, flush) */
 312                 /* I wish the block layer simplified cmd_type/cmd_flags/cmd[]
 313                  * into a clearly defined set of RPC commands:
 314                  * read, write, flush, scsi command, power mgmt req,
 315                  * driver-specific, etc.
 316                  */
 317
 318                 do_flush = (req_op(rq) == REQ_OP_FLUSH);
 319                 do_write = (rq_data_dir(rq) == WRITE);
 320
 321                 if (!do_flush) { /* osd_flush does not use a bio */
 322                         /* a bio clone to be passed down to OSD request */
 323                         bio = bio_chain_clone(rq->bio, GFP_ATOMIC);
 324                         if (!bio)
 325                                 break;
 326                 } else
 327                         bio = NULL;
 328
 329                 /* alloc internal OSD request, for OSD command execution */
 330                 or = osd_start_request(osdev->osd, GFP_ATOMIC);
 331                 if (!or) {
 332                         bio_chain_put(bio);
 333                         OSDBLK_DEBUG("osd_start_request with err\n");
 334                         break;
 335                 }
 336
 337                 orq = &osdev->req[rq->tag];
 338                 orq->rq = rq;
 339                 orq->bio = bio;
 340                 orq->osdev = osdev;
 341
 342                 /* init OSD command: flush, write or read */
 343                 if (do_flush)
 344                         osd_req_flush_object(or, &osdev->obj,
 345                                              OSD_CDB_FLUSH_ALL, 0, 0);
 346                 else if (do_write)
 347                         osd_req_write(or, &osdev->obj, blk_rq_pos(rq) * 512ULL,
 348                                       bio, blk_rq_bytes(rq));
 349                 else
 350                         osd_req_read(or, &osdev->obj, blk_rq_pos(rq) * 512ULL,
 351                                      bio, blk_rq_bytes(rq));
 352
 353                 OSDBLK_DEBUG("%s 0x%x bytes at 0x%llx\n",
 354                         do_flush ? "flush" : do_write ?
 355                                 "write" : "read", blk_rq_bytes(rq),
 356                         blk_rq_pos(rq) * 512ULL);
 357
 358                 /* begin OSD command execution */
 359                 if (osd_async_op(or, osdblk_osd_complete, orq,
 360                                  osdev->obj_cred)) {
 361                         osd_end_request(or);
 362                         blk_requeue_request(q, rq);
 363                         bio_chain_put(bio);
 364                         OSDBLK_DEBUG("osd_execute_request_async with err\n");
 365                         break;
 366                 }
 367
 368                 /* remove the special 'flush' marker, now that the command
 369                  * is executing
 370                  */
 371                 rq->special = NULL;
 372         }
 373 }
 374
 375 static void osdblk_free_disk(struct osdblk_device *osdev)
 376 {
 377         struct gendisk *disk = osdev->disk;
 378
 379         if (!disk)
 380                 return;
 381
 382         if (disk->flags & GENHD_FL_UP)
 383                 del_gendisk(disk);
 384         if (disk->queue)
 385                 blk_cleanup_queue(disk->queue);
 386         put_disk(disk);
 387 }
 388
 389 static int osdblk_init_disk(struct osdblk_device *osdev)
 390 {
 391         struct gendisk *disk;
 392         struct request_queue *q;
 393         int rc;
 394         u64 obj_size = 0;
 395
 396         /* contact OSD, request size info about the object being mapped */
 397         rc = osdblk_get_obj_size(osdev, &obj_size);
 398         if (rc)
 399                 return rc;
 400
 401         /* create gendisk info */
 402         disk = alloc_disk(OSDBLK_MINORS_PER_MAJOR);
 403         if (!disk)
 404                 return -ENOMEM;
 405
 406         sprintf(disk->disk_name, DRV_NAME "%d", osdev->id);
 407         disk->major = osdev->major;
 408         disk->first_minor = 0;
 409         disk->fops = &osdblk_bd_ops;
 410         disk->private_data = osdev;
 411
 412         /* init rq */
 413         q = blk_init_queue(osdblk_rq_fn, &osdev->lock);
 414         if (!q) {
 415                 put_disk(disk);
 416                 return -ENOMEM;
 417         }
 418
 419         /* switch queue to TCQ mode; allocate tag map */
 420         rc = blk_queue_init_tags(q, OSDBLK_MAX_REQ, NULL, BLK_TAG_ALLOC_FIFO);
 421         if (rc) {
 422                 blk_cleanup_queue(q);
 423                 put_disk(disk);
 424                 return rc;
 425         }
 426
 427         /* Set our limits to the lower device limits, because osdblk cannot
 428          * sleep when allocating a lower-request and therefore cannot be
 429          * bouncing.
 430          */
 431         blk_queue_stack_limits(q, osd_request_queue(osdev->osd));
 432
 433         blk_queue_prep_rq(q, blk_queue_start_tag);
 434         blk_queue_write_cache(q, true, false);
 435
 436         disk->queue = q;
 437
 438         q->queuedata = osdev;
 439
 440         osdev->disk = disk;
 441         osdev->q = q;
 442
 443         /* finally, announce the disk to the world */
 444         set_capacity(disk, obj_size / 512ULL);
 445         add_disk(disk);
 446
 447         printk(KERN_INFO "%s: Added of size 0x%llx\n",
 448                 disk->disk_name, (unsigned long long)obj_size);
 449
 450         return 0;
 451 }
 452
 453 /********************************************************************
 454  * /sys/class/osdblk/
 455  *                   add        map OSD object to blkdev
 456  *                   remove     unmap OSD object
 457  *                   list       show mappings
 458  *******************************************************************/
 459
 460 static void class_osdblk_release(struct class *cls)
 461 {
 462         kfree(cls);
 463 }
 464
 465 static ssize_t class_osdblk_list(struct class *c,
 466                                 struct class_attribute *attr,
 467                                 char *data)
 468 {
 469         int n = 0;
 470         struct list_head *tmp;
 471
 472         mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
 473
 474         list_for_each(tmp, &osdblkdev_list) {
 475                 struct osdblk_device *osdev;
 476
 477                 osdev = list_entry(tmp, struct osdblk_device, node);
 478
 479                 n += sprintf(data+n, "%d %d %llu %llu %s\n",
 480                         osdev->id,
 481                         osdev->major,
 482                         osdev->obj.partition,
 483                         osdev->obj.id,
 484                         osdev->osd_path);
 485         }
 486
 487         mutex_unlock(&ctl_mutex);
 488         return n;
 489 }
 490
 491 static ssize_t class_osdblk_add(struct class *c,
 492                                 struct class_attribute *attr,
 493                                 const char *buf, size_t count)
 494 {
 495         struct osdblk_device *osdev;
 496         ssize_t rc;
 497         int irc, new_id = 0;
 498         struct list_head *tmp;
 499
 500         if (!try_module_get(THIS_MODULE))
 501                 return -ENODEV;
 502
 503         /* new osdblk_device object */
 504         osdev = kzalloc(sizeof(*osdev) + strlen(buf) + 1, GFP_KERNEL);
 505         if (!osdev) {
 506                 rc = -ENOMEM;
 507                 goto err_out_mod;
 508         }
 509
 510         /* static osdblk_device initialization */
 511         spin_lock_init(&osdev->lock);
 512         INIT_LIST_HEAD(&osdev->node);
 513
 514         /* generate unique id: find highest unique id, add one */
 515
 516         mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
 517
 518         list_for_each(tmp, &osdblkdev_list) {
 519                 struct osdblk_device *osdev;
 520
 521                 osdev = list_entry(tmp, struct osdblk_device, node);
 522                 if (osdev->id > new_id)
 523                         new_id = osdev->id + 1;
 524         }
 525
 526         osdev->id = new_id;
 527
 528         /* add to global list */
 529         list_add_tail(&osdev->node, &osdblkdev_list);
 530
 531         mutex_unlock(&ctl_mutex);
 532
 533         /* parse add command */
 534         if (sscanf(buf, "%llu %llu %s", &osdev->obj.partition, &osdev->obj.id,
 535                    osdev->osd_path) != 3) {
 536                 rc = -EINVAL;
 537                 goto err_out_slot;
 538         }
 539
 540         /* initialize rest of new object */
 541         sprintf(osdev->name, DRV_NAME "%d", osdev->id);
 542
 543         /* contact requested OSD */
 544         osdev->osd = osduld_path_lookup(osdev->osd_path);
 545         if (IS_ERR(osdev->osd)) {
 546                 rc = PTR_ERR(osdev->osd);
 547                 goto err_out_slot;
 548         }
 549
 550         /* build OSD credential */
 551         osdblk_make_credential(osdev->obj_cred, &osdev->obj);
 552
 553         /* register our block device */
 554         irc = register_blkdev(0, osdev->name);
 555         if (irc < 0) {
 556                 rc = irc;
 557                 goto err_out_osd;
 558         }
 559
 560         osdev->major = irc;
 561
 562         /* set up and announce blkdev mapping */
 563         rc = osdblk_init_disk(osdev);
 564         if (rc)
 565                 goto err_out_blkdev;
 566
 567         return count;
 568
 569 err_out_blkdev:
 570         unregister_blkdev(osdev->major, osdev->name);
 571 err_out_osd:
 572         osduld_put_device(osdev->osd);
 573 err_out_slot:
 574         mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
 575         list_del_init(&osdev->node);
 576         mutex_unlock(&ctl_mutex);
 577
 578         kfree(osdev);
 579 err_out_mod:
 580         OSDBLK_DEBUG("Error adding device %s\n", buf);
 581         module_put(THIS_MODULE);
 582         return rc;
 583 }
 584
 585 static ssize_t class_osdblk_remove(struct class *c,
 586                                         struct class_attribute *attr,
 587                                         const char *buf,
 588                                         size_t count)
 589 {
 590         struct osdblk_device *osdev = NULL;
 591         int target_id, rc;
 592         unsigned long ul;
 593         struct list_head *tmp;
 594
 595         rc = kstrtoul(buf, 10, &ul);
 596         if (rc)
 597                 return rc;
 598
 599         /* convert to int; abort if we lost anything in the conversion */
 600         target_id = (int) ul;
 601         if (target_id != ul)
 602                 return -EINVAL;
 603
 604         /* remove object from list immediately */
 605         mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
 606
 607         list_for_each(tmp, &osdblkdev_list) {
 608                 osdev = list_entry(tmp, struct osdblk_device, node);
 609                 if (osdev->id == target_id) {
 610                         list_del_init(&osdev->node);
 611                         break;
 612                 }
 613                 osdev = NULL;
 614         }
 615
 616         mutex_unlock(&ctl_mutex);
 617
 618         if (!osdev)
 619                 return -ENOENT;
 620
 621         /* clean up and free blkdev and associated OSD connection */
 622         osdblk_free_disk(osdev);
 623         unregister_blkdev(osdev->major, osdev->name);
 624         osduld_put_device(osdev->osd);
 625         kfree(osdev);
 626
 627         /* release module ref */
 628         module_put(THIS_MODULE);
 629
 630         return count;
 631 }
 632
 633 static struct class_attribute class_osdblk_attrs[] = {
 634         __ATTR(add,     0200, NULL, class_osdblk_add),
 635         __ATTR(remove,  0200, NULL, class_osdblk_remove),
 636         __ATTR(list,    0444, class_osdblk_list, NULL),
 637         __ATTR_NULL
 638 };
 639
 640 static int osdblk_sysfs_init(void)
 641 {
 642         int ret = 0;
 643
 644         /*
 645          * create control files in sysfs
 646          * /sys/class/osdblk/...
 647          */
 648         class_osdblk = kzalloc(sizeof(*class_osdblk), GFP_KERNEL);
 649         if (!class_osdblk)
 650                 return -ENOMEM;
 651
 652         class_osdblk->name = DRV_NAME;
 653         class_osdblk->owner = THIS_MODULE;
 654         class_osdblk->class_release = class_osdblk_release;
 655         class_osdblk->class_attrs = class_osdblk_attrs;
 656
 657         ret = class_register(class_osdblk);
 658         if (ret) {
 659                 kfree(class_osdblk);
 660                 class_osdblk = NULL;
 661                 printk(PFX "failed to create class osdblk\n");
 662                 return ret;
 663         }
 664
 665         return 0;
 666 }
 667
 668 static void osdblk_sysfs_cleanup(void)
 669 {
 670         if (class_osdblk)
 671                 class_destroy(class_osdblk);
 672         class_osdblk = NULL;
 673 }
 674
 675 static int __init osdblk_init(void)
 676 {
 677         int rc;
 678
 679         rc = osdblk_sysfs_init();
 680         if (rc)
 681                 return rc;
 682
 683         return 0;
 684 }
 685
 686 static void __exit osdblk_exit(void)
 687 {
 688         osdblk_sysfs_cleanup();
 689 }
 690
 691 module_init(osdblk_init);
 692 module_exit(osdblk_exit);
 693