?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
xtensa: support DMA buffers in high memory
[cris-mirror.git] / drivers / md / dm-mpath.c
blob7d3e572072f51f865d07a79ae5a0ac974bd98e4a
1 /*
2 * Copyright (C) 2003 Sistina Software Limited.
3 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8 #include <linux/device-mapper.h>
9
10 #include "dm-rq.h"
11 #include "dm-bio-record.h"
12 #include "dm-path-selector.h"
13 #include "dm-uevent.h"
14
15 #include <linux/blkdev.h>
16 #include <linux/ctype.h>
17 #include <linux/init.h>
18 #include <linux/mempool.h>
19 #include <linux/module.h>
20 #include <linux/pagemap.h>
21 #include <linux/slab.h>
22 #include <linux/time.h>
23 #include <linux/workqueue.h>
24 #include <linux/delay.h>
25 #include <scsi/scsi_dh.h>
26 #include <linux/atomic.h>
27 #include <linux/blk-mq.h>
28
29 #define DM_MSG_PREFIX "multipath"
30 #define DM_PG_INIT_DELAY_MSECS 2000
31 #define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1)
32
33 /* Path properties */
34 struct pgpath {
35 struct list_head list;
36
37 struct priority_group *pg; /* Owning PG */
38 unsigned fail_count; /* Cumulative failure count */
39
40 struct dm_path path;
41 struct delayed_work activate_path;
42
43 bool is_active:1; /* Path status */
44 };
45
46 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
47
48 /*
49 * Paths are grouped into Priority Groups and numbered from 1 upwards.
50 * Each has a path selector which controls which path gets used.
51 */
52 struct priority_group {
53 struct list_head list;
54
55 struct multipath *m; /* Owning multipath instance */
56 struct path_selector ps;
57
58 unsigned pg_num; /* Reference number */
59 unsigned nr_pgpaths; /* Number of paths in PG */
60 struct list_head pgpaths;
61
62 bool bypassed:1; /* Temporarily bypass this PG? */
63 };
64
65 /* Multipath context */
66 struct multipath {
67 unsigned long flags; /* Multipath state flags */
68
69 spinlock_t lock;
70 enum dm_queue_mode queue_mode;
71
72 struct pgpath *current_pgpath;
73 struct priority_group *current_pg;
74 struct priority_group *next_pg; /* Switch to this PG if set */
75
76 atomic_t nr_valid_paths; /* Total number of usable paths */
77 unsigned nr_priority_groups;
78 struct list_head priority_groups;
79
80 const char *hw_handler_name;
81 char *hw_handler_params;
82 wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
83 unsigned pg_init_retries; /* Number of times to retry pg_init */
84 unsigned pg_init_delay_msecs; /* Number of msecs before pg_init retry */
85 atomic_t pg_init_in_progress; /* Only one pg_init allowed at once */
86 atomic_t pg_init_count; /* Number of times pg_init called */
87
88 struct mutex work_mutex;
89 struct work_struct trigger_event;
90 struct dm_target *ti;
91
92 struct work_struct process_queued_bios;
93 struct bio_list queued_bios;
94 };
95
96 /*
97 * Context information attached to each io we process.
98 */
99 struct dm_mpath_io {
100 struct pgpath *pgpath;
101 size_t nr_bytes;
102 };
103
104 typedef int (*action_fn) (struct pgpath *pgpath);
105
106 static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
107 static void trigger_event(struct work_struct *work);
108 static void activate_or_offline_path(struct pgpath *pgpath);
109 static void activate_path_work(struct work_struct *work);
110 static void process_queued_bios(struct work_struct *work);
111
112 /*-----------------------------------------------
113 * Multipath state flags.
114 *-----------------------------------------------*/
115
116 #define MPATHF_QUEUE_IO 0 /* Must we queue all I/O? */
117 #define MPATHF_QUEUE_IF_NO_PATH 1 /* Queue I/O if last path fails? */
118 #define MPATHF_SAVED_QUEUE_IF_NO_PATH 2 /* Saved state during suspension */
119 #define MPATHF_RETAIN_ATTACHED_HW_HANDLER 3 /* If there's already a hw_handler present, don't change it. */
120 #define MPATHF_PG_INIT_DISABLED 4 /* pg_init is not currently allowed */
121 #define MPATHF_PG_INIT_REQUIRED 5 /* pg_init needs calling? */
122 #define MPATHF_PG_INIT_DELAY_RETRY 6 /* Delay pg_init retry? */
123
124 /*-----------------------------------------------
125 * Allocation routines
126 *-----------------------------------------------*/
127
128 static struct pgpath *alloc_pgpath(void)
129 {
130 struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
131
132 if (!pgpath)
133 return NULL;
134
135 pgpath->is_active = true;
136
137 return pgpath;
138 }
139
140 static void free_pgpath(struct pgpath *pgpath)
141 {
142 kfree(pgpath);
143 }
144
145 static struct priority_group *alloc_priority_group(void)
146 {
147 struct priority_group *pg;
148
149 pg = kzalloc(sizeof(*pg), GFP_KERNEL);
150
151 if (pg)
152 INIT_LIST_HEAD(&pg->pgpaths);
153
154 return pg;
155 }
156
157 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
158 {
159 struct pgpath *pgpath, *tmp;
160
161 list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
162 list_del(&pgpath->list);
163 dm_put_device(ti, pgpath->path.dev);
164 free_pgpath(pgpath);
165 }
166 }
167
168 static void free_priority_group(struct priority_group *pg,
169 struct dm_target *ti)
170 {
171 struct path_selector *ps = &pg->ps;
172
173 if (ps->type) {
174 ps->type->destroy(ps);
175 dm_put_path_selector(ps->type);
176 }
177
178 free_pgpaths(&pg->pgpaths, ti);
179 kfree(pg);
180 }
181
182 static struct multipath *alloc_multipath(struct dm_target *ti)
183 {
184 struct multipath *m;
185
186 m = kzalloc(sizeof(*m), GFP_KERNEL);
187 if (m) {
188 INIT_LIST_HEAD(&m->priority_groups);
189 spin_lock_init(&m->lock);
190 atomic_set(&m->nr_valid_paths, 0);
191 INIT_WORK(&m->trigger_event, trigger_event);
192 mutex_init(&m->work_mutex);
193
194 m->queue_mode = DM_TYPE_NONE;
195
196 m->ti = ti;
197 ti->private = m;
198 }
199
200 return m;
201 }
202
203 static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m)
204 {
205 if (m->queue_mode == DM_TYPE_NONE) {
206 /*
207 * Default to request-based.
208 */
209 if (dm_use_blk_mq(dm_table_get_md(ti->table)))
210 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
211 else
212 m->queue_mode = DM_TYPE_REQUEST_BASED;
213
214 } else if (m->queue_mode == DM_TYPE_BIO_BASED ||
215 m->queue_mode == DM_TYPE_NVME_BIO_BASED) {
216 INIT_WORK(&m->process_queued_bios, process_queued_bios);
217
218 if (m->queue_mode == DM_TYPE_BIO_BASED) {
219 /*
220 * bio-based doesn't support any direct scsi_dh management;
221 * it just discovers if a scsi_dh is attached.
222 */
223 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
224 }
225 }
226
227 if (m->queue_mode != DM_TYPE_NVME_BIO_BASED) {
228 set_bit(MPATHF_QUEUE_IO, &m->flags);
229 atomic_set(&m->pg_init_in_progress, 0);
230 atomic_set(&m->pg_init_count, 0);
231 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
232 init_waitqueue_head(&m->pg_init_wait);
233 }
234
235 dm_table_set_type(ti->table, m->queue_mode);
236
237 return 0;
238 }
239
240 static void free_multipath(struct multipath *m)
241 {
242 struct priority_group *pg, *tmp;
243
244 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
245 list_del(&pg->list);
246 free_priority_group(pg, m->ti);
247 }
248
249 kfree(m->hw_handler_name);
250 kfree(m->hw_handler_params);
251 mutex_destroy(&m->work_mutex);
252 kfree(m);
253 }
254
255 static struct dm_mpath_io *get_mpio(union map_info *info)
256 {
257 return info->ptr;
258 }
259
260 static size_t multipath_per_bio_data_size(void)
261 {
262 return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details);
263 }
264
265 static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio)
266 {
267 return dm_per_bio_data(bio, multipath_per_bio_data_size());
268 }
269
270 static struct dm_bio_details *get_bio_details_from_mpio(struct dm_mpath_io *mpio)
271 {
272 /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */
273 void *bio_details = mpio + 1;
274 return bio_details;
275 }
276
277 static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p)
278 {
279 struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
280 struct dm_bio_details *bio_details = get_bio_details_from_mpio(mpio);
281
282 mpio->nr_bytes = bio->bi_iter.bi_size;
283 mpio->pgpath = NULL;
284 *mpio_p = mpio;
285
286 dm_bio_record(bio_details, bio);
287 }
288
289 /*-----------------------------------------------
290 * Path selection
291 *-----------------------------------------------*/
292
293 static int __pg_init_all_paths(struct multipath *m)
294 {
295 struct pgpath *pgpath;
296 unsigned long pg_init_delay = 0;
297
298 lockdep_assert_held(&m->lock);
299
300 if (atomic_read(&m->pg_init_in_progress) || test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
301 return 0;
302
303 atomic_inc(&m->pg_init_count);
304 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
305
306 /* Check here to reset pg_init_required */
307 if (!m->current_pg)
308 return 0;
309
310 if (test_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags))
311 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ?
312 m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS);
313 list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
314 /* Skip failed paths */
315 if (!pgpath->is_active)
316 continue;
317 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path,
318 pg_init_delay))
319 atomic_inc(&m->pg_init_in_progress);
320 }
321 return atomic_read(&m->pg_init_in_progress);
322 }
323
324 static int pg_init_all_paths(struct multipath *m)
325 {
326 int ret;
327 unsigned long flags;
328
329 spin_lock_irqsave(&m->lock, flags);
330 ret = __pg_init_all_paths(m);
331 spin_unlock_irqrestore(&m->lock, flags);
332
333 return ret;
334 }
335
336 static void __switch_pg(struct multipath *m, struct priority_group *pg)
337 {
338 m->current_pg = pg;
339
340 if (m->queue_mode == DM_TYPE_NVME_BIO_BASED)
341 return;
342
343 /* Must we initialise the PG first, and queue I/O till it's ready? */
344 if (m->hw_handler_name) {
345 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
346 set_bit(MPATHF_QUEUE_IO, &m->flags);
347 } else {
348 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
349 clear_bit(MPATHF_QUEUE_IO, &m->flags);
350 }
351
352 atomic_set(&m->pg_init_count, 0);
353 }
354
355 static struct pgpath *choose_path_in_pg(struct multipath *m,
356 struct priority_group *pg,
357 size_t nr_bytes)
358 {
359 unsigned long flags;
360 struct dm_path *path;
361 struct pgpath *pgpath;
362
363 path = pg->ps.type->select_path(&pg->ps, nr_bytes);
364 if (!path)
365 return ERR_PTR(-ENXIO);
366
367 pgpath = path_to_pgpath(path);
368
369 if (unlikely(READ_ONCE(m->current_pg) != pg)) {
370 /* Only update current_pgpath if pg changed */
371 spin_lock_irqsave(&m->lock, flags);
372 m->current_pgpath = pgpath;
373 __switch_pg(m, pg);
374 spin_unlock_irqrestore(&m->lock, flags);
375 }
376
377 return pgpath;
378 }
379
380 static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes)
381 {
382 unsigned long flags;
383 struct priority_group *pg;
384 struct pgpath *pgpath;
385 unsigned bypassed = 1;
386
387 if (!atomic_read(&m->nr_valid_paths)) {
388 if (m->queue_mode != DM_TYPE_NVME_BIO_BASED)
389 clear_bit(MPATHF_QUEUE_IO, &m->flags);
390 goto failed;
391 }
392
393 /* Were we instructed to switch PG? */
394 if (READ_ONCE(m->next_pg)) {
395 spin_lock_irqsave(&m->lock, flags);
396 pg = m->next_pg;
397 if (!pg) {
398 spin_unlock_irqrestore(&m->lock, flags);
399 goto check_current_pg;
400 }
401 m->next_pg = NULL;
402 spin_unlock_irqrestore(&m->lock, flags);
403 pgpath = choose_path_in_pg(m, pg, nr_bytes);
404 if (!IS_ERR_OR_NULL(pgpath))
405 return pgpath;
406 }
407
408 /* Don't change PG until it has no remaining paths */
409 check_current_pg:
410 pg = READ_ONCE(m->current_pg);
411 if (pg) {
412 pgpath = choose_path_in_pg(m, pg, nr_bytes);
413 if (!IS_ERR_OR_NULL(pgpath))
414 return pgpath;
415 }
416
417 /*
418 * Loop through priority groups until we find a valid path.
419 * First time we skip PGs marked 'bypassed'.
420 * Second time we only try the ones we skipped, but set
421 * pg_init_delay_retry so we do not hammer controllers.
422 */
423 do {
424 list_for_each_entry(pg, &m->priority_groups, list) {
425 if (pg->bypassed == !!bypassed)
426 continue;
427 pgpath = choose_path_in_pg(m, pg, nr_bytes);
428 if (!IS_ERR_OR_NULL(pgpath)) {
429 if (!bypassed)
430 set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
431 return pgpath;
432 }
433 }
434 } while (bypassed--);
435
436 failed:
437 spin_lock_irqsave(&m->lock, flags);
438 m->current_pgpath = NULL;
439 m->current_pg = NULL;
440 spin_unlock_irqrestore(&m->lock, flags);
441
442 return NULL;
443 }
444
445 /*
446 * dm_report_EIO() is a macro instead of a function to make pr_debug()
447 * report the function name and line number of the function from which
448 * it has been invoked.
449 */
450 #define dm_report_EIO(m) \
451 do { \
452 struct mapped_device *md = dm_table_get_md((m)->ti->table); \
453 \
454 pr_debug("%s: returning EIO; QIFNP = %d; SQIFNP = %d; DNFS = %d\n", \
455 dm_device_name(md), \
456 test_bit(MPATHF_QUEUE_IF_NO_PATH, &(m)->flags), \
457 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &(m)->flags), \
458 dm_noflush_suspending((m)->ti)); \
459 } while (0)
460
461 /*
462 * Check whether bios must be queued in the device-mapper core rather
463 * than here in the target.
464 *
465 * If MPATHF_QUEUE_IF_NO_PATH and MPATHF_SAVED_QUEUE_IF_NO_PATH hold
466 * the same value then we are not between multipath_presuspend()
467 * and multipath_resume() calls and we have no need to check
468 * for the DMF_NOFLUSH_SUSPENDING flag.
469 */
470 static bool __must_push_back(struct multipath *m, unsigned long flags)
471 {
472 return ((test_bit(MPATHF_QUEUE_IF_NO_PATH, &flags) !=
473 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &flags)) &&
474 dm_noflush_suspending(m->ti));
475 }
476
477 /*
478 * Following functions use READ_ONCE to get atomic access to
479 * all m->flags to avoid taking spinlock
480 */
481 static bool must_push_back_rq(struct multipath *m)
482 {
483 unsigned long flags = READ_ONCE(m->flags);
484 return test_bit(MPATHF_QUEUE_IF_NO_PATH, &flags) || __must_push_back(m, flags);
485 }
486
487 static bool must_push_back_bio(struct multipath *m)
488 {
489 unsigned long flags = READ_ONCE(m->flags);
490 return __must_push_back(m, flags);
491 }
492
493 /*
494 * Map cloned requests (request-based multipath)
495 */
496 static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
497 union map_info *map_context,
498 struct request **__clone)
499 {
500 struct multipath *m = ti->private;
501 size_t nr_bytes = blk_rq_bytes(rq);
502 struct pgpath *pgpath;
503 struct block_device *bdev;
504 struct dm_mpath_io *mpio = get_mpio(map_context);
505 struct request_queue *q;
506 struct request *clone;
507
508 /* Do we need to select a new pgpath? */
509 pgpath = READ_ONCE(m->current_pgpath);
510 if (!pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags))
511 pgpath = choose_pgpath(m, nr_bytes);
512
513 if (!pgpath) {
514 if (must_push_back_rq(m))
515 return DM_MAPIO_DELAY_REQUEUE;
516 dm_report_EIO(m); /* Failed */
517 return DM_MAPIO_KILL;
518 } else if (test_bit(MPATHF_QUEUE_IO, &m->flags) ||
519 test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
520 pg_init_all_paths(m);
521 return DM_MAPIO_DELAY_REQUEUE;
522 }
523
524 mpio->pgpath = pgpath;
525 mpio->nr_bytes = nr_bytes;
526
527 bdev = pgpath->path.dev->bdev;
528 q = bdev_get_queue(bdev);
529 clone = blk_get_request(q, rq->cmd_flags | REQ_NOMERGE, GFP_ATOMIC);
530 if (IS_ERR(clone)) {
531 /* EBUSY, ENODEV or EWOULDBLOCK: requeue */
532 if (blk_queue_dying(q)) {
533 atomic_inc(&m->pg_init_in_progress);
534 activate_or_offline_path(pgpath);
535 return DM_MAPIO_DELAY_REQUEUE;
536 }
537
538 /*
539 * blk-mq's SCHED_RESTART can cover this requeue, so we
540 * needn't deal with it by DELAY_REQUEUE. More importantly,
541 * we have to return DM_MAPIO_REQUEUE so that blk-mq can
542 * get the queue busy feedback (via BLK_STS_RESOURCE),
543 * otherwise I/O merging can suffer.
544 */
545 if (q->mq_ops)
546 return DM_MAPIO_REQUEUE;
547 else
548 return DM_MAPIO_DELAY_REQUEUE;
549 }
550 clone->bio = clone->biotail = NULL;
551 clone->rq_disk = bdev->bd_disk;
552 clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
553 *__clone = clone;
554
555 if (pgpath->pg->ps.type->start_io)
556 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
557 &pgpath->path,
558 nr_bytes);
559 return DM_MAPIO_REMAPPED;
560 }
561
562 static void multipath_release_clone(struct request *clone)
563 {
564 blk_put_request(clone);
565 }
566
567 /*
568 * Map cloned bios (bio-based multipath)
569 */
570
571 static struct pgpath *__map_bio(struct multipath *m, struct bio *bio)
572 {
573 struct pgpath *pgpath;
574 unsigned long flags;
575 bool queue_io;
576
577 /* Do we need to select a new pgpath? */
578 pgpath = READ_ONCE(m->current_pgpath);
579 queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags);
580 if (!pgpath || !queue_io)
581 pgpath = choose_pgpath(m, bio->bi_iter.bi_size);
582
583 if ((pgpath && queue_io) ||
584 (!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))) {
585 /* Queue for the daemon to resubmit */
586 spin_lock_irqsave(&m->lock, flags);
587 bio_list_add(&m->queued_bios, bio);
588 spin_unlock_irqrestore(&m->lock, flags);
589
590 /* PG_INIT_REQUIRED cannot be set without QUEUE_IO */
591 if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
592 pg_init_all_paths(m);
593 else if (!queue_io)
594 queue_work(kmultipathd, &m->process_queued_bios);
595
596 return ERR_PTR(-EAGAIN);
597 }
598
599 return pgpath;
600 }
601
602 static struct pgpath *__map_bio_nvme(struct multipath *m, struct bio *bio)
603 {
604 struct pgpath *pgpath;
605 unsigned long flags;
606
607 /* Do we need to select a new pgpath? */
608 /*
609 * FIXME: currently only switching path if no path (due to failure, etc)
610 * - which negates the point of using a path selector
611 */
612 pgpath = READ_ONCE(m->current_pgpath);
613 if (!pgpath)
614 pgpath = choose_pgpath(m, bio->bi_iter.bi_size);
615
616 if (!pgpath) {
617 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
618 /* Queue for the daemon to resubmit */
619 spin_lock_irqsave(&m->lock, flags);
620 bio_list_add(&m->queued_bios, bio);
621 spin_unlock_irqrestore(&m->lock, flags);
622 queue_work(kmultipathd, &m->process_queued_bios);
623
624 return ERR_PTR(-EAGAIN);
625 }
626 return NULL;
627 }
628
629 return pgpath;
630 }
631
632 static int __multipath_map_bio(struct multipath *m, struct bio *bio,
633 struct dm_mpath_io *mpio)
634 {
635 struct pgpath *pgpath;
636
637 if (m->queue_mode == DM_TYPE_NVME_BIO_BASED)
638 pgpath = __map_bio_nvme(m, bio);
639 else
640 pgpath = __map_bio(m, bio);
641
642 if (IS_ERR(pgpath))
643 return DM_MAPIO_SUBMITTED;
644
645 if (!pgpath) {
646 if (must_push_back_bio(m))
647 return DM_MAPIO_REQUEUE;
648 dm_report_EIO(m);
649 return DM_MAPIO_KILL;
650 }
651
652 mpio->pgpath = pgpath;
653
654 bio->bi_status = 0;
655 bio_set_dev(bio, pgpath->path.dev->bdev);
656 bio->bi_opf |= REQ_FAILFAST_TRANSPORT;
657
658 if (pgpath->pg->ps.type->start_io)
659 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
660 &pgpath->path,
661 mpio->nr_bytes);
662 return DM_MAPIO_REMAPPED;
663 }
664
665 static int multipath_map_bio(struct dm_target *ti, struct bio *bio)
666 {
667 struct multipath *m = ti->private;
668 struct dm_mpath_io *mpio = NULL;
669
670 multipath_init_per_bio_data(bio, &mpio);
671 return __multipath_map_bio(m, bio, mpio);
672 }
673
674 static void process_queued_io_list(struct multipath *m)
675 {
676 if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED)
677 dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table));
678 else if (m->queue_mode == DM_TYPE_BIO_BASED ||
679 m->queue_mode == DM_TYPE_NVME_BIO_BASED)
680 queue_work(kmultipathd, &m->process_queued_bios);
681 }
682
683 static void process_queued_bios(struct work_struct *work)
684 {
685 int r;
686 unsigned long flags;
687 struct bio *bio;
688 struct bio_list bios;
689 struct blk_plug plug;
690 struct multipath *m =
691 container_of(work, struct multipath, process_queued_bios);
692
693 bio_list_init(&bios);
694
695 spin_lock_irqsave(&m->lock, flags);
696
697 if (bio_list_empty(&m->queued_bios)) {
698 spin_unlock_irqrestore(&m->lock, flags);
699 return;
700 }
701
702 bio_list_merge(&bios, &m->queued_bios);
703 bio_list_init(&m->queued_bios);
704
705 spin_unlock_irqrestore(&m->lock, flags);
706
707 blk_start_plug(&plug);
708 while ((bio = bio_list_pop(&bios))) {
709 struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
710 dm_bio_restore(get_bio_details_from_mpio(mpio), bio);
711 r = __multipath_map_bio(m, bio, mpio);
712 switch (r) {
713 case DM_MAPIO_KILL:
714 bio->bi_status = BLK_STS_IOERR;
715 bio_endio(bio);
716 break;
717 case DM_MAPIO_REQUEUE:
718 bio->bi_status = BLK_STS_DM_REQUEUE;
719 bio_endio(bio);
720 break;
721 case DM_MAPIO_REMAPPED:
722 generic_make_request(bio);
723 break;
724 case 0:
725 break;
726 default:
727 WARN_ONCE(true, "__multipath_map_bio() returned %d\n", r);
728 }
729 }
730 blk_finish_plug(&plug);
731 }
732
733 /*
734 * If we run out of usable paths, should we queue I/O or error it?
735 */
736 static int queue_if_no_path(struct multipath *m, bool queue_if_no_path,
737 bool save_old_value)
738 {
739 unsigned long flags;
740
741 spin_lock_irqsave(&m->lock, flags);
742 assign_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags,
743 (save_old_value && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) ||
744 (!save_old_value && queue_if_no_path));
745 assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags, queue_if_no_path);
746 spin_unlock_irqrestore(&m->lock, flags);
747
748 if (!queue_if_no_path) {
749 dm_table_run_md_queue_async(m->ti->table);
750 process_queued_io_list(m);
751 }
752
753 return 0;
754 }
755
756 /*
757 * An event is triggered whenever a path is taken out of use.
758 * Includes path failure and PG bypass.
759 */
760 static void trigger_event(struct work_struct *work)
761 {
762 struct multipath *m =
763 container_of(work, struct multipath, trigger_event);
764
765 dm_table_event(m->ti->table);
766 }
767
768 /*-----------------------------------------------------------------
769 * Constructor/argument parsing:
770 * <#multipath feature args> [<arg>]*
771 * <#hw_handler args> [hw_handler [<arg>]*]
772 * <#priority groups>
773 * <initial priority group>
774 * [<selector> <#selector args> [<arg>]*
775 * <#paths> <#per-path selector args>
776 * [<path> [<arg>]* ]+ ]+
777 *---------------------------------------------------------------*/
778 static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
779 struct dm_target *ti)
780 {
781 int r;
782 struct path_selector_type *pst;
783 unsigned ps_argc;
784
785 static const struct dm_arg _args[] = {
786 {0, 1024, "invalid number of path selector args"},
787 };
788
789 pst = dm_get_path_selector(dm_shift_arg(as));
790 if (!pst) {
791 ti->error = "unknown path selector type";
792 return -EINVAL;
793 }
794
795 r = dm_read_arg_group(_args, as, &ps_argc, &ti->error);
796 if (r) {
797 dm_put_path_selector(pst);
798 return -EINVAL;
799 }
800
801 r = pst->create(&pg->ps, ps_argc, as->argv);
802 if (r) {
803 dm_put_path_selector(pst);
804 ti->error = "path selector constructor failed";
805 return r;
806 }
807
808 pg->ps.type = pst;
809 dm_consume_args(as, ps_argc);
810
811 return 0;
812 }
813
814 static int setup_scsi_dh(struct block_device *bdev, struct multipath *m, char **error)
815 {
816 struct request_queue *q = bdev_get_queue(bdev);
817 const char *attached_handler_name;
818 int r;
819
820 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) {
821 retain:
822 attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
823 if (attached_handler_name) {
824 /*
825 * Clear any hw_handler_params associated with a
826 * handler that isn't already attached.
827 */
828 if (m->hw_handler_name && strcmp(attached_handler_name, m->hw_handler_name)) {
829 kfree(m->hw_handler_params);
830 m->hw_handler_params = NULL;
831 }
832
833 /*
834 * Reset hw_handler_name to match the attached handler
835 *
836 * NB. This modifies the table line to show the actual
837 * handler instead of the original table passed in.
838 */
839 kfree(m->hw_handler_name);
840 m->hw_handler_name = attached_handler_name;
841 }
842 }
843
844 if (m->hw_handler_name) {
845 r = scsi_dh_attach(q, m->hw_handler_name);
846 if (r == -EBUSY) {
847 char b[BDEVNAME_SIZE];
848
849 printk(KERN_INFO "dm-mpath: retaining handler on device %s\n",
850 bdevname(bdev, b));
851 goto retain;
852 }
853 if (r < 0) {
854 *error = "error attaching hardware handler";
855 return r;
856 }
857
858 if (m->hw_handler_params) {
859 r = scsi_dh_set_params(q, m->hw_handler_params);
860 if (r < 0) {
861 *error = "unable to set hardware handler parameters";
862 return r;
863 }
864 }
865 }
866
867 return 0;
868 }
869
870 static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
871 struct dm_target *ti)
872 {
873 int r;
874 struct pgpath *p;
875 struct multipath *m = ti->private;
876
877 /* we need at least a path arg */
878 if (as->argc < 1) {
879 ti->error = "no device given";
880 return ERR_PTR(-EINVAL);
881 }
882
883 p = alloc_pgpath();
884 if (!p)
885 return ERR_PTR(-ENOMEM);
886
887 r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
888 &p->path.dev);
889 if (r) {
890 ti->error = "error getting device";
891 goto bad;
892 }
893
894 if (m->queue_mode != DM_TYPE_NVME_BIO_BASED) {
895 INIT_DELAYED_WORK(&p->activate_path, activate_path_work);
896 r = setup_scsi_dh(p->path.dev->bdev, m, &ti->error);
897 if (r) {
898 dm_put_device(ti, p->path.dev);
899 goto bad;
900 }
901 }
902
903 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
904 if (r) {
905 dm_put_device(ti, p->path.dev);
906 goto bad;
907 }
908
909 return p;
910 bad:
911 free_pgpath(p);
912 return ERR_PTR(r);
913 }
914
915 static struct priority_group *parse_priority_group(struct dm_arg_set *as,
916 struct multipath *m)
917 {
918 static const struct dm_arg _args[] = {
919 {1, 1024, "invalid number of paths"},
920 {0, 1024, "invalid number of selector args"}
921 };
922
923 int r;
924 unsigned i, nr_selector_args, nr_args;
925 struct priority_group *pg;
926 struct dm_target *ti = m->ti;
927
928 if (as->argc < 2) {
929 as->argc = 0;
930 ti->error = "not enough priority group arguments";
931 return ERR_PTR(-EINVAL);
932 }
933
934 pg = alloc_priority_group();
935 if (!pg) {
936 ti->error = "couldn't allocate priority group";
937 return ERR_PTR(-ENOMEM);
938 }
939 pg->m = m;
940
941 r = parse_path_selector(as, pg, ti);
942 if (r)
943 goto bad;
944
945 /*
946 * read the paths
947 */
948 r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error);
949 if (r)
950 goto bad;
951
952 r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error);
953 if (r)
954 goto bad;
955
956 nr_args = 1 + nr_selector_args;
957 for (i = 0; i < pg->nr_pgpaths; i++) {
958 struct pgpath *pgpath;
959 struct dm_arg_set path_args;
960
961 if (as->argc < nr_args) {
962 ti->error = "not enough path parameters";
963 r = -EINVAL;
964 goto bad;
965 }
966
967 path_args.argc = nr_args;
968 path_args.argv = as->argv;
969
970 pgpath = parse_path(&path_args, &pg->ps, ti);
971 if (IS_ERR(pgpath)) {
972 r = PTR_ERR(pgpath);
973 goto bad;
974 }
975
976 pgpath->pg = pg;
977 list_add_tail(&pgpath->list, &pg->pgpaths);
978 dm_consume_args(as, nr_args);
979 }
980
981 return pg;
982
983 bad:
984 free_priority_group(pg, ti);
985 return ERR_PTR(r);
986 }
987
988 static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
989 {
990 unsigned hw_argc;
991 int ret;
992 struct dm_target *ti = m->ti;
993
994 static const struct dm_arg _args[] = {
995 {0, 1024, "invalid number of hardware handler args"},
996 };
997
998 if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
999 return -EINVAL;
1000
1001 if (!hw_argc)
1002 return 0;
1003
1004 if (m->queue_mode == DM_TYPE_BIO_BASED ||
1005 m->queue_mode == DM_TYPE_NVME_BIO_BASED) {
1006 dm_consume_args(as, hw_argc);
1007 DMERR("bio-based multipath doesn't allow hardware handler args");
1008 return 0;
1009 }
1010
1011 m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
1012 if (!m->hw_handler_name)
1013 return -EINVAL;
1014
1015 if (hw_argc > 1) {
1016 char *p;
1017 int i, j, len = 4;
1018
1019 for (i = 0; i <= hw_argc - 2; i++)
1020 len += strlen(as->argv[i]) + 1;
1021 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
1022 if (!p) {
1023 ti->error = "memory allocation failed";
1024 ret = -ENOMEM;
1025 goto fail;
1026 }
1027 j = sprintf(p, "%d", hw_argc - 1);
1028 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
1029 j = sprintf(p, "%s", as->argv[i]);
1030 }
1031 dm_consume_args(as, hw_argc - 1);
1032
1033 return 0;
1034 fail:
1035 kfree(m->hw_handler_name);
1036 m->hw_handler_name = NULL;
1037 return ret;
1038 }
1039
1040 static int parse_features(struct dm_arg_set *as, struct multipath *m)
1041 {
1042 int r;
1043 unsigned argc;
1044 struct dm_target *ti = m->ti;
1045 const char *arg_name;
1046
1047 static const struct dm_arg _args[] = {
1048 {0, 8, "invalid number of feature args"},
1049 {1, 50, "pg_init_retries must be between 1 and 50"},
1050 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
1051 };
1052
1053 r = dm_read_arg_group(_args, as, &argc, &ti->error);
1054 if (r)
1055 return -EINVAL;
1056
1057 if (!argc)
1058 return 0;
1059
1060 do {
1061 arg_name = dm_shift_arg(as);
1062 argc--;
1063
1064 if (!strcasecmp(arg_name, "queue_if_no_path")) {
1065 r = queue_if_no_path(m, true, false);
1066 continue;
1067 }
1068
1069 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) {
1070 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
1071 continue;
1072 }
1073
1074 if (!strcasecmp(arg_name, "pg_init_retries") &&
1075 (argc >= 1)) {
1076 r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error);
1077 argc--;
1078 continue;
1079 }
1080
1081 if (!strcasecmp(arg_name, "pg_init_delay_msecs") &&
1082 (argc >= 1)) {
1083 r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error);
1084 argc--;
1085 continue;
1086 }
1087
1088 if (!strcasecmp(arg_name, "queue_mode") &&
1089 (argc >= 1)) {
1090 const char *queue_mode_name = dm_shift_arg(as);
1091
1092 if (!strcasecmp(queue_mode_name, "bio"))
1093 m->queue_mode = DM_TYPE_BIO_BASED;
1094 else if (!strcasecmp(queue_mode_name, "nvme"))
1095 m->queue_mode = DM_TYPE_NVME_BIO_BASED;
1096 else if (!strcasecmp(queue_mode_name, "rq"))
1097 m->queue_mode = DM_TYPE_REQUEST_BASED;
1098 else if (!strcasecmp(queue_mode_name, "mq"))
1099 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
1100 else {
1101 ti->error = "Unknown 'queue_mode' requested";
1102 r = -EINVAL;
1103 }
1104 argc--;
1105 continue;
1106 }
1107
1108 ti->error = "Unrecognised multipath feature request";
1109 r = -EINVAL;
1110 } while (argc && !r);
1111
1112 return r;
1113 }
1114
1115 static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv)
1116 {
1117 /* target arguments */
1118 static const struct dm_arg _args[] = {
1119 {0, 1024, "invalid number of priority groups"},
1120 {0, 1024, "invalid initial priority group number"},
1121 };
1122
1123 int r;
1124 struct multipath *m;
1125 struct dm_arg_set as;
1126 unsigned pg_count = 0;
1127 unsigned next_pg_num;
1128
1129 as.argc = argc;
1130 as.argv = argv;
1131
1132 m = alloc_multipath(ti);
1133 if (!m) {
1134 ti->error = "can't allocate multipath";
1135 return -EINVAL;
1136 }
1137
1138 r = parse_features(&as, m);
1139 if (r)
1140 goto bad;
1141
1142 r = alloc_multipath_stage2(ti, m);
1143 if (r)
1144 goto bad;
1145
1146 r = parse_hw_handler(&as, m);
1147 if (r)
1148 goto bad;
1149
1150 r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error);
1151 if (r)
1152 goto bad;
1153
1154 r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error);
1155 if (r)
1156 goto bad;
1157
1158 if ((!m->nr_priority_groups && next_pg_num) ||
1159 (m->nr_priority_groups && !next_pg_num)) {
1160 ti->error = "invalid initial priority group";
1161 r = -EINVAL;
1162 goto bad;
1163 }
1164
1165 /* parse the priority groups */
1166 while (as.argc) {
1167 struct priority_group *pg;
1168 unsigned nr_valid_paths = atomic_read(&m->nr_valid_paths);
1169
1170 pg = parse_priority_group(&as, m);
1171 if (IS_ERR(pg)) {
1172 r = PTR_ERR(pg);
1173 goto bad;
1174 }
1175
1176 nr_valid_paths += pg->nr_pgpaths;
1177 atomic_set(&m->nr_valid_paths, nr_valid_paths);
1178
1179 list_add_tail(&pg->list, &m->priority_groups);
1180 pg_count++;
1181 pg->pg_num = pg_count;
1182 if (!--next_pg_num)
1183 m->next_pg = pg;
1184 }
1185
1186 if (pg_count != m->nr_priority_groups) {
1187 ti->error = "priority group count mismatch";
1188 r = -EINVAL;
1189 goto bad;
1190 }
1191
1192 ti->num_flush_bios = 1;
1193 ti->num_discard_bios = 1;
1194 ti->num_write_same_bios = 1;
1195 ti->num_write_zeroes_bios = 1;
1196 if (m->queue_mode == DM_TYPE_BIO_BASED || m->queue_mode == DM_TYPE_NVME_BIO_BASED)
1197 ti->per_io_data_size = multipath_per_bio_data_size();
1198 else
1199 ti->per_io_data_size = sizeof(struct dm_mpath_io);
1200
1201 return 0;
1202
1203 bad:
1204 free_multipath(m);
1205 return r;
1206 }
1207
1208 static void multipath_wait_for_pg_init_completion(struct multipath *m)
1209 {
1210 DEFINE_WAIT(wait);
1211
1212 while (1) {
1213 prepare_to_wait(&m->pg_init_wait, &wait, TASK_UNINTERRUPTIBLE);
1214
1215 if (!atomic_read(&m->pg_init_in_progress))
1216 break;
1217
1218 io_schedule();
1219 }
1220 finish_wait(&m->pg_init_wait, &wait);
1221 }
1222
1223 static void flush_multipath_work(struct multipath *m)
1224 {
1225 if (m->hw_handler_name) {
1226 set_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1227 smp_mb__after_atomic();
1228
1229 flush_workqueue(kmpath_handlerd);
1230 multipath_wait_for_pg_init_completion(m);
1231
1232 clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1233 smp_mb__after_atomic();
1234 }
1235
1236 flush_workqueue(kmultipathd);
1237 flush_work(&m->trigger_event);
1238 }
1239
1240 static void multipath_dtr(struct dm_target *ti)
1241 {
1242 struct multipath *m = ti->private;
1243
1244 flush_multipath_work(m);
1245 free_multipath(m);
1246 }
1247
1248 /*
1249 * Take a path out of use.
1250 */
1251 static int fail_path(struct pgpath *pgpath)
1252 {
1253 unsigned long flags;
1254 struct multipath *m = pgpath->pg->m;
1255
1256 spin_lock_irqsave(&m->lock, flags);
1257
1258 if (!pgpath->is_active)
1259 goto out;
1260
1261 DMWARN("Failing path %s.", pgpath->path.dev->name);
1262
1263 pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
1264 pgpath->is_active = false;
1265 pgpath->fail_count++;
1266
1267 atomic_dec(&m->nr_valid_paths);
1268
1269 if (pgpath == m->current_pgpath)
1270 m->current_pgpath = NULL;
1271
1272 dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
1273 pgpath->path.dev->name, atomic_read(&m->nr_valid_paths));
1274
1275 schedule_work(&m->trigger_event);
1276
1277 out:
1278 spin_unlock_irqrestore(&m->lock, flags);
1279
1280 return 0;
1281 }
1282
1283 /*
1284 * Reinstate a previously-failed path
1285 */
1286 static int reinstate_path(struct pgpath *pgpath)
1287 {
1288 int r = 0, run_queue = 0;
1289 unsigned long flags;
1290 struct multipath *m = pgpath->pg->m;
1291 unsigned nr_valid_paths;
1292
1293 spin_lock_irqsave(&m->lock, flags);
1294
1295 if (pgpath->is_active)
1296 goto out;
1297
1298 DMWARN("Reinstating path %s.", pgpath->path.dev->name);
1299
1300 r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
1301 if (r)
1302 goto out;
1303
1304 pgpath->is_active = true;
1305
1306 nr_valid_paths = atomic_inc_return(&m->nr_valid_paths);
1307 if (nr_valid_paths == 1) {
1308 m->current_pgpath = NULL;
1309 run_queue = 1;
1310 } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
1311 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
1312 atomic_inc(&m->pg_init_in_progress);
1313 }
1314
1315 dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
1316 pgpath->path.dev->name, nr_valid_paths);
1317
1318 schedule_work(&m->trigger_event);
1319
1320 out:
1321 spin_unlock_irqrestore(&m->lock, flags);
1322 if (run_queue) {
1323 dm_table_run_md_queue_async(m->ti->table);
1324 process_queued_io_list(m);
1325 }
1326
1327 return r;
1328 }
1329
1330 /*
1331 * Fail or reinstate all paths that match the provided struct dm_dev.
1332 */
1333 static int action_dev(struct multipath *m, struct dm_dev *dev,
1334 action_fn action)
1335 {
1336 int r = -EINVAL;
1337 struct pgpath *pgpath;
1338 struct priority_group *pg;
1339
1340 list_for_each_entry(pg, &m->priority_groups, list) {
1341 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1342 if (pgpath->path.dev == dev)
1343 r = action(pgpath);
1344 }
1345 }
1346
1347 return r;
1348 }
1349
1350 /*
1351 * Temporarily try to avoid having to use the specified PG
1352 */
1353 static void bypass_pg(struct multipath *m, struct priority_group *pg,
1354 bool bypassed)
1355 {
1356 unsigned long flags;
1357
1358 spin_lock_irqsave(&m->lock, flags);
1359
1360 pg->bypassed = bypassed;
1361 m->current_pgpath = NULL;
1362 m->current_pg = NULL;
1363
1364 spin_unlock_irqrestore(&m->lock, flags);
1365
1366 schedule_work(&m->trigger_event);
1367 }
1368
1369 /*
1370 * Switch to using the specified PG from the next I/O that gets mapped
1371 */
1372 static int switch_pg_num(struct multipath *m, const char *pgstr)
1373 {
1374 struct priority_group *pg;
1375 unsigned pgnum;
1376 unsigned long flags;
1377 char dummy;
1378
1379 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1380 !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
1381 DMWARN("invalid PG number supplied to switch_pg_num");
1382 return -EINVAL;
1383 }
1384
1385 spin_lock_irqsave(&m->lock, flags);
1386 list_for_each_entry(pg, &m->priority_groups, list) {
1387 pg->bypassed = false;
1388 if (--pgnum)
1389 continue;
1390
1391 m->current_pgpath = NULL;
1392 m->current_pg = NULL;
1393 m->next_pg = pg;
1394 }
1395 spin_unlock_irqrestore(&m->lock, flags);
1396
1397 schedule_work(&m->trigger_event);
1398 return 0;
1399 }
1400
1401 /*
1402 * Set/clear bypassed status of a PG.
1403 * PGs are numbered upwards from 1 in the order they were declared.
1404 */
1405 static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed)
1406 {
1407 struct priority_group *pg;
1408 unsigned pgnum;
1409 char dummy;
1410
1411 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1412 !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
1413 DMWARN("invalid PG number supplied to bypass_pg");
1414 return -EINVAL;
1415 }
1416
1417 list_for_each_entry(pg, &m->priority_groups, list) {
1418 if (!--pgnum)
1419 break;
1420 }
1421
1422 bypass_pg(m, pg, bypassed);
1423 return 0;
1424 }
1425
1426 /*
1427 * Should we retry pg_init immediately?
1428 */
1429 static bool pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
1430 {
1431 unsigned long flags;
1432 bool limit_reached = false;
1433
1434 spin_lock_irqsave(&m->lock, flags);
1435
1436 if (atomic_read(&m->pg_init_count) <= m->pg_init_retries &&
1437 !test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
1438 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
1439 else
1440 limit_reached = true;
1441
1442 spin_unlock_irqrestore(&m->lock, flags);
1443
1444 return limit_reached;
1445 }
1446
1447 static void pg_init_done(void *data, int errors)
1448 {
1449 struct pgpath *pgpath = data;
1450 struct priority_group *pg = pgpath->pg;
1451 struct multipath *m = pg->m;
1452 unsigned long flags;
1453 bool delay_retry = false;
1454
1455 /* device or driver problems */
1456 switch (errors) {
1457 case SCSI_DH_OK:
1458 break;
1459 case SCSI_DH_NOSYS:
1460 if (!m->hw_handler_name) {
1461 errors = 0;
1462 break;
1463 }
1464 DMERR("Could not failover the device: Handler scsi_dh_%s "
1465 "Error %d.", m->hw_handler_name, errors);
1466 /*
1467 * Fail path for now, so we do not ping pong
1468 */
1469 fail_path(pgpath);
1470 break;
1471 case SCSI_DH_DEV_TEMP_BUSY:
1472 /*
1473 * Probably doing something like FW upgrade on the
1474 * controller so try the other pg.
1475 */
1476 bypass_pg(m, pg, true);
1477 break;
1478 case SCSI_DH_RETRY:
1479 /* Wait before retrying. */
1480 delay_retry = 1;
1481 /* fall through */
1482 case SCSI_DH_IMM_RETRY:
1483 case SCSI_DH_RES_TEMP_UNAVAIL:
1484 if (pg_init_limit_reached(m, pgpath))
1485 fail_path(pgpath);
1486 errors = 0;
1487 break;
1488 case SCSI_DH_DEV_OFFLINED:
1489 default:
1490 /*
1491 * We probably do not want to fail the path for a device
1492 * error, but this is what the old dm did. In future
1493 * patches we can do more advanced handling.
1494 */
1495 fail_path(pgpath);
1496 }
1497
1498 spin_lock_irqsave(&m->lock, flags);
1499 if (errors) {
1500 if (pgpath == m->current_pgpath) {
1501 DMERR("Could not failover device. Error %d.", errors);
1502 m->current_pgpath = NULL;
1503 m->current_pg = NULL;
1504 }
1505 } else if (!test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
1506 pg->bypassed = false;
1507
1508 if (atomic_dec_return(&m->pg_init_in_progress) > 0)
1509 /* Activations of other paths are still on going */
1510 goto out;
1511
1512 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
1513 if (delay_retry)
1514 set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1515 else
1516 clear_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1517
1518 if (__pg_init_all_paths(m))
1519 goto out;
1520 }
1521 clear_bit(MPATHF_QUEUE_IO, &m->flags);
1522
1523 process_queued_io_list(m);
1524
1525 /*
1526 * Wake up any thread waiting to suspend.
1527 */
1528 wake_up(&m->pg_init_wait);
1529
1530 out:
1531 spin_unlock_irqrestore(&m->lock, flags);
1532 }
1533
1534 static void activate_or_offline_path(struct pgpath *pgpath)
1535 {
1536 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1537
1538 if (pgpath->is_active && !blk_queue_dying(q))
1539 scsi_dh_activate(q, pg_init_done, pgpath);
1540 else
1541 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED);
1542 }
1543
1544 static void activate_path_work(struct work_struct *work)
1545 {
1546 struct pgpath *pgpath =
1547 container_of(work, struct pgpath, activate_path.work);
1548
1549 activate_or_offline_path(pgpath);
1550 }
1551
1552 static int multipath_end_io(struct dm_target *ti, struct request *clone,
1553 blk_status_t error, union map_info *map_context)
1554 {
1555 struct dm_mpath_io *mpio = get_mpio(map_context);
1556 struct pgpath *pgpath = mpio->pgpath;
1557 int r = DM_ENDIO_DONE;
1558
1559 /*
1560 * We don't queue any clone request inside the multipath target
1561 * during end I/O handling, since those clone requests don't have
1562 * bio clones. If we queue them inside the multipath target,
1563 * we need to make bio clones, that requires memory allocation.
1564 * (See drivers/md/dm-rq.c:end_clone_bio() about why the clone requests
1565 * don't have bio clones.)
1566 * Instead of queueing the clone request here, we queue the original
1567 * request into dm core, which will remake a clone request and
1568 * clone bios for it and resubmit it later.
1569 */
1570 if (error && blk_path_error(error)) {
1571 struct multipath *m = ti->private;
1572
1573 if (error == BLK_STS_RESOURCE)
1574 r = DM_ENDIO_DELAY_REQUEUE;
1575 else
1576 r = DM_ENDIO_REQUEUE;
1577
1578 if (pgpath)
1579 fail_path(pgpath);
1580
1581 if (atomic_read(&m->nr_valid_paths) == 0 &&
1582 !must_push_back_rq(m)) {
1583 if (error == BLK_STS_IOERR)
1584 dm_report_EIO(m);
1585 /* complete with the original error */
1586 r = DM_ENDIO_DONE;
1587 }
1588 }
1589
1590 if (pgpath) {
1591 struct path_selector *ps = &pgpath->pg->ps;
1592
1593 if (ps->type->end_io)
1594 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1595 }
1596
1597 return r;
1598 }
1599
1600 static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone,
1601 blk_status_t *error)
1602 {
1603 struct multipath *m = ti->private;
1604 struct dm_mpath_io *mpio = get_mpio_from_bio(clone);
1605 struct pgpath *pgpath = mpio->pgpath;
1606 unsigned long flags;
1607 int r = DM_ENDIO_DONE;
1608
1609 if (!*error || !blk_path_error(*error))
1610 goto done;
1611
1612 if (pgpath)
1613 fail_path(pgpath);
1614
1615 if (atomic_read(&m->nr_valid_paths) == 0 &&
1616 !test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
1617 if (must_push_back_bio(m)) {
1618 r = DM_ENDIO_REQUEUE;
1619 } else {
1620 dm_report_EIO(m);
1621 *error = BLK_STS_IOERR;
1622 }
1623 goto done;
1624 }
1625
1626 spin_lock_irqsave(&m->lock, flags);
1627 bio_list_add(&m->queued_bios, clone);
1628 spin_unlock_irqrestore(&m->lock, flags);
1629 if (!test_bit(MPATHF_QUEUE_IO, &m->flags))
1630 queue_work(kmultipathd, &m->process_queued_bios);
1631
1632 r = DM_ENDIO_INCOMPLETE;
1633 done:
1634 if (pgpath) {
1635 struct path_selector *ps = &pgpath->pg->ps;
1636
1637 if (ps->type->end_io)
1638 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1639 }
1640
1641 return r;
1642 }
1643
1644 /*
1645 * Suspend can't complete until all the I/O is processed so if
1646 * the last path fails we must error any remaining I/O.
1647 * Note that if the freeze_bdev fails while suspending, the
1648 * queue_if_no_path state is lost - userspace should reset it.
1649 */
1650 static void multipath_presuspend(struct dm_target *ti)
1651 {
1652 struct multipath *m = ti->private;
1653
1654 queue_if_no_path(m, false, true);
1655 }
1656
1657 static void multipath_postsuspend(struct dm_target *ti)
1658 {
1659 struct multipath *m = ti->private;
1660
1661 mutex_lock(&m->work_mutex);
1662 flush_multipath_work(m);
1663 mutex_unlock(&m->work_mutex);
1664 }
1665
1666 /*
1667 * Restore the queue_if_no_path setting.
1668 */
1669 static void multipath_resume(struct dm_target *ti)
1670 {
1671 struct multipath *m = ti->private;
1672 unsigned long flags;
1673
1674 spin_lock_irqsave(&m->lock, flags);
1675 assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags,
1676 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags));
1677 spin_unlock_irqrestore(&m->lock, flags);
1678 }
1679
1680 /*
1681 * Info output has the following format:
1682 * num_multipath_feature_args [multipath_feature_args]*
1683 * num_handler_status_args [handler_status_args]*
1684 * num_groups init_group_number
1685 * [A|D|E num_ps_status_args [ps_status_args]*
1686 * num_paths num_selector_args
1687 * [path_dev A|F fail_count [selector_args]* ]+ ]+
1688 *
1689 * Table output has the following format (identical to the constructor string):
1690 * num_feature_args [features_args]*
1691 * num_handler_args hw_handler [hw_handler_args]*
1692 * num_groups init_group_number
1693 * [priority selector-name num_ps_args [ps_args]*
1694 * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1695 */
1696 static void multipath_status(struct dm_target *ti, status_type_t type,
1697 unsigned status_flags, char *result, unsigned maxlen)
1698 {
1699 int sz = 0;
1700 unsigned long flags;
1701 struct multipath *m = ti->private;
1702 struct priority_group *pg;
1703 struct pgpath *p;
1704 unsigned pg_num;
1705 char state;
1706
1707 spin_lock_irqsave(&m->lock, flags);
1708
1709 /* Features */
1710 if (type == STATUSTYPE_INFO)
1711 DMEMIT("2 %u %u ", test_bit(MPATHF_QUEUE_IO, &m->flags),
1712 atomic_read(&m->pg_init_count));
1713 else {
1714 DMEMIT("%u ", test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) +
1715 (m->pg_init_retries > 0) * 2 +
1716 (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 +
1717 test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) +
1718 (m->queue_mode != DM_TYPE_REQUEST_BASED) * 2);
1719
1720 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1721 DMEMIT("queue_if_no_path ");
1722 if (m->pg_init_retries)
1723 DMEMIT("pg_init_retries %u ", m->pg_init_retries);
1724 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT)
1725 DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
1726 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags))
1727 DMEMIT("retain_attached_hw_handler ");
1728 if (m->queue_mode != DM_TYPE_REQUEST_BASED) {
1729 switch(m->queue_mode) {
1730 case DM_TYPE_BIO_BASED:
1731 DMEMIT("queue_mode bio ");
1732 break;
1733 case DM_TYPE_NVME_BIO_BASED:
1734 DMEMIT("queue_mode nvme ");
1735 break;
1736 case DM_TYPE_MQ_REQUEST_BASED:
1737 DMEMIT("queue_mode mq ");
1738 break;
1739 default:
1740 WARN_ON_ONCE(true);
1741 break;
1742 }
1743 }
1744 }
1745
1746 if (!m->hw_handler_name || type == STATUSTYPE_INFO)
1747 DMEMIT("0 ");
1748 else
1749 DMEMIT("1 %s ", m->hw_handler_name);
1750
1751 DMEMIT("%u ", m->nr_priority_groups);
1752
1753 if (m->next_pg)
1754 pg_num = m->next_pg->pg_num;
1755 else if (m->current_pg)
1756 pg_num = m->current_pg->pg_num;
1757 else
1758 pg_num = (m->nr_priority_groups ? 1 : 0);
1759
1760 DMEMIT("%u ", pg_num);
1761
1762 switch (type) {
1763 case STATUSTYPE_INFO:
1764 list_for_each_entry(pg, &m->priority_groups, list) {
1765 if (pg->bypassed)
1766 state = 'D'; /* Disabled */
1767 else if (pg == m->current_pg)
1768 state = 'A'; /* Currently Active */
1769 else
1770 state = 'E'; /* Enabled */
1771
1772 DMEMIT("%c ", state);
1773
1774 if (pg->ps.type->status)
1775 sz += pg->ps.type->status(&pg->ps, NULL, type,
1776 result + sz,
1777 maxlen - sz);
1778 else
1779 DMEMIT("0 ");
1780
1781 DMEMIT("%u %u ", pg->nr_pgpaths,
1782 pg->ps.type->info_args);
1783
1784 list_for_each_entry(p, &pg->pgpaths, list) {
1785 DMEMIT("%s %s %u ", p->path.dev->name,
1786 p->is_active ? "A" : "F",
1787 p->fail_count);
1788 if (pg->ps.type->status)
1789 sz += pg->ps.type->status(&pg->ps,
1790 &p->path, type, result + sz,
1791 maxlen - sz);
1792 }
1793 }
1794 break;
1795
1796 case STATUSTYPE_TABLE:
1797 list_for_each_entry(pg, &m->priority_groups, list) {
1798 DMEMIT("%s ", pg->ps.type->name);
1799
1800 if (pg->ps.type->status)
1801 sz += pg->ps.type->status(&pg->ps, NULL, type,
1802 result + sz,
1803 maxlen - sz);
1804 else
1805 DMEMIT("0 ");
1806
1807 DMEMIT("%u %u ", pg->nr_pgpaths,
1808 pg->ps.type->table_args);
1809
1810 list_for_each_entry(p, &pg->pgpaths, list) {
1811 DMEMIT("%s ", p->path.dev->name);
1812 if (pg->ps.type->status)
1813 sz += pg->ps.type->status(&pg->ps,
1814 &p->path, type, result + sz,
1815 maxlen - sz);
1816 }
1817 }
1818 break;
1819 }
1820
1821 spin_unlock_irqrestore(&m->lock, flags);
1822 }
1823
1824 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
1825 {
1826 int r = -EINVAL;
1827 struct dm_dev *dev;
1828 struct multipath *m = ti->private;
1829 action_fn action;
1830
1831 mutex_lock(&m->work_mutex);
1832
1833 if (dm_suspended(ti)) {
1834 r = -EBUSY;
1835 goto out;
1836 }
1837
1838 if (argc == 1) {
1839 if (!strcasecmp(argv[0], "queue_if_no_path")) {
1840 r = queue_if_no_path(m, true, false);
1841 goto out;
1842 } else if (!strcasecmp(argv[0], "fail_if_no_path")) {
1843 r = queue_if_no_path(m, false, false);
1844 goto out;
1845 }
1846 }
1847
1848 if (argc != 2) {
1849 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc);
1850 goto out;
1851 }
1852
1853 if (!strcasecmp(argv[0], "disable_group")) {
1854 r = bypass_pg_num(m, argv[1], true);
1855 goto out;
1856 } else if (!strcasecmp(argv[0], "enable_group")) {
1857 r = bypass_pg_num(m, argv[1], false);
1858 goto out;
1859 } else if (!strcasecmp(argv[0], "switch_group")) {
1860 r = switch_pg_num(m, argv[1]);
1861 goto out;
1862 } else if (!strcasecmp(argv[0], "reinstate_path"))
1863 action = reinstate_path;
1864 else if (!strcasecmp(argv[0], "fail_path"))
1865 action = fail_path;
1866 else {
1867 DMWARN("Unrecognised multipath message received: %s", argv[0]);
1868 goto out;
1869 }
1870
1871 r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
1872 if (r) {
1873 DMWARN("message: error getting device %s",
1874 argv[1]);
1875 goto out;
1876 }
1877
1878 r = action_dev(m, dev, action);
1879
1880 dm_put_device(ti, dev);
1881
1882 out:
1883 mutex_unlock(&m->work_mutex);
1884 return r;
1885 }
1886
1887 static int multipath_prepare_ioctl(struct dm_target *ti,
1888 struct block_device **bdev, fmode_t *mode)
1889 {
1890 struct multipath *m = ti->private;
1891 struct pgpath *current_pgpath;
1892 int r;
1893
1894 current_pgpath = READ_ONCE(m->current_pgpath);
1895 if (!current_pgpath)
1896 current_pgpath = choose_pgpath(m, 0);
1897
1898 if (current_pgpath) {
1899 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) {
1900 *bdev = current_pgpath->path.dev->bdev;
1901 *mode = current_pgpath->path.dev->mode;
1902 r = 0;
1903 } else {
1904 /* pg_init has not started or completed */
1905 r = -ENOTCONN;
1906 }
1907 } else {
1908 /* No path is available */
1909 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1910 r = -ENOTCONN;
1911 else
1912 r = -EIO;
1913 }
1914
1915 if (r == -ENOTCONN) {
1916 if (!READ_ONCE(m->current_pg)) {
1917 /* Path status changed, redo selection */
1918 (void) choose_pgpath(m, 0);
1919 }
1920 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
1921 pg_init_all_paths(m);
1922 dm_table_run_md_queue_async(m->ti->table);
1923 process_queued_io_list(m);
1924 }
1925
1926 /*
1927 * Only pass ioctls through if the device sizes match exactly.
1928 */
1929 if (!r && ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT)
1930 return 1;
1931 return r;
1932 }
1933
1934 static int multipath_iterate_devices(struct dm_target *ti,
1935 iterate_devices_callout_fn fn, void *data)
1936 {
1937 struct multipath *m = ti->private;
1938 struct priority_group *pg;
1939 struct pgpath *p;
1940 int ret = 0;
1941
1942 list_for_each_entry(pg, &m->priority_groups, list) {
1943 list_for_each_entry(p, &pg->pgpaths, list) {
1944 ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
1945 if (ret)
1946 goto out;
1947 }
1948 }
1949
1950 out:
1951 return ret;
1952 }
1953
1954 static int pgpath_busy(struct pgpath *pgpath)
1955 {
1956 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1957
1958 return blk_lld_busy(q);
1959 }
1960
1961 /*
1962 * We return "busy", only when we can map I/Os but underlying devices
1963 * are busy (so even if we map I/Os now, the I/Os will wait on
1964 * the underlying queue).
1965 * In other words, if we want to kill I/Os or queue them inside us
1966 * due to map unavailability, we don't return "busy". Otherwise,
1967 * dm core won't give us the I/Os and we can't do what we want.
1968 */
1969 static int multipath_busy(struct dm_target *ti)
1970 {
1971 bool busy = false, has_active = false;
1972 struct multipath *m = ti->private;
1973 struct priority_group *pg, *next_pg;
1974 struct pgpath *pgpath;
1975
1976 /* pg_init in progress */
1977 if (atomic_read(&m->pg_init_in_progress))
1978 return true;
1979
1980 /* no paths available, for blk-mq: rely on IO mapping to delay requeue */
1981 if (!atomic_read(&m->nr_valid_paths) && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1982 return (m->queue_mode != DM_TYPE_MQ_REQUEST_BASED);
1983
1984 /* Guess which priority_group will be used at next mapping time */
1985 pg = READ_ONCE(m->current_pg);
1986 next_pg = READ_ONCE(m->next_pg);
1987 if (unlikely(!READ_ONCE(m->current_pgpath) && next_pg))
1988 pg = next_pg;
1989
1990 if (!pg) {
1991 /*
1992 * We don't know which pg will be used at next mapping time.
1993 * We don't call choose_pgpath() here to avoid to trigger
1994 * pg_init just by busy checking.
1995 * So we don't know whether underlying devices we will be using
1996 * at next mapping time are busy or not. Just try mapping.
1997 */
1998 return busy;
1999 }
2000
2001 /*
2002 * If there is one non-busy active path at least, the path selector
2003 * will be able to select it. So we consider such a pg as not busy.
2004 */
2005 busy = true;
2006 list_for_each_entry(pgpath, &pg->pgpaths, list) {
2007 if (pgpath->is_active) {
2008 has_active = true;
2009 if (!pgpath_busy(pgpath)) {
2010 busy = false;
2011 break;
2012 }
2013 }
2014 }
2015
2016 if (!has_active) {
2017 /*
2018 * No active path in this pg, so this pg won't be used and
2019 * the current_pg will be changed at next mapping time.
2020 * We need to try mapping to determine it.
2021 */
2022 busy = false;
2023 }
2024
2025 return busy;
2026 }
2027
2028 /*-----------------------------------------------------------------
2029 * Module setup
2030 *---------------------------------------------------------------*/
2031 static struct target_type multipath_target = {
2032 .name = "multipath",
2033 .version = {1, 12, 0},
2034 .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE,
2035 .module = THIS_MODULE,
2036 .ctr = multipath_ctr,
2037 .dtr = multipath_dtr,
2038 .clone_and_map_rq = multipath_clone_and_map,
2039 .release_clone_rq = multipath_release_clone,
2040 .rq_end_io = multipath_end_io,
2041 .map = multipath_map_bio,
2042 .end_io = multipath_end_io_bio,
2043 .presuspend = multipath_presuspend,
2044 .postsuspend = multipath_postsuspend,
2045 .resume = multipath_resume,
2046 .status = multipath_status,
2047 .message = multipath_message,
2048 .prepare_ioctl = multipath_prepare_ioctl,
2049 .iterate_devices = multipath_iterate_devices,
2050 .busy = multipath_busy,
2051 };
2052
2053 static int __init dm_multipath_init(void)
2054 {
2055 int r;
2056
2057 kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
2058 if (!kmultipathd) {
2059 DMERR("failed to create workqueue kmpathd");
2060 r = -ENOMEM;
2061 goto bad_alloc_kmultipathd;
2062 }
2063
2064 /*
2065 * A separate workqueue is used to handle the device handlers
2066 * to avoid overloading existing workqueue. Overloading the
2067 * old workqueue would also create a bottleneck in the
2068 * path of the storage hardware device activation.
2069 */
2070 kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd",
2071 WQ_MEM_RECLAIM);
2072 if (!kmpath_handlerd) {
2073 DMERR("failed to create workqueue kmpath_handlerd");
2074 r = -ENOMEM;
2075 goto bad_alloc_kmpath_handlerd;
2076 }
2077
2078 r = dm_register_target(&multipath_target);
2079 if (r < 0) {
2080 DMERR("request-based register failed %d", r);
2081 r = -EINVAL;
2082 goto bad_register_target;
2083 }
2084
2085 return 0;
2086
2087 bad_register_target:
2088 destroy_workqueue(kmpath_handlerd);
2089 bad_alloc_kmpath_handlerd:
2090 destroy_workqueue(kmultipathd);
2091 bad_alloc_kmultipathd:
2092 return r;
2093 }
2094
2095 static void __exit dm_multipath_exit(void)
2096 {
2097 destroy_workqueue(kmpath_handlerd);
2098 destroy_workqueue(kmultipathd);
2099
2100 dm_unregister_target(&multipath_target);
2101 }
2102
2103 module_init(dm_multipath_init);
2104 module_exit(dm_multipath_exit);
2105
2106 MODULE_DESCRIPTION(DM_NAME " multipath target");
2107 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
2108 MODULE_LICENSE("GPL");
CRIS linux kernel tree