drivers/mmc/host/mmc_hsq.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *
   4  * MMC software queue support based on command queue interfaces
   5  *
   6  * Copyright (C) 2019 Linaro, Inc.
   7  * Author: Baolin Wang <baolin.wang@linaro.org>
   8  */
   9
  10 #include <linux/mmc/card.h>
  11 #include <linux/mmc/host.h>
  12 #include <linux/module.h>
  13
  14 #include "mmc_hsq.h"
  15
  16 static void mmc_hsq_retry_handler(struct work_struct *work)
  17 {
  18         struct mmc_hsq *hsq = container_of(work, struct mmc_hsq, retry_work);
  19         struct mmc_host *mmc = hsq->mmc;
  20
  21         mmc->ops->request(mmc, hsq->mrq);
  22 }
  23
  24 static void mmc_hsq_modify_threshold(struct mmc_hsq *hsq)
  25 {
  26         struct mmc_host *mmc = hsq->mmc;
  27         struct mmc_request *mrq;
  28         unsigned int tag, need_change = 0;
  29
  30         mmc->hsq_depth = HSQ_NORMAL_DEPTH;
  31         for (tag = 0; tag < HSQ_NUM_SLOTS; tag++) {
  32                 mrq = hsq->slot[tag].mrq;
  33                 if (mrq && mrq->data &&
  34                    (mrq->data->blksz * mrq->data->blocks == 4096) &&
  35                    (mrq->data->flags & MMC_DATA_WRITE) &&
  36                    (++need_change == 2)) {
  37                         mmc->hsq_depth = HSQ_PERFORMANCE_DEPTH;
  38                         break;
  39                 }
  40         }
  41 }
  42
  43 static void mmc_hsq_pump_requests(struct mmc_hsq *hsq)
  44 {
  45         struct mmc_host *mmc = hsq->mmc;
  46         struct hsq_slot *slot;
  47         unsigned long flags;
  48         int ret = 0;
  49
  50         spin_lock_irqsave(&hsq->lock, flags);
  51
  52         /* Make sure we are not already running a request now */
  53         if (hsq->mrq || hsq->recovery_halt) {
  54                 spin_unlock_irqrestore(&hsq->lock, flags);
  55                 return;
  56         }
  57
  58         /* Make sure there are remain requests need to pump */
  59         if (!hsq->qcnt || !hsq->enabled) {
  60                 spin_unlock_irqrestore(&hsq->lock, flags);
  61                 return;
  62         }
  63
  64         mmc_hsq_modify_threshold(hsq);
  65
  66         slot = &hsq->slot[hsq->next_tag];
  67         hsq->mrq = slot->mrq;
  68         hsq->qcnt--;
  69
  70         spin_unlock_irqrestore(&hsq->lock, flags);
  71
  72         if (mmc->ops->request_atomic)
  73                 ret = mmc->ops->request_atomic(mmc, hsq->mrq);
  74         else
  75                 mmc->ops->request(mmc, hsq->mrq);
  76
  77         /*
  78          * If returning BUSY from request_atomic(), which means the card
  79          * may be busy now, and we should change to non-atomic context to
  80          * try again for this unusual case, to avoid time-consuming operations
  81          * in the atomic context.
  82          *
  83          * Note: we just give a warning for other error cases, since the host
  84          * driver will handle them.
  85          */
  86         if (ret == -EBUSY)
  87                 schedule_work(&hsq->retry_work);
  88         else
  89                 WARN_ON_ONCE(ret);
  90 }
  91
  92 static void mmc_hsq_update_next_tag(struct mmc_hsq *hsq, int remains)
  93 {
  94         int tag;
  95
  96         /*
  97          * If there are no remain requests in software queue, then set a invalid
  98          * tag.
  99          */
 100         if (!remains) {
 101                 hsq->next_tag = HSQ_INVALID_TAG;
 102                 hsq->tail_tag = HSQ_INVALID_TAG;
 103                 return;
 104         }
 105
 106         tag = hsq->tag_slot[hsq->next_tag];
 107         hsq->tag_slot[hsq->next_tag] = HSQ_INVALID_TAG;
 108         hsq->next_tag = tag;
 109 }
 110
 111 static void mmc_hsq_post_request(struct mmc_hsq *hsq)
 112 {
 113         unsigned long flags;
 114         int remains;
 115
 116         spin_lock_irqsave(&hsq->lock, flags);
 117
 118         remains = hsq->qcnt;
 119         hsq->mrq = NULL;
 120
 121         /* Update the next available tag to be queued. */
 122         mmc_hsq_update_next_tag(hsq, remains);
 123
 124         if (hsq->waiting_for_idle && !remains) {
 125                 hsq->waiting_for_idle = false;
 126                 wake_up(&hsq->wait_queue);
 127         }
 128
 129         /* Do not pump new request in recovery mode. */
 130         if (hsq->recovery_halt) {
 131                 spin_unlock_irqrestore(&hsq->lock, flags);
 132                 return;
 133         }
 134
 135         spin_unlock_irqrestore(&hsq->lock, flags);
 136
 137          /*
 138           * Try to pump new request to host controller as fast as possible,
 139           * after completing previous request.
 140           */
 141         if (remains > 0)
 142                 mmc_hsq_pump_requests(hsq);
 143 }
 144
 145 /**
 146  * mmc_hsq_finalize_request - finalize one request if the request is done
 147  * @mmc: the host controller
 148  * @mrq: the request need to be finalized
 149  *
 150  * Return true if we finalized the corresponding request in software queue,
 151  * otherwise return false.
 152  */
 153 bool mmc_hsq_finalize_request(struct mmc_host *mmc, struct mmc_request *mrq)
 154 {
 155         struct mmc_hsq *hsq = mmc->cqe_private;
 156         unsigned long flags;
 157
 158         spin_lock_irqsave(&hsq->lock, flags);
 159
 160         if (!hsq->enabled || !hsq->mrq || hsq->mrq != mrq) {
 161                 spin_unlock_irqrestore(&hsq->lock, flags);
 162                 return false;
 163         }
 164
 165         /*
 166          * Clear current completed slot request to make a room for new request.
 167          */
 168         hsq->slot[hsq->next_tag].mrq = NULL;
 169
 170         spin_unlock_irqrestore(&hsq->lock, flags);
 171
 172         mmc_cqe_request_done(mmc, hsq->mrq);
 173
 174         mmc_hsq_post_request(hsq);
 175
 176         return true;
 177 }
 178 EXPORT_SYMBOL_GPL(mmc_hsq_finalize_request);
 179
 180 static void mmc_hsq_recovery_start(struct mmc_host *mmc)
 181 {
 182         struct mmc_hsq *hsq = mmc->cqe_private;
 183         unsigned long flags;
 184
 185         spin_lock_irqsave(&hsq->lock, flags);
 186
 187         hsq->recovery_halt = true;
 188
 189         spin_unlock_irqrestore(&hsq->lock, flags);
 190 }
 191
 192 static void mmc_hsq_recovery_finish(struct mmc_host *mmc)
 193 {
 194         struct mmc_hsq *hsq = mmc->cqe_private;
 195         int remains;
 196
 197         spin_lock_irq(&hsq->lock);
 198
 199         hsq->recovery_halt = false;
 200         remains = hsq->qcnt;
 201
 202         spin_unlock_irq(&hsq->lock);
 203
 204         /*
 205          * Try to pump new request if there are request pending in software
 206          * queue after finishing recovery.
 207          */
 208         if (remains > 0)
 209                 mmc_hsq_pump_requests(hsq);
 210 }
 211
 212 static int mmc_hsq_request(struct mmc_host *mmc, struct mmc_request *mrq)
 213 {
 214         struct mmc_hsq *hsq = mmc->cqe_private;
 215         int tag = mrq->tag;
 216
 217         spin_lock_irq(&hsq->lock);
 218
 219         if (!hsq->enabled) {
 220                 spin_unlock_irq(&hsq->lock);
 221                 return -ESHUTDOWN;
 222         }
 223
 224         /* Do not queue any new requests in recovery mode. */
 225         if (hsq->recovery_halt) {
 226                 spin_unlock_irq(&hsq->lock);
 227                 return -EBUSY;
 228         }
 229
 230         hsq->slot[tag].mrq = mrq;
 231
 232         /*
 233          * Set the next tag as current request tag if no available
 234          * next tag.
 235          */
 236         if (hsq->next_tag == HSQ_INVALID_TAG) {
 237                 hsq->next_tag = tag;
 238                 hsq->tail_tag = tag;
 239                 hsq->tag_slot[hsq->tail_tag] = HSQ_INVALID_TAG;
 240         } else {
 241                 hsq->tag_slot[hsq->tail_tag] = tag;
 242                 hsq->tail_tag = tag;
 243         }
 244
 245         hsq->qcnt++;
 246
 247         spin_unlock_irq(&hsq->lock);
 248
 249         mmc_hsq_pump_requests(hsq);
 250
 251         return 0;
 252 }
 253
 254 static void mmc_hsq_post_req(struct mmc_host *mmc, struct mmc_request *mrq)
 255 {
 256         if (mmc->ops->post_req)
 257                 mmc->ops->post_req(mmc, mrq, 0);
 258 }
 259
 260 static bool mmc_hsq_queue_is_idle(struct mmc_hsq *hsq, int *ret)
 261 {
 262         bool is_idle;
 263
 264         spin_lock_irq(&hsq->lock);
 265
 266         is_idle = (!hsq->mrq && !hsq->qcnt) ||
 267                 hsq->recovery_halt;
 268
 269         *ret = hsq->recovery_halt ? -EBUSY : 0;
 270         hsq->waiting_for_idle = !is_idle;
 271
 272         spin_unlock_irq(&hsq->lock);
 273
 274         return is_idle;
 275 }
 276
 277 static int mmc_hsq_wait_for_idle(struct mmc_host *mmc)
 278 {
 279         struct mmc_hsq *hsq = mmc->cqe_private;
 280         int ret;
 281
 282         wait_event(hsq->wait_queue,
 283                    mmc_hsq_queue_is_idle(hsq, &ret));
 284
 285         return ret;
 286 }
 287
 288 static void mmc_hsq_disable(struct mmc_host *mmc)
 289 {
 290         struct mmc_hsq *hsq = mmc->cqe_private;
 291         u32 timeout = 500;
 292         int ret;
 293
 294         spin_lock_irq(&hsq->lock);
 295
 296         if (!hsq->enabled) {
 297                 spin_unlock_irq(&hsq->lock);
 298                 return;
 299         }
 300
 301         spin_unlock_irq(&hsq->lock);
 302
 303         ret = wait_event_timeout(hsq->wait_queue,
 304                                  mmc_hsq_queue_is_idle(hsq, &ret),
 305                                  msecs_to_jiffies(timeout));
 306         if (ret == 0) {
 307                 pr_warn("could not stop mmc software queue\n");
 308                 return;
 309         }
 310
 311         spin_lock_irq(&hsq->lock);
 312
 313         hsq->enabled = false;
 314
 315         spin_unlock_irq(&hsq->lock);
 316 }
 317
 318 static int mmc_hsq_enable(struct mmc_host *mmc, struct mmc_card *card)
 319 {
 320         struct mmc_hsq *hsq = mmc->cqe_private;
 321
 322         spin_lock_irq(&hsq->lock);
 323
 324         if (hsq->enabled) {
 325                 spin_unlock_irq(&hsq->lock);
 326                 return -EBUSY;
 327         }
 328
 329         hsq->enabled = true;
 330
 331         spin_unlock_irq(&hsq->lock);
 332
 333         return 0;
 334 }
 335
 336 static const struct mmc_cqe_ops mmc_hsq_ops = {
 337         .cqe_enable = mmc_hsq_enable,
 338         .cqe_disable = mmc_hsq_disable,
 339         .cqe_request = mmc_hsq_request,
 340         .cqe_post_req = mmc_hsq_post_req,
 341         .cqe_wait_for_idle = mmc_hsq_wait_for_idle,
 342         .cqe_recovery_start = mmc_hsq_recovery_start,
 343         .cqe_recovery_finish = mmc_hsq_recovery_finish,
 344 };
 345
 346 int mmc_hsq_init(struct mmc_hsq *hsq, struct mmc_host *mmc)
 347 {
 348         int i;
 349         hsq->num_slots = HSQ_NUM_SLOTS;
 350         hsq->next_tag = HSQ_INVALID_TAG;
 351         hsq->tail_tag = HSQ_INVALID_TAG;
 352
 353         hsq->slot = devm_kcalloc(mmc_dev(mmc), hsq->num_slots,
 354                                  sizeof(struct hsq_slot), GFP_KERNEL);
 355         if (!hsq->slot)
 356                 return -ENOMEM;
 357
 358         hsq->mmc = mmc;
 359         hsq->mmc->cqe_private = hsq;
 360         mmc->cqe_ops = &mmc_hsq_ops;
 361         mmc->hsq_depth = HSQ_NORMAL_DEPTH;
 362
 363         for (i = 0; i < HSQ_NUM_SLOTS; i++)
 364                 hsq->tag_slot[i] = HSQ_INVALID_TAG;
 365
 366         INIT_WORK(&hsq->retry_work, mmc_hsq_retry_handler);
 367         spin_lock_init(&hsq->lock);
 368         init_waitqueue_head(&hsq->wait_queue);
 369
 370         return 0;
 371 }
 372 EXPORT_SYMBOL_GPL(mmc_hsq_init);
 373
 374 void mmc_hsq_suspend(struct mmc_host *mmc)
 375 {
 376         mmc_hsq_disable(mmc);
 377 }
 378 EXPORT_SYMBOL_GPL(mmc_hsq_suspend);
 379
 380 int mmc_hsq_resume(struct mmc_host *mmc)
 381 {
 382         return mmc_hsq_enable(mmc, NULL);
 383 }
 384 EXPORT_SYMBOL_GPL(mmc_hsq_resume);
 385
 386 MODULE_DESCRIPTION("MMC Host Software Queue support");
 387 MODULE_LICENSE("GPL v2");