| blob | 52740b60d78692ade175244dbd0c38ebe684f0ba |
1 /*
2 * Universal Flash Storage Host controller driver Core
3 *
4 * This code is based on drivers/scsi/ufs/ufshcd.c
5 * Copyright (C) 2011-2013 Samsung India Software Operations
6 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7 *
8 * Authors:
9 * Santosh Yaraganavi <santosh.sy@samsung.com>
10 * Vinayak Holikatti <h.vinayak@samsung.com>
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 * See the COPYING file in the top-level directory or visit
17 * <http://www.gnu.org/licenses/gpl-2.0.html>
18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 *
24 * This program is provided "AS IS" and "WITH ALL FAULTS" and
25 * without warranty of any kind. You are solely responsible for
26 * determining the appropriateness of using and distributing
27 * the program and assume all risks associated with your exercise
28 * of rights with respect to the program, including but not limited
29 * to infringement of third party rights, the risks and costs of
30 * program errors, damage to or loss of data, programs or equipment,
31 * and unavailability or interruption of operations. Under no
32 * circumstances will the contributor of this Program be liable for
33 * any damages of any kind arising from your use or distribution of
34 * this program.
35 *
36 * The Linux Foundation chooses to take subject only to the GPLv2
37 * license terms, and distributes only under these terms.
38 */
40 #include <linux/async.h>
41 #include <linux/devfreq.h>
42 #include <linux/nls.h>
43 #include <linux/of.h>
44 #include <linux/bitfield.h>
45 #include <linux/blk-pm.h>
52 #define CREATE_TRACE_POINTS
53 #include <trace/events/ufs.h>
55 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
56 UTP_TASK_REQ_COMPL |\
57 UFSHCD_ERROR_MASK)
58 /* UIC command timeout, unit: ms */
59 #define UIC_CMD_TIMEOUT 500
61 /* NOP OUT retries waiting for NOP IN response */
62 #define NOP_OUT_RETRIES 10
63 /* Timeout after 30 msecs if NOP OUT hangs without response */
66 /* Query request retries */
67 #define QUERY_REQ_RETRIES 3
68 /* Query request timeout */
71 /* Task management command timeout */
74 /* maximum number of retries for a general UIC command */
75 #define UFS_UIC_COMMAND_RETRIES 3
77 /* maximum number of link-startup retries */
78 #define DME_LINKSTARTUP_RETRIES 3
80 /* Maximum retries for Hibern8 enter */
81 #define UIC_HIBERN8_ENTER_RETRIES 3
83 /* maximum number of reset retries before giving up */
84 #define MAX_HOST_RESET_RETRIES 5
86 /* Expose the flag value from utp_upiu_query.value */
87 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
89 /* Interrupt aggregation default timeout, unit: 40us */
90 #define INT_AGGR_DEF_TO 0x02
92 /* default delay of autosuspend: 2000 ms */
93 #define RPM_AUTOSUSPEND_DELAY_MS 2000
95 /* Default value of wait time before gating device ref clock */
98 #define ufshcd_toggle_vreg(_dev, _vreg, _on) \
99 ({ \
100 int _ret; \
101 if (_on) \
102 _ret = ufshcd_enable_vreg(_dev, _vreg); \
103 else \
104 _ret = ufshcd_disable_vreg(_dev, _vreg); \
105 _ret; \
106 })
108 #define ufshcd_hex_dump(prefix_str, buf, len) do { \
109 size_t __len = (len); \
110 print_hex_dump(KERN_ERR, prefix_str, \
111 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
112 16, 4, buf, __len, false); \
113 } while (0)
117 {
135 }
143 };
145 /* UFSHCD states */
147 UFSHCD_STATE_RESET,
148 UFSHCD_STATE_ERROR,
149 UFSHCD_STATE_OPERATIONAL,
150 UFSHCD_STATE_EH_SCHEDULED,
151 };
153 /* UFSHCD error handling flags */
156 };
158 /* UFSHCD UIC layer error flags */
166 };
168 #define ufshcd_set_eh_in_progress(h) \
169 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
170 #define ufshcd_eh_in_progress(h) \
171 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
172 #define ufshcd_clear_eh_in_progress(h) \
173 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
182 };
186 {
188 }
192 {
194 }
199 {
206 }
208 /* if no match found, return the level 0 */
210 }
213 /* UFS cards deviations table */
215 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
217 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
219 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
221 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
223 UFS_DEVICE_QUIRK_PA_TACTIVATE),
225 UFS_DEVICE_QUIRK_PA_TACTIVATE),
227 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
229 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
231 END_FIX
232 };
255 {
257 }
260 {
264 }
265 }
268 {
272 }
273 }
276 {
279 }
282 {
285 }
289 {
293 }
297 {
301 }
305 {
311 }
315 {
324 /* trace UPIU W/O tracing command */
328 }
331 /* trace UPIU also */
335 /*
336 * Currently we only fully trace read(10) and write(10)
337 * commands
338 */
343 }
344 }
350 }
353 {
365 }
366 }
371 {
383 }
387 }
390 {
422 }
424 static
426 {
463 }
464 }
467 {
475 }
476 }
479 {
498 }
500 /**
501 * ufshcd_print_pwr_info - print power params as saved in hba
502 * power info
503 * @hba: per-adapter instance
504 */
506 {
515 };
518 __func__,
524 }
527 {
533 else
535 }
538 /*
539 * ufshcd_wait_for_register - wait for register value to change
540 * @hba - per-adapter interface
541 * @reg - mmio register offset
542 * @mask - mask to apply to read register value
543 * @val - wait condition
544 * @interval_us - polling interval in microsecs
545 * @timeout_ms - timeout in millisecs
546 * @can_sleep - perform sleep or just spin
547 *
548 * Returns -ETIMEDOUT on error, zero on success
549 */
553 {
557 /* ignore bits that we don't intend to wait on */
563 else
569 }
570 }
573 }
575 /**
576 * ufshcd_get_intr_mask - Get the interrupt bit mask
577 * @hba: Pointer to adapter instance
578 *
579 * Returns interrupt bit mask per version
580 */
582 {
597 }
600 }
602 /**
603 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
604 * @hba: Pointer to adapter instance
605 *
606 * Returns UFSHCI version supported by the controller
607 */
609 {
614 }
616 /**
617 * ufshcd_is_device_present - Check if any device connected to
618 * the host controller
619 * @hba: pointer to adapter instance
620 *
621 * Returns true if device present, false if no device detected
622 */
624 {
627 }
629 /**
630 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
631 * @lrbp: pointer to local command reference block
632 *
633 * This function is used to get the OCS field from UTRD
634 * Returns the OCS field in the UTRD
635 */
637 {
639 }
641 /**
642 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
643 * @hba: per adapter instance
644 * @pos: position of the bit to be cleared
645 */
647 {
649 }
651 /**
652 * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
653 * @hba: per adapter instance
654 * @pos: position of the bit to be cleared
655 */
657 {
659 }
661 /**
662 * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
663 * @hba: per adapter instance
664 * @tag: position of the bit to be cleared
665 */
667 {
669 }
671 /**
672 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
673 * @reg: Register value of host controller status
674 *
675 * Returns integer, 0 on Success and positive value if failed
676 */
678 {
680 }
682 /**
683 * ufshcd_get_uic_cmd_result - Get the UIC command result
684 * @hba: Pointer to adapter instance
685 *
686 * This function gets the result of UIC command completion
687 * Returns 0 on success, non zero value on error
688 */
690 {
692 MASK_UIC_COMMAND_RESULT;
693 }
695 /**
696 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
697 * @hba: Pointer to adapter instance
698 *
699 * This function gets UIC command argument3
700 * Returns 0 on success, non zero value on error
701 */
703 {
705 }
707 /**
708 * ufshcd_get_req_rsp - returns the TR response transaction type
709 * @ucd_rsp_ptr: pointer to response UPIU
710 */
713 {
715 }
717 /**
718 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
719 * @ucd_rsp_ptr: pointer to response UPIU
720 *
721 * This function gets the response status and scsi_status from response UPIU
722 * Returns the response result code.
723 */
726 {
728 }
730 /*
731 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
732 * from response UPIU
733 * @ucd_rsp_ptr: pointer to response UPIU
734 *
735 * Return the data segment length.
736 */
739 {
741 MASK_RSP_UPIU_DATA_SEG_LEN;
742 }
744 /**
745 * ufshcd_is_exception_event - Check if the device raised an exception event
746 * @ucd_rsp_ptr: pointer to response UPIU
747 *
748 * The function checks if the device raised an exception event indicated in
749 * the Device Information field of response UPIU.
750 *
751 * Returns true if exception is raised, false otherwise.
752 */
754 {
757 }
759 /**
760 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
761 * @hba: per adapter instance
762 */
765 {
767 INT_AGGR_COUNTER_AND_TIMER_RESET,
768 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
769 }
771 /**
772 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
773 * @hba: per adapter instance
774 * @cnt: Interrupt aggregation counter threshold
775 * @tmout: Interrupt aggregation timeout value
776 */
779 {
783 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
784 }
786 /**
787 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
788 * @hba: per adapter instance
789 */
791 {
793 }
795 /**
796 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
797 * When run-stop registers are set to 1, it indicates the
798 * host controller that it can process the requests
799 * @hba: per adapter instance
800 */
802 {
804 REG_UTP_TASK_REQ_LIST_RUN_STOP);
806 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
807 }
809 /**
810 * ufshcd_hba_start - Start controller initialization sequence
811 * @hba: per adapter instance
812 */
814 {
816 }
818 /**
819 * ufshcd_is_hba_active - Get controller state
820 * @hba: per adapter instance
821 *
822 * Returns false if controller is active, true otherwise
823 */
825 {
828 }
831 {
832 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
836 else
838 }
842 {
843 /*
844 * If both host and device support UniPro ver1.6 or later, PA layer
845 * parameters tuning happens during link startup itself.
846 *
847 * We can manually tune PA layer parameters if either host or device
848 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
849 * logic simple, we will only do manual tuning if local unipro version
850 * doesn't support ver1.6 or later.
851 */
854 else
856 }
858 /**
859 * ufshcd_set_clk_freq - set UFS controller clock frequencies
860 * @hba: per adapter instance
861 * @scale_up: If True, set max possible frequency othewise set low frequency
862 *
863 * Returns 0 if successful
864 * Returns < 0 for any other errors
865 */
867 {
887 }
905 }
911 }
912 }
915 }
917 out:
919 }
921 /**
922 * ufshcd_scale_clks - scale up or scale down UFS controller clocks
923 * @hba: per adapter instance
924 * @scale_up: True if scaling up and false if scaling down
925 *
926 * Returns 0 if successful
927 * Returns < 0 for any other errors
928 */
930 {
946 out:
951 }
953 /**
954 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
955 * @hba: per adapter instance
956 * @scale_up: True if scaling up and false if scaling down
957 *
958 * Returns true if scaling is required, false otherwise.
959 */
962 {
979 }
980 }
981 }
984 }
987 u64 wait_timeout_us)
988 {
991 u32 tm_doorbell;
992 u32 tr_doorbell;
994 ktime_t start;
998 /*
999 * Wait for all the outstanding tasks/transfer requests.
1000 * Verify by checking the doorbell registers are clear.
1001 */
1007 }
1016 }
1021 wait_timeout_us) {
1023 /*
1024 * We might have scheduled out for long time so make
1025 * sure to check if doorbells are cleared by this time
1026 * or not.
1027 */
1029 }
1038 }
1039 out:
1043 }
1045 /**
1046 * ufshcd_scale_gear - scale up/down UFS gear
1047 * @hba: per adapter instance
1048 * @scale_up: True for scaling up gear and false for scaling down
1049 *
1050 * Returns 0 for success,
1051 * Returns -EBUSY if scaling can't happen at this time
1052 * Returns non-zero for any other errors
1053 */
1055 {
1056 #define UFS_MIN_GEAR_TO_SCALE_DOWN UFS_HS_G1
1069 /* save the current power mode */
1074 /* scale down gear */
1077 }
1078 }
1080 /* check if the power mode needs to be changed or not? */
1089 }
1092 {
1095 /*
1096 * make sure that there are no outstanding requests when
1097 * clock scaling is in progress
1098 */
1105 }
1108 }
1111 {
1114 }
1116 /**
1117 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1118 * @hba: per adapter instance
1119 * @scale_up: True for scaling up and false for scalin down
1120 *
1121 * Returns 0 for success,
1122 * Returns -EBUSY if scaling can't happen at this time
1123 * Returns non-zero for any other errors
1124 */
1126 {
1129 /* let's not get into low power until clock scaling is completed */
1136 /* scale down the gear before scaling down clocks */
1141 }
1148 }
1150 /* scale up the gear after scaling up clocks */
1155 }
1157 out_unprepare:
1159 out:
1162 }
1165 {
1174 }
1179 }
1182 {
1191 }
1196 }
1200 {
1203 ktime_t start;
1213 /* Override with the closest supported frequency */
1219 }
1227 }
1229 /* Decide based on the rounded-off frequency and update */
1233 /* Update the frequency */
1238 }
1248 out:
1254 }
1257 {
1263 }
1265 /* Whether or not any tag is in use by a request that is in progress. */
1267 {
1273 }
1277 {
1294 /*
1295 * If current frequency is 0, then the ondemand governor considers
1296 * there's no initial frequency set. And it always requests to set
1297 * to max. frequency.
1298 */
1307 start_window:
1317 }
1320 }
1326 };
1328 #if IS_ENABLED(CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND)
1332 };
1335 #else
1337 #endif
1340 {
1346 /* Skip devfreq if we don't have any clocks in the list */
1355 gov_data);
1358 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1359 gov_data);
1367 }
1372 }
1375 {
1388 }
1391 {
1398 }
1401 {
1412 }
1417 }
1420 {
1431 }
1436 }
1440 {
1444 }
1448 {
1450 u32 value;
1476 }
1480 out:
1482 }
1485 {
1493 }
1496 {
1508 }
1515 /* Exit from hibern8 */
1517 /* Prevent gating in this path */
1524 else
1526 }
1528 }
1529 unblock_reqs:
1531 }
1533 /**
1534 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1535 * Also, exit from hibern8 mode and set the link as active.
1536 * @hba: per adapter instance
1537 * @async: This indicates whether caller should ungate clocks asynchronously.
1538 */
1540 {
1552 }
1554 start:
1557 /*
1558 * Wait for the ungate work to complete if in progress.
1559 * Though the clocks may be in ON state, the link could
1560 * still be in hibner8 state if hibern8 is allowed
1561 * during clock gating.
1562 * Make sure we exit hibern8 state also in addition to
1563 * clocks being ON.
1564 */
1571 }
1576 }
1584 }
1585 /*
1586 * If we are here, it means gating work is either done or
1587 * currently running. Hence, fall through to cancel gating
1588 * work and to enable clocks.
1589 */
1590 /* fallthrough */
1598 /*
1599 * fall through to check if we should wait for this
1600 * work to be done or not.
1601 */
1602 /* fallthrough */
1608 }
1612 /* Make sure state is CLKS_ON before returning */
1619 }
1621 out:
1623 }
1627 {
1633 /*
1634 * In case you are here to cancel this work the gating state
1635 * would be marked as REQ_CLKS_ON. In this case save time by
1636 * skipping the gating work and exit after changing the clock
1637 * state to CLKS_ON.
1638 */
1645 }
1655 /* put the link into hibern8 mode before turning off clocks */
1662 }
1664 }
1670 else
1671 /* If link is active, device ref_clk can't be switched off */
1674 /*
1675 * In case you are here to cancel this work the gating state
1676 * would be marked as REQ_CLKS_ON. In this case keep the state
1677 * as REQ_CLKS_ON which would anyway imply that clocks are off
1678 * and a request to turn them on is pending. By doing this way,
1679 * we keep the state machine in tact and this would ultimately
1680 * prevent from doing cancel work multiple times when there are
1681 * new requests arriving before the current cancel work is done.
1682 */
1688 }
1689 rel_lock:
1691 out:
1693 }
1695 /* host lock must be held before calling this variant */
1697 {
1715 }
1718 {
1724 }
1729 {
1733 }
1737 {
1748 }
1752 {
1756 }
1760 {
1763 u32 value;
1778 }
1781 out:
1783 }
1786 {
1793 ufshcd_clk_scaling_suspend_work);
1795 ufshcd_clk_scaling_resume_work);
1802 }
1805 {
1811 }
1814 {
1827 WQ_MEM_RECLAIM);
1846 }
1849 {
1857 }
1859 /* Must be called with host lock acquired */
1861 {
1881 }
1886 }
1887 }
1890 {
1901 }
1902 }
1903 /**
1904 * ufshcd_send_command - Send SCSI or device management commands
1905 * @hba: per adapter instance
1906 * @task_tag: Task tag of the command
1907 */
1910 {
1917 /* Make sure that doorbell is committed immediately */
1919 }
1921 /**
1922 * ufshcd_copy_sense_data - Copy sense data in case of check condition
1923 * @lrbp: pointer to local reference block
1924 */
1926 {
1936 len_to_copy);
1937 }
1938 }
1940 /**
1941 * ufshcd_copy_query_response() - Copy the Query Response and the data
1942 * descriptor
1943 * @hba: per adapter instance
1944 * @lrbp: pointer to local reference block
1945 */
1946 static
1948 {
1953 /* Get the descriptor */
1957 GENERAL_UPIU_REQUEST_SIZE;
1958 u16 resp_len;
1959 u16 buf_len;
1961 /* data segment length */
1963 MASK_QUERY_DATA_SEG_LEN;
1973 }
1974 }
1977 }
1979 /**
1980 * ufshcd_hba_capabilities - Read controller capabilities
1981 * @hba: per adapter instance
1982 */
1984 {
1987 /* nutrs and nutmrs are 0 based values */
1991 }
1993 /**
1994 * ufshcd_ready_for_uic_cmd - Check if controller is ready
1995 * to accept UIC commands
1996 * @hba: per adapter instance
1997 * Return true on success, else false
1998 */
2000 {
2003 else
2005 }
2007 /**
2008 * ufshcd_get_upmcrs - Get the power mode change request status
2009 * @hba: Pointer to adapter instance
2010 *
2011 * This function gets the UPMCRS field of HCS register
2012 * Returns value of UPMCRS field
2013 */
2015 {
2017 }
2019 /**
2020 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
2021 * @hba: per adapter instance
2022 * @uic_cmd: UIC command
2023 *
2024 * Mutex must be held.
2025 */
2028 {
2033 /* Write Args */
2038 /* Write UIC Cmd */
2040 REG_UIC_COMMAND);
2041 }
2043 /**
2044 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2045 * @hba: per adapter instance
2046 * @uic_cmd: UIC command
2047 *
2048 * Must be called with mutex held.
2049 * Returns 0 only if success.
2050 */
2051 static int
2053 {
2060 else
2068 }
2070 /**
2071 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2072 * @hba: per adapter instance
2073 * @uic_cmd: UIC command
2074 * @completion: initialize the completion only if this is set to true
2075 *
2076 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
2077 * with mutex held and host_lock locked.
2078 * Returns 0 only if success.
2079 */
2080 static int
2083 {
2088 }
2096 }
2098 /**
2099 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2100 * @hba: per adapter instance
2101 * @uic_cmd: UIC command
2102 *
2103 * Returns 0 only if success.
2104 */
2106 {
2124 }
2126 /**
2127 * ufshcd_map_sg - Map scatter-gather list to prdt
2128 * @hba: per adapter instance
2129 * @lrbp: pointer to local reference block
2130 *
2131 * Returns 0 in case of success, non-zero value in case of failure
2132 */
2134 {
2160 }
2163 }
2166 }
2168 /**
2169 * ufshcd_enable_intr - enable interrupts
2170 * @hba: per adapter instance
2171 * @intrs: interrupt bits
2172 */
2174 {
2178 u32 rw;
2183 }
2186 }
2188 /**
2189 * ufshcd_disable_intr - disable interrupts
2190 * @hba: per adapter instance
2191 * @intrs: interrupt bits
2192 */
2194 {
2198 u32 rw;
2205 }
2208 }
2210 /**
2211 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2212 * descriptor according to request
2213 * @lrbp: pointer to local reference block
2214 * @upiu_flags: flags required in the header
2215 * @cmd_dir: requests data direction
2216 */
2219 {
2221 u32 data_direction;
2222 u32 dword_0;
2233 }
2240 /* Transfer request descriptor header fields */
2242 /* dword_1 is reserved, hence it is set to 0 */
2244 /*
2245 * assigning invalid value for command status. Controller
2246 * updates OCS on command completion, with the command
2247 * status
2248 */
2251 /* dword_3 is reserved, hence it is set to 0 */
2255 }
2257 /**
2258 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2259 * for scsi commands
2260 * @lrbp: local reference block pointer
2261 * @upiu_flags: flags
2262 */
2263 static
2265 {
2270 /* command descriptor fields */
2277 /* Total EHS length and Data segment length will be zero */
2287 }
2289 /**
2290 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2291 * for query requsts
2292 * @hba: UFS hba
2293 * @lrbp: local reference block pointer
2294 * @upiu_flags: flags
2295 */
2298 {
2303 /* Query request header */
2310 /* Data segment length only need for WRITE_DESC */
2314 else
2317 /* Copy the Query Request buffer as is */
2319 QUERY_OSF_SIZE);
2321 /* Copy the Descriptor */
2326 }
2329 {
2334 /* command descriptor fields */
2338 /* clear rest of the fields of basic header */
2343 }
2345 /**
2346 * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
2347 * for Device Management Purposes
2348 * @hba: per adapter instance
2349 * @lrbp: pointer to local reference block
2350 */
2352 {
2353 u32 upiu_flags;
2359 else
2367 else
2371 }
2373 /**
2374 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2375 * for SCSI Purposes
2376 * @hba: per adapter instance
2377 * @lrbp: pointer to local reference block
2378 */
2380 {
2381 u32 upiu_flags;
2387 else
2396 }
2399 }
2401 /**
2402 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2403 * @upiu_wlun_id: UPIU W-LUN id
2404 *
2405 * Returns SCSI W-LUN id
2406 */
2408 {
2410 }
2413 {
2419 response_upiu);
2431 }
2433 /**
2434 * ufshcd_queuecommand - main entry point for SCSI requests
2435 * @host: SCSI host pointer
2436 * @cmd: command from SCSI Midlayer
2437 *
2438 * Returns 0 for success, non-zero in case of failure
2439 */
2441 {
2456 }
2479 }
2481 /* if error handling is in progress, don't issue commands */
2486 }
2495 }
2516 }
2517 /* Make sure descriptors are ready before ringing the doorbell */
2520 /* issue command to the controller */
2524 out_unlock:
2526 out:
2529 }
2533 {
2543 }
2545 static int
2547 {
2552 /* clear outstanding transaction before retry */
2557 /*
2558 * wait for for h/w to clear corresponding bit in door-bell.
2559 * max. wait is 1 sec.
2560 */
2562 REG_UTP_TRANSFER_REQ_DOOR_BELL,
2566 }
2568 static int
2570 {
2573 /* Get the UPIU response */
2575 UPIU_RSP_CODE_OFFSET;
2577 }
2579 /**
2580 * ufshcd_dev_cmd_completion() - handles device management command responses
2581 * @hba: per adapter instance
2582 * @lrbp: pointer to local reference block
2583 */
2584 static int
2586 {
2599 }
2607 /* TODO: handle Reject UPIU Response */
2610 __func__);
2617 }
2620 }
2624 {
2632 /* Make sure descriptors are ready before ringing the doorbell */
2640 }
2648 /* successfully cleared the command, retry if needed */
2650 /*
2651 * in case of an error, after clearing the doorbell,
2652 * we also need to clear the outstanding_request
2653 * field in hba
2654 */
2656 }
2659 }
2661 /**
2662 * ufshcd_exec_dev_cmd - API for sending device management requests
2663 * @hba: UFS hba
2664 * @cmd_type: specifies the type (NOP, Query...)
2665 * @timeout: time in seconds
2666 *
2667 * NOTE: Since there is only one available tag for device management commands,
2668 * it is expected you hold the hba->dev_cmd.lock mutex.
2669 */
2672 {
2683 /*
2684 * Get free slot, sleep if slots are unavailable.
2685 * Even though we use wait_event() which sleeps indefinitely,
2686 * the maximum wait time is bounded by SCSI request timeout.
2687 */
2692 }
2706 /* Make sure descriptors are ready before ringing the doorbell */
2718 out_put_tag:
2720 out_unlock:
2723 }
2725 /**
2726 * ufshcd_init_query() - init the query response and request parameters
2727 * @hba: per-adapter instance
2728 * @request: address of the request pointer to be initialized
2729 * @response: address of the response pointer to be initialized
2730 * @opcode: operation to perform
2731 * @idn: flag idn to access
2732 * @index: LU number to access
2733 * @selector: query/flag/descriptor further identification
2734 */
2738 {
2747 }
2751 {
2761 else
2763 }
2770 }
2772 /**
2773 * ufshcd_query_flag() - API function for sending flag query requests
2774 * @hba: per-adapter instance
2775 * @opcode: flag query to perform
2776 * @idn: flag idn to access
2777 * @flag_res: the flag value after the query request completes
2778 *
2779 * Returns 0 for success, non-zero in case of failure
2780 */
2783 {
2794 selector);
2805 /* No dummy reads */
2807 __func__);
2810 }
2818 }
2827 }
2833 out_unlock:
2837 }
2839 /**
2840 * ufshcd_query_attr - API function for sending attribute requests
2841 * @hba: per-adapter instance
2842 * @opcode: attribute opcode
2843 * @idn: attribute idn to access
2844 * @index: index field
2845 * @selector: selector field
2846 * @attr_val: the attribute value after the query request completes
2847 *
2848 * Returns 0 for success, non-zero in case of failure
2849 */
2852 {
2865 }
2869 selector);
2884 }
2892 }
2896 out_unlock:
2898 out:
2901 }
2903 /**
2904 * ufshcd_query_attr_retry() - API function for sending query
2905 * attribute with retries
2906 * @hba: per-adapter instance
2907 * @opcode: attribute opcode
2908 * @idn: attribute idn to access
2909 * @index: index field
2910 * @selector: selector field
2911 * @attr_val: the attribute value after the query request
2912 * completes
2913 *
2914 * Returns 0 for success, non-zero in case of failure
2915 */
2919 {
2921 u32 retries;
2929 else
2931 }
2938 }
2943 {
2956 }
2963 }
2967 selector);
2984 }
2992 }
2996 out_unlock:
2999 out:
3002 }
3004 /**
3005 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3006 * @hba: per-adapter instance
3007 * @opcode: attribute opcode
3008 * @idn: attribute idn to access
3009 * @index: index field
3010 * @selector: selector field
3011 * @desc_buf: the buffer that contains the descriptor
3012 * @buf_len: length parameter passed to the device
3013 *
3014 * Returns 0 for success, non-zero in case of failure.
3015 * The buf_len parameter will contain, on return, the length parameter
3016 * received on the response.
3017 */
3021 u8 selector,
3023 {
3032 }
3035 }
3037 /**
3038 * ufshcd_read_desc_length - read the specified descriptor length from header
3039 * @hba: Pointer to adapter instance
3040 * @desc_id: descriptor idn value
3041 * @desc_index: descriptor index
3042 * @desc_length: pointer to variable to read the length of descriptor
3043 *
3044 * Return 0 in case of success, non-zero otherwise
3045 */
3050 {
3069 desc_id);
3071 }
3076 }
3078 /**
3079 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3080 * @hba: Pointer to adapter instance
3081 * @desc_id: descriptor idn value
3082 * @desc_len: mapped desc length (out)
3083 *
3084 * Return 0 in case of success, non-zero otherwise
3085 */
3088 {
3121 }
3123 }
3126 /**
3127 * ufshcd_read_desc_param - read the specified descriptor parameter
3128 * @hba: Pointer to adapter instance
3129 * @desc_id: descriptor idn value
3130 * @desc_index: descriptor index
3131 * @param_offset: offset of the parameter to read
3132 * @param_read_buf: pointer to buffer where parameter would be read
3133 * @param_size: sizeof(param_read_buf)
3134 *
3135 * Return 0 in case of success, non-zero otherwise
3136 */
3140 u8 param_offset,
3142 u8 param_size)
3143 {
3149 /* Safety check */
3153 /* Get the max length of descriptor from structure filled up at probe
3154 * time.
3155 */
3158 /* Sanity checks */
3161 __func__);
3163 }
3165 /* Check whether we need temp memory */
3173 }
3175 /* Request for full descriptor */
3181 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3184 }
3186 /* Sanity check */
3192 }
3194 /* Check wherher we will not copy more data, than available */
3200 out:
3204 }
3210 u32 size)
3211 {
3213 }
3216 /**
3217 * struct uc_string_id - unicode string
3218 *
3219 * @len: size of this descriptor inclusive
3220 * @type: descriptor type
3221 * @uc: unicode string character
3222 */
3224 u8 len;
3225 u8 type;
3229 /* replace non-printable or non-ASCII characters with spaces */
3231 {
3233 }
3235 /**
3236 * ufshcd_read_string_desc - read string descriptor
3237 * @hba: pointer to adapter instance
3238 * @desc_index: descriptor index
3239 * @buf: pointer to buffer where descriptor would be read,
3240 * the caller should free the memory.
3241 * @ascii: if true convert from unicode to ascii characters
3242 * null terminated string.
3243 *
3244 * Return:
3245 * * string size on success.
3246 * * -ENOMEM: on allocation failure
3247 * * -EINVAL: on a wrong parameter
3248 */
3251 {
3265 QUERY_DESC_MAX_SIZE);
3271 }
3278 }
3281 ssize_t ascii_len;
3283 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3289 }
3291 /*
3292 * the descriptor contains string in UTF16 format
3293 * we need to convert to utf-8 so it can be displayed
3294 */
3299 /* replace non-printable or non-ASCII characters with spaces */
3310 }
3312 }
3313 out:
3317 }
3319 /**
3320 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3321 * @hba: Pointer to adapter instance
3322 * @lun: lun id
3323 * @param_offset: offset of the parameter to read
3324 * @param_read_buf: pointer to buffer where parameter would be read
3325 * @param_size: sizeof(param_read_buf)
3326 *
3327 * Return 0 in case of success, non-zero otherwise
3328 */
3333 u32 param_size)
3334 {
3335 /*
3336 * Unit descriptors are only available for general purpose LUs (LUN id
3337 * from 0 to 7) and RPMB Well known LU.
3338 */
3344 }
3347 {
3362 gating_wait);
3363 }
3366 }
3369 }
3371 /**
3372 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3373 * @hba: per adapter instance
3374 *
3375 * 1. Allocate DMA memory for Command Descriptor array
3376 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3377 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3378 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3379 * (UTMRDL)
3380 * 4. Allocate memory for local reference block(lrb).
3381 *
3382 * Returns 0 for success, non-zero in case of failure
3383 */
3385 {
3388 /* Allocate memory for UTP command descriptors */
3391 ucdl_size,
3393 GFP_KERNEL);
3395 /*
3396 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3397 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3398 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3399 * be aligned to 128 bytes as well
3400 */
3406 }
3408 /*
3409 * Allocate memory for UTP Transfer descriptors
3410 * UFSHCI requires 1024 byte alignment of UTRD
3411 */
3414 utrdl_size,
3416 GFP_KERNEL);
3422 }
3424 /*
3425 * Allocate memory for UTP Task Management descriptors
3426 * UFSHCI requires 1024 byte alignment of UTMRD
3427 */
3430 utmrdl_size,
3432 GFP_KERNEL);
3438 }
3440 /* Allocate memory for local reference block */
3443 GFP_KERNEL);
3447 }
3449 out:
3451 }
3453 /**
3454 * ufshcd_host_memory_configure - configure local reference block with
3455 * memory offsets
3456 * @hba: per adapter instance
3457 *
3458 * Configure Host memory space
3459 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3460 * address.
3461 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3462 * and PRDT offset.
3463 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3464 * into local reference block.
3465 */
3467 {
3469 dma_addr_t cmd_desc_dma_addr;
3470 dma_addr_t cmd_desc_element_addr;
3471 u16 response_offset;
3472 u16 prdt_offset;
3478 response_offset =
3480 prdt_offset =
3487 /* Configure UTRD with command descriptor base address */
3488 cmd_desc_element_addr =
3495 /* Response upiu and prdt offset should be in double words */
3503 }
3504 }
3506 /**
3507 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3508 * @hba: per adapter instance
3509 *
3510 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3511 * in order to initialize the Unipro link startup procedure.
3512 * Once the Unipro links are up, the device connected to the controller
3513 * is detected.
3514 *
3515 * Returns 0 on success, non-zero value on failure
3516 */
3518 {
3529 }
3532 {
3533 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3539 /*
3540 * last_dme_cmd_tstamp will be 0 only for 1st call to
3541 * this function
3542 */
3552 min_sleep_time_us =
3554 else
3556 }
3558 /* allow sleep for extra 50us if needed */
3560 }
3562 /**
3563 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3564 * @hba: per adapter instance
3565 * @attr_sel: uic command argument1
3566 * @attr_set: attribute set type as uic command argument2
3567 * @mib_val: setting value as uic command argument3
3568 * @peer: indicate whether peer or local
3569 *
3570 * Returns 0 on success, non-zero value on failure
3571 */
3574 {
3578 "dme-peer-set"
3579 };
3591 /* for peer attributes we retry upon failure */
3604 }
3607 /**
3608 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3609 * @hba: per adapter instance
3610 * @attr_sel: uic command argument1
3611 * @mib_val: the value of the attribute as returned by the UIC command
3612 * @peer: indicate whether peer or local
3613 *
3614 * Returns 0 on success, non-zero value on failure
3615 */
3618 {
3622 "dme-peer-get"
3623 };
3645 }
3650 }
3651 }
3658 /* for peer attributes we retry upon failure */
3676 out:
3678 }
3681 /**
3682 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3683 * state) and waits for it to take effect.
3684 *
3685 * @hba: per adapter instance
3686 * @cmd: UIC command to execute
3687 *
3688 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3689 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3690 * and device UniPro link and hence it's final completion would be indicated by
3691 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3692 * addition to normal UIC command completion Status (UCCS). This function only
3693 * returns after the relevant status bits indicate the completion.
3694 *
3695 * Returns 0 on success, non-zero value on failure
3696 */
3698 {
3701 u8 status;
3713 /*
3714 * Make sure UIC command completion interrupt is disabled before
3715 * issuing UIC command.
3716 */
3719 }
3727 }
3736 }
3744 }
3745 out:
3750 }
3761 }
3763 /**
3764 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
3765 * using DME_SET primitives.
3766 * @hba: per adapter instance
3767 * @mode: powr mode value
3768 *
3769 * Returns 0 on success, non-zero value on failure
3770 */
3772 {
3783 }
3784 }
3793 out:
3795 }
3798 {
3807 /* Reset the attached device */
3823 }
3827 {
3845 /*
3846 * If link recovery fails then return error code returned from
3847 * ufshcd_link_recovery().
3848 * If link recovery succeeds then return -EAGAIN to attempt
3849 * hibern8 enter retry again.
3850 */
3857 }
3860 POST_CHANGE);
3863 }
3866 {
3873 }
3874 out:
3876 }
3879 {
3897 POST_CHANGE);
3900 }
3903 }
3907 {
3918 }
3927 }
3928 }
3932 {
3941 }
3943 /**
3944 * ufshcd_init_pwr_info - setting the POR (power on reset)
3945 * values in hba power info
3946 * @hba: per-adapter instance
3947 */
3949 {
3957 }
3959 /**
3960 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
3961 * @hba: per-adapter instance
3962 */
3964 {
3974 /* Get the connected lane count */
3982 __func__,
3986 }
3988 /*
3989 * First, get the maximum gears of HS speed.
3990 * If a zero value, it means there is no HSGEAR capability.
3991 * Then, get the maximum gears of PWM speed.
3992 */
4001 }
4003 }
4014 }
4016 }
4020 }
4024 {
4027 /* if already configured to the requested pwr_mode */
4037 }
4039 /*
4040 * Configure attributes for power mode change with below.
4041 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4042 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4043 * - PA_HSSERIES
4044 */
4051 else
4060 else
4071 DL_FC0ProtectionTimeOutVal_Default);
4073 DL_TC0ReplayTimeOutVal_Default);
4075 DL_AFC0ReqTimeOutVal_Default);
4077 DL_FC1ProtectionTimeOutVal_Default);
4079 DL_TC1ReplayTimeOutVal_Default);
4081 DL_AFC1ReqTimeOutVal_Default);
4084 DL_FC0ProtectionTimeOutVal_Default);
4086 DL_TC0ReplayTimeOutVal_Default);
4088 DL_AFC0ReqTimeOutVal_Default);
4098 pwr_mode);
4102 }
4105 }
4107 /**
4108 * ufshcd_config_pwr_mode - configure a new power mode
4109 * @hba: per-adapter instance
4110 * @desired_pwr_mode: desired power configuration
4111 */
4114 {
4127 }
4130 /**
4131 * ufshcd_complete_dev_init() - checks device readiness
4132 * @hba: per-adapter instance
4133 *
4134 * Set fDeviceInit flag and poll until device toggles it.
4135 */
4137 {
4149 }
4151 /* poll for max. 1000 iterations for fDeviceInit flag to clear */
4163 __func__);
4165 out:
4167 }
4169 /**
4170 * ufshcd_make_hba_operational - Make UFS controller operational
4171 * @hba: per adapter instance
4172 *
4173 * To bring UFS host controller to operational state,
4174 * 1. Enable required interrupts
4175 * 2. Configure interrupt aggregation
4176 * 3. Program UTRL and UTMRL base address
4177 * 4. Configure run-stop-registers
4178 *
4179 * Returns 0 on success, non-zero value on failure
4180 */
4182 {
4184 u32 reg;
4186 /* Enable required interrupts */
4189 /* Configure interrupt aggregation */
4192 else
4195 /* Configure UTRL and UTMRL base address registers */
4197 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4199 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4201 REG_UTP_TASK_REQ_LIST_BASE_L);
4203 REG_UTP_TASK_REQ_LIST_BASE_H);
4205 /*
4206 * Make sure base address and interrupt setup are updated before
4207 * enabling the run/stop registers below.
4208 */
4211 /*
4212 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4213 */
4222 }
4224 out:
4226 }
4229 /**
4230 * ufshcd_hba_stop - Send controller to reset state
4231 * @hba: per adapter instance
4232 * @can_sleep: perform sleep or just spin
4233 */
4235 {
4244 }
4246 /**
4247 * ufshcd_hba_enable - initialize the controller
4248 * @hba: per adapter instance
4249 *
4250 * The controller resets itself and controller firmware initialization
4251 * sequence kicks off. When controller is ready it will set
4252 * the Host Controller Enable bit to 1.
4253 *
4254 * Returns 0 on success, non-zero value on failure
4255 */
4257 {
4261 /* change controller state to "reset state" */
4264 /* UniPro link is disabled at this point */
4269 /* start controller initialization sequence */
4272 /*
4273 * To initialize a UFS host controller HCE bit must be set to 1.
4274 * During initialization the HCE bit value changes from 1->0->1.
4275 * When the host controller completes initialization sequence
4276 * it sets the value of HCE bit to 1. The same HCE bit is read back
4277 * to check if the controller has completed initialization sequence.
4278 * So without this delay the value HCE = 1, set in the previous
4279 * instruction might be read back.
4280 * This delay can be changed based on the controller.
4281 */
4284 /* wait for the host controller to complete initialization */
4288 retry--;
4293 }
4295 }
4297 /* enable UIC related interrupts */
4303 }
4307 {
4313 else
4322 else
4331 }
4332 }
4335 }
4338 {
4340 }
4343 u32 reg)
4344 {
4348 }
4351 /**
4352 * ufshcd_link_startup - Initialize unipro link startup
4353 * @hba: per adapter instance
4354 *
4355 * Returns 0 for success, non-zero in case of failure
4356 */
4358 {
4363 /*
4364 * If UFS device isn't active then we will have to issue link startup
4365 * 2 times to make sure the device state move to active.
4366 */
4370 link_startup:
4376 /* check if device is detected by inter-connect layer */
4383 }
4385 /*
4386 * DME link lost indication is only received when link is up,
4387 * but we can't be sure if the link is up until link startup
4388 * succeeds. So reset the local Uni-Pro and try again.
4389 */
4394 }
4398 /* failed to get the link up... retire */
4402 }
4408 }
4410 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4418 }
4420 /* Include any host controller configuration via UIC commands */
4426 out:
4432 }
4434 }
4436 /**
4437 * ufshcd_verify_dev_init() - Verify device initialization
4438 * @hba: per-adapter instance
4439 *
4440 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4441 * device Transport Protocol (UTP) layer is ready after a reset.
4442 * If the UTP layer at the device side is not initialized, it may
4443 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4444 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4445 */
4447 {
4455 NOP_OUT_TIMEOUT);
4461 }
4468 }
4470 /**
4471 * ufshcd_set_queue_depth - set lun queue depth
4472 * @sdev: pointer to SCSI device
4473 *
4474 * Read bLUQueueDepth value and activate scsi tagged command
4475 * queueing. For WLUN, queue depth is set to 1. For best-effort
4476 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4477 * value that host can queue.
4478 */
4480 {
4482 u8 lun_qdepth;
4490 UNIT_DESC_PARAM_LU_Q_DEPTH,
4494 /* Some WLUN doesn't support unit descriptor */
4498 /* eventually, we can figure out the real queue depth */
4500 else
4506 }
4508 /*
4509 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4510 * @hba: per-adapter instance
4511 * @lun: UFS device lun id
4512 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4513 *
4514 * Returns 0 in case of success and b_lu_write_protect status would be returned
4515 * @b_lu_write_protect parameter.
4516 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4517 * Returns -EINVAL in case of invalid parameters passed to this function.
4518 */
4520 u8 lun,
4522 {
4527 /*
4528 * According to UFS device spec, RPMB LU can't be write
4529 * protected so skip reading bLUWriteProtect parameter for
4530 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4531 */
4534 else
4536 lun,
4537 UNIT_DESC_PARAM_LU_WR_PROTECT,
4538 b_lu_write_protect,
4541 }
4543 /**
4544 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4545 * status
4546 * @hba: per-adapter instance
4547 * @sdev: pointer to SCSI device
4548 *
4549 */
4552 {
4555 u8 b_lu_write_protect;
4561 }
4562 }
4564 /**
4565 * ufshcd_slave_alloc - handle initial SCSI device configurations
4566 * @sdev: pointer to SCSI device
4567 *
4568 * Returns success
4569 */
4571 {
4576 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4579 /* DBD field should be set to 1 in mode sense(10) */
4582 /* allow SCSI layer to restart the device in case of errors */
4585 /* REPORT SUPPORTED OPERATION CODES is not supported */
4588 /* WRITE_SAME command is not supported */
4596 }
4598 /**
4599 * ufshcd_change_queue_depth - change queue depth
4600 * @sdev: pointer to SCSI device
4601 * @depth: required depth to set
4602 *
4603 * Change queue depth and make sure the max. limits are not crossed.
4604 */
4606 {
4612 }
4614 /**
4615 * ufshcd_slave_configure - adjust SCSI device configurations
4616 * @sdev: pointer to SCSI device
4617 */
4619 {
4629 }
4631 /**
4632 * ufshcd_slave_destroy - remove SCSI device configurations
4633 * @sdev: pointer to SCSI device
4634 */
4636 {
4640 /* Drop the reference as it won't be needed anymore */
4647 }
4648 }
4650 /**
4651 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
4652 * @lrbp: pointer to local reference block of completed command
4653 * @scsi_status: SCSI command status
4654 *
4655 * Returns value base on SCSI command status
4656 */
4659 {
4665 /* fallthrough */
4669 scsi_status;
4683 }
4685 /**
4686 * ufshcd_transfer_rsp_status - Get overall status of the response
4687 * @hba: per adapter instance
4688 * @lrbp: pointer to local reference block of completed command
4689 *
4690 * Returns result of the command to notify SCSI midlayer
4691 */
4694 {
4699 /* overall command status of utrd */
4708 /*
4709 * get the response UPIU result to extract
4710 * the SCSI command status
4711 */
4714 /*
4715 * get the result based on SCSI status response
4716 * to notify the SCSI midlayer of the command status
4717 */
4721 /*
4722 * Currently we are only supporting BKOPs exception
4723 * events hence we can ignore BKOPs exception event
4724 * during power management callbacks. BKOPs exception
4725 * event is not expected to be raised in runtime suspend
4726 * callback as it allows the urgent bkops.
4727 * During system suspend, we are anyway forcefully
4728 * disabling the bkops and if urgent bkops is needed
4729 * it will be enabled on system resume. Long term
4730 * solution could be to abort the system suspend if
4731 * UFS device needs urgent BKOPs.
4732 */
4736 /*
4737 * Prevent suspend once eeh_work is scheduled
4738 * to avoid deadlock between ufshcd_suspend
4739 * and exception event handler.
4740 */
4742 }
4745 /* TODO: handle Reject UPIU Response */
4753 result);
4756 }
4783 }
4785 /**
4786 * ufshcd_uic_cmd_compl - handle completion of uic command
4787 * @hba: per adapter instance
4788 * @intr_status: interrupt status generated by the controller
4789 *
4790 * Returns
4791 * IRQ_HANDLED - If interrupt is valid
4792 * IRQ_NONE - If invalid interrupt
4793 */
4795 {
4805 }
4810 }
4812 }
4814 /**
4815 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
4816 * @hba: per adapter instance
4817 * @completed_reqs: requests to complete
4818 */
4821 {
4835 /* Mark completed command as NULL in LRB */
4838 /* Do not touch lrbp after scsi done */
4848 }
4849 }
4852 }
4854 /* clear corresponding bits of completed commands */
4858 }
4860 /**
4861 * ufshcd_transfer_req_compl - handle SCSI and query command completion
4862 * @hba: per adapter instance
4863 *
4864 * Returns
4865 * IRQ_HANDLED - If interrupt is valid
4866 * IRQ_NONE - If invalid interrupt
4867 */
4869 {
4871 u32 tr_doorbell;
4873 /* Resetting interrupt aggregation counters first and reading the
4874 * DOOR_BELL afterward allows us to handle all the completed requests.
4875 * In order to prevent other interrupts starvation the DB is read once
4876 * after reset. The down side of this solution is the possibility of
4877 * false interrupt if device completes another request after resetting
4878 * aggregation and before reading the DB.
4879 */
4891 }
4892 }
4894 /**
4895 * ufshcd_disable_ee - disable exception event
4896 * @hba: per-adapter instance
4897 * @mask: exception event to disable
4898 *
4899 * Disables exception event in the device so that the EVENT_ALERT
4900 * bit is not set.
4901 *
4902 * Returns zero on success, non-zero error value on failure.
4903 */
4905 {
4907 u32 val;
4918 out:
4920 }
4922 /**
4923 * ufshcd_enable_ee - enable exception event
4924 * @hba: per-adapter instance
4925 * @mask: exception event to enable
4926 *
4927 * Enable corresponding exception event in the device to allow
4928 * device to alert host in critical scenarios.
4929 *
4930 * Returns zero on success, non-zero error value on failure.
4931 */
4933 {
4935 u32 val;
4946 out:
4948 }
4950 /**
4951 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
4952 * @hba: per-adapter instance
4953 *
4954 * Allow device to manage background operations on its own. Enabling
4955 * this might lead to inconsistent latencies during normal data transfers
4956 * as the device is allowed to manage its own way of handling background
4957 * operations.
4958 *
4959 * Returns zero on success, non-zero on failure.
4960 */
4962 {
4974 }
4979 /* No need of URGENT_BKOPS exception from the device */
4984 out:
4986 }
4988 /**
4989 * ufshcd_disable_auto_bkops - block device in doing background operations
4990 * @hba: per-adapter instance
4991 *
4992 * Disabling background operations improves command response latency but
4993 * has drawback of device moving into critical state where the device is
4994 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
4995 * host is idle so that BKOPS are managed effectively without any negative
4996 * impacts.
4997 *
4998 * Returns zero on success, non-zero on failure.
4999 */
5001 {
5007 /*
5008 * If host assisted BKOPs is to be enabled, make sure
5009 * urgent bkops exception is allowed.
5010 */
5016 }
5025 }
5030 out:
5032 }
5034 /**
5035 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5036 * @hba: per adapter instance
5037 *
5038 * After a device reset the device may toggle the BKOPS_EN flag
5039 * to default value. The s/w tracking variables should be updated
5040 * as well. This function would change the auto-bkops state based on
5041 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
5042 */
5044 {
5053 }
5055 }
5058 {
5061 }
5063 /**
5064 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5065 * @hba: per-adapter instance
5066 * @status: bkops_status value
5067 *
5068 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5069 * flag in the device to permit background operations if the device
5070 * bkops_status is greater than or equal to "status" argument passed to
5071 * this function, disable otherwise.
5072 *
5073 * Returns 0 for success, non-zero in case of failure.
5074 *
5075 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5076 * to know whether auto bkops is enabled or disabled after this function
5077 * returns control to it.
5078 */
5081 {
5095 }
5099 else
5101 out:
5103 }
5105 /**
5106 * ufshcd_urgent_bkops - handle urgent bkops exception event
5107 * @hba: per-adapter instance
5108 *
5109 * Enable fBackgroundOpsEn flag in the device to permit background
5110 * operations.
5111 *
5112 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5113 * and negative error value for any other failure.
5114 */
5116 {
5118 }
5121 {
5124 }
5127 {
5139 }
5141 /*
5142 * We are seeing that some devices are raising the urgent bkops
5143 * exception events even when BKOPS status doesn't indicate performace
5144 * impacted or critical. Handle these device by determining their urgent
5145 * bkops status at runtime.
5146 */
5150 /* update the current status as the urgent bkops level */
5153 }
5155 enable_auto_bkops:
5157 out:
5161 }
5163 /**
5164 * ufshcd_exception_event_handler - handle exceptions raised by device
5165 * @work: pointer to work data
5166 *
5167 * Read bExceptionEventStatus attribute from the device and handle the
5168 * exception event accordingly.
5169 */
5171 {
5184 }
5191 out:
5193 /*
5194 * pm_runtime_get_noresume is called while scheduling
5195 * eeh_work to avoid suspend racing with exception work.
5196 * Hence decrement usage counter using pm_runtime_put_noidle
5197 * to allow suspend on completion of exception event handler.
5198 */
5202 }
5204 /* Complete requests that have door-bell cleared */
5206 {
5209 }
5211 /**
5212 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5213 * to recover from the DL NAC errors or not.
5214 * @hba: per-adapter instance
5215 *
5216 * Returns true if error handling is required, false otherwise
5217 */
5219 {
5224 /*
5225 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5226 * device fatal error and/or DL NAC & REPLAY timeout errors.
5227 */
5239 /*
5240 * wait for 50ms to see if we can get any other errors or not.
5241 */
5246 /*
5247 * now check if we have got any other severe errors other than
5248 * DL NAC error?
5249 */
5255 /*
5256 * As DL NAC is the only error received so far, send out NOP
5257 * command to confirm if link is still active or not.
5258 * - If we don't get any response then do error recovery.
5259 * - If we get response then clear the DL NAC error bit.
5260 */
5269 /* Link seems to be alive hence ignore the DL NAC errors */
5272 /* clear NAC error */
5277 }
5278 }
5279 out:
5282 }
5284 /**
5285 * ufshcd_err_handler - handle UFS errors that require s/w attention
5286 * @work: pointer to work structure
5287 */
5289 {
5310 /* Complete requests that have door-bell cleared by h/w */
5317 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
5322 }
5327 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5328 UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
5331 /*
5332 * if host reset is required then skip clearing the pending
5333 * transfers forcefully because they will get cleared during
5334 * host reset and restore
5335 */
5339 /* release lock as clear command might sleep */
5341 /* Clear pending transfer requests */
5346 }
5347 }
5349 /* Clear pending task management requests */
5354 }
5355 }
5357 lock_skip_pending_xfer_clear:
5360 /* Complete the requests that are cleared by s/w */
5366 skip_pending_xfer_clear:
5367 /* Fatal errors need reset */
5371 /*
5372 * ufshcd_reset_and_restore() does the link reinitialization
5373 * which will need atleast one empty doorbell slot to send the
5374 * device management commands (NOP and query commands).
5375 * If there is no slot empty at this moment then free up last
5376 * slot forcefully.
5377 */
5387 __func__);
5389 }
5390 /*
5391 * Inform scsi mid-layer that we did reset and allow to handle
5392 * Unit Attention properly.
5393 */
5397 }
5399 skip_err_handling:
5405 }
5409 out:
5414 }
5416 /**
5417 * ufshcd_update_uic_error - check and set fatal UIC error flags.
5418 * @hba: per-adapter instance
5419 *
5420 * Returns
5421 * IRQ_HANDLED - If interrupt is valid
5422 * IRQ_NONE - If invalid interrupt
5423 */
5425 {
5426 u32 reg;
5429 /* PHY layer lane error */
5431 /* Ignore LINERESET indication, as this is not an error */
5434 /*
5435 * To know whether this error is fatal or not, DB timeout
5436 * must be checked but this error is handled separately.
5437 */
5441 }
5443 /* PA_INIT_ERROR is fatal and needs UIC reset */
5452 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5455 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5458 }
5460 }
5462 /* UIC NL/TL/DME errors needs software retry */
5469 }
5477 }
5485 }
5490 }
5493 u32 intr_mask)
5494 {
5508 }
5510 /**
5511 * ufshcd_check_errors - Check for errors that need s/w attention
5512 * @hba: per-adapter instance
5513 *
5514 * Returns
5515 * IRQ_HANDLED - If interrupt is valid
5516 * IRQ_NONE - If invalid interrupt
5517 */
5519 {
5526 }
5533 }
5544 }
5547 /*
5548 * update the transfer error masks to sticky bits, let's do this
5549 * irrespective of current ufshcd_state.
5550 */
5554 /* handle fatal errors only when link is functional */
5556 /* block commands from scsi mid-layer */
5561 /* dump controller state before resetting */
5564 SYSTEM_BUS_FATAL_ERROR);
5574 pr_prdt);
5575 }
5577 }
5579 }
5580 /*
5581 * if (!queue_eh_work) -
5582 * Other errors are either non-fatal where host recovers
5583 * itself without s/w intervention or errors that will be
5584 * handled by the SCSI core layer.
5585 */
5587 }
5593 };
5596 {
5608 }
5610 /**
5611 * ufshcd_tmc_handler - handle task management function completion
5612 * @hba: per adapter instance
5613 *
5614 * Returns
5615 * IRQ_HANDLED - If interrupt is valid
5616 * IRQ_NONE - If invalid interrupt
5617 */
5619 {
5624 };
5628 }
5630 /**
5631 * ufshcd_sl_intr - Interrupt service routine
5632 * @hba: per adapter instance
5633 * @intr_status: contains interrupts generated by the controller
5634 *
5635 * Returns
5636 * IRQ_HANDLED - If interrupt is valid
5637 * IRQ_NONE - If invalid interrupt
5638 */
5640 {
5661 }
5663 /**
5664 * ufshcd_intr - Main interrupt service routine
5665 * @irq: irq number
5666 * @__hba: pointer to adapter instance
5667 *
5668 * Returns
5669 * IRQ_HANDLED - If interrupt is valid
5670 * IRQ_NONE - If invalid interrupt
5671 */
5673 {
5682 /*
5683 * There could be max of hba->nutrs reqs in flight and in worst case
5684 * if the reqs get finished 1 by 1 after the interrupt status is
5685 * read, make sure we handle them by checking the interrupt status
5686 * again in a loop until we process all of the reqs before returning.
5687 */
5689 enabled_intr_status =
5703 }
5707 }
5710 {
5722 /* poll for max. 1 sec to clear door bell register by h/w */
5724 REG_UTP_TASK_REQ_DOOR_BELL,
5726 out:
5728 }
5732 {
5740 /*
5741 * Get free slot, sleep if slots are unavailable.
5742 * Even though we use wait_event() which sleeps indefinitely,
5743 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
5744 */
5759 /* send command to the controller */
5762 /* Make sure descriptors are ready before ringing the task doorbell */
5766 /* Make sure that doorbell is committed immediately */
5773 /* wait until the task management command is completed */
5777 /*
5778 * Make sure that ufshcd_compl_tm() does not trigger a
5779 * use-after-free.
5780 */
5794 }
5804 }
5806 /**
5807 * ufshcd_issue_tm_cmd - issues task management commands to controller
5808 * @hba: per adapter instance
5809 * @lun_id: LUN ID to which TM command is sent
5810 * @task_id: task ID to which the TM command is applicable
5811 * @tm_function: task management function opcode
5812 * @tm_response: task management service response return value
5813 *
5814 * Returns non-zero value on error, zero on success.
5815 */
5818 {
5822 /* Configure task request descriptor */
5826 /* Configure task request UPIU */
5831 /*
5832 * The host shall provide the same value for LUN field in the basic
5833 * header and for Input Parameter.
5834 */
5848 MASK_TM_SERVICE_RESP;
5850 }
5852 /**
5853 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
5854 * @hba: per-adapter instance
5855 * @req_upiu: upiu request
5856 * @rsp_upiu: upiu reply
5857 * @desc_buff: pointer to descriptor buffer, NULL if NA
5858 * @buff_len: descriptor size, 0 if NA
5859 * @cmd_type: specifies the type (NOP, Query...)
5860 * @desc_op: descriptor operation
5861 *
5862 * Those type of requests uses UTP Transfer Request Descriptor - utrd.
5863 * Therefore, it "rides" the device management infrastructure: uses its tag and
5864 * tasks work queues.
5865 *
5866 * Since there is only one available tag for device management commands,
5867 * the caller is expected to hold the hba->dev_cmd.lock mutex.
5868 */
5875 {
5883 u32 upiu_flags;
5891 }
5915 }
5917 /* update the task tag in the request upiu */
5922 /* just copy the upiu request as it is */
5925 /* The Data Segment Area is optional depending upon the query
5926 * function value. for WRITE DESCRIPTOR, the data segment
5927 * follows right after the tsf.
5928 */
5931 }
5937 /* Make sure descriptors are ready before ringing the doorbell */
5943 /*
5944 * ignore the returning value here - ufshcd_check_query_response is
5945 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
5946 * read the response directly ignoring all errors.
5947 */
5950 /* just copy the upiu response as it is */
5955 MASK_QUERY_DATA_SEG_LEN;
5966 }
5967 }
5970 out_unlock:
5973 }
5975 /**
5976 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
5977 * @hba: per-adapter instance
5978 * @req_upiu: upiu request
5979 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands
5980 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
5981 * @desc_buff: pointer to descriptor buffer, NULL if NA
5982 * @buff_len: descriptor size, 0 if NA
5983 * @desc_op: descriptor operation
5984 *
5985 * Supports UTP Transfer requests (nop and query), and UTP Task
5986 * Management requests.
5987 * It is up to the caller to fill the upiu conent properly, as it will
5988 * be copied without any further input validations.
5989 */
5996 {
6006 /* fall through */
6030 ocs_value);
6032 }
6041 }
6044 }
6046 /**
6047 * ufshcd_eh_device_reset_handler - device reset handler registered to
6048 * scsi layer.
6049 * @cmd: SCSI command pointer
6050 *
6051 * Returns SUCCESS/FAILED
6052 */
6054 {
6058 u32 pos;
6074 }
6076 /* clear the commands that were pending for corresponding LUN */
6082 }
6083 }
6088 out:
6096 }
6098 }
6101 {
6108 }
6109 }
6111 /**
6112 * ufshcd_abort - abort a specific command
6113 * @cmd: SCSI command pointer
6114 *
6115 * Abort the pending command in device by sending UFS_ABORT_TASK task management
6116 * command, and in host controller by clearing the door-bell register. There can
6117 * be race between controller sending the command to the device while abort is
6118 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
6119 * really issued and then try to abort it.
6120 *
6121 * Returns SUCCESS/FAILED
6122 */
6124 {
6133 u32 reg;
6144 }
6146 /*
6147 * Task abort to the device W-LUN is illegal. When this command
6148 * will fail, due to spec violation, scsi err handling next step
6149 * will be to send LU reset which, again, is a spec violation.
6150 * To avoid these unnecessary/illegal step we skip to the last error
6151 * handling stage: reset and restore.
6152 */
6158 /* If command is already aborted/completed, return SUCCESS */
6164 }
6170 }
6172 /* Print Transfer Request of aborted task */
6175 /*
6176 * Print detailed info about aborted request.
6177 * As more than one request might get aborted at the same time,
6178 * print full information only for the first aborted request in order
6179 * to reduce repeated printouts. For other aborted requests only print
6180 * basic details.
6181 */
6191 }
6194 /* Skip task abort in case previous aborts failed and report failure */
6198 }
6204 /* cmd pending in the device */
6209 /*
6210 * cmd not pending in the device, check if it is
6211 * in transition.
6212 */
6217 /* sleep for max. 200us to stabilize */
6220 }
6221 /* command completed already */
6232 }
6233 }
6238 }
6247 }
6249 }
6256 }
6265 out:
6272 }
6274 /*
6275 * This ufshcd_release() corresponds to the original scsi cmd that got
6276 * aborted here (as we won't get any IRQ for it).
6277 */
6280 }
6282 /**
6283 * ufshcd_host_reset_and_restore - reset and restore host controller
6284 * @hba: per-adapter instance
6285 *
6286 * Note that host controller reset may issue DME_RESET to
6287 * local and remote (device) Uni-Pro stack and the attributes
6288 * are reset to default state.
6289 *
6290 * Returns zero on success, non-zero on failure
6291 */
6293 {
6297 /*
6298 * Stop the host controller and complete the requests
6299 * cleared by h/w
6300 */
6308 /* scale up clocks to max frequency before full reinitialization */
6315 /* Establish the link again and restore the device */
6320 out:
6325 }
6327 /**
6328 * ufshcd_reset_and_restore - reset and re-initialize host/device
6329 * @hba: per-adapter instance
6330 *
6331 * Reset and recover device, host and re-establish link. This
6332 * is helpful to recover the communication in fatal error conditions.
6333 *
6334 * Returns zero on success, non-zero on failure
6335 */
6337 {
6342 /* Reset the attached device */
6349 }
6351 /**
6352 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
6353 * @cmd: SCSI command pointer
6354 *
6355 * Returns SUCCESS/FAILED
6356 */
6358 {
6366 /*
6367 * Check if there is any race with fatal error handling.
6368 * If so, wait for it to complete. Even though fatal error
6369 * handling does reset and restore in some cases, don't assume
6370 * anything out of it. We are just avoiding race here.
6371 */
6396 }
6402 }
6404 /**
6405 * ufshcd_get_max_icc_level - calculate the ICC level
6406 * @sup_curr_uA: max. current supported by the regulator
6407 * @start_scan: row at the desc table to start scan from
6408 * @buff: power descriptor buffer
6409 *
6410 * Returns calculated max ICC level for specific regulator
6411 */
6413 {
6416 u16 data;
6417 u16 unit;
6422 ATTR_ICC_LVL_UNIT_OFFSET;
6437 }
6440 }
6444 }
6447 }
6449 /**
6450 * ufshcd_calc_icc_level - calculate the max ICC level
6451 * In case regulators are not initialized we'll return 0
6452 * @hba: per-adapter instance
6453 * @desc_buf: power descriptor buffer to extract ICC levels from.
6454 * @len: length of desc_buff
6455 *
6456 * Returns calculated ICC level
6457 */
6460 {
6469 }
6480 icc_level,
6486 icc_level,
6488 out:
6490 }
6493 {
6497 u32 icc_level;
6510 }
6513 buff_len);
6524 out:
6526 }
6529 {
6534 RPM_AUTOSUSPEND_DELAY_MS);
6536 }
6538 /**
6539 * ufshcd_scsi_add_wlus - Adds required W-LUs
6540 * @hba: per-adapter instance
6541 *
6542 * UFS device specification requires the UFS devices to support 4 well known
6543 * logical units:
6544 * "REPORT_LUNS" (address: 01h)
6545 * "UFS Device" (address: 50h)
6546 * "RPMB" (address: 44h)
6547 * "BOOT" (address: 30h)
6548 * UFS device's power management needs to be controlled by "POWER CONDITION"
6549 * field of SSU (START STOP UNIT) command. But this "power condition" field
6550 * will take effect only when its sent to "UFS device" well known logical unit
6551 * hence we require the scsi_device instance to represent this logical unit in
6552 * order for the UFS host driver to send the SSU command for power management.
6553 *
6554 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
6555 * Block) LU so user space process can control this LU. User space may also
6556 * want to have access to BOOT LU.
6557 *
6558 * This function adds scsi device instances for each of all well known LUs
6559 * (except "REPORT LUNS" LU).
6560 *
6561 * Returns zero on success (all required W-LUs are added successfully),
6562 * non-zero error value on failure (if failed to add any of the required W-LU).
6563 */
6565 {
6576 }
6585 }
6596 }
6599 remove_sdev_ufs_device:
6601 out:
6603 }
6606 {
6609 u8 model_index;
6619 }
6627 }
6629 /*
6630 * getting vendor (manufacturerID) and Bank Index in big endian
6631 * format
6632 */
6636 /* getting Specification Version in big endian format */
6647 }
6649 /*
6650 * ufshcd_read_string_desc returns size of the string
6651 * reset the error value
6652 */
6655 out:
6658 }
6661 {
6666 }
6669 {
6680 }
6681 }
6683 /**
6684 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
6685 * @hba: per-adapter instance
6686 *
6687 * PA_TActivate parameter can be tuned manually if UniPro version is less than
6688 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
6689 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
6690 * the hibern8 exit latency.
6691 *
6692 * Returns zero on success, non-zero error value on failure.
6693 */
6695 {
6701 RX_MIN_ACTIVATETIME_CAPABILITY,
6707 /* make sure proper unit conversion is applied */
6708 tuned_pa_tactivate =
6712 tuned_pa_tactivate);
6714 out:
6716 }
6718 /**
6719 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
6720 * @hba: per-adapter instance
6721 *
6722 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
6723 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
6724 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
6725 * This optimal value can help reduce the hibern8 exit latency.
6726 *
6727 * Returns zero on success, non-zero error value on failure.
6728 */
6730 {
6750 peer_rx_hibern8_time_cap);
6751 /* make sure proper unit conversion is applied */
6755 tuned_pa_hibern8time);
6756 out:
6758 }
6760 /**
6761 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
6762 * less than device PA_TACTIVATE time.
6763 * @hba: per-adapter instance
6764 *
6765 * Some UFS devices require host PA_TACTIVATE to be lower than device
6766 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
6767 * for such devices.
6768 *
6769 * Returns zero on success, non-zero error value on failure.
6770 */
6772 {
6794 }
6801 }
6817 u32 new_peer_pa_tactivate;
6821 new_peer_pa_tactivate++;
6823 new_peer_pa_tactivate);
6824 }
6826 out:
6828 }
6831 {
6835 }
6840 /* set 1ms timeout for PA_TACTIVATE */
6845 }
6848 {
6852 }
6855 {
6892 }
6895 {
6905 }
6913 }
6920 out:
6923 }
6931 };
6933 static enum ufs_ref_clk_freq
6935 {
6943 }
6946 {
6957 }
6960 {
6962 u32 ref_clk;
6970 err);
6972 }
6984 }
6989 out:
6991 }
6994 {
6998 /* Clear any previous UFS device information */
7001 /* Init check for device descriptor sizes */
7004 /* Init UFS geometry descriptor related parameters */
7009 /* Check and apply UFS device quirks */
7015 }
7025 /* Probe maximum power mode co-supported by both UFS host and device */
7029 __func__);
7030 out:
7032 }
7034 /**
7035 * ufshcd_add_lus - probe and add UFS logical units
7036 * @hba: per-adapter instance
7037 */
7039 {
7042 /* Add required well known logical units to scsi mid layer */
7047 /* Initialize devfreq after UFS device is detected */
7057 }
7060 }
7066 out:
7068 }
7070 /**
7071 * ufshcd_probe_hba - probe hba to detect device and initialize
7072 * @hba: per-adapter instance
7073 * @async: asynchronous execution or not
7074 *
7075 * Execute link-startup and verify device initialization
7076 */
7078 {
7086 /* set the default level for urgent bkops */
7090 /* Debug counters initialization */
7093 /* UniPro link is active now */
7096 /* Verify device initialization by sending NOP OUT UPIU */
7101 /* Initiate UFS initialization, and waiting until completion */
7106 /*
7107 * Initialize UFS device parameters used by driver, these
7108 * parameters are associated with UFS descriptors.
7109 */
7114 }
7118 /* UFS device is also active now */
7123 /* Gear up to HS gear if supported */
7125 /*
7126 * Set the right value to bRefClkFreq before attempting to
7127 * switch to HS gears.
7128 */
7136 }
7138 }
7140 /*
7141 * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
7142 * and for removable UFS card as well, hence always set the parameter.
7143 * Note: Error handler may issue the device reset hence resetting
7144 * bActiveICCLevel as well so it is always safe to set this here.
7145 */
7148 /* set the state as operational after switching to desired gear */
7151 /* Enable Auto-Hibernate if configured */
7154 out:
7160 }
7162 /**
7163 * ufshcd_async_scan - asynchronous execution for probing hba
7164 * @data: data pointer to pass to this function
7165 * @cookie: cookie data
7166 */
7168 {
7172 /* Initialize hba, detect and initialize UFS device */
7177 /* Probe and add UFS logical units */
7179 out:
7180 /*
7181 * If we failed to initialize the device or the device is not
7182 * present, turn off the power/clocks etc.
7183 */
7188 }
7189 }
7194 NULL,
7195 };
7219 };
7223 {
7229 /*
7230 * "set_load" operation shall be required on those regulators
7231 * which specifically configured current limitation. Otherwise
7232 * zero max_uA may cause unexpected behavior when regulator is
7233 * enabled or set as high power mode.
7234 */
7242 }
7245 }
7249 {
7251 }
7255 {
7260 }
7264 {
7289 }
7290 }
7291 }
7292 out:
7294 }
7297 {
7309 else
7312 out:
7314 }
7317 {
7326 /* ignore errors on applying disable config */
7332 }
7333 out:
7335 }
7338 {
7355 out:
7360 }
7362 }
7365 {
7369 }
7372 {
7383 }
7384 out:
7386 }
7389 {
7403 out:
7405 }
7408 {
7415 }
7419 {
7446 }
7449 }
7453 }
7454 }
7460 out:
7465 }
7472 }
7479 }
7482 {
7484 }
7487 {
7506 }
7508 /*
7509 * Parse device ref clk freq as per device tree "ref_clk".
7510 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
7511 * in ufshcd_alloc_host().
7512 */
7523 }
7525 }
7528 }
7529 out:
7531 }
7534 {
7550 out_exit:
7552 out:
7557 }
7560 {
7567 }
7570 {
7573 /*
7574 * Handle host controller power separately from the UFS device power
7575 * rails as it will help controlling the UFS host controller power
7576 * collapse easily which is different than UFS device power collapse.
7577 * Also, enable the host controller power before we go ahead with rest
7578 * of the initialization here.
7579 */
7611 out_disable_vreg:
7613 out_disable_clks:
7615 out_disable_hba_vreg:
7617 out:
7619 }
7622 {
7634 }
7635 }
7637 static int
7639 {
7644 UFS_SENSE_SIZE,
7653 }
7662 out:
7664 }
7666 /**
7667 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
7668 * power mode
7669 * @hba: per adapter instance
7670 * @pwr_mode: device power mode to set
7671 *
7672 * Returns 0 if requested power mode is set successfully
7673 * Returns non-zero if failed to set the requested power mode
7674 */
7677 {
7691 }
7694 }
7700 /*
7701 * If scsi commands fail, the scsi mid-layer schedules scsi error-
7702 * handling, which would wait for host to be resumed. Since we know
7703 * we are functional while we are here, skip host resume in error
7704 * handling context.
7705 */
7711 /* Unit attention condition is cleared now */
7713 }
7717 /*
7718 * Current function would be generally called from the power management
7719 * callbacks hence set the RQF_PM flag so that it doesn't resume the
7720 * already suspended childs.
7721 */
7730 }
7734 out:
7738 }
7743 {
7753 else
7755 }
7756 /*
7757 * If autobkops is enabled, link can't be turned off because
7758 * turning off the link would also turn off the device.
7759 */
7762 /*
7763 * Let's make sure that link is in low power mode, we are doing
7764 * this currently by putting the link in Hibern8. Otherway to
7765 * put the link in low power mode is to send the DME end point
7766 * to device and then send the DME reset command to local
7767 * unipro. But putting the link in hibern8 is much faster.
7768 */
7772 /*
7773 * Change controller state to "reset state" which
7774 * should also put the link in off/reset state
7775 */
7777 /*
7778 * TODO: Check if we need any delay to make sure that
7779 * controller is reset
7780 */
7782 }
7784 out:
7786 }
7789 {
7790 /*
7791 * It seems some UFS devices may keep drawing more than sleep current
7792 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
7793 * To avoid this situation, add 2ms delay before putting these UFS
7794 * rails in LPM mode.
7795 */
7800 /*
7801 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
7802 * power.
7803 *
7804 * If UFS device and link is in OFF state, all power supplies (VCC,
7805 * VCCQ, VCCQ2) can be turned off if power on write protect is not
7806 * required. If UFS link is inactive (Hibern8 or OFF state) and device
7807 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
7808 *
7809 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
7810 * in low power state which would save some power.
7811 */
7820 }
7821 }
7822 }
7825 {
7839 }
7841 }
7844 vccq_lpm:
7846 vcc_disable:
7848 out:
7850 }
7853 {
7856 }
7859 {
7862 }
7864 /**
7865 * ufshcd_suspend - helper function for suspend operations
7866 * @hba: per adapter instance
7867 * @pm_op: desired low power operation type
7868 *
7869 * This function will try to put the UFS device and link into low power
7870 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
7871 * (System PM level).
7872 *
7873 * If this function is called during shutdown, it will make sure that
7874 * both UFS device and UFS link is powered off.
7875 *
7876 * NOTE: UFS device & link must be active before we enter in this function.
7877 *
7878 * Returns 0 for success and non-zero for failure
7879 */
7881 {
7896 }
7898 /*
7899 * If we can't transition into any of the low power modes
7900 * just gate the clocks.
7901 */
7909 }
7914 }
7920 /* UFS device & link must be active before we enter in this function */
7924 }
7928 /*
7929 * The device is idle with no requests in the queue,
7930 * allow background operations if bkops status shows
7931 * that performance might be impacted.
7932 */
7937 /* make sure that auto bkops is disabled */
7939 }
7940 }
7945 /* ensure that bkops is disabled */
7950 }
7959 disable_clks:
7960 /*
7961 * Call vendor specific suspend callback. As these callbacks may access
7962 * vendor specific host controller register space call them before the
7963 * host clocks are ON.
7964 */
7968 /*
7969 * Disable the host irq as host controller as there won't be any
7970 * host controller transaction expected till resume.
7971 */
7976 else
7977 /* If link is active, device ref_clk can't be switched off */
7983 /* Put the host controller in low power mode if possible */
7987 set_link_active:
7995 set_dev_active:
7998 enable_gating:
8003 out:
8008 }
8010 /**
8011 * ufshcd_resume - helper function for resume operations
8012 * @hba: per adapter instance
8013 * @pm_op: runtime PM or system PM
8014 *
8015 * This function basically brings the UFS device, UniPro link and controller
8016 * to active state.
8017 *
8018 * Returns 0 for success and non-zero for failure
8019 */
8021 {
8029 /* Make sure clocks are enabled before accessing controller */
8034 /* enable the host irq as host controller would be active soon */
8041 /*
8042 * Call vendor specific resume callback. As these callbacks may access
8043 * vendor specific host controller register space call them when the
8044 * host clocks are ON.
8045 */
8054 else
8058 /*
8059 * ufshcd_host_reset_and_restore() should have already
8060 * set the link state as active
8061 */
8064 }
8070 }
8074 else
8075 /*
8076 * If BKOPs operations are urgently needed at this moment then
8077 * keep auto-bkops enabled or else disable it.
8078 */
8086 /* Enable Auto-Hibernate if configured */
8089 /* Schedule clock gating in case of no access to UFS device yet */
8094 set_old_link_state:
8096 vendor_suspend:
8098 disable_vreg:
8100 disable_irq_and_vops_clks:
8105 out:
8110 }
8112 /**
8113 * ufshcd_system_suspend - system suspend routine
8114 * @hba: per adapter instance
8115 *
8116 * Check the description of ufshcd_suspend() function for more details.
8117 *
8118 * Returns 0 for success and non-zero for failure
8119 */
8121 {
8135 /*
8136 * UFS device and/or UFS link low power states during runtime
8137 * suspend seems to be different than what is expected during
8138 * system suspend. Hence runtime resume the devic & link and
8139 * let the system suspend low power states to take effect.
8140 * TODO: If resume takes longer time, we might have optimize
8141 * it in future by not resuming everything if possible.
8142 */
8146 }
8149 out:
8156 }
8159 /**
8160 * ufshcd_system_resume - system resume routine
8161 * @hba: per adapter instance
8162 *
8163 * Returns 0 for success and non-zero for failure
8164 */
8167 {
8175 /*
8176 * Let the runtime resume take care of resuming
8177 * if runtime suspended.
8178 */
8180 else
8182 out:
8189 }
8192 /**
8193 * ufshcd_runtime_suspend - runtime suspend routine
8194 * @hba: per adapter instance
8195 *
8196 * Check the description of ufshcd_suspend() function for more details.
8197 *
8198 * Returns 0 for success and non-zero for failure
8199 */
8201 {
8210 else
8212 out:
8217 }
8220 /**
8221 * ufshcd_runtime_resume - runtime resume routine
8222 * @hba: per adapter instance
8223 *
8224 * This function basically brings the UFS device, UniPro link and controller
8225 * to active state. Following operations are done in this function:
8226 *
8227 * 1. Turn on all the controller related clocks
8228 * 2. Bring the UniPro link out of Hibernate state
8229 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
8230 * to active state.
8231 * 4. If auto-bkops is enabled on the device, disable it.
8232 *
8233 * So following would be the possible power state after this function return
8234 * successfully:
8235 * S1: UFS device in Active state with VCC rail ON
8236 * UniPro link in Active state
8237 * All the UFS/UniPro controller clocks are ON
8238 *
8239 * Returns 0 for success and non-zero for failure
8240 */
8242 {
8251 else
8253 out:
8258 }
8262 {
8264 }
8267 /**
8268 * ufshcd_shutdown - shutdown routine
8269 * @hba: per adapter instance
8270 *
8271 * This function would power off both UFS device and UFS link.
8272 *
8273 * Returns 0 always to allow force shutdown even in case of errors.
8274 */
8276 {
8289 }
8292 out:
8295 /* allow force shutdown even in case of errors */
8297 }
8300 /**
8301 * ufshcd_remove - de-allocate SCSI host and host memory space
8302 * data structure memory
8303 * @hba: per adapter instance
8304 */
8306 {
8313 /* disable interrupts */
8322 }
8325 /**
8326 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
8327 * @hba: pointer to Host Bus Adapter (HBA)
8328 */
8330 {
8332 }
8335 /**
8336 * ufshcd_set_dma_mask - Set dma mask based on the controller
8337 * addressing capability
8338 * @hba: per adapter instance
8339 *
8340 * Returns 0 for success, non-zero for failure
8341 */
8343 {
8347 }
8349 }
8351 /**
8352 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
8353 * @dev: pointer to device handle
8354 * @hba_handle: driver private handle
8355 * Returns 0 on success, non-zero value on failure
8356 */
8358 {
8368 }
8376 }
8385 out_error:
8387 }
8390 /* This function exists because blk_mq_alloc_tag_set() requires this. */
8393 {
8396 }
8400 };
8402 /**
8403 * ufshcd_init - Driver initialization routine
8404 * @hba: per-adapter instance
8405 * @mmio_base: base register address
8406 * @irq: Interrupt line of device
8407 * Returns 0 on success, non-zero value on failure
8408 */
8410 {
8420 }
8430 /* Read capabilities registers */
8433 /* Get UFS version supported by the controller */
8443 /* Get Interrupt bit mask per version */
8450 }
8452 /* Allocate memory for host memory space */
8457 }
8459 /* Configure LRB */
8472 /* Initialize work queues */
8476 /* Initialize UIC command mutex */
8479 /* Initialize mutex for device management commands */
8488 /*
8489 * In order to avoid any spurious interrupt immediately after
8490 * registering UFS controller interrupt handler, clear any pending UFS
8491 * interrupt status and disable all the UFS interrupts.
8492 */
8494 REG_INTERRUPT_STATUS);
8496 /*
8497 * Make sure that UFS interrupts are disabled and any pending interrupt
8498 * status is cleared before registering UFS interrupt handler.
8499 */
8502 /* IRQ registration */
8509 }
8515 }
8521 }
8528 };
8536 }
8538 /* Reset the attached device */
8541 /* Host controller enable */
8548 }
8550 /*
8551 * Set the default power management level for runtime and system PM.
8552 * Default power saving mode is to keep UFS link in Hibern8 state
8553 * and UFS device in sleep state.
8554 */
8556 UFS_SLEEP_PWR_MODE,
8557 UIC_LINK_HIBERN8_STATE);
8559 UFS_SLEEP_PWR_MODE,
8560 UIC_LINK_HIBERN8_STATE);
8562 /* Set the default auto-hiberate idle timer value to 150 ms */
8566 }
8568 /* Hold auto suspend until async scan completes */
8571 /*
8572 * We are assuming that device wasn't put in sleep/power-down
8573 * state exclusively during the boot stage before kernel.
8574 * This assumption helps avoid doing link startup twice during
8575 * ufshcd_probe_hba().
8576 */
8584 free_tmf_queue:
8586 free_tmf_tag_set:
8588 free_cmd_queue:
8590 out_remove_scsi_host:
8592 exit_gating:
8595 out_disable:
8598 out_error:
8600 }