| blob | abd0e6b05f7913a9c3ba3dbac66a57abb078f74d |
1 /*
2 * Universal Flash Storage Host controller driver Core
3 *
4 * This code is based on drivers/scsi/ufs/ufshcd.c
5 * Copyright (C) 2011-2013 Samsung India Software Operations
6 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7 *
8 * Authors:
9 * Santosh Yaraganavi <santosh.sy@samsung.com>
10 * Vinayak Holikatti <h.vinayak@samsung.com>
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 * See the COPYING file in the top-level directory or visit
17 * <http://www.gnu.org/licenses/gpl-2.0.html>
18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 *
24 * This program is provided "AS IS" and "WITH ALL FAULTS" and
25 * without warranty of any kind. You are solely responsible for
26 * determining the appropriateness of using and distributing
27 * the program and assume all risks associated with your exercise
28 * of rights with respect to the program, including but not limited
29 * to infringement of third party rights, the risks and costs of
30 * program errors, damage to or loss of data, programs or equipment,
31 * and unavailability or interruption of operations. Under no
32 * circumstances will the contributor of this Program be liable for
33 * any damages of any kind arising from your use or distribution of
34 * this program.
35 *
36 * The Linux Foundation chooses to take subject only to the GPLv2
37 * license terms, and distributes only under these terms.
38 */
40 #include <linux/async.h>
41 #include <linux/devfreq.h>
42 #include <linux/nls.h>
43 #include <linux/of.h>
44 #include <linux/bitfield.h>
51 #define CREATE_TRACE_POINTS
52 #include <trace/events/ufs.h>
54 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
55 UTP_TASK_REQ_COMPL |\
56 UFSHCD_ERROR_MASK)
57 /* UIC command timeout, unit: ms */
58 #define UIC_CMD_TIMEOUT 500
60 /* NOP OUT retries waiting for NOP IN response */
61 #define NOP_OUT_RETRIES 10
62 /* Timeout after 30 msecs if NOP OUT hangs without response */
65 /* Query request retries */
66 #define QUERY_REQ_RETRIES 3
67 /* Query request timeout */
70 /* Task management command timeout */
73 /* maximum number of retries for a general UIC command */
74 #define UFS_UIC_COMMAND_RETRIES 3
76 /* maximum number of link-startup retries */
77 #define DME_LINKSTARTUP_RETRIES 3
79 /* Maximum retries for Hibern8 enter */
80 #define UIC_HIBERN8_ENTER_RETRIES 3
82 /* maximum number of reset retries before giving up */
83 #define MAX_HOST_RESET_RETRIES 5
85 /* Expose the flag value from utp_upiu_query.value */
86 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
88 /* Interrupt aggregation default timeout, unit: 40us */
89 #define INT_AGGR_DEF_TO 0x02
91 /* default delay of autosuspend: 2000 ms */
92 #define RPM_AUTOSUSPEND_DELAY_MS 2000
94 #define ufshcd_toggle_vreg(_dev, _vreg, _on) \
95 ({ \
96 int _ret; \
97 if (_on) \
98 _ret = ufshcd_enable_vreg(_dev, _vreg); \
99 else \
100 _ret = ufshcd_disable_vreg(_dev, _vreg); \
101 _ret; \
102 })
104 #define ufshcd_hex_dump(prefix_str, buf, len) do { \
105 size_t __len = (len); \
106 print_hex_dump(KERN_ERR, prefix_str, \
107 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
108 16, 4, buf, __len, false); \
109 } while (0)
113 {
131 }
139 };
141 /* UFSHCD states */
143 UFSHCD_STATE_RESET,
144 UFSHCD_STATE_ERROR,
145 UFSHCD_STATE_OPERATIONAL,
146 UFSHCD_STATE_EH_SCHEDULED,
147 };
149 /* UFSHCD error handling flags */
152 };
154 /* UFSHCD UIC layer error flags */
162 };
164 #define ufshcd_set_eh_in_progress(h) \
165 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
166 #define ufshcd_eh_in_progress(h) \
167 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
168 #define ufshcd_clear_eh_in_progress(h) \
169 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
171 #define ufshcd_set_ufs_dev_active(h) \
172 ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
173 #define ufshcd_set_ufs_dev_sleep(h) \
174 ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
175 #define ufshcd_set_ufs_dev_poweroff(h) \
176 ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
177 #define ufshcd_is_ufs_dev_active(h) \
178 ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
179 #define ufshcd_is_ufs_dev_sleep(h) \
180 ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
181 #define ufshcd_is_ufs_dev_poweroff(h) \
182 ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
191 };
195 {
197 }
201 {
203 }
208 {
215 }
217 /* if no match found, return the level 0 */
219 }
222 /* UFS cards deviations table */
224 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
226 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
228 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
230 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
232 UFS_DEVICE_QUIRK_PA_TACTIVATE),
234 UFS_DEVICE_QUIRK_PA_TACTIVATE),
236 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
238 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
240 END_FIX
241 };
264 {
266 }
269 {
273 }
274 }
277 {
281 }
282 }
285 {
288 }
291 {
294 }
298 {
302 }
306 {
310 }
314 {
320 }
324 {
333 /* trace UPIU W/O tracing command */
337 }
340 /* trace UPIU also */
344 /*
345 * Currently we only fully trace read(10) and write(10)
346 * commands
347 */
352 }
353 }
359 }
362 {
374 }
375 }
380 {
392 }
396 }
399 {
431 }
433 static
435 {
472 }
473 }
476 {
484 }
485 }
488 {
507 }
509 /**
510 * ufshcd_print_pwr_info - print power params as saved in hba
511 * power info
512 * @hba: per-adapter instance
513 */
515 {
524 };
527 __func__,
533 }
535 /*
536 * ufshcd_wait_for_register - wait for register value to change
537 * @hba - per-adapter interface
538 * @reg - mmio register offset
539 * @mask - mask to apply to read register value
540 * @val - wait condition
541 * @interval_us - polling interval in microsecs
542 * @timeout_ms - timeout in millisecs
543 * @can_sleep - perform sleep or just spin
544 *
545 * Returns -ETIMEDOUT on error, zero on success
546 */
550 {
554 /* ignore bits that we don't intend to wait on */
560 else
566 }
567 }
570 }
572 /**
573 * ufshcd_get_intr_mask - Get the interrupt bit mask
574 * @hba: Pointer to adapter instance
575 *
576 * Returns interrupt bit mask per version
577 */
579 {
594 }
597 }
599 /**
600 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
601 * @hba: Pointer to adapter instance
602 *
603 * Returns UFSHCI version supported by the controller
604 */
606 {
611 }
613 /**
614 * ufshcd_is_device_present - Check if any device connected to
615 * the host controller
616 * @hba: pointer to adapter instance
617 *
618 * Returns true if device present, false if no device detected
619 */
621 {
624 }
626 /**
627 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
628 * @lrbp: pointer to local command reference block
629 *
630 * This function is used to get the OCS field from UTRD
631 * Returns the OCS field in the UTRD
632 */
634 {
636 }
638 /**
639 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
640 * @hba: per adapter instance
641 * @pos: position of the bit to be cleared
642 */
644 {
647 else
649 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
650 }
652 /**
653 * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
654 * @hba: per adapter instance
655 * @pos: position of the bit to be cleared
656 */
658 {
661 else
663 }
665 /**
666 * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
667 * @hba: per adapter instance
668 * @tag: position of the bit to be cleared
669 */
671 {
673 }
675 /**
676 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
677 * @reg: Register value of host controller status
678 *
679 * Returns integer, 0 on Success and positive value if failed
680 */
682 {
684 }
686 /**
687 * ufshcd_get_uic_cmd_result - Get the UIC command result
688 * @hba: Pointer to adapter instance
689 *
690 * This function gets the result of UIC command completion
691 * Returns 0 on success, non zero value on error
692 */
694 {
696 MASK_UIC_COMMAND_RESULT;
697 }
699 /**
700 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
701 * @hba: Pointer to adapter instance
702 *
703 * This function gets UIC command argument3
704 * Returns 0 on success, non zero value on error
705 */
707 {
709 }
711 /**
712 * ufshcd_get_req_rsp - returns the TR response transaction type
713 * @ucd_rsp_ptr: pointer to response UPIU
714 */
717 {
719 }
721 /**
722 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
723 * @ucd_rsp_ptr: pointer to response UPIU
724 *
725 * This function gets the response status and scsi_status from response UPIU
726 * Returns the response result code.
727 */
730 {
732 }
734 /*
735 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
736 * from response UPIU
737 * @ucd_rsp_ptr: pointer to response UPIU
738 *
739 * Return the data segment length.
740 */
743 {
745 MASK_RSP_UPIU_DATA_SEG_LEN;
746 }
748 /**
749 * ufshcd_is_exception_event - Check if the device raised an exception event
750 * @ucd_rsp_ptr: pointer to response UPIU
751 *
752 * The function checks if the device raised an exception event indicated in
753 * the Device Information field of response UPIU.
754 *
755 * Returns true if exception is raised, false otherwise.
756 */
758 {
761 }
763 /**
764 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
765 * @hba: per adapter instance
766 */
769 {
771 INT_AGGR_COUNTER_AND_TIMER_RESET,
772 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
773 }
775 /**
776 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
777 * @hba: per adapter instance
778 * @cnt: Interrupt aggregation counter threshold
779 * @tmout: Interrupt aggregation timeout value
780 */
783 {
787 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
788 }
790 /**
791 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
792 * @hba: per adapter instance
793 */
795 {
797 }
799 /**
800 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
801 * When run-stop registers are set to 1, it indicates the
802 * host controller that it can process the requests
803 * @hba: per adapter instance
804 */
806 {
808 REG_UTP_TASK_REQ_LIST_RUN_STOP);
810 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
811 }
813 /**
814 * ufshcd_hba_start - Start controller initialization sequence
815 * @hba: per adapter instance
816 */
818 {
820 }
822 /**
823 * ufshcd_is_hba_active - Get controller state
824 * @hba: per adapter instance
825 *
826 * Returns false if controller is active, true otherwise
827 */
829 {
832 }
835 {
836 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
840 else
842 }
846 {
847 /*
848 * If both host and device support UniPro ver1.6 or later, PA layer
849 * parameters tuning happens during link startup itself.
850 *
851 * We can manually tune PA layer parameters if either host or device
852 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
853 * logic simple, we will only do manual tuning if local unipro version
854 * doesn't support ver1.6 or later.
855 */
858 else
860 }
863 {
890 }
909 }
915 }
916 }
919 }
923 out:
929 }
931 /**
932 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
933 * @hba: per adapter instance
934 * @scale_up: True if scaling up and false if scaling down
935 *
936 * Returns true if scaling is required, false otherwise.
937 */
940 {
957 }
958 }
959 }
962 }
965 u64 wait_timeout_us)
966 {
969 u32 tm_doorbell;
970 u32 tr_doorbell;
972 ktime_t start;
976 /*
977 * Wait for all the outstanding tasks/transfer requests.
978 * Verify by checking the doorbell registers are clear.
979 */
985 }
994 }
999 wait_timeout_us) {
1001 /*
1002 * We might have scheduled out for long time so make
1003 * sure to check if doorbells are cleared by this time
1004 * or not.
1005 */
1007 }
1016 }
1017 out:
1021 }
1023 /**
1024 * ufshcd_scale_gear - scale up/down UFS gear
1025 * @hba: per adapter instance
1026 * @scale_up: True for scaling up gear and false for scaling down
1027 *
1028 * Returns 0 for success,
1029 * Returns -EBUSY if scaling can't happen at this time
1030 * Returns non-zero for any other errors
1031 */
1033 {
1034 #define UFS_MIN_GEAR_TO_SCALE_DOWN UFS_HS_G1
1047 /* save the current power mode */
1052 /* scale down gear */
1055 }
1056 }
1058 /* check if the power mode needs to be changed or not? */
1068 }
1071 {
1074 /*
1075 * make sure that there are no outstanding requests when
1076 * clock scaling is in progress
1077 */
1084 }
1087 }
1090 {
1093 }
1095 /**
1096 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1097 * @hba: per adapter instance
1098 * @scale_up: True for scaling up and false for scalin down
1099 *
1100 * Returns 0 for success,
1101 * Returns -EBUSY if scaling can't happen at this time
1102 * Returns non-zero for any other errors
1103 */
1105 {
1108 /* let's not get into low power until clock scaling is completed */
1115 /* scale down the gear before scaling down clocks */
1120 }
1127 }
1129 /* scale up the gear after scaling up clocks */
1135 }
1136 }
1140 out:
1144 }
1147 {
1156 }
1161 }
1164 {
1173 }
1178 }
1182 {
1185 ktime_t start;
1198 }
1206 }
1214 }
1224 out:
1230 }
1233 {
1239 }
1241 /* Whether or not any tag is in use by a request that is in progress. */
1243 {
1249 }
1253 {
1274 start_window:
1284 }
1287 }
1293 };
1296 {
1302 /* Skip devfreq if we don't have any clocks in the list */
1312 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1313 NULL);
1321 }
1326 }
1329 {
1342 }
1345 {
1352 }
1355 {
1366 }
1371 }
1374 {
1385 }
1390 }
1394 {
1398 }
1402 {
1404 u32 value;
1430 }
1434 out:
1436 }
1439 {
1447 }
1450 {
1462 }
1469 /* Exit from hibern8 */
1471 /* Prevent gating in this path */
1478 else
1480 }
1482 }
1483 unblock_reqs:
1485 }
1487 /**
1488 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1489 * Also, exit from hibern8 mode and set the link as active.
1490 * @hba: per adapter instance
1491 * @async: This indicates whether caller should ungate clocks asynchronously.
1492 */
1494 {
1506 }
1508 start:
1511 /*
1512 * Wait for the ungate work to complete if in progress.
1513 * Though the clocks may be in ON state, the link could
1514 * still be in hibner8 state if hibern8 is allowed
1515 * during clock gating.
1516 * Make sure we exit hibern8 state also in addition to
1517 * clocks being ON.
1518 */
1525 }
1533 }
1534 /*
1535 * If we are here, it means gating work is either done or
1536 * currently running. Hence, fall through to cancel gating
1537 * work and to enable clocks.
1538 */
1539 /* fallthrough */
1547 /*
1548 * fall through to check if we should wait for this
1549 * work to be done or not.
1550 */
1551 /* fallthrough */
1557 }
1561 /* Make sure state is CLKS_ON before returning */
1568 }
1570 out:
1572 }
1576 {
1582 /*
1583 * In case you are here to cancel this work the gating state
1584 * would be marked as REQ_CLKS_ON. In this case save time by
1585 * skipping the gating work and exit after changing the clock
1586 * state to CLKS_ON.
1587 */
1594 }
1604 /* put the link into hibern8 mode before turning off clocks */
1611 }
1613 }
1619 else
1620 /* If link is active, device ref_clk can't be switched off */
1623 /*
1624 * In case you are here to cancel this work the gating state
1625 * would be marked as REQ_CLKS_ON. In this case keep the state
1626 * as REQ_CLKS_ON which would anyway imply that clocks are off
1627 * and a request to turn them on is pending. By doing this way,
1628 * we keep the state machine in tact and this would ultimately
1629 * prevent from doing cancel work multiple times when there are
1630 * new requests arriving before the current cancel work is done.
1631 */
1637 }
1638 rel_lock:
1640 out:
1642 }
1644 /* host lock must be held before calling this variant */
1646 {
1664 }
1667 {
1673 }
1678 {
1682 }
1686 {
1697 }
1701 {
1705 }
1709 {
1712 u32 value;
1727 }
1730 out:
1732 }
1735 {
1742 ufshcd_clk_scaling_suspend_work);
1744 ufshcd_clk_scaling_resume_work);
1751 }
1754 {
1760 }
1763 {
1776 WQ_MEM_RECLAIM);
1795 }
1798 {
1806 }
1808 /* Must be called with host lock acquired */
1810 {
1830 }
1835 }
1836 }
1839 {
1850 }
1851 }
1852 /**
1853 * ufshcd_send_command - Send SCSI or device management commands
1854 * @hba: per adapter instance
1855 * @task_tag: Task tag of the command
1856 */
1859 {
1866 /* Make sure that doorbell is committed immediately */
1868 }
1870 /**
1871 * ufshcd_copy_sense_data - Copy sense data in case of check condition
1872 * @lrbp: pointer to local reference block
1873 */
1875 {
1885 len_to_copy);
1886 }
1887 }
1889 /**
1890 * ufshcd_copy_query_response() - Copy the Query Response and the data
1891 * descriptor
1892 * @hba: per adapter instance
1893 * @lrbp: pointer to local reference block
1894 */
1895 static
1897 {
1902 /* Get the descriptor */
1906 GENERAL_UPIU_REQUEST_SIZE;
1907 u16 resp_len;
1908 u16 buf_len;
1910 /* data segment length */
1912 MASK_QUERY_DATA_SEG_LEN;
1922 }
1923 }
1926 }
1928 /**
1929 * ufshcd_hba_capabilities - Read controller capabilities
1930 * @hba: per adapter instance
1931 */
1933 {
1936 /* nutrs and nutmrs are 0 based values */
1940 }
1942 /**
1943 * ufshcd_ready_for_uic_cmd - Check if controller is ready
1944 * to accept UIC commands
1945 * @hba: per adapter instance
1946 * Return true on success, else false
1947 */
1949 {
1952 else
1954 }
1956 /**
1957 * ufshcd_get_upmcrs - Get the power mode change request status
1958 * @hba: Pointer to adapter instance
1959 *
1960 * This function gets the UPMCRS field of HCS register
1961 * Returns value of UPMCRS field
1962 */
1964 {
1966 }
1968 /**
1969 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
1970 * @hba: per adapter instance
1971 * @uic_cmd: UIC command
1972 *
1973 * Mutex must be held.
1974 */
1977 {
1982 /* Write Args */
1987 /* Write UIC Cmd */
1989 REG_UIC_COMMAND);
1990 }
1992 /**
1993 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
1994 * @hba: per adapter instance
1995 * @uic_cmd: UIC command
1996 *
1997 * Must be called with mutex held.
1998 * Returns 0 only if success.
1999 */
2000 static int
2002 {
2009 else
2017 }
2019 /**
2020 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2021 * @hba: per adapter instance
2022 * @uic_cmd: UIC command
2023 * @completion: initialize the completion only if this is set to true
2024 *
2025 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
2026 * with mutex held and host_lock locked.
2027 * Returns 0 only if success.
2028 */
2029 static int
2032 {
2037 }
2045 }
2047 /**
2048 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2049 * @hba: per adapter instance
2050 * @uic_cmd: UIC command
2051 *
2052 * Returns 0 only if success.
2053 */
2055 {
2073 }
2075 /**
2076 * ufshcd_map_sg - Map scatter-gather list to prdt
2077 * @hba: per adapter instance
2078 * @lrbp: pointer to local reference block
2079 *
2080 * Returns 0 in case of success, non-zero value in case of failure
2081 */
2083 {
2100 else
2114 }
2117 }
2120 }
2122 /**
2123 * ufshcd_enable_intr - enable interrupts
2124 * @hba: per adapter instance
2125 * @intrs: interrupt bits
2126 */
2128 {
2132 u32 rw;
2137 }
2140 }
2142 /**
2143 * ufshcd_disable_intr - disable interrupts
2144 * @hba: per adapter instance
2145 * @intrs: interrupt bits
2146 */
2148 {
2152 u32 rw;
2159 }
2162 }
2164 /**
2165 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2166 * descriptor according to request
2167 * @lrbp: pointer to local reference block
2168 * @upiu_flags: flags required in the header
2169 * @cmd_dir: requests data direction
2170 */
2173 {
2175 u32 data_direction;
2176 u32 dword_0;
2187 }
2194 /* Transfer request descriptor header fields */
2196 /* dword_1 is reserved, hence it is set to 0 */
2198 /*
2199 * assigning invalid value for command status. Controller
2200 * updates OCS on command completion, with the command
2201 * status
2202 */
2205 /* dword_3 is reserved, hence it is set to 0 */
2209 }
2211 /**
2212 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2213 * for scsi commands
2214 * @lrbp: local reference block pointer
2215 * @upiu_flags: flags
2216 */
2217 static
2219 {
2224 /* command descriptor fields */
2231 /* Total EHS length and Data segment length will be zero */
2241 }
2243 /**
2244 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2245 * for query requsts
2246 * @hba: UFS hba
2247 * @lrbp: local reference block pointer
2248 * @upiu_flags: flags
2249 */
2252 {
2257 /* Query request header */
2264 /* Data segment length only need for WRITE_DESC */
2268 else
2271 /* Copy the Query Request buffer as is */
2273 QUERY_OSF_SIZE);
2275 /* Copy the Descriptor */
2280 }
2283 {
2288 /* command descriptor fields */
2292 /* clear rest of the fields of basic header */
2297 }
2299 /**
2300 * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
2301 * for Device Management Purposes
2302 * @hba: per adapter instance
2303 * @lrbp: pointer to local reference block
2304 */
2306 {
2307 u32 upiu_flags;
2313 else
2321 else
2325 }
2327 /**
2328 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2329 * for SCSI Purposes
2330 * @hba: per adapter instance
2331 * @lrbp: pointer to local reference block
2332 */
2334 {
2335 u32 upiu_flags;
2341 else
2350 }
2353 }
2355 /**
2356 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2357 * @upiu_wlun_id: UPIU W-LUN id
2358 *
2359 * Returns SCSI W-LUN id
2360 */
2362 {
2364 }
2366 /**
2367 * ufshcd_queuecommand - main entry point for SCSI requests
2368 * @host: SCSI host pointer
2369 * @cmd: command from SCSI Midlayer
2370 *
2371 * Returns 0 for success, non-zero in case of failure
2372 */
2374 {
2389 }
2412 }
2414 /* if error handling is in progress, don't issue commands */
2419 }
2428 }
2449 }
2450 /* Make sure descriptors are ready before ringing the doorbell */
2453 /* issue command to the controller */
2457 out_unlock:
2459 out:
2462 }
2466 {
2476 }
2478 static int
2480 {
2485 /* clear outstanding transaction before retry */
2490 /*
2491 * wait for for h/w to clear corresponding bit in door-bell.
2492 * max. wait is 1 sec.
2493 */
2495 REG_UTP_TRANSFER_REQ_DOOR_BELL,
2499 }
2501 static int
2503 {
2506 /* Get the UPIU response */
2508 UPIU_RSP_CODE_OFFSET;
2510 }
2512 /**
2513 * ufshcd_dev_cmd_completion() - handles device management command responses
2514 * @hba: per adapter instance
2515 * @lrbp: pointer to local reference block
2516 */
2517 static int
2519 {
2532 }
2540 /* TODO: handle Reject UPIU Response */
2543 __func__);
2550 }
2553 }
2557 {
2565 /* Make sure descriptors are ready before ringing the doorbell */
2573 }
2581 /* successfully cleared the command, retry if needed */
2583 /*
2584 * in case of an error, after clearing the doorbell,
2585 * we also need to clear the outstanding_request
2586 * field in hba
2587 */
2589 }
2592 }
2594 /**
2595 * ufshcd_exec_dev_cmd - API for sending device management requests
2596 * @hba: UFS hba
2597 * @cmd_type: specifies the type (NOP, Query...)
2598 * @timeout: time in seconds
2599 *
2600 * NOTE: Since there is only one available tag for device management commands,
2601 * it is expected you hold the hba->dev_cmd.lock mutex.
2602 */
2605 {
2616 /*
2617 * Get free slot, sleep if slots are unavailable.
2618 * Even though we use wait_event() which sleeps indefinitely,
2619 * the maximum wait time is bounded by SCSI request timeout.
2620 */
2625 }
2639 /* Make sure descriptors are ready before ringing the doorbell */
2651 out_put_tag:
2653 out_unlock:
2656 }
2658 /**
2659 * ufshcd_init_query() - init the query response and request parameters
2660 * @hba: per-adapter instance
2661 * @request: address of the request pointer to be initialized
2662 * @response: address of the response pointer to be initialized
2663 * @opcode: operation to perform
2664 * @idn: flag idn to access
2665 * @index: LU number to access
2666 * @selector: query/flag/descriptor further identification
2667 */
2671 {
2680 }
2684 {
2694 else
2696 }
2703 }
2705 /**
2706 * ufshcd_query_flag() - API function for sending flag query requests
2707 * @hba: per-adapter instance
2708 * @opcode: flag query to perform
2709 * @idn: flag idn to access
2710 * @flag_res: the flag value after the query request completes
2711 *
2712 * Returns 0 for success, non-zero in case of failure
2713 */
2716 {
2727 selector);
2738 /* No dummy reads */
2740 __func__);
2743 }
2751 }
2760 }
2766 out_unlock:
2770 }
2772 /**
2773 * ufshcd_query_attr - API function for sending attribute requests
2774 * @hba: per-adapter instance
2775 * @opcode: attribute opcode
2776 * @idn: attribute idn to access
2777 * @index: index field
2778 * @selector: selector field
2779 * @attr_val: the attribute value after the query request completes
2780 *
2781 * Returns 0 for success, non-zero in case of failure
2782 */
2785 {
2798 }
2802 selector);
2817 }
2825 }
2829 out_unlock:
2831 out:
2834 }
2836 /**
2837 * ufshcd_query_attr_retry() - API function for sending query
2838 * attribute with retries
2839 * @hba: per-adapter instance
2840 * @opcode: attribute opcode
2841 * @idn: attribute idn to access
2842 * @index: index field
2843 * @selector: selector field
2844 * @attr_val: the attribute value after the query request
2845 * completes
2846 *
2847 * Returns 0 for success, non-zero in case of failure
2848 */
2852 {
2854 u32 retries;
2862 else
2864 }
2871 }
2876 {
2889 }
2896 }
2900 selector);
2917 }
2925 }
2929 out_unlock:
2932 out:
2935 }
2937 /**
2938 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
2939 * @hba: per-adapter instance
2940 * @opcode: attribute opcode
2941 * @idn: attribute idn to access
2942 * @index: index field
2943 * @selector: selector field
2944 * @desc_buf: the buffer that contains the descriptor
2945 * @buf_len: length parameter passed to the device
2946 *
2947 * Returns 0 for success, non-zero in case of failure.
2948 * The buf_len parameter will contain, on return, the length parameter
2949 * received on the response.
2950 */
2954 u8 selector,
2956 {
2965 }
2968 }
2970 /**
2971 * ufshcd_read_desc_length - read the specified descriptor length from header
2972 * @hba: Pointer to adapter instance
2973 * @desc_id: descriptor idn value
2974 * @desc_index: descriptor index
2975 * @desc_length: pointer to variable to read the length of descriptor
2976 *
2977 * Return 0 in case of success, non-zero otherwise
2978 */
2983 {
3002 desc_id);
3004 }
3009 }
3011 /**
3012 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3013 * @hba: Pointer to adapter instance
3014 * @desc_id: descriptor idn value
3015 * @desc_len: mapped desc length (out)
3016 *
3017 * Return 0 in case of success, non-zero otherwise
3018 */
3021 {
3054 }
3056 }
3059 /**
3060 * ufshcd_read_desc_param - read the specified descriptor parameter
3061 * @hba: Pointer to adapter instance
3062 * @desc_id: descriptor idn value
3063 * @desc_index: descriptor index
3064 * @param_offset: offset of the parameter to read
3065 * @param_read_buf: pointer to buffer where parameter would be read
3066 * @param_size: sizeof(param_read_buf)
3067 *
3068 * Return 0 in case of success, non-zero otherwise
3069 */
3073 u8 param_offset,
3075 u8 param_size)
3076 {
3082 /* Safety check */
3086 /* Get the max length of descriptor from structure filled up at probe
3087 * time.
3088 */
3091 /* Sanity checks */
3094 __func__);
3096 }
3098 /* Check whether we need temp memory */
3106 }
3108 /* Request for full descriptor */
3114 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3117 }
3119 /* Sanity check */
3125 }
3127 /* Check wherher we will not copy more data, than available */
3133 out:
3137 }
3143 u32 size)
3144 {
3146 }
3149 /**
3150 * struct uc_string_id - unicode string
3151 *
3152 * @len: size of this descriptor inclusive
3153 * @type: descriptor type
3154 * @uc: unicode string character
3155 */
3157 u8 len;
3158 u8 type;
3162 /* replace non-printable or non-ASCII characters with spaces */
3164 {
3166 }
3168 /**
3169 * ufshcd_read_string_desc - read string descriptor
3170 * @hba: pointer to adapter instance
3171 * @desc_index: descriptor index
3172 * @buf: pointer to buffer where descriptor would be read,
3173 * the caller should free the memory.
3174 * @ascii: if true convert from unicode to ascii characters
3175 * null terminated string.
3176 *
3177 * Return:
3178 * * string size on success.
3179 * * -ENOMEM: on allocation failure
3180 * * -EINVAL: on a wrong parameter
3181 */
3184 {
3198 QUERY_DESC_MAX_SIZE);
3204 }
3211 }
3214 ssize_t ascii_len;
3216 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3222 }
3224 /*
3225 * the descriptor contains string in UTF16 format
3226 * we need to convert to utf-8 so it can be displayed
3227 */
3232 /* replace non-printable or non-ASCII characters with spaces */
3243 }
3245 }
3246 out:
3250 }
3252 /**
3253 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3254 * @hba: Pointer to adapter instance
3255 * @lun: lun id
3256 * @param_offset: offset of the parameter to read
3257 * @param_read_buf: pointer to buffer where parameter would be read
3258 * @param_size: sizeof(param_read_buf)
3259 *
3260 * Return 0 in case of success, non-zero otherwise
3261 */
3266 u32 param_size)
3267 {
3268 /*
3269 * Unit descriptors are only available for general purpose LUs (LUN id
3270 * from 0 to 7) and RPMB Well known LU.
3271 */
3277 }
3279 /**
3280 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3281 * @hba: per adapter instance
3282 *
3283 * 1. Allocate DMA memory for Command Descriptor array
3284 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3285 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3286 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3287 * (UTMRDL)
3288 * 4. Allocate memory for local reference block(lrb).
3289 *
3290 * Returns 0 for success, non-zero in case of failure
3291 */
3293 {
3296 /* Allocate memory for UTP command descriptors */
3299 ucdl_size,
3301 GFP_KERNEL);
3303 /*
3304 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3305 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3306 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3307 * be aligned to 128 bytes as well
3308 */
3314 }
3316 /*
3317 * Allocate memory for UTP Transfer descriptors
3318 * UFSHCI requires 1024 byte alignment of UTRD
3319 */
3322 utrdl_size,
3324 GFP_KERNEL);
3330 }
3332 /*
3333 * Allocate memory for UTP Task Management descriptors
3334 * UFSHCI requires 1024 byte alignment of UTMRD
3335 */
3338 utmrdl_size,
3340 GFP_KERNEL);
3346 }
3348 /* Allocate memory for local reference block */
3351 GFP_KERNEL);
3355 }
3357 out:
3359 }
3361 /**
3362 * ufshcd_host_memory_configure - configure local reference block with
3363 * memory offsets
3364 * @hba: per adapter instance
3365 *
3366 * Configure Host memory space
3367 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3368 * address.
3369 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3370 * and PRDT offset.
3371 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3372 * into local reference block.
3373 */
3375 {
3378 dma_addr_t cmd_desc_dma_addr;
3379 dma_addr_t cmd_desc_element_addr;
3380 u16 response_offset;
3381 u16 prdt_offset;
3388 response_offset =
3390 prdt_offset =
3397 /* Configure UTRD with command descriptor base address */
3398 cmd_desc_element_addr =
3405 /* Response upiu and prdt offset should be in double words */
3420 }
3431 response_offset;
3435 prdt_offset;
3436 }
3437 }
3439 /**
3440 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3441 * @hba: per adapter instance
3442 *
3443 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3444 * in order to initialize the Unipro link startup procedure.
3445 * Once the Unipro links are up, the device connected to the controller
3446 * is detected.
3447 *
3448 * Returns 0 on success, non-zero value on failure
3449 */
3451 {
3462 }
3463 /**
3464 * ufshcd_dme_reset - UIC command for DME_RESET
3465 * @hba: per adapter instance
3466 *
3467 * DME_RESET command is issued in order to reset UniPro stack.
3468 * This function now deal with cold reset.
3469 *
3470 * Returns 0 on success, non-zero value on failure
3471 */
3473 {
3485 }
3487 /**
3488 * ufshcd_dme_enable - UIC command for DME_ENABLE
3489 * @hba: per adapter instance
3490 *
3491 * DME_ENABLE command is issued in order to enable UniPro stack.
3492 *
3493 * Returns 0 on success, non-zero value on failure
3494 */
3496 {
3508 }
3511 {
3512 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3518 /*
3519 * last_dme_cmd_tstamp will be 0 only for 1st call to
3520 * this function
3521 */
3531 min_sleep_time_us =
3533 else
3535 }
3537 /* allow sleep for extra 50us if needed */
3539 }
3541 /**
3542 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3543 * @hba: per adapter instance
3544 * @attr_sel: uic command argument1
3545 * @attr_set: attribute set type as uic command argument2
3546 * @mib_val: setting value as uic command argument3
3547 * @peer: indicate whether peer or local
3548 *
3549 * Returns 0 on success, non-zero value on failure
3550 */
3553 {
3557 "dme-peer-set"
3558 };
3570 /* for peer attributes we retry upon failure */
3583 }
3586 /**
3587 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3588 * @hba: per adapter instance
3589 * @attr_sel: uic command argument1
3590 * @mib_val: the value of the attribute as returned by the UIC command
3591 * @peer: indicate whether peer or local
3592 *
3593 * Returns 0 on success, non-zero value on failure
3594 */
3597 {
3601 "dme-peer-get"
3602 };
3624 }
3629 }
3630 }
3637 /* for peer attributes we retry upon failure */
3655 out:
3657 }
3660 /**
3661 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3662 * state) and waits for it to take effect.
3663 *
3664 * @hba: per adapter instance
3665 * @cmd: UIC command to execute
3666 *
3667 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3668 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3669 * and device UniPro link and hence it's final completion would be indicated by
3670 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3671 * addition to normal UIC command completion Status (UCCS). This function only
3672 * returns after the relevant status bits indicate the completion.
3673 *
3674 * Returns 0 on success, non-zero value on failure
3675 */
3677 {
3680 u8 status;
3692 /*
3693 * Make sure UIC command completion interrupt is disabled before
3694 * issuing UIC command.
3695 */
3698 }
3706 }
3715 }
3723 }
3724 out:
3729 }
3740 }
3742 /**
3743 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
3744 * using DME_SET primitives.
3745 * @hba: per adapter instance
3746 * @mode: powr mode value
3747 *
3748 * Returns 0 on success, non-zero value on failure
3749 */
3751 {
3762 }
3763 }
3772 out:
3774 }
3777 {
3786 /* Reset the attached device */
3802 }
3805 {
3823 /*
3824 * If link recovery fails then return error code returned from
3825 * ufshcd_link_recovery().
3826 * If link recovery succeeds then return -EAGAIN to attempt
3827 * hibern8 enter retry again.
3828 */
3835 }
3838 POST_CHANGE);
3841 }
3844 {
3851 }
3852 out:
3854 }
3857 {
3875 POST_CHANGE);
3878 }
3881 }
3885 {
3897 out_unlock:
3899 }
3903 {
3912 }
3914 /**
3915 * ufshcd_init_pwr_info - setting the POR (power on reset)
3916 * values in hba power info
3917 * @hba: per-adapter instance
3918 */
3920 {
3928 }
3930 /**
3931 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
3932 * @hba: per-adapter instance
3933 */
3935 {
3945 /* Get the connected lane count */
3953 __func__,
3957 }
3959 /*
3960 * First, get the maximum gears of HS speed.
3961 * If a zero value, it means there is no HSGEAR capability.
3962 * Then, get the maximum gears of PWM speed.
3963 */
3972 }
3974 }
3985 }
3987 }
3991 }
3995 {
3998 /* if already configured to the requested pwr_mode */
4008 }
4010 /*
4011 * Configure attributes for power mode change with below.
4012 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4013 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4014 * - PA_HSSERIES
4015 */
4022 else
4031 else
4042 DL_FC0ProtectionTimeOutVal_Default);
4044 DL_TC0ReplayTimeOutVal_Default);
4046 DL_AFC0ReqTimeOutVal_Default);
4048 DL_FC1ProtectionTimeOutVal_Default);
4050 DL_TC1ReplayTimeOutVal_Default);
4052 DL_AFC1ReqTimeOutVal_Default);
4055 DL_FC0ProtectionTimeOutVal_Default);
4057 DL_TC0ReplayTimeOutVal_Default);
4059 DL_AFC0ReqTimeOutVal_Default);
4069 pwr_mode);
4073 }
4076 }
4078 /**
4079 * ufshcd_config_pwr_mode - configure a new power mode
4080 * @hba: per-adapter instance
4081 * @desired_pwr_mode: desired power configuration
4082 */
4085 {
4100 }
4103 /**
4104 * ufshcd_complete_dev_init() - checks device readiness
4105 * @hba: per-adapter instance
4106 *
4107 * Set fDeviceInit flag and poll until device toggles it.
4108 */
4110 {
4122 }
4124 /* poll for max. 1000 iterations for fDeviceInit flag to clear */
4136 __func__);
4138 out:
4140 }
4142 /**
4143 * ufshcd_make_hba_operational - Make UFS controller operational
4144 * @hba: per adapter instance
4145 *
4146 * To bring UFS host controller to operational state,
4147 * 1. Enable required interrupts
4148 * 2. Configure interrupt aggregation
4149 * 3. Program UTRL and UTMRL base address
4150 * 4. Configure run-stop-registers
4151 *
4152 * Returns 0 on success, non-zero value on failure
4153 */
4155 {
4157 u32 reg;
4159 /* Enable required interrupts */
4162 /* Configure interrupt aggregation */
4165 else
4168 /* Configure UTRL and UTMRL base address registers */
4170 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4172 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4174 REG_UTP_TASK_REQ_LIST_BASE_L);
4176 REG_UTP_TASK_REQ_LIST_BASE_H);
4178 /*
4179 * Make sure base address and interrupt setup are updated before
4180 * enabling the run/stop registers below.
4181 */
4184 /*
4185 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4186 */
4195 }
4197 out:
4199 }
4202 /**
4203 * ufshcd_hba_stop - Send controller to reset state
4204 * @hba: per adapter instance
4205 * @can_sleep: perform sleep or just spin
4206 */
4208 {
4217 }
4219 /**
4220 * ufshcd_hba_execute_hce - initialize the controller
4221 * @hba: per adapter instance
4222 *
4223 * The controller resets itself and controller firmware initialization
4224 * sequence kicks off. When controller is ready it will set
4225 * the Host Controller Enable bit to 1.
4226 *
4227 * Returns 0 on success, non-zero value on failure
4228 */
4230 {
4234 /* change controller state to "reset state" */
4237 /* UniPro link is disabled at this point */
4242 /* start controller initialization sequence */
4245 /*
4246 * To initialize a UFS host controller HCE bit must be set to 1.
4247 * During initialization the HCE bit value changes from 1->0->1.
4248 * When the host controller completes initialization sequence
4249 * it sets the value of HCE bit to 1. The same HCE bit is read back
4250 * to check if the controller has completed initialization sequence.
4251 * So without this delay the value HCE = 1, set in the previous
4252 * instruction might be read back.
4253 * This delay can be changed based on the controller.
4254 */
4257 /* wait for the host controller to complete initialization */
4261 retry--;
4266 }
4268 }
4270 /* enable UIC related interrupts */
4276 }
4279 {
4286 /* enable UIC related interrupts */
4296 }
4299 }
4302 }
4306 {
4312 else
4321 else
4330 }
4331 }
4334 }
4337 {
4339 }
4342 u32 reg)
4343 {
4347 }
4350 /**
4351 * ufshcd_link_startup - Initialize unipro link startup
4352 * @hba: per adapter instance
4353 *
4354 * Returns 0 for success, non-zero in case of failure
4355 */
4357 {
4362 /*
4363 * If UFS device isn't active then we will have to issue link startup
4364 * 2 times to make sure the device state move to active.
4365 */
4369 link_startup:
4375 /* check if device is detected by inter-connect layer */
4382 }
4384 /*
4385 * DME link lost indication is only received when link is up,
4386 * but we can't be sure if the link is up until link startup
4387 * succeeds. So reset the local Uni-Pro and try again.
4388 */
4393 }
4397 /* failed to get the link up... retire */
4401 }
4407 }
4409 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4417 }
4419 /* Include any host controller configuration via UIC commands */
4425 out:
4431 }
4433 }
4435 /**
4436 * ufshcd_verify_dev_init() - Verify device initialization
4437 * @hba: per-adapter instance
4438 *
4439 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4440 * device Transport Protocol (UTP) layer is ready after a reset.
4441 * If the UTP layer at the device side is not initialized, it may
4442 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4443 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4444 */
4446 {
4454 NOP_OUT_TIMEOUT);
4460 }
4467 }
4469 /**
4470 * ufshcd_set_queue_depth - set lun queue depth
4471 * @sdev: pointer to SCSI device
4472 *
4473 * Read bLUQueueDepth value and activate scsi tagged command
4474 * queueing. For WLUN, queue depth is set to 1. For best-effort
4475 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4476 * value that host can queue.
4477 */
4479 {
4481 u8 lun_qdepth;
4489 UNIT_DESC_PARAM_LU_Q_DEPTH,
4493 /* Some WLUN doesn't support unit descriptor */
4497 /* eventually, we can figure out the real queue depth */
4499 else
4505 }
4507 /*
4508 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4509 * @hba: per-adapter instance
4510 * @lun: UFS device lun id
4511 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4512 *
4513 * Returns 0 in case of success and b_lu_write_protect status would be returned
4514 * @b_lu_write_protect parameter.
4515 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4516 * Returns -EINVAL in case of invalid parameters passed to this function.
4517 */
4519 u8 lun,
4521 {
4526 /*
4527 * According to UFS device spec, RPMB LU can't be write
4528 * protected so skip reading bLUWriteProtect parameter for
4529 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4530 */
4533 else
4535 lun,
4536 UNIT_DESC_PARAM_LU_WR_PROTECT,
4537 b_lu_write_protect,
4540 }
4542 /**
4543 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4544 * status
4545 * @hba: per-adapter instance
4546 * @sdev: pointer to SCSI device
4547 *
4548 */
4551 {
4554 u8 b_lu_write_protect;
4560 }
4561 }
4563 /**
4564 * ufshcd_slave_alloc - handle initial SCSI device configurations
4565 * @sdev: pointer to SCSI device
4566 *
4567 * Returns success
4568 */
4570 {
4575 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4578 /* DBD field should be set to 1 in mode sense(10) */
4581 /* allow SCSI layer to restart the device in case of errors */
4584 /* REPORT SUPPORTED OPERATION CODES is not supported */
4587 /* WRITE_SAME command is not supported */
4595 }
4597 /**
4598 * ufshcd_change_queue_depth - change queue depth
4599 * @sdev: pointer to SCSI device
4600 * @depth: required depth to set
4601 *
4602 * Change queue depth and make sure the max. limits are not crossed.
4603 */
4605 {
4611 }
4613 /**
4614 * ufshcd_slave_configure - adjust SCSI device configurations
4615 * @sdev: pointer to SCSI device
4616 */
4618 {
4628 }
4630 /**
4631 * ufshcd_slave_destroy - remove SCSI device configurations
4632 * @sdev: pointer to SCSI device
4633 */
4635 {
4639 /* Drop the reference as it won't be needed anymore */
4646 }
4647 }
4649 /**
4650 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
4651 * @lrbp: pointer to local reference block of completed command
4652 * @scsi_status: SCSI command status
4653 *
4654 * Returns value base on SCSI command status
4655 */
4658 {
4664 /* fallthrough */
4668 scsi_status;
4682 }
4684 /**
4685 * ufshcd_transfer_rsp_status - Get overall status of the response
4686 * @hba: per adapter instance
4687 * @lrbp: pointer to local reference block of completed command
4688 *
4689 * Returns result of the command to notify SCSI midlayer
4690 */
4693 {
4698 /* overall command status of utrd */
4707 /*
4708 * get the response UPIU result to extract
4709 * the SCSI command status
4710 */
4713 /*
4714 * get the result based on SCSI status response
4715 * to notify the SCSI midlayer of the command status
4716 */
4720 /*
4721 * Currently we are only supporting BKOPs exception
4722 * events hence we can ignore BKOPs exception event
4723 * during power management callbacks. BKOPs exception
4724 * event is not expected to be raised in runtime suspend
4725 * callback as it allows the urgent bkops.
4726 * During system suspend, we are anyway forcefully
4727 * disabling the bkops and if urgent bkops is needed
4728 * it will be enabled on system resume. Long term
4729 * solution could be to abort the system suspend if
4730 * UFS device needs urgent BKOPs.
4731 */
4737 /* TODO: handle Reject UPIU Response */
4745 result);
4748 }
4775 }
4777 /**
4778 * ufshcd_uic_cmd_compl - handle completion of uic command
4779 * @hba: per adapter instance
4780 * @intr_status: interrupt status generated by the controller
4781 *
4782 * Returns
4783 * IRQ_HANDLED - If interrupt is valid
4784 * IRQ_NONE - If invalid interrupt
4785 */
4787 {
4797 }
4802 }
4804 }
4806 /**
4807 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
4808 * @hba: per adapter instance
4809 * @completed_reqs: requests to complete
4810 */
4813 {
4827 /* Mark completed command as NULL in LRB */
4830 /* Do not touch lrbp after scsi done */
4840 }
4841 }
4844 }
4846 /* clear corresponding bits of completed commands */
4850 }
4852 /**
4853 * ufshcd_transfer_req_compl - handle SCSI and query command completion
4854 * @hba: per adapter instance
4855 *
4856 * Returns
4857 * IRQ_HANDLED - If interrupt is valid
4858 * IRQ_NONE - If invalid interrupt
4859 */
4861 {
4863 u32 tr_doorbell;
4865 /* Resetting interrupt aggregation counters first and reading the
4866 * DOOR_BELL afterward allows us to handle all the completed requests.
4867 * In order to prevent other interrupts starvation the DB is read once
4868 * after reset. The down side of this solution is the possibility of
4869 * false interrupt if device completes another request after resetting
4870 * aggregation and before reading the DB.
4871 */
4884 }
4885 }
4887 /**
4888 * ufshcd_disable_ee - disable exception event
4889 * @hba: per-adapter instance
4890 * @mask: exception event to disable
4891 *
4892 * Disables exception event in the device so that the EVENT_ALERT
4893 * bit is not set.
4894 *
4895 * Returns zero on success, non-zero error value on failure.
4896 */
4898 {
4900 u32 val;
4911 out:
4913 }
4915 /**
4916 * ufshcd_enable_ee - enable exception event
4917 * @hba: per-adapter instance
4918 * @mask: exception event to enable
4919 *
4920 * Enable corresponding exception event in the device to allow
4921 * device to alert host in critical scenarios.
4922 *
4923 * Returns zero on success, non-zero error value on failure.
4924 */
4926 {
4928 u32 val;
4939 out:
4941 }
4943 /**
4944 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
4945 * @hba: per-adapter instance
4946 *
4947 * Allow device to manage background operations on its own. Enabling
4948 * this might lead to inconsistent latencies during normal data transfers
4949 * as the device is allowed to manage its own way of handling background
4950 * operations.
4951 *
4952 * Returns zero on success, non-zero on failure.
4953 */
4955 {
4967 }
4972 /* No need of URGENT_BKOPS exception from the device */
4977 out:
4979 }
4981 /**
4982 * ufshcd_disable_auto_bkops - block device in doing background operations
4983 * @hba: per-adapter instance
4984 *
4985 * Disabling background operations improves command response latency but
4986 * has drawback of device moving into critical state where the device is
4987 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
4988 * host is idle so that BKOPS are managed effectively without any negative
4989 * impacts.
4990 *
4991 * Returns zero on success, non-zero on failure.
4992 */
4994 {
5000 /*
5001 * If host assisted BKOPs is to be enabled, make sure
5002 * urgent bkops exception is allowed.
5003 */
5009 }
5018 }
5023 out:
5025 }
5027 /**
5028 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5029 * @hba: per adapter instance
5030 *
5031 * After a device reset the device may toggle the BKOPS_EN flag
5032 * to default value. The s/w tracking variables should be updated
5033 * as well. This function would change the auto-bkops state based on
5034 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
5035 */
5037 {
5046 }
5048 }
5051 {
5054 }
5056 /**
5057 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5058 * @hba: per-adapter instance
5059 * @status: bkops_status value
5060 *
5061 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5062 * flag in the device to permit background operations if the device
5063 * bkops_status is greater than or equal to "status" argument passed to
5064 * this function, disable otherwise.
5065 *
5066 * Returns 0 for success, non-zero in case of failure.
5067 *
5068 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5069 * to know whether auto bkops is enabled or disabled after this function
5070 * returns control to it.
5071 */
5074 {
5088 }
5092 else
5095 out:
5097 }
5099 /**
5100 * ufshcd_urgent_bkops - handle urgent bkops exception event
5101 * @hba: per-adapter instance
5102 *
5103 * Enable fBackgroundOpsEn flag in the device to permit background
5104 * operations.
5105 *
5106 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5107 * and negative error value for any other failure.
5108 */
5110 {
5112 }
5115 {
5118 }
5121 {
5133 }
5135 /*
5136 * We are seeing that some devices are raising the urgent bkops
5137 * exception events even when BKOPS status doesn't indicate performace
5138 * impacted or critical. Handle these device by determining their urgent
5139 * bkops status at runtime.
5140 */
5144 /* update the current status as the urgent bkops level */
5147 }
5149 enable_auto_bkops:
5151 out:
5155 }
5157 /**
5158 * ufshcd_exception_event_handler - handle exceptions raised by device
5159 * @work: pointer to work data
5160 *
5161 * Read bExceptionEventStatus attribute from the device and handle the
5162 * exception event accordingly.
5163 */
5165 {
5178 }
5185 out:
5189 }
5191 /* Complete requests that have door-bell cleared */
5193 {
5196 }
5198 /**
5199 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5200 * to recover from the DL NAC errors or not.
5201 * @hba: per-adapter instance
5202 *
5203 * Returns true if error handling is required, false otherwise
5204 */
5206 {
5211 /*
5212 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5213 * device fatal error and/or DL NAC & REPLAY timeout errors.
5214 */
5226 /*
5227 * wait for 50ms to see if we can get any other errors or not.
5228 */
5233 /*
5234 * now check if we have got any other severe errors other than
5235 * DL NAC error?
5236 */
5242 /*
5243 * As DL NAC is the only error received so far, send out NOP
5244 * command to confirm if link is still active or not.
5245 * - If we don't get any response then do error recovery.
5246 * - If we get response then clear the DL NAC error bit.
5247 */
5256 /* Link seems to be alive hence ignore the DL NAC errors */
5259 /* clear NAC error */
5264 }
5265 }
5266 out:
5269 }
5271 /**
5272 * ufshcd_err_handler - handle UFS errors that require s/w attention
5273 * @work: pointer to work structure
5274 */
5276 {
5297 /* Complete requests that have door-bell cleared by h/w */
5304 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
5309 }
5314 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5315 UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
5318 /*
5319 * if host reset is required then skip clearing the pending
5320 * transfers forcefully because they will get cleared during
5321 * host reset and restore
5322 */
5326 /* release lock as clear command might sleep */
5328 /* Clear pending transfer requests */
5333 }
5334 }
5336 /* Clear pending task management requests */
5341 }
5342 }
5344 lock_skip_pending_xfer_clear:
5347 /* Complete the requests that are cleared by s/w */
5353 skip_pending_xfer_clear:
5354 /* Fatal errors need reset */
5358 /*
5359 * ufshcd_reset_and_restore() does the link reinitialization
5360 * which will need atleast one empty doorbell slot to send the
5361 * device management commands (NOP and query commands).
5362 * If there is no slot empty at this moment then free up last
5363 * slot forcefully.
5364 */
5374 __func__);
5376 }
5377 /*
5378 * Inform scsi mid-layer that we did reset and allow to handle
5379 * Unit Attention properly.
5380 */
5384 }
5386 skip_err_handling:
5392 }
5396 out:
5401 }
5403 /**
5404 * ufshcd_update_uic_error - check and set fatal UIC error flags.
5405 * @hba: per-adapter instance
5406 *
5407 * Returns
5408 * IRQ_HANDLED - If interrupt is valid
5409 * IRQ_NONE - If invalid interrupt
5410 */
5412 {
5413 u32 reg;
5416 /* PHY layer lane error */
5418 /* Ignore LINERESET indication, as this is not an error */
5421 /*
5422 * To know whether this error is fatal or not, DB timeout
5423 * must be checked but this error is handled separately.
5424 */
5428 }
5430 /* PA_INIT_ERROR is fatal and needs UIC reset */
5439 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5442 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5445 }
5447 }
5449 /* UIC NL/TL/DME errors needs software retry */
5456 }
5464 }
5472 }
5477 }
5480 u32 intr_mask)
5481 {
5494 }
5496 /**
5497 * ufshcd_check_errors - Check for errors that need s/w attention
5498 * @hba: per-adapter instance
5499 *
5500 * Returns
5501 * IRQ_HANDLED - If interrupt is valid
5502 * IRQ_NONE - If invalid interrupt
5503 */
5505 {
5512 }
5519 }
5530 }
5533 /*
5534 * update the transfer error masks to sticky bits, let's do this
5535 * irrespective of current ufshcd_state.
5536 */
5540 /* handle fatal errors only when link is functional */
5542 /* block commands from scsi mid-layer */
5547 /* dump controller state before resetting */
5550 SYSTEM_BUS_FATAL_ERROR);
5560 pr_prdt);
5561 }
5563 }
5565 }
5566 /*
5567 * if (!queue_eh_work) -
5568 * Other errors are either non-fatal where host recovers
5569 * itself without s/w intervention or errors that will be
5570 * handled by the SCSI core layer.
5571 */
5573 }
5579 };
5582 {
5594 }
5596 /**
5597 * ufshcd_tmc_handler - handle task management function completion
5598 * @hba: per adapter instance
5599 *
5600 * Returns
5601 * IRQ_HANDLED - If interrupt is valid
5602 * IRQ_NONE - If invalid interrupt
5603 */
5605 {
5610 };
5614 }
5616 /**
5617 * ufshcd_sl_intr - Interrupt service routine
5618 * @hba: per adapter instance
5619 * @intr_status: contains interrupts generated by the controller
5620 *
5621 * Returns
5622 * IRQ_HANDLED - If interrupt is valid
5623 * IRQ_NONE - If invalid interrupt
5624 */
5626 {
5647 }
5649 /**
5650 * ufshcd_intr - Main interrupt service routine
5651 * @irq: irq number
5652 * @__hba: pointer to adapter instance
5653 *
5654 * Returns
5655 * IRQ_HANDLED - If interrupt is valid
5656 * IRQ_NONE - If invalid interrupt
5657 */
5659 {
5668 /*
5669 * There could be max of hba->nutrs reqs in flight and in worst case
5670 * if the reqs get finished 1 by 1 after the interrupt status is
5671 * read, make sure we handle them by checking the interrupt status
5672 * again in a loop until we process all of the reqs before returning.
5673 */
5675 enabled_intr_status =
5689 }
5693 }
5696 {
5708 /* poll for max. 1 sec to clear door bell register by h/w */
5710 REG_UTP_TASK_REQ_DOOR_BELL,
5712 out:
5714 }
5718 {
5726 /*
5727 * Get free slot, sleep if slots are unavailable.
5728 * Even though we use wait_event() which sleeps indefinitely,
5729 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
5730 */
5745 /* send command to the controller */
5748 /* Make sure descriptors are ready before ringing the task doorbell */
5752 /* Make sure that doorbell is committed immediately */
5759 /* wait until the task management command is completed */
5763 /*
5764 * Make sure that ufshcd_compl_tm() does not trigger a
5765 * use-after-free.
5766 */
5780 }
5790 }
5792 /**
5793 * ufshcd_issue_tm_cmd - issues task management commands to controller
5794 * @hba: per adapter instance
5795 * @lun_id: LUN ID to which TM command is sent
5796 * @task_id: task ID to which the TM command is applicable
5797 * @tm_function: task management function opcode
5798 * @tm_response: task management service response return value
5799 *
5800 * Returns non-zero value on error, zero on success.
5801 */
5804 {
5808 /* Configure task request descriptor */
5812 /* Configure task request UPIU */
5817 /*
5818 * The host shall provide the same value for LUN field in the basic
5819 * header and for Input Parameter.
5820 */
5834 MASK_TM_SERVICE_RESP;
5836 }
5838 /**
5839 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
5840 * @hba: per-adapter instance
5841 * @req_upiu: upiu request
5842 * @rsp_upiu: upiu reply
5843 * @desc_buff: pointer to descriptor buffer, NULL if NA
5844 * @buff_len: descriptor size, 0 if NA
5845 * @cmd_type: specifies the type (NOP, Query...)
5846 * @desc_op: descriptor operation
5847 *
5848 * Those type of requests uses UTP Transfer Request Descriptor - utrd.
5849 * Therefore, it "rides" the device management infrastructure: uses its tag and
5850 * tasks work queues.
5851 *
5852 * Since there is only one available tag for device management commands,
5853 * the caller is expected to hold the hba->dev_cmd.lock mutex.
5854 */
5861 {
5869 u32 upiu_flags;
5877 }
5901 }
5903 /* update the task tag in the request upiu */
5908 /* just copy the upiu request as it is */
5911 /* The Data Segment Area is optional depending upon the query
5912 * function value. for WRITE DESCRIPTOR, the data segment
5913 * follows right after the tsf.
5914 */
5917 }
5923 /* Make sure descriptors are ready before ringing the doorbell */
5929 /*
5930 * ignore the returning value here - ufshcd_check_query_response is
5931 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
5932 * read the response directly ignoring all errors.
5933 */
5936 /* just copy the upiu response as it is */
5941 MASK_QUERY_DATA_SEG_LEN;
5952 }
5953 }
5956 out_unlock:
5959 }
5961 /**
5962 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
5963 * @hba: per-adapter instance
5964 * @req_upiu: upiu request
5965 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands
5966 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
5967 * @desc_buff: pointer to descriptor buffer, NULL if NA
5968 * @buff_len: descriptor size, 0 if NA
5969 * @desc_op: descriptor operation
5970 *
5971 * Supports UTP Transfer requests (nop and query), and UTP Task
5972 * Management requests.
5973 * It is up to the caller to fill the upiu conent properly, as it will
5974 * be copied without any further input validations.
5975 */
5982 {
5992 /* fall through */
6016 ocs_value);
6018 }
6027 }
6030 }
6032 /**
6033 * ufshcd_eh_device_reset_handler - device reset handler registered to
6034 * scsi layer.
6035 * @cmd: SCSI command pointer
6036 *
6037 * Returns SUCCESS/FAILED
6038 */
6040 {
6044 u32 pos;
6060 }
6062 /* clear the commands that were pending for corresponding LUN */
6068 }
6069 }
6074 out:
6082 }
6084 }
6087 {
6094 }
6095 }
6097 /**
6098 * ufshcd_abort - abort a specific command
6099 * @cmd: SCSI command pointer
6100 *
6101 * Abort the pending command in device by sending UFS_ABORT_TASK task management
6102 * command, and in host controller by clearing the door-bell register. There can
6103 * be race between controller sending the command to the device while abort is
6104 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
6105 * really issued and then try to abort it.
6106 *
6107 * Returns SUCCESS/FAILED
6108 */
6110 {
6119 u32 reg;
6130 }
6132 /*
6133 * Task abort to the device W-LUN is illegal. When this command
6134 * will fail, due to spec violation, scsi err handling next step
6135 * will be to send LU reset which, again, is a spec violation.
6136 * To avoid these unnecessary/illegal step we skip to the last error
6137 * handling stage: reset and restore.
6138 */
6144 /* If command is already aborted/completed, return SUCCESS */
6150 }
6156 }
6158 /* Print Transfer Request of aborted task */
6161 /*
6162 * Print detailed info about aborted request.
6163 * As more than one request might get aborted at the same time,
6164 * print full information only for the first aborted request in order
6165 * to reduce repeated printouts. For other aborted requests only print
6166 * basic details.
6167 */
6177 }
6180 /* Skip task abort in case previous aborts failed and report failure */
6184 }
6190 /* cmd pending in the device */
6195 /*
6196 * cmd not pending in the device, check if it is
6197 * in transition.
6198 */
6203 /* sleep for max. 200us to stabilize */
6206 }
6207 /* command completed already */
6218 }
6219 }
6224 }
6233 }
6235 }
6242 }
6251 out:
6258 }
6260 /*
6261 * This ufshcd_release() corresponds to the original scsi cmd that got
6262 * aborted here (as we won't get any IRQ for it).
6263 */
6266 }
6268 /**
6269 * ufshcd_host_reset_and_restore - reset and restore host controller
6270 * @hba: per-adapter instance
6271 *
6272 * Note that host controller reset may issue DME_RESET to
6273 * local and remote (device) Uni-Pro stack and the attributes
6274 * are reset to default state.
6275 *
6276 * Returns zero on success, non-zero on failure
6277 */
6279 {
6283 /*
6284 * Stop the host controller and complete the requests
6285 * cleared by h/w
6286 */
6294 /* scale up clocks to max frequency before full reinitialization */
6301 /* Establish the link again and restore the device */
6306 out:
6311 }
6313 /**
6314 * ufshcd_reset_and_restore - reset and re-initialize host/device
6315 * @hba: per-adapter instance
6316 *
6317 * Reset and recover device, host and re-establish link. This
6318 * is helpful to recover the communication in fatal error conditions.
6319 *
6320 * Returns zero on success, non-zero on failure
6321 */
6323 {
6328 /* Reset the attached device */
6335 }
6337 /**
6338 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
6339 * @cmd: SCSI command pointer
6340 *
6341 * Returns SUCCESS/FAILED
6342 */
6344 {
6352 /*
6353 * Check if there is any race with fatal error handling.
6354 * If so, wait for it to complete. Even though fatal error
6355 * handling does reset and restore in some cases, don't assume
6356 * anything out of it. We are just avoiding race here.
6357 */
6382 }
6388 }
6390 /**
6391 * ufshcd_get_max_icc_level - calculate the ICC level
6392 * @sup_curr_uA: max. current supported by the regulator
6393 * @start_scan: row at the desc table to start scan from
6394 * @buff: power descriptor buffer
6395 *
6396 * Returns calculated max ICC level for specific regulator
6397 */
6399 {
6402 u16 data;
6403 u16 unit;
6408 ATTR_ICC_LVL_UNIT_OFFSET;
6423 }
6426 }
6430 }
6433 }
6435 /**
6436 * ufshcd_calc_icc_level - calculate the max ICC level
6437 * In case regulators are not initialized we'll return 0
6438 * @hba: per-adapter instance
6439 * @desc_buf: power descriptor buffer to extract ICC levels from.
6440 * @len: length of desc_buff
6441 *
6442 * Returns calculated ICC level
6443 */
6446 {
6455 }
6466 icc_level,
6472 icc_level,
6474 out:
6476 }
6479 {
6495 }
6512 out:
6514 }
6516 /**
6517 * ufshcd_scsi_add_wlus - Adds required W-LUs
6518 * @hba: per-adapter instance
6519 *
6520 * UFS device specification requires the UFS devices to support 4 well known
6521 * logical units:
6522 * "REPORT_LUNS" (address: 01h)
6523 * "UFS Device" (address: 50h)
6524 * "RPMB" (address: 44h)
6525 * "BOOT" (address: 30h)
6526 * UFS device's power management needs to be controlled by "POWER CONDITION"
6527 * field of SSU (START STOP UNIT) command. But this "power condition" field
6528 * will take effect only when its sent to "UFS device" well known logical unit
6529 * hence we require the scsi_device instance to represent this logical unit in
6530 * order for the UFS host driver to send the SSU command for power management.
6531 *
6532 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
6533 * Block) LU so user space process can control this LU. User space may also
6534 * want to have access to BOOT LU.
6535 *
6536 * This function adds scsi device instances for each of all well known LUs
6537 * (except "REPORT LUNS" LU).
6538 *
6539 * Returns zero on success (all required W-LUs are added successfully),
6540 * non-zero error value on failure (if failed to add any of the required W-LU).
6541 */
6543 {
6554 }
6562 }
6569 else
6573 remove_sdev_ufs_device:
6575 out:
6577 }
6580 {
6583 u8 model_index;
6593 }
6601 }
6603 /*
6604 * getting vendor (manufacturerID) and Bank Index in big endian
6605 * format
6606 */
6617 }
6619 /*
6620 * ufshcd_read_string_desc returns size of the string
6621 * reset the error value
6622 */
6625 out:
6628 }
6631 {
6636 }
6639 {
6650 }
6651 }
6653 /**
6654 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
6655 * @hba: per-adapter instance
6656 *
6657 * PA_TActivate parameter can be tuned manually if UniPro version is less than
6658 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
6659 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
6660 * the hibern8 exit latency.
6661 *
6662 * Returns zero on success, non-zero error value on failure.
6663 */
6665 {
6671 RX_MIN_ACTIVATETIME_CAPABILITY,
6677 /* make sure proper unit conversion is applied */
6678 tuned_pa_tactivate =
6682 tuned_pa_tactivate);
6684 out:
6686 }
6688 /**
6689 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
6690 * @hba: per-adapter instance
6691 *
6692 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
6693 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
6694 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
6695 * This optimal value can help reduce the hibern8 exit latency.
6696 *
6697 * Returns zero on success, non-zero error value on failure.
6698 */
6700 {
6720 peer_rx_hibern8_time_cap);
6721 /* make sure proper unit conversion is applied */
6725 tuned_pa_hibern8time);
6726 out:
6728 }
6730 /**
6731 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
6732 * less than device PA_TACTIVATE time.
6733 * @hba: per-adapter instance
6734 *
6735 * Some UFS devices require host PA_TACTIVATE to be lower than device
6736 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
6737 * for such devices.
6738 *
6739 * Returns zero on success, non-zero error value on failure.
6740 */
6742 {
6764 }
6771 }
6787 u32 new_peer_pa_tactivate;
6791 new_peer_pa_tactivate++;
6793 new_peer_pa_tactivate);
6794 }
6796 out:
6798 }
6801 {
6805 }
6808 /* set 1ms timeout for PA_TACTIVATE */
6815 }
6818 {
6822 }
6825 {
6862 }
6865 {
6875 }
6883 }
6890 out:
6893 }
6901 };
6903 static enum ufs_ref_clk_freq
6905 {
6913 }
6916 {
6927 }
6930 {
6932 u32 ref_clk;
6940 err);
6942 }
6954 }
6959 out:
6961 }
6964 {
6968 /* Clear any previous UFS device information */
6971 /* Init check for device descriptor sizes */
6974 /* Init UFS geometry descriptor related parameters */
6979 /* Check and apply UFS device quirks */
6985 }
6993 /* Probe maximum power mode co-supported by both UFS host and device */
6997 __func__);
6998 out:
7000 }
7002 /**
7003 * ufshcd_add_lus - probe and add UFS logical units
7004 * @hba: per-adapter instance
7005 */
7007 {
7012 /* Add required well known logical units to scsi mid layer */
7017 /* Initialize devfreq after UFS device is detected */
7027 }
7030 }
7036 out:
7038 }
7040 /**
7041 * ufshcd_probe_hba - probe hba to detect device and initialize
7042 * @hba: per-adapter instance
7043 * @async: asynchronous execution or not
7044 *
7045 * Execute link-startup and verify device initialization
7046 */
7048 {
7056 /* set the default level for urgent bkops */
7060 /* Debug counters initialization */
7063 /* UniPro link is active now */
7066 /* Verify device initialization by sending NOP OUT UPIU */
7071 /* Initiate UFS initialization, and waiting until completion */
7076 /*
7077 * Initialize UFS device parameters used by driver, these
7078 * parameters are associated with UFS descriptors.
7079 */
7084 }
7088 /* UFS device is also active now */
7093 /* Gear up to HS gear if supported */
7095 /*
7096 * Set the right value to bRefClkFreq before attempting to
7097 * switch to HS gears.
7098 */
7106 }
7107 }
7109 /* set the state as operational after switching to desired gear */
7112 /* Enable Auto-Hibernate if configured */
7115 out:
7121 }
7123 /**
7124 * ufshcd_async_scan - asynchronous execution for probing hba
7125 * @data: data pointer to pass to this function
7126 * @cookie: cookie data
7127 */
7129 {
7133 /* Initialize hba, detect and initialize UFS device */
7138 /* Probe and add UFS logical units */
7140 out:
7141 /*
7142 * If we failed to initialize the device or the device is not
7143 * present, turn off the power/clocks etc.
7144 */
7149 }
7150 }
7155 NULL,
7156 };
7180 };
7184 {
7190 /*
7191 * "set_load" operation shall be required on those regulators
7192 * which specifically configured current limitation. Otherwise
7193 * zero max_uA may cause unexpected behavior when regulator is
7194 * enabled or set as high power mode.
7195 */
7203 }
7206 }
7210 {
7212 }
7216 {
7221 }
7225 {
7245 }
7246 }
7252 }
7253 out:
7255 }
7258 {
7270 else
7273 out:
7275 }
7278 {
7287 /* ignore errors on applying disable config */
7293 }
7294 out:
7296 }
7299 {
7316 out:
7321 }
7323 }
7326 {
7330 }
7333 {
7344 }
7345 out:
7347 }
7350 {
7364 out:
7366 }
7369 {
7376 }
7380 {
7391 /*
7392 * vendor specific setup_clocks ops may depend on clocks managed by
7393 * this standard driver hence call the vendor specific setup_clocks
7394 * before disabling the clocks managed here.
7395 */
7400 }
7414 }
7417 }
7421 }
7422 }
7424 /*
7425 * vendor specific setup_clocks ops may depend on clocks managed by
7426 * this standard driver hence call the vendor specific setup_clocks
7427 * after enabling the clocks managed here.
7428 */
7433 }
7435 out:
7440 }
7447 }
7454 }
7457 {
7459 }
7462 {
7481 }
7483 /*
7484 * Parse device ref clk freq as per device tree "ref_clk".
7485 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
7486 * in ufshcd_alloc_host().
7487 */
7498 }
7500 }
7503 }
7504 out:
7506 }
7509 {
7525 out_exit:
7527 out:
7532 }
7535 {
7542 }
7545 {
7548 /*
7549 * Handle host controller power separately from the UFS device power
7550 * rails as it will help controlling the UFS host controller power
7551 * collapse easily which is different than UFS device power collapse.
7552 * Also, enable the host controller power before we go ahead with rest
7553 * of the initialization here.
7554 */
7586 out_disable_vreg:
7588 out_disable_clks:
7590 out_disable_hba_vreg:
7592 out:
7594 }
7597 {
7609 }
7610 }
7612 static int
7614 {
7619 UFS_SENSE_SIZE,
7628 }
7637 out:
7639 }
7641 /**
7642 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
7643 * power mode
7644 * @hba: per adapter instance
7645 * @pwr_mode: device power mode to set
7646 *
7647 * Returns 0 if requested power mode is set successfully
7648 * Returns non-zero if failed to set the requested power mode
7649 */
7652 {
7666 }
7669 }
7675 /*
7676 * If scsi commands fail, the scsi mid-layer schedules scsi error-
7677 * handling, which would wait for host to be resumed. Since we know
7678 * we are functional while we are here, skip host resume in error
7679 * handling context.
7680 */
7686 /* Unit attention condition is cleared now */
7688 }
7692 /*
7693 * Current function would be generally called from the power management
7694 * callbacks hence set the RQF_PM flag so that it doesn't resume the
7695 * already suspended childs.
7696 */
7705 }
7709 out:
7713 }
7718 {
7728 else
7730 }
7731 /*
7732 * If autobkops is enabled, link can't be turned off because
7733 * turning off the link would also turn off the device.
7734 */
7737 /*
7738 * Let's make sure that link is in low power mode, we are doing
7739 * this currently by putting the link in Hibern8. Otherway to
7740 * put the link in low power mode is to send the DME end point
7741 * to device and then send the DME reset command to local
7742 * unipro. But putting the link in hibern8 is much faster.
7743 */
7747 /*
7748 * Change controller state to "reset state" which
7749 * should also put the link in off/reset state
7750 */
7752 /*
7753 * TODO: Check if we need any delay to make sure that
7754 * controller is reset
7755 */
7757 }
7759 out:
7761 }
7764 {
7765 /*
7766 * It seems some UFS devices may keep drawing more than sleep current
7767 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
7768 * To avoid this situation, add 2ms delay before putting these UFS
7769 * rails in LPM mode.
7770 */
7775 /*
7776 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
7777 * power.
7778 *
7779 * If UFS device and link is in OFF state, all power supplies (VCC,
7780 * VCCQ, VCCQ2) can be turned off if power on write protect is not
7781 * required. If UFS link is inactive (Hibern8 or OFF state) and device
7782 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
7783 *
7784 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
7785 * in low power state which would save some power.
7786 */
7795 }
7796 }
7797 }
7800 {
7814 }
7816 }
7819 vccq_lpm:
7821 vcc_disable:
7823 out:
7825 }
7828 {
7831 }
7834 {
7837 }
7839 /**
7840 * ufshcd_suspend - helper function for suspend operations
7841 * @hba: per adapter instance
7842 * @pm_op: desired low power operation type
7843 *
7844 * This function will try to put the UFS device and link into low power
7845 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
7846 * (System PM level).
7847 *
7848 * If this function is called during shutdown, it will make sure that
7849 * both UFS device and UFS link is powered off.
7850 *
7851 * NOTE: UFS device & link must be active before we enter in this function.
7852 *
7853 * Returns 0 for success and non-zero for failure
7854 */
7856 {
7871 }
7873 /*
7874 * If we can't transition into any of the low power modes
7875 * just gate the clocks.
7876 */
7884 }
7889 }
7895 /* UFS device & link must be active before we enter in this function */
7899 }
7903 /*
7904 * The device is idle with no requests in the queue,
7905 * allow background operations if bkops status shows
7906 * that performance might be impacted.
7907 */
7912 /* make sure that auto bkops is disabled */
7914 }
7915 }
7920 /* ensure that bkops is disabled */
7925 }
7933 disable_clks:
7934 /*
7935 * Call vendor specific suspend callback. As these callbacks may access
7936 * vendor specific host controller register space call them before the
7937 * host clocks are ON.
7938 */
7942 /*
7943 * Disable the host irq as host controller as there won't be any
7944 * host controller transaction expected till resume.
7945 */
7950 else
7951 /* If link is active, device ref_clk can't be switched off */
7957 /* Put the host controller in low power mode if possible */
7961 set_link_active:
7969 set_dev_active:
7972 enable_gating:
7977 out:
7982 }
7984 /**
7985 * ufshcd_resume - helper function for resume operations
7986 * @hba: per adapter instance
7987 * @pm_op: runtime PM or system PM
7988 *
7989 * This function basically brings the UFS device, UniPro link and controller
7990 * to active state.
7991 *
7992 * Returns 0 for success and non-zero for failure
7993 */
7995 {
8003 /* Make sure clocks are enabled before accessing controller */
8008 /* enable the host irq as host controller would be active soon */
8015 /*
8016 * Call vendor specific resume callback. As these callbacks may access
8017 * vendor specific host controller register space call them when the
8018 * host clocks are ON.
8019 */
8028 else
8032 /*
8033 * ufshcd_host_reset_and_restore() should have already
8034 * set the link state as active
8035 */
8038 }
8044 }
8048 else
8049 /*
8050 * If BKOPs operations are urgently needed at this moment then
8051 * keep auto-bkops enabled or else disable it.
8052 */
8060 /* Enable Auto-Hibernate if configured */
8063 /* Schedule clock gating in case of no access to UFS device yet */
8068 set_old_link_state:
8070 vendor_suspend:
8072 disable_vreg:
8074 disable_irq_and_vops_clks:
8079 out:
8084 }
8086 /**
8087 * ufshcd_system_suspend - system suspend routine
8088 * @hba: per adapter instance
8089 *
8090 * Check the description of ufshcd_suspend() function for more details.
8091 *
8092 * Returns 0 for success and non-zero for failure
8093 */
8095 {
8109 /*
8110 * UFS device and/or UFS link low power states during runtime
8111 * suspend seems to be different than what is expected during
8112 * system suspend. Hence runtime resume the devic & link and
8113 * let the system suspend low power states to take effect.
8114 * TODO: If resume takes longer time, we might have optimize
8115 * it in future by not resuming everything if possible.
8116 */
8120 }
8123 out:
8130 }
8133 /**
8134 * ufshcd_system_resume - system resume routine
8135 * @hba: per adapter instance
8136 *
8137 * Returns 0 for success and non-zero for failure
8138 */
8141 {
8149 /*
8150 * Let the runtime resume take care of resuming
8151 * if runtime suspended.
8152 */
8154 else
8156 out:
8163 }
8166 /**
8167 * ufshcd_runtime_suspend - runtime suspend routine
8168 * @hba: per adapter instance
8169 *
8170 * Check the description of ufshcd_suspend() function for more details.
8171 *
8172 * Returns 0 for success and non-zero for failure
8173 */
8175 {
8184 else
8186 out:
8191 }
8194 /**
8195 * ufshcd_runtime_resume - runtime resume routine
8196 * @hba: per adapter instance
8197 *
8198 * This function basically brings the UFS device, UniPro link and controller
8199 * to active state. Following operations are done in this function:
8200 *
8201 * 1. Turn on all the controller related clocks
8202 * 2. Bring the UniPro link out of Hibernate state
8203 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
8204 * to active state.
8205 * 4. If auto-bkops is enabled on the device, disable it.
8206 *
8207 * So following would be the possible power state after this function return
8208 * successfully:
8209 * S1: UFS device in Active state with VCC rail ON
8210 * UniPro link in Active state
8211 * All the UFS/UniPro controller clocks are ON
8212 *
8213 * Returns 0 for success and non-zero for failure
8214 */
8216 {
8225 else
8227 out:
8232 }
8236 {
8238 }
8241 /**
8242 * ufshcd_shutdown - shutdown routine
8243 * @hba: per adapter instance
8244 *
8245 * This function would power off both UFS device and UFS link.
8246 *
8247 * Returns 0 always to allow force shutdown even in case of errors.
8248 */
8250 {
8263 }
8266 out:
8269 /* allow force shutdown even in case of errors */
8271 }
8274 /**
8275 * ufshcd_remove - de-allocate SCSI host and host memory space
8276 * data structure memory
8277 * @hba: per adapter instance
8278 */
8280 {
8287 /* disable interrupts */
8296 }
8299 /**
8300 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
8301 * @hba: pointer to Host Bus Adapter (HBA)
8302 */
8304 {
8306 }
8309 /**
8310 * ufshcd_set_dma_mask - Set dma mask based on the controller
8311 * addressing capability
8312 * @hba: per adapter instance
8313 *
8314 * Returns 0 for success, non-zero for failure
8315 */
8317 {
8321 }
8323 }
8325 /**
8326 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
8327 * @dev: pointer to device handle
8328 * @hba_handle: driver private handle
8329 * Returns 0 on success, non-zero value on failure
8330 */
8332 {
8342 }
8350 }
8359 out_error:
8361 }
8364 /* This function exists because blk_mq_alloc_tag_set() requires this. */
8367 {
8370 }
8374 };
8376 /**
8377 * ufshcd_init - Driver initialization routine
8378 * @hba: per-adapter instance
8379 * @mmio_base: base register address
8380 * @irq: Interrupt line of device
8381 * Returns 0 on success, non-zero value on failure
8382 */
8384 {
8394 }
8403 /* Read capabilities registers */
8406 /* Get UFS version supported by the controller */
8416 /* Get Interrupt bit mask per version */
8423 }
8425 /* Allocate memory for host memory space */
8430 }
8432 /* Configure LRB */
8445 /* Initialize work queues */
8449 /* Initialize UIC command mutex */
8452 /* Initialize mutex for device management commands */
8461 /*
8462 * In order to avoid any spurious interrupt immediately after
8463 * registering UFS controller interrupt handler, clear any pending UFS
8464 * interrupt status and disable all the UFS interrupts.
8465 */
8467 REG_INTERRUPT_STATUS);
8469 /*
8470 * Make sure that UFS interrupts are disabled and any pending interrupt
8471 * status is cleared before registering UFS interrupt handler.
8472 */
8475 /* IRQ registration */
8482 }
8488 }
8494 }
8501 };
8509 }
8511 /* Reset the attached device */
8514 /* Host controller enable */
8521 }
8523 /*
8524 * Set the default power management level for runtime and system PM.
8525 * Default power saving mode is to keep UFS link in Hibern8 state
8526 * and UFS device in sleep state.
8527 */
8529 UFS_SLEEP_PWR_MODE,
8530 UIC_LINK_HIBERN8_STATE);
8532 UFS_SLEEP_PWR_MODE,
8533 UIC_LINK_HIBERN8_STATE);
8535 /* Set the default auto-hiberate idle timer value to 150 ms */
8539 }
8541 /* Hold auto suspend until async scan completes */
8544 /*
8545 * We are assuming that device wasn't put in sleep/power-down
8546 * state exclusively during the boot stage before kernel.
8547 * This assumption helps avoid doing link startup twice during
8548 * ufshcd_probe_hba().
8549 */
8557 free_tmf_queue:
8559 free_tmf_tag_set:
8561 free_cmd_queue:
8563 out_remove_scsi_host:
8565 exit_gating:
8568 out_disable:
8571 out_error:
8573 }