| blob | bd21c9cdf8183f2ebb6503e9bac982d46573d92e |
1 /*
2 * Universal Flash Storage Host controller driver Core
3 *
4 * This code is based on drivers/scsi/ufs/ufshcd.c
5 * Copyright (C) 2011-2013 Samsung India Software Operations
6 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7 *
8 * Authors:
9 * Santosh Yaraganavi <santosh.sy@samsung.com>
10 * Vinayak Holikatti <h.vinayak@samsung.com>
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 * See the COPYING file in the top-level directory or visit
17 * <http://www.gnu.org/licenses/gpl-2.0.html>
18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 *
24 * This program is provided "AS IS" and "WITH ALL FAULTS" and
25 * without warranty of any kind. You are solely responsible for
26 * determining the appropriateness of using and distributing
27 * the program and assume all risks associated with your exercise
28 * of rights with respect to the program, including but not limited
29 * to infringement of third party rights, the risks and costs of
30 * program errors, damage to or loss of data, programs or equipment,
31 * and unavailability or interruption of operations. Under no
32 * circumstances will the contributor of this Program be liable for
33 * any damages of any kind arising from your use or distribution of
34 * this program.
35 *
36 * The Linux Foundation chooses to take subject only to the GPLv2
37 * license terms, and distributes only under these terms.
38 */
40 #include <linux/async.h>
41 #include <linux/devfreq.h>
42 #include <linux/nls.h>
43 #include <linux/of.h>
44 #include <linux/bitfield.h>
50 #define CREATE_TRACE_POINTS
51 #include <trace/events/ufs.h>
53 #define UFSHCD_REQ_SENSE_SIZE 18
55 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
56 UTP_TASK_REQ_COMPL |\
57 UFSHCD_ERROR_MASK)
58 /* UIC command timeout, unit: ms */
59 #define UIC_CMD_TIMEOUT 500
61 /* NOP OUT retries waiting for NOP IN response */
62 #define NOP_OUT_RETRIES 10
63 /* Timeout after 30 msecs if NOP OUT hangs without response */
66 /* Query request retries */
67 #define QUERY_REQ_RETRIES 3
68 /* Query request timeout */
71 /* Task management command timeout */
74 /* maximum number of retries for a general UIC command */
75 #define UFS_UIC_COMMAND_RETRIES 3
77 /* maximum number of link-startup retries */
78 #define DME_LINKSTARTUP_RETRIES 3
80 /* Maximum retries for Hibern8 enter */
81 #define UIC_HIBERN8_ENTER_RETRIES 3
83 /* maximum number of reset retries before giving up */
84 #define MAX_HOST_RESET_RETRIES 5
86 /* Expose the flag value from utp_upiu_query.value */
87 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
89 /* Interrupt aggregation default timeout, unit: 40us */
90 #define INT_AGGR_DEF_TO 0x02
92 #define ufshcd_toggle_vreg(_dev, _vreg, _on) \
93 ({ \
94 int _ret; \
95 if (_on) \
96 _ret = ufshcd_enable_vreg(_dev, _vreg); \
97 else \
98 _ret = ufshcd_disable_vreg(_dev, _vreg); \
99 _ret; \
100 })
102 #define ufshcd_hex_dump(prefix_str, buf, len) do { \
103 size_t __len = (len); \
104 print_hex_dump(KERN_ERR, prefix_str, \
105 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
106 16, 4, buf, __len, false); \
107 } while (0)
111 {
129 }
137 };
139 /* UFSHCD states */
141 UFSHCD_STATE_RESET,
142 UFSHCD_STATE_ERROR,
143 UFSHCD_STATE_OPERATIONAL,
144 UFSHCD_STATE_EH_SCHEDULED,
145 };
147 /* UFSHCD error handling flags */
150 };
152 /* UFSHCD UIC layer error flags */
160 };
162 #define ufshcd_set_eh_in_progress(h) \
163 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
164 #define ufshcd_eh_in_progress(h) \
165 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
166 #define ufshcd_clear_eh_in_progress(h) \
167 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
169 #define ufshcd_set_ufs_dev_active(h) \
170 ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
171 #define ufshcd_set_ufs_dev_sleep(h) \
172 ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
173 #define ufshcd_set_ufs_dev_poweroff(h) \
174 ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
175 #define ufshcd_is_ufs_dev_active(h) \
176 ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
177 #define ufshcd_is_ufs_dev_sleep(h) \
178 ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
179 #define ufshcd_is_ufs_dev_poweroff(h) \
180 ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
189 };
193 {
195 }
199 {
201 }
206 {
213 }
215 /* if no match found, return the level 0 */
217 }
220 /* UFS cards deviations table */
222 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
225 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
227 UFS_DEVICE_NO_FASTAUTO),
229 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
231 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
233 UFS_DEVICE_QUIRK_PA_TACTIVATE),
235 UFS_DEVICE_QUIRK_PA_TACTIVATE),
238 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
240 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
242 END_FIX
243 };
268 {
270 }
273 {
278 hba);
283 }
286 }
289 {
293 }
294 }
297 {
300 }
303 {
306 }
308 /* replace non-printable or non-ASCII characters with spaces */
310 {
316 }
320 {
324 }
328 {
332 }
336 {
345 }
349 {
357 /* trace UPIU W/O tracing command */
361 }
364 /* trace UPIU also */
368 /*
369 * Currently we only fully trace read(10) and write(10)
370 * commands
371 */
373 lba =
377 }
378 }
384 }
387 {
399 }
400 }
404 {
414 }
415 }
418 {
440 }
442 static
444 {
481 }
482 }
485 {
495 tag);
499 tag);
502 }
503 }
506 {
525 }
527 /**
528 * ufshcd_print_pwr_info - print power params as saved in hba
529 * power info
530 * @hba: per-adapter instance
531 */
533 {
542 };
545 __func__,
551 }
553 /*
554 * ufshcd_wait_for_register - wait for register value to change
555 * @hba - per-adapter interface
556 * @reg - mmio register offset
557 * @mask - mask to apply to read register value
558 * @val - wait condition
559 * @interval_us - polling interval in microsecs
560 * @timeout_ms - timeout in millisecs
561 * @can_sleep - perform sleep or just spin
562 *
563 * Returns -ETIMEDOUT on error, zero on success
564 */
568 {
572 /* ignore bits that we don't intend to wait on */
578 else
584 }
585 }
588 }
590 /**
591 * ufshcd_get_intr_mask - Get the interrupt bit mask
592 * @hba: Pointer to adapter instance
593 *
594 * Returns interrupt bit mask per version
595 */
597 {
612 }
615 }
617 /**
618 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
619 * @hba: Pointer to adapter instance
620 *
621 * Returns UFSHCI version supported by the controller
622 */
624 {
629 }
631 /**
632 * ufshcd_is_device_present - Check if any device connected to
633 * the host controller
634 * @hba: pointer to adapter instance
635 *
636 * Returns true if device present, false if no device detected
637 */
639 {
642 }
644 /**
645 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
646 * @lrbp: pointer to local command reference block
647 *
648 * This function is used to get the OCS field from UTRD
649 * Returns the OCS field in the UTRD
650 */
652 {
654 }
656 /**
657 * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status
658 * @task_req_descp: pointer to utp_task_req_desc structure
659 *
660 * This function is used to get the OCS field from UTMRD
661 * Returns the OCS field in the UTMRD
662 */
665 {
667 }
669 /**
670 * ufshcd_get_tm_free_slot - get a free slot for task management request
671 * @hba: per adapter instance
672 * @free_slot: pointer to variable with available slot value
673 *
674 * Get a free tag and lock it until ufshcd_put_tm_slot() is called.
675 * Returns 0 if free slot is not available, else return 1 with tag value
676 * in @free_slot.
677 */
679 {
694 out:
696 }
699 {
701 }
703 /**
704 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
705 * @hba: per adapter instance
706 * @pos: position of the bit to be cleared
707 */
709 {
712 else
714 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
715 }
717 /**
718 * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
719 * @hba: per adapter instance
720 * @pos: position of the bit to be cleared
721 */
723 {
726 else
728 }
730 /**
731 * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
732 * @hba: per adapter instance
733 * @tag: position of the bit to be cleared
734 */
736 {
738 }
740 /**
741 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
742 * @reg: Register value of host controller status
743 *
744 * Returns integer, 0 on Success and positive value if failed
745 */
747 {
749 }
751 /**
752 * ufshcd_get_uic_cmd_result - Get the UIC command result
753 * @hba: Pointer to adapter instance
754 *
755 * This function gets the result of UIC command completion
756 * Returns 0 on success, non zero value on error
757 */
759 {
761 MASK_UIC_COMMAND_RESULT;
762 }
764 /**
765 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
766 * @hba: Pointer to adapter instance
767 *
768 * This function gets UIC command argument3
769 * Returns 0 on success, non zero value on error
770 */
772 {
774 }
776 /**
777 * ufshcd_get_req_rsp - returns the TR response transaction type
778 * @ucd_rsp_ptr: pointer to response UPIU
779 */
782 {
784 }
786 /**
787 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
788 * @ucd_rsp_ptr: pointer to response UPIU
789 *
790 * This function gets the response status and scsi_status from response UPIU
791 * Returns the response result code.
792 */
795 {
797 }
799 /*
800 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
801 * from response UPIU
802 * @ucd_rsp_ptr: pointer to response UPIU
803 *
804 * Return the data segment length.
805 */
808 {
810 MASK_RSP_UPIU_DATA_SEG_LEN;
811 }
813 /**
814 * ufshcd_is_exception_event - Check if the device raised an exception event
815 * @ucd_rsp_ptr: pointer to response UPIU
816 *
817 * The function checks if the device raised an exception event indicated in
818 * the Device Information field of response UPIU.
819 *
820 * Returns true if exception is raised, false otherwise.
821 */
823 {
826 }
828 /**
829 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
830 * @hba: per adapter instance
831 */
834 {
836 INT_AGGR_COUNTER_AND_TIMER_RESET,
837 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
838 }
840 /**
841 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
842 * @hba: per adapter instance
843 * @cnt: Interrupt aggregation counter threshold
844 * @tmout: Interrupt aggregation timeout value
845 */
848 {
852 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
853 }
855 /**
856 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
857 * @hba: per adapter instance
858 */
860 {
862 }
864 /**
865 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
866 * When run-stop registers are set to 1, it indicates the
867 * host controller that it can process the requests
868 * @hba: per adapter instance
869 */
871 {
873 REG_UTP_TASK_REQ_LIST_RUN_STOP);
875 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
876 }
878 /**
879 * ufshcd_hba_start - Start controller initialization sequence
880 * @hba: per adapter instance
881 */
883 {
885 }
887 /**
888 * ufshcd_is_hba_active - Get controller state
889 * @hba: per adapter instance
890 *
891 * Returns false if controller is active, true otherwise
892 */
894 {
897 }
900 {
901 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
905 else
907 }
911 {
912 /*
913 * If both host and device support UniPro ver1.6 or later, PA layer
914 * parameters tuning happens during link startup itself.
915 *
916 * We can manually tune PA layer parameters if either host or device
917 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
918 * logic simple, we will only do manual tuning if local unipro version
919 * doesn't support ver1.6 or later.
920 */
923 else
925 }
928 {
955 }
974 }
980 }
981 }
984 }
988 out:
994 }
996 /**
997 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
998 * @hba: per adapter instance
999 * @scale_up: True if scaling up and false if scaling down
1000 *
1001 * Returns true if scaling is required, false otherwise.
1002 */
1005 {
1022 }
1023 }
1024 }
1027 }
1030 u64 wait_timeout_us)
1031 {
1034 u32 tm_doorbell;
1035 u32 tr_doorbell;
1037 ktime_t start;
1041 /*
1042 * Wait for all the outstanding tasks/transfer requests.
1043 * Verify by checking the doorbell registers are clear.
1044 */
1050 }
1059 }
1064 wait_timeout_us) {
1066 /*
1067 * We might have scheduled out for long time so make
1068 * sure to check if doorbells are cleared by this time
1069 * or not.
1070 */
1072 }
1081 }
1082 out:
1086 }
1088 /**
1089 * ufshcd_scale_gear - scale up/down UFS gear
1090 * @hba: per adapter instance
1091 * @scale_up: True for scaling up gear and false for scaling down
1092 *
1093 * Returns 0 for success,
1094 * Returns -EBUSY if scaling can't happen at this time
1095 * Returns non-zero for any other errors
1096 */
1098 {
1099 #define UFS_MIN_GEAR_TO_SCALE_DOWN UFS_HS_G1
1112 /* save the current power mode */
1117 /* scale down gear */
1120 }
1121 }
1123 /* check if the power mode needs to be changed or not? */
1133 }
1136 {
1139 /*
1140 * make sure that there are no outstanding requests when
1141 * clock scaling is in progress
1142 */
1149 }
1152 }
1155 {
1158 }
1160 /**
1161 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1162 * @hba: per adapter instance
1163 * @scale_up: True for scaling up and false for scalin down
1164 *
1165 * Returns 0 for success,
1166 * Returns -EBUSY if scaling can't happen at this time
1167 * Returns non-zero for any other errors
1168 */
1170 {
1173 /* let's not get into low power until clock scaling is completed */
1180 /* scale down the gear before scaling down clocks */
1185 }
1192 }
1194 /* scale up the gear after scaling up clocks */
1200 }
1201 }
1205 out:
1209 }
1212 {
1221 }
1226 }
1229 {
1238 }
1243 }
1247 {
1250 ktime_t start;
1263 }
1271 }
1279 }
1289 out:
1295 }
1300 {
1321 start_window:
1331 }
1334 }
1340 };
1343 {
1349 /* Skip devfreq if we don't have any clocks in the list */
1359 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1360 NULL);
1368 }
1373 }
1376 {
1389 }
1392 {
1399 }
1402 {
1413 }
1418 }
1421 {
1432 }
1437 }
1441 {
1445 }
1449 {
1451 u32 value;
1477 }
1481 out:
1483 }
1486 {
1494 }
1497 {
1509 }
1514 /* Exit from hibern8 */
1516 /* Prevent gating in this path */
1523 else
1525 }
1527 }
1528 unblock_reqs:
1530 }
1532 /**
1533 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1534 * Also, exit from hibern8 mode and set the link as active.
1535 * @hba: per adapter instance
1536 * @async: This indicates whether caller should ungate clocks asynchronously.
1537 */
1539 {
1551 }
1553 start:
1556 /*
1557 * Wait for the ungate work to complete if in progress.
1558 * Though the clocks may be in ON state, the link could
1559 * still be in hibner8 state if hibern8 is allowed
1560 * during clock gating.
1561 * Make sure we exit hibern8 state also in addition to
1562 * clocks being ON.
1563 */
1570 }
1575 }
1583 }
1584 /*
1585 * If we are here, it means gating work is either done or
1586 * currently running. Hence, fall through to cancel gating
1587 * work and to enable clocks.
1588 */
1596 /*
1597 * fall through to check if we should wait for this
1598 * work to be done or not.
1599 */
1605 }
1609 /* Make sure state is CLKS_ON before returning */
1616 }
1618 out:
1620 }
1624 {
1630 /*
1631 * In case you are here to cancel this work the gating state
1632 * would be marked as REQ_CLKS_ON. In this case save time by
1633 * skipping the gating work and exit after changing the clock
1634 * state to CLKS_ON.
1635 */
1642 }
1652 /* put the link into hibern8 mode before turning off clocks */
1659 }
1661 }
1665 else
1666 /* If link is active, device ref_clk can't be switched off */
1669 /*
1670 * In case you are here to cancel this work the gating state
1671 * would be marked as REQ_CLKS_ON. In this case keep the state
1672 * as REQ_CLKS_ON which would anyway imply that clocks are off
1673 * and a request to turn them on is pending. By doing this way,
1674 * we keep the state machine in tact and this would ultimately
1675 * prevent from doing cancel work multiple times when there are
1676 * new requests arriving before the current cancel work is done.
1677 */
1683 }
1684 rel_lock:
1686 out:
1688 }
1690 /* host lock must be held before calling this variant */
1692 {
1710 }
1713 {
1719 }
1724 {
1728 }
1732 {
1743 }
1747 {
1751 }
1755 {
1758 u32 value;
1773 }
1776 out:
1778 }
1781 {
1788 ufshcd_clk_scaling_suspend_work);
1790 ufshcd_clk_scaling_resume_work);
1797 }
1800 {
1806 }
1809 {
1822 WQ_MEM_RECLAIM);
1841 }
1844 {
1852 }
1854 /* Must be called with host lock acquired */
1856 {
1876 }
1881 }
1882 }
1885 {
1896 }
1897 }
1898 /**
1899 * ufshcd_send_command - Send SCSI or device management commands
1900 * @hba: per adapter instance
1901 * @task_tag: Task tag of the command
1902 */
1905 {
1911 /* Make sure that doorbell is committed immediately */
1914 }
1916 /**
1917 * ufshcd_copy_sense_data - Copy sense data in case of check condition
1918 * @lrbp: pointer to local reference block
1919 */
1921 {
1933 }
1934 }
1936 /**
1937 * ufshcd_copy_query_response() - Copy the Query Response and the data
1938 * descriptor
1939 * @hba: per adapter instance
1940 * @lrbp: pointer to local reference block
1941 */
1942 static
1944 {
1949 /* Get the descriptor */
1953 GENERAL_UPIU_REQUEST_SIZE;
1954 u16 resp_len;
1955 u16 buf_len;
1957 /* data segment length */
1959 MASK_QUERY_DATA_SEG_LEN;
1967 __func__);
1969 }
1970 }
1973 }
1975 /**
1976 * ufshcd_hba_capabilities - Read controller capabilities
1977 * @hba: per adapter instance
1978 */
1980 {
1983 /* nutrs and nutmrs are 0 based values */
1987 }
1989 /**
1990 * ufshcd_ready_for_uic_cmd - Check if controller is ready
1991 * to accept UIC commands
1992 * @hba: per adapter instance
1993 * Return true on success, else false
1994 */
1996 {
1999 else
2001 }
2003 /**
2004 * ufshcd_get_upmcrs - Get the power mode change request status
2005 * @hba: Pointer to adapter instance
2006 *
2007 * This function gets the UPMCRS field of HCS register
2008 * Returns value of UPMCRS field
2009 */
2011 {
2013 }
2015 /**
2016 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
2017 * @hba: per adapter instance
2018 * @uic_cmd: UIC command
2019 *
2020 * Mutex must be held.
2021 */
2024 {
2029 /* Write Args */
2034 /* Write UIC Cmd */
2036 REG_UIC_COMMAND);
2037 }
2039 /**
2040 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2041 * @hba: per adapter instance
2042 * @uic_cmd: UIC command
2043 *
2044 * Must be called with mutex held.
2045 * Returns 0 only if success.
2046 */
2047 static int
2049 {
2056 else
2064 }
2066 /**
2067 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2068 * @hba: per adapter instance
2069 * @uic_cmd: UIC command
2070 * @completion: initialize the completion only if this is set to true
2071 *
2072 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
2073 * with mutex held and host_lock locked.
2074 * Returns 0 only if success.
2075 */
2076 static int
2079 {
2084 }
2092 }
2094 /**
2095 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2096 * @hba: per adapter instance
2097 * @uic_cmd: UIC command
2098 *
2099 * Returns 0 only if success.
2100 */
2101 static int
2103 {
2121 }
2123 /**
2124 * ufshcd_map_sg - Map scatter-gather list to prdt
2125 * @hba: per adapter instance
2126 * @lrbp: pointer to local reference block
2127 *
2128 * Returns 0 in case of success, non-zero value in case of failure
2129 */
2131 {
2148 else
2162 }
2165 }
2168 }
2170 /**
2171 * ufshcd_enable_intr - enable interrupts
2172 * @hba: per adapter instance
2173 * @intrs: interrupt bits
2174 */
2176 {
2180 u32 rw;
2185 }
2188 }
2190 /**
2191 * ufshcd_disable_intr - disable interrupts
2192 * @hba: per adapter instance
2193 * @intrs: interrupt bits
2194 */
2196 {
2200 u32 rw;
2207 }
2210 }
2212 /**
2213 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2214 * descriptor according to request
2215 * @lrbp: pointer to local reference block
2216 * @upiu_flags: flags required in the header
2217 * @cmd_dir: requests data direction
2218 */
2221 {
2223 u32 data_direction;
2224 u32 dword_0;
2235 }
2242 /* Transfer request descriptor header fields */
2244 /* dword_1 is reserved, hence it is set to 0 */
2246 /*
2247 * assigning invalid value for command status. Controller
2248 * updates OCS on command completion, with the command
2249 * status
2250 */
2253 /* dword_3 is reserved, hence it is set to 0 */
2257 }
2259 /**
2260 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2261 * for scsi commands
2262 * @lrbp: local reference block pointer
2263 * @upiu_flags: flags
2264 */
2265 static
2267 {
2271 /* command descriptor fields */
2278 /* Total EHS length and Data segment length will be zero */
2289 }
2291 /**
2292 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2293 * for query requsts
2294 * @hba: UFS hba
2295 * @lrbp: local reference block pointer
2296 * @upiu_flags: flags
2297 */
2300 {
2306 /* Query request header */
2313 /* Data segment length only need for WRITE_DESC */
2317 else
2320 /* Copy the Query Request buffer as is */
2322 QUERY_OSF_SIZE);
2324 /* Copy the Descriptor */
2329 }
2332 {
2337 /* command descriptor fields */
2341 /* clear rest of the fields of basic header */
2346 }
2348 /**
2349 * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
2350 * for Device Management Purposes
2351 * @hba: per adapter instance
2352 * @lrbp: pointer to local reference block
2353 */
2355 {
2356 u32 upiu_flags;
2362 else
2370 else
2374 }
2376 /**
2377 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2378 * for SCSI Purposes
2379 * @hba: per adapter instance
2380 * @lrbp: pointer to local reference block
2381 */
2383 {
2384 u32 upiu_flags;
2390 else
2399 }
2402 }
2404 /**
2405 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2406 * @upiu_wlun_id: UPIU W-LUN id
2407 *
2408 * Returns SCSI W-LUN id
2409 */
2411 {
2413 }
2415 /**
2416 * ufshcd_queuecommand - main entry point for SCSI requests
2417 * @host: SCSI host pointer
2418 * @cmd: command from SCSI Midlayer
2419 *
2420 * Returns 0 for success, non-zero in case of failure
2421 */
2423 {
2438 }
2461 }
2463 /* if error handling is in progress, don't issue commands */
2468 }
2473 /* acquire the tag to make sure device cmds don't use it */
2475 /*
2476 * Dev manage command in progress, requeue the command.
2477 * Requeuing the command helps in cases where the request *may*
2478 * find different tag instead of waiting for dev manage command
2479 * completion.
2480 */
2483 }
2490 }
2512 }
2513 /* Make sure descriptors are ready before ringing the doorbell */
2516 /* issue command to the controller */
2520 out_unlock:
2522 out:
2525 }
2529 {
2539 }
2541 static int
2543 {
2548 /* clear outstanding transaction before retry */
2553 /*
2554 * wait for for h/w to clear corresponding bit in door-bell.
2555 * max. wait is 1 sec.
2556 */
2558 REG_UTP_TRANSFER_REQ_DOOR_BELL,
2562 }
2564 static int
2566 {
2569 /* Get the UPIU response */
2571 UPIU_RSP_CODE_OFFSET;
2573 }
2575 /**
2576 * ufshcd_dev_cmd_completion() - handles device management command responses
2577 * @hba: per adapter instance
2578 * @lrbp: pointer to local reference block
2579 */
2580 static int
2582 {
2595 }
2603 /* TODO: handle Reject UPIU Response */
2606 __func__);
2613 }
2616 }
2620 {
2628 /* Make sure descriptors are ready before ringing the doorbell */
2636 }
2644 /* successfully cleared the command, retry if needed */
2646 /*
2647 * in case of an error, after clearing the doorbell,
2648 * we also need to clear the outstanding_request
2649 * field in hba
2650 */
2652 }
2655 }
2657 /**
2658 * ufshcd_get_dev_cmd_tag - Get device management command tag
2659 * @hba: per-adapter instance
2660 * @tag_out: pointer to variable with available slot value
2661 *
2662 * Get a free slot and lock it until device management command
2663 * completes.
2664 *
2665 * Returns false if free slot is unavailable for locking, else
2666 * return true with tag value in @tag.
2667 */
2669 {
2686 out:
2688 }
2691 {
2693 }
2695 /**
2696 * ufshcd_exec_dev_cmd - API for sending device management requests
2697 * @hba: UFS hba
2698 * @cmd_type: specifies the type (NOP, Query...)
2699 * @timeout: time in seconds
2700 *
2701 * NOTE: Since there is only one available tag for device management commands,
2702 * it is expected you hold the hba->dev_cmd.lock mutex.
2703 */
2706 {
2715 /*
2716 * Get free slot, sleep if slots are unavailable.
2717 * Even though we use wait_event() which sleeps indefinitely,
2718 * the maximum wait time is bounded by SCSI request timeout.
2719 */
2732 /* Make sure descriptors are ready before ringing the doorbell */
2744 out_put_tag:
2749 }
2751 /**
2752 * ufshcd_init_query() - init the query response and request parameters
2753 * @hba: per-adapter instance
2754 * @request: address of the request pointer to be initialized
2755 * @response: address of the response pointer to be initialized
2756 * @opcode: operation to perform
2757 * @idn: flag idn to access
2758 * @index: LU number to access
2759 * @selector: query/flag/descriptor further identification
2760 */
2764 {
2773 }
2777 {
2787 else
2789 }
2796 }
2798 /**
2799 * ufshcd_query_flag() - API function for sending flag query requests
2800 * @hba: per-adapter instance
2801 * @opcode: flag query to perform
2802 * @idn: flag idn to access
2803 * @flag_res: the flag value after the query request completes
2804 *
2805 * Returns 0 for success, non-zero in case of failure
2806 */
2809 {
2820 selector);
2831 /* No dummy reads */
2833 __func__);
2836 }
2844 }
2853 }
2859 out_unlock:
2863 }
2865 /**
2866 * ufshcd_query_attr - API function for sending attribute requests
2867 * @hba: per-adapter instance
2868 * @opcode: attribute opcode
2869 * @idn: attribute idn to access
2870 * @index: index field
2871 * @selector: selector field
2872 * @attr_val: the attribute value after the query request completes
2873 *
2874 * Returns 0 for success, non-zero in case of failure
2875 */
2878 {
2891 }
2895 selector);
2910 }
2918 }
2922 out_unlock:
2924 out:
2927 }
2929 /**
2930 * ufshcd_query_attr_retry() - API function for sending query
2931 * attribute with retries
2932 * @hba: per-adapter instance
2933 * @opcode: attribute opcode
2934 * @idn: attribute idn to access
2935 * @index: index field
2936 * @selector: selector field
2937 * @attr_val: the attribute value after the query request
2938 * completes
2939 *
2940 * Returns 0 for success, non-zero in case of failure
2941 */
2945 {
2947 u32 retries;
2955 else
2957 }
2964 }
2969 {
2982 }
2989 }
2993 selector);
3010 }
3018 }
3022 out_unlock:
3025 out:
3028 }
3030 /**
3031 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3032 * @hba: per-adapter instance
3033 * @opcode: attribute opcode
3034 * @idn: attribute idn to access
3035 * @index: index field
3036 * @selector: selector field
3037 * @desc_buf: the buffer that contains the descriptor
3038 * @buf_len: length parameter passed to the device
3039 *
3040 * Returns 0 for success, non-zero in case of failure.
3041 * The buf_len parameter will contain, on return, the length parameter
3042 * received on the response.
3043 */
3047 u8 selector,
3049 {
3058 }
3061 }
3063 /**
3064 * ufshcd_read_desc_length - read the specified descriptor length from header
3065 * @hba: Pointer to adapter instance
3066 * @desc_id: descriptor idn value
3067 * @desc_index: descriptor index
3068 * @desc_length: pointer to variable to read the length of descriptor
3069 *
3070 * Return 0 in case of success, non-zero otherwise
3071 */
3076 {
3095 desc_id);
3097 }
3102 }
3104 /**
3105 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3106 * @hba: Pointer to adapter instance
3107 * @desc_id: descriptor idn value
3108 * @desc_len: mapped desc length (out)
3109 *
3110 * Return 0 in case of success, non-zero otherwise
3111 */
3114 {
3147 }
3149 }
3152 /**
3153 * ufshcd_read_desc_param - read the specified descriptor parameter
3154 * @hba: Pointer to adapter instance
3155 * @desc_id: descriptor idn value
3156 * @desc_index: descriptor index
3157 * @param_offset: offset of the parameter to read
3158 * @param_read_buf: pointer to buffer where parameter would be read
3159 * @param_size: sizeof(param_read_buf)
3160 *
3161 * Return 0 in case of success, non-zero otherwise
3162 */
3166 u8 param_offset,
3168 u8 param_size)
3169 {
3175 /* Safety check */
3179 /* Get the max length of descriptor from structure filled up at probe
3180 * time.
3181 */
3184 /* Sanity checks */
3187 __func__);
3189 }
3191 /* Check whether we need temp memory */
3199 }
3201 /* Request for full descriptor */
3207 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3210 }
3212 /* Sanity check */
3218 }
3220 /* Check wherher we will not copy more data, than available */
3226 out:
3230 }
3236 u32 size)
3237 {
3239 }
3243 u32 size)
3244 {
3246 }
3249 {
3251 }
3253 /**
3254 * ufshcd_read_string_desc - read string descriptor
3255 * @hba: pointer to adapter instance
3256 * @desc_index: descriptor index
3257 * @buf: pointer to buffer where descriptor would be read
3258 * @size: size of buf
3259 * @ascii: if true convert from unicode to ascii characters
3260 *
3261 * Return 0 in case of success, non-zero otherwise
3262 */
3265 {
3275 }
3284 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3288 __func__);
3291 }
3297 }
3299 /*
3300 * the descriptor contains string in UTF16 format
3301 * we need to convert to utf-8 so it can be displayed
3302 */
3307 /* replace non-printable or non-ASCII characters with spaces */
3316 }
3317 out:
3319 }
3321 /**
3322 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3323 * @hba: Pointer to adapter instance
3324 * @lun: lun id
3325 * @param_offset: offset of the parameter to read
3326 * @param_read_buf: pointer to buffer where parameter would be read
3327 * @param_size: sizeof(param_read_buf)
3328 *
3329 * Return 0 in case of success, non-zero otherwise
3330 */
3335 u32 param_size)
3336 {
3337 /*
3338 * Unit descriptors are only available for general purpose LUs (LUN id
3339 * from 0 to 7) and RPMB Well known LU.
3340 */
3346 }
3348 /**
3349 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3350 * @hba: per adapter instance
3351 *
3352 * 1. Allocate DMA memory for Command Descriptor array
3353 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3354 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3355 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3356 * (UTMRDL)
3357 * 4. Allocate memory for local reference block(lrb).
3358 *
3359 * Returns 0 for success, non-zero in case of failure
3360 */
3362 {
3365 /* Allocate memory for UTP command descriptors */
3368 ucdl_size,
3370 GFP_KERNEL);
3372 /*
3373 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3374 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3375 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3376 * be aligned to 128 bytes as well
3377 */
3383 }
3385 /*
3386 * Allocate memory for UTP Transfer descriptors
3387 * UFSHCI requires 1024 byte alignment of UTRD
3388 */
3391 utrdl_size,
3393 GFP_KERNEL);
3399 }
3401 /*
3402 * Allocate memory for UTP Task Management descriptors
3403 * UFSHCI requires 1024 byte alignment of UTMRD
3404 */
3407 utmrdl_size,
3409 GFP_KERNEL);
3415 }
3417 /* Allocate memory for local reference block */
3420 GFP_KERNEL);
3424 }
3426 out:
3428 }
3430 /**
3431 * ufshcd_host_memory_configure - configure local reference block with
3432 * memory offsets
3433 * @hba: per adapter instance
3434 *
3435 * Configure Host memory space
3436 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3437 * address.
3438 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3439 * and PRDT offset.
3440 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3441 * into local reference block.
3442 */
3444 {
3447 dma_addr_t cmd_desc_dma_addr;
3448 dma_addr_t cmd_desc_element_addr;
3449 u16 response_offset;
3450 u16 prdt_offset;
3457 response_offset =
3459 prdt_offset =
3466 /* Configure UTRD with command descriptor base address */
3467 cmd_desc_element_addr =
3474 /* Response upiu and prdt offset should be in double words */
3489 }
3500 response_offset;
3504 prdt_offset;
3505 }
3506 }
3508 /**
3509 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3510 * @hba: per adapter instance
3511 *
3512 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3513 * in order to initialize the Unipro link startup procedure.
3514 * Once the Unipro links are up, the device connected to the controller
3515 * is detected.
3516 *
3517 * Returns 0 on success, non-zero value on failure
3518 */
3520 {
3531 }
3532 /**
3533 * ufshcd_dme_reset - UIC command for DME_RESET
3534 * @hba: per adapter instance
3535 *
3536 * DME_RESET command is issued in order to reset UniPro stack.
3537 * This function now deal with cold reset.
3538 *
3539 * Returns 0 on success, non-zero value on failure
3540 */
3542 {
3554 }
3556 /**
3557 * ufshcd_dme_enable - UIC command for DME_ENABLE
3558 * @hba: per adapter instance
3559 *
3560 * DME_ENABLE command is issued in order to enable UniPro stack.
3561 *
3562 * Returns 0 on success, non-zero value on failure
3563 */
3565 {
3577 }
3580 {
3581 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3587 /*
3588 * last_dme_cmd_tstamp will be 0 only for 1st call to
3589 * this function
3590 */
3600 min_sleep_time_us =
3602 else
3604 }
3606 /* allow sleep for extra 50us if needed */
3608 }
3610 /**
3611 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3612 * @hba: per adapter instance
3613 * @attr_sel: uic command argument1
3614 * @attr_set: attribute set type as uic command argument2
3615 * @mib_val: setting value as uic command argument3
3616 * @peer: indicate whether peer or local
3617 *
3618 * Returns 0 on success, non-zero value on failure
3619 */
3622 {
3626 "dme-peer-set"
3627 };
3639 /* for peer attributes we retry upon failure */
3652 }
3655 /**
3656 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3657 * @hba: per adapter instance
3658 * @attr_sel: uic command argument1
3659 * @mib_val: the value of the attribute as returned by the UIC command
3660 * @peer: indicate whether peer or local
3661 *
3662 * Returns 0 on success, non-zero value on failure
3663 */
3666 {
3670 "dme-peer-get"
3671 };
3693 }
3698 }
3699 }
3706 /* for peer attributes we retry upon failure */
3724 out:
3726 }
3729 /**
3730 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3731 * state) and waits for it to take effect.
3732 *
3733 * @hba: per adapter instance
3734 * @cmd: UIC command to execute
3735 *
3736 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3737 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3738 * and device UniPro link and hence it's final completion would be indicated by
3739 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3740 * addition to normal UIC command completion Status (UCCS). This function only
3741 * returns after the relevant status bits indicate the completion.
3742 *
3743 * Returns 0 on success, non-zero value on failure
3744 */
3746 {
3749 u8 status;
3761 /*
3762 * Make sure UIC command completion interrupt is disabled before
3763 * issuing UIC command.
3764 */
3767 }
3775 }
3784 }
3792 }
3793 out:
3798 }
3809 }
3811 /**
3812 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
3813 * using DME_SET primitives.
3814 * @hba: per adapter instance
3815 * @mode: powr mode value
3816 *
3817 * Returns 0 on success, non-zero value on failure
3818 */
3820 {
3831 }
3832 }
3841 out:
3843 }
3846 {
3868 }
3871 {
3889 /*
3890 * If link recovery fails then return error code returned from
3891 * ufshcd_link_recovery().
3892 * If link recovery succeeds then return -EAGAIN to attempt
3893 * hibern8 enter retry again.
3894 */
3901 }
3904 POST_CHANGE);
3907 }
3910 {
3917 }
3918 out:
3920 }
3923 {
3941 POST_CHANGE);
3944 }
3947 }
3950 {
3959 }
3961 /**
3962 * ufshcd_init_pwr_info - setting the POR (power on reset)
3963 * values in hba power info
3964 * @hba: per-adapter instance
3965 */
3967 {
3975 }
3977 /**
3978 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
3979 * @hba: per-adapter instance
3980 */
3982 {
3992 /* Get the connected lane count */
4000 __func__,
4004 }
4006 /*
4007 * First, get the maximum gears of HS speed.
4008 * If a zero value, it means there is no HSGEAR capability.
4009 * Then, get the maximum gears of PWM speed.
4010 */
4019 }
4021 }
4032 }
4034 }
4038 }
4042 {
4045 /* if already configured to the requested pwr_mode */
4055 }
4057 /*
4058 * Configure attributes for power mode change with below.
4059 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4060 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4061 * - PA_HSSERIES
4062 */
4069 else
4078 else
4096 pwr_mode);
4100 }
4103 }
4105 /**
4106 * ufshcd_config_pwr_mode - configure a new power mode
4107 * @hba: per-adapter instance
4108 * @desired_pwr_mode: desired power configuration
4109 */
4112 {
4127 }
4130 /**
4131 * ufshcd_complete_dev_init() - checks device readiness
4132 * @hba: per-adapter instance
4133 *
4134 * Set fDeviceInit flag and poll until device toggles it.
4135 */
4137 {
4149 }
4151 /* poll for max. 1000 iterations for fDeviceInit flag to clear */
4163 __func__);
4165 out:
4167 }
4169 /**
4170 * ufshcd_make_hba_operational - Make UFS controller operational
4171 * @hba: per adapter instance
4172 *
4173 * To bring UFS host controller to operational state,
4174 * 1. Enable required interrupts
4175 * 2. Configure interrupt aggregation
4176 * 3. Program UTRL and UTMRL base address
4177 * 4. Configure run-stop-registers
4178 *
4179 * Returns 0 on success, non-zero value on failure
4180 */
4182 {
4184 u32 reg;
4186 /* Enable required interrupts */
4189 /* Configure interrupt aggregation */
4192 else
4195 /* Configure UTRL and UTMRL base address registers */
4197 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4199 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4201 REG_UTP_TASK_REQ_LIST_BASE_L);
4203 REG_UTP_TASK_REQ_LIST_BASE_H);
4205 /*
4206 * Make sure base address and interrupt setup are updated before
4207 * enabling the run/stop registers below.
4208 */
4211 /*
4212 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4213 */
4222 }
4224 out:
4226 }
4228 /**
4229 * ufshcd_hba_stop - Send controller to reset state
4230 * @hba: per adapter instance
4231 * @can_sleep: perform sleep or just spin
4232 */
4234 {
4243 }
4245 /**
4246 * ufshcd_hba_execute_hce - initialize the controller
4247 * @hba: per adapter instance
4248 *
4249 * The controller resets itself and controller firmware initialization
4250 * sequence kicks off. When controller is ready it will set
4251 * the Host Controller Enable bit to 1.
4252 *
4253 * Returns 0 on success, non-zero value on failure
4254 */
4256 {
4259 /*
4260 * msleep of 1 and 5 used in this function might result in msleep(20),
4261 * but it was necessary to send the UFS FPGA to reset mode during
4262 * development and testing of this driver. msleep can be changed to
4263 * mdelay and retry count can be reduced based on the controller.
4264 */
4266 /* change controller state to "reset state" */
4269 /* UniPro link is disabled at this point */
4274 /* start controller initialization sequence */
4277 /*
4278 * To initialize a UFS host controller HCE bit must be set to 1.
4279 * During initialization the HCE bit value changes from 1->0->1.
4280 * When the host controller completes initialization sequence
4281 * it sets the value of HCE bit to 1. The same HCE bit is read back
4282 * to check if the controller has completed initialization sequence.
4283 * So without this delay the value HCE = 1, set in the previous
4284 * instruction might be read back.
4285 * This delay can be changed based on the controller.
4286 */
4289 /* wait for the host controller to complete initialization */
4293 retry--;
4298 }
4300 }
4302 /* enable UIC related interrupts */
4308 }
4311 {
4318 /* enable UIC related interrupts */
4328 }
4331 }
4334 }
4336 {
4342 else
4351 else
4360 }
4361 }
4364 }
4367 {
4369 }
4371 /**
4372 * ufshcd_link_startup - Initialize unipro link startup
4373 * @hba: per adapter instance
4374 *
4375 * Returns 0 for success, non-zero in case of failure
4376 */
4378 {
4383 /*
4384 * If UFS device isn't active then we will have to issue link startup
4385 * 2 times to make sure the device state move to active.
4386 */
4390 link_startup:
4396 /* check if device is detected by inter-connect layer */
4401 }
4403 /*
4404 * DME link lost indication is only received when link is up,
4405 * but we can't be sure if the link is up until link startup
4406 * succeeds. So reset the local Uni-Pro and try again.
4407 */
4413 /* failed to get the link up... retire */
4420 }
4422 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4430 }
4432 /* Include any host controller configuration via UIC commands */
4438 out:
4444 }
4446 }
4448 /**
4449 * ufshcd_verify_dev_init() - Verify device initialization
4450 * @hba: per-adapter instance
4451 *
4452 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4453 * device Transport Protocol (UTP) layer is ready after a reset.
4454 * If the UTP layer at the device side is not initialized, it may
4455 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4456 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4457 */
4459 {
4467 NOP_OUT_TIMEOUT);
4473 }
4480 }
4482 /**
4483 * ufshcd_set_queue_depth - set lun queue depth
4484 * @sdev: pointer to SCSI device
4485 *
4486 * Read bLUQueueDepth value and activate scsi tagged command
4487 * queueing. For WLUN, queue depth is set to 1. For best-effort
4488 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4489 * value that host can queue.
4490 */
4492 {
4494 u8 lun_qdepth;
4502 UNIT_DESC_PARAM_LU_Q_DEPTH,
4506 /* Some WLUN doesn't support unit descriptor */
4510 /* eventually, we can figure out the real queue depth */
4512 else
4518 }
4520 /*
4521 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4522 * @hba: per-adapter instance
4523 * @lun: UFS device lun id
4524 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4525 *
4526 * Returns 0 in case of success and b_lu_write_protect status would be returned
4527 * @b_lu_write_protect parameter.
4528 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4529 * Returns -EINVAL in case of invalid parameters passed to this function.
4530 */
4532 u8 lun,
4534 {
4539 /*
4540 * According to UFS device spec, RPMB LU can't be write
4541 * protected so skip reading bLUWriteProtect parameter for
4542 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4543 */
4546 else
4548 lun,
4549 UNIT_DESC_PARAM_LU_WR_PROTECT,
4550 b_lu_write_protect,
4553 }
4555 /**
4556 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4557 * status
4558 * @hba: per-adapter instance
4559 * @sdev: pointer to SCSI device
4560 *
4561 */
4564 {
4567 u8 b_lu_write_protect;
4573 }
4574 }
4576 /**
4577 * ufshcd_slave_alloc - handle initial SCSI device configurations
4578 * @sdev: pointer to SCSI device
4579 *
4580 * Returns success
4581 */
4583 {
4588 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4591 /* allow SCSI layer to restart the device in case of errors */
4594 /* REPORT SUPPORTED OPERATION CODES is not supported */
4597 /* WRITE_SAME command is not supported */
4605 }
4607 /**
4608 * ufshcd_change_queue_depth - change queue depth
4609 * @sdev: pointer to SCSI device
4610 * @depth: required depth to set
4611 *
4612 * Change queue depth and make sure the max. limits are not crossed.
4613 */
4615 {
4621 }
4623 /**
4624 * ufshcd_slave_configure - adjust SCSI device configurations
4625 * @sdev: pointer to SCSI device
4626 */
4628 {
4635 }
4637 /**
4638 * ufshcd_slave_destroy - remove SCSI device configurations
4639 * @sdev: pointer to SCSI device
4640 */
4642 {
4646 /* Drop the reference as it won't be needed anymore */
4653 }
4654 }
4656 /**
4657 * ufshcd_task_req_compl - handle task management request completion
4658 * @hba: per adapter instance
4659 * @index: index of the completed request
4660 * @resp: task management service response
4661 *
4662 * Returns non-zero value on error, zero on success
4663 */
4665 {
4674 /* Clear completed tasks from outstanding_tasks */
4690 }
4694 }
4696 /**
4697 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
4698 * @lrbp: pointer to local reference block of completed command
4699 * @scsi_status: SCSI command status
4700 *
4701 * Returns value base on SCSI command status
4702 */
4705 {
4714 scsi_status;
4728 }
4730 /**
4731 * ufshcd_transfer_rsp_status - Get overall status of the response
4732 * @hba: per adapter instance
4733 * @lrbp: pointer to local reference block of completed command
4734 *
4735 * Returns result of the command to notify SCSI midlayer
4736 */
4739 {
4744 /* overall command status of utrd */
4753 /*
4754 * get the response UPIU result to extract
4755 * the SCSI command status
4756 */
4759 /*
4760 * get the result based on SCSI status response
4761 * to notify the SCSI midlayer of the command status
4762 */
4766 /*
4767 * Currently we are only supporting BKOPs exception
4768 * events hence we can ignore BKOPs exception event
4769 * during power management callbacks. BKOPs exception
4770 * event is not expected to be raised in runtime suspend
4771 * callback as it allows the urgent bkops.
4772 * During system suspend, we are anyway forcefully
4773 * disabling the bkops and if urgent bkops is needed
4774 * it will be enabled on system resume. Long term
4775 * solution could be to abort the system suspend if
4776 * UFS device needs urgent BKOPs.
4777 */
4783 /* TODO: handle Reject UPIU Response */
4792 result);
4794 }
4821 }
4823 /**
4824 * ufshcd_uic_cmd_compl - handle completion of uic command
4825 * @hba: per adapter instance
4826 * @intr_status: interrupt status generated by the controller
4827 */
4829 {
4836 }
4840 }
4842 /**
4843 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
4844 * @hba: per adapter instance
4845 * @completed_reqs: requests to complete
4846 */
4849 {
4863 /* Mark completed command as NULL in LRB */
4866 /* Do not touch lrbp after scsi done */
4875 }
4876 }
4881 }
4883 /* clear corresponding bits of completed commands */
4888 /* we might have free'd some tags above */
4890 }
4892 /**
4893 * ufshcd_transfer_req_compl - handle SCSI and query command completion
4894 * @hba: per adapter instance
4895 */
4897 {
4899 u32 tr_doorbell;
4901 /* Resetting interrupt aggregation counters first and reading the
4902 * DOOR_BELL afterward allows us to handle all the completed requests.
4903 * In order to prevent other interrupts starvation the DB is read once
4904 * after reset. The down side of this solution is the possibility of
4905 * false interrupt if device completes another request after resetting
4906 * aggregation and before reading the DB.
4907 */
4916 }
4918 /**
4919 * ufshcd_disable_ee - disable exception event
4920 * @hba: per-adapter instance
4921 * @mask: exception event to disable
4922 *
4923 * Disables exception event in the device so that the EVENT_ALERT
4924 * bit is not set.
4925 *
4926 * Returns zero on success, non-zero error value on failure.
4927 */
4929 {
4931 u32 val;
4942 out:
4944 }
4946 /**
4947 * ufshcd_enable_ee - enable exception event
4948 * @hba: per-adapter instance
4949 * @mask: exception event to enable
4950 *
4951 * Enable corresponding exception event in the device to allow
4952 * device to alert host in critical scenarios.
4953 *
4954 * Returns zero on success, non-zero error value on failure.
4955 */
4957 {
4959 u32 val;
4970 out:
4972 }
4974 /**
4975 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
4976 * @hba: per-adapter instance
4977 *
4978 * Allow device to manage background operations on its own. Enabling
4979 * this might lead to inconsistent latencies during normal data transfers
4980 * as the device is allowed to manage its own way of handling background
4981 * operations.
4982 *
4983 * Returns zero on success, non-zero on failure.
4984 */
4986 {
4998 }
5003 /* No need of URGENT_BKOPS exception from the device */
5008 out:
5010 }
5012 /**
5013 * ufshcd_disable_auto_bkops - block device in doing background operations
5014 * @hba: per-adapter instance
5015 *
5016 * Disabling background operations improves command response latency but
5017 * has drawback of device moving into critical state where the device is
5018 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5019 * host is idle so that BKOPS are managed effectively without any negative
5020 * impacts.
5021 *
5022 * Returns zero on success, non-zero on failure.
5023 */
5025 {
5031 /*
5032 * If host assisted BKOPs is to be enabled, make sure
5033 * urgent bkops exception is allowed.
5034 */
5040 }
5049 }
5054 out:
5056 }
5058 /**
5059 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5060 * @hba: per adapter instance
5061 *
5062 * After a device reset the device may toggle the BKOPS_EN flag
5063 * to default value. The s/w tracking variables should be updated
5064 * as well. This function would change the auto-bkops state based on
5065 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
5066 */
5068 {
5077 }
5079 }
5082 {
5085 }
5087 /**
5088 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5089 * @hba: per-adapter instance
5090 * @status: bkops_status value
5091 *
5092 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5093 * flag in the device to permit background operations if the device
5094 * bkops_status is greater than or equal to "status" argument passed to
5095 * this function, disable otherwise.
5096 *
5097 * Returns 0 for success, non-zero in case of failure.
5098 *
5099 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5100 * to know whether auto bkops is enabled or disabled after this function
5101 * returns control to it.
5102 */
5105 {
5119 }
5123 else
5125 out:
5127 }
5129 /**
5130 * ufshcd_urgent_bkops - handle urgent bkops exception event
5131 * @hba: per-adapter instance
5132 *
5133 * Enable fBackgroundOpsEn flag in the device to permit background
5134 * operations.
5135 *
5136 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5137 * and negative error value for any other failure.
5138 */
5140 {
5142 }
5145 {
5148 }
5151 {
5163 }
5165 /*
5166 * We are seeing that some devices are raising the urgent bkops
5167 * exception events even when BKOPS status doesn't indicate performace
5168 * impacted or critical. Handle these device by determining their urgent
5169 * bkops status at runtime.
5170 */
5174 /* update the current status as the urgent bkops level */
5177 }
5179 enable_auto_bkops:
5181 out:
5185 }
5187 /**
5188 * ufshcd_exception_event_handler - handle exceptions raised by device
5189 * @work: pointer to work data
5190 *
5191 * Read bExceptionEventStatus attribute from the device and handle the
5192 * exception event accordingly.
5193 */
5195 {
5208 }
5215 out:
5219 }
5221 /* Complete requests that have door-bell cleared */
5223 {
5226 }
5228 /**
5229 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5230 * to recover from the DL NAC errors or not.
5231 * @hba: per-adapter instance
5232 *
5233 * Returns true if error handling is required, false otherwise
5234 */
5236 {
5241 /*
5242 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5243 * device fatal error and/or DL NAC & REPLAY timeout errors.
5244 */
5256 /*
5257 * wait for 50ms to see if we can get any other errors or not.
5258 */
5263 /*
5264 * now check if we have got any other severe errors other than
5265 * DL NAC error?
5266 */
5272 /*
5273 * As DL NAC is the only error received so far, send out NOP
5274 * command to confirm if link is still active or not.
5275 * - If we don't get any response then do error recovery.
5276 * - If we get response then clear the DL NAC error bit.
5277 */
5286 /* Link seems to be alive hence ignore the DL NAC errors */
5289 /* clear NAC error */
5294 }
5295 }
5296 out:
5299 }
5301 /**
5302 * ufshcd_err_handler - handle UFS errors that require s/w attention
5303 * @work: pointer to work structure
5304 */
5306 {
5327 /* Complete requests that have door-bell cleared by h/w */
5334 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
5339 }
5343 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5344 UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
5347 /*
5348 * if host reset is required then skip clearing the pending
5349 * transfers forcefully because they will get cleared during
5350 * host reset and restore
5351 */
5355 /* release lock as clear command might sleep */
5357 /* Clear pending transfer requests */
5362 }
5363 }
5365 /* Clear pending task management requests */
5370 }
5371 }
5373 lock_skip_pending_xfer_clear:
5376 /* Complete the requests that are cleared by s/w */
5382 skip_pending_xfer_clear:
5383 /* Fatal errors need reset */
5387 /*
5388 * ufshcd_reset_and_restore() does the link reinitialization
5389 * which will need atleast one empty doorbell slot to send the
5390 * device management commands (NOP and query commands).
5391 * If there is no slot empty at this moment then free up last
5392 * slot forcefully.
5393 */
5403 __func__);
5405 }
5406 /*
5407 * Inform scsi mid-layer that we did reset and allow to handle
5408 * Unit Attention properly.
5409 */
5413 }
5415 skip_err_handling:
5421 }
5425 out:
5430 }
5433 u32 reg)
5434 {
5438 }
5440 /**
5441 * ufshcd_update_uic_error - check and set fatal UIC error flags.
5442 * @hba: per-adapter instance
5443 */
5445 {
5446 u32 reg;
5448 /* PHY layer lane error */
5450 /* Ignore LINERESET indication, as this is not an error */
5453 /*
5454 * To know whether this error is fatal or not, DB timeout
5455 * must be checked but this error is handled separately.
5456 */
5459 }
5461 /* PA_INIT_ERROR is fatal and needs UIC reset */
5469 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5472 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5475 }
5477 /* UIC NL/TL/DME errors needs software retry */
5482 }
5488 }
5494 }
5498 }
5500 /**
5501 * ufshcd_check_errors - Check for errors that need s/w attention
5502 * @hba: per-adapter instance
5503 */
5505 {
5516 }
5519 /*
5520 * update the transfer error masks to sticky bits, let's do this
5521 * irrespective of current ufshcd_state.
5522 */
5526 /* handle fatal errors only when link is functional */
5528 /* block commands from scsi mid-layer */
5533 /* dump controller state before resetting */
5536 SYSTEM_BUS_FATAL_ERROR);
5546 pr_prdt);
5547 }
5549 }
5550 }
5551 /*
5552 * if (!queue_eh_work) -
5553 * Other errors are either non-fatal where host recovers
5554 * itself without s/w intervention or errors that will be
5555 * handled by the SCSI core layer.
5556 */
5557 }
5559 /**
5560 * ufshcd_tmc_handler - handle task management function completion
5561 * @hba: per adapter instance
5562 */
5564 {
5565 u32 tm_doorbell;
5570 }
5572 /**
5573 * ufshcd_sl_intr - Interrupt service routine
5574 * @hba: per adapter instance
5575 * @intr_status: contains interrupts generated by the controller
5576 */
5578 {
5591 }
5593 /**
5594 * ufshcd_intr - Main interrupt service routine
5595 * @irq: irq number
5596 * @__hba: pointer to adapter instance
5597 *
5598 * Returns IRQ_HANDLED - If interrupt is valid
5599 * IRQ_NONE - If invalid interrupt
5600 */
5602 {
5611 /*
5612 * There could be max of hba->nutrs reqs in flight and in worst case
5613 * if the reqs get finished 1 by 1 after the interrupt status is
5614 * read, make sure we handle them by checking the interrupt status
5615 * again in a loop until we process all of the reqs before returning.
5616 */
5618 enabled_intr_status =
5625 }
5632 }
5635 {
5647 /* poll for max. 1 sec to clear door bell register by h/w */
5649 REG_UTP_TASK_REQ_DOOR_BELL,
5651 out:
5653 }
5655 /**
5656 * ufshcd_issue_tm_cmd - issues task management commands to controller
5657 * @hba: per adapter instance
5658 * @lun_id: LUN ID to which TM command is sent
5659 * @task_id: task ID to which the TM command is applicable
5660 * @tm_function: task management function opcode
5661 * @tm_response: task management service response return value
5662 *
5663 * Returns non-zero value on error, zero on success.
5664 */
5667 {
5678 /*
5679 * Get free slot, sleep if slots are unavailable.
5680 * Even though we use wait_event() which sleeps indefinitely,
5681 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
5682 */
5690 /* Configure task request descriptor */
5695 /* Configure task request UPIU */
5696 task_req_upiup =
5704 /*
5705 * The host shall provide the same value for LUN field in the basic
5706 * header and for Input Parameter.
5707 */
5713 /* send command to the controller */
5716 /* Make sure descriptors are ready before ringing the task doorbell */
5720 /* Make sure that doorbell is committed immediately */
5727 /* wait until the task management command is completed */
5742 }
5750 }
5752 /**
5753 * ufshcd_eh_device_reset_handler - device reset handler registered to
5754 * scsi layer.
5755 * @cmd: SCSI command pointer
5756 *
5757 * Returns SUCCESS/FAILED
5758 */
5760 {
5764 u32 pos;
5780 }
5782 /* clear the commands that were pending for corresponding LUN */
5788 }
5789 }
5794 out:
5801 }
5803 }
5806 {
5813 }
5814 }
5816 /**
5817 * ufshcd_abort - abort a specific command
5818 * @cmd: SCSI command pointer
5819 *
5820 * Abort the pending command in device by sending UFS_ABORT_TASK task management
5821 * command, and in host controller by clearing the door-bell register. There can
5822 * be race between controller sending the command to the device while abort is
5823 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
5824 * really issued and then try to abort it.
5825 *
5826 * Returns SUCCESS/FAILED
5827 */
5829 {
5838 u32 reg;
5849 }
5851 /*
5852 * Task abort to the device W-LUN is illegal. When this command
5853 * will fail, due to spec violation, scsi err handling next step
5854 * will be to send LU reset which, again, is a spec violation.
5855 * To avoid these unnecessary/illegal step we skip to the last error
5856 * handling stage: reset and restore.
5857 */
5863 /* If command is already aborted/completed, return SUCCESS */
5869 }
5875 }
5877 /* Print Transfer Request of aborted task */
5880 /*
5881 * Print detailed info about aborted request.
5882 * As more than one request might get aborted at the same time,
5883 * print full information only for the first aborted request in order
5884 * to reduce repeated printouts. For other aborted requests only print
5885 * basic details.
5886 */
5895 }
5898 /* Skip task abort in case previous aborts failed and report failure */
5902 }
5908 /* cmd pending in the device */
5913 /*
5914 * cmd not pending in the device, check if it is
5915 * in transition.
5916 */
5921 /* sleep for max. 200us to stabilize */
5924 }
5925 /* command completed already */
5936 }
5937 }
5942 }
5951 }
5953 }
5960 }
5972 out:
5979 }
5981 /*
5982 * This ufshcd_release() corresponds to the original scsi cmd that got
5983 * aborted here (as we won't get any IRQ for it).
5984 */
5987 }
5989 /**
5990 * ufshcd_host_reset_and_restore - reset and restore host controller
5991 * @hba: per-adapter instance
5992 *
5993 * Note that host controller reset may issue DME_RESET to
5994 * local and remote (device) Uni-Pro stack and the attributes
5995 * are reset to default state.
5996 *
5997 * Returns zero on success, non-zero on failure
5998 */
6000 {
6004 /*
6005 * Stop the host controller and complete the requests
6006 * cleared by h/w
6007 */
6015 /* scale up clocks to max frequency before full reinitialization */
6022 /* Establish the link again and restore the device */
6027 out:
6032 }
6034 /**
6035 * ufshcd_reset_and_restore - reset and re-initialize host/device
6036 * @hba: per-adapter instance
6037 *
6038 * Reset and recover device, host and re-establish link. This
6039 * is helpful to recover the communication in fatal error conditions.
6040 *
6041 * Returns zero on success, non-zero on failure
6042 */
6044 {
6053 }
6055 /**
6056 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
6057 * @cmd: SCSI command pointer
6058 *
6059 * Returns SUCCESS/FAILED
6060 */
6062 {
6070 /*
6071 * Check if there is any race with fatal error handling.
6072 * If so, wait for it to complete. Even though fatal error
6073 * handling does reset and restore in some cases, don't assume
6074 * anything out of it. We are just avoiding race here.
6075 */
6100 }
6106 }
6108 /**
6109 * ufshcd_get_max_icc_level - calculate the ICC level
6110 * @sup_curr_uA: max. current supported by the regulator
6111 * @start_scan: row at the desc table to start scan from
6112 * @buff: power descriptor buffer
6113 *
6114 * Returns calculated max ICC level for specific regulator
6115 */
6117 {
6120 u16 data;
6121 u16 unit;
6126 ATTR_ICC_LVL_UNIT_OFFSET;
6141 }
6144 }
6148 }
6151 }
6153 /**
6154 * ufshcd_calc_icc_level - calculate the max ICC level
6155 * In case regulators are not initialized we'll return 0
6156 * @hba: per-adapter instance
6157 * @desc_buf: power descriptor buffer to extract ICC levels from.
6158 * @len: length of desc_buff
6159 *
6160 * Returns calculated ICC level
6161 */
6164 {
6173 }
6184 icc_level,
6190 icc_level,
6192 out:
6194 }
6197 {
6212 }
6229 out:
6231 }
6233 /**
6234 * ufshcd_scsi_add_wlus - Adds required W-LUs
6235 * @hba: per-adapter instance
6236 *
6237 * UFS device specification requires the UFS devices to support 4 well known
6238 * logical units:
6239 * "REPORT_LUNS" (address: 01h)
6240 * "UFS Device" (address: 50h)
6241 * "RPMB" (address: 44h)
6242 * "BOOT" (address: 30h)
6243 * UFS device's power management needs to be controlled by "POWER CONDITION"
6244 * field of SSU (START STOP UNIT) command. But this "power condition" field
6245 * will take effect only when its sent to "UFS device" well known logical unit
6246 * hence we require the scsi_device instance to represent this logical unit in
6247 * order for the UFS host driver to send the SSU command for power management.
6248 *
6249 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
6250 * Block) LU so user space process can control this LU. User space may also
6251 * want to have access to BOOT LU.
6252 *
6253 * This function adds scsi device instances for each of all well known LUs
6254 * (except "REPORT LUNS" LU).
6255 *
6256 * Returns zero on success (all required W-LUs are added successfully),
6257 * non-zero error value on failure (if failed to add any of the required W-LU).
6258 */
6260 {
6271 }
6279 }
6286 else
6290 remove_sdev_ufs_device:
6292 out:
6294 }
6298 {
6301 u8 model_index;
6310 }
6317 }
6319 /*
6320 * getting vendor (manufacturerID) and Bank Index in big endian
6321 * format
6322 */
6328 /* Zero-pad entire buffer for string termination. */
6337 }
6342 MAX_MODEL_LEN));
6344 /* Null terminate the model string */
6347 out:
6350 }
6354 {
6363 }
6364 }
6366 /**
6367 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
6368 * @hba: per-adapter instance
6369 *
6370 * PA_TActivate parameter can be tuned manually if UniPro version is less than
6371 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
6372 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
6373 * the hibern8 exit latency.
6374 *
6375 * Returns zero on success, non-zero error value on failure.
6376 */
6378 {
6384 RX_MIN_ACTIVATETIME_CAPABILITY,
6390 /* make sure proper unit conversion is applied */
6391 tuned_pa_tactivate =
6395 tuned_pa_tactivate);
6397 out:
6399 }
6401 /**
6402 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
6403 * @hba: per-adapter instance
6404 *
6405 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
6406 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
6407 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
6408 * This optimal value can help reduce the hibern8 exit latency.
6409 *
6410 * Returns zero on success, non-zero error value on failure.
6411 */
6413 {
6433 peer_rx_hibern8_time_cap);
6434 /* make sure proper unit conversion is applied */
6438 tuned_pa_hibern8time);
6439 out:
6441 }
6443 /**
6444 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
6445 * less than device PA_TACTIVATE time.
6446 * @hba: per-adapter instance
6447 *
6448 * Some UFS devices require host PA_TACTIVATE to be lower than device
6449 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
6450 * for such devices.
6451 *
6452 * Returns zero on success, non-zero error value on failure.
6453 */
6455 {
6477 }
6484 }
6500 u32 new_peer_pa_tactivate;
6504 new_peer_pa_tactivate++;
6506 new_peer_pa_tactivate);
6507 }
6509 out:
6511 }
6514 {
6518 }
6521 /* set 1ms timeout for PA_TACTIVATE */
6528 }
6531 {
6544 }
6547 {
6583 }
6586 {
6594 }
6596 /**
6597 * ufshcd_probe_hba - probe hba to detect device and initialize
6598 * @hba: per-adapter instance
6599 *
6600 * Execute link-startup and verify device initialization
6601 */
6603 {
6612 /* set the default level for urgent bkops */
6616 /* Debug counters initialization */
6619 /* UniPro link is active now */
6622 /* Enable Auto-Hibernate if configured */
6633 /* Init check for device descriptor sizes */
6641 }
6651 /* UFS device is also active now */
6659 __func__);
6666 }
6667 }
6669 /* set the state as operational after switching to desired gear */
6672 /*
6673 * If we are in error handling context or in power management callbacks
6674 * context, no need to scan the host
6675 */
6679 /* clear any previous UFS device information */
6688 /* Add required well known logical units to scsi mid layer */
6693 /* Initialize devfreq after UFS device is detected */
6703 }
6705 }
6709 }
6714 out:
6715 /*
6716 * If we failed to initialize the device or the device is not
6717 * present, turn off the power/clocks etc.
6718 */
6723 }
6729 }
6731 /**
6732 * ufshcd_async_scan - asynchronous execution for probing hba
6733 * @data: data pointer to pass to this function
6734 * @cookie: cookie data
6735 */
6737 {
6741 }
6744 {
6765 }
6766 }
6770 /*
6771 * Bypass SCSI error handling and reset the block layer timer if this
6772 * SCSI command was not actually dispatched to UFS driver, otherwise
6773 * let SCSI layer handle the error as usual.
6774 */
6776 }
6781 NULL,
6782 };
6804 };
6808 {
6814 /*
6815 * "set_load" operation shall be required on those regulators
6816 * which specifically configured current limitation. Otherwise
6817 * zero max_uA may cause unexpected behavior when regulator is
6818 * enabled or set as high power mode.
6819 */
6827 }
6830 }
6834 {
6839 else
6841 UFS_VREG_LPM_LOAD_UA);
6842 }
6846 {
6851 else
6853 }
6857 {
6877 }
6878 }
6884 }
6885 out:
6887 }
6890 {
6904 else
6907 out:
6909 }
6912 {
6923 /* ignore errors on applying disable config */
6929 }
6930 out:
6932 }
6935 {
6955 out:
6960 }
6962 }
6965 {
6972 }
6975 {
6986 }
6987 out:
6989 }
6992 {
7009 out:
7011 }
7014 {
7021 }
7024 {
7034 /* shut off the rail here */
7036 /*
7037 * Mark this rail as no longer used, so it doesn't get enabled
7038 * later by mistake
7039 */
7043 /*
7044 * rail should have been already enabled hence just make sure
7045 * that unused flag is cleared.
7046 */
7048 }
7049 out:
7051 }
7055 {
7066 /*
7067 * vendor specific setup_clocks ops may depend on clocks managed by
7068 * this standard driver hence call the vendor specific setup_clocks
7069 * before disabling the clocks managed here.
7070 */
7075 }
7089 }
7092 }
7096 }
7097 }
7099 /*
7100 * vendor specific setup_clocks ops may depend on clocks managed by
7101 * this standard driver hence call the vendor specific setup_clocks
7102 * after enabling the clocks managed here.
7103 */
7108 }
7110 out:
7115 }
7122 }
7129 }
7132 {
7134 }
7137 {
7156 }
7165 }
7167 }
7170 }
7171 out:
7173 }
7176 {
7192 out_exit:
7194 out:
7199 }
7202 {
7209 }
7212 {
7215 /*
7216 * Handle host controller power separately from the UFS device power
7217 * rails as it will help controlling the UFS host controller power
7218 * collapse easily which is different than UFS device power collapse.
7219 * Also, enable the host controller power before we go ahead with rest
7220 * of the initialization here.
7221 */
7253 out_disable_vreg:
7255 out_disable_clks:
7257 out_disable_hba_vreg:
7259 out:
7261 }
7264 {
7275 }
7276 }
7278 static int
7280 {
7285 UFSHCD_REQ_SENSE_SIZE,
7294 }
7303 out:
7305 }
7307 /**
7308 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
7309 * power mode
7310 * @hba: per adapter instance
7311 * @pwr_mode: device power mode to set
7312 *
7313 * Returns 0 if requested power mode is set successfully
7314 * Returns non-zero if failed to set the requested power mode
7315 */
7318 {
7332 }
7335 }
7341 /*
7342 * If scsi commands fail, the scsi mid-layer schedules scsi error-
7343 * handling, which would wait for host to be resumed. Since we know
7344 * we are functional while we are here, skip host resume in error
7345 * handling context.
7346 */
7352 /* Unit attention condition is cleared now */
7354 }
7358 /*
7359 * Current function would be generally called from the power management
7360 * callbacks hence set the RQF_PM flag so that it doesn't resume the
7361 * already suspended childs.
7362 */
7371 }
7375 out:
7379 }
7384 {
7394 else
7396 }
7397 /*
7398 * If autobkops is enabled, link can't be turned off because
7399 * turning off the link would also turn off the device.
7400 */
7404 /*
7405 * Let's make sure that link is in low power mode, we are doing
7406 * this currently by putting the link in Hibern8. Otherway to
7407 * put the link in low power mode is to send the DME end point
7408 * to device and then send the DME reset command to local
7409 * unipro. But putting the link in hibern8 is much faster.
7410 */
7414 /*
7415 * Change controller state to "reset state" which
7416 * should also put the link in off/reset state
7417 */
7419 /*
7420 * TODO: Check if we need any delay to make sure that
7421 * controller is reset
7422 */
7424 }
7426 out:
7428 }
7431 {
7432 /*
7433 * It seems some UFS devices may keep drawing more than sleep current
7434 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
7435 * To avoid this situation, add 2ms delay before putting these UFS
7436 * rails in LPM mode.
7437 */
7442 /*
7443 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
7444 * power.
7445 *
7446 * If UFS device and link is in OFF state, all power supplies (VCC,
7447 * VCCQ, VCCQ2) can be turned off if power on write protect is not
7448 * required. If UFS link is inactive (Hibern8 or OFF state) and device
7449 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
7450 *
7451 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
7452 * in low power state which would save some power.
7453 */
7462 }
7463 }
7464 }
7467 {
7481 }
7483 }
7486 vccq_lpm:
7488 vcc_disable:
7490 out:
7492 }
7495 {
7498 }
7501 {
7504 }
7506 /**
7507 * ufshcd_suspend - helper function for suspend operations
7508 * @hba: per adapter instance
7509 * @pm_op: desired low power operation type
7510 *
7511 * This function will try to put the UFS device and link into low power
7512 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
7513 * (System PM level).
7514 *
7515 * If this function is called during shutdown, it will make sure that
7516 * both UFS device and UFS link is powered off.
7517 *
7518 * NOTE: UFS device & link must be active before we enter in this function.
7519 *
7520 * Returns 0 for success and non-zero for failure
7521 */
7523 {
7538 }
7540 /*
7541 * If we can't transition into any of the low power modes
7542 * just gate the clocks.
7543 */
7551 }
7556 }
7562 /* UFS device & link must be active before we enter in this function */
7566 }
7570 /*
7571 * The device is idle with no requests in the queue,
7572 * allow background operations if bkops status shows
7573 * that performance might be impacted.
7574 */
7579 /* make sure that auto bkops is disabled */
7581 }
7582 }
7587 /* ensure that bkops is disabled */
7592 }
7600 disable_clks:
7601 /*
7602 * Call vendor specific suspend callback. As these callbacks may access
7603 * vendor specific host controller register space call them before the
7604 * host clocks are ON.
7605 */
7612 else
7613 /* If link is active, device ref_clk can't be switched off */
7618 /*
7619 * Disable the host irq as host controller as there won't be any
7620 * host controller transaction expected till resume.
7621 */
7623 /* Put the host controller in low power mode if possible */
7627 set_link_active:
7635 set_dev_active:
7638 enable_gating:
7643 out:
7646 }
7648 /**
7649 * ufshcd_resume - helper function for resume operations
7650 * @hba: per adapter instance
7651 * @pm_op: runtime PM or system PM
7652 *
7653 * This function basically brings the UFS device, UniPro link and controller
7654 * to active state.
7655 *
7656 * Returns 0 for success and non-zero for failure
7657 */
7659 {
7667 /* Make sure clocks are enabled before accessing controller */
7672 /* enable the host irq as host controller would be active soon */
7681 /*
7682 * Call vendor specific resume callback. As these callbacks may access
7683 * vendor specific host controller register space call them when the
7684 * host clocks are ON.
7685 */
7694 else
7698 /*
7699 * ufshcd_host_reset_and_restore() should have already
7700 * set the link state as active
7701 */
7704 }
7710 }
7714 else
7715 /*
7716 * If BKOPs operations are urgently needed at this moment then
7717 * keep auto-bkops enabled or else disable it.
7718 */
7726 /* Schedule clock gating in case of no access to UFS device yet */
7729 /* Enable Auto-Hibernate if configured */
7734 set_old_link_state:
7736 vendor_suspend:
7738 disable_vreg:
7740 disable_irq_and_vops_clks:
7745 out:
7748 }
7750 /**
7751 * ufshcd_system_suspend - system suspend routine
7752 * @hba: per adapter instance
7753 *
7754 * Check the description of ufshcd_suspend() function for more details.
7755 *
7756 * Returns 0 for success and non-zero for failure
7757 */
7759 {
7773 /*
7774 * UFS device and/or UFS link low power states during runtime
7775 * suspend seems to be different than what is expected during
7776 * system suspend. Hence runtime resume the devic & link and
7777 * let the system suspend low power states to take effect.
7778 * TODO: If resume takes longer time, we might have optimize
7779 * it in future by not resuming everything if possible.
7780 */
7784 }
7787 out:
7794 }
7797 /**
7798 * ufshcd_system_resume - system resume routine
7799 * @hba: per adapter instance
7800 *
7801 * Returns 0 for success and non-zero for failure
7802 */
7805 {
7813 /*
7814 * Let the runtime resume take care of resuming
7815 * if runtime suspended.
7816 */
7818 else
7820 out:
7827 }
7830 /**
7831 * ufshcd_runtime_suspend - runtime suspend routine
7832 * @hba: per adapter instance
7833 *
7834 * Check the description of ufshcd_suspend() function for more details.
7835 *
7836 * Returns 0 for success and non-zero for failure
7837 */
7839 {
7848 else
7850 out:
7855 }
7858 /**
7859 * ufshcd_runtime_resume - runtime resume routine
7860 * @hba: per adapter instance
7861 *
7862 * This function basically brings the UFS device, UniPro link and controller
7863 * to active state. Following operations are done in this function:
7864 *
7865 * 1. Turn on all the controller related clocks
7866 * 2. Bring the UniPro link out of Hibernate state
7867 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
7868 * to active state.
7869 * 4. If auto-bkops is enabled on the device, disable it.
7870 *
7871 * So following would be the possible power state after this function return
7872 * successfully:
7873 * S1: UFS device in Active state with VCC rail ON
7874 * UniPro link in Active state
7875 * All the UFS/UniPro controller clocks are ON
7876 *
7877 * Returns 0 for success and non-zero for failure
7878 */
7880 {
7889 else
7891 out:
7896 }
7900 {
7902 }
7905 /**
7906 * ufshcd_shutdown - shutdown routine
7907 * @hba: per adapter instance
7908 *
7909 * This function would power off both UFS device and UFS link.
7910 *
7911 * Returns 0 always to allow force shutdown even in case of errors.
7912 */
7914 {
7927 }
7930 out:
7933 /* allow force shutdown even in case of errors */
7935 }
7938 /**
7939 * ufshcd_remove - de-allocate SCSI host and host memory space
7940 * data structure memory
7941 * @hba: per adapter instance
7942 */
7944 {
7947 /* disable interrupts */
7956 }
7959 /**
7960 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
7961 * @hba: pointer to Host Bus Adapter (HBA)
7962 */
7964 {
7966 }
7969 /**
7970 * ufshcd_set_dma_mask - Set dma mask based on the controller
7971 * addressing capability
7972 * @hba: per adapter instance
7973 *
7974 * Returns 0 for success, non-zero for failure
7975 */
7977 {
7981 }
7983 }
7985 /**
7986 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
7987 * @dev: pointer to device handle
7988 * @hba_handle: driver private handle
7989 * Returns 0 on success, non-zero value on failure
7990 */
7992 {
8002 }
8010 }
8012 /*
8013 * Do not use blk-mq at this time because blk-mq does not support
8014 * runtime pm.
8015 */
8025 out_error:
8027 }
8030 /**
8031 * ufshcd_init - Driver initialization routine
8032 * @hba: per-adapter instance
8033 * @mmio_base: base register address
8034 * @irq: Interrupt line of device
8035 * Returns 0 on success, non-zero value on failure
8036 */
8038 {
8048 }
8053 /* Set descriptor lengths to specification defaults */
8060 /* Read capabilities registers */
8063 /* Get UFS version supported by the controller */
8073 /* Get Interrupt bit mask per version */
8080 }
8082 /* Allocate memory for host memory space */
8087 }
8089 /* Configure LRB */
8102 /* Initailize wait queue for task management */
8106 /* Initialize work queues */
8110 /* Initialize UIC command mutex */
8113 /* Initialize mutex for device management commands */
8118 /* Initialize device management tag acquire wait queue */
8125 /*
8126 * In order to avoid any spurious interrupt immediately after
8127 * registering UFS controller interrupt handler, clear any pending UFS
8128 * interrupt status and disable all the UFS interrupts.
8129 */
8131 REG_INTERRUPT_STATUS);
8133 /*
8134 * Make sure that UFS interrupts are disabled and any pending interrupt
8135 * status is cleared before registering UFS interrupt handler.
8136 */
8139 /* IRQ registration */
8146 }
8152 }
8154 /* Host controller enable */
8161 }
8163 /*
8164 * Set the default power management level for runtime and system PM.
8165 * Default power saving mode is to keep UFS link in Hibern8 state
8166 * and UFS device in sleep state.
8167 */
8169 UFS_SLEEP_PWR_MODE,
8170 UIC_LINK_HIBERN8_STATE);
8172 UFS_SLEEP_PWR_MODE,
8173 UIC_LINK_HIBERN8_STATE);
8175 /* Set the default auto-hiberate idle timer value to 150 ms */
8179 }
8181 /* Hold auto suspend until async scan completes */
8184 /*
8185 * We are assuming that device wasn't put in sleep/power-down
8186 * state exclusively during the boot stage before kernel.
8187 * This assumption helps avoid doing link startup twice during
8188 * ufshcd_probe_hba().
8189 */
8197 out_remove_scsi_host:
8199 exit_gating:
8202 out_disable:
8205 out_error:
8207 }