| blob | b5966faf3e984190f7af8f60daebcc32cbccf5dc |
1 /*
2 * Universal Flash Storage Host controller driver Core
3 *
4 * This code is based on drivers/scsi/ufs/ufshcd.c
5 * Copyright (C) 2011-2013 Samsung India Software Operations
6 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7 *
8 * Authors:
9 * Santosh Yaraganavi <santosh.sy@samsung.com>
10 * Vinayak Holikatti <h.vinayak@samsung.com>
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 * See the COPYING file in the top-level directory or visit
17 * <http://www.gnu.org/licenses/gpl-2.0.html>
18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 *
24 * This program is provided "AS IS" and "WITH ALL FAULTS" and
25 * without warranty of any kind. You are solely responsible for
26 * determining the appropriateness of using and distributing
27 * the program and assume all risks associated with your exercise
28 * of rights with respect to the program, including but not limited
29 * to infringement of third party rights, the risks and costs of
30 * program errors, damage to or loss of data, programs or equipment,
31 * and unavailability or interruption of operations. Under no
32 * circumstances will the contributor of this Program be liable for
33 * any damages of any kind arising from your use or distribution of
34 * this program.
35 *
36 * The Linux Foundation chooses to take subject only to the GPLv2
37 * license terms, and distributes only under these terms.
38 */
40 #include <linux/async.h>
41 #include <linux/devfreq.h>
42 #include <linux/nls.h>
43 #include <linux/of.h>
44 #include <linux/bitfield.h>
51 #define CREATE_TRACE_POINTS
52 #include <trace/events/ufs.h>
54 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
55 UTP_TASK_REQ_COMPL |\
56 UFSHCD_ERROR_MASK)
57 /* UIC command timeout, unit: ms */
58 #define UIC_CMD_TIMEOUT 500
60 /* NOP OUT retries waiting for NOP IN response */
61 #define NOP_OUT_RETRIES 10
62 /* Timeout after 30 msecs if NOP OUT hangs without response */
65 /* Query request retries */
66 #define QUERY_REQ_RETRIES 3
67 /* Query request timeout */
70 /* Task management command timeout */
73 /* maximum number of retries for a general UIC command */
74 #define UFS_UIC_COMMAND_RETRIES 3
76 /* maximum number of link-startup retries */
77 #define DME_LINKSTARTUP_RETRIES 3
79 /* Maximum retries for Hibern8 enter */
80 #define UIC_HIBERN8_ENTER_RETRIES 3
82 /* maximum number of reset retries before giving up */
83 #define MAX_HOST_RESET_RETRIES 5
85 /* Expose the flag value from utp_upiu_query.value */
86 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
88 /* Interrupt aggregation default timeout, unit: 40us */
89 #define INT_AGGR_DEF_TO 0x02
91 /* default delay of autosuspend: 2000 ms */
92 #define RPM_AUTOSUSPEND_DELAY_MS 2000
94 #define ufshcd_toggle_vreg(_dev, _vreg, _on) \
95 ({ \
96 int _ret; \
97 if (_on) \
98 _ret = ufshcd_enable_vreg(_dev, _vreg); \
99 else \
100 _ret = ufshcd_disable_vreg(_dev, _vreg); \
101 _ret; \
102 })
104 #define ufshcd_hex_dump(prefix_str, buf, len) do { \
105 size_t __len = (len); \
106 print_hex_dump(KERN_ERR, prefix_str, \
107 __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
108 16, 4, buf, __len, false); \
109 } while (0)
113 {
131 }
139 };
141 /* UFSHCD states */
143 UFSHCD_STATE_RESET,
144 UFSHCD_STATE_ERROR,
145 UFSHCD_STATE_OPERATIONAL,
146 UFSHCD_STATE_EH_SCHEDULED,
147 };
149 /* UFSHCD error handling flags */
152 };
154 /* UFSHCD UIC layer error flags */
162 };
164 #define ufshcd_set_eh_in_progress(h) \
165 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
166 #define ufshcd_eh_in_progress(h) \
167 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
168 #define ufshcd_clear_eh_in_progress(h) \
169 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
171 #define ufshcd_set_ufs_dev_active(h) \
172 ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
173 #define ufshcd_set_ufs_dev_sleep(h) \
174 ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
175 #define ufshcd_set_ufs_dev_poweroff(h) \
176 ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
177 #define ufshcd_is_ufs_dev_active(h) \
178 ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
179 #define ufshcd_is_ufs_dev_sleep(h) \
180 ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
181 #define ufshcd_is_ufs_dev_poweroff(h) \
182 ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
191 };
195 {
197 }
201 {
203 }
208 {
215 }
217 /* if no match found, return the level 0 */
219 }
222 /* UFS cards deviations table */
224 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
226 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
228 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
230 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
232 UFS_DEVICE_QUIRK_PA_TACTIVATE),
234 UFS_DEVICE_QUIRK_PA_TACTIVATE),
236 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
238 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
240 END_FIX
241 };
265 {
267 }
270 {
275 hba);
280 }
283 }
286 {
290 }
291 }
294 {
297 }
300 {
303 }
307 {
311 }
315 {
319 }
323 {
329 }
333 {
341 /* trace UPIU W/O tracing command */
345 }
348 /* trace UPIU also */
352 /*
353 * Currently we only fully trace read(10) and write(10)
354 * commands
355 */
357 lba =
361 }
362 }
368 }
371 {
383 }
384 }
389 {
401 }
405 }
408 {
441 }
443 static
445 {
482 }
483 }
486 {
494 }
495 }
498 {
517 }
519 /**
520 * ufshcd_print_pwr_info - print power params as saved in hba
521 * power info
522 * @hba: per-adapter instance
523 */
525 {
534 };
537 __func__,
543 }
545 /*
546 * ufshcd_wait_for_register - wait for register value to change
547 * @hba - per-adapter interface
548 * @reg - mmio register offset
549 * @mask - mask to apply to read register value
550 * @val - wait condition
551 * @interval_us - polling interval in microsecs
552 * @timeout_ms - timeout in millisecs
553 * @can_sleep - perform sleep or just spin
554 *
555 * Returns -ETIMEDOUT on error, zero on success
556 */
560 {
564 /* ignore bits that we don't intend to wait on */
570 else
576 }
577 }
580 }
582 /**
583 * ufshcd_get_intr_mask - Get the interrupt bit mask
584 * @hba: Pointer to adapter instance
585 *
586 * Returns interrupt bit mask per version
587 */
589 {
604 }
607 }
609 /**
610 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
611 * @hba: Pointer to adapter instance
612 *
613 * Returns UFSHCI version supported by the controller
614 */
616 {
621 }
623 /**
624 * ufshcd_is_device_present - Check if any device connected to
625 * the host controller
626 * @hba: pointer to adapter instance
627 *
628 * Returns true if device present, false if no device detected
629 */
631 {
634 }
636 /**
637 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
638 * @lrbp: pointer to local command reference block
639 *
640 * This function is used to get the OCS field from UTRD
641 * Returns the OCS field in the UTRD
642 */
644 {
646 }
648 /**
649 * ufshcd_get_tm_free_slot - get a free slot for task management request
650 * @hba: per adapter instance
651 * @free_slot: pointer to variable with available slot value
652 *
653 * Get a free tag and lock it until ufshcd_put_tm_slot() is called.
654 * Returns 0 if free slot is not available, else return 1 with tag value
655 * in @free_slot.
656 */
658 {
673 out:
675 }
678 {
680 }
682 /**
683 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
684 * @hba: per adapter instance
685 * @pos: position of the bit to be cleared
686 */
688 {
691 else
693 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
694 }
696 /**
697 * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
698 * @hba: per adapter instance
699 * @pos: position of the bit to be cleared
700 */
702 {
705 else
707 }
709 /**
710 * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
711 * @hba: per adapter instance
712 * @tag: position of the bit to be cleared
713 */
715 {
717 }
719 /**
720 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
721 * @reg: Register value of host controller status
722 *
723 * Returns integer, 0 on Success and positive value if failed
724 */
726 {
728 }
730 /**
731 * ufshcd_get_uic_cmd_result - Get the UIC command result
732 * @hba: Pointer to adapter instance
733 *
734 * This function gets the result of UIC command completion
735 * Returns 0 on success, non zero value on error
736 */
738 {
740 MASK_UIC_COMMAND_RESULT;
741 }
743 /**
744 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
745 * @hba: Pointer to adapter instance
746 *
747 * This function gets UIC command argument3
748 * Returns 0 on success, non zero value on error
749 */
751 {
753 }
755 /**
756 * ufshcd_get_req_rsp - returns the TR response transaction type
757 * @ucd_rsp_ptr: pointer to response UPIU
758 */
761 {
763 }
765 /**
766 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
767 * @ucd_rsp_ptr: pointer to response UPIU
768 *
769 * This function gets the response status and scsi_status from response UPIU
770 * Returns the response result code.
771 */
774 {
776 }
778 /*
779 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
780 * from response UPIU
781 * @ucd_rsp_ptr: pointer to response UPIU
782 *
783 * Return the data segment length.
784 */
787 {
789 MASK_RSP_UPIU_DATA_SEG_LEN;
790 }
792 /**
793 * ufshcd_is_exception_event - Check if the device raised an exception event
794 * @ucd_rsp_ptr: pointer to response UPIU
795 *
796 * The function checks if the device raised an exception event indicated in
797 * the Device Information field of response UPIU.
798 *
799 * Returns true if exception is raised, false otherwise.
800 */
802 {
805 }
807 /**
808 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
809 * @hba: per adapter instance
810 */
813 {
815 INT_AGGR_COUNTER_AND_TIMER_RESET,
816 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
817 }
819 /**
820 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
821 * @hba: per adapter instance
822 * @cnt: Interrupt aggregation counter threshold
823 * @tmout: Interrupt aggregation timeout value
824 */
827 {
831 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
832 }
834 /**
835 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
836 * @hba: per adapter instance
837 */
839 {
841 }
843 /**
844 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
845 * When run-stop registers are set to 1, it indicates the
846 * host controller that it can process the requests
847 * @hba: per adapter instance
848 */
850 {
852 REG_UTP_TASK_REQ_LIST_RUN_STOP);
854 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
855 }
857 /**
858 * ufshcd_hba_start - Start controller initialization sequence
859 * @hba: per adapter instance
860 */
862 {
864 }
866 /**
867 * ufshcd_is_hba_active - Get controller state
868 * @hba: per adapter instance
869 *
870 * Returns false if controller is active, true otherwise
871 */
873 {
876 }
879 {
880 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
884 else
886 }
890 {
891 /*
892 * If both host and device support UniPro ver1.6 or later, PA layer
893 * parameters tuning happens during link startup itself.
894 *
895 * We can manually tune PA layer parameters if either host or device
896 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
897 * logic simple, we will only do manual tuning if local unipro version
898 * doesn't support ver1.6 or later.
899 */
902 else
904 }
907 {
934 }
953 }
959 }
960 }
963 }
967 out:
973 }
975 /**
976 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
977 * @hba: per adapter instance
978 * @scale_up: True if scaling up and false if scaling down
979 *
980 * Returns true if scaling is required, false otherwise.
981 */
984 {
1001 }
1002 }
1003 }
1006 }
1009 u64 wait_timeout_us)
1010 {
1013 u32 tm_doorbell;
1014 u32 tr_doorbell;
1016 ktime_t start;
1020 /*
1021 * Wait for all the outstanding tasks/transfer requests.
1022 * Verify by checking the doorbell registers are clear.
1023 */
1029 }
1038 }
1043 wait_timeout_us) {
1045 /*
1046 * We might have scheduled out for long time so make
1047 * sure to check if doorbells are cleared by this time
1048 * or not.
1049 */
1051 }
1060 }
1061 out:
1065 }
1067 /**
1068 * ufshcd_scale_gear - scale up/down UFS gear
1069 * @hba: per adapter instance
1070 * @scale_up: True for scaling up gear and false for scaling down
1071 *
1072 * Returns 0 for success,
1073 * Returns -EBUSY if scaling can't happen at this time
1074 * Returns non-zero for any other errors
1075 */
1077 {
1078 #define UFS_MIN_GEAR_TO_SCALE_DOWN UFS_HS_G1
1091 /* save the current power mode */
1096 /* scale down gear */
1099 }
1100 }
1102 /* check if the power mode needs to be changed or not? */
1112 }
1115 {
1118 /*
1119 * make sure that there are no outstanding requests when
1120 * clock scaling is in progress
1121 */
1128 }
1131 }
1134 {
1137 }
1139 /**
1140 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1141 * @hba: per adapter instance
1142 * @scale_up: True for scaling up and false for scalin down
1143 *
1144 * Returns 0 for success,
1145 * Returns -EBUSY if scaling can't happen at this time
1146 * Returns non-zero for any other errors
1147 */
1149 {
1152 /* let's not get into low power until clock scaling is completed */
1159 /* scale down the gear before scaling down clocks */
1164 }
1171 }
1173 /* scale up the gear after scaling up clocks */
1179 }
1180 }
1184 out:
1188 }
1191 {
1200 }
1205 }
1208 {
1217 }
1222 }
1226 {
1229 ktime_t start;
1242 }
1250 }
1258 }
1268 out:
1274 }
1279 {
1300 start_window:
1310 }
1313 }
1319 };
1322 {
1328 /* Skip devfreq if we don't have any clocks in the list */
1338 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1339 NULL);
1347 }
1352 }
1355 {
1368 }
1371 {
1378 }
1381 {
1392 }
1397 }
1400 {
1411 }
1416 }
1420 {
1424 }
1428 {
1430 u32 value;
1456 }
1460 out:
1462 }
1465 {
1473 }
1476 {
1488 }
1493 /* Exit from hibern8 */
1495 /* Prevent gating in this path */
1502 else
1504 }
1506 }
1507 unblock_reqs:
1509 }
1511 /**
1512 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1513 * Also, exit from hibern8 mode and set the link as active.
1514 * @hba: per adapter instance
1515 * @async: This indicates whether caller should ungate clocks asynchronously.
1516 */
1518 {
1530 }
1532 start:
1535 /*
1536 * Wait for the ungate work to complete if in progress.
1537 * Though the clocks may be in ON state, the link could
1538 * still be in hibner8 state if hibern8 is allowed
1539 * during clock gating.
1540 * Make sure we exit hibern8 state also in addition to
1541 * clocks being ON.
1542 */
1549 }
1557 }
1558 /*
1559 * If we are here, it means gating work is either done or
1560 * currently running. Hence, fall through to cancel gating
1561 * work and to enable clocks.
1562 */
1563 /* fallthrough */
1571 /*
1572 * fall through to check if we should wait for this
1573 * work to be done or not.
1574 */
1575 /* fallthrough */
1581 }
1585 /* Make sure state is CLKS_ON before returning */
1592 }
1594 out:
1596 }
1600 {
1606 /*
1607 * In case you are here to cancel this work the gating state
1608 * would be marked as REQ_CLKS_ON. In this case save time by
1609 * skipping the gating work and exit after changing the clock
1610 * state to CLKS_ON.
1611 */
1618 }
1628 /* put the link into hibern8 mode before turning off clocks */
1635 }
1637 }
1641 else
1642 /* If link is active, device ref_clk can't be switched off */
1645 /*
1646 * In case you are here to cancel this work the gating state
1647 * would be marked as REQ_CLKS_ON. In this case keep the state
1648 * as REQ_CLKS_ON which would anyway imply that clocks are off
1649 * and a request to turn them on is pending. By doing this way,
1650 * we keep the state machine in tact and this would ultimately
1651 * prevent from doing cancel work multiple times when there are
1652 * new requests arriving before the current cancel work is done.
1653 */
1659 }
1660 rel_lock:
1662 out:
1664 }
1666 /* host lock must be held before calling this variant */
1668 {
1686 }
1689 {
1695 }
1700 {
1704 }
1708 {
1719 }
1723 {
1727 }
1731 {
1734 u32 value;
1749 }
1752 out:
1754 }
1757 {
1764 ufshcd_clk_scaling_suspend_work);
1766 ufshcd_clk_scaling_resume_work);
1773 }
1776 {
1782 }
1785 {
1798 WQ_MEM_RECLAIM);
1817 }
1820 {
1828 }
1830 /* Must be called with host lock acquired */
1832 {
1852 }
1857 }
1858 }
1861 {
1872 }
1873 }
1874 /**
1875 * ufshcd_send_command - Send SCSI or device management commands
1876 * @hba: per adapter instance
1877 * @task_tag: Task tag of the command
1878 */
1881 {
1887 /* Make sure that doorbell is committed immediately */
1890 }
1892 /**
1893 * ufshcd_copy_sense_data - Copy sense data in case of check condition
1894 * @lrbp: pointer to local reference block
1895 */
1897 {
1907 len_to_copy);
1908 }
1909 }
1911 /**
1912 * ufshcd_copy_query_response() - Copy the Query Response and the data
1913 * descriptor
1914 * @hba: per adapter instance
1915 * @lrbp: pointer to local reference block
1916 */
1917 static
1919 {
1924 /* Get the descriptor */
1928 GENERAL_UPIU_REQUEST_SIZE;
1929 u16 resp_len;
1930 u16 buf_len;
1932 /* data segment length */
1934 MASK_QUERY_DATA_SEG_LEN;
1944 }
1945 }
1948 }
1950 /**
1951 * ufshcd_hba_capabilities - Read controller capabilities
1952 * @hba: per adapter instance
1953 */
1955 {
1958 /* nutrs and nutmrs are 0 based values */
1962 }
1964 /**
1965 * ufshcd_ready_for_uic_cmd - Check if controller is ready
1966 * to accept UIC commands
1967 * @hba: per adapter instance
1968 * Return true on success, else false
1969 */
1971 {
1974 else
1976 }
1978 /**
1979 * ufshcd_get_upmcrs - Get the power mode change request status
1980 * @hba: Pointer to adapter instance
1981 *
1982 * This function gets the UPMCRS field of HCS register
1983 * Returns value of UPMCRS field
1984 */
1986 {
1988 }
1990 /**
1991 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
1992 * @hba: per adapter instance
1993 * @uic_cmd: UIC command
1994 *
1995 * Mutex must be held.
1996 */
1999 {
2004 /* Write Args */
2009 /* Write UIC Cmd */
2011 REG_UIC_COMMAND);
2012 }
2014 /**
2015 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2016 * @hba: per adapter instance
2017 * @uic_cmd: UIC command
2018 *
2019 * Must be called with mutex held.
2020 * Returns 0 only if success.
2021 */
2022 static int
2024 {
2031 else
2039 }
2041 /**
2042 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2043 * @hba: per adapter instance
2044 * @uic_cmd: UIC command
2045 * @completion: initialize the completion only if this is set to true
2046 *
2047 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
2048 * with mutex held and host_lock locked.
2049 * Returns 0 only if success.
2050 */
2051 static int
2054 {
2059 }
2067 }
2069 /**
2070 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2071 * @hba: per adapter instance
2072 * @uic_cmd: UIC command
2073 *
2074 * Returns 0 only if success.
2075 */
2077 {
2095 }
2097 /**
2098 * ufshcd_map_sg - Map scatter-gather list to prdt
2099 * @hba: per adapter instance
2100 * @lrbp: pointer to local reference block
2101 *
2102 * Returns 0 in case of success, non-zero value in case of failure
2103 */
2105 {
2122 else
2136 }
2139 }
2142 }
2144 /**
2145 * ufshcd_enable_intr - enable interrupts
2146 * @hba: per adapter instance
2147 * @intrs: interrupt bits
2148 */
2150 {
2154 u32 rw;
2159 }
2162 }
2164 /**
2165 * ufshcd_disable_intr - disable interrupts
2166 * @hba: per adapter instance
2167 * @intrs: interrupt bits
2168 */
2170 {
2174 u32 rw;
2181 }
2184 }
2186 /**
2187 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2188 * descriptor according to request
2189 * @lrbp: pointer to local reference block
2190 * @upiu_flags: flags required in the header
2191 * @cmd_dir: requests data direction
2192 */
2195 {
2197 u32 data_direction;
2198 u32 dword_0;
2209 }
2216 /* Transfer request descriptor header fields */
2218 /* dword_1 is reserved, hence it is set to 0 */
2220 /*
2221 * assigning invalid value for command status. Controller
2222 * updates OCS on command completion, with the command
2223 * status
2224 */
2227 /* dword_3 is reserved, hence it is set to 0 */
2231 }
2233 /**
2234 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2235 * for scsi commands
2236 * @lrbp: local reference block pointer
2237 * @upiu_flags: flags
2238 */
2239 static
2241 {
2245 /* command descriptor fields */
2252 /* Total EHS length and Data segment length will be zero */
2263 }
2265 /**
2266 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2267 * for query requsts
2268 * @hba: UFS hba
2269 * @lrbp: local reference block pointer
2270 * @upiu_flags: flags
2271 */
2274 {
2279 /* Query request header */
2286 /* Data segment length only need for WRITE_DESC */
2290 else
2293 /* Copy the Query Request buffer as is */
2295 QUERY_OSF_SIZE);
2297 /* Copy the Descriptor */
2302 }
2305 {
2310 /* command descriptor fields */
2314 /* clear rest of the fields of basic header */
2319 }
2321 /**
2322 * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
2323 * for Device Management Purposes
2324 * @hba: per adapter instance
2325 * @lrbp: pointer to local reference block
2326 */
2328 {
2329 u32 upiu_flags;
2335 else
2343 else
2347 }
2349 /**
2350 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2351 * for SCSI Purposes
2352 * @hba: per adapter instance
2353 * @lrbp: pointer to local reference block
2354 */
2356 {
2357 u32 upiu_flags;
2363 else
2372 }
2375 }
2377 /**
2378 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2379 * @upiu_wlun_id: UPIU W-LUN id
2380 *
2381 * Returns SCSI W-LUN id
2382 */
2384 {
2386 }
2388 /**
2389 * ufshcd_queuecommand - main entry point for SCSI requests
2390 * @host: SCSI host pointer
2391 * @cmd: command from SCSI Midlayer
2392 *
2393 * Returns 0 for success, non-zero in case of failure
2394 */
2396 {
2411 }
2434 }
2436 /* if error handling is in progress, don't issue commands */
2441 }
2446 /* acquire the tag to make sure device cmds don't use it */
2448 /*
2449 * Dev manage command in progress, requeue the command.
2450 * Requeuing the command helps in cases where the request *may*
2451 * find different tag instead of waiting for dev manage command
2452 * completion.
2453 */
2456 }
2463 }
2484 }
2485 /* Make sure descriptors are ready before ringing the doorbell */
2488 /* issue command to the controller */
2492 out_unlock:
2494 out:
2497 }
2501 {
2511 }
2513 static int
2515 {
2520 /* clear outstanding transaction before retry */
2525 /*
2526 * wait for for h/w to clear corresponding bit in door-bell.
2527 * max. wait is 1 sec.
2528 */
2530 REG_UTP_TRANSFER_REQ_DOOR_BELL,
2534 }
2536 static int
2538 {
2541 /* Get the UPIU response */
2543 UPIU_RSP_CODE_OFFSET;
2545 }
2547 /**
2548 * ufshcd_dev_cmd_completion() - handles device management command responses
2549 * @hba: per adapter instance
2550 * @lrbp: pointer to local reference block
2551 */
2552 static int
2554 {
2567 }
2575 /* TODO: handle Reject UPIU Response */
2578 __func__);
2585 }
2588 }
2592 {
2600 /* Make sure descriptors are ready before ringing the doorbell */
2608 }
2616 /* successfully cleared the command, retry if needed */
2618 /*
2619 * in case of an error, after clearing the doorbell,
2620 * we also need to clear the outstanding_request
2621 * field in hba
2622 */
2624 }
2627 }
2629 /**
2630 * ufshcd_get_dev_cmd_tag - Get device management command tag
2631 * @hba: per-adapter instance
2632 * @tag_out: pointer to variable with available slot value
2633 *
2634 * Get a free slot and lock it until device management command
2635 * completes.
2636 *
2637 * Returns false if free slot is unavailable for locking, else
2638 * return true with tag value in @tag.
2639 */
2641 {
2658 out:
2660 }
2663 {
2665 }
2667 /**
2668 * ufshcd_exec_dev_cmd - API for sending device management requests
2669 * @hba: UFS hba
2670 * @cmd_type: specifies the type (NOP, Query...)
2671 * @timeout: time in seconds
2672 *
2673 * NOTE: Since there is only one available tag for device management commands,
2674 * it is expected you hold the hba->dev_cmd.lock mutex.
2675 */
2678 {
2687 /*
2688 * Get free slot, sleep if slots are unavailable.
2689 * Even though we use wait_event() which sleeps indefinitely,
2690 * the maximum wait time is bounded by SCSI request timeout.
2691 */
2704 /* Make sure descriptors are ready before ringing the doorbell */
2716 out_put_tag:
2721 }
2723 /**
2724 * ufshcd_init_query() - init the query response and request parameters
2725 * @hba: per-adapter instance
2726 * @request: address of the request pointer to be initialized
2727 * @response: address of the response pointer to be initialized
2728 * @opcode: operation to perform
2729 * @idn: flag idn to access
2730 * @index: LU number to access
2731 * @selector: query/flag/descriptor further identification
2732 */
2736 {
2745 }
2749 {
2759 else
2761 }
2768 }
2770 /**
2771 * ufshcd_query_flag() - API function for sending flag query requests
2772 * @hba: per-adapter instance
2773 * @opcode: flag query to perform
2774 * @idn: flag idn to access
2775 * @flag_res: the flag value after the query request completes
2776 *
2777 * Returns 0 for success, non-zero in case of failure
2778 */
2781 {
2792 selector);
2803 /* No dummy reads */
2805 __func__);
2808 }
2816 }
2825 }
2831 out_unlock:
2835 }
2837 /**
2838 * ufshcd_query_attr - API function for sending attribute requests
2839 * @hba: per-adapter instance
2840 * @opcode: attribute opcode
2841 * @idn: attribute idn to access
2842 * @index: index field
2843 * @selector: selector field
2844 * @attr_val: the attribute value after the query request completes
2845 *
2846 * Returns 0 for success, non-zero in case of failure
2847 */
2850 {
2863 }
2867 selector);
2882 }
2890 }
2894 out_unlock:
2896 out:
2899 }
2901 /**
2902 * ufshcd_query_attr_retry() - API function for sending query
2903 * attribute with retries
2904 * @hba: per-adapter instance
2905 * @opcode: attribute opcode
2906 * @idn: attribute idn to access
2907 * @index: index field
2908 * @selector: selector field
2909 * @attr_val: the attribute value after the query request
2910 * completes
2911 *
2912 * Returns 0 for success, non-zero in case of failure
2913 */
2917 {
2919 u32 retries;
2927 else
2929 }
2936 }
2941 {
2954 }
2961 }
2965 selector);
2982 }
2990 }
2994 out_unlock:
2997 out:
3000 }
3002 /**
3003 * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3004 * @hba: per-adapter instance
3005 * @opcode: attribute opcode
3006 * @idn: attribute idn to access
3007 * @index: index field
3008 * @selector: selector field
3009 * @desc_buf: the buffer that contains the descriptor
3010 * @buf_len: length parameter passed to the device
3011 *
3012 * Returns 0 for success, non-zero in case of failure.
3013 * The buf_len parameter will contain, on return, the length parameter
3014 * received on the response.
3015 */
3019 u8 selector,
3021 {
3030 }
3033 }
3035 /**
3036 * ufshcd_read_desc_length - read the specified descriptor length from header
3037 * @hba: Pointer to adapter instance
3038 * @desc_id: descriptor idn value
3039 * @desc_index: descriptor index
3040 * @desc_length: pointer to variable to read the length of descriptor
3041 *
3042 * Return 0 in case of success, non-zero otherwise
3043 */
3048 {
3067 desc_id);
3069 }
3074 }
3076 /**
3077 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3078 * @hba: Pointer to adapter instance
3079 * @desc_id: descriptor idn value
3080 * @desc_len: mapped desc length (out)
3081 *
3082 * Return 0 in case of success, non-zero otherwise
3083 */
3086 {
3119 }
3121 }
3124 /**
3125 * ufshcd_read_desc_param - read the specified descriptor parameter
3126 * @hba: Pointer to adapter instance
3127 * @desc_id: descriptor idn value
3128 * @desc_index: descriptor index
3129 * @param_offset: offset of the parameter to read
3130 * @param_read_buf: pointer to buffer where parameter would be read
3131 * @param_size: sizeof(param_read_buf)
3132 *
3133 * Return 0 in case of success, non-zero otherwise
3134 */
3138 u8 param_offset,
3140 u8 param_size)
3141 {
3147 /* Safety check */
3151 /* Get the max length of descriptor from structure filled up at probe
3152 * time.
3153 */
3156 /* Sanity checks */
3159 __func__);
3161 }
3163 /* Check whether we need temp memory */
3171 }
3173 /* Request for full descriptor */
3179 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3182 }
3184 /* Sanity check */
3190 }
3192 /* Check wherher we will not copy more data, than available */
3198 out:
3202 }
3208 u32 size)
3209 {
3211 }
3215 u32 size)
3216 {
3218 }
3221 {
3223 }
3225 /**
3226 * struct uc_string_id - unicode string
3227 *
3228 * @len: size of this descriptor inclusive
3229 * @type: descriptor type
3230 * @uc: unicode string character
3231 */
3233 u8 len;
3234 u8 type;
3238 /* replace non-printable or non-ASCII characters with spaces */
3240 {
3242 }
3244 /**
3245 * ufshcd_read_string_desc - read string descriptor
3246 * @hba: pointer to adapter instance
3247 * @desc_index: descriptor index
3248 * @buf: pointer to buffer where descriptor would be read,
3249 * the caller should free the memory.
3250 * @ascii: if true convert from unicode to ascii characters
3251 * null terminated string.
3252 *
3253 * Return:
3254 * * string size on success.
3255 * * -ENOMEM: on allocation failure
3256 * * -EINVAL: on a wrong parameter
3257 */
3260 {
3274 QUERY_DESC_MAX_SIZE);
3280 }
3287 }
3290 ssize_t ascii_len;
3292 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3298 }
3300 /*
3301 * the descriptor contains string in UTF16 format
3302 * we need to convert to utf-8 so it can be displayed
3303 */
3308 /* replace non-printable or non-ASCII characters with spaces */
3319 }
3321 }
3322 out:
3326 }
3328 /**
3329 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3330 * @hba: Pointer to adapter instance
3331 * @lun: lun id
3332 * @param_offset: offset of the parameter to read
3333 * @param_read_buf: pointer to buffer where parameter would be read
3334 * @param_size: sizeof(param_read_buf)
3335 *
3336 * Return 0 in case of success, non-zero otherwise
3337 */
3342 u32 param_size)
3343 {
3344 /*
3345 * Unit descriptors are only available for general purpose LUs (LUN id
3346 * from 0 to 7) and RPMB Well known LU.
3347 */
3353 }
3355 /**
3356 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3357 * @hba: per adapter instance
3358 *
3359 * 1. Allocate DMA memory for Command Descriptor array
3360 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3361 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3362 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3363 * (UTMRDL)
3364 * 4. Allocate memory for local reference block(lrb).
3365 *
3366 * Returns 0 for success, non-zero in case of failure
3367 */
3369 {
3372 /* Allocate memory for UTP command descriptors */
3375 ucdl_size,
3377 GFP_KERNEL);
3379 /*
3380 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3381 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3382 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3383 * be aligned to 128 bytes as well
3384 */
3390 }
3392 /*
3393 * Allocate memory for UTP Transfer descriptors
3394 * UFSHCI requires 1024 byte alignment of UTRD
3395 */
3398 utrdl_size,
3400 GFP_KERNEL);
3406 }
3408 /*
3409 * Allocate memory for UTP Task Management descriptors
3410 * UFSHCI requires 1024 byte alignment of UTMRD
3411 */
3414 utmrdl_size,
3416 GFP_KERNEL);
3422 }
3424 /* Allocate memory for local reference block */
3427 GFP_KERNEL);
3431 }
3433 out:
3435 }
3437 /**
3438 * ufshcd_host_memory_configure - configure local reference block with
3439 * memory offsets
3440 * @hba: per adapter instance
3441 *
3442 * Configure Host memory space
3443 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3444 * address.
3445 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3446 * and PRDT offset.
3447 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3448 * into local reference block.
3449 */
3451 {
3454 dma_addr_t cmd_desc_dma_addr;
3455 dma_addr_t cmd_desc_element_addr;
3456 u16 response_offset;
3457 u16 prdt_offset;
3464 response_offset =
3466 prdt_offset =
3473 /* Configure UTRD with command descriptor base address */
3474 cmd_desc_element_addr =
3481 /* Response upiu and prdt offset should be in double words */
3496 }
3507 response_offset;
3511 prdt_offset;
3512 }
3513 }
3515 /**
3516 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3517 * @hba: per adapter instance
3518 *
3519 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3520 * in order to initialize the Unipro link startup procedure.
3521 * Once the Unipro links are up, the device connected to the controller
3522 * is detected.
3523 *
3524 * Returns 0 on success, non-zero value on failure
3525 */
3527 {
3538 }
3539 /**
3540 * ufshcd_dme_reset - UIC command for DME_RESET
3541 * @hba: per adapter instance
3542 *
3543 * DME_RESET command is issued in order to reset UniPro stack.
3544 * This function now deal with cold reset.
3545 *
3546 * Returns 0 on success, non-zero value on failure
3547 */
3549 {
3561 }
3563 /**
3564 * ufshcd_dme_enable - UIC command for DME_ENABLE
3565 * @hba: per adapter instance
3566 *
3567 * DME_ENABLE command is issued in order to enable UniPro stack.
3568 *
3569 * Returns 0 on success, non-zero value on failure
3570 */
3572 {
3584 }
3587 {
3588 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3594 /*
3595 * last_dme_cmd_tstamp will be 0 only for 1st call to
3596 * this function
3597 */
3607 min_sleep_time_us =
3609 else
3611 }
3613 /* allow sleep for extra 50us if needed */
3615 }
3617 /**
3618 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3619 * @hba: per adapter instance
3620 * @attr_sel: uic command argument1
3621 * @attr_set: attribute set type as uic command argument2
3622 * @mib_val: setting value as uic command argument3
3623 * @peer: indicate whether peer or local
3624 *
3625 * Returns 0 on success, non-zero value on failure
3626 */
3629 {
3633 "dme-peer-set"
3634 };
3646 /* for peer attributes we retry upon failure */
3659 }
3662 /**
3663 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3664 * @hba: per adapter instance
3665 * @attr_sel: uic command argument1
3666 * @mib_val: the value of the attribute as returned by the UIC command
3667 * @peer: indicate whether peer or local
3668 *
3669 * Returns 0 on success, non-zero value on failure
3670 */
3673 {
3677 "dme-peer-get"
3678 };
3700 }
3705 }
3706 }
3713 /* for peer attributes we retry upon failure */
3731 out:
3733 }
3736 /**
3737 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3738 * state) and waits for it to take effect.
3739 *
3740 * @hba: per adapter instance
3741 * @cmd: UIC command to execute
3742 *
3743 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3744 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3745 * and device UniPro link and hence it's final completion would be indicated by
3746 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3747 * addition to normal UIC command completion Status (UCCS). This function only
3748 * returns after the relevant status bits indicate the completion.
3749 *
3750 * Returns 0 on success, non-zero value on failure
3751 */
3753 {
3756 u8 status;
3768 /*
3769 * Make sure UIC command completion interrupt is disabled before
3770 * issuing UIC command.
3771 */
3774 }
3782 }
3791 }
3799 }
3800 out:
3805 }
3816 }
3818 /**
3819 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
3820 * using DME_SET primitives.
3821 * @hba: per adapter instance
3822 * @mode: powr mode value
3823 *
3824 * Returns 0 on success, non-zero value on failure
3825 */
3827 {
3838 }
3839 }
3848 out:
3850 }
3853 {
3862 /* Reset the attached device */
3878 }
3881 {
3899 /*
3900 * If link recovery fails then return error code returned from
3901 * ufshcd_link_recovery().
3902 * If link recovery succeeds then return -EAGAIN to attempt
3903 * hibern8 enter retry again.
3904 */
3911 }
3914 POST_CHANGE);
3917 }
3920 {
3927 }
3928 out:
3930 }
3933 {
3951 POST_CHANGE);
3954 }
3957 }
3960 {
3969 }
3971 /**
3972 * ufshcd_init_pwr_info - setting the POR (power on reset)
3973 * values in hba power info
3974 * @hba: per-adapter instance
3975 */
3977 {
3985 }
3987 /**
3988 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
3989 * @hba: per-adapter instance
3990 */
3992 {
4002 /* Get the connected lane count */
4010 __func__,
4014 }
4016 /*
4017 * First, get the maximum gears of HS speed.
4018 * If a zero value, it means there is no HSGEAR capability.
4019 * Then, get the maximum gears of PWM speed.
4020 */
4029 }
4031 }
4042 }
4044 }
4048 }
4052 {
4055 /* if already configured to the requested pwr_mode */
4065 }
4067 /*
4068 * Configure attributes for power mode change with below.
4069 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4070 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4071 * - PA_HSSERIES
4072 */
4079 else
4088 else
4106 pwr_mode);
4110 }
4113 }
4115 /**
4116 * ufshcd_config_pwr_mode - configure a new power mode
4117 * @hba: per-adapter instance
4118 * @desired_pwr_mode: desired power configuration
4119 */
4122 {
4137 }
4140 /**
4141 * ufshcd_complete_dev_init() - checks device readiness
4142 * @hba: per-adapter instance
4143 *
4144 * Set fDeviceInit flag and poll until device toggles it.
4145 */
4147 {
4159 }
4161 /* poll for max. 1000 iterations for fDeviceInit flag to clear */
4173 __func__);
4175 out:
4177 }
4179 /**
4180 * ufshcd_make_hba_operational - Make UFS controller operational
4181 * @hba: per adapter instance
4182 *
4183 * To bring UFS host controller to operational state,
4184 * 1. Enable required interrupts
4185 * 2. Configure interrupt aggregation
4186 * 3. Program UTRL and UTMRL base address
4187 * 4. Configure run-stop-registers
4188 *
4189 * Returns 0 on success, non-zero value on failure
4190 */
4192 {
4194 u32 reg;
4196 /* Enable required interrupts */
4199 /* Configure interrupt aggregation */
4202 else
4205 /* Configure UTRL and UTMRL base address registers */
4207 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4209 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4211 REG_UTP_TASK_REQ_LIST_BASE_L);
4213 REG_UTP_TASK_REQ_LIST_BASE_H);
4215 /*
4216 * Make sure base address and interrupt setup are updated before
4217 * enabling the run/stop registers below.
4218 */
4221 /*
4222 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4223 */
4232 }
4234 out:
4236 }
4238 /**
4239 * ufshcd_hba_stop - Send controller to reset state
4240 * @hba: per adapter instance
4241 * @can_sleep: perform sleep or just spin
4242 */
4244 {
4253 }
4255 /**
4256 * ufshcd_hba_execute_hce - initialize the controller
4257 * @hba: per adapter instance
4258 *
4259 * The controller resets itself and controller firmware initialization
4260 * sequence kicks off. When controller is ready it will set
4261 * the Host Controller Enable bit to 1.
4262 *
4263 * Returns 0 on success, non-zero value on failure
4264 */
4266 {
4270 /* change controller state to "reset state" */
4273 /* UniPro link is disabled at this point */
4278 /* start controller initialization sequence */
4281 /*
4282 * To initialize a UFS host controller HCE bit must be set to 1.
4283 * During initialization the HCE bit value changes from 1->0->1.
4284 * When the host controller completes initialization sequence
4285 * it sets the value of HCE bit to 1. The same HCE bit is read back
4286 * to check if the controller has completed initialization sequence.
4287 * So without this delay the value HCE = 1, set in the previous
4288 * instruction might be read back.
4289 * This delay can be changed based on the controller.
4290 */
4293 /* wait for the host controller to complete initialization */
4297 retry--;
4302 }
4304 }
4306 /* enable UIC related interrupts */
4312 }
4315 {
4322 /* enable UIC related interrupts */
4332 }
4335 }
4338 }
4340 {
4346 else
4355 else
4364 }
4365 }
4368 }
4371 {
4373 }
4376 u32 reg)
4377 {
4381 }
4383 /**
4384 * ufshcd_link_startup - Initialize unipro link startup
4385 * @hba: per adapter instance
4386 *
4387 * Returns 0 for success, non-zero in case of failure
4388 */
4390 {
4395 /*
4396 * If UFS device isn't active then we will have to issue link startup
4397 * 2 times to make sure the device state move to active.
4398 */
4402 link_startup:
4408 /* check if device is detected by inter-connect layer */
4415 }
4417 /*
4418 * DME link lost indication is only received when link is up,
4419 * but we can't be sure if the link is up until link startup
4420 * succeeds. So reset the local Uni-Pro and try again.
4421 */
4426 }
4430 /* failed to get the link up... retire */
4434 }
4440 }
4442 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4450 }
4452 /* Include any host controller configuration via UIC commands */
4458 out:
4464 }
4466 }
4468 /**
4469 * ufshcd_verify_dev_init() - Verify device initialization
4470 * @hba: per-adapter instance
4471 *
4472 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4473 * device Transport Protocol (UTP) layer is ready after a reset.
4474 * If the UTP layer at the device side is not initialized, it may
4475 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4476 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4477 */
4479 {
4487 NOP_OUT_TIMEOUT);
4493 }
4500 }
4502 /**
4503 * ufshcd_set_queue_depth - set lun queue depth
4504 * @sdev: pointer to SCSI device
4505 *
4506 * Read bLUQueueDepth value and activate scsi tagged command
4507 * queueing. For WLUN, queue depth is set to 1. For best-effort
4508 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4509 * value that host can queue.
4510 */
4512 {
4514 u8 lun_qdepth;
4522 UNIT_DESC_PARAM_LU_Q_DEPTH,
4526 /* Some WLUN doesn't support unit descriptor */
4530 /* eventually, we can figure out the real queue depth */
4532 else
4538 }
4540 /*
4541 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4542 * @hba: per-adapter instance
4543 * @lun: UFS device lun id
4544 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4545 *
4546 * Returns 0 in case of success and b_lu_write_protect status would be returned
4547 * @b_lu_write_protect parameter.
4548 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4549 * Returns -EINVAL in case of invalid parameters passed to this function.
4550 */
4552 u8 lun,
4554 {
4559 /*
4560 * According to UFS device spec, RPMB LU can't be write
4561 * protected so skip reading bLUWriteProtect parameter for
4562 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4563 */
4566 else
4568 lun,
4569 UNIT_DESC_PARAM_LU_WR_PROTECT,
4570 b_lu_write_protect,
4573 }
4575 /**
4576 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4577 * status
4578 * @hba: per-adapter instance
4579 * @sdev: pointer to SCSI device
4580 *
4581 */
4584 {
4587 u8 b_lu_write_protect;
4593 }
4594 }
4596 /**
4597 * ufshcd_slave_alloc - handle initial SCSI device configurations
4598 * @sdev: pointer to SCSI device
4599 *
4600 * Returns success
4601 */
4603 {
4608 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4611 /* allow SCSI layer to restart the device in case of errors */
4614 /* REPORT SUPPORTED OPERATION CODES is not supported */
4617 /* WRITE_SAME command is not supported */
4625 }
4627 /**
4628 * ufshcd_change_queue_depth - change queue depth
4629 * @sdev: pointer to SCSI device
4630 * @depth: required depth to set
4631 *
4632 * Change queue depth and make sure the max. limits are not crossed.
4633 */
4635 {
4641 }
4643 /**
4644 * ufshcd_slave_configure - adjust SCSI device configurations
4645 * @sdev: pointer to SCSI device
4646 */
4648 {
4658 }
4660 /**
4661 * ufshcd_slave_destroy - remove SCSI device configurations
4662 * @sdev: pointer to SCSI device
4663 */
4665 {
4669 /* Drop the reference as it won't be needed anymore */
4676 }
4677 }
4679 /**
4680 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
4681 * @lrbp: pointer to local reference block of completed command
4682 * @scsi_status: SCSI command status
4683 *
4684 * Returns value base on SCSI command status
4685 */
4688 {
4694 /* fallthrough */
4698 scsi_status;
4712 }
4714 /**
4715 * ufshcd_transfer_rsp_status - Get overall status of the response
4716 * @hba: per adapter instance
4717 * @lrbp: pointer to local reference block of completed command
4718 *
4719 * Returns result of the command to notify SCSI midlayer
4720 */
4723 {
4728 /* overall command status of utrd */
4737 /*
4738 * get the response UPIU result to extract
4739 * the SCSI command status
4740 */
4743 /*
4744 * get the result based on SCSI status response
4745 * to notify the SCSI midlayer of the command status
4746 */
4750 /*
4751 * Currently we are only supporting BKOPs exception
4752 * events hence we can ignore BKOPs exception event
4753 * during power management callbacks. BKOPs exception
4754 * event is not expected to be raised in runtime suspend
4755 * callback as it allows the urgent bkops.
4756 * During system suspend, we are anyway forcefully
4757 * disabling the bkops and if urgent bkops is needed
4758 * it will be enabled on system resume. Long term
4759 * solution could be to abort the system suspend if
4760 * UFS device needs urgent BKOPs.
4761 */
4767 /* TODO: handle Reject UPIU Response */
4775 result);
4778 }
4805 }
4807 /**
4808 * ufshcd_uic_cmd_compl - handle completion of uic command
4809 * @hba: per adapter instance
4810 * @intr_status: interrupt status generated by the controller
4811 *
4812 * Returns
4813 * IRQ_HANDLED - If interrupt is valid
4814 * IRQ_NONE - If invalid interrupt
4815 */
4817 {
4827 }
4832 }
4834 }
4836 /**
4837 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
4838 * @hba: per adapter instance
4839 * @completed_reqs: requests to complete
4840 */
4843 {
4857 /* Mark completed command as NULL in LRB */
4860 /* Do not touch lrbp after scsi done */
4869 }
4870 }
4875 }
4877 /* clear corresponding bits of completed commands */
4882 /* we might have free'd some tags above */
4884 }
4886 /**
4887 * ufshcd_transfer_req_compl - handle SCSI and query command completion
4888 * @hba: per adapter instance
4889 *
4890 * Returns
4891 * IRQ_HANDLED - If interrupt is valid
4892 * IRQ_NONE - If invalid interrupt
4893 */
4895 {
4897 u32 tr_doorbell;
4899 /* Resetting interrupt aggregation counters first and reading the
4900 * DOOR_BELL afterward allows us to handle all the completed requests.
4901 * In order to prevent other interrupts starvation the DB is read once
4902 * after reset. The down side of this solution is the possibility of
4903 * false interrupt if device completes another request after resetting
4904 * aggregation and before reading the DB.
4905 */
4918 }
4919 }
4921 /**
4922 * ufshcd_disable_ee - disable exception event
4923 * @hba: per-adapter instance
4924 * @mask: exception event to disable
4925 *
4926 * Disables exception event in the device so that the EVENT_ALERT
4927 * bit is not set.
4928 *
4929 * Returns zero on success, non-zero error value on failure.
4930 */
4932 {
4934 u32 val;
4945 out:
4947 }
4949 /**
4950 * ufshcd_enable_ee - enable exception event
4951 * @hba: per-adapter instance
4952 * @mask: exception event to enable
4953 *
4954 * Enable corresponding exception event in the device to allow
4955 * device to alert host in critical scenarios.
4956 *
4957 * Returns zero on success, non-zero error value on failure.
4958 */
4960 {
4962 u32 val;
4973 out:
4975 }
4977 /**
4978 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
4979 * @hba: per-adapter instance
4980 *
4981 * Allow device to manage background operations on its own. Enabling
4982 * this might lead to inconsistent latencies during normal data transfers
4983 * as the device is allowed to manage its own way of handling background
4984 * operations.
4985 *
4986 * Returns zero on success, non-zero on failure.
4987 */
4989 {
5001 }
5006 /* No need of URGENT_BKOPS exception from the device */
5011 out:
5013 }
5015 /**
5016 * ufshcd_disable_auto_bkops - block device in doing background operations
5017 * @hba: per-adapter instance
5018 *
5019 * Disabling background operations improves command response latency but
5020 * has drawback of device moving into critical state where the device is
5021 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5022 * host is idle so that BKOPS are managed effectively without any negative
5023 * impacts.
5024 *
5025 * Returns zero on success, non-zero on failure.
5026 */
5028 {
5034 /*
5035 * If host assisted BKOPs is to be enabled, make sure
5036 * urgent bkops exception is allowed.
5037 */
5043 }
5052 }
5056 out:
5058 }
5060 /**
5061 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5062 * @hba: per adapter instance
5063 *
5064 * After a device reset the device may toggle the BKOPS_EN flag
5065 * to default value. The s/w tracking variables should be updated
5066 * as well. This function would change the auto-bkops state based on
5067 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
5068 */
5070 {
5079 }
5080 }
5083 {
5086 }
5088 /**
5089 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5090 * @hba: per-adapter instance
5091 * @status: bkops_status value
5092 *
5093 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5094 * flag in the device to permit background operations if the device
5095 * bkops_status is greater than or equal to "status" argument passed to
5096 * this function, disable otherwise.
5097 *
5098 * Returns 0 for success, non-zero in case of failure.
5099 *
5100 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5101 * to know whether auto bkops is enabled or disabled after this function
5102 * returns control to it.
5103 */
5106 {
5120 }
5124 else
5126 out:
5128 }
5130 /**
5131 * ufshcd_urgent_bkops - handle urgent bkops exception event
5132 * @hba: per-adapter instance
5133 *
5134 * Enable fBackgroundOpsEn flag in the device to permit background
5135 * operations.
5136 *
5137 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5138 * and negative error value for any other failure.
5139 */
5141 {
5143 }
5146 {
5149 }
5152 {
5164 }
5166 /*
5167 * We are seeing that some devices are raising the urgent bkops
5168 * exception events even when BKOPS status doesn't indicate performace
5169 * impacted or critical. Handle these device by determining their urgent
5170 * bkops status at runtime.
5171 */
5175 /* update the current status as the urgent bkops level */
5178 }
5180 enable_auto_bkops:
5182 out:
5186 }
5188 /**
5189 * ufshcd_exception_event_handler - handle exceptions raised by device
5190 * @work: pointer to work data
5191 *
5192 * Read bExceptionEventStatus attribute from the device and handle the
5193 * exception event accordingly.
5194 */
5196 {
5209 }
5216 out:
5220 }
5222 /* Complete requests that have door-bell cleared */
5224 {
5227 }
5229 /**
5230 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5231 * to recover from the DL NAC errors or not.
5232 * @hba: per-adapter instance
5233 *
5234 * Returns true if error handling is required, false otherwise
5235 */
5237 {
5242 /*
5243 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5244 * device fatal error and/or DL NAC & REPLAY timeout errors.
5245 */
5257 /*
5258 * wait for 50ms to see if we can get any other errors or not.
5259 */
5264 /*
5265 * now check if we have got any other severe errors other than
5266 * DL NAC error?
5267 */
5273 /*
5274 * As DL NAC is the only error received so far, send out NOP
5275 * command to confirm if link is still active or not.
5276 * - If we don't get any response then do error recovery.
5277 * - If we get response then clear the DL NAC error bit.
5278 */
5287 /* Link seems to be alive hence ignore the DL NAC errors */
5290 /* clear NAC error */
5295 }
5296 }
5297 out:
5300 }
5302 /**
5303 * ufshcd_err_handler - handle UFS errors that require s/w attention
5304 * @work: pointer to work structure
5305 */
5307 {
5328 /* Complete requests that have door-bell cleared by h/w */
5335 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
5340 }
5345 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5346 UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
5349 /*
5350 * if host reset is required then skip clearing the pending
5351 * transfers forcefully because they will automatically get
5352 * cleared after link startup.
5353 */
5357 /* release lock as clear command might sleep */
5359 /* Clear pending transfer requests */
5364 }
5365 }
5367 /* Clear pending task management requests */
5372 }
5373 }
5375 lock_skip_pending_xfer_clear:
5378 /* Complete the requests that are cleared by s/w */
5384 skip_pending_xfer_clear:
5385 /* Fatal errors need reset */
5389 /*
5390 * ufshcd_reset_and_restore() does the link reinitialization
5391 * which will need atleast one empty doorbell slot to send the
5392 * device management commands (NOP and query commands).
5393 * If there is no slot empty at this moment then free up last
5394 * slot forcefully.
5395 */
5405 __func__);
5407 }
5408 /*
5409 * Inform scsi mid-layer that we did reset and allow to handle
5410 * Unit Attention properly.
5411 */
5415 }
5417 skip_err_handling:
5423 }
5427 out:
5432 }
5434 /**
5435 * ufshcd_update_uic_error - check and set fatal UIC error flags.
5436 * @hba: per-adapter instance
5437 *
5438 * Returns
5439 * IRQ_HANDLED - If interrupt is valid
5440 * IRQ_NONE - If invalid interrupt
5441 */
5443 {
5444 u32 reg;
5447 /* PHY layer lane error */
5449 /* Ignore LINERESET indication, as this is not an error */
5452 /*
5453 * To know whether this error is fatal or not, DB timeout
5454 * must be checked but this error is handled separately.
5455 */
5459 }
5461 /* PA_INIT_ERROR is fatal and needs UIC reset */
5470 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5473 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5476 }
5478 }
5480 /* UIC NL/TL/DME errors needs software retry */
5487 }
5495 }
5503 }
5508 }
5511 u32 intr_mask)
5512 {
5525 }
5527 /**
5528 * ufshcd_check_errors - Check for errors that need s/w attention
5529 * @hba: per-adapter instance
5530 *
5531 * Returns
5532 * IRQ_HANDLED - If interrupt is valid
5533 * IRQ_NONE - If invalid interrupt
5534 */
5536 {
5543 }
5550 }
5561 }
5564 /*
5565 * update the transfer error masks to sticky bits, let's do this
5566 * irrespective of current ufshcd_state.
5567 */
5571 /* handle fatal errors only when link is functional */
5573 /* block commands from scsi mid-layer */
5578 /* dump controller state before resetting */
5581 SYSTEM_BUS_FATAL_ERROR);
5591 pr_prdt);
5592 }
5594 }
5596 }
5597 /*
5598 * if (!queue_eh_work) -
5599 * Other errors are either non-fatal where host recovers
5600 * itself without s/w intervention or errors that will be
5601 * handled by the SCSI core layer.
5602 */
5604 }
5606 /**
5607 * ufshcd_tmc_handler - handle task management function completion
5608 * @hba: per adapter instance
5609 *
5610 * Returns
5611 * IRQ_HANDLED - If interrupt is valid
5612 * IRQ_NONE - If invalid interrupt
5613 */
5615 {
5616 u32 tm_doorbell;
5625 }
5626 }
5628 /**
5629 * ufshcd_sl_intr - Interrupt service routine
5630 * @hba: per adapter instance
5631 * @intr_status: contains interrupts generated by the controller
5632 *
5633 * Returns
5634 * IRQ_HANDLED - If interrupt is valid
5635 * IRQ_NONE - If invalid interrupt
5636 */
5638 {
5659 }
5661 /**
5662 * ufshcd_intr - Main interrupt service routine
5663 * @irq: irq number
5664 * @__hba: pointer to adapter instance
5665 *
5666 * Returns
5667 * IRQ_HANDLED - If interrupt is valid
5668 * IRQ_NONE - If invalid interrupt
5669 */
5671 {
5680 /*
5681 * There could be max of hba->nutrs reqs in flight and in worst case
5682 * if the reqs get finished 1 by 1 after the interrupt status is
5683 * read, make sure we handle them by checking the interrupt status
5684 * again in a loop until we process all of the reqs before returning.
5685 */
5687 enabled_intr_status =
5701 }
5705 }
5708 {
5720 /* poll for max. 1 sec to clear door bell register by h/w */
5722 REG_UTP_TASK_REQ_DOOR_BELL,
5724 out:
5726 }
5730 {
5735 /*
5736 * Get free slot, sleep if slots are unavailable.
5737 * Even though we use wait_event() which sleeps indefinitely,
5738 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
5739 */
5751 /* send command to the controller */
5754 /* Make sure descriptors are ready before ringing the task doorbell */
5758 /* Make sure that doorbell is committed immediately */
5765 /* wait until the task management command is completed */
5782 }
5794 }
5796 /**
5797 * ufshcd_issue_tm_cmd - issues task management commands to controller
5798 * @hba: per adapter instance
5799 * @lun_id: LUN ID to which TM command is sent
5800 * @task_id: task ID to which the TM command is applicable
5801 * @tm_function: task management function opcode
5802 * @tm_response: task management service response return value
5803 *
5804 * Returns non-zero value on error, zero on success.
5805 */
5808 {
5812 /* Configure task request descriptor */
5816 /* Configure task request UPIU */
5821 /*
5822 * The host shall provide the same value for LUN field in the basic
5823 * header and for Input Parameter.
5824 */
5838 MASK_TM_SERVICE_RESP;
5840 }
5842 /**
5843 * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
5844 * @hba: per-adapter instance
5845 * @req_upiu: upiu request
5846 * @rsp_upiu: upiu reply
5847 * @desc_buff: pointer to descriptor buffer, NULL if NA
5848 * @buff_len: descriptor size, 0 if NA
5849 * @cmd_type: specifies the type (NOP, Query...)
5850 * @desc_op: descriptor operation
5851 *
5852 * Those type of requests uses UTP Transfer Request Descriptor - utrd.
5853 * Therefore, it "rides" the device management infrastructure: uses its tag and
5854 * tasks work queues.
5855 *
5856 * Since there is only one available tag for device management commands,
5857 * the caller is expected to hold the hba->dev_cmd.lock mutex.
5858 */
5865 {
5871 u32 upiu_flags;
5897 }
5899 /* update the task tag in the request upiu */
5904 /* just copy the upiu request as it is */
5907 /* The Data Segment Area is optional depending upon the query
5908 * function value. for WRITE DESCRIPTOR, the data segment
5909 * follows right after the tsf.
5910 */
5913 }
5919 /* Make sure descriptors are ready before ringing the doorbell */
5925 /*
5926 * ignore the returning value here - ufshcd_check_query_response is
5927 * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
5928 * read the response directly ignoring all errors.
5929 */
5932 /* just copy the upiu response as it is */
5937 MASK_QUERY_DATA_SEG_LEN;
5948 }
5949 }
5955 }
5957 /**
5958 * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
5959 * @hba: per-adapter instance
5960 * @req_upiu: upiu request
5961 * @rsp_upiu: upiu reply - only 8 DW as we do not support scsi commands
5962 * @msgcode: message code, one of UPIU Transaction Codes Initiator to Target
5963 * @desc_buff: pointer to descriptor buffer, NULL if NA
5964 * @buff_len: descriptor size, 0 if NA
5965 * @desc_op: descriptor operation
5966 *
5967 * Supports UTP Transfer requests (nop and query), and UTP Task
5968 * Management requests.
5969 * It is up to the caller to fill the upiu conent properly, as it will
5970 * be copied without any further input validations.
5971 */
5978 {
5988 /* fall through */
6012 ocs_value);
6014 }
6023 }
6026 }
6028 /**
6029 * ufshcd_eh_device_reset_handler - device reset handler registered to
6030 * scsi layer.
6031 * @cmd: SCSI command pointer
6032 *
6033 * Returns SUCCESS/FAILED
6034 */
6036 {
6040 u32 pos;
6056 }
6058 /* clear the commands that were pending for corresponding LUN */
6064 }
6065 }
6070 out:
6078 }
6080 }
6083 {
6090 }
6091 }
6093 /**
6094 * ufshcd_abort - abort a specific command
6095 * @cmd: SCSI command pointer
6096 *
6097 * Abort the pending command in device by sending UFS_ABORT_TASK task management
6098 * command, and in host controller by clearing the door-bell register. There can
6099 * be race between controller sending the command to the device while abort is
6100 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
6101 * really issued and then try to abort it.
6102 *
6103 * Returns SUCCESS/FAILED
6104 */
6106 {
6115 u32 reg;
6126 }
6128 /*
6129 * Task abort to the device W-LUN is illegal. When this command
6130 * will fail, due to spec violation, scsi err handling next step
6131 * will be to send LU reset which, again, is a spec violation.
6132 * To avoid these unnecessary/illegal step we skip to the last error
6133 * handling stage: reset and restore.
6134 */
6140 /* If command is already aborted/completed, return SUCCESS */
6146 }
6152 }
6154 /* Print Transfer Request of aborted task */
6157 /*
6158 * Print detailed info about aborted request.
6159 * As more than one request might get aborted at the same time,
6160 * print full information only for the first aborted request in order
6161 * to reduce repeated printouts. For other aborted requests only print
6162 * basic details.
6163 */
6173 }
6176 /* Skip task abort in case previous aborts failed and report failure */
6180 }
6186 /* cmd pending in the device */
6191 /*
6192 * cmd not pending in the device, check if it is
6193 * in transition.
6194 */
6199 /* sleep for max. 200us to stabilize */
6202 }
6203 /* command completed already */
6214 }
6215 }
6220 }
6229 }
6231 }
6238 }
6250 out:
6257 }
6259 /*
6260 * This ufshcd_release() corresponds to the original scsi cmd that got
6261 * aborted here (as we won't get any IRQ for it).
6262 */
6265 }
6267 /**
6268 * ufshcd_host_reset_and_restore - reset and restore host controller
6269 * @hba: per-adapter instance
6270 *
6271 * Note that host controller reset may issue DME_RESET to
6272 * local and remote (device) Uni-Pro stack and the attributes
6273 * are reset to default state.
6274 *
6275 * Returns zero on success, non-zero on failure
6276 */
6278 {
6282 /* Reset the host controller */
6287 /* scale up clocks to max frequency before full reinitialization */
6294 /* Establish the link again and restore the device */
6299 out:
6304 }
6306 /**
6307 * ufshcd_reset_and_restore - reset and re-initialize host/device
6308 * @hba: per-adapter instance
6309 *
6310 * Reset and recover device, host and re-establish link. This
6311 * is helpful to recover the communication in fatal error conditions.
6312 *
6313 * Returns zero on success, non-zero on failure
6314 */
6316 {
6322 /* Reset the attached device */
6328 /*
6329 * After reset the door-bell might be cleared, complete
6330 * outstanding requests in s/w here.
6331 */
6338 }
6340 /**
6341 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
6342 * @cmd: SCSI command pointer
6343 *
6344 * Returns SUCCESS/FAILED
6345 */
6347 {
6355 /*
6356 * Check if there is any race with fatal error handling.
6357 * If so, wait for it to complete. Even though fatal error
6358 * handling does reset and restore in some cases, don't assume
6359 * anything out of it. We are just avoiding race here.
6360 */
6385 }
6391 }
6393 /**
6394 * ufshcd_get_max_icc_level - calculate the ICC level
6395 * @sup_curr_uA: max. current supported by the regulator
6396 * @start_scan: row at the desc table to start scan from
6397 * @buff: power descriptor buffer
6398 *
6399 * Returns calculated max ICC level for specific regulator
6400 */
6402 {
6405 u16 data;
6406 u16 unit;
6411 ATTR_ICC_LVL_UNIT_OFFSET;
6426 }
6429 }
6433 }
6436 }
6438 /**
6439 * ufshcd_calc_icc_level - calculate the max ICC level
6440 * In case regulators are not initialized we'll return 0
6441 * @hba: per-adapter instance
6442 * @desc_buf: power descriptor buffer to extract ICC levels from.
6443 * @len: length of desc_buff
6444 *
6445 * Returns calculated ICC level
6446 */
6449 {
6458 }
6469 icc_level,
6475 icc_level,
6477 out:
6479 }
6482 {
6497 }
6514 out:
6516 }
6518 /**
6519 * ufshcd_scsi_add_wlus - Adds required W-LUs
6520 * @hba: per-adapter instance
6521 *
6522 * UFS device specification requires the UFS devices to support 4 well known
6523 * logical units:
6524 * "REPORT_LUNS" (address: 01h)
6525 * "UFS Device" (address: 50h)
6526 * "RPMB" (address: 44h)
6527 * "BOOT" (address: 30h)
6528 * UFS device's power management needs to be controlled by "POWER CONDITION"
6529 * field of SSU (START STOP UNIT) command. But this "power condition" field
6530 * will take effect only when its sent to "UFS device" well known logical unit
6531 * hence we require the scsi_device instance to represent this logical unit in
6532 * order for the UFS host driver to send the SSU command for power management.
6533 *
6534 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
6535 * Block) LU so user space process can control this LU. User space may also
6536 * want to have access to BOOT LU.
6537 *
6538 * This function adds scsi device instances for each of all well known LUs
6539 * (except "REPORT LUNS" LU).
6540 *
6541 * Returns zero on success (all required W-LUs are added successfully),
6542 * non-zero error value on failure (if failed to add any of the required W-LU).
6543 */
6545 {
6556 }
6564 }
6571 else
6575 remove_sdev_ufs_device:
6577 out:
6579 }
6583 {
6586 u8 model_index;
6598 }
6605 }
6607 /*
6608 * getting vendor (manufacturerID) and Bank Index in big endian
6609 * format
6610 */
6621 }
6623 /*
6624 * ufshcd_read_string_desc returns size of the string
6625 * reset the error value
6626 */
6629 out:
6632 }
6635 {
6638 }
6642 {
6652 }
6653 }
6655 /**
6656 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
6657 * @hba: per-adapter instance
6658 *
6659 * PA_TActivate parameter can be tuned manually if UniPro version is less than
6660 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
6661 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
6662 * the hibern8 exit latency.
6663 *
6664 * Returns zero on success, non-zero error value on failure.
6665 */
6667 {
6673 RX_MIN_ACTIVATETIME_CAPABILITY,
6679 /* make sure proper unit conversion is applied */
6680 tuned_pa_tactivate =
6684 tuned_pa_tactivate);
6686 out:
6688 }
6690 /**
6691 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
6692 * @hba: per-adapter instance
6693 *
6694 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
6695 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
6696 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
6697 * This optimal value can help reduce the hibern8 exit latency.
6698 *
6699 * Returns zero on success, non-zero error value on failure.
6700 */
6702 {
6722 peer_rx_hibern8_time_cap);
6723 /* make sure proper unit conversion is applied */
6727 tuned_pa_hibern8time);
6728 out:
6730 }
6732 /**
6733 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
6734 * less than device PA_TACTIVATE time.
6735 * @hba: per-adapter instance
6736 *
6737 * Some UFS devices require host PA_TACTIVATE to be lower than device
6738 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
6739 * for such devices.
6740 *
6741 * Returns zero on success, non-zero error value on failure.
6742 */
6744 {
6766 }
6773 }
6789 u32 new_peer_pa_tactivate;
6793 new_peer_pa_tactivate++;
6795 new_peer_pa_tactivate);
6796 }
6798 out:
6800 }
6803 {
6807 }
6810 /* set 1ms timeout for PA_TACTIVATE */
6817 }
6820 {
6824 }
6827 {
6864 }
6872 };
6874 static enum ufs_ref_clk_freq
6876 {
6884 }
6887 {
6898 }
6901 {
6903 u32 ref_clk;
6911 err);
6913 }
6925 }
6930 out:
6932 }
6934 /**
6935 * ufshcd_probe_hba - probe hba to detect device and initialize
6936 * @hba: per-adapter instance
6937 *
6938 * Execute link-startup and verify device initialization
6939 */
6941 {
6950 /* set the default level for urgent bkops */
6954 /* Debug counters initialization */
6957 /* UniPro link is active now */
6968 /* Init check for device descriptor sizes */
6976 }
6983 /* UFS device is also active now */
6991 __func__);
6993 /*
6994 * Set the right value to bRefClkFreq before attempting to
6995 * switch to HS gears.
6996 */
7004 }
7005 }
7007 /* set the state as operational after switching to desired gear */
7010 /* Enable Auto-Hibernate if configured */
7013 /*
7014 * If we are in error handling context or in power management callbacks
7015 * context, no need to scan the host
7016 */
7020 /* clear any previous UFS device information */
7029 /* Add required well known logical units to scsi mid layer */
7033 /* Initialize devfreq after UFS device is detected */
7043 }
7045 }
7051 }
7056 out:
7057 /*
7058 * If we failed to initialize the device or the device is not
7059 * present, turn off the power/clocks etc.
7060 */
7065 }
7071 }
7073 /**
7074 * ufshcd_async_scan - asynchronous execution for probing hba
7075 * @data: data pointer to pass to this function
7076 * @cookie: cookie data
7077 */
7079 {
7083 }
7086 {
7107 }
7108 }
7112 /*
7113 * Bypass SCSI error handling and reset the block layer timer if this
7114 * SCSI command was not actually dispatched to UFS driver, otherwise
7115 * let SCSI layer handle the error as usual.
7116 */
7118 }
7123 NULL,
7124 };
7149 };
7153 {
7159 /*
7160 * "set_load" operation shall be required on those regulators
7161 * which specifically configured current limitation. Otherwise
7162 * zero max_uA may cause unexpected behavior when regulator is
7163 * enabled or set as high power mode.
7164 */
7172 }
7175 }
7179 {
7181 }
7185 {
7190 }
7194 {
7214 }
7215 }
7221 }
7222 out:
7224 }
7227 {
7239 else
7242 out:
7244 }
7247 {
7256 /* ignore errors on applying disable config */
7262 }
7263 out:
7265 }
7268 {
7285 out:
7290 }
7292 }
7295 {
7299 }
7302 {
7313 }
7314 out:
7316 }
7319 {
7333 out:
7335 }
7338 {
7345 }
7349 {
7360 /*
7361 * vendor specific setup_clocks ops may depend on clocks managed by
7362 * this standard driver hence call the vendor specific setup_clocks
7363 * before disabling the clocks managed here.
7364 */
7369 }
7383 }
7386 }
7390 }
7391 }
7393 /*
7394 * vendor specific setup_clocks ops may depend on clocks managed by
7395 * this standard driver hence call the vendor specific setup_clocks
7396 * after enabling the clocks managed here.
7397 */
7402 }
7404 out:
7409 }
7416 }
7423 }
7426 {
7428 }
7431 {
7450 }
7452 /*
7453 * Parse device ref clk freq as per device tree "ref_clk".
7454 * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
7455 * in ufshcd_alloc_host().
7456 */
7467 }
7469 }
7472 }
7473 out:
7475 }
7478 {
7494 out_exit:
7496 out:
7501 }
7504 {
7511 }
7514 {
7517 /*
7518 * Handle host controller power separately from the UFS device power
7519 * rails as it will help controlling the UFS host controller power
7520 * collapse easily which is different than UFS device power collapse.
7521 * Also, enable the host controller power before we go ahead with rest
7522 * of the initialization here.
7523 */
7555 out_disable_vreg:
7557 out_disable_clks:
7559 out_disable_hba_vreg:
7561 out:
7563 }
7566 {
7577 }
7578 }
7580 static int
7582 {
7587 UFS_SENSE_SIZE,
7596 }
7605 out:
7607 }
7609 /**
7610 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
7611 * power mode
7612 * @hba: per adapter instance
7613 * @pwr_mode: device power mode to set
7614 *
7615 * Returns 0 if requested power mode is set successfully
7616 * Returns non-zero if failed to set the requested power mode
7617 */
7620 {
7634 }
7637 }
7643 /*
7644 * If scsi commands fail, the scsi mid-layer schedules scsi error-
7645 * handling, which would wait for host to be resumed. Since we know
7646 * we are functional while we are here, skip host resume in error
7647 * handling context.
7648 */
7654 /* Unit attention condition is cleared now */
7656 }
7660 /*
7661 * Current function would be generally called from the power management
7662 * callbacks hence set the RQF_PM flag so that it doesn't resume the
7663 * already suspended childs.
7664 */
7673 }
7677 out:
7681 }
7686 {
7696 else
7698 }
7699 /*
7700 * If autobkops is enabled, link can't be turned off because
7701 * turning off the link would also turn off the device.
7702 */
7706 /*
7707 * Let's make sure that link is in low power mode, we are doing
7708 * this currently by putting the link in Hibern8. Otherway to
7709 * put the link in low power mode is to send the DME end point
7710 * to device and then send the DME reset command to local
7711 * unipro. But putting the link in hibern8 is much faster.
7712 */
7716 /*
7717 * Change controller state to "reset state" which
7718 * should also put the link in off/reset state
7719 */
7721 /*
7722 * TODO: Check if we need any delay to make sure that
7723 * controller is reset
7724 */
7726 }
7728 out:
7730 }
7733 {
7734 /*
7735 * It seems some UFS devices may keep drawing more than sleep current
7736 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
7737 * To avoid this situation, add 2ms delay before putting these UFS
7738 * rails in LPM mode.
7739 */
7744 /*
7745 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
7746 * power.
7747 *
7748 * If UFS device and link is in OFF state, all power supplies (VCC,
7749 * VCCQ, VCCQ2) can be turned off if power on write protect is not
7750 * required. If UFS link is inactive (Hibern8 or OFF state) and device
7751 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
7752 *
7753 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
7754 * in low power state which would save some power.
7755 */
7764 }
7765 }
7766 }
7769 {
7783 }
7785 }
7788 vccq_lpm:
7790 vcc_disable:
7792 out:
7794 }
7797 {
7800 }
7803 {
7806 }
7808 /**
7809 * ufshcd_suspend - helper function for suspend operations
7810 * @hba: per adapter instance
7811 * @pm_op: desired low power operation type
7812 *
7813 * This function will try to put the UFS device and link into low power
7814 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
7815 * (System PM level).
7816 *
7817 * If this function is called during shutdown, it will make sure that
7818 * both UFS device and UFS link is powered off.
7819 *
7820 * NOTE: UFS device & link must be active before we enter in this function.
7821 *
7822 * Returns 0 for success and non-zero for failure
7823 */
7825 {
7840 }
7842 /*
7843 * If we can't transition into any of the low power modes
7844 * just gate the clocks.
7845 */
7853 }
7858 }
7864 /* UFS device & link must be active before we enter in this function */
7868 }
7872 /*
7873 * The device is idle with no requests in the queue,
7874 * allow background operations if bkops status shows
7875 * that performance might be impacted.
7876 */
7881 /* make sure that auto bkops is disabled */
7883 }
7884 }
7889 /* ensure that bkops is disabled */
7894 }
7902 disable_clks:
7903 /*
7904 * Call vendor specific suspend callback. As these callbacks may access
7905 * vendor specific host controller register space call them before the
7906 * host clocks are ON.
7907 */
7914 else
7915 /* If link is active, device ref_clk can't be switched off */
7920 /*
7921 * Disable the host irq as host controller as there won't be any
7922 * host controller transaction expected till resume.
7923 */
7925 /* Put the host controller in low power mode if possible */
7929 set_link_active:
7937 set_dev_active:
7940 enable_gating:
7945 out:
7950 }
7952 /**
7953 * ufshcd_resume - helper function for resume operations
7954 * @hba: per adapter instance
7955 * @pm_op: runtime PM or system PM
7956 *
7957 * This function basically brings the UFS device, UniPro link and controller
7958 * to active state.
7959 *
7960 * Returns 0 for success and non-zero for failure
7961 */
7963 {
7971 /* Make sure clocks are enabled before accessing controller */
7976 /* enable the host irq as host controller would be active soon */
7985 /*
7986 * Call vendor specific resume callback. As these callbacks may access
7987 * vendor specific host controller register space call them when the
7988 * host clocks are ON.
7989 */
7998 else
8002 /*
8003 * ufshcd_host_reset_and_restore() should have already
8004 * set the link state as active
8005 */
8008 }
8014 }
8018 else
8019 /*
8020 * If BKOPs operations are urgently needed at this moment then
8021 * keep auto-bkops enabled or else disable it.
8022 */
8030 /* Enable Auto-Hibernate if configured */
8033 /* Schedule clock gating in case of no access to UFS device yet */
8038 set_old_link_state:
8040 vendor_suspend:
8042 disable_vreg:
8044 disable_irq_and_vops_clks:
8049 out:
8054 }
8056 /**
8057 * ufshcd_system_suspend - system suspend routine
8058 * @hba: per adapter instance
8059 *
8060 * Check the description of ufshcd_suspend() function for more details.
8061 *
8062 * Returns 0 for success and non-zero for failure
8063 */
8065 {
8079 /*
8080 * UFS device and/or UFS link low power states during runtime
8081 * suspend seems to be different than what is expected during
8082 * system suspend. Hence runtime resume the devic & link and
8083 * let the system suspend low power states to take effect.
8084 * TODO: If resume takes longer time, we might have optimize
8085 * it in future by not resuming everything if possible.
8086 */
8090 }
8093 out:
8100 }
8103 /**
8104 * ufshcd_system_resume - system resume routine
8105 * @hba: per adapter instance
8106 *
8107 * Returns 0 for success and non-zero for failure
8108 */
8111 {
8119 /*
8120 * Let the runtime resume take care of resuming
8121 * if runtime suspended.
8122 */
8124 else
8126 out:
8133 }
8136 /**
8137 * ufshcd_runtime_suspend - runtime suspend routine
8138 * @hba: per adapter instance
8139 *
8140 * Check the description of ufshcd_suspend() function for more details.
8141 *
8142 * Returns 0 for success and non-zero for failure
8143 */
8145 {
8154 else
8156 out:
8161 }
8164 /**
8165 * ufshcd_runtime_resume - runtime resume routine
8166 * @hba: per adapter instance
8167 *
8168 * This function basically brings the UFS device, UniPro link and controller
8169 * to active state. Following operations are done in this function:
8170 *
8171 * 1. Turn on all the controller related clocks
8172 * 2. Bring the UniPro link out of Hibernate state
8173 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
8174 * to active state.
8175 * 4. If auto-bkops is enabled on the device, disable it.
8176 *
8177 * So following would be the possible power state after this function return
8178 * successfully:
8179 * S1: UFS device in Active state with VCC rail ON
8180 * UniPro link in Active state
8181 * All the UFS/UniPro controller clocks are ON
8182 *
8183 * Returns 0 for success and non-zero for failure
8184 */
8186 {
8195 else
8197 out:
8202 }
8206 {
8208 }
8211 /**
8212 * ufshcd_shutdown - shutdown routine
8213 * @hba: per adapter instance
8214 *
8215 * This function would power off both UFS device and UFS link.
8216 *
8217 * Returns 0 always to allow force shutdown even in case of errors.
8218 */
8220 {
8233 }
8236 out:
8239 /* allow force shutdown even in case of errors */
8241 }
8244 /**
8245 * ufshcd_remove - de-allocate SCSI host and host memory space
8246 * data structure memory
8247 * @hba: per adapter instance
8248 */
8250 {
8254 /* disable interrupts */
8263 }
8266 /**
8267 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
8268 * @hba: pointer to Host Bus Adapter (HBA)
8269 */
8271 {
8273 }
8276 /**
8277 * ufshcd_set_dma_mask - Set dma mask based on the controller
8278 * addressing capability
8279 * @hba: per adapter instance
8280 *
8281 * Returns 0 for success, non-zero for failure
8282 */
8284 {
8288 }
8290 }
8292 /**
8293 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
8294 * @dev: pointer to device handle
8295 * @hba_handle: driver private handle
8296 * Returns 0 on success, non-zero value on failure
8297 */
8299 {
8309 }
8317 }
8326 out_error:
8328 }
8331 /**
8332 * ufshcd_init - Driver initialization routine
8333 * @hba: per-adapter instance
8334 * @mmio_base: base register address
8335 * @irq: Interrupt line of device
8336 * Returns 0 on success, non-zero value on failure
8337 */
8339 {
8349 }
8358 /* Read capabilities registers */
8361 /* Get UFS version supported by the controller */
8371 /* Get Interrupt bit mask per version */
8378 }
8380 /* Allocate memory for host memory space */
8385 }
8387 /* Configure LRB */
8400 /* Initailize wait queue for task management */
8404 /* Initialize work queues */
8408 /* Initialize UIC command mutex */
8411 /* Initialize mutex for device management commands */
8416 /* Initialize device management tag acquire wait queue */
8423 /*
8424 * In order to avoid any spurious interrupt immediately after
8425 * registering UFS controller interrupt handler, clear any pending UFS
8426 * interrupt status and disable all the UFS interrupts.
8427 */
8429 REG_INTERRUPT_STATUS);
8431 /*
8432 * Make sure that UFS interrupts are disabled and any pending interrupt
8433 * status is cleared before registering UFS interrupt handler.
8434 */
8437 /* IRQ registration */
8444 }
8450 }
8452 /* Reset the attached device */
8455 /* Host controller enable */
8462 }
8464 /*
8465 * Set the default power management level for runtime and system PM.
8466 * Default power saving mode is to keep UFS link in Hibern8 state
8467 * and UFS device in sleep state.
8468 */
8470 UFS_SLEEP_PWR_MODE,
8471 UIC_LINK_HIBERN8_STATE);
8473 UFS_SLEEP_PWR_MODE,
8474 UIC_LINK_HIBERN8_STATE);
8476 /* Set the default auto-hiberate idle timer value to 150 ms */
8480 }
8482 /* Hold auto suspend until async scan completes */
8485 /*
8486 * We are assuming that device wasn't put in sleep/power-down
8487 * state exclusively during the boot stage before kernel.
8488 * This assumption helps avoid doing link startup twice during
8489 * ufshcd_probe_hba().
8490 */
8498 out_remove_scsi_host:
8500 exit_gating:
8503 out_disable:
8506 out_error:
8508 }