| blob | c7da2c185990615f906e847def5b6513a1338801 |
1 /*
2 * Universal Flash Storage Host controller driver Core
3 *
4 * This code is based on drivers/scsi/ufs/ufshcd.c
5 * Copyright (C) 2011-2013 Samsung India Software Operations
6 * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
7 *
8 * Authors:
9 * Santosh Yaraganavi <santosh.sy@samsung.com>
10 * Vinayak Holikatti <h.vinayak@samsung.com>
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 * See the COPYING file in the top-level directory or visit
17 * <http://www.gnu.org/licenses/gpl-2.0.html>
18 *
19 * This program is distributed in the hope that it will be useful,
20 * but WITHOUT ANY WARRANTY; without even the implied warranty of
21 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 * GNU General Public License for more details.
23 *
24 * This program is provided "AS IS" and "WITH ALL FAULTS" and
25 * without warranty of any kind. You are solely responsible for
26 * determining the appropriateness of using and distributing
27 * the program and assume all risks associated with your exercise
28 * of rights with respect to the program, including but not limited
29 * to infringement of third party rights, the risks and costs of
30 * program errors, damage to or loss of data, programs or equipment,
31 * and unavailability or interruption of operations. Under no
32 * circumstances will the contributor of this Program be liable for
33 * any damages of any kind arising from your use or distribution of
34 * this program.
35 *
36 * The Linux Foundation chooses to take subject only to the GPLv2
37 * license terms, and distributes only under these terms.
38 */
40 #include <linux/async.h>
41 #include <linux/devfreq.h>
42 #include <linux/nls.h>
43 #include <linux/of.h>
48 #define CREATE_TRACE_POINTS
49 #include <trace/events/ufs.h>
51 #define UFSHCD_REQ_SENSE_SIZE 18
53 #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
54 UTP_TASK_REQ_COMPL |\
55 UFSHCD_ERROR_MASK)
56 /* UIC command timeout, unit: ms */
57 #define UIC_CMD_TIMEOUT 500
59 /* NOP OUT retries waiting for NOP IN response */
60 #define NOP_OUT_RETRIES 10
61 /* Timeout after 30 msecs if NOP OUT hangs without response */
64 /* Query request retries */
65 #define QUERY_REQ_RETRIES 3
66 /* Query request timeout */
69 /* Task management command timeout */
72 /* maximum number of retries for a general UIC command */
73 #define UFS_UIC_COMMAND_RETRIES 3
75 /* maximum number of link-startup retries */
76 #define DME_LINKSTARTUP_RETRIES 3
78 /* Maximum retries for Hibern8 enter */
79 #define UIC_HIBERN8_ENTER_RETRIES 3
81 /* maximum number of reset retries before giving up */
82 #define MAX_HOST_RESET_RETRIES 5
84 /* Expose the flag value from utp_upiu_query.value */
85 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
87 /* Interrupt aggregation default timeout, unit: 40us */
88 #define INT_AGGR_DEF_TO 0x02
90 #define ufshcd_toggle_vreg(_dev, _vreg, _on) \
91 ({ \
92 int _ret; \
93 if (_on) \
94 _ret = ufshcd_enable_vreg(_dev, _vreg); \
95 else \
96 _ret = ufshcd_disable_vreg(_dev, _vreg); \
97 _ret; \
98 })
100 #define ufshcd_hex_dump(prefix_str, buf, len) \
101 print_hex_dump(KERN_ERR, prefix_str, DUMP_PREFIX_OFFSET, 16, 4, buf, len, false)
108 };
110 /* UFSHCD states */
112 UFSHCD_STATE_RESET,
113 UFSHCD_STATE_ERROR,
114 UFSHCD_STATE_OPERATIONAL,
115 UFSHCD_STATE_EH_SCHEDULED,
116 };
118 /* UFSHCD error handling flags */
121 };
123 /* UFSHCD UIC layer error flags */
131 };
133 #define ufshcd_set_eh_in_progress(h) \
134 ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
135 #define ufshcd_eh_in_progress(h) \
136 ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
137 #define ufshcd_clear_eh_in_progress(h) \
138 ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
140 #define ufshcd_set_ufs_dev_active(h) \
141 ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
142 #define ufshcd_set_ufs_dev_sleep(h) \
143 ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
144 #define ufshcd_set_ufs_dev_poweroff(h) \
145 ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
146 #define ufshcd_is_ufs_dev_active(h) \
147 ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
148 #define ufshcd_is_ufs_dev_sleep(h) \
149 ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
150 #define ufshcd_is_ufs_dev_poweroff(h) \
151 ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
160 };
164 {
166 }
170 {
172 }
177 {
184 }
186 /* if no match found, return the level 0 */
188 }
191 /* UFS cards deviations table */
193 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
196 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
198 UFS_DEVICE_NO_FASTAUTO),
200 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
202 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
204 UFS_DEVICE_QUIRK_PA_TACTIVATE),
206 UFS_DEVICE_QUIRK_PA_TACTIVATE),
209 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
211 END_FIX
212 };
239 {
241 }
244 {
249 hba);
254 }
257 }
260 {
264 }
265 }
267 /* replace non-printable or non-ASCII characters with spaces */
269 {
275 }
279 {
294 /*
295 * Currently we only fully trace read(10) and write(10)
296 * commands
297 */
299 lba =
303 }
304 }
310 }
313 {
325 }
326 }
330 {
340 }
341 }
344 {
345 /*
346 * hex_dump reads its data without the readl macro. This might
347 * cause inconsistency issues on some platform, as the printed
348 * values may be from cache and not the most recent value.
349 * To know whether you are looking at an un-cached version verify
350 * that IORESOURCE_MEM flag is on when xxx_get_resource() is invoked
351 * during platform/pci probe function.
352 */
374 }
376 static
378 {
415 }
416 }
419 {
429 tag);
433 tag);
436 }
437 }
440 {
459 }
461 /**
462 * ufshcd_print_pwr_info - print power params as saved in hba
463 * power info
464 * @hba: per-adapter instance
465 */
467 {
476 };
479 __func__,
485 }
487 /*
488 * ufshcd_wait_for_register - wait for register value to change
489 * @hba - per-adapter interface
490 * @reg - mmio register offset
491 * @mask - mask to apply to read register value
492 * @val - wait condition
493 * @interval_us - polling interval in microsecs
494 * @timeout_ms - timeout in millisecs
495 * @can_sleep - perform sleep or just spin
496 *
497 * Returns -ETIMEDOUT on error, zero on success
498 */
502 {
506 /* ignore bits that we don't intend to wait on */
512 else
518 }
519 }
522 }
524 /**
525 * ufshcd_get_intr_mask - Get the interrupt bit mask
526 * @hba - Pointer to adapter instance
527 *
528 * Returns interrupt bit mask per version
529 */
531 {
546 }
549 }
551 /**
552 * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
553 * @hba - Pointer to adapter instance
554 *
555 * Returns UFSHCI version supported by the controller
556 */
558 {
563 }
565 /**
566 * ufshcd_is_device_present - Check if any device connected to
567 * the host controller
568 * @hba: pointer to adapter instance
569 *
570 * Returns true if device present, false if no device detected
571 */
573 {
576 }
578 /**
579 * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
580 * @lrb: pointer to local command reference block
581 *
582 * This function is used to get the OCS field from UTRD
583 * Returns the OCS field in the UTRD
584 */
586 {
588 }
590 /**
591 * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status
592 * @task_req_descp: pointer to utp_task_req_desc structure
593 *
594 * This function is used to get the OCS field from UTMRD
595 * Returns the OCS field in the UTMRD
596 */
599 {
601 }
603 /**
604 * ufshcd_get_tm_free_slot - get a free slot for task management request
605 * @hba: per adapter instance
606 * @free_slot: pointer to variable with available slot value
607 *
608 * Get a free tag and lock it until ufshcd_put_tm_slot() is called.
609 * Returns 0 if free slot is not available, else return 1 with tag value
610 * in @free_slot.
611 */
613 {
628 out:
630 }
633 {
635 }
637 /**
638 * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
639 * @hba: per adapter instance
640 * @pos: position of the bit to be cleared
641 */
643 {
645 }
647 /**
648 * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
649 * @hba: per adapter instance
650 * @tag: position of the bit to be cleared
651 */
653 {
655 }
657 /**
658 * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
659 * @reg: Register value of host controller status
660 *
661 * Returns integer, 0 on Success and positive value if failed
662 */
664 {
666 }
668 /**
669 * ufshcd_get_uic_cmd_result - Get the UIC command result
670 * @hba: Pointer to adapter instance
671 *
672 * This function gets the result of UIC command completion
673 * Returns 0 on success, non zero value on error
674 */
676 {
678 MASK_UIC_COMMAND_RESULT;
679 }
681 /**
682 * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
683 * @hba: Pointer to adapter instance
684 *
685 * This function gets UIC command argument3
686 * Returns 0 on success, non zero value on error
687 */
689 {
691 }
693 /**
694 * ufshcd_get_req_rsp - returns the TR response transaction type
695 * @ucd_rsp_ptr: pointer to response UPIU
696 */
699 {
701 }
703 /**
704 * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
705 * @ucd_rsp_ptr: pointer to response UPIU
706 *
707 * This function gets the response status and scsi_status from response UPIU
708 * Returns the response result code.
709 */
712 {
714 }
716 /*
717 * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
718 * from response UPIU
719 * @ucd_rsp_ptr: pointer to response UPIU
720 *
721 * Return the data segment length.
722 */
725 {
727 MASK_RSP_UPIU_DATA_SEG_LEN;
728 }
730 /**
731 * ufshcd_is_exception_event - Check if the device raised an exception event
732 * @ucd_rsp_ptr: pointer to response UPIU
733 *
734 * The function checks if the device raised an exception event indicated in
735 * the Device Information field of response UPIU.
736 *
737 * Returns true if exception is raised, false otherwise.
738 */
740 {
743 }
745 /**
746 * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
747 * @hba: per adapter instance
748 */
751 {
753 INT_AGGR_COUNTER_AND_TIMER_RESET,
754 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
755 }
757 /**
758 * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
759 * @hba: per adapter instance
760 * @cnt: Interrupt aggregation counter threshold
761 * @tmout: Interrupt aggregation timeout value
762 */
765 {
769 REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
770 }
772 /**
773 * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
774 * @hba: per adapter instance
775 */
777 {
779 }
781 /**
782 * ufshcd_enable_run_stop_reg - Enable run-stop registers,
783 * When run-stop registers are set to 1, it indicates the
784 * host controller that it can process the requests
785 * @hba: per adapter instance
786 */
788 {
790 REG_UTP_TASK_REQ_LIST_RUN_STOP);
792 REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
793 }
795 /**
796 * ufshcd_hba_start - Start controller initialization sequence
797 * @hba: per adapter instance
798 */
800 {
802 }
804 /**
805 * ufshcd_is_hba_active - Get controller state
806 * @hba: per adapter instance
807 *
808 * Returns false if controller is active, true otherwise
809 */
811 {
814 }
818 {
824 }
825 }
829 {
835 }
836 }
839 {
840 /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
844 else
846 }
850 {
851 /*
852 * If both host and device support UniPro ver1.6 or later, PA layer
853 * parameters tuning happens during link startup itself.
854 *
855 * We can manually tune PA layer parameters if either host or device
856 * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
857 * logic simple, we will only do manual tuning if local unipro version
858 * doesn't support ver1.6 or later.
859 */
862 else
864 }
867 {
894 }
913 }
919 }
920 }
923 }
927 out:
933 }
935 /**
936 * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
937 * @hba: per adapter instance
938 * @scale_up: True if scaling up and false if scaling down
939 *
940 * Returns true if scaling is required, false otherwise.
941 */
944 {
961 }
962 }
963 }
966 }
969 u64 wait_timeout_us)
970 {
973 u32 tm_doorbell;
974 u32 tr_doorbell;
976 ktime_t start;
980 /*
981 * Wait for all the outstanding tasks/transfer requests.
982 * Verify by checking the doorbell registers are clear.
983 */
989 }
998 }
1003 wait_timeout_us) {
1005 /*
1006 * We might have scheduled out for long time so make
1007 * sure to check if doorbells are cleared by this time
1008 * or not.
1009 */
1011 }
1020 }
1021 out:
1025 }
1027 /**
1028 * ufshcd_scale_gear - scale up/down UFS gear
1029 * @hba: per adapter instance
1030 * @scale_up: True for scaling up gear and false for scaling down
1031 *
1032 * Returns 0 for success,
1033 * Returns -EBUSY if scaling can't happen at this time
1034 * Returns non-zero for any other errors
1035 */
1037 {
1038 #define UFS_MIN_GEAR_TO_SCALE_DOWN UFS_HS_G1
1051 /* save the current power mode */
1056 /* scale down gear */
1059 }
1060 }
1062 /* check if the power mode needs to be changed or not? */
1072 }
1075 {
1078 /*
1079 * make sure that there are no outstanding requests when
1080 * clock scaling is in progress
1081 */
1088 }
1091 }
1094 {
1097 }
1099 /**
1100 * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1101 * @hba: per adapter instance
1102 * @scale_up: True for scaling up and false for scalin down
1103 *
1104 * Returns 0 for success,
1105 * Returns -EBUSY if scaling can't happen at this time
1106 * Returns non-zero for any other errors
1107 */
1109 {
1112 /* let's not get into low power until clock scaling is completed */
1119 /* scale down the gear before scaling down clocks */
1124 }
1131 }
1133 /* scale up the gear after scaling up clocks */
1139 }
1140 }
1144 out:
1148 }
1151 {
1160 }
1165 }
1168 {
1177 }
1182 }
1186 {
1189 ktime_t start;
1199 }
1205 }
1215 }
1225 out:
1231 }
1236 {
1257 start_window:
1267 }
1270 }
1276 };
1279 {
1286 }
1289 {
1300 }
1305 }
1308 {
1319 }
1324 }
1328 {
1332 }
1336 {
1338 u32 value;
1364 }
1368 out:
1370 }
1373 {
1381 }
1384 {
1396 }
1401 /* Exit from hibern8 */
1403 /* Prevent gating in this path */
1410 else
1412 }
1414 }
1415 unblock_reqs:
1417 }
1419 /**
1420 * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1421 * Also, exit from hibern8 mode and set the link as active.
1422 * @hba: per adapter instance
1423 * @async: This indicates whether caller should ungate clocks asynchronously.
1424 */
1426 {
1438 }
1440 start:
1443 /*
1444 * Wait for the ungate work to complete if in progress.
1445 * Though the clocks may be in ON state, the link could
1446 * still be in hibner8 state if hibern8 is allowed
1447 * during clock gating.
1448 * Make sure we exit hibern8 state also in addition to
1449 * clocks being ON.
1450 */
1457 }
1465 }
1466 /*
1467 * If we are here, it means gating work is either done or
1468 * currently running. Hence, fall through to cancel gating
1469 * work and to enable clocks.
1470 */
1477 /*
1478 * fall through to check if we should wait for this
1479 * work to be done or not.
1480 */
1486 }
1490 /* Make sure state is CLKS_ON before returning */
1497 }
1499 out:
1501 }
1505 {
1511 /*
1512 * In case you are here to cancel this work the gating state
1513 * would be marked as REQ_CLKS_ON. In this case save time by
1514 * skipping the gating work and exit after changing the clock
1515 * state to CLKS_ON.
1516 */
1523 }
1533 /* put the link into hibern8 mode before turning off clocks */
1540 }
1542 }
1546 else
1547 /* If link is active, device ref_clk can't be switched off */
1550 /*
1551 * In case you are here to cancel this work the gating state
1552 * would be marked as REQ_CLKS_ON. In this case keep the state
1553 * as REQ_CLKS_ON which would anyway imply that clocks are off
1554 * and a request to turn them on is pending. By doing this way,
1555 * we keep the state machine in tact and this would ultimately
1556 * prevent from doing cancel work multiple times when there are
1557 * new requests arriving before the current cancel work is done.
1558 */
1564 }
1565 rel_lock:
1567 out:
1569 }
1571 /* host lock must be held before calling this variant */
1573 {
1590 }
1593 {
1599 }
1604 {
1608 }
1612 {
1623 }
1627 {
1631 }
1635 {
1638 u32 value;
1653 }
1656 out:
1658 }
1661 {
1686 }
1689 {
1696 }
1698 /* Must be called with host lock acquired */
1700 {
1720 }
1725 }
1726 }
1729 {
1740 }
1741 }
1742 /**
1743 * ufshcd_send_command - Send SCSI or device management commands
1744 * @hba: per adapter instance
1745 * @task_tag: Task tag of the command
1746 */
1749 {
1755 /* Make sure that doorbell is committed immediately */
1758 }
1760 /**
1761 * ufshcd_copy_sense_data - Copy sense data in case of check condition
1762 * @lrb - pointer to local reference block
1763 */
1765 {
1777 }
1778 }
1780 /**
1781 * ufshcd_copy_query_response() - Copy the Query Response and the data
1782 * descriptor
1783 * @hba: per adapter instance
1784 * @lrb - pointer to local reference block
1785 */
1786 static
1788 {
1793 /* Get the descriptor */
1796 GENERAL_UPIU_REQUEST_SIZE;
1797 u16 resp_len;
1798 u16 buf_len;
1800 /* data segment length */
1802 MASK_QUERY_DATA_SEG_LEN;
1810 __func__);
1812 }
1813 }
1816 }
1818 /**
1819 * ufshcd_hba_capabilities - Read controller capabilities
1820 * @hba: per adapter instance
1821 */
1823 {
1826 /* nutrs and nutmrs are 0 based values */
1830 }
1832 /**
1833 * ufshcd_ready_for_uic_cmd - Check if controller is ready
1834 * to accept UIC commands
1835 * @hba: per adapter instance
1836 * Return true on success, else false
1837 */
1839 {
1842 else
1844 }
1846 /**
1847 * ufshcd_get_upmcrs - Get the power mode change request status
1848 * @hba: Pointer to adapter instance
1849 *
1850 * This function gets the UPMCRS field of HCS register
1851 * Returns value of UPMCRS field
1852 */
1854 {
1856 }
1858 /**
1859 * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
1860 * @hba: per adapter instance
1861 * @uic_cmd: UIC command
1862 *
1863 * Mutex must be held.
1864 */
1867 {
1872 /* Write Args */
1877 /* Write UIC Cmd */
1879 REG_UIC_COMMAND);
1880 }
1882 /**
1883 * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
1884 * @hba: per adapter instance
1885 * @uic_command: UIC command
1886 *
1887 * Must be called with mutex held.
1888 * Returns 0 only if success.
1889 */
1890 static int
1892 {
1899 else
1907 }
1909 /**
1910 * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
1911 * @hba: per adapter instance
1912 * @uic_cmd: UIC command
1913 * @completion: initialize the completion only if this is set to true
1914 *
1915 * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
1916 * with mutex held and host_lock locked.
1917 * Returns 0 only if success.
1918 */
1919 static int
1922 {
1927 }
1935 }
1937 /**
1938 * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
1939 * @hba: per adapter instance
1940 * @uic_cmd: UIC command
1941 *
1942 * Returns 0 only if success.
1943 */
1944 static int
1946 {
1964 }
1966 /**
1967 * ufshcd_map_sg - Map scatter-gather list to prdt
1968 * @lrbp - pointer to local reference block
1969 *
1970 * Returns 0 in case of success, non-zero value in case of failure
1971 */
1973 {
1990 else
2004 }
2007 }
2010 }
2012 /**
2013 * ufshcd_enable_intr - enable interrupts
2014 * @hba: per adapter instance
2015 * @intrs: interrupt bits
2016 */
2018 {
2022 u32 rw;
2027 }
2030 }
2032 /**
2033 * ufshcd_disable_intr - disable interrupts
2034 * @hba: per adapter instance
2035 * @intrs: interrupt bits
2036 */
2038 {
2042 u32 rw;
2049 }
2052 }
2054 /**
2055 * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2056 * descriptor according to request
2057 * @lrbp: pointer to local reference block
2058 * @upiu_flags: flags required in the header
2059 * @cmd_dir: requests data direction
2060 */
2063 {
2065 u32 data_direction;
2066 u32 dword_0;
2077 }
2084 /* Transfer request descriptor header fields */
2086 /* dword_1 is reserved, hence it is set to 0 */
2088 /*
2089 * assigning invalid value for command status. Controller
2090 * updates OCS on command completion, with the command
2091 * status
2092 */
2095 /* dword_3 is reserved, hence it is set to 0 */
2099 }
2101 /**
2102 * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2103 * for scsi commands
2104 * @lrbp - local reference block pointer
2105 * @upiu_flags - flags
2106 */
2107 static
2109 {
2113 /* command descriptor fields */
2120 /* Total EHS length and Data segment length will be zero */
2131 }
2133 /**
2134 * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2135 * for query requsts
2136 * @hba: UFS hba
2137 * @lrbp: local reference block pointer
2138 * @upiu_flags: flags
2139 */
2142 {
2148 /* Query request header */
2155 /* Data segment length only need for WRITE_DESC */
2159 else
2162 /* Copy the Query Request buffer as is */
2164 QUERY_OSF_SIZE);
2166 /* Copy the Descriptor */
2171 }
2174 {
2179 /* command descriptor fields */
2183 /* clear rest of the fields of basic header */
2188 }
2190 /**
2191 * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
2192 * for Device Management Purposes
2193 * @hba - per adapter instance
2194 * @lrb - pointer to local reference block
2195 */
2197 {
2198 u32 upiu_flags;
2204 else
2212 else
2216 }
2218 /**
2219 * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2220 * for SCSI Purposes
2221 * @hba - per adapter instance
2222 * @lrb - pointer to local reference block
2223 */
2225 {
2226 u32 upiu_flags;
2232 else
2241 }
2244 }
2246 /*
2247 * ufshcd_scsi_to_upiu_lun - maps scsi LUN to UPIU LUN
2248 * @scsi_lun: scsi LUN id
2249 *
2250 * Returns UPIU LUN id
2251 */
2253 {
2257 else
2259 }
2261 /**
2262 * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2263 * @scsi_lun: UPIU W-LUN id
2264 *
2265 * Returns SCSI W-LUN id
2266 */
2268 {
2270 }
2272 /**
2273 * ufshcd_queuecommand - main entry point for SCSI requests
2274 * @cmd: command from SCSI Midlayer
2275 * @done: call back function
2276 *
2277 * Returns 0 for success, non-zero in case of failure
2278 */
2280 {
2295 }
2318 }
2320 /* if error handling is in progress, don't issue commands */
2325 }
2330 /* acquire the tag to make sure device cmds don't use it */
2332 /*
2333 * Dev manage command in progress, requeue the command.
2334 * Requeuing the command helps in cases where the request *may*
2335 * find different tag instead of waiting for dev manage command
2336 * completion.
2337 */
2340 }
2347 }
2368 }
2369 /* Make sure descriptors are ready before ringing the doorbell */
2372 /* issue command to the controller */
2376 out_unlock:
2378 out:
2381 }
2385 {
2395 }
2397 static int
2399 {
2404 /* clear outstanding transaction before retry */
2409 /*
2410 * wait for for h/w to clear corresponding bit in door-bell.
2411 * max. wait is 1 sec.
2412 */
2414 REG_UTP_TRANSFER_REQ_DOOR_BELL,
2418 }
2420 static int
2422 {
2425 /* Get the UPIU response */
2427 UPIU_RSP_CODE_OFFSET;
2429 }
2431 /**
2432 * ufshcd_dev_cmd_completion() - handles device management command responses
2433 * @hba: per adapter instance
2434 * @lrbp: pointer to local reference block
2435 */
2436 static int
2438 {
2451 }
2459 /* TODO: handle Reject UPIU Response */
2462 __func__);
2469 }
2472 }
2476 {
2484 /* Make sure descriptors are ready before ringing the doorbell */
2492 }
2500 /* successfully cleared the command, retry if needed */
2502 /*
2503 * in case of an error, after clearing the doorbell,
2504 * we also need to clear the outstanding_request
2505 * field in hba
2506 */
2508 }
2511 }
2513 /**
2514 * ufshcd_get_dev_cmd_tag - Get device management command tag
2515 * @hba: per-adapter instance
2516 * @tag: pointer to variable with available slot value
2517 *
2518 * Get a free slot and lock it until device management command
2519 * completes.
2520 *
2521 * Returns false if free slot is unavailable for locking, else
2522 * return true with tag value in @tag.
2523 */
2525 {
2542 out:
2544 }
2547 {
2549 }
2551 /**
2552 * ufshcd_exec_dev_cmd - API for sending device management requests
2553 * @hba - UFS hba
2554 * @cmd_type - specifies the type (NOP, Query...)
2555 * @timeout - time in seconds
2556 *
2557 * NOTE: Since there is only one available tag for device management commands,
2558 * it is expected you hold the hba->dev_cmd.lock mutex.
2559 */
2562 {
2571 /*
2572 * Get free slot, sleep if slots are unavailable.
2573 * Even though we use wait_event() which sleeps indefinitely,
2574 * the maximum wait time is bounded by SCSI request timeout.
2575 */
2587 /* Make sure descriptors are ready before ringing the doorbell */
2596 out_put_tag:
2601 }
2603 /**
2604 * ufshcd_init_query() - init the query response and request parameters
2605 * @hba: per-adapter instance
2606 * @request: address of the request pointer to be initialized
2607 * @response: address of the response pointer to be initialized
2608 * @opcode: operation to perform
2609 * @idn: flag idn to access
2610 * @index: LU number to access
2611 * @selector: query/flag/descriptor further identification
2612 */
2616 {
2625 }
2629 {
2639 else
2641 }
2648 }
2650 /**
2651 * ufshcd_query_flag() - API function for sending flag query requests
2652 * hba: per-adapter instance
2653 * query_opcode: flag query to perform
2654 * idn: flag idn to access
2655 * flag_res: the flag value after the query request completes
2656 *
2657 * Returns 0 for success, non-zero in case of failure
2658 */
2661 {
2672 selector);
2683 /* No dummy reads */
2685 __func__);
2688 }
2696 }
2705 }
2711 out_unlock:
2715 }
2717 /**
2718 * ufshcd_query_attr - API function for sending attribute requests
2719 * hba: per-adapter instance
2720 * opcode: attribute opcode
2721 * idn: attribute idn to access
2722 * index: index field
2723 * selector: selector field
2724 * attr_val: the attribute value after the query request completes
2725 *
2726 * Returns 0 for success, non-zero in case of failure
2727 */
2730 {
2743 }
2747 selector);
2762 }
2770 }
2774 out_unlock:
2776 out:
2779 }
2781 /**
2782 * ufshcd_query_attr_retry() - API function for sending query
2783 * attribute with retries
2784 * @hba: per-adapter instance
2785 * @opcode: attribute opcode
2786 * @idn: attribute idn to access
2787 * @index: index field
2788 * @selector: selector field
2789 * @attr_val: the attribute value after the query request
2790 * completes
2791 *
2792 * Returns 0 for success, non-zero in case of failure
2793 */
2797 {
2799 u32 retries;
2807 else
2809 }
2816 }
2821 {
2834 }
2841 }
2845 selector);
2862 }
2870 }
2875 out_unlock:
2877 out:
2880 }
2882 /**
2883 * ufshcd_query_descriptor_retry - API function for sending descriptor
2884 * requests
2885 * hba: per-adapter instance
2886 * opcode: attribute opcode
2887 * idn: attribute idn to access
2888 * index: index field
2889 * selector: selector field
2890 * desc_buf: the buffer that contains the descriptor
2891 * buf_len: length parameter passed to the device
2892 *
2893 * Returns 0 for success, non-zero in case of failure.
2894 * The buf_len parameter will contain, on return, the length parameter
2895 * received on the response.
2896 */
2900 u8 selector,
2902 {
2911 }
2914 }
2916 /**
2917 * ufshcd_read_desc_length - read the specified descriptor length from header
2918 * @hba: Pointer to adapter instance
2919 * @desc_id: descriptor idn value
2920 * @desc_index: descriptor index
2921 * @desc_length: pointer to variable to read the length of descriptor
2922 *
2923 * Return 0 in case of success, non-zero otherwise
2924 */
2929 {
2948 desc_id);
2950 }
2955 }
2957 /**
2958 * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
2959 * @hba: Pointer to adapter instance
2960 * @desc_id: descriptor idn value
2961 * @desc_len: mapped desc length (out)
2962 *
2963 * Return 0 in case of success, non-zero otherwise
2964 */
2967 {
2997 }
2999 }
3002 /**
3003 * ufshcd_read_desc_param - read the specified descriptor parameter
3004 * @hba: Pointer to adapter instance
3005 * @desc_id: descriptor idn value
3006 * @desc_index: descriptor index
3007 * @param_offset: offset of the parameter to read
3008 * @param_read_buf: pointer to buffer where parameter would be read
3009 * @param_size: sizeof(param_read_buf)
3010 *
3011 * Return 0 in case of success, non-zero otherwise
3012 */
3016 u8 param_offset,
3018 u8 param_size)
3019 {
3025 /* Safety check */
3029 /* Get the max length of descriptor from structure filled up at probe
3030 * time.
3031 */
3034 /* Sanity checks */
3037 __func__);
3039 }
3041 /* Check whether we need temp memory */
3049 }
3051 /* Request for full descriptor */
3057 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3060 }
3062 /* Sanity check */
3068 }
3070 /* Check wherher we will not copy more data, than available */
3076 out:
3080 }
3086 u32 size)
3087 {
3089 }
3093 u32 size)
3094 {
3096 }
3099 {
3101 }
3103 /**
3104 * ufshcd_read_string_desc - read string descriptor
3105 * @hba: pointer to adapter instance
3106 * @desc_index: descriptor index
3107 * @buf: pointer to buffer where descriptor would be read
3108 * @size: size of buf
3109 * @ascii: if true convert from unicode to ascii characters
3110 *
3111 * Return 0 in case of success, non-zero otherwise
3112 */
3113 #define ASCII_STD true
3116 {
3126 }
3135 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3139 __func__);
3142 }
3148 }
3150 /*
3151 * the descriptor contains string in UTF16 format
3152 * we need to convert to utf-8 so it can be displayed
3153 */
3158 /* replace non-printable or non-ASCII characters with spaces */
3167 }
3168 out:
3170 }
3172 /**
3173 * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3174 * @hba: Pointer to adapter instance
3175 * @lun: lun id
3176 * @param_offset: offset of the parameter to read
3177 * @param_read_buf: pointer to buffer where parameter would be read
3178 * @param_size: sizeof(param_read_buf)
3179 *
3180 * Return 0 in case of success, non-zero otherwise
3181 */
3186 u32 param_size)
3187 {
3188 /*
3189 * Unit descriptors are only available for general purpose LUs (LUN id
3190 * from 0 to 7) and RPMB Well known LU.
3191 */
3197 }
3199 /**
3200 * ufshcd_memory_alloc - allocate memory for host memory space data structures
3201 * @hba: per adapter instance
3202 *
3203 * 1. Allocate DMA memory for Command Descriptor array
3204 * Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3205 * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3206 * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3207 * (UTMRDL)
3208 * 4. Allocate memory for local reference block(lrb).
3209 *
3210 * Returns 0 for success, non-zero in case of failure
3211 */
3213 {
3216 /* Allocate memory for UTP command descriptors */
3219 ucdl_size,
3221 GFP_KERNEL);
3223 /*
3224 * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3225 * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3226 * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3227 * be aligned to 128 bytes as well
3228 */
3234 }
3236 /*
3237 * Allocate memory for UTP Transfer descriptors
3238 * UFSHCI requires 1024 byte alignment of UTRD
3239 */
3242 utrdl_size,
3244 GFP_KERNEL);
3250 }
3252 /*
3253 * Allocate memory for UTP Task Management descriptors
3254 * UFSHCI requires 1024 byte alignment of UTMRD
3255 */
3258 utmrdl_size,
3260 GFP_KERNEL);
3266 }
3268 /* Allocate memory for local reference block */
3271 GFP_KERNEL);
3275 }
3277 out:
3279 }
3281 /**
3282 * ufshcd_host_memory_configure - configure local reference block with
3283 * memory offsets
3284 * @hba: per adapter instance
3285 *
3286 * Configure Host memory space
3287 * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3288 * address.
3289 * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3290 * and PRDT offset.
3291 * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3292 * into local reference block.
3293 */
3295 {
3298 dma_addr_t cmd_desc_dma_addr;
3299 dma_addr_t cmd_desc_element_addr;
3300 u16 response_offset;
3301 u16 prdt_offset;
3308 response_offset =
3310 prdt_offset =
3317 /* Configure UTRD with command descriptor base address */
3318 cmd_desc_element_addr =
3325 /* Response upiu and prdt offset should be in double words */
3340 }
3351 response_offset;
3355 prdt_offset;
3356 }
3357 }
3359 /**
3360 * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3361 * @hba: per adapter instance
3362 *
3363 * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3364 * in order to initialize the Unipro link startup procedure.
3365 * Once the Unipro links are up, the device connected to the controller
3366 * is detected.
3367 *
3368 * Returns 0 on success, non-zero value on failure
3369 */
3371 {
3382 }
3385 {
3386 #define MIN_DELAY_BEFORE_DME_CMDS_US 1000
3392 /*
3393 * last_dme_cmd_tstamp will be 0 only for 1st call to
3394 * this function
3395 */
3405 min_sleep_time_us =
3407 else
3409 }
3411 /* allow sleep for extra 50us if needed */
3413 }
3415 /**
3416 * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3417 * @hba: per adapter instance
3418 * @attr_sel: uic command argument1
3419 * @attr_set: attribute set type as uic command argument2
3420 * @mib_val: setting value as uic command argument3
3421 * @peer: indicate whether peer or local
3422 *
3423 * Returns 0 on success, non-zero value on failure
3424 */
3427 {
3431 "dme-peer-set"
3432 };
3444 /* for peer attributes we retry upon failure */
3457 }
3460 /**
3461 * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3462 * @hba: per adapter instance
3463 * @attr_sel: uic command argument1
3464 * @mib_val: the value of the attribute as returned by the UIC command
3465 * @peer: indicate whether peer or local
3466 *
3467 * Returns 0 on success, non-zero value on failure
3468 */
3471 {
3475 "dme-peer-get"
3476 };
3498 }
3503 }
3504 }
3511 /* for peer attributes we retry upon failure */
3529 out:
3531 }
3534 /**
3535 * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3536 * state) and waits for it to take effect.
3537 *
3538 * @hba: per adapter instance
3539 * @cmd: UIC command to execute
3540 *
3541 * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3542 * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3543 * and device UniPro link and hence it's final completion would be indicated by
3544 * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3545 * addition to normal UIC command completion Status (UCCS). This function only
3546 * returns after the relevant status bits indicate the completion.
3547 *
3548 * Returns 0 on success, non-zero value on failure
3549 */
3551 {
3554 u8 status;
3566 /*
3567 * Make sure UIC command completion interrupt is disabled before
3568 * issuing UIC command.
3569 */
3572 }
3580 }
3589 }
3597 }
3598 out:
3603 }
3614 }
3616 /**
3617 * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
3618 * using DME_SET primitives.
3619 * @hba: per adapter instance
3620 * @mode: powr mode value
3621 *
3622 * Returns 0 on success, non-zero value on failure
3623 */
3625 {
3636 }
3637 }
3646 out:
3648 }
3651 {
3673 }
3676 {
3692 /*
3693 * If link recovery fails then return error so that caller
3694 * don't retry the hibern8 enter again.
3695 */
3700 POST_CHANGE);
3703 }
3706 {
3713 }
3714 out:
3716 }
3719 {
3737 POST_CHANGE);
3740 }
3743 }
3745 /**
3746 * ufshcd_init_pwr_info - setting the POR (power on reset)
3747 * values in hba power info
3748 * @hba: per-adapter instance
3749 */
3751 {
3759 }
3761 /**
3762 * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
3763 * @hba: per-adapter instance
3764 */
3766 {
3776 /* Get the connected lane count */
3784 __func__,
3788 }
3790 /*
3791 * First, get the maximum gears of HS speed.
3792 * If a zero value, it means there is no HSGEAR capability.
3793 * Then, get the maximum gears of PWM speed.
3794 */
3803 }
3805 }
3816 }
3818 }
3822 }
3826 {
3829 /* if already configured to the requested pwr_mode */
3839 }
3841 /*
3842 * Configure attributes for power mode change with below.
3843 * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
3844 * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
3845 * - PA_HSSERIES
3846 */
3853 else
3862 else
3880 pwr_mode);
3884 }
3887 }
3889 /**
3890 * ufshcd_config_pwr_mode - configure a new power mode
3891 * @hba: per-adapter instance
3892 * @desired_pwr_mode: desired power configuration
3893 */
3896 {
3911 }
3913 /**
3914 * ufshcd_complete_dev_init() - checks device readiness
3915 * hba: per-adapter instance
3916 *
3917 * Set fDeviceInit flag and poll until device toggles it.
3918 */
3920 {
3932 }
3934 /* poll for max. 1000 iterations for fDeviceInit flag to clear */
3946 __func__);
3948 out:
3950 }
3952 /**
3953 * ufshcd_make_hba_operational - Make UFS controller operational
3954 * @hba: per adapter instance
3955 *
3956 * To bring UFS host controller to operational state,
3957 * 1. Enable required interrupts
3958 * 2. Configure interrupt aggregation
3959 * 3. Program UTRL and UTMRL base address
3960 * 4. Configure run-stop-registers
3961 *
3962 * Returns 0 on success, non-zero value on failure
3963 */
3965 {
3967 u32 reg;
3969 /* Enable required interrupts */
3972 /* Configure interrupt aggregation */
3975 else
3978 /* Configure UTRL and UTMRL base address registers */
3980 REG_UTP_TRANSFER_REQ_LIST_BASE_L);
3982 REG_UTP_TRANSFER_REQ_LIST_BASE_H);
3984 REG_UTP_TASK_REQ_LIST_BASE_L);
3986 REG_UTP_TASK_REQ_LIST_BASE_H);
3988 /*
3989 * Make sure base address and interrupt setup are updated before
3990 * enabling the run/stop registers below.
3991 */
3994 /*
3995 * UCRDY, UTMRLDY and UTRLRDY bits must be 1
3996 */
4005 }
4007 out:
4009 }
4011 /**
4012 * ufshcd_hba_stop - Send controller to reset state
4013 * @hba: per adapter instance
4014 * @can_sleep: perform sleep or just spin
4015 */
4017 {
4026 }
4028 /**
4029 * ufshcd_hba_enable - initialize the controller
4030 * @hba: per adapter instance
4031 *
4032 * The controller resets itself and controller firmware initialization
4033 * sequence kicks off. When controller is ready it will set
4034 * the Host Controller Enable bit to 1.
4035 *
4036 * Returns 0 on success, non-zero value on failure
4037 */
4039 {
4042 /*
4043 * msleep of 1 and 5 used in this function might result in msleep(20),
4044 * but it was necessary to send the UFS FPGA to reset mode during
4045 * development and testing of this driver. msleep can be changed to
4046 * mdelay and retry count can be reduced based on the controller.
4047 */
4049 /* change controller state to "reset state" */
4052 /* UniPro link is disabled at this point */
4057 /* start controller initialization sequence */
4060 /*
4061 * To initialize a UFS host controller HCE bit must be set to 1.
4062 * During initialization the HCE bit value changes from 1->0->1.
4063 * When the host controller completes initialization sequence
4064 * it sets the value of HCE bit to 1. The same HCE bit is read back
4065 * to check if the controller has completed initialization sequence.
4066 * So without this delay the value HCE = 1, set in the previous
4067 * instruction might be read back.
4068 * This delay can be changed based on the controller.
4069 */
4072 /* wait for the host controller to complete initialization */
4076 retry--;
4081 }
4083 }
4085 /* enable UIC related interrupts */
4091 }
4094 {
4100 else
4109 else
4118 }
4119 }
4122 }
4125 {
4127 }
4129 /**
4130 * ufshcd_link_startup - Initialize unipro link startup
4131 * @hba: per adapter instance
4132 *
4133 * Returns 0 for success, non-zero in case of failure
4134 */
4136 {
4141 /*
4142 * If UFS device isn't active then we will have to issue link startup
4143 * 2 times to make sure the device state move to active.
4144 */
4148 link_startup:
4154 /* check if device is detected by inter-connect layer */
4159 }
4161 /*
4162 * DME link lost indication is only received when link is up,
4163 * but we can't be sure if the link is up until link startup
4164 * succeeds. So reset the local Uni-Pro and try again.
4165 */
4171 /* failed to get the link up... retire */
4178 }
4180 /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4188 }
4190 /* Include any host controller configuration via UIC commands */
4196 out:
4202 }
4204 }
4206 /**
4207 * ufshcd_verify_dev_init() - Verify device initialization
4208 * @hba: per-adapter instance
4209 *
4210 * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4211 * device Transport Protocol (UTP) layer is ready after a reset.
4212 * If the UTP layer at the device side is not initialized, it may
4213 * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4214 * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4215 */
4217 {
4225 NOP_OUT_TIMEOUT);
4231 }
4238 }
4240 /**
4241 * ufshcd_set_queue_depth - set lun queue depth
4242 * @sdev: pointer to SCSI device
4243 *
4244 * Read bLUQueueDepth value and activate scsi tagged command
4245 * queueing. For WLUN, queue depth is set to 1. For best-effort
4246 * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4247 * value that host can queue.
4248 */
4250 {
4252 u8 lun_qdepth;
4260 UNIT_DESC_PARAM_LU_Q_DEPTH,
4264 /* Some WLUN doesn't support unit descriptor */
4268 /* eventually, we can figure out the real queue depth */
4270 else
4276 }
4278 /*
4279 * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4280 * @hba: per-adapter instance
4281 * @lun: UFS device lun id
4282 * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4283 *
4284 * Returns 0 in case of success and b_lu_write_protect status would be returned
4285 * @b_lu_write_protect parameter.
4286 * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4287 * Returns -EINVAL in case of invalid parameters passed to this function.
4288 */
4290 u8 lun,
4292 {
4297 /*
4298 * According to UFS device spec, RPMB LU can't be write
4299 * protected so skip reading bLUWriteProtect parameter for
4300 * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4301 */
4304 else
4306 lun,
4307 UNIT_DESC_PARAM_LU_WR_PROTECT,
4308 b_lu_write_protect,
4311 }
4313 /**
4314 * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4315 * status
4316 * @hba: per-adapter instance
4317 * @sdev: pointer to SCSI device
4318 *
4319 */
4322 {
4325 u8 b_lu_write_protect;
4331 }
4332 }
4334 /**
4335 * ufshcd_slave_alloc - handle initial SCSI device configurations
4336 * @sdev: pointer to SCSI device
4337 *
4338 * Returns success
4339 */
4341 {
4346 /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4349 /* allow SCSI layer to restart the device in case of errors */
4352 /* REPORT SUPPORTED OPERATION CODES is not supported */
4355 /* WRITE_SAME command is not supported */
4363 }
4365 /**
4366 * ufshcd_change_queue_depth - change queue depth
4367 * @sdev: pointer to SCSI device
4368 * @depth: required depth to set
4369 *
4370 * Change queue depth and make sure the max. limits are not crossed.
4371 */
4373 {
4379 }
4381 /**
4382 * ufshcd_slave_configure - adjust SCSI device configurations
4383 * @sdev: pointer to SCSI device
4384 */
4386 {
4393 }
4395 /**
4396 * ufshcd_slave_destroy - remove SCSI device configurations
4397 * @sdev: pointer to SCSI device
4398 */
4400 {
4404 /* Drop the reference as it won't be needed anymore */
4411 }
4412 }
4414 /**
4415 * ufshcd_task_req_compl - handle task management request completion
4416 * @hba: per adapter instance
4417 * @index: index of the completed request
4418 * @resp: task management service response
4419 *
4420 * Returns non-zero value on error, zero on success
4421 */
4423 {
4432 /* Clear completed tasks from outstanding_tasks */
4448 }
4452 }
4454 /**
4455 * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
4456 * @lrb: pointer to local reference block of completed command
4457 * @scsi_status: SCSI command status
4458 *
4459 * Returns value base on SCSI command status
4460 */
4463 {
4472 scsi_status;
4486 }
4488 /**
4489 * ufshcd_transfer_rsp_status - Get overall status of the response
4490 * @hba: per adapter instance
4491 * @lrb: pointer to local reference block of completed command
4492 *
4493 * Returns result of the command to notify SCSI midlayer
4494 */
4497 {
4502 /* overall command status of utrd */
4511 /*
4512 * get the response UPIU result to extract
4513 * the SCSI command status
4514 */
4517 /*
4518 * get the result based on SCSI status response
4519 * to notify the SCSI midlayer of the command status
4520 */
4524 /*
4525 * Currently we are only supporting BKOPs exception
4526 * events hence we can ignore BKOPs exception event
4527 * during power management callbacks. BKOPs exception
4528 * event is not expected to be raised in runtime suspend
4529 * callback as it allows the urgent bkops.
4530 * During system suspend, we are anyway forcefully
4531 * disabling the bkops and if urgent bkops is needed
4532 * it will be enabled on system resume. Long term
4533 * solution could be to abort the system suspend if
4534 * UFS device needs urgent BKOPs.
4535 */
4541 /* TODO: handle Reject UPIU Response */
4550 result);
4552 }
4579 }
4581 /**
4582 * ufshcd_uic_cmd_compl - handle completion of uic command
4583 * @hba: per adapter instance
4584 * @intr_status: interrupt status generated by the controller
4585 */
4587 {
4594 }
4598 }
4600 /**
4601 * __ufshcd_transfer_req_compl - handle SCSI and query command completion
4602 * @hba: per adapter instance
4603 * @completed_reqs: requests to complete
4604 */
4607 {
4621 /* Mark completed command as NULL in LRB */
4624 /* Do not touch lrbp after scsi done */
4633 }
4634 }
4639 }
4641 /* clear corresponding bits of completed commands */
4646 /* we might have free'd some tags above */
4648 }
4650 /**
4651 * ufshcd_transfer_req_compl - handle SCSI and query command completion
4652 * @hba: per adapter instance
4653 */
4655 {
4657 u32 tr_doorbell;
4659 /* Resetting interrupt aggregation counters first and reading the
4660 * DOOR_BELL afterward allows us to handle all the completed requests.
4661 * In order to prevent other interrupts starvation the DB is read once
4662 * after reset. The down side of this solution is the possibility of
4663 * false interrupt if device completes another request after resetting
4664 * aggregation and before reading the DB.
4665 */
4673 }
4675 /**
4676 * ufshcd_disable_ee - disable exception event
4677 * @hba: per-adapter instance
4678 * @mask: exception event to disable
4679 *
4680 * Disables exception event in the device so that the EVENT_ALERT
4681 * bit is not set.
4682 *
4683 * Returns zero on success, non-zero error value on failure.
4684 */
4686 {
4688 u32 val;
4699 out:
4701 }
4703 /**
4704 * ufshcd_enable_ee - enable exception event
4705 * @hba: per-adapter instance
4706 * @mask: exception event to enable
4707 *
4708 * Enable corresponding exception event in the device to allow
4709 * device to alert host in critical scenarios.
4710 *
4711 * Returns zero on success, non-zero error value on failure.
4712 */
4714 {
4716 u32 val;
4727 out:
4729 }
4731 /**
4732 * ufshcd_enable_auto_bkops - Allow device managed BKOPS
4733 * @hba: per-adapter instance
4734 *
4735 * Allow device to manage background operations on its own. Enabling
4736 * this might lead to inconsistent latencies during normal data transfers
4737 * as the device is allowed to manage its own way of handling background
4738 * operations.
4739 *
4740 * Returns zero on success, non-zero on failure.
4741 */
4743 {
4755 }
4760 /* No need of URGENT_BKOPS exception from the device */
4765 out:
4767 }
4769 /**
4770 * ufshcd_disable_auto_bkops - block device in doing background operations
4771 * @hba: per-adapter instance
4772 *
4773 * Disabling background operations improves command response latency but
4774 * has drawback of device moving into critical state where the device is
4775 * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
4776 * host is idle so that BKOPS are managed effectively without any negative
4777 * impacts.
4778 *
4779 * Returns zero on success, non-zero on failure.
4780 */
4782 {
4788 /*
4789 * If host assisted BKOPs is to be enabled, make sure
4790 * urgent bkops exception is allowed.
4791 */
4797 }
4806 }
4810 out:
4812 }
4814 /**
4815 * ufshcd_force_reset_auto_bkops - force reset auto bkops state
4816 * @hba: per adapter instance
4817 *
4818 * After a device reset the device may toggle the BKOPS_EN flag
4819 * to default value. The s/w tracking variables should be updated
4820 * as well. This function would change the auto-bkops state based on
4821 * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
4822 */
4824 {
4833 }
4834 }
4837 {
4840 }
4842 /**
4843 * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
4844 * @hba: per-adapter instance
4845 * @status: bkops_status value
4846 *
4847 * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
4848 * flag in the device to permit background operations if the device
4849 * bkops_status is greater than or equal to "status" argument passed to
4850 * this function, disable otherwise.
4851 *
4852 * Returns 0 for success, non-zero in case of failure.
4853 *
4854 * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
4855 * to know whether auto bkops is enabled or disabled after this function
4856 * returns control to it.
4857 */
4860 {
4874 }
4878 else
4880 out:
4882 }
4884 /**
4885 * ufshcd_urgent_bkops - handle urgent bkops exception event
4886 * @hba: per-adapter instance
4887 *
4888 * Enable fBackgroundOpsEn flag in the device to permit background
4889 * operations.
4890 *
4891 * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
4892 * and negative error value for any other failure.
4893 */
4895 {
4897 }
4900 {
4903 }
4906 {
4918 }
4920 /*
4921 * We are seeing that some devices are raising the urgent bkops
4922 * exception events even when BKOPS status doesn't indicate performace
4923 * impacted or critical. Handle these device by determining their urgent
4924 * bkops status at runtime.
4925 */
4929 /* update the current status as the urgent bkops level */
4932 }
4934 enable_auto_bkops:
4936 out:
4940 }
4942 /**
4943 * ufshcd_exception_event_handler - handle exceptions raised by device
4944 * @work: pointer to work data
4945 *
4946 * Read bExceptionEventStatus attribute from the device and handle the
4947 * exception event accordingly.
4948 */
4950 {
4962 }
4969 out:
4972 }
4974 /* Complete requests that have door-bell cleared */
4976 {
4979 }
4981 /**
4982 * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
4983 * to recover from the DL NAC errors or not.
4984 * @hba: per-adapter instance
4985 *
4986 * Returns true if error handling is required, false otherwise
4987 */
4989 {
4994 /*
4995 * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
4996 * device fatal error and/or DL NAC & REPLAY timeout errors.
4997 */
5009 /*
5010 * wait for 50ms to see if we can get any other errors or not.
5011 */
5016 /*
5017 * now check if we have got any other severe errors other than
5018 * DL NAC error?
5019 */
5025 /*
5026 * As DL NAC is the only error received so far, send out NOP
5027 * command to confirm if link is still active or not.
5028 * - If we don't get any response then do error recovery.
5029 * - If we get response then clear the DL NAC error bit.
5030 */
5039 /* Link seems to be alive hence ignore the DL NAC errors */
5042 /* clear NAC error */
5047 }
5048 }
5049 out:
5052 }
5054 /**
5055 * ufshcd_err_handler - handle UFS errors that require s/w attention
5056 * @work: pointer to work structure
5057 */
5059 {
5080 /* Complete requests that have door-bell cleared by h/w */
5087 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
5092 }
5096 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5097 UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
5100 /*
5101 * if host reset is required then skip clearing the pending
5102 * transfers forcefully because they will automatically get
5103 * cleared after link startup.
5104 */
5108 /* release lock as clear command might sleep */
5110 /* Clear pending transfer requests */
5115 }
5116 }
5118 /* Clear pending task management requests */
5123 }
5124 }
5126 lock_skip_pending_xfer_clear:
5129 /* Complete the requests that are cleared by s/w */
5135 skip_pending_xfer_clear:
5136 /* Fatal errors need reset */
5140 /*
5141 * ufshcd_reset_and_restore() does the link reinitialization
5142 * which will need atleast one empty doorbell slot to send the
5143 * device management commands (NOP and query commands).
5144 * If there is no slot empty at this moment then free up last
5145 * slot forcefully.
5146 */
5156 __func__);
5158 }
5159 /*
5160 * Inform scsi mid-layer that we did reset and allow to handle
5161 * Unit Attention properly.
5162 */
5166 }
5168 skip_err_handling:
5174 }
5178 out:
5183 }
5186 u32 reg)
5187 {
5191 }
5193 /**
5194 * ufshcd_update_uic_error - check and set fatal UIC error flags.
5195 * @hba: per-adapter instance
5196 */
5198 {
5199 u32 reg;
5201 /* PHY layer lane error */
5203 /* Ignore LINERESET indication, as this is not an error */
5206 /*
5207 * To know whether this error is fatal or not, DB timeout
5208 * must be checked but this error is handled separately.
5209 */
5212 }
5214 /* PA_INIT_ERROR is fatal and needs UIC reset */
5222 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5225 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5228 }
5230 /* UIC NL/TL/DME errors needs software retry */
5235 }
5241 }
5247 }
5251 }
5253 /**
5254 * ufshcd_check_errors - Check for errors that need s/w attention
5255 * @hba: per-adapter instance
5256 */
5258 {
5269 }
5272 /*
5273 * update the transfer error masks to sticky bits, let's do this
5274 * irrespective of current ufshcd_state.
5275 */
5279 /* handle fatal errors only when link is functional */
5281 /* block commands from scsi mid-layer */
5286 /* dump controller state before resetting */
5289 SYSTEM_BUS_FATAL_ERROR);
5299 pr_prdt);
5300 }
5302 }
5303 }
5304 /*
5305 * if (!queue_eh_work) -
5306 * Other errors are either non-fatal where host recovers
5307 * itself without s/w intervention or errors that will be
5308 * handled by the SCSI core layer.
5309 */
5310 }
5312 /**
5313 * ufshcd_tmc_handler - handle task management function completion
5314 * @hba: per adapter instance
5315 */
5317 {
5318 u32 tm_doorbell;
5323 }
5325 /**
5326 * ufshcd_sl_intr - Interrupt service routine
5327 * @hba: per adapter instance
5328 * @intr_status: contains interrupts generated by the controller
5329 */
5331 {
5344 }
5346 /**
5347 * ufshcd_intr - Main interrupt service routine
5348 * @irq: irq number
5349 * @__hba: pointer to adapter instance
5350 *
5351 * Returns IRQ_HANDLED - If interrupt is valid
5352 * IRQ_NONE - If invalid interrupt
5353 */
5355 {
5362 enabled_intr_status =
5371 }
5374 }
5377 {
5389 /* poll for max. 1 sec to clear door bell register by h/w */
5391 REG_UTP_TASK_REQ_DOOR_BELL,
5393 out:
5395 }
5397 /**
5398 * ufshcd_issue_tm_cmd - issues task management commands to controller
5399 * @hba: per adapter instance
5400 * @lun_id: LUN ID to which TM command is sent
5401 * @task_id: task ID to which the TM command is applicable
5402 * @tm_function: task management function opcode
5403 * @tm_response: task management service response return value
5404 *
5405 * Returns non-zero value on error, zero on success.
5406 */
5409 {
5420 /*
5421 * Get free slot, sleep if slots are unavailable.
5422 * Even though we use wait_event() which sleeps indefinitely,
5423 * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
5424 */
5432 /* Configure task request descriptor */
5437 /* Configure task request UPIU */
5438 task_req_upiup =
5446 /*
5447 * The host shall provide the same value for LUN field in the basic
5448 * header and for Input Parameter.
5449 */
5455 /* send command to the controller */
5458 /* Make sure descriptors are ready before ringing the task doorbell */
5462 /* Make sure that doorbell is committed immediately */
5467 /* wait until the task management command is completed */
5480 }
5488 }
5490 /**
5491 * ufshcd_eh_device_reset_handler - device reset handler registered to
5492 * scsi layer.
5493 * @cmd: SCSI command pointer
5494 *
5495 * Returns SUCCESS/FAILED
5496 */
5498 {
5502 u32 pos;
5518 }
5520 /* clear the commands that were pending for corresponding LUN */
5526 }
5527 }
5532 out:
5539 }
5541 }
5544 {
5551 }
5552 }
5554 /**
5555 * ufshcd_abort - abort a specific command
5556 * @cmd: SCSI command pointer
5557 *
5558 * Abort the pending command in device by sending UFS_ABORT_TASK task management
5559 * command, and in host controller by clearing the door-bell register. There can
5560 * be race between controller sending the command to the device while abort is
5561 * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
5562 * really issued and then try to abort it.
5563 *
5564 * Returns SUCCESS/FAILED
5565 */
5567 {
5576 u32 reg;
5587 }
5589 /*
5590 * Task abort to the device W-LUN is illegal. When this command
5591 * will fail, due to spec violation, scsi err handling next step
5592 * will be to send LU reset which, again, is a spec violation.
5593 * To avoid these unnecessary/illegal step we skip to the last error
5594 * handling stage: reset and restore.
5595 */
5601 /* If command is already aborted/completed, return SUCCESS */
5607 }
5613 }
5615 /* Print Transfer Request of aborted task */
5618 /*
5619 * Print detailed info about aborted request.
5620 * As more than one request might get aborted at the same time,
5621 * print full information only for the first aborted request in order
5622 * to reduce repeated printouts. For other aborted requests only print
5623 * basic details.
5624 */
5633 }
5636 /* Skip task abort in case previous aborts failed and report failure */
5640 }
5646 /* cmd pending in the device */
5651 /*
5652 * cmd not pending in the device, check if it is
5653 * in transition.
5654 */
5659 /* sleep for max. 200us to stabilize */
5662 }
5663 /* command completed already */
5674 }
5675 }
5680 }
5689 }
5691 }
5698 }
5710 out:
5717 }
5719 /*
5720 * This ufshcd_release() corresponds to the original scsi cmd that got
5721 * aborted here (as we won't get any IRQ for it).
5722 */
5725 }
5727 /**
5728 * ufshcd_host_reset_and_restore - reset and restore host controller
5729 * @hba: per-adapter instance
5730 *
5731 * Note that host controller reset may issue DME_RESET to
5732 * local and remote (device) Uni-Pro stack and the attributes
5733 * are reset to default state.
5734 *
5735 * Returns zero on success, non-zero on failure
5736 */
5738 {
5742 /* Reset the host controller */
5747 /* scale up clocks to max frequency before full reinitialization */
5754 /* Establish the link again and restore the device */
5759 out:
5764 }
5766 /**
5767 * ufshcd_reset_and_restore - reset and re-initialize host/device
5768 * @hba: per-adapter instance
5769 *
5770 * Reset and recover device, host and re-establish link. This
5771 * is helpful to recover the communication in fatal error conditions.
5772 *
5773 * Returns zero on success, non-zero on failure
5774 */
5776 {
5785 /*
5786 * After reset the door-bell might be cleared, complete
5787 * outstanding requests in s/w here.
5788 */
5795 }
5797 /**
5798 * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
5799 * @cmd - SCSI command pointer
5800 *
5801 * Returns SUCCESS/FAILED
5802 */
5804 {
5812 /*
5813 * Check if there is any race with fatal error handling.
5814 * If so, wait for it to complete. Even though fatal error
5815 * handling does reset and restore in some cases, don't assume
5816 * anything out of it. We are just avoiding race here.
5817 */
5842 }
5848 }
5850 /**
5851 * ufshcd_get_max_icc_level - calculate the ICC level
5852 * @sup_curr_uA: max. current supported by the regulator
5853 * @start_scan: row at the desc table to start scan from
5854 * @buff: power descriptor buffer
5855 *
5856 * Returns calculated max ICC level for specific regulator
5857 */
5859 {
5862 u16 data;
5863 u16 unit;
5868 ATTR_ICC_LVL_UNIT_OFFSET;
5883 }
5886 }
5890 }
5893 }
5895 /**
5896 * ufshcd_calc_icc_level - calculate the max ICC level
5897 * In case regulators are not initialized we'll return 0
5898 * @hba: per-adapter instance
5899 * @desc_buf: power descriptor buffer to extract ICC levels from.
5900 * @len: length of desc_buff
5901 *
5902 * Returns calculated ICC level
5903 */
5906 {
5915 }
5926 icc_level,
5932 icc_level,
5934 out:
5936 }
5939 {
5950 }
5967 }
5969 /**
5970 * ufshcd_scsi_add_wlus - Adds required W-LUs
5971 * @hba: per-adapter instance
5972 *
5973 * UFS device specification requires the UFS devices to support 4 well known
5974 * logical units:
5975 * "REPORT_LUNS" (address: 01h)
5976 * "UFS Device" (address: 50h)
5977 * "RPMB" (address: 44h)
5978 * "BOOT" (address: 30h)
5979 * UFS device's power management needs to be controlled by "POWER CONDITION"
5980 * field of SSU (START STOP UNIT) command. But this "power condition" field
5981 * will take effect only when its sent to "UFS device" well known logical unit
5982 * hence we require the scsi_device instance to represent this logical unit in
5983 * order for the UFS host driver to send the SSU command for power management.
5985 * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
5986 * Block) LU so user space process can control this LU. User space may also
5987 * want to have access to BOOT LU.
5989 * This function adds scsi device instances for each of all well known LUs
5990 * (except "REPORT LUNS" LU).
5991 *
5992 * Returns zero on success (all required W-LUs are added successfully),
5993 * non-zero error value on failure (if failed to add any of the required W-LU).
5994 */
5996 {
6007 }
6015 }
6022 else
6026 remove_sdev_ufs_device:
6028 out:
6030 }
6034 {
6036 u8 model_index;
6045 }
6047 /*
6048 * getting vendor (manufacturerID) and Bank Index in big endian
6049 * format
6050 */
6062 }
6067 MAX_MODEL_LEN));
6069 /* Null terminate the model string */
6072 out:
6074 }
6078 {
6087 }
6088 }
6090 /**
6091 * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
6092 * @hba: per-adapter instance
6093 *
6094 * PA_TActivate parameter can be tuned manually if UniPro version is less than
6095 * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
6096 * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
6097 * the hibern8 exit latency.
6098 *
6099 * Returns zero on success, non-zero error value on failure.
6100 */
6102 {
6108 RX_MIN_ACTIVATETIME_CAPABILITY,
6114 /* make sure proper unit conversion is applied */
6115 tuned_pa_tactivate =
6119 tuned_pa_tactivate);
6121 out:
6123 }
6125 /**
6126 * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
6127 * @hba: per-adapter instance
6128 *
6129 * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
6130 * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
6131 * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
6132 * This optimal value can help reduce the hibern8 exit latency.
6133 *
6134 * Returns zero on success, non-zero error value on failure.
6135 */
6137 {
6157 peer_rx_hibern8_time_cap);
6158 /* make sure proper unit conversion is applied */
6162 tuned_pa_hibern8time);
6163 out:
6165 }
6167 /**
6168 * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
6169 * less than device PA_TACTIVATE time.
6170 * @hba: per-adapter instance
6171 *
6172 * Some UFS devices require host PA_TACTIVATE to be lower than device
6173 * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
6174 * for such devices.
6175 *
6176 * Returns zero on success, non-zero error value on failure.
6177 */
6179 {
6201 }
6208 }
6224 u32 new_peer_pa_tactivate;
6228 new_peer_pa_tactivate++;
6230 new_peer_pa_tactivate);
6231 }
6233 out:
6235 }
6238 {
6242 }
6245 /* set 1ms timeout for PA_TACTIVATE */
6252 }
6255 {
6268 }
6271 {
6303 }
6306 {
6313 }
6315 /**
6316 * ufshcd_probe_hba - probe hba to detect device and initialize
6317 * @hba: per-adapter instance
6318 *
6319 * Execute link-startup and verify device initialization
6320 */
6322 {
6331 /* set the default level for urgent bkops */
6335 /* Debug counters initialization */
6338 /* UniPro link is active now */
6349 /* Init check for device descriptor sizes */
6357 }
6367 /* UFS device is also active now */
6375 __func__);
6382 }
6383 }
6385 /* set the state as operational after switching to desired gear */
6388 /*
6389 * If we are in error handling context or in power management callbacks
6390 * context, no need to scan the host
6391 */
6395 /* clear any previous UFS device information */
6404 /* Add required well known logical units to scsi mid layer */
6408 /* Initialize devfreq after UFS device is detected */
6418 NULL);
6422 ret);
6424 }
6425 }
6427 }
6431 }
6436 out:
6437 /*
6438 * If we failed to initialize the device or the device is not
6439 * present, turn off the power/clocks etc.
6440 */
6444 }
6450 }
6452 /**
6453 * ufshcd_async_scan - asynchronous execution for probing hba
6454 * @data: data pointer to pass to this function
6455 * @cookie: cookie data
6456 */
6458 {
6462 }
6465 {
6486 }
6487 }
6491 /*
6492 * Bypass SCSI error handling and reset the block layer timer if this
6493 * SCSI command was not actually dispatched to UFS driver, otherwise
6494 * let SCSI layer handle the error as usual.
6495 */
6497 }
6518 };
6522 {
6532 }
6535 }
6539 {
6544 else
6546 UFS_VREG_LPM_LOAD_UA);
6547 }
6551 {
6556 else
6558 }
6562 {
6580 }
6586 }
6587 out:
6589 }
6592 {
6606 else
6609 out:
6611 }
6614 {
6625 /* ignore errors on applying disable config */
6631 }
6632 out:
6634 }
6637 {
6657 out:
6662 }
6664 }
6667 {
6674 }
6677 {
6688 }
6689 out:
6691 }
6694 {
6711 out:
6713 }
6716 {
6723 }
6726 {
6736 /* shut off the rail here */
6738 /*
6739 * Mark this rail as no longer used, so it doesn't get enabled
6740 * later by mistake
6741 */
6745 /*
6746 * rail should have been already enabled hence just make sure
6747 * that unused flag is cleared.
6748 */
6750 }
6751 out:
6753 }
6757 {
6784 }
6787 }
6791 }
6792 }
6798 out:
6803 }
6810 }
6817 }
6820 {
6822 }
6825 {
6844 }
6853 }
6855 }
6858 }
6859 out:
6861 }
6864 {
6880 out_exit:
6882 out:
6887 }
6890 {
6897 }
6900 {
6903 /*
6904 * Handle host controller power separately from the UFS device power
6905 * rails as it will help controlling the UFS host controller power
6906 * collapse easily which is different than UFS device power collapse.
6907 * Also, enable the host controller power before we go ahead with rest
6908 * of the initialization here.
6909 */
6941 out_disable_vreg:
6943 out_disable_clks:
6945 out_disable_hba_vreg:
6947 out:
6949 }
6952 {
6961 }
6965 }
6966 }
6968 static int
6970 {
6975 UFSHCD_REQ_SENSE_SIZE,
6984 }
6993 out:
6995 }
6997 /**
6998 * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
6999 * power mode
7000 * @hba: per adapter instance
7001 * @pwr_mode: device power mode to set
7002 *
7003 * Returns 0 if requested power mode is set successfully
7004 * Returns non-zero if failed to set the requested power mode
7005 */
7008 {
7022 }
7025 }
7031 /*
7032 * If scsi commands fail, the scsi mid-layer schedules scsi error-
7033 * handling, which would wait for host to be resumed. Since we know
7034 * we are functional while we are here, skip host resume in error
7035 * handling context.
7036 */
7042 /* Unit attention condition is cleared now */
7044 }
7048 /*
7049 * Current function would be generally called from the power management
7050 * callbacks hence set the RQF_PM flag so that it doesn't resume the
7051 * already suspended childs.
7052 */
7061 }
7065 out:
7069 }
7074 {
7084 else
7086 }
7087 /*
7088 * If autobkops is enabled, link can't be turned off because
7089 * turning off the link would also turn off the device.
7090 */
7094 /*
7095 * Let's make sure that link is in low power mode, we are doing
7096 * this currently by putting the link in Hibern8. Otherway to
7097 * put the link in low power mode is to send the DME end point
7098 * to device and then send the DME reset command to local
7099 * unipro. But putting the link in hibern8 is much faster.
7100 */
7104 /*
7105 * Change controller state to "reset state" which
7106 * should also put the link in off/reset state
7107 */
7109 /*
7110 * TODO: Check if we need any delay to make sure that
7111 * controller is reset
7112 */
7114 }
7116 out:
7118 }
7121 {
7122 /*
7123 * It seems some UFS devices may keep drawing more than sleep current
7124 * (atleast for 500us) from UFS rails (especially from VCCQ rail).
7125 * To avoid this situation, add 2ms delay before putting these UFS
7126 * rails in LPM mode.
7127 */
7132 /*
7133 * If UFS device is either in UFS_Sleep turn off VCC rail to save some
7134 * power.
7135 *
7136 * If UFS device and link is in OFF state, all power supplies (VCC,
7137 * VCCQ, VCCQ2) can be turned off if power on write protect is not
7138 * required. If UFS link is inactive (Hibern8 or OFF state) and device
7139 * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
7140 *
7141 * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
7142 * in low power state which would save some power.
7143 */
7152 }
7153 }
7154 }
7157 {
7171 }
7173 }
7176 vccq_lpm:
7178 vcc_disable:
7180 out:
7182 }
7185 {
7188 }
7191 {
7194 }
7196 /**
7197 * ufshcd_suspend - helper function for suspend operations
7198 * @hba: per adapter instance
7199 * @pm_op: desired low power operation type
7200 *
7201 * This function will try to put the UFS device and link into low power
7202 * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
7203 * (System PM level).
7204 *
7205 * If this function is called during shutdown, it will make sure that
7206 * both UFS device and UFS link is powered off.
7207 *
7208 * NOTE: UFS device & link must be active before we enter in this function.
7209 *
7210 * Returns 0 for success and non-zero for failure
7211 */
7213 {
7228 }
7230 /*
7231 * If we can't transition into any of the low power modes
7232 * just gate the clocks.
7233 */
7241 }
7246 }
7252 /* UFS device & link must be active before we enter in this function */
7256 }
7260 /*
7261 * The device is idle with no requests in the queue,
7262 * allow background operations if bkops status shows
7263 * that performance might be impacted.
7264 */
7269 /* make sure that auto bkops is disabled */
7271 }
7272 }
7277 /* ensure that bkops is disabled */
7282 }
7290 disable_clks:
7291 /*
7292 * Call vendor specific suspend callback. As these callbacks may access
7293 * vendor specific host controller register space call them before the
7294 * host clocks are ON.
7295 */
7302 else
7303 /* If link is active, device ref_clk can't be switched off */
7308 /*
7309 * Disable the host irq as host controller as there won't be any
7310 * host controller transaction expected till resume.
7311 */
7313 /* Put the host controller in low power mode if possible */
7317 set_link_active:
7325 set_dev_active:
7328 enable_gating:
7333 out:
7336 }
7338 /**
7339 * ufshcd_resume - helper function for resume operations
7340 * @hba: per adapter instance
7341 * @pm_op: runtime PM or system PM
7342 *
7343 * This function basically brings the UFS device, UniPro link and controller
7344 * to active state.
7345 *
7346 * Returns 0 for success and non-zero for failure
7347 */
7349 {
7357 /* Make sure clocks are enabled before accessing controller */
7362 /* enable the host irq as host controller would be active soon */
7371 /*
7372 * Call vendor specific resume callback. As these callbacks may access
7373 * vendor specific host controller register space call them when the
7374 * host clocks are ON.
7375 */
7384 else
7388 /*
7389 * ufshcd_host_reset_and_restore() should have already
7390 * set the link state as active
7391 */
7394 }
7400 }
7404 else
7405 /*
7406 * If BKOPs operations are urgently needed at this moment then
7407 * keep auto-bkops enabled or else disable it.
7408 */
7416 /* Schedule clock gating in case of no access to UFS device yet */
7420 set_old_link_state:
7422 vendor_suspend:
7424 disable_vreg:
7426 disable_irq_and_vops_clks:
7431 out:
7434 }
7436 /**
7437 * ufshcd_system_suspend - system suspend routine
7438 * @hba: per adapter instance
7439 * @pm_op: runtime PM or system PM
7440 *
7441 * Check the description of ufshcd_suspend() function for more details.
7442 *
7443 * Returns 0 for success and non-zero for failure
7444 */
7446 {
7460 /*
7461 * UFS device and/or UFS link low power states during runtime
7462 * suspend seems to be different than what is expected during
7463 * system suspend. Hence runtime resume the devic & link and
7464 * let the system suspend low power states to take effect.
7465 * TODO: If resume takes longer time, we might have optimize
7466 * it in future by not resuming everything if possible.
7467 */
7471 }
7474 out:
7481 }
7484 /**
7485 * ufshcd_system_resume - system resume routine
7486 * @hba: per adapter instance
7487 *
7488 * Returns 0 for success and non-zero for failure
7489 */
7492 {
7500 /*
7501 * Let the runtime resume take care of resuming
7502 * if runtime suspended.
7503 */
7505 else
7507 out:
7512 }
7515 /**
7516 * ufshcd_runtime_suspend - runtime suspend routine
7517 * @hba: per adapter instance
7518 *
7519 * Check the description of ufshcd_suspend() function for more details.
7520 *
7521 * Returns 0 for success and non-zero for failure
7522 */
7524 {
7533 else
7535 out:
7540 }
7543 /**
7544 * ufshcd_runtime_resume - runtime resume routine
7545 * @hba: per adapter instance
7546 *
7547 * This function basically brings the UFS device, UniPro link and controller
7548 * to active state. Following operations are done in this function:
7549 *
7550 * 1. Turn on all the controller related clocks
7551 * 2. Bring the UniPro link out of Hibernate state
7552 * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
7553 * to active state.
7554 * 4. If auto-bkops is enabled on the device, disable it.
7555 *
7556 * So following would be the possible power state after this function return
7557 * successfully:
7558 * S1: UFS device in Active state with VCC rail ON
7559 * UniPro link in Active state
7560 * All the UFS/UniPro controller clocks are ON
7561 *
7562 * Returns 0 for success and non-zero for failure
7563 */
7565 {
7574 else
7576 out:
7581 }
7585 {
7587 }
7594 {
7607 else
7611 }
7615 {
7618 u8 lvl;
7633 lvl,
7640 }
7644 {
7646 }
7649 {
7657 }
7661 {
7664 u8 lvl;
7679 lvl,
7686 }
7690 {
7692 }
7695 {
7703 }
7706 {
7709 }
7712 {
7715 }
7717 /**
7718 * ufshcd_shutdown - shutdown routine
7719 * @hba: per adapter instance
7720 *
7721 * This function would power off both UFS device and UFS link.
7722 *
7723 * Returns 0 always to allow force shutdown even in case of errors.
7724 */
7726 {
7736 }
7739 out:
7742 /* allow force shutdown even in case of errors */
7744 }
7747 /**
7748 * ufshcd_remove - de-allocate SCSI host and host memory space
7749 * data structure memory
7750 * @hba - per adapter instance
7751 */
7753 {
7756 /* disable interrupts */
7764 }
7767 /**
7768 * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
7769 * @hba: pointer to Host Bus Adapter (HBA)
7770 */
7772 {
7774 }
7777 /**
7778 * ufshcd_set_dma_mask - Set dma mask based on the controller
7779 * addressing capability
7780 * @hba: per adapter instance
7781 *
7782 * Returns 0 for success, non-zero for failure
7783 */
7785 {
7789 }
7791 }
7793 /**
7794 * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
7795 * @dev: pointer to device handle
7796 * @hba_handle: driver private handle
7797 * Returns 0 on success, non-zero value on failure
7798 */
7800 {
7810 }
7818 }
7826 out_error:
7828 }
7831 /**
7832 * ufshcd_init - Driver initialization routine
7833 * @hba: per-adapter instance
7834 * @mmio_base: base register address
7835 * @irq: Interrupt line of device
7836 * Returns 0 on success, non-zero value on failure
7837 */
7839 {
7849 }
7854 /* Set descriptor lengths to specification defaults */
7861 /* Read capabilities registers */
7864 /* Get UFS version supported by the controller */
7874 /* Get Interrupt bit mask per version */
7881 }
7883 /* Allocate memory for host memory space */
7888 }
7890 /* Configure LRB */
7903 /* Initailize wait queue for task management */
7907 /* Initialize work queues */
7911 /* Initialize UIC command mutex */
7914 /* Initialize mutex for device management commands */
7919 /* Initialize device management tag acquire wait queue */
7924 /*
7925 * In order to avoid any spurious interrupt immediately after
7926 * registering UFS controller interrupt handler, clear any pending UFS
7927 * interrupt status and disable all the UFS interrupts.
7928 */
7930 REG_INTERRUPT_STATUS);
7932 /*
7933 * Make sure that UFS interrupts are disabled and any pending interrupt
7934 * status is cleared before registering UFS interrupt handler.
7935 */
7938 /* IRQ registration */
7945 }
7951 }
7953 /* Host controller enable */
7960 }
7966 ufshcd_clk_scaling_suspend_work);
7968 ufshcd_clk_scaling_resume_work);
7975 }
7977 /*
7978 * Set the default power management level for runtime and system PM.
7979 * Default power saving mode is to keep UFS link in Hibern8 state
7980 * and UFS device in sleep state.
7981 */
7983 UFS_SLEEP_PWR_MODE,
7984 UIC_LINK_HIBERN8_STATE);
7986 UFS_SLEEP_PWR_MODE,
7987 UIC_LINK_HIBERN8_STATE);
7989 /* Hold auto suspend until async scan completes */
7992 /*
7993 * We are assuming that device wasn't put in sleep/power-down
7994 * state exclusively during the boot stage before kernel.
7995 * This assumption helps avoid doing link startup twice during
7996 * ufshcd_probe_hba().
7997 */
8005 out_remove_scsi_host:
8007 exit_gating:
8009 out_disable:
8012 out_error:
8014 }