?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
accel/ivpu: Move recovery work to system_unbound_wq
[drm/drm-misc.git] / drivers / net / wireless / intel / iwlwifi / pcie / trans.c
blob86f1d87a909c5c0d66c8df7b6dd59ae5a52e56c7
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /*
3 * Copyright (C) 2007-2015, 2018-2024 Intel Corporation
4 * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
5 * Copyright (C) 2016-2017 Intel Deutschland GmbH
6 */
7 #include <linux/pci.h>
8 #include <linux/interrupt.h>
9 #include <linux/debugfs.h>
10 #include <linux/sched.h>
11 #include <linux/bitops.h>
12 #include <linux/gfp.h>
13 #include <linux/vmalloc.h>
14 #include <linux/module.h>
15 #include <linux/wait.h>
16 #include <linux/seq_file.h>
17
18 #include "iwl-drv.h"
19 #include "iwl-trans.h"
20 #include "iwl-csr.h"
21 #include "iwl-prph.h"
22 #include "iwl-scd.h"
23 #include "iwl-agn-hw.h"
24 #include "fw/error-dump.h"
25 #include "fw/dbg.h"
26 #include "fw/api/tx.h"
27 #include "mei/iwl-mei.h"
28 #include "internal.h"
29 #include "iwl-fh.h"
30 #include "iwl-context-info-gen3.h"
31
32 /* extended range in FW SRAM */
33 #define IWL_FW_MEM_EXTENDED_START 0x40000
34 #define IWL_FW_MEM_EXTENDED_END 0x57FFF
35
36 void iwl_trans_pcie_dump_regs(struct iwl_trans *trans)
37 {
38 #define PCI_DUMP_SIZE 352
39 #define PCI_MEM_DUMP_SIZE 64
40 #define PCI_PARENT_DUMP_SIZE 524
41 #define PREFIX_LEN 32
42 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
43 struct pci_dev *pdev = trans_pcie->pci_dev;
44 u32 i, pos, alloc_size, *ptr, *buf;
45 char *prefix;
46
47 if (trans_pcie->pcie_dbg_dumped_once)
48 return;
49
50 /* Should be a multiple of 4 */
51 BUILD_BUG_ON(PCI_DUMP_SIZE > 4096 || PCI_DUMP_SIZE & 0x3);
52 BUILD_BUG_ON(PCI_MEM_DUMP_SIZE > 4096 || PCI_MEM_DUMP_SIZE & 0x3);
53 BUILD_BUG_ON(PCI_PARENT_DUMP_SIZE > 4096 || PCI_PARENT_DUMP_SIZE & 0x3);
54
55 /* Alloc a max size buffer */
56 alloc_size = PCI_ERR_ROOT_ERR_SRC + 4 + PREFIX_LEN;
57 alloc_size = max_t(u32, alloc_size, PCI_DUMP_SIZE + PREFIX_LEN);
58 alloc_size = max_t(u32, alloc_size, PCI_MEM_DUMP_SIZE + PREFIX_LEN);
59 alloc_size = max_t(u32, alloc_size, PCI_PARENT_DUMP_SIZE + PREFIX_LEN);
60
61 buf = kmalloc(alloc_size, GFP_ATOMIC);
62 if (!buf)
63 return;
64 prefix = (char *)buf + alloc_size - PREFIX_LEN;
65
66 IWL_ERR(trans, "iwlwifi transaction failed, dumping registers\n");
67
68 /* Print wifi device registers */
69 sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
70 IWL_ERR(trans, "iwlwifi device config registers:\n");
71 for (i = 0, ptr = buf; i < PCI_DUMP_SIZE; i += 4, ptr++)
72 if (pci_read_config_dword(pdev, i, ptr))
73 goto err_read;
74 print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
75
76 IWL_ERR(trans, "iwlwifi device memory mapped registers:\n");
77 for (i = 0, ptr = buf; i < PCI_MEM_DUMP_SIZE; i += 4, ptr++)
78 *ptr = iwl_read32(trans, i);
79 print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
80
81 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
82 if (pos) {
83 IWL_ERR(trans, "iwlwifi device AER capability structure:\n");
84 for (i = 0, ptr = buf; i < PCI_ERR_ROOT_COMMAND; i += 4, ptr++)
85 if (pci_read_config_dword(pdev, pos + i, ptr))
86 goto err_read;
87 print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET,
88 32, 4, buf, i, 0);
89 }
90
91 /* Print parent device registers next */
92 if (!pdev->bus->self)
93 goto out;
94
95 pdev = pdev->bus->self;
96 sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
97
98 IWL_ERR(trans, "iwlwifi parent port (%s) config registers:\n",
99 pci_name(pdev));
100 for (i = 0, ptr = buf; i < PCI_PARENT_DUMP_SIZE; i += 4, ptr++)
101 if (pci_read_config_dword(pdev, i, ptr))
102 goto err_read;
103 print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
104
105 /* Print root port AER registers */
106 pos = 0;
107 pdev = pcie_find_root_port(pdev);
108 if (pdev)
109 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
110 if (pos) {
111 IWL_ERR(trans, "iwlwifi root port (%s) AER cap structure:\n",
112 pci_name(pdev));
113 sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
114 for (i = 0, ptr = buf; i <= PCI_ERR_ROOT_ERR_SRC; i += 4, ptr++)
115 if (pci_read_config_dword(pdev, pos + i, ptr))
116 goto err_read;
117 print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32,
118 4, buf, i, 0);
119 }
120 goto out;
121
122 err_read:
123 print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
124 IWL_ERR(trans, "Read failed at 0x%X\n", i);
125 out:
126 trans_pcie->pcie_dbg_dumped_once = 1;
127 kfree(buf);
128 }
129
130 int iwl_trans_pcie_sw_reset(struct iwl_trans *trans, bool retake_ownership)
131 {
132 /* Reset entire device - do controller reset (results in SHRD_HW_RST) */
133 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
134 iwl_set_bit(trans, CSR_GP_CNTRL,
135 CSR_GP_CNTRL_REG_FLAG_SW_RESET);
136 usleep_range(10000, 20000);
137 } else {
138 iwl_set_bit(trans, CSR_RESET,
139 CSR_RESET_REG_FLAG_SW_RESET);
140 usleep_range(5000, 6000);
141 }
142
143 if (retake_ownership)
144 return iwl_pcie_prepare_card_hw(trans);
145
146 return 0;
147 }
148
149 static void iwl_pcie_free_fw_monitor(struct iwl_trans *trans)
150 {
151 struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon;
152
153 if (!fw_mon->size)
154 return;
155
156 dma_free_coherent(trans->dev, fw_mon->size, fw_mon->block,
157 fw_mon->physical);
158
159 fw_mon->block = NULL;
160 fw_mon->physical = 0;
161 fw_mon->size = 0;
162 }
163
164 static void iwl_pcie_alloc_fw_monitor_block(struct iwl_trans *trans,
165 u8 max_power)
166 {
167 struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon;
168 void *block = NULL;
169 dma_addr_t physical = 0;
170 u32 size = 0;
171 u8 power;
172
173 if (fw_mon->size) {
174 memset(fw_mon->block, 0, fw_mon->size);
175 return;
176 }
177
178 /* need at least 2 KiB, so stop at 11 */
179 for (power = max_power; power >= 11; power--) {
180 size = BIT(power);
181 block = dma_alloc_coherent(trans->dev, size, &physical,
182 GFP_KERNEL | __GFP_NOWARN);
183 if (!block)
184 continue;
185
186 IWL_INFO(trans,
187 "Allocated 0x%08x bytes for firmware monitor.\n",
188 size);
189 break;
190 }
191
192 if (WARN_ON_ONCE(!block))
193 return;
194
195 if (power != max_power)
196 IWL_ERR(trans,
197 "Sorry - debug buffer is only %luK while you requested %luK\n",
198 (unsigned long)BIT(power - 10),
199 (unsigned long)BIT(max_power - 10));
200
201 fw_mon->block = block;
202 fw_mon->physical = physical;
203 fw_mon->size = size;
204 }
205
206 void iwl_pcie_alloc_fw_monitor(struct iwl_trans *trans, u8 max_power)
207 {
208 if (!max_power) {
209 /* default max_power is maximum */
210 max_power = 26;
211 } else {
212 max_power += 11;
213 }
214
215 if (WARN(max_power > 26,
216 "External buffer size for monitor is too big %d, check the FW TLV\n",
217 max_power))
218 return;
219
220 iwl_pcie_alloc_fw_monitor_block(trans, max_power);
221 }
222
223 static u32 iwl_trans_pcie_read_shr(struct iwl_trans *trans, u32 reg)
224 {
225 iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_CTRL_REG,
226 ((reg & 0x0000ffff) | (2 << 28)));
227 return iwl_read32(trans, HEEP_CTRL_WRD_PCIEX_DATA_REG);
228 }
229
230 static void iwl_trans_pcie_write_shr(struct iwl_trans *trans, u32 reg, u32 val)
231 {
232 iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_DATA_REG, val);
233 iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_CTRL_REG,
234 ((reg & 0x0000ffff) | (3 << 28)));
235 }
236
237 static void iwl_pcie_set_pwr(struct iwl_trans *trans, bool vaux)
238 {
239 if (trans->cfg->apmg_not_supported)
240 return;
241
242 if (vaux && pci_pme_capable(to_pci_dev(trans->dev), PCI_D3cold))
243 iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG,
244 APMG_PS_CTRL_VAL_PWR_SRC_VAUX,
245 ~APMG_PS_CTRL_MSK_PWR_SRC);
246 else
247 iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG,
248 APMG_PS_CTRL_VAL_PWR_SRC_VMAIN,
249 ~APMG_PS_CTRL_MSK_PWR_SRC);
250 }
251
252 /* PCI registers */
253 #define PCI_CFG_RETRY_TIMEOUT 0x041
254
255 void iwl_pcie_apm_config(struct iwl_trans *trans)
256 {
257 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
258 u16 lctl;
259 u16 cap;
260
261 /*
262 * L0S states have been found to be unstable with our devices
263 * and in newer hardware they are not officially supported at
264 * all, so we must always set the L0S_DISABLED bit.
265 */
266 iwl_set_bit(trans, CSR_GIO_REG, CSR_GIO_REG_VAL_L0S_DISABLED);
267
268 pcie_capability_read_word(trans_pcie->pci_dev, PCI_EXP_LNKCTL, &lctl);
269 trans->pm_support = !(lctl & PCI_EXP_LNKCTL_ASPM_L0S);
270
271 pcie_capability_read_word(trans_pcie->pci_dev, PCI_EXP_DEVCTL2, &cap);
272 trans->ltr_enabled = cap & PCI_EXP_DEVCTL2_LTR_EN;
273 IWL_DEBUG_POWER(trans, "L1 %sabled - LTR %sabled\n",
274 (lctl & PCI_EXP_LNKCTL_ASPM_L1) ? "En" : "Dis",
275 trans->ltr_enabled ? "En" : "Dis");
276 }
277
278 /*
279 * Start up NIC's basic functionality after it has been reset
280 * (e.g. after platform boot, or shutdown via iwl_pcie_apm_stop())
281 * NOTE: This does not load uCode nor start the embedded processor
282 */
283 static int iwl_pcie_apm_init(struct iwl_trans *trans)
284 {
285 int ret;
286
287 IWL_DEBUG_INFO(trans, "Init card's basic functions\n");
288
289 /*
290 * Use "set_bit" below rather than "write", to preserve any hardware
291 * bits already set by default after reset.
292 */
293
294 /* Disable L0S exit timer (platform NMI Work/Around) */
295 if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_8000)
296 iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
297 CSR_GIO_CHICKEN_BITS_REG_BIT_DIS_L0S_EXIT_TIMER);
298
299 /*
300 * Disable L0s without affecting L1;
301 * don't wait for ICH L0s (ICH bug W/A)
302 */
303 iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
304 CSR_GIO_CHICKEN_BITS_REG_BIT_L1A_NO_L0S_RX);
305
306 /* Set FH wait threshold to maximum (HW error during stress W/A) */
307 iwl_set_bit(trans, CSR_DBG_HPET_MEM_REG, CSR_DBG_HPET_MEM_REG_VAL);
308
309 /*
310 * Enable HAP INTA (interrupt from management bus) to
311 * wake device's PCI Express link L1a -> L0s
312 */
313 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
314 CSR_HW_IF_CONFIG_REG_BIT_HAP_WAKE_L1A);
315
316 iwl_pcie_apm_config(trans);
317
318 /* Configure analog phase-lock-loop before activating to D0A */
319 if (trans->trans_cfg->base_params->pll_cfg)
320 iwl_set_bit(trans, CSR_ANA_PLL_CFG, CSR50_ANA_PLL_CFG_VAL);
321
322 ret = iwl_finish_nic_init(trans);
323 if (ret)
324 return ret;
325
326 if (trans->cfg->host_interrupt_operation_mode) {
327 /*
328 * This is a bit of an abuse - This is needed for 7260 / 3160
329 * only check host_interrupt_operation_mode even if this is
330 * not related to host_interrupt_operation_mode.
331 *
332 * Enable the oscillator to count wake up time for L1 exit. This
333 * consumes slightly more power (100uA) - but allows to be sure
334 * that we wake up from L1 on time.
335 *
336 * This looks weird: read twice the same register, discard the
337 * value, set a bit, and yet again, read that same register
338 * just to discard the value. But that's the way the hardware
339 * seems to like it.
340 */
341 iwl_read_prph(trans, OSC_CLK);
342 iwl_read_prph(trans, OSC_CLK);
343 iwl_set_bits_prph(trans, OSC_CLK, OSC_CLK_FORCE_CONTROL);
344 iwl_read_prph(trans, OSC_CLK);
345 iwl_read_prph(trans, OSC_CLK);
346 }
347
348 /*
349 * Enable DMA clock and wait for it to stabilize.
350 *
351 * Write to "CLK_EN_REG"; "1" bits enable clocks, while "0"
352 * bits do not disable clocks. This preserves any hardware
353 * bits already set by default in "CLK_CTRL_REG" after reset.
354 */
355 if (!trans->cfg->apmg_not_supported) {
356 iwl_write_prph(trans, APMG_CLK_EN_REG,
357 APMG_CLK_VAL_DMA_CLK_RQT);
358 udelay(20);
359
360 /* Disable L1-Active */
361 iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG,
362 APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
363
364 /* Clear the interrupt in APMG if the NIC is in RFKILL */
365 iwl_write_prph(trans, APMG_RTC_INT_STT_REG,
366 APMG_RTC_INT_STT_RFKILL);
367 }
368
369 set_bit(STATUS_DEVICE_ENABLED, &trans->status);
370
371 return 0;
372 }
373
374 /*
375 * Enable LP XTAL to avoid HW bug where device may consume much power if
376 * FW is not loaded after device reset. LP XTAL is disabled by default
377 * after device HW reset. Do it only if XTAL is fed by internal source.
378 * Configure device's "persistence" mode to avoid resetting XTAL again when
379 * SHRD_HW_RST occurs in S3.
380 */
381 static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans)
382 {
383 int ret;
384 u32 apmg_gp1_reg;
385 u32 apmg_xtal_cfg_reg;
386 u32 dl_cfg_reg;
387
388 /* Force XTAL ON */
389 __iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL,
390 CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
391
392 ret = iwl_trans_pcie_sw_reset(trans, true);
393
394 if (!ret)
395 ret = iwl_finish_nic_init(trans);
396
397 if (WARN_ON(ret)) {
398 /* Release XTAL ON request */
399 __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
400 CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
401 return;
402 }
403
404 /*
405 * Clear "disable persistence" to avoid LP XTAL resetting when
406 * SHRD_HW_RST is applied in S3.
407 */
408 iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG,
409 APMG_PCIDEV_STT_VAL_PERSIST_DIS);
410
411 /*
412 * Force APMG XTAL to be active to prevent its disabling by HW
413 * caused by APMG idle state.
414 */
415 apmg_xtal_cfg_reg = iwl_trans_pcie_read_shr(trans,
416 SHR_APMG_XTAL_CFG_REG);
417 iwl_trans_pcie_write_shr(trans, SHR_APMG_XTAL_CFG_REG,
418 apmg_xtal_cfg_reg |
419 SHR_APMG_XTAL_CFG_XTAL_ON_REQ);
420
421 ret = iwl_trans_pcie_sw_reset(trans, true);
422 if (ret)
423 IWL_ERR(trans,
424 "iwl_pcie_apm_lp_xtal_enable: failed to retake NIC ownership\n");
425
426 /* Enable LP XTAL by indirect access through CSR */
427 apmg_gp1_reg = iwl_trans_pcie_read_shr(trans, SHR_APMG_GP1_REG);
428 iwl_trans_pcie_write_shr(trans, SHR_APMG_GP1_REG, apmg_gp1_reg |
429 SHR_APMG_GP1_WF_XTAL_LP_EN |
430 SHR_APMG_GP1_CHICKEN_BIT_SELECT);
431
432 /* Clear delay line clock power up */
433 dl_cfg_reg = iwl_trans_pcie_read_shr(trans, SHR_APMG_DL_CFG_REG);
434 iwl_trans_pcie_write_shr(trans, SHR_APMG_DL_CFG_REG, dl_cfg_reg &
435 ~SHR_APMG_DL_CFG_DL_CLOCK_POWER_UP);
436
437 /*
438 * Enable persistence mode to avoid LP XTAL resetting when
439 * SHRD_HW_RST is applied in S3.
440 */
441 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
442 CSR_HW_IF_CONFIG_REG_PERSIST_MODE);
443
444 /*
445 * Clear "initialization complete" bit to move adapter from
446 * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
447 */
448 iwl_clear_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
449
450 /* Activates XTAL resources monitor */
451 __iwl_trans_pcie_set_bit(trans, CSR_MONITOR_CFG_REG,
452 CSR_MONITOR_XTAL_RESOURCES);
453
454 /* Release XTAL ON request */
455 __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
456 CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
457 udelay(10);
458
459 /* Release APMG XTAL */
460 iwl_trans_pcie_write_shr(trans, SHR_APMG_XTAL_CFG_REG,
461 apmg_xtal_cfg_reg &
462 ~SHR_APMG_XTAL_CFG_XTAL_ON_REQ);
463 }
464
465 void iwl_pcie_apm_stop_master(struct iwl_trans *trans)
466 {
467 int ret;
468
469 /* stop device's busmaster DMA activity */
470
471 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
472 iwl_set_bit(trans, CSR_GP_CNTRL,
473 CSR_GP_CNTRL_REG_FLAG_BUS_MASTER_DISABLE_REQ);
474
475 ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
476 CSR_GP_CNTRL_REG_FLAG_BUS_MASTER_DISABLE_STATUS,
477 CSR_GP_CNTRL_REG_FLAG_BUS_MASTER_DISABLE_STATUS,
478 100);
479 usleep_range(10000, 20000);
480 } else {
481 iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_STOP_MASTER);
482
483 ret = iwl_poll_bit(trans, CSR_RESET,
484 CSR_RESET_REG_FLAG_MASTER_DISABLED,
485 CSR_RESET_REG_FLAG_MASTER_DISABLED, 100);
486 }
487
488 if (ret < 0)
489 IWL_WARN(trans, "Master Disable Timed Out, 100 usec\n");
490
491 IWL_DEBUG_INFO(trans, "stop master\n");
492 }
493
494 static void iwl_pcie_apm_stop(struct iwl_trans *trans, bool op_mode_leave)
495 {
496 IWL_DEBUG_INFO(trans, "Stop card, put in low power state\n");
497
498 if (op_mode_leave) {
499 if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status))
500 iwl_pcie_apm_init(trans);
501
502 /* inform ME that we are leaving */
503 if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000)
504 iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG,
505 APMG_PCIDEV_STT_VAL_WAKE_ME);
506 else if (trans->trans_cfg->device_family >=
507 IWL_DEVICE_FAMILY_8000) {
508 iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
509 CSR_RESET_LINK_PWR_MGMT_DISABLED);
510 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
511 CSR_HW_IF_CONFIG_REG_PREPARE |
512 CSR_HW_IF_CONFIG_REG_ENABLE_PME);
513 mdelay(1);
514 iwl_clear_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
515 CSR_RESET_LINK_PWR_MGMT_DISABLED);
516 }
517 mdelay(5);
518 }
519
520 clear_bit(STATUS_DEVICE_ENABLED, &trans->status);
521
522 /* Stop device's DMA activity */
523 iwl_pcie_apm_stop_master(trans);
524
525 if (trans->cfg->lp_xtal_workaround) {
526 iwl_pcie_apm_lp_xtal_enable(trans);
527 return;
528 }
529
530 iwl_trans_pcie_sw_reset(trans, false);
531
532 /*
533 * Clear "initialization complete" bit to move adapter from
534 * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
535 */
536 iwl_clear_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
537 }
538
539 static int iwl_pcie_nic_init(struct iwl_trans *trans)
540 {
541 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
542 int ret;
543
544 /* nic_init */
545 spin_lock_bh(&trans_pcie->irq_lock);
546 ret = iwl_pcie_apm_init(trans);
547 spin_unlock_bh(&trans_pcie->irq_lock);
548
549 if (ret)
550 return ret;
551
552 iwl_pcie_set_pwr(trans, false);
553
554 iwl_op_mode_nic_config(trans->op_mode);
555
556 /* Allocate the RX queue, or reset if it is already allocated */
557 ret = iwl_pcie_rx_init(trans);
558 if (ret)
559 return ret;
560
561 /* Allocate or reset and init all Tx and Command queues */
562 if (iwl_pcie_tx_init(trans)) {
563 iwl_pcie_rx_free(trans);
564 return -ENOMEM;
565 }
566
567 if (trans->trans_cfg->base_params->shadow_reg_enable) {
568 /* enable shadow regs in HW */
569 iwl_set_bit(trans, CSR_MAC_SHADOW_REG_CTRL, 0x800FFFFF);
570 IWL_DEBUG_INFO(trans, "Enabling shadow registers in device\n");
571 }
572
573 return 0;
574 }
575
576 #define HW_READY_TIMEOUT (50)
577
578 /* Note: returns poll_bit return value, which is >= 0 if success */
579 static int iwl_pcie_set_hw_ready(struct iwl_trans *trans)
580 {
581 int ret;
582
583 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
584 CSR_HW_IF_CONFIG_REG_BIT_NIC_READY);
585
586 /* See if we got it */
587 ret = iwl_poll_bit(trans, CSR_HW_IF_CONFIG_REG,
588 CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
589 CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
590 HW_READY_TIMEOUT);
591
592 if (ret >= 0)
593 iwl_set_bit(trans, CSR_MBOX_SET_REG, CSR_MBOX_SET_REG_OS_ALIVE);
594
595 IWL_DEBUG_INFO(trans, "hardware%s ready\n", ret < 0 ? " not" : "");
596 return ret;
597 }
598
599 /* Note: returns standard 0/-ERROR code */
600 int iwl_pcie_prepare_card_hw(struct iwl_trans *trans)
601 {
602 int ret;
603 int iter;
604
605 IWL_DEBUG_INFO(trans, "iwl_trans_prepare_card_hw enter\n");
606
607 ret = iwl_pcie_set_hw_ready(trans);
608 /* If the card is ready, exit 0 */
609 if (ret >= 0) {
610 trans->csme_own = false;
611 return 0;
612 }
613
614 iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
615 CSR_RESET_LINK_PWR_MGMT_DISABLED);
616 usleep_range(1000, 2000);
617
618 for (iter = 0; iter < 10; iter++) {
619 int t = 0;
620
621 /* If HW is not ready, prepare the conditions to check again */
622 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
623 CSR_HW_IF_CONFIG_REG_PREPARE);
624
625 do {
626 ret = iwl_pcie_set_hw_ready(trans);
627 if (ret >= 0) {
628 trans->csme_own = false;
629 return 0;
630 }
631
632 if (iwl_mei_is_connected()) {
633 IWL_DEBUG_INFO(trans,
634 "Couldn't prepare the card but SAP is connected\n");
635 trans->csme_own = true;
636 if (trans->trans_cfg->device_family !=
637 IWL_DEVICE_FAMILY_9000)
638 IWL_ERR(trans,
639 "SAP not supported for this NIC family\n");
640
641 return -EBUSY;
642 }
643
644 usleep_range(200, 1000);
645 t += 200;
646 } while (t < 150000);
647 msleep(25);
648 }
649
650 IWL_ERR(trans, "Couldn't prepare the card\n");
651
652 return ret;
653 }
654
655 /*
656 * ucode
657 */
658 static void iwl_pcie_load_firmware_chunk_fh(struct iwl_trans *trans,
659 u32 dst_addr, dma_addr_t phy_addr,
660 u32 byte_cnt)
661 {
662 iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
663 FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);
664
665 iwl_write32(trans, FH_SRVC_CHNL_SRAM_ADDR_REG(FH_SRVC_CHNL),
666 dst_addr);
667
668 iwl_write32(trans, FH_TFDIB_CTRL0_REG(FH_SRVC_CHNL),
669 phy_addr & FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);
670
671 iwl_write32(trans, FH_TFDIB_CTRL1_REG(FH_SRVC_CHNL),
672 (iwl_get_dma_hi_addr(phy_addr)
673 << FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);
674
675 iwl_write32(trans, FH_TCSR_CHNL_TX_BUF_STS_REG(FH_SRVC_CHNL),
676 BIT(FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM) |
677 BIT(FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX) |
678 FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);
679
680 iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
681 FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
682 FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE |
683 FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
684 }
685
686 static int iwl_pcie_load_firmware_chunk(struct iwl_trans *trans,
687 u32 dst_addr, dma_addr_t phy_addr,
688 u32 byte_cnt)
689 {
690 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
691 int ret;
692
693 trans_pcie->ucode_write_complete = false;
694
695 if (!iwl_trans_grab_nic_access(trans))
696 return -EIO;
697
698 iwl_pcie_load_firmware_chunk_fh(trans, dst_addr, phy_addr,
699 byte_cnt);
700 iwl_trans_release_nic_access(trans);
701
702 ret = wait_event_timeout(trans_pcie->ucode_write_waitq,
703 trans_pcie->ucode_write_complete, 5 * HZ);
704 if (!ret) {
705 IWL_ERR(trans, "Failed to load firmware chunk!\n");
706 iwl_trans_pcie_dump_regs(trans);
707 return -ETIMEDOUT;
708 }
709
710 return 0;
711 }
712
713 static int iwl_pcie_load_section(struct iwl_trans *trans, u8 section_num,
714 const struct fw_desc *section)
715 {
716 u8 *v_addr;
717 dma_addr_t p_addr;
718 u32 offset, chunk_sz = min_t(u32, FH_MEM_TB_MAX_LENGTH, section->len);
719 int ret = 0;
720
721 IWL_DEBUG_FW(trans, "[%d] uCode section being loaded...\n",
722 section_num);
723
724 v_addr = dma_alloc_coherent(trans->dev, chunk_sz, &p_addr,
725 GFP_KERNEL | __GFP_NOWARN);
726 if (!v_addr) {
727 IWL_DEBUG_INFO(trans, "Falling back to small chunks of DMA\n");
728 chunk_sz = PAGE_SIZE;
729 v_addr = dma_alloc_coherent(trans->dev, chunk_sz,
730 &p_addr, GFP_KERNEL);
731 if (!v_addr)
732 return -ENOMEM;
733 }
734
735 for (offset = 0; offset < section->len; offset += chunk_sz) {
736 u32 copy_size, dst_addr;
737 bool extended_addr = false;
738
739 copy_size = min_t(u32, chunk_sz, section->len - offset);
740 dst_addr = section->offset + offset;
741
742 if (dst_addr >= IWL_FW_MEM_EXTENDED_START &&
743 dst_addr <= IWL_FW_MEM_EXTENDED_END)
744 extended_addr = true;
745
746 if (extended_addr)
747 iwl_set_bits_prph(trans, LMPM_CHICK,
748 LMPM_CHICK_EXTENDED_ADDR_SPACE);
749
750 memcpy(v_addr, (const u8 *)section->data + offset, copy_size);
751 ret = iwl_pcie_load_firmware_chunk(trans, dst_addr, p_addr,
752 copy_size);
753
754 if (extended_addr)
755 iwl_clear_bits_prph(trans, LMPM_CHICK,
756 LMPM_CHICK_EXTENDED_ADDR_SPACE);
757
758 if (ret) {
759 IWL_ERR(trans,
760 "Could not load the [%d] uCode section\n",
761 section_num);
762 break;
763 }
764 }
765
766 dma_free_coherent(trans->dev, chunk_sz, v_addr, p_addr);
767 return ret;
768 }
769
770 static int iwl_pcie_load_cpu_sections_8000(struct iwl_trans *trans,
771 const struct fw_img *image,
772 int cpu,
773 int *first_ucode_section)
774 {
775 int shift_param;
776 int i, ret = 0, sec_num = 0x1;
777 u32 val, last_read_idx = 0;
778
779 if (cpu == 1) {
780 shift_param = 0;
781 *first_ucode_section = 0;
782 } else {
783 shift_param = 16;
784 (*first_ucode_section)++;
785 }
786
787 for (i = *first_ucode_section; i < image->num_sec; i++) {
788 last_read_idx = i;
789
790 /*
791 * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
792 * CPU1 to CPU2.
793 * PAGING_SEPARATOR_SECTION delimiter - separate between
794 * CPU2 non paged to CPU2 paging sec.
795 */
796 if (!image->sec[i].data ||
797 image->sec[i].offset == CPU1_CPU2_SEPARATOR_SECTION ||
798 image->sec[i].offset == PAGING_SEPARATOR_SECTION) {
799 IWL_DEBUG_FW(trans,
800 "Break since Data not valid or Empty section, sec = %d\n",
801 i);
802 break;
803 }
804
805 ret = iwl_pcie_load_section(trans, i, &image->sec[i]);
806 if (ret)
807 return ret;
808
809 /* Notify ucode of loaded section number and status */
810 val = iwl_read_direct32(trans, FH_UCODE_LOAD_STATUS);
811 val = val | (sec_num << shift_param);
812 iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS, val);
813
814 sec_num = (sec_num << 1) | 0x1;
815 }
816
817 *first_ucode_section = last_read_idx;
818
819 iwl_enable_interrupts(trans);
820
821 if (trans->trans_cfg->gen2) {
822 if (cpu == 1)
823 iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS,
824 0xFFFF);
825 else
826 iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS,
827 0xFFFFFFFF);
828 } else {
829 if (cpu == 1)
830 iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS,
831 0xFFFF);
832 else
833 iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS,
834 0xFFFFFFFF);
835 }
836
837 return 0;
838 }
839
840 static int iwl_pcie_load_cpu_sections(struct iwl_trans *trans,
841 const struct fw_img *image,
842 int cpu,
843 int *first_ucode_section)
844 {
845 int i, ret = 0;
846 u32 last_read_idx = 0;
847
848 if (cpu == 1)
849 *first_ucode_section = 0;
850 else
851 (*first_ucode_section)++;
852
853 for (i = *first_ucode_section; i < image->num_sec; i++) {
854 last_read_idx = i;
855
856 /*
857 * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
858 * CPU1 to CPU2.
859 * PAGING_SEPARATOR_SECTION delimiter - separate between
860 * CPU2 non paged to CPU2 paging sec.
861 */
862 if (!image->sec[i].data ||
863 image->sec[i].offset == CPU1_CPU2_SEPARATOR_SECTION ||
864 image->sec[i].offset == PAGING_SEPARATOR_SECTION) {
865 IWL_DEBUG_FW(trans,
866 "Break since Data not valid or Empty section, sec = %d\n",
867 i);
868 break;
869 }
870
871 ret = iwl_pcie_load_section(trans, i, &image->sec[i]);
872 if (ret)
873 return ret;
874 }
875
876 *first_ucode_section = last_read_idx;
877
878 return 0;
879 }
880
881 static void iwl_pcie_apply_destination_ini(struct iwl_trans *trans)
882 {
883 enum iwl_fw_ini_allocation_id alloc_id = IWL_FW_INI_ALLOCATION_ID_DBGC1;
884 struct iwl_fw_ini_allocation_tlv *fw_mon_cfg =
885 &trans->dbg.fw_mon_cfg[alloc_id];
886 struct iwl_dram_data *frag;
887
888 if (!iwl_trans_dbg_ini_valid(trans))
889 return;
890
891 if (le32_to_cpu(fw_mon_cfg->buf_location) ==
892 IWL_FW_INI_LOCATION_SRAM_PATH) {
893 IWL_DEBUG_FW(trans, "WRT: Applying SMEM buffer destination\n");
894 /* set sram monitor by enabling bit 7 */
895 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
896 CSR_HW_IF_CONFIG_REG_BIT_MONITOR_SRAM);
897
898 return;
899 }
900
901 if (le32_to_cpu(fw_mon_cfg->buf_location) !=
902 IWL_FW_INI_LOCATION_DRAM_PATH ||
903 !trans->dbg.fw_mon_ini[alloc_id].num_frags)
904 return;
905
906 frag = &trans->dbg.fw_mon_ini[alloc_id].frags[0];
907
908 IWL_DEBUG_FW(trans, "WRT: Applying DRAM destination (alloc_id=%u)\n",
909 alloc_id);
910
911 iwl_write_umac_prph(trans, MON_BUFF_BASE_ADDR_VER2,
912 frag->physical >> MON_BUFF_SHIFT_VER2);
913 iwl_write_umac_prph(trans, MON_BUFF_END_ADDR_VER2,
914 (frag->physical + frag->size - 256) >>
915 MON_BUFF_SHIFT_VER2);
916 }
917
918 void iwl_pcie_apply_destination(struct iwl_trans *trans)
919 {
920 const struct iwl_fw_dbg_dest_tlv_v1 *dest = trans->dbg.dest_tlv;
921 const struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon;
922 int i;
923
924 if (iwl_trans_dbg_ini_valid(trans)) {
925 iwl_pcie_apply_destination_ini(trans);
926 return;
927 }
928
929 IWL_INFO(trans, "Applying debug destination %s\n",
930 get_fw_dbg_mode_string(dest->monitor_mode));
931
932 if (dest->monitor_mode == EXTERNAL_MODE)
933 iwl_pcie_alloc_fw_monitor(trans, dest->size_power);
934 else
935 IWL_WARN(trans, "PCI should have external buffer debug\n");
936
937 for (i = 0; i < trans->dbg.n_dest_reg; i++) {
938 u32 addr = le32_to_cpu(dest->reg_ops[i].addr);
939 u32 val = le32_to_cpu(dest->reg_ops[i].val);
940
941 switch (dest->reg_ops[i].op) {
942 case CSR_ASSIGN:
943 iwl_write32(trans, addr, val);
944 break;
945 case CSR_SETBIT:
946 iwl_set_bit(trans, addr, BIT(val));
947 break;
948 case CSR_CLEARBIT:
949 iwl_clear_bit(trans, addr, BIT(val));
950 break;
951 case PRPH_ASSIGN:
952 iwl_write_prph(trans, addr, val);
953 break;
954 case PRPH_SETBIT:
955 iwl_set_bits_prph(trans, addr, BIT(val));
956 break;
957 case PRPH_CLEARBIT:
958 iwl_clear_bits_prph(trans, addr, BIT(val));
959 break;
960 case PRPH_BLOCKBIT:
961 if (iwl_read_prph(trans, addr) & BIT(val)) {
962 IWL_ERR(trans,
963 "BIT(%u) in address 0x%x is 1, stopping FW configuration\n",
964 val, addr);
965 goto monitor;
966 }
967 break;
968 default:
969 IWL_ERR(trans, "FW debug - unknown OP %d\n",
970 dest->reg_ops[i].op);
971 break;
972 }
973 }
974
975 monitor:
976 if (dest->monitor_mode == EXTERNAL_MODE && fw_mon->size) {
977 iwl_write_prph(trans, le32_to_cpu(dest->base_reg),
978 fw_mon->physical >> dest->base_shift);
979 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
980 iwl_write_prph(trans, le32_to_cpu(dest->end_reg),
981 (fw_mon->physical + fw_mon->size -
982 256) >> dest->end_shift);
983 else
984 iwl_write_prph(trans, le32_to_cpu(dest->end_reg),
985 (fw_mon->physical + fw_mon->size) >>
986 dest->end_shift);
987 }
988 }
989
990 static int iwl_pcie_load_given_ucode(struct iwl_trans *trans,
991 const struct fw_img *image)
992 {
993 int ret = 0;
994 int first_ucode_section;
995
996 IWL_DEBUG_FW(trans, "working with %s CPU\n",
997 image->is_dual_cpus ? "Dual" : "Single");
998
999 /* load to FW the binary non secured sections of CPU1 */
1000 ret = iwl_pcie_load_cpu_sections(trans, image, 1, &first_ucode_section);
1001 if (ret)
1002 return ret;
1003
1004 if (image->is_dual_cpus) {
1005 /* set CPU2 header address */
1006 iwl_write_prph(trans,
1007 LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR,
1008 LMPM_SECURE_CPU2_HDR_MEM_SPACE);
1009
1010 /* load to FW the binary sections of CPU2 */
1011 ret = iwl_pcie_load_cpu_sections(trans, image, 2,
1012 &first_ucode_section);
1013 if (ret)
1014 return ret;
1015 }
1016
1017 if (iwl_pcie_dbg_on(trans))
1018 iwl_pcie_apply_destination(trans);
1019
1020 iwl_enable_interrupts(trans);
1021
1022 /* release CPU reset */
1023 iwl_write32(trans, CSR_RESET, 0);
1024
1025 return 0;
1026 }
1027
1028 static int iwl_pcie_load_given_ucode_8000(struct iwl_trans *trans,
1029 const struct fw_img *image)
1030 {
1031 int ret = 0;
1032 int first_ucode_section;
1033
1034 IWL_DEBUG_FW(trans, "working with %s CPU\n",
1035 image->is_dual_cpus ? "Dual" : "Single");
1036
1037 if (iwl_pcie_dbg_on(trans))
1038 iwl_pcie_apply_destination(trans);
1039
1040 IWL_DEBUG_POWER(trans, "Original WFPM value = 0x%08X\n",
1041 iwl_read_prph(trans, WFPM_GP2));
1042
1043 /*
1044 * Set default value. On resume reading the values that were
1045 * zeored can provide debug data on the resume flow.
1046 * This is for debugging only and has no functional impact.
1047 */
1048 iwl_write_prph(trans, WFPM_GP2, 0x01010101);
1049
1050 /* configure the ucode to be ready to get the secured image */
1051 /* release CPU reset */
1052 iwl_write_prph(trans, RELEASE_CPU_RESET, RELEASE_CPU_RESET_BIT);
1053
1054 /* load to FW the binary Secured sections of CPU1 */
1055 ret = iwl_pcie_load_cpu_sections_8000(trans, image, 1,
1056 &first_ucode_section);
1057 if (ret)
1058 return ret;
1059
1060 /* load to FW the binary sections of CPU2 */
1061 return iwl_pcie_load_cpu_sections_8000(trans, image, 2,
1062 &first_ucode_section);
1063 }
1064
1065 bool iwl_pcie_check_hw_rf_kill(struct iwl_trans *trans)
1066 {
1067 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1068 bool hw_rfkill = iwl_is_rfkill_set(trans);
1069 bool prev = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1070 bool report;
1071
1072 if (hw_rfkill) {
1073 set_bit(STATUS_RFKILL_HW, &trans->status);
1074 set_bit(STATUS_RFKILL_OPMODE, &trans->status);
1075 } else {
1076 clear_bit(STATUS_RFKILL_HW, &trans->status);
1077 if (trans_pcie->opmode_down)
1078 clear_bit(STATUS_RFKILL_OPMODE, &trans->status);
1079 }
1080
1081 report = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1082
1083 if (prev != report)
1084 iwl_trans_pcie_rf_kill(trans, report, false);
1085
1086 return hw_rfkill;
1087 }
1088
1089 struct iwl_causes_list {
1090 u16 mask_reg;
1091 u8 bit;
1092 u8 addr;
1093 };
1094
1095 #define IWL_CAUSE(reg, mask) \
1096 { \
1097 .mask_reg = reg, \
1098 .bit = ilog2(mask), \
1099 .addr = ilog2(mask) + \
1100 ((reg) == CSR_MSIX_FH_INT_MASK_AD ? -16 : \
1101 (reg) == CSR_MSIX_HW_INT_MASK_AD ? 16 : \
1102 0xffff), /* causes overflow warning */ \
1103 }
1104
1105 static const struct iwl_causes_list causes_list_common[] = {
1106 IWL_CAUSE(CSR_MSIX_FH_INT_MASK_AD, MSIX_FH_INT_CAUSES_D2S_CH0_NUM),
1107 IWL_CAUSE(CSR_MSIX_FH_INT_MASK_AD, MSIX_FH_INT_CAUSES_D2S_CH1_NUM),
1108 IWL_CAUSE(CSR_MSIX_FH_INT_MASK_AD, MSIX_FH_INT_CAUSES_S2D),
1109 IWL_CAUSE(CSR_MSIX_FH_INT_MASK_AD, MSIX_FH_INT_CAUSES_FH_ERR),
1110 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_ALIVE),
1111 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_WAKEUP),
1112 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_RESET_DONE),
1113 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_TOP_FATAL_ERR),
1114 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_CT_KILL),
1115 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_RF_KILL),
1116 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_PERIODIC),
1117 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_SCD),
1118 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_FH_TX),
1119 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_HW_ERR),
1120 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_HAP),
1121 };
1122
1123 static const struct iwl_causes_list causes_list_pre_bz[] = {
1124 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_SW_ERR),
1125 };
1126
1127 static const struct iwl_causes_list causes_list_bz[] = {
1128 IWL_CAUSE(CSR_MSIX_HW_INT_MASK_AD, MSIX_HW_INT_CAUSES_REG_SW_ERR_BZ),
1129 };
1130
1131 static void iwl_pcie_map_list(struct iwl_trans *trans,
1132 const struct iwl_causes_list *causes,
1133 int arr_size, int val)
1134 {
1135 int i;
1136
1137 for (i = 0; i < arr_size; i++) {
1138 iwl_write8(trans, CSR_MSIX_IVAR(causes[i].addr), val);
1139 iwl_clear_bit(trans, causes[i].mask_reg,
1140 BIT(causes[i].bit));
1141 }
1142 }
1143
1144 static void iwl_pcie_map_non_rx_causes(struct iwl_trans *trans)
1145 {
1146 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1147 int val = trans_pcie->def_irq | MSIX_NON_AUTO_CLEAR_CAUSE;
1148 /*
1149 * Access all non RX causes and map them to the default irq.
1150 * In case we are missing at least one interrupt vector,
1151 * the first interrupt vector will serve non-RX and FBQ causes.
1152 */
1153 iwl_pcie_map_list(trans, causes_list_common,
1154 ARRAY_SIZE(causes_list_common), val);
1155 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
1156 iwl_pcie_map_list(trans, causes_list_bz,
1157 ARRAY_SIZE(causes_list_bz), val);
1158 else
1159 iwl_pcie_map_list(trans, causes_list_pre_bz,
1160 ARRAY_SIZE(causes_list_pre_bz), val);
1161 }
1162
1163 static void iwl_pcie_map_rx_causes(struct iwl_trans *trans)
1164 {
1165 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1166 u32 offset =
1167 trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 1 : 0;
1168 u32 val, idx;
1169
1170 /*
1171 * The first RX queue - fallback queue, which is designated for
1172 * management frame, command responses etc, is always mapped to the
1173 * first interrupt vector. The other RX queues are mapped to
1174 * the other (N - 2) interrupt vectors.
1175 */
1176 val = BIT(MSIX_FH_INT_CAUSES_Q(0));
1177 for (idx = 1; idx < trans->num_rx_queues; idx++) {
1178 iwl_write8(trans, CSR_MSIX_RX_IVAR(idx),
1179 MSIX_FH_INT_CAUSES_Q(idx - offset));
1180 val |= BIT(MSIX_FH_INT_CAUSES_Q(idx));
1181 }
1182 iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD, ~val);
1183
1184 val = MSIX_FH_INT_CAUSES_Q(0);
1185 if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX)
1186 val |= MSIX_NON_AUTO_CLEAR_CAUSE;
1187 iwl_write8(trans, CSR_MSIX_RX_IVAR(0), val);
1188
1189 if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS)
1190 iwl_write8(trans, CSR_MSIX_RX_IVAR(1), val);
1191 }
1192
1193 void iwl_pcie_conf_msix_hw(struct iwl_trans_pcie *trans_pcie)
1194 {
1195 struct iwl_trans *trans = trans_pcie->trans;
1196
1197 if (!trans_pcie->msix_enabled) {
1198 if (trans->trans_cfg->mq_rx_supported &&
1199 test_bit(STATUS_DEVICE_ENABLED, &trans->status))
1200 iwl_write_umac_prph(trans, UREG_CHICK,
1201 UREG_CHICK_MSI_ENABLE);
1202 return;
1203 }
1204 /*
1205 * The IVAR table needs to be configured again after reset,
1206 * but if the device is disabled, we can't write to
1207 * prph.
1208 */
1209 if (test_bit(STATUS_DEVICE_ENABLED, &trans->status))
1210 iwl_write_umac_prph(trans, UREG_CHICK, UREG_CHICK_MSIX_ENABLE);
1211
1212 /*
1213 * Each cause from the causes list above and the RX causes is
1214 * represented as a byte in the IVAR table. The first nibble
1215 * represents the bound interrupt vector of the cause, the second
1216 * represents no auto clear for this cause. This will be set if its
1217 * interrupt vector is bound to serve other causes.
1218 */
1219 iwl_pcie_map_rx_causes(trans);
1220
1221 iwl_pcie_map_non_rx_causes(trans);
1222 }
1223
1224 static void iwl_pcie_init_msix(struct iwl_trans_pcie *trans_pcie)
1225 {
1226 struct iwl_trans *trans = trans_pcie->trans;
1227
1228 iwl_pcie_conf_msix_hw(trans_pcie);
1229
1230 if (!trans_pcie->msix_enabled)
1231 return;
1232
1233 trans_pcie->fh_init_mask = ~iwl_read32(trans, CSR_MSIX_FH_INT_MASK_AD);
1234 trans_pcie->fh_mask = trans_pcie->fh_init_mask;
1235 trans_pcie->hw_init_mask = ~iwl_read32(trans, CSR_MSIX_HW_INT_MASK_AD);
1236 trans_pcie->hw_mask = trans_pcie->hw_init_mask;
1237 }
1238
1239 static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans, bool from_irq)
1240 {
1241 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1242
1243 lockdep_assert_held(&trans_pcie->mutex);
1244
1245 if (trans_pcie->is_down)
1246 return;
1247
1248 trans_pcie->is_down = true;
1249
1250 /* tell the device to stop sending interrupts */
1251 iwl_disable_interrupts(trans);
1252
1253 /* device going down, Stop using ICT table */
1254 iwl_pcie_disable_ict(trans);
1255
1256 /*
1257 * If a HW restart happens during firmware loading,
1258 * then the firmware loading might call this function
1259 * and later it might be called again due to the
1260 * restart. So don't process again if the device is
1261 * already dead.
1262 */
1263 if (test_and_clear_bit(STATUS_DEVICE_ENABLED, &trans->status)) {
1264 IWL_DEBUG_INFO(trans,
1265 "DEVICE_ENABLED bit was set and is now cleared\n");
1266 if (!from_irq)
1267 iwl_pcie_synchronize_irqs(trans);
1268 iwl_pcie_rx_napi_sync(trans);
1269 iwl_pcie_tx_stop(trans);
1270 iwl_pcie_rx_stop(trans);
1271
1272 /* Power-down device's busmaster DMA clocks */
1273 if (!trans->cfg->apmg_not_supported) {
1274 iwl_write_prph(trans, APMG_CLK_DIS_REG,
1275 APMG_CLK_VAL_DMA_CLK_RQT);
1276 udelay(5);
1277 }
1278 }
1279
1280 /* Make sure (redundant) we've released our request to stay awake */
1281 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
1282 iwl_clear_bit(trans, CSR_GP_CNTRL,
1283 CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ);
1284 else
1285 iwl_clear_bit(trans, CSR_GP_CNTRL,
1286 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1287
1288 /* Stop the device, and put it in low power state */
1289 iwl_pcie_apm_stop(trans, false);
1290
1291 /* re-take ownership to prevent other users from stealing the device */
1292 iwl_trans_pcie_sw_reset(trans, true);
1293
1294 /*
1295 * Upon stop, the IVAR table gets erased, so msi-x won't
1296 * work. This causes a bug in RF-KILL flows, since the interrupt
1297 * that enables radio won't fire on the correct irq, and the
1298 * driver won't be able to handle the interrupt.
1299 * Configure the IVAR table again after reset.
1300 */
1301 iwl_pcie_conf_msix_hw(trans_pcie);
1302
1303 /*
1304 * Upon stop, the APM issues an interrupt if HW RF kill is set.
1305 * This is a bug in certain verions of the hardware.
1306 * Certain devices also keep sending HW RF kill interrupt all
1307 * the time, unless the interrupt is ACKed even if the interrupt
1308 * should be masked. Re-ACK all the interrupts here.
1309 */
1310 iwl_disable_interrupts(trans);
1311
1312 /* clear all status bits */
1313 clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
1314 clear_bit(STATUS_INT_ENABLED, &trans->status);
1315 clear_bit(STATUS_TPOWER_PMI, &trans->status);
1316
1317 /*
1318 * Even if we stop the HW, we still want the RF kill
1319 * interrupt
1320 */
1321 iwl_enable_rfkill_int(trans);
1322 }
1323
1324 void iwl_pcie_synchronize_irqs(struct iwl_trans *trans)
1325 {
1326 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1327
1328 if (trans_pcie->msix_enabled) {
1329 int i;
1330
1331 for (i = 0; i < trans_pcie->alloc_vecs; i++)
1332 synchronize_irq(trans_pcie->msix_entries[i].vector);
1333 } else {
1334 synchronize_irq(trans_pcie->pci_dev->irq);
1335 }
1336 }
1337
1338 int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
1339 const struct fw_img *fw, bool run_in_rfkill)
1340 {
1341 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1342 bool hw_rfkill;
1343 int ret;
1344
1345 /* This may fail if AMT took ownership of the device */
1346 if (iwl_pcie_prepare_card_hw(trans)) {
1347 IWL_WARN(trans, "Exit HW not ready\n");
1348 return -EIO;
1349 }
1350
1351 iwl_enable_rfkill_int(trans);
1352
1353 iwl_write32(trans, CSR_INT, 0xFFFFFFFF);
1354
1355 /*
1356 * We enabled the RF-Kill interrupt and the handler may very
1357 * well be running. Disable the interrupts to make sure no other
1358 * interrupt can be fired.
1359 */
1360 iwl_disable_interrupts(trans);
1361
1362 /* Make sure it finished running */
1363 iwl_pcie_synchronize_irqs(trans);
1364
1365 mutex_lock(&trans_pcie->mutex);
1366
1367 /* If platform's RF_KILL switch is NOT set to KILL */
1368 hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
1369 if (hw_rfkill && !run_in_rfkill) {
1370 ret = -ERFKILL;
1371 goto out;
1372 }
1373
1374 /* Someone called stop_device, don't try to start_fw */
1375 if (trans_pcie->is_down) {
1376 IWL_WARN(trans,
1377 "Can't start_fw since the HW hasn't been started\n");
1378 ret = -EIO;
1379 goto out;
1380 }
1381
1382 /* make sure rfkill handshake bits are cleared */
1383 iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
1384 iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR,
1385 CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);
1386
1387 /* clear (again), then enable host interrupts */
1388 iwl_write32(trans, CSR_INT, 0xFFFFFFFF);
1389
1390 ret = iwl_pcie_nic_init(trans);
1391 if (ret) {
1392 IWL_ERR(trans, "Unable to init nic\n");
1393 goto out;
1394 }
1395
1396 /*
1397 * Now, we load the firmware and don't want to be interrupted, even
1398 * by the RF-Kill interrupt (hence mask all the interrupt besides the
1399 * FH_TX interrupt which is needed to load the firmware). If the
1400 * RF-Kill switch is toggled, we will find out after having loaded
1401 * the firmware and return the proper value to the caller.
1402 */
1403 iwl_enable_fw_load_int(trans);
1404
1405 /* really make sure rfkill handshake bits are cleared */
1406 iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
1407 iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
1408
1409 /* Load the given image to the HW */
1410 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
1411 ret = iwl_pcie_load_given_ucode_8000(trans, fw);
1412 else
1413 ret = iwl_pcie_load_given_ucode(trans, fw);
1414
1415 /* re-check RF-Kill state since we may have missed the interrupt */
1416 hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
1417 if (hw_rfkill && !run_in_rfkill)
1418 ret = -ERFKILL;
1419
1420 out:
1421 mutex_unlock(&trans_pcie->mutex);
1422 return ret;
1423 }
1424
1425 void iwl_trans_pcie_fw_alive(struct iwl_trans *trans, u32 scd_addr)
1426 {
1427 iwl_pcie_reset_ict(trans);
1428 iwl_pcie_tx_start(trans, scd_addr);
1429 }
1430
1431 void iwl_trans_pcie_handle_stop_rfkill(struct iwl_trans *trans,
1432 bool was_in_rfkill)
1433 {
1434 bool hw_rfkill;
1435
1436 /*
1437 * Check again since the RF kill state may have changed while
1438 * all the interrupts were disabled, in this case we couldn't
1439 * receive the RF kill interrupt and update the state in the
1440 * op_mode.
1441 * Don't call the op_mode if the rkfill state hasn't changed.
1442 * This allows the op_mode to call stop_device from the rfkill
1443 * notification without endless recursion. Under very rare
1444 * circumstances, we might have a small recursion if the rfkill
1445 * state changed exactly now while we were called from stop_device.
1446 * This is very unlikely but can happen and is supported.
1447 */
1448 hw_rfkill = iwl_is_rfkill_set(trans);
1449 if (hw_rfkill) {
1450 set_bit(STATUS_RFKILL_HW, &trans->status);
1451 set_bit(STATUS_RFKILL_OPMODE, &trans->status);
1452 } else {
1453 clear_bit(STATUS_RFKILL_HW, &trans->status);
1454 clear_bit(STATUS_RFKILL_OPMODE, &trans->status);
1455 }
1456 if (hw_rfkill != was_in_rfkill)
1457 iwl_trans_pcie_rf_kill(trans, hw_rfkill, false);
1458 }
1459
1460 void iwl_trans_pcie_stop_device(struct iwl_trans *trans)
1461 {
1462 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1463 bool was_in_rfkill;
1464
1465 iwl_op_mode_time_point(trans->op_mode,
1466 IWL_FW_INI_TIME_POINT_HOST_DEVICE_DISABLE,
1467 NULL);
1468
1469 mutex_lock(&trans_pcie->mutex);
1470 trans_pcie->opmode_down = true;
1471 was_in_rfkill = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1472 _iwl_trans_pcie_stop_device(trans, false);
1473 iwl_trans_pcie_handle_stop_rfkill(trans, was_in_rfkill);
1474 mutex_unlock(&trans_pcie->mutex);
1475 }
1476
1477 void iwl_trans_pcie_rf_kill(struct iwl_trans *trans, bool state, bool from_irq)
1478 {
1479 struct iwl_trans_pcie __maybe_unused *trans_pcie =
1480 IWL_TRANS_GET_PCIE_TRANS(trans);
1481
1482 lockdep_assert_held(&trans_pcie->mutex);
1483
1484 IWL_WARN(trans, "reporting RF_KILL (radio %s)\n",
1485 state ? "disabled" : "enabled");
1486 if (iwl_op_mode_hw_rf_kill(trans->op_mode, state) &&
1487 !WARN_ON(trans->trans_cfg->gen2))
1488 _iwl_trans_pcie_stop_device(trans, from_irq);
1489 }
1490
1491 void iwl_pcie_d3_complete_suspend(struct iwl_trans *trans,
1492 bool test, bool reset)
1493 {
1494 iwl_disable_interrupts(trans);
1495
1496 /*
1497 * in testing mode, the host stays awake and the
1498 * hardware won't be reset (not even partially)
1499 */
1500 if (test)
1501 return;
1502
1503 iwl_pcie_disable_ict(trans);
1504
1505 iwl_pcie_synchronize_irqs(trans);
1506
1507 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
1508 iwl_clear_bit(trans, CSR_GP_CNTRL,
1509 CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ);
1510 iwl_clear_bit(trans, CSR_GP_CNTRL,
1511 CSR_GP_CNTRL_REG_FLAG_MAC_INIT);
1512 } else {
1513 iwl_clear_bit(trans, CSR_GP_CNTRL,
1514 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1515 iwl_clear_bit(trans, CSR_GP_CNTRL,
1516 CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
1517 }
1518
1519 if (reset) {
1520 /*
1521 * reset TX queues -- some of their registers reset during S3
1522 * so if we don't reset everything here the D3 image would try
1523 * to execute some invalid memory upon resume
1524 */
1525 iwl_trans_pcie_tx_reset(trans);
1526 }
1527
1528 iwl_pcie_set_pwr(trans, true);
1529 }
1530
1531 static int iwl_pcie_d3_handshake(struct iwl_trans *trans, bool suspend)
1532 {
1533 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1534 int ret;
1535
1536 if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_AX210)
1537 iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
1538 suspend ? UREG_DOORBELL_TO_ISR6_SUSPEND :
1539 UREG_DOORBELL_TO_ISR6_RESUME);
1540 else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
1541 iwl_write32(trans, CSR_IPC_SLEEP_CONTROL,
1542 suspend ? CSR_IPC_SLEEP_CONTROL_SUSPEND :
1543 CSR_IPC_SLEEP_CONTROL_RESUME);
1544 else
1545 return 0;
1546
1547 ret = wait_event_timeout(trans_pcie->sx_waitq,
1548 trans_pcie->sx_complete, 2 * HZ);
1549
1550 /* Invalidate it toward next suspend or resume */
1551 trans_pcie->sx_complete = false;
1552
1553 if (!ret) {
1554 IWL_ERR(trans, "Timeout %s D3\n",
1555 suspend ? "entering" : "exiting");
1556 return -ETIMEDOUT;
1557 }
1558
1559 return 0;
1560 }
1561
1562 int iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test, bool reset)
1563 {
1564 int ret;
1565
1566 if (!reset)
1567 /* Enable persistence mode to avoid reset */
1568 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
1569 CSR_HW_IF_CONFIG_REG_PERSIST_MODE);
1570
1571 ret = iwl_pcie_d3_handshake(trans, true);
1572 if (ret)
1573 return ret;
1574
1575 iwl_pcie_d3_complete_suspend(trans, test, reset);
1576
1577 return 0;
1578 }
1579
1580 int iwl_trans_pcie_d3_resume(struct iwl_trans *trans,
1581 enum iwl_d3_status *status,
1582 bool test, bool reset)
1583 {
1584 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1585 u32 val;
1586 int ret;
1587
1588 if (test) {
1589 iwl_enable_interrupts(trans);
1590 *status = IWL_D3_STATUS_ALIVE;
1591 ret = 0;
1592 goto out;
1593 }
1594
1595 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
1596 iwl_set_bit(trans, CSR_GP_CNTRL,
1597 CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ);
1598 else
1599 iwl_set_bit(trans, CSR_GP_CNTRL,
1600 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1601
1602 ret = iwl_finish_nic_init(trans);
1603 if (ret)
1604 return ret;
1605
1606 /*
1607 * Reconfigure IVAR table in case of MSIX or reset ict table in
1608 * MSI mode since HW reset erased it.
1609 * Also enables interrupts - none will happen as
1610 * the device doesn't know we're waking it up, only when
1611 * the opmode actually tells it after this call.
1612 */
1613 iwl_pcie_conf_msix_hw(trans_pcie);
1614 if (!trans_pcie->msix_enabled)
1615 iwl_pcie_reset_ict(trans);
1616 iwl_enable_interrupts(trans);
1617
1618 iwl_pcie_set_pwr(trans, false);
1619
1620 if (!reset) {
1621 iwl_clear_bit(trans, CSR_GP_CNTRL,
1622 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1623 } else {
1624 iwl_trans_pcie_tx_reset(trans);
1625
1626 ret = iwl_pcie_rx_init(trans);
1627 if (ret) {
1628 IWL_ERR(trans,
1629 "Failed to resume the device (RX reset)\n");
1630 return ret;
1631 }
1632 }
1633
1634 IWL_DEBUG_POWER(trans, "WFPM value upon resume = 0x%08X\n",
1635 iwl_read_umac_prph(trans, WFPM_GP2));
1636
1637 val = iwl_read32(trans, CSR_RESET);
1638 if (val & CSR_RESET_REG_FLAG_NEVO_RESET)
1639 *status = IWL_D3_STATUS_RESET;
1640 else
1641 *status = IWL_D3_STATUS_ALIVE;
1642
1643 out:
1644 if (*status == IWL_D3_STATUS_ALIVE)
1645 ret = iwl_pcie_d3_handshake(trans, false);
1646 else
1647 trans->state = IWL_TRANS_NO_FW;
1648
1649 return ret;
1650 }
1651
1652 static void
1653 iwl_pcie_set_interrupt_capa(struct pci_dev *pdev,
1654 struct iwl_trans *trans,
1655 const struct iwl_cfg_trans_params *cfg_trans)
1656 {
1657 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1658 int max_irqs, num_irqs, i, ret;
1659 u16 pci_cmd;
1660 u32 max_rx_queues = IWL_MAX_RX_HW_QUEUES;
1661
1662 if (!cfg_trans->mq_rx_supported)
1663 goto enable_msi;
1664
1665 if (cfg_trans->device_family <= IWL_DEVICE_FAMILY_9000)
1666 max_rx_queues = IWL_9000_MAX_RX_HW_QUEUES;
1667
1668 max_irqs = min_t(u32, num_online_cpus() + 2, max_rx_queues);
1669 for (i = 0; i < max_irqs; i++)
1670 trans_pcie->msix_entries[i].entry = i;
1671
1672 num_irqs = pci_enable_msix_range(pdev, trans_pcie->msix_entries,
1673 MSIX_MIN_INTERRUPT_VECTORS,
1674 max_irqs);
1675 if (num_irqs < 0) {
1676 IWL_DEBUG_INFO(trans,
1677 "Failed to enable msi-x mode (ret %d). Moving to msi mode.\n",
1678 num_irqs);
1679 goto enable_msi;
1680 }
1681 trans_pcie->def_irq = (num_irqs == max_irqs) ? num_irqs - 1 : 0;
1682
1683 IWL_DEBUG_INFO(trans,
1684 "MSI-X enabled. %d interrupt vectors were allocated\n",
1685 num_irqs);
1686
1687 /*
1688 * In case the OS provides fewer interrupts than requested, different
1689 * causes will share the same interrupt vector as follows:
1690 * One interrupt less: non rx causes shared with FBQ.
1691 * Two interrupts less: non rx causes shared with FBQ and RSS.
1692 * More than two interrupts: we will use fewer RSS queues.
1693 */
1694 if (num_irqs <= max_irqs - 2) {
1695 trans_pcie->trans->num_rx_queues = num_irqs + 1;
1696 trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX |
1697 IWL_SHARED_IRQ_FIRST_RSS;
1698 } else if (num_irqs == max_irqs - 1) {
1699 trans_pcie->trans->num_rx_queues = num_irqs;
1700 trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX;
1701 } else {
1702 trans_pcie->trans->num_rx_queues = num_irqs - 1;
1703 }
1704
1705 IWL_DEBUG_INFO(trans,
1706 "MSI-X enabled with rx queues %d, vec mask 0x%x\n",
1707 trans_pcie->trans->num_rx_queues, trans_pcie->shared_vec_mask);
1708
1709 WARN_ON(trans_pcie->trans->num_rx_queues > IWL_MAX_RX_HW_QUEUES);
1710
1711 trans_pcie->alloc_vecs = num_irqs;
1712 trans_pcie->msix_enabled = true;
1713 return;
1714
1715 enable_msi:
1716 ret = pci_enable_msi(pdev);
1717 if (ret) {
1718 dev_err(&pdev->dev, "pci_enable_msi failed - %d\n", ret);
1719 /* enable rfkill interrupt: hw bug w/a */
1720 pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd);
1721 if (pci_cmd & PCI_COMMAND_INTX_DISABLE) {
1722 pci_cmd &= ~PCI_COMMAND_INTX_DISABLE;
1723 pci_write_config_word(pdev, PCI_COMMAND, pci_cmd);
1724 }
1725 }
1726 }
1727
1728 static void iwl_pcie_irq_set_affinity(struct iwl_trans *trans)
1729 {
1730 #if defined(CONFIG_SMP)
1731 int iter_rx_q, i, ret, cpu, offset;
1732 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1733
1734 i = trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 0 : 1;
1735 iter_rx_q = trans_pcie->trans->num_rx_queues - 1 + i;
1736 offset = 1 + i;
1737 for (; i < iter_rx_q ; i++) {
1738 /*
1739 * Get the cpu prior to the place to search
1740 * (i.e. return will be > i - 1).
1741 */
1742 cpu = cpumask_next(i - offset, cpu_online_mask);
1743 cpumask_set_cpu(cpu, &trans_pcie->affinity_mask[i]);
1744 ret = irq_set_affinity_hint(trans_pcie->msix_entries[i].vector,
1745 &trans_pcie->affinity_mask[i]);
1746 if (ret)
1747 IWL_ERR(trans_pcie->trans,
1748 "Failed to set affinity mask for IRQ %d\n",
1749 trans_pcie->msix_entries[i].vector);
1750 }
1751 #endif
1752 }
1753
1754 static int iwl_pcie_init_msix_handler(struct pci_dev *pdev,
1755 struct iwl_trans_pcie *trans_pcie)
1756 {
1757 int i;
1758
1759 for (i = 0; i < trans_pcie->alloc_vecs; i++) {
1760 int ret;
1761 struct msix_entry *msix_entry;
1762 const char *qname = queue_name(&pdev->dev, trans_pcie, i);
1763
1764 if (!qname)
1765 return -ENOMEM;
1766
1767 msix_entry = &trans_pcie->msix_entries[i];
1768 ret = devm_request_threaded_irq(&pdev->dev,
1769 msix_entry->vector,
1770 iwl_pcie_msix_isr,
1771 (i == trans_pcie->def_irq) ?
1772 iwl_pcie_irq_msix_handler :
1773 iwl_pcie_irq_rx_msix_handler,
1774 IRQF_SHARED,
1775 qname,
1776 msix_entry);
1777 if (ret) {
1778 IWL_ERR(trans_pcie->trans,
1779 "Error allocating IRQ %d\n", i);
1780
1781 return ret;
1782 }
1783 }
1784 iwl_pcie_irq_set_affinity(trans_pcie->trans);
1785
1786 return 0;
1787 }
1788
1789 static int iwl_trans_pcie_clear_persistence_bit(struct iwl_trans *trans)
1790 {
1791 u32 hpm, wprot;
1792
1793 switch (trans->trans_cfg->device_family) {
1794 case IWL_DEVICE_FAMILY_9000:
1795 wprot = PREG_PRPH_WPROT_9000;
1796 break;
1797 case IWL_DEVICE_FAMILY_22000:
1798 wprot = PREG_PRPH_WPROT_22000;
1799 break;
1800 default:
1801 return 0;
1802 }
1803
1804 hpm = iwl_read_umac_prph_no_grab(trans, HPM_DEBUG);
1805 if (!iwl_trans_is_hw_error_value(hpm) && (hpm & PERSISTENCE_BIT)) {
1806 u32 wprot_val = iwl_read_umac_prph_no_grab(trans, wprot);
1807
1808 if (wprot_val & PREG_WFPM_ACCESS) {
1809 IWL_ERR(trans,
1810 "Error, can not clear persistence bit\n");
1811 return -EPERM;
1812 }
1813 iwl_write_umac_prph_no_grab(trans, HPM_DEBUG,
1814 hpm & ~PERSISTENCE_BIT);
1815 }
1816
1817 return 0;
1818 }
1819
1820 static int iwl_pcie_gen2_force_power_gating(struct iwl_trans *trans)
1821 {
1822 int ret;
1823
1824 ret = iwl_finish_nic_init(trans);
1825 if (ret < 0)
1826 return ret;
1827
1828 iwl_set_bits_prph(trans, HPM_HIPM_GEN_CFG,
1829 HPM_HIPM_GEN_CFG_CR_FORCE_ACTIVE);
1830 udelay(20);
1831 iwl_set_bits_prph(trans, HPM_HIPM_GEN_CFG,
1832 HPM_HIPM_GEN_CFG_CR_PG_EN |
1833 HPM_HIPM_GEN_CFG_CR_SLP_EN);
1834 udelay(20);
1835 iwl_clear_bits_prph(trans, HPM_HIPM_GEN_CFG,
1836 HPM_HIPM_GEN_CFG_CR_FORCE_ACTIVE);
1837
1838 return iwl_trans_pcie_sw_reset(trans, true);
1839 }
1840
1841 static int _iwl_trans_pcie_start_hw(struct iwl_trans *trans)
1842 {
1843 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1844 int err;
1845
1846 lockdep_assert_held(&trans_pcie->mutex);
1847
1848 err = iwl_pcie_prepare_card_hw(trans);
1849 if (err) {
1850 IWL_ERR(trans, "Error while preparing HW: %d\n", err);
1851 return err;
1852 }
1853
1854 err = iwl_trans_pcie_clear_persistence_bit(trans);
1855 if (err)
1856 return err;
1857
1858 err = iwl_trans_pcie_sw_reset(trans, true);
1859 if (err)
1860 return err;
1861
1862 if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_22000 &&
1863 trans->trans_cfg->integrated) {
1864 err = iwl_pcie_gen2_force_power_gating(trans);
1865 if (err)
1866 return err;
1867 }
1868
1869 err = iwl_pcie_apm_init(trans);
1870 if (err)
1871 return err;
1872
1873 iwl_pcie_init_msix(trans_pcie);
1874
1875 /* From now on, the op_mode will be kept updated about RF kill state */
1876 iwl_enable_rfkill_int(trans);
1877
1878 trans_pcie->opmode_down = false;
1879
1880 /* Set is_down to false here so that...*/
1881 trans_pcie->is_down = false;
1882
1883 /* ...rfkill can call stop_device and set it false if needed */
1884 iwl_pcie_check_hw_rf_kill(trans);
1885
1886 return 0;
1887 }
1888
1889 int iwl_trans_pcie_start_hw(struct iwl_trans *trans)
1890 {
1891 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1892 int ret;
1893
1894 mutex_lock(&trans_pcie->mutex);
1895 ret = _iwl_trans_pcie_start_hw(trans);
1896 mutex_unlock(&trans_pcie->mutex);
1897
1898 return ret;
1899 }
1900
1901 void iwl_trans_pcie_op_mode_leave(struct iwl_trans *trans)
1902 {
1903 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1904
1905 mutex_lock(&trans_pcie->mutex);
1906
1907 /* disable interrupts - don't enable HW RF kill interrupt */
1908 iwl_disable_interrupts(trans);
1909
1910 iwl_pcie_apm_stop(trans, true);
1911
1912 iwl_disable_interrupts(trans);
1913
1914 iwl_pcie_disable_ict(trans);
1915
1916 mutex_unlock(&trans_pcie->mutex);
1917
1918 iwl_pcie_synchronize_irqs(trans);
1919 }
1920
1921 void iwl_trans_pcie_write8(struct iwl_trans *trans, u32 ofs, u8 val)
1922 {
1923 writeb(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1924 }
1925
1926 void iwl_trans_pcie_write32(struct iwl_trans *trans, u32 ofs, u32 val)
1927 {
1928 writel(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1929 }
1930
1931 u32 iwl_trans_pcie_read32(struct iwl_trans *trans, u32 ofs)
1932 {
1933 return readl(IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1934 }
1935
1936 static u32 iwl_trans_pcie_prph_msk(struct iwl_trans *trans)
1937 {
1938 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
1939 return 0x00FFFFFF;
1940 else
1941 return 0x000FFFFF;
1942 }
1943
1944 u32 iwl_trans_pcie_read_prph(struct iwl_trans *trans, u32 reg)
1945 {
1946 u32 mask = iwl_trans_pcie_prph_msk(trans);
1947
1948 iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_RADDR,
1949 ((reg & mask) | (3 << 24)));
1950 return iwl_trans_pcie_read32(trans, HBUS_TARG_PRPH_RDAT);
1951 }
1952
1953 void iwl_trans_pcie_write_prph(struct iwl_trans *trans, u32 addr, u32 val)
1954 {
1955 u32 mask = iwl_trans_pcie_prph_msk(trans);
1956
1957 iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WADDR,
1958 ((addr & mask) | (3 << 24)));
1959 iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WDAT, val);
1960 }
1961
1962 void iwl_trans_pcie_configure(struct iwl_trans *trans,
1963 const struct iwl_trans_config *trans_cfg)
1964 {
1965 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1966
1967 /* free all first - we might be reconfigured for a different size */
1968 iwl_pcie_free_rbs_pool(trans);
1969
1970 trans_pcie->txqs.cmd.q_id = trans_cfg->cmd_queue;
1971 trans_pcie->txqs.cmd.fifo = trans_cfg->cmd_fifo;
1972 trans_pcie->txqs.page_offs = trans_cfg->cb_data_offs;
1973 trans_pcie->txqs.dev_cmd_offs = trans_cfg->cb_data_offs + sizeof(void *);
1974 trans_pcie->txqs.queue_alloc_cmd_ver = trans_cfg->queue_alloc_cmd_ver;
1975
1976 if (WARN_ON(trans_cfg->n_no_reclaim_cmds > MAX_NO_RECLAIM_CMDS))
1977 trans_pcie->n_no_reclaim_cmds = 0;
1978 else
1979 trans_pcie->n_no_reclaim_cmds = trans_cfg->n_no_reclaim_cmds;
1980 if (trans_pcie->n_no_reclaim_cmds)
1981 memcpy(trans_pcie->no_reclaim_cmds, trans_cfg->no_reclaim_cmds,
1982 trans_pcie->n_no_reclaim_cmds * sizeof(u8));
1983
1984 trans_pcie->rx_buf_size = trans_cfg->rx_buf_size;
1985 trans_pcie->rx_page_order =
1986 iwl_trans_get_rb_size_order(trans_pcie->rx_buf_size);
1987 trans_pcie->rx_buf_bytes =
1988 iwl_trans_get_rb_size(trans_pcie->rx_buf_size);
1989 trans_pcie->supported_dma_mask = DMA_BIT_MASK(12);
1990 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
1991 trans_pcie->supported_dma_mask = DMA_BIT_MASK(11);
1992
1993 trans_pcie->txqs.bc_table_dword = trans_cfg->bc_table_dword;
1994 trans_pcie->scd_set_active = trans_cfg->scd_set_active;
1995
1996 trans->command_groups = trans_cfg->command_groups;
1997 trans->command_groups_size = trans_cfg->command_groups_size;
1998
1999
2000 trans_pcie->fw_reset_handshake = trans_cfg->fw_reset_handshake;
2001 }
2002
2003 void iwl_trans_pcie_free_pnvm_dram_regions(struct iwl_dram_regions *dram_regions,
2004 struct device *dev)
2005 {
2006 u8 i;
2007 struct iwl_dram_data *desc_dram = &dram_regions->prph_scratch_mem_desc;
2008
2009 /* free DRAM payloads */
2010 for (i = 0; i < dram_regions->n_regions; i++) {
2011 dma_free_coherent(dev, dram_regions->drams[i].size,
2012 dram_regions->drams[i].block,
2013 dram_regions->drams[i].physical);
2014 }
2015 dram_regions->n_regions = 0;
2016
2017 /* free DRAM addresses array */
2018 if (desc_dram->block) {
2019 dma_free_coherent(dev, desc_dram->size,
2020 desc_dram->block,
2021 desc_dram->physical);
2022 }
2023 memset(desc_dram, 0, sizeof(*desc_dram));
2024 }
2025
2026 static void iwl_pcie_free_invalid_tx_cmd(struct iwl_trans *trans)
2027 {
2028 iwl_pcie_free_dma_ptr(trans, &trans->invalid_tx_cmd);
2029 }
2030
2031 static int iwl_pcie_alloc_invalid_tx_cmd(struct iwl_trans *trans)
2032 {
2033 struct iwl_cmd_header_wide bad_cmd = {
2034 .cmd = INVALID_WR_PTR_CMD,
2035 .group_id = DEBUG_GROUP,
2036 .sequence = cpu_to_le16(0xffff),
2037 .length = cpu_to_le16(0),
2038 .version = 0,
2039 };
2040 int ret;
2041
2042 ret = iwl_pcie_alloc_dma_ptr(trans, &trans->invalid_tx_cmd,
2043 sizeof(bad_cmd));
2044 if (ret)
2045 return ret;
2046 memcpy(trans->invalid_tx_cmd.addr, &bad_cmd, sizeof(bad_cmd));
2047 return 0;
2048 }
2049
2050 void iwl_trans_pcie_free(struct iwl_trans *trans)
2051 {
2052 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2053 int i;
2054
2055 iwl_pcie_synchronize_irqs(trans);
2056
2057 if (trans->trans_cfg->gen2)
2058 iwl_txq_gen2_tx_free(trans);
2059 else
2060 iwl_pcie_tx_free(trans);
2061 iwl_pcie_rx_free(trans);
2062
2063 if (trans_pcie->rba.alloc_wq) {
2064 destroy_workqueue(trans_pcie->rba.alloc_wq);
2065 trans_pcie->rba.alloc_wq = NULL;
2066 }
2067
2068 if (trans_pcie->msix_enabled) {
2069 for (i = 0; i < trans_pcie->alloc_vecs; i++) {
2070 irq_set_affinity_hint(
2071 trans_pcie->msix_entries[i].vector,
2072 NULL);
2073 }
2074
2075 trans_pcie->msix_enabled = false;
2076 } else {
2077 iwl_pcie_free_ict(trans);
2078 }
2079
2080 free_netdev(trans_pcie->napi_dev);
2081
2082 iwl_pcie_free_invalid_tx_cmd(trans);
2083
2084 iwl_pcie_free_fw_monitor(trans);
2085
2086 iwl_trans_pcie_free_pnvm_dram_regions(&trans_pcie->pnvm_data,
2087 trans->dev);
2088 iwl_trans_pcie_free_pnvm_dram_regions(&trans_pcie->reduced_tables_data,
2089 trans->dev);
2090
2091 mutex_destroy(&trans_pcie->mutex);
2092
2093 if (trans_pcie->txqs.tso_hdr_page) {
2094 for_each_possible_cpu(i) {
2095 struct iwl_tso_hdr_page *p =
2096 per_cpu_ptr(trans_pcie->txqs.tso_hdr_page, i);
2097
2098 if (p && p->page)
2099 __free_page(p->page);
2100 }
2101
2102 free_percpu(trans_pcie->txqs.tso_hdr_page);
2103 }
2104
2105 iwl_trans_free(trans);
2106 }
2107
2108 struct iwl_trans_pcie_removal {
2109 struct pci_dev *pdev;
2110 struct work_struct work;
2111 bool rescan;
2112 };
2113
2114 static void iwl_trans_pcie_removal_wk(struct work_struct *wk)
2115 {
2116 struct iwl_trans_pcie_removal *removal =
2117 container_of(wk, struct iwl_trans_pcie_removal, work);
2118 struct pci_dev *pdev = removal->pdev;
2119 static char *prop[] = {"EVENT=INACCESSIBLE", NULL};
2120 struct pci_bus *bus;
2121
2122 pci_lock_rescan_remove();
2123
2124 bus = pdev->bus;
2125 /* in this case, something else already removed the device */
2126 if (!bus)
2127 goto out;
2128
2129 dev_err(&pdev->dev, "Device gone - attempting removal\n");
2130
2131 kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, prop);
2132
2133 pci_stop_and_remove_bus_device(pdev);
2134 pci_dev_put(pdev);
2135
2136 if (removal->rescan) {
2137 if (bus->parent)
2138 bus = bus->parent;
2139 pci_rescan_bus(bus);
2140 }
2141
2142 out:
2143 pci_unlock_rescan_remove();
2144
2145 kfree(removal);
2146 module_put(THIS_MODULE);
2147 }
2148
2149 void iwl_trans_pcie_remove(struct iwl_trans *trans, bool rescan)
2150 {
2151 struct iwl_trans_pcie_removal *removal;
2152
2153 if (test_bit(STATUS_TRANS_DEAD, &trans->status))
2154 return;
2155
2156 IWL_ERR(trans, "Device gone - scheduling removal!\n");
2157 iwl_pcie_dump_csr(trans);
2158
2159 /*
2160 * get a module reference to avoid doing this
2161 * while unloading anyway and to avoid
2162 * scheduling a work with code that's being
2163 * removed.
2164 */
2165 if (!try_module_get(THIS_MODULE)) {
2166 IWL_ERR(trans,
2167 "Module is being unloaded - abort\n");
2168 return;
2169 }
2170
2171 removal = kzalloc(sizeof(*removal), GFP_ATOMIC);
2172 if (!removal) {
2173 module_put(THIS_MODULE);
2174 return;
2175 }
2176 /*
2177 * we don't need to clear this flag, because
2178 * the trans will be freed and reallocated.
2179 */
2180 set_bit(STATUS_TRANS_DEAD, &trans->status);
2181
2182 removal->pdev = to_pci_dev(trans->dev);
2183 removal->rescan = rescan;
2184 INIT_WORK(&removal->work, iwl_trans_pcie_removal_wk);
2185 pci_dev_get(removal->pdev);
2186 schedule_work(&removal->work);
2187 }
2188 EXPORT_SYMBOL(iwl_trans_pcie_remove);
2189
2190 /*
2191 * This version doesn't disable BHs but rather assumes they're
2192 * already disabled.
2193 */
2194 bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans)
2195 {
2196 int ret;
2197 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2198 u32 write = CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ;
2199 u32 mask = CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY |
2200 CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP;
2201 u32 poll = CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN;
2202
2203 if (test_bit(STATUS_TRANS_DEAD, &trans->status))
2204 return false;
2205
2206 spin_lock(&trans_pcie->reg_lock);
2207
2208 if (trans_pcie->cmd_hold_nic_awake)
2209 goto out;
2210
2211 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ) {
2212 write = CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ;
2213 mask = CSR_GP_CNTRL_REG_FLAG_MAC_STATUS;
2214 poll = CSR_GP_CNTRL_REG_FLAG_MAC_STATUS;
2215 }
2216
2217 /* this bit wakes up the NIC */
2218 __iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL, write);
2219 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_8000)
2220 udelay(2);
2221
2222 /*
2223 * These bits say the device is running, and should keep running for
2224 * at least a short while (at least as long as MAC_ACCESS_REQ stays 1),
2225 * but they do not indicate that embedded SRAM is restored yet;
2226 * HW with volatile SRAM must save/restore contents to/from
2227 * host DRAM when sleeping/waking for power-saving.
2228 * Each direction takes approximately 1/4 millisecond; with this
2229 * overhead, it's a good idea to grab and hold MAC_ACCESS_REQUEST if a
2230 * series of register accesses are expected (e.g. reading Event Log),
2231 * to keep device from sleeping.
2232 *
2233 * CSR_UCODE_DRV_GP1 register bit MAC_SLEEP == 0 indicates that
2234 * SRAM is okay/restored. We don't check that here because this call
2235 * is just for hardware register access; but GP1 MAC_SLEEP
2236 * check is a good idea before accessing the SRAM of HW with
2237 * volatile SRAM (e.g. reading Event Log).
2238 *
2239 * 5000 series and later (including 1000 series) have non-volatile SRAM,
2240 * and do not save/restore SRAM when power cycling.
2241 */
2242 ret = iwl_poll_bit(trans, CSR_GP_CNTRL, poll, mask, 15000);
2243 if (unlikely(ret < 0)) {
2244 u32 cntrl = iwl_read32(trans, CSR_GP_CNTRL);
2245
2246 WARN_ONCE(1,
2247 "Timeout waiting for hardware access (CSR_GP_CNTRL 0x%08x)\n",
2248 cntrl);
2249
2250 iwl_trans_pcie_dump_regs(trans);
2251
2252 if (iwlwifi_mod_params.remove_when_gone && cntrl == ~0U)
2253 iwl_trans_pcie_remove(trans, false);
2254 else
2255 iwl_write32(trans, CSR_RESET,
2256 CSR_RESET_REG_FLAG_FORCE_NMI);
2257
2258 spin_unlock(&trans_pcie->reg_lock);
2259 return false;
2260 }
2261
2262 out:
2263 /*
2264 * Fool sparse by faking we release the lock - sparse will
2265 * track nic_access anyway.
2266 */
2267 __release(&trans_pcie->reg_lock);
2268 return true;
2269 }
2270
2271 bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans)
2272 {
2273 bool ret;
2274
2275 local_bh_disable();
2276 ret = __iwl_trans_pcie_grab_nic_access(trans);
2277 if (ret) {
2278 /* keep BHs disabled until iwl_trans_pcie_release_nic_access */
2279 return ret;
2280 }
2281 local_bh_enable();
2282 return false;
2283 }
2284
2285 void iwl_trans_pcie_release_nic_access(struct iwl_trans *trans)
2286 {
2287 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2288
2289 lockdep_assert_held(&trans_pcie->reg_lock);
2290
2291 /*
2292 * Fool sparse by faking we acquiring the lock - sparse will
2293 * track nic_access anyway.
2294 */
2295 __acquire(&trans_pcie->reg_lock);
2296
2297 if (trans_pcie->cmd_hold_nic_awake)
2298 goto out;
2299 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
2300 __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
2301 CSR_GP_CNTRL_REG_FLAG_BZ_MAC_ACCESS_REQ);
2302 else
2303 __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
2304 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
2305 /*
2306 * Above we read the CSR_GP_CNTRL register, which will flush
2307 * any previous writes, but we need the write that clears the
2308 * MAC_ACCESS_REQ bit to be performed before any other writes
2309 * scheduled on different CPUs (after we drop reg_lock).
2310 */
2311 out:
2312 spin_unlock_bh(&trans_pcie->reg_lock);
2313 }
2314
2315 int iwl_trans_pcie_read_mem(struct iwl_trans *trans, u32 addr,
2316 void *buf, int dwords)
2317 {
2318 #define IWL_MAX_HW_ERRS 5
2319 unsigned int num_consec_hw_errors = 0;
2320 int offs = 0;
2321 u32 *vals = buf;
2322
2323 while (offs < dwords) {
2324 /* limit the time we spin here under lock to 1/2s */
2325 unsigned long end = jiffies + HZ / 2;
2326 bool resched = false;
2327
2328 if (iwl_trans_grab_nic_access(trans)) {
2329 iwl_write32(trans, HBUS_TARG_MEM_RADDR,
2330 addr + 4 * offs);
2331
2332 while (offs < dwords) {
2333 vals[offs] = iwl_read32(trans,
2334 HBUS_TARG_MEM_RDAT);
2335
2336 if (iwl_trans_is_hw_error_value(vals[offs]))
2337 num_consec_hw_errors++;
2338 else
2339 num_consec_hw_errors = 0;
2340
2341 if (num_consec_hw_errors >= IWL_MAX_HW_ERRS) {
2342 iwl_trans_release_nic_access(trans);
2343 return -EIO;
2344 }
2345
2346 offs++;
2347
2348 if (time_after(jiffies, end)) {
2349 resched = true;
2350 break;
2351 }
2352 }
2353 iwl_trans_release_nic_access(trans);
2354
2355 if (resched)
2356 cond_resched();
2357 } else {
2358 return -EBUSY;
2359 }
2360 }
2361
2362 return 0;
2363 }
2364
2365 int iwl_trans_pcie_write_mem(struct iwl_trans *trans, u32 addr,
2366 const void *buf, int dwords)
2367 {
2368 int offs, ret = 0;
2369 const u32 *vals = buf;
2370
2371 if (iwl_trans_grab_nic_access(trans)) {
2372 iwl_write32(trans, HBUS_TARG_MEM_WADDR, addr);
2373 for (offs = 0; offs < dwords; offs++)
2374 iwl_write32(trans, HBUS_TARG_MEM_WDAT,
2375 vals ? vals[offs] : 0);
2376 iwl_trans_release_nic_access(trans);
2377 } else {
2378 ret = -EBUSY;
2379 }
2380 return ret;
2381 }
2382
2383 int iwl_trans_pcie_read_config32(struct iwl_trans *trans, u32 ofs,
2384 u32 *val)
2385 {
2386 return pci_read_config_dword(IWL_TRANS_GET_PCIE_TRANS(trans)->pci_dev,
2387 ofs, val);
2388 }
2389
2390 #define IWL_FLUSH_WAIT_MS 2000
2391
2392 int iwl_trans_pcie_rxq_dma_data(struct iwl_trans *trans, int queue,
2393 struct iwl_trans_rxq_dma_data *data)
2394 {
2395 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2396
2397 if (queue >= trans->num_rx_queues || !trans_pcie->rxq)
2398 return -EINVAL;
2399
2400 data->fr_bd_cb = trans_pcie->rxq[queue].bd_dma;
2401 data->urbd_stts_wrptr = trans_pcie->rxq[queue].rb_stts_dma;
2402 data->ur_bd_cb = trans_pcie->rxq[queue].used_bd_dma;
2403 data->fr_bd_wid = 0;
2404
2405 return 0;
2406 }
2407
2408 int iwl_trans_pcie_wait_txq_empty(struct iwl_trans *trans, int txq_idx)
2409 {
2410 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2411 struct iwl_txq *txq;
2412 unsigned long now = jiffies;
2413 bool overflow_tx;
2414 u8 wr_ptr;
2415
2416 /* Make sure the NIC is still alive in the bus */
2417 if (test_bit(STATUS_TRANS_DEAD, &trans->status))
2418 return -ENODEV;
2419
2420 if (!test_bit(txq_idx, trans_pcie->txqs.queue_used))
2421 return -EINVAL;
2422
2423 IWL_DEBUG_TX_QUEUES(trans, "Emptying queue %d...\n", txq_idx);
2424 txq = trans_pcie->txqs.txq[txq_idx];
2425
2426 spin_lock_bh(&txq->lock);
2427 overflow_tx = txq->overflow_tx ||
2428 !skb_queue_empty(&txq->overflow_q);
2429 spin_unlock_bh(&txq->lock);
2430
2431 wr_ptr = READ_ONCE(txq->write_ptr);
2432
2433 while ((txq->read_ptr != READ_ONCE(txq->write_ptr) ||
2434 overflow_tx) &&
2435 !time_after(jiffies,
2436 now + msecs_to_jiffies(IWL_FLUSH_WAIT_MS))) {
2437 u8 write_ptr = READ_ONCE(txq->write_ptr);
2438
2439 /*
2440 * If write pointer moved during the wait, warn only
2441 * if the TX came from op mode. In case TX came from
2442 * trans layer (overflow TX) don't warn.
2443 */
2444 if (WARN_ONCE(wr_ptr != write_ptr && !overflow_tx,
2445 "WR pointer moved while flushing %d -> %d\n",
2446 wr_ptr, write_ptr))
2447 return -ETIMEDOUT;
2448 wr_ptr = write_ptr;
2449
2450 usleep_range(1000, 2000);
2451
2452 spin_lock_bh(&txq->lock);
2453 overflow_tx = txq->overflow_tx ||
2454 !skb_queue_empty(&txq->overflow_q);
2455 spin_unlock_bh(&txq->lock);
2456 }
2457
2458 if (txq->read_ptr != txq->write_ptr) {
2459 IWL_ERR(trans,
2460 "fail to flush all tx fifo queues Q %d\n", txq_idx);
2461 iwl_txq_log_scd_error(trans, txq);
2462 return -ETIMEDOUT;
2463 }
2464
2465 IWL_DEBUG_TX_QUEUES(trans, "Queue %d is now empty.\n", txq_idx);
2466
2467 return 0;
2468 }
2469
2470 int iwl_trans_pcie_wait_txqs_empty(struct iwl_trans *trans, u32 txq_bm)
2471 {
2472 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2473 int cnt;
2474 int ret = 0;
2475
2476 /* waiting for all the tx frames complete might take a while */
2477 for (cnt = 0;
2478 cnt < trans->trans_cfg->base_params->num_of_queues;
2479 cnt++) {
2480
2481 if (cnt == trans_pcie->txqs.cmd.q_id)
2482 continue;
2483 if (!test_bit(cnt, trans_pcie->txqs.queue_used))
2484 continue;
2485 if (!(BIT(cnt) & txq_bm))
2486 continue;
2487
2488 ret = iwl_trans_pcie_wait_txq_empty(trans, cnt);
2489 if (ret)
2490 break;
2491 }
2492
2493 return ret;
2494 }
2495
2496 void iwl_trans_pcie_set_bits_mask(struct iwl_trans *trans, u32 reg,
2497 u32 mask, u32 value)
2498 {
2499 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2500
2501 spin_lock_bh(&trans_pcie->reg_lock);
2502 __iwl_trans_pcie_set_bits_mask(trans, reg, mask, value);
2503 spin_unlock_bh(&trans_pcie->reg_lock);
2504 }
2505
2506 static const char *get_csr_string(int cmd)
2507 {
2508 #define IWL_CMD(x) case x: return #x
2509 switch (cmd) {
2510 IWL_CMD(CSR_HW_IF_CONFIG_REG);
2511 IWL_CMD(CSR_INT_COALESCING);
2512 IWL_CMD(CSR_INT);
2513 IWL_CMD(CSR_INT_MASK);
2514 IWL_CMD(CSR_FH_INT_STATUS);
2515 IWL_CMD(CSR_GPIO_IN);
2516 IWL_CMD(CSR_RESET);
2517 IWL_CMD(CSR_GP_CNTRL);
2518 IWL_CMD(CSR_HW_REV);
2519 IWL_CMD(CSR_EEPROM_REG);
2520 IWL_CMD(CSR_EEPROM_GP);
2521 IWL_CMD(CSR_OTP_GP_REG);
2522 IWL_CMD(CSR_GIO_REG);
2523 IWL_CMD(CSR_GP_UCODE_REG);
2524 IWL_CMD(CSR_GP_DRIVER_REG);
2525 IWL_CMD(CSR_UCODE_DRV_GP1);
2526 IWL_CMD(CSR_UCODE_DRV_GP2);
2527 IWL_CMD(CSR_LED_REG);
2528 IWL_CMD(CSR_DRAM_INT_TBL_REG);
2529 IWL_CMD(CSR_GIO_CHICKEN_BITS);
2530 IWL_CMD(CSR_ANA_PLL_CFG);
2531 IWL_CMD(CSR_HW_REV_WA_REG);
2532 IWL_CMD(CSR_MONITOR_STATUS_REG);
2533 IWL_CMD(CSR_DBG_HPET_MEM_REG);
2534 default:
2535 return "UNKNOWN";
2536 }
2537 #undef IWL_CMD
2538 }
2539
2540 void iwl_pcie_dump_csr(struct iwl_trans *trans)
2541 {
2542 int i;
2543 static const u32 csr_tbl[] = {
2544 CSR_HW_IF_CONFIG_REG,
2545 CSR_INT_COALESCING,
2546 CSR_INT,
2547 CSR_INT_MASK,
2548 CSR_FH_INT_STATUS,
2549 CSR_GPIO_IN,
2550 CSR_RESET,
2551 CSR_GP_CNTRL,
2552 CSR_HW_REV,
2553 CSR_EEPROM_REG,
2554 CSR_EEPROM_GP,
2555 CSR_OTP_GP_REG,
2556 CSR_GIO_REG,
2557 CSR_GP_UCODE_REG,
2558 CSR_GP_DRIVER_REG,
2559 CSR_UCODE_DRV_GP1,
2560 CSR_UCODE_DRV_GP2,
2561 CSR_LED_REG,
2562 CSR_DRAM_INT_TBL_REG,
2563 CSR_GIO_CHICKEN_BITS,
2564 CSR_ANA_PLL_CFG,
2565 CSR_MONITOR_STATUS_REG,
2566 CSR_HW_REV_WA_REG,
2567 CSR_DBG_HPET_MEM_REG
2568 };
2569 IWL_ERR(trans, "CSR values:\n");
2570 IWL_ERR(trans, "(2nd byte of CSR_INT_COALESCING is "
2571 "CSR_INT_PERIODIC_REG)\n");
2572 for (i = 0; i < ARRAY_SIZE(csr_tbl); i++) {
2573 IWL_ERR(trans, " %25s: 0X%08x\n",
2574 get_csr_string(csr_tbl[i]),
2575 iwl_read32(trans, csr_tbl[i]));
2576 }
2577 }
2578
2579 #ifdef CONFIG_IWLWIFI_DEBUGFS
2580 /* create and remove of files */
2581 #define DEBUGFS_ADD_FILE(name, parent, mode) do { \
2582 debugfs_create_file(#name, mode, parent, trans, \
2583 &iwl_dbgfs_##name##_ops); \
2584 } while (0)
2585
2586 /* file operation */
2587 #define DEBUGFS_READ_FILE_OPS(name) \
2588 static const struct file_operations iwl_dbgfs_##name##_ops = { \
2589 .read = iwl_dbgfs_##name##_read, \
2590 .open = simple_open, \
2591 .llseek = generic_file_llseek, \
2592 };
2593
2594 #define DEBUGFS_WRITE_FILE_OPS(name) \
2595 static const struct file_operations iwl_dbgfs_##name##_ops = { \
2596 .write = iwl_dbgfs_##name##_write, \
2597 .open = simple_open, \
2598 .llseek = generic_file_llseek, \
2599 };
2600
2601 #define DEBUGFS_READ_WRITE_FILE_OPS(name) \
2602 static const struct file_operations iwl_dbgfs_##name##_ops = { \
2603 .write = iwl_dbgfs_##name##_write, \
2604 .read = iwl_dbgfs_##name##_read, \
2605 .open = simple_open, \
2606 .llseek = generic_file_llseek, \
2607 };
2608
2609 struct iwl_dbgfs_tx_queue_priv {
2610 struct iwl_trans *trans;
2611 };
2612
2613 struct iwl_dbgfs_tx_queue_state {
2614 loff_t pos;
2615 };
2616
2617 static void *iwl_dbgfs_tx_queue_seq_start(struct seq_file *seq, loff_t *pos)
2618 {
2619 struct iwl_dbgfs_tx_queue_priv *priv = seq->private;
2620 struct iwl_dbgfs_tx_queue_state *state;
2621
2622 if (*pos >= priv->trans->trans_cfg->base_params->num_of_queues)
2623 return NULL;
2624
2625 state = kmalloc(sizeof(*state), GFP_KERNEL);
2626 if (!state)
2627 return NULL;
2628 state->pos = *pos;
2629 return state;
2630 }
2631
2632 static void *iwl_dbgfs_tx_queue_seq_next(struct seq_file *seq,
2633 void *v, loff_t *pos)
2634 {
2635 struct iwl_dbgfs_tx_queue_priv *priv = seq->private;
2636 struct iwl_dbgfs_tx_queue_state *state = v;
2637
2638 *pos = ++state->pos;
2639
2640 if (*pos >= priv->trans->trans_cfg->base_params->num_of_queues)
2641 return NULL;
2642
2643 return state;
2644 }
2645
2646 static void iwl_dbgfs_tx_queue_seq_stop(struct seq_file *seq, void *v)
2647 {
2648 kfree(v);
2649 }
2650
2651 static int iwl_dbgfs_tx_queue_seq_show(struct seq_file *seq, void *v)
2652 {
2653 struct iwl_dbgfs_tx_queue_priv *priv = seq->private;
2654 struct iwl_dbgfs_tx_queue_state *state = v;
2655 struct iwl_trans *trans = priv->trans;
2656 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2657 struct iwl_txq *txq = trans_pcie->txqs.txq[state->pos];
2658
2659 seq_printf(seq, "hwq %.3u: used=%d stopped=%d ",
2660 (unsigned int)state->pos,
2661 !!test_bit(state->pos, trans_pcie->txqs.queue_used),
2662 !!test_bit(state->pos, trans_pcie->txqs.queue_stopped));
2663 if (txq)
2664 seq_printf(seq,
2665 "read=%u write=%u need_update=%d frozen=%d n_window=%d ampdu=%d",
2666 txq->read_ptr, txq->write_ptr,
2667 txq->need_update, txq->frozen,
2668 txq->n_window, txq->ampdu);
2669 else
2670 seq_puts(seq, "(unallocated)");
2671
2672 if (state->pos == trans_pcie->txqs.cmd.q_id)
2673 seq_puts(seq, " (HCMD)");
2674 seq_puts(seq, "\n");
2675
2676 return 0;
2677 }
2678
2679 static const struct seq_operations iwl_dbgfs_tx_queue_seq_ops = {
2680 .start = iwl_dbgfs_tx_queue_seq_start,
2681 .next = iwl_dbgfs_tx_queue_seq_next,
2682 .stop = iwl_dbgfs_tx_queue_seq_stop,
2683 .show = iwl_dbgfs_tx_queue_seq_show,
2684 };
2685
2686 static int iwl_dbgfs_tx_queue_open(struct inode *inode, struct file *filp)
2687 {
2688 struct iwl_dbgfs_tx_queue_priv *priv;
2689
2690 priv = __seq_open_private(filp, &iwl_dbgfs_tx_queue_seq_ops,
2691 sizeof(*priv));
2692
2693 if (!priv)
2694 return -ENOMEM;
2695
2696 priv->trans = inode->i_private;
2697 return 0;
2698 }
2699
2700 static ssize_t iwl_dbgfs_rx_queue_read(struct file *file,
2701 char __user *user_buf,
2702 size_t count, loff_t *ppos)
2703 {
2704 struct iwl_trans *trans = file->private_data;
2705 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2706 char *buf;
2707 int pos = 0, i, ret;
2708 size_t bufsz;
2709
2710 bufsz = sizeof(char) * 121 * trans->num_rx_queues;
2711
2712 if (!trans_pcie->rxq)
2713 return -EAGAIN;
2714
2715 buf = kzalloc(bufsz, GFP_KERNEL);
2716 if (!buf)
2717 return -ENOMEM;
2718
2719 for (i = 0; i < trans->num_rx_queues && pos < bufsz; i++) {
2720 struct iwl_rxq *rxq = &trans_pcie->rxq[i];
2721
2722 pos += scnprintf(buf + pos, bufsz - pos, "queue#: %2d\n",
2723 i);
2724 pos += scnprintf(buf + pos, bufsz - pos, "\tread: %u\n",
2725 rxq->read);
2726 pos += scnprintf(buf + pos, bufsz - pos, "\twrite: %u\n",
2727 rxq->write);
2728 pos += scnprintf(buf + pos, bufsz - pos, "\twrite_actual: %u\n",
2729 rxq->write_actual);
2730 pos += scnprintf(buf + pos, bufsz - pos, "\tneed_update: %2d\n",
2731 rxq->need_update);
2732 pos += scnprintf(buf + pos, bufsz - pos, "\tfree_count: %u\n",
2733 rxq->free_count);
2734 if (rxq->rb_stts) {
2735 u32 r = iwl_get_closed_rb_stts(trans, rxq);
2736 pos += scnprintf(buf + pos, bufsz - pos,
2737 "\tclosed_rb_num: %u\n", r);
2738 } else {
2739 pos += scnprintf(buf + pos, bufsz - pos,
2740 "\tclosed_rb_num: Not Allocated\n");
2741 }
2742 }
2743 ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
2744 kfree(buf);
2745
2746 return ret;
2747 }
2748
2749 static ssize_t iwl_dbgfs_interrupt_read(struct file *file,
2750 char __user *user_buf,
2751 size_t count, loff_t *ppos)
2752 {
2753 struct iwl_trans *trans = file->private_data;
2754 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2755 struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
2756
2757 int pos = 0;
2758 char *buf;
2759 int bufsz = 24 * 64; /* 24 items * 64 char per item */
2760 ssize_t ret;
2761
2762 buf = kzalloc(bufsz, GFP_KERNEL);
2763 if (!buf)
2764 return -ENOMEM;
2765
2766 pos += scnprintf(buf + pos, bufsz - pos,
2767 "Interrupt Statistics Report:\n");
2768
2769 pos += scnprintf(buf + pos, bufsz - pos, "HW Error:\t\t\t %u\n",
2770 isr_stats->hw);
2771 pos += scnprintf(buf + pos, bufsz - pos, "SW Error:\t\t\t %u\n",
2772 isr_stats->sw);
2773 if (isr_stats->sw || isr_stats->hw) {
2774 pos += scnprintf(buf + pos, bufsz - pos,
2775 "\tLast Restarting Code: 0x%X\n",
2776 isr_stats->err_code);
2777 }
2778 #ifdef CONFIG_IWLWIFI_DEBUG
2779 pos += scnprintf(buf + pos, bufsz - pos, "Frame transmitted:\t\t %u\n",
2780 isr_stats->sch);
2781 pos += scnprintf(buf + pos, bufsz - pos, "Alive interrupt:\t\t %u\n",
2782 isr_stats->alive);
2783 #endif
2784 pos += scnprintf(buf + pos, bufsz - pos,
2785 "HW RF KILL switch toggled:\t %u\n", isr_stats->rfkill);
2786
2787 pos += scnprintf(buf + pos, bufsz - pos, "CT KILL:\t\t\t %u\n",
2788 isr_stats->ctkill);
2789
2790 pos += scnprintf(buf + pos, bufsz - pos, "Wakeup Interrupt:\t\t %u\n",
2791 isr_stats->wakeup);
2792
2793 pos += scnprintf(buf + pos, bufsz - pos,
2794 "Rx command responses:\t\t %u\n", isr_stats->rx);
2795
2796 pos += scnprintf(buf + pos, bufsz - pos, "Tx/FH interrupt:\t\t %u\n",
2797 isr_stats->tx);
2798
2799 pos += scnprintf(buf + pos, bufsz - pos, "Unexpected INTA:\t\t %u\n",
2800 isr_stats->unhandled);
2801
2802 ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
2803 kfree(buf);
2804 return ret;
2805 }
2806
2807 static ssize_t iwl_dbgfs_interrupt_write(struct file *file,
2808 const char __user *user_buf,
2809 size_t count, loff_t *ppos)
2810 {
2811 struct iwl_trans *trans = file->private_data;
2812 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2813 struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
2814 u32 reset_flag;
2815 int ret;
2816
2817 ret = kstrtou32_from_user(user_buf, count, 16, &reset_flag);
2818 if (ret)
2819 return ret;
2820 if (reset_flag == 0)
2821 memset(isr_stats, 0, sizeof(*isr_stats));
2822
2823 return count;
2824 }
2825
2826 static ssize_t iwl_dbgfs_csr_write(struct file *file,
2827 const char __user *user_buf,
2828 size_t count, loff_t *ppos)
2829 {
2830 struct iwl_trans *trans = file->private_data;
2831
2832 iwl_pcie_dump_csr(trans);
2833
2834 return count;
2835 }
2836
2837 static ssize_t iwl_dbgfs_fh_reg_read(struct file *file,
2838 char __user *user_buf,
2839 size_t count, loff_t *ppos)
2840 {
2841 struct iwl_trans *trans = file->private_data;
2842 char *buf = NULL;
2843 ssize_t ret;
2844
2845 ret = iwl_dump_fh(trans, &buf);
2846 if (ret < 0)
2847 return ret;
2848 if (!buf)
2849 return -EINVAL;
2850 ret = simple_read_from_buffer(user_buf, count, ppos, buf, ret);
2851 kfree(buf);
2852 return ret;
2853 }
2854
2855 static ssize_t iwl_dbgfs_rfkill_read(struct file *file,
2856 char __user *user_buf,
2857 size_t count, loff_t *ppos)
2858 {
2859 struct iwl_trans *trans = file->private_data;
2860 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2861 char buf[100];
2862 int pos;
2863
2864 pos = scnprintf(buf, sizeof(buf), "debug: %d\nhw: %d\n",
2865 trans_pcie->debug_rfkill,
2866 !(iwl_read32(trans, CSR_GP_CNTRL) &
2867 CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW));
2868
2869 return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
2870 }
2871
2872 static ssize_t iwl_dbgfs_rfkill_write(struct file *file,
2873 const char __user *user_buf,
2874 size_t count, loff_t *ppos)
2875 {
2876 struct iwl_trans *trans = file->private_data;
2877 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2878 bool new_value;
2879 int ret;
2880
2881 ret = kstrtobool_from_user(user_buf, count, &new_value);
2882 if (ret)
2883 return ret;
2884 if (new_value == trans_pcie->debug_rfkill)
2885 return count;
2886 IWL_WARN(trans, "changing debug rfkill %d->%d\n",
2887 trans_pcie->debug_rfkill, new_value);
2888 trans_pcie->debug_rfkill = new_value;
2889 iwl_pcie_handle_rfkill_irq(trans, false);
2890
2891 return count;
2892 }
2893
2894 static int iwl_dbgfs_monitor_data_open(struct inode *inode,
2895 struct file *file)
2896 {
2897 struct iwl_trans *trans = inode->i_private;
2898 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2899
2900 if (!trans->dbg.dest_tlv ||
2901 trans->dbg.dest_tlv->monitor_mode != EXTERNAL_MODE) {
2902 IWL_ERR(trans, "Debug destination is not set to DRAM\n");
2903 return -ENOENT;
2904 }
2905
2906 if (trans_pcie->fw_mon_data.state != IWL_FW_MON_DBGFS_STATE_CLOSED)
2907 return -EBUSY;
2908
2909 trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_OPEN;
2910 return simple_open(inode, file);
2911 }
2912
2913 static int iwl_dbgfs_monitor_data_release(struct inode *inode,
2914 struct file *file)
2915 {
2916 struct iwl_trans_pcie *trans_pcie =
2917 IWL_TRANS_GET_PCIE_TRANS(inode->i_private);
2918
2919 if (trans_pcie->fw_mon_data.state == IWL_FW_MON_DBGFS_STATE_OPEN)
2920 trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_CLOSED;
2921 return 0;
2922 }
2923
2924 static bool iwl_write_to_user_buf(char __user *user_buf, ssize_t count,
2925 void *buf, ssize_t *size,
2926 ssize_t *bytes_copied)
2927 {
2928 ssize_t buf_size_left = count - *bytes_copied;
2929
2930 buf_size_left = buf_size_left - (buf_size_left % sizeof(u32));
2931 if (*size > buf_size_left)
2932 *size = buf_size_left;
2933
2934 *size -= copy_to_user(user_buf, buf, *size);
2935 *bytes_copied += *size;
2936
2937 if (buf_size_left == *size)
2938 return true;
2939 return false;
2940 }
2941
2942 static ssize_t iwl_dbgfs_monitor_data_read(struct file *file,
2943 char __user *user_buf,
2944 size_t count, loff_t *ppos)
2945 {
2946 struct iwl_trans *trans = file->private_data;
2947 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2948 u8 *cpu_addr = (void *)trans->dbg.fw_mon.block, *curr_buf;
2949 struct cont_rec *data = &trans_pcie->fw_mon_data;
2950 u32 write_ptr_addr, wrap_cnt_addr, write_ptr, wrap_cnt;
2951 ssize_t size, bytes_copied = 0;
2952 bool b_full;
2953
2954 if (trans->dbg.dest_tlv) {
2955 write_ptr_addr =
2956 le32_to_cpu(trans->dbg.dest_tlv->write_ptr_reg);
2957 wrap_cnt_addr = le32_to_cpu(trans->dbg.dest_tlv->wrap_count);
2958 } else {
2959 write_ptr_addr = MON_BUFF_WRPTR;
2960 wrap_cnt_addr = MON_BUFF_CYCLE_CNT;
2961 }
2962
2963 if (unlikely(!trans->dbg.rec_on))
2964 return 0;
2965
2966 mutex_lock(&data->mutex);
2967 if (data->state ==
2968 IWL_FW_MON_DBGFS_STATE_DISABLED) {
2969 mutex_unlock(&data->mutex);
2970 return 0;
2971 }
2972
2973 /* write_ptr position in bytes rather then DW */
2974 write_ptr = iwl_read_prph(trans, write_ptr_addr) * sizeof(u32);
2975 wrap_cnt = iwl_read_prph(trans, wrap_cnt_addr);
2976
2977 if (data->prev_wrap_cnt == wrap_cnt) {
2978 size = write_ptr - data->prev_wr_ptr;
2979 curr_buf = cpu_addr + data->prev_wr_ptr;
2980 b_full = iwl_write_to_user_buf(user_buf, count,
2981 curr_buf, &size,
2982 &bytes_copied);
2983 data->prev_wr_ptr += size;
2984
2985 } else if (data->prev_wrap_cnt == wrap_cnt - 1 &&
2986 write_ptr < data->prev_wr_ptr) {
2987 size = trans->dbg.fw_mon.size - data->prev_wr_ptr;
2988 curr_buf = cpu_addr + data->prev_wr_ptr;
2989 b_full = iwl_write_to_user_buf(user_buf, count,
2990 curr_buf, &size,
2991 &bytes_copied);
2992 data->prev_wr_ptr += size;
2993
2994 if (!b_full) {
2995 size = write_ptr;
2996 b_full = iwl_write_to_user_buf(user_buf, count,
2997 cpu_addr, &size,
2998 &bytes_copied);
2999 data->prev_wr_ptr = size;
3000 data->prev_wrap_cnt++;
3001 }
3002 } else {
3003 if (data->prev_wrap_cnt == wrap_cnt - 1 &&
3004 write_ptr > data->prev_wr_ptr)
3005 IWL_WARN(trans,
3006 "write pointer passed previous write pointer, start copying from the beginning\n");
3007 else if (!unlikely(data->prev_wrap_cnt == 0 &&
3008 data->prev_wr_ptr == 0))
3009 IWL_WARN(trans,
3010 "monitor data is out of sync, start copying from the beginning\n");
3011
3012 size = write_ptr;
3013 b_full = iwl_write_to_user_buf(user_buf, count,
3014 cpu_addr, &size,
3015 &bytes_copied);
3016 data->prev_wr_ptr = size;
3017 data->prev_wrap_cnt = wrap_cnt;
3018 }
3019
3020 mutex_unlock(&data->mutex);
3021
3022 return bytes_copied;
3023 }
3024
3025 static ssize_t iwl_dbgfs_rf_read(struct file *file,
3026 char __user *user_buf,
3027 size_t count, loff_t *ppos)
3028 {
3029 struct iwl_trans *trans = file->private_data;
3030 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3031
3032 if (!trans_pcie->rf_name[0])
3033 return -ENODEV;
3034
3035 return simple_read_from_buffer(user_buf, count, ppos,
3036 trans_pcie->rf_name,
3037 strlen(trans_pcie->rf_name));
3038 }
3039
3040 DEBUGFS_READ_WRITE_FILE_OPS(interrupt);
3041 DEBUGFS_READ_FILE_OPS(fh_reg);
3042 DEBUGFS_READ_FILE_OPS(rx_queue);
3043 DEBUGFS_WRITE_FILE_OPS(csr);
3044 DEBUGFS_READ_WRITE_FILE_OPS(rfkill);
3045 DEBUGFS_READ_FILE_OPS(rf);
3046
3047 static const struct file_operations iwl_dbgfs_tx_queue_ops = {
3048 .owner = THIS_MODULE,
3049 .open = iwl_dbgfs_tx_queue_open,
3050 .read = seq_read,
3051 .llseek = seq_lseek,
3052 .release = seq_release_private,
3053 };
3054
3055 static const struct file_operations iwl_dbgfs_monitor_data_ops = {
3056 .read = iwl_dbgfs_monitor_data_read,
3057 .open = iwl_dbgfs_monitor_data_open,
3058 .release = iwl_dbgfs_monitor_data_release,
3059 };
3060
3061 /* Create the debugfs files and directories */
3062 void iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans)
3063 {
3064 struct dentry *dir = trans->dbgfs_dir;
3065
3066 DEBUGFS_ADD_FILE(rx_queue, dir, 0400);
3067 DEBUGFS_ADD_FILE(tx_queue, dir, 0400);
3068 DEBUGFS_ADD_FILE(interrupt, dir, 0600);
3069 DEBUGFS_ADD_FILE(csr, dir, 0200);
3070 DEBUGFS_ADD_FILE(fh_reg, dir, 0400);
3071 DEBUGFS_ADD_FILE(rfkill, dir, 0600);
3072 DEBUGFS_ADD_FILE(monitor_data, dir, 0400);
3073 DEBUGFS_ADD_FILE(rf, dir, 0400);
3074 }
3075
3076 void iwl_trans_pcie_debugfs_cleanup(struct iwl_trans *trans)
3077 {
3078 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3079 struct cont_rec *data = &trans_pcie->fw_mon_data;
3080
3081 mutex_lock(&data->mutex);
3082 data->state = IWL_FW_MON_DBGFS_STATE_DISABLED;
3083 mutex_unlock(&data->mutex);
3084 }
3085 #endif /*CONFIG_IWLWIFI_DEBUGFS */
3086
3087 static u32 iwl_trans_pcie_get_cmdlen(struct iwl_trans *trans, void *tfd)
3088 {
3089 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3090 u32 cmdlen = 0;
3091 int i;
3092
3093 for (i = 0; i < trans_pcie->txqs.tfd.max_tbs; i++)
3094 cmdlen += iwl_txq_gen1_tfd_tb_get_len(trans, tfd, i);
3095
3096 return cmdlen;
3097 }
3098
3099 static u32 iwl_trans_pcie_dump_rbs(struct iwl_trans *trans,
3100 struct iwl_fw_error_dump_data **data,
3101 int allocated_rb_nums)
3102 {
3103 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3104 int max_len = trans_pcie->rx_buf_bytes;
3105 /* Dump RBs is supported only for pre-9000 devices (1 queue) */
3106 struct iwl_rxq *rxq = &trans_pcie->rxq[0];
3107 u32 i, r, j, rb_len = 0;
3108
3109 spin_lock_bh(&rxq->lock);
3110
3111 r = iwl_get_closed_rb_stts(trans, rxq);
3112
3113 for (i = rxq->read, j = 0;
3114 i != r && j < allocated_rb_nums;
3115 i = (i + 1) & RX_QUEUE_MASK, j++) {
3116 struct iwl_rx_mem_buffer *rxb = rxq->queue[i];
3117 struct iwl_fw_error_dump_rb *rb;
3118
3119 dma_sync_single_for_cpu(trans->dev, rxb->page_dma,
3120 max_len, DMA_FROM_DEVICE);
3121
3122 rb_len += sizeof(**data) + sizeof(*rb) + max_len;
3123
3124 (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RB);
3125 (*data)->len = cpu_to_le32(sizeof(*rb) + max_len);
3126 rb = (void *)(*data)->data;
3127 rb->index = cpu_to_le32(i);
3128 memcpy(rb->data, page_address(rxb->page), max_len);
3129
3130 *data = iwl_fw_error_next_data(*data);
3131 }
3132
3133 spin_unlock_bh(&rxq->lock);
3134
3135 return rb_len;
3136 }
3137 #define IWL_CSR_TO_DUMP (0x250)
3138
3139 static u32 iwl_trans_pcie_dump_csr(struct iwl_trans *trans,
3140 struct iwl_fw_error_dump_data **data)
3141 {
3142 u32 csr_len = sizeof(**data) + IWL_CSR_TO_DUMP;
3143 __le32 *val;
3144 int i;
3145
3146 (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_CSR);
3147 (*data)->len = cpu_to_le32(IWL_CSR_TO_DUMP);
3148 val = (void *)(*data)->data;
3149
3150 for (i = 0; i < IWL_CSR_TO_DUMP; i += 4)
3151 *val++ = cpu_to_le32(iwl_trans_pcie_read32(trans, i));
3152
3153 *data = iwl_fw_error_next_data(*data);
3154
3155 return csr_len;
3156 }
3157
3158 static u32 iwl_trans_pcie_fh_regs_dump(struct iwl_trans *trans,
3159 struct iwl_fw_error_dump_data **data)
3160 {
3161 u32 fh_regs_len = FH_MEM_UPPER_BOUND - FH_MEM_LOWER_BOUND;
3162 __le32 *val;
3163 int i;
3164
3165 if (!iwl_trans_grab_nic_access(trans))
3166 return 0;
3167
3168 (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_FH_REGS);
3169 (*data)->len = cpu_to_le32(fh_regs_len);
3170 val = (void *)(*data)->data;
3171
3172 if (!trans->trans_cfg->gen2)
3173 for (i = FH_MEM_LOWER_BOUND; i < FH_MEM_UPPER_BOUND;
3174 i += sizeof(u32))
3175 *val++ = cpu_to_le32(iwl_trans_pcie_read32(trans, i));
3176 else
3177 for (i = iwl_umac_prph(trans, FH_MEM_LOWER_BOUND_GEN2);
3178 i < iwl_umac_prph(trans, FH_MEM_UPPER_BOUND_GEN2);
3179 i += sizeof(u32))
3180 *val++ = cpu_to_le32(iwl_trans_pcie_read_prph(trans,
3181 i));
3182
3183 iwl_trans_release_nic_access(trans);
3184
3185 *data = iwl_fw_error_next_data(*data);
3186
3187 return sizeof(**data) + fh_regs_len;
3188 }
3189
3190 static u32
3191 iwl_trans_pci_dump_marbh_monitor(struct iwl_trans *trans,
3192 struct iwl_fw_error_dump_fw_mon *fw_mon_data,
3193 u32 monitor_len)
3194 {
3195 u32 buf_size_in_dwords = (monitor_len >> 2);
3196 u32 *buffer = (u32 *)fw_mon_data->data;
3197 u32 i;
3198
3199 if (!iwl_trans_grab_nic_access(trans))
3200 return 0;
3201
3202 iwl_write_umac_prph_no_grab(trans, MON_DMARB_RD_CTL_ADDR, 0x1);
3203 for (i = 0; i < buf_size_in_dwords; i++)
3204 buffer[i] = iwl_read_umac_prph_no_grab(trans,
3205 MON_DMARB_RD_DATA_ADDR);
3206 iwl_write_umac_prph_no_grab(trans, MON_DMARB_RD_CTL_ADDR, 0x0);
3207
3208 iwl_trans_release_nic_access(trans);
3209
3210 return monitor_len;
3211 }
3212
3213 static void
3214 iwl_trans_pcie_dump_pointers(struct iwl_trans *trans,
3215 struct iwl_fw_error_dump_fw_mon *fw_mon_data)
3216 {
3217 u32 base, base_high, write_ptr, write_ptr_val, wrap_cnt;
3218
3219 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
3220 base = DBGC_CUR_DBGBUF_BASE_ADDR_LSB;
3221 base_high = DBGC_CUR_DBGBUF_BASE_ADDR_MSB;
3222 write_ptr = DBGC_CUR_DBGBUF_STATUS;
3223 wrap_cnt = DBGC_DBGBUF_WRAP_AROUND;
3224 } else if (trans->dbg.dest_tlv) {
3225 write_ptr = le32_to_cpu(trans->dbg.dest_tlv->write_ptr_reg);
3226 wrap_cnt = le32_to_cpu(trans->dbg.dest_tlv->wrap_count);
3227 base = le32_to_cpu(trans->dbg.dest_tlv->base_reg);
3228 } else {
3229 base = MON_BUFF_BASE_ADDR;
3230 write_ptr = MON_BUFF_WRPTR;
3231 wrap_cnt = MON_BUFF_CYCLE_CNT;
3232 }
3233
3234 write_ptr_val = iwl_read_prph(trans, write_ptr);
3235 fw_mon_data->fw_mon_cycle_cnt =
3236 cpu_to_le32(iwl_read_prph(trans, wrap_cnt));
3237 fw_mon_data->fw_mon_base_ptr =
3238 cpu_to_le32(iwl_read_prph(trans, base));
3239 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
3240 fw_mon_data->fw_mon_base_high_ptr =
3241 cpu_to_le32(iwl_read_prph(trans, base_high));
3242 write_ptr_val &= DBGC_CUR_DBGBUF_STATUS_OFFSET_MSK;
3243 /* convert wrtPtr to DWs, to align with all HWs */
3244 write_ptr_val >>= 2;
3245 }
3246 fw_mon_data->fw_mon_wr_ptr = cpu_to_le32(write_ptr_val);
3247 }
3248
3249 static u32
3250 iwl_trans_pcie_dump_monitor(struct iwl_trans *trans,
3251 struct iwl_fw_error_dump_data **data,
3252 u32 monitor_len)
3253 {
3254 struct iwl_dram_data *fw_mon = &trans->dbg.fw_mon;
3255 u32 len = 0;
3256
3257 if (trans->dbg.dest_tlv ||
3258 (fw_mon->size &&
3259 (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000 ||
3260 trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210))) {
3261 struct iwl_fw_error_dump_fw_mon *fw_mon_data;
3262
3263 (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_FW_MONITOR);
3264 fw_mon_data = (void *)(*data)->data;
3265
3266 iwl_trans_pcie_dump_pointers(trans, fw_mon_data);
3267
3268 len += sizeof(**data) + sizeof(*fw_mon_data);
3269 if (fw_mon->size) {
3270 memcpy(fw_mon_data->data, fw_mon->block, fw_mon->size);
3271 monitor_len = fw_mon->size;
3272 } else if (trans->dbg.dest_tlv->monitor_mode == SMEM_MODE) {
3273 u32 base = le32_to_cpu(fw_mon_data->fw_mon_base_ptr);
3274 /*
3275 * Update pointers to reflect actual values after
3276 * shifting
3277 */
3278 if (trans->dbg.dest_tlv->version) {
3279 base = (iwl_read_prph(trans, base) &
3280 IWL_LDBG_M2S_BUF_BA_MSK) <<
3281 trans->dbg.dest_tlv->base_shift;
3282 base *= IWL_M2S_UNIT_SIZE;
3283 base += trans->cfg->smem_offset;
3284 } else {
3285 base = iwl_read_prph(trans, base) <<
3286 trans->dbg.dest_tlv->base_shift;
3287 }
3288
3289 iwl_trans_read_mem(trans, base, fw_mon_data->data,
3290 monitor_len / sizeof(u32));
3291 } else if (trans->dbg.dest_tlv->monitor_mode == MARBH_MODE) {
3292 monitor_len =
3293 iwl_trans_pci_dump_marbh_monitor(trans,
3294 fw_mon_data,
3295 monitor_len);
3296 } else {
3297 /* Didn't match anything - output no monitor data */
3298 monitor_len = 0;
3299 }
3300
3301 len += monitor_len;
3302 (*data)->len = cpu_to_le32(monitor_len + sizeof(*fw_mon_data));
3303 }
3304
3305 return len;
3306 }
3307
3308 static int iwl_trans_get_fw_monitor_len(struct iwl_trans *trans, u32 *len)
3309 {
3310 if (trans->dbg.fw_mon.size) {
3311 *len += sizeof(struct iwl_fw_error_dump_data) +
3312 sizeof(struct iwl_fw_error_dump_fw_mon) +
3313 trans->dbg.fw_mon.size;
3314 return trans->dbg.fw_mon.size;
3315 } else if (trans->dbg.dest_tlv) {
3316 u32 base, end, cfg_reg, monitor_len;
3317
3318 if (trans->dbg.dest_tlv->version == 1) {
3319 cfg_reg = le32_to_cpu(trans->dbg.dest_tlv->base_reg);
3320 cfg_reg = iwl_read_prph(trans, cfg_reg);
3321 base = (cfg_reg & IWL_LDBG_M2S_BUF_BA_MSK) <<
3322 trans->dbg.dest_tlv->base_shift;
3323 base *= IWL_M2S_UNIT_SIZE;
3324 base += trans->cfg->smem_offset;
3325
3326 monitor_len =
3327 (cfg_reg & IWL_LDBG_M2S_BUF_SIZE_MSK) >>
3328 trans->dbg.dest_tlv->end_shift;
3329 monitor_len *= IWL_M2S_UNIT_SIZE;
3330 } else {
3331 base = le32_to_cpu(trans->dbg.dest_tlv->base_reg);
3332 end = le32_to_cpu(trans->dbg.dest_tlv->end_reg);
3333
3334 base = iwl_read_prph(trans, base) <<
3335 trans->dbg.dest_tlv->base_shift;
3336 end = iwl_read_prph(trans, end) <<
3337 trans->dbg.dest_tlv->end_shift;
3338
3339 /* Make "end" point to the actual end */
3340 if (trans->trans_cfg->device_family >=
3341 IWL_DEVICE_FAMILY_8000 ||
3342 trans->dbg.dest_tlv->monitor_mode == MARBH_MODE)
3343 end += (1 << trans->dbg.dest_tlv->end_shift);
3344 monitor_len = end - base;
3345 }
3346 *len += sizeof(struct iwl_fw_error_dump_data) +
3347 sizeof(struct iwl_fw_error_dump_fw_mon) +
3348 monitor_len;
3349 return monitor_len;
3350 }
3351 return 0;
3352 }
3353
3354 struct iwl_trans_dump_data *
3355 iwl_trans_pcie_dump_data(struct iwl_trans *trans, u32 dump_mask,
3356 const struct iwl_dump_sanitize_ops *sanitize_ops,
3357 void *sanitize_ctx)
3358 {
3359 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3360 struct iwl_fw_error_dump_data *data;
3361 struct iwl_txq *cmdq = trans_pcie->txqs.txq[trans_pcie->txqs.cmd.q_id];
3362 struct iwl_fw_error_dump_txcmd *txcmd;
3363 struct iwl_trans_dump_data *dump_data;
3364 u32 len, num_rbs = 0, monitor_len = 0;
3365 int i, ptr;
3366 bool dump_rbs = test_bit(STATUS_FW_ERROR, &trans->status) &&
3367 !trans->trans_cfg->mq_rx_supported &&
3368 dump_mask & BIT(IWL_FW_ERROR_DUMP_RB);
3369
3370 if (!dump_mask)
3371 return NULL;
3372
3373 /* transport dump header */
3374 len = sizeof(*dump_data);
3375
3376 /* host commands */
3377 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_TXCMD) && cmdq)
3378 len += sizeof(*data) +
3379 cmdq->n_window * (sizeof(*txcmd) +
3380 TFD_MAX_PAYLOAD_SIZE);
3381
3382 /* FW monitor */
3383 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FW_MONITOR))
3384 monitor_len = iwl_trans_get_fw_monitor_len(trans, &len);
3385
3386 /* CSR registers */
3387 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_CSR))
3388 len += sizeof(*data) + IWL_CSR_TO_DUMP;
3389
3390 /* FH registers */
3391 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FH_REGS)) {
3392 if (trans->trans_cfg->gen2)
3393 len += sizeof(*data) +
3394 (iwl_umac_prph(trans, FH_MEM_UPPER_BOUND_GEN2) -
3395 iwl_umac_prph(trans, FH_MEM_LOWER_BOUND_GEN2));
3396 else
3397 len += sizeof(*data) +
3398 (FH_MEM_UPPER_BOUND -
3399 FH_MEM_LOWER_BOUND);
3400 }
3401
3402 if (dump_rbs) {
3403 /* Dump RBs is supported only for pre-9000 devices (1 queue) */
3404 struct iwl_rxq *rxq = &trans_pcie->rxq[0];
3405 /* RBs */
3406 num_rbs = iwl_get_closed_rb_stts(trans, rxq);
3407 num_rbs = (num_rbs - rxq->read) & RX_QUEUE_MASK;
3408 len += num_rbs * (sizeof(*data) +
3409 sizeof(struct iwl_fw_error_dump_rb) +
3410 (PAGE_SIZE << trans_pcie->rx_page_order));
3411 }
3412
3413 /* Paged memory for gen2 HW */
3414 if (trans->trans_cfg->gen2 && dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING))
3415 for (i = 0; i < trans->init_dram.paging_cnt; i++)
3416 len += sizeof(*data) +
3417 sizeof(struct iwl_fw_error_dump_paging) +
3418 trans->init_dram.paging[i].size;
3419
3420 dump_data = vzalloc(len);
3421 if (!dump_data)
3422 return NULL;
3423
3424 len = 0;
3425 data = (void *)dump_data->data;
3426
3427 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_TXCMD) && cmdq) {
3428 u16 tfd_size = trans_pcie->txqs.tfd.size;
3429
3430 data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_TXCMD);
3431 txcmd = (void *)data->data;
3432 spin_lock_bh(&cmdq->lock);
3433 ptr = cmdq->write_ptr;
3434 for (i = 0; i < cmdq->n_window; i++) {
3435 u8 idx = iwl_txq_get_cmd_index(cmdq, ptr);
3436 u8 tfdidx;
3437 u32 caplen, cmdlen;
3438
3439 if (trans->trans_cfg->gen2)
3440 tfdidx = idx;
3441 else
3442 tfdidx = ptr;
3443
3444 cmdlen = iwl_trans_pcie_get_cmdlen(trans,
3445 (u8 *)cmdq->tfds +
3446 tfd_size * tfdidx);
3447 caplen = min_t(u32, TFD_MAX_PAYLOAD_SIZE, cmdlen);
3448
3449 if (cmdlen) {
3450 len += sizeof(*txcmd) + caplen;
3451 txcmd->cmdlen = cpu_to_le32(cmdlen);
3452 txcmd->caplen = cpu_to_le32(caplen);
3453 memcpy(txcmd->data, cmdq->entries[idx].cmd,
3454 caplen);
3455 if (sanitize_ops && sanitize_ops->frob_hcmd)
3456 sanitize_ops->frob_hcmd(sanitize_ctx,
3457 txcmd->data,
3458 caplen);
3459 txcmd = (void *)((u8 *)txcmd->data + caplen);
3460 }
3461
3462 ptr = iwl_txq_dec_wrap(trans, ptr);
3463 }
3464 spin_unlock_bh(&cmdq->lock);
3465
3466 data->len = cpu_to_le32(len);
3467 len += sizeof(*data);
3468 data = iwl_fw_error_next_data(data);
3469 }
3470
3471 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_CSR))
3472 len += iwl_trans_pcie_dump_csr(trans, &data);
3473 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FH_REGS))
3474 len += iwl_trans_pcie_fh_regs_dump(trans, &data);
3475 if (dump_rbs)
3476 len += iwl_trans_pcie_dump_rbs(trans, &data, num_rbs);
3477
3478 /* Paged memory for gen2 HW */
3479 if (trans->trans_cfg->gen2 &&
3480 dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING)) {
3481 for (i = 0; i < trans->init_dram.paging_cnt; i++) {
3482 struct iwl_fw_error_dump_paging *paging;
3483 u32 page_len = trans->init_dram.paging[i].size;
3484
3485 data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PAGING);
3486 data->len = cpu_to_le32(sizeof(*paging) + page_len);
3487 paging = (void *)data->data;
3488 paging->index = cpu_to_le32(i);
3489 memcpy(paging->data,
3490 trans->init_dram.paging[i].block, page_len);
3491 data = iwl_fw_error_next_data(data);
3492
3493 len += sizeof(*data) + sizeof(*paging) + page_len;
3494 }
3495 }
3496 if (dump_mask & BIT(IWL_FW_ERROR_DUMP_FW_MONITOR))
3497 len += iwl_trans_pcie_dump_monitor(trans, &data, monitor_len);
3498
3499 dump_data->len = len;
3500
3501 return dump_data;
3502 }
3503
3504 void iwl_trans_pci_interrupts(struct iwl_trans *trans, bool enable)
3505 {
3506 if (enable)
3507 iwl_enable_interrupts(trans);
3508 else
3509 iwl_disable_interrupts(trans);
3510 }
3511
3512 void iwl_trans_pcie_sync_nmi(struct iwl_trans *trans)
3513 {
3514 u32 inta_addr, sw_err_bit;
3515 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3516
3517 if (trans_pcie->msix_enabled) {
3518 inta_addr = CSR_MSIX_HW_INT_CAUSES_AD;
3519 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
3520 sw_err_bit = MSIX_HW_INT_CAUSES_REG_SW_ERR_BZ;
3521 else
3522 sw_err_bit = MSIX_HW_INT_CAUSES_REG_SW_ERR;
3523 } else {
3524 inta_addr = CSR_INT;
3525 sw_err_bit = CSR_INT_BIT_SW_ERR;
3526 }
3527
3528 iwl_trans_sync_nmi_with_addr(trans, inta_addr, sw_err_bit);
3529 }
3530
3531 struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
3532 const struct pci_device_id *ent,
3533 const struct iwl_cfg_trans_params *cfg_trans)
3534 {
3535 struct iwl_trans_pcie *trans_pcie, **priv;
3536 struct iwl_trans *trans;
3537 int ret, addr_size;
3538 u32 bar0;
3539
3540 /* reassign our BAR 0 if invalid due to possible runtime PM races */
3541 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_0, &bar0);
3542 if (bar0 == PCI_BASE_ADDRESS_MEM_TYPE_64) {
3543 ret = pci_assign_resource(pdev, 0);
3544 if (ret)
3545 return ERR_PTR(ret);
3546 }
3547
3548 ret = pcim_enable_device(pdev);
3549 if (ret)
3550 return ERR_PTR(ret);
3551
3552 trans = iwl_trans_alloc(sizeof(struct iwl_trans_pcie), &pdev->dev,
3553 cfg_trans);
3554 if (!trans)
3555 return ERR_PTR(-ENOMEM);
3556
3557 trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3558
3559 if (trans->trans_cfg->gen2) {
3560 trans_pcie->txqs.tfd.addr_size = 64;
3561 trans_pcie->txqs.tfd.max_tbs = IWL_TFH_NUM_TBS;
3562 trans_pcie->txqs.tfd.size = sizeof(struct iwl_tfh_tfd);
3563 } else {
3564 trans_pcie->txqs.tfd.addr_size = 36;
3565 trans_pcie->txqs.tfd.max_tbs = IWL_NUM_OF_TBS;
3566 trans_pcie->txqs.tfd.size = sizeof(struct iwl_tfd);
3567 }
3568 trans->max_skb_frags = IWL_TRANS_PCIE_MAX_FRAGS(trans_pcie);
3569
3570 /* Set a short watchdog for the command queue */
3571 trans_pcie->txqs.cmd.wdg_timeout = IWL_DEF_WD_TIMEOUT;
3572
3573 trans_pcie->txqs.tso_hdr_page = alloc_percpu(struct iwl_tso_hdr_page);
3574 if (!trans_pcie->txqs.tso_hdr_page) {
3575 ret = -ENOMEM;
3576 goto out_free_trans;
3577 }
3578
3579 if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
3580 trans_pcie->txqs.bc_tbl_size =
3581 sizeof(struct iwl_gen3_bc_tbl_entry) * TFD_QUEUE_BC_SIZE_GEN3_BZ;
3582 else if (trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210)
3583 trans_pcie->txqs.bc_tbl_size =
3584 sizeof(struct iwl_gen3_bc_tbl_entry) * TFD_QUEUE_BC_SIZE_GEN3_AX210;
3585 else
3586 trans_pcie->txqs.bc_tbl_size = sizeof(struct iwlagn_scd_bc_tbl);
3587 /*
3588 * For gen2 devices, we use a single allocation for each byte-count
3589 * table, but they're pretty small (1k) so use a DMA pool that we
3590 * allocate here.
3591 */
3592 if (trans->trans_cfg->gen2) {
3593 trans_pcie->txqs.bc_pool =
3594 dmam_pool_create("iwlwifi:bc", trans->dev,
3595 trans_pcie->txqs.bc_tbl_size,
3596 256, 0);
3597 if (!trans_pcie->txqs.bc_pool) {
3598 ret = -ENOMEM;
3599 goto out_free_tso;
3600 }
3601 }
3602
3603 /* Some things must not change even if the config does */
3604 WARN_ON(trans_pcie->txqs.tfd.addr_size !=
3605 (trans->trans_cfg->gen2 ? 64 : 36));
3606
3607 /* Initialize NAPI here - it should be before registering to mac80211
3608 * in the opmode but after the HW struct is allocated.
3609 */
3610 trans_pcie->napi_dev = alloc_netdev_dummy(sizeof(struct iwl_trans_pcie *));
3611 if (!trans_pcie->napi_dev) {
3612 ret = -ENOMEM;
3613 goto out_free_tso;
3614 }
3615 /* The private struct in netdev is a pointer to struct iwl_trans_pcie */
3616 priv = netdev_priv(trans_pcie->napi_dev);
3617 *priv = trans_pcie;
3618
3619 trans_pcie->trans = trans;
3620 trans_pcie->opmode_down = true;
3621 spin_lock_init(&trans_pcie->irq_lock);
3622 spin_lock_init(&trans_pcie->reg_lock);
3623 spin_lock_init(&trans_pcie->alloc_page_lock);
3624 mutex_init(&trans_pcie->mutex);
3625 init_waitqueue_head(&trans_pcie->ucode_write_waitq);
3626 init_waitqueue_head(&trans_pcie->fw_reset_waitq);
3627 init_waitqueue_head(&trans_pcie->imr_waitq);
3628
3629 trans_pcie->rba.alloc_wq = alloc_workqueue("rb_allocator",
3630 WQ_HIGHPRI | WQ_UNBOUND, 0);
3631 if (!trans_pcie->rba.alloc_wq) {
3632 ret = -ENOMEM;
3633 goto out_free_ndev;
3634 }
3635 INIT_WORK(&trans_pcie->rba.rx_alloc, iwl_pcie_rx_allocator_work);
3636
3637 trans_pcie->debug_rfkill = -1;
3638
3639 if (!cfg_trans->base_params->pcie_l1_allowed) {
3640 /*
3641 * W/A - seems to solve weird behavior. We need to remove this
3642 * if we don't want to stay in L1 all the time. This wastes a
3643 * lot of power.
3644 */
3645 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S |
3646 PCIE_LINK_STATE_L1 |
3647 PCIE_LINK_STATE_CLKPM);
3648 }
3649
3650 pci_set_master(pdev);
3651
3652 addr_size = trans_pcie->txqs.tfd.addr_size;
3653 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(addr_size));
3654 if (ret) {
3655 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3656 /* both attempts failed: */
3657 if (ret) {
3658 dev_err(&pdev->dev, "No suitable DMA available\n");
3659 goto out_no_pci;
3660 }
3661 }
3662
3663 ret = pcim_request_all_regions(pdev, DRV_NAME);
3664 if (ret) {
3665 dev_err(&pdev->dev, "Requesting all PCI BARs failed.\n");
3666 goto out_no_pci;
3667 }
3668
3669 trans_pcie->hw_base = pcim_iomap(pdev, 0, 0);
3670 if (!trans_pcie->hw_base) {
3671 dev_err(&pdev->dev, "Could not ioremap PCI BAR 0.\n");
3672 ret = -ENODEV;
3673 goto out_no_pci;
3674 }
3675
3676 /* We disable the RETRY_TIMEOUT register (0x41) to keep
3677 * PCI Tx retries from interfering with C3 CPU state */
3678 pci_write_config_byte(pdev, PCI_CFG_RETRY_TIMEOUT, 0x00);
3679
3680 trans_pcie->pci_dev = pdev;
3681 iwl_disable_interrupts(trans);
3682
3683 trans->hw_rev = iwl_read32(trans, CSR_HW_REV);
3684 if (trans->hw_rev == 0xffffffff) {
3685 dev_err(&pdev->dev, "HW_REV=0xFFFFFFFF, PCI issues?\n");
3686 ret = -EIO;
3687 goto out_no_pci;
3688 }
3689
3690 /*
3691 * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have
3692 * changed, and now the revision step also includes bit 0-1 (no more
3693 * "dash" value). To keep hw_rev backwards compatible - we'll store it
3694 * in the old format.
3695 */
3696 if (cfg_trans->device_family >= IWL_DEVICE_FAMILY_8000)
3697 trans->hw_rev_step = trans->hw_rev & 0xF;
3698 else
3699 trans->hw_rev_step = (trans->hw_rev & 0xC) >> 2;
3700
3701 IWL_DEBUG_INFO(trans, "HW REV: 0x%0x\n", trans->hw_rev);
3702
3703 iwl_pcie_set_interrupt_capa(pdev, trans, cfg_trans);
3704 trans->hw_id = (pdev->device << 16) + pdev->subsystem_device;
3705 snprintf(trans->hw_id_str, sizeof(trans->hw_id_str),
3706 "PCI ID: 0x%04X:0x%04X", pdev->device, pdev->subsystem_device);
3707
3708 init_waitqueue_head(&trans_pcie->sx_waitq);
3709
3710 ret = iwl_pcie_alloc_invalid_tx_cmd(trans);
3711 if (ret)
3712 goto out_no_pci;
3713
3714 if (trans_pcie->msix_enabled) {
3715 ret = iwl_pcie_init_msix_handler(pdev, trans_pcie);
3716 if (ret)
3717 goto out_no_pci;
3718 } else {
3719 ret = iwl_pcie_alloc_ict(trans);
3720 if (ret)
3721 goto out_no_pci;
3722
3723 ret = devm_request_threaded_irq(&pdev->dev, pdev->irq,
3724 iwl_pcie_isr,
3725 iwl_pcie_irq_handler,
3726 IRQF_SHARED, DRV_NAME, trans);
3727 if (ret) {
3728 IWL_ERR(trans, "Error allocating IRQ %d\n", pdev->irq);
3729 goto out_free_ict;
3730 }
3731 }
3732
3733 #ifdef CONFIG_IWLWIFI_DEBUGFS
3734 trans_pcie->fw_mon_data.state = IWL_FW_MON_DBGFS_STATE_CLOSED;
3735 mutex_init(&trans_pcie->fw_mon_data.mutex);
3736 #endif
3737
3738 iwl_dbg_tlv_init(trans);
3739
3740 return trans;
3741
3742 out_free_ict:
3743 iwl_pcie_free_ict(trans);
3744 out_no_pci:
3745 destroy_workqueue(trans_pcie->rba.alloc_wq);
3746 out_free_ndev:
3747 free_netdev(trans_pcie->napi_dev);
3748 out_free_tso:
3749 free_percpu(trans_pcie->txqs.tso_hdr_page);
3750 out_free_trans:
3751 iwl_trans_free(trans);
3752 return ERR_PTR(ret);
3753 }
3754
3755 void iwl_trans_pcie_copy_imr_fh(struct iwl_trans *trans,
3756 u32 dst_addr, u64 src_addr, u32 byte_cnt)
3757 {
3758 iwl_write_prph(trans, IMR_UREG_CHICK,
3759 iwl_read_prph(trans, IMR_UREG_CHICK) |
3760 IMR_UREG_CHICK_HALT_UMAC_PERMANENTLY_MSK);
3761 iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_SRAM_ADDR, dst_addr);
3762 iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_DRAM_ADDR_LSB,
3763 (u32)(src_addr & 0xFFFFFFFF));
3764 iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_DRAM_ADDR_MSB,
3765 iwl_get_dma_hi_addr(src_addr));
3766 iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_BC, byte_cnt);
3767 iwl_write_prph(trans, IMR_TFH_SRV_DMA_CHNL0_CTRL,
3768 IMR_TFH_SRV_DMA_CHNL0_CTRL_D2S_IRQ_TARGET_POS |
3769 IMR_TFH_SRV_DMA_CHNL0_CTRL_D2S_DMA_EN_POS |
3770 IMR_TFH_SRV_DMA_CHNL0_CTRL_D2S_RS_MSK);
3771 }
3772
3773 int iwl_trans_pcie_copy_imr(struct iwl_trans *trans,
3774 u32 dst_addr, u64 src_addr, u32 byte_cnt)
3775 {
3776 struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3777 int ret = -1;
3778
3779 trans_pcie->imr_status = IMR_D2S_REQUESTED;
3780 iwl_trans_pcie_copy_imr_fh(trans, dst_addr, src_addr, byte_cnt);
3781 ret = wait_event_timeout(trans_pcie->imr_waitq,
3782 trans_pcie->imr_status !=
3783 IMR_D2S_REQUESTED, 5 * HZ);
3784 if (!ret || trans_pcie->imr_status == IMR_D2S_ERROR) {
3785 IWL_ERR(trans, "Failed to copy IMR Memory chunk!\n");
3786 iwl_trans_pcie_dump_regs(trans);
3787 return -ETIMEDOUT;
3788 }
3789 trans_pcie->imr_status = IMR_D2S_IDLE;
3790 return 0;
3791 }
Kernel DRM miscellaneous fixes and cross-tree changes