?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.19.133
[linux/fpc-iii.git] / drivers / ata / sata_nv.c
blob761577d57ff3720bc129c1ca16315447a4e2d5bf
1 /*
2 * sata_nv.c - NVIDIA nForce SATA
3 *
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
6 *
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
21 *
22 *
23 * libata documentation is available via 'make {ps|pdf}docs',
24 * as Documentation/driver-api/libata.rst
25 *
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
30 * hotplug info, etc.
31 *
32 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
36 *
37 */
38
39 #include <linux/kernel.h>
40 #include <linux/module.h>
41 #include <linux/gfp.h>
42 #include <linux/pci.h>
43 #include <linux/blkdev.h>
44 #include <linux/delay.h>
45 #include <linux/interrupt.h>
46 #include <linux/device.h>
47 #include <scsi/scsi_host.h>
48 #include <scsi/scsi_device.h>
49 #include <linux/libata.h>
50
51 #define DRV_NAME "sata_nv"
52 #define DRV_VERSION "3.5"
53
54 #define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
55
56 enum {
57 NV_MMIO_BAR = 5,
58
59 NV_PORTS = 2,
60 NV_PIO_MASK = ATA_PIO4,
61 NV_MWDMA_MASK = ATA_MWDMA2,
62 NV_UDMA_MASK = ATA_UDMA6,
63 NV_PORT0_SCR_REG_OFFSET = 0x00,
64 NV_PORT1_SCR_REG_OFFSET = 0x40,
65
66 /* INT_STATUS/ENABLE */
67 NV_INT_STATUS = 0x10,
68 NV_INT_ENABLE = 0x11,
69 NV_INT_STATUS_CK804 = 0x440,
70 NV_INT_ENABLE_CK804 = 0x441,
71
72 /* INT_STATUS/ENABLE bits */
73 NV_INT_DEV = 0x01,
74 NV_INT_PM = 0x02,
75 NV_INT_ADDED = 0x04,
76 NV_INT_REMOVED = 0x08,
77
78 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
79
80 NV_INT_ALL = 0x0f,
81 NV_INT_MASK = NV_INT_DEV |
82 NV_INT_ADDED | NV_INT_REMOVED,
83
84 /* INT_CONFIG */
85 NV_INT_CONFIG = 0x12,
86 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
87
88 // For PCI config register 20
89 NV_MCP_SATA_CFG_20 = 0x50,
90 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
91 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
92 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
93 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
94 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
95
96 NV_ADMA_MAX_CPBS = 32,
97 NV_ADMA_CPB_SZ = 128,
98 NV_ADMA_APRD_SZ = 16,
99 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
100 NV_ADMA_APRD_SZ,
101 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
102 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
103 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
104 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
105
106 /* BAR5 offset to ADMA general registers */
107 NV_ADMA_GEN = 0x400,
108 NV_ADMA_GEN_CTL = 0x00,
109 NV_ADMA_NOTIFIER_CLEAR = 0x30,
110
111 /* BAR5 offset to ADMA ports */
112 NV_ADMA_PORT = 0x480,
113
114 /* size of ADMA port register space */
115 NV_ADMA_PORT_SIZE = 0x100,
116
117 /* ADMA port registers */
118 NV_ADMA_CTL = 0x40,
119 NV_ADMA_CPB_COUNT = 0x42,
120 NV_ADMA_NEXT_CPB_IDX = 0x43,
121 NV_ADMA_STAT = 0x44,
122 NV_ADMA_CPB_BASE_LOW = 0x48,
123 NV_ADMA_CPB_BASE_HIGH = 0x4C,
124 NV_ADMA_APPEND = 0x50,
125 NV_ADMA_NOTIFIER = 0x68,
126 NV_ADMA_NOTIFIER_ERROR = 0x6C,
127
128 /* NV_ADMA_CTL register bits */
129 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
130 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
131 NV_ADMA_CTL_GO = (1 << 7),
132 NV_ADMA_CTL_AIEN = (1 << 8),
133 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
134 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
135
136 /* CPB response flag bits */
137 NV_CPB_RESP_DONE = (1 << 0),
138 NV_CPB_RESP_ATA_ERR = (1 << 3),
139 NV_CPB_RESP_CMD_ERR = (1 << 4),
140 NV_CPB_RESP_CPB_ERR = (1 << 7),
141
142 /* CPB control flag bits */
143 NV_CPB_CTL_CPB_VALID = (1 << 0),
144 NV_CPB_CTL_QUEUE = (1 << 1),
145 NV_CPB_CTL_APRD_VALID = (1 << 2),
146 NV_CPB_CTL_IEN = (1 << 3),
147 NV_CPB_CTL_FPDMA = (1 << 4),
148
149 /* APRD flags */
150 NV_APRD_WRITE = (1 << 1),
151 NV_APRD_END = (1 << 2),
152 NV_APRD_CONT = (1 << 3),
153
154 /* NV_ADMA_STAT flags */
155 NV_ADMA_STAT_TIMEOUT = (1 << 0),
156 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
157 NV_ADMA_STAT_HOTPLUG = (1 << 2),
158 NV_ADMA_STAT_CPBERR = (1 << 4),
159 NV_ADMA_STAT_SERROR = (1 << 5),
160 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
161 NV_ADMA_STAT_IDLE = (1 << 8),
162 NV_ADMA_STAT_LEGACY = (1 << 9),
163 NV_ADMA_STAT_STOPPED = (1 << 10),
164 NV_ADMA_STAT_DONE = (1 << 12),
165 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
166 NV_ADMA_STAT_TIMEOUT,
167
168 /* port flags */
169 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
170 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
171
172 /* MCP55 reg offset */
173 NV_CTL_MCP55 = 0x400,
174 NV_INT_STATUS_MCP55 = 0x440,
175 NV_INT_ENABLE_MCP55 = 0x444,
176 NV_NCQ_REG_MCP55 = 0x448,
177
178 /* MCP55 */
179 NV_INT_ALL_MCP55 = 0xffff,
180 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */
181 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd,
182
183 /* SWNCQ ENABLE BITS*/
184 NV_CTL_PRI_SWNCQ = 0x02,
185 NV_CTL_SEC_SWNCQ = 0x04,
186
187 /* SW NCQ status bits*/
188 NV_SWNCQ_IRQ_DEV = (1 << 0),
189 NV_SWNCQ_IRQ_PM = (1 << 1),
190 NV_SWNCQ_IRQ_ADDED = (1 << 2),
191 NV_SWNCQ_IRQ_REMOVED = (1 << 3),
192
193 NV_SWNCQ_IRQ_BACKOUT = (1 << 4),
194 NV_SWNCQ_IRQ_SDBFIS = (1 << 5),
195 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6),
196 NV_SWNCQ_IRQ_DMASETUP = (1 << 7),
197
198 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED |
199 NV_SWNCQ_IRQ_REMOVED,
200
201 };
202
203 /* ADMA Physical Region Descriptor - one SG segment */
204 struct nv_adma_prd {
205 __le64 addr;
206 __le32 len;
207 u8 flags;
208 u8 packet_len;
209 __le16 reserved;
210 };
211
212 enum nv_adma_regbits {
213 CMDEND = (1 << 15), /* end of command list */
214 WNB = (1 << 14), /* wait-not-BSY */
215 IGN = (1 << 13), /* ignore this entry */
216 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
217 DA2 = (1 << (2 + 8)),
218 DA1 = (1 << (1 + 8)),
219 DA0 = (1 << (0 + 8)),
220 };
221
222 /* ADMA Command Parameter Block
223 The first 5 SG segments are stored inside the Command Parameter Block itself.
224 If there are more than 5 segments the remainder are stored in a separate
225 memory area indicated by next_aprd. */
226 struct nv_adma_cpb {
227 u8 resp_flags; /* 0 */
228 u8 reserved1; /* 1 */
229 u8 ctl_flags; /* 2 */
230 /* len is length of taskfile in 64 bit words */
231 u8 len; /* 3 */
232 u8 tag; /* 4 */
233 u8 next_cpb_idx; /* 5 */
234 __le16 reserved2; /* 6-7 */
235 __le16 tf[12]; /* 8-31 */
236 struct nv_adma_prd aprd[5]; /* 32-111 */
237 __le64 next_aprd; /* 112-119 */
238 __le64 reserved3; /* 120-127 */
239 };
240
241
242 struct nv_adma_port_priv {
243 struct nv_adma_cpb *cpb;
244 dma_addr_t cpb_dma;
245 struct nv_adma_prd *aprd;
246 dma_addr_t aprd_dma;
247 void __iomem *ctl_block;
248 void __iomem *gen_block;
249 void __iomem *notifier_clear_block;
250 u64 adma_dma_mask;
251 u8 flags;
252 int last_issue_ncq;
253 };
254
255 struct nv_host_priv {
256 unsigned long type;
257 };
258
259 struct defer_queue {
260 u32 defer_bits;
261 unsigned int head;
262 unsigned int tail;
263 unsigned int tag[ATA_MAX_QUEUE];
264 };
265
266 enum ncq_saw_flag_list {
267 ncq_saw_d2h = (1U << 0),
268 ncq_saw_dmas = (1U << 1),
269 ncq_saw_sdb = (1U << 2),
270 ncq_saw_backout = (1U << 3),
271 };
272
273 struct nv_swncq_port_priv {
274 struct ata_bmdma_prd *prd; /* our SG list */
275 dma_addr_t prd_dma; /* and its DMA mapping */
276 void __iomem *sactive_block;
277 void __iomem *irq_block;
278 void __iomem *tag_block;
279 u32 qc_active;
280
281 unsigned int last_issue_tag;
282
283 /* fifo circular queue to store deferral command */
284 struct defer_queue defer_queue;
285
286 /* for NCQ interrupt analysis */
287 u32 dhfis_bits;
288 u32 dmafis_bits;
289 u32 sdbfis_bits;
290
291 unsigned int ncq_flags;
292 };
293
294
295 #define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
296
297 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
298 #ifdef CONFIG_PM_SLEEP
299 static int nv_pci_device_resume(struct pci_dev *pdev);
300 #endif
301 static void nv_ck804_host_stop(struct ata_host *host);
302 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
303 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
304 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
305 static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
306 static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
307
308 static int nv_hardreset(struct ata_link *link, unsigned int *class,
309 unsigned long deadline);
310 static void nv_nf2_freeze(struct ata_port *ap);
311 static void nv_nf2_thaw(struct ata_port *ap);
312 static void nv_ck804_freeze(struct ata_port *ap);
313 static void nv_ck804_thaw(struct ata_port *ap);
314 static int nv_adma_slave_config(struct scsi_device *sdev);
315 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
316 static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
317 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
318 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
319 static void nv_adma_irq_clear(struct ata_port *ap);
320 static int nv_adma_port_start(struct ata_port *ap);
321 static void nv_adma_port_stop(struct ata_port *ap);
322 #ifdef CONFIG_PM
323 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
324 static int nv_adma_port_resume(struct ata_port *ap);
325 #endif
326 static void nv_adma_freeze(struct ata_port *ap);
327 static void nv_adma_thaw(struct ata_port *ap);
328 static void nv_adma_error_handler(struct ata_port *ap);
329 static void nv_adma_host_stop(struct ata_host *host);
330 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
331 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
332
333 static void nv_mcp55_thaw(struct ata_port *ap);
334 static void nv_mcp55_freeze(struct ata_port *ap);
335 static void nv_swncq_error_handler(struct ata_port *ap);
336 static int nv_swncq_slave_config(struct scsi_device *sdev);
337 static int nv_swncq_port_start(struct ata_port *ap);
338 static void nv_swncq_qc_prep(struct ata_queued_cmd *qc);
339 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
340 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
341 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
342 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
343 #ifdef CONFIG_PM
344 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
345 static int nv_swncq_port_resume(struct ata_port *ap);
346 #endif
347
348 enum nv_host_type
349 {
350 GENERIC,
351 NFORCE2,
352 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
353 CK804,
354 ADMA,
355 MCP5x,
356 SWNCQ,
357 };
358
359 static const struct pci_device_id nv_pci_tbl[] = {
360 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
361 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
362 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
363 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
364 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
365 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
366 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
367 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), MCP5x },
368 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), MCP5x },
369 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), MCP5x },
370 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), MCP5x },
371 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
372 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
373 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
374
375 { } /* terminate list */
376 };
377
378 static struct pci_driver nv_pci_driver = {
379 .name = DRV_NAME,
380 .id_table = nv_pci_tbl,
381 .probe = nv_init_one,
382 #ifdef CONFIG_PM_SLEEP
383 .suspend = ata_pci_device_suspend,
384 .resume = nv_pci_device_resume,
385 #endif
386 .remove = ata_pci_remove_one,
387 };
388
389 static struct scsi_host_template nv_sht = {
390 ATA_BMDMA_SHT(DRV_NAME),
391 };
392
393 static struct scsi_host_template nv_adma_sht = {
394 ATA_NCQ_SHT(DRV_NAME),
395 .can_queue = NV_ADMA_MAX_CPBS,
396 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
397 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
398 .slave_configure = nv_adma_slave_config,
399 };
400
401 static struct scsi_host_template nv_swncq_sht = {
402 ATA_NCQ_SHT(DRV_NAME),
403 .can_queue = ATA_MAX_QUEUE - 1,
404 .sg_tablesize = LIBATA_MAX_PRD,
405 .dma_boundary = ATA_DMA_BOUNDARY,
406 .slave_configure = nv_swncq_slave_config,
407 };
408
409 /*
410 * NV SATA controllers have various different problems with hardreset
411 * protocol depending on the specific controller and device.
412 *
413 * GENERIC:
414 *
415 * bko11195 reports that link doesn't come online after hardreset on
416 * generic nv's and there have been several other similar reports on
417 * linux-ide.
418 *
419 * bko12351#c23 reports that warmplug on MCP61 doesn't work with
420 * softreset.
421 *
422 * NF2/3:
423 *
424 * bko3352 reports nf2/3 controllers can't determine device signature
425 * reliably after hardreset. The following thread reports detection
426 * failure on cold boot with the standard debouncing timing.
427 *
428 * http://thread.gmane.org/gmane.linux.ide/34098
429 *
430 * bko12176 reports that hardreset fails to bring up the link during
431 * boot on nf2.
432 *
433 * CK804:
434 *
435 * For initial probing after boot and hot plugging, hardreset mostly
436 * works fine on CK804 but curiously, reprobing on the initial port
437 * by rescanning or rmmod/insmod fails to acquire the initial D2H Reg
438 * FIS in somewhat undeterministic way.
439 *
440 * SWNCQ:
441 *
442 * bko12351 reports that when SWNCQ is enabled, for hotplug to work,
443 * hardreset should be used and hardreset can't report proper
444 * signature, which suggests that mcp5x is closer to nf2 as long as
445 * reset quirkiness is concerned.
446 *
447 * bko12703 reports that boot probing fails for intel SSD with
448 * hardreset. Link fails to come online. Softreset works fine.
449 *
450 * The failures are varied but the following patterns seem true for
451 * all flavors.
452 *
453 * - Softreset during boot always works.
454 *
455 * - Hardreset during boot sometimes fails to bring up the link on
456 * certain comibnations and device signature acquisition is
457 * unreliable.
458 *
459 * - Hardreset is often necessary after hotplug.
460 *
461 * So, preferring softreset for boot probing and error handling (as
462 * hardreset might bring down the link) but using hardreset for
463 * post-boot probing should work around the above issues in most
464 * cases. Define nv_hardreset() which only kicks in for post-boot
465 * probing and use it for all variants.
466 */
467 static struct ata_port_operations nv_generic_ops = {
468 .inherits = &ata_bmdma_port_ops,
469 .lost_interrupt = ATA_OP_NULL,
470 .scr_read = nv_scr_read,
471 .scr_write = nv_scr_write,
472 .hardreset = nv_hardreset,
473 };
474
475 static struct ata_port_operations nv_nf2_ops = {
476 .inherits = &nv_generic_ops,
477 .freeze = nv_nf2_freeze,
478 .thaw = nv_nf2_thaw,
479 };
480
481 static struct ata_port_operations nv_ck804_ops = {
482 .inherits = &nv_generic_ops,
483 .freeze = nv_ck804_freeze,
484 .thaw = nv_ck804_thaw,
485 .host_stop = nv_ck804_host_stop,
486 };
487
488 static struct ata_port_operations nv_adma_ops = {
489 .inherits = &nv_ck804_ops,
490
491 .check_atapi_dma = nv_adma_check_atapi_dma,
492 .sff_tf_read = nv_adma_tf_read,
493 .qc_defer = ata_std_qc_defer,
494 .qc_prep = nv_adma_qc_prep,
495 .qc_issue = nv_adma_qc_issue,
496 .sff_irq_clear = nv_adma_irq_clear,
497
498 .freeze = nv_adma_freeze,
499 .thaw = nv_adma_thaw,
500 .error_handler = nv_adma_error_handler,
501 .post_internal_cmd = nv_adma_post_internal_cmd,
502
503 .port_start = nv_adma_port_start,
504 .port_stop = nv_adma_port_stop,
505 #ifdef CONFIG_PM
506 .port_suspend = nv_adma_port_suspend,
507 .port_resume = nv_adma_port_resume,
508 #endif
509 .host_stop = nv_adma_host_stop,
510 };
511
512 static struct ata_port_operations nv_swncq_ops = {
513 .inherits = &nv_generic_ops,
514
515 .qc_defer = ata_std_qc_defer,
516 .qc_prep = nv_swncq_qc_prep,
517 .qc_issue = nv_swncq_qc_issue,
518
519 .freeze = nv_mcp55_freeze,
520 .thaw = nv_mcp55_thaw,
521 .error_handler = nv_swncq_error_handler,
522
523 #ifdef CONFIG_PM
524 .port_suspend = nv_swncq_port_suspend,
525 .port_resume = nv_swncq_port_resume,
526 #endif
527 .port_start = nv_swncq_port_start,
528 };
529
530 struct nv_pi_priv {
531 irq_handler_t irq_handler;
532 struct scsi_host_template *sht;
533 };
534
535 #define NV_PI_PRIV(_irq_handler, _sht) \
536 &(struct nv_pi_priv){ .irq_handler = _irq_handler, .sht = _sht }
537
538 static const struct ata_port_info nv_port_info[] = {
539 /* generic */
540 {
541 .flags = ATA_FLAG_SATA,
542 .pio_mask = NV_PIO_MASK,
543 .mwdma_mask = NV_MWDMA_MASK,
544 .udma_mask = NV_UDMA_MASK,
545 .port_ops = &nv_generic_ops,
546 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
547 },
548 /* nforce2/3 */
549 {
550 .flags = ATA_FLAG_SATA,
551 .pio_mask = NV_PIO_MASK,
552 .mwdma_mask = NV_MWDMA_MASK,
553 .udma_mask = NV_UDMA_MASK,
554 .port_ops = &nv_nf2_ops,
555 .private_data = NV_PI_PRIV(nv_nf2_interrupt, &nv_sht),
556 },
557 /* ck804 */
558 {
559 .flags = ATA_FLAG_SATA,
560 .pio_mask = NV_PIO_MASK,
561 .mwdma_mask = NV_MWDMA_MASK,
562 .udma_mask = NV_UDMA_MASK,
563 .port_ops = &nv_ck804_ops,
564 .private_data = NV_PI_PRIV(nv_ck804_interrupt, &nv_sht),
565 },
566 /* ADMA */
567 {
568 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ,
569 .pio_mask = NV_PIO_MASK,
570 .mwdma_mask = NV_MWDMA_MASK,
571 .udma_mask = NV_UDMA_MASK,
572 .port_ops = &nv_adma_ops,
573 .private_data = NV_PI_PRIV(nv_adma_interrupt, &nv_adma_sht),
574 },
575 /* MCP5x */
576 {
577 .flags = ATA_FLAG_SATA,
578 .pio_mask = NV_PIO_MASK,
579 .mwdma_mask = NV_MWDMA_MASK,
580 .udma_mask = NV_UDMA_MASK,
581 .port_ops = &nv_generic_ops,
582 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
583 },
584 /* SWNCQ */
585 {
586 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ,
587 .pio_mask = NV_PIO_MASK,
588 .mwdma_mask = NV_MWDMA_MASK,
589 .udma_mask = NV_UDMA_MASK,
590 .port_ops = &nv_swncq_ops,
591 .private_data = NV_PI_PRIV(nv_swncq_interrupt, &nv_swncq_sht),
592 },
593 };
594
595 MODULE_AUTHOR("NVIDIA");
596 MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
597 MODULE_LICENSE("GPL");
598 MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
599 MODULE_VERSION(DRV_VERSION);
600
601 static bool adma_enabled;
602 static bool swncq_enabled = true;
603 static bool msi_enabled;
604
605 static void nv_adma_register_mode(struct ata_port *ap)
606 {
607 struct nv_adma_port_priv *pp = ap->private_data;
608 void __iomem *mmio = pp->ctl_block;
609 u16 tmp, status;
610 int count = 0;
611
612 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
613 return;
614
615 status = readw(mmio + NV_ADMA_STAT);
616 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
617 ndelay(50);
618 status = readw(mmio + NV_ADMA_STAT);
619 count++;
620 }
621 if (count == 20)
622 ata_port_warn(ap, "timeout waiting for ADMA IDLE, stat=0x%hx\n",
623 status);
624
625 tmp = readw(mmio + NV_ADMA_CTL);
626 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
627
628 count = 0;
629 status = readw(mmio + NV_ADMA_STAT);
630 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
631 ndelay(50);
632 status = readw(mmio + NV_ADMA_STAT);
633 count++;
634 }
635 if (count == 20)
636 ata_port_warn(ap,
637 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
638 status);
639
640 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
641 }
642
643 static void nv_adma_mode(struct ata_port *ap)
644 {
645 struct nv_adma_port_priv *pp = ap->private_data;
646 void __iomem *mmio = pp->ctl_block;
647 u16 tmp, status;
648 int count = 0;
649
650 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
651 return;
652
653 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
654
655 tmp = readw(mmio + NV_ADMA_CTL);
656 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
657
658 status = readw(mmio + NV_ADMA_STAT);
659 while (((status & NV_ADMA_STAT_LEGACY) ||
660 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
661 ndelay(50);
662 status = readw(mmio + NV_ADMA_STAT);
663 count++;
664 }
665 if (count == 20)
666 ata_port_warn(ap,
667 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
668 status);
669
670 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
671 }
672
673 static int nv_adma_slave_config(struct scsi_device *sdev)
674 {
675 struct ata_port *ap = ata_shost_to_port(sdev->host);
676 struct nv_adma_port_priv *pp = ap->private_data;
677 struct nv_adma_port_priv *port0, *port1;
678 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
679 unsigned long segment_boundary, flags;
680 unsigned short sg_tablesize;
681 int rc;
682 int adma_enable;
683 u32 current_reg, new_reg, config_mask;
684
685 rc = ata_scsi_slave_config(sdev);
686
687 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
688 /* Not a proper libata device, ignore */
689 return rc;
690
691 spin_lock_irqsave(ap->lock, flags);
692
693 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
694 /*
695 * NVIDIA reports that ADMA mode does not support ATAPI commands.
696 * Therefore ATAPI commands are sent through the legacy interface.
697 * However, the legacy interface only supports 32-bit DMA.
698 * Restrict DMA parameters as required by the legacy interface
699 * when an ATAPI device is connected.
700 */
701 segment_boundary = ATA_DMA_BOUNDARY;
702 /* Subtract 1 since an extra entry may be needed for padding, see
703 libata-scsi.c */
704 sg_tablesize = LIBATA_MAX_PRD - 1;
705
706 /* Since the legacy DMA engine is in use, we need to disable ADMA
707 on the port. */
708 adma_enable = 0;
709 nv_adma_register_mode(ap);
710 } else {
711 segment_boundary = NV_ADMA_DMA_BOUNDARY;
712 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
713 adma_enable = 1;
714 }
715
716 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
717
718 if (ap->port_no == 1)
719 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
720 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
721 else
722 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
723 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
724
725 if (adma_enable) {
726 new_reg = current_reg | config_mask;
727 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
728 } else {
729 new_reg = current_reg & ~config_mask;
730 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
731 }
732
733 if (current_reg != new_reg)
734 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
735
736 port0 = ap->host->ports[0]->private_data;
737 port1 = ap->host->ports[1]->private_data;
738 if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
739 (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
740 /*
741 * We have to set the DMA mask to 32-bit if either port is in
742 * ATAPI mode, since they are on the same PCI device which is
743 * used for DMA mapping. If either SCSI device is not allocated
744 * yet, it's OK since that port will discover its correct
745 * setting when it does get allocated.
746 */
747 rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
748 } else {
749 rc = dma_set_mask(&pdev->dev, pp->adma_dma_mask);
750 }
751
752 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
753 blk_queue_max_segments(sdev->request_queue, sg_tablesize);
754 ata_port_info(ap,
755 "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
756 (unsigned long long)*ap->host->dev->dma_mask,
757 segment_boundary, sg_tablesize);
758
759 spin_unlock_irqrestore(ap->lock, flags);
760
761 return rc;
762 }
763
764 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
765 {
766 struct nv_adma_port_priv *pp = qc->ap->private_data;
767 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
768 }
769
770 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
771 {
772 /* Other than when internal or pass-through commands are executed,
773 the only time this function will be called in ADMA mode will be
774 if a command fails. In the failure case we don't care about going
775 into register mode with ADMA commands pending, as the commands will
776 all shortly be aborted anyway. We assume that NCQ commands are not
777 issued via passthrough, which is the only way that switching into
778 ADMA mode could abort outstanding commands. */
779 nv_adma_register_mode(ap);
780
781 ata_sff_tf_read(ap, tf);
782 }
783
784 static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
785 {
786 unsigned int idx = 0;
787
788 if (tf->flags & ATA_TFLAG_ISADDR) {
789 if (tf->flags & ATA_TFLAG_LBA48) {
790 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
791 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
792 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
793 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
794 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
795 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
796 } else
797 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
798
799 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
800 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
801 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
802 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
803 }
804
805 if (tf->flags & ATA_TFLAG_DEVICE)
806 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
807
808 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
809
810 while (idx < 12)
811 cpb[idx++] = cpu_to_le16(IGN);
812
813 return idx;
814 }
815
816 static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
817 {
818 struct nv_adma_port_priv *pp = ap->private_data;
819 u8 flags = pp->cpb[cpb_num].resp_flags;
820
821 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
822
823 if (unlikely((force_err ||
824 flags & (NV_CPB_RESP_ATA_ERR |
825 NV_CPB_RESP_CMD_ERR |
826 NV_CPB_RESP_CPB_ERR)))) {
827 struct ata_eh_info *ehi = &ap->link.eh_info;
828 int freeze = 0;
829
830 ata_ehi_clear_desc(ehi);
831 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
832 if (flags & NV_CPB_RESP_ATA_ERR) {
833 ata_ehi_push_desc(ehi, "ATA error");
834 ehi->err_mask |= AC_ERR_DEV;
835 } else if (flags & NV_CPB_RESP_CMD_ERR) {
836 ata_ehi_push_desc(ehi, "CMD error");
837 ehi->err_mask |= AC_ERR_DEV;
838 } else if (flags & NV_CPB_RESP_CPB_ERR) {
839 ata_ehi_push_desc(ehi, "CPB error");
840 ehi->err_mask |= AC_ERR_SYSTEM;
841 freeze = 1;
842 } else {
843 /* notifier error, but no error in CPB flags? */
844 ata_ehi_push_desc(ehi, "unknown");
845 ehi->err_mask |= AC_ERR_OTHER;
846 freeze = 1;
847 }
848 /* Kill all commands. EH will determine what actually failed. */
849 if (freeze)
850 ata_port_freeze(ap);
851 else
852 ata_port_abort(ap);
853 return -1;
854 }
855
856 if (likely(flags & NV_CPB_RESP_DONE))
857 return 1;
858 return 0;
859 }
860
861 static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
862 {
863 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
864
865 /* freeze if hotplugged */
866 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
867 ata_port_freeze(ap);
868 return 1;
869 }
870
871 /* bail out if not our interrupt */
872 if (!(irq_stat & NV_INT_DEV))
873 return 0;
874
875 /* DEV interrupt w/ no active qc? */
876 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
877 ata_sff_check_status(ap);
878 return 1;
879 }
880
881 /* handle interrupt */
882 return ata_bmdma_port_intr(ap, qc);
883 }
884
885 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
886 {
887 struct ata_host *host = dev_instance;
888 int i, handled = 0;
889 u32 notifier_clears[2];
890
891 spin_lock(&host->lock);
892
893 for (i = 0; i < host->n_ports; i++) {
894 struct ata_port *ap = host->ports[i];
895 struct nv_adma_port_priv *pp = ap->private_data;
896 void __iomem *mmio = pp->ctl_block;
897 u16 status;
898 u32 gen_ctl;
899 u32 notifier, notifier_error;
900
901 notifier_clears[i] = 0;
902
903 /* if ADMA is disabled, use standard ata interrupt handler */
904 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
905 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
906 >> (NV_INT_PORT_SHIFT * i);
907 handled += nv_host_intr(ap, irq_stat);
908 continue;
909 }
910
911 /* if in ATA register mode, check for standard interrupts */
912 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
913 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
914 >> (NV_INT_PORT_SHIFT * i);
915 if (ata_tag_valid(ap->link.active_tag))
916 /** NV_INT_DEV indication seems unreliable
917 at times at least in ADMA mode. Force it
918 on always when a command is active, to
919 prevent losing interrupts. */
920 irq_stat |= NV_INT_DEV;
921 handled += nv_host_intr(ap, irq_stat);
922 }
923
924 notifier = readl(mmio + NV_ADMA_NOTIFIER);
925 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
926 notifier_clears[i] = notifier | notifier_error;
927
928 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
929
930 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
931 !notifier_error)
932 /* Nothing to do */
933 continue;
934
935 status = readw(mmio + NV_ADMA_STAT);
936
937 /*
938 * Clear status. Ensure the controller sees the
939 * clearing before we start looking at any of the CPB
940 * statuses, so that any CPB completions after this
941 * point in the handler will raise another interrupt.
942 */
943 writew(status, mmio + NV_ADMA_STAT);
944 readw(mmio + NV_ADMA_STAT); /* flush posted write */
945 rmb();
946
947 handled++; /* irq handled if we got here */
948
949 /* freeze if hotplugged or controller error */
950 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
951 NV_ADMA_STAT_HOTUNPLUG |
952 NV_ADMA_STAT_TIMEOUT |
953 NV_ADMA_STAT_SERROR))) {
954 struct ata_eh_info *ehi = &ap->link.eh_info;
955
956 ata_ehi_clear_desc(ehi);
957 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
958 if (status & NV_ADMA_STAT_TIMEOUT) {
959 ehi->err_mask |= AC_ERR_SYSTEM;
960 ata_ehi_push_desc(ehi, "timeout");
961 } else if (status & NV_ADMA_STAT_HOTPLUG) {
962 ata_ehi_hotplugged(ehi);
963 ata_ehi_push_desc(ehi, "hotplug");
964 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
965 ata_ehi_hotplugged(ehi);
966 ata_ehi_push_desc(ehi, "hot unplug");
967 } else if (status & NV_ADMA_STAT_SERROR) {
968 /* let EH analyze SError and figure out cause */
969 ata_ehi_push_desc(ehi, "SError");
970 } else
971 ata_ehi_push_desc(ehi, "unknown");
972 ata_port_freeze(ap);
973 continue;
974 }
975
976 if (status & (NV_ADMA_STAT_DONE |
977 NV_ADMA_STAT_CPBERR |
978 NV_ADMA_STAT_CMD_COMPLETE)) {
979 u32 check_commands = notifier_clears[i];
980 u32 done_mask = 0;
981 int pos, rc;
982
983 if (status & NV_ADMA_STAT_CPBERR) {
984 /* check all active commands */
985 if (ata_tag_valid(ap->link.active_tag))
986 check_commands = 1 <<
987 ap->link.active_tag;
988 else
989 check_commands = ap->link.sactive;
990 }
991
992 /* check CPBs for completed commands */
993 while ((pos = ffs(check_commands))) {
994 pos--;
995 rc = nv_adma_check_cpb(ap, pos,
996 notifier_error & (1 << pos));
997 if (rc > 0)
998 done_mask |= 1 << pos;
999 else if (unlikely(rc < 0))
1000 check_commands = 0;
1001 check_commands &= ~(1 << pos);
1002 }
1003 ata_qc_complete_multiple(ap, ata_qc_get_active(ap) ^ done_mask);
1004 }
1005 }
1006
1007 if (notifier_clears[0] || notifier_clears[1]) {
1008 /* Note: Both notifier clear registers must be written
1009 if either is set, even if one is zero, according to NVIDIA. */
1010 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
1011 writel(notifier_clears[0], pp->notifier_clear_block);
1012 pp = host->ports[1]->private_data;
1013 writel(notifier_clears[1], pp->notifier_clear_block);
1014 }
1015
1016 spin_unlock(&host->lock);
1017
1018 return IRQ_RETVAL(handled);
1019 }
1020
1021 static void nv_adma_freeze(struct ata_port *ap)
1022 {
1023 struct nv_adma_port_priv *pp = ap->private_data;
1024 void __iomem *mmio = pp->ctl_block;
1025 u16 tmp;
1026
1027 nv_ck804_freeze(ap);
1028
1029 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1030 return;
1031
1032 /* clear any outstanding CK804 notifications */
1033 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1034 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1035
1036 /* Disable interrupt */
1037 tmp = readw(mmio + NV_ADMA_CTL);
1038 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1039 mmio + NV_ADMA_CTL);
1040 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1041 }
1042
1043 static void nv_adma_thaw(struct ata_port *ap)
1044 {
1045 struct nv_adma_port_priv *pp = ap->private_data;
1046 void __iomem *mmio = pp->ctl_block;
1047 u16 tmp;
1048
1049 nv_ck804_thaw(ap);
1050
1051 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1052 return;
1053
1054 /* Enable interrupt */
1055 tmp = readw(mmio + NV_ADMA_CTL);
1056 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1057 mmio + NV_ADMA_CTL);
1058 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1059 }
1060
1061 static void nv_adma_irq_clear(struct ata_port *ap)
1062 {
1063 struct nv_adma_port_priv *pp = ap->private_data;
1064 void __iomem *mmio = pp->ctl_block;
1065 u32 notifier_clears[2];
1066
1067 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
1068 ata_bmdma_irq_clear(ap);
1069 return;
1070 }
1071
1072 /* clear any outstanding CK804 notifications */
1073 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1074 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1075
1076 /* clear ADMA status */
1077 writew(0xffff, mmio + NV_ADMA_STAT);
1078
1079 /* clear notifiers - note both ports need to be written with
1080 something even though we are only clearing on one */
1081 if (ap->port_no == 0) {
1082 notifier_clears[0] = 0xFFFFFFFF;
1083 notifier_clears[1] = 0;
1084 } else {
1085 notifier_clears[0] = 0;
1086 notifier_clears[1] = 0xFFFFFFFF;
1087 }
1088 pp = ap->host->ports[0]->private_data;
1089 writel(notifier_clears[0], pp->notifier_clear_block);
1090 pp = ap->host->ports[1]->private_data;
1091 writel(notifier_clears[1], pp->notifier_clear_block);
1092 }
1093
1094 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
1095 {
1096 struct nv_adma_port_priv *pp = qc->ap->private_data;
1097
1098 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
1099 ata_bmdma_post_internal_cmd(qc);
1100 }
1101
1102 static int nv_adma_port_start(struct ata_port *ap)
1103 {
1104 struct device *dev = ap->host->dev;
1105 struct nv_adma_port_priv *pp;
1106 int rc;
1107 void *mem;
1108 dma_addr_t mem_dma;
1109 void __iomem *mmio;
1110 struct pci_dev *pdev = to_pci_dev(dev);
1111 u16 tmp;
1112
1113 VPRINTK("ENTER\n");
1114
1115 /*
1116 * Ensure DMA mask is set to 32-bit before allocating legacy PRD and
1117 * pad buffers.
1118 */
1119 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1120 if (rc)
1121 return rc;
1122
1123 /* we might fallback to bmdma, allocate bmdma resources */
1124 rc = ata_bmdma_port_start(ap);
1125 if (rc)
1126 return rc;
1127
1128 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1129 if (!pp)
1130 return -ENOMEM;
1131
1132 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
1133 ap->port_no * NV_ADMA_PORT_SIZE;
1134 pp->ctl_block = mmio;
1135 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
1136 pp->notifier_clear_block = pp->gen_block +
1137 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1138
1139 /*
1140 * Now that the legacy PRD and padding buffer are allocated we can
1141 * try to raise the DMA mask to allocate the CPB/APRD table.
1142 */
1143 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1144 if (rc) {
1145 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1146 if (rc)
1147 return rc;
1148 }
1149 pp->adma_dma_mask = *dev->dma_mask;
1150
1151 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1152 &mem_dma, GFP_KERNEL);
1153 if (!mem)
1154 return -ENOMEM;
1155 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1156
1157 /*
1158 * First item in chunk of DMA memory:
1159 * 128-byte command parameter block (CPB)
1160 * one for each command tag
1161 */
1162 pp->cpb = mem;
1163 pp->cpb_dma = mem_dma;
1164
1165 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1166 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1167
1168 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1169 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1170
1171 /*
1172 * Second item: block of ADMA_SGTBL_LEN s/g entries
1173 */
1174 pp->aprd = mem;
1175 pp->aprd_dma = mem_dma;
1176
1177 ap->private_data = pp;
1178
1179 /* clear any outstanding interrupt conditions */
1180 writew(0xffff, mmio + NV_ADMA_STAT);
1181
1182 /* initialize port variables */
1183 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1184
1185 /* clear CPB fetch count */
1186 writew(0, mmio + NV_ADMA_CPB_COUNT);
1187
1188 /* clear GO for register mode, enable interrupt */
1189 tmp = readw(mmio + NV_ADMA_CTL);
1190 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1191 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1192
1193 tmp = readw(mmio + NV_ADMA_CTL);
1194 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1195 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1196 udelay(1);
1197 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1198 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1199
1200 return 0;
1201 }
1202
1203 static void nv_adma_port_stop(struct ata_port *ap)
1204 {
1205 struct nv_adma_port_priv *pp = ap->private_data;
1206 void __iomem *mmio = pp->ctl_block;
1207
1208 VPRINTK("ENTER\n");
1209 writew(0, mmio + NV_ADMA_CTL);
1210 }
1211
1212 #ifdef CONFIG_PM
1213 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1214 {
1215 struct nv_adma_port_priv *pp = ap->private_data;
1216 void __iomem *mmio = pp->ctl_block;
1217
1218 /* Go to register mode - clears GO */
1219 nv_adma_register_mode(ap);
1220
1221 /* clear CPB fetch count */
1222 writew(0, mmio + NV_ADMA_CPB_COUNT);
1223
1224 /* disable interrupt, shut down port */
1225 writew(0, mmio + NV_ADMA_CTL);
1226
1227 return 0;
1228 }
1229
1230 static int nv_adma_port_resume(struct ata_port *ap)
1231 {
1232 struct nv_adma_port_priv *pp = ap->private_data;
1233 void __iomem *mmio = pp->ctl_block;
1234 u16 tmp;
1235
1236 /* set CPB block location */
1237 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1238 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1239
1240 /* clear any outstanding interrupt conditions */
1241 writew(0xffff, mmio + NV_ADMA_STAT);
1242
1243 /* initialize port variables */
1244 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1245
1246 /* clear CPB fetch count */
1247 writew(0, mmio + NV_ADMA_CPB_COUNT);
1248
1249 /* clear GO for register mode, enable interrupt */
1250 tmp = readw(mmio + NV_ADMA_CTL);
1251 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1252 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1253
1254 tmp = readw(mmio + NV_ADMA_CTL);
1255 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1256 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1257 udelay(1);
1258 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1259 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1260
1261 return 0;
1262 }
1263 #endif
1264
1265 static void nv_adma_setup_port(struct ata_port *ap)
1266 {
1267 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1268 struct ata_ioports *ioport = &ap->ioaddr;
1269
1270 VPRINTK("ENTER\n");
1271
1272 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
1273
1274 ioport->cmd_addr = mmio;
1275 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
1276 ioport->error_addr =
1277 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1278 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1279 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1280 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1281 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1282 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
1283 ioport->status_addr =
1284 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
1285 ioport->altstatus_addr =
1286 ioport->ctl_addr = mmio + 0x20;
1287 }
1288
1289 static int nv_adma_host_init(struct ata_host *host)
1290 {
1291 struct pci_dev *pdev = to_pci_dev(host->dev);
1292 unsigned int i;
1293 u32 tmp32;
1294
1295 VPRINTK("ENTER\n");
1296
1297 /* enable ADMA on the ports */
1298 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1299 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1300 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1301 NV_MCP_SATA_CFG_20_PORT1_EN |
1302 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1303
1304 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1305
1306 for (i = 0; i < host->n_ports; i++)
1307 nv_adma_setup_port(host->ports[i]);
1308
1309 return 0;
1310 }
1311
1312 static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1313 struct scatterlist *sg,
1314 int idx,
1315 struct nv_adma_prd *aprd)
1316 {
1317 u8 flags = 0;
1318 if (qc->tf.flags & ATA_TFLAG_WRITE)
1319 flags |= NV_APRD_WRITE;
1320 if (idx == qc->n_elem - 1)
1321 flags |= NV_APRD_END;
1322 else if (idx != 4)
1323 flags |= NV_APRD_CONT;
1324
1325 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1326 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
1327 aprd->flags = flags;
1328 aprd->packet_len = 0;
1329 }
1330
1331 static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1332 {
1333 struct nv_adma_port_priv *pp = qc->ap->private_data;
1334 struct nv_adma_prd *aprd;
1335 struct scatterlist *sg;
1336 unsigned int si;
1337
1338 VPRINTK("ENTER\n");
1339
1340 for_each_sg(qc->sg, sg, qc->n_elem, si) {
1341 aprd = (si < 5) ? &cpb->aprd[si] :
1342 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->hw_tag + (si-5)];
1343 nv_adma_fill_aprd(qc, sg, si, aprd);
1344 }
1345 if (si > 5)
1346 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->hw_tag)));
1347 else
1348 cpb->next_aprd = cpu_to_le64(0);
1349 }
1350
1351 static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1352 {
1353 struct nv_adma_port_priv *pp = qc->ap->private_data;
1354
1355 /* ADMA engine can only be used for non-ATAPI DMA commands,
1356 or interrupt-driven no-data commands. */
1357 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
1358 (qc->tf.flags & ATA_TFLAG_POLLING))
1359 return 1;
1360
1361 if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
1362 (qc->tf.protocol == ATA_PROT_NODATA))
1363 return 0;
1364
1365 return 1;
1366 }
1367
1368 static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1369 {
1370 struct nv_adma_port_priv *pp = qc->ap->private_data;
1371 struct nv_adma_cpb *cpb = &pp->cpb[qc->hw_tag];
1372 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
1373 NV_CPB_CTL_IEN;
1374
1375 if (nv_adma_use_reg_mode(qc)) {
1376 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1377 (qc->flags & ATA_QCFLAG_DMAMAP));
1378 nv_adma_register_mode(qc->ap);
1379 ata_bmdma_qc_prep(qc);
1380 return;
1381 }
1382
1383 cpb->resp_flags = NV_CPB_RESP_DONE;
1384 wmb();
1385 cpb->ctl_flags = 0;
1386 wmb();
1387
1388 cpb->len = 3;
1389 cpb->tag = qc->hw_tag;
1390 cpb->next_cpb_idx = 0;
1391
1392 /* turn on NCQ flags for NCQ commands */
1393 if (qc->tf.protocol == ATA_PROT_NCQ)
1394 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1395
1396 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1397
1398 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1399
1400 if (qc->flags & ATA_QCFLAG_DMAMAP) {
1401 nv_adma_fill_sg(qc, cpb);
1402 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1403 } else
1404 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
1405
1406 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1407 until we are finished filling in all of the contents */
1408 wmb();
1409 cpb->ctl_flags = ctl_flags;
1410 wmb();
1411 cpb->resp_flags = 0;
1412 }
1413
1414 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1415 {
1416 struct nv_adma_port_priv *pp = qc->ap->private_data;
1417 void __iomem *mmio = pp->ctl_block;
1418 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
1419
1420 VPRINTK("ENTER\n");
1421
1422 /* We can't handle result taskfile with NCQ commands, since
1423 retrieving the taskfile switches us out of ADMA mode and would abort
1424 existing commands. */
1425 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1426 (qc->flags & ATA_QCFLAG_RESULT_TF))) {
1427 ata_dev_err(qc->dev, "NCQ w/ RESULT_TF not allowed\n");
1428 return AC_ERR_SYSTEM;
1429 }
1430
1431 if (nv_adma_use_reg_mode(qc)) {
1432 /* use ATA register mode */
1433 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
1434 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1435 (qc->flags & ATA_QCFLAG_DMAMAP));
1436 nv_adma_register_mode(qc->ap);
1437 return ata_bmdma_qc_issue(qc);
1438 } else
1439 nv_adma_mode(qc->ap);
1440
1441 /* write append register, command tag in lower 8 bits
1442 and (number of cpbs to append -1) in top 8 bits */
1443 wmb();
1444
1445 if (curr_ncq != pp->last_issue_ncq) {
1446 /* Seems to need some delay before switching between NCQ and
1447 non-NCQ commands, else we get command timeouts and such. */
1448 udelay(20);
1449 pp->last_issue_ncq = curr_ncq;
1450 }
1451
1452 writew(qc->hw_tag, mmio + NV_ADMA_APPEND);
1453
1454 DPRINTK("Issued tag %u\n", qc->hw_tag);
1455
1456 return 0;
1457 }
1458
1459 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1460 {
1461 struct ata_host *host = dev_instance;
1462 unsigned int i;
1463 unsigned int handled = 0;
1464 unsigned long flags;
1465
1466 spin_lock_irqsave(&host->lock, flags);
1467
1468 for (i = 0; i < host->n_ports; i++) {
1469 struct ata_port *ap = host->ports[i];
1470 struct ata_queued_cmd *qc;
1471
1472 qc = ata_qc_from_tag(ap, ap->link.active_tag);
1473 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING))) {
1474 handled += ata_bmdma_port_intr(ap, qc);
1475 } else {
1476 /*
1477 * No request pending? Clear interrupt status
1478 * anyway, in case there's one pending.
1479 */
1480 ap->ops->sff_check_status(ap);
1481 }
1482 }
1483
1484 spin_unlock_irqrestore(&host->lock, flags);
1485
1486 return IRQ_RETVAL(handled);
1487 }
1488
1489 static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
1490 {
1491 int i, handled = 0;
1492
1493 for (i = 0; i < host->n_ports; i++) {
1494 handled += nv_host_intr(host->ports[i], irq_stat);
1495 irq_stat >>= NV_INT_PORT_SHIFT;
1496 }
1497
1498 return IRQ_RETVAL(handled);
1499 }
1500
1501 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
1502 {
1503 struct ata_host *host = dev_instance;
1504 u8 irq_stat;
1505 irqreturn_t ret;
1506
1507 spin_lock(&host->lock);
1508 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
1509 ret = nv_do_interrupt(host, irq_stat);
1510 spin_unlock(&host->lock);
1511
1512 return ret;
1513 }
1514
1515 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
1516 {
1517 struct ata_host *host = dev_instance;
1518 u8 irq_stat;
1519 irqreturn_t ret;
1520
1521 spin_lock(&host->lock);
1522 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1523 ret = nv_do_interrupt(host, irq_stat);
1524 spin_unlock(&host->lock);
1525
1526 return ret;
1527 }
1528
1529 static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
1530 {
1531 if (sc_reg > SCR_CONTROL)
1532 return -EINVAL;
1533
1534 *val = ioread32(link->ap->ioaddr.scr_addr + (sc_reg * 4));
1535 return 0;
1536 }
1537
1538 static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
1539 {
1540 if (sc_reg > SCR_CONTROL)
1541 return -EINVAL;
1542
1543 iowrite32(val, link->ap->ioaddr.scr_addr + (sc_reg * 4));
1544 return 0;
1545 }
1546
1547 static int nv_hardreset(struct ata_link *link, unsigned int *class,
1548 unsigned long deadline)
1549 {
1550 struct ata_eh_context *ehc = &link->eh_context;
1551
1552 /* Do hardreset iff it's post-boot probing, please read the
1553 * comment above port ops for details.
1554 */
1555 if (!(link->ap->pflags & ATA_PFLAG_LOADING) &&
1556 !ata_dev_enabled(link->device))
1557 sata_link_hardreset(link, sata_deb_timing_hotplug, deadline,
1558 NULL, NULL);
1559 else {
1560 const unsigned long *timing = sata_ehc_deb_timing(ehc);
1561 int rc;
1562
1563 if (!(ehc->i.flags & ATA_EHI_QUIET))
1564 ata_link_info(link,
1565 "nv: skipping hardreset on occupied port\n");
1566
1567 /* make sure the link is online */
1568 rc = sata_link_resume(link, timing, deadline);
1569 /* whine about phy resume failure but proceed */
1570 if (rc && rc != -EOPNOTSUPP)
1571 ata_link_warn(link, "failed to resume link (errno=%d)\n",
1572 rc);
1573 }
1574
1575 /* device signature acquisition is unreliable */
1576 return -EAGAIN;
1577 }
1578
1579 static void nv_nf2_freeze(struct ata_port *ap)
1580 {
1581 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1582 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1583 u8 mask;
1584
1585 mask = ioread8(scr_addr + NV_INT_ENABLE);
1586 mask &= ~(NV_INT_ALL << shift);
1587 iowrite8(mask, scr_addr + NV_INT_ENABLE);
1588 }
1589
1590 static void nv_nf2_thaw(struct ata_port *ap)
1591 {
1592 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1593 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1594 u8 mask;
1595
1596 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
1597
1598 mask = ioread8(scr_addr + NV_INT_ENABLE);
1599 mask |= (NV_INT_MASK << shift);
1600 iowrite8(mask, scr_addr + NV_INT_ENABLE);
1601 }
1602
1603 static void nv_ck804_freeze(struct ata_port *ap)
1604 {
1605 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1606 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1607 u8 mask;
1608
1609 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1610 mask &= ~(NV_INT_ALL << shift);
1611 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1612 }
1613
1614 static void nv_ck804_thaw(struct ata_port *ap)
1615 {
1616 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1617 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1618 u8 mask;
1619
1620 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1621
1622 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1623 mask |= (NV_INT_MASK << shift);
1624 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1625 }
1626
1627 static void nv_mcp55_freeze(struct ata_port *ap)
1628 {
1629 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1630 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1631 u32 mask;
1632
1633 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1634
1635 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1636 mask &= ~(NV_INT_ALL_MCP55 << shift);
1637 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1638 }
1639
1640 static void nv_mcp55_thaw(struct ata_port *ap)
1641 {
1642 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1643 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1644 u32 mask;
1645
1646 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1647
1648 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1649 mask |= (NV_INT_MASK_MCP55 << shift);
1650 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1651 }
1652
1653 static void nv_adma_error_handler(struct ata_port *ap)
1654 {
1655 struct nv_adma_port_priv *pp = ap->private_data;
1656 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
1657 void __iomem *mmio = pp->ctl_block;
1658 int i;
1659 u16 tmp;
1660
1661 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
1662 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1663 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1664 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1665 u32 status = readw(mmio + NV_ADMA_STAT);
1666 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1667 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
1668
1669 ata_port_err(ap,
1670 "EH in ADMA mode, notifier 0x%X "
1671 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1672 "next cpb count 0x%X next cpb idx 0x%x\n",
1673 notifier, notifier_error, gen_ctl, status,
1674 cpb_count, next_cpb_idx);
1675
1676 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
1677 struct nv_adma_cpb *cpb = &pp->cpb[i];
1678 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
1679 ap->link.sactive & (1 << i))
1680 ata_port_err(ap,
1681 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1682 i, cpb->ctl_flags, cpb->resp_flags);
1683 }
1684 }
1685
1686 /* Push us back into port register mode for error handling. */
1687 nv_adma_register_mode(ap);
1688
1689 /* Mark all of the CPBs as invalid to prevent them from
1690 being executed */
1691 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
1692 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1693
1694 /* clear CPB fetch count */
1695 writew(0, mmio + NV_ADMA_CPB_COUNT);
1696
1697 /* Reset channel */
1698 tmp = readw(mmio + NV_ADMA_CTL);
1699 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1700 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1701 udelay(1);
1702 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1703 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1704 }
1705
1706 ata_bmdma_error_handler(ap);
1707 }
1708
1709 static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1710 {
1711 struct nv_swncq_port_priv *pp = ap->private_data;
1712 struct defer_queue *dq = &pp->defer_queue;
1713
1714 /* queue is full */
1715 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
1716 dq->defer_bits |= (1 << qc->hw_tag);
1717 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->hw_tag;
1718 }
1719
1720 static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1721 {
1722 struct nv_swncq_port_priv *pp = ap->private_data;
1723 struct defer_queue *dq = &pp->defer_queue;
1724 unsigned int tag;
1725
1726 if (dq->head == dq->tail) /* null queue */
1727 return NULL;
1728
1729 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1730 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1731 WARN_ON(!(dq->defer_bits & (1 << tag)));
1732 dq->defer_bits &= ~(1 << tag);
1733
1734 return ata_qc_from_tag(ap, tag);
1735 }
1736
1737 static void nv_swncq_fis_reinit(struct ata_port *ap)
1738 {
1739 struct nv_swncq_port_priv *pp = ap->private_data;
1740
1741 pp->dhfis_bits = 0;
1742 pp->dmafis_bits = 0;
1743 pp->sdbfis_bits = 0;
1744 pp->ncq_flags = 0;
1745 }
1746
1747 static void nv_swncq_pp_reinit(struct ata_port *ap)
1748 {
1749 struct nv_swncq_port_priv *pp = ap->private_data;
1750 struct defer_queue *dq = &pp->defer_queue;
1751
1752 dq->head = 0;
1753 dq->tail = 0;
1754 dq->defer_bits = 0;
1755 pp->qc_active = 0;
1756 pp->last_issue_tag = ATA_TAG_POISON;
1757 nv_swncq_fis_reinit(ap);
1758 }
1759
1760 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1761 {
1762 struct nv_swncq_port_priv *pp = ap->private_data;
1763
1764 writew(fis, pp->irq_block);
1765 }
1766
1767 static void __ata_bmdma_stop(struct ata_port *ap)
1768 {
1769 struct ata_queued_cmd qc;
1770
1771 qc.ap = ap;
1772 ata_bmdma_stop(&qc);
1773 }
1774
1775 static void nv_swncq_ncq_stop(struct ata_port *ap)
1776 {
1777 struct nv_swncq_port_priv *pp = ap->private_data;
1778 unsigned int i;
1779 u32 sactive;
1780 u32 done_mask;
1781
1782 ata_port_err(ap, "EH in SWNCQ mode,QC:qc_active 0x%llX sactive 0x%X\n",
1783 ap->qc_active, ap->link.sactive);
1784 ata_port_err(ap,
1785 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n "
1786 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1787 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1788 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1789
1790 ata_port_err(ap, "ATA_REG 0x%X ERR_REG 0x%X\n",
1791 ap->ops->sff_check_status(ap),
1792 ioread8(ap->ioaddr.error_addr));
1793
1794 sactive = readl(pp->sactive_block);
1795 done_mask = pp->qc_active ^ sactive;
1796
1797 ata_port_err(ap, "tag : dhfis dmafis sdbfis sactive\n");
1798 for (i = 0; i < ATA_MAX_QUEUE; i++) {
1799 u8 err = 0;
1800 if (pp->qc_active & (1 << i))
1801 err = 0;
1802 else if (done_mask & (1 << i))
1803 err = 1;
1804 else
1805 continue;
1806
1807 ata_port_err(ap,
1808 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1809 (pp->dhfis_bits >> i) & 0x1,
1810 (pp->dmafis_bits >> i) & 0x1,
1811 (pp->sdbfis_bits >> i) & 0x1,
1812 (sactive >> i) & 0x1,
1813 (err ? "error! tag doesn't exit" : " "));
1814 }
1815
1816 nv_swncq_pp_reinit(ap);
1817 ap->ops->sff_irq_clear(ap);
1818 __ata_bmdma_stop(ap);
1819 nv_swncq_irq_clear(ap, 0xffff);
1820 }
1821
1822 static void nv_swncq_error_handler(struct ata_port *ap)
1823 {
1824 struct ata_eh_context *ehc = &ap->link.eh_context;
1825
1826 if (ap->link.sactive) {
1827 nv_swncq_ncq_stop(ap);
1828 ehc->i.action |= ATA_EH_RESET;
1829 }
1830
1831 ata_bmdma_error_handler(ap);
1832 }
1833
1834 #ifdef CONFIG_PM
1835 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1836 {
1837 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1838 u32 tmp;
1839
1840 /* clear irq */
1841 writel(~0, mmio + NV_INT_STATUS_MCP55);
1842
1843 /* disable irq */
1844 writel(0, mmio + NV_INT_ENABLE_MCP55);
1845
1846 /* disable swncq */
1847 tmp = readl(mmio + NV_CTL_MCP55);
1848 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1849 writel(tmp, mmio + NV_CTL_MCP55);
1850
1851 return 0;
1852 }
1853
1854 static int nv_swncq_port_resume(struct ata_port *ap)
1855 {
1856 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1857 u32 tmp;
1858
1859 /* clear irq */
1860 writel(~0, mmio + NV_INT_STATUS_MCP55);
1861
1862 /* enable irq */
1863 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1864
1865 /* enable swncq */
1866 tmp = readl(mmio + NV_CTL_MCP55);
1867 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1868
1869 return 0;
1870 }
1871 #endif
1872
1873 static void nv_swncq_host_init(struct ata_host *host)
1874 {
1875 u32 tmp;
1876 void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1877 struct pci_dev *pdev = to_pci_dev(host->dev);
1878 u8 regval;
1879
1880 /* disable ECO 398 */
1881 pci_read_config_byte(pdev, 0x7f, &regval);
1882 regval &= ~(1 << 7);
1883 pci_write_config_byte(pdev, 0x7f, regval);
1884
1885 /* enable swncq */
1886 tmp = readl(mmio + NV_CTL_MCP55);
1887 VPRINTK("HOST_CTL:0x%X\n", tmp);
1888 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1889
1890 /* enable irq intr */
1891 tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1892 VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1893 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1894
1895 /* clear port irq */
1896 writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1897 }
1898
1899 static int nv_swncq_slave_config(struct scsi_device *sdev)
1900 {
1901 struct ata_port *ap = ata_shost_to_port(sdev->host);
1902 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1903 struct ata_device *dev;
1904 int rc;
1905 u8 rev;
1906 u8 check_maxtor = 0;
1907 unsigned char model_num[ATA_ID_PROD_LEN + 1];
1908
1909 rc = ata_scsi_slave_config(sdev);
1910 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1911 /* Not a proper libata device, ignore */
1912 return rc;
1913
1914 dev = &ap->link.device[sdev->id];
1915 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1916 return rc;
1917
1918 /* if MCP51 and Maxtor, then disable ncq */
1919 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1920 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1921 check_maxtor = 1;
1922
1923 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1924 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1925 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1926 pci_read_config_byte(pdev, 0x8, &rev);
1927 if (rev <= 0xa2)
1928 check_maxtor = 1;
1929 }
1930
1931 if (!check_maxtor)
1932 return rc;
1933
1934 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1935
1936 if (strncmp(model_num, "Maxtor", 6) == 0) {
1937 ata_scsi_change_queue_depth(sdev, 1);
1938 ata_dev_notice(dev, "Disabling SWNCQ mode (depth %x)\n",
1939 sdev->queue_depth);
1940 }
1941
1942 return rc;
1943 }
1944
1945 static int nv_swncq_port_start(struct ata_port *ap)
1946 {
1947 struct device *dev = ap->host->dev;
1948 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1949 struct nv_swncq_port_priv *pp;
1950 int rc;
1951
1952 /* we might fallback to bmdma, allocate bmdma resources */
1953 rc = ata_bmdma_port_start(ap);
1954 if (rc)
1955 return rc;
1956
1957 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1958 if (!pp)
1959 return -ENOMEM;
1960
1961 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1962 &pp->prd_dma, GFP_KERNEL);
1963 if (!pp->prd)
1964 return -ENOMEM;
1965 memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
1966
1967 ap->private_data = pp;
1968 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1969 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1970 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1971
1972 return 0;
1973 }
1974
1975 static void nv_swncq_qc_prep(struct ata_queued_cmd *qc)
1976 {
1977 if (qc->tf.protocol != ATA_PROT_NCQ) {
1978 ata_bmdma_qc_prep(qc);
1979 return;
1980 }
1981
1982 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
1983 return;
1984
1985 nv_swncq_fill_sg(qc);
1986 }
1987
1988 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
1989 {
1990 struct ata_port *ap = qc->ap;
1991 struct scatterlist *sg;
1992 struct nv_swncq_port_priv *pp = ap->private_data;
1993 struct ata_bmdma_prd *prd;
1994 unsigned int si, idx;
1995
1996 prd = pp->prd + ATA_MAX_PRD * qc->hw_tag;
1997
1998 idx = 0;
1999 for_each_sg(qc->sg, sg, qc->n_elem, si) {
2000 u32 addr, offset;
2001 u32 sg_len, len;
2002
2003 addr = (u32)sg_dma_address(sg);
2004 sg_len = sg_dma_len(sg);
2005
2006 while (sg_len) {
2007 offset = addr & 0xffff;
2008 len = sg_len;
2009 if ((offset + sg_len) > 0x10000)
2010 len = 0x10000 - offset;
2011
2012 prd[idx].addr = cpu_to_le32(addr);
2013 prd[idx].flags_len = cpu_to_le32(len & 0xffff);
2014
2015 idx++;
2016 sg_len -= len;
2017 addr += len;
2018 }
2019 }
2020
2021 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
2022 }
2023
2024 static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
2025 struct ata_queued_cmd *qc)
2026 {
2027 struct nv_swncq_port_priv *pp = ap->private_data;
2028
2029 if (qc == NULL)
2030 return 0;
2031
2032 DPRINTK("Enter\n");
2033
2034 writel((1 << qc->hw_tag), pp->sactive_block);
2035 pp->last_issue_tag = qc->hw_tag;
2036 pp->dhfis_bits &= ~(1 << qc->hw_tag);
2037 pp->dmafis_bits &= ~(1 << qc->hw_tag);
2038 pp->qc_active |= (0x1 << qc->hw_tag);
2039
2040 ap->ops->sff_tf_load(ap, &qc->tf); /* load tf registers */
2041 ap->ops->sff_exec_command(ap, &qc->tf);
2042
2043 DPRINTK("Issued tag %u\n", qc->hw_tag);
2044
2045 return 0;
2046 }
2047
2048 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2049 {
2050 struct ata_port *ap = qc->ap;
2051 struct nv_swncq_port_priv *pp = ap->private_data;
2052
2053 if (qc->tf.protocol != ATA_PROT_NCQ)
2054 return ata_bmdma_qc_issue(qc);
2055
2056 DPRINTK("Enter\n");
2057
2058 if (!pp->qc_active)
2059 nv_swncq_issue_atacmd(ap, qc);
2060 else
2061 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */
2062
2063 return 0;
2064 }
2065
2066 static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2067 {
2068 u32 serror;
2069 struct ata_eh_info *ehi = &ap->link.eh_info;
2070
2071 ata_ehi_clear_desc(ehi);
2072
2073 /* AHCI needs SError cleared; otherwise, it might lock up */
2074 sata_scr_read(&ap->link, SCR_ERROR, &serror);
2075 sata_scr_write(&ap->link, SCR_ERROR, serror);
2076
2077 /* analyze @irq_stat */
2078 if (fis & NV_SWNCQ_IRQ_ADDED)
2079 ata_ehi_push_desc(ehi, "hot plug");
2080 else if (fis & NV_SWNCQ_IRQ_REMOVED)
2081 ata_ehi_push_desc(ehi, "hot unplug");
2082
2083 ata_ehi_hotplugged(ehi);
2084
2085 /* okay, let's hand over to EH */
2086 ehi->serror |= serror;
2087
2088 ata_port_freeze(ap);
2089 }
2090
2091 static int nv_swncq_sdbfis(struct ata_port *ap)
2092 {
2093 struct ata_queued_cmd *qc;
2094 struct nv_swncq_port_priv *pp = ap->private_data;
2095 struct ata_eh_info *ehi = &ap->link.eh_info;
2096 u32 sactive;
2097 u32 done_mask;
2098 u8 host_stat;
2099 u8 lack_dhfis = 0;
2100
2101 host_stat = ap->ops->bmdma_status(ap);
2102 if (unlikely(host_stat & ATA_DMA_ERR)) {
2103 /* error when transferring data to/from memory */
2104 ata_ehi_clear_desc(ehi);
2105 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2106 ehi->err_mask |= AC_ERR_HOST_BUS;
2107 ehi->action |= ATA_EH_RESET;
2108 return -EINVAL;
2109 }
2110
2111 ap->ops->sff_irq_clear(ap);
2112 __ata_bmdma_stop(ap);
2113
2114 sactive = readl(pp->sactive_block);
2115 done_mask = pp->qc_active ^ sactive;
2116
2117 pp->qc_active &= ~done_mask;
2118 pp->dhfis_bits &= ~done_mask;
2119 pp->dmafis_bits &= ~done_mask;
2120 pp->sdbfis_bits |= done_mask;
2121 ata_qc_complete_multiple(ap, ap->qc_active ^ done_mask);
2122
2123 if (!ap->qc_active) {
2124 DPRINTK("over\n");
2125 nv_swncq_pp_reinit(ap);
2126 return 0;
2127 }
2128
2129 if (pp->qc_active & pp->dhfis_bits)
2130 return 0;
2131
2132 if ((pp->ncq_flags & ncq_saw_backout) ||
2133 (pp->qc_active ^ pp->dhfis_bits))
2134 /* if the controller can't get a device to host register FIS,
2135 * The driver needs to reissue the new command.
2136 */
2137 lack_dhfis = 1;
2138
2139 DPRINTK("id 0x%x QC: qc_active 0x%x,"
2140 "SWNCQ:qc_active 0x%X defer_bits %X "
2141 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2142 ap->print_id, ap->qc_active, pp->qc_active,
2143 pp->defer_queue.defer_bits, pp->dhfis_bits,
2144 pp->dmafis_bits, pp->last_issue_tag);
2145
2146 nv_swncq_fis_reinit(ap);
2147
2148 if (lack_dhfis) {
2149 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2150 nv_swncq_issue_atacmd(ap, qc);
2151 return 0;
2152 }
2153
2154 if (pp->defer_queue.defer_bits) {
2155 /* send deferral queue command */
2156 qc = nv_swncq_qc_from_dq(ap);
2157 WARN_ON(qc == NULL);
2158 nv_swncq_issue_atacmd(ap, qc);
2159 }
2160
2161 return 0;
2162 }
2163
2164 static inline u32 nv_swncq_tag(struct ata_port *ap)
2165 {
2166 struct nv_swncq_port_priv *pp = ap->private_data;
2167 u32 tag;
2168
2169 tag = readb(pp->tag_block) >> 2;
2170 return (tag & 0x1f);
2171 }
2172
2173 static void nv_swncq_dmafis(struct ata_port *ap)
2174 {
2175 struct ata_queued_cmd *qc;
2176 unsigned int rw;
2177 u8 dmactl;
2178 u32 tag;
2179 struct nv_swncq_port_priv *pp = ap->private_data;
2180
2181 __ata_bmdma_stop(ap);
2182 tag = nv_swncq_tag(ap);
2183
2184 DPRINTK("dma setup tag 0x%x\n", tag);
2185 qc = ata_qc_from_tag(ap, tag);
2186
2187 if (unlikely(!qc))
2188 return;
2189
2190 rw = qc->tf.flags & ATA_TFLAG_WRITE;
2191
2192 /* load PRD table addr. */
2193 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->hw_tag,
2194 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2195
2196 /* specify data direction, triple-check start bit is clear */
2197 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2198 dmactl &= ~ATA_DMA_WR;
2199 if (!rw)
2200 dmactl |= ATA_DMA_WR;
2201
2202 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2203 }
2204
2205 static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2206 {
2207 struct nv_swncq_port_priv *pp = ap->private_data;
2208 struct ata_queued_cmd *qc;
2209 struct ata_eh_info *ehi = &ap->link.eh_info;
2210 u32 serror;
2211 u8 ata_stat;
2212
2213 ata_stat = ap->ops->sff_check_status(ap);
2214 nv_swncq_irq_clear(ap, fis);
2215 if (!fis)
2216 return;
2217
2218 if (ap->pflags & ATA_PFLAG_FROZEN)
2219 return;
2220
2221 if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2222 nv_swncq_hotplug(ap, fis);
2223 return;
2224 }
2225
2226 if (!pp->qc_active)
2227 return;
2228
2229 if (ap->ops->scr_read(&ap->link, SCR_ERROR, &serror))
2230 return;
2231 ap->ops->scr_write(&ap->link, SCR_ERROR, serror);
2232
2233 if (ata_stat & ATA_ERR) {
2234 ata_ehi_clear_desc(ehi);
2235 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2236 ehi->err_mask |= AC_ERR_DEV;
2237 ehi->serror |= serror;
2238 ehi->action |= ATA_EH_RESET;
2239 ata_port_freeze(ap);
2240 return;
2241 }
2242
2243 if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2244 /* If the IRQ is backout, driver must issue
2245 * the new command again some time later.
2246 */
2247 pp->ncq_flags |= ncq_saw_backout;
2248 }
2249
2250 if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2251 pp->ncq_flags |= ncq_saw_sdb;
2252 DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2253 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2254 ap->print_id, pp->qc_active, pp->dhfis_bits,
2255 pp->dmafis_bits, readl(pp->sactive_block));
2256 if (nv_swncq_sdbfis(ap) < 0)
2257 goto irq_error;
2258 }
2259
2260 if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2261 /* The interrupt indicates the new command
2262 * was transmitted correctly to the drive.
2263 */
2264 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2265 pp->ncq_flags |= ncq_saw_d2h;
2266 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2267 ata_ehi_push_desc(ehi, "illegal fis transaction");
2268 ehi->err_mask |= AC_ERR_HSM;
2269 ehi->action |= ATA_EH_RESET;
2270 goto irq_error;
2271 }
2272
2273 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2274 !(pp->ncq_flags & ncq_saw_dmas)) {
2275 ata_stat = ap->ops->sff_check_status(ap);
2276 if (ata_stat & ATA_BUSY)
2277 goto irq_exit;
2278
2279 if (pp->defer_queue.defer_bits) {
2280 DPRINTK("send next command\n");
2281 qc = nv_swncq_qc_from_dq(ap);
2282 nv_swncq_issue_atacmd(ap, qc);
2283 }
2284 }
2285 }
2286
2287 if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2288 /* program the dma controller with appropriate PRD buffers
2289 * and start the DMA transfer for requested command.
2290 */
2291 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2292 pp->ncq_flags |= ncq_saw_dmas;
2293 nv_swncq_dmafis(ap);
2294 }
2295
2296 irq_exit:
2297 return;
2298 irq_error:
2299 ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2300 ata_port_freeze(ap);
2301 return;
2302 }
2303
2304 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2305 {
2306 struct ata_host *host = dev_instance;
2307 unsigned int i;
2308 unsigned int handled = 0;
2309 unsigned long flags;
2310 u32 irq_stat;
2311
2312 spin_lock_irqsave(&host->lock, flags);
2313
2314 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2315
2316 for (i = 0; i < host->n_ports; i++) {
2317 struct ata_port *ap = host->ports[i];
2318
2319 if (ap->link.sactive) {
2320 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2321 handled = 1;
2322 } else {
2323 if (irq_stat) /* reserve Hotplug */
2324 nv_swncq_irq_clear(ap, 0xfff0);
2325
2326 handled += nv_host_intr(ap, (u8)irq_stat);
2327 }
2328 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2329 }
2330
2331 spin_unlock_irqrestore(&host->lock, flags);
2332
2333 return IRQ_RETVAL(handled);
2334 }
2335
2336 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
2337 {
2338 const struct ata_port_info *ppi[] = { NULL, NULL };
2339 struct nv_pi_priv *ipriv;
2340 struct ata_host *host;
2341 struct nv_host_priv *hpriv;
2342 int rc;
2343 u32 bar;
2344 void __iomem *base;
2345 unsigned long type = ent->driver_data;
2346
2347 // Make sure this is a SATA controller by counting the number of bars
2348 // (NVIDIA SATA controllers will always have six bars). Otherwise,
2349 // it's an IDE controller and we ignore it.
2350 for (bar = 0; bar < 6; bar++)
2351 if (pci_resource_start(pdev, bar) == 0)
2352 return -ENODEV;
2353
2354 ata_print_version_once(&pdev->dev, DRV_VERSION);
2355
2356 rc = pcim_enable_device(pdev);
2357 if (rc)
2358 return rc;
2359
2360 /* determine type and allocate host */
2361 if (type == CK804 && adma_enabled) {
2362 dev_notice(&pdev->dev, "Using ADMA mode\n");
2363 type = ADMA;
2364 } else if (type == MCP5x && swncq_enabled) {
2365 dev_notice(&pdev->dev, "Using SWNCQ mode\n");
2366 type = SWNCQ;
2367 }
2368
2369 ppi[0] = &nv_port_info[type];
2370 ipriv = ppi[0]->private_data;
2371 rc = ata_pci_bmdma_prepare_host(pdev, ppi, &host);
2372 if (rc)
2373 return rc;
2374
2375 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
2376 if (!hpriv)
2377 return -ENOMEM;
2378 hpriv->type = type;
2379 host->private_data = hpriv;
2380
2381 /* request and iomap NV_MMIO_BAR */
2382 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2383 if (rc)
2384 return rc;
2385
2386 /* configure SCR access */
2387 base = host->iomap[NV_MMIO_BAR];
2388 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2389 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
2390
2391 /* enable SATA space for CK804 */
2392 if (type >= CK804) {
2393 u8 regval;
2394
2395 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2396 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2397 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2398 }
2399
2400 /* init ADMA */
2401 if (type == ADMA) {
2402 rc = nv_adma_host_init(host);
2403 if (rc)
2404 return rc;
2405 } else if (type == SWNCQ)
2406 nv_swncq_host_init(host);
2407
2408 if (msi_enabled) {
2409 dev_notice(&pdev->dev, "Using MSI\n");
2410 pci_enable_msi(pdev);
2411 }
2412
2413 pci_set_master(pdev);
2414 return ata_pci_sff_activate_host(host, ipriv->irq_handler, ipriv->sht);
2415 }
2416
2417 #ifdef CONFIG_PM_SLEEP
2418 static int nv_pci_device_resume(struct pci_dev *pdev)
2419 {
2420 struct ata_host *host = pci_get_drvdata(pdev);
2421 struct nv_host_priv *hpriv = host->private_data;
2422 int rc;
2423
2424 rc = ata_pci_device_do_resume(pdev);
2425 if (rc)
2426 return rc;
2427
2428 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
2429 if (hpriv->type >= CK804) {
2430 u8 regval;
2431
2432 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2433 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2434 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2435 }
2436 if (hpriv->type == ADMA) {
2437 u32 tmp32;
2438 struct nv_adma_port_priv *pp;
2439 /* enable/disable ADMA on the ports appropriately */
2440 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2441
2442 pp = host->ports[0]->private_data;
2443 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2444 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2445 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2446 else
2447 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
2448 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2449 pp = host->ports[1]->private_data;
2450 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2451 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
2452 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2453 else
2454 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
2455 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2456
2457 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2458 }
2459 }
2460
2461 ata_host_resume(host);
2462
2463 return 0;
2464 }
2465 #endif
2466
2467 static void nv_ck804_host_stop(struct ata_host *host)
2468 {
2469 struct pci_dev *pdev = to_pci_dev(host->dev);
2470 u8 regval;
2471
2472 /* disable SATA space for CK804 */
2473 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2474 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2475 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2476 }
2477
2478 static void nv_adma_host_stop(struct ata_host *host)
2479 {
2480 struct pci_dev *pdev = to_pci_dev(host->dev);
2481 u32 tmp32;
2482
2483 /* disable ADMA on the ports */
2484 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2485 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2486 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2487 NV_MCP_SATA_CFG_20_PORT1_EN |
2488 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2489
2490 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2491
2492 nv_ck804_host_stop(host);
2493 }
2494
2495 module_pci_driver(nv_pci_driver);
2496
2497 module_param_named(adma, adma_enabled, bool, 0444);
2498 MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: false)");
2499 module_param_named(swncq, swncq_enabled, bool, 0444);
2500 MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: true)");
2501 module_param_named(msi, msi_enabled, bool, 0444);
2502 MODULE_PARM_DESC(msi, "Enable use of MSI (Default: false)");
Linux with added FPC-III support