?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
iwlwifi: introduce host commands callbacks
[linux/fpc-iii.git] / drivers / ata / sata_nv.c
blobed5473bf7a0a2005ea5e156cc1eff7cc723ca54d
1 /*
2 * sata_nv.c - NVIDIA nForce SATA
3 *
4 * Copyright 2004 NVIDIA Corp. All rights reserved.
5 * Copyright 2004 Andrew Chew
6 *
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; see the file COPYING. If not, write to
20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
21 *
22 *
23 * libata documentation is available via 'make {ps|pdf}docs',
24 * as Documentation/DocBook/libata.*
25 *
26 * No hardware documentation available outside of NVIDIA.
27 * This driver programs the NVIDIA SATA controller in a similar
28 * fashion as with other PCI IDE BMDMA controllers, with a few
29 * NV-specific details such as register offsets, SATA phy location,
30 * hotplug info, etc.
31 *
32 * CK804/MCP04 controllers support an alternate programming interface
33 * similar to the ADMA specification (with some modifications).
34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
35 * sent through the legacy interface.
36 *
37 */
38
39 #include <linux/kernel.h>
40 #include <linux/module.h>
41 #include <linux/pci.h>
42 #include <linux/init.h>
43 #include <linux/blkdev.h>
44 #include <linux/delay.h>
45 #include <linux/interrupt.h>
46 #include <linux/device.h>
47 #include <scsi/scsi_host.h>
48 #include <scsi/scsi_device.h>
49 #include <linux/libata.h>
50
51 #define DRV_NAME "sata_nv"
52 #define DRV_VERSION "3.5"
53
54 #define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
55
56 enum {
57 NV_MMIO_BAR = 5,
58
59 NV_PORTS = 2,
60 NV_PIO_MASK = 0x1f,
61 NV_MWDMA_MASK = 0x07,
62 NV_UDMA_MASK = 0x7f,
63 NV_PORT0_SCR_REG_OFFSET = 0x00,
64 NV_PORT1_SCR_REG_OFFSET = 0x40,
65
66 /* INT_STATUS/ENABLE */
67 NV_INT_STATUS = 0x10,
68 NV_INT_ENABLE = 0x11,
69 NV_INT_STATUS_CK804 = 0x440,
70 NV_INT_ENABLE_CK804 = 0x441,
71
72 /* INT_STATUS/ENABLE bits */
73 NV_INT_DEV = 0x01,
74 NV_INT_PM = 0x02,
75 NV_INT_ADDED = 0x04,
76 NV_INT_REMOVED = 0x08,
77
78 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
79
80 NV_INT_ALL = 0x0f,
81 NV_INT_MASK = NV_INT_DEV |
82 NV_INT_ADDED | NV_INT_REMOVED,
83
84 /* INT_CONFIG */
85 NV_INT_CONFIG = 0x12,
86 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
87
88 // For PCI config register 20
89 NV_MCP_SATA_CFG_20 = 0x50,
90 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
91 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
92 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
93 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
94 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
95
96 NV_ADMA_MAX_CPBS = 32,
97 NV_ADMA_CPB_SZ = 128,
98 NV_ADMA_APRD_SZ = 16,
99 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
100 NV_ADMA_APRD_SZ,
101 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
102 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
103 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
104 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
105
106 /* BAR5 offset to ADMA general registers */
107 NV_ADMA_GEN = 0x400,
108 NV_ADMA_GEN_CTL = 0x00,
109 NV_ADMA_NOTIFIER_CLEAR = 0x30,
110
111 /* BAR5 offset to ADMA ports */
112 NV_ADMA_PORT = 0x480,
113
114 /* size of ADMA port register space */
115 NV_ADMA_PORT_SIZE = 0x100,
116
117 /* ADMA port registers */
118 NV_ADMA_CTL = 0x40,
119 NV_ADMA_CPB_COUNT = 0x42,
120 NV_ADMA_NEXT_CPB_IDX = 0x43,
121 NV_ADMA_STAT = 0x44,
122 NV_ADMA_CPB_BASE_LOW = 0x48,
123 NV_ADMA_CPB_BASE_HIGH = 0x4C,
124 NV_ADMA_APPEND = 0x50,
125 NV_ADMA_NOTIFIER = 0x68,
126 NV_ADMA_NOTIFIER_ERROR = 0x6C,
127
128 /* NV_ADMA_CTL register bits */
129 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
130 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
131 NV_ADMA_CTL_GO = (1 << 7),
132 NV_ADMA_CTL_AIEN = (1 << 8),
133 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
134 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
135
136 /* CPB response flag bits */
137 NV_CPB_RESP_DONE = (1 << 0),
138 NV_CPB_RESP_ATA_ERR = (1 << 3),
139 NV_CPB_RESP_CMD_ERR = (1 << 4),
140 NV_CPB_RESP_CPB_ERR = (1 << 7),
141
142 /* CPB control flag bits */
143 NV_CPB_CTL_CPB_VALID = (1 << 0),
144 NV_CPB_CTL_QUEUE = (1 << 1),
145 NV_CPB_CTL_APRD_VALID = (1 << 2),
146 NV_CPB_CTL_IEN = (1 << 3),
147 NV_CPB_CTL_FPDMA = (1 << 4),
148
149 /* APRD flags */
150 NV_APRD_WRITE = (1 << 1),
151 NV_APRD_END = (1 << 2),
152 NV_APRD_CONT = (1 << 3),
153
154 /* NV_ADMA_STAT flags */
155 NV_ADMA_STAT_TIMEOUT = (1 << 0),
156 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
157 NV_ADMA_STAT_HOTPLUG = (1 << 2),
158 NV_ADMA_STAT_CPBERR = (1 << 4),
159 NV_ADMA_STAT_SERROR = (1 << 5),
160 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
161 NV_ADMA_STAT_IDLE = (1 << 8),
162 NV_ADMA_STAT_LEGACY = (1 << 9),
163 NV_ADMA_STAT_STOPPED = (1 << 10),
164 NV_ADMA_STAT_DONE = (1 << 12),
165 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
166 NV_ADMA_STAT_TIMEOUT,
167
168 /* port flags */
169 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
170 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
171
172 /* MCP55 reg offset */
173 NV_CTL_MCP55 = 0x400,
174 NV_INT_STATUS_MCP55 = 0x440,
175 NV_INT_ENABLE_MCP55 = 0x444,
176 NV_NCQ_REG_MCP55 = 0x448,
177
178 /* MCP55 */
179 NV_INT_ALL_MCP55 = 0xffff,
180 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */
181 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd,
182
183 /* SWNCQ ENABLE BITS*/
184 NV_CTL_PRI_SWNCQ = 0x02,
185 NV_CTL_SEC_SWNCQ = 0x04,
186
187 /* SW NCQ status bits*/
188 NV_SWNCQ_IRQ_DEV = (1 << 0),
189 NV_SWNCQ_IRQ_PM = (1 << 1),
190 NV_SWNCQ_IRQ_ADDED = (1 << 2),
191 NV_SWNCQ_IRQ_REMOVED = (1 << 3),
192
193 NV_SWNCQ_IRQ_BACKOUT = (1 << 4),
194 NV_SWNCQ_IRQ_SDBFIS = (1 << 5),
195 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6),
196 NV_SWNCQ_IRQ_DMASETUP = (1 << 7),
197
198 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED |
199 NV_SWNCQ_IRQ_REMOVED,
200
201 };
202
203 /* ADMA Physical Region Descriptor - one SG segment */
204 struct nv_adma_prd {
205 __le64 addr;
206 __le32 len;
207 u8 flags;
208 u8 packet_len;
209 __le16 reserved;
210 };
211
212 enum nv_adma_regbits {
213 CMDEND = (1 << 15), /* end of command list */
214 WNB = (1 << 14), /* wait-not-BSY */
215 IGN = (1 << 13), /* ignore this entry */
216 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
217 DA2 = (1 << (2 + 8)),
218 DA1 = (1 << (1 + 8)),
219 DA0 = (1 << (0 + 8)),
220 };
221
222 /* ADMA Command Parameter Block
223 The first 5 SG segments are stored inside the Command Parameter Block itself.
224 If there are more than 5 segments the remainder are stored in a separate
225 memory area indicated by next_aprd. */
226 struct nv_adma_cpb {
227 u8 resp_flags; /* 0 */
228 u8 reserved1; /* 1 */
229 u8 ctl_flags; /* 2 */
230 /* len is length of taskfile in 64 bit words */
231 u8 len; /* 3 */
232 u8 tag; /* 4 */
233 u8 next_cpb_idx; /* 5 */
234 __le16 reserved2; /* 6-7 */
235 __le16 tf[12]; /* 8-31 */
236 struct nv_adma_prd aprd[5]; /* 32-111 */
237 __le64 next_aprd; /* 112-119 */
238 __le64 reserved3; /* 120-127 */
239 };
240
241
242 struct nv_adma_port_priv {
243 struct nv_adma_cpb *cpb;
244 dma_addr_t cpb_dma;
245 struct nv_adma_prd *aprd;
246 dma_addr_t aprd_dma;
247 void __iomem *ctl_block;
248 void __iomem *gen_block;
249 void __iomem *notifier_clear_block;
250 u64 adma_dma_mask;
251 u8 flags;
252 int last_issue_ncq;
253 };
254
255 struct nv_host_priv {
256 unsigned long type;
257 };
258
259 struct defer_queue {
260 u32 defer_bits;
261 unsigned int head;
262 unsigned int tail;
263 unsigned int tag[ATA_MAX_QUEUE];
264 };
265
266 enum ncq_saw_flag_list {
267 ncq_saw_d2h = (1U << 0),
268 ncq_saw_dmas = (1U << 1),
269 ncq_saw_sdb = (1U << 2),
270 ncq_saw_backout = (1U << 3),
271 };
272
273 struct nv_swncq_port_priv {
274 struct ata_prd *prd; /* our SG list */
275 dma_addr_t prd_dma; /* and its DMA mapping */
276 void __iomem *sactive_block;
277 void __iomem *irq_block;
278 void __iomem *tag_block;
279 u32 qc_active;
280
281 unsigned int last_issue_tag;
282
283 /* fifo circular queue to store deferral command */
284 struct defer_queue defer_queue;
285
286 /* for NCQ interrupt analysis */
287 u32 dhfis_bits;
288 u32 dmafis_bits;
289 u32 sdbfis_bits;
290
291 unsigned int ncq_flags;
292 };
293
294
295 #define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
296
297 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
298 #ifdef CONFIG_PM
299 static int nv_pci_device_resume(struct pci_dev *pdev);
300 #endif
301 static void nv_ck804_host_stop(struct ata_host *host);
302 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
303 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
304 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
305 static int nv_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
306 static int nv_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
307
308 static void nv_nf2_freeze(struct ata_port *ap);
309 static void nv_nf2_thaw(struct ata_port *ap);
310 static void nv_ck804_freeze(struct ata_port *ap);
311 static void nv_ck804_thaw(struct ata_port *ap);
312 static void nv_error_handler(struct ata_port *ap);
313 static int nv_adma_slave_config(struct scsi_device *sdev);
314 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
315 static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
316 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
317 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
318 static void nv_adma_irq_clear(struct ata_port *ap);
319 static int nv_adma_port_start(struct ata_port *ap);
320 static void nv_adma_port_stop(struct ata_port *ap);
321 #ifdef CONFIG_PM
322 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
323 static int nv_adma_port_resume(struct ata_port *ap);
324 #endif
325 static void nv_adma_freeze(struct ata_port *ap);
326 static void nv_adma_thaw(struct ata_port *ap);
327 static void nv_adma_error_handler(struct ata_port *ap);
328 static void nv_adma_host_stop(struct ata_host *host);
329 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
330 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
331
332 static void nv_mcp55_thaw(struct ata_port *ap);
333 static void nv_mcp55_freeze(struct ata_port *ap);
334 static void nv_swncq_error_handler(struct ata_port *ap);
335 static int nv_swncq_slave_config(struct scsi_device *sdev);
336 static int nv_swncq_port_start(struct ata_port *ap);
337 static void nv_swncq_qc_prep(struct ata_queued_cmd *qc);
338 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
339 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
340 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
341 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
342 #ifdef CONFIG_PM
343 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
344 static int nv_swncq_port_resume(struct ata_port *ap);
345 #endif
346
347 enum nv_host_type
348 {
349 GENERIC,
350 NFORCE2,
351 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
352 CK804,
353 ADMA,
354 SWNCQ,
355 };
356
357 static const struct pci_device_id nv_pci_tbl[] = {
358 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
359 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
360 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
361 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
362 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
363 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
364 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
365 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), SWNCQ },
366 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), SWNCQ },
367 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), SWNCQ },
368 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), SWNCQ },
369 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
370 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
371 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
372
373 { } /* terminate list */
374 };
375
376 static struct pci_driver nv_pci_driver = {
377 .name = DRV_NAME,
378 .id_table = nv_pci_tbl,
379 .probe = nv_init_one,
380 #ifdef CONFIG_PM
381 .suspend = ata_pci_device_suspend,
382 .resume = nv_pci_device_resume,
383 #endif
384 .remove = ata_pci_remove_one,
385 };
386
387 static struct scsi_host_template nv_sht = {
388 .module = THIS_MODULE,
389 .name = DRV_NAME,
390 .ioctl = ata_scsi_ioctl,
391 .queuecommand = ata_scsi_queuecmd,
392 .can_queue = ATA_DEF_QUEUE,
393 .this_id = ATA_SHT_THIS_ID,
394 .sg_tablesize = LIBATA_MAX_PRD,
395 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
396 .emulated = ATA_SHT_EMULATED,
397 .use_clustering = ATA_SHT_USE_CLUSTERING,
398 .proc_name = DRV_NAME,
399 .dma_boundary = ATA_DMA_BOUNDARY,
400 .slave_configure = ata_scsi_slave_config,
401 .slave_destroy = ata_scsi_slave_destroy,
402 .bios_param = ata_std_bios_param,
403 };
404
405 static struct scsi_host_template nv_adma_sht = {
406 .module = THIS_MODULE,
407 .name = DRV_NAME,
408 .ioctl = ata_scsi_ioctl,
409 .queuecommand = ata_scsi_queuecmd,
410 .change_queue_depth = ata_scsi_change_queue_depth,
411 .can_queue = NV_ADMA_MAX_CPBS,
412 .this_id = ATA_SHT_THIS_ID,
413 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
414 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
415 .emulated = ATA_SHT_EMULATED,
416 .use_clustering = ATA_SHT_USE_CLUSTERING,
417 .proc_name = DRV_NAME,
418 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
419 .slave_configure = nv_adma_slave_config,
420 .slave_destroy = ata_scsi_slave_destroy,
421 .bios_param = ata_std_bios_param,
422 };
423
424 static struct scsi_host_template nv_swncq_sht = {
425 .module = THIS_MODULE,
426 .name = DRV_NAME,
427 .ioctl = ata_scsi_ioctl,
428 .queuecommand = ata_scsi_queuecmd,
429 .change_queue_depth = ata_scsi_change_queue_depth,
430 .can_queue = ATA_MAX_QUEUE,
431 .this_id = ATA_SHT_THIS_ID,
432 .sg_tablesize = LIBATA_MAX_PRD,
433 .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
434 .emulated = ATA_SHT_EMULATED,
435 .use_clustering = ATA_SHT_USE_CLUSTERING,
436 .proc_name = DRV_NAME,
437 .dma_boundary = ATA_DMA_BOUNDARY,
438 .slave_configure = nv_swncq_slave_config,
439 .slave_destroy = ata_scsi_slave_destroy,
440 .bios_param = ata_std_bios_param,
441 };
442
443 static const struct ata_port_operations nv_generic_ops = {
444 .tf_load = ata_tf_load,
445 .tf_read = ata_tf_read,
446 .exec_command = ata_exec_command,
447 .check_status = ata_check_status,
448 .dev_select = ata_std_dev_select,
449 .bmdma_setup = ata_bmdma_setup,
450 .bmdma_start = ata_bmdma_start,
451 .bmdma_stop = ata_bmdma_stop,
452 .bmdma_status = ata_bmdma_status,
453 .qc_prep = ata_qc_prep,
454 .qc_issue = ata_qc_issue_prot,
455 .freeze = ata_bmdma_freeze,
456 .thaw = ata_bmdma_thaw,
457 .error_handler = nv_error_handler,
458 .post_internal_cmd = ata_bmdma_post_internal_cmd,
459 .data_xfer = ata_data_xfer,
460 .irq_clear = ata_bmdma_irq_clear,
461 .irq_on = ata_irq_on,
462 .scr_read = nv_scr_read,
463 .scr_write = nv_scr_write,
464 .port_start = ata_port_start,
465 };
466
467 static const struct ata_port_operations nv_nf2_ops = {
468 .tf_load = ata_tf_load,
469 .tf_read = ata_tf_read,
470 .exec_command = ata_exec_command,
471 .check_status = ata_check_status,
472 .dev_select = ata_std_dev_select,
473 .bmdma_setup = ata_bmdma_setup,
474 .bmdma_start = ata_bmdma_start,
475 .bmdma_stop = ata_bmdma_stop,
476 .bmdma_status = ata_bmdma_status,
477 .qc_prep = ata_qc_prep,
478 .qc_issue = ata_qc_issue_prot,
479 .freeze = nv_nf2_freeze,
480 .thaw = nv_nf2_thaw,
481 .error_handler = nv_error_handler,
482 .post_internal_cmd = ata_bmdma_post_internal_cmd,
483 .data_xfer = ata_data_xfer,
484 .irq_clear = ata_bmdma_irq_clear,
485 .irq_on = ata_irq_on,
486 .scr_read = nv_scr_read,
487 .scr_write = nv_scr_write,
488 .port_start = ata_port_start,
489 };
490
491 static const struct ata_port_operations nv_ck804_ops = {
492 .tf_load = ata_tf_load,
493 .tf_read = ata_tf_read,
494 .exec_command = ata_exec_command,
495 .check_status = ata_check_status,
496 .dev_select = ata_std_dev_select,
497 .bmdma_setup = ata_bmdma_setup,
498 .bmdma_start = ata_bmdma_start,
499 .bmdma_stop = ata_bmdma_stop,
500 .bmdma_status = ata_bmdma_status,
501 .qc_prep = ata_qc_prep,
502 .qc_issue = ata_qc_issue_prot,
503 .freeze = nv_ck804_freeze,
504 .thaw = nv_ck804_thaw,
505 .error_handler = nv_error_handler,
506 .post_internal_cmd = ata_bmdma_post_internal_cmd,
507 .data_xfer = ata_data_xfer,
508 .irq_clear = ata_bmdma_irq_clear,
509 .irq_on = ata_irq_on,
510 .scr_read = nv_scr_read,
511 .scr_write = nv_scr_write,
512 .port_start = ata_port_start,
513 .host_stop = nv_ck804_host_stop,
514 };
515
516 static const struct ata_port_operations nv_adma_ops = {
517 .tf_load = ata_tf_load,
518 .tf_read = nv_adma_tf_read,
519 .check_atapi_dma = nv_adma_check_atapi_dma,
520 .exec_command = ata_exec_command,
521 .check_status = ata_check_status,
522 .dev_select = ata_std_dev_select,
523 .bmdma_setup = ata_bmdma_setup,
524 .bmdma_start = ata_bmdma_start,
525 .bmdma_stop = ata_bmdma_stop,
526 .bmdma_status = ata_bmdma_status,
527 .qc_defer = ata_std_qc_defer,
528 .qc_prep = nv_adma_qc_prep,
529 .qc_issue = nv_adma_qc_issue,
530 .freeze = nv_adma_freeze,
531 .thaw = nv_adma_thaw,
532 .error_handler = nv_adma_error_handler,
533 .post_internal_cmd = nv_adma_post_internal_cmd,
534 .data_xfer = ata_data_xfer,
535 .irq_clear = nv_adma_irq_clear,
536 .irq_on = ata_irq_on,
537 .scr_read = nv_scr_read,
538 .scr_write = nv_scr_write,
539 .port_start = nv_adma_port_start,
540 .port_stop = nv_adma_port_stop,
541 #ifdef CONFIG_PM
542 .port_suspend = nv_adma_port_suspend,
543 .port_resume = nv_adma_port_resume,
544 #endif
545 .host_stop = nv_adma_host_stop,
546 };
547
548 static const struct ata_port_operations nv_swncq_ops = {
549 .tf_load = ata_tf_load,
550 .tf_read = ata_tf_read,
551 .exec_command = ata_exec_command,
552 .check_status = ata_check_status,
553 .dev_select = ata_std_dev_select,
554 .bmdma_setup = ata_bmdma_setup,
555 .bmdma_start = ata_bmdma_start,
556 .bmdma_stop = ata_bmdma_stop,
557 .bmdma_status = ata_bmdma_status,
558 .qc_defer = ata_std_qc_defer,
559 .qc_prep = nv_swncq_qc_prep,
560 .qc_issue = nv_swncq_qc_issue,
561 .freeze = nv_mcp55_freeze,
562 .thaw = nv_mcp55_thaw,
563 .error_handler = nv_swncq_error_handler,
564 .post_internal_cmd = ata_bmdma_post_internal_cmd,
565 .data_xfer = ata_data_xfer,
566 .irq_clear = ata_bmdma_irq_clear,
567 .irq_on = ata_irq_on,
568 .scr_read = nv_scr_read,
569 .scr_write = nv_scr_write,
570 #ifdef CONFIG_PM
571 .port_suspend = nv_swncq_port_suspend,
572 .port_resume = nv_swncq_port_resume,
573 #endif
574 .port_start = nv_swncq_port_start,
575 };
576
577 static const struct ata_port_info nv_port_info[] = {
578 /* generic */
579 {
580 .sht = &nv_sht,
581 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
582 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
583 .pio_mask = NV_PIO_MASK,
584 .mwdma_mask = NV_MWDMA_MASK,
585 .udma_mask = NV_UDMA_MASK,
586 .port_ops = &nv_generic_ops,
587 .irq_handler = nv_generic_interrupt,
588 },
589 /* nforce2/3 */
590 {
591 .sht = &nv_sht,
592 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
593 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
594 .pio_mask = NV_PIO_MASK,
595 .mwdma_mask = NV_MWDMA_MASK,
596 .udma_mask = NV_UDMA_MASK,
597 .port_ops = &nv_nf2_ops,
598 .irq_handler = nv_nf2_interrupt,
599 },
600 /* ck804 */
601 {
602 .sht = &nv_sht,
603 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY,
604 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
605 .pio_mask = NV_PIO_MASK,
606 .mwdma_mask = NV_MWDMA_MASK,
607 .udma_mask = NV_UDMA_MASK,
608 .port_ops = &nv_ck804_ops,
609 .irq_handler = nv_ck804_interrupt,
610 },
611 /* ADMA */
612 {
613 .sht = &nv_adma_sht,
614 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
615 ATA_FLAG_MMIO | ATA_FLAG_NCQ,
616 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
617 .pio_mask = NV_PIO_MASK,
618 .mwdma_mask = NV_MWDMA_MASK,
619 .udma_mask = NV_UDMA_MASK,
620 .port_ops = &nv_adma_ops,
621 .irq_handler = nv_adma_interrupt,
622 },
623 /* SWNCQ */
624 {
625 .sht = &nv_swncq_sht,
626 .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
627 ATA_FLAG_NCQ,
628 .link_flags = ATA_LFLAG_HRST_TO_RESUME,
629 .pio_mask = NV_PIO_MASK,
630 .mwdma_mask = NV_MWDMA_MASK,
631 .udma_mask = NV_UDMA_MASK,
632 .port_ops = &nv_swncq_ops,
633 .irq_handler = nv_swncq_interrupt,
634 },
635 };
636
637 MODULE_AUTHOR("NVIDIA");
638 MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
639 MODULE_LICENSE("GPL");
640 MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
641 MODULE_VERSION(DRV_VERSION);
642
643 static int adma_enabled = 1;
644 static int swncq_enabled;
645
646 static void nv_adma_register_mode(struct ata_port *ap)
647 {
648 struct nv_adma_port_priv *pp = ap->private_data;
649 void __iomem *mmio = pp->ctl_block;
650 u16 tmp, status;
651 int count = 0;
652
653 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
654 return;
655
656 status = readw(mmio + NV_ADMA_STAT);
657 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
658 ndelay(50);
659 status = readw(mmio + NV_ADMA_STAT);
660 count++;
661 }
662 if (count == 20)
663 ata_port_printk(ap, KERN_WARNING,
664 "timeout waiting for ADMA IDLE, stat=0x%hx\n",
665 status);
666
667 tmp = readw(mmio + NV_ADMA_CTL);
668 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
669
670 count = 0;
671 status = readw(mmio + NV_ADMA_STAT);
672 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
673 ndelay(50);
674 status = readw(mmio + NV_ADMA_STAT);
675 count++;
676 }
677 if (count == 20)
678 ata_port_printk(ap, KERN_WARNING,
679 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
680 status);
681
682 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
683 }
684
685 static void nv_adma_mode(struct ata_port *ap)
686 {
687 struct nv_adma_port_priv *pp = ap->private_data;
688 void __iomem *mmio = pp->ctl_block;
689 u16 tmp, status;
690 int count = 0;
691
692 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
693 return;
694
695 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
696
697 tmp = readw(mmio + NV_ADMA_CTL);
698 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
699
700 status = readw(mmio + NV_ADMA_STAT);
701 while (((status & NV_ADMA_STAT_LEGACY) ||
702 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
703 ndelay(50);
704 status = readw(mmio + NV_ADMA_STAT);
705 count++;
706 }
707 if (count == 20)
708 ata_port_printk(ap, KERN_WARNING,
709 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
710 status);
711
712 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
713 }
714
715 static int nv_adma_slave_config(struct scsi_device *sdev)
716 {
717 struct ata_port *ap = ata_shost_to_port(sdev->host);
718 struct nv_adma_port_priv *pp = ap->private_data;
719 struct nv_adma_port_priv *port0, *port1;
720 struct scsi_device *sdev0, *sdev1;
721 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
722 unsigned long segment_boundary, flags;
723 unsigned short sg_tablesize;
724 int rc;
725 int adma_enable;
726 u32 current_reg, new_reg, config_mask;
727
728 rc = ata_scsi_slave_config(sdev);
729
730 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
731 /* Not a proper libata device, ignore */
732 return rc;
733
734 spin_lock_irqsave(ap->lock, flags);
735
736 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
737 /*
738 * NVIDIA reports that ADMA mode does not support ATAPI commands.
739 * Therefore ATAPI commands are sent through the legacy interface.
740 * However, the legacy interface only supports 32-bit DMA.
741 * Restrict DMA parameters as required by the legacy interface
742 * when an ATAPI device is connected.
743 */
744 segment_boundary = ATA_DMA_BOUNDARY;
745 /* Subtract 1 since an extra entry may be needed for padding, see
746 libata-scsi.c */
747 sg_tablesize = LIBATA_MAX_PRD - 1;
748
749 /* Since the legacy DMA engine is in use, we need to disable ADMA
750 on the port. */
751 adma_enable = 0;
752 nv_adma_register_mode(ap);
753 } else {
754 segment_boundary = NV_ADMA_DMA_BOUNDARY;
755 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
756 adma_enable = 1;
757 }
758
759 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
760
761 if (ap->port_no == 1)
762 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
763 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
764 else
765 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
766 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
767
768 if (adma_enable) {
769 new_reg = current_reg | config_mask;
770 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
771 } else {
772 new_reg = current_reg & ~config_mask;
773 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
774 }
775
776 if (current_reg != new_reg)
777 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
778
779 port0 = ap->host->ports[0]->private_data;
780 port1 = ap->host->ports[1]->private_data;
781 sdev0 = ap->host->ports[0]->link.device[0].sdev;
782 sdev1 = ap->host->ports[1]->link.device[0].sdev;
783 if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
784 (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
785 /** We have to set the DMA mask to 32-bit if either port is in
786 ATAPI mode, since they are on the same PCI device which is
787 used for DMA mapping. If we set the mask we also need to set
788 the bounce limit on both ports to ensure that the block
789 layer doesn't feed addresses that cause DMA mapping to
790 choke. If either SCSI device is not allocated yet, it's OK
791 since that port will discover its correct setting when it
792 does get allocated.
793 Note: Setting 32-bit mask should not fail. */
794 if (sdev0)
795 blk_queue_bounce_limit(sdev0->request_queue,
796 ATA_DMA_MASK);
797 if (sdev1)
798 blk_queue_bounce_limit(sdev1->request_queue,
799 ATA_DMA_MASK);
800
801 pci_set_dma_mask(pdev, ATA_DMA_MASK);
802 } else {
803 /** This shouldn't fail as it was set to this value before */
804 pci_set_dma_mask(pdev, pp->adma_dma_mask);
805 if (sdev0)
806 blk_queue_bounce_limit(sdev0->request_queue,
807 pp->adma_dma_mask);
808 if (sdev1)
809 blk_queue_bounce_limit(sdev1->request_queue,
810 pp->adma_dma_mask);
811 }
812
813 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
814 blk_queue_max_hw_segments(sdev->request_queue, sg_tablesize);
815 ata_port_printk(ap, KERN_INFO,
816 "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
817 (unsigned long long)*ap->host->dev->dma_mask,
818 segment_boundary, sg_tablesize);
819
820 spin_unlock_irqrestore(ap->lock, flags);
821
822 return rc;
823 }
824
825 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
826 {
827 struct nv_adma_port_priv *pp = qc->ap->private_data;
828 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
829 }
830
831 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
832 {
833 /* Other than when internal or pass-through commands are executed,
834 the only time this function will be called in ADMA mode will be
835 if a command fails. In the failure case we don't care about going
836 into register mode with ADMA commands pending, as the commands will
837 all shortly be aborted anyway. We assume that NCQ commands are not
838 issued via passthrough, which is the only way that switching into
839 ADMA mode could abort outstanding commands. */
840 nv_adma_register_mode(ap);
841
842 ata_tf_read(ap, tf);
843 }
844
845 static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
846 {
847 unsigned int idx = 0;
848
849 if (tf->flags & ATA_TFLAG_ISADDR) {
850 if (tf->flags & ATA_TFLAG_LBA48) {
851 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
852 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
853 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
854 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
855 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
856 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
857 } else
858 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
859
860 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
861 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
862 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
863 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
864 }
865
866 if (tf->flags & ATA_TFLAG_DEVICE)
867 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
868
869 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
870
871 while (idx < 12)
872 cpb[idx++] = cpu_to_le16(IGN);
873
874 return idx;
875 }
876
877 static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
878 {
879 struct nv_adma_port_priv *pp = ap->private_data;
880 u8 flags = pp->cpb[cpb_num].resp_flags;
881
882 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
883
884 if (unlikely((force_err ||
885 flags & (NV_CPB_RESP_ATA_ERR |
886 NV_CPB_RESP_CMD_ERR |
887 NV_CPB_RESP_CPB_ERR)))) {
888 struct ata_eh_info *ehi = &ap->link.eh_info;
889 int freeze = 0;
890
891 ata_ehi_clear_desc(ehi);
892 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
893 if (flags & NV_CPB_RESP_ATA_ERR) {
894 ata_ehi_push_desc(ehi, "ATA error");
895 ehi->err_mask |= AC_ERR_DEV;
896 } else if (flags & NV_CPB_RESP_CMD_ERR) {
897 ata_ehi_push_desc(ehi, "CMD error");
898 ehi->err_mask |= AC_ERR_DEV;
899 } else if (flags & NV_CPB_RESP_CPB_ERR) {
900 ata_ehi_push_desc(ehi, "CPB error");
901 ehi->err_mask |= AC_ERR_SYSTEM;
902 freeze = 1;
903 } else {
904 /* notifier error, but no error in CPB flags? */
905 ata_ehi_push_desc(ehi, "unknown");
906 ehi->err_mask |= AC_ERR_OTHER;
907 freeze = 1;
908 }
909 /* Kill all commands. EH will determine what actually failed. */
910 if (freeze)
911 ata_port_freeze(ap);
912 else
913 ata_port_abort(ap);
914 return 1;
915 }
916
917 if (likely(flags & NV_CPB_RESP_DONE)) {
918 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, cpb_num);
919 VPRINTK("CPB flags done, flags=0x%x\n", flags);
920 if (likely(qc)) {
921 DPRINTK("Completing qc from tag %d\n", cpb_num);
922 ata_qc_complete(qc);
923 } else {
924 struct ata_eh_info *ehi = &ap->link.eh_info;
925 /* Notifier bits set without a command may indicate the drive
926 is misbehaving. Raise host state machine violation on this
927 condition. */
928 ata_port_printk(ap, KERN_ERR,
929 "notifier for tag %d with no cmd?\n",
930 cpb_num);
931 ehi->err_mask |= AC_ERR_HSM;
932 ehi->action |= ATA_EH_SOFTRESET;
933 ata_port_freeze(ap);
934 return 1;
935 }
936 }
937 return 0;
938 }
939
940 static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
941 {
942 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
943
944 /* freeze if hotplugged */
945 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
946 ata_port_freeze(ap);
947 return 1;
948 }
949
950 /* bail out if not our interrupt */
951 if (!(irq_stat & NV_INT_DEV))
952 return 0;
953
954 /* DEV interrupt w/ no active qc? */
955 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
956 ata_check_status(ap);
957 return 1;
958 }
959
960 /* handle interrupt */
961 return ata_host_intr(ap, qc);
962 }
963
964 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
965 {
966 struct ata_host *host = dev_instance;
967 int i, handled = 0;
968 u32 notifier_clears[2];
969
970 spin_lock(&host->lock);
971
972 for (i = 0; i < host->n_ports; i++) {
973 struct ata_port *ap = host->ports[i];
974 notifier_clears[i] = 0;
975
976 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
977 struct nv_adma_port_priv *pp = ap->private_data;
978 void __iomem *mmio = pp->ctl_block;
979 u16 status;
980 u32 gen_ctl;
981 u32 notifier, notifier_error;
982
983 /* if ADMA is disabled, use standard ata interrupt handler */
984 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
985 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
986 >> (NV_INT_PORT_SHIFT * i);
987 handled += nv_host_intr(ap, irq_stat);
988 continue;
989 }
990
991 /* if in ATA register mode, check for standard interrupts */
992 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
993 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
994 >> (NV_INT_PORT_SHIFT * i);
995 if (ata_tag_valid(ap->link.active_tag))
996 /** NV_INT_DEV indication seems unreliable at times
997 at least in ADMA mode. Force it on always when a
998 command is active, to prevent losing interrupts. */
999 irq_stat |= NV_INT_DEV;
1000 handled += nv_host_intr(ap, irq_stat);
1001 }
1002
1003 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1004 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1005 notifier_clears[i] = notifier | notifier_error;
1006
1007 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1008
1009 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
1010 !notifier_error)
1011 /* Nothing to do */
1012 continue;
1013
1014 status = readw(mmio + NV_ADMA_STAT);
1015
1016 /* Clear status. Ensure the controller sees the clearing before we start
1017 looking at any of the CPB statuses, so that any CPB completions after
1018 this point in the handler will raise another interrupt. */
1019 writew(status, mmio + NV_ADMA_STAT);
1020 readw(mmio + NV_ADMA_STAT); /* flush posted write */
1021 rmb();
1022
1023 handled++; /* irq handled if we got here */
1024
1025 /* freeze if hotplugged or controller error */
1026 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
1027 NV_ADMA_STAT_HOTUNPLUG |
1028 NV_ADMA_STAT_TIMEOUT |
1029 NV_ADMA_STAT_SERROR))) {
1030 struct ata_eh_info *ehi = &ap->link.eh_info;
1031
1032 ata_ehi_clear_desc(ehi);
1033 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
1034 if (status & NV_ADMA_STAT_TIMEOUT) {
1035 ehi->err_mask |= AC_ERR_SYSTEM;
1036 ata_ehi_push_desc(ehi, "timeout");
1037 } else if (status & NV_ADMA_STAT_HOTPLUG) {
1038 ata_ehi_hotplugged(ehi);
1039 ata_ehi_push_desc(ehi, "hotplug");
1040 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
1041 ata_ehi_hotplugged(ehi);
1042 ata_ehi_push_desc(ehi, "hot unplug");
1043 } else if (status & NV_ADMA_STAT_SERROR) {
1044 /* let libata analyze SError and figure out the cause */
1045 ata_ehi_push_desc(ehi, "SError");
1046 } else
1047 ata_ehi_push_desc(ehi, "unknown");
1048 ata_port_freeze(ap);
1049 continue;
1050 }
1051
1052 if (status & (NV_ADMA_STAT_DONE |
1053 NV_ADMA_STAT_CPBERR |
1054 NV_ADMA_STAT_CMD_COMPLETE)) {
1055 u32 check_commands = notifier_clears[i];
1056 int pos, error = 0;
1057
1058 if (status & NV_ADMA_STAT_CPBERR) {
1059 /* Check all active commands */
1060 if (ata_tag_valid(ap->link.active_tag))
1061 check_commands = 1 <<
1062 ap->link.active_tag;
1063 else
1064 check_commands = ap->
1065 link.sactive;
1066 }
1067
1068 /** Check CPBs for completed commands */
1069 while ((pos = ffs(check_commands)) && !error) {
1070 pos--;
1071 error = nv_adma_check_cpb(ap, pos,
1072 notifier_error & (1 << pos));
1073 check_commands &= ~(1 << pos);
1074 }
1075 }
1076 }
1077 }
1078
1079 if (notifier_clears[0] || notifier_clears[1]) {
1080 /* Note: Both notifier clear registers must be written
1081 if either is set, even if one is zero, according to NVIDIA. */
1082 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
1083 writel(notifier_clears[0], pp->notifier_clear_block);
1084 pp = host->ports[1]->private_data;
1085 writel(notifier_clears[1], pp->notifier_clear_block);
1086 }
1087
1088 spin_unlock(&host->lock);
1089
1090 return IRQ_RETVAL(handled);
1091 }
1092
1093 static void nv_adma_freeze(struct ata_port *ap)
1094 {
1095 struct nv_adma_port_priv *pp = ap->private_data;
1096 void __iomem *mmio = pp->ctl_block;
1097 u16 tmp;
1098
1099 nv_ck804_freeze(ap);
1100
1101 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1102 return;
1103
1104 /* clear any outstanding CK804 notifications */
1105 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1106 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1107
1108 /* Disable interrupt */
1109 tmp = readw(mmio + NV_ADMA_CTL);
1110 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1111 mmio + NV_ADMA_CTL);
1112 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1113 }
1114
1115 static void nv_adma_thaw(struct ata_port *ap)
1116 {
1117 struct nv_adma_port_priv *pp = ap->private_data;
1118 void __iomem *mmio = pp->ctl_block;
1119 u16 tmp;
1120
1121 nv_ck804_thaw(ap);
1122
1123 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1124 return;
1125
1126 /* Enable interrupt */
1127 tmp = readw(mmio + NV_ADMA_CTL);
1128 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
1129 mmio + NV_ADMA_CTL);
1130 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1131 }
1132
1133 static void nv_adma_irq_clear(struct ata_port *ap)
1134 {
1135 struct nv_adma_port_priv *pp = ap->private_data;
1136 void __iomem *mmio = pp->ctl_block;
1137 u32 notifier_clears[2];
1138
1139 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
1140 ata_bmdma_irq_clear(ap);
1141 return;
1142 }
1143
1144 /* clear any outstanding CK804 notifications */
1145 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
1146 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1147
1148 /* clear ADMA status */
1149 writew(0xffff, mmio + NV_ADMA_STAT);
1150
1151 /* clear notifiers - note both ports need to be written with
1152 something even though we are only clearing on one */
1153 if (ap->port_no == 0) {
1154 notifier_clears[0] = 0xFFFFFFFF;
1155 notifier_clears[1] = 0;
1156 } else {
1157 notifier_clears[0] = 0;
1158 notifier_clears[1] = 0xFFFFFFFF;
1159 }
1160 pp = ap->host->ports[0]->private_data;
1161 writel(notifier_clears[0], pp->notifier_clear_block);
1162 pp = ap->host->ports[1]->private_data;
1163 writel(notifier_clears[1], pp->notifier_clear_block);
1164 }
1165
1166 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
1167 {
1168 struct nv_adma_port_priv *pp = qc->ap->private_data;
1169
1170 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
1171 ata_bmdma_post_internal_cmd(qc);
1172 }
1173
1174 static int nv_adma_port_start(struct ata_port *ap)
1175 {
1176 struct device *dev = ap->host->dev;
1177 struct nv_adma_port_priv *pp;
1178 int rc;
1179 void *mem;
1180 dma_addr_t mem_dma;
1181 void __iomem *mmio;
1182 struct pci_dev *pdev = to_pci_dev(dev);
1183 u16 tmp;
1184
1185 VPRINTK("ENTER\n");
1186
1187 /* Ensure DMA mask is set to 32-bit before allocating legacy PRD and
1188 pad buffers */
1189 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1190 if (rc)
1191 return rc;
1192 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1193 if (rc)
1194 return rc;
1195
1196 rc = ata_port_start(ap);
1197 if (rc)
1198 return rc;
1199
1200 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1201 if (!pp)
1202 return -ENOMEM;
1203
1204 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
1205 ap->port_no * NV_ADMA_PORT_SIZE;
1206 pp->ctl_block = mmio;
1207 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
1208 pp->notifier_clear_block = pp->gen_block +
1209 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1210
1211 /* Now that the legacy PRD and padding buffer are allocated we can
1212 safely raise the DMA mask to allocate the CPB/APRD table.
1213 These are allowed to fail since we store the value that ends up
1214 being used to set as the bounce limit in slave_config later if
1215 needed. */
1216 pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1217 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1218 pp->adma_dma_mask = *dev->dma_mask;
1219
1220 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1221 &mem_dma, GFP_KERNEL);
1222 if (!mem)
1223 return -ENOMEM;
1224 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);
1225
1226 /*
1227 * First item in chunk of DMA memory:
1228 * 128-byte command parameter block (CPB)
1229 * one for each command tag
1230 */
1231 pp->cpb = mem;
1232 pp->cpb_dma = mem_dma;
1233
1234 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1235 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1236
1237 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1238 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1239
1240 /*
1241 * Second item: block of ADMA_SGTBL_LEN s/g entries
1242 */
1243 pp->aprd = mem;
1244 pp->aprd_dma = mem_dma;
1245
1246 ap->private_data = pp;
1247
1248 /* clear any outstanding interrupt conditions */
1249 writew(0xffff, mmio + NV_ADMA_STAT);
1250
1251 /* initialize port variables */
1252 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1253
1254 /* clear CPB fetch count */
1255 writew(0, mmio + NV_ADMA_CPB_COUNT);
1256
1257 /* clear GO for register mode, enable interrupt */
1258 tmp = readw(mmio + NV_ADMA_CTL);
1259 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1260 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1261
1262 tmp = readw(mmio + NV_ADMA_CTL);
1263 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1264 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1265 udelay(1);
1266 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1267 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1268
1269 return 0;
1270 }
1271
1272 static void nv_adma_port_stop(struct ata_port *ap)
1273 {
1274 struct nv_adma_port_priv *pp = ap->private_data;
1275 void __iomem *mmio = pp->ctl_block;
1276
1277 VPRINTK("ENTER\n");
1278 writew(0, mmio + NV_ADMA_CTL);
1279 }
1280
1281 #ifdef CONFIG_PM
1282 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1283 {
1284 struct nv_adma_port_priv *pp = ap->private_data;
1285 void __iomem *mmio = pp->ctl_block;
1286
1287 /* Go to register mode - clears GO */
1288 nv_adma_register_mode(ap);
1289
1290 /* clear CPB fetch count */
1291 writew(0, mmio + NV_ADMA_CPB_COUNT);
1292
1293 /* disable interrupt, shut down port */
1294 writew(0, mmio + NV_ADMA_CTL);
1295
1296 return 0;
1297 }
1298
1299 static int nv_adma_port_resume(struct ata_port *ap)
1300 {
1301 struct nv_adma_port_priv *pp = ap->private_data;
1302 void __iomem *mmio = pp->ctl_block;
1303 u16 tmp;
1304
1305 /* set CPB block location */
1306 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
1307 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
1308
1309 /* clear any outstanding interrupt conditions */
1310 writew(0xffff, mmio + NV_ADMA_STAT);
1311
1312 /* initialize port variables */
1313 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1314
1315 /* clear CPB fetch count */
1316 writew(0, mmio + NV_ADMA_CPB_COUNT);
1317
1318 /* clear GO for register mode, enable interrupt */
1319 tmp = readw(mmio + NV_ADMA_CTL);
1320 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1321 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
1322
1323 tmp = readw(mmio + NV_ADMA_CTL);
1324 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1325 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1326 udelay(1);
1327 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1328 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1329
1330 return 0;
1331 }
1332 #endif
1333
1334 static void nv_adma_setup_port(struct ata_port *ap)
1335 {
1336 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1337 struct ata_ioports *ioport = &ap->ioaddr;
1338
1339 VPRINTK("ENTER\n");
1340
1341 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
1342
1343 ioport->cmd_addr = mmio;
1344 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
1345 ioport->error_addr =
1346 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1347 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1348 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1349 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1350 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1351 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
1352 ioport->status_addr =
1353 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
1354 ioport->altstatus_addr =
1355 ioport->ctl_addr = mmio + 0x20;
1356 }
1357
1358 static int nv_adma_host_init(struct ata_host *host)
1359 {
1360 struct pci_dev *pdev = to_pci_dev(host->dev);
1361 unsigned int i;
1362 u32 tmp32;
1363
1364 VPRINTK("ENTER\n");
1365
1366 /* enable ADMA on the ports */
1367 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1368 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1369 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1370 NV_MCP_SATA_CFG_20_PORT1_EN |
1371 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1372
1373 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1374
1375 for (i = 0; i < host->n_ports; i++)
1376 nv_adma_setup_port(host->ports[i]);
1377
1378 return 0;
1379 }
1380
1381 static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1382 struct scatterlist *sg,
1383 int idx,
1384 struct nv_adma_prd *aprd)
1385 {
1386 u8 flags = 0;
1387 if (qc->tf.flags & ATA_TFLAG_WRITE)
1388 flags |= NV_APRD_WRITE;
1389 if (idx == qc->n_elem - 1)
1390 flags |= NV_APRD_END;
1391 else if (idx != 4)
1392 flags |= NV_APRD_CONT;
1393
1394 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1395 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
1396 aprd->flags = flags;
1397 aprd->packet_len = 0;
1398 }
1399
1400 static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1401 {
1402 struct nv_adma_port_priv *pp = qc->ap->private_data;
1403 struct nv_adma_prd *aprd;
1404 struct scatterlist *sg;
1405 unsigned int si;
1406
1407 VPRINTK("ENTER\n");
1408
1409 for_each_sg(qc->sg, sg, qc->n_elem, si) {
1410 aprd = (si < 5) ? &cpb->aprd[si] :
1411 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (si-5)];
1412 nv_adma_fill_aprd(qc, sg, si, aprd);
1413 }
1414 if (si > 5)
1415 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag)));
1416 else
1417 cpb->next_aprd = cpu_to_le64(0);
1418 }
1419
1420 static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1421 {
1422 struct nv_adma_port_priv *pp = qc->ap->private_data;
1423
1424 /* ADMA engine can only be used for non-ATAPI DMA commands,
1425 or interrupt-driven no-data commands. */
1426 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
1427 (qc->tf.flags & ATA_TFLAG_POLLING))
1428 return 1;
1429
1430 if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
1431 (qc->tf.protocol == ATA_PROT_NODATA))
1432 return 0;
1433
1434 return 1;
1435 }
1436
1437 static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
1438 {
1439 struct nv_adma_port_priv *pp = qc->ap->private_data;
1440 struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
1441 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
1442 NV_CPB_CTL_IEN;
1443
1444 if (nv_adma_use_reg_mode(qc)) {
1445 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1446 (qc->flags & ATA_QCFLAG_DMAMAP));
1447 nv_adma_register_mode(qc->ap);
1448 ata_qc_prep(qc);
1449 return;
1450 }
1451
1452 cpb->resp_flags = NV_CPB_RESP_DONE;
1453 wmb();
1454 cpb->ctl_flags = 0;
1455 wmb();
1456
1457 cpb->len = 3;
1458 cpb->tag = qc->tag;
1459 cpb->next_cpb_idx = 0;
1460
1461 /* turn on NCQ flags for NCQ commands */
1462 if (qc->tf.protocol == ATA_PROT_NCQ)
1463 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1464
1465 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1466
1467 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1468
1469 if (qc->flags & ATA_QCFLAG_DMAMAP) {
1470 nv_adma_fill_sg(qc, cpb);
1471 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1472 } else
1473 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
1474
1475 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1476 until we are finished filling in all of the contents */
1477 wmb();
1478 cpb->ctl_flags = ctl_flags;
1479 wmb();
1480 cpb->resp_flags = 0;
1481 }
1482
1483 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1484 {
1485 struct nv_adma_port_priv *pp = qc->ap->private_data;
1486 void __iomem *mmio = pp->ctl_block;
1487 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
1488
1489 VPRINTK("ENTER\n");
1490
1491 /* We can't handle result taskfile with NCQ commands, since
1492 retrieving the taskfile switches us out of ADMA mode and would abort
1493 existing commands. */
1494 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1495 (qc->flags & ATA_QCFLAG_RESULT_TF))) {
1496 ata_dev_printk(qc->dev, KERN_ERR,
1497 "NCQ w/ RESULT_TF not allowed\n");
1498 return AC_ERR_SYSTEM;
1499 }
1500
1501 if (nv_adma_use_reg_mode(qc)) {
1502 /* use ATA register mode */
1503 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
1504 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1505 (qc->flags & ATA_QCFLAG_DMAMAP));
1506 nv_adma_register_mode(qc->ap);
1507 return ata_qc_issue_prot(qc);
1508 } else
1509 nv_adma_mode(qc->ap);
1510
1511 /* write append register, command tag in lower 8 bits
1512 and (number of cpbs to append -1) in top 8 bits */
1513 wmb();
1514
1515 if (curr_ncq != pp->last_issue_ncq) {
1516 /* Seems to need some delay before switching between NCQ and
1517 non-NCQ commands, else we get command timeouts and such. */
1518 udelay(20);
1519 pp->last_issue_ncq = curr_ncq;
1520 }
1521
1522 writew(qc->tag, mmio + NV_ADMA_APPEND);
1523
1524 DPRINTK("Issued tag %u\n", qc->tag);
1525
1526 return 0;
1527 }
1528
1529 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1530 {
1531 struct ata_host *host = dev_instance;
1532 unsigned int i;
1533 unsigned int handled = 0;
1534 unsigned long flags;
1535
1536 spin_lock_irqsave(&host->lock, flags);
1537
1538 for (i = 0; i < host->n_ports; i++) {
1539 struct ata_port *ap;
1540
1541 ap = host->ports[i];
1542 if (ap &&
1543 !(ap->flags & ATA_FLAG_DISABLED)) {
1544 struct ata_queued_cmd *qc;
1545
1546 qc = ata_qc_from_tag(ap, ap->link.active_tag);
1547 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING)))
1548 handled += ata_host_intr(ap, qc);
1549 else
1550 // No request pending? Clear interrupt status
1551 // anyway, in case there's one pending.
1552 ap->ops->check_status(ap);
1553 }
1554
1555 }
1556
1557 spin_unlock_irqrestore(&host->lock, flags);
1558
1559 return IRQ_RETVAL(handled);
1560 }
1561
1562 static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
1563 {
1564 int i, handled = 0;
1565
1566 for (i = 0; i < host->n_ports; i++) {
1567 struct ata_port *ap = host->ports[i];
1568
1569 if (ap && !(ap->flags & ATA_FLAG_DISABLED))
1570 handled += nv_host_intr(ap, irq_stat);
1571
1572 irq_stat >>= NV_INT_PORT_SHIFT;
1573 }
1574
1575 return IRQ_RETVAL(handled);
1576 }
1577
1578 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
1579 {
1580 struct ata_host *host = dev_instance;
1581 u8 irq_stat;
1582 irqreturn_t ret;
1583
1584 spin_lock(&host->lock);
1585 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
1586 ret = nv_do_interrupt(host, irq_stat);
1587 spin_unlock(&host->lock);
1588
1589 return ret;
1590 }
1591
1592 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
1593 {
1594 struct ata_host *host = dev_instance;
1595 u8 irq_stat;
1596 irqreturn_t ret;
1597
1598 spin_lock(&host->lock);
1599 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1600 ret = nv_do_interrupt(host, irq_stat);
1601 spin_unlock(&host->lock);
1602
1603 return ret;
1604 }
1605
1606 static int nv_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
1607 {
1608 if (sc_reg > SCR_CONTROL)
1609 return -EINVAL;
1610
1611 *val = ioread32(ap->ioaddr.scr_addr + (sc_reg * 4));
1612 return 0;
1613 }
1614
1615 static int nv_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
1616 {
1617 if (sc_reg > SCR_CONTROL)
1618 return -EINVAL;
1619
1620 iowrite32(val, ap->ioaddr.scr_addr + (sc_reg * 4));
1621 return 0;
1622 }
1623
1624 static void nv_nf2_freeze(struct ata_port *ap)
1625 {
1626 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1627 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1628 u8 mask;
1629
1630 mask = ioread8(scr_addr + NV_INT_ENABLE);
1631 mask &= ~(NV_INT_ALL << shift);
1632 iowrite8(mask, scr_addr + NV_INT_ENABLE);
1633 }
1634
1635 static void nv_nf2_thaw(struct ata_port *ap)
1636 {
1637 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
1638 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1639 u8 mask;
1640
1641 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
1642
1643 mask = ioread8(scr_addr + NV_INT_ENABLE);
1644 mask |= (NV_INT_MASK << shift);
1645 iowrite8(mask, scr_addr + NV_INT_ENABLE);
1646 }
1647
1648 static void nv_ck804_freeze(struct ata_port *ap)
1649 {
1650 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1651 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1652 u8 mask;
1653
1654 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1655 mask &= ~(NV_INT_ALL << shift);
1656 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1657 }
1658
1659 static void nv_ck804_thaw(struct ata_port *ap)
1660 {
1661 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1662 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1663 u8 mask;
1664
1665 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1666
1667 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1668 mask |= (NV_INT_MASK << shift);
1669 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1670 }
1671
1672 static void nv_mcp55_freeze(struct ata_port *ap)
1673 {
1674 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1675 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1676 u32 mask;
1677
1678 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1679
1680 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1681 mask &= ~(NV_INT_ALL_MCP55 << shift);
1682 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1683 ata_bmdma_freeze(ap);
1684 }
1685
1686 static void nv_mcp55_thaw(struct ata_port *ap)
1687 {
1688 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1689 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1690 u32 mask;
1691
1692 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1693
1694 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1695 mask |= (NV_INT_MASK_MCP55 << shift);
1696 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
1697 ata_bmdma_thaw(ap);
1698 }
1699
1700 static int nv_hardreset(struct ata_link *link, unsigned int *class,
1701 unsigned long deadline)
1702 {
1703 unsigned int dummy;
1704
1705 /* SATA hardreset fails to retrieve proper device signature on
1706 * some controllers. Don't classify on hardreset. For more
1707 * info, see http://bugzilla.kernel.org/show_bug.cgi?id=3352
1708 */
1709 return sata_std_hardreset(link, &dummy, deadline);
1710 }
1711
1712 static void nv_error_handler(struct ata_port *ap)
1713 {
1714 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1715 nv_hardreset, ata_std_postreset);
1716 }
1717
1718 static void nv_adma_error_handler(struct ata_port *ap)
1719 {
1720 struct nv_adma_port_priv *pp = ap->private_data;
1721 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
1722 void __iomem *mmio = pp->ctl_block;
1723 int i;
1724 u16 tmp;
1725
1726 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
1727 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1728 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1729 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1730 u32 status = readw(mmio + NV_ADMA_STAT);
1731 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1732 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
1733
1734 ata_port_printk(ap, KERN_ERR,
1735 "EH in ADMA mode, notifier 0x%X "
1736 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1737 "next cpb count 0x%X next cpb idx 0x%x\n",
1738 notifier, notifier_error, gen_ctl, status,
1739 cpb_count, next_cpb_idx);
1740
1741 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
1742 struct nv_adma_cpb *cpb = &pp->cpb[i];
1743 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
1744 ap->link.sactive & (1 << i))
1745 ata_port_printk(ap, KERN_ERR,
1746 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1747 i, cpb->ctl_flags, cpb->resp_flags);
1748 }
1749 }
1750
1751 /* Push us back into port register mode for error handling. */
1752 nv_adma_register_mode(ap);
1753
1754 /* Mark all of the CPBs as invalid to prevent them from
1755 being executed */
1756 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
1757 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1758
1759 /* clear CPB fetch count */
1760 writew(0, mmio + NV_ADMA_CPB_COUNT);
1761
1762 /* Reset channel */
1763 tmp = readw(mmio + NV_ADMA_CTL);
1764 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1765 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1766 udelay(1);
1767 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
1768 readw(mmio + NV_ADMA_CTL); /* flush posted write */
1769 }
1770
1771 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1772 nv_hardreset, ata_std_postreset);
1773 }
1774
1775 static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1776 {
1777 struct nv_swncq_port_priv *pp = ap->private_data;
1778 struct defer_queue *dq = &pp->defer_queue;
1779
1780 /* queue is full */
1781 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
1782 dq->defer_bits |= (1 << qc->tag);
1783 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->tag;
1784 }
1785
1786 static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1787 {
1788 struct nv_swncq_port_priv *pp = ap->private_data;
1789 struct defer_queue *dq = &pp->defer_queue;
1790 unsigned int tag;
1791
1792 if (dq->head == dq->tail) /* null queue */
1793 return NULL;
1794
1795 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1796 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1797 WARN_ON(!(dq->defer_bits & (1 << tag)));
1798 dq->defer_bits &= ~(1 << tag);
1799
1800 return ata_qc_from_tag(ap, tag);
1801 }
1802
1803 static void nv_swncq_fis_reinit(struct ata_port *ap)
1804 {
1805 struct nv_swncq_port_priv *pp = ap->private_data;
1806
1807 pp->dhfis_bits = 0;
1808 pp->dmafis_bits = 0;
1809 pp->sdbfis_bits = 0;
1810 pp->ncq_flags = 0;
1811 }
1812
1813 static void nv_swncq_pp_reinit(struct ata_port *ap)
1814 {
1815 struct nv_swncq_port_priv *pp = ap->private_data;
1816 struct defer_queue *dq = &pp->defer_queue;
1817
1818 dq->head = 0;
1819 dq->tail = 0;
1820 dq->defer_bits = 0;
1821 pp->qc_active = 0;
1822 pp->last_issue_tag = ATA_TAG_POISON;
1823 nv_swncq_fis_reinit(ap);
1824 }
1825
1826 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1827 {
1828 struct nv_swncq_port_priv *pp = ap->private_data;
1829
1830 writew(fis, pp->irq_block);
1831 }
1832
1833 static void __ata_bmdma_stop(struct ata_port *ap)
1834 {
1835 struct ata_queued_cmd qc;
1836
1837 qc.ap = ap;
1838 ata_bmdma_stop(&qc);
1839 }
1840
1841 static void nv_swncq_ncq_stop(struct ata_port *ap)
1842 {
1843 struct nv_swncq_port_priv *pp = ap->private_data;
1844 unsigned int i;
1845 u32 sactive;
1846 u32 done_mask;
1847
1848 ata_port_printk(ap, KERN_ERR,
1849 "EH in SWNCQ mode,QC:qc_active 0x%X sactive 0x%X\n",
1850 ap->qc_active, ap->link.sactive);
1851 ata_port_printk(ap, KERN_ERR,
1852 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n "
1853 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1854 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1855 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1856
1857 ata_port_printk(ap, KERN_ERR, "ATA_REG 0x%X ERR_REG 0x%X\n",
1858 ap->ops->check_status(ap),
1859 ioread8(ap->ioaddr.error_addr));
1860
1861 sactive = readl(pp->sactive_block);
1862 done_mask = pp->qc_active ^ sactive;
1863
1864 ata_port_printk(ap, KERN_ERR, "tag : dhfis dmafis sdbfis sacitve\n");
1865 for (i = 0; i < ATA_MAX_QUEUE; i++) {
1866 u8 err = 0;
1867 if (pp->qc_active & (1 << i))
1868 err = 0;
1869 else if (done_mask & (1 << i))
1870 err = 1;
1871 else
1872 continue;
1873
1874 ata_port_printk(ap, KERN_ERR,
1875 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1876 (pp->dhfis_bits >> i) & 0x1,
1877 (pp->dmafis_bits >> i) & 0x1,
1878 (pp->sdbfis_bits >> i) & 0x1,
1879 (sactive >> i) & 0x1,
1880 (err ? "error! tag doesn't exit" : " "));
1881 }
1882
1883 nv_swncq_pp_reinit(ap);
1884 ap->ops->irq_clear(ap);
1885 __ata_bmdma_stop(ap);
1886 nv_swncq_irq_clear(ap, 0xffff);
1887 }
1888
1889 static void nv_swncq_error_handler(struct ata_port *ap)
1890 {
1891 struct ata_eh_context *ehc = &ap->link.eh_context;
1892
1893 if (ap->link.sactive) {
1894 nv_swncq_ncq_stop(ap);
1895 ehc->i.action |= ATA_EH_HARDRESET;
1896 }
1897
1898 ata_bmdma_drive_eh(ap, ata_std_prereset, ata_std_softreset,
1899 nv_hardreset, ata_std_postreset);
1900 }
1901
1902 #ifdef CONFIG_PM
1903 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1904 {
1905 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1906 u32 tmp;
1907
1908 /* clear irq */
1909 writel(~0, mmio + NV_INT_STATUS_MCP55);
1910
1911 /* disable irq */
1912 writel(0, mmio + NV_INT_ENABLE_MCP55);
1913
1914 /* disable swncq */
1915 tmp = readl(mmio + NV_CTL_MCP55);
1916 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1917 writel(tmp, mmio + NV_CTL_MCP55);
1918
1919 return 0;
1920 }
1921
1922 static int nv_swncq_port_resume(struct ata_port *ap)
1923 {
1924 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1925 u32 tmp;
1926
1927 /* clear irq */
1928 writel(~0, mmio + NV_INT_STATUS_MCP55);
1929
1930 /* enable irq */
1931 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1932
1933 /* enable swncq */
1934 tmp = readl(mmio + NV_CTL_MCP55);
1935 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1936
1937 return 0;
1938 }
1939 #endif
1940
1941 static void nv_swncq_host_init(struct ata_host *host)
1942 {
1943 u32 tmp;
1944 void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1945 struct pci_dev *pdev = to_pci_dev(host->dev);
1946 u8 regval;
1947
1948 /* disable ECO 398 */
1949 pci_read_config_byte(pdev, 0x7f, &regval);
1950 regval &= ~(1 << 7);
1951 pci_write_config_byte(pdev, 0x7f, regval);
1952
1953 /* enable swncq */
1954 tmp = readl(mmio + NV_CTL_MCP55);
1955 VPRINTK("HOST_CTL:0x%X\n", tmp);
1956 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1957
1958 /* enable irq intr */
1959 tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1960 VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1961 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1962
1963 /* clear port irq */
1964 writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1965 }
1966
1967 static int nv_swncq_slave_config(struct scsi_device *sdev)
1968 {
1969 struct ata_port *ap = ata_shost_to_port(sdev->host);
1970 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1971 struct ata_device *dev;
1972 int rc;
1973 u8 rev;
1974 u8 check_maxtor = 0;
1975 unsigned char model_num[ATA_ID_PROD_LEN + 1];
1976
1977 rc = ata_scsi_slave_config(sdev);
1978 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1979 /* Not a proper libata device, ignore */
1980 return rc;
1981
1982 dev = &ap->link.device[sdev->id];
1983 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1984 return rc;
1985
1986 /* if MCP51 and Maxtor, then disable ncq */
1987 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1988 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1989 check_maxtor = 1;
1990
1991 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1992 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1993 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1994 pci_read_config_byte(pdev, 0x8, &rev);
1995 if (rev <= 0xa2)
1996 check_maxtor = 1;
1997 }
1998
1999 if (!check_maxtor)
2000 return rc;
2001
2002 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
2003
2004 if (strncmp(model_num, "Maxtor", 6) == 0) {
2005 ata_scsi_change_queue_depth(sdev, 1);
2006 ata_dev_printk(dev, KERN_NOTICE,
2007 "Disabling SWNCQ mode (depth %x)\n", sdev->queue_depth);
2008 }
2009
2010 return rc;
2011 }
2012
2013 static int nv_swncq_port_start(struct ata_port *ap)
2014 {
2015 struct device *dev = ap->host->dev;
2016 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
2017 struct nv_swncq_port_priv *pp;
2018 int rc;
2019
2020 rc = ata_port_start(ap);
2021 if (rc)
2022 return rc;
2023
2024 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
2025 if (!pp)
2026 return -ENOMEM;
2027
2028 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
2029 &pp->prd_dma, GFP_KERNEL);
2030 if (!pp->prd)
2031 return -ENOMEM;
2032 memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE);
2033
2034 ap->private_data = pp;
2035 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
2036 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
2037 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
2038
2039 return 0;
2040 }
2041
2042 static void nv_swncq_qc_prep(struct ata_queued_cmd *qc)
2043 {
2044 if (qc->tf.protocol != ATA_PROT_NCQ) {
2045 ata_qc_prep(qc);
2046 return;
2047 }
2048
2049 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
2050 return;
2051
2052 nv_swncq_fill_sg(qc);
2053 }
2054
2055 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
2056 {
2057 struct ata_port *ap = qc->ap;
2058 struct scatterlist *sg;
2059 struct nv_swncq_port_priv *pp = ap->private_data;
2060 struct ata_prd *prd;
2061 unsigned int si, idx;
2062
2063 prd = pp->prd + ATA_MAX_PRD * qc->tag;
2064
2065 idx = 0;
2066 for_each_sg(qc->sg, sg, qc->n_elem, si) {
2067 u32 addr, offset;
2068 u32 sg_len, len;
2069
2070 addr = (u32)sg_dma_address(sg);
2071 sg_len = sg_dma_len(sg);
2072
2073 while (sg_len) {
2074 offset = addr & 0xffff;
2075 len = sg_len;
2076 if ((offset + sg_len) > 0x10000)
2077 len = 0x10000 - offset;
2078
2079 prd[idx].addr = cpu_to_le32(addr);
2080 prd[idx].flags_len = cpu_to_le32(len & 0xffff);
2081
2082 idx++;
2083 sg_len -= len;
2084 addr += len;
2085 }
2086 }
2087
2088 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
2089 }
2090
2091 static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
2092 struct ata_queued_cmd *qc)
2093 {
2094 struct nv_swncq_port_priv *pp = ap->private_data;
2095
2096 if (qc == NULL)
2097 return 0;
2098
2099 DPRINTK("Enter\n");
2100
2101 writel((1 << qc->tag), pp->sactive_block);
2102 pp->last_issue_tag = qc->tag;
2103 pp->dhfis_bits &= ~(1 << qc->tag);
2104 pp->dmafis_bits &= ~(1 << qc->tag);
2105 pp->qc_active |= (0x1 << qc->tag);
2106
2107 ap->ops->tf_load(ap, &qc->tf); /* load tf registers */
2108 ap->ops->exec_command(ap, &qc->tf);
2109
2110 DPRINTK("Issued tag %u\n", qc->tag);
2111
2112 return 0;
2113 }
2114
2115 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2116 {
2117 struct ata_port *ap = qc->ap;
2118 struct nv_swncq_port_priv *pp = ap->private_data;
2119
2120 if (qc->tf.protocol != ATA_PROT_NCQ)
2121 return ata_qc_issue_prot(qc);
2122
2123 DPRINTK("Enter\n");
2124
2125 if (!pp->qc_active)
2126 nv_swncq_issue_atacmd(ap, qc);
2127 else
2128 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */
2129
2130 return 0;
2131 }
2132
2133 static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2134 {
2135 u32 serror;
2136 struct ata_eh_info *ehi = &ap->link.eh_info;
2137
2138 ata_ehi_clear_desc(ehi);
2139
2140 /* AHCI needs SError cleared; otherwise, it might lock up */
2141 sata_scr_read(&ap->link, SCR_ERROR, &serror);
2142 sata_scr_write(&ap->link, SCR_ERROR, serror);
2143
2144 /* analyze @irq_stat */
2145 if (fis & NV_SWNCQ_IRQ_ADDED)
2146 ata_ehi_push_desc(ehi, "hot plug");
2147 else if (fis & NV_SWNCQ_IRQ_REMOVED)
2148 ata_ehi_push_desc(ehi, "hot unplug");
2149
2150 ata_ehi_hotplugged(ehi);
2151
2152 /* okay, let's hand over to EH */
2153 ehi->serror |= serror;
2154
2155 ata_port_freeze(ap);
2156 }
2157
2158 static int nv_swncq_sdbfis(struct ata_port *ap)
2159 {
2160 struct ata_queued_cmd *qc;
2161 struct nv_swncq_port_priv *pp = ap->private_data;
2162 struct ata_eh_info *ehi = &ap->link.eh_info;
2163 u32 sactive;
2164 int nr_done = 0;
2165 u32 done_mask;
2166 int i;
2167 u8 host_stat;
2168 u8 lack_dhfis = 0;
2169
2170 host_stat = ap->ops->bmdma_status(ap);
2171 if (unlikely(host_stat & ATA_DMA_ERR)) {
2172 /* error when transfering data to/from memory */
2173 ata_ehi_clear_desc(ehi);
2174 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2175 ehi->err_mask |= AC_ERR_HOST_BUS;
2176 ehi->action |= ATA_EH_SOFTRESET;
2177 return -EINVAL;
2178 }
2179
2180 ap->ops->irq_clear(ap);
2181 __ata_bmdma_stop(ap);
2182
2183 sactive = readl(pp->sactive_block);
2184 done_mask = pp->qc_active ^ sactive;
2185
2186 if (unlikely(done_mask & sactive)) {
2187 ata_ehi_clear_desc(ehi);
2188 ata_ehi_push_desc(ehi, "illegal SWNCQ:qc_active transition"
2189 "(%08x->%08x)", pp->qc_active, sactive);
2190 ehi->err_mask |= AC_ERR_HSM;
2191 ehi->action |= ATA_EH_HARDRESET;
2192 return -EINVAL;
2193 }
2194 for (i = 0; i < ATA_MAX_QUEUE; i++) {
2195 if (!(done_mask & (1 << i)))
2196 continue;
2197
2198 qc = ata_qc_from_tag(ap, i);
2199 if (qc) {
2200 ata_qc_complete(qc);
2201 pp->qc_active &= ~(1 << i);
2202 pp->dhfis_bits &= ~(1 << i);
2203 pp->dmafis_bits &= ~(1 << i);
2204 pp->sdbfis_bits |= (1 << i);
2205 nr_done++;
2206 }
2207 }
2208
2209 if (!ap->qc_active) {
2210 DPRINTK("over\n");
2211 nv_swncq_pp_reinit(ap);
2212 return nr_done;
2213 }
2214
2215 if (pp->qc_active & pp->dhfis_bits)
2216 return nr_done;
2217
2218 if ((pp->ncq_flags & ncq_saw_backout) ||
2219 (pp->qc_active ^ pp->dhfis_bits))
2220 /* if the controller cann't get a device to host register FIS,
2221 * The driver needs to reissue the new command.
2222 */
2223 lack_dhfis = 1;
2224
2225 DPRINTK("id 0x%x QC: qc_active 0x%x,"
2226 "SWNCQ:qc_active 0x%X defer_bits %X "
2227 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2228 ap->print_id, ap->qc_active, pp->qc_active,
2229 pp->defer_queue.defer_bits, pp->dhfis_bits,
2230 pp->dmafis_bits, pp->last_issue_tag);
2231
2232 nv_swncq_fis_reinit(ap);
2233
2234 if (lack_dhfis) {
2235 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2236 nv_swncq_issue_atacmd(ap, qc);
2237 return nr_done;
2238 }
2239
2240 if (pp->defer_queue.defer_bits) {
2241 /* send deferral queue command */
2242 qc = nv_swncq_qc_from_dq(ap);
2243 WARN_ON(qc == NULL);
2244 nv_swncq_issue_atacmd(ap, qc);
2245 }
2246
2247 return nr_done;
2248 }
2249
2250 static inline u32 nv_swncq_tag(struct ata_port *ap)
2251 {
2252 struct nv_swncq_port_priv *pp = ap->private_data;
2253 u32 tag;
2254
2255 tag = readb(pp->tag_block) >> 2;
2256 return (tag & 0x1f);
2257 }
2258
2259 static int nv_swncq_dmafis(struct ata_port *ap)
2260 {
2261 struct ata_queued_cmd *qc;
2262 unsigned int rw;
2263 u8 dmactl;
2264 u32 tag;
2265 struct nv_swncq_port_priv *pp = ap->private_data;
2266
2267 __ata_bmdma_stop(ap);
2268 tag = nv_swncq_tag(ap);
2269
2270 DPRINTK("dma setup tag 0x%x\n", tag);
2271 qc = ata_qc_from_tag(ap, tag);
2272
2273 if (unlikely(!qc))
2274 return 0;
2275
2276 rw = qc->tf.flags & ATA_TFLAG_WRITE;
2277
2278 /* load PRD table addr. */
2279 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag,
2280 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2281
2282 /* specify data direction, triple-check start bit is clear */
2283 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2284 dmactl &= ~ATA_DMA_WR;
2285 if (!rw)
2286 dmactl |= ATA_DMA_WR;
2287
2288 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2289
2290 return 1;
2291 }
2292
2293 static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2294 {
2295 struct nv_swncq_port_priv *pp = ap->private_data;
2296 struct ata_queued_cmd *qc;
2297 struct ata_eh_info *ehi = &ap->link.eh_info;
2298 u32 serror;
2299 u8 ata_stat;
2300 int rc = 0;
2301
2302 ata_stat = ap->ops->check_status(ap);
2303 nv_swncq_irq_clear(ap, fis);
2304 if (!fis)
2305 return;
2306
2307 if (ap->pflags & ATA_PFLAG_FROZEN)
2308 return;
2309
2310 if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2311 nv_swncq_hotplug(ap, fis);
2312 return;
2313 }
2314
2315 if (!pp->qc_active)
2316 return;
2317
2318 if (ap->ops->scr_read(ap, SCR_ERROR, &serror))
2319 return;
2320 ap->ops->scr_write(ap, SCR_ERROR, serror);
2321
2322 if (ata_stat & ATA_ERR) {
2323 ata_ehi_clear_desc(ehi);
2324 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2325 ehi->err_mask |= AC_ERR_DEV;
2326 ehi->serror |= serror;
2327 ehi->action |= ATA_EH_SOFTRESET;
2328 ata_port_freeze(ap);
2329 return;
2330 }
2331
2332 if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2333 /* If the IRQ is backout, driver must issue
2334 * the new command again some time later.
2335 */
2336 pp->ncq_flags |= ncq_saw_backout;
2337 }
2338
2339 if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2340 pp->ncq_flags |= ncq_saw_sdb;
2341 DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2342 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2343 ap->print_id, pp->qc_active, pp->dhfis_bits,
2344 pp->dmafis_bits, readl(pp->sactive_block));
2345 rc = nv_swncq_sdbfis(ap);
2346 if (rc < 0)
2347 goto irq_error;
2348 }
2349
2350 if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2351 /* The interrupt indicates the new command
2352 * was transmitted correctly to the drive.
2353 */
2354 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2355 pp->ncq_flags |= ncq_saw_d2h;
2356 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2357 ata_ehi_push_desc(ehi, "illegal fis transaction");
2358 ehi->err_mask |= AC_ERR_HSM;
2359 ehi->action |= ATA_EH_HARDRESET;
2360 goto irq_error;
2361 }
2362
2363 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2364 !(pp->ncq_flags & ncq_saw_dmas)) {
2365 ata_stat = ap->ops->check_status(ap);
2366 if (ata_stat & ATA_BUSY)
2367 goto irq_exit;
2368
2369 if (pp->defer_queue.defer_bits) {
2370 DPRINTK("send next command\n");
2371 qc = nv_swncq_qc_from_dq(ap);
2372 nv_swncq_issue_atacmd(ap, qc);
2373 }
2374 }
2375 }
2376
2377 if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2378 /* program the dma controller with appropriate PRD buffers
2379 * and start the DMA transfer for requested command.
2380 */
2381 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2382 pp->ncq_flags |= ncq_saw_dmas;
2383 rc = nv_swncq_dmafis(ap);
2384 }
2385
2386 irq_exit:
2387 return;
2388 irq_error:
2389 ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2390 ata_port_freeze(ap);
2391 return;
2392 }
2393
2394 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2395 {
2396 struct ata_host *host = dev_instance;
2397 unsigned int i;
2398 unsigned int handled = 0;
2399 unsigned long flags;
2400 u32 irq_stat;
2401
2402 spin_lock_irqsave(&host->lock, flags);
2403
2404 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2405
2406 for (i = 0; i < host->n_ports; i++) {
2407 struct ata_port *ap = host->ports[i];
2408
2409 if (ap && !(ap->flags & ATA_FLAG_DISABLED)) {
2410 if (ap->link.sactive) {
2411 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2412 handled = 1;
2413 } else {
2414 if (irq_stat) /* reserve Hotplug */
2415 nv_swncq_irq_clear(ap, 0xfff0);
2416
2417 handled += nv_host_intr(ap, (u8)irq_stat);
2418 }
2419 }
2420 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2421 }
2422
2423 spin_unlock_irqrestore(&host->lock, flags);
2424
2425 return IRQ_RETVAL(handled);
2426 }
2427
2428 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
2429 {
2430 static int printed_version;
2431 const struct ata_port_info *ppi[] = { NULL, NULL };
2432 struct ata_host *host;
2433 struct nv_host_priv *hpriv;
2434 int rc;
2435 u32 bar;
2436 void __iomem *base;
2437 unsigned long type = ent->driver_data;
2438
2439 // Make sure this is a SATA controller by counting the number of bars
2440 // (NVIDIA SATA controllers will always have six bars). Otherwise,
2441 // it's an IDE controller and we ignore it.
2442 for (bar = 0; bar < 6; bar++)
2443 if (pci_resource_start(pdev, bar) == 0)
2444 return -ENODEV;
2445
2446 if (!printed_version++)
2447 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
2448
2449 rc = pcim_enable_device(pdev);
2450 if (rc)
2451 return rc;
2452
2453 /* determine type and allocate host */
2454 if (type == CK804 && adma_enabled) {
2455 dev_printk(KERN_NOTICE, &pdev->dev, "Using ADMA mode\n");
2456 type = ADMA;
2457 }
2458
2459 if (type == SWNCQ) {
2460 if (swncq_enabled)
2461 dev_printk(KERN_NOTICE, &pdev->dev,
2462 "Using SWNCQ mode\n");
2463 else
2464 type = GENERIC;
2465 }
2466
2467 ppi[0] = &nv_port_info[type];
2468 rc = ata_pci_prepare_sff_host(pdev, ppi, &host);
2469 if (rc)
2470 return rc;
2471
2472 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
2473 if (!hpriv)
2474 return -ENOMEM;
2475 hpriv->type = type;
2476 host->private_data = hpriv;
2477
2478 /* request and iomap NV_MMIO_BAR */
2479 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2480 if (rc)
2481 return rc;
2482
2483 /* configure SCR access */
2484 base = host->iomap[NV_MMIO_BAR];
2485 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2486 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
2487
2488 /* enable SATA space for CK804 */
2489 if (type >= CK804) {
2490 u8 regval;
2491
2492 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2493 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2494 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2495 }
2496
2497 /* init ADMA */
2498 if (type == ADMA) {
2499 rc = nv_adma_host_init(host);
2500 if (rc)
2501 return rc;
2502 } else if (type == SWNCQ)
2503 nv_swncq_host_init(host);
2504
2505 pci_set_master(pdev);
2506 return ata_host_activate(host, pdev->irq, ppi[0]->irq_handler,
2507 IRQF_SHARED, ppi[0]->sht);
2508 }
2509
2510 #ifdef CONFIG_PM
2511 static int nv_pci_device_resume(struct pci_dev *pdev)
2512 {
2513 struct ata_host *host = dev_get_drvdata(&pdev->dev);
2514 struct nv_host_priv *hpriv = host->private_data;
2515 int rc;
2516
2517 rc = ata_pci_device_do_resume(pdev);
2518 if (rc)
2519 return rc;
2520
2521 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
2522 if (hpriv->type >= CK804) {
2523 u8 regval;
2524
2525 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2526 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2527 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2528 }
2529 if (hpriv->type == ADMA) {
2530 u32 tmp32;
2531 struct nv_adma_port_priv *pp;
2532 /* enable/disable ADMA on the ports appropriately */
2533 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2534
2535 pp = host->ports[0]->private_data;
2536 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2537 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2538 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2539 else
2540 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
2541 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
2542 pp = host->ports[1]->private_data;
2543 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
2544 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
2545 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2546 else
2547 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
2548 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2549
2550 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2551 }
2552 }
2553
2554 ata_host_resume(host);
2555
2556 return 0;
2557 }
2558 #endif
2559
2560 static void nv_ck804_host_stop(struct ata_host *host)
2561 {
2562 struct pci_dev *pdev = to_pci_dev(host->dev);
2563 u8 regval;
2564
2565 /* disable SATA space for CK804 */
2566 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2567 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2568 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2569 }
2570
2571 static void nv_adma_host_stop(struct ata_host *host)
2572 {
2573 struct pci_dev *pdev = to_pci_dev(host->dev);
2574 u32 tmp32;
2575
2576 /* disable ADMA on the ports */
2577 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2578 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2579 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2580 NV_MCP_SATA_CFG_20_PORT1_EN |
2581 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2582
2583 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2584
2585 nv_ck804_host_stop(host);
2586 }
2587
2588 static int __init nv_init(void)
2589 {
2590 return pci_register_driver(&nv_pci_driver);
2591 }
2592
2593 static void __exit nv_exit(void)
2594 {
2595 pci_unregister_driver(&nv_pci_driver);
2596 }
2597
2598 module_init(nv_init);
2599 module_exit(nv_exit);
2600 module_param_named(adma, adma_enabled, bool, 0444);
2601 MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: true)");
2602 module_param_named(swncq, swncq_enabled, bool, 0444);
2603 MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: false)");
2604
Linux with added FPC-III support