?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6
[wrt350n-kernel.git] / drivers / crypto / hifn_795x.c
blob6f6df0f2eeec716e173f985e65e47ba2f3da8801
1 /*
2 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
3 * All rights reserved.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/moduleparam.h>
23 #include <linux/mod_devicetable.h>
24 #include <linux/interrupt.h>
25 #include <linux/pci.h>
26 #include <linux/slab.h>
27 #include <linux/delay.h>
28 #include <linux/mm.h>
29 #include <linux/dma-mapping.h>
30 #include <linux/scatterlist.h>
31 #include <linux/highmem.h>
32 #include <linux/interrupt.h>
33 #include <linux/crypto.h>
34 #include <linux/hw_random.h>
35 #include <linux/ktime.h>
36
37 #include <crypto/algapi.h>
38 #include <crypto/des.h>
39
40 #include <asm/kmap_types.h>
41
42 #undef dprintk
43
44 #define HIFN_TEST
45 //#define HIFN_DEBUG
46
47 #ifdef HIFN_DEBUG
48 #define dprintk(f, a...) printk(f, ##a)
49 #else
50 #define dprintk(f, a...) do {} while (0)
51 #endif
52
53 static char hifn_pll_ref[sizeof("extNNN")] = "ext";
54 module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444);
55 MODULE_PARM_DESC(hifn_pll_ref,
56 "PLL reference clock (pci[freq] or ext[freq], default ext)");
57
58 static atomic_t hifn_dev_number;
59
60 #define ACRYPTO_OP_DECRYPT 0
61 #define ACRYPTO_OP_ENCRYPT 1
62 #define ACRYPTO_OP_HMAC 2
63 #define ACRYPTO_OP_RNG 3
64
65 #define ACRYPTO_MODE_ECB 0
66 #define ACRYPTO_MODE_CBC 1
67 #define ACRYPTO_MODE_CFB 2
68 #define ACRYPTO_MODE_OFB 3
69
70 #define ACRYPTO_TYPE_AES_128 0
71 #define ACRYPTO_TYPE_AES_192 1
72 #define ACRYPTO_TYPE_AES_256 2
73 #define ACRYPTO_TYPE_3DES 3
74 #define ACRYPTO_TYPE_DES 4
75
76 #define PCI_VENDOR_ID_HIFN 0x13A3
77 #define PCI_DEVICE_ID_HIFN_7955 0x0020
78 #define PCI_DEVICE_ID_HIFN_7956 0x001d
79
80 /* I/O region sizes */
81
82 #define HIFN_BAR0_SIZE 0x1000
83 #define HIFN_BAR1_SIZE 0x2000
84 #define HIFN_BAR2_SIZE 0x8000
85
86 /* DMA registres */
87
88 #define HIFN_DMA_CRA 0x0C /* DMA Command Ring Address */
89 #define HIFN_DMA_SDRA 0x1C /* DMA Source Data Ring Address */
90 #define HIFN_DMA_RRA 0x2C /* DMA Result Ring Address */
91 #define HIFN_DMA_DDRA 0x3C /* DMA Destination Data Ring Address */
92 #define HIFN_DMA_STCTL 0x40 /* DMA Status and Control */
93 #define HIFN_DMA_INTREN 0x44 /* DMA Interrupt Enable */
94 #define HIFN_DMA_CFG1 0x48 /* DMA Configuration #1 */
95 #define HIFN_DMA_CFG2 0x6C /* DMA Configuration #2 */
96 #define HIFN_CHIP_ID 0x98 /* Chip ID */
97
98 /*
99 * Processing Unit Registers (offset from BASEREG0)
100 */
101 #define HIFN_0_PUDATA 0x00 /* Processing Unit Data */
102 #define HIFN_0_PUCTRL 0x04 /* Processing Unit Control */
103 #define HIFN_0_PUISR 0x08 /* Processing Unit Interrupt Status */
104 #define HIFN_0_PUCNFG 0x0c /* Processing Unit Configuration */
105 #define HIFN_0_PUIER 0x10 /* Processing Unit Interrupt Enable */
106 #define HIFN_0_PUSTAT 0x14 /* Processing Unit Status/Chip ID */
107 #define HIFN_0_FIFOSTAT 0x18 /* FIFO Status */
108 #define HIFN_0_FIFOCNFG 0x1c /* FIFO Configuration */
109 #define HIFN_0_SPACESIZE 0x20 /* Register space size */
110
111 /* Processing Unit Control Register (HIFN_0_PUCTRL) */
112 #define HIFN_PUCTRL_CLRSRCFIFO 0x0010 /* clear source fifo */
113 #define HIFN_PUCTRL_STOP 0x0008 /* stop pu */
114 #define HIFN_PUCTRL_LOCKRAM 0x0004 /* lock ram */
115 #define HIFN_PUCTRL_DMAENA 0x0002 /* enable dma */
116 #define HIFN_PUCTRL_RESET 0x0001 /* Reset processing unit */
117
118 /* Processing Unit Interrupt Status Register (HIFN_0_PUISR) */
119 #define HIFN_PUISR_CMDINVAL 0x8000 /* Invalid command interrupt */
120 #define HIFN_PUISR_DATAERR 0x4000 /* Data error interrupt */
121 #define HIFN_PUISR_SRCFIFO 0x2000 /* Source FIFO ready interrupt */
122 #define HIFN_PUISR_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */
123 #define HIFN_PUISR_DSTOVER 0x0200 /* Destination overrun interrupt */
124 #define HIFN_PUISR_SRCCMD 0x0080 /* Source command interrupt */
125 #define HIFN_PUISR_SRCCTX 0x0040 /* Source context interrupt */
126 #define HIFN_PUISR_SRCDATA 0x0020 /* Source data interrupt */
127 #define HIFN_PUISR_DSTDATA 0x0010 /* Destination data interrupt */
128 #define HIFN_PUISR_DSTRESULT 0x0004 /* Destination result interrupt */
129
130 /* Processing Unit Configuration Register (HIFN_0_PUCNFG) */
131 #define HIFN_PUCNFG_DRAMMASK 0xe000 /* DRAM size mask */
132 #define HIFN_PUCNFG_DSZ_256K 0x0000 /* 256k dram */
133 #define HIFN_PUCNFG_DSZ_512K 0x2000 /* 512k dram */
134 #define HIFN_PUCNFG_DSZ_1M 0x4000 /* 1m dram */
135 #define HIFN_PUCNFG_DSZ_2M 0x6000 /* 2m dram */
136 #define HIFN_PUCNFG_DSZ_4M 0x8000 /* 4m dram */
137 #define HIFN_PUCNFG_DSZ_8M 0xa000 /* 8m dram */
138 #define HIFN_PUNCFG_DSZ_16M 0xc000 /* 16m dram */
139 #define HIFN_PUCNFG_DSZ_32M 0xe000 /* 32m dram */
140 #define HIFN_PUCNFG_DRAMREFRESH 0x1800 /* DRAM refresh rate mask */
141 #define HIFN_PUCNFG_DRFR_512 0x0000 /* 512 divisor of ECLK */
142 #define HIFN_PUCNFG_DRFR_256 0x0800 /* 256 divisor of ECLK */
143 #define HIFN_PUCNFG_DRFR_128 0x1000 /* 128 divisor of ECLK */
144 #define HIFN_PUCNFG_TCALLPHASES 0x0200 /* your guess is as good as mine... */
145 #define HIFN_PUCNFG_TCDRVTOTEM 0x0100 /* your guess is as good as mine... */
146 #define HIFN_PUCNFG_BIGENDIAN 0x0080 /* DMA big endian mode */
147 #define HIFN_PUCNFG_BUS32 0x0040 /* Bus width 32bits */
148 #define HIFN_PUCNFG_BUS16 0x0000 /* Bus width 16 bits */
149 #define HIFN_PUCNFG_CHIPID 0x0020 /* Allow chipid from PUSTAT */
150 #define HIFN_PUCNFG_DRAM 0x0010 /* Context RAM is DRAM */
151 #define HIFN_PUCNFG_SRAM 0x0000 /* Context RAM is SRAM */
152 #define HIFN_PUCNFG_COMPSING 0x0004 /* Enable single compression context */
153 #define HIFN_PUCNFG_ENCCNFG 0x0002 /* Encryption configuration */
154
155 /* Processing Unit Interrupt Enable Register (HIFN_0_PUIER) */
156 #define HIFN_PUIER_CMDINVAL 0x8000 /* Invalid command interrupt */
157 #define HIFN_PUIER_DATAERR 0x4000 /* Data error interrupt */
158 #define HIFN_PUIER_SRCFIFO 0x2000 /* Source FIFO ready interrupt */
159 #define HIFN_PUIER_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */
160 #define HIFN_PUIER_DSTOVER 0x0200 /* Destination overrun interrupt */
161 #define HIFN_PUIER_SRCCMD 0x0080 /* Source command interrupt */
162 #define HIFN_PUIER_SRCCTX 0x0040 /* Source context interrupt */
163 #define HIFN_PUIER_SRCDATA 0x0020 /* Source data interrupt */
164 #define HIFN_PUIER_DSTDATA 0x0010 /* Destination data interrupt */
165 #define HIFN_PUIER_DSTRESULT 0x0004 /* Destination result interrupt */
166
167 /* Processing Unit Status Register/Chip ID (HIFN_0_PUSTAT) */
168 #define HIFN_PUSTAT_CMDINVAL 0x8000 /* Invalid command interrupt */
169 #define HIFN_PUSTAT_DATAERR 0x4000 /* Data error interrupt */
170 #define HIFN_PUSTAT_SRCFIFO 0x2000 /* Source FIFO ready interrupt */
171 #define HIFN_PUSTAT_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */
172 #define HIFN_PUSTAT_DSTOVER 0x0200 /* Destination overrun interrupt */
173 #define HIFN_PUSTAT_SRCCMD 0x0080 /* Source command interrupt */
174 #define HIFN_PUSTAT_SRCCTX 0x0040 /* Source context interrupt */
175 #define HIFN_PUSTAT_SRCDATA 0x0020 /* Source data interrupt */
176 #define HIFN_PUSTAT_DSTDATA 0x0010 /* Destination data interrupt */
177 #define HIFN_PUSTAT_DSTRESULT 0x0004 /* Destination result interrupt */
178 #define HIFN_PUSTAT_CHIPREV 0x00ff /* Chip revision mask */
179 #define HIFN_PUSTAT_CHIPENA 0xff00 /* Chip enabled mask */
180 #define HIFN_PUSTAT_ENA_2 0x1100 /* Level 2 enabled */
181 #define HIFN_PUSTAT_ENA_1 0x1000 /* Level 1 enabled */
182 #define HIFN_PUSTAT_ENA_0 0x3000 /* Level 0 enabled */
183 #define HIFN_PUSTAT_REV_2 0x0020 /* 7751 PT6/2 */
184 #define HIFN_PUSTAT_REV_3 0x0030 /* 7751 PT6/3 */
185
186 /* FIFO Status Register (HIFN_0_FIFOSTAT) */
187 #define HIFN_FIFOSTAT_SRC 0x7f00 /* Source FIFO available */
188 #define HIFN_FIFOSTAT_DST 0x007f /* Destination FIFO available */
189
190 /* FIFO Configuration Register (HIFN_0_FIFOCNFG) */
191 #define HIFN_FIFOCNFG_THRESHOLD 0x0400 /* must be written as 1 */
192
193 /*
194 * DMA Interface Registers (offset from BASEREG1)
195 */
196 #define HIFN_1_DMA_CRAR 0x0c /* DMA Command Ring Address */
197 #define HIFN_1_DMA_SRAR 0x1c /* DMA Source Ring Address */
198 #define HIFN_1_DMA_RRAR 0x2c /* DMA Result Ring Address */
199 #define HIFN_1_DMA_DRAR 0x3c /* DMA Destination Ring Address */
200 #define HIFN_1_DMA_CSR 0x40 /* DMA Status and Control */
201 #define HIFN_1_DMA_IER 0x44 /* DMA Interrupt Enable */
202 #define HIFN_1_DMA_CNFG 0x48 /* DMA Configuration */
203 #define HIFN_1_PLL 0x4c /* 795x: PLL config */
204 #define HIFN_1_7811_RNGENA 0x60 /* 7811: rng enable */
205 #define HIFN_1_7811_RNGCFG 0x64 /* 7811: rng config */
206 #define HIFN_1_7811_RNGDAT 0x68 /* 7811: rng data */
207 #define HIFN_1_7811_RNGSTS 0x6c /* 7811: rng status */
208 #define HIFN_1_7811_MIPSRST 0x94 /* 7811: MIPS reset */
209 #define HIFN_1_REVID 0x98 /* Revision ID */
210 #define HIFN_1_UNLOCK_SECRET1 0xf4
211 #define HIFN_1_UNLOCK_SECRET2 0xfc
212 #define HIFN_1_PUB_RESET 0x204 /* Public/RNG Reset */
213 #define HIFN_1_PUB_BASE 0x300 /* Public Base Address */
214 #define HIFN_1_PUB_OPLEN 0x304 /* Public Operand Length */
215 #define HIFN_1_PUB_OP 0x308 /* Public Operand */
216 #define HIFN_1_PUB_STATUS 0x30c /* Public Status */
217 #define HIFN_1_PUB_IEN 0x310 /* Public Interrupt enable */
218 #define HIFN_1_RNG_CONFIG 0x314 /* RNG config */
219 #define HIFN_1_RNG_DATA 0x318 /* RNG data */
220 #define HIFN_1_PUB_MEM 0x400 /* start of Public key memory */
221 #define HIFN_1_PUB_MEMEND 0xbff /* end of Public key memory */
222
223 /* DMA Status and Control Register (HIFN_1_DMA_CSR) */
224 #define HIFN_DMACSR_D_CTRLMASK 0xc0000000 /* Destinition Ring Control */
225 #define HIFN_DMACSR_D_CTRL_NOP 0x00000000 /* Dest. Control: no-op */
226 #define HIFN_DMACSR_D_CTRL_DIS 0x40000000 /* Dest. Control: disable */
227 #define HIFN_DMACSR_D_CTRL_ENA 0x80000000 /* Dest. Control: enable */
228 #define HIFN_DMACSR_D_ABORT 0x20000000 /* Destinition Ring PCIAbort */
229 #define HIFN_DMACSR_D_DONE 0x10000000 /* Destinition Ring Done */
230 #define HIFN_DMACSR_D_LAST 0x08000000 /* Destinition Ring Last */
231 #define HIFN_DMACSR_D_WAIT 0x04000000 /* Destinition Ring Waiting */
232 #define HIFN_DMACSR_D_OVER 0x02000000 /* Destinition Ring Overflow */
233 #define HIFN_DMACSR_R_CTRL 0x00c00000 /* Result Ring Control */
234 #define HIFN_DMACSR_R_CTRL_NOP 0x00000000 /* Result Control: no-op */
235 #define HIFN_DMACSR_R_CTRL_DIS 0x00400000 /* Result Control: disable */
236 #define HIFN_DMACSR_R_CTRL_ENA 0x00800000 /* Result Control: enable */
237 #define HIFN_DMACSR_R_ABORT 0x00200000 /* Result Ring PCI Abort */
238 #define HIFN_DMACSR_R_DONE 0x00100000 /* Result Ring Done */
239 #define HIFN_DMACSR_R_LAST 0x00080000 /* Result Ring Last */
240 #define HIFN_DMACSR_R_WAIT 0x00040000 /* Result Ring Waiting */
241 #define HIFN_DMACSR_R_OVER 0x00020000 /* Result Ring Overflow */
242 #define HIFN_DMACSR_S_CTRL 0x0000c000 /* Source Ring Control */
243 #define HIFN_DMACSR_S_CTRL_NOP 0x00000000 /* Source Control: no-op */
244 #define HIFN_DMACSR_S_CTRL_DIS 0x00004000 /* Source Control: disable */
245 #define HIFN_DMACSR_S_CTRL_ENA 0x00008000 /* Source Control: enable */
246 #define HIFN_DMACSR_S_ABORT 0x00002000 /* Source Ring PCI Abort */
247 #define HIFN_DMACSR_S_DONE 0x00001000 /* Source Ring Done */
248 #define HIFN_DMACSR_S_LAST 0x00000800 /* Source Ring Last */
249 #define HIFN_DMACSR_S_WAIT 0x00000400 /* Source Ring Waiting */
250 #define HIFN_DMACSR_ILLW 0x00000200 /* Illegal write (7811 only) */
251 #define HIFN_DMACSR_ILLR 0x00000100 /* Illegal read (7811 only) */
252 #define HIFN_DMACSR_C_CTRL 0x000000c0 /* Command Ring Control */
253 #define HIFN_DMACSR_C_CTRL_NOP 0x00000000 /* Command Control: no-op */
254 #define HIFN_DMACSR_C_CTRL_DIS 0x00000040 /* Command Control: disable */
255 #define HIFN_DMACSR_C_CTRL_ENA 0x00000080 /* Command Control: enable */
256 #define HIFN_DMACSR_C_ABORT 0x00000020 /* Command Ring PCI Abort */
257 #define HIFN_DMACSR_C_DONE 0x00000010 /* Command Ring Done */
258 #define HIFN_DMACSR_C_LAST 0x00000008 /* Command Ring Last */
259 #define HIFN_DMACSR_C_WAIT 0x00000004 /* Command Ring Waiting */
260 #define HIFN_DMACSR_PUBDONE 0x00000002 /* Public op done (7951 only) */
261 #define HIFN_DMACSR_ENGINE 0x00000001 /* Command Ring Engine IRQ */
262
263 /* DMA Interrupt Enable Register (HIFN_1_DMA_IER) */
264 #define HIFN_DMAIER_D_ABORT 0x20000000 /* Destination Ring PCIAbort */
265 #define HIFN_DMAIER_D_DONE 0x10000000 /* Destination Ring Done */
266 #define HIFN_DMAIER_D_LAST 0x08000000 /* Destination Ring Last */
267 #define HIFN_DMAIER_D_WAIT 0x04000000 /* Destination Ring Waiting */
268 #define HIFN_DMAIER_D_OVER 0x02000000 /* Destination Ring Overflow */
269 #define HIFN_DMAIER_R_ABORT 0x00200000 /* Result Ring PCI Abort */
270 #define HIFN_DMAIER_R_DONE 0x00100000 /* Result Ring Done */
271 #define HIFN_DMAIER_R_LAST 0x00080000 /* Result Ring Last */
272 #define HIFN_DMAIER_R_WAIT 0x00040000 /* Result Ring Waiting */
273 #define HIFN_DMAIER_R_OVER 0x00020000 /* Result Ring Overflow */
274 #define HIFN_DMAIER_S_ABORT 0x00002000 /* Source Ring PCI Abort */
275 #define HIFN_DMAIER_S_DONE 0x00001000 /* Source Ring Done */
276 #define HIFN_DMAIER_S_LAST 0x00000800 /* Source Ring Last */
277 #define HIFN_DMAIER_S_WAIT 0x00000400 /* Source Ring Waiting */
278 #define HIFN_DMAIER_ILLW 0x00000200 /* Illegal write (7811 only) */
279 #define HIFN_DMAIER_ILLR 0x00000100 /* Illegal read (7811 only) */
280 #define HIFN_DMAIER_C_ABORT 0x00000020 /* Command Ring PCI Abort */
281 #define HIFN_DMAIER_C_DONE 0x00000010 /* Command Ring Done */
282 #define HIFN_DMAIER_C_LAST 0x00000008 /* Command Ring Last */
283 #define HIFN_DMAIER_C_WAIT 0x00000004 /* Command Ring Waiting */
284 #define HIFN_DMAIER_PUBDONE 0x00000002 /* public op done (7951 only) */
285 #define HIFN_DMAIER_ENGINE 0x00000001 /* Engine IRQ */
286
287 /* DMA Configuration Register (HIFN_1_DMA_CNFG) */
288 #define HIFN_DMACNFG_BIGENDIAN 0x10000000 /* big endian mode */
289 #define HIFN_DMACNFG_POLLFREQ 0x00ff0000 /* Poll frequency mask */
290 #define HIFN_DMACNFG_UNLOCK 0x00000800
291 #define HIFN_DMACNFG_POLLINVAL 0x00000700 /* Invalid Poll Scalar */
292 #define HIFN_DMACNFG_LAST 0x00000010 /* Host control LAST bit */
293 #define HIFN_DMACNFG_MODE 0x00000004 /* DMA mode */
294 #define HIFN_DMACNFG_DMARESET 0x00000002 /* DMA Reset # */
295 #define HIFN_DMACNFG_MSTRESET 0x00000001 /* Master Reset # */
296
297 /* PLL configuration register */
298 #define HIFN_PLL_REF_CLK_HBI 0x00000000 /* HBI reference clock */
299 #define HIFN_PLL_REF_CLK_PLL 0x00000001 /* PLL reference clock */
300 #define HIFN_PLL_BP 0x00000002 /* Reference clock bypass */
301 #define HIFN_PLL_PK_CLK_HBI 0x00000000 /* PK engine HBI clock */
302 #define HIFN_PLL_PK_CLK_PLL 0x00000008 /* PK engine PLL clock */
303 #define HIFN_PLL_PE_CLK_HBI 0x00000000 /* PE engine HBI clock */
304 #define HIFN_PLL_PE_CLK_PLL 0x00000010 /* PE engine PLL clock */
305 #define HIFN_PLL_RESERVED_1 0x00000400 /* Reserved bit, must be 1 */
306 #define HIFN_PLL_ND_SHIFT 11 /* Clock multiplier shift */
307 #define HIFN_PLL_ND_MULT_2 0x00000000 /* PLL clock multiplier 2 */
308 #define HIFN_PLL_ND_MULT_4 0x00000800 /* PLL clock multiplier 4 */
309 #define HIFN_PLL_ND_MULT_6 0x00001000 /* PLL clock multiplier 6 */
310 #define HIFN_PLL_ND_MULT_8 0x00001800 /* PLL clock multiplier 8 */
311 #define HIFN_PLL_ND_MULT_10 0x00002000 /* PLL clock multiplier 10 */
312 #define HIFN_PLL_ND_MULT_12 0x00002800 /* PLL clock multiplier 12 */
313 #define HIFN_PLL_IS_1_8 0x00000000 /* charge pump (mult. 1-8) */
314 #define HIFN_PLL_IS_9_12 0x00010000 /* charge pump (mult. 9-12) */
315
316 #define HIFN_PLL_FCK_MAX 266 /* Maximum PLL frequency */
317
318 /* Public key reset register (HIFN_1_PUB_RESET) */
319 #define HIFN_PUBRST_RESET 0x00000001 /* reset public/rng unit */
320
321 /* Public base address register (HIFN_1_PUB_BASE) */
322 #define HIFN_PUBBASE_ADDR 0x00003fff /* base address */
323
324 /* Public operand length register (HIFN_1_PUB_OPLEN) */
325 #define HIFN_PUBOPLEN_MOD_M 0x0000007f /* modulus length mask */
326 #define HIFN_PUBOPLEN_MOD_S 0 /* modulus length shift */
327 #define HIFN_PUBOPLEN_EXP_M 0x0003ff80 /* exponent length mask */
328 #define HIFN_PUBOPLEN_EXP_S 7 /* exponent lenght shift */
329 #define HIFN_PUBOPLEN_RED_M 0x003c0000 /* reducend length mask */
330 #define HIFN_PUBOPLEN_RED_S 18 /* reducend length shift */
331
332 /* Public operation register (HIFN_1_PUB_OP) */
333 #define HIFN_PUBOP_AOFFSET_M 0x0000007f /* A offset mask */
334 #define HIFN_PUBOP_AOFFSET_S 0 /* A offset shift */
335 #define HIFN_PUBOP_BOFFSET_M 0x00000f80 /* B offset mask */
336 #define HIFN_PUBOP_BOFFSET_S 7 /* B offset shift */
337 #define HIFN_PUBOP_MOFFSET_M 0x0003f000 /* M offset mask */
338 #define HIFN_PUBOP_MOFFSET_S 12 /* M offset shift */
339 #define HIFN_PUBOP_OP_MASK 0x003c0000 /* Opcode: */
340 #define HIFN_PUBOP_OP_NOP 0x00000000 /* NOP */
341 #define HIFN_PUBOP_OP_ADD 0x00040000 /* ADD */
342 #define HIFN_PUBOP_OP_ADDC 0x00080000 /* ADD w/carry */
343 #define HIFN_PUBOP_OP_SUB 0x000c0000 /* SUB */
344 #define HIFN_PUBOP_OP_SUBC 0x00100000 /* SUB w/carry */
345 #define HIFN_PUBOP_OP_MODADD 0x00140000 /* Modular ADD */
346 #define HIFN_PUBOP_OP_MODSUB 0x00180000 /* Modular SUB */
347 #define HIFN_PUBOP_OP_INCA 0x001c0000 /* INC A */
348 #define HIFN_PUBOP_OP_DECA 0x00200000 /* DEC A */
349 #define HIFN_PUBOP_OP_MULT 0x00240000 /* MULT */
350 #define HIFN_PUBOP_OP_MODMULT 0x00280000 /* Modular MULT */
351 #define HIFN_PUBOP_OP_MODRED 0x002c0000 /* Modular RED */
352 #define HIFN_PUBOP_OP_MODEXP 0x00300000 /* Modular EXP */
353
354 /* Public status register (HIFN_1_PUB_STATUS) */
355 #define HIFN_PUBSTS_DONE 0x00000001 /* operation done */
356 #define HIFN_PUBSTS_CARRY 0x00000002 /* carry */
357
358 /* Public interrupt enable register (HIFN_1_PUB_IEN) */
359 #define HIFN_PUBIEN_DONE 0x00000001 /* operation done interrupt */
360
361 /* Random number generator config register (HIFN_1_RNG_CONFIG) */
362 #define HIFN_RNGCFG_ENA 0x00000001 /* enable rng */
363
364 #define HIFN_NAMESIZE 32
365 #define HIFN_MAX_RESULT_ORDER 5
366
367 #define HIFN_D_CMD_RSIZE 24*4
368 #define HIFN_D_SRC_RSIZE 80*4
369 #define HIFN_D_DST_RSIZE 80*4
370 #define HIFN_D_RES_RSIZE 24*4
371
372 #define HIFN_QUEUE_LENGTH HIFN_D_CMD_RSIZE-5
373
374 #define AES_MIN_KEY_SIZE 16
375 #define AES_MAX_KEY_SIZE 32
376
377 #define HIFN_DES_KEY_LENGTH 8
378 #define HIFN_3DES_KEY_LENGTH 24
379 #define HIFN_MAX_CRYPT_KEY_LENGTH AES_MAX_KEY_SIZE
380 #define HIFN_IV_LENGTH 8
381 #define HIFN_AES_IV_LENGTH 16
382 #define HIFN_MAX_IV_LENGTH HIFN_AES_IV_LENGTH
383
384 #define HIFN_MAC_KEY_LENGTH 64
385 #define HIFN_MD5_LENGTH 16
386 #define HIFN_SHA1_LENGTH 20
387 #define HIFN_MAC_TRUNC_LENGTH 12
388
389 #define HIFN_MAX_COMMAND (8 + 8 + 8 + 64 + 260)
390 #define HIFN_MAX_RESULT (8 + 4 + 4 + 20 + 4)
391 #define HIFN_USED_RESULT 12
392
393 struct hifn_desc
394 {
395 volatile u32 l;
396 volatile u32 p;
397 };
398
399 struct hifn_dma {
400 struct hifn_desc cmdr[HIFN_D_CMD_RSIZE+1];
401 struct hifn_desc srcr[HIFN_D_SRC_RSIZE+1];
402 struct hifn_desc dstr[HIFN_D_DST_RSIZE+1];
403 struct hifn_desc resr[HIFN_D_RES_RSIZE+1];
404
405 u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND];
406 u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT];
407
408 u64 test_src, test_dst;
409
410 /*
411 * Our current positions for insertion and removal from the descriptor
412 * rings.
413 */
414 volatile int cmdi, srci, dsti, resi;
415 volatile int cmdu, srcu, dstu, resu;
416 int cmdk, srck, dstk, resk;
417 };
418
419 #define HIFN_FLAG_CMD_BUSY (1<<0)
420 #define HIFN_FLAG_SRC_BUSY (1<<1)
421 #define HIFN_FLAG_DST_BUSY (1<<2)
422 #define HIFN_FLAG_RES_BUSY (1<<3)
423 #define HIFN_FLAG_OLD_KEY (1<<4)
424
425 #define HIFN_DEFAULT_ACTIVE_NUM 5
426
427 struct hifn_device
428 {
429 char name[HIFN_NAMESIZE];
430
431 int irq;
432
433 struct pci_dev *pdev;
434 void __iomem *bar[3];
435
436 unsigned long result_mem;
437 dma_addr_t dst;
438
439 void *desc_virt;
440 dma_addr_t desc_dma;
441
442 u32 dmareg;
443
444 void *sa[HIFN_D_RES_RSIZE];
445
446 spinlock_t lock;
447
448 void *priv;
449
450 u32 flags;
451 int active, started;
452 struct delayed_work work;
453 unsigned long reset;
454 unsigned long success;
455 unsigned long prev_success;
456
457 u8 snum;
458
459 struct tasklet_struct tasklet;
460
461 struct crypto_queue queue;
462 struct list_head alg_list;
463
464 unsigned int pk_clk_freq;
465
466 <<<<<<< HEAD:drivers/crypto/hifn_795x.c
467 #ifdef CRYPTO_DEV_HIFN_795X_RNG
468 =======
469 #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
470 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:drivers/crypto/hifn_795x.c
471 unsigned int rng_wait_time;
472 ktime_t rngtime;
473 struct hwrng rng;
474 #endif
475 };
476
477 #define HIFN_D_LENGTH 0x0000ffff
478 #define HIFN_D_NOINVALID 0x01000000
479 #define HIFN_D_MASKDONEIRQ 0x02000000
480 #define HIFN_D_DESTOVER 0x04000000
481 #define HIFN_D_OVER 0x08000000
482 #define HIFN_D_LAST 0x20000000
483 #define HIFN_D_JUMP 0x40000000
484 #define HIFN_D_VALID 0x80000000
485
486 struct hifn_base_command
487 {
488 volatile u16 masks;
489 volatile u16 session_num;
490 volatile u16 total_source_count;
491 volatile u16 total_dest_count;
492 };
493
494 #define HIFN_BASE_CMD_COMP 0x0100 /* enable compression engine */
495 #define HIFN_BASE_CMD_PAD 0x0200 /* enable padding engine */
496 #define HIFN_BASE_CMD_MAC 0x0400 /* enable MAC engine */
497 #define HIFN_BASE_CMD_CRYPT 0x0800 /* enable crypt engine */
498 #define HIFN_BASE_CMD_DECODE 0x2000
499 #define HIFN_BASE_CMD_SRCLEN_M 0xc000
500 #define HIFN_BASE_CMD_SRCLEN_S 14
501 #define HIFN_BASE_CMD_DSTLEN_M 0x3000
502 #define HIFN_BASE_CMD_DSTLEN_S 12
503 #define HIFN_BASE_CMD_LENMASK_HI 0x30000
504 #define HIFN_BASE_CMD_LENMASK_LO 0x0ffff
505
506 /*
507 * Structure to help build up the command data structure.
508 */
509 struct hifn_crypt_command
510 {
511 volatile u16 masks;
512 volatile u16 header_skip;
513 volatile u16 source_count;
514 volatile u16 reserved;
515 };
516
517 #define HIFN_CRYPT_CMD_ALG_MASK 0x0003 /* algorithm: */
518 #define HIFN_CRYPT_CMD_ALG_DES 0x0000 /* DES */
519 #define HIFN_CRYPT_CMD_ALG_3DES 0x0001 /* 3DES */
520 #define HIFN_CRYPT_CMD_ALG_RC4 0x0002 /* RC4 */
521 #define HIFN_CRYPT_CMD_ALG_AES 0x0003 /* AES */
522 #define HIFN_CRYPT_CMD_MODE_MASK 0x0018 /* Encrypt mode: */
523 #define HIFN_CRYPT_CMD_MODE_ECB 0x0000 /* ECB */
524 #define HIFN_CRYPT_CMD_MODE_CBC 0x0008 /* CBC */
525 #define HIFN_CRYPT_CMD_MODE_CFB 0x0010 /* CFB */
526 #define HIFN_CRYPT_CMD_MODE_OFB 0x0018 /* OFB */
527 #define HIFN_CRYPT_CMD_CLR_CTX 0x0040 /* clear context */
528 #define HIFN_CRYPT_CMD_KSZ_MASK 0x0600 /* AES key size: */
529 #define HIFN_CRYPT_CMD_KSZ_128 0x0000 /* 128 bit */
530 #define HIFN_CRYPT_CMD_KSZ_192 0x0200 /* 192 bit */
531 #define HIFN_CRYPT_CMD_KSZ_256 0x0400 /* 256 bit */
532 #define HIFN_CRYPT_CMD_NEW_KEY 0x0800 /* expect new key */
533 #define HIFN_CRYPT_CMD_NEW_IV 0x1000 /* expect new iv */
534 #define HIFN_CRYPT_CMD_SRCLEN_M 0xc000
535 #define HIFN_CRYPT_CMD_SRCLEN_S 14
536
537 /*
538 * Structure to help build up the command data structure.
539 */
540 struct hifn_mac_command
541 {
542 volatile u16 masks;
543 volatile u16 header_skip;
544 volatile u16 source_count;
545 volatile u16 reserved;
546 };
547
548 #define HIFN_MAC_CMD_ALG_MASK 0x0001
549 #define HIFN_MAC_CMD_ALG_SHA1 0x0000
550 #define HIFN_MAC_CMD_ALG_MD5 0x0001
551 #define HIFN_MAC_CMD_MODE_MASK 0x000c
552 #define HIFN_MAC_CMD_MODE_HMAC 0x0000
553 #define HIFN_MAC_CMD_MODE_SSL_MAC 0x0004
554 #define HIFN_MAC_CMD_MODE_HASH 0x0008
555 #define HIFN_MAC_CMD_MODE_FULL 0x0004
556 #define HIFN_MAC_CMD_TRUNC 0x0010
557 #define HIFN_MAC_CMD_RESULT 0x0020
558 #define HIFN_MAC_CMD_APPEND 0x0040
559 #define HIFN_MAC_CMD_SRCLEN_M 0xc000
560 #define HIFN_MAC_CMD_SRCLEN_S 14
561
562 /*
563 * MAC POS IPsec initiates authentication after encryption on encodes
564 * and before decryption on decodes.
565 */
566 #define HIFN_MAC_CMD_POS_IPSEC 0x0200
567 #define HIFN_MAC_CMD_NEW_KEY 0x0800
568
569 struct hifn_comp_command
570 {
571 volatile u16 masks;
572 volatile u16 header_skip;
573 volatile u16 source_count;
574 volatile u16 reserved;
575 };
576
577 #define HIFN_COMP_CMD_SRCLEN_M 0xc000
578 #define HIFN_COMP_CMD_SRCLEN_S 14
579 #define HIFN_COMP_CMD_ONE 0x0100 /* must be one */
580 #define HIFN_COMP_CMD_CLEARHIST 0x0010 /* clear history */
581 #define HIFN_COMP_CMD_UPDATEHIST 0x0008 /* update history */
582 #define HIFN_COMP_CMD_LZS_STRIP0 0x0004 /* LZS: strip zero */
583 #define HIFN_COMP_CMD_MPPC_RESTART 0x0004 /* MPPC: restart */
584 #define HIFN_COMP_CMD_ALG_MASK 0x0001 /* compression mode: */
585 #define HIFN_COMP_CMD_ALG_MPPC 0x0001 /* MPPC */
586 #define HIFN_COMP_CMD_ALG_LZS 0x0000 /* LZS */
587
588 struct hifn_base_result
589 {
590 volatile u16 flags;
591 volatile u16 session;
592 volatile u16 src_cnt; /* 15:0 of source count */
593 volatile u16 dst_cnt; /* 15:0 of dest count */
594 };
595
596 #define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */
597 #define HIFN_BASE_RES_SRCLEN_M 0xc000 /* 17:16 of source count */
598 #define HIFN_BASE_RES_SRCLEN_S 14
599 #define HIFN_BASE_RES_DSTLEN_M 0x3000 /* 17:16 of dest count */
600 #define HIFN_BASE_RES_DSTLEN_S 12
601
602 struct hifn_comp_result
603 {
604 volatile u16 flags;
605 volatile u16 crc;
606 };
607
608 #define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */
609 #define HIFN_COMP_RES_LCB_S 8
610 #define HIFN_COMP_RES_RESTART 0x0004 /* MPPC: restart */
611 #define HIFN_COMP_RES_ENDMARKER 0x0002 /* LZS: end marker seen */
612 #define HIFN_COMP_RES_SRC_NOTZERO 0x0001 /* source expired */
613
614 struct hifn_mac_result
615 {
616 volatile u16 flags;
617 volatile u16 reserved;
618 /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */
619 };
620
621 #define HIFN_MAC_RES_MISCOMPARE 0x0002 /* compare failed */
622 #define HIFN_MAC_RES_SRC_NOTZERO 0x0001 /* source expired */
623
624 struct hifn_crypt_result
625 {
626 volatile u16 flags;
627 volatile u16 reserved;
628 };
629
630 #define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */
631
632 #ifndef HIFN_POLL_FREQUENCY
633 #define HIFN_POLL_FREQUENCY 0x1
634 #endif
635
636 #ifndef HIFN_POLL_SCALAR
637 #define HIFN_POLL_SCALAR 0x0
638 #endif
639
640 #define HIFN_MAX_SEGLEN 0xffff /* maximum dma segment len */
641 #define HIFN_MAX_DMALEN 0x3ffff /* maximum dma length */
642
643 struct hifn_crypto_alg
644 {
645 struct list_head entry;
646 struct crypto_alg alg;
647 struct hifn_device *dev;
648 };
649
650 #define ASYNC_SCATTERLIST_CACHE 16
651
652 #define ASYNC_FLAGS_MISALIGNED (1<<0)
653
654 struct ablkcipher_walk
655 {
656 struct scatterlist cache[ASYNC_SCATTERLIST_CACHE];
657 u32 flags;
658 int num;
659 };
660
661 struct hifn_context
662 {
663 u8 key[HIFN_MAX_CRYPT_KEY_LENGTH], *iv;
664 struct hifn_device *dev;
665 unsigned int keysize, ivsize;
666 u8 op, type, mode, unused;
667 struct ablkcipher_walk walk;
668 atomic_t sg_num;
669 };
670
671 #define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg)
672
673 static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg)
674 {
675 u32 ret;
676
677 ret = readl((char *)(dev->bar[0]) + reg);
678
679 return ret;
680 }
681
682 static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg)
683 {
684 u32 ret;
685
686 ret = readl((char *)(dev->bar[1]) + reg);
687
688 return ret;
689 }
690
691 static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val)
692 {
693 writel(val, (char *)(dev->bar[0]) + reg);
694 }
695
696 static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val)
697 {
698 writel(val, (char *)(dev->bar[1]) + reg);
699 }
700
701 static void hifn_wait_puc(struct hifn_device *dev)
702 {
703 int i;
704 u32 ret;
705
706 for (i=10000; i > 0; --i) {
707 ret = hifn_read_0(dev, HIFN_0_PUCTRL);
708 if (!(ret & HIFN_PUCTRL_RESET))
709 break;
710
711 udelay(1);
712 }
713
714 if (!i)
715 dprintk("%s: Failed to reset PUC unit.\n", dev->name);
716 }
717
718 static void hifn_reset_puc(struct hifn_device *dev)
719 {
720 hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
721 hifn_wait_puc(dev);
722 }
723
724 static void hifn_stop_device(struct hifn_device *dev)
725 {
726 hifn_write_1(dev, HIFN_1_DMA_CSR,
727 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
728 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS);
729 hifn_write_0(dev, HIFN_0_PUIER, 0);
730 hifn_write_1(dev, HIFN_1_DMA_IER, 0);
731 }
732
733 static void hifn_reset_dma(struct hifn_device *dev, int full)
734 {
735 hifn_stop_device(dev);
736
737 /*
738 * Setting poll frequency and others to 0.
739 */
740 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
741 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
742 mdelay(1);
743
744 /*
745 * Reset DMA.
746 */
747 if (full) {
748 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
749 mdelay(1);
750 } else {
751 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE |
752 HIFN_DMACNFG_MSTRESET);
753 hifn_reset_puc(dev);
754 }
755
756 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
757 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
758
759 hifn_reset_puc(dev);
760 }
761
762 static u32 hifn_next_signature(u_int32_t a, u_int cnt)
763 {
764 int i;
765 u32 v;
766
767 for (i = 0; i < cnt; i++) {
768
769 /* get the parity */
770 v = a & 0x80080125;
771 v ^= v >> 16;
772 v ^= v >> 8;
773 v ^= v >> 4;
774 v ^= v >> 2;
775 v ^= v >> 1;
776
777 a = (v & 1) ^ (a << 1);
778 }
779
780 return a;
781 }
782
783 static struct pci2id {
784 u_short pci_vendor;
785 u_short pci_prod;
786 char card_id[13];
787 } pci2id[] = {
788 {
789 PCI_VENDOR_ID_HIFN,
790 PCI_DEVICE_ID_HIFN_7955,
791 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
792 0x00, 0x00, 0x00, 0x00, 0x00 }
793 },
794 {
795 PCI_VENDOR_ID_HIFN,
796 PCI_DEVICE_ID_HIFN_7956,
797 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
798 0x00, 0x00, 0x00, 0x00, 0x00 }
799 }
800 };
801
802 <<<<<<< HEAD:drivers/crypto/hifn_795x.c
803 #ifdef CRYPTO_DEV_HIFN_795X_RNG
804 =======
805 #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
806 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:drivers/crypto/hifn_795x.c
807 static int hifn_rng_data_present(struct hwrng *rng, int wait)
808 {
809 struct hifn_device *dev = (struct hifn_device *)rng->priv;
810 s64 nsec;
811
812 nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime));
813 nsec -= dev->rng_wait_time;
814 if (nsec <= 0)
815 return 1;
816 if (!wait)
817 return 0;
818 ndelay(nsec);
819 return 1;
820 }
821
822 static int hifn_rng_data_read(struct hwrng *rng, u32 *data)
823 {
824 struct hifn_device *dev = (struct hifn_device *)rng->priv;
825
826 *data = hifn_read_1(dev, HIFN_1_RNG_DATA);
827 dev->rngtime = ktime_get();
828 return 4;
829 }
830
831 static int hifn_register_rng(struct hifn_device *dev)
832 {
833 /*
834 * We must wait at least 256 Pk_clk cycles between two reads of the rng.
835 */
836 dev->rng_wait_time = DIV_ROUND_UP(NSEC_PER_SEC, dev->pk_clk_freq) *
837 256;
838
839 dev->rng.name = dev->name;
840 dev->rng.data_present = hifn_rng_data_present,
841 dev->rng.data_read = hifn_rng_data_read,
842 dev->rng.priv = (unsigned long)dev;
843
844 return hwrng_register(&dev->rng);
845 }
846
847 static void hifn_unregister_rng(struct hifn_device *dev)
848 {
849 hwrng_unregister(&dev->rng);
850 }
851 #else
852 #define hifn_register_rng(dev) 0
853 #define hifn_unregister_rng(dev)
854 #endif
855
856 static int hifn_init_pubrng(struct hifn_device *dev)
857 {
858 int i;
859
860 hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) |
861 HIFN_PUBRST_RESET);
862
863 for (i=100; i > 0; --i) {
864 mdelay(1);
865
866 if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0)
867 break;
868 }
869
870 if (!i)
871 dprintk("Chip %s: Failed to initialise public key engine.\n",
872 dev->name);
873 else {
874 hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
875 dev->dmareg |= HIFN_DMAIER_PUBDONE;
876 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
877
878 dprintk("Chip %s: Public key engine has been sucessfully "
879 "initialised.\n", dev->name);
880 }
881
882 /*
883 * Enable RNG engine.
884 */
885
886 hifn_write_1(dev, HIFN_1_RNG_CONFIG,
887 hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA);
888 dprintk("Chip %s: RNG engine has been successfully initialised.\n",
889 dev->name);
890
891 <<<<<<< HEAD:drivers/crypto/hifn_795x.c
892 #ifdef CRYPTO_DEV_HIFN_795X_RNG
893 =======
894 #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
895 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:drivers/crypto/hifn_795x.c
896 /* First value must be discarded */
897 hifn_read_1(dev, HIFN_1_RNG_DATA);
898 dev->rngtime = ktime_get();
899 #endif
900 return 0;
901 }
902
903 static int hifn_enable_crypto(struct hifn_device *dev)
904 {
905 u32 dmacfg, addr;
906 char *offtbl = NULL;
907 int i;
908
909 for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) {
910 if (pci2id[i].pci_vendor == dev->pdev->vendor &&
911 pci2id[i].pci_prod == dev->pdev->device) {
912 offtbl = pci2id[i].card_id;
913 break;
914 }
915 }
916
917 if (offtbl == NULL) {
918 dprintk("Chip %s: Unknown card!\n", dev->name);
919 return -ENODEV;
920 }
921
922 dmacfg = hifn_read_1(dev, HIFN_1_DMA_CNFG);
923
924 hifn_write_1(dev, HIFN_1_DMA_CNFG,
925 HIFN_DMACNFG_UNLOCK | HIFN_DMACNFG_MSTRESET |
926 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
927 mdelay(1);
928 addr = hifn_read_1(dev, HIFN_1_UNLOCK_SECRET1);
929 mdelay(1);
930 hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0);
931 mdelay(1);
932
933 for (i=0; i<12; ++i) {
934 addr = hifn_next_signature(addr, offtbl[i] + 0x101);
935 hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr);
936
937 mdelay(1);
938 }
939 hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg);
940
941 dprintk("Chip %s: %s.\n", dev->name, pci_name(dev->pdev));
942
943 return 0;
944 }
945
946 static void hifn_init_dma(struct hifn_device *dev)
947 {
948 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
949 u32 dptr = dev->desc_dma;
950 int i;
951
952 for (i=0; i<HIFN_D_CMD_RSIZE; ++i)
953 dma->cmdr[i].p = __cpu_to_le32(dptr +
954 offsetof(struct hifn_dma, command_bufs[i][0]));
955 for (i=0; i<HIFN_D_RES_RSIZE; ++i)
956 dma->resr[i].p = __cpu_to_le32(dptr +
957 offsetof(struct hifn_dma, result_bufs[i][0]));
958
959 /*
960 * Setup LAST descriptors.
961 */
962 dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr +
963 offsetof(struct hifn_dma, cmdr[0]));
964 dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr +
965 offsetof(struct hifn_dma, srcr[0]));
966 dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr +
967 offsetof(struct hifn_dma, dstr[0]));
968 dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr +
969 offsetof(struct hifn_dma, resr[0]));
970
971 dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0;
972 dma->cmdi = dma->srci = dma->dsti = dma->resi = 0;
973 dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
974 }
975
976 /*
977 * Initialize the PLL. We need to know the frequency of the reference clock
978 * to calculate the optimal multiplier. For PCI we assume 66MHz, since that
979 * allows us to operate without the risk of overclocking the chip. If it
980 * actually uses 33MHz, the chip will operate at half the speed, this can be
981 * overriden by specifying the frequency as module parameter (pci33).
982 *
983 * Unfortunately the PCI clock is not very suitable since the HIFN needs a
984 * stable clock and the PCI clock frequency may vary, so the default is the
985 * external clock. There is no way to find out its frequency, we default to
986 * 66MHz since according to Mike Ham of HiFn, almost every board in existence
987 * has an external crystal populated at 66MHz.
988 */
989 static void hifn_init_pll(struct hifn_device *dev)
990 {
991 unsigned int freq, m;
992 u32 pllcfg;
993
994 pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1;
995
996 if (strncmp(hifn_pll_ref, "ext", 3) == 0)
997 pllcfg |= HIFN_PLL_REF_CLK_PLL;
998 else
999 pllcfg |= HIFN_PLL_REF_CLK_HBI;
1000
1001 if (hifn_pll_ref[3] != '\0')
1002 freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
1003 else {
1004 freq = 66;
1005 printk(KERN_INFO "hifn795x: assuming %uMHz clock speed, "
1006 "override with hifn_pll_ref=%.3s<frequency>\n",
1007 freq, hifn_pll_ref);
1008 }
1009
1010 m = HIFN_PLL_FCK_MAX / freq;
1011
1012 pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT;
1013 if (m <= 8)
1014 pllcfg |= HIFN_PLL_IS_1_8;
1015 else
1016 pllcfg |= HIFN_PLL_IS_9_12;
1017
1018 /* Select clock source and enable clock bypass */
1019 hifn_write_1(dev, HIFN_1_PLL, pllcfg |
1020 HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP);
1021
1022 /* Let the chip lock to the input clock */
1023 mdelay(10);
1024
1025 /* Disable clock bypass */
1026 hifn_write_1(dev, HIFN_1_PLL, pllcfg |
1027 HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI);
1028
1029 /* Switch the engines to the PLL */
1030 hifn_write_1(dev, HIFN_1_PLL, pllcfg |
1031 HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL);
1032
1033 /*
1034 * The Fpk_clk runs at half the total speed. Its frequency is needed to
1035 * calculate the minimum time between two reads of the rng. Since 33MHz
1036 * is actually 33.333... we overestimate the frequency here, resulting
1037 * in slightly larger intervals.
1038 */
1039 dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2;
1040 }
1041
1042 static void hifn_init_registers(struct hifn_device *dev)
1043 {
1044 u32 dptr = dev->desc_dma;
1045
1046 /* Initialization magic... */
1047 hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
1048 hifn_write_0(dev, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
1049 hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
1050
1051 /* write all 4 ring address registers */
1052 hifn_write_1(dev, HIFN_1_DMA_CRAR, __cpu_to_le32(dptr +
1053 offsetof(struct hifn_dma, cmdr[0])));
1054 hifn_write_1(dev, HIFN_1_DMA_SRAR, __cpu_to_le32(dptr +
1055 offsetof(struct hifn_dma, srcr[0])));
1056 hifn_write_1(dev, HIFN_1_DMA_DRAR, __cpu_to_le32(dptr +
1057 offsetof(struct hifn_dma, dstr[0])));
1058 hifn_write_1(dev, HIFN_1_DMA_RRAR, __cpu_to_le32(dptr +
1059 offsetof(struct hifn_dma, resr[0])));
1060
1061 mdelay(2);
1062 #if 0
1063 hifn_write_1(dev, HIFN_1_DMA_CSR,
1064 HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
1065 HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
1066 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
1067 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
1068 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
1069 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
1070 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
1071 HIFN_DMACSR_S_WAIT |
1072 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1073 HIFN_DMACSR_C_WAIT |
1074 HIFN_DMACSR_ENGINE |
1075 HIFN_DMACSR_PUBDONE);
1076 #else
1077 hifn_write_1(dev, HIFN_1_DMA_CSR,
1078 HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1079 HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA |
1080 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
1081 HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
1082 HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
1083 HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
1084 HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
1085 HIFN_DMACSR_S_WAIT |
1086 HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1087 HIFN_DMACSR_C_WAIT |
1088 HIFN_DMACSR_ENGINE |
1089 HIFN_DMACSR_PUBDONE);
1090 #endif
1091 hifn_read_1(dev, HIFN_1_DMA_CSR);
1092
1093 dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
1094 HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
1095 HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
1096 HIFN_DMAIER_ENGINE;
1097 dev->dmareg &= ~HIFN_DMAIER_C_WAIT;
1098
1099 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
1100 hifn_read_1(dev, HIFN_1_DMA_IER);
1101 #if 0
1102 hifn_write_0(dev, HIFN_0_PUCNFG, HIFN_PUCNFG_ENCCNFG |
1103 HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
1104 HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
1105 HIFN_PUCNFG_DRAM);
1106 #else
1107 hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342);
1108 #endif
1109 hifn_init_pll(dev);
1110
1111 hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1112 hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
1113 HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
1114 ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
1115 ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
1116 }
1117
1118 static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf,
1119 unsigned dlen, unsigned slen, u16 mask, u8 snum)
1120 {
1121 struct hifn_base_command *base_cmd;
1122 u8 *buf_pos = buf;
1123
1124 base_cmd = (struct hifn_base_command *)buf_pos;
1125 base_cmd->masks = __cpu_to_le16(mask);
1126 base_cmd->total_source_count =
1127 __cpu_to_le16(slen & HIFN_BASE_CMD_LENMASK_LO);
1128 base_cmd->total_dest_count =
1129 __cpu_to_le16(dlen & HIFN_BASE_CMD_LENMASK_LO);
1130
1131 dlen >>= 16;
1132 slen >>= 16;
1133 base_cmd->session_num = __cpu_to_le16(snum |
1134 ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
1135 ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
1136
1137 return sizeof(struct hifn_base_command);
1138 }
1139
1140 static int hifn_setup_crypto_command(struct hifn_device *dev,
1141 u8 *buf, unsigned dlen, unsigned slen,
1142 u8 *key, int keylen, u8 *iv, int ivsize, u16 mode)
1143 {
1144 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1145 struct hifn_crypt_command *cry_cmd;
1146 u8 *buf_pos = buf;
1147 u16 cmd_len;
1148
1149 cry_cmd = (struct hifn_crypt_command *)buf_pos;
1150
1151 cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff);
1152 dlen >>= 16;
1153 cry_cmd->masks = __cpu_to_le16(mode |
1154 ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) &
1155 HIFN_CRYPT_CMD_SRCLEN_M));
1156 cry_cmd->header_skip = 0;
1157 cry_cmd->reserved = 0;
1158
1159 buf_pos += sizeof(struct hifn_crypt_command);
1160
1161 dma->cmdu++;
1162 if (dma->cmdu > 1) {
1163 dev->dmareg |= HIFN_DMAIER_C_WAIT;
1164 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
1165 }
1166
1167 if (keylen) {
1168 memcpy(buf_pos, key, keylen);
1169 buf_pos += keylen;
1170 }
1171 if (ivsize) {
1172 memcpy(buf_pos, iv, ivsize);
1173 buf_pos += ivsize;
1174 }
1175
1176 cmd_len = buf_pos - buf;
1177
1178 return cmd_len;
1179 }
1180
1181 static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
1182 unsigned int offset, unsigned int size)
1183 {
1184 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1185 int idx;
1186 dma_addr_t addr;
1187
1188 addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE);
1189
1190 idx = dma->srci;
1191
1192 dma->srcr[idx].p = __cpu_to_le32(addr);
1193 dma->srcr[idx].l = __cpu_to_le32(size) | HIFN_D_VALID |
1194 HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST;
1195
1196 if (++idx == HIFN_D_SRC_RSIZE) {
1197 dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1198 HIFN_D_JUMP |
1199 HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
1200 idx = 0;
1201 }
1202
1203 dma->srci = idx;
1204 dma->srcu++;
1205
1206 if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {
1207 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
1208 dev->flags |= HIFN_FLAG_SRC_BUSY;
1209 }
1210
1211 return size;
1212 }
1213
1214 static void hifn_setup_res_desc(struct hifn_device *dev)
1215 {
1216 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1217
1218 dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
1219 HIFN_D_VALID | HIFN_D_LAST);
1220 /*
1221 * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |
1222 * HIFN_D_LAST | HIFN_D_NOINVALID);
1223 */
1224
1225 if (++dma->resi == HIFN_D_RES_RSIZE) {
1226 dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |
1227 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
1228 dma->resi = 0;
1229 }
1230
1231 dma->resu++;
1232
1233 if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {
1234 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
1235 dev->flags |= HIFN_FLAG_RES_BUSY;
1236 }
1237 }
1238
1239 static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
1240 unsigned offset, unsigned size)
1241 {
1242 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1243 int idx;
1244 dma_addr_t addr;
1245
1246 addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE);
1247
1248 idx = dma->dsti;
1249 dma->dstr[idx].p = __cpu_to_le32(addr);
1250 dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
1251 HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST);
1252
1253 if (++idx == HIFN_D_DST_RSIZE) {
1254 dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |
1255 HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
1256 HIFN_D_LAST | HIFN_D_NOINVALID);
1257 idx = 0;
1258 }
1259 dma->dsti = idx;
1260 dma->dstu++;
1261
1262 if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {
1263 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
1264 dev->flags |= HIFN_FLAG_DST_BUSY;
1265 }
1266 }
1267
1268 static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned int soff,
1269 struct page *dpage, unsigned int doff, unsigned int nbytes, void *priv,
1270 struct hifn_context *ctx)
1271 {
1272 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1273 int cmd_len, sa_idx;
1274 u8 *buf, *buf_pos;
1275 u16 mask;
1276
1277 dprintk("%s: spage: %p, soffset: %u, dpage: %p, doffset: %u, nbytes: %u, priv: %p, ctx: %p.\n",
1278 dev->name, spage, soff, dpage, doff, nbytes, priv, ctx);
1279
1280 sa_idx = dma->resi;
1281
1282 hifn_setup_src_desc(dev, spage, soff, nbytes);
1283
1284 buf_pos = buf = dma->command_bufs[dma->cmdi];
1285
1286 mask = 0;
1287 switch (ctx->op) {
1288 case ACRYPTO_OP_DECRYPT:
1289 mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE;
1290 break;
1291 case ACRYPTO_OP_ENCRYPT:
1292 mask = HIFN_BASE_CMD_CRYPT;
1293 break;
1294 case ACRYPTO_OP_HMAC:
1295 mask = HIFN_BASE_CMD_MAC;
1296 break;
1297 default:
1298 goto err_out;
1299 }
1300
1301 buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes,
1302 nbytes, mask, dev->snum);
1303
1304 if (ctx->op == ACRYPTO_OP_ENCRYPT || ctx->op == ACRYPTO_OP_DECRYPT) {
1305 u16 md = 0;
1306
1307 if (ctx->keysize)
1308 md |= HIFN_CRYPT_CMD_NEW_KEY;
1309 if (ctx->iv && ctx->mode != ACRYPTO_MODE_ECB)
1310 md |= HIFN_CRYPT_CMD_NEW_IV;
1311
1312 switch (ctx->mode) {
1313 case ACRYPTO_MODE_ECB:
1314 md |= HIFN_CRYPT_CMD_MODE_ECB;
1315 break;
1316 case ACRYPTO_MODE_CBC:
1317 md |= HIFN_CRYPT_CMD_MODE_CBC;
1318 break;
1319 case ACRYPTO_MODE_CFB:
1320 md |= HIFN_CRYPT_CMD_MODE_CFB;
1321 break;
1322 case ACRYPTO_MODE_OFB:
1323 md |= HIFN_CRYPT_CMD_MODE_OFB;
1324 break;
1325 default:
1326 goto err_out;
1327 }
1328
1329 switch (ctx->type) {
1330 case ACRYPTO_TYPE_AES_128:
1331 if (ctx->keysize != 16)
1332 goto err_out;
1333 md |= HIFN_CRYPT_CMD_KSZ_128 |
1334 HIFN_CRYPT_CMD_ALG_AES;
1335 break;
1336 case ACRYPTO_TYPE_AES_192:
1337 if (ctx->keysize != 24)
1338 goto err_out;
1339 md |= HIFN_CRYPT_CMD_KSZ_192 |
1340 HIFN_CRYPT_CMD_ALG_AES;
1341 break;
1342 case ACRYPTO_TYPE_AES_256:
1343 if (ctx->keysize != 32)
1344 goto err_out;
1345 md |= HIFN_CRYPT_CMD_KSZ_256 |
1346 HIFN_CRYPT_CMD_ALG_AES;
1347 break;
1348 case ACRYPTO_TYPE_3DES:
1349 if (ctx->keysize != 24)
1350 goto err_out;
1351 md |= HIFN_CRYPT_CMD_ALG_3DES;
1352 break;
1353 case ACRYPTO_TYPE_DES:
1354 if (ctx->keysize != 8)
1355 goto err_out;
1356 md |= HIFN_CRYPT_CMD_ALG_DES;
1357 break;
1358 default:
1359 goto err_out;
1360 }
1361
1362 buf_pos += hifn_setup_crypto_command(dev, buf_pos,
1363 nbytes, nbytes, ctx->key, ctx->keysize,
1364 ctx->iv, ctx->ivsize, md);
1365 }
1366
1367 dev->sa[sa_idx] = priv;
1368
1369 cmd_len = buf_pos - buf;
1370 dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID |
1371 HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
1372
1373 if (++dma->cmdi == HIFN_D_CMD_RSIZE) {
1374 dma->cmdr[dma->cmdi].l = __cpu_to_le32(HIFN_MAX_COMMAND |
1375 HIFN_D_VALID | HIFN_D_LAST |
1376 HIFN_D_MASKDONEIRQ | HIFN_D_JUMP);
1377 dma->cmdi = 0;
1378 } else
1379 dma->cmdr[dma->cmdi-1].l |= __cpu_to_le32(HIFN_D_VALID);
1380
1381 if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) {
1382 hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
1383 dev->flags |= HIFN_FLAG_CMD_BUSY;
1384 }
1385
1386 hifn_setup_dst_desc(dev, dpage, doff, nbytes);
1387 hifn_setup_res_desc(dev);
1388
1389 return 0;
1390
1391 err_out:
1392 return -EINVAL;
1393 }
1394
1395 static int ablkcipher_walk_init(struct ablkcipher_walk *w,
1396 int num, gfp_t gfp_flags)
1397 {
1398 int i;
1399
1400 num = min(ASYNC_SCATTERLIST_CACHE, num);
1401 sg_init_table(w->cache, num);
1402
1403 w->num = 0;
1404 for (i=0; i<num; ++i) {
1405 struct page *page = alloc_page(gfp_flags);
1406 struct scatterlist *s;
1407
1408 if (!page)
1409 break;
1410
1411 s = &w->cache[i];
1412
1413 sg_set_page(s, page, PAGE_SIZE, 0);
1414 w->num++;
1415 }
1416
1417 return i;
1418 }
1419
1420 static void ablkcipher_walk_exit(struct ablkcipher_walk *w)
1421 {
1422 int i;
1423
1424 for (i=0; i<w->num; ++i) {
1425 struct scatterlist *s = &w->cache[i];
1426
1427 __free_page(sg_page(s));
1428
1429 s->length = 0;
1430 }
1431
1432 w->num = 0;
1433 }
1434
1435 static int ablkcipher_add(void *daddr, unsigned int *drestp, struct scatterlist *src,
1436 unsigned int size, unsigned int *nbytesp)
1437 {
1438 unsigned int copy, drest = *drestp, nbytes = *nbytesp;
1439 int idx = 0;
1440 void *saddr;
1441
1442 if (drest < size || size > nbytes)
1443 return -EINVAL;
1444
1445 while (size) {
1446 copy = min(drest, src->length);
1447
1448 saddr = kmap_atomic(sg_page(src), KM_SOFTIRQ1);
1449 memcpy(daddr, saddr + src->offset, copy);
1450 kunmap_atomic(saddr, KM_SOFTIRQ1);
1451
1452 size -= copy;
1453 drest -= copy;
1454 nbytes -= copy;
1455 daddr += copy;
1456
1457 dprintk("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n",
1458 __func__, copy, size, drest, nbytes);
1459
1460 src++;
1461 idx++;
1462 }
1463
1464 *nbytesp = nbytes;
1465 *drestp = drest;
1466
1467 return idx;
1468 }
1469
1470 static int ablkcipher_walk(struct ablkcipher_request *req,
1471 struct ablkcipher_walk *w)
1472 {
1473 unsigned blocksize =
1474 crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req));
1475 unsigned alignmask =
1476 crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req));
1477 struct scatterlist *src, *dst, *t;
1478 void *daddr;
1479 unsigned int nbytes = req->nbytes, offset, copy, diff;
1480 int idx, tidx, err;
1481
1482 tidx = idx = 0;
1483 offset = 0;
1484 while (nbytes) {
1485 if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED))
1486 return -EINVAL;
1487
1488 src = &req->src[idx];
1489 dst = &req->dst[idx];
1490
1491 dprintk("\n%s: slen: %u, dlen: %u, soff: %u, doff: %u, offset: %u, "
1492 "blocksize: %u, nbytes: %u.\n",
1493 __func__, src->length, dst->length, src->offset,
1494 dst->offset, offset, blocksize, nbytes);
1495
1496 if (src->length & (blocksize - 1) ||
1497 src->offset & (alignmask - 1) ||
1498 dst->length & (blocksize - 1) ||
1499 dst->offset & (alignmask - 1) ||
1500 offset) {
1501 unsigned slen = src->length - offset;
1502 unsigned dlen = PAGE_SIZE;
1503
1504 t = &w->cache[idx];
1505
1506 daddr = kmap_atomic(sg_page(t), KM_SOFTIRQ0);
1507 err = ablkcipher_add(daddr, &dlen, src, slen, &nbytes);
1508 if (err < 0)
1509 goto err_out_unmap;
1510
1511 idx += err;
1512
1513 copy = slen & ~(blocksize - 1);
1514 diff = slen & (blocksize - 1);
1515
1516 if (dlen < nbytes) {
1517 /*
1518 * Destination page does not have enough space
1519 * to put there additional blocksized chunk,
1520 * so we mark that page as containing only
1521 * blocksize aligned chunks:
1522 * t->length = (slen & ~(blocksize - 1));
1523 * and increase number of bytes to be processed
1524 * in next chunk:
1525 * nbytes += diff;
1526 */
1527 nbytes += diff;
1528
1529 /*
1530 * Temporary of course...
1531 * Kick author if you will catch this one.
1532 */
1533 printk(KERN_ERR "%s: dlen: %u, nbytes: %u,"
1534 "slen: %u, offset: %u.\n",
1535 __func__, dlen, nbytes, slen, offset);
1536 printk(KERN_ERR "%s: please contact author to fix this "
1537 "issue, generally you should not catch "
1538 "this path under any condition but who "
1539 "knows how did you use crypto code.\n"
1540 "Thank you.\n", __func__);
1541 BUG();
1542 } else {
1543 copy += diff + nbytes;
1544
1545 src = &req->src[idx];
1546
1547 err = ablkcipher_add(daddr + slen, &dlen, src, nbytes, &nbytes);
1548 if (err < 0)
1549 goto err_out_unmap;
1550
1551 idx += err;
1552 }
1553
1554 t->length = copy;
1555 t->offset = offset;
1556
1557 kunmap_atomic(daddr, KM_SOFTIRQ0);
1558 } else {
1559 nbytes -= src->length;
1560 idx++;
1561 }
1562
1563 tidx++;
1564 }
1565
1566 return tidx;
1567
1568 err_out_unmap:
1569 kunmap_atomic(daddr, KM_SOFTIRQ0);
1570 return err;
1571 }
1572
1573 static int hifn_setup_session(struct ablkcipher_request *req)
1574 {
1575 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1576 struct hifn_device *dev = ctx->dev;
1577 struct page *spage, *dpage;
1578 unsigned long soff, doff, flags;
1579 unsigned int nbytes = req->nbytes, idx = 0, len;
1580 int err = -EINVAL, sg_num;
1581 struct scatterlist *src, *dst, *t;
1582 unsigned blocksize =
1583 crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req));
1584 unsigned alignmask =
1585 crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req));
1586
1587 if (ctx->iv && !ctx->ivsize && ctx->mode != ACRYPTO_MODE_ECB)
1588 goto err_out_exit;
1589
1590 ctx->walk.flags = 0;
1591
1592 while (nbytes) {
1593 src = &req->src[idx];
1594 dst = &req->dst[idx];
1595
1596 if (src->length & (blocksize - 1) ||
1597 src->offset & (alignmask - 1) ||
1598 dst->length & (blocksize - 1) ||
1599 dst->offset & (alignmask - 1)) {
1600 ctx->walk.flags |= ASYNC_FLAGS_MISALIGNED;
1601 }
1602
1603 nbytes -= src->length;
1604 idx++;
1605 }
1606
1607 if (ctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1608 err = ablkcipher_walk_init(&ctx->walk, idx, GFP_ATOMIC);
1609 if (err < 0)
1610 return err;
1611 }
1612
1613 nbytes = req->nbytes;
1614 idx = 0;
1615
1616 sg_num = ablkcipher_walk(req, &ctx->walk);
1617
1618 atomic_set(&ctx->sg_num, sg_num);
1619
1620 spin_lock_irqsave(&dev->lock, flags);
1621 if (dev->started + sg_num > HIFN_QUEUE_LENGTH) {
1622 err = -EAGAIN;
1623 goto err_out;
1624 }
1625
1626 dev->snum++;
1627 dev->started += sg_num;
1628
1629 while (nbytes) {
1630 src = &req->src[idx];
1631 dst = &req->dst[idx];
1632 t = &ctx->walk.cache[idx];
1633
1634 if (t->length) {
1635 spage = dpage = sg_page(t);
1636 soff = doff = 0;
1637 len = t->length;
1638 } else {
1639 spage = sg_page(src);
1640 soff = src->offset;
1641
1642 dpage = sg_page(dst);
1643 doff = dst->offset;
1644
1645 len = dst->length;
1646 }
1647
1648 idx++;
1649
1650 err = hifn_setup_dma(dev, spage, soff, dpage, doff, nbytes,
1651 req, ctx);
1652 if (err)
1653 goto err_out;
1654
1655 nbytes -= len;
1656 }
1657
1658 dev->active = HIFN_DEFAULT_ACTIVE_NUM;
1659 spin_unlock_irqrestore(&dev->lock, flags);
1660
1661 return 0;
1662
1663 err_out:
1664 spin_unlock_irqrestore(&dev->lock, flags);
1665 err_out_exit:
1666 if (err && printk_ratelimit())
1667 dprintk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, "
1668 "type: %u, err: %d.\n",
1669 dev->name, ctx->iv, ctx->ivsize,
1670 ctx->key, ctx->keysize,
1671 ctx->mode, ctx->op, ctx->type, err);
1672
1673 return err;
1674 }
1675
1676 static int hifn_test(struct hifn_device *dev, int encdec, u8 snum)
1677 {
1678 int n, err;
1679 u8 src[16];
1680 struct hifn_context ctx;
1681 u8 fips_aes_ecb_from_zero[16] = {
1682 0x66, 0xE9, 0x4B, 0xD4,
1683 0xEF, 0x8A, 0x2C, 0x3B,
1684 0x88, 0x4C, 0xFA, 0x59,
1685 0xCA, 0x34, 0x2B, 0x2E};
1686
1687 memset(src, 0, sizeof(src));
1688 memset(ctx.key, 0, sizeof(ctx.key));
1689
1690 ctx.dev = dev;
1691 ctx.keysize = 16;
1692 ctx.ivsize = 0;
1693 ctx.iv = NULL;
1694 ctx.op = (encdec)?ACRYPTO_OP_ENCRYPT:ACRYPTO_OP_DECRYPT;
1695 ctx.mode = ACRYPTO_MODE_ECB;
1696 ctx.type = ACRYPTO_TYPE_AES_128;
1697 atomic_set(&ctx.sg_num, 1);
1698
1699 err = hifn_setup_dma(dev,
1700 virt_to_page(src), offset_in_page(src),
1701 virt_to_page(src), offset_in_page(src),
1702 sizeof(src), NULL, &ctx);
1703 if (err)
1704 goto err_out;
1705
1706 msleep(200);
1707
1708 dprintk("%s: decoded: ", dev->name);
1709 for (n=0; n<sizeof(src); ++n)
1710 dprintk("%02x ", src[n]);
1711 dprintk("\n");
1712 dprintk("%s: FIPS : ", dev->name);
1713 for (n=0; n<sizeof(fips_aes_ecb_from_zero); ++n)
1714 dprintk("%02x ", fips_aes_ecb_from_zero[n]);
1715 dprintk("\n");
1716
1717 if (!memcmp(src, fips_aes_ecb_from_zero, sizeof(fips_aes_ecb_from_zero))) {
1718 printk(KERN_INFO "%s: AES 128 ECB test has been successfully "
1719 "passed.\n", dev->name);
1720 return 0;
1721 }
1722
1723 err_out:
1724 printk(KERN_INFO "%s: AES 128 ECB test has been failed.\n", dev->name);
1725 return -1;
1726 }
1727
1728 static int hifn_start_device(struct hifn_device *dev)
1729 {
1730 int err;
1731
1732 hifn_reset_dma(dev, 1);
1733
1734 err = hifn_enable_crypto(dev);
1735 if (err)
1736 return err;
1737
1738 hifn_reset_puc(dev);
1739
1740 hifn_init_dma(dev);
1741
1742 hifn_init_registers(dev);
1743
1744 hifn_init_pubrng(dev);
1745
1746 return 0;
1747 }
1748
1749 static int ablkcipher_get(void *saddr, unsigned int *srestp, unsigned int offset,
1750 struct scatterlist *dst, unsigned int size, unsigned int *nbytesp)
1751 {
1752 unsigned int srest = *srestp, nbytes = *nbytesp, copy;
1753 void *daddr;
1754 int idx = 0;
1755
1756 if (srest < size || size > nbytes)
1757 return -EINVAL;
1758
1759 while (size) {
1760
1761 copy = min(dst->length, srest);
1762
1763 daddr = kmap_atomic(sg_page(dst), KM_IRQ0);
1764 memcpy(daddr + dst->offset + offset, saddr, copy);
1765 kunmap_atomic(daddr, KM_IRQ0);
1766
1767 nbytes -= copy;
1768 size -= copy;
1769 srest -= copy;
1770 saddr += copy;
1771 offset = 0;
1772
1773 dprintk("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n",
1774 __func__, copy, size, srest, nbytes);
1775
1776 dst++;
1777 idx++;
1778 }
1779
1780 *nbytesp = nbytes;
1781 *srestp = srest;
1782
1783 return idx;
1784 }
1785
1786 static void hifn_process_ready(struct ablkcipher_request *req, int error)
1787 {
1788 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
1789 struct hifn_device *dev;
1790
1791 dprintk("%s: req: %p, ctx: %p.\n", __func__, req, ctx);
1792
1793 dev = ctx->dev;
1794 dprintk("%s: req: %p, started: %d, sg_num: %d.\n",
1795 __func__, req, dev->started, atomic_read(&ctx->sg_num));
1796
1797 if (--dev->started < 0)
1798 BUG();
1799
1800 if (atomic_dec_and_test(&ctx->sg_num)) {
1801 unsigned int nbytes = req->nbytes;
1802 int idx = 0, err;
1803 struct scatterlist *dst, *t;
1804 void *saddr;
1805
1806 if (ctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
1807 while (nbytes) {
1808 t = &ctx->walk.cache[idx];
1809 dst = &req->dst[idx];
1810
1811 dprintk("\n%s: sg_page(t): %p, t->length: %u, "
1812 "sg_page(dst): %p, dst->length: %u, "
1813 "nbytes: %u.\n",
1814 __func__, sg_page(t), t->length,
1815 sg_page(dst), dst->length, nbytes);
1816
1817 if (!t->length) {
1818 nbytes -= dst->length;
1819 idx++;
1820 continue;
1821 }
1822
1823 saddr = kmap_atomic(sg_page(t), KM_IRQ1);
1824
1825 err = ablkcipher_get(saddr, &t->length, t->offset,
1826 dst, nbytes, &nbytes);
1827 if (err < 0) {
1828 kunmap_atomic(saddr, KM_IRQ1);
1829 break;
1830 }
1831
1832 idx += err;
1833 kunmap_atomic(saddr, KM_IRQ1);
1834 }
1835
1836 ablkcipher_walk_exit(&ctx->walk);
1837 }
1838
1839 req->base.complete(&req->base, error);
1840 }
1841 }
1842
1843 static void hifn_check_for_completion(struct hifn_device *dev, int error)
1844 {
1845 int i;
1846 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1847
1848 for (i=0; i<HIFN_D_RES_RSIZE; ++i) {
1849 struct hifn_desc *d = &dma->resr[i];
1850
1851 if (!(d->l & __cpu_to_le32(HIFN_D_VALID)) && dev->sa[i]) {
1852 dev->success++;
1853 dev->reset = 0;
1854 hifn_process_ready(dev->sa[i], error);
1855 dev->sa[i] = NULL;
1856 }
1857
1858 if (d->l & __cpu_to_le32(HIFN_D_DESTOVER | HIFN_D_OVER))
1859 if (printk_ratelimit())
1860 printk("%s: overflow detected [d: %u, o: %u] "
1861 "at %d resr: l: %08x, p: %08x.\n",
1862 dev->name,
1863 !!(d->l & __cpu_to_le32(HIFN_D_DESTOVER)),
1864 !!(d->l & __cpu_to_le32(HIFN_D_OVER)),
1865 i, d->l, d->p);
1866 }
1867 }
1868
1869 static void hifn_clear_rings(struct hifn_device *dev)
1870 {
1871 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1872 int i, u;
1873
1874 dprintk("%s: ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
1875 "k: %d.%d.%d.%d.\n",
1876 dev->name,
1877 dma->cmdi, dma->srci, dma->dsti, dma->resi,
1878 dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1879 dma->cmdk, dma->srck, dma->dstk, dma->resk);
1880
1881 i = dma->resk; u = dma->resu;
1882 while (u != 0) {
1883 if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID))
1884 break;
1885
1886 if (i != HIFN_D_RES_RSIZE)
1887 u--;
1888
1889 if (++i == (HIFN_D_RES_RSIZE + 1))
1890 i = 0;
1891 }
1892 dma->resk = i; dma->resu = u;
1893
1894 i = dma->srck; u = dma->srcu;
1895 while (u != 0) {
1896 if (i == HIFN_D_SRC_RSIZE)
1897 i = 0;
1898 if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID))
1899 break;
1900 i++, u--;
1901 }
1902 dma->srck = i; dma->srcu = u;
1903
1904 i = dma->cmdk; u = dma->cmdu;
1905 while (u != 0) {
1906 if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID))
1907 break;
1908 if (i != HIFN_D_CMD_RSIZE)
1909 u--;
1910 if (++i == (HIFN_D_CMD_RSIZE + 1))
1911 i = 0;
1912 }
1913 dma->cmdk = i; dma->cmdu = u;
1914
1915 i = dma->dstk; u = dma->dstu;
1916 while (u != 0) {
1917 if (i == HIFN_D_DST_RSIZE)
1918 i = 0;
1919 if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID))
1920 break;
1921 i++, u--;
1922 }
1923 dma->dstk = i; dma->dstu = u;
1924
1925 dprintk("%s: ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
1926 "k: %d.%d.%d.%d.\n",
1927 dev->name,
1928 dma->cmdi, dma->srci, dma->dsti, dma->resi,
1929 dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1930 dma->cmdk, dma->srck, dma->dstk, dma->resk);
1931 }
1932
1933 static void hifn_work(struct work_struct *work)
1934 {
1935 struct delayed_work *dw = container_of(work, struct delayed_work, work);
1936 struct hifn_device *dev = container_of(dw, struct hifn_device, work);
1937 unsigned long flags;
1938 int reset = 0;
1939 u32 r = 0;
1940
1941 spin_lock_irqsave(&dev->lock, flags);
1942 if (dev->active == 0) {
1943 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1944
1945 if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) {
1946 dev->flags &= ~HIFN_FLAG_CMD_BUSY;
1947 r |= HIFN_DMACSR_C_CTRL_DIS;
1948 }
1949 if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) {
1950 dev->flags &= ~HIFN_FLAG_SRC_BUSY;
1951 r |= HIFN_DMACSR_S_CTRL_DIS;
1952 }
1953 if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) {
1954 dev->flags &= ~HIFN_FLAG_DST_BUSY;
1955 r |= HIFN_DMACSR_D_CTRL_DIS;
1956 }
1957 if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) {
1958 dev->flags &= ~HIFN_FLAG_RES_BUSY;
1959 r |= HIFN_DMACSR_R_CTRL_DIS;
1960 }
1961 if (r)
1962 hifn_write_1(dev, HIFN_1_DMA_CSR, r);
1963 } else
1964 dev->active--;
1965
1966 if (dev->prev_success == dev->success && dev->started)
1967 reset = 1;
1968 dev->prev_success = dev->success;
1969 spin_unlock_irqrestore(&dev->lock, flags);
1970
1971 if (reset) {
1972 dprintk("%s: r: %08x, active: %d, started: %d, "
1973 "success: %lu: reset: %d.\n",
1974 dev->name, r, dev->active, dev->started,
1975 dev->success, reset);
1976
1977 if (++dev->reset >= 5) {
1978 dprintk("%s: really hard reset.\n", dev->name);
1979 hifn_reset_dma(dev, 1);
1980 hifn_stop_device(dev);
1981 hifn_start_device(dev);
1982 dev->reset = 0;
1983 }
1984
1985 spin_lock_irqsave(&dev->lock, flags);
1986 hifn_check_for_completion(dev, -EBUSY);
1987 hifn_clear_rings(dev);
1988 dev->started = 0;
1989 spin_unlock_irqrestore(&dev->lock, flags);
1990 }
1991
1992 schedule_delayed_work(&dev->work, HZ);
1993 }
1994
1995 static irqreturn_t hifn_interrupt(int irq, void *data)
1996 {
1997 struct hifn_device *dev = (struct hifn_device *)data;
1998 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
1999 u32 dmacsr, restart;
2000
2001 dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR);
2002
2003 dprintk("%s: 1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], "
2004 "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n",
2005 dev->name, dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi,
2006 dma->cmdu, dma->srcu, dma->dstu, dma->resu,
2007 dma->cmdi, dma->srci, dma->dsti, dma->resi);
2008
2009 if ((dmacsr & dev->dmareg) == 0)
2010 return IRQ_NONE;
2011
2012 hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg);
2013
2014 if (dmacsr & HIFN_DMACSR_ENGINE)
2015 hifn_write_0(dev, HIFN_0_PUISR, hifn_read_0(dev, HIFN_0_PUISR));
2016 if (dmacsr & HIFN_DMACSR_PUBDONE)
2017 hifn_write_1(dev, HIFN_1_PUB_STATUS,
2018 hifn_read_1(dev, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
2019
2020 restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER);
2021 if (restart) {
2022 u32 puisr = hifn_read_0(dev, HIFN_0_PUISR);
2023
2024 if (printk_ratelimit())
2025 printk("%s: overflow: r: %d, d: %d, puisr: %08x, d: %u.\n",
2026 dev->name, !!(dmacsr & HIFN_DMACSR_R_OVER),
2027 !!(dmacsr & HIFN_DMACSR_D_OVER),
2028 puisr, !!(puisr & HIFN_PUISR_DSTOVER));
2029 if (!!(puisr & HIFN_PUISR_DSTOVER))
2030 hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
2031 hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER |
2032 HIFN_DMACSR_D_OVER));
2033 }
2034
2035 restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
2036 HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
2037 if (restart) {
2038 if (printk_ratelimit())
2039 printk("%s: abort: c: %d, s: %d, d: %d, r: %d.\n",
2040 dev->name, !!(dmacsr & HIFN_DMACSR_C_ABORT),
2041 !!(dmacsr & HIFN_DMACSR_S_ABORT),
2042 !!(dmacsr & HIFN_DMACSR_D_ABORT),
2043 !!(dmacsr & HIFN_DMACSR_R_ABORT));
2044 hifn_reset_dma(dev, 1);
2045 hifn_init_dma(dev);
2046 hifn_init_registers(dev);
2047 }
2048
2049 if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) {
2050 dprintk("%s: wait on command.\n", dev->name);
2051 dev->dmareg &= ~(HIFN_DMAIER_C_WAIT);
2052 hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
2053 }
2054
2055 tasklet_schedule(&dev->tasklet);
2056 hifn_clear_rings(dev);
2057
2058 return IRQ_HANDLED;
2059 }
2060
2061 static void hifn_flush(struct hifn_device *dev)
2062 {
2063 unsigned long flags;
2064 struct crypto_async_request *async_req;
2065 struct hifn_context *ctx;
2066 struct ablkcipher_request *req;
2067 struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
2068 int i;
2069
2070 spin_lock_irqsave(&dev->lock, flags);
2071 for (i=0; i<HIFN_D_RES_RSIZE; ++i) {
2072 struct hifn_desc *d = &dma->resr[i];
2073
2074 if (dev->sa[i]) {
2075 hifn_process_ready(dev->sa[i],
2076 (d->l & __cpu_to_le32(HIFN_D_VALID))?-ENODEV:0);
2077 }
2078 }
2079
2080 while ((async_req = crypto_dequeue_request(&dev->queue))) {
2081 ctx = crypto_tfm_ctx(async_req->tfm);
2082 req = container_of(async_req, struct ablkcipher_request, base);
2083
2084 hifn_process_ready(req, -ENODEV);
2085 }
2086 spin_unlock_irqrestore(&dev->lock, flags);
2087 }
2088
2089 static int hifn_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
2090 unsigned int len)
2091 {
2092 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
2093 struct hifn_context *ctx = crypto_tfm_ctx(tfm);
2094 struct hifn_device *dev = ctx->dev;
2095
2096 if (len > HIFN_MAX_CRYPT_KEY_LENGTH) {
2097 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
2098 return -1;
2099 }
2100
2101 if (len == HIFN_DES_KEY_LENGTH) {
2102 u32 tmp[DES_EXPKEY_WORDS];
2103 int ret = des_ekey(tmp, key);
2104
2105 if (unlikely(ret == 0) && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
2106 tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
2107 return -EINVAL;
2108 }
2109 }
2110
2111 dev->flags &= ~HIFN_FLAG_OLD_KEY;
2112
2113 memcpy(ctx->key, key, len);
2114 ctx->keysize = len;
2115
2116 return 0;
2117 }
2118
2119 static int hifn_handle_req(struct ablkcipher_request *req)
2120 {
2121 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
2122 struct hifn_device *dev = ctx->dev;
2123 int err = -EAGAIN;
2124
2125 if (dev->started + DIV_ROUND_UP(req->nbytes, PAGE_SIZE) <= HIFN_QUEUE_LENGTH)
2126 err = hifn_setup_session(req);
2127
2128 if (err == -EAGAIN) {
2129 unsigned long flags;
2130
2131 spin_lock_irqsave(&dev->lock, flags);
2132 err = ablkcipher_enqueue_request(&dev->queue, req);
2133 spin_unlock_irqrestore(&dev->lock, flags);
2134 }
2135
2136 return err;
2137 }
2138
2139 static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op,
2140 u8 type, u8 mode)
2141 {
2142 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
2143 unsigned ivsize;
2144
2145 ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req));
2146
2147 if (req->info && mode != ACRYPTO_MODE_ECB) {
2148 if (type == ACRYPTO_TYPE_AES_128)
2149 ivsize = HIFN_AES_IV_LENGTH;
2150 else if (type == ACRYPTO_TYPE_DES)
2151 ivsize = HIFN_DES_KEY_LENGTH;
2152 else if (type == ACRYPTO_TYPE_3DES)
2153 ivsize = HIFN_3DES_KEY_LENGTH;
2154 }
2155
2156 if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) {
2157 if (ctx->keysize == 24)
2158 type = ACRYPTO_TYPE_AES_192;
2159 else if (ctx->keysize == 32)
2160 type = ACRYPTO_TYPE_AES_256;
2161 }
2162
2163 ctx->op = op;
2164 ctx->mode = mode;
2165 ctx->type = type;
2166 ctx->iv = req->info;
2167 ctx->ivsize = ivsize;
2168
2169 /*
2170 * HEAVY TODO: needs to kick Herbert XU to write documentation.
2171 * HEAVY TODO: needs to kick Herbert XU to write documentation.
2172 * HEAVY TODO: needs to kick Herbert XU to write documentation.
2173 */
2174
2175 return hifn_handle_req(req);
2176 }
2177
2178 static int hifn_process_queue(struct hifn_device *dev)
2179 {
2180 struct crypto_async_request *async_req;
2181 struct hifn_context *ctx;
2182 struct ablkcipher_request *req;
2183 unsigned long flags;
2184 int err = 0;
2185
2186 while (dev->started < HIFN_QUEUE_LENGTH) {
2187 spin_lock_irqsave(&dev->lock, flags);
2188 async_req = crypto_dequeue_request(&dev->queue);
2189 spin_unlock_irqrestore(&dev->lock, flags);
2190
2191 if (!async_req)
2192 break;
2193
2194 ctx = crypto_tfm_ctx(async_req->tfm);
2195 req = container_of(async_req, struct ablkcipher_request, base);
2196
2197 err = hifn_handle_req(req);
2198 if (err)
2199 break;
2200 }
2201
2202 return err;
2203 }
2204
2205 static int hifn_setup_crypto(struct ablkcipher_request *req, u8 op,
2206 u8 type, u8 mode)
2207 {
2208 int err;
2209 struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
2210 struct hifn_device *dev = ctx->dev;
2211
2212 err = hifn_setup_crypto_req(req, op, type, mode);
2213 if (err)
2214 return err;
2215
2216 if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)
2217 err = hifn_process_queue(dev);
2218
2219 return err;
2220 }
2221
2222 /*
2223 * AES ecryption functions.
2224 */
2225 static inline int hifn_encrypt_aes_ecb(struct ablkcipher_request *req)
2226 {
2227 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2228 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
2229 }
2230 static inline int hifn_encrypt_aes_cbc(struct ablkcipher_request *req)
2231 {
2232 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2233 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
2234 }
2235 static inline int hifn_encrypt_aes_cfb(struct ablkcipher_request *req)
2236 {
2237 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2238 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB);
2239 }
2240 static inline int hifn_encrypt_aes_ofb(struct ablkcipher_request *req)
2241 {
2242 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2243 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB);
2244 }
2245
2246 /*
2247 * AES decryption functions.
2248 */
2249 static inline int hifn_decrypt_aes_ecb(struct ablkcipher_request *req)
2250 {
2251 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2252 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
2253 }
2254 static inline int hifn_decrypt_aes_cbc(struct ablkcipher_request *req)
2255 {
2256 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2257 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
2258 }
2259 static inline int hifn_decrypt_aes_cfb(struct ablkcipher_request *req)
2260 {
2261 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2262 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB);
2263 }
2264 static inline int hifn_decrypt_aes_ofb(struct ablkcipher_request *req)
2265 {
2266 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2267 ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB);
2268 }
2269
2270 /*
2271 * DES ecryption functions.
2272 */
2273 static inline int hifn_encrypt_des_ecb(struct ablkcipher_request *req)
2274 {
2275 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2276 ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
2277 }
2278 static inline int hifn_encrypt_des_cbc(struct ablkcipher_request *req)
2279 {
2280 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2281 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
2282 }
2283 static inline int hifn_encrypt_des_cfb(struct ablkcipher_request *req)
2284 {
2285 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2286 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB);
2287 }
2288 static inline int hifn_encrypt_des_ofb(struct ablkcipher_request *req)
2289 {
2290 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2291 ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB);
2292 }
2293
2294 /*
2295 * DES decryption functions.
2296 */
2297 static inline int hifn_decrypt_des_ecb(struct ablkcipher_request *req)
2298 {
2299 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2300 ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
2301 }
2302 static inline int hifn_decrypt_des_cbc(struct ablkcipher_request *req)
2303 {
2304 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2305 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
2306 }
2307 static inline int hifn_decrypt_des_cfb(struct ablkcipher_request *req)
2308 {
2309 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2310 ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB);
2311 }
2312 static inline int hifn_decrypt_des_ofb(struct ablkcipher_request *req)
2313 {
2314 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2315 ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB);
2316 }
2317
2318 /*
2319 * 3DES ecryption functions.
2320 */
2321 static inline int hifn_encrypt_3des_ecb(struct ablkcipher_request *req)
2322 {
2323 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2324 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
2325 }
2326 static inline int hifn_encrypt_3des_cbc(struct ablkcipher_request *req)
2327 {
2328 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2329 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
2330 }
2331 static inline int hifn_encrypt_3des_cfb(struct ablkcipher_request *req)
2332 {
2333 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2334 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB);
2335 }
2336 static inline int hifn_encrypt_3des_ofb(struct ablkcipher_request *req)
2337 {
2338 return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
2339 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB);
2340 }
2341
2342 /*
2343 * 3DES decryption functions.
2344 */
2345 static inline int hifn_decrypt_3des_ecb(struct ablkcipher_request *req)
2346 {
2347 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2348 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
2349 }
2350 static inline int hifn_decrypt_3des_cbc(struct ablkcipher_request *req)
2351 {
2352 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2353 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
2354 }
2355 static inline int hifn_decrypt_3des_cfb(struct ablkcipher_request *req)
2356 {
2357 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2358 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB);
2359 }
2360 static inline int hifn_decrypt_3des_ofb(struct ablkcipher_request *req)
2361 {
2362 return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
2363 ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB);
2364 }
2365
2366 struct hifn_alg_template
2367 {
2368 char name[CRYPTO_MAX_ALG_NAME];
2369 char drv_name[CRYPTO_MAX_ALG_NAME];
2370 unsigned int bsize;
2371 struct ablkcipher_alg ablkcipher;
2372 };
2373
2374 static struct hifn_alg_template hifn_alg_templates[] = {
2375 /*
2376 * 3DES ECB, CBC, CFB and OFB modes.
2377 */
2378 {
2379 .name = "cfb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
2380 .ablkcipher = {
2381 .min_keysize = HIFN_3DES_KEY_LENGTH,
2382 .max_keysize = HIFN_3DES_KEY_LENGTH,
2383 .setkey = hifn_setkey,
2384 .encrypt = hifn_encrypt_3des_cfb,
2385 .decrypt = hifn_decrypt_3des_cfb,
2386 },
2387 },
2388 {
2389 .name = "ofb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
2390 .ablkcipher = {
2391 .min_keysize = HIFN_3DES_KEY_LENGTH,
2392 .max_keysize = HIFN_3DES_KEY_LENGTH,
2393 .setkey = hifn_setkey,
2394 .encrypt = hifn_encrypt_3des_ofb,
2395 .decrypt = hifn_decrypt_3des_ofb,
2396 },
2397 },
2398 {
2399 .name = "cbc(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
2400 .ablkcipher = {
2401 .min_keysize = HIFN_3DES_KEY_LENGTH,
2402 .max_keysize = HIFN_3DES_KEY_LENGTH,
2403 .setkey = hifn_setkey,
2404 .encrypt = hifn_encrypt_3des_cbc,
2405 .decrypt = hifn_decrypt_3des_cbc,
2406 },
2407 },
2408 {
2409 .name = "ecb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
2410 .ablkcipher = {
2411 .min_keysize = HIFN_3DES_KEY_LENGTH,
2412 .max_keysize = HIFN_3DES_KEY_LENGTH,
2413 .setkey = hifn_setkey,
2414 .encrypt = hifn_encrypt_3des_ecb,
2415 .decrypt = hifn_decrypt_3des_ecb,
2416 },
2417 },
2418
2419 /*
2420 * DES ECB, CBC, CFB and OFB modes.
2421 */
2422 {
2423 .name = "cfb(des)", .drv_name = "hifn-des", .bsize = 8,
2424 .ablkcipher = {
2425 .min_keysize = HIFN_DES_KEY_LENGTH,
2426 .max_keysize = HIFN_DES_KEY_LENGTH,
2427 .setkey = hifn_setkey,
2428 .encrypt = hifn_encrypt_des_cfb,
2429 .decrypt = hifn_decrypt_des_cfb,
2430 },
2431 },
2432 {
2433 .name = "ofb(des)", .drv_name = "hifn-des", .bsize = 8,
2434 .ablkcipher = {
2435 .min_keysize = HIFN_DES_KEY_LENGTH,
2436 .max_keysize = HIFN_DES_KEY_LENGTH,
2437 .setkey = hifn_setkey,
2438 .encrypt = hifn_encrypt_des_ofb,
2439 .decrypt = hifn_decrypt_des_ofb,
2440 },
2441 },
2442 {
2443 .name = "cbc(des)", .drv_name = "hifn-des", .bsize = 8,
2444 .ablkcipher = {
2445 .min_keysize = HIFN_DES_KEY_LENGTH,
2446 .max_keysize = HIFN_DES_KEY_LENGTH,
2447 .setkey = hifn_setkey,
2448 .encrypt = hifn_encrypt_des_cbc,
2449 .decrypt = hifn_decrypt_des_cbc,
2450 },
2451 },
2452 {
2453 .name = "ecb(des)", .drv_name = "hifn-des", .bsize = 8,
2454 .ablkcipher = {
2455 .min_keysize = HIFN_DES_KEY_LENGTH,
2456 .max_keysize = HIFN_DES_KEY_LENGTH,
2457 .setkey = hifn_setkey,
2458 .encrypt = hifn_encrypt_des_ecb,
2459 .decrypt = hifn_decrypt_des_ecb,
2460 },
2461 },
2462
2463 /*
2464 * AES ECB, CBC, CFB and OFB modes.
2465 */
2466 {
2467 .name = "ecb(aes)", .drv_name = "hifn-aes", .bsize = 16,
2468 .ablkcipher = {
2469 .min_keysize = AES_MIN_KEY_SIZE,
2470 .max_keysize = AES_MAX_KEY_SIZE,
2471 .setkey = hifn_setkey,
2472 .encrypt = hifn_encrypt_aes_ecb,
2473 .decrypt = hifn_decrypt_aes_ecb,
2474 },
2475 },
2476 {
2477 .name = "cbc(aes)", .drv_name = "hifn-aes", .bsize = 16,
2478 .ablkcipher = {
2479 .min_keysize = AES_MIN_KEY_SIZE,
2480 .max_keysize = AES_MAX_KEY_SIZE,
2481 .setkey = hifn_setkey,
2482 .encrypt = hifn_encrypt_aes_cbc,
2483 .decrypt = hifn_decrypt_aes_cbc,
2484 },
2485 },
2486 {
2487 .name = "cfb(aes)", .drv_name = "hifn-aes", .bsize = 16,
2488 .ablkcipher = {
2489 .min_keysize = AES_MIN_KEY_SIZE,
2490 .max_keysize = AES_MAX_KEY_SIZE,
2491 .setkey = hifn_setkey,
2492 .encrypt = hifn_encrypt_aes_cfb,
2493 .decrypt = hifn_decrypt_aes_cfb,
2494 },
2495 },
2496 {
2497 .name = "ofb(aes)", .drv_name = "hifn-aes", .bsize = 16,
2498 .ablkcipher = {
2499 .min_keysize = AES_MIN_KEY_SIZE,
2500 .max_keysize = AES_MAX_KEY_SIZE,
2501 .setkey = hifn_setkey,
2502 .encrypt = hifn_encrypt_aes_ofb,
2503 .decrypt = hifn_decrypt_aes_ofb,
2504 },
2505 },
2506 };
2507
2508 static int hifn_cra_init(struct crypto_tfm *tfm)
2509 {
2510 struct crypto_alg *alg = tfm->__crt_alg;
2511 struct hifn_crypto_alg *ha = crypto_alg_to_hifn(alg);
2512 struct hifn_context *ctx = crypto_tfm_ctx(tfm);
2513
2514 ctx->dev = ha->dev;
2515
2516 return 0;
2517 }
2518
2519 static int hifn_alg_alloc(struct hifn_device *dev, struct hifn_alg_template *t)
2520 {
2521 struct hifn_crypto_alg *alg;
2522 int err;
2523
2524 alg = kzalloc(sizeof(struct hifn_crypto_alg), GFP_KERNEL);
2525 if (!alg)
2526 return -ENOMEM;
2527
2528 snprintf(alg->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s", t->name);
2529 snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", t->drv_name);
2530
2531 alg->alg.cra_priority = 300;
2532 alg->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
2533 alg->alg.cra_blocksize = t->bsize;
2534 alg->alg.cra_ctxsize = sizeof(struct hifn_context);
2535 alg->alg.cra_alignmask = 15;
2536 if (t->bsize == 8)
2537 alg->alg.cra_alignmask = 3;
2538 alg->alg.cra_type = &crypto_ablkcipher_type;
2539 alg->alg.cra_module = THIS_MODULE;
2540 alg->alg.cra_u.ablkcipher = t->ablkcipher;
2541 alg->alg.cra_init = hifn_cra_init;
2542
2543 alg->dev = dev;
2544
2545 list_add_tail(&alg->entry, &dev->alg_list);
2546
2547 err = crypto_register_alg(&alg->alg);
2548 if (err) {
2549 list_del(&alg->entry);
2550 kfree(alg);
2551 }
2552
2553 return err;
2554 }
2555
2556 static void hifn_unregister_alg(struct hifn_device *dev)
2557 {
2558 struct hifn_crypto_alg *a, *n;
2559
2560 list_for_each_entry_safe(a, n, &dev->alg_list, entry) {
2561 list_del(&a->entry);
2562 crypto_unregister_alg(&a->alg);
2563 kfree(a);
2564 }
2565 }
2566
2567 static int hifn_register_alg(struct hifn_device *dev)
2568 {
2569 int i, err;
2570
2571 for (i=0; i<ARRAY_SIZE(hifn_alg_templates); ++i) {
2572 err = hifn_alg_alloc(dev, &hifn_alg_templates[i]);
2573 if (err)
2574 goto err_out_exit;
2575 }
2576
2577 return 0;
2578
2579 err_out_exit:
2580 hifn_unregister_alg(dev);
2581 return err;
2582 }
2583
2584 static void hifn_tasklet_callback(unsigned long data)
2585 {
2586 struct hifn_device *dev = (struct hifn_device *)data;
2587
2588 /*
2589 * This is ok to call this without lock being held,
2590 * althogh it modifies some parameters used in parallel,
2591 * (like dev->success), but they are used in process
2592 * context or update is atomic (like setting dev->sa[i] to NULL).
2593 */
2594 hifn_check_for_completion(dev, 0);
2595 }
2596
2597 static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2598 {
2599 int err, i;
2600 struct hifn_device *dev;
2601 char name[8];
2602
2603 err = pci_enable_device(pdev);
2604 if (err)
2605 return err;
2606 pci_set_master(pdev);
2607
2608 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
2609 if (err)
2610 goto err_out_disable_pci_device;
2611
2612 snprintf(name, sizeof(name), "hifn%d",
2613 atomic_inc_return(&hifn_dev_number)-1);
2614
2615 err = pci_request_regions(pdev, name);
2616 if (err)
2617 goto err_out_disable_pci_device;
2618
2619 if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE ||
2620 pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE ||
2621 pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) {
2622 dprintk("%s: Broken hardware - I/O regions are too small.\n",
2623 pci_name(pdev));
2624 err = -ENODEV;
2625 goto err_out_free_regions;
2626 }
2627
2628 dev = kzalloc(sizeof(struct hifn_device) + sizeof(struct crypto_alg),
2629 GFP_KERNEL);
2630 if (!dev) {
2631 err = -ENOMEM;
2632 goto err_out_free_regions;
2633 }
2634
2635 INIT_LIST_HEAD(&dev->alg_list);
2636
2637 snprintf(dev->name, sizeof(dev->name), "%s", name);
2638 spin_lock_init(&dev->lock);
2639
2640 for (i=0; i<3; ++i) {
2641 unsigned long addr, size;
2642
2643 addr = pci_resource_start(pdev, i);
2644 size = pci_resource_len(pdev, i);
2645
2646 dev->bar[i] = ioremap_nocache(addr, size);
2647 if (!dev->bar[i])
2648 goto err_out_unmap_bars;
2649 }
2650
2651 dev->result_mem = __get_free_pages(GFP_KERNEL, HIFN_MAX_RESULT_ORDER);
2652 if (!dev->result_mem) {
2653 dprintk("Failed to allocate %d pages for result_mem.\n",
2654 HIFN_MAX_RESULT_ORDER);
2655 goto err_out_unmap_bars;
2656 }
2657 memset((void *)dev->result_mem, 0, PAGE_SIZE*(1<<HIFN_MAX_RESULT_ORDER));
2658
2659 dev->dst = pci_map_single(pdev, (void *)dev->result_mem,
2660 PAGE_SIZE << HIFN_MAX_RESULT_ORDER, PCI_DMA_FROMDEVICE);
2661
2662 dev->desc_virt = pci_alloc_consistent(pdev, sizeof(struct hifn_dma),
2663 &dev->desc_dma);
2664 if (!dev->desc_virt) {
2665 dprintk("Failed to allocate descriptor rings.\n");
2666 goto err_out_free_result_pages;
2667 }
2668 memset(dev->desc_virt, 0, sizeof(struct hifn_dma));
2669
2670 dev->pdev = pdev;
2671 dev->irq = pdev->irq;
2672
2673 for (i=0; i<HIFN_D_RES_RSIZE; ++i)
2674 dev->sa[i] = NULL;
2675
2676 pci_set_drvdata(pdev, dev);
2677
2678 tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev);
2679
2680 crypto_init_queue(&dev->queue, 1);
2681
2682 err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev);
2683 if (err) {
2684 dprintk("Failed to request IRQ%d: err: %d.\n", dev->irq, err);
2685 dev->irq = 0;
2686 goto err_out_free_desc;
2687 }
2688
2689 err = hifn_start_device(dev);
2690 if (err)
2691 goto err_out_free_irq;
2692
2693 err = hifn_test(dev, 1, 0);
2694 if (err)
2695 goto err_out_stop_device;
2696
2697 err = hifn_register_rng(dev);
2698 if (err)
2699 goto err_out_stop_device;
2700
2701 err = hifn_register_alg(dev);
2702 if (err)
2703 goto err_out_unregister_rng;
2704
2705 INIT_DELAYED_WORK(&dev->work, hifn_work);
2706 schedule_delayed_work(&dev->work, HZ);
2707
2708 dprintk("HIFN crypto accelerator card at %s has been "
2709 "successfully registered as %s.\n",
2710 pci_name(pdev), dev->name);
2711
2712 return 0;
2713
2714 err_out_unregister_rng:
2715 hifn_unregister_rng(dev);
2716 err_out_stop_device:
2717 hifn_reset_dma(dev, 1);
2718 hifn_stop_device(dev);
2719 err_out_free_irq:
2720 free_irq(dev->irq, dev->name);
2721 tasklet_kill(&dev->tasklet);
2722 err_out_free_desc:
2723 pci_free_consistent(pdev, sizeof(struct hifn_dma),
2724 dev->desc_virt, dev->desc_dma);
2725
2726 err_out_free_result_pages:
2727 pci_unmap_single(pdev, dev->dst, PAGE_SIZE << HIFN_MAX_RESULT_ORDER,
2728 PCI_DMA_FROMDEVICE);
2729 free_pages(dev->result_mem, HIFN_MAX_RESULT_ORDER);
2730
2731 err_out_unmap_bars:
2732 for (i=0; i<3; ++i)
2733 if (dev->bar[i])
2734 iounmap(dev->bar[i]);
2735
2736 err_out_free_regions:
2737 pci_release_regions(pdev);
2738
2739 err_out_disable_pci_device:
2740 pci_disable_device(pdev);
2741
2742 return err;
2743 }
2744
2745 static void hifn_remove(struct pci_dev *pdev)
2746 {
2747 int i;
2748 struct hifn_device *dev;
2749
2750 dev = pci_get_drvdata(pdev);
2751
2752 if (dev) {
2753 cancel_delayed_work(&dev->work);
2754 flush_scheduled_work();
2755
2756 hifn_unregister_rng(dev);
2757 hifn_unregister_alg(dev);
2758 hifn_reset_dma(dev, 1);
2759 hifn_stop_device(dev);
2760
2761 free_irq(dev->irq, dev->name);
2762 tasklet_kill(&dev->tasklet);
2763
2764 hifn_flush(dev);
2765
2766 pci_free_consistent(pdev, sizeof(struct hifn_dma),
2767 dev->desc_virt, dev->desc_dma);
2768 pci_unmap_single(pdev, dev->dst,
2769 PAGE_SIZE << HIFN_MAX_RESULT_ORDER,
2770 PCI_DMA_FROMDEVICE);
2771 free_pages(dev->result_mem, HIFN_MAX_RESULT_ORDER);
2772 for (i=0; i<3; ++i)
2773 if (dev->bar[i])
2774 iounmap(dev->bar[i]);
2775
2776 kfree(dev);
2777 }
2778
2779 pci_release_regions(pdev);
2780 pci_disable_device(pdev);
2781 }
2782
2783 static struct pci_device_id hifn_pci_tbl[] = {
2784 { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7955) },
2785 { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7956) },
2786 { 0 }
2787 };
2788 MODULE_DEVICE_TABLE(pci, hifn_pci_tbl);
2789
2790 static struct pci_driver hifn_pci_driver = {
2791 .name = "hifn795x",
2792 .id_table = hifn_pci_tbl,
2793 .probe = hifn_probe,
2794 .remove = __devexit_p(hifn_remove),
2795 };
2796
2797 static int __devinit hifn_init(void)
2798 {
2799 unsigned int freq;
2800 int err;
2801
2802 if (strncmp(hifn_pll_ref, "ext", 3) &&
2803 strncmp(hifn_pll_ref, "pci", 3)) {
2804 printk(KERN_ERR "hifn795x: invalid hifn_pll_ref clock, "
2805 "must be pci or ext");
2806 return -EINVAL;
2807 }
2808
2809 /*
2810 * For the 7955/7956 the reference clock frequency must be in the
2811 * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz,
2812 * but this chip is currently not supported.
2813 */
2814 if (hifn_pll_ref[3] != '\0') {
2815 freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
2816 if (freq < 20 || freq > 100) {
2817 printk(KERN_ERR "hifn795x: invalid hifn_pll_ref "
2818 "frequency, must be in the range "
2819 "of 20-100");
2820 return -EINVAL;
2821 }
2822 }
2823
2824 err = pci_register_driver(&hifn_pci_driver);
2825 if (err < 0) {
2826 dprintk("Failed to register PCI driver for %s device.\n",
2827 hifn_pci_driver.name);
2828 return -ENODEV;
2829 }
2830
2831 printk(KERN_INFO "Driver for HIFN 795x crypto accelerator chip "
2832 "has been successfully registered.\n");
2833
2834 return 0;
2835 }
2836
2837 static void __devexit hifn_fini(void)
2838 {
2839 pci_unregister_driver(&hifn_pci_driver);
2840
2841 printk(KERN_INFO "Driver for HIFN 795x crypto accelerator chip "
2842 "has been successfully unregistered.\n");
2843 }
2844
2845 module_init(hifn_init);
2846 module_exit(hifn_fini);
2847
2848 MODULE_LICENSE("GPL");
2849 MODULE_AUTHOR("Evgeniy Polyakov <johnpol@2ka.mipt.ru>");
2850 MODULE_DESCRIPTION("Driver for HIFN 795x crypto accelerator chip.");
WRT350N Kernel Patches