?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
WIP FPC-III support
[linux/fpc-iii.git] / drivers / crypto / qat / qat_common / qat_uclo.c
blob1fb5fc852f6b86d361056ea20caf5fba142a861e
1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/slab.h>
4 #include <linux/ctype.h>
5 #include <linux/kernel.h>
6 #include <linux/delay.h>
7 #include <linux/pci_ids.h>
8 #include "adf_accel_devices.h"
9 #include "adf_common_drv.h"
10 #include "icp_qat_uclo.h"
11 #include "icp_qat_hal.h"
12 #include "icp_qat_fw_loader_handle.h"
13
14 #define UWORD_CPYBUF_SIZE 1024
15 #define INVLD_UWORD 0xffffffffffull
16 #define PID_MINOR_REV 0xf
17 #define PID_MAJOR_REV (0xf << 4)
18
19 static int qat_uclo_init_ae_data(struct icp_qat_uclo_objhandle *obj_handle,
20 unsigned int ae, unsigned int image_num)
21 {
22 struct icp_qat_uclo_aedata *ae_data;
23 struct icp_qat_uclo_encapme *encap_image;
24 struct icp_qat_uclo_page *page = NULL;
25 struct icp_qat_uclo_aeslice *ae_slice = NULL;
26
27 ae_data = &obj_handle->ae_data[ae];
28 encap_image = &obj_handle->ae_uimage[image_num];
29 ae_slice = &ae_data->ae_slices[ae_data->slice_num];
30 ae_slice->encap_image = encap_image;
31
32 if (encap_image->img_ptr) {
33 ae_slice->ctx_mask_assigned =
34 encap_image->img_ptr->ctx_assigned;
35 ae_data->eff_ustore_size = obj_handle->ustore_phy_size;
36 } else {
37 ae_slice->ctx_mask_assigned = 0;
38 }
39 ae_slice->region = kzalloc(sizeof(*ae_slice->region), GFP_KERNEL);
40 if (!ae_slice->region)
41 return -ENOMEM;
42 ae_slice->page = kzalloc(sizeof(*ae_slice->page), GFP_KERNEL);
43 if (!ae_slice->page)
44 goto out_err;
45 page = ae_slice->page;
46 page->encap_page = encap_image->page;
47 ae_slice->page->region = ae_slice->region;
48 ae_data->slice_num++;
49 return 0;
50 out_err:
51 kfree(ae_slice->region);
52 ae_slice->region = NULL;
53 return -ENOMEM;
54 }
55
56 static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data)
57 {
58 unsigned int i;
59
60 if (!ae_data) {
61 pr_err("QAT: bad argument, ae_data is NULL\n ");
62 return -EINVAL;
63 }
64
65 for (i = 0; i < ae_data->slice_num; i++) {
66 kfree(ae_data->ae_slices[i].region);
67 ae_data->ae_slices[i].region = NULL;
68 kfree(ae_data->ae_slices[i].page);
69 ae_data->ae_slices[i].page = NULL;
70 }
71 return 0;
72 }
73
74 static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table,
75 unsigned int str_offset)
76 {
77 if (!str_table->table_len || str_offset > str_table->table_len)
78 return NULL;
79 return (char *)(((uintptr_t)(str_table->strings)) + str_offset);
80 }
81
82 static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr)
83 {
84 int maj = hdr->maj_ver & 0xff;
85 int min = hdr->min_ver & 0xff;
86
87 if (hdr->file_id != ICP_QAT_UOF_FID) {
88 pr_err("QAT: Invalid header 0x%x\n", hdr->file_id);
89 return -EINVAL;
90 }
91 if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) {
92 pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n",
93 maj, min);
94 return -EINVAL;
95 }
96 return 0;
97 }
98
99 static int qat_uclo_check_suof_format(struct icp_qat_suof_filehdr *suof_hdr)
100 {
101 int maj = suof_hdr->maj_ver & 0xff;
102 int min = suof_hdr->min_ver & 0xff;
103
104 if (suof_hdr->file_id != ICP_QAT_SUOF_FID) {
105 pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id);
106 return -EINVAL;
107 }
108 if (suof_hdr->fw_type != 0) {
109 pr_err("QAT: unsupported firmware type\n");
110 return -EINVAL;
111 }
112 if (suof_hdr->num_chunks <= 0x1) {
113 pr_err("QAT: SUOF chunk amount is incorrect\n");
114 return -EINVAL;
115 }
116 if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) {
117 pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n",
118 maj, min);
119 return -EINVAL;
120 }
121 return 0;
122 }
123
124 static void qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle,
125 unsigned int addr, unsigned int *val,
126 unsigned int num_in_bytes)
127 {
128 unsigned int outval;
129 unsigned char *ptr = (unsigned char *)val;
130
131 while (num_in_bytes) {
132 memcpy(&outval, ptr, 4);
133 SRAM_WRITE(handle, addr, outval);
134 num_in_bytes -= 4;
135 ptr += 4;
136 addr += 4;
137 }
138 }
139
140 static void qat_uclo_wr_umem_by_words(struct icp_qat_fw_loader_handle *handle,
141 unsigned char ae, unsigned int addr,
142 unsigned int *val,
143 unsigned int num_in_bytes)
144 {
145 unsigned int outval;
146 unsigned char *ptr = (unsigned char *)val;
147
148 addr >>= 0x2; /* convert to uword address */
149
150 while (num_in_bytes) {
151 memcpy(&outval, ptr, 4);
152 qat_hal_wr_umem(handle, ae, addr++, 1, &outval);
153 num_in_bytes -= 4;
154 ptr += 4;
155 }
156 }
157
158 static void qat_uclo_batch_wr_umem(struct icp_qat_fw_loader_handle *handle,
159 unsigned char ae,
160 struct icp_qat_uof_batch_init
161 *umem_init_header)
162 {
163 struct icp_qat_uof_batch_init *umem_init;
164
165 if (!umem_init_header)
166 return;
167 umem_init = umem_init_header->next;
168 while (umem_init) {
169 unsigned int addr, *value, size;
170
171 ae = umem_init->ae;
172 addr = umem_init->addr;
173 value = umem_init->value;
174 size = umem_init->size;
175 qat_uclo_wr_umem_by_words(handle, ae, addr, value, size);
176 umem_init = umem_init->next;
177 }
178 }
179
180 static void
181 qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle,
182 struct icp_qat_uof_batch_init **base)
183 {
184 struct icp_qat_uof_batch_init *umem_init;
185
186 umem_init = *base;
187 while (umem_init) {
188 struct icp_qat_uof_batch_init *pre;
189
190 pre = umem_init;
191 umem_init = umem_init->next;
192 kfree(pre);
193 }
194 *base = NULL;
195 }
196
197 static int qat_uclo_parse_num(char *str, unsigned int *num)
198 {
199 char buf[16] = {0};
200 unsigned long ae = 0;
201 int i;
202
203 strncpy(buf, str, 15);
204 for (i = 0; i < 16; i++) {
205 if (!isdigit(buf[i])) {
206 buf[i] = '\0';
207 break;
208 }
209 }
210 if ((kstrtoul(buf, 10, &ae)))
211 return -EFAULT;
212
213 *num = (unsigned int)ae;
214 return 0;
215 }
216
217 static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle,
218 struct icp_qat_uof_initmem *init_mem,
219 unsigned int size_range, unsigned int *ae)
220 {
221 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
222 char *str;
223
224 if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) {
225 pr_err("QAT: initmem is out of range");
226 return -EINVAL;
227 }
228 if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) {
229 pr_err("QAT: Memory scope for init_mem error\n");
230 return -EINVAL;
231 }
232 str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name);
233 if (!str) {
234 pr_err("QAT: AE name assigned in UOF init table is NULL\n");
235 return -EINVAL;
236 }
237 if (qat_uclo_parse_num(str, ae)) {
238 pr_err("QAT: Parse num for AE number failed\n");
239 return -EINVAL;
240 }
241 if (*ae >= ICP_QAT_UCLO_MAX_AE) {
242 pr_err("QAT: ae %d out of range\n", *ae);
243 return -EINVAL;
244 }
245 return 0;
246 }
247
248 static int qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle
249 *handle, struct icp_qat_uof_initmem
250 *init_mem, unsigned int ae,
251 struct icp_qat_uof_batch_init
252 **init_tab_base)
253 {
254 struct icp_qat_uof_batch_init *init_header, *tail;
255 struct icp_qat_uof_batch_init *mem_init, *tail_old;
256 struct icp_qat_uof_memvar_attr *mem_val_attr;
257 unsigned int i, flag = 0;
258
259 mem_val_attr =
260 (struct icp_qat_uof_memvar_attr *)((uintptr_t)init_mem +
261 sizeof(struct icp_qat_uof_initmem));
262
263 init_header = *init_tab_base;
264 if (!init_header) {
265 init_header = kzalloc(sizeof(*init_header), GFP_KERNEL);
266 if (!init_header)
267 return -ENOMEM;
268 init_header->size = 1;
269 *init_tab_base = init_header;
270 flag = 1;
271 }
272 tail_old = init_header;
273 while (tail_old->next)
274 tail_old = tail_old->next;
275 tail = tail_old;
276 for (i = 0; i < init_mem->val_attr_num; i++) {
277 mem_init = kzalloc(sizeof(*mem_init), GFP_KERNEL);
278 if (!mem_init)
279 goto out_err;
280 mem_init->ae = ae;
281 mem_init->addr = init_mem->addr + mem_val_attr->offset_in_byte;
282 mem_init->value = &mem_val_attr->value;
283 mem_init->size = 4;
284 mem_init->next = NULL;
285 tail->next = mem_init;
286 tail = mem_init;
287 init_header->size += qat_hal_get_ins_num();
288 mem_val_attr++;
289 }
290 return 0;
291 out_err:
292 /* Do not free the list head unless we allocated it. */
293 tail_old = tail_old->next;
294 if (flag) {
295 kfree(*init_tab_base);
296 *init_tab_base = NULL;
297 }
298
299 while (tail_old) {
300 mem_init = tail_old->next;
301 kfree(tail_old);
302 tail_old = mem_init;
303 }
304 return -ENOMEM;
305 }
306
307 static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle,
308 struct icp_qat_uof_initmem *init_mem)
309 {
310 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
311 unsigned int ae;
312
313 if (qat_uclo_fetch_initmem_ae(handle, init_mem,
314 handle->chip_info->lm_size, &ae))
315 return -EINVAL;
316 if (qat_uclo_create_batch_init_list(handle, init_mem, ae,
317 &obj_handle->lm_init_tab[ae]))
318 return -EINVAL;
319 return 0;
320 }
321
322 static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle,
323 struct icp_qat_uof_initmem *init_mem)
324 {
325 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
326 unsigned int ae, ustore_size, uaddr, i;
327 struct icp_qat_uclo_aedata *aed;
328
329 ustore_size = obj_handle->ustore_phy_size;
330 if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae))
331 return -EINVAL;
332 if (qat_uclo_create_batch_init_list(handle, init_mem, ae,
333 &obj_handle->umem_init_tab[ae]))
334 return -EINVAL;
335 /* set the highest ustore address referenced */
336 uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2;
337 aed = &obj_handle->ae_data[ae];
338 for (i = 0; i < aed->slice_num; i++) {
339 if (aed->ae_slices[i].encap_image->uwords_num < uaddr)
340 aed->ae_slices[i].encap_image->uwords_num = uaddr;
341 }
342 return 0;
343 }
344
345 #define ICP_DH895XCC_PESRAM_BAR_SIZE 0x80000
346 static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle,
347 struct icp_qat_uof_initmem *init_mem)
348 {
349 switch (init_mem->region) {
350 case ICP_QAT_UOF_LMEM_REGION:
351 if (qat_uclo_init_lmem_seg(handle, init_mem))
352 return -EINVAL;
353 break;
354 case ICP_QAT_UOF_UMEM_REGION:
355 if (qat_uclo_init_umem_seg(handle, init_mem))
356 return -EINVAL;
357 break;
358 default:
359 pr_err("QAT: initmem region error. region type=0x%x\n",
360 init_mem->region);
361 return -EINVAL;
362 }
363 return 0;
364 }
365
366 static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle,
367 struct icp_qat_uclo_encapme *image)
368 {
369 unsigned int i;
370 struct icp_qat_uclo_encap_page *page;
371 struct icp_qat_uof_image *uof_image;
372 unsigned char ae;
373 unsigned int ustore_size;
374 unsigned int patt_pos;
375 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
376 unsigned long ae_mask = handle->hal_handle->ae_mask;
377 unsigned long cfg_ae_mask = handle->cfg_ae_mask;
378 u64 *fill_data;
379
380 uof_image = image->img_ptr;
381 fill_data = kcalloc(ICP_QAT_UCLO_MAX_USTORE, sizeof(u64),
382 GFP_KERNEL);
383 if (!fill_data)
384 return -ENOMEM;
385 for (i = 0; i < ICP_QAT_UCLO_MAX_USTORE; i++)
386 memcpy(&fill_data[i], &uof_image->fill_pattern,
387 sizeof(u64));
388 page = image->page;
389
390 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
391 if (!test_bit(ae, (unsigned long *)&uof_image->ae_assigned))
392 continue;
393
394 if (!test_bit(ae, &cfg_ae_mask))
395 continue;
396
397 ustore_size = obj_handle->ae_data[ae].eff_ustore_size;
398 patt_pos = page->beg_addr_p + page->micro_words_num;
399
400 qat_hal_wr_uwords(handle, (unsigned char)ae, 0,
401 page->beg_addr_p, &fill_data[0]);
402 qat_hal_wr_uwords(handle, (unsigned char)ae, patt_pos,
403 ustore_size - patt_pos + 1,
404 &fill_data[page->beg_addr_p]);
405 }
406 kfree(fill_data);
407 return 0;
408 }
409
410 static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle)
411 {
412 int i, ae;
413 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
414 struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem;
415 unsigned long ae_mask = handle->hal_handle->ae_mask;
416
417 for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) {
418 if (initmem->num_in_bytes) {
419 if (qat_uclo_init_ae_memory(handle, initmem))
420 return -EINVAL;
421 }
422 initmem = (struct icp_qat_uof_initmem *)((uintptr_t)(
423 (uintptr_t)initmem +
424 sizeof(struct icp_qat_uof_initmem)) +
425 (sizeof(struct icp_qat_uof_memvar_attr) *
426 initmem->val_attr_num));
427 }
428
429 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
430 if (qat_hal_batch_wr_lm(handle, ae,
431 obj_handle->lm_init_tab[ae])) {
432 pr_err("QAT: fail to batch init lmem for AE %d\n", ae);
433 return -EINVAL;
434 }
435 qat_uclo_cleanup_batch_init_list(handle,
436 &obj_handle->lm_init_tab[ae]);
437 qat_uclo_batch_wr_umem(handle, ae,
438 obj_handle->umem_init_tab[ae]);
439 qat_uclo_cleanup_batch_init_list(handle,
440 &obj_handle->
441 umem_init_tab[ae]);
442 }
443 return 0;
444 }
445
446 static void *qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr,
447 char *chunk_id, void *cur)
448 {
449 int i;
450 struct icp_qat_uof_chunkhdr *chunk_hdr =
451 (struct icp_qat_uof_chunkhdr *)
452 ((uintptr_t)obj_hdr + sizeof(struct icp_qat_uof_objhdr));
453
454 for (i = 0; i < obj_hdr->num_chunks; i++) {
455 if ((cur < (void *)&chunk_hdr[i]) &&
456 !strncmp(chunk_hdr[i].chunk_id, chunk_id,
457 ICP_QAT_UOF_OBJID_LEN)) {
458 return &chunk_hdr[i];
459 }
460 }
461 return NULL;
462 }
463
464 static unsigned int qat_uclo_calc_checksum(unsigned int reg, int ch)
465 {
466 int i;
467 unsigned int topbit = 1 << 0xF;
468 unsigned int inbyte = (unsigned int)((reg >> 0x18) ^ ch);
469
470 reg ^= inbyte << 0x8;
471 for (i = 0; i < 0x8; i++) {
472 if (reg & topbit)
473 reg = (reg << 1) ^ 0x1021;
474 else
475 reg <<= 1;
476 }
477 return reg & 0xFFFF;
478 }
479
480 static unsigned int qat_uclo_calc_str_checksum(char *ptr, int num)
481 {
482 unsigned int chksum = 0;
483
484 if (ptr)
485 while (num--)
486 chksum = qat_uclo_calc_checksum(chksum, *ptr++);
487 return chksum;
488 }
489
490 static struct icp_qat_uclo_objhdr *
491 qat_uclo_map_chunk(char *buf, struct icp_qat_uof_filehdr *file_hdr,
492 char *chunk_id)
493 {
494 struct icp_qat_uof_filechunkhdr *file_chunk;
495 struct icp_qat_uclo_objhdr *obj_hdr;
496 char *chunk;
497 int i;
498
499 file_chunk = (struct icp_qat_uof_filechunkhdr *)
500 (buf + sizeof(struct icp_qat_uof_filehdr));
501 for (i = 0; i < file_hdr->num_chunks; i++) {
502 if (!strncmp(file_chunk->chunk_id, chunk_id,
503 ICP_QAT_UOF_OBJID_LEN)) {
504 chunk = buf + file_chunk->offset;
505 if (file_chunk->checksum != qat_uclo_calc_str_checksum(
506 chunk, file_chunk->size))
507 break;
508 obj_hdr = kzalloc(sizeof(*obj_hdr), GFP_KERNEL);
509 if (!obj_hdr)
510 break;
511 obj_hdr->file_buff = chunk;
512 obj_hdr->checksum = file_chunk->checksum;
513 obj_hdr->size = file_chunk->size;
514 return obj_hdr;
515 }
516 file_chunk++;
517 }
518 return NULL;
519 }
520
521 static unsigned int
522 qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj,
523 struct icp_qat_uof_image *image)
524 {
525 struct icp_qat_uof_objtable *uc_var_tab, *imp_var_tab, *imp_expr_tab;
526 struct icp_qat_uof_objtable *neigh_reg_tab;
527 struct icp_qat_uof_code_page *code_page;
528
529 code_page = (struct icp_qat_uof_code_page *)
530 ((char *)image + sizeof(struct icp_qat_uof_image));
531 uc_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
532 code_page->uc_var_tab_offset);
533 imp_var_tab = (struct icp_qat_uof_objtable *)(encap_uof_obj->beg_uof +
534 code_page->imp_var_tab_offset);
535 imp_expr_tab = (struct icp_qat_uof_objtable *)
536 (encap_uof_obj->beg_uof +
537 code_page->imp_expr_tab_offset);
538 if (uc_var_tab->entry_num || imp_var_tab->entry_num ||
539 imp_expr_tab->entry_num) {
540 pr_err("QAT: UOF can't contain imported variable to be parsed\n");
541 return -EINVAL;
542 }
543 neigh_reg_tab = (struct icp_qat_uof_objtable *)
544 (encap_uof_obj->beg_uof +
545 code_page->neigh_reg_tab_offset);
546 if (neigh_reg_tab->entry_num) {
547 pr_err("QAT: UOF can't contain neighbor register table\n");
548 return -EINVAL;
549 }
550 if (image->numpages > 1) {
551 pr_err("QAT: UOF can't contain multiple pages\n");
552 return -EINVAL;
553 }
554 if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) {
555 pr_err("QAT: UOF can't use shared control store feature\n");
556 return -EFAULT;
557 }
558 if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) {
559 pr_err("QAT: UOF can't use reloadable feature\n");
560 return -EFAULT;
561 }
562 return 0;
563 }
564
565 static void qat_uclo_map_image_page(struct icp_qat_uof_encap_obj
566 *encap_uof_obj,
567 struct icp_qat_uof_image *img,
568 struct icp_qat_uclo_encap_page *page)
569 {
570 struct icp_qat_uof_code_page *code_page;
571 struct icp_qat_uof_code_area *code_area;
572 struct icp_qat_uof_objtable *uword_block_tab;
573 struct icp_qat_uof_uword_block *uwblock;
574 int i;
575
576 code_page = (struct icp_qat_uof_code_page *)
577 ((char *)img + sizeof(struct icp_qat_uof_image));
578 page->def_page = code_page->def_page;
579 page->page_region = code_page->page_region;
580 page->beg_addr_v = code_page->beg_addr_v;
581 page->beg_addr_p = code_page->beg_addr_p;
582 code_area = (struct icp_qat_uof_code_area *)(encap_uof_obj->beg_uof +
583 code_page->code_area_offset);
584 page->micro_words_num = code_area->micro_words_num;
585 uword_block_tab = (struct icp_qat_uof_objtable *)
586 (encap_uof_obj->beg_uof +
587 code_area->uword_block_tab);
588 page->uwblock_num = uword_block_tab->entry_num;
589 uwblock = (struct icp_qat_uof_uword_block *)((char *)uword_block_tab +
590 sizeof(struct icp_qat_uof_objtable));
591 page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock;
592 for (i = 0; i < uword_block_tab->entry_num; i++)
593 page->uwblock[i].micro_words =
594 (uintptr_t)encap_uof_obj->beg_uof + uwblock[i].uword_offset;
595 }
596
597 static int qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle,
598 struct icp_qat_uclo_encapme *ae_uimage,
599 int max_image)
600 {
601 int i, j;
602 struct icp_qat_uof_chunkhdr *chunk_hdr = NULL;
603 struct icp_qat_uof_image *image;
604 struct icp_qat_uof_objtable *ae_regtab;
605 struct icp_qat_uof_objtable *init_reg_sym_tab;
606 struct icp_qat_uof_objtable *sbreak_tab;
607 struct icp_qat_uof_encap_obj *encap_uof_obj =
608 &obj_handle->encap_uof_obj;
609
610 for (j = 0; j < max_image; j++) {
611 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr,
612 ICP_QAT_UOF_IMAG, chunk_hdr);
613 if (!chunk_hdr)
614 break;
615 image = (struct icp_qat_uof_image *)(encap_uof_obj->beg_uof +
616 chunk_hdr->offset);
617 ae_regtab = (struct icp_qat_uof_objtable *)
618 (image->reg_tab_offset +
619 obj_handle->obj_hdr->file_buff);
620 ae_uimage[j].ae_reg_num = ae_regtab->entry_num;
621 ae_uimage[j].ae_reg = (struct icp_qat_uof_ae_reg *)
622 (((char *)ae_regtab) +
623 sizeof(struct icp_qat_uof_objtable));
624 init_reg_sym_tab = (struct icp_qat_uof_objtable *)
625 (image->init_reg_sym_tab +
626 obj_handle->obj_hdr->file_buff);
627 ae_uimage[j].init_regsym_num = init_reg_sym_tab->entry_num;
628 ae_uimage[j].init_regsym = (struct icp_qat_uof_init_regsym *)
629 (((char *)init_reg_sym_tab) +
630 sizeof(struct icp_qat_uof_objtable));
631 sbreak_tab = (struct icp_qat_uof_objtable *)
632 (image->sbreak_tab + obj_handle->obj_hdr->file_buff);
633 ae_uimage[j].sbreak_num = sbreak_tab->entry_num;
634 ae_uimage[j].sbreak = (struct icp_qat_uof_sbreak *)
635 (((char *)sbreak_tab) +
636 sizeof(struct icp_qat_uof_objtable));
637 ae_uimage[j].img_ptr = image;
638 if (qat_uclo_check_image_compat(encap_uof_obj, image))
639 goto out_err;
640 ae_uimage[j].page =
641 kzalloc(sizeof(struct icp_qat_uclo_encap_page),
642 GFP_KERNEL);
643 if (!ae_uimage[j].page)
644 goto out_err;
645 qat_uclo_map_image_page(encap_uof_obj, image,
646 ae_uimage[j].page);
647 }
648 return j;
649 out_err:
650 for (i = 0; i < j; i++)
651 kfree(ae_uimage[i].page);
652 return 0;
653 }
654
655 static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae)
656 {
657 int i, ae;
658 int mflag = 0;
659 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
660 unsigned long ae_mask = handle->hal_handle->ae_mask;
661 unsigned long cfg_ae_mask = handle->cfg_ae_mask;
662
663 for_each_set_bit(ae, &ae_mask, max_ae) {
664 if (!test_bit(ae, &cfg_ae_mask))
665 continue;
666
667 for (i = 0; i < obj_handle->uimage_num; i++) {
668 if (!test_bit(ae, (unsigned long *)
669 &obj_handle->ae_uimage[i].img_ptr->ae_assigned))
670 continue;
671 mflag = 1;
672 if (qat_uclo_init_ae_data(obj_handle, ae, i))
673 return -EINVAL;
674 }
675 }
676 if (!mflag) {
677 pr_err("QAT: uimage uses AE not set\n");
678 return -EINVAL;
679 }
680 return 0;
681 }
682
683 static struct icp_qat_uof_strtable *
684 qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr,
685 char *tab_name, struct icp_qat_uof_strtable *str_table)
686 {
687 struct icp_qat_uof_chunkhdr *chunk_hdr;
688
689 chunk_hdr = qat_uclo_find_chunk((struct icp_qat_uof_objhdr *)
690 obj_hdr->file_buff, tab_name, NULL);
691 if (chunk_hdr) {
692 int hdr_size;
693
694 memcpy(&str_table->table_len, obj_hdr->file_buff +
695 chunk_hdr->offset, sizeof(str_table->table_len));
696 hdr_size = (char *)&str_table->strings - (char *)str_table;
697 str_table->strings = (uintptr_t)obj_hdr->file_buff +
698 chunk_hdr->offset + hdr_size;
699 return str_table;
700 }
701 return NULL;
702 }
703
704 static void
705 qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj,
706 struct icp_qat_uclo_init_mem_table *init_mem_tab)
707 {
708 struct icp_qat_uof_chunkhdr *chunk_hdr;
709
710 chunk_hdr = qat_uclo_find_chunk(encap_uof_obj->obj_hdr,
711 ICP_QAT_UOF_IMEM, NULL);
712 if (chunk_hdr) {
713 memmove(&init_mem_tab->entry_num, encap_uof_obj->beg_uof +
714 chunk_hdr->offset, sizeof(unsigned int));
715 init_mem_tab->init_mem = (struct icp_qat_uof_initmem *)
716 (encap_uof_obj->beg_uof + chunk_hdr->offset +
717 sizeof(unsigned int));
718 }
719 }
720
721 static unsigned int
722 qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle)
723 {
724 switch (handle->pci_dev->device) {
725 case PCI_DEVICE_ID_INTEL_QAT_DH895XCC:
726 return ICP_QAT_AC_895XCC_DEV_TYPE;
727 case PCI_DEVICE_ID_INTEL_QAT_C62X:
728 return ICP_QAT_AC_C62X_DEV_TYPE;
729 case PCI_DEVICE_ID_INTEL_QAT_C3XXX:
730 return ICP_QAT_AC_C3XXX_DEV_TYPE;
731 case ADF_4XXX_PCI_DEVICE_ID:
732 return ICP_QAT_AC_4XXX_A_DEV_TYPE;
733 default:
734 pr_err("QAT: unsupported device 0x%x\n",
735 handle->pci_dev->device);
736 return 0;
737 }
738 }
739
740 static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle)
741 {
742 unsigned int maj_ver, prod_type = obj_handle->prod_type;
743
744 if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) {
745 pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n",
746 obj_handle->encap_uof_obj.obj_hdr->ac_dev_type,
747 prod_type);
748 return -EINVAL;
749 }
750 maj_ver = obj_handle->prod_rev & 0xff;
751 if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver ||
752 obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) {
753 pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver);
754 return -EINVAL;
755 }
756 return 0;
757 }
758
759 static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle,
760 unsigned char ae, unsigned char ctx_mask,
761 enum icp_qat_uof_regtype reg_type,
762 unsigned short reg_addr, unsigned int value)
763 {
764 switch (reg_type) {
765 case ICP_GPA_ABS:
766 case ICP_GPB_ABS:
767 ctx_mask = 0;
768 fallthrough;
769 case ICP_GPA_REL:
770 case ICP_GPB_REL:
771 return qat_hal_init_gpr(handle, ae, ctx_mask, reg_type,
772 reg_addr, value);
773 case ICP_SR_ABS:
774 case ICP_DR_ABS:
775 case ICP_SR_RD_ABS:
776 case ICP_DR_RD_ABS:
777 ctx_mask = 0;
778 fallthrough;
779 case ICP_SR_REL:
780 case ICP_DR_REL:
781 case ICP_SR_RD_REL:
782 case ICP_DR_RD_REL:
783 return qat_hal_init_rd_xfer(handle, ae, ctx_mask, reg_type,
784 reg_addr, value);
785 case ICP_SR_WR_ABS:
786 case ICP_DR_WR_ABS:
787 ctx_mask = 0;
788 fallthrough;
789 case ICP_SR_WR_REL:
790 case ICP_DR_WR_REL:
791 return qat_hal_init_wr_xfer(handle, ae, ctx_mask, reg_type,
792 reg_addr, value);
793 case ICP_NEIGH_REL:
794 return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value);
795 default:
796 pr_err("QAT: UOF uses not supported reg type 0x%x\n", reg_type);
797 return -EFAULT;
798 }
799 return 0;
800 }
801
802 static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle,
803 unsigned int ae,
804 struct icp_qat_uclo_encapme *encap_ae)
805 {
806 unsigned int i;
807 unsigned char ctx_mask;
808 struct icp_qat_uof_init_regsym *init_regsym;
809
810 if (ICP_QAT_CTX_MODE(encap_ae->img_ptr->ae_mode) ==
811 ICP_QAT_UCLO_MAX_CTX)
812 ctx_mask = 0xff;
813 else
814 ctx_mask = 0x55;
815
816 for (i = 0; i < encap_ae->init_regsym_num; i++) {
817 unsigned int exp_res;
818
819 init_regsym = &encap_ae->init_regsym[i];
820 exp_res = init_regsym->value;
821 switch (init_regsym->init_type) {
822 case ICP_QAT_UOF_INIT_REG:
823 qat_uclo_init_reg(handle, ae, ctx_mask,
824 (enum icp_qat_uof_regtype)
825 init_regsym->reg_type,
826 (unsigned short)init_regsym->reg_addr,
827 exp_res);
828 break;
829 case ICP_QAT_UOF_INIT_REG_CTX:
830 /* check if ctx is appropriate for the ctxMode */
831 if (!((1 << init_regsym->ctx) & ctx_mask)) {
832 pr_err("QAT: invalid ctx num = 0x%x\n",
833 init_regsym->ctx);
834 return -EINVAL;
835 }
836 qat_uclo_init_reg(handle, ae,
837 (unsigned char)
838 (1 << init_regsym->ctx),
839 (enum icp_qat_uof_regtype)
840 init_regsym->reg_type,
841 (unsigned short)init_regsym->reg_addr,
842 exp_res);
843 break;
844 case ICP_QAT_UOF_INIT_EXPR:
845 pr_err("QAT: INIT_EXPR feature not supported\n");
846 return -EINVAL;
847 case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP:
848 pr_err("QAT: INIT_EXPR_ENDIAN_SWAP feature not supported\n");
849 return -EINVAL;
850 default:
851 break;
852 }
853 }
854 return 0;
855 }
856
857 static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle)
858 {
859 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
860 unsigned long ae_mask = handle->hal_handle->ae_mask;
861 struct icp_qat_uclo_aedata *aed;
862 unsigned int s, ae;
863
864 if (obj_handle->global_inited)
865 return 0;
866 if (obj_handle->init_mem_tab.entry_num) {
867 if (qat_uclo_init_memory(handle)) {
868 pr_err("QAT: initialize memory failed\n");
869 return -EINVAL;
870 }
871 }
872
873 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
874 aed = &obj_handle->ae_data[ae];
875 for (s = 0; s < aed->slice_num; s++) {
876 if (!aed->ae_slices[s].encap_image)
877 continue;
878 if (qat_uclo_init_reg_sym(handle, ae, aed->ae_slices[s].encap_image))
879 return -EINVAL;
880 }
881 }
882 obj_handle->global_inited = 1;
883 return 0;
884 }
885
886 static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle,
887 struct icp_qat_uclo_objhandle *obj_handle,
888 unsigned char ae,
889 struct icp_qat_uof_image *uof_image)
890 {
891 unsigned char mode;
892 int ret;
893
894 mode = ICP_QAT_CTX_MODE(uof_image->ae_mode);
895 ret = qat_hal_set_ae_ctx_mode(handle, ae, mode);
896 if (ret) {
897 pr_err("QAT: qat_hal_set_ae_ctx_mode error\n");
898 return ret;
899 }
900 if (handle->chip_info->nn) {
901 mode = ICP_QAT_NN_MODE(uof_image->ae_mode);
902 ret = qat_hal_set_ae_nn_mode(handle, ae, mode);
903 if (ret) {
904 pr_err("QAT: qat_hal_set_ae_nn_mode error\n");
905 return ret;
906 }
907 }
908 mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode);
909 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode);
910 if (ret) {
911 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n");
912 return ret;
913 }
914 mode = ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode);
915 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, mode);
916 if (ret) {
917 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n");
918 return ret;
919 }
920 if (handle->chip_info->lm2lm3) {
921 mode = ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode);
922 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, mode);
923 if (ret) {
924 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n");
925 return ret;
926 }
927 mode = ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode);
928 ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, mode);
929 if (ret) {
930 pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n");
931 return ret;
932 }
933 mode = ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode);
934 qat_hal_set_ae_tindex_mode(handle, ae, mode);
935 }
936 return 0;
937 }
938
939 static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle)
940 {
941 struct icp_qat_uof_image *uof_image;
942 struct icp_qat_uclo_aedata *ae_data;
943 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
944 unsigned long ae_mask = handle->hal_handle->ae_mask;
945 unsigned long cfg_ae_mask = handle->cfg_ae_mask;
946 unsigned char ae, s;
947 int error;
948
949 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
950 if (!test_bit(ae, &cfg_ae_mask))
951 continue;
952
953 ae_data = &obj_handle->ae_data[ae];
954 for (s = 0; s < min_t(unsigned int, ae_data->slice_num,
955 ICP_QAT_UCLO_MAX_CTX); s++) {
956 if (!obj_handle->ae_data[ae].ae_slices[s].encap_image)
957 continue;
958 uof_image = ae_data->ae_slices[s].encap_image->img_ptr;
959 error = qat_hal_set_modes(handle, obj_handle, ae,
960 uof_image);
961 if (error)
962 return error;
963 }
964 }
965 return 0;
966 }
967
968 static void qat_uclo_init_uword_num(struct icp_qat_fw_loader_handle *handle)
969 {
970 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
971 struct icp_qat_uclo_encapme *image;
972 int a;
973
974 for (a = 0; a < obj_handle->uimage_num; a++) {
975 image = &obj_handle->ae_uimage[a];
976 image->uwords_num = image->page->beg_addr_p +
977 image->page->micro_words_num;
978 }
979 }
980
981 static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle)
982 {
983 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
984 unsigned int ae;
985
986 obj_handle->encap_uof_obj.beg_uof = obj_handle->obj_hdr->file_buff;
987 obj_handle->encap_uof_obj.obj_hdr = (struct icp_qat_uof_objhdr *)
988 obj_handle->obj_hdr->file_buff;
989 obj_handle->uword_in_bytes = 6;
990 obj_handle->prod_type = qat_uclo_get_dev_type(handle);
991 obj_handle->prod_rev = PID_MAJOR_REV |
992 (PID_MINOR_REV & handle->hal_handle->revision_id);
993 if (qat_uclo_check_uof_compat(obj_handle)) {
994 pr_err("QAT: UOF incompatible\n");
995 return -EINVAL;
996 }
997 obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(u64),
998 GFP_KERNEL);
999 if (!obj_handle->uword_buf)
1000 return -ENOMEM;
1001 obj_handle->ustore_phy_size = ICP_QAT_UCLO_MAX_USTORE;
1002 if (!obj_handle->obj_hdr->file_buff ||
1003 !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT,
1004 &obj_handle->str_table)) {
1005 pr_err("QAT: UOF doesn't have effective images\n");
1006 goto out_err;
1007 }
1008 obj_handle->uimage_num =
1009 qat_uclo_map_uimage(obj_handle, obj_handle->ae_uimage,
1010 ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX);
1011 if (!obj_handle->uimage_num)
1012 goto out_err;
1013 if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) {
1014 pr_err("QAT: Bad object\n");
1015 goto out_check_uof_aemask_err;
1016 }
1017 qat_uclo_init_uword_num(handle);
1018 qat_uclo_map_initmem_table(&obj_handle->encap_uof_obj,
1019 &obj_handle->init_mem_tab);
1020 if (qat_uclo_set_ae_mode(handle))
1021 goto out_check_uof_aemask_err;
1022 return 0;
1023 out_check_uof_aemask_err:
1024 for (ae = 0; ae < obj_handle->uimage_num; ae++)
1025 kfree(obj_handle->ae_uimage[ae].page);
1026 out_err:
1027 kfree(obj_handle->uword_buf);
1028 return -EFAULT;
1029 }
1030
1031 static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle,
1032 struct icp_qat_suof_filehdr *suof_ptr,
1033 int suof_size)
1034 {
1035 unsigned int check_sum = 0;
1036 unsigned int min_ver_offset = 0;
1037 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
1038
1039 suof_handle->file_id = ICP_QAT_SUOF_FID;
1040 suof_handle->suof_buf = (char *)suof_ptr;
1041 suof_handle->suof_size = suof_size;
1042 min_ver_offset = suof_size - offsetof(struct icp_qat_suof_filehdr,
1043 min_ver);
1044 check_sum = qat_uclo_calc_str_checksum((char *)&suof_ptr->min_ver,
1045 min_ver_offset);
1046 if (check_sum != suof_ptr->check_sum) {
1047 pr_err("QAT: incorrect SUOF checksum\n");
1048 return -EINVAL;
1049 }
1050 suof_handle->check_sum = suof_ptr->check_sum;
1051 suof_handle->min_ver = suof_ptr->min_ver;
1052 suof_handle->maj_ver = suof_ptr->maj_ver;
1053 suof_handle->fw_type = suof_ptr->fw_type;
1054 return 0;
1055 }
1056
1057 static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle,
1058 struct icp_qat_suof_img_hdr *suof_img_hdr,
1059 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr)
1060 {
1061 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
1062 struct icp_qat_simg_ae_mode *ae_mode;
1063 struct icp_qat_suof_objhdr *suof_objhdr;
1064
1065 suof_img_hdr->simg_buf = (suof_handle->suof_buf +
1066 suof_chunk_hdr->offset +
1067 sizeof(*suof_objhdr));
1068 suof_img_hdr->simg_len = ((struct icp_qat_suof_objhdr *)(uintptr_t)
1069 (suof_handle->suof_buf +
1070 suof_chunk_hdr->offset))->img_length;
1071
1072 suof_img_hdr->css_header = suof_img_hdr->simg_buf;
1073 suof_img_hdr->css_key = (suof_img_hdr->css_header +
1074 sizeof(struct icp_qat_css_hdr));
1075 suof_img_hdr->css_signature = suof_img_hdr->css_key +
1076 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) +
1077 ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle);
1078 suof_img_hdr->css_simg = suof_img_hdr->css_signature +
1079 ICP_QAT_CSS_SIGNATURE_LEN(handle);
1080
1081 ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg);
1082 suof_img_hdr->ae_mask = ae_mode->ae_mask;
1083 suof_img_hdr->simg_name = (unsigned long)&ae_mode->simg_name;
1084 suof_img_hdr->appmeta_data = (unsigned long)&ae_mode->appmeta_data;
1085 suof_img_hdr->fw_type = ae_mode->fw_type;
1086 }
1087
1088 static void
1089 qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle *suof_handle,
1090 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr)
1091 {
1092 char **sym_str = (char **)&suof_handle->sym_str;
1093 unsigned int *sym_size = &suof_handle->sym_size;
1094 struct icp_qat_suof_strtable *str_table_obj;
1095
1096 *sym_size = *(unsigned int *)(uintptr_t)
1097 (suof_chunk_hdr->offset + suof_handle->suof_buf);
1098 *sym_str = (char *)(uintptr_t)
1099 (suof_handle->suof_buf + suof_chunk_hdr->offset +
1100 sizeof(str_table_obj->tab_length));
1101 }
1102
1103 static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle,
1104 struct icp_qat_suof_img_hdr *img_hdr)
1105 {
1106 struct icp_qat_simg_ae_mode *img_ae_mode = NULL;
1107 unsigned int prod_rev, maj_ver, prod_type;
1108
1109 prod_type = qat_uclo_get_dev_type(handle);
1110 img_ae_mode = (struct icp_qat_simg_ae_mode *)img_hdr->css_simg;
1111 prod_rev = PID_MAJOR_REV |
1112 (PID_MINOR_REV & handle->hal_handle->revision_id);
1113 if (img_ae_mode->dev_type != prod_type) {
1114 pr_err("QAT: incompatible product type %x\n",
1115 img_ae_mode->dev_type);
1116 return -EINVAL;
1117 }
1118 maj_ver = prod_rev & 0xff;
1119 if (maj_ver > img_ae_mode->devmax_ver ||
1120 maj_ver < img_ae_mode->devmin_ver) {
1121 pr_err("QAT: incompatible device majver 0x%x\n", maj_ver);
1122 return -EINVAL;
1123 }
1124 return 0;
1125 }
1126
1127 static void qat_uclo_del_suof(struct icp_qat_fw_loader_handle *handle)
1128 {
1129 struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle;
1130
1131 kfree(sobj_handle->img_table.simg_hdr);
1132 sobj_handle->img_table.simg_hdr = NULL;
1133 kfree(handle->sobj_handle);
1134 handle->sobj_handle = NULL;
1135 }
1136
1137 static void qat_uclo_tail_img(struct icp_qat_suof_img_hdr *suof_img_hdr,
1138 unsigned int img_id, unsigned int num_simgs)
1139 {
1140 struct icp_qat_suof_img_hdr img_header;
1141
1142 if (img_id != num_simgs - 1) {
1143 memcpy(&img_header, &suof_img_hdr[num_simgs - 1],
1144 sizeof(*suof_img_hdr));
1145 memcpy(&suof_img_hdr[num_simgs - 1], &suof_img_hdr[img_id],
1146 sizeof(*suof_img_hdr));
1147 memcpy(&suof_img_hdr[img_id], &img_header,
1148 sizeof(*suof_img_hdr));
1149 }
1150 }
1151
1152 static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle,
1153 struct icp_qat_suof_filehdr *suof_ptr,
1154 int suof_size)
1155 {
1156 struct icp_qat_suof_handle *suof_handle = handle->sobj_handle;
1157 struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL;
1158 struct icp_qat_suof_img_hdr *suof_img_hdr = NULL;
1159 int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE;
1160 unsigned int i = 0;
1161 struct icp_qat_suof_img_hdr img_header;
1162
1163 if (!suof_ptr || suof_size == 0) {
1164 pr_err("QAT: input parameter SUOF pointer/size is NULL\n");
1165 return -EINVAL;
1166 }
1167 if (qat_uclo_check_suof_format(suof_ptr))
1168 return -EINVAL;
1169 ret = qat_uclo_map_suof_file_hdr(handle, suof_ptr, suof_size);
1170 if (ret)
1171 return ret;
1172 suof_chunk_hdr = (struct icp_qat_suof_chunk_hdr *)
1173 ((uintptr_t)suof_ptr + sizeof(*suof_ptr));
1174
1175 qat_uclo_map_suof_symobjs(suof_handle, suof_chunk_hdr);
1176 suof_handle->img_table.num_simgs = suof_ptr->num_chunks - 1;
1177
1178 if (suof_handle->img_table.num_simgs != 0) {
1179 suof_img_hdr = kcalloc(suof_handle->img_table.num_simgs,
1180 sizeof(img_header),
1181 GFP_KERNEL);
1182 if (!suof_img_hdr)
1183 return -ENOMEM;
1184 suof_handle->img_table.simg_hdr = suof_img_hdr;
1185
1186 for (i = 0; i < suof_handle->img_table.num_simgs; i++) {
1187 qat_uclo_map_simg(handle, &suof_img_hdr[i],
1188 &suof_chunk_hdr[1 + i]);
1189 ret = qat_uclo_check_simg_compat(handle,
1190 &suof_img_hdr[i]);
1191 if (ret)
1192 return ret;
1193 suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask;
1194 if ((suof_img_hdr[i].ae_mask & 0x1) != 0)
1195 ae0_img = i;
1196 }
1197
1198 if (!handle->chip_info->tgroup_share_ustore) {
1199 qat_uclo_tail_img(suof_img_hdr, ae0_img,
1200 suof_handle->img_table.num_simgs);
1201 }
1202 }
1203 return 0;
1204 }
1205
1206 #define ADD_ADDR(high, low) ((((u64)high) << 32) + low)
1207 #define BITS_IN_DWORD 32
1208
1209 static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle,
1210 struct icp_qat_fw_auth_desc *desc)
1211 {
1212 u32 fcu_sts, retry = 0;
1213 u32 fcu_ctl_csr, fcu_sts_csr;
1214 u32 fcu_dram_hi_csr, fcu_dram_lo_csr;
1215 u64 bus_addr;
1216
1217 bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low)
1218 - sizeof(struct icp_qat_auth_chunk);
1219
1220 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
1221 fcu_sts_csr = handle->chip_info->fcu_sts_csr;
1222 fcu_dram_hi_csr = handle->chip_info->fcu_dram_addr_hi;
1223 fcu_dram_lo_csr = handle->chip_info->fcu_dram_addr_lo;
1224
1225 SET_CAP_CSR(handle, fcu_dram_hi_csr, (bus_addr >> BITS_IN_DWORD));
1226 SET_CAP_CSR(handle, fcu_dram_lo_csr, bus_addr);
1227 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH);
1228
1229 do {
1230 msleep(FW_AUTH_WAIT_PERIOD);
1231 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
1232 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL)
1233 goto auth_fail;
1234 if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1))
1235 if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE)
1236 return 0;
1237 } while (retry++ < FW_AUTH_MAX_RETRY);
1238 auth_fail:
1239 pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n",
1240 fcu_sts & FCU_AUTH_STS_MASK, retry);
1241 return -EINVAL;
1242 }
1243
1244 static bool qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle,
1245 int imgid)
1246 {
1247 struct icp_qat_suof_handle *sobj_handle;
1248
1249 if (!handle->chip_info->tgroup_share_ustore)
1250 return false;
1251
1252 sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle;
1253 if (handle->hal_handle->admin_ae_mask &
1254 sobj_handle->img_table.simg_hdr[imgid].ae_mask)
1255 return false;
1256
1257 return true;
1258 }
1259
1260 static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle,
1261 struct icp_qat_fw_auth_desc *desc)
1262 {
1263 unsigned long ae_mask = handle->hal_handle->ae_mask;
1264 unsigned long desc_ae_mask = desc->ae_mask;
1265 u32 fcu_sts, ae_broadcast_mask = 0;
1266 u32 fcu_loaded_csr, ae_loaded;
1267 u32 fcu_sts_csr, fcu_ctl_csr;
1268 unsigned int ae, retry = 0;
1269
1270 if (handle->chip_info->tgroup_share_ustore) {
1271 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
1272 fcu_sts_csr = handle->chip_info->fcu_sts_csr;
1273 fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr;
1274 } else {
1275 pr_err("Chip 0x%x doesn't support broadcast load\n",
1276 handle->pci_dev->device);
1277 return -EINVAL;
1278 }
1279
1280 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
1281 if (qat_hal_check_ae_active(handle, (unsigned char)ae)) {
1282 pr_err("QAT: Broadcast load failed. AE is not enabled or active.\n");
1283 return -EINVAL;
1284 }
1285
1286 if (test_bit(ae, &desc_ae_mask))
1287 ae_broadcast_mask |= 1 << ae;
1288 }
1289
1290 if (ae_broadcast_mask) {
1291 SET_CAP_CSR(handle, FCU_ME_BROADCAST_MASK_TYPE,
1292 ae_broadcast_mask);
1293
1294 SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_LOAD);
1295
1296 do {
1297 msleep(FW_AUTH_WAIT_PERIOD);
1298 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
1299 fcu_sts &= FCU_AUTH_STS_MASK;
1300
1301 if (fcu_sts == FCU_STS_LOAD_FAIL) {
1302 pr_err("Broadcast load failed: 0x%x)\n", fcu_sts);
1303 return -EINVAL;
1304 } else if (fcu_sts == FCU_STS_LOAD_DONE) {
1305 ae_loaded = GET_CAP_CSR(handle, fcu_loaded_csr);
1306 ae_loaded >>= handle->chip_info->fcu_loaded_ae_pos;
1307
1308 if ((ae_loaded & ae_broadcast_mask) == ae_broadcast_mask)
1309 break;
1310 }
1311 } while (retry++ < FW_AUTH_MAX_RETRY);
1312
1313 if (retry > FW_AUTH_MAX_RETRY) {
1314 pr_err("QAT: broadcast load failed timeout %d\n", retry);
1315 return -EINVAL;
1316 }
1317 }
1318 return 0;
1319 }
1320
1321 static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle,
1322 struct icp_firml_dram_desc *dram_desc,
1323 unsigned int size)
1324 {
1325 void *vptr;
1326 dma_addr_t ptr;
1327
1328 vptr = dma_alloc_coherent(&handle->pci_dev->dev,
1329 size, &ptr, GFP_KERNEL);
1330 if (!vptr)
1331 return -ENOMEM;
1332 dram_desc->dram_base_addr_v = vptr;
1333 dram_desc->dram_bus_addr = ptr;
1334 dram_desc->dram_size = size;
1335 return 0;
1336 }
1337
1338 static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle,
1339 struct icp_firml_dram_desc *dram_desc)
1340 {
1341 if (handle && dram_desc && dram_desc->dram_base_addr_v) {
1342 dma_free_coherent(&handle->pci_dev->dev,
1343 (size_t)(dram_desc->dram_size),
1344 dram_desc->dram_base_addr_v,
1345 dram_desc->dram_bus_addr);
1346 }
1347
1348 if (dram_desc)
1349 memset(dram_desc, 0, sizeof(*dram_desc));
1350 }
1351
1352 static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle,
1353 struct icp_qat_fw_auth_desc **desc)
1354 {
1355 struct icp_firml_dram_desc dram_desc;
1356
1357 if (*desc) {
1358 dram_desc.dram_base_addr_v = *desc;
1359 dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *)
1360 (*desc))->chunk_bus_addr;
1361 dram_desc.dram_size = ((struct icp_qat_auth_chunk *)
1362 (*desc))->chunk_size;
1363 qat_uclo_simg_free(handle, &dram_desc);
1364 }
1365 }
1366
1367 static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle,
1368 char *image, unsigned int size,
1369 struct icp_qat_fw_auth_desc **desc)
1370 {
1371 struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image;
1372 struct icp_qat_fw_auth_desc *auth_desc;
1373 struct icp_qat_auth_chunk *auth_chunk;
1374 u64 virt_addr, bus_addr, virt_base;
1375 unsigned int length, simg_offset = sizeof(*auth_chunk);
1376 struct icp_qat_simg_ae_mode *simg_ae_mode;
1377 struct icp_firml_dram_desc img_desc;
1378
1379 if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_MAX_IMAGE_LEN)) {
1380 pr_err("QAT: error, input image size overflow %d\n", size);
1381 return -EINVAL;
1382 }
1383 length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ?
1384 ICP_QAT_CSS_AE_SIMG_LEN(handle) + simg_offset :
1385 size + ICP_QAT_CSS_FWSK_PAD_LEN(handle) + simg_offset;
1386 if (qat_uclo_simg_alloc(handle, &img_desc, length)) {
1387 pr_err("QAT: error, allocate continuous dram fail\n");
1388 return -ENOMEM;
1389 }
1390
1391 auth_chunk = img_desc.dram_base_addr_v;
1392 auth_chunk->chunk_size = img_desc.dram_size;
1393 auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr;
1394 virt_base = (uintptr_t)img_desc.dram_base_addr_v + simg_offset;
1395 bus_addr = img_desc.dram_bus_addr + simg_offset;
1396 auth_desc = img_desc.dram_base_addr_v;
1397 auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1398 auth_desc->css_hdr_low = (unsigned int)bus_addr;
1399 virt_addr = virt_base;
1400
1401 memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr));
1402 /* pub key */
1403 bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) +
1404 sizeof(*css_hdr);
1405 virt_addr = virt_addr + sizeof(*css_hdr);
1406
1407 auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1408 auth_desc->fwsk_pub_low = (unsigned int)bus_addr;
1409
1410 memcpy((void *)(uintptr_t)virt_addr,
1411 (void *)(image + sizeof(*css_hdr)),
1412 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle));
1413 /* padding */
1414 memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)),
1415 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle));
1416
1417 /* exponent */
1418 memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) +
1419 ICP_QAT_CSS_FWSK_PAD_LEN(handle)),
1420 (void *)(image + sizeof(*css_hdr) +
1421 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)),
1422 sizeof(unsigned int));
1423
1424 /* signature */
1425 bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high,
1426 auth_desc->fwsk_pub_low) +
1427 ICP_QAT_CSS_FWSK_PUB_LEN(handle);
1428 virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle);
1429 auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1430 auth_desc->signature_low = (unsigned int)bus_addr;
1431
1432 memcpy((void *)(uintptr_t)virt_addr,
1433 (void *)(image + sizeof(*css_hdr) +
1434 ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) +
1435 ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)),
1436 ICP_QAT_CSS_SIGNATURE_LEN(handle));
1437
1438 bus_addr = ADD_ADDR(auth_desc->signature_high,
1439 auth_desc->signature_low) +
1440 ICP_QAT_CSS_SIGNATURE_LEN(handle);
1441 virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle);
1442
1443 auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD);
1444 auth_desc->img_low = (unsigned int)bus_addr;
1445 auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle);
1446 memcpy((void *)(uintptr_t)virt_addr,
1447 (void *)(image + ICP_QAT_AE_IMG_OFFSET(handle)),
1448 auth_desc->img_len);
1449 virt_addr = virt_base;
1450 /* AE firmware */
1451 if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type ==
1452 CSS_AE_FIRMWARE) {
1453 auth_desc->img_ae_mode_data_high = auth_desc->img_high;
1454 auth_desc->img_ae_mode_data_low = auth_desc->img_low;
1455 bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high,
1456 auth_desc->img_ae_mode_data_low) +
1457 sizeof(struct icp_qat_simg_ae_mode);
1458
1459 auth_desc->img_ae_init_data_high = (unsigned int)
1460 (bus_addr >> BITS_IN_DWORD);
1461 auth_desc->img_ae_init_data_low = (unsigned int)bus_addr;
1462 bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN;
1463 auth_desc->img_ae_insts_high = (unsigned int)
1464 (bus_addr >> BITS_IN_DWORD);
1465 auth_desc->img_ae_insts_low = (unsigned int)bus_addr;
1466 virt_addr += sizeof(struct icp_qat_css_hdr);
1467 virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle);
1468 virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle);
1469 simg_ae_mode = (struct icp_qat_simg_ae_mode *)(uintptr_t)virt_addr;
1470 auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask;
1471 } else {
1472 auth_desc->img_ae_insts_high = auth_desc->img_high;
1473 auth_desc->img_ae_insts_low = auth_desc->img_low;
1474 }
1475 *desc = auth_desc;
1476 return 0;
1477 }
1478
1479 static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle,
1480 struct icp_qat_fw_auth_desc *desc)
1481 {
1482 unsigned long ae_mask = handle->hal_handle->ae_mask;
1483 u32 fcu_sts_csr, fcu_ctl_csr;
1484 u32 loaded_aes, loaded_csr;
1485 unsigned int i;
1486 u32 fcu_sts;
1487
1488 fcu_ctl_csr = handle->chip_info->fcu_ctl_csr;
1489 fcu_sts_csr = handle->chip_info->fcu_sts_csr;
1490 loaded_csr = handle->chip_info->fcu_loaded_ae_csr;
1491
1492 for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) {
1493 int retry = 0;
1494
1495 if (!((desc->ae_mask >> i) & 0x1))
1496 continue;
1497 if (qat_hal_check_ae_active(handle, i)) {
1498 pr_err("QAT: AE %d is active\n", i);
1499 return -EINVAL;
1500 }
1501 SET_CAP_CSR(handle, fcu_ctl_csr,
1502 (FCU_CTRL_CMD_LOAD |
1503 (1 << FCU_CTRL_BROADCAST_POS) |
1504 (i << FCU_CTRL_AE_POS)));
1505
1506 do {
1507 msleep(FW_AUTH_WAIT_PERIOD);
1508 fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr);
1509 if ((fcu_sts & FCU_AUTH_STS_MASK) ==
1510 FCU_STS_LOAD_DONE) {
1511 loaded_aes = GET_CAP_CSR(handle, loaded_csr);
1512 loaded_aes >>= handle->chip_info->fcu_loaded_ae_pos;
1513 if (loaded_aes & (1 << i))
1514 break;
1515 }
1516 } while (retry++ < FW_AUTH_MAX_RETRY);
1517 if (retry > FW_AUTH_MAX_RETRY) {
1518 pr_err("QAT: firmware load failed timeout %x\n", retry);
1519 return -EINVAL;
1520 }
1521 }
1522 return 0;
1523 }
1524
1525 static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle,
1526 void *addr_ptr, int mem_size)
1527 {
1528 struct icp_qat_suof_handle *suof_handle;
1529
1530 suof_handle = kzalloc(sizeof(*suof_handle), GFP_KERNEL);
1531 if (!suof_handle)
1532 return -ENOMEM;
1533 handle->sobj_handle = suof_handle;
1534 if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) {
1535 qat_uclo_del_suof(handle);
1536 pr_err("QAT: map SUOF failed\n");
1537 return -EINVAL;
1538 }
1539 return 0;
1540 }
1541
1542 int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle,
1543 void *addr_ptr, int mem_size)
1544 {
1545 struct icp_qat_fw_auth_desc *desc = NULL;
1546 int status = 0;
1547
1548 if (handle->chip_info->fw_auth) {
1549 if (!qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc))
1550 status = qat_uclo_auth_fw(handle, desc);
1551 qat_uclo_ummap_auth_fw(handle, &desc);
1552 } else {
1553 if (!handle->chip_info->sram_visible) {
1554 dev_dbg(&handle->pci_dev->dev,
1555 "QAT MMP fw not loaded for device 0x%x",
1556 handle->pci_dev->device);
1557 return status;
1558 }
1559 qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size);
1560 }
1561 return status;
1562 }
1563
1564 static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle,
1565 void *addr_ptr, int mem_size)
1566 {
1567 struct icp_qat_uof_filehdr *filehdr;
1568 struct icp_qat_uclo_objhandle *objhdl;
1569
1570 objhdl = kzalloc(sizeof(*objhdl), GFP_KERNEL);
1571 if (!objhdl)
1572 return -ENOMEM;
1573 objhdl->obj_buf = kmemdup(addr_ptr, mem_size, GFP_KERNEL);
1574 if (!objhdl->obj_buf)
1575 goto out_objbuf_err;
1576 filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf;
1577 if (qat_uclo_check_uof_format(filehdr))
1578 goto out_objhdr_err;
1579 objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr,
1580 ICP_QAT_UOF_OBJS);
1581 if (!objhdl->obj_hdr) {
1582 pr_err("QAT: object file chunk is null\n");
1583 goto out_objhdr_err;
1584 }
1585 handle->obj_handle = objhdl;
1586 if (qat_uclo_parse_uof_obj(handle))
1587 goto out_overlay_obj_err;
1588 return 0;
1589
1590 out_overlay_obj_err:
1591 handle->obj_handle = NULL;
1592 kfree(objhdl->obj_hdr);
1593 out_objhdr_err:
1594 kfree(objhdl->obj_buf);
1595 out_objbuf_err:
1596 kfree(objhdl);
1597 return -ENOMEM;
1598 }
1599
1600 static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle,
1601 struct icp_qat_mof_file_hdr *mof_ptr,
1602 u32 mof_size)
1603 {
1604 struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle;
1605 unsigned int min_ver_offset;
1606 unsigned int checksum;
1607
1608 mobj_handle->file_id = ICP_QAT_MOF_FID;
1609 mobj_handle->mof_buf = (char *)mof_ptr;
1610 mobj_handle->mof_size = mof_size;
1611
1612 min_ver_offset = mof_size - offsetof(struct icp_qat_mof_file_hdr,
1613 min_ver);
1614 checksum = qat_uclo_calc_str_checksum(&mof_ptr->min_ver,
1615 min_ver_offset);
1616 if (checksum != mof_ptr->checksum) {
1617 pr_err("QAT: incorrect MOF checksum\n");
1618 return -EINVAL;
1619 }
1620
1621 mobj_handle->checksum = mof_ptr->checksum;
1622 mobj_handle->min_ver = mof_ptr->min_ver;
1623 mobj_handle->maj_ver = mof_ptr->maj_ver;
1624 return 0;
1625 }
1626
1627 static void qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle)
1628 {
1629 struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle;
1630
1631 kfree(mobj_handle->obj_table.obj_hdr);
1632 mobj_handle->obj_table.obj_hdr = NULL;
1633 kfree(handle->mobj_handle);
1634 handle->mobj_handle = NULL;
1635 }
1636
1637 static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle,
1638 char *obj_name, char **obj_ptr,
1639 unsigned int *obj_size)
1640 {
1641 struct icp_qat_mof_objhdr *obj_hdr = mobj_handle->obj_table.obj_hdr;
1642 unsigned int i;
1643
1644 for (i = 0; i < mobj_handle->obj_table.num_objs; i++) {
1645 if (!strncmp(obj_hdr[i].obj_name, obj_name,
1646 ICP_QAT_SUOF_OBJ_NAME_LEN)) {
1647 *obj_ptr = obj_hdr[i].obj_buf;
1648 *obj_size = obj_hdr[i].obj_size;
1649 return 0;
1650 }
1651 }
1652
1653 pr_err("QAT: object %s is not found inside MOF\n", obj_name);
1654 return -EINVAL;
1655 }
1656
1657 static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle,
1658 struct icp_qat_mof_objhdr *mobj_hdr,
1659 struct icp_qat_mof_obj_chunkhdr *obj_chunkhdr)
1660 {
1661 u8 *obj;
1662
1663 if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_UOF_IMAG,
1664 ICP_QAT_MOF_OBJ_CHUNKID_LEN)) {
1665 obj = mobj_handle->uobjs_hdr + obj_chunkhdr->offset;
1666 } else if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_SUOF_IMAG,
1667 ICP_QAT_MOF_OBJ_CHUNKID_LEN)) {
1668 obj = mobj_handle->sobjs_hdr + obj_chunkhdr->offset;
1669 } else {
1670 pr_err("QAT: unsupported chunk id\n");
1671 return -EINVAL;
1672 }
1673 mobj_hdr->obj_buf = obj;
1674 mobj_hdr->obj_size = (unsigned int)obj_chunkhdr->size;
1675 mobj_hdr->obj_name = obj_chunkhdr->name + mobj_handle->sym_str;
1676 return 0;
1677 }
1678
1679 static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle)
1680 {
1681 struct icp_qat_mof_obj_chunkhdr *uobj_chunkhdr;
1682 struct icp_qat_mof_obj_chunkhdr *sobj_chunkhdr;
1683 struct icp_qat_mof_obj_hdr *uobj_hdr;
1684 struct icp_qat_mof_obj_hdr *sobj_hdr;
1685 struct icp_qat_mof_objhdr *mobj_hdr;
1686 unsigned int uobj_chunk_num = 0;
1687 unsigned int sobj_chunk_num = 0;
1688 unsigned int *valid_chunk;
1689 int ret, i;
1690
1691 uobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->uobjs_hdr;
1692 sobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->sobjs_hdr;
1693 if (uobj_hdr)
1694 uobj_chunk_num = uobj_hdr->num_chunks;
1695 if (sobj_hdr)
1696 sobj_chunk_num = sobj_hdr->num_chunks;
1697
1698 mobj_hdr = kzalloc((uobj_chunk_num + sobj_chunk_num) *
1699 sizeof(*mobj_hdr), GFP_KERNEL);
1700 if (!mobj_hdr)
1701 return -ENOMEM;
1702
1703 mobj_handle->obj_table.obj_hdr = mobj_hdr;
1704 valid_chunk = &mobj_handle->obj_table.num_objs;
1705 uobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *)
1706 ((uintptr_t)uobj_hdr + sizeof(*uobj_hdr));
1707 sobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *)
1708 ((uintptr_t)sobj_hdr + sizeof(*sobj_hdr));
1709
1710 /* map uof objects */
1711 for (i = 0; i < uobj_chunk_num; i++) {
1712 ret = qat_uclo_map_obj_from_mof(mobj_handle,
1713 &mobj_hdr[*valid_chunk],
1714 &uobj_chunkhdr[i]);
1715 if (ret)
1716 return ret;
1717 (*valid_chunk)++;
1718 }
1719
1720 /* map suof objects */
1721 for (i = 0; i < sobj_chunk_num; i++) {
1722 ret = qat_uclo_map_obj_from_mof(mobj_handle,
1723 &mobj_hdr[*valid_chunk],
1724 &sobj_chunkhdr[i]);
1725 if (ret)
1726 return ret;
1727 (*valid_chunk)++;
1728 }
1729
1730 if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunk) {
1731 pr_err("QAT: inconsistent UOF/SUOF chunk amount\n");
1732 return -EINVAL;
1733 }
1734 return 0;
1735 }
1736
1737 static void qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle *mobj_handle,
1738 struct icp_qat_mof_chunkhdr *mof_chunkhdr)
1739 {
1740 char **sym_str = (char **)&mobj_handle->sym_str;
1741 unsigned int *sym_size = &mobj_handle->sym_size;
1742 struct icp_qat_mof_str_table *str_table_obj;
1743
1744 *sym_size = *(unsigned int *)(uintptr_t)
1745 (mof_chunkhdr->offset + mobj_handle->mof_buf);
1746 *sym_str = (char *)(uintptr_t)
1747 (mobj_handle->mof_buf + mof_chunkhdr->offset +
1748 sizeof(str_table_obj->tab_len));
1749 }
1750
1751 static void qat_uclo_map_mof_chunk(struct icp_qat_mof_handle *mobj_handle,
1752 struct icp_qat_mof_chunkhdr *mof_chunkhdr)
1753 {
1754 char *chunk_id = mof_chunkhdr->chunk_id;
1755
1756 if (!strncmp(chunk_id, ICP_QAT_MOF_SYM_OBJS, ICP_QAT_MOF_OBJ_ID_LEN))
1757 qat_uclo_map_mof_symobjs(mobj_handle, mof_chunkhdr);
1758 else if (!strncmp(chunk_id, ICP_QAT_UOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN))
1759 mobj_handle->uobjs_hdr = mobj_handle->mof_buf +
1760 mof_chunkhdr->offset;
1761 else if (!strncmp(chunk_id, ICP_QAT_SUOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN))
1762 mobj_handle->sobjs_hdr = mobj_handle->mof_buf +
1763 mof_chunkhdr->offset;
1764 }
1765
1766 static int qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr *mof_hdr)
1767 {
1768 int maj = mof_hdr->maj_ver & 0xff;
1769 int min = mof_hdr->min_ver & 0xff;
1770
1771 if (mof_hdr->file_id != ICP_QAT_MOF_FID) {
1772 pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id);
1773 return -EINVAL;
1774 }
1775
1776 if (mof_hdr->num_chunks <= 0x1) {
1777 pr_err("QAT: MOF chunk amount is incorrect\n");
1778 return -EINVAL;
1779 }
1780 if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) {
1781 pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n",
1782 maj, min);
1783 return -EINVAL;
1784 }
1785 return 0;
1786 }
1787
1788 static int qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle *handle,
1789 struct icp_qat_mof_file_hdr *mof_ptr,
1790 u32 mof_size, char *obj_name, char **obj_ptr,
1791 unsigned int *obj_size)
1792 {
1793 struct icp_qat_mof_chunkhdr *mof_chunkhdr;
1794 unsigned int file_id = mof_ptr->file_id;
1795 struct icp_qat_mof_handle *mobj_handle;
1796 unsigned short chunks_num;
1797 unsigned int i;
1798 int ret;
1799
1800 if (file_id == ICP_QAT_UOF_FID || file_id == ICP_QAT_SUOF_FID) {
1801 if (obj_ptr)
1802 *obj_ptr = (char *)mof_ptr;
1803 if (obj_size)
1804 *obj_size = mof_size;
1805 return 0;
1806 }
1807 if (qat_uclo_check_mof_format(mof_ptr))
1808 return -EINVAL;
1809
1810 mobj_handle = kzalloc(sizeof(*mobj_handle), GFP_KERNEL);
1811 if (!mobj_handle)
1812 return -ENOMEM;
1813
1814 handle->mobj_handle = mobj_handle;
1815 ret = qat_uclo_map_mof_file_hdr(handle, mof_ptr, mof_size);
1816 if (ret)
1817 return ret;
1818
1819 mof_chunkhdr = (void *)mof_ptr + sizeof(*mof_ptr);
1820 chunks_num = mof_ptr->num_chunks;
1821
1822 /* Parse MOF file chunks */
1823 for (i = 0; i < chunks_num; i++)
1824 qat_uclo_map_mof_chunk(mobj_handle, &mof_chunkhdr[i]);
1825
1826 /* All sym_objs uobjs and sobjs should be available */
1827 if (!mobj_handle->sym_str ||
1828 (!mobj_handle->uobjs_hdr && !mobj_handle->sobjs_hdr))
1829 return -EINVAL;
1830
1831 ret = qat_uclo_map_objs_from_mof(mobj_handle);
1832 if (ret)
1833 return ret;
1834
1835 /* Seek specified uof object in MOF */
1836 return qat_uclo_seek_obj_inside_mof(mobj_handle, obj_name,
1837 obj_ptr, obj_size);
1838 }
1839
1840 int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle,
1841 void *addr_ptr, u32 mem_size, char *obj_name)
1842 {
1843 char *obj_addr;
1844 u32 obj_size;
1845 int ret;
1846
1847 BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >=
1848 (sizeof(handle->hal_handle->ae_mask) * 8));
1849
1850 if (!handle || !addr_ptr || mem_size < 24)
1851 return -EINVAL;
1852
1853 if (obj_name) {
1854 ret = qat_uclo_map_mof_obj(handle, addr_ptr, mem_size, obj_name,
1855 &obj_addr, &obj_size);
1856 if (ret)
1857 return ret;
1858 } else {
1859 obj_addr = addr_ptr;
1860 obj_size = mem_size;
1861 }
1862
1863 return (handle->chip_info->fw_auth) ?
1864 qat_uclo_map_suof_obj(handle, obj_addr, obj_size) :
1865 qat_uclo_map_uof_obj(handle, obj_addr, obj_size);
1866 }
1867
1868 void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle)
1869 {
1870 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
1871 unsigned int a;
1872
1873 if (handle->mobj_handle)
1874 qat_uclo_del_mof(handle);
1875 if (handle->sobj_handle)
1876 qat_uclo_del_suof(handle);
1877 if (!obj_handle)
1878 return;
1879
1880 kfree(obj_handle->uword_buf);
1881 for (a = 0; a < obj_handle->uimage_num; a++)
1882 kfree(obj_handle->ae_uimage[a].page);
1883
1884 for (a = 0; a < handle->hal_handle->ae_max_num; a++)
1885 qat_uclo_free_ae_data(&obj_handle->ae_data[a]);
1886
1887 kfree(obj_handle->obj_hdr);
1888 kfree(obj_handle->obj_buf);
1889 kfree(obj_handle);
1890 handle->obj_handle = NULL;
1891 }
1892
1893 static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle,
1894 struct icp_qat_uclo_encap_page *encap_page,
1895 u64 *uword, unsigned int addr_p,
1896 unsigned int raddr, u64 fill)
1897 {
1898 unsigned int i, addr;
1899 u64 uwrd = 0;
1900
1901 if (!encap_page) {
1902 *uword = fill;
1903 return;
1904 }
1905 addr = (encap_page->page_region) ? raddr : addr_p;
1906 for (i = 0; i < encap_page->uwblock_num; i++) {
1907 if (addr >= encap_page->uwblock[i].start_addr &&
1908 addr <= encap_page->uwblock[i].start_addr +
1909 encap_page->uwblock[i].words_num - 1) {
1910 addr -= encap_page->uwblock[i].start_addr;
1911 addr *= obj_handle->uword_in_bytes;
1912 memcpy(&uwrd, (void *)(((uintptr_t)
1913 encap_page->uwblock[i].micro_words) + addr),
1914 obj_handle->uword_in_bytes);
1915 uwrd = uwrd & GENMASK_ULL(43, 0);
1916 }
1917 }
1918 *uword = uwrd;
1919 if (*uword == INVLD_UWORD)
1920 *uword = fill;
1921 }
1922
1923 static void qat_uclo_wr_uimage_raw_page(struct icp_qat_fw_loader_handle *handle,
1924 struct icp_qat_uclo_encap_page
1925 *encap_page, unsigned int ae)
1926 {
1927 unsigned int uw_physical_addr, uw_relative_addr, i, words_num, cpylen;
1928 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
1929 u64 fill_pat;
1930
1931 /* load the page starting at appropriate ustore address */
1932 /* get fill-pattern from an image -- they are all the same */
1933 memcpy(&fill_pat, obj_handle->ae_uimage[0].img_ptr->fill_pattern,
1934 sizeof(u64));
1935 uw_physical_addr = encap_page->beg_addr_p;
1936 uw_relative_addr = 0;
1937 words_num = encap_page->micro_words_num;
1938 while (words_num) {
1939 if (words_num < UWORD_CPYBUF_SIZE)
1940 cpylen = words_num;
1941 else
1942 cpylen = UWORD_CPYBUF_SIZE;
1943
1944 /* load the buffer */
1945 for (i = 0; i < cpylen; i++)
1946 qat_uclo_fill_uwords(obj_handle, encap_page,
1947 &obj_handle->uword_buf[i],
1948 uw_physical_addr + i,
1949 uw_relative_addr + i, fill_pat);
1950
1951 /* copy the buffer to ustore */
1952 qat_hal_wr_uwords(handle, (unsigned char)ae,
1953 uw_physical_addr, cpylen,
1954 obj_handle->uword_buf);
1955
1956 uw_physical_addr += cpylen;
1957 uw_relative_addr += cpylen;
1958 words_num -= cpylen;
1959 }
1960 }
1961
1962 static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle,
1963 struct icp_qat_uof_image *image)
1964 {
1965 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
1966 unsigned long ae_mask = handle->hal_handle->ae_mask;
1967 unsigned long cfg_ae_mask = handle->cfg_ae_mask;
1968 unsigned long ae_assigned = image->ae_assigned;
1969 struct icp_qat_uclo_aedata *aed;
1970 unsigned int ctx_mask, s;
1971 struct icp_qat_uclo_page *page;
1972 unsigned char ae;
1973 int ctx;
1974
1975 if (ICP_QAT_CTX_MODE(image->ae_mode) == ICP_QAT_UCLO_MAX_CTX)
1976 ctx_mask = 0xff;
1977 else
1978 ctx_mask = 0x55;
1979 /* load the default page and set assigned CTX PC
1980 * to the entrypoint address */
1981 for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) {
1982 if (!test_bit(ae, &cfg_ae_mask))
1983 continue;
1984
1985 if (!test_bit(ae, &ae_assigned))
1986 continue;
1987
1988 aed = &obj_handle->ae_data[ae];
1989 /* find the slice to which this image is assigned */
1990 for (s = 0; s < aed->slice_num; s++) {
1991 if (image->ctx_assigned &
1992 aed->ae_slices[s].ctx_mask_assigned)
1993 break;
1994 }
1995 if (s >= aed->slice_num)
1996 continue;
1997 page = aed->ae_slices[s].page;
1998 if (!page->encap_page->def_page)
1999 continue;
2000 qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae);
2001
2002 page = aed->ae_slices[s].page;
2003 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++)
2004 aed->ae_slices[s].cur_page[ctx] =
2005 (ctx_mask & (1 << ctx)) ? page : NULL;
2006 qat_hal_set_live_ctx(handle, (unsigned char)ae,
2007 image->ctx_assigned);
2008 qat_hal_set_pc(handle, (unsigned char)ae, image->ctx_assigned,
2009 image->entry_address);
2010 }
2011 }
2012
2013 static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle)
2014 {
2015 unsigned int i;
2016 struct icp_qat_fw_auth_desc *desc = NULL;
2017 struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle;
2018 struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr;
2019
2020 for (i = 0; i < sobj_handle->img_table.num_simgs; i++) {
2021 if (qat_uclo_map_auth_fw(handle,
2022 (char *)simg_hdr[i].simg_buf,
2023 (unsigned int)
2024 simg_hdr[i].simg_len,
2025 &desc))
2026 goto wr_err;
2027 if (qat_uclo_auth_fw(handle, desc))
2028 goto wr_err;
2029 if (qat_uclo_is_broadcast(handle, i)) {
2030 if (qat_uclo_broadcast_load_fw(handle, desc))
2031 goto wr_err;
2032 } else {
2033 if (qat_uclo_load_fw(handle, desc))
2034 goto wr_err;
2035 }
2036 qat_uclo_ummap_auth_fw(handle, &desc);
2037 }
2038 return 0;
2039 wr_err:
2040 qat_uclo_ummap_auth_fw(handle, &desc);
2041 return -EINVAL;
2042 }
2043
2044 static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle)
2045 {
2046 struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle;
2047 unsigned int i;
2048
2049 if (qat_uclo_init_globals(handle))
2050 return -EINVAL;
2051 for (i = 0; i < obj_handle->uimage_num; i++) {
2052 if (!obj_handle->ae_uimage[i].img_ptr)
2053 return -EINVAL;
2054 if (qat_uclo_init_ustore(handle, &obj_handle->ae_uimage[i]))
2055 return -EINVAL;
2056 qat_uclo_wr_uimage_page(handle,
2057 obj_handle->ae_uimage[i].img_ptr);
2058 }
2059 return 0;
2060 }
2061
2062 int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle)
2063 {
2064 return (handle->chip_info->fw_auth) ? qat_uclo_wr_suof_img(handle) :
2065 qat_uclo_wr_uof_img(handle);
2066 }
2067
2068 int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle,
2069 unsigned int cfg_ae_mask)
2070 {
2071 if (!cfg_ae_mask)
2072 return -EINVAL;
2073
2074 handle->cfg_ae_mask = cfg_ae_mask;
2075 return 0;
2076 }
Linux with added FPC-III support