| blob | 03e21f1864671fcea65ea662ab5706693faf9032 |
1 /*
2 * Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
21 #include "asm.S"
23 .fpu neon
25 .macro transpose_8x8 r0 r1 r2 r3 r4 r5 r6 r7
26 vtrn.32 \r0, \r4
27 vtrn.32 \r1, \r5
28 vtrn.32 \r2, \r6
29 vtrn.32 \r3, \r7
30 vtrn.16 \r0, \r2
31 vtrn.16 \r1, \r3
32 vtrn.16 \r4, \r6
33 vtrn.16 \r5, \r7
34 vtrn.8 \r0, \r1
35 vtrn.8 \r2, \r3
36 vtrn.8 \r4, \r5
37 vtrn.8 \r6, \r7
38 .endm
40 .macro transpose_4x4 r0 r1 r2 r3
41 vtrn.16 \r0, \r2
42 vtrn.16 \r1, \r3
43 vtrn.8 \r0, \r1
44 vtrn.8 \r2, \r3
45 .endm
47 .macro swap4 r0 r1 r2 r3 r4 r5 r6 r7
48 vswp \r0, \r4
49 vswp \r1, \r5
50 vswp \r2, \r6
51 vswp \r3, \r7
52 .endm
54 .macro transpose16_4x4 r0 r1 r2 r3 r4 r5 r6 r7
55 vtrn.32 \r0, \r2
56 vtrn.32 \r1, \r3
57 vtrn.32 \r4, \r6
58 vtrn.32 \r5, \r7
59 vtrn.16 \r0, \r1
60 vtrn.16 \r2, \r3
61 vtrn.16 \r4, \r5
62 vtrn.16 \r6, \r7
63 .endm
65 /* chroma_mc8(uint8_t *dst, uint8_t *src, int stride, int h, int x, int y) */
66 .macro h264_chroma_mc8 type
67 function ff_\type\()_h264_chroma_mc8_neon, export=1
68 push {r4-r7, lr}
69 ldrd r4, [sp, #20]
70 .ifc \type,avg
71 mov lr, r0
72 .endif
73 pld [r1]
74 pld [r1, r2]
76 muls r7, r4, r5
77 rsb r6, r7, r5, lsl #3
78 rsb ip, r7, r4, lsl #3
79 sub r4, r7, r4, lsl #3
80 sub r4, r4, r5, lsl #3
81 add r4, r4, #64
83 beq 2f
85 add r5, r1, r2
87 vdup.8 d0, r4
88 lsl r4, r2, #1
89 vdup.8 d1, ip
90 vld1.64 {d4, d5}, [r1], r4
91 vdup.8 d2, r6
92 vld1.64 {d6, d7}, [r5], r4
93 vdup.8 d3, r7
95 vext.8 d5, d4, d5, #1
96 vext.8 d7, d6, d7, #1
98 1: pld [r5]
99 vmull.u8 q8, d4, d0
100 vmlal.u8 q8, d5, d1
101 vld1.64 {d4, d5}, [r1], r4
102 vmlal.u8 q8, d6, d2
103 vext.8 d5, d4, d5, #1
104 vmlal.u8 q8, d7, d3
105 vmull.u8 q9, d6, d0
106 subs r3, r3, #2
107 vmlal.u8 q9, d7, d1
108 vmlal.u8 q9, d4, d2
109 vmlal.u8 q9, d5, d3
110 vrshrn.u16 d16, q8, #6
111 vld1.64 {d6, d7}, [r5], r4
112 pld [r1]
113 vrshrn.u16 d17, q9, #6
114 .ifc \type,avg
115 vld1.64 {d20}, [lr,:64], r2
116 vld1.64 {d21}, [lr,:64], r2
117 vrhadd.u8 q8, q8, q10
118 .endif
119 vext.8 d7, d6, d7, #1
120 vst1.64 {d16}, [r0,:64], r2
121 vst1.64 {d17}, [r0,:64], r2
122 bgt 1b
124 pop {r4-r7, pc}
126 2: tst r6, r6
127 add ip, ip, r6
128 vdup.8 d0, r4
129 vdup.8 d1, ip
131 beq 4f
133 add r5, r1, r2
134 lsl r4, r2, #1
135 vld1.64 {d4}, [r1], r4
136 vld1.64 {d6}, [r5], r4
138 3: pld [r5]
139 vmull.u8 q8, d4, d0
140 vmlal.u8 q8, d6, d1
141 vld1.64 {d4}, [r1], r4
142 vmull.u8 q9, d6, d0
143 vmlal.u8 q9, d4, d1
144 vld1.64 {d6}, [r5], r4
145 vrshrn.u16 d16, q8, #6
146 vrshrn.u16 d17, q9, #6
147 .ifc \type,avg
148 vld1.64 {d20}, [lr,:64], r2
149 vld1.64 {d21}, [lr,:64], r2
150 vrhadd.u8 q8, q8, q10
151 .endif
152 subs r3, r3, #2
153 pld [r1]
154 vst1.64 {d16}, [r0,:64], r2
155 vst1.64 {d17}, [r0,:64], r2
156 bgt 3b
158 pop {r4-r7, pc}
160 4: vld1.64 {d4, d5}, [r1], r2
161 vld1.64 {d6, d7}, [r1], r2
162 vext.8 d5, d4, d5, #1
163 vext.8 d7, d6, d7, #1
165 5: pld [r1]
166 subs r3, r3, #2
167 vmull.u8 q8, d4, d0
168 vmlal.u8 q8, d5, d1
169 vld1.64 {d4, d5}, [r1], r2
170 vmull.u8 q9, d6, d0
171 vmlal.u8 q9, d7, d1
172 pld [r1]
173 vext.8 d5, d4, d5, #1
174 vrshrn.u16 d16, q8, #6
175 vrshrn.u16 d17, q9, #6
176 .ifc \type,avg
177 vld1.64 {d20}, [lr,:64], r2
178 vld1.64 {d21}, [lr,:64], r2
179 vrhadd.u8 q8, q8, q10
180 .endif
181 vld1.64 {d6, d7}, [r1], r2
182 vext.8 d7, d6, d7, #1
183 vst1.64 {d16}, [r0,:64], r2
184 vst1.64 {d17}, [r0,:64], r2
185 bgt 5b
187 pop {r4-r7, pc}
188 .endfunc
189 .endm
191 /* chroma_mc4(uint8_t *dst, uint8_t *src, int stride, int h, int x, int y) */
192 .macro h264_chroma_mc4 type
193 function ff_\type\()_h264_chroma_mc4_neon, export=1
194 push {r4-r7, lr}
195 ldrd r4, [sp, #20]
196 .ifc \type,avg
197 mov lr, r0
198 .endif
199 pld [r1]
200 pld [r1, r2]
202 muls r7, r4, r5
203 rsb r6, r7, r5, lsl #3
204 rsb ip, r7, r4, lsl #3
205 sub r4, r7, r4, lsl #3
206 sub r4, r4, r5, lsl #3
207 add r4, r4, #64
209 beq 2f
211 add r5, r1, r2
213 vdup.8 d0, r4
214 lsl r4, r2, #1
215 vdup.8 d1, ip
216 vld1.64 {d4}, [r1], r4
217 vdup.8 d2, r6
218 vld1.64 {d6}, [r5], r4
219 vdup.8 d3, r7
221 vext.8 d5, d4, d5, #1
222 vext.8 d7, d6, d7, #1
223 vtrn.32 d4, d5
224 vtrn.32 d6, d7
226 vtrn.32 d0, d1
227 vtrn.32 d2, d3
229 1: pld [r5]
230 vmull.u8 q8, d4, d0
231 vmlal.u8 q8, d6, d2
232 vld1.64 {d4}, [r1], r4
233 vext.8 d5, d4, d5, #1
234 vtrn.32 d4, d5
235 vmull.u8 q9, d6, d0
236 vmlal.u8 q9, d4, d2
237 vld1.64 {d6}, [r5], r4
238 vadd.i16 d16, d16, d17
239 vadd.i16 d17, d18, d19
240 vrshrn.u16 d16, q8, #6
241 subs r3, r3, #2
242 pld [r1]
243 .ifc \type,avg
244 vld1.32 {d20[0]}, [lr,:32], r2
245 vld1.32 {d20[1]}, [lr,:32], r2
246 vrhadd.u8 d16, d16, d20
247 .endif
248 vext.8 d7, d6, d7, #1
249 vtrn.32 d6, d7
250 vst1.32 {d16[0]}, [r0,:32], r2
251 vst1.32 {d16[1]}, [r0,:32], r2
252 bgt 1b
254 pop {r4-r7, pc}
256 2: tst r6, r6
257 add ip, ip, r6
258 vdup.8 d0, r4
259 vdup.8 d1, ip
260 vtrn.32 d0, d1
262 beq 4f
264 vext.32 d1, d0, d1, #1
265 add r5, r1, r2
266 lsl r4, r2, #1
267 vld1.32 {d4[0]}, [r1], r4
268 vld1.32 {d4[1]}, [r5], r4
270 3: pld [r5]
271 vmull.u8 q8, d4, d0
272 vld1.32 {d4[0]}, [r1], r4
273 vmull.u8 q9, d4, d1
274 vld1.32 {d4[1]}, [r5], r4
275 vadd.i16 d16, d16, d17
276 vadd.i16 d17, d18, d19
277 vrshrn.u16 d16, q8, #6
278 .ifc \type,avg
279 vld1.32 {d20[0]}, [lr,:32], r2
280 vld1.32 {d20[1]}, [lr,:32], r2
281 vrhadd.u8 d16, d16, d20
282 .endif
283 subs r3, r3, #2
284 pld [r1]
285 vst1.32 {d16[0]}, [r0,:32], r2
286 vst1.32 {d16[1]}, [r0,:32], r2
287 bgt 3b
289 pop {r4-r7, pc}
291 4: vld1.64 {d4}, [r1], r2
292 vld1.64 {d6}, [r1], r2
293 vext.8 d5, d4, d5, #1
294 vext.8 d7, d6, d7, #1
295 vtrn.32 d4, d5
296 vtrn.32 d6, d7
298 5: vmull.u8 q8, d4, d0
299 vmull.u8 q9, d6, d0
300 subs r3, r3, #2
301 vld1.64 {d4}, [r1], r2
302 vext.8 d5, d4, d5, #1
303 vtrn.32 d4, d5
304 vadd.i16 d16, d16, d17
305 vadd.i16 d17, d18, d19
306 pld [r1]
307 vrshrn.u16 d16, q8, #6
308 .ifc \type,avg
309 vld1.32 {d20[0]}, [lr,:32], r2
310 vld1.32 {d20[1]}, [lr,:32], r2
311 vrhadd.u8 d16, d16, d20
312 .endif
313 vld1.64 {d6}, [r1], r2
314 vext.8 d7, d6, d7, #1
315 vtrn.32 d6, d7
316 pld [r1]
317 vst1.32 {d16[0]}, [r0,:32], r2
318 vst1.32 {d16[1]}, [r0,:32], r2
319 bgt 5b
321 pop {r4-r7, pc}
322 .endfunc
323 .endm
325 .text
326 .align
328 h264_chroma_mc8 put
329 h264_chroma_mc8 avg
330 h264_chroma_mc4 put
331 h264_chroma_mc4 avg
333 /* H.264 loop filter */
335 .macro h264_loop_filter_start
336 ldr ip, [sp]
337 tst r2, r2
338 ldr ip, [ip]
339 tstne r3, r3
340 vmov.32 d24[0], ip
341 and ip, ip, ip, lsl #16
342 bxeq lr
343 ands ip, ip, ip, lsl #8
344 bxlt lr
345 .endm
347 .macro align_push_regs
348 and ip, sp, #15
349 add ip, ip, #32
350 sub sp, sp, ip
351 vst1.64 {d12-d15}, [sp,:128]
352 sub sp, sp, #32
353 vst1.64 {d8-d11}, [sp,:128]
354 .endm
356 .macro align_pop_regs
357 vld1.64 {d8-d11}, [sp,:128]!
358 vld1.64 {d12-d15}, [sp,:128], ip
359 .endm
361 .macro h264_loop_filter_luma
362 vdup.8 q11, r2 @ alpha
363 vmovl.u8 q12, d24
364 vabd.u8 q6, q8, q0 @ abs(p0 - q0)
365 vmovl.u16 q12, d24
366 vabd.u8 q14, q9, q8 @ abs(p1 - p0)
367 vsli.16 q12, q12, #8
368 vabd.u8 q15, q1, q0 @ abs(q1 - q0)
369 vsli.32 q12, q12, #16
370 vclt.u8 q6, q6, q11 @ < alpha
371 vdup.8 q11, r3 @ beta
372 vclt.s8 q7, q12, #0
373 vclt.u8 q14, q14, q11 @ < beta
374 vclt.u8 q15, q15, q11 @ < beta
375 vbic q6, q6, q7
376 vabd.u8 q4, q10, q8 @ abs(p2 - p0)
377 vand q6, q6, q14
378 vabd.u8 q5, q2, q0 @ abs(q2 - q0)
379 vclt.u8 q4, q4, q11 @ < beta
380 vand q6, q6, q15
381 vclt.u8 q5, q5, q11 @ < beta
382 vand q4, q4, q6
383 vand q5, q5, q6
384 vand q12, q12, q6
385 vrhadd.u8 q14, q8, q0
386 vsub.i8 q6, q12, q4
387 vqadd.u8 q7, q9, q12
388 vhadd.u8 q10, q10, q14
389 vsub.i8 q6, q6, q5
390 vhadd.u8 q14, q2, q14
391 vmin.u8 q7, q7, q10
392 vqsub.u8 q11, q9, q12
393 vqadd.u8 q2, q1, q12
394 vmax.u8 q7, q7, q11
395 vqsub.u8 q11, q1, q12
396 vmin.u8 q14, q2, q14
397 vmovl.u8 q2, d0
398 vmax.u8 q14, q14, q11
399 vmovl.u8 q10, d1
400 vsubw.u8 q2, q2, d16
401 vsubw.u8 q10, q10, d17
402 vshl.i16 q2, q2, #2
403 vshl.i16 q10, q10, #2
404 vaddw.u8 q2, q2, d18
405 vaddw.u8 q10, q10, d19
406 vsubw.u8 q2, q2, d2
407 vsubw.u8 q10, q10, d3
408 vrshrn.i16 d4, q2, #3
409 vrshrn.i16 d5, q10, #3
410 vbsl q4, q7, q9
411 vbsl q5, q14, q1
412 vneg.s8 q7, q6
413 vmovl.u8 q14, d16
414 vmin.s8 q2, q2, q6
415 vmovl.u8 q6, d17
416 vmax.s8 q2, q2, q7
417 vmovl.u8 q11, d0
418 vmovl.u8 q12, d1
419 vaddw.s8 q14, q14, d4
420 vaddw.s8 q6, q6, d5
421 vsubw.s8 q11, q11, d4
422 vsubw.s8 q12, q12, d5
423 vqmovun.s16 d16, q14
424 vqmovun.s16 d17, q6
425 vqmovun.s16 d0, q11
426 vqmovun.s16 d1, q12
427 .endm
429 function ff_h264_v_loop_filter_luma_neon, export=1
430 h264_loop_filter_start
432 vld1.64 {d0, d1}, [r0,:128], r1
433 vld1.64 {d2, d3}, [r0,:128], r1
434 vld1.64 {d4, d5}, [r0,:128], r1
435 sub r0, r0, r1, lsl #2
436 sub r0, r0, r1, lsl #1
437 vld1.64 {d20,d21}, [r0,:128], r1
438 vld1.64 {d18,d19}, [r0,:128], r1
439 vld1.64 {d16,d17}, [r0,:128], r1
441 align_push_regs
443 h264_loop_filter_luma
445 sub r0, r0, r1, lsl #1
446 vst1.64 {d8, d9}, [r0,:128], r1
447 vst1.64 {d16,d17}, [r0,:128], r1
448 vst1.64 {d0, d1}, [r0,:128], r1
449 vst1.64 {d10,d11}, [r0,:128]
451 align_pop_regs
452 bx lr
453 .endfunc
455 function ff_h264_h_loop_filter_luma_neon, export=1
456 h264_loop_filter_start
458 sub r0, r0, #4
459 vld1.64 {d6}, [r0], r1
460 vld1.64 {d20}, [r0], r1
461 vld1.64 {d18}, [r0], r1
462 vld1.64 {d16}, [r0], r1
463 vld1.64 {d0}, [r0], r1
464 vld1.64 {d2}, [r0], r1
465 vld1.64 {d4}, [r0], r1
466 vld1.64 {d26}, [r0], r1
467 vld1.64 {d7}, [r0], r1
468 vld1.64 {d21}, [r0], r1
469 vld1.64 {d19}, [r0], r1
470 vld1.64 {d17}, [r0], r1
471 vld1.64 {d1}, [r0], r1
472 vld1.64 {d3}, [r0], r1
473 vld1.64 {d5}, [r0], r1
474 vld1.64 {d27}, [r0], r1
476 transpose_8x8 q3, q10, q9, q8, q0, q1, q2, q13
478 align_push_regs
480 h264_loop_filter_luma
482 transpose_4x4 q4, q8, q0, q5
484 sub r0, r0, r1, lsl #4
485 add r0, r0, #2
486 vst1.32 {d8[0]}, [r0], r1
487 vst1.32 {d16[0]}, [r0], r1
488 vst1.32 {d0[0]}, [r0], r1
489 vst1.32 {d10[0]}, [r0], r1
490 vst1.32 {d8[1]}, [r0], r1
491 vst1.32 {d16[1]}, [r0], r1
492 vst1.32 {d0[1]}, [r0], r1
493 vst1.32 {d10[1]}, [r0], r1
494 vst1.32 {d9[0]}, [r0], r1
495 vst1.32 {d17[0]}, [r0], r1
496 vst1.32 {d1[0]}, [r0], r1
497 vst1.32 {d11[0]}, [r0], r1
498 vst1.32 {d9[1]}, [r0], r1
499 vst1.32 {d17[1]}, [r0], r1
500 vst1.32 {d1[1]}, [r0], r1
501 vst1.32 {d11[1]}, [r0], r1
503 align_pop_regs
504 bx lr
505 .endfunc
507 .macro h264_loop_filter_chroma
508 vdup.8 d22, r2 @ alpha
509 vmovl.u8 q12, d24
510 vabd.u8 d26, d16, d0 @ abs(p0 - q0)
511 vmovl.u8 q2, d0
512 vabd.u8 d28, d18, d16 @ abs(p1 - p0)
513 vsubw.u8 q2, q2, d16
514 vsli.16 d24, d24, #8
515 vshl.i16 q2, q2, #2
516 vabd.u8 d30, d2, d0 @ abs(q1 - q0)
517 vaddw.u8 q2, q2, d18
518 vclt.u8 d26, d26, d22 @ < alpha
519 vsubw.u8 q2, q2, d2
520 vdup.8 d22, r3 @ beta
521 vclt.s8 d25, d24, #0
522 vrshrn.i16 d4, q2, #3
523 vclt.u8 d28, d28, d22 @ < beta
524 vbic d26, d26, d25
525 vclt.u8 d30, d30, d22 @ < beta
526 vand d26, d26, d28
527 vneg.s8 d25, d24
528 vand d26, d26, d30
529 vmin.s8 d4, d4, d24
530 vmovl.u8 q14, d16
531 vand d4, d4, d26
532 vmax.s8 d4, d4, d25
533 vmovl.u8 q11, d0
534 vaddw.s8 q14, q14, d4
535 vsubw.s8 q11, q11, d4
536 vqmovun.s16 d16, q14
537 vqmovun.s16 d0, q11
538 .endm
540 function ff_h264_v_loop_filter_chroma_neon, export=1
541 h264_loop_filter_start
543 sub r0, r0, r1, lsl #1
544 vld1.64 {d18}, [r0,:64], r1
545 vld1.64 {d16}, [r0,:64], r1
546 vld1.64 {d0}, [r0,:64], r1
547 vld1.64 {d2}, [r0,:64]
549 h264_loop_filter_chroma
551 sub r0, r0, r1, lsl #1
552 vst1.64 {d16}, [r0,:64], r1
553 vst1.64 {d0}, [r0,:64], r1
555 bx lr
556 .endfunc
558 function ff_h264_h_loop_filter_chroma_neon, export=1
559 h264_loop_filter_start
561 sub r0, r0, #2
562 vld1.32 {d18[0]}, [r0], r1
563 vld1.32 {d16[0]}, [r0], r1
564 vld1.32 {d0[0]}, [r0], r1
565 vld1.32 {d2[0]}, [r0], r1
566 vld1.32 {d18[1]}, [r0], r1
567 vld1.32 {d16[1]}, [r0], r1
568 vld1.32 {d0[1]}, [r0], r1
569 vld1.32 {d2[1]}, [r0], r1
571 vtrn.16 d18, d0
572 vtrn.16 d16, d2
573 vtrn.8 d18, d16
574 vtrn.8 d0, d2
576 h264_loop_filter_chroma
578 vtrn.16 d18, d0
579 vtrn.16 d16, d2
580 vtrn.8 d18, d16
581 vtrn.8 d0, d2
583 sub r0, r0, r1, lsl #3
584 vst1.32 {d18[0]}, [r0], r1
585 vst1.32 {d16[0]}, [r0], r1
586 vst1.32 {d0[0]}, [r0], r1
587 vst1.32 {d2[0]}, [r0], r1
588 vst1.32 {d18[1]}, [r0], r1
589 vst1.32 {d16[1]}, [r0], r1
590 vst1.32 {d0[1]}, [r0], r1
591 vst1.32 {d2[1]}, [r0], r1
593 bx lr
594 .endfunc
596 /* H.264 qpel MC */
598 .macro lowpass_const r
599 movw \r, #5
600 movt \r, #20
601 vmov.32 d6[0], \r
602 .endm
604 .macro lowpass_8 r0, r1, r2, r3, d0, d1, narrow=1
605 .if \narrow
606 t0 .req q0
607 t1 .req q8
608 .else
609 t0 .req \d0
610 t1 .req \d1
611 .endif
612 vext.8 d2, \r0, \r1, #2
613 vext.8 d3, \r0, \r1, #3
614 vaddl.u8 q1, d2, d3
615 vext.8 d4, \r0, \r1, #1
616 vext.8 d5, \r0, \r1, #4
617 vaddl.u8 q2, d4, d5
618 vext.8 d30, \r0, \r1, #5
619 vaddl.u8 t0, \r0, d30
620 vext.8 d18, \r2, \r3, #2
621 vmla.i16 t0, q1, d6[1]
622 vext.8 d19, \r2, \r3, #3
623 vaddl.u8 q9, d18, d19
624 vext.8 d20, \r2, \r3, #1
625 vmls.i16 t0, q2, d6[0]
626 vext.8 d21, \r2, \r3, #4
627 vaddl.u8 q10, d20, d21
628 vext.8 d31, \r2, \r3, #5
629 vaddl.u8 t1, \r2, d31
630 vmla.i16 t1, q9, d6[1]
631 vmls.i16 t1, q10, d6[0]
632 .if \narrow
633 vqrshrun.s16 \d0, t0, #5
634 vqrshrun.s16 \d1, t1, #5
635 .endif
636 .unreq t0
637 .unreq t1
638 .endm
640 .macro lowpass_8_1 r0, r1, d0, narrow=1
641 .if \narrow
642 t0 .req q0
643 .else
644 t0 .req \d0
645 .endif
646 vext.8 d2, \r0, \r1, #2
647 vext.8 d3, \r0, \r1, #3
648 vaddl.u8 q1, d2, d3
649 vext.8 d4, \r0, \r1, #1
650 vext.8 d5, \r0, \r1, #4
651 vaddl.u8 q2, d4, d5
652 vext.8 d30, \r0, \r1, #5
653 vaddl.u8 t0, \r0, d30
654 vmla.i16 t0, q1, d6[1]
655 vmls.i16 t0, q2, d6[0]
656 .if \narrow
657 vqrshrun.s16 \d0, t0, #5
658 .endif
659 .unreq t0
660 .endm
662 .macro lowpass_8.16 r0, r1, l0, h0, l1, h1, d
663 vext.16 q1, \r0, \r1, #2
664 vext.16 q0, \r0, \r1, #3
665 vaddl.s16 q9, d2, d0
666 vext.16 q2, \r0, \r1, #1
667 vaddl.s16 q1, d3, d1
668 vext.16 q3, \r0, \r1, #4
669 vaddl.s16 q10, d4, d6
670 vext.16 \r1, \r0, \r1, #5
671 vaddl.s16 q2, d5, d7
672 vaddl.s16 q0, \h0, \h1
673 vaddl.s16 q8, \l0, \l1
675 vshl.i32 q3, q9, #4
676 vshl.i32 q9, q9, #2
677 vshl.i32 q15, q10, #2
678 vadd.i32 q9, q9, q3
679 vadd.i32 q10, q10, q15
681 vshl.i32 q3, q1, #4
682 vshl.i32 q1, q1, #2
683 vshl.i32 q15, q2, #2
684 vadd.i32 q1, q1, q3
685 vadd.i32 q2, q2, q15
687 vadd.i32 q9, q9, q8
688 vsub.i32 q9, q9, q10
690 vadd.i32 q1, q1, q0
691 vsub.i32 q1, q1, q2
693 vrshrn.s32 d18, q9, #10
694 vrshrn.s32 d19, q1, #10
696 vqmovun.s16 \d, q9
697 .endm
699 function put_h264_qpel16_h_lowpass_neon_packed
700 mov r4, lr
701 mov ip, #16
702 mov r3, #8
703 bl put_h264_qpel8_h_lowpass_neon
704 sub r1, r1, r2, lsl #4
705 add r1, r1, #8
706 mov ip, #16
707 mov lr, r4
708 b put_h264_qpel8_h_lowpass_neon
709 .endfunc
711 function put_h264_qpel16_h_lowpass_neon
712 push {lr}
713 mov ip, #16
714 bl put_h264_qpel8_h_lowpass_neon
715 sub r0, r0, r3, lsl #4
716 sub r1, r1, r2, lsl #4
717 add r0, r0, #8
718 add r1, r1, #8
719 mov ip, #16
720 pop {lr}
721 .endfunc
723 function put_h264_qpel8_h_lowpass_neon
724 1: vld1.64 {d0, d1}, [r1], r2
725 vld1.64 {d16,d17}, [r1], r2
726 subs ip, ip, #2
727 lowpass_8 d0, d1, d16, d17, d0, d16
728 vst1.64 {d0}, [r0,:64], r3
729 vst1.64 {d16}, [r0,:64], r3
730 bne 1b
731 bx lr
732 .endfunc
734 function put_h264_qpel16_h_lowpass_l2_neon
735 push {lr}
736 mov ip, #16
737 bl put_h264_qpel8_h_lowpass_l2_neon
738 sub r0, r0, r2, lsl #4
739 sub r1, r1, r2, lsl #4
740 sub r3, r3, r2, lsl #4
741 add r0, r0, #8
742 add r1, r1, #8
743 add r3, r3, #8
744 mov ip, #16
745 pop {lr}
746 .endfunc
748 function put_h264_qpel8_h_lowpass_l2_neon
749 1: vld1.64 {d0, d1}, [r1], r2
750 vld1.64 {d16,d17}, [r1], r2
751 vld1.64 {d28}, [r3], r2
752 vld1.64 {d29}, [r3], r2
753 subs ip, ip, #2
754 lowpass_8 d0, d1, d16, d17, d0, d1
755 vrhadd.u8 q0, q0, q14
756 vst1.64 {d0}, [r0,:64], r2
757 vst1.64 {d1}, [r0,:64], r2
758 bne 1b
759 bx lr
760 .endfunc
762 function put_h264_qpel16_v_lowpass_neon_packed
763 mov r4, lr
764 mov r2, #8
765 bl put_h264_qpel8_v_lowpass_neon
766 sub r1, r1, r3, lsl #2
767 bl put_h264_qpel8_v_lowpass_neon
768 sub r1, r1, r3, lsl #4
769 sub r1, r1, r3, lsl #2
770 add r1, r1, #8
771 bl put_h264_qpel8_v_lowpass_neon
772 sub r1, r1, r3, lsl #2
773 mov lr, r4
774 b put_h264_qpel8_v_lowpass_neon
775 .endfunc
777 function put_h264_qpel16_v_lowpass_neon
778 mov r4, lr
779 bl put_h264_qpel8_v_lowpass_neon
780 sub r1, r1, r3, lsl #2
781 bl put_h264_qpel8_v_lowpass_neon
782 sub r0, r0, r2, lsl #4
783 add r0, r0, #8
784 sub r1, r1, r3, lsl #4
785 sub r1, r1, r3, lsl #2
786 add r1, r1, #8
787 bl put_h264_qpel8_v_lowpass_neon
788 sub r1, r1, r3, lsl #2
789 mov lr, r4
790 .endfunc
792 function put_h264_qpel8_v_lowpass_neon
793 vld1.64 {d8}, [r1], r3
794 vld1.64 {d10}, [r1], r3
795 vld1.64 {d12}, [r1], r3
796 vld1.64 {d14}, [r1], r3
797 vld1.64 {d22}, [r1], r3
798 vld1.64 {d24}, [r1], r3
799 vld1.64 {d26}, [r1], r3
800 vld1.64 {d28}, [r1], r3
801 vld1.64 {d9}, [r1], r3
802 vld1.64 {d11}, [r1], r3
803 vld1.64 {d13}, [r1], r3
804 vld1.64 {d15}, [r1], r3
805 vld1.64 {d23}, [r1]
807 transpose_8x8 q4, q5, q6, q7, q11, q12, q13, q14
808 lowpass_8 d8, d9, d10, d11, d8, d10
809 lowpass_8 d12, d13, d14, d15, d12, d14
810 lowpass_8 d22, d23, d24, d25, d22, d24
811 lowpass_8 d26, d27, d28, d29, d26, d28
812 transpose_8x8 d8, d10, d12, d14, d22, d24, d26, d28
814 vst1.64 {d8}, [r0,:64], r2
815 vst1.64 {d10}, [r0,:64], r2
816 vst1.64 {d12}, [r0,:64], r2
817 vst1.64 {d14}, [r0,:64], r2
818 vst1.64 {d22}, [r0,:64], r2
819 vst1.64 {d24}, [r0,:64], r2
820 vst1.64 {d26}, [r0,:64], r2
821 vst1.64 {d28}, [r0,:64], r2
823 bx lr
824 .endfunc
826 function put_h264_qpel16_v_lowpass_l2_neon
827 mov r4, lr
828 bl put_h264_qpel8_v_lowpass_l2_neon
829 sub r1, r1, r3, lsl #2
830 bl put_h264_qpel8_v_lowpass_l2_neon
831 sub r0, r0, r3, lsl #4
832 sub ip, ip, r2, lsl #4
833 add r0, r0, #8
834 add ip, ip, #8
835 sub r1, r1, r3, lsl #4
836 sub r1, r1, r3, lsl #2
837 add r1, r1, #8
838 bl put_h264_qpel8_v_lowpass_l2_neon
839 sub r1, r1, r3, lsl #2
840 mov lr, r4
841 .endfunc
843 function put_h264_qpel8_v_lowpass_l2_neon
844 vld1.64 {d8}, [r1], r3
845 vld1.64 {d10}, [r1], r3
846 vld1.64 {d12}, [r1], r3
847 vld1.64 {d14}, [r1], r3
848 vld1.64 {d22}, [r1], r3
849 vld1.64 {d24}, [r1], r3
850 vld1.64 {d26}, [r1], r3
851 vld1.64 {d28}, [r1], r3
852 vld1.64 {d9}, [r1], r3
853 vld1.64 {d11}, [r1], r3
854 vld1.64 {d13}, [r1], r3
855 vld1.64 {d15}, [r1], r3
856 vld1.64 {d23}, [r1]
858 transpose_8x8 q4, q5, q6, q7, q11, q12, q13, q14
859 lowpass_8 d8, d9, d10, d11, d8, d9
860 lowpass_8 d12, d13, d14, d15, d12, d13
861 lowpass_8 d22, d23, d24, d25, d22, d23
862 lowpass_8 d26, d27, d28, d29, d26, d27
863 transpose_8x8 d8, d9, d12, d13, d22, d23, d26, d27
865 vld1.64 {d0}, [ip], r2
866 vld1.64 {d1}, [ip], r2
867 vld1.64 {d2}, [ip], r2
868 vld1.64 {d3}, [ip], r2
869 vld1.64 {d4}, [ip], r2
870 vrhadd.u8 q0, q0, q4
871 vld1.64 {d5}, [ip], r2
872 vrhadd.u8 q1, q1, q6
873 vld1.64 {d10}, [ip], r2
874 vrhadd.u8 q2, q2, q11
875 vld1.64 {d11}, [ip], r2
877 vst1.64 {d0}, [r0,:64], r3
878 vst1.64 {d1}, [r0,:64], r3
879 vrhadd.u8 q5, q5, q13
880 vst1.64 {d2}, [r0,:64], r3
881 vst1.64 {d3}, [r0,:64], r3
882 vst1.64 {d4}, [r0,:64], r3
883 vst1.64 {d5}, [r0,:64], r3
884 vst1.64 {d10}, [r0,:64], r3
885 vst1.64 {d11}, [r0,:64], r3
887 bx lr
888 .endfunc
890 function put_h264_qpel8_hv_lowpass_neon_top
891 lowpass_const ip
892 mov ip, #12
893 1: vld1.64 {d0, d1}, [r1], r3
894 vld1.64 {d16,d17}, [r1], r3
895 subs ip, ip, #2
896 lowpass_8 d0, d1, d16, d17, q11, q12, narrow=0
897 vst1.64 {d22-d25}, [r4,:128]!
898 bne 1b
900 vld1.64 {d0, d1}, [r1]
901 lowpass_8_1 d0, d1, q12, narrow=0
903 mov ip, #-16
904 add r4, r4, ip
905 vld1.64 {d30,d31}, [r4,:128], ip
906 vld1.64 {d20,d21}, [r4,:128], ip
907 vld1.64 {d18,d19}, [r4,:128], ip
908 vld1.64 {d16,d17}, [r4,:128], ip
909 vld1.64 {d14,d15}, [r4,:128], ip
910 vld1.64 {d12,d13}, [r4,:128], ip
911 vld1.64 {d10,d11}, [r4,:128], ip
912 vld1.64 {d8, d9}, [r4,:128], ip
913 vld1.64 {d6, d7}, [r4,:128], ip
914 vld1.64 {d4, d5}, [r4,:128], ip
915 vld1.64 {d2, d3}, [r4,:128], ip
916 vld1.64 {d0, d1}, [r4,:128]
918 swap4 d1, d3, d5, d7, d8, d10, d12, d14
919 transpose16_4x4 q0, q1, q2, q3, q4, q5, q6, q7
921 swap4 d17, d19, d21, d31, d24, d26, d28, d22
922 transpose16_4x4 q8, q9, q10, q15, q12, q13, q14, q11
924 vst1.64 {d30,d31}, [r4,:128]!
925 vst1.64 {d6, d7}, [r4,:128]!
926 vst1.64 {d20,d21}, [r4,:128]!
927 vst1.64 {d4, d5}, [r4,:128]!
928 vst1.64 {d18,d19}, [r4,:128]!
929 vst1.64 {d2, d3}, [r4,:128]!
930 vst1.64 {d16,d17}, [r4,:128]!
931 vst1.64 {d0, d1}, [r4,:128]
933 lowpass_8.16 q4, q12, d8, d9, d24, d25, d8
934 lowpass_8.16 q5, q13, d10, d11, d26, d27, d9
935 lowpass_8.16 q6, q14, d12, d13, d28, d29, d10
936 lowpass_8.16 q7, q11, d14, d15, d22, d23, d11
938 vld1.64 {d16,d17}, [r4,:128], ip
939 vld1.64 {d30,d31}, [r4,:128], ip
940 lowpass_8.16 q8, q15, d16, d17, d30, d31, d12
941 vld1.64 {d16,d17}, [r4,:128], ip
942 vld1.64 {d30,d31}, [r4,:128], ip
943 lowpass_8.16 q8, q15, d16, d17, d30, d31, d13
944 vld1.64 {d16,d17}, [r4,:128], ip
945 vld1.64 {d30,d31}, [r4,:128], ip
946 lowpass_8.16 q8, q15, d16, d17, d30, d31, d14
947 vld1.64 {d16,d17}, [r4,:128], ip
948 vld1.64 {d30,d31}, [r4,:128]
949 lowpass_8.16 q8, q15, d16, d17, d30, d31, d15
951 transpose_8x8 d12, d13, d14, d15, d8, d9, d10, d11
953 bx lr
954 .endfunc
956 function put_h264_qpel8_hv_lowpass_neon
957 mov r10, lr
958 bl put_h264_qpel8_hv_lowpass_neon_top
959 vst1.64 {d12}, [r0,:64], r2
960 vst1.64 {d13}, [r0,:64], r2
961 vst1.64 {d14}, [r0,:64], r2
962 vst1.64 {d15}, [r0,:64], r2
963 vst1.64 {d8}, [r0,:64], r2
964 vst1.64 {d9}, [r0,:64], r2
965 vst1.64 {d10}, [r0,:64], r2
966 vst1.64 {d11}, [r0,:64], r2
968 mov lr, r10
969 bx lr
970 .endfunc
972 function put_h264_qpel8_hv_lowpass_l2_neon
973 mov r10, lr
974 bl put_h264_qpel8_hv_lowpass_neon_top
976 vld1.64 {d0, d1}, [r2,:128]!
977 vld1.64 {d2, d3}, [r2,:128]!
978 vrhadd.u8 q0, q0, q6
979 vld1.64 {d4, d5}, [r2,:128]!
980 vrhadd.u8 q1, q1, q7
981 vld1.64 {d6, d7}, [r2,:128]!
982 vrhadd.u8 q2, q2, q4
984 vst1.64 {d0}, [r0,:64], r3
985 vrhadd.u8 q3, q3, q5
986 vst1.64 {d1}, [r0,:64], r3
987 vst1.64 {d2}, [r0,:64], r3
988 vst1.64 {d3}, [r0,:64], r3
989 vst1.64 {d4}, [r0,:64], r3
990 vst1.64 {d5}, [r0,:64], r3
991 vst1.64 {d6}, [r0,:64], r3
992 vst1.64 {d7}, [r0,:64], r3
994 mov lr, r10
995 bx lr
996 .endfunc
998 function put_h264_qpel16_hv_lowpass_neon
999 mov r9, lr
1000 bl put_h264_qpel8_hv_lowpass_neon
1001 sub r1, r1, r3, lsl #2
1002 bl put_h264_qpel8_hv_lowpass_neon
1003 sub r1, r1, r3, lsl #4
1004 sub r1, r1, r3, lsl #2
1005 add r1, r1, #8
1006 sub r0, r0, r2, lsl #4
1007 add r0, r0, #8
1008 bl put_h264_qpel8_hv_lowpass_neon
1009 sub r1, r1, r3, lsl #2
1010 mov lr, r9
1011 b put_h264_qpel8_hv_lowpass_neon
1012 .endfunc
1014 function put_h264_qpel16_hv_lowpass_l2_neon
1015 mov r9, lr
1016 sub r2, r4, #256
1017 bl put_h264_qpel8_hv_lowpass_l2_neon
1018 sub r1, r1, r3, lsl #2
1019 bl put_h264_qpel8_hv_lowpass_l2_neon
1020 sub r1, r1, r3, lsl #4
1021 sub r1, r1, r3, lsl #2
1022 add r1, r1, #8
1023 sub r0, r0, r3, lsl #4
1024 add r0, r0, #8
1025 bl put_h264_qpel8_hv_lowpass_l2_neon
1026 sub r1, r1, r3, lsl #2
1027 mov lr, r9
1028 b put_h264_qpel8_hv_lowpass_l2_neon
1029 .endfunc
1031 function ff_put_h264_qpel8_mc10_neon, export=1
1032 lowpass_const r3
1033 mov r3, r1
1034 sub r1, r1, #2
1035 mov ip, #8
1036 b put_h264_qpel8_h_lowpass_l2_neon
1037 .endfunc
1039 function ff_put_h264_qpel8_mc20_neon, export=1
1040 lowpass_const r3
1041 sub r1, r1, #2
1042 mov r3, r2
1043 mov ip, #8
1044 b put_h264_qpel8_h_lowpass_neon
1045 .endfunc
1047 function ff_put_h264_qpel8_mc30_neon, export=1
1048 lowpass_const r3
1049 add r3, r1, #1
1050 sub r1, r1, #2
1051 mov ip, #8
1052 b put_h264_qpel8_h_lowpass_l2_neon
1053 .endfunc
1055 function ff_put_h264_qpel8_mc01_neon, export=1
1056 push {lr}
1057 mov ip, r1
1058 put_h264_qpel8_mc01:
1059 lowpass_const r3
1060 mov r3, r2
1061 sub r1, r1, r2, lsl #1
1062 vpush {d8-d15}
1063 bl put_h264_qpel8_v_lowpass_l2_neon
1064 vpop {d8-d15}
1065 pop {pc}
1066 .endfunc
1068 function ff_put_h264_qpel8_mc11_neon, export=1
1069 push {r0, r1, r2, lr}
1070 put_h264_qpel8_mc11:
1071 lowpass_const r3
1072 sub sp, sp, #64
1073 mov r0, sp
1074 sub r1, r1, #2
1075 mov r3, #8
1076 mov ip, #8
1077 vpush {d8-d15}
1078 bl put_h264_qpel8_h_lowpass_neon
1079 ldrd r0, [sp, #128]
1080 mov r3, r2
1081 add ip, sp, #64
1082 sub r1, r1, r2, lsl #1
1083 mov r2, #8
1084 bl put_h264_qpel8_v_lowpass_l2_neon
1085 vpop {d8-d15}
1086 add sp, sp, #76
1087 pop {pc}
1088 .endfunc
1090 function ff_put_h264_qpel8_mc21_neon, export=1
1091 push {r0, r1, r4, r10, r11, lr}
1092 put_h264_qpel8_mc21:
1093 lowpass_const r3
1094 mov r11, sp
1095 bic sp, sp, #15
1096 sub sp, sp, #(8*8+16*12)
1097 sub r1, r1, #2
1098 mov r3, #8
1099 mov r0, sp
1100 mov ip, #8
1101 vpush {d8-d15}
1102 bl put_h264_qpel8_h_lowpass_neon
1103 mov r4, r0
1104 ldrd r0, [r11]
1105 sub r1, r1, r2, lsl #1
1106 sub r1, r1, #2
1107 mov r3, r2
1108 sub r2, r4, #64
1109 bl put_h264_qpel8_hv_lowpass_l2_neon
1110 vpop {d8-d15}
1111 add sp, r11, #8
1112 pop {r4, r10, r11, pc}
1113 .endfunc
1115 function ff_put_h264_qpel8_mc31_neon, export=1
1116 add r1, r1, #1
1117 push {r0, r1, r2, lr}
1118 sub r1, r1, #1
1119 b put_h264_qpel8_mc11
1120 .endfunc
1122 function ff_put_h264_qpel8_mc02_neon, export=1
1123 push {lr}
1124 lowpass_const r3
1125 sub r1, r1, r2, lsl #1
1126 mov r3, r2
1127 vpush {d8-d15}
1128 bl put_h264_qpel8_v_lowpass_neon
1129 vpop {d8-d15}
1130 pop {pc}
1131 .endfunc
1133 function ff_put_h264_qpel8_mc12_neon, export=1
1134 push {r0, r1, r4, r10, r11, lr}
1135 put_h264_qpel8_mc12:
1136 lowpass_const r3
1137 mov r11, sp
1138 bic sp, sp, #15
1139 sub sp, sp, #(8*8+16*12)
1140 sub r1, r1, r2, lsl #1
1141 mov r3, r2
1142 mov r2, #8
1143 mov r0, sp
1144 vpush {d8-d15}
1145 bl put_h264_qpel8_v_lowpass_neon
1146 mov r4, r0
1147 ldrd r0, [r11]
1148 sub r1, r1, r3, lsl #1
1149 sub r1, r1, #2
1150 sub r2, r4, #64
1151 bl put_h264_qpel8_hv_lowpass_l2_neon
1152 vpop {d8-d15}
1153 add sp, r11, #8
1154 pop {r4, r10, r11, pc}
1155 .endfunc
1157 function ff_put_h264_qpel8_mc22_neon, export=1
1158 push {r4, r10, r11, lr}
1159 mov r11, sp
1160 bic sp, sp, #15
1161 sub r1, r1, r2, lsl #1
1162 sub r1, r1, #2
1163 mov r3, r2
1164 sub sp, sp, #(16*12)
1165 mov r4, sp
1166 vpush {d8-d15}
1167 bl put_h264_qpel8_hv_lowpass_neon
1168 vpop {d8-d15}
1169 mov sp, r11
1170 pop {r4, r10, r11, pc}
1171 .endfunc
1173 function ff_put_h264_qpel8_mc32_neon, export=1
1174 push {r0, r1, r4, r10, r11, lr}
1175 add r1, r1, #1
1176 b put_h264_qpel8_mc12
1177 .endfunc
1179 function ff_put_h264_qpel8_mc03_neon, export=1
1180 push {lr}
1181 add ip, r1, r2
1182 b put_h264_qpel8_mc01
1183 .endfunc
1185 function ff_put_h264_qpel8_mc13_neon, export=1
1186 push {r0, r1, r2, lr}
1187 add r1, r1, r2
1188 b put_h264_qpel8_mc11
1189 .endfunc
1191 function ff_put_h264_qpel8_mc23_neon, export=1
1192 push {r0, r1, r4, r10, r11, lr}
1193 add r1, r1, r2
1194 b put_h264_qpel8_mc21
1195 .endfunc
1197 function ff_put_h264_qpel8_mc33_neon, export=1
1198 add r1, r1, #1
1199 push {r0, r1, r2, lr}
1200 add r1, r1, r2
1201 sub r1, r1, #1
1202 b put_h264_qpel8_mc11
1203 .endfunc
1205 function ff_put_h264_qpel16_mc10_neon, export=1
1206 lowpass_const r3
1207 mov r3, r1
1208 sub r1, r1, #2
1209 b put_h264_qpel16_h_lowpass_l2_neon
1210 .endfunc
1212 function ff_put_h264_qpel16_mc20_neon, export=1
1213 lowpass_const r3
1214 sub r1, r1, #2
1215 mov r3, r2
1216 b put_h264_qpel16_h_lowpass_neon
1217 .endfunc
1219 function ff_put_h264_qpel16_mc30_neon, export=1
1220 lowpass_const r3
1221 add r3, r1, #1
1222 sub r1, r1, #2
1223 b put_h264_qpel16_h_lowpass_l2_neon
1224 .endfunc
1226 function ff_put_h264_qpel16_mc01_neon, export=1
1227 push {r4, lr}
1228 mov ip, r1
1229 put_h264_qpel16_mc01:
1230 lowpass_const r3
1231 mov r3, r2
1232 sub r1, r1, r2, lsl #1
1233 vpush {d8-d15}
1234 bl put_h264_qpel16_v_lowpass_l2_neon
1235 vpop {d8-d15}
1236 pop {r4, pc}
1237 .endfunc
1239 function ff_put_h264_qpel16_mc11_neon, export=1
1240 push {r0, r1, r4, lr}
1241 put_h264_qpel16_mc11:
1242 lowpass_const r3
1243 sub sp, sp, #256
1244 mov r0, sp
1245 sub r1, r1, #2
1246 mov r3, #16
1247 vpush {d8-d15}
1248 bl put_h264_qpel16_h_lowpass_neon
1249 add r0, sp, #256
1250 ldrd r0, [r0, #64]
1251 mov r3, r2
1252 add ip, sp, #64
1253 sub r1, r1, r2, lsl #1
1254 mov r2, #16
1255 bl put_h264_qpel16_v_lowpass_l2_neon
1256 vpop {d8-d15}
1257 add sp, sp, #(256+8)
1258 pop {r4, pc}
1259 .endfunc
1261 function ff_put_h264_qpel16_mc21_neon, export=1
1262 push {r0, r1, r4-r5, r9-r11, lr}
1263 put_h264_qpel16_mc21:
1264 lowpass_const r3
1265 mov r11, sp
1266 bic sp, sp, #15
1267 sub sp, sp, #(16*16+16*12)
1268 sub r1, r1, #2
1269 mov r0, sp
1270 vpush {d8-d15}
1271 bl put_h264_qpel16_h_lowpass_neon_packed
1272 mov r4, r0
1273 ldrd r0, [r11]
1274 sub r1, r1, r2, lsl #1
1275 sub r1, r1, #2
1276 mov r3, r2
1277 bl put_h264_qpel16_hv_lowpass_l2_neon
1278 vpop {d8-d15}
1279 add sp, r11, #8
1280 pop {r4-r5, r9-r11, pc}
1281 .endfunc
1283 function ff_put_h264_qpel16_mc31_neon, export=1
1284 add r1, r1, #1
1285 push {r0, r1, r4, lr}
1286 sub r1, r1, #1
1287 b put_h264_qpel16_mc11
1288 .endfunc
1290 function ff_put_h264_qpel16_mc02_neon, export=1
1291 push {r4, lr}
1292 lowpass_const r3
1293 sub r1, r1, r2, lsl #1
1294 mov r3, r2
1295 vpush {d8-d15}
1296 bl put_h264_qpel16_v_lowpass_neon
1297 vpop {d8-d15}
1298 pop {r4, pc}
1299 .endfunc
1301 function ff_put_h264_qpel16_mc12_neon, export=1
1302 push {r0, r1, r4-r5, r9-r11, lr}
1303 put_h264_qpel16_mc12:
1304 lowpass_const r3
1305 mov r11, sp
1306 bic sp, sp, #15
1307 sub sp, sp, #(16*16+16*12)
1308 sub r1, r1, r2, lsl #1
1309 mov r0, sp
1310 mov r3, r2
1311 vpush {d8-d15}
1312 bl put_h264_qpel16_v_lowpass_neon_packed
1313 mov r4, r0
1314 ldrd r0, [r11]
1315 sub r1, r1, r3, lsl #1
1316 sub r1, r1, #2
1317 mov r2, r3
1318 bl put_h264_qpel16_hv_lowpass_l2_neon
1319 vpop {d8-d15}
1320 add sp, r11, #8
1321 pop {r4-r5, r9-r11, pc}
1322 .endfunc
1324 function ff_put_h264_qpel16_mc22_neon, export=1
1325 push {r4, r9-r11, lr}
1326 lowpass_const r3
1327 mov r11, sp
1328 bic sp, sp, #15
1329 sub r1, r1, r2, lsl #1
1330 sub r1, r1, #2
1331 mov r3, r2
1332 sub sp, sp, #(16*12)
1333 mov r4, sp
1334 vpush {d8-d15}
1335 bl put_h264_qpel16_hv_lowpass_neon
1336 vpop {d8-d15}
1337 mov sp, r11
1338 pop {r4, r9-r11, pc}
1339 .endfunc
1341 function ff_put_h264_qpel16_mc32_neon, export=1
1342 push {r0, r1, r4-r5, r9-r11, lr}
1343 add r1, r1, #1
1344 b put_h264_qpel16_mc12
1345 .endfunc
1347 function ff_put_h264_qpel16_mc03_neon, export=1
1348 push {r4, lr}
1349 add ip, r1, r2
1350 b put_h264_qpel16_mc01
1351 .endfunc
1353 function ff_put_h264_qpel16_mc13_neon, export=1
1354 push {r0, r1, r4, lr}
1355 add r1, r1, r2
1356 b put_h264_qpel16_mc11
1357 .endfunc
1359 function ff_put_h264_qpel16_mc23_neon, export=1
1360 push {r0, r1, r4-r5, r9-r11, lr}
1361 add r1, r1, r2
1362 b put_h264_qpel16_mc21
1363 .endfunc
1365 function ff_put_h264_qpel16_mc33_neon, export=1
1366 add r1, r1, #1
1367 push {r0, r1, r4, lr}
1368 add r1, r1, r2
1369 sub r1, r1, #1
1370 b put_h264_qpel16_mc11
1371 .endfunc
1373 @ Biweighted prediction
1375 .macro biweight_16 macs, macd
1376 vdup.8 d0, r4
1377 vdup.8 d1, r5
1378 vmov q2, q8
1379 vmov q3, q8
1380 1: subs ip, ip, #2
1381 vld1.8 {d20-d21},[r0,:128], r2
1382 \macd q2, d0, d20
1383 pld [r0]
1384 \macd q3, d0, d21
1385 vld1.8 {d22-d23},[r1,:128], r2
1386 \macs q2, d1, d22
1387 pld [r1]
1388 \macs q3, d1, d23
1389 vmov q12, q8
1390 vld1.8 {d28-d29},[r0,:128], r2
1391 vmov q13, q8
1392 \macd q12, d0, d28
1393 pld [r0]
1394 \macd q13, d0, d29
1395 vld1.8 {d30-d31},[r1,:128], r2
1396 \macs q12, d1, d30
1397 pld [r1]
1398 \macs q13, d1, d31
1399 vshl.s16 q2, q2, q9
1400 vshl.s16 q3, q3, q9
1401 vqmovun.s16 d4, q2
1402 vqmovun.s16 d5, q3
1403 vshl.s16 q12, q12, q9
1404 vshl.s16 q13, q13, q9
1405 vqmovun.s16 d24, q12
1406 vqmovun.s16 d25, q13
1407 vmov q3, q8
1408 vst1.8 {d4- d5}, [r6,:128], r2
1409 vmov q2, q8
1410 vst1.8 {d24-d25},[r6,:128], r2
1411 bne 1b
1412 pop {r4-r6, pc}
1413 .endm
1415 .macro biweight_8 macs, macd
1416 vdup.8 d0, r4
1417 vdup.8 d1, r5
1418 vmov q1, q8
1419 vmov q10, q8
1420 1: subs ip, ip, #2
1421 vld1.8 {d4},[r0,:64], r2
1422 \macd q1, d0, d4
1423 pld [r0]
1424 vld1.8 {d5},[r1,:64], r2
1425 \macs q1, d1, d5
1426 pld [r1]
1427 vld1.8 {d6},[r0,:64], r2
1428 \macd q10, d0, d6
1429 pld [r0]
1430 vld1.8 {d7},[r1,:64], r2
1431 \macs q10, d1, d7
1432 pld [r1]
1433 vshl.s16 q1, q1, q9
1434 vqmovun.s16 d2, q1
1435 vshl.s16 q10, q10, q9
1436 vqmovun.s16 d4, q10
1437 vmov q10, q8
1438 vst1.8 {d2},[r6,:64], r2
1439 vmov q1, q8
1440 vst1.8 {d4},[r6,:64], r2
1441 bne 1b
1442 pop {r4-r6, pc}
1443 .endm
1445 .macro biweight_4 macs, macd
1446 vdup.8 d0, r4
1447 vdup.8 d1, r5
1448 vmov q1, q8
1449 vmov q10, q8
1450 1: subs ip, ip, #4
1451 vld1.32 {d4[0]},[r0,:32], r2
1452 vld1.32 {d4[1]},[r0,:32], r2
1453 \macd q1, d0, d4
1454 pld [r0]
1455 vld1.32 {d5[0]},[r1,:32], r2
1456 vld1.32 {d5[1]},[r1,:32], r2
1457 \macs q1, d1, d5
1458 pld [r1]
1459 blt 2f
1460 vld1.32 {d6[0]},[r0,:32], r2
1461 vld1.32 {d6[1]},[r0,:32], r2
1462 \macd q10, d0, d6
1463 pld [r0]
1464 vld1.32 {d7[0]},[r1,:32], r2
1465 vld1.32 {d7[1]},[r1,:32], r2
1466 \macs q10, d1, d7
1467 pld [r1]
1468 vshl.s16 q1, q1, q9
1469 vqmovun.s16 d2, q1
1470 vshl.s16 q10, q10, q9
1471 vqmovun.s16 d4, q10
1472 vmov q10, q8
1473 vst1.32 {d2[0]},[r6,:32], r2
1474 vst1.32 {d2[1]},[r6,:32], r2
1475 vmov q1, q8
1476 vst1.32 {d4[0]},[r6,:32], r2
1477 vst1.32 {d4[1]},[r6,:32], r2
1478 bne 1b
1479 pop {r4-r6, pc}
1480 2: vshl.s16 q1, q1, q9
1481 vqmovun.s16 d2, q1
1482 vst1.32 {d2[0]},[r6,:32], r2
1483 vst1.32 {d2[1]},[r6,:32], r2
1484 pop {r4-r6, pc}
1485 .endm
1487 .macro biweight_func w
1488 function biweight_h264_pixels_\w\()_neon
1489 push {r4-r6, lr}
1490 add r4, sp, #16
1491 ldm r4, {r4-r6}
1492 lsr lr, r4, #31
1493 add r6, r6, #1
1494 eors lr, lr, r5, lsr #30
1495 orr r6, r6, #1
1496 vdup.16 q9, r3
1497 lsl r6, r6, r3
1498 vmvn q9, q9
1499 vdup.16 q8, r6
1500 mov r6, r0
1501 beq 10f
1502 subs lr, lr, #1
1503 beq 20f
1504 subs lr, lr, #1
1505 beq 30f
1506 b 40f
1507 10: biweight_\w vmlal.u8, vmlal.u8
1508 20: rsb r4, r4, #0
1509 biweight_\w vmlal.u8, vmlsl.u8
1510 30: rsb r4, r4, #0
1511 rsb r5, r5, #0
1512 biweight_\w vmlsl.u8, vmlsl.u8
1513 40: rsb r5, r5, #0
1514 biweight_\w vmlsl.u8, vmlal.u8
1515 .endfunc
1516 .endm
1518 .macro biweight_entry w, h, b=1
1519 function ff_biweight_h264_pixels_\w\()x\h\()_neon, export=1
1520 mov ip, #\h
1521 .if \b
1522 b biweight_h264_pixels_\w\()_neon
1523 .endif
1524 .endfunc
1525 .endm
1527 biweight_entry 16, 8
1528 biweight_entry 16, 16, b=0
1529 biweight_func 16
1531 biweight_entry 8, 16
1532 biweight_entry 8, 4
1533 biweight_entry 8, 8, b=0
1534 biweight_func 8
1536 biweight_entry 4, 8
1537 biweight_entry 4, 2
1538 biweight_entry 4, 4, b=0
1539 biweight_func 4
1541 @ Weighted prediction
1543 .macro weight_16 add
1544 vdup.8 d0, r3
1545 1: subs ip, ip, #2
1546 vld1.8 {d20-d21},[r0,:128], r1
1547 vmull.u8 q2, d0, d20
1548 pld [r0]
1549 vmull.u8 q3, d0, d21
1550 vld1.8 {d28-d29},[r0,:128], r1
1551 vmull.u8 q12, d0, d28
1552 pld [r0]
1553 vmull.u8 q13, d0, d29
1554 \add q2, q8, q2
1555 vrshl.s16 q2, q2, q9
1556 \add q3, q8, q3
1557 vrshl.s16 q3, q3, q9
1558 vqmovun.s16 d4, q2
1559 vqmovun.s16 d5, q3
1560 \add q12, q8, q12
1561 vrshl.s16 q12, q12, q9
1562 \add q13, q8, q13
1563 vrshl.s16 q13, q13, q9
1564 vqmovun.s16 d24, q12
1565 vqmovun.s16 d25, q13
1566 vst1.8 {d4- d5}, [r4,:128], r1
1567 vst1.8 {d24-d25},[r4,:128], r1
1568 bne 1b
1569 pop {r4, pc}
1570 .endm
1572 .macro weight_8 add
1573 vdup.8 d0, r3
1574 1: subs ip, ip, #2
1575 vld1.8 {d4},[r0,:64], r1
1576 vmull.u8 q1, d0, d4
1577 pld [r0]
1578 vld1.8 {d6},[r0,:64], r1
1579 vmull.u8 q10, d0, d6
1580 \add q1, q8, q1
1581 pld [r0]
1582 vrshl.s16 q1, q1, q9
1583 vqmovun.s16 d2, q1
1584 \add q10, q8, q10
1585 vrshl.s16 q10, q10, q9
1586 vqmovun.s16 d4, q10
1587 vst1.8 {d2},[r4,:64], r1
1588 vst1.8 {d4},[r4,:64], r1
1589 bne 1b
1590 pop {r4, pc}
1591 .endm
1593 .macro weight_4 add
1594 vdup.8 d0, r3
1595 vmov q1, q8
1596 vmov q10, q8
1597 1: subs ip, ip, #4
1598 vld1.32 {d4[0]},[r0,:32], r1
1599 vld1.32 {d4[1]},[r0,:32], r1
1600 vmull.u8 q1, d0, d4
1601 pld [r0]
1602 blt 2f
1603 vld1.32 {d6[0]},[r0,:32], r1
1604 vld1.32 {d6[1]},[r0,:32], r1
1605 vmull.u8 q10, d0, d6
1606 pld [r0]
1607 \add q1, q8, q1
1608 vrshl.s16 q1, q1, q9
1609 vqmovun.s16 d2, q1
1610 \add q10, q8, q10
1611 vrshl.s16 q10, q10, q9
1612 vqmovun.s16 d4, q10
1613 vmov q10, q8
1614 vst1.32 {d2[0]},[r4,:32], r1
1615 vst1.32 {d2[1]},[r4,:32], r1
1616 vmov q1, q8
1617 vst1.32 {d4[0]},[r4,:32], r1
1618 vst1.32 {d4[1]},[r4,:32], r1
1619 bne 1b
1620 pop {r4, pc}
1621 2: \add q1, q8, q1
1622 vrshl.s16 q1, q1, q9
1623 vqmovun.s16 d2, q1
1624 vst1.32 {d2[0]},[r4,:32], r1
1625 vst1.32 {d2[1]},[r4,:32], r1
1626 pop {r4, pc}
1627 .endm
1629 .macro weight_func w
1630 function weight_h264_pixels_\w\()_neon
1631 push {r4, lr}
1632 ldr r4, [sp, #8]
1633 cmp r2, #1
1634 lsl r4, r4, r2
1635 vdup.16 q8, r4
1636 mov r4, r0
1637 ble 20f
1638 rsb lr, r2, #1
1639 vdup.16 q9, lr
1640 cmp r3, #0
1641 blt 10f
1642 weight_\w vhadd.s16
1643 10: rsb r3, r3, #0
1644 weight_\w vhsub.s16
1645 20: rsb lr, r2, #0
1646 vdup.16 q9, lr
1647 cmp r3, #0
1648 blt 10f
1649 weight_\w vadd.s16
1650 10: rsb r3, r3, #0
1651 weight_\w vsub.s16
1652 .endfunc
1653 .endm
1655 .macro weight_entry w, h, b=1
1656 function ff_weight_h264_pixels_\w\()x\h\()_neon, export=1
1657 mov ip, #\h
1658 .if \b
1659 b weight_h264_pixels_\w\()_neon
1660 .endif
1661 .endfunc
1662 .endm
1664 weight_entry 16, 8
1665 weight_entry 16, 16, b=0
1666 weight_func 16
1668 weight_entry 8, 16
1669 weight_entry 8, 4
1670 weight_entry 8, 8, b=0
1671 weight_func 8
1673 weight_entry 4, 8
1674 weight_entry 4, 2
1675 weight_entry 4, 4, b=0
1676 weight_func 4