avcodec/jpegxl_parse{,r}: fix integer overflow for some malformed files
[FFMpeg-mirror.git] / libavfilter / x86 / vf_w3fdif.asm
blob3010469f97fb7a267d2b598c40c61243aff4146e
1 ;*****************************************************************************
2 ;* x86-optimized functions for w3fdif filter
3 ;*
4 ;* Copyright (c) 2015 Paul B Mahol
5 ;*
6 ;* This file is part of FFmpeg.
7 ;*
8 ;* FFmpeg is free software; you can redistribute it and/or
9 ;* modify it under the terms of the GNU Lesser General Public
10 ;* License as published by the Free Software Foundation; either
11 ;* version 2.1 of the License, or (at your option) any later version.
13 ;* FFmpeg is distributed in the hope that it will be useful,
14 ;* but WITHOUT ANY WARRANTY; without even the implied warranty of
15 ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 ;* Lesser General Public License for more details.
18 ;* You should have received a copy of the GNU Lesser General Public
19 ;* License along with FFmpeg; if not, write to the Free Software
20 ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 ;******************************************************************************
23 %include "libavutil/x86/x86util.asm"
25 SECTION .text
27 INIT_XMM sse2
28 cglobal w3fdif_scale, 3, 3, 2, 0, out_pixel, work_pixel, linesize
29 .loop:
30 mova m0, [work_pixelq]
31 mova m1, [work_pixelq+mmsize]
32 psrad m0, 15
33 psrad m1, 15
34 packssdw m0, m1
35 packuswb m0, m0
36 movh [out_pixelq], m0
37 add out_pixelq, mmsize/2
38 add work_pixelq, mmsize*2
39 sub linesized, mmsize/2
40 jg .loop
41 RET
43 cglobal w3fdif_simple_low, 4, 5, 6, 0, work_line, in_lines_cur0, coef, linesize, offset
44 movd m1, [coefq]
45 DEFINE_ARGS work_line, in_lines_cur0, in_lines_cur1, linesize, offset
46 SPLATW m0, m1, 0
47 SPLATW m1, m1, 1
48 pxor m4, m4
49 mov offsetq, 0
50 mov in_lines_cur1q, [in_lines_cur0q + gprsize]
51 mov in_lines_cur0q, [in_lines_cur0q]
53 .loop:
54 movh m2, [in_lines_cur0q+offsetq]
55 movh m3, [in_lines_cur1q+offsetq]
56 punpcklbw m2, m4
57 punpcklbw m3, m4
58 SBUTTERFLY wd, 2, 3, 5
59 pmaddwd m2, m0
60 pmaddwd m3, m1
61 mova [work_lineq+offsetq*4], m2
62 mova [work_lineq+offsetq*4+mmsize], m3
63 add offsetq, mmsize/2
64 sub linesized, mmsize/2
65 jg .loop
66 RET
68 cglobal w3fdif_complex_low, 4, 7, 8, 0, work_line, in_lines_cur0, coef, linesize
69 movq m0, [coefq]
70 DEFINE_ARGS work_line, in_lines_cur0, in_lines_cur1, linesize, offset, in_lines_cur2, in_lines_cur3
71 pshufd m2, m0, q1111
72 SPLATD m0
73 pxor m1, m1
74 mov offsetq, 0
75 mov in_lines_cur3q, [in_lines_cur0q+gprsize*3]
76 mov in_lines_cur2q, [in_lines_cur0q+gprsize*2]
77 mov in_lines_cur1q, [in_lines_cur0q+gprsize]
78 mov in_lines_cur0q, [in_lines_cur0q]
80 .loop:
81 movh m4, [in_lines_cur0q+offsetq]
82 movh m5, [in_lines_cur1q+offsetq]
83 punpcklbw m4, m1
84 punpcklbw m5, m1
85 SBUTTERFLY wd, 4, 5, 7
86 pmaddwd m4, m0
87 pmaddwd m5, m0
88 movh m6, [in_lines_cur2q+offsetq]
89 movh m3, [in_lines_cur3q+offsetq]
90 punpcklbw m6, m1
91 punpcklbw m3, m1
92 SBUTTERFLY wd, 6, 3, 7
93 pmaddwd m6, m2
94 pmaddwd m3, m2
95 paddd m4, m6
96 paddd m5, m3
97 mova [work_lineq+offsetq*4], m4
98 mova [work_lineq+offsetq*4+mmsize], m5
99 add offsetq, mmsize/2
100 sub linesized, mmsize/2
101 jg .loop
104 %if ARCH_X86_64
105 cglobal w3fdif_simple_high, 5, 9, 8, 0, work_line, in_lines_cur0, in_lines_adj0, coef, linesize
106 %else
107 cglobal w3fdif_simple_high, 4, 7, 8, 0, work_line, in_lines_cur0, in_lines_adj0, coef, linesize
108 %endif
109 movq m2, [coefq]
110 %if ARCH_X86_64
111 DEFINE_ARGS work_line, in_lines_cur0, in_lines_adj0, in_lines_cur1, linesize, offset, in_lines_cur2, in_lines_adj1, in_lines_adj2
112 xor offsetq, offsetq
113 %else
114 DEFINE_ARGS work_line, in_lines_cur0, in_lines_adj0, in_lines_cur1, in_lines_cur2, in_lines_adj1, in_lines_adj2
115 %define linesized r4mp
116 %endif
118 pshufd m0, m2, q0000
119 SPLATW m2, m2, 2
120 pxor m7, m7
121 mov in_lines_cur2q, [in_lines_cur0q+gprsize*2]
122 mov in_lines_cur1q, [in_lines_cur0q+gprsize]
123 mov in_lines_cur0q, [in_lines_cur0q]
124 mov in_lines_adj2q, [in_lines_adj0q+gprsize*2]
125 mov in_lines_adj1q, [in_lines_adj0q+gprsize]
126 mov in_lines_adj0q, [in_lines_adj0q]
128 %if ARCH_X86_32
129 sub in_lines_cur1q, in_lines_cur0q
130 sub in_lines_cur2q, in_lines_cur0q
131 sub in_lines_adj0q, in_lines_cur0q
132 sub in_lines_adj1q, in_lines_cur0q
133 sub in_lines_adj2q, in_lines_cur0q
134 %define offsetq in_lines_cur0q
135 %endif
137 .loop:
138 %if ARCH_X86_64
139 movh m3, [in_lines_cur0q+offsetq]
140 %else
141 movh m3, [in_lines_cur0q]
142 %endif
143 movh m4, [in_lines_cur1q+offsetq]
144 punpcklbw m3, m7
145 punpcklbw m4, m7
146 SBUTTERFLY wd, 3, 4, 1
147 pmaddwd m3, m0
148 pmaddwd m4, m0
149 movh m5, [in_lines_adj0q+offsetq]
150 movh m6, [in_lines_adj1q+offsetq]
151 punpcklbw m5, m7
152 punpcklbw m6, m7
153 SBUTTERFLY wd, 5, 6, 1
154 pmaddwd m5, m0
155 pmaddwd m6, m0
156 paddd m3, m5
157 paddd m4, m6
158 movh m5, [in_lines_cur2q+offsetq]
159 movh m6, [in_lines_adj2q+offsetq]
160 punpcklbw m5, m7
161 punpcklbw m6, m7
162 SBUTTERFLY wd, 5, 6, 1
163 pmaddwd m5, m2
164 pmaddwd m6, m2
165 paddd m3, m5
166 paddd m4, m6
167 %if ARCH_X86_64
168 paddd m3, [work_lineq+offsetq*4]
169 paddd m4, [work_lineq+offsetq*4+mmsize]
170 mova [work_lineq+offsetq*4], m3
171 mova [work_lineq+offsetq*4+mmsize], m4
172 %else
173 paddd m3, [work_lineq]
174 paddd m4, [work_lineq+mmsize]
175 mova [work_lineq], m3
176 mova [work_lineq+mmsize], m4
177 add work_lineq, mmsize*2
178 %endif
179 add offsetq, mmsize/2
180 sub linesized, mmsize/2
181 jg .loop
184 %if ARCH_X86_64
186 cglobal w3fdif_complex_high, 5, 13, 10, 0, work_line, in_lines_cur0, in_lines_adj0, coef, linesize
187 movq m0, [coefq+0]
188 movd m4, [coefq+8]
189 DEFINE_ARGS work_line, in_lines_cur0, in_lines_adj0, in_lines_cur1, linesize, offset, in_lines_cur2, in_lines_cur3, in_lines_cur4, in_lines_adj1, in_lines_adj2, in_lines_adj3, in_lines_adj4
190 pshufd m1, m0, q1111
191 SPLATD m0
192 SPLATW m4, m4
193 pxor m3, m3
194 mov offsetq, 0
195 mov in_lines_cur4q, [in_lines_cur0q+gprsize*4]
196 mov in_lines_cur3q, [in_lines_cur0q+gprsize*3]
197 mov in_lines_cur2q, [in_lines_cur0q+gprsize*2]
198 mov in_lines_cur1q, [in_lines_cur0q+gprsize]
199 mov in_lines_cur0q, [in_lines_cur0q]
200 mov in_lines_adj4q, [in_lines_adj0q+gprsize*4]
201 mov in_lines_adj3q, [in_lines_adj0q+gprsize*3]
202 mov in_lines_adj2q, [in_lines_adj0q+gprsize*2]
203 mov in_lines_adj1q, [in_lines_adj0q+gprsize]
204 mov in_lines_adj0q, [in_lines_adj0q]
206 .loop:
207 movh m5, [in_lines_cur0q+offsetq]
208 movh m6, [in_lines_cur1q+offsetq]
209 punpcklbw m5, m3
210 punpcklbw m6, m3
211 SBUTTERFLY wd, 5, 6, 2
212 pmaddwd m5, m0
213 pmaddwd m6, m0
214 movh m8, [in_lines_cur2q+offsetq]
215 movh m9, [in_lines_cur3q+offsetq]
216 punpcklbw m8, m3
217 punpcklbw m9, m3
218 SBUTTERFLY wd, 8, 9, 2
219 pmaddwd m8, m1
220 pmaddwd m9, m1
221 paddd m5, m8
222 paddd m6, m9
223 movh m8, [in_lines_adj0q+offsetq]
224 movh m9, [in_lines_adj1q+offsetq]
225 punpcklbw m8, m3
226 punpcklbw m9, m3
227 SBUTTERFLY wd, 8, 9, 2
228 pmaddwd m8, m0
229 pmaddwd m9, m0
230 paddd m5, m8
231 paddd m6, m9
232 movh m8, [in_lines_adj2q+offsetq]
233 movh m9, [in_lines_adj3q+offsetq]
234 punpcklbw m8, m3
235 punpcklbw m9, m3
236 SBUTTERFLY wd, 8, 9, 2
237 pmaddwd m8, m1
238 pmaddwd m9, m1
239 paddd m5, m8
240 paddd m6, m9
241 movh m8, [in_lines_cur4q+offsetq]
242 movh m9, [in_lines_adj4q+offsetq]
243 punpcklbw m8, m3
244 punpcklbw m9, m3
245 SBUTTERFLY wd, 8, 9, 2
246 pmaddwd m8, m4
247 pmaddwd m9, m4
248 paddd m5, m8
249 paddd m6, m9
250 paddd m5, [work_lineq+offsetq*4]
251 paddd m6, [work_lineq+offsetq*4+mmsize]
252 mova [work_lineq+offsetq*4], m5
253 mova [work_lineq+offsetq*4+mmsize], m6
254 add offsetq, mmsize/2
255 sub linesized, mmsize/2
256 jg .loop
259 %endif