store quant_shift as an unsigned char
[libvpx.git] / vp8 / encoder / quantize.c
blob86ed267fb084c3613154d019dffaac92a96fc9b9
1 /*
2 * Copyright (c) 2010 The WebM project authors. All Rights Reserved.
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
12 #include <math.h>
13 #include "vpx_mem/vpx_mem.h"
15 #include "quantize.h"
16 #include "vp8/common/entropy.h"
18 #define EXACT_QUANT
20 #ifdef EXACT_FASTQUANT
21 void vp8_fast_quantize_b_c(BLOCK *b, BLOCKD *d)
23 int i, rc, eob;
24 int zbin;
25 int x, y, z, sz;
26 short *coeff_ptr = b->coeff;
27 short *zbin_ptr = b->zbin;
28 short *round_ptr = b->round;
29 short *quant_ptr = b->quant_fast;
30 unsigned char *quant_shift_ptr = b->quant_shift;
31 short *qcoeff_ptr = d->qcoeff;
32 short *dqcoeff_ptr = d->dqcoeff;
33 short *dequant_ptr = d->dequant;
35 vpx_memset(qcoeff_ptr, 0, 32);
36 vpx_memset(dqcoeff_ptr, 0, 32);
38 eob = -1;
40 for (i = 0; i < 16; i++)
42 rc = vp8_default_zig_zag1d[i];
43 z = coeff_ptr[rc];
44 zbin = zbin_ptr[rc] ;
46 sz = (z >> 31); // sign of z
47 x = (z ^ sz) - sz; // x = abs(z)
49 if (x >= zbin)
51 x += round_ptr[rc];
52 y = (((x * quant_ptr[rc]) >> 16) + x)
53 >> quant_shift_ptr[rc]; // quantize (x)
54 x = (y ^ sz) - sz; // get the sign back
55 qcoeff_ptr[rc] = x; // write to destination
56 dqcoeff_ptr[rc] = x * dequant_ptr[rc]; // dequantized value
58 if (y)
60 eob = i; // last nonzero coeffs
64 d->eob = eob + 1;
67 #else
69 void vp8_fast_quantize_b_c(BLOCK *b, BLOCKD *d)
71 int i, rc, eob;
72 int x, y, z, sz;
73 short *coeff_ptr = b->coeff;
74 short *round_ptr = b->round;
75 short *quant_ptr = b->quant_fast;
76 short *qcoeff_ptr = d->qcoeff;
77 short *dqcoeff_ptr = d->dqcoeff;
78 short *dequant_ptr = d->dequant;
80 eob = -1;
81 for (i = 0; i < 16; i++)
83 rc = vp8_default_zig_zag1d[i];
84 z = coeff_ptr[rc];
86 sz = (z >> 31); // sign of z
87 x = (z ^ sz) - sz; // x = abs(z)
89 y = ((x + round_ptr[rc]) * quant_ptr[rc]) >> 16; // quantize (x)
90 x = (y ^ sz) - sz; // get the sign back
91 qcoeff_ptr[rc] = x; // write to destination
92 dqcoeff_ptr[rc] = x * dequant_ptr[rc]; // dequantized value
94 if (y)
96 eob = i; // last nonzero coeffs
99 d->eob = eob + 1;
102 #endif
104 #ifdef EXACT_QUANT
105 void vp8_regular_quantize_b(BLOCK *b, BLOCKD *d)
107 int i, rc, eob;
108 int zbin;
109 int x, y, z, sz;
110 short *zbin_boost_ptr = b->zrun_zbin_boost;
111 short *coeff_ptr = b->coeff;
112 short *zbin_ptr = b->zbin;
113 short *round_ptr = b->round;
114 short *quant_ptr = b->quant;
115 unsigned char *quant_shift_ptr = b->quant_shift;
116 short *qcoeff_ptr = d->qcoeff;
117 short *dqcoeff_ptr = d->dqcoeff;
118 short *dequant_ptr = d->dequant;
119 short zbin_oq_value = b->zbin_extra;
121 vpx_memset(qcoeff_ptr, 0, 32);
122 vpx_memset(dqcoeff_ptr, 0, 32);
124 eob = -1;
126 for (i = 0; i < 16; i++)
128 rc = vp8_default_zig_zag1d[i];
129 z = coeff_ptr[rc];
131 zbin = zbin_ptr[rc] + *zbin_boost_ptr + zbin_oq_value;
133 zbin_boost_ptr ++;
134 sz = (z >> 31); // sign of z
135 x = (z ^ sz) - sz; // x = abs(z)
137 if (x >= zbin)
139 x += round_ptr[rc];
140 y = (((x * quant_ptr[rc]) >> 16) + x)
141 >> quant_shift_ptr[rc]; // quantize (x)
142 x = (y ^ sz) - sz; // get the sign back
143 qcoeff_ptr[rc] = x; // write to destination
144 dqcoeff_ptr[rc] = x * dequant_ptr[rc]; // dequantized value
146 if (y)
148 eob = i; // last nonzero coeffs
149 zbin_boost_ptr = b->zrun_zbin_boost; // reset zero runlength
154 d->eob = eob + 1;
157 /* Perform regular quantization, with unbiased rounding and no zero bin. */
158 void vp8_strict_quantize_b(BLOCK *b, BLOCKD *d)
160 int i;
161 int rc;
162 int eob;
163 int x;
164 int y;
165 int z;
166 int sz;
167 short *coeff_ptr;
168 short *quant_ptr;
169 unsigned char *quant_shift_ptr;
170 short *qcoeff_ptr;
171 short *dqcoeff_ptr;
172 short *dequant_ptr;
174 coeff_ptr = b->coeff;
175 quant_ptr = b->quant;
176 quant_shift_ptr = b->quant_shift;
177 qcoeff_ptr = d->qcoeff;
178 dqcoeff_ptr = d->dqcoeff;
179 dequant_ptr = d->dequant;
180 eob = - 1;
181 vpx_memset(qcoeff_ptr, 0, 32);
182 vpx_memset(dqcoeff_ptr, 0, 32);
183 for (i = 0; i < 16; i++)
185 int dq;
186 int round;
188 /*TODO: These arrays should be stored in zig-zag order.*/
189 rc = vp8_default_zig_zag1d[i];
190 z = coeff_ptr[rc];
191 dq = dequant_ptr[rc];
192 round = dq >> 1;
193 /* Sign of z. */
194 sz = -(z < 0);
195 x = (z + sz) ^ sz;
196 x += round;
197 if (x >= dq)
199 /* Quantize x. */
200 y = (((x * quant_ptr[rc]) >> 16) + x) >> quant_shift_ptr[rc];
201 /* Put the sign back. */
202 x = (y + sz) ^ sz;
203 /* Save the coefficient and its dequantized value. */
204 qcoeff_ptr[rc] = x;
205 dqcoeff_ptr[rc] = x * dq;
206 /* Remember the last non-zero coefficient. */
207 if (y)
208 eob = i;
212 d->eob = eob + 1;
215 #else
217 void vp8_regular_quantize_b(BLOCK *b, BLOCKD *d)
219 int i, rc, eob;
220 int zbin;
221 int x, y, z, sz;
222 short *zbin_boost_ptr = b->zrun_zbin_boost;
223 short *coeff_ptr = b->coeff;
224 short *zbin_ptr = b->zbin;
225 short *round_ptr = b->round;
226 short *quant_ptr = b->quant;
227 short *qcoeff_ptr = d->qcoeff;
228 short *dqcoeff_ptr = d->dqcoeff;
229 short *dequant_ptr = d->dequant;
230 short zbin_oq_value = b->zbin_extra;
232 vpx_memset(qcoeff_ptr, 0, 32);
233 vpx_memset(dqcoeff_ptr, 0, 32);
235 eob = -1;
237 for (i = 0; i < 16; i++)
239 rc = vp8_default_zig_zag1d[i];
240 z = coeff_ptr[rc];
242 //if ( i == 0 )
243 // zbin = zbin_ptr[rc] + *zbin_boost_ptr + zbin_oq_value/2;
244 //else
245 zbin = zbin_ptr[rc] + *zbin_boost_ptr + zbin_oq_value;
247 zbin_boost_ptr ++;
248 sz = (z >> 31); // sign of z
249 x = (z ^ sz) - sz; // x = abs(z)
251 if (x >= zbin)
253 y = ((x + round_ptr[rc]) * quant_ptr[rc]) >> 16; // quantize (x)
254 x = (y ^ sz) - sz; // get the sign back
255 qcoeff_ptr[rc] = x; // write to destination
256 dqcoeff_ptr[rc] = x * dequant_ptr[rc]; // dequantized value
258 if (y)
260 eob = i; // last nonzero coeffs
261 zbin_boost_ptr = &b->zrun_zbin_boost[0]; // reset zero runlength
266 d->eob = eob + 1;
269 #endif
271 void vp8_quantize_mby(MACROBLOCK *x)
273 int i;
274 int has_2nd_order = (x->e_mbd.mode_info_context->mbmi.mode != B_PRED
275 && x->e_mbd.mode_info_context->mbmi.mode != SPLITMV);
277 for (i = 0; i < 16; i++)
278 x->quantize_b(&x->block[i], &x->e_mbd.block[i]);
280 if(has_2nd_order)
281 x->quantize_b(&x->block[24], &x->e_mbd.block[24]);
284 void vp8_quantize_mb(MACROBLOCK *x)
286 int i;
287 int has_2nd_order=(x->e_mbd.mode_info_context->mbmi.mode != B_PRED
288 && x->e_mbd.mode_info_context->mbmi.mode != SPLITMV);
290 for (i = 0; i < 24+has_2nd_order; i++)
291 x->quantize_b(&x->block[i], &x->e_mbd.block[i]);
295 void vp8_quantize_mbuv(MACROBLOCK *x)
297 int i;
299 for (i = 16; i < 24; i++)
300 x->quantize_b(&x->block[i], &x->e_mbd.block[i]);