1 /***********************************************************************
2 * Routines for packing INT_32, INT_16, FLOAT_32, FLOAT_64,
3 * STEIM1 and STEIM2 data records.
6 * Seismological Laboratory
7 * University of California, Berkeley
8 * doug@seismo.berkeley.edu
12 * - Optimize Steim 1 & 2 packing routines using small, re-used buffers.
15 * - Reworked and cleaned routines for use within libmseed.
16 * - Added float32 and float64 packing routines.
18 * Modified by Chad Trabant, IRIS Data Management Center
21 ************************************************************************/
24 * Copyright (c) 1996-2004 The Regents of the University of California.
25 * All Rights Reserved.
27 * Permission to use, copy, modify, and distribute this software and its
28 * documentation for educational, research and non-profit purposes,
29 * without fee, and without a written agreement is hereby granted,
30 * provided that the above copyright notice, this paragraph and the
31 * following three paragraphs appear in all copies.
33 * Permission to incorporate this software into commercial products may
34 * be obtained from the Office of Technology Licensing, 2150 Shattuck
35 * Avenue, Suite 510, Berkeley, CA 94704.
37 * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY
38 * FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES,
39 * INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND
40 * ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN
41 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43 * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES,
44 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
45 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE
46 * PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF
47 * CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT,
48 * UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
58 static int pad_steim_frame (DFRAMES
*, int, int, int, int, int);
60 #define EMPTY_BLOCK(fn,wn) (fn+wn == 0)
62 #define X0 dframes->f[0].w[0].fw
63 #define XN dframes->f[0].w[1].fw
65 #define BIT4PACK(i,points_remaining) \
66 (points_remaining >= 7 && \
67 (minbits[i] <= 4) && (minbits[i+1] <= 4) && \
68 (minbits[i+2] <= 4) && (minbits[i+3] <= 4) && \
69 (minbits[i+4] <= 4) && (minbits[i+5] <= 4) && \
72 #define BIT5PACK(i,points_remaining) \
73 (points_remaining >= 6 && \
74 (minbits[i] <= 5) && (minbits[i+1] <= 5) && \
75 (minbits[i+2] <= 5) && (minbits[i+3] <= 5) && \
76 (minbits[i+4] <= 5) && (minbits[i+5] <= 5))
78 #define BIT6PACK(i,points_remaining) \
79 (points_remaining >= 5 && \
80 (minbits[i] <= 6) && (minbits[i+1] <= 6) && \
81 (minbits[i+2] <= 6) && (minbits[i+3] <= 6) && \
84 #define BYTEPACK(i,points_remaining) \
85 (points_remaining >= 4 && \
86 (minbits[i] <= 8) && (minbits[i+1] <= 8) && \
87 (minbits[i+2] <= 8) && (minbits[i+3] <= 8))
89 #define BIT10PACK(i,points_remaining) \
90 (points_remaining >= 3 && \
91 (minbits[i] <= 10) && (minbits[i+1] <= 10) && \
94 #define BIT15PACK(i,points_remaining) \
95 (points_remaining >= 2 && \
96 (minbits[i] <= 15) && (minbits[i+1] <= 15))
98 #define HALFPACK(i,points_remaining) \
99 (points_remaining >= 2 && (minbits[i] <= 16) && (minbits[i+1] <= 16))
101 #define BIT30PACK(i,points_remaining) \
102 (points_remaining >= 1 && \
105 #define MINBITS(diff,minbits) \
106 if (diff >= -8 && diff < 8) minbits = 4; \
107 else if (diff >= -16 && diff < 16) minbits = 5; \
108 else if (diff >= -32 && diff < 32) minbits = 6; \
109 else if (diff >= -128 && diff < 128) minbits = 8; \
110 else if (diff >= -512 && diff < 512) minbits = 10; \
111 else if (diff >= -16384 && diff < 16384) minbits = 15; \
112 else if (diff >= -32768 && diff < 32768) minbits = 16; \
113 else if (diff >= -536870912 && diff < 536870912) minbits = 30; \
116 #define PACK(bits,n,m1,m2) { \
118 unsigned int val = 0; \
119 for (i=0;i<n;i++) { \
120 val = (val<<bits) | (diff[i]&m1); \
122 val |= ((unsigned int)m2 << 30); \
123 dframes->f[fn].w[wn].fw = val; }
126 /************************************************************************
128 * Pack integer data into INT_16 format. *
129 * Return: 0 on success, -1 on failure. *
130 ************************************************************************/
132 (int16_t *packed
, /* output data array - packed */
133 int32_t *data
, /* input data array */
134 int ns
, /* desired number of samples to pack */
135 int max_bytes
, /* max # of bytes for output buffer */
136 int pad
, /* flag to specify padding to max_bytes */
137 int *pnbytes
, /* number of bytes actually packed */
138 int *pnsamples
, /* number of samples actually packed */
139 int swapflag
) /* if data should be swapped */
141 int points_remaining
= ns
; /* number of samples remaining to pack */
144 while (points_remaining
> 0 && max_bytes
>= 2)
145 { /* Pack the next available data into INT_16 format */
146 if ( data
[i
] < -32768 || data
[i
] > 32767 )
147 ms_log (2, "msr_pack_int_16(%s): input sample out of range: %d\n",
148 PACK_SRCNAME
, data
[i
]);
151 if ( swapflag
) ms_gswap2 (packed
);
159 *pnbytes
= (ns
- points_remaining
) * 2;
161 /* Pad miniSEED block if necessary */
164 memset ((void *)packed
, 0, max_bytes
);
165 *pnbytes
+= max_bytes
;
168 *pnsamples
= ns
- points_remaining
;
174 /************************************************************************
176 * Pack integer data into INT_32 format. *
177 * Return: 0 on success, -1 on failure. *
178 ************************************************************************/
180 (int32_t *packed
, /* output data array - packed */
181 int32_t *data
, /* input data array - unpacked */
182 int ns
, /* desired number of samples to pack */
183 int max_bytes
, /* max # of bytes for output buffer */
184 int pad
, /* flag to specify padding to max_bytes */
185 int *pnbytes
, /* number of bytes actually packed */
186 int *pnsamples
, /* number of samples actually packed */
187 int swapflag
) /* if data should be swapped */
189 int points_remaining
= ns
; /* number of samples remaining to pack */
192 while (points_remaining
> 0 && max_bytes
>= 4)
193 { /* Pack the next available data into INT_32 format */
195 if ( swapflag
) ms_gswap4 (packed
);
203 *pnbytes
= (ns
- points_remaining
) * 4;
205 /* Pad miniSEED block if necessary */
208 memset ((void *)packed
, 0, max_bytes
);
209 *pnbytes
+= max_bytes
;
212 *pnsamples
= ns
- points_remaining
;
218 /************************************************************************
219 * msr_pack_float_32: *
220 * Pack float data into FLOAT32 format. *
221 * Return: 0 on success, -1 on error. *
222 ************************************************************************/
223 int msr_pack_float_32
224 (float *packed
, /* output data array - packed */
225 float *data
, /* input data array - unpacked */
226 int ns
, /* desired number of samples to pack */
227 int max_bytes
, /* max # of bytes for output buffer */
228 int pad
, /* flag to specify padding to max_bytes */
229 int *pnbytes
, /* number of bytes actually packed */
230 int *pnsamples
, /* number of samples actually packed */
231 int swapflag
) /* if data should be swapped */
233 int points_remaining
= ns
; /* number of samples remaining to pack */
236 while (points_remaining
> 0 && max_bytes
>= 4)
239 if ( swapflag
) ms_gswap4 (packed
);
247 *pnbytes
= (ns
- points_remaining
) * 4;
249 /* Pad miniSEED block if necessary */
252 memset ((void *)packed
, 0, max_bytes
);
253 *pnbytes
+= max_bytes
;
256 *pnsamples
= ns
- points_remaining
;
262 /************************************************************************
263 * msr_pack_float_64: *
264 * Pack double data into FLOAT64 format. *
265 * Return: 0 on success, -1 on error. *
266 ************************************************************************/
267 int msr_pack_float_64
268 (double *packed
, /* output data array - packed */
269 double *data
, /* input data array - unpacked */
270 int ns
, /* desired number of samples to pack */
271 int max_bytes
, /* max # of bytes for output buffer */
272 int pad
, /* flag to specify padding to max_bytes */
273 int *pnbytes
, /* number of bytes actually packed */
274 int *pnsamples
, /* number of samples actually packed */
275 int swapflag
) /* if data should be swapped */
277 int points_remaining
= ns
; /* number of samples remaining to pack */
280 while (points_remaining
> 0 && max_bytes
>= 8)
283 if ( swapflag
) ms_gswap8 (packed
);
291 *pnbytes
= (ns
- points_remaining
) * 8;
293 /* Pad miniSEED block if necessary */
296 memset ((void *)packed
, 0, max_bytes
);
297 *pnbytes
+= max_bytes
;
300 *pnsamples
= ns
- points_remaining
;
306 /************************************************************************
308 * Pack data into STEIM1 data frames. *
312 ************************************************************************/
314 (DFRAMES
*dframes
, /* ptr to data frames */
315 int32_t *data
, /* ptr to unpacked data array */
316 int32_t d0
, /* first difference value */
317 int ns
, /* number of samples to pack */
318 int nf
, /* total number of data frames */
319 int pad
, /* flag to specify padding to nf */
320 int *pnframes
, /* number of frames actually packed */
321 int *pnsamples
, /* number of samples actually packed */
322 int swapflag
) /* if data should be swapped */
324 int points_remaining
= ns
;
325 int points_packed
= 0;
326 int32_t diff
[4]; /* array of differences */
327 uint8_t minbits
[4]; /* array of minimum bits for diffs */
330 int ipt
= 0; /* index of initial data to pack. */
331 int fn
= 0; /* index of initial frame to pack. */
332 int wn
= 2; /* index of initial word to pack. */
336 /* Calculate initial difference and minbits buffers */
338 MINBITS(diff
[0],minbits
[0]);
339 for (i
=1; i
< 4 && i
< ns
; i
++)
341 diff
[i
] = data
[i
] - data
[i
-1];
342 MINBITS(diff
[i
],minbits
[i
]);
345 dframes
->f
[fn
].ctrl
= 0;
347 /* Set X0 and XN values in first frame */
349 if ( swapflag
) ms_gswap4 (&X0
);
350 dframes
->f
[0].ctrl
= (dframes
->f
[0].ctrl
<<2) | STEIM1_SPECIAL_MASK
;
352 if ( swapflag
) ms_gswap4 (&XN
);
353 dframes
->f
[0].ctrl
= (dframes
->f
[0].ctrl
<<2) | STEIM1_SPECIAL_MASK
;
355 while (points_remaining
> 0)
359 /* Pack the next available data into the most compact form */
360 if (BYTEPACK(0,points_remaining
))
362 mask
= STEIM1_BYTE_MASK
;
363 for (j
=0; j
<4; j
++) { dframes
->f
[fn
].w
[wn
].byte
[j
] = diff
[j
]; }
366 else if (HALFPACK(0,points_remaining
))
368 mask
= STEIM1_HALFWORD_MASK
;
372 if ( swapflag
) ms_gswap2 (&stmp
);
373 dframes
->f
[fn
].w
[wn
].hw
[j
] = stmp
;
379 mask
= STEIM1_FULLWORD_MASK
;
381 if ( swapflag
) ms_gswap4 (&itmp
);
382 dframes
->f
[fn
].w
[wn
].fw
= itmp
;
386 /* Append mask for this word to current mask */
387 dframes
->f
[fn
].ctrl
= (dframes
->f
[fn
].ctrl
<<2) | mask
;
389 points_remaining
-= points_packed
;
390 ipt
+= points_packed
;
392 /* Check for full frame or full block */
393 if (++wn
>= VALS_PER_FRAME
)
395 if ( swapflag
) ms_gswap4 (&dframes
->f
[fn
].ctrl
);
396 /* Reset output index to beginning of frame */
398 /* If block is full, output block and reinitialize */
399 if (++fn
>= nf
) break;
400 dframes
->f
[fn
].ctrl
= 0;
403 /* Shift and re-fill difference and minbits buffers */
404 for ( i
=points_packed
; i
< 4; i
++ )
406 /* Shift remaining buffer entries */
407 diff
[i
-points_packed
] = diff
[i
];
408 minbits
[i
-points_packed
] = minbits
[i
];
410 for ( i
=4-points_packed
,j
=ipt
+(4-points_packed
); i
< 4 && j
< ns
; i
++,j
++ )
412 /* Re-fill entries */
413 diff
[i
] = data
[j
] - data
[j
-1];
414 MINBITS(diff
[i
],minbits
[i
]);
418 /* Update XN value in first frame */
419 XN
= data
[(ns
-1)-points_remaining
];
420 if ( swapflag
) ms_gswap4 (&XN
);
422 /* End of data. Pad current frame and optionally rest of block */
423 /* Do not pad and output a completely empty block */
424 if ( ! EMPTY_BLOCK(fn
,wn
) )
426 *pnframes
= pad_steim_frame (dframes
, fn
, wn
, nf
, swapflag
, pad
);
433 *pnsamples
= ns
- points_remaining
;
439 /************************************************************************
441 * Pack data into STEIM1 data frames. *
445 ************************************************************************/
447 (DFRAMES
*dframes
, /* ptr to data frames */
448 int32_t *data
, /* ptr to unpacked data array */
449 int32_t d0
, /* first difference value */
450 int ns
, /* number of samples to pack */
451 int nf
, /* total number of data frames to pack */
452 int pad
, /* flag to specify padding to nf */
453 int *pnframes
, /* number of frames actually packed */
454 int *pnsamples
, /* number of samples actually packed */
455 int swapflag
) /* if data should be swapped */
457 int points_remaining
= ns
;
458 int points_packed
= 0;
459 int32_t diff
[7]; /* array of differences */
460 uint8_t minbits
[7]; /* array of minimum bits for diffs */
463 int ipt
= 0; /* index of initial data to pack. */
464 int fn
= 0; /* index of initial frame to pack. */
465 int wn
= 2; /* index of initial word to pack. */
467 /* Calculate initial difference and minbits buffers */
469 MINBITS(diff
[0],minbits
[0]);
470 for (i
=1; i
< 7 && i
< ns
; i
++)
472 diff
[i
] = data
[i
] - data
[i
-1];
473 MINBITS(diff
[i
],minbits
[i
]);
476 dframes
->f
[fn
].ctrl
= 0;
478 /* Set X0 and XN values in first frame */
480 if ( swapflag
) ms_gswap4 (&X0
);
481 dframes
->f
[0].ctrl
= (dframes
->f
[0].ctrl
<<2) | STEIM2_SPECIAL_MASK
;
483 if ( swapflag
) ms_gswap4 (&XN
);
484 dframes
->f
[0].ctrl
= (dframes
->f
[0].ctrl
<<2) | STEIM2_SPECIAL_MASK
;
486 while (points_remaining
> 0)
490 /* Pack the next available datapoints into the most compact form */
491 if (BIT4PACK(0,points_remaining
))
493 PACK(4,7,0x0000000f,02)
494 if ( swapflag
) ms_gswap4 (&dframes
->f
[fn
].w
[wn
].fw
);
495 mask
= STEIM2_567_MASK
;
498 else if (BIT5PACK(0,points_remaining
))
500 PACK(5,6,0x0000001f,01)
501 if ( swapflag
) ms_gswap4 (&dframes
->f
[fn
].w
[wn
].fw
);
502 mask
= STEIM2_567_MASK
;
505 else if (BIT6PACK(0,points_remaining
))
507 PACK(6,5,0x0000003f,00)
508 if ( swapflag
) ms_gswap4 (&dframes
->f
[fn
].w
[wn
].fw
);
509 mask
= STEIM2_567_MASK
;
512 else if (BYTEPACK(0,points_remaining
))
514 mask
= STEIM2_BYTE_MASK
;
515 for (j
=0; j
<4; j
++) dframes
->f
[fn
].w
[wn
].byte
[j
] = diff
[j
];
518 else if (BIT10PACK(0,points_remaining
))
520 PACK(10,3,0x000003ff,03)
521 if ( swapflag
) ms_gswap4 (&dframes
->f
[fn
].w
[wn
].fw
);
522 mask
= STEIM2_123_MASK
;
525 else if (BIT15PACK(0,points_remaining
))
527 PACK(15,2,0x00007fff,02)
528 if ( swapflag
) ms_gswap4 (&dframes
->f
[fn
].w
[wn
].fw
);
529 mask
= STEIM2_123_MASK
;
532 else if (BIT30PACK(0,points_remaining
))
534 PACK(30,1,0x3fffffff,01)
535 if ( swapflag
) ms_gswap4 (&dframes
->f
[fn
].w
[wn
].fw
);
536 mask
= STEIM2_123_MASK
;
541 ms_log (2, "msr_pack_steim2(%s): Unable to represent difference in <= 30 bits\n",
546 /* Append mask for this word to current mask */
547 dframes
->f
[fn
].ctrl
= (dframes
->f
[fn
].ctrl
<<2) | mask
;
549 points_remaining
-= points_packed
;
550 ipt
+= points_packed
;
552 /* Check for full frame or full block */
553 if (++wn
>= VALS_PER_FRAME
)
555 if ( swapflag
) ms_gswap4 (&dframes
->f
[fn
].ctrl
);
556 /* Reset output index to beginning of frame */
558 /* If block is full, output block and reinitialize */
559 if (++fn
>= nf
) break;
560 dframes
->f
[fn
].ctrl
= 0;
563 /* Shift and re-fill difference and minbits buffers */
564 for ( i
=points_packed
; i
< 7; i
++ )
566 /* Shift remaining buffer entries */
567 diff
[i
-points_packed
] = diff
[i
];
568 minbits
[i
-points_packed
] = minbits
[i
];
570 for ( i
=7-points_packed
,j
=ipt
+(7-points_packed
); i
< 7 && j
< ns
; i
++,j
++ )
572 /* Re-fill entries */
573 diff
[i
] = data
[j
] - data
[j
-1];
574 MINBITS(diff
[i
],minbits
[i
]);
578 /* Update XN value in first frame */
579 XN
= data
[(ns
-1)-points_remaining
];
580 if ( swapflag
) ms_gswap4 (&XN
);
582 /* End of data. Pad current frame and optionally rest of block */
583 /* Do not pad and output a completely empty block */
584 if ( ! EMPTY_BLOCK(fn
,wn
) )
586 *pnframes
= pad_steim_frame (dframes
, fn
, wn
, nf
, swapflag
, pad
);
593 *pnsamples
= ns
- points_remaining
;
599 /************************************************************************
601 * Pad the rest of the data record with null values, *
602 * and optionally the rest of the total number of frames. *
604 * total number of frames in record. *
605 ************************************************************************/
606 static int pad_steim_frame
608 int fn
, /* current frame number. */
609 int wn
, /* current work number. */
610 int nf
, /* total number of data frames. */
611 int swapflag
, /* flag to swap byte order of data. */
612 int pad
) /* flag to pad # frames to nf. */
614 /* Finish off the current frame */
615 if (wn
< VALS_PER_FRAME
&& fn
< nf
)
617 for (; wn
< VALS_PER_FRAME
; wn
++)
619 dframes
->f
[fn
].w
[wn
].fw
= 0;
620 dframes
->f
[fn
].ctrl
= (dframes
->f
[fn
].ctrl
<<2) | STEIM1_SPECIAL_MASK
;
622 if ( swapflag
) ms_gswap4 (&dframes
->f
[fn
].ctrl
);
626 /* Fill the remaining frames in the block */
631 dframes
->f
[fn
].ctrl
= STEIM1_SPECIAL_MASK
; /* mask for ctrl */
632 for (wn
=0; wn
<VALS_PER_FRAME
; wn
++)
634 dframes
->f
[fn
].w
[wn
].fw
= 0;
635 dframes
->f
[fn
].ctrl
= (dframes
->f
[fn
].ctrl
<<2) | STEIM1_SPECIAL_MASK
;
638 if ( swapflag
) ms_gswap4 (&dframes
->f
[fn
].ctrl
);
646 /************************************************************************
648 * Pack text data into text format. Split input data on line *
649 * breaks so as to not split lines between records. *
650 * Return: 0 on success, -1 on error. *
651 ************************************************************************/
653 (char *packed
, /* output data array - packed. */
654 char *data
, /* input data array - unpacked. */
655 int ns
, /* desired number of samples to pack. */
656 int max_bytes
, /* max # of bytes for output buffer. */
657 int pad
, /* flag to specify padding to max_bytes.*/
658 int *pnbytes
, /* number of bytes actually packed. */
659 int *pnsamples
) /* number of samples actually packed. */
661 int points_remaining
= ns
; /* number of samples remaining to pack. */
666 /* Split lines only if a single line will not fit in 1 record */
667 if (points_remaining
> max_bytes
)
669 /* Look for the last newline that will fit in output buffer */
670 for (i
=max_bytes
-1; i
>=0; i
--)
672 if (data
[i
] == '\n') {
677 if (last
< 0) last
= max_bytes
- 1;
680 if (last
< 0) last
= points_remaining
- 1;
682 memcpy (packed
, data
, nbytes
);
687 points_remaining
-= nbytes
;
689 /* Pad miniSEED block if necessary */
692 memset ((void *)packed
, 0, max_bytes
);
693 *pnbytes
+= max_bytes
;
696 *pnsamples
= ns
- points_remaining
;