libvncclient/zrle.c

   1 /*
   2  *  Copyright (C) 2005 Johannes E. Schindelin.  All Rights Reserved.
   3  *
   4  *  This is free software; you can redistribute it and/or modify
   5  *  it under the terms of the GNU General Public License as published by
   6  *  the Free Software Foundation; either version 2 of the License, or
   7  *  (at your option) any later version.
   8  *
   9  *  This software is distributed in the hope that it will be useful,
  10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  *  GNU General Public License for more details.
  13  *
  14  *  You should have received a copy of the GNU General Public License
  15  *  along with this software; if not, write to the Free Software
  16  *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
  17  *  USA.
  18  */
  19
  20 #ifdef LIBVNCSERVER_HAVE_LIBZ
  21
  22 /*
  23  * zrle.c - handle zrle encoding.
  24  *
  25  * This file shouldn't be compiled directly.  It is included multiple times by
  26  * rfbproto.c, each time with a different definition of the macro BPP.  For
  27  * each value of BPP, this file defines a function which handles an zrle
  28  * encoded rectangle with BPP bits per pixel.
  29  */
  30
  31 #ifndef REALBPP
  32 #define REALBPP BPP
  33 #endif
  34
  35 #if !defined(UNCOMP) || UNCOMP==0
  36 #define HandleZRLE CONCAT2E(HandleZRLE,REALBPP)
  37 #define HandleZRLETile CONCAT2E(HandleZRLETile,REALBPP)
  38 #elif UNCOMP>0
  39 #define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Down)
  40 #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Down)
  41 #else
  42 #define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Up)
  43 #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Up)
  44 #endif
  45 #define CARDBPP CONCAT3E(uint,BPP,_t)
  46 #define CARDREALBPP CONCAT3E(uint,REALBPP,_t)
  47
  48 #define ENDIAN_LITTLE 0
  49 #define ENDIAN_BIG 1
  50 #define ENDIAN_NO 2
  51 #define ZYWRLE_ENDIAN ENDIAN_LITTLE
  52 #undef END_FIX
  53 #if ZYWRLE_ENDIAN == ENDIAN_LITTLE
  54 #  define END_FIX LE
  55 #elif ZYWRLE_ENDIAN == ENDIAN_BIG
  56 #  define END_FIX BE
  57 #else
  58 #  define END_FIX NE
  59 #endif
  60 #define __RFB_CONCAT3E(a,b,c) CONCAT3E(a,b,c)
  61 #define __RFB_CONCAT2E(a,b) CONCAT2E(a,b)
  62 #undef CPIXEL
  63 #if REALBPP != BPP
  64 #if UNCOMP == 0
  65 #define CPIXEL REALBPP
  66 #elif UNCOMP>0
  67 #define CPIXEL CONCAT2E(REALBPP,Down)
  68 #else
  69 #define CPIXEL CONCAT2E(REALBPP,Up)
  70 #endif
  71 #endif
  72 #define PIXEL_T __RFB_CONCAT3E(uint,BPP,_t)
  73 #if BPP!=8
  74 #define ZYWRLE_DECODE 1
  75 #include "zywrletemplate.c"
  76 #endif
  77 #undef CPIXEL
  78
  79 static int HandleZRLETile(rfbClient* client,
  80         uint8_t* buffer,size_t buffer_length,
  81         int x,int y,int w,int h);
  82
  83 static rfbBool
  84 HandleZRLE (rfbClient* client, int rx, int ry, int rw, int rh)
  85 {
  86         rfbZRLEHeader header;
  87         int remaining;
  88         int inflateResult;
  89         int toRead;
  90         int min_buffer_size = rw * rh * (REALBPP / 8) * 2;
  91
  92         /* First make sure we have a large enough raw buffer to hold the
  93          * decompressed data.  In practice, with a fixed REALBPP, fixed frame
  94          * buffer size and the first update containing the entire frame
  95          * buffer, this buffer allocation should only happen once, on the
  96          * first update.
  97          */
  98         if ( client->raw_buffer_size < min_buffer_size) {
  99
 100                 if ( client->raw_buffer != NULL ) {
 101
 102                         free( client->raw_buffer );
 103
 104                 }
 105
 106                 client->raw_buffer_size = min_buffer_size;
 107                 client->raw_buffer = (char*) malloc( client->raw_buffer_size );
 108
 109         }
 110
 111         if (!ReadFromRFBServer(client, (char *)&header, sz_rfbZRLEHeader))
 112                 return FALSE;
 113
 114         remaining = rfbClientSwap32IfLE(header.length);
 115
 116         /* Need to initialize the decompressor state. */
 117         client->decompStream.next_in   = ( Bytef * )client->buffer;
 118         client->decompStream.avail_in  = 0;
 119         client->decompStream.next_out  = ( Bytef * )client->raw_buffer;
 120         client->decompStream.avail_out = client->raw_buffer_size;
 121         client->decompStream.data_type = Z_BINARY;
 122
 123         /* Initialize the decompression stream structures on the first invocation. */
 124         if ( client->decompStreamInited == FALSE ) {
 125
 126                 inflateResult = inflateInit( &client->decompStream );
 127
 128                 if ( inflateResult != Z_OK ) {
 129                         rfbClientLog(
 130                                         "inflateInit returned error: %d, msg: %s\n",
 131                                         inflateResult,
 132                                         client->decompStream.msg);
 133                         return FALSE;
 134                 }
 135
 136                 client->decompStreamInited = TRUE;
 137
 138         }
 139
 140         inflateResult = Z_OK;
 141
 142         /* Process buffer full of data until no more to process, or
 143          * some type of inflater error, or Z_STREAM_END.
 144          */
 145         while (( remaining > 0 ) &&
 146                         ( inflateResult == Z_OK )) {
 147
 148                 if ( remaining > RFB_BUFFER_SIZE ) {
 149                         toRead = RFB_BUFFER_SIZE;
 150                 }
 151                 else {
 152                         toRead = remaining;
 153                 }
 154
 155                 /* Fill the buffer, obtaining data from the server. */
 156                 if (!ReadFromRFBServer(client, client->buffer,toRead))
 157                         return FALSE;
 158
 159                 client->decompStream.next_in  = ( Bytef * )client->buffer;
 160                 client->decompStream.avail_in = toRead;
 161
 162                 /* Need to uncompress buffer full. */
 163                 inflateResult = inflate( &client->decompStream, Z_SYNC_FLUSH );
 164
 165                 /* We never supply a dictionary for compression. */
 166                 if ( inflateResult == Z_NEED_DICT ) {
 167                         rfbClientLog("zlib inflate needs a dictionary!\n");
 168                         return FALSE;
 169                 }
 170                 if ( inflateResult < 0 ) {
 171                         rfbClientLog(
 172                                         "zlib inflate returned error: %d, msg: %s\n",
 173                                         inflateResult,
 174                                         client->decompStream.msg);
 175                         return FALSE;
 176                 }
 177
 178                 /* Result buffer allocated to be at least large enough.  We should
 179                  * never run out of space!
 180                  */
 181                 if (( client->decompStream.avail_in > 0 ) &&
 182                                 ( client->decompStream.avail_out <= 0 )) {
 183                         rfbClientLog("zlib inflate ran out of space!\n");
 184                         return FALSE;
 185                 }
 186
 187                 remaining -= toRead;
 188
 189         } /* while ( remaining > 0 ) */
 190
 191         if ( inflateResult == Z_OK ) {
 192                 void* buf=client->raw_buffer;
 193                 int i,j;
 194
 195                 remaining = client->raw_buffer_size-client->decompStream.avail_out;
 196
 197                 for(j=0; j<rh; j+=rfbZRLETileHeight)
 198                         for(i=0; i<rw; i+=rfbZRLETileWidth) {
 199                                 int subWidth=(i+rfbZRLETileWidth>rw)?rw-i:rfbZRLETileWidth;
 200                                 int subHeight=(j+rfbZRLETileHeight>rh)?rh-j:rfbZRLETileHeight;
 201                                 int result=HandleZRLETile(client,buf,remaining,rx+i,ry+j,subWidth,subHeight);
 202
 203                                 if(result<0) {
 204                                         rfbClientLog("ZRLE decoding failed (%d)\n",result);
 205 return TRUE;
 206                                         return FALSE;
 207                                 }
 208
 209                                 buf+=result;
 210                                 remaining-=result;
 211                         }
 212         }
 213         else {
 214
 215                 rfbClientLog(
 216                                 "zlib inflate returned error: %d, msg: %s\n",
 217                                 inflateResult,
 218                                 client->decompStream.msg);
 219                 return FALSE;
 220
 221         }
 222
 223         return TRUE;
 224 }
 225
 226 #if REALBPP!=BPP && defined(UNCOMP) && UNCOMP!=0
 227 #if UNCOMP>0
 228 #define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)>>UNCOMP)
 229 #else
 230 #define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)<<(-(UNCOMP)))
 231 #endif
 232 #else
 233 #define UncompressCPixel(pointer) (*(CARDBPP*)pointer)
 234 #endif
 235
 236 static int HandleZRLETile(rfbClient* client,
 237                 uint8_t* buffer,size_t buffer_length,
 238                 int x,int y,int w,int h) {
 239         uint8_t* buffer_copy = buffer;
 240         uint8_t* buffer_end = buffer+buffer_length;
 241         uint8_t type;
 242 #if BPP!=8
 243         uint8_t zywrle_level = (client->appData.qualityLevel & 0x80) ?
 244                 0 : (3 - client->appData.qualityLevel / 3);
 245 #endif
 246
 247         if(buffer_length<1)
 248                 return -2;
 249
 250         type = *buffer;
 251         buffer++;
 252         {
 253                 if( type == 0 ) /* raw */
 254 #if BPP!=8
 255           if( zywrle_level > 0 ){
 256                         CARDBPP* pFrame = (CARDBPP*)client->frameBuffer + y*client->width+x;
 257                         int ret;
 258                         client->appData.qualityLevel |= 0x80;
 259                         ret = HandleZRLETile(client, buffer, buffer_end-buffer, x, y, w, h);
 260                     client->appData.qualityLevel &= 0x7F;
 261                         if( ret < 0 ){
 262                                 return ret;
 263                         }
 264                         ZYWRLE_SYNTHESIZE( pFrame, pFrame, w, h, client->width, zywrle_level, (int*)client->zlib_buffer );
 265                         buffer += ret;
 266                   }else
 267 #endif
 268                 {
 269 #if REALBPP!=BPP
 270                         int i,j;
 271
 272                         if(1+w*h*REALBPP/8>buffer_length) {
 273                                 rfbClientLog("expected %d bytes, got only %d (%dx%d)\n",1+w*h*REALBPP/8,buffer_length,w,h);
 274                                 return -3;
 275                         }
 276
 277                         for(j=y*client->width; j<(y+h)*client->width; j+=client->width)
 278                                 for(i=x; i<x+w; i++,buffer+=REALBPP/8)
 279                                         ((CARDBPP*)client->frameBuffer)[j+i] = UncompressCPixel(buffer);
 280 #else
 281                         CopyRectangle(client, buffer, x, y, w, h);
 282                         buffer+=w*h*REALBPP/8;
 283 #endif
 284                 }
 285                 else if( type == 1 ) /* solid */
 286                 {
 287                         CARDBPP color = UncompressCPixel(buffer);
 288
 289                         if(1+REALBPP/8>buffer_length)
 290                                 return -4;
 291
 292                         FillRectangle(client, x, y, w, h, color);
 293
 294                         buffer+=REALBPP/8;
 295
 296                 }
 297                 else if( (type >= 2)&&(type <= 127) ) /* packed Palette */
 298                 {
 299                         CARDBPP palette[16];
 300                         int i,j,shift,
 301                                 bpp=(type>4?(type>16?8:4):(type>2?2:1)),
 302                                 mask=(1<<bpp)-1,
 303                                 divider=(8/bpp);
 304
 305                         if(1+type*REALBPP/8+((w+divider-1)/divider)*h>buffer_length)
 306                                 return -5;
 307
 308                         /* read palette */
 309                         for(i=0; i<type; i++,buffer+=REALBPP/8)
 310                                 palette[i] = UncompressCPixel(buffer);
 311
 312                         /* read palettized pixels */
 313                         for(j=y*client->width; j<(y+h)*client->width; j+=client->width) {
 314                                 for(i=x,shift=8-bpp; i<x+w; i++) {
 315                                         ((CARDBPP*)client->frameBuffer)[j+i] = palette[((*buffer)>>shift)&mask];
 316                                         shift-=bpp;
 317                                         if(shift<0) {
 318                                                 shift=8-bpp;
 319                                                 buffer++;
 320                                         }
 321                                 }
 322                                 if(shift<8-bpp)
 323                                         buffer++;
 324                         }
 325
 326                 }
 327                 /* case 17 ... 127: not used, but valid */
 328                 else if( type == 128 ) /* plain RLE */
 329                 {
 330                         int i=0,j=0;
 331                         while(j<h) {
 332                                 int color,length;
 333                                 /* read color */
 334                                 if(buffer+REALBPP/8+1>buffer_end)
 335                                         return -7;
 336                                 color = UncompressCPixel(buffer);
 337                                 buffer+=REALBPP/8;
 338                                 /* read run length */
 339                                 length=1;
 340                                 while(*buffer==0xff) {
 341                                         if(buffer+1>=buffer_end)
 342                                                 return -8;
 343                                         length+=*buffer;
 344                                         buffer++;
 345                                 }
 346                                 length+=*buffer;
 347                                 buffer++;
 348                                 while(j<h && length>0) {
 349                                         ((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color;
 350                                         length--;
 351                                         i++;
 352                                         if(i>=w) {
 353                                                 i=0;
 354                                                 j++;
 355                                         }
 356                                 }
 357                                 if(length>0)
 358                                         rfbClientLog("Warning: possible ZRLE corruption\n");
 359                         }
 360
 361                 }
 362                 else if( type == 129 ) /* unused */
 363                 {
 364                         return -8;
 365                 }
 366                 else if( type >= 130 ) /* palette RLE */
 367                 {
 368                         CARDBPP palette[128];
 369                         int i,j;
 370
 371                         if(2+(type-128)*REALBPP/8>buffer_length)
 372                                 return -9;
 373
 374                         /* read palette */
 375                         for(i=0; i<type-128; i++,buffer+=REALBPP/8)
 376                                 palette[i] = UncompressCPixel(buffer);
 377                         /* read palettized pixels */
 378                         i=j=0;
 379                         while(j<h) {
 380                                 int color,length;
 381                                 /* read color */
 382                                 if(buffer>=buffer_end)
 383                                         return -10;
 384                                 color = palette[(*buffer)&0x7f];
 385                                 length=1;
 386                                 if(*buffer&0x80) {
 387                                         if(buffer+1>=buffer_end)
 388                                                 return -11;
 389                                         buffer++;
 390                                         /* read run length */
 391                                         while(*buffer==0xff) {
 392                                                 if(buffer+1>=buffer_end)
 393                                                         return -8;
 394                                                 length+=*buffer;
 395                                                 buffer++;
 396                                         }
 397                                         length+=*buffer;
 398                                 }
 399                                 buffer++;
 400                                 while(j<h && length>0) {
 401                                         ((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color;
 402                                         length--;
 403                                         i++;
 404                                         if(i>=w) {
 405                                                 i=0;
 406                                                 j++;
 407                                         }
 408                                 }
 409                                 if(length>0)
 410                                         rfbClientLog("Warning: possible ZRLE corruption\n");
 411                         }
 412                 }
 413         }
 414
 415         return buffer-buffer_copy;
 416 }
 417
 418 #undef CARDBPP
 419 #undef CARDREALBPP
 420 #undef HandleZRLE
 421 #undef HandleZRLETile
 422 #undef UncompressCPixel
 423 #undef REALBPP
 424
 425 #endif
 426
 427 #undef UNCOMP