addons/AVCodec/source/IoAVCodec.c

   1
   2 //metadoc AVCodec copyright Steve Dekorte, 2004
   3 //metadoc AVCodec license BSD revised
   4 //metadoc AVCodec category Media
   5 /*metadoc AVCodec description
   6 An object for encoding and decoding audio and video streams.
   7 */
   8
   9 #include "IoAVCodec.h"
  10 #include "List.h"
  11 #include "IoState.h"
  12 #include "IoNumber.h"
  13 #include "IoSeq.h"
  14 #include "IoList.h"
  15 #include <limits.h>
  16 //#include <math.h>
  17
  18
  19 #define DATA(self) ((IoAVCodecData *)IoObject_dataPointer(self))
  20
  21 void IoAVCodec_registerIfNeeded(IoAVCodec *self)
  22 {
  23         avcodec_init();
  24         avcodec_register_all();
  25         av_register_all();
  26 }
  27
  28 IoTag *IoAVCodec_newTag(void *state)
  29 {
  30         IoTag *tag = IoTag_newWithName_("AVCodec");
  31         IoTag_state_(tag, state);
  32         IoTag_cloneFunc_(tag, (IoTagCloneFunc *)IoAVCodec_rawClone);
  33         IoTag_markFunc_(tag, (IoTagMarkFunc *)IoAVCodec_mark);
  34         IoTag_freeFunc_(tag, (IoTagFreeFunc *)IoAVCodec_free);
  35         return tag;
  36 }
  37
  38 IoAVCodec *IoAVCodec_proto(void *state)
  39 {
  40         IoObject *self = IoObject_new(state);
  41         IoObject_tag_(self, IoAVCodec_newTag(state));
  42
  43         IoObject_setDataPointer_(self, calloc(1, sizeof(IoAVCodecData)));
  44
  45         DATA(self)->inputBuffer  = IoSeq_new(state);
  46         DATA(self)->outputBuffer = IoSeq_new(state);
  47
  48         IoState_registerProtoWithFunc_(state, self, IoAVCodec_proto);
  49
  50         IoAVCodec_registerIfNeeded(self);
  51
  52         {
  53                 IoMethodTable methodTable[] = {
  54
  55                 {"audioInputBuffer",  IoAVCodec_audioInputBuffer},
  56                 {"audioOutputBuffer", IoAVCodec_audioOutputBuffer},
  57
  58                 {"encodeCodecNames",  IoAVCodec_encodeCodecNames},
  59                 {"decodeCodecNames",  IoAVCodec_decodeCodecNames},
  60
  61                 {"open", IoAVCodec_open},
  62                 {"close", IoAVCodec_close},
  63
  64                 {"decode", IoAVCodec_decode},
  65                 {"isAtEnd", IoAVCodec_isAtEnd},
  66                 //{"encode",     IoAVCodec_startEncoding},
  67
  68                 {NULL, NULL},
  69                 };
  70                 IoObject_addMethodTable_(self, methodTable);
  71         }
  72         return self;
  73 }
  74
  75 IoAVCodec *IoAVCodec_rawClone(IoAVCodec *proto)
  76 {
  77         IoObject *self = IoObject_rawClonePrimitive(proto);
  78         IoObject_setDataPointer_(self, calloc(1, sizeof(IoAVCodecData)));
  79
  80         DATA(self)->inputBuffer        = IOCLONE(DATA(proto)->inputBuffer);
  81         DATA(self)->outputBuffer       = IOCLONE(DATA(proto)->outputBuffer);
  82
  83         return self;
  84 }
  85
  86 IoAVCodec *IoAVCodec_new(void *state)
  87 {
  88         IoObject *proto = IoState_protoWithInitFunction_(state, IoAVCodec_proto);
  89         return IOCLONE(proto);
  90 }
  91
  92 // -----------------------------------------------------------
  93
  94 void IoAVCodec_createContextIfNeeded(IoAVCodec *self)
  95 {
  96         if(!DATA(self)->packet)
  97         {
  98                 DATA(self)->packet = calloc(1, sizeof(AVPacket));
  99         }
 100
 101         // video
 102
 103         // frames
 104         if (!DATA(self)->frames)
 105         {
 106                 DATA(self)->frames = IoList_new(IOSTATE);
 107                 IoObject_setSlot_to_(self, IOSYMBOL("frames"), DATA(self)->frames);
 108         }
 109
 110         // videoSize
 111         {
 112                 UArray *sizeUArray = UArray_newWithData_type_encoding_size_copy_("", CTYPE_float32_t, CENCODING_NUMBER, 2, 1);
 113                 IoSeq *sizeSeq = IoSeq_newWithUArray_copy_(IOSTATE, sizeUArray, 0);
 114                 IoObject_setSlot_to_(self, IOSYMBOL("videoSize"), sizeSeq);
 115         }
 116
 117         if (!DATA(self)->decodedFrame)
 118         {
 119                 DATA(self)->decodedFrame = avcodec_alloc_frame();
 120         }
 121
 122         // audio
 123
 124         if(!DATA(self)->audioOutBuffer)
 125         {
 126                 DATA(self)->audioOutBuffer = malloc(AVCODEC_MAX_AUDIO_FRAME_SIZE);
 127         }
 128 }
 129
 130 AVPicture *IoAVCode_allocDstPictureIfNeeded(IoAVCodec *self, int pix_fmt, int width, int height)
 131 {
 132         if(!DATA(self)->rgbPicture)
 133         {
 134                 DATA(self)->rgbPicture = calloc(1, sizeof(AVPicture));
 135                 avpicture_alloc(DATA(self)->rgbPicture, PIX_FMT_RGB24, width, height);
 136         }
 137
 138         return DATA(self)->rgbPicture;
 139 }
 140
 141
 142 void IoAVCodec_freeContextIfNeeded(IoAVCodec *self)
 143 {
 144         //printf("IoAVCodec_freeContextIfNeeded\n");
 145
 146         DATA(self)->audioContext = NULL;
 147         DATA(self)->videoContext = NULL;
 148
 149         if (DATA(self)->audioContext)
 150         {
 151                 //avcodec_close(DATA(self)->audioContext);
 152                 //av_free(DATA(self)->audioContext);
 153                 DATA(self)->audioContext = NULL;
 154         }
 155
 156         if (DATA(self)->videoContext)
 157         {
 158                 //avcodec_close(DATA(self)->audioContext);
 159                 //av_free(DATA(self)->audioContext);
 160                 DATA(self)->audioContext = NULL;
 161         }
 162
 163         if (DATA(self)->formatContext)
 164         {
 165                 av_close_input_file(DATA(self)->formatContext);
 166                 //av_free(DATA(self)->formatContext);
 167                 DATA(self)->formatContext = NULL;
 168         }
 169
 170         if(DATA(self)->packet)
 171         {
 172                 //free(DATA(self)->packet);
 173                 DATA(self)->packet = NULL;
 174         }
 175
 176         if (DATA(self)->audioOutBuffer)
 177         {
 178                 //free(DATA(self)->audioOutBuffer);
 179                 DATA(self)->audioOutBuffer = NULL;
 180         }
 181
 182         if(DATA(self)->decodedFrame)
 183         {
 184                 //av_free(DATA(self)->decodedFrame);
 185                 DATA(self)->decodedFrame = NULL;
 186         }
 187
 188         if(DATA(self)->rgbPicture)
 189         {
 190                 avpicture_free(DATA(self)->rgbPicture);
 191                 //free(DATA(self)->rgbPicture);
 192                 DATA(self)->rgbPicture = NULL;
 193         }
 194
 195         //printf("IoAVCodec_freeContextIfNeeded done\n");
 196 }
 197
 198 void IoAVCodec_free(IoAVCodec *self)
 199 {
 200         IoAVCodec_freeContextIfNeeded(self);
 201         free(IoObject_dataPointer(self));
 202 }
 203
 204 void IoAVCodec_mark(IoAVCodec *self)
 205 {
 206         if(DATA(self)->frames) IoObject_shouldMark(DATA(self)->frames);
 207
 208         IoObject_shouldMark(DATA(self)->inputBuffer);
 209         IoObject_shouldMark(DATA(self)->outputBuffer);
 210 }
 211
 212 // -----------------------------------------------------------
 213
 214 void IoAVCodec_error_(IoAVCodec *self, IoMessage *m, char *s)
 215 {
 216         fprintf(stderr, s);
 217         IoState_error_(IOSTATE, m, s);
 218 }
 219
 220 IoObject *IoAVCodec_audioInputBuffer(IoAVCodec *self, IoObject *locals, IoMessage *m)
 221 {
 222         /*doc AVCodec inputBuffer
 223         Returns the input buffer.
 224         */
 225         return DATA(self)->inputBuffer;
 226 }
 227
 228 IoObject *IoAVCodec_audioOutputBuffer(IoAVCodec *self, IoObject *locals, IoMessage *m)
 229 {
 230         /*doc AVCodec outputBuffer
 231         Returns the output buffer.
 232         */
 233         return DATA(self)->outputBuffer;
 234 }
 235
 236 IoObject *IoAVCodec_decodeCodecNames(IoAVCodec *self, IoObject *locals, IoMessage *m)
 237 {
 238         /*doc AVCodec decodeCodecNames
 239         Returns a list of strings with the names of the decode codecs.
 240         */
 241
 242         AVCodec *p = first_avcodec;
 243         IoList *names = IoList_new(IOSTATE);
 244
 245         while (p)
 246         {
 247                 if (p->decode)
 248                 {
 249                         IoList_rawAppend_(names, IOSYMBOL(p->name));
 250                 }
 251
 252                 p = p->next;
 253         }
 254
 255         return names;
 256 }
 257
 258 IoObject *IoAVCodec_encodeCodecNames(IoAVCodec *self, IoObject *locals, IoMessage *m)
 259 {
 260         /*doc AVCodec encodeCodecNames
 261         Returns a list of strings with the names of the encode codecs.
 262         */
 263
 264         AVCodec *p = first_avcodec;
 265         IoList *names = IoList_new(IOSTATE);
 266
 267         while (p)
 268         {
 269                 if (p->encode)
 270                 {
 271                         IoList_rawAppend_(names, IOSYMBOL(p->name));
 272                 }
 273
 274                 p = p->next;
 275         }
 276
 277         return names;
 278 }
 279
 280 void IoAVCodec_ConvertShortToFloat(short *s, float *f, size_t sampleCount)
 281 {
 282         size_t i;
 283
 284         for (i = 0; i < sampleCount; i ++)
 285         {
 286                 *f = (float)(*s) / (float)SHRT_MAX;
 287                 f ++;
 288                 *f = (float)(*s) / (float)SHRT_MAX;
 289                 s ++;
 290         }
 291 }
 292
 293 void IoAVCodec_ConvertFloatToShort(float *f, short *s, size_t sampleCount)
 294 {
 295         size_t i;
 296
 297         for (i = 0; i < sampleCount; i ++)
 298         {
 299                 *s = (short)((*f) * SHRT_MAX);
 300                 s ++;
 301                 *s = (short)((*f) * SHRT_MAX);
 302                 f ++;
 303         }
 304 }
 305
 306 // -----------------------------------------------------------
 307
 308 #define INBUF_SIZE 4096
 309
 310 // ----------------------------------------------------------------------------------------------
 311
 312 IoObject *IoAVCodec_close(IoAVCodec *self, IoObject *locals, IoMessage *m)
 313 {
 314         IoAVCodec_registerIfNeeded(self);
 315         IoAVCodec_freeContextIfNeeded(self);
 316         return self;
 317 }
 318
 319 int IoAVCodec_openFile(IoAVCodec *self)
 320 {
 321         //AVInputFormat *inputFormat;
 322         IoObject *fileName = IoObject_symbolGetSlot_(self, IOSYMBOL("path"));
 323         int err = av_open_input_file(&DATA(self)->formatContext, CSTRING(fileName), NULL, 0, NULL);
 324         return err;
 325 }
 326
 327 IoObject *IoAVCodec_open(IoAVCodec *self, IoObject *locals, IoMessage *m)
 328 {
 329         int err;
 330
 331         IoAVCodec_registerIfNeeded(self);
 332         IoAVCodec_freeContextIfNeeded(self);
 333         IoAVCodec_createContextIfNeeded(self);
 334
 335         DATA(self)->isAtEnd = 0;
 336
 337         err = IoAVCodec_openFile(self);
 338
 339         if (err != 0)
 340         {
 341                 IoObject *fileName = IoObject_symbolGetSlot_(self, IOSYMBOL("path"));
 342                 IoState_error_(IOSTATE, m, "error %i opening file %s\n", err, CSTRING(fileName));
 343                 return IONIL(self);
 344         }
 345
 346         IoAVCodec_findStreams(self);
 347         av_read_play(DATA(self)->formatContext);
 348
 349         return self;
 350 }
 351
 352 int IoAVCodec_findStreams(IoAVCodec *self)
 353 {
 354         AVFormatContext *formatContext = DATA(self)->formatContext;
 355         int i;
 356
 357         av_find_stream_info(formatContext);
 358
 359         //printf("formatContext = %p streams = %i\n", (void *)formatContext, formatContext->nb_streams);
 360
 361         for(i = 0; i < formatContext->nb_streams; i++)
 362         {
 363                 AVStream *stream = formatContext->streams[i];
 364                 AVCodecContext *codecContext = stream->codec;
 365
 366                 switch(codecContext->codec_type)
 367                 {
 368                         case CODEC_TYPE_AUDIO:
 369                                 DATA(self)->audioStreamIndex = i;
 370                                 {
 371                                         AVCodec *codec = avcodec_find_decoder(codecContext->codec_id);
 372
 373                                         if (codec)
 374                                         {
 375                                                 int err = avcodec_open(codecContext, codec);
 376
 377                                                 if (err == 0)
 378                                                 {
 379                                                         DATA(self)->audioContext = codecContext;
 380                                                 }
 381                                         }
 382                                 }
 383
 384                                 //printf("audioStreamIndex = %i\n", DATA(self)->audioStreamIndex);
 385                                 IoObject_setSlot_to_(self, IOSYMBOL("audioChannels"),   IONUMBER(codecContext->channels));
 386                                 IoObject_setSlot_to_(self, IOSYMBOL("audioSampleRate"), IONUMBER(codecContext->sample_rate));
 387                                 IoObject_setSlot_to_(self, IOSYMBOL("audioBitRate"),    IONUMBER(codecContext->bit_rate));
 388                                 IoObject_setSlot_to_(self, IOSYMBOL("audioDuration"),   IONUMBER(stream->duration));
 389                                 IoObject_setSlot_to_(self, IOSYMBOL("audioFrameCount"), IONUMBER(stream->nb_frames));
 390                                 break;
 391
 392                         case CODEC_TYPE_VIDEO:
 393                                 {
 394                                 DATA(self)->videoStreamIndex = i;
 395
 396                                 {
 397                                         AVCodec *codec = avcodec_find_decoder(codecContext->codec_id);
 398
 399                                         if (codec)
 400                                         {
 401                                                 int err = avcodec_open(codecContext, codec);
 402
 403                                                 if (err == 0)
 404                                                 {
 405                                                         DATA(self)->videoContext = codecContext;
 406                                                 }
 407                                         }
 408                                 }
 409
 410                                 //printf("videoStreamIndex = %i\n", DATA(self)->videoStreamIndex);
 411                                 {
 412                                         float framePeriod = (((float)codecContext->time_base.num)/((float)codecContext->time_base.den));
 413                                         //UArray *sizeUArray = UArray_newWithData_type_encoding_size_copy_("", CTYPE_float32_t, CENCODING_NUMBER, 2, 1);
 414                                         IoObject_setSlot_to_(self, IOSYMBOL("framePeriod"),     IONUMBER(framePeriod));
 415                                         IoObject_setSlot_to_(self, IOSYMBOL("videoDuration"),   IONUMBER(stream->duration));
 416                                         IoObject_setSlot_to_(self, IOSYMBOL("videoFrameCount"), IONUMBER(stream->nb_frames));
 417                                 }
 418
 419                                 {
 420                                         UArray *sizeUArray = UArray_newWithData_type_encoding_size_copy_("", CTYPE_float32_t, CENCODING_NUMBER, 2, 1);
 421                                         IoSeq *sizeSeq = IoSeq_newWithUArray_copy_(IOSTATE, sizeUArray, 0);
 422                                         UArray_at_putDouble_(sizeUArray, 0, codecContext->width);
 423                                         UArray_at_putDouble_(sizeUArray, 1, codecContext->height);
 424                                         IoObject_setSlot_to_(self, IOSYMBOL("videoSize"), sizeSeq);
 425                                 }
 426
 427                                 break;
 428                                 }
 429                         case CODEC_TYPE_UNKNOWN:
 430                         case CODEC_TYPE_DATA:
 431                         case CODEC_TYPE_SUBTITLE:
 432                         case CODEC_TYPE_NB:
 433                         case default:
 434                 }
 435         }
 436
 437         return 0;
 438 }
 439
 440 IoObject *IoAVCodec_isAtEnd(IoAVCodec *self, IoObject *locals, IoMessage *m)
 441 {
 442         return IOBOOL(self, DATA(self)->isAtEnd);
 443 }
 444
 445 IoObject *IoAVCodec_decode(IoAVCodec *self, IoObject *locals, IoMessage *m)
 446 {
 447         AVFormatContext *formatContext = DATA(self)->formatContext;
 448         int audioStreamIndex = DATA(self)->audioStreamIndex;
 449         int videoStreamIndex = DATA(self)->videoStreamIndex;
 450         AVPacket *packet = DATA(self)->packet;
 451         int ret;
 452
 453
 454         if(DATA(self)->audioContext == NULL && DATA(self)->videoContext == NULL)
 455         {
 456                 //printf("not open\n");
 457                 return IONIL(self);
 458         }
 459
 460         ret = av_read_frame(formatContext, packet);
 461
 462         if (ret < 0)
 463         {
 464                 //printf("av_read_frame ret = %i\n", ret);
 465
 466                 if(ret == AVERROR_IO)
 467                 {
 468                         DATA(self)->isAtEnd = 1;
 469                 }
 470
 471                 return IONIL(self);
 472         }
 473
 474         if (packet->stream_index == audioStreamIndex && DATA(self)->audioContext)
 475         {
 476                 IoAVCodec_decodeAudioPacket(self,
 477                         formatContext->streams[audioStreamIndex]->codec,
 478                         packet->data, packet->size);
 479         }
 480         else if (packet->stream_index == videoStreamIndex && DATA(self)->videoContext)
 481         {
 482                 IoAVCodec_decodeVideoPacket(self,
 483                         formatContext->streams[videoStreamIndex]->codec,
 484                         packet->data, packet->size);
 485         }
 486         else
 487         {
 488                 av_free_packet(packet);
 489         }
 490
 491         return self;
 492 }
 493
 494 int IoAVCodec_decodeAudioPacket(IoAVCodec *self, AVCodecContext *c, uint8_t *inbuf, size_t size)
 495 {
 496         UArray  *outba  = IoSeq_rawUArray(DATA(self)->outputBuffer);
 497         uint8_t *outbuf = DATA(self)->audioOutBuffer;
 498
 499         //UArray_setItemType_(outba, CTYPE_float32_t);
 500
 501         while (size > 0)
 502         {
 503                 int outSize;
 504                 int len = avcodec_decode_audio(c, (int16_t *)outbuf, &outSize, inbuf, size);
 505
 506                 if (len < 0)
 507                 {
 508                         printf("Error while decoding audio packet\n");
 509                         return -1;
 510                 }
 511
 512                 if (outSize > 0)
 513                 {
 514                         // if a frame has been decoded, output it
 515                         // convert short ints to floats
 516
 517                         size_t sampleCount = outSize / c->channels;
 518                         size_t oldSize = UArray_size(outba);
 519                         //UArray_setSize_(outba, oldSize + sampleCount); // knows it's a float32 array
 520                         UArray_setSize_(outba, oldSize + sampleCount * sizeof(float)); // knows it's a float32 array
 521
 522                         IoAVCodec_ConvertShortToFloat((short *)outbuf, (float *)(UArray_bytes(outba) + oldSize), sampleCount);
 523                 }
 524
 525                 size -= len;
 526                 inbuf += len;
 527         }
 528
 529         return 0;
 530 }
 531
 532 int IoAVCodec_decodeVideoPacket(IoAVCodec *self, AVCodecContext *c, uint8_t *inbuf, size_t size)
 533 {
 534         AVFrame *decodeFrame = DATA(self)->decodedFrame;
 535
 536         while (size > 0)
 537         {
 538                 int got_picture;
 539                 size_t len = avcodec_decode_video(c, DATA(self)->decodedFrame, &got_picture, inbuf, size);
 540
 541                 if (len < 0)
 542                 {
 543                         printf("Error while decoding video packet\n");
 544                         return -1;
 545                 }
 546
 547                 if (got_picture)
 548                 {
 549                         IoList_rawAppend_(DATA(self)->frames, IoAVCode_frameSeqForAVFrame_(self, decodeFrame, c->pix_fmt, c->width, c->height));
 550                 }
 551
 552                 size -= len;
 553                 inbuf += len;
 554         }
 555
 556         return 0;
 557 }
 558
 559 IoSeq *IoAVCode_frameSeqForAVFrame_(IoAVCodec *self, AVFrame *avframe, int srcPixelFormat, int width, int height)
 560 {
 561         AVPicture *rgbPicture = IoAVCode_allocDstPictureIfNeeded(self, srcPixelFormat, width, height);
 562         AVPicture srcPicture;
 563         int result;
 564
 565         memcpy(srcPicture.data,     avframe->data,     sizeof(uint8_t *) * 4);
 566         memcpy(srcPicture.linesize, avframe->linesize, sizeof(int)       * 4);
 567
 568         result = img_convert(rgbPicture, PIX_FMT_RGB24, &srcPicture, srcPixelFormat, width, height);
 569
 570         if (result)
 571         {
 572                 printf("AVCodec: img_convert error?\n");
 573         }
 574
 575         UArray *data = UArray_newWithData_type_encoding_size_copy_(rgbPicture->data[0], CTYPE_uint8_t, CENCODING_NUMBER, width * height * 3, 1);
 576
 577         return IoSeq_newWithUArray_copy_(IOSTATE, data, 0);
 578 }
 579