addons/AVCodec/source/IoAVCodec.c

   1 /*#io
   2 AVCodec ioDoc(
   3                     docCopyright("Steve Dekorte", 2004)
   4                     docLicense("BSD revised")
   5                     docCategory("Media")
   6                     docDescription("""An object for encoding and decoding audio and video streams.""")
   7 */
   8
   9 #include "IoAVCodec.h"
  10 #include "List.h"
  11 #include "IoState.h"
  12 #include "IoNumber.h"
  13 #include "IoSeq.h"
  14 #include "IoList.h"
  15 #include <limits.h>
  16 //#include <math.h>
  17
  18
  19 #define DATA(self) ((IoAVCodecData *)IoObject_dataPointer(self))
  20 #define IVAR(name) (((IoAVCodecData *)IoObject_dataPointer(self))->name)
  21
  22 void IoAVCodec_registerIfNeeded(IoAVCodec *self)
  23 {
  24         avcodec_init();
  25         avcodec_register_all();
  26         av_register_all();
  27 }
  28
  29 IoTag *IoAVCodec_newTag(void *state)
  30 {
  31         IoTag *tag = IoTag_newWithName_("AVCodec");
  32         IoTag_state_(tag, state);
  33         IoTag_cloneFunc_(tag, (IoTagCloneFunc *)IoAVCodec_rawClone);
  34         IoTag_markFunc_(tag, (IoTagMarkFunc *)IoAVCodec_mark);
  35         IoTag_freeFunc_(tag, (IoTagFreeFunc *)IoAVCodec_free);
  36         return tag;
  37 }
  38
  39 IoAVCodec *IoAVCodec_proto(void *state)
  40 {
  41         IoObject *self = IoObject_new(state);
  42         IoObject_tag_(self, IoAVCodec_newTag(state));
  43
  44         IoObject_setDataPointer_(self, calloc(1, sizeof(IoAVCodecData)));
  45
  46         IVAR(inputBuffer)  = IoSeq_new(state);
  47         IVAR(outputBuffer) = IoSeq_new(state);
  48
  49         IoState_registerProtoWithFunc_(state, self, IoAVCodec_proto);
  50
  51         IoAVCodec_registerIfNeeded(self);
  52
  53         {
  54                 IoMethodTable methodTable[] = {
  55
  56                 {"audioInputBuffer",  IoAVCodec_audioInputBuffer},
  57                 {"audioOutputBuffer", IoAVCodec_audioOutputBuffer},
  58
  59                 {"encodeCodecNames",  IoAVCodec_encodeCodecNames},
  60                 {"decodeCodecNames",  IoAVCodec_decodeCodecNames},
  61
  62                 {"open", IoAVCodec_open},
  63                 {"close", IoAVCodec_close},
  64
  65                 {"decode", IoAVCodec_decode},
  66                 {"isAtEnd", IoAVCodec_isAtEnd},
  67                 //{"encode",     IoAVCodec_startEncoding},
  68
  69                 {NULL, NULL},
  70                 };
  71                 IoObject_addMethodTable_(self, methodTable);
  72         }
  73         return self;
  74 }
  75
  76 IoAVCodec *IoAVCodec_rawClone(IoAVCodec *proto)
  77 {
  78         IoObject *self = IoObject_rawClonePrimitive(proto);
  79         IoObject_setDataPointer_(self, calloc(1, sizeof(IoAVCodecData)));
  80
  81         IVAR(inputBuffer)        = IOCLONE(DATA(proto)->inputBuffer);
  82         IVAR(outputBuffer)       = IOCLONE(DATA(proto)->outputBuffer);
  83
  84         return self;
  85 }
  86
  87 IoAVCodec *IoAVCodec_new(void *state)
  88 {
  89         IoObject *proto = IoState_protoWithInitFunction_(state, IoAVCodec_proto);
  90         return IOCLONE(proto);
  91 }
  92
  93 // -----------------------------------------------------------
  94
  95 void IoAVCodec_createContextIfNeeded(IoAVCodec *self)
  96 {
  97         if(!IVAR(packet))
  98         {
  99                 IVAR(packet) = calloc(1, sizeof(AVPacket));
 100         }
 101
 102         // video
 103
 104         // frames
 105         if (!IVAR(frames))
 106         {
 107                 IVAR(frames) = IoList_new(IOSTATE);
 108                 IoObject_setSlot_to_(self, IOSYMBOL("frames"), IVAR(frames));
 109         }
 110
 111         // videoSize
 112         {
 113                 UArray *sizeUArray = UArray_newWithData_type_encoding_size_copy_("", CTYPE_float32_t, CENCODING_NUMBER, 2, 1);
 114                 IoSeq *sizeSeq = IoSeq_newWithUArray_copy_(IOSTATE, sizeUArray, 0);
 115                 IoObject_setSlot_to_(self, IOSYMBOL("videoSize"), sizeSeq);
 116         }
 117
 118         if (!IVAR(decodedFrame))
 119         {
 120                 IVAR(decodedFrame) = avcodec_alloc_frame();
 121         }
 122
 123         // audio
 124
 125         if(!IVAR(audioOutBuffer))
 126         {
 127                 IVAR(audioOutBuffer) = malloc(AVCODEC_MAX_AUDIO_FRAME_SIZE);
 128         }
 129 }
 130
 131 AVPicture *IoAVCode_allocDstPictureIfNeeded(IoAVCodec *self, int pix_fmt, int width, int height)
 132 {
 133         if(!IVAR(rgbPicture))
 134         {
 135                 IVAR(rgbPicture) = calloc(1, sizeof(AVPicture));
 136                 avpicture_alloc(IVAR(rgbPicture), PIX_FMT_RGB24, width, height);
 137         }
 138
 139         return IVAR(rgbPicture);
 140 }
 141
 142
 143 void IoAVCodec_freeContextIfNeeded(IoAVCodec *self)
 144 {
 145         //printf("IoAVCodec_freeContextIfNeeded\n");
 146
 147         IVAR(audioContext) = NULL;
 148         IVAR(videoContext) = NULL;
 149
 150         if (IVAR(audioContext))
 151         {
 152                 //avcodec_close(IVAR(audioContext));
 153                 //av_free(IVAR(audioContext));
 154                 IVAR(audioContext) = NULL;
 155         }
 156
 157         if (IVAR(videoContext))
 158         {
 159                 //avcodec_close(IVAR(audioContext));
 160                 //av_free(IVAR(audioContext));
 161                 IVAR(audioContext) = NULL;
 162         }
 163
 164         if (IVAR(formatContext))
 165         {
 166                 av_close_input_file(IVAR(formatContext));
 167                 //av_free(IVAR(formatContext));
 168                 IVAR(formatContext) = NULL;
 169         }
 170
 171         if(IVAR(packet))
 172         {
 173                 //free(IVAR(packet));
 174                 IVAR(packet) = NULL;
 175         }
 176
 177         if (IVAR(audioOutBuffer))
 178         {
 179                 //free(IVAR(audioOutBuffer));
 180                 IVAR(audioOutBuffer) = NULL;
 181         }
 182
 183         if(IVAR(decodedFrame))
 184         {
 185                 //av_free(IVAR(decodedFrame));
 186                 IVAR(decodedFrame) = NULL;
 187         }
 188
 189         if(IVAR(rgbPicture))
 190         {
 191                 avpicture_free(IVAR(rgbPicture));
 192                 //free(IVAR(rgbPicture));
 193                 IVAR(rgbPicture) = NULL;
 194         }
 195
 196         //printf("IoAVCodec_freeContextIfNeeded done\n");
 197 }
 198
 199 void IoAVCodec_free(IoAVCodec *self)
 200 {
 201         IoAVCodec_freeContextIfNeeded(self);
 202         free(IoObject_dataPointer(self));
 203 }
 204
 205 void IoAVCodec_mark(IoAVCodec *self)
 206 {
 207         if(IVAR(frames)) IoObject_shouldMark(IVAR(frames));
 208
 209         IoObject_shouldMark(IVAR(inputBuffer));
 210         IoObject_shouldMark(IVAR(outputBuffer));
 211 }
 212
 213 // -----------------------------------------------------------
 214
 215 void IoAVCodec_error_(IoAVCodec *self, IoMessage *m, char *s)
 216 {
 217         fprintf(stderr, s);
 218         IoState_error_(IOSTATE, m, s);
 219 }
 220
 221 IoObject *IoAVCodec_audioInputBuffer(IoAVCodec *self, IoObject *locals, IoMessage *m)
 222 {
 223         /*#io
 224         docSlot("inputBuffer", "Returns the input buffer.")
 225         */
 226         return IVAR(inputBuffer);
 227 }
 228
 229 IoObject *IoAVCodec_audioOutputBuffer(IoAVCodec *self, IoObject *locals, IoMessage *m)
 230 {
 231         /*#io
 232         docSlot("outputBuffer", "Returns the output buffer.")
 233         */
 234         return IVAR(outputBuffer);
 235 }
 236
 237 IoObject *IoAVCodec_decodeCodecNames(IoAVCodec *self, IoObject *locals, IoMessage *m)
 238 {
 239         /*#io
 240         docSlot("decodeCodecNames", "Returns a list of strings with the names of the decode codecs.")
 241         */
 242
 243         AVCodec *p = first_avcodec;
 244         IoList *names = IoList_new(IOSTATE);
 245
 246         while (p)
 247         {
 248                 if (p->decode)
 249                 {
 250                         IoList_rawAppend_(names, IOSYMBOL(p->name));
 251                 }
 252
 253                 p = p->next;
 254         }
 255
 256         return names;
 257 }
 258
 259 IoObject *IoAVCodec_encodeCodecNames(IoAVCodec *self, IoObject *locals, IoMessage *m)
 260 {
 261         /*#io
 262         docSlot("encodeCodecNames", "Returns a list of strings with the names of the encode codecs.")
 263         */
 264
 265         AVCodec *p = first_avcodec;
 266         IoList *names = IoList_new(IOSTATE);
 267
 268         while (p)
 269         {
 270                 if (p->encode)
 271                 {
 272                         IoList_rawAppend_(names, IOSYMBOL(p->name));
 273                 }
 274
 275                 p = p->next;
 276         }
 277
 278         return names;
 279 }
 280
 281 void IoAVCodec_ConvertShortToFloat(short *s, float *f, size_t sampleCount)
 282 {
 283         size_t i;
 284
 285         for (i = 0; i < sampleCount; i ++)
 286         {
 287                 *f = (float)(*s) / (float)SHRT_MAX;
 288                 f ++;
 289                 *f = (float)(*s) / (float)SHRT_MAX;
 290                 s ++;
 291         }
 292 }
 293
 294 void IoAVCodec_ConvertFloatToShort(float *f, short *s, size_t sampleCount)
 295 {
 296         size_t i;
 297
 298         for (i = 0; i < sampleCount; i ++)
 299         {
 300                 *s = (short)((*f) * SHRT_MAX);
 301                 s ++;
 302                 *s = (short)((*f) * SHRT_MAX);
 303                 f ++;
 304         }
 305 }
 306
 307 // -----------------------------------------------------------
 308
 309 #define INBUF_SIZE 4096
 310
 311 // ----------------------------------------------------------------------------------------------
 312
 313 IoObject *IoAVCodec_close(IoAVCodec *self, IoObject *locals, IoMessage *m)
 314 {
 315         IoAVCodec_registerIfNeeded(self);
 316         IoAVCodec_freeContextIfNeeded(self);
 317         return self;
 318 }
 319
 320 int IoAVCodec_openFile(IoAVCodec *self)
 321 {
 322         AVInputFormat *inputFormat;
 323         IoObject *fileName = IoObject_symbolGetSlot_(self, IOSYMBOL("path"));
 324     int err = av_open_input_file(&IVAR(formatContext), CSTRING(fileName), NULL, 0, NULL);
 325         return err;
 326 }
 327
 328 IoObject *IoAVCodec_open(IoAVCodec *self, IoObject *locals, IoMessage *m)
 329 {
 330         int err;
 331
 332         IoAVCodec_registerIfNeeded(self);
 333         IoAVCodec_freeContextIfNeeded(self);
 334         IoAVCodec_createContextIfNeeded(self);
 335
 336         IVAR(isAtEnd) = 0;
 337
 338         err = IoAVCodec_openFile(self);
 339
 340     if (err != 0)
 341         {
 342                 IoObject *fileName = IoObject_symbolGetSlot_(self, IOSYMBOL("path"));
 343                 IoState_error_(IOSTATE, m, "error %i opening file %s\n", err, CSTRING(fileName));
 344                 return IONIL(self);
 345         }
 346
 347         IoAVCodec_findStreams(self);
 348         av_read_play(IVAR(formatContext));
 349
 350         return self;
 351 }
 352
 353 int IoAVCodec_findStreams(IoAVCodec *self)
 354 {
 355     AVFormatContext *formatContext = IVAR(formatContext);
 356         int i;
 357
 358         av_find_stream_info(formatContext);
 359
 360         //printf("formatContext = %p streams = %i\n", (void *)formatContext, formatContext->nb_streams);
 361
 362     for(i = 0; i < formatContext->nb_streams; i++)
 363         {
 364                 AVStream *stream = formatContext->streams[i];
 365         AVCodecContext *codecContext = stream->codec;
 366
 367         switch(codecContext->codec_type)
 368                 {
 369                         case CODEC_TYPE_AUDIO:
 370                                 IVAR(audioStreamIndex) = i;
 371                                 {
 372                                         AVCodec *codec = avcodec_find_decoder(codecContext->codec_id);
 373
 374                                         if (codec)
 375                                         {
 376                                                 int err = avcodec_open(codecContext, codec);
 377
 378                                                 if (err == 0)
 379                                                 {
 380                                                         IVAR(audioContext) = codecContext;
 381                                                 }
 382                                         }
 383                                 }
 384
 385                                 //printf("audioStreamIndex = %i\n", IVAR(audioStreamIndex));
 386                                 IoObject_setSlot_to_(self, IOSYMBOL("audioChannels"),   IONUMBER(codecContext->channels));
 387                                 IoObject_setSlot_to_(self, IOSYMBOL("audioSampleRate"), IONUMBER(codecContext->sample_rate));
 388                                 IoObject_setSlot_to_(self, IOSYMBOL("audioBitRate"),    IONUMBER(codecContext->bit_rate));
 389                                 IoObject_setSlot_to_(self, IOSYMBOL("audioDuration"),   IONUMBER(stream->duration));
 390                                 IoObject_setSlot_to_(self, IOSYMBOL("audioFrameCount"), IONUMBER(stream->nb_frames));
 391                                 break;
 392
 393                         case CODEC_TYPE_VIDEO:
 394                                 {
 395                                 IVAR(videoStreamIndex) = i;
 396
 397                                 {
 398                                         AVCodec *codec = avcodec_find_decoder(codecContext->codec_id);
 399
 400                                         if (codec)
 401                                         {
 402                                                 int err = avcodec_open(codecContext, codec);
 403
 404                                                 if (err == 0)
 405                                                 {
 406                                                         IVAR(videoContext) = codecContext;
 407                                                 }
 408                                         }
 409                                 }
 410
 411                                 //printf("videoStreamIndex = %i\n", IVAR(videoStreamIndex));
 412                                 {
 413                                         float framePeriod = (((float)codecContext->time_base.num)/((float)codecContext->time_base.den));
 414                                         UArray *sizeUArray = UArray_newWithData_type_encoding_size_copy_("", CTYPE_float32_t, CENCODING_NUMBER, 2, 1);
 415                                         IoObject_setSlot_to_(self, IOSYMBOL("framePeriod"),     IONUMBER(framePeriod));
 416                                         IoObject_setSlot_to_(self, IOSYMBOL("videoDuration"),   IONUMBER(stream->duration));
 417                                         IoObject_setSlot_to_(self, IOSYMBOL("videoFrameCount"), IONUMBER(stream->nb_frames));
 418                                 }
 419
 420                                 {
 421                                         UArray *sizeUArray = UArray_newWithData_type_encoding_size_copy_("", CTYPE_float32_t, CENCODING_NUMBER, 2, 1);
 422                                         IoSeq *sizeSeq = IoSeq_newWithUArray_copy_(IOSTATE, sizeUArray, 0);
 423                                         UArray_at_putDouble_(sizeUArray, 0, codecContext->width);
 424                                         UArray_at_putDouble_(sizeUArray, 1, codecContext->height);
 425                                         IoObject_setSlot_to_(self, IOSYMBOL("videoSize"), sizeSeq);
 426                                 }
 427
 428                                 break;
 429                                 }
 430         }
 431     }
 432
 433         return 0;
 434 }
 435
 436 IoObject *IoAVCodec_isAtEnd(IoAVCodec *self, IoObject *locals, IoMessage *m)
 437 {
 438         return IOBOOL(self, IVAR(isAtEnd));
 439 }
 440
 441 IoObject *IoAVCodec_decode(IoAVCodec *self, IoObject *locals, IoMessage *m)
 442 {
 443         AVFormatContext *formatContext = IVAR(formatContext);
 444         int audioStreamIndex = IVAR(audioStreamIndex);
 445         int videoStreamIndex = IVAR(videoStreamIndex);
 446     AVPacket *packet = IVAR(packet);
 447         int ret;
 448
 449
 450         if(IVAR(audioContext) == NULL && IVAR(videoContext) == NULL)
 451         {
 452                 //printf("not open\n");
 453                 return IONIL(self);
 454         }
 455
 456         ret = av_read_frame(formatContext, packet);
 457
 458         if (ret < 0)
 459         {
 460                 //printf("av_read_frame ret = %i\n", ret);
 461
 462                 if(ret == AVERROR_IO)
 463                 {
 464                         IVAR(isAtEnd) = 1;
 465                 }
 466
 467                 return IONIL(self);
 468         }
 469
 470         if (packet->stream_index == audioStreamIndex && IVAR(audioContext))
 471         {
 472                 IoAVCodec_decodeAudioPacket(self,
 473                         formatContext->streams[audioStreamIndex]->codec,
 474                         packet->data, packet->size);
 475         }
 476         else if (packet->stream_index == videoStreamIndex && IVAR(videoContext))
 477         {
 478                 IoAVCodec_decodeVideoPacket(self,
 479                         formatContext->streams[videoStreamIndex]->codec,
 480                         packet->data, packet->size);
 481         }
 482         else
 483         {
 484                 av_free_packet(packet);
 485         }
 486
 487         return self;
 488 }
 489
 490 int IoAVCodec_decodeAudioPacket(IoAVCodec *self, AVCodecContext *c, uint8_t *inbuf, size_t size)
 491 {
 492         UArray  *outba  = IoSeq_rawUArray(IVAR(outputBuffer));
 493         uint8_t *outbuf = IVAR(audioOutBuffer);
 494
 495         //UArray_setItemType_(outba, CTYPE_float32_t);
 496
 497         while (size > 0)
 498         {
 499                 int outSize;
 500                 int len = avcodec_decode_audio(c, (int16_t *)outbuf, &outSize, inbuf, size);
 501
 502                 if (len < 0)
 503                 {
 504                         printf("Error while decoding audio packet\n");
 505                         return -1;
 506                 }
 507
 508                 if (outSize > 0)
 509                 {
 510                         // if a frame has been decoded, output it
 511                         // convert short ints to floats
 512
 513                         size_t sampleCount = outSize / c->channels;
 514                         size_t oldSize = UArray_size(outba);
 515                         //UArray_setSize_(outba, oldSize + sampleCount); // knows it's a float32 array
 516                         UArray_setSize_(outba, oldSize + sampleCount * sizeof(float)); // knows it's a float32 array
 517
 518                         IoAVCodec_ConvertShortToFloat((short *)outbuf, (float *)(UArray_bytes(outba) + oldSize), sampleCount);
 519                 }
 520
 521                 size -= len;
 522                 inbuf += len;
 523         }
 524
 525         return 0;
 526 }
 527
 528 int IoAVCodec_decodeVideoPacket(IoAVCodec *self, AVCodecContext *c, uint8_t *inbuf, size_t size)
 529 {
 530     AVFrame *decodeFrame = IVAR(decodedFrame);
 531
 532         while (size > 0)
 533         {
 534                 int got_picture;
 535                 size_t len = avcodec_decode_video(c, IVAR(decodedFrame), &got_picture, inbuf, size);
 536
 537                 if (len < 0)
 538                 {
 539                         printf("Error while decoding video packet\n");
 540                         return -1;
 541                 }
 542
 543                 if (got_picture)
 544                 {
 545                         IoList_rawAppend_(IVAR(frames), IoAVCode_frameSeqForAVFrame_(self, decodeFrame, c->pix_fmt, c->width, c->height));
 546                 }
 547
 548                 size -= len;
 549                 inbuf += len;
 550         }
 551
 552         return 0;
 553 }
 554
 555 IoSeq *IoAVCode_frameSeqForAVFrame_(IoAVCodec *self, AVFrame *avframe, int srcPixelFormat, int width, int height)
 556 {
 557         AVPicture *rgbPicture = IoAVCode_allocDstPictureIfNeeded(self, srcPixelFormat, width, height);
 558         AVPicture srcPicture;
 559         int result;
 560
 561         memcpy(srcPicture.data,     avframe->data,     sizeof(uint8_t *) * 4);
 562         memcpy(srcPicture.linesize, avframe->linesize, sizeof(int)       * 4);
 563
 564         result = img_convert(rgbPicture, PIX_FMT_RGB24, &srcPicture, srcPixelFormat, width, height);
 565
 566         if (result)
 567         {
 568                 printf("AVCodec: img_convert error?\n");
 569         }
 570
 571         UArray *data = UArray_newWithData_type_encoding_size_copy_(rgbPicture->data[0], CTYPE_uint8_t, CENCODING_NUMBER, width * height * 3, 1);
 572
 573         return IoSeq_newWithUArray_copy_(IOSTATE, data, 0);
 574 }
 575