libavfilter/vf_lcevc.c

   1 /*
   2  * This file is part of FFmpeg.
   3  *
   4  * Copyright (c) 2024 James Almer <jamrial@gmail.com>
   5  *
   6  * FFmpeg is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * FFmpeg is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with FFmpeg; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 #include <stdint.h>
  22
  23 #include <LCEVC/lcevc_dec.h>
  24
  25 #include "libavutil/internal.h"
  26 #include "libavutil/opt.h"
  27 #include "filters.h"
  28 #include "video.h"
  29
  30 typedef struct LCEVCContext {
  31     LCEVC_DecoderHandle decoder;
  32     int w, h;
  33 } LCEVCContext;
  34
  35 static LCEVC_ColorFormat map_format(int format)
  36 {
  37     switch (format) {
  38     case AV_PIX_FMT_YUV420P:
  39         return LCEVC_I420_8;
  40     case AV_PIX_FMT_YUV420P10:
  41         return LCEVC_I420_10_LE;
  42     case AV_PIX_FMT_NV12:
  43         return LCEVC_NV12_8;
  44     case AV_PIX_FMT_NV21:
  45         return LCEVC_NV21_8;
  46     case AV_PIX_FMT_GRAY8:
  47         return LCEVC_GRAY_8;
  48     case AV_PIX_FMT_GRAY10LE:
  49         return LCEVC_GRAY_10_LE;
  50     }
  51
  52     return LCEVC_ColorFormat_Unknown;
  53 }
  54
  55 static inline LCEVC_ColorRange map_range(int range)
  56 {
  57     switch (range) {
  58     case AVCOL_RANGE_MPEG:
  59         return LCEVC_ColorRange_Limited;
  60     case AVCOL_RANGE_JPEG:
  61         return LCEVC_ColorRange_Full;
  62     }
  63
  64     return LCEVC_ColorRange_Unknown;
  65 }
  66
  67 static inline enum AVColorRange map_av_range(int range)
  68 {
  69     switch (range) {
  70     case LCEVC_ColorRange_Limited:
  71         return AVCOL_RANGE_MPEG;
  72     case LCEVC_ColorRange_Full:
  73         return AVCOL_RANGE_JPEG;
  74     }
  75
  76     return AVCOL_RANGE_UNSPECIFIED;
  77 }
  78
  79 static int alloc_base_frame(AVFilterLink *inlink, const AVFrame *in,
  80                             LCEVC_PictureHandle *picture)
  81 {
  82     AVFilterContext *ctx = inlink->dst;
  83     LCEVCContext *lcevc = ctx->priv;
  84     LCEVC_PictureDesc desc;
  85     LCEVC_PicturePlaneDesc planes[AV_VIDEO_MAX_PLANES] = { 0 };
  86     LCEVC_ColorFormat fmt = map_format(in->format);
  87     int width = in->width - in->crop_left - in->crop_right;
  88     int height = in->height - in->crop_top - in->crop_bottom;
  89     LCEVC_ReturnCode res;
  90
  91     res = LCEVC_DefaultPictureDesc(&desc, fmt, width, height);
  92     if (res != LCEVC_Success) {
  93         av_log(ctx, AV_LOG_ERROR, "LCEVC_DefaultPictureDesc failed\n");
  94         return AVERROR_EXTERNAL;
  95     }
  96
  97     for (int i = 0; i < AV_VIDEO_MAX_PLANES; i++) {
  98         planes[i].firstSample = in->data[i];
  99         planes[i].rowByteStride = in->linesize[i];
 100     }
 101
 102     desc.cropTop    = in->crop_top;
 103     desc.cropBottom = in->crop_bottom;
 104     desc.cropLeft   = in->crop_left;
 105     desc.cropRight  = in->crop_right;
 106     desc.sampleAspectRatioNum = in->sample_aspect_ratio.num;
 107     desc.sampleAspectRatioDen = in->sample_aspect_ratio.den;
 108     desc.colorRange = map_range(in->color_range);
 109     desc.colorPrimaries = (LCEVC_ColorPrimaries)in->color_primaries;
 110     desc.matrixCoefficients = (LCEVC_MatrixCoefficients)in->colorspace;
 111     desc.transferCharacteristics = (LCEVC_TransferCharacteristics)in->color_trc;
 112     av_log(ctx, AV_LOG_DEBUG, "in  PTS %"PRId64", %dx%d, "
 113                               "%"SIZE_SPECIFIER"/%"SIZE_SPECIFIER"/%"SIZE_SPECIFIER"/%"SIZE_SPECIFIER", "
 114                               "SAR %d:%d\n",
 115            in->pts, in->width, in->height,
 116            in->crop_top, in->crop_bottom, in->crop_left, in->crop_right,
 117            in->sample_aspect_ratio.num, in->sample_aspect_ratio.den);
 118
 119     res = LCEVC_AllocPictureExternal(lcevc->decoder, &desc, NULL, planes, picture);
 120     if (res != LCEVC_Success) {
 121         av_log(ctx, AV_LOG_ERROR, "LCEVC_AllocPictureExternal to allocate a buffer for a base frame\n");
 122         return AVERROR_EXTERNAL;
 123     }
 124
 125     return 0;
 126 }
 127
 128 static int send_frame(AVFilterLink *inlink, AVFrame *in)
 129 {
 130     AVFilterContext *ctx = inlink->dst;
 131     LCEVCContext *lcevc = ctx->priv;
 132     LCEVC_PictureHandle picture;
 133     const AVFrameSideData *sd = av_frame_get_side_data(in, AV_FRAME_DATA_LCEVC);
 134     LCEVC_ReturnCode res;
 135     int ret;
 136
 137     ret = alloc_base_frame(inlink, in, &picture);
 138     if (ret < 0)
 139         return ret;
 140
 141     if (sd) {
 142         res = LCEVC_SendDecoderEnhancementData(lcevc->decoder, in->pts, 0, sd->data, sd->size);
 143         if (res == LCEVC_Again)
 144             return AVERROR(EAGAIN);
 145         else if (res != LCEVC_Success) {
 146             av_log(ctx, AV_LOG_ERROR, "LCEVC_SendDecoderEnhancementData failed\n");
 147             return AVERROR_EXTERNAL;
 148         }
 149     }
 150
 151     res = LCEVC_SendDecoderBase(lcevc->decoder, in->pts, 0, picture, -1, in);
 152     if (res != LCEVC_Success) {
 153         av_log(ctx, AV_LOG_ERROR, "LCEVC_SendDecoderBase failed\n");
 154         LCEVC_FreePicture(lcevc->decoder, picture);
 155         return AVERROR_EXTERNAL;
 156     }
 157
 158     return 0;
 159 }
 160
 161 static int alloc_enhanced_frame(AVFilterLink *inlink, const AVFrame *out,
 162                                 LCEVC_PictureHandle *picture)
 163 {
 164     AVFilterContext *ctx = inlink->dst;
 165     LCEVCContext *lcevc = ctx->priv;
 166     LCEVC_PictureDesc desc;
 167     LCEVC_PicturePlaneDesc planes[AV_VIDEO_MAX_PLANES] = { 0 };
 168     LCEVC_ColorFormat fmt = map_format(out->format);
 169     LCEVC_ReturnCode res;
 170
 171     res = LCEVC_DefaultPictureDesc(&desc, fmt, out->width, out->height);
 172     if (res != LCEVC_Success)
 173         return AVERROR_EXTERNAL;
 174
 175     for (int i = 0; i < AV_VIDEO_MAX_PLANES; i++) {
 176         planes[i].firstSample = out->data[i];
 177         planes[i].rowByteStride = out->linesize[i];
 178     }
 179
 180     res = LCEVC_AllocPictureExternal(lcevc->decoder, &desc, NULL, planes, picture);
 181     if (res != LCEVC_Success) {
 182         av_log(ctx, AV_LOG_ERROR, "LCEVC_AllocPictureExternal to allocate a buffer for an enhanced frame\n");
 183         return AVERROR_EXTERNAL;
 184     }
 185
 186     return 0;
 187 }
 188
 189 static int generate_output(AVFilterLink *inlink, AVFrame *out)
 190 {
 191     AVFilterContext *ctx = inlink->dst;
 192     AVFilterLink *outlink = ctx->outputs[0];
 193     LCEVCContext *lcevc = ctx->priv;
 194     LCEVC_PictureDesc desc;
 195     LCEVC_DecodeInformation info;
 196     LCEVC_PictureHandle picture;
 197     LCEVC_ReturnCode res;
 198
 199     res = LCEVC_ReceiveDecoderPicture(lcevc->decoder, &picture, &info);
 200     if (res == LCEVC_Again) {
 201         int64_t pts;
 202         int status;
 203         if (ff_inlink_acknowledge_status(inlink, &status, &pts)) {
 204             av_frame_free(&out);
 205             ff_outlink_set_status(outlink, status, pts);
 206             return 0;
 207         }
 208         // this shouldn't be reachable, but instead of asserting, just error out
 209         return AVERROR_BUG;
 210     } else if (res != LCEVC_Success) {
 211         av_log(ctx, AV_LOG_ERROR, "LCEVC_ReceiveDecoderPicture failed\n");
 212         return AVERROR_EXTERNAL;
 213     }
 214
 215     av_frame_copy_props(out, (AVFrame *)info.baseUserData);
 216     av_frame_remove_side_data(out, AV_FRAME_DATA_LCEVC);
 217
 218     av_frame_free((AVFrame **)&info.baseUserData);
 219
 220     res = LCEVC_GetPictureDesc(lcevc->decoder, picture, &desc);
 221     LCEVC_FreePicture(lcevc->decoder, picture);
 222
 223     out->crop_top = desc.cropTop;
 224     out->crop_bottom = desc.cropBottom;
 225     out->crop_left = desc.cropLeft;
 226     out->crop_right = desc.cropRight;
 227     out->sample_aspect_ratio.num = outlink->sample_aspect_ratio.num = desc.sampleAspectRatioNum;
 228     out->sample_aspect_ratio.den = outlink->sample_aspect_ratio.den = desc.sampleAspectRatioDen;
 229     out->color_range = map_range(desc.colorRange);
 230     out->color_primaries = (enum AVColorPrimaries)desc.colorPrimaries;
 231     out->colorspace = (enum AVColorSpace)desc.matrixCoefficients;
 232     out->color_trc = (enum AVColorTransferCharacteristic)desc.transferCharacteristics;
 233     out->width = outlink->w = desc.width + out->crop_left + out->crop_right;
 234     out->height = outlink->h = desc.height + out->crop_top + out->crop_bottom;
 235
 236     av_log(ctx, AV_LOG_DEBUG, "out PTS %"PRId64", %dx%d, "
 237                               "%"SIZE_SPECIFIER"/%"SIZE_SPECIFIER"/%"SIZE_SPECIFIER"/%"SIZE_SPECIFIER", "
 238                               "SAR %d:%d, "
 239                               "hasEnhancement %d, enhanced %d\n",
 240            out->pts, out->width, out->height,
 241            out->crop_top, out->crop_bottom, out->crop_left, out->crop_right,
 242            out->sample_aspect_ratio.num, out->sample_aspect_ratio.den,
 243            info.hasEnhancement, info.enhanced);
 244
 245     return ff_filter_frame(outlink, out);
 246 }
 247
 248 static int receive_frame(AVFilterLink *inlink, AVFrame *out)
 249 {
 250     AVFilterContext *ctx = inlink->dst;
 251     LCEVCContext *lcevc = ctx->priv;
 252     LCEVC_PictureHandle picture;
 253     LCEVC_ReturnCode res;
 254     int ret;
 255
 256     ret = alloc_enhanced_frame(inlink, out, &picture);
 257     if (ret < 0)
 258         return ret;
 259
 260     res = LCEVC_SendDecoderPicture(lcevc->decoder, picture);
 261     if (res != LCEVC_Success) {
 262         av_log(ctx, AV_LOG_ERROR, "LCEVC_SendDecoderPicture failed\n");
 263         return AVERROR_EXTERNAL;
 264     }
 265
 266     return generate_output(inlink, out);
 267 }
 268
 269 static int config_props(AVFilterLink *outlink)
 270 {
 271     AVFilterContext *ctx = outlink->src;
 272     AVFilterLink *inlink = ctx->inputs[0];
 273     LCEVCContext *lcevc = ctx->priv;
 274
 275     outlink->w = lcevc->w = inlink->w * 2 / FFMAX(inlink->sample_aspect_ratio.den, 1);
 276     outlink->h = lcevc->h = inlink->h * 2 / FFMAX(inlink->sample_aspect_ratio.den, 1);
 277     outlink->sample_aspect_ratio = (AVRational) { 0, 1 };
 278
 279     return 0;
 280 }
 281
 282 static void flush_bases(AVFilterContext *ctx)
 283 {
 284     LCEVCContext *lcevc = ctx->priv;
 285     LCEVC_PictureHandle picture;
 286
 287     while (LCEVC_ReceiveDecoderBase(lcevc->decoder, &picture) == LCEVC_Success)
 288         LCEVC_FreePicture(lcevc->decoder, picture);
 289 }
 290
 291 static int activate(AVFilterContext *ctx)
 292 {
 293     LCEVCContext *lcevc   = ctx->priv;
 294     AVFilterLink *inlink  = ctx->inputs[0];
 295     AVFilterLink *outlink = ctx->outputs[0];
 296     AVFrame *in, *out;
 297     int status, ret;
 298
 299     FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
 300
 301     ret = ff_inlink_consume_frame(inlink, &in);
 302     if (ret < 0)
 303         return ret;
 304     if (!ret) {
 305         int64_t pts;
 306         if (ff_inlink_acknowledge_status(inlink, &status, &pts)) {
 307             if (!status)
 308                 ff_outlink_set_status(outlink, status, pts);
 309         }
 310         if (!status)
 311             FF_FILTER_FORWARD_WANTED(outlink, inlink);
 312     }
 313
 314     if (in) {
 315        if (in->width  != inlink->w ||
 316             in->height != inlink->h ||
 317             in->sample_aspect_ratio.den != inlink->sample_aspect_ratio.den ||
 318             in->sample_aspect_ratio.num != inlink->sample_aspect_ratio.num) {
 319             inlink->dst->inputs[0]->w                       = in->width;
 320             inlink->dst->inputs[0]->h                       = in->height;
 321             inlink->dst->inputs[0]->sample_aspect_ratio.den = in->sample_aspect_ratio.den;
 322             inlink->dst->inputs[0]->sample_aspect_ratio.num = in->sample_aspect_ratio.num;
 323
 324             config_props(outlink);
 325         }
 326
 327         ret = send_frame(inlink, in);
 328         if (ret < 0)
 329             return ret;
 330     }
 331
 332     out = ff_get_video_buffer(outlink, lcevc->w, lcevc->h);
 333     if (!out)
 334         return AVERROR(ENOMEM);
 335
 336     ret = receive_frame(inlink, out);
 337     if (ret < 0) {
 338         av_frame_free(&out);
 339         return ret;
 340     }
 341
 342     flush_bases(ctx);
 343
 344     return ret;
 345 }
 346
 347 static void log_callback(LCEVC_DecoderHandle dec, LCEVC_Event event,
 348                          LCEVC_PictureHandle pic, const LCEVC_DecodeInformation *info,
 349                          const uint8_t *data, uint32_t size, void *logctx)
 350 {
 351     if (event != LCEVC_Log) // shouldn't happen
 352         return;
 353
 354     if (strlen(data) != size) // sanitize input
 355         return;
 356
 357     av_log(logctx, AV_LOG_INFO, "LCEVC Log: %s\n", data);
 358 }
 359
 360 static av_cold int init(AVFilterContext *ctx)
 361 {
 362     LCEVCContext *lcevc = ctx->priv;
 363     LCEVC_AccelContextHandle dummy = { 0 };
 364     const int32_t event = LCEVC_Log;
 365     LCEVC_ReturnCode res;
 366
 367     res = LCEVC_CreateDecoder(&lcevc->decoder, dummy);
 368     if (res != LCEVC_Success) {
 369         av_log(ctx, AV_LOG_ERROR, "LCEVC_CreateDecoder failed\n");
 370         return AVERROR_EXTERNAL;
 371     }
 372
 373     res = LCEVC_ConfigureDecoderInt(lcevc->decoder, "log_level", 4);
 374     if (res != LCEVC_Success) {
 375         av_log(ctx, AV_LOG_ERROR, "LCEVC_ConfigureDecoderInt failed to set \"log_level\"\n");
 376         return AVERROR_EXTERNAL;
 377     }
 378     res = LCEVC_ConfigureDecoderIntArray(lcevc->decoder, "events", 1, &event);
 379     if (res != LCEVC_Success) {
 380         av_log(ctx, AV_LOG_ERROR, "LCEVC_ConfigureDecoderIntArray failed to set \"events\"\n");
 381         return AVERROR_EXTERNAL;
 382     }
 383     res = LCEVC_SetDecoderEventCallback(lcevc->decoder, log_callback, ctx);
 384     if (res != LCEVC_Success) {
 385         av_log(ctx, AV_LOG_ERROR, "LCEVC_SetDecoderEventCallback failed\n");
 386         return AVERROR_EXTERNAL;
 387     }
 388
 389     res = LCEVC_InitializeDecoder(lcevc->decoder);
 390     if (res != LCEVC_Success) {
 391         av_log(ctx, AV_LOG_ERROR, "LCEVC_InitializeDecoder failed\n");
 392         return AVERROR_EXTERNAL;
 393     }
 394
 395     return 0;
 396 }
 397
 398 static av_cold void uninit(AVFilterContext *ctx)
 399 {
 400     LCEVCContext *lcevc = ctx->priv;
 401
 402     LCEVC_DestroyDecoder(lcevc->decoder);
 403 }
 404
 405 static const AVFilterPad lcevc_outputs[] = {
 406     {
 407         .name         = "default",
 408         .type         = AVMEDIA_TYPE_VIDEO,
 409         .config_props = config_props,
 410     },
 411 };
 412
 413 static const enum AVPixelFormat pix_fmts[] = {
 414     AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV420P10LE,
 415     AV_PIX_FMT_NV12, AV_PIX_FMT_NV21,
 416     AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY10LE,
 417     AV_PIX_FMT_NONE
 418 };
 419
 420 const FFFilter ff_vf_lcevc = {
 421     .p.name        = "lcevc",
 422     .p.description = NULL_IF_CONFIG_SMALL("LCEVC"),
 423     .activate      = activate,
 424     FILTER_INPUTS(ff_video_default_filterpad),
 425     FILTER_OUTPUTS(lcevc_outputs),
 426     FILTER_PIXFMTS_ARRAY(pix_fmts),
 427     .priv_size     = sizeof(LCEVCContext),
 428     .init          = init,
 429     .uninit        = uninit,
 430 };