libavfilter/vf_tiltandshift.c

   1 /*
   2  * Copyright (c) 2014 Vittorio Giovara
   3  *
   4  * This file is part of FFmpeg.
   5  *
   6  * FFmpeg is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * FFmpeg is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with FFmpeg; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 /**
  22  * @file vf_tiltandshift.c
  23  * Simple time and space inverter.
  24  */
  25
  26 #include <string.h>
  27
  28 #include "libavutil/imgutils.h"
  29 #include "libavutil/mem.h"
  30 #include "libavutil/opt.h"
  31 #include "libavutil/pixdesc.h"
  32
  33 #include "avfilter.h"
  34 #include "filters.h"
  35 #include "video.h"
  36
  37 enum PaddingOption {
  38     TILT_NONE,
  39     TILT_FRAME,
  40     TILT_BLACK,
  41     TILT_OPT_MAX,
  42 };
  43
  44 typedef struct TiltandshiftContext {
  45     const AVClass *class;
  46
  47     /* set when all input frames have been processed and we have to
  48      * empty buffers, pad and then return */
  49     int eof_recv;
  50
  51     /* live or static sliding */
  52     int tilt;
  53
  54     /* initial or final actions to perform (pad/hold a frame/black/nothing) */
  55     enum PaddingOption start;
  56     enum PaddingOption end;
  57
  58     /* columns to hold or pad at the beginning or at the end (respectively) */
  59     int hold;
  60     int pad;
  61
  62     /* buffers for black columns */
  63     uint8_t *black_buffers[4];
  64     int black_linesizes[4];
  65
  66     /* list containing all input frames */
  67     size_t input_size;
  68     AVFrame *input;
  69     AVFrame *prev;
  70
  71     const AVPixFmtDescriptor *desc;
  72 } TiltandshiftContext;
  73
  74 static int list_add_frame(TiltandshiftContext *s, AVFrame *frame)
  75 {
  76     if (s->input == NULL) {
  77         s->input = frame;
  78     } else {
  79         AVFrame *head = s->input;
  80         while (head->opaque)
  81             head = head->opaque;
  82         head->opaque = frame;
  83     }
  84     s->input_size++;
  85     return 0;
  86 }
  87
  88 static void list_remove_head(TiltandshiftContext *s)
  89 {
  90     AVFrame *head = s->input;
  91     if (head) {
  92         s->input = head->opaque;
  93         av_frame_free(&head);
  94     }
  95     s->input_size--;
  96 }
  97
  98 static const enum AVPixelFormat pix_fmts[] = {
  99     AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P,
 100     AV_PIX_FMT_YUV410P,
 101     AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVJ444P,
 102     AV_PIX_FMT_YUVJ440P,
 103     AV_PIX_FMT_NONE
 104 };
 105
 106 static av_cold void uninit(AVFilterContext *ctx)
 107 {
 108     TiltandshiftContext *s = ctx->priv;
 109     while (s->input)
 110         list_remove_head(s);
 111     av_freep(&s->black_buffers);
 112 }
 113
 114 static int config_props(AVFilterLink *outlink)
 115 {
 116     AVFilterContext *ctx = outlink->src;
 117     TiltandshiftContext *s = ctx->priv;
 118
 119     outlink->w = ctx->inputs[0]->w;
 120     outlink->h = ctx->inputs[0]->h;
 121     outlink->format = ctx->inputs[0]->format;
 122
 123     // when we have to pad black or a frame at the start, skip navigating
 124     // the list and use either the frame or black for the requested value
 125     if (s->start != TILT_NONE && !s->hold)
 126         s->hold = outlink->w;
 127
 128     // Init black buffers if we pad with black at the start or at the end.
 129     // For the end, we always have to init on NONE and BLACK because we never
 130     // know if there are going to be enough input frames to fill an output one.
 131     if (s->start == TILT_BLACK || s->end != TILT_FRAME) {
 132         int i, j, ret;
 133         uint8_t black_data[] = { 0x10, 0x80, 0x80, 0x10 };
 134         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(outlink->format);
 135         if (!desc)
 136             return AVERROR_BUG;
 137
 138         if (outlink->format == AV_PIX_FMT_YUVJ420P ||
 139             outlink->format == AV_PIX_FMT_YUVJ422P ||
 140             outlink->format == AV_PIX_FMT_YUVJ444P ||
 141             outlink->format == AV_PIX_FMT_YUVJ440P ||
 142             outlink->color_range == AVCOL_RANGE_JPEG)
 143             black_data[0] = black_data[3] = 0;
 144
 145         ret = av_image_alloc(s->black_buffers, s->black_linesizes, 1,
 146                              outlink->h, outlink->format, 1);
 147         if (ret < 0)
 148             return ret;
 149
 150         for (i = 0; i < FFMIN(desc->nb_components, 4); i++)
 151             for (j = 0; j < (!i ? outlink->h
 152                                 : -((-outlink->h) >> desc->log2_chroma_h)); j++)
 153                 memset(s->black_buffers[i] + j * s->black_linesizes[i],
 154                        black_data[i], 1);
 155
 156         av_log(ctx, AV_LOG_VERBOSE, "Padding buffers initialized.\n");
 157     }
 158
 159     s->desc = av_pix_fmt_desc_get(outlink->format);
 160     if (!s->desc)
 161         return AVERROR_BUG;
 162
 163     return 0;
 164 }
 165
 166
 167 static void copy_column(AVFilterLink *outlink,
 168                         uint8_t *dst_data[4], int dst_linesizes[4],
 169                         const uint8_t *src_data[4], const int src_linesizes[4],
 170                         int ncol, int tilt)
 171 {
 172     AVFilterContext *ctx = outlink->src;
 173     TiltandshiftContext *s = ctx->priv;
 174     uint8_t *dst[4];
 175     const uint8_t *src[4];
 176
 177     dst[0] = dst_data[0] + ncol;
 178     dst[1] = dst_data[1] + (ncol >> s->desc->log2_chroma_w);
 179     dst[2] = dst_data[2] + (ncol >> s->desc->log2_chroma_w);
 180
 181     if (!tilt)
 182         ncol = 0;
 183     src[0] = src_data[0] + ncol;
 184     src[1] = src_data[1] + (ncol >> s->desc->log2_chroma_w);
 185     src[2] = src_data[2] + (ncol >> s->desc->log2_chroma_w);
 186
 187     av_image_copy(dst, dst_linesizes, src, src_linesizes, outlink->format, 1, outlink->h);
 188 }
 189
 190 static int output_frame(AVFilterLink *outlink)
 191 {
 192     TiltandshiftContext *s = outlink->src->priv;
 193     AVFrame *head;
 194     int ret;
 195
 196     int ncol = 0;
 197     AVFrame *dst = ff_get_video_buffer(outlink, outlink->w, outlink->h);
 198     if (!dst)
 199         return AVERROR(ENOMEM);
 200
 201     // in case we have to do any initial black padding
 202     if (s->start == TILT_BLACK) {
 203         for ( ; ncol < s->hold; ncol++)
 204             copy_column(outlink, dst->data, dst->linesize,
 205                         (const uint8_t **)s->black_buffers, s->black_linesizes,
 206                         ncol, 0);
 207     }
 208
 209     head = s->input;
 210     // copy a column from each input frame
 211     for ( ; ncol < s->input_size; ncol++) {
 212         AVFrame *src = head;
 213
 214         copy_column(outlink, dst->data, dst->linesize,
 215                     (const uint8_t **)src->data, src->linesize,
 216                     ncol, s->tilt);
 217
 218         // keep track of the last known frame in case we need it below
 219         s->prev = head;
 220         // advance to the next frame unless we have to hold it
 221         if (s->hold <= ncol)
 222             head = head->opaque;
 223     }
 224
 225     // pad any remaining space with black or last frame
 226     if (s->end == TILT_FRAME) {
 227         for ( ; ncol < outlink->w; ncol++)
 228             copy_column(outlink, dst->data, dst->linesize,
 229                         (const uint8_t **)s->prev->data,
 230                         s->prev->linesize, ncol, 1);
 231     } else { // TILT_BLACK and TILT_NONE
 232         for ( ; ncol < outlink->w; ncol++)
 233             copy_column(outlink, dst->data, dst->linesize,
 234                         (const uint8_t **)s->black_buffers, s->black_linesizes,
 235                         ncol, 0);
 236     }
 237
 238     // set correct timestamps and props as long as there is proper input
 239     ret = av_frame_copy_props(dst, s->input);
 240     if (ret < 0) {
 241         av_frame_free(&dst);
 242         return ret;
 243     }
 244
 245     // discard frame at the top of the list since it has been fully processed
 246     list_remove_head(s);
 247     // and it is safe to reduce the hold value (even if unused)
 248     s->hold--;
 249
 250     // output
 251     return ff_filter_frame(outlink, dst);
 252 }
 253
 254 // This function just polls for new frames and queues them on a list
 255 static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
 256 {
 257     AVFilterLink *outlink = inlink->dst->outputs[0];
 258     AVFilterContext *ctx = outlink->src;
 259     TiltandshiftContext *s = inlink->dst->priv;
 260
 261     int ret = list_add_frame(s, frame);
 262     if (ret < 0) {
 263         return ret;
 264     }
 265
 266     // load up enough frames to fill a frame and keep the queue filled on subsequent
 267     // calls, until we receive EOF, and then we either pad or end
 268     if (!s->eof_recv && s->input_size < outlink->w - s->pad) {
 269         av_log(ctx, AV_LOG_DEBUG, "Not enough frames in the list (%zu/%d), waiting for more.\n", s->input_size, outlink->w - s->pad);
 270         return 0;
 271     }
 272
 273     return output_frame(outlink);
 274 }
 275
 276 static int request_frame(AVFilterLink *outlink)
 277 {
 278     AVFilterContext *ctx = outlink->src;
 279     TiltandshiftContext *s = ctx->priv;
 280     int ret;
 281
 282     // signal job finished when list is empty or when padding is either
 283     // limited or disabled and eof was received
 284     if ((s->input_size <= 0 || s->input_size == outlink->w - s->pad || s->end == TILT_NONE) && s->eof_recv) {
 285         return AVERROR_EOF;
 286     }
 287
 288     ret = ff_request_frame(ctx->inputs[0]);
 289     if (ret == AVERROR_EOF) {
 290         s->eof_recv = 1;
 291     } else if (ret < 0) {
 292         return ret;
 293     }
 294
 295     if (s->eof_recv) {
 296         while (s->input_size) {
 297             av_log(ctx, AV_LOG_DEBUG, "Emptying buffers (%zu/%d).\n", s->input_size, outlink->w - s->pad);
 298             ret = output_frame(outlink);
 299             if (ret < 0) {
 300                 return ret;
 301             }
 302         }
 303     }
 304
 305     return 0;
 306 }
 307
 308 #define OFFSET(x) offsetof(TiltandshiftContext, x)
 309 #define V AV_OPT_FLAG_VIDEO_PARAM
 310 static const AVOption tiltandshift_options[] = {
 311     { "tilt", "Tilt the video horizontally while shifting", OFFSET(tilt), AV_OPT_TYPE_INT,
 312         { .i64 = 1 }, 0, 1, .flags = V, .unit = "tilt" },
 313
 314     { "start", "Action at the start of input", OFFSET(start), AV_OPT_TYPE_INT,
 315         { .i64 = TILT_NONE }, 0, TILT_OPT_MAX, .flags = V, .unit = "start" },
 316     { "none", "Start immediately (default)", 0, AV_OPT_TYPE_CONST,
 317         { .i64 = TILT_NONE }, INT_MIN, INT_MAX, .flags = V, .unit = "start" },
 318     { "frame", "Use the first frames", 0, AV_OPT_TYPE_CONST,
 319         { .i64 = TILT_FRAME }, INT_MIN, INT_MAX, .flags = V, .unit = "start" },
 320     { "black", "Fill with black", 0, AV_OPT_TYPE_CONST,
 321         { .i64 = TILT_BLACK }, INT_MIN, INT_MAX, .flags = V, .unit = "start" },
 322
 323     { "end", "Action at the end of input", OFFSET(end), AV_OPT_TYPE_INT,
 324         { .i64 = TILT_NONE }, 0, TILT_OPT_MAX, .flags = V, .unit = "end" },
 325     { "none", "Do not pad at the end (default)", 0, AV_OPT_TYPE_CONST,
 326         { .i64 = TILT_NONE }, INT_MIN, INT_MAX, .flags = V, .unit = "end" },
 327     { "frame", "Use the last frame", 0, AV_OPT_TYPE_CONST,
 328         { .i64 = TILT_FRAME }, INT_MIN, INT_MAX, .flags = V, .unit = "end" },
 329     { "black", "Fill with black", 0, AV_OPT_TYPE_CONST,
 330         { .i64 = TILT_BLACK }, INT_MIN, INT_MAX, .flags = V, .unit = "end" },
 331
 332     { "hold", "Number of columns to hold at the start of the video", OFFSET(hold), AV_OPT_TYPE_INT,
 333         { .i64 = 0 }, 0, INT_MAX, .flags = V, .unit = "hold" },
 334     { "pad", "Number of columns to pad at the end of the video", OFFSET(pad), AV_OPT_TYPE_INT,
 335         { .i64 = 0 }, 0, INT_MAX, .flags = V, .unit = "pad" },
 336
 337     { NULL },
 338 };
 339
 340 AVFILTER_DEFINE_CLASS(tiltandshift);
 341
 342 static const AVFilterPad tiltandshift_inputs[] = {
 343     {
 344         .name         = "in",
 345         .type         = AVMEDIA_TYPE_VIDEO,
 346         .filter_frame = filter_frame,
 347     },
 348 };
 349
 350 static const AVFilterPad tiltandshift_outputs[] = {
 351     {
 352         .name          = "out",
 353         .type          = AVMEDIA_TYPE_VIDEO,
 354         .config_props  = config_props,
 355         .request_frame = request_frame,
 356     },
 357 };
 358
 359 const FFFilter ff_vf_tiltandshift = {
 360     .p.name        = "tiltandshift",
 361     .p.description = NULL_IF_CONFIG_SMALL("Generate a tilt-and-shift'd video."),
 362     .p.priv_class  = &tiltandshift_class,
 363     .priv_size     = sizeof(TiltandshiftContext),
 364     .uninit        = uninit,
 365     FILTER_INPUTS(tiltandshift_inputs),
 366     FILTER_OUTPUTS(tiltandshift_outputs),
 367     FILTER_PIXFMTS_ARRAY(pix_fmts),
 368 };