libavfilter/vf_photosensitivity.c

   1 /*
   2  * Copyright (c) 2019 Vladimir Panteleev
   3  *
   4  * This file is part of FFmpeg.
   5  *
   6  * FFmpeg is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * FFmpeg is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with FFmpeg; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 #include <float.h>
  22
  23 #include "libavutil/opt.h"
  24 #include "avfilter.h"
  25
  26 #include "filters.h"
  27 #include "video.h"
  28
  29 #define MAX_FRAMES 240
  30 #define GRID_SIZE 8
  31 #define NUM_CHANNELS 3
  32
  33 typedef struct PhotosensitivityFrame {
  34     uint8_t grid[GRID_SIZE][GRID_SIZE][4];
  35 } PhotosensitivityFrame;
  36
  37 typedef struct PhotosensitivityContext {
  38     const AVClass *class;
  39
  40     int nb_frames;
  41     int skip;
  42     float threshold_multiplier;
  43     int bypass;
  44
  45     int badness_threshold;
  46
  47     /* Circular buffer */
  48     int history[MAX_FRAMES];
  49     int history_pos;
  50
  51     PhotosensitivityFrame last_frame_e;
  52     AVFrame *last_frame_av;
  53 } PhotosensitivityContext;
  54
  55 #define OFFSET(x) offsetof(PhotosensitivityContext, x)
  56 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
  57
  58 static const AVOption photosensitivity_options[] = {
  59     { "frames",    "set how many frames to use",                          OFFSET(nb_frames),            AV_OPT_TYPE_INT,   {.i64=30}, 2, MAX_FRAMES, FLAGS },
  60     { "f",         "set how many frames to use",                          OFFSET(nb_frames),            AV_OPT_TYPE_INT,   {.i64=30}, 2, MAX_FRAMES, FLAGS },
  61     { "threshold", "set detection threshold factor (lower is stricter)",  OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl=1},  0.1, FLT_MAX,  FLAGS },
  62     { "t",         "set detection threshold factor (lower is stricter)",  OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl=1},  0.1, FLT_MAX,  FLAGS },
  63     { "skip",      "set pixels to skip when sampling frames",             OFFSET(skip),                 AV_OPT_TYPE_INT,   {.i64=1},  1, 1024,       FLAGS },
  64     { "bypass",    "leave frames unchanged",                              OFFSET(bypass),               AV_OPT_TYPE_BOOL,  {.i64=0},  0, 1,          FLAGS },
  65     { NULL }
  66 };
  67
  68 AVFILTER_DEFINE_CLASS(photosensitivity);
  69
  70 typedef struct ThreadData_convert_frame
  71 {
  72     AVFrame *in;
  73     PhotosensitivityFrame *out;
  74     int skip;
  75 } ThreadData_convert_frame;
  76
  77 #define NUM_CELLS (GRID_SIZE * GRID_SIZE)
  78
  79 static int convert_frame_partial(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
  80 {
  81     int cell, gx, gy, x0, x1, y0, y1, x, y, c, area;
  82     int sum[NUM_CHANNELS];
  83     const uint8_t *p;
  84
  85     ThreadData_convert_frame *td = arg;
  86
  87     const int slice_start = (NUM_CELLS * jobnr) / nb_jobs;
  88     const int slice_end = (NUM_CELLS * (jobnr+1)) / nb_jobs;
  89
  90     int width = td->in->width, height = td->in->height, linesize = td->in->linesize[0], skip = td->skip;
  91     const uint8_t *data = td->in->data[0];
  92
  93     for (cell = slice_start; cell < slice_end; cell++) {
  94         gx = cell % GRID_SIZE;
  95         gy = cell / GRID_SIZE;
  96
  97         x0 = width  *  gx    / GRID_SIZE;
  98         x1 = width  * (gx+1) / GRID_SIZE;
  99         y0 = height *  gy    / GRID_SIZE;
 100         y1 = height * (gy+1) / GRID_SIZE;
 101
 102         for (c = 0; c < NUM_CHANNELS; c++) {
 103             sum[c] = 0;
 104         }
 105         for (y = y0; y < y1; y += skip) {
 106             p = data + y * linesize + x0 * NUM_CHANNELS;
 107             for (x = x0; x < x1; x += skip) {
 108                 //av_log(NULL, AV_LOG_VERBOSE, "%d %d %d : (%d,%d) (%d,%d) -> %d,%d | *%d\n", c, gx, gy, x0, y0, x1, y1, x, y, (int)row);
 109                 sum[0] += p[0];
 110                 sum[1] += p[1];
 111                 sum[2] += p[2];
 112                 p += NUM_CHANNELS * skip;
 113                 // TODO: variable size
 114             }
 115         }
 116
 117         area = ((x1 - x0 + skip - 1) / skip) * ((y1 - y0 + skip - 1) / skip);
 118         for (c = 0; c < NUM_CHANNELS; c++) {
 119             if (area)
 120                 sum[c] /= area;
 121             td->out->grid[gy][gx][c] = sum[c];
 122         }
 123     }
 124     return 0;
 125 }
 126
 127 static void convert_frame(AVFilterContext *ctx, AVFrame *in, PhotosensitivityFrame *out, int skip)
 128 {
 129     ThreadData_convert_frame td;
 130     td.in = in;
 131     td.out = out;
 132     td.skip = skip;
 133     ff_filter_execute(ctx, convert_frame_partial, &td, NULL,
 134                       FFMIN(NUM_CELLS, ff_filter_get_nb_threads(ctx)));
 135 }
 136
 137 typedef struct ThreadData_blend_frame
 138 {
 139     AVFrame *target;
 140     AVFrame *source;
 141     uint16_t s_mul;
 142 } ThreadData_blend_frame;
 143
 144 static int blend_frame_partial(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
 145 {
 146     int x, y;
 147     uint8_t *t, *s;
 148
 149     ThreadData_blend_frame *td = arg;
 150     const uint16_t s_mul = td->s_mul;
 151     const uint16_t t_mul = 0x100 - s_mul;
 152     const int slice_start = (td->target->height * jobnr) / nb_jobs;
 153     const int slice_end = (td->target->height * (jobnr+1)) / nb_jobs;
 154     const int linesize = td->target->linesize[0];
 155
 156     for (y = slice_start; y < slice_end; y++) {
 157         t = td->target->data[0] + y * td->target->linesize[0];
 158         s = td->source->data[0] + y * td->source->linesize[0];
 159         for (x = 0; x < linesize; x++) {
 160             *t = (*t * t_mul + *s * s_mul) >> 8;
 161             t++; s++;
 162         }
 163     }
 164     return 0;
 165 }
 166
 167 static void blend_frame(AVFilterContext *ctx, AVFrame *target, AVFrame *source, float factor)
 168 {
 169     ThreadData_blend_frame td;
 170     td.target = target;
 171     td.source = source;
 172     td.s_mul = (uint16_t)(factor * 0x100);
 173     ff_filter_execute(ctx, blend_frame_partial, &td, NULL,
 174                       FFMIN(ctx->outputs[0]->h, ff_filter_get_nb_threads(ctx)));
 175 }
 176
 177 static int get_badness(PhotosensitivityFrame *a, PhotosensitivityFrame *b)
 178 {
 179     int badness, x, y, c;
 180     badness = 0;
 181     for (c = 0; c < NUM_CHANNELS; c++) {
 182         for (y = 0; y < GRID_SIZE; y++) {
 183             for (x = 0; x < GRID_SIZE; x++) {
 184                 badness += abs((int)a->grid[y][x][c] - (int)b->grid[y][x][c]);
 185                 //av_log(NULL, AV_LOG_VERBOSE, "%d - %d -> %d \n", a->grid[y][x], b->grid[y][x], badness);
 186                 //av_log(NULL, AV_LOG_VERBOSE, "%d -> %d \n", abs((int)a->grid[y][x] - (int)b->grid[y][x]), badness);
 187             }
 188         }
 189     }
 190     return badness;
 191 }
 192
 193 static int config_input(AVFilterLink *inlink)
 194 {
 195     /* const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); */
 196     AVFilterContext *ctx = inlink->dst;
 197     PhotosensitivityContext *s = ctx->priv;
 198
 199     s->badness_threshold = (int)(GRID_SIZE * GRID_SIZE * 4 * 256 * s->nb_frames * s->threshold_multiplier / 128);
 200
 201     return 0;
 202 }
 203
 204 static int filter_frame(AVFilterLink *inlink, AVFrame *in)
 205 {
 206     int this_badness, current_badness, fixed_badness, new_badness, i, res;
 207     PhotosensitivityFrame ef;
 208     AVFrame *src, *out;
 209     int free_in = 0;
 210     float factor;
 211     AVDictionary **metadata;
 212
 213     AVFilterContext *ctx = inlink->dst;
 214     AVFilterLink *outlink = ctx->outputs[0];
 215     PhotosensitivityContext *s = ctx->priv;
 216
 217     /* weighted moving average */
 218     current_badness = 0;
 219     for (i = 1; i < s->nb_frames; i++)
 220         current_badness += i * s->history[(s->history_pos + i) % s->nb_frames];
 221     current_badness /= s->nb_frames;
 222
 223     convert_frame(ctx, in, &ef, s->skip);
 224     this_badness = get_badness(&ef, &s->last_frame_e);
 225     new_badness = current_badness + this_badness;
 226     av_log(s, AV_LOG_VERBOSE, "badness: %6d -> %6d / %6d (%3d%% - %s)\n",
 227         current_badness, new_badness, s->badness_threshold,
 228         100 * new_badness / s->badness_threshold, new_badness < s->badness_threshold ? "OK" : "EXCEEDED");
 229
 230     fixed_badness = new_badness;
 231     if (new_badness < s->badness_threshold || !s->last_frame_av || s->bypass) {
 232         factor = 1; /* for metadata */
 233         av_frame_free(&s->last_frame_av);
 234         s->last_frame_av = src = in;
 235         s->last_frame_e = ef;
 236         s->history[s->history_pos] = this_badness;
 237     } else {
 238         factor = (float)(s->badness_threshold - current_badness) / (new_badness - current_badness);
 239         if (factor <= 0) {
 240             /* just duplicate the frame */
 241             s->history[s->history_pos] = 0; /* frame was duplicated, thus, delta is zero */
 242         } else {
 243             res = ff_inlink_make_frame_writable(inlink, &s->last_frame_av);
 244             if (res) {
 245                 av_frame_free(&in);
 246                 return res;
 247             }
 248             blend_frame(ctx, s->last_frame_av, in, factor);
 249
 250             convert_frame(ctx, s->last_frame_av, &ef, s->skip);
 251             this_badness = get_badness(&ef, &s->last_frame_e);
 252             fixed_badness = current_badness + this_badness;
 253             av_log(s, AV_LOG_VERBOSE, "  fixed: %6d -> %6d / %6d (%3d%%) factor=%5.3f\n",
 254                 current_badness, fixed_badness, s->badness_threshold,
 255                 100 * new_badness / s->badness_threshold, factor);
 256             s->last_frame_e = ef;
 257             s->history[s->history_pos] = this_badness;
 258         }
 259         src = s->last_frame_av;
 260         free_in = 1;
 261     }
 262     s->history_pos = (s->history_pos + 1) % s->nb_frames;
 263
 264     out = ff_get_video_buffer(outlink, in->width, in->height);
 265     if (!out) {
 266         if (free_in == 1)
 267             av_frame_free(&in);
 268         return AVERROR(ENOMEM);
 269     }
 270     av_frame_copy_props(out, in);
 271     metadata = &out->metadata;
 272     if (metadata) {
 273         char value[128];
 274
 275         snprintf(value, sizeof(value), "%f", (float)new_badness / s->badness_threshold);
 276         av_dict_set(metadata, "lavfi.photosensitivity.badness", value, 0);
 277
 278         snprintf(value, sizeof(value), "%f", (float)fixed_badness / s->badness_threshold);
 279         av_dict_set(metadata, "lavfi.photosensitivity.fixed-badness", value, 0);
 280
 281         snprintf(value, sizeof(value), "%f", (float)this_badness / s->badness_threshold);
 282         av_dict_set(metadata, "lavfi.photosensitivity.frame-badness", value, 0);
 283
 284         snprintf(value, sizeof(value), "%f", factor);
 285         av_dict_set(metadata, "lavfi.photosensitivity.factor", value, 0);
 286     }
 287     av_frame_copy(out, src);
 288     if (free_in == 1)
 289         av_frame_free(&in);
 290     return ff_filter_frame(outlink, out);
 291 }
 292
 293 static av_cold void uninit(AVFilterContext *ctx)
 294 {
 295     PhotosensitivityContext *s = ctx->priv;
 296
 297     av_frame_free(&s->last_frame_av);
 298 }
 299
 300 static const AVFilterPad inputs[] = {
 301     {
 302         .name         = "default",
 303         .type         = AVMEDIA_TYPE_VIDEO,
 304         .filter_frame = filter_frame,
 305         .config_props = config_input,
 306     },
 307 };
 308
 309 const FFFilter ff_vf_photosensitivity = {
 310     .p.name        = "photosensitivity",
 311     .p.description = NULL_IF_CONFIG_SMALL("Filter out photosensitive epilepsy seizure-inducing flashes."),
 312     .p.priv_class  = &photosensitivity_class,
 313     .priv_size     = sizeof(PhotosensitivityContext),
 314     .uninit        = uninit,
 315     FILTER_INPUTS(inputs),
 316     FILTER_OUTPUTS(ff_video_default_filterpad),
 317     FILTER_PIXFMTS(AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24),
 318 };