libavfilter/vf_readeia608.c

   1 /*
   2  * Copyright (c) 2017 Paul B Mahol
   3  *
   4  * This file is part of FFmpeg.
   5  *
   6  * FFmpeg is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * FFmpeg is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with FFmpeg; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 /**
  22  * @file
  23  * Filter for reading closed captioning data (EIA-608).
  24  * See also https://en.wikipedia.org/wiki/EIA-608
  25  */
  26
  27 #include <string.h>
  28
  29 #include "libavutil/internal.h"
  30 #include "libavutil/mem.h"
  31 #include "libavutil/opt.h"
  32 #include "libavutil/pixdesc.h"
  33
  34 #include "avfilter.h"
  35 #include "filters.h"
  36 #include "video.h"
  37
  38 #define LAG 25
  39 #define CLOCK_BITSIZE_MIN 0.2f
  40 #define CLOCK_BITSIZE_MAX 1.5f
  41 #define SYNC_BITSIZE_MIN 12.f
  42 #define SYNC_BITSIZE_MAX 15.f
  43
  44 typedef struct LineItem {
  45     int   input;
  46     int   output;
  47
  48     float unfiltered;
  49     float filtered;
  50     float average;
  51     float deviation;
  52 } LineItem;
  53
  54 typedef struct CodeItem {
  55     uint8_t bit;
  56     int size;
  57 } CodeItem;
  58
  59 typedef struct ScanItem {
  60     int nb_line;
  61     int found;
  62     int white;
  63     int black;
  64     uint64_t *histogram;
  65     uint8_t byte[2];
  66
  67     CodeItem *code;
  68     LineItem *line;
  69 } ScanItem;
  70
  71 typedef struct ReadEIA608Context {
  72     const AVClass *class;
  73
  74     int start, end;
  75     float spw;
  76     int chp;
  77     int lp;
  78
  79     int depth;
  80     int max;
  81     int nb_allocated;
  82     ScanItem *scan;
  83
  84     void (*read_line[2])(AVFrame *in, int nb_line,
  85                          LineItem *line, int lp, int w);
  86 } ReadEIA608Context;
  87
  88 #define OFFSET(x) offsetof(ReadEIA608Context, x)
  89 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_RUNTIME_PARAM
  90
  91 static const AVOption readeia608_options[] = {
  92     { "scan_min", "set from which line to scan for codes",               OFFSET(start), AV_OPT_TYPE_INT,   {.i64=0},     0, INT_MAX, FLAGS },
  93     { "scan_max", "set to which line to scan for codes",                 OFFSET(end),   AV_OPT_TYPE_INT,   {.i64=29},    0, INT_MAX, FLAGS },
  94     { "spw",      "set ratio of width reserved for sync code detection", OFFSET(spw),   AV_OPT_TYPE_FLOAT, {.dbl=.27}, 0.1,     0.7, FLAGS },
  95     { "chp",      "check and apply parity bit",                          OFFSET(chp),   AV_OPT_TYPE_BOOL,  {.i64= 0},    0,       1, FLAGS },
  96     { "lp",       "lowpass line prior to processing",                    OFFSET(lp),    AV_OPT_TYPE_BOOL,  {.i64= 1},    0,       1, FLAGS },
  97     { NULL }
  98 };
  99
 100 AVFILTER_DEFINE_CLASS(readeia608);
 101
 102 static const enum AVPixelFormat pixel_fmts[] = {
 103     AV_PIX_FMT_GRAY8, AV_PIX_FMT_GRAY9,
 104     AV_PIX_FMT_GRAY10, AV_PIX_FMT_GRAY12, AV_PIX_FMT_GRAY14,
 105     AV_PIX_FMT_GRAY16,
 106     AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P,
 107     AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P,
 108     AV_PIX_FMT_YUV440P, AV_PIX_FMT_YUV444P,
 109     AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P,
 110     AV_PIX_FMT_YUVJ440P, AV_PIX_FMT_YUVJ444P,
 111     AV_PIX_FMT_YUVJ411P,
 112     AV_PIX_FMT_YUV420P9, AV_PIX_FMT_YUV422P9, AV_PIX_FMT_YUV444P9,
 113     AV_PIX_FMT_YUV420P10, AV_PIX_FMT_YUV422P10, AV_PIX_FMT_YUV444P10,
 114     AV_PIX_FMT_YUV440P10,
 115     AV_PIX_FMT_YUV444P12, AV_PIX_FMT_YUV422P12, AV_PIX_FMT_YUV420P12,
 116     AV_PIX_FMT_YUV440P12,
 117     AV_PIX_FMT_YUV444P14, AV_PIX_FMT_YUV422P14, AV_PIX_FMT_YUV420P14,
 118     AV_PIX_FMT_YUV420P16, AV_PIX_FMT_YUV422P16, AV_PIX_FMT_YUV444P16,
 119     AV_PIX_FMT_YUVA420P,  AV_PIX_FMT_YUVA422P,   AV_PIX_FMT_YUVA444P,
 120     AV_PIX_FMT_YUVA444P9, AV_PIX_FMT_YUVA444P10, AV_PIX_FMT_YUVA444P12, AV_PIX_FMT_YUVA444P16,
 121     AV_PIX_FMT_YUVA422P9, AV_PIX_FMT_YUVA422P10, AV_PIX_FMT_YUVA422P12, AV_PIX_FMT_YUVA422P16,
 122     AV_PIX_FMT_YUVA420P9, AV_PIX_FMT_YUVA420P10, AV_PIX_FMT_YUVA420P16,
 123     AV_PIX_FMT_NONE
 124 };
 125
 126 static int config_filter(AVFilterContext *ctx, int start, int end)
 127 {
 128     ReadEIA608Context *s = ctx->priv;
 129     AVFilterLink *inlink = ctx->inputs[0];
 130     int size = inlink->w + LAG;
 131
 132     if (end >= inlink->h) {
 133         av_log(ctx, AV_LOG_WARNING, "Last line to scan too large, clipping.\n");
 134         end = inlink->h - 1;
 135     }
 136
 137     if (start > end) {
 138         av_log(ctx, AV_LOG_ERROR, "Invalid range.\n");
 139         return AVERROR(EINVAL);
 140     }
 141
 142     if (s->nb_allocated < end - start + 1) {
 143         const int diff = end - start + 1 - s->nb_allocated;
 144
 145         s->scan = av_realloc_f(s->scan, end - start + 1, sizeof(*s->scan));
 146         if (!s->scan)
 147             return AVERROR(ENOMEM);
 148         memset(&s->scan[s->nb_allocated], 0, diff * sizeof(*s->scan));
 149         s->nb_allocated = end - start + 1;
 150     }
 151
 152     for (int i = 0; i < s->nb_allocated; i++) {
 153         ScanItem *scan = &s->scan[i];
 154
 155         if (!scan->histogram)
 156             scan->histogram = av_calloc(s->max + 1, sizeof(*scan->histogram));
 157         if (!scan->line)
 158             scan->line = av_calloc(size, sizeof(*scan->line));
 159         if (!scan->code)
 160             scan->code = av_calloc(size, sizeof(*scan->code));
 161         if (!scan->line || !scan->code || !scan->histogram)
 162             return AVERROR(ENOMEM);
 163     }
 164
 165     s->start = start;
 166     s->end = end;
 167
 168     return 0;
 169 }
 170
 171 static void build_histogram(ReadEIA608Context *s, ScanItem *scan, const LineItem *line, int len)
 172 {
 173     memset(scan->histogram, 0, (s->max + 1) * sizeof(*scan->histogram));
 174
 175     for (int i = LAG; i < len + LAG; i++)
 176         scan->histogram[line[i].input]++;
 177 }
 178
 179 static void find_black_and_white(ReadEIA608Context *s, ScanItem *scan)
 180 {
 181     const int max = s->max;
 182     int start = 0, end = 0, middle;
 183     int black = 0, white = 0;
 184     int cnt;
 185
 186     for (int i = 0; i <= max; i++) {
 187         if (scan->histogram[i]) {
 188             start = i;
 189             break;
 190         }
 191     }
 192
 193     for (int i = max; i >= 0; i--) {
 194         if (scan->histogram[i]) {
 195             end = i;
 196             break;
 197         }
 198     }
 199
 200     middle = start + (end - start) / 2;
 201
 202     cnt = 0;
 203     for (int i = start; i <= middle; i++) {
 204         if (scan->histogram[i] > cnt) {
 205             cnt = scan->histogram[i];
 206             black = i;
 207         }
 208     }
 209
 210     cnt = 0;
 211     for (int i = end; i >= middle; i--) {
 212         if (scan->histogram[i] > cnt) {
 213             cnt = scan->histogram[i];
 214             white = i;
 215         }
 216     }
 217
 218     scan->black = black;
 219     scan->white = white;
 220 }
 221
 222 static float meanf(const LineItem *line, int len)
 223 {
 224     float sum = 0.0, mean = 0.0;
 225
 226     for (int i = 0; i < len; i++)
 227         sum += line[i].filtered;
 228
 229     mean = sum / len;
 230
 231     return mean;
 232 }
 233
 234 static float stddevf(const LineItem *line, int len)
 235 {
 236     float m = meanf(line, len);
 237     float standard_deviation = 0.f;
 238
 239     for (int i = 0; i < len; i++)
 240         standard_deviation += (line[i].filtered - m) * (line[i].filtered - m);
 241
 242     return sqrtf(standard_deviation / (len - 1));
 243 }
 244
 245 static void thresholding(ReadEIA608Context *s, ScanItem *scan, LineItem *line,
 246                          int lag, float threshold, float influence, int len)
 247 {
 248     for (int i = lag; i < len + lag; i++) {
 249         line[i].unfiltered = line[i].input / 255.f;
 250         line[i].filtered = line[i].unfiltered;
 251     }
 252
 253     for (int i = 0; i < lag; i++) {
 254         line[i].unfiltered = meanf(line, len * s->spw);
 255         line[i].filtered = line[i].unfiltered;
 256     }
 257
 258     line[lag - 1].average   = meanf(line, lag);
 259     line[lag - 1].deviation = stddevf(line, lag);
 260
 261     for (int i = lag; i < len + lag; i++) {
 262         if (fabsf(line[i].unfiltered - line[i-1].average) > threshold * line[i-1].deviation) {
 263             if (line[i].unfiltered > line[i-1].average) {
 264                 line[i].output = 255;
 265             } else {
 266                 line[i].output = 0;
 267             }
 268
 269             line[i].filtered = influence * line[i].unfiltered + (1.f - influence) * line[i-1].filtered;
 270         } else {
 271             int distance_from_black, distance_from_white;
 272
 273             distance_from_black = FFABS(line[i].input - scan->black);
 274             distance_from_white = FFABS(line[i].input - scan->white);
 275
 276             line[i].output = distance_from_black <= distance_from_white ? 0 : 255;
 277         }
 278
 279         line[i].average   = meanf(line + i - lag, lag);
 280         line[i].deviation = stddevf(line + i - lag, lag);
 281     }
 282 }
 283
 284 static int periods(const LineItem *line, CodeItem *code, int len)
 285 {
 286     int hold = line[LAG].output, cnt = 0;
 287     int last = LAG;
 288
 289     memset(code, 0, len * sizeof(*code));
 290
 291     for (int i = LAG + 1; i < len + LAG; i++) {
 292         if (line[i].output != hold) {
 293             code[cnt].size = i - last;
 294             code[cnt].bit = hold;
 295             hold = line[i].output;
 296             last = i;
 297             cnt++;
 298         }
 299     }
 300
 301     code[cnt].size = LAG + len - last;
 302     code[cnt].bit = hold;
 303
 304     return cnt + 1;
 305 }
 306
 307 static void dump_code(AVFilterContext *ctx, ScanItem *scan, int len, int item)
 308 {
 309     av_log(ctx, AV_LOG_DEBUG, "%d:", item);
 310     for (int i = 0; i < len; i++) {
 311         av_log(ctx, AV_LOG_DEBUG, " %03d", scan->code[i].size);
 312     }
 313     av_log(ctx, AV_LOG_DEBUG, "\n");
 314 }
 315
 316 #define READ_LINE(type, name)                                                 \
 317 static void read_##name(AVFrame *in, int nb_line, LineItem *line, int lp, int w) \
 318 {                                                                             \
 319     const type *src = (const type *)(&in->data[0][nb_line * in->linesize[0]]);\
 320                                                                               \
 321     if (lp) {                                                                 \
 322         for (int i = 0; i < w; i++) {                                         \
 323             int a = FFMAX(i - 3, 0);                                          \
 324             int b = FFMAX(i - 2, 0);                                          \
 325             int c = FFMAX(i - 1, 0);                                          \
 326             int d = FFMIN(i + 3, w-1);                                        \
 327             int e = FFMIN(i + 2, w-1);                                        \
 328             int f = FFMIN(i + 1, w-1);                                        \
 329                                                                               \
 330             line[LAG + i].input = (src[a] + src[b] + src[c] + src[i] +        \
 331                                    src[d] + src[e] + src[f] + 6) / 7;         \
 332         }                                                                     \
 333     } else {                                                                  \
 334         for (int i = 0; i < w; i++) {                                         \
 335             line[LAG + i].input = src[i];                                     \
 336         }                                                                     \
 337     }                                                                         \
 338 }
 339
 340 READ_LINE(uint8_t, byte)
 341 READ_LINE(uint16_t, word)
 342
 343 static int config_input(AVFilterLink *inlink)
 344 {
 345     AVFilterContext *ctx = inlink->dst;
 346     ReadEIA608Context *s = ctx->priv;
 347     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
 348
 349     if (!desc)
 350         return AVERROR_BUG;
 351     s->depth = desc->comp[0].depth;
 352     s->max = (1 << desc->comp[0].depth) - 1;
 353     s->read_line[0] = read_byte;
 354     s->read_line[1] = read_word;
 355
 356     return config_filter(ctx, s->start, s->end);
 357 }
 358
 359 static void extract_line(AVFilterContext *ctx, AVFrame *in, ScanItem *scan, int w, int nb_line)
 360 {
 361     ReadEIA608Context *s = ctx->priv;
 362     LineItem *line = scan->line;
 363     int i, j, ch, len;
 364     uint8_t codes[19] = { 0 };
 365     float bit_size = 0.f;
 366     int parity;
 367
 368     memset(line, 0, (w + LAG) * sizeof(*line));
 369     scan->byte[0] = scan->byte[1] = 0;
 370     scan->found = 0;
 371
 372     s->read_line[s->depth > 8](in, nb_line, line, s->lp, w);
 373
 374     build_histogram(s, scan, line, w);
 375     find_black_and_white(s, scan);
 376     if (scan->white - scan->black < 5)
 377         return;
 378
 379     thresholding(s, scan, line, LAG, 1, 0, w);
 380     len = periods(line, scan->code, w);
 381     dump_code(ctx, scan, len, nb_line);
 382     if (len < 15 ||
 383         scan->code[14].bit != 0 ||
 384         w / (float)scan->code[14].size < SYNC_BITSIZE_MIN ||
 385         w / (float)scan->code[14].size > SYNC_BITSIZE_MAX) {
 386         return;
 387     }
 388
 389     for (i = 14; i < len; i++) {
 390         bit_size += scan->code[i].size;
 391     }
 392
 393     bit_size /= 19.f;
 394     for (i = 1; i < 14; i++) {
 395         if (scan->code[i].size / bit_size > CLOCK_BITSIZE_MAX ||
 396             scan->code[i].size / bit_size < CLOCK_BITSIZE_MIN) {
 397             return;
 398         }
 399     }
 400
 401     if (scan->code[15].size / bit_size < 0.45f) {
 402         return;
 403     }
 404
 405     for (j = 0, i = 14; i < len; i++) {
 406         int run, bit;
 407
 408         run = lrintf(scan->code[i].size / bit_size);
 409         bit = scan->code[i].bit;
 410
 411         for (int k = 0; j < 19 && k < run; k++) {
 412             codes[j++] = bit;
 413         }
 414
 415         if (j >= 19)
 416             break;
 417     }
 418
 419     for (ch = 0; ch < 2; ch++) {
 420         for (parity = 0, i = 0; i < 8; i++) {
 421             int b = codes[3 + ch * 8 + i];
 422
 423             if (b == 255) {
 424                 parity++;
 425                 b = 1;
 426             } else {
 427                 b = 0;
 428             }
 429             scan->byte[ch] |= b << i;
 430         }
 431
 432         if (s->chp) {
 433             if (!(parity & 1)) {
 434                 scan->byte[ch] = 0x7F;
 435             }
 436         }
 437     }
 438
 439     scan->nb_line = nb_line;
 440     scan->found = 1;
 441 }
 442
 443 static int extract_lines(AVFilterContext *ctx, void *arg,
 444                          int job, int nb_jobs)
 445 {
 446     ReadEIA608Context *s = ctx->priv;
 447     AVFilterLink *inlink = ctx->inputs[0];
 448     const int h = s->end - s->start + 1;
 449     const int start = (h * job) / nb_jobs;
 450     const int end   = (h * (job+1)) / nb_jobs;
 451     AVFrame *in = arg;
 452
 453     for (int i = start; i < end; i++) {
 454         ScanItem *scan = &s->scan[i];
 455
 456         extract_line(ctx, in, scan, inlink->w, s->start + i);
 457     }
 458
 459     return 0;
 460 }
 461
 462 static int filter_frame(AVFilterLink *inlink, AVFrame *in)
 463 {
 464     AVFilterContext *ctx  = inlink->dst;
 465     AVFilterLink *outlink = ctx->outputs[0];
 466     ReadEIA608Context *s = ctx->priv;
 467     int nb_found;
 468
 469     ff_filter_execute(ctx, extract_lines, in, NULL,
 470                       FFMIN(FFMAX(s->end - s->start + 1, 1), ff_filter_get_nb_threads(ctx)));
 471
 472     nb_found = 0;
 473     for (int i = 0; i < s->end - s->start + 1; i++) {
 474         ScanItem *scan = &s->scan[i];
 475         uint8_t key[128], value[128];
 476
 477         if (!scan->found)
 478             continue;
 479
 480         //snprintf(key, sizeof(key), "lavfi.readeia608.%d.bits", nb_found);
 481         //snprintf(value, sizeof(value), "0b%d%d%d%d%d%d%d%d 0b%d%d%d%d%d%d%d%d", codes[3]==255,codes[4]==255,codes[5]==255,codes[6]==255,codes[7]==255,codes[8]==255,codes[9]==255,codes[10]==255,codes[11]==255,codes[12]==255,codes[13]==255,codes[14]==255,codes[15]==255,codes[16]==255,codes[17]==255,codes[18]==255);
 482         //av_dict_set(&in->metadata, key, value, 0);
 483
 484         snprintf(key, sizeof(key), "lavfi.readeia608.%d.cc", nb_found);
 485         snprintf(value, sizeof(value), "0x%02X%02X", scan->byte[0], scan->byte[1]);
 486         av_dict_set(&in->metadata, key, value, 0);
 487
 488         snprintf(key, sizeof(key), "lavfi.readeia608.%d.line", nb_found);
 489         av_dict_set_int(&in->metadata, key, scan->nb_line, 0);
 490
 491         nb_found++;
 492     }
 493
 494     return ff_filter_frame(outlink, in);
 495 }
 496
 497 static av_cold void uninit(AVFilterContext *ctx)
 498 {
 499     ReadEIA608Context *s = ctx->priv;
 500
 501     for (int i = 0; i < s->nb_allocated; i++) {
 502         ScanItem *scan = &s->scan[i];
 503
 504         av_freep(&scan->histogram);
 505         av_freep(&scan->code);
 506         av_freep(&scan->line);
 507     }
 508
 509     s->nb_allocated = 0;
 510     av_freep(&s->scan);
 511 }
 512
 513 static int process_command(AVFilterContext *ctx, const char *cmd, const char *args,
 514                            char *res, int res_len, int flags)
 515 {
 516     ReadEIA608Context *s = ctx->priv;
 517     int ret, start = s->start, end = s->end;
 518
 519     ret = ff_filter_process_command(ctx, cmd, args, res, res_len, flags);
 520     if (ret < 0)
 521         return ret;
 522
 523     ret = config_filter(ctx, s->start, s->end);
 524     if (ret < 0) {
 525         s->start = start;
 526         s->end = end;
 527     }
 528
 529     return 0;
 530 }
 531
 532 static const AVFilterPad readeia608_inputs[] = {
 533     {
 534         .name         = "default",
 535         .type         = AVMEDIA_TYPE_VIDEO,
 536         .filter_frame = filter_frame,
 537         .config_props = config_input,
 538     },
 539 };
 540
 541 const FFFilter ff_vf_readeia608 = {
 542     .p.name        = "readeia608",
 543     .p.description = NULL_IF_CONFIG_SMALL("Read EIA-608 Closed Caption codes from input video and write them to frame metadata."),
 544     .p.priv_class  = &readeia608_class,
 545     .p.flags       = AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC |
 546                      AVFILTER_FLAG_SLICE_THREADS            |
 547                      AVFILTER_FLAG_METADATA_ONLY,
 548     .priv_size     = sizeof(ReadEIA608Context),
 549     FILTER_INPUTS(readeia608_inputs),
 550     FILTER_OUTPUTS(ff_video_default_filterpad),
 551     FILTER_PIXFMTS_ARRAY(pixel_fmts),
 552     .uninit        = uninit,
 553     .process_command = process_command,
 554 };