libavfilter/vf_bwdif_vulkan.c

   1 /*
   2  * Copyright (c) Lynne
   3  * Copyright (C) 2018 Philip Langdale <philipl@overt.org>
   4  * Copyright (C) 2016 Thomas Mundt <loudmax@yahoo.de>
   5  *
   6  * This file is part of FFmpeg.
   7  *
   8  * FFmpeg is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2.1 of the License, or (at your option) any later version.
  12  *
  13  * FFmpeg is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with FFmpeg; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  21  */
  22
  23 #include "libavutil/random_seed.h"
  24 #include "libavutil/opt.h"
  25 #include "libavutil/vulkan_spirv.h"
  26 #include "vulkan_filter.h"
  27 #include "yadif.h"
  28 #include "filters.h"
  29
  30 typedef struct BWDIFVulkanContext {
  31     YADIFContext yadif;
  32     FFVulkanContext vkctx;
  33
  34     int initialized;
  35     FFVkExecPool e;
  36     AVVulkanDeviceQueueFamily *qf;
  37     VkSampler sampler;
  38     FFVulkanShader shd;
  39 } BWDIFVulkanContext;
  40
  41 typedef struct BWDIFParameters {
  42     int parity;
  43     int tff;
  44     int current_field;
  45 } BWDIFParameters;
  46
  47 extern const char *ff_source_bwdif_comp;
  48
  49 static av_cold int init_filter(AVFilterContext *ctx)
  50 {
  51     int err;
  52     uint8_t *spv_data;
  53     size_t spv_len;
  54     void *spv_opaque = NULL;
  55     BWDIFVulkanContext *s = ctx->priv;
  56     FFVulkanContext *vkctx = &s->vkctx;
  57     const int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
  58     FFVulkanShader *shd;
  59     FFVkSPIRVCompiler *spv;
  60     FFVulkanDescriptorSetBinding *desc;
  61
  62     spv = ff_vk_spirv_init();
  63     if (!spv) {
  64         av_log(ctx, AV_LOG_ERROR, "Unable to initialize SPIR-V compiler!\n");
  65         return AVERROR_EXTERNAL;
  66     }
  67
  68     s->qf = ff_vk_qf_find(vkctx, VK_QUEUE_COMPUTE_BIT, 0);
  69     if (!s->qf) {
  70         av_log(ctx, AV_LOG_ERROR, "Device has no compute queues\n");
  71         err = AVERROR(ENOTSUP);
  72         goto fail;
  73     }
  74
  75     RET(ff_vk_exec_pool_init(vkctx, s->qf, &s->e, s->qf->num*4, 0, 0, 0, NULL));
  76     RET(ff_vk_init_sampler(vkctx, &s->sampler, 1, VK_FILTER_NEAREST));
  77
  78     RET(ff_vk_shader_init(vkctx, &s->shd, "bwdif",
  79                           VK_SHADER_STAGE_COMPUTE_BIT,
  80                           NULL, 0,
  81                           1, 64, 1,
  82                           0));
  83     shd = &s->shd;
  84
  85     desc = (FFVulkanDescriptorSetBinding []) {
  86         {
  87             .name       = "prev",
  88             .type       = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
  89             .dimensions = 2,
  90             .elems      = planes,
  91             .stages     = VK_SHADER_STAGE_COMPUTE_BIT,
  92             .samplers   = DUP_SAMPLER(s->sampler),
  93         },
  94         {
  95             .name       = "cur",
  96             .type       = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
  97             .dimensions = 2,
  98             .elems      = planes,
  99             .stages     = VK_SHADER_STAGE_COMPUTE_BIT,
 100             .samplers   = DUP_SAMPLER(s->sampler),
 101         },
 102         {
 103             .name       = "next",
 104             .type       = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
 105             .dimensions = 2,
 106             .elems      = planes,
 107             .stages     = VK_SHADER_STAGE_COMPUTE_BIT,
 108             .samplers   = DUP_SAMPLER(s->sampler),
 109         },
 110         {
 111             .name       = "dst",
 112             .type       = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 113             .mem_layout = ff_vk_shader_rep_fmt(s->vkctx.output_format, FF_VK_REP_FLOAT),
 114             .mem_quali  = "writeonly",
 115             .dimensions = 2,
 116             .elems      = planes,
 117             .stages     = VK_SHADER_STAGE_COMPUTE_BIT,
 118         },
 119     };
 120
 121     RET(ff_vk_shader_add_descriptor_set(vkctx, &s->shd, desc, 4, 0, 0));
 122
 123     GLSLC(0, layout(push_constant, std430) uniform pushConstants {                 );
 124     GLSLC(1,    int parity;                                                        );
 125     GLSLC(1,    int tff;                                                           );
 126     GLSLC(1,    int current_field;                                                 );
 127     GLSLC(0, };                                                                    );
 128
 129     ff_vk_shader_add_push_const(&s->shd, 0, sizeof(BWDIFParameters),
 130                                 VK_SHADER_STAGE_COMPUTE_BIT);
 131
 132     GLSLD(ff_source_bwdif_comp                                                     );
 133     GLSLC(0, void main()                                                           );
 134     GLSLC(0, {                                                                     );
 135     GLSLC(1,     ivec2 size;                                                       );
 136     GLSLC(1,     const ivec2 pos = ivec2(gl_GlobalInvocationID.xy);                );
 137     GLSLC(1,     bool filter_field = ((pos.y ^ parity) & 1) == 1;                  );
 138     GLSLF(1,     bool is_intra = filter_field && (current_field == %i);           ,YADIF_FIELD_END);
 139     GLSLC(1,     bool field_parity = (parity ^ tff) != 0;                          );
 140     GLSLC(0,                                                                       );
 141     GLSLC(1,     size = imageSize(dst[0]);                                         );
 142     GLSLC(1,     if (!IS_WITHIN(pos, size)) {                                      );
 143     GLSLC(2,         return;                                                       );
 144     GLSLC(1,     } else if (is_intra) {                                            );
 145     for (int i = 0; i < planes; i++) {
 146         if (i == 1) {
 147             GLSLF(2, size = imageSize(dst[%i]);                                    ,i);
 148             GLSLC(2, if (!IS_WITHIN(pos, size))                                    );
 149             GLSLC(3,     return;                                                   );
 150         }
 151         GLSLF(2,     process_plane_intra(%i, pos);                                 ,i);
 152     }
 153     GLSLC(1,     } else if (filter_field) {                                        );
 154     for (int i = 0; i < planes; i++) {
 155         if (i == 1) {
 156             GLSLF(2, size = imageSize(dst[%i]);                                    ,i);
 157             GLSLC(2, if (!IS_WITHIN(pos, size))                                    );
 158             GLSLC(3,     return;                                                   );
 159         }
 160         GLSLF(2,     process_plane(%i, pos, filter_field, is_intra, field_parity); ,i);
 161     }
 162     GLSLC(1,     } else {                                                          );
 163     for (int i = 0; i < planes; i++) {
 164         if (i == 1) {
 165             GLSLF(2, size = imageSize(dst[%i]);                                    ,i);
 166             GLSLC(2, if (!IS_WITHIN(pos, size))                                    );
 167             GLSLC(3,     return;                                                   );
 168         }
 169         GLSLF(2,     imageStore(dst[%i], pos, texture(cur[%i], pos));              ,i, i);
 170     }
 171     GLSLC(1,     }                                                                 );
 172     GLSLC(0, }                                                                     );
 173
 174     RET(spv->compile_shader(vkctx, spv, &s->shd, &spv_data, &spv_len, "main",
 175                             &spv_opaque));
 176     RET(ff_vk_shader_link(vkctx, &s->shd, spv_data, spv_len, "main"));
 177
 178     RET(ff_vk_shader_register_exec(vkctx, &s->e, &s->shd));
 179
 180     s->initialized = 1;
 181
 182 fail:
 183     if (spv_opaque)
 184         spv->free_shader(spv, &spv_opaque);
 185     if (spv)
 186         spv->uninit(&spv);
 187
 188     return err;
 189 }
 190
 191 static void bwdif_vulkan_filter_frame(AVFilterContext *ctx, AVFrame *dst,
 192                                       int parity, int tff)
 193 {
 194     BWDIFVulkanContext *s = ctx->priv;
 195     YADIFContext *y = &s->yadif;
 196     BWDIFParameters params = {
 197         .parity = parity,
 198         .tff = tff,
 199         .current_field = y->current_field,
 200     };
 201
 202     ff_vk_filter_process_Nin(&s->vkctx, &s->e, &s->shd, dst,
 203                              (AVFrame *[]){ y->prev, y->cur, y->next }, 3,
 204                              s->sampler, &params, sizeof(params));
 205
 206     if (y->current_field == YADIF_FIELD_END)
 207         y->current_field = YADIF_FIELD_NORMAL;
 208 }
 209
 210 static void bwdif_vulkan_uninit(AVFilterContext *avctx)
 211 {
 212     BWDIFVulkanContext *s = avctx->priv;
 213     FFVulkanContext *vkctx = &s->vkctx;
 214     FFVulkanFunctions *vk = &vkctx->vkfn;
 215
 216     ff_vk_exec_pool_free(vkctx, &s->e);
 217     ff_vk_shader_free(vkctx, &s->shd);
 218
 219     if (s->sampler)
 220         vk->DestroySampler(vkctx->hwctx->act_dev, s->sampler,
 221                            vkctx->hwctx->alloc);
 222
 223     ff_vk_uninit(&s->vkctx);
 224
 225     ff_yadif_uninit(avctx);
 226
 227     s->initialized = 0;
 228 }
 229
 230 static int bwdif_vulkan_config_input(AVFilterLink *inlink)
 231 {
 232     FilterLink *l = ff_filter_link(inlink);
 233     AVHWFramesContext *input_frames;
 234     AVFilterContext *avctx = inlink->dst;
 235     BWDIFVulkanContext *s = avctx->priv;
 236     FFVulkanContext *vkctx = &s->vkctx;
 237
 238     if (!l->hw_frames_ctx) {
 239         av_log(inlink->dst, AV_LOG_ERROR, "Vulkan filtering requires a "
 240                "hardware frames context on the input.\n");
 241         return AVERROR(EINVAL);
 242     }
 243
 244     input_frames = (AVHWFramesContext *)l->hw_frames_ctx->data;
 245     if (input_frames->format != AV_PIX_FMT_VULKAN)
 246         return AVERROR(EINVAL);
 247
 248     /* Extract the device and default output format from the first input. */
 249     if (avctx->inputs[0] != inlink)
 250         return 0;
 251
 252     /* Save the ref, without reffing it */
 253     vkctx->input_frames_ref = l->hw_frames_ctx;
 254
 255     /* Defaults */
 256     vkctx->output_format = input_frames->sw_format;
 257     vkctx->output_width  = inlink->w;
 258     vkctx->output_height = inlink->h;
 259
 260     return 0;
 261 }
 262
 263 static int bwdif_vulkan_config_output(AVFilterLink *outlink)
 264 {
 265     FilterLink *l = ff_filter_link(outlink);
 266     int err;
 267     AVFilterContext *avctx = outlink->src;
 268     BWDIFVulkanContext *s = avctx->priv;
 269     YADIFContext *y = &s->yadif;
 270     FFVulkanContext *vkctx = &s->vkctx;
 271
 272     av_buffer_unref(&l->hw_frames_ctx);
 273
 274     err = ff_vk_filter_init_context(avctx, vkctx, vkctx->input_frames_ref,
 275                                     vkctx->output_width, vkctx->output_height,
 276                                     vkctx->output_format);
 277     if (err < 0)
 278         return err;
 279
 280     /* For logging */
 281     vkctx->class = y->class;
 282
 283     l->hw_frames_ctx = av_buffer_ref(vkctx->frames_ref);
 284     if (!l->hw_frames_ctx)
 285         return AVERROR(ENOMEM);
 286
 287     err = ff_yadif_config_output_common(outlink);
 288     if (err < 0)
 289         return err;
 290
 291     y->csp = av_pix_fmt_desc_get(vkctx->frames->sw_format);
 292     y->filter = bwdif_vulkan_filter_frame;
 293
 294     if (AV_CEIL_RSHIFT(outlink->w, y->csp->log2_chroma_w) < 4 || AV_CEIL_RSHIFT(outlink->h, y->csp->log2_chroma_h) < 4) {
 295         av_log(avctx, AV_LOG_ERROR, "Video with planes less than 4 columns or lines is not supported\n");
 296         return AVERROR(EINVAL);
 297     }
 298
 299     return init_filter(avctx);
 300 }
 301
 302 static const AVClass bwdif_vulkan_class = {
 303     .class_name = "bwdif_vulkan",
 304     .item_name  = av_default_item_name,
 305     .option     = ff_yadif_options,
 306     .version    = LIBAVUTIL_VERSION_INT,
 307     .category   = AV_CLASS_CATEGORY_FILTER,
 308 };
 309
 310 static const AVFilterPad bwdif_vulkan_inputs[] = {
 311     {
 312         .name         = "default",
 313         .type         = AVMEDIA_TYPE_VIDEO,
 314         .filter_frame  = ff_yadif_filter_frame,
 315         .config_props = &bwdif_vulkan_config_input,
 316     },
 317 };
 318
 319 static const AVFilterPad bwdif_vulkan_outputs[] = {
 320     {
 321         .name = "default",
 322         .type = AVMEDIA_TYPE_VIDEO,
 323         .request_frame = ff_yadif_request_frame,
 324         .config_props = &bwdif_vulkan_config_output,
 325     },
 326 };
 327
 328 const FFFilter ff_vf_bwdif_vulkan = {
 329     .p.name         = "bwdif_vulkan",
 330     .p.description  = NULL_IF_CONFIG_SMALL("Deinterlace Vulkan frames via bwdif"),
 331     .p.priv_class   = &bwdif_vulkan_class,
 332     .p.flags        = AVFILTER_FLAG_HWDEVICE |
 333                       AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL,
 334     .priv_size      = sizeof(BWDIFVulkanContext),
 335     .init           = &ff_vk_filter_init,
 336     .uninit         = &bwdif_vulkan_uninit,
 337     FILTER_INPUTS(bwdif_vulkan_inputs),
 338     FILTER_OUTPUTS(bwdif_vulkan_outputs),
 339     FILTER_SINGLE_PIXFMT(AV_PIX_FMT_VULKAN),
 340     .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 341 };