libavfilter/vulkan_filter.c

   1 /*
   2  * Copyright (c) Lynne
   3  *
   4  * This file is part of FFmpeg.
   5  *
   6  * FFmpeg is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * FFmpeg is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with FFmpeg; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 #include "filters.h"
  22 #include "vulkan_filter.h"
  23 #include "libavutil/vulkan_loader.h"
  24
  25 int ff_vk_filter_init_context(AVFilterContext *avctx, FFVulkanContext *s,
  26                               AVBufferRef *frames_ref,
  27                               int width, int height, enum AVPixelFormat sw_format)
  28 {
  29     int err;
  30     AVHWFramesContext *frames_ctx;
  31     AVHWDeviceContext *device_ctx;
  32     AVVulkanFramesContext *vk_frames;
  33     AVVulkanDeviceContext *vk_dev;
  34     AVBufferRef *device_ref = avctx->hw_device_ctx;
  35
  36     /* Check if context is reusable as-is */
  37     if (frames_ref) {
  38         int no_storage = 0;
  39         FFVulkanFunctions *vk;
  40         VkImageUsageFlagBits usage_req;
  41         const VkFormat *sub = av_vkfmt_from_pixfmt(sw_format);
  42
  43         frames_ctx = (AVHWFramesContext *)frames_ref->data;
  44         device_ctx = (AVHWDeviceContext *)frames_ctx->device_ref->data;
  45         vk_frames = frames_ctx->hwctx;
  46         vk_dev = device_ctx->hwctx;
  47
  48         /* Width and height mismatch */
  49         if (width != frames_ctx->width ||
  50             height != frames_ctx->height)
  51             goto skip;
  52
  53         /* Format mismatch */
  54         if (sw_format != frames_ctx->sw_format)
  55             goto skip;
  56
  57         /* Don't let linear through. */
  58         if (vk_frames->tiling == VK_IMAGE_TILING_LINEAR)
  59             goto skip;
  60
  61         s->extensions = ff_vk_extensions_to_mask(vk_dev->enabled_dev_extensions,
  62                                                  vk_dev->nb_enabled_dev_extensions);
  63
  64         /* More advanced format checks */
  65         err = ff_vk_load_functions(device_ctx, &s->vkfn, s->extensions, 1, 1);
  66         if (err < 0)
  67             return err;
  68         vk = &s->vkfn;
  69
  70         /* Usage mismatch */
  71         usage_req = VK_IMAGE_USAGE_SAMPLED_BIT |
  72                     VK_IMAGE_USAGE_STORAGE_BIT;
  73
  74         /* If format supports hardware encoding, make sure
  75          * the context includes it. */
  76         if (vk_frames->format[1] == VK_FORMAT_UNDEFINED &&
  77             (s->extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |
  78                               FF_VK_EXT_VIDEO_MAINTENANCE_1))) {
  79             VkFormatProperties3 fprops = {
  80                 .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3,
  81             };
  82             VkFormatProperties2 prop = {
  83                 .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
  84                 .pNext = &fprops,
  85             };
  86             vk->GetPhysicalDeviceFormatProperties2(vk_dev->phys_dev,
  87                                                    vk_frames->format[0],
  88                                                    &prop);
  89             if (fprops.optimalTilingFeatures & VK_FORMAT_FEATURE_2_VIDEO_ENCODE_INPUT_BIT_KHR)
  90                 usage_req |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
  91         }
  92
  93         if ((vk_frames->usage & usage_req) != usage_req)
  94             goto skip;
  95
  96         /* Check if the subformats can do storage */
  97         for (int i = 0; sub[i] != VK_FORMAT_UNDEFINED; i++) {
  98             VkFormatProperties2 prop = {
  99                 .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
 100             };
 101             vk->GetPhysicalDeviceFormatProperties2(vk_dev->phys_dev, sub[i],
 102                                                    &prop);
 103             no_storage |= !(prop.formatProperties.optimalTilingFeatures &
 104                             VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
 105         }
 106
 107         /* Check if it's usable */
 108         if (no_storage) {
 109 skip:
 110             av_log(avctx, AV_LOG_VERBOSE, "Cannot reuse context, creating a new one\n");
 111             device_ref = frames_ctx->device_ref;
 112             frames_ref = NULL;
 113         } else {
 114             av_log(avctx, AV_LOG_VERBOSE, "Reusing existing frames context\n");
 115             frames_ref = av_buffer_ref(frames_ref);
 116             if (!frames_ref)
 117                 return AVERROR(ENOMEM);
 118         }
 119     }
 120
 121     if (!frames_ref) {
 122         if (!device_ref) {
 123             av_log(avctx, AV_LOG_ERROR,
 124                    "Vulkan filtering requires a device context!\n");
 125             return AVERROR(EINVAL);
 126         }
 127
 128         frames_ref = av_hwframe_ctx_alloc(device_ref);
 129
 130         frames_ctx = (AVHWFramesContext *)frames_ref->data;
 131         frames_ctx->format    = AV_PIX_FMT_VULKAN;
 132         frames_ctx->sw_format = sw_format;
 133         frames_ctx->width     = width;
 134         frames_ctx->height    = height;
 135
 136         vk_frames = frames_ctx->hwctx;
 137         vk_frames->tiling = VK_IMAGE_TILING_OPTIMAL;
 138         vk_frames->usage  = VK_IMAGE_USAGE_SAMPLED_BIT |
 139                             VK_IMAGE_USAGE_STORAGE_BIT |
 140                             VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
 141
 142         err = av_hwframe_ctx_init(frames_ref);
 143         if (err < 0) {
 144             av_buffer_unref(&frames_ref);
 145             return err;
 146         }
 147
 148         device_ctx = (AVHWDeviceContext *)frames_ctx->device_ref->data;
 149         vk_dev = device_ctx->hwctx;
 150     }
 151
 152     s->extensions = ff_vk_extensions_to_mask(vk_dev->enabled_dev_extensions,
 153                                              vk_dev->nb_enabled_dev_extensions);
 154     s->extensions |= ff_vk_extensions_to_mask(vk_dev->enabled_inst_extensions,
 155                                               vk_dev->nb_enabled_inst_extensions);
 156
 157     err = ff_vk_load_functions(device_ctx, &s->vkfn, s->extensions, 1, 1);
 158     if (err < 0) {
 159         av_buffer_unref(&frames_ref);
 160         return err;
 161     }
 162
 163     s->frames_ref = frames_ref;
 164     s->frames = frames_ctx;
 165     s->hwfc = vk_frames;
 166     s->device = device_ctx;
 167     s->hwctx = device_ctx->hwctx;
 168
 169     err = ff_vk_load_props(s);
 170     if (err < 0)
 171         av_buffer_unref(&s->frames_ref);
 172
 173     return err;
 174 }
 175
 176 int ff_vk_filter_config_input(AVFilterLink *inlink)
 177 {
 178     FilterLink *l = ff_filter_link(inlink);
 179     AVHWFramesContext *input_frames;
 180     AVFilterContext *avctx = inlink->dst;
 181     FFVulkanContext *s = inlink->dst->priv;
 182
 183     if (!l->hw_frames_ctx) {
 184         av_log(inlink->dst, AV_LOG_ERROR, "Vulkan filtering requires a "
 185                "hardware frames context on the input.\n");
 186         return AVERROR(EINVAL);
 187     }
 188
 189     input_frames = (AVHWFramesContext *)l->hw_frames_ctx->data;
 190     if (input_frames->format != AV_PIX_FMT_VULKAN)
 191         return AVERROR(EINVAL);
 192
 193     /* Extract the device and default output format from the first input. */
 194     if (avctx->inputs[0] != inlink)
 195         return 0;
 196
 197     /* Save the ref, without reffing it */
 198     s->input_frames_ref = l->hw_frames_ctx;
 199
 200     /* Defaults */
 201     s->input_format = input_frames->sw_format;
 202     s->output_format = input_frames->sw_format;
 203     s->output_width = inlink->w;
 204     s->output_height = inlink->h;
 205
 206     return 0;
 207 }
 208
 209 int ff_vk_filter_config_output(AVFilterLink *outlink)
 210 {
 211     int err;
 212     FilterLink *l = ff_filter_link(outlink);
 213     FFVulkanContext *s = outlink->src->priv;
 214
 215     av_buffer_unref(&l->hw_frames_ctx);
 216
 217     err = ff_vk_filter_init_context(outlink->src, s, s->input_frames_ref,
 218                                     s->output_width, s->output_height,
 219                                     s->output_format);
 220     if (err < 0)
 221         return err;
 222
 223     l->hw_frames_ctx = av_buffer_ref(s->frames_ref);
 224     if (!l->hw_frames_ctx)
 225         return AVERROR(ENOMEM);
 226
 227     outlink->w = s->output_width;
 228     outlink->h = s->output_height;
 229
 230     return err;
 231 }
 232
 233 int ff_vk_filter_init(AVFilterContext *avctx)
 234 {
 235     FFVulkanContext *s = avctx->priv;
 236
 237     s->output_format = AV_PIX_FMT_NONE;
 238
 239     return 0;
 240 }
 241
 242 int ff_vk_filter_process_simple(FFVulkanContext *vkctx, FFVkExecPool *e,
 243                                 FFVulkanShader *shd, AVFrame *out_f, AVFrame *in_f,
 244                                 VkSampler sampler, void *push_src, size_t push_size)
 245 {
 246     int err = 0;
 247     FFVulkanFunctions *vk = &vkctx->vkfn;
 248     VkImageView in_views[AV_NUM_DATA_POINTERS];
 249     VkImageView out_views[AV_NUM_DATA_POINTERS];
 250     VkImageMemoryBarrier2 img_bar[37];
 251     int nb_img_bar = 0;
 252
 253     /* Update descriptors and init the exec context */
 254     FFVkExecContext *exec = ff_vk_exec_get(vkctx, e);
 255     ff_vk_exec_start(vkctx, exec);
 256
 257     RET(ff_vk_exec_add_dep_frame(vkctx, exec, out_f,
 258                                  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
 259                                  VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT));
 260     RET(ff_vk_create_imageviews(vkctx, exec, out_views, out_f, FF_VK_REP_FLOAT));
 261     ff_vk_shader_update_img_array(vkctx, exec, shd, out_f, out_views, 0, !!in_f,
 262                                   VK_IMAGE_LAYOUT_GENERAL,
 263                                   VK_NULL_HANDLE);
 264     if (in_f) {
 265         RET(ff_vk_exec_add_dep_frame(vkctx, exec, in_f,
 266                                      VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
 267                                      VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT));
 268         RET(ff_vk_create_imageviews(vkctx, exec, in_views,  in_f, FF_VK_REP_FLOAT));
 269         ff_vk_shader_update_img_array(vkctx, exec, shd,  in_f,  in_views, 0, 0,
 270                                           VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
 271                                           sampler);
 272     }
 273
 274     /* Bind pipeline, update push data */
 275     ff_vk_exec_bind_shader(vkctx, exec, shd);
 276     if (push_src)
 277         ff_vk_shader_update_push_const(vkctx, exec, shd, VK_SHADER_STAGE_COMPUTE_BIT,
 278                                        0, push_size, push_src);
 279
 280     /* Add data sync barriers */
 281     ff_vk_frame_barrier(vkctx, exec, out_f, img_bar, &nb_img_bar,
 282                         VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
 283                         VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
 284                         VK_ACCESS_SHADER_WRITE_BIT,
 285                         VK_IMAGE_LAYOUT_GENERAL,
 286                         VK_QUEUE_FAMILY_IGNORED);
 287     if (in_f)
 288         ff_vk_frame_barrier(vkctx, exec, in_f, img_bar, &nb_img_bar,
 289                             VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
 290                             VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
 291                             VK_ACCESS_SHADER_READ_BIT,
 292                             VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
 293                             VK_QUEUE_FAMILY_IGNORED);
 294
 295     vk->CmdPipelineBarrier2(exec->buf, &(VkDependencyInfo) {
 296             .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
 297             .pImageMemoryBarriers = img_bar,
 298             .imageMemoryBarrierCount = nb_img_bar,
 299         });
 300
 301     vk->CmdDispatch(exec->buf,
 302                     FFALIGN(vkctx->output_width,  shd->lg_size[0])/shd->lg_size[0],
 303                     FFALIGN(vkctx->output_height, shd->lg_size[1])/shd->lg_size[1],
 304                     shd->lg_size[2]);
 305
 306     return ff_vk_exec_submit(vkctx, exec);
 307 fail:
 308     ff_vk_exec_discard_deps(vkctx, exec);
 309     return err;
 310 }
 311
 312 int ff_vk_filter_process_2pass(FFVulkanContext *vkctx, FFVkExecPool *e,
 313                                FFVulkanShader *shd_list[2],
 314                                AVFrame *out, AVFrame *tmp, AVFrame *in,
 315                                VkSampler sampler, void *push_src, size_t push_size)
 316 {
 317     int err = 0;
 318     FFVulkanFunctions *vk = &vkctx->vkfn;
 319     VkImageView in_views[AV_NUM_DATA_POINTERS];
 320     VkImageView tmp_views[AV_NUM_DATA_POINTERS];
 321     VkImageView out_views[AV_NUM_DATA_POINTERS];
 322     VkImageMemoryBarrier2 img_bar[37];
 323     int nb_img_bar = 0;
 324
 325     /* Update descriptors and init the exec context */
 326     FFVkExecContext *exec = ff_vk_exec_get(vkctx, e);
 327     ff_vk_exec_start(vkctx, exec);
 328
 329     RET(ff_vk_exec_add_dep_frame(vkctx, exec, in,
 330                                  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
 331                                  VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT));
 332     RET(ff_vk_exec_add_dep_frame(vkctx, exec, tmp,
 333                                  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
 334                                  VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT));
 335     RET(ff_vk_exec_add_dep_frame(vkctx, exec, out,
 336                                  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
 337                                  VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT));
 338
 339     RET(ff_vk_create_imageviews(vkctx, exec, in_views,  in, FF_VK_REP_FLOAT));
 340     RET(ff_vk_create_imageviews(vkctx, exec, tmp_views, tmp, FF_VK_REP_FLOAT));
 341     RET(ff_vk_create_imageviews(vkctx, exec, out_views, out, FF_VK_REP_FLOAT));
 342
 343     ff_vk_frame_barrier(vkctx, exec, in, img_bar, &nb_img_bar,
 344                         VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
 345                         VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
 346                         VK_ACCESS_SHADER_READ_BIT,
 347                         VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
 348                         VK_QUEUE_FAMILY_IGNORED);
 349     ff_vk_frame_barrier(vkctx, exec, tmp, img_bar, &nb_img_bar,
 350                         VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
 351                         VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
 352                         VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT,
 353                         VK_IMAGE_LAYOUT_GENERAL,
 354                         VK_QUEUE_FAMILY_IGNORED);
 355     ff_vk_frame_barrier(vkctx, exec, out, img_bar, &nb_img_bar,
 356                         VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
 357                         VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
 358                         VK_ACCESS_SHADER_WRITE_BIT,
 359                         VK_IMAGE_LAYOUT_GENERAL,
 360                         VK_QUEUE_FAMILY_IGNORED);
 361
 362     vk->CmdPipelineBarrier2(exec->buf, &(VkDependencyInfo) {
 363             .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
 364             .pImageMemoryBarriers = img_bar,
 365             .imageMemoryBarrierCount = nb_img_bar,
 366         });
 367
 368     for (int i = 0; i < 2; i++) {
 369         FFVulkanShader *shd = shd_list[i];
 370         AVFrame *src_f = !i ? in : tmp;
 371         AVFrame *dst_f = !i ? tmp : out;
 372         VkImageView *src_views = !i ? in_views : tmp_views;
 373         VkImageView *dst_views = !i ? tmp_views : out_views;
 374
 375         ff_vk_shader_update_img_array(vkctx, exec, shd, src_f, src_views, 0, 0,
 376                                       !i ? VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL :
 377                                            VK_IMAGE_LAYOUT_GENERAL,
 378                                       sampler);
 379         ff_vk_shader_update_img_array(vkctx, exec, shd, dst_f, dst_views, 0, 1,
 380                                       VK_IMAGE_LAYOUT_GENERAL,
 381                                       VK_NULL_HANDLE);
 382
 383         /* Bind pipeline, update push data */
 384         ff_vk_exec_bind_shader(vkctx, exec, shd);
 385         if (push_src)
 386             ff_vk_shader_update_push_const(vkctx, exec, shd, VK_SHADER_STAGE_COMPUTE_BIT,
 387                                            0, push_size, push_src);
 388
 389         vk->CmdDispatch(exec->buf,
 390                         FFALIGN(vkctx->output_width,  shd->lg_size[0])/shd->lg_size[0],
 391                         FFALIGN(vkctx->output_height, shd->lg_size[1])/shd->lg_size[1],
 392                         shd->lg_size[2]);
 393     }
 394
 395     return ff_vk_exec_submit(vkctx, exec);
 396 fail:
 397     ff_vk_exec_discard_deps(vkctx, exec);
 398     return err;
 399 }
 400
 401 int ff_vk_filter_process_Nin(FFVulkanContext *vkctx, FFVkExecPool *e,
 402                              FFVulkanShader *shd,
 403                              AVFrame *out, AVFrame *in[], int nb_in,
 404                              VkSampler sampler, void *push_src, size_t push_size)
 405 {
 406     int err = 0;
 407     FFVulkanFunctions *vk = &vkctx->vkfn;
 408     VkImageView in_views[16][AV_NUM_DATA_POINTERS];
 409     VkImageView out_views[AV_NUM_DATA_POINTERS];
 410     VkImageMemoryBarrier2 img_bar[128];
 411     int nb_img_bar = 0;
 412
 413     /* Update descriptors and init the exec context */
 414     FFVkExecContext *exec = ff_vk_exec_get(vkctx, e);
 415     ff_vk_exec_start(vkctx, exec);
 416
 417     /* Add deps and create temporary imageviews */
 418     RET(ff_vk_exec_add_dep_frame(vkctx, exec, out,
 419                                  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
 420                                  VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT));
 421     RET(ff_vk_create_imageviews(vkctx, exec, out_views, out, FF_VK_REP_FLOAT));
 422     for (int i = 0; i < nb_in; i++) {
 423         RET(ff_vk_exec_add_dep_frame(vkctx, exec, in[i],
 424                                      VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
 425                                      VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT));
 426         RET(ff_vk_create_imageviews(vkctx, exec, in_views[i], in[i], FF_VK_REP_FLOAT));
 427     }
 428
 429     /* Update descriptor sets */
 430     ff_vk_shader_update_img_array(vkctx, exec, shd, out, out_views, 0, nb_in,
 431                                   VK_IMAGE_LAYOUT_GENERAL,
 432                                   VK_NULL_HANDLE);
 433     for (int i = 0; i < nb_in; i++)
 434         ff_vk_shader_update_img_array(vkctx, exec, shd, in[i], in_views[i], 0, i,
 435                                       VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
 436                                       sampler);
 437
 438     /* Bind pipeline, update push data */
 439     ff_vk_exec_bind_shader(vkctx, exec, shd);
 440     if (push_src)
 441         ff_vk_shader_update_push_const(vkctx, exec, shd, VK_SHADER_STAGE_COMPUTE_BIT,
 442                                        0, push_size, push_src);
 443
 444     /* Add data sync barriers */
 445     ff_vk_frame_barrier(vkctx, exec, out, img_bar, &nb_img_bar,
 446                         VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
 447                         VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
 448                         VK_ACCESS_SHADER_WRITE_BIT,
 449                         VK_IMAGE_LAYOUT_GENERAL,
 450                         VK_QUEUE_FAMILY_IGNORED);
 451     for (int i = 0; i < nb_in; i++)
 452         ff_vk_frame_barrier(vkctx, exec, in[i], img_bar, &nb_img_bar,
 453                             VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
 454                             VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
 455                             VK_ACCESS_SHADER_READ_BIT,
 456                             VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
 457                             VK_QUEUE_FAMILY_IGNORED);
 458
 459     vk->CmdPipelineBarrier2(exec->buf, &(VkDependencyInfo) {
 460             .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
 461             .pImageMemoryBarriers = img_bar,
 462             .imageMemoryBarrierCount = nb_img_bar,
 463         });
 464
 465     vk->CmdDispatch(exec->buf,
 466                     FFALIGN(vkctx->output_width,  shd->lg_size[0])/shd->lg_size[0],
 467                     FFALIGN(vkctx->output_height, shd->lg_size[1])/shd->lg_size[1],
 468                     shd->lg_size[2]);
 469
 470     return ff_vk_exec_submit(vkctx, exec);
 471 fail:
 472     ff_vk_exec_discard_deps(vkctx, exec);
 473     return err;
 474 }