overlay_vulkan: port for the rewrite

Lynne committed 3 years ago

Commit 8e9ceb11 parent 59f21c01 file · +123 −275Browse files

Unified · Split

@@ -1,4 +1,6 @@
1	1	/*
	2	+ * Copyright (c) Lynne
	3	+ *
2	4	* This file is part of FFmpeg.
3	5	*
4	6	* FFmpeg is free software; you can redistribute it and/or
@@ -19,26 +21,26 @@ · 14 unmodified lines
19	21	#include "libavutil/random_seed.h"
20	22	#include "libavutil/opt.h"
21	23	#include "vulkan_filter.h"
	24	+#include "vulkan_spirv.h"
22	25	#include "internal.h"
23	26	#include "framesync.h"
24	27
25		−#define CGROUPS (int [3]){ 32, 32, 1 }
26		−
27	28	typedef struct OverlayVulkanContext {
28	29	FFVulkanContext vkctx;
	30	+ FFFrameSync fs;
29	31
30	32	int initialized;
	33	+ FFVulkanPipeline pl;
	34	+ FFVkExecPool e;
31	35	FFVkQueueFamilyCtx qf;
32		− FFVkExecContext *exec;
33		− FFVulkanPipeline *pl;
34		− FFFrameSync fs;
35		− FFVkBuffer params_buf;
	36	+ FFVkSPIRVShader shd;
	37	+ VkSampler sampler;
36	38
37		− /* Shader updators, must be in the main filter struct */
38		− VkDescriptorImageInfo main_images[3];
39		− VkDescriptorImageInfo overlay_images[3];
40		− VkDescriptorImageInfo output_images[3];
41		− VkDescriptorBufferInfo params_desc;
	39	+ /* Push constants / options */
	40	+ struct {
	41	+ int32_t o_offset[2*3];
	42	+ int32_t o_size[2*3];
	43	+ } opts;
42	44
43	45	int overlay_x;
44	46	int overlay_y;
@@ -80,279 +82,114 @@ static const char overlay_alpha[] = { · 35 unmodified lines
80	82	static av_cold int init_filter(AVFilterContext *ctx)
81	83	{
82	84	int err;
83		− FFVkSampler *sampler;
	85	+ uint8_t *spv_data;
	86	+ size_t spv_len;
	87	+ void *spv_opaque = NULL;
84	88	OverlayVulkanContext *s = ctx->priv;
85	89	FFVulkanContext *vkctx = &s->vkctx;
86	90	const int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
87		−
88		− ff_vk_qf_init(vkctx, &s->qf, VK_QUEUE_COMPUTE_BIT, 0);
89		−
90		− sampler = ff_vk_init_sampler(vkctx, 1, VK_FILTER_NEAREST);
91		− if (!sampler)
	91	+ const int ialpha = av_pix_fmt_desc_get(s->vkctx.input_format)->flags & AV_PIX_FMT_FLAG_ALPHA;
	92	+ const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(s->vkctx.output_format);
	93	+ FFVkSPIRVShader *shd = &s->shd;
	94	+ FFVkSPIRVCompiler *spv;
	95	+ FFVulkanDescriptorSetBinding *desc;
	96	+
	97	+ spv = ff_vk_spirv_init();
	98	+ if (!spv) {
	99	+ av_log(ctx, AV_LOG_ERROR, "Unable to initialize SPIR-V compiler!\n");
92	100	return AVERROR_EXTERNAL;
93		−
94		− s->pl = ff_vk_create_pipeline(vkctx, &s->qf);
95		− if (!s->pl)
96		− return AVERROR(ENOMEM);
97		−
98		− { /* Create the shader */
99		− const int ialpha = av_pix_fmt_desc_get(s->vkctx.input_format)->flags & AV_PIX_FMT_FLAG_ALPHA;
100		−
101		− FFVulkanDescriptorSetBinding desc_i[3] = {
102		− {
103		− .name = "main_img",
104		− .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
105		− .dimensions = 2,
106		− .elems = planes,
107		− .stages = VK_SHADER_STAGE_COMPUTE_BIT,
108		− .updater = s->main_images,
109		− .sampler = sampler,
110		− },
111		− {
112		− .name = "overlay_img",
113		− .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
114		− .dimensions = 2,
115		− .elems = planes,
116		− .stages = VK_SHADER_STAGE_COMPUTE_BIT,
117		− .updater = s->overlay_images,
118		− .sampler = sampler,
119		− },
120		− {
121		− .name = "output_img",
122		− .type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
123		− .mem_layout = ff_vk_shader_rep_fmt(s->vkctx.output_format),
124		− .mem_quali = "writeonly",
125		− .dimensions = 2,
126		− .elems = planes,
127		− .stages = VK_SHADER_STAGE_COMPUTE_BIT,
128		− .updater = s->output_images,
129		− },
130		− };
131		−
132		− FFVulkanDescriptorSetBinding desc_b = {
133		− .name = "params",
134		− .type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
135		− .mem_quali = "readonly",
136		− .mem_layout = "std430",
137		− .stages = VK_SHADER_STAGE_COMPUTE_BIT,
138		− .updater = &s->params_desc,
139		− .buf_content = "ivec2 o_offset[3], o_size[3];",
140		− };
141		−
142		− FFVkSPIRVShader *shd = ff_vk_init_shader(s->pl, "overlay_compute",
143		− VK_SHADER_STAGE_COMPUTE_BIT);
144		− if (!shd)
145		− return AVERROR(ENOMEM);
146		−
147		− ff_vk_set_compute_shader_sizes(shd, CGROUPS);
148		−
149		− RET(ff_vk_add_descriptor_set(vkctx, s->pl, shd, desc_i, FF_ARRAY_ELEMS(desc_i), 0)); /* set 0 */
150		− RET(ff_vk_add_descriptor_set(vkctx, s->pl, shd, &desc_b, 1, 0)); /* set 1 */
151		−
152		− GLSLD( overlay_noalpha );
153		− GLSLD( overlay_alpha );
154		− GLSLC(0, void main() );
155		− GLSLC(0, { );
156		− GLSLC(1, ivec2 pos = ivec2(gl_GlobalInvocationID.xy); );
157		− GLSLF(1, int planes = %i; ,planes);
158		− GLSLC(1, for (int i = 0; i < planes; i++) { );
159		− if (ialpha)
160		− GLSLC(2, overlay_alpha_opaque(i, pos); );
161		− else
162		− GLSLC(2, overlay_noalpha(i, pos); );
163		− GLSLC(1, } );
164		− GLSLC(0, } );
165		−
166		− RET(ff_vk_compile_shader(vkctx, shd, "main"));
167		− }
168		−
169		− RET(ff_vk_init_pipeline_layout(vkctx, s->pl));
170		− RET(ff_vk_init_compute_pipeline(vkctx, s->pl));
171		−
172		− { /* Create and update buffer */
173		− const AVPixFmtDescriptor *desc;
174		−
175		− /* NOTE: std430 requires the same identical struct layout, padding and
176		− * alignment as C, so we're allowed to do this, as this will map
177		− * exactly to what the shader recieves */
178		− struct {
179		− int32_t o_offset[2*3];
180		− int32_t o_size[2*3];
181		− } *par;
182		−
183		− err = ff_vk_create_buf(vkctx, &s->params_buf,
184		− sizeof(*par), NULL,
185		− VK_BUFFER_USAGE_STORAGE_BUFFER_BIT,
186		− VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);
187		− if (err)
188		− return err;
189		−
190		− err = ff_vk_map_buffers(vkctx, &s->params_buf, (uint8_t **)&par, 1, 0);
191		− if (err)
192		− return err;
193		−
194		− desc = av_pix_fmt_desc_get(s->vkctx.output_format);
195		−
196		− par->o_offset[0] = s->overlay_x;
197		− par->o_offset[1] = s->overlay_y;
198		− par->o_offset[2] = par->o_offset[0] >> desc->log2_chroma_w;
199		− par->o_offset[3] = par->o_offset[1] >> desc->log2_chroma_h;
200		− par->o_offset[4] = par->o_offset[0] >> desc->log2_chroma_w;
201		− par->o_offset[5] = par->o_offset[1] >> desc->log2_chroma_h;
202		−
203		− par->o_size[0] = s->overlay_w;
204		− par->o_size[1] = s->overlay_h;
205		− par->o_size[2] = par->o_size[0] >> desc->log2_chroma_w;
206		− par->o_size[3] = par->o_size[1] >> desc->log2_chroma_h;
207		− par->o_size[4] = par->o_size[0] >> desc->log2_chroma_w;
208		− par->o_size[5] = par->o_size[1] >> desc->log2_chroma_h;
209		−
210		− err = ff_vk_unmap_buffers(vkctx, &s->params_buf, 1, 1);
211		− if (err)
212		− return err;
213		−
214		− s->params_desc.buffer = s->params_buf.buf;
215		− s->params_desc.range = VK_WHOLE_SIZE;
216		−
217		− ff_vk_update_descriptor_set(vkctx, s->pl, 1);
218	101	}
219	102
220		− /* Execution context */
221		− RET(ff_vk_create_exec_ctx(vkctx, &s->exec, &s->qf));
	103	+ ff_vk_qf_init(vkctx, &s->qf, VK_QUEUE_COMPUTE_BIT);
	104	+ RET(ff_vk_exec_pool_init(vkctx, &s->qf, &s->e, s->qf.nb_queues*4, 0, 0, 0, NULL));
	105	+ RET(ff_vk_init_sampler(vkctx, &s->sampler, 1, VK_FILTER_NEAREST));
	106	+ RET(ff_vk_shader_init(&s->pl, &s->shd, "overlay_compute",
	107	+ VK_SHADER_STAGE_COMPUTE_BIT, 0));
	108	+
	109	+ ff_vk_shader_set_compute_sizes(&s->shd, 32, 32, 1);
	110	+
	111	+ GLSLC(0, layout(push_constant, std430) uniform pushConstants { );
	112	+ GLSLC(1, ivec2 o_offset[3]; );
	113	+ GLSLC(1, ivec2 o_size[3]; );
	114	+ GLSLC(0, }; );
	115	+ GLSLC(0, );
	116	+
	117	+ ff_vk_add_push_constant(&s->pl, 0, sizeof(s->opts),
	118	+ VK_SHADER_STAGE_COMPUTE_BIT);
	119	+
	120	+ desc = (FFVulkanDescriptorSetBinding []) {
	121	+ {
	122	+ .name = "main_img",
	123	+ .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
	124	+ .dimensions = 2,
	125	+ .elems = planes,
	126	+ .stages = VK_SHADER_STAGE_COMPUTE_BIT,
	127	+ .samplers = DUP_SAMPLER(s->sampler),
	128	+ },
	129	+ {
	130	+ .name = "overlay_img",
	131	+ .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
	132	+ .dimensions = 2,
	133	+ .elems = planes,
	134	+ .stages = VK_SHADER_STAGE_COMPUTE_BIT,
	135	+ .samplers = DUP_SAMPLER(s->sampler),
	136	+ },
	137	+ {
	138	+ .name = "output_img",
	139	+ .type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
	140	+ .mem_layout = ff_vk_shader_rep_fmt(s->vkctx.output_format),
	141	+ .mem_quali = "writeonly",
	142	+ .dimensions = 2,
	143	+ .elems = planes,
	144	+ .stages = VK_SHADER_STAGE_COMPUTE_BIT,
	145	+ },
	146	+ };
	147	+
	148	+ RET(ff_vk_pipeline_descriptor_set_add(vkctx, &s->pl, shd, desc, 3, 0, 0));
	149	+
	150	+ GLSLD( overlay_noalpha );
	151	+ GLSLD( overlay_alpha );
	152	+ GLSLC(0, void main() );
	153	+ GLSLC(0, { );
	154	+ GLSLC(1, ivec2 pos = ivec2(gl_GlobalInvocationID.xy); );
	155	+ GLSLF(1, int planes = %i; ,planes);
	156	+ GLSLC(1, for (int i = 0; i < planes; i++) { );
	157	+ if (ialpha)
	158	+ GLSLC(2, overlay_alpha_opaque(i, pos); );
	159	+ else
	160	+ GLSLC(2, overlay_noalpha(i, pos); );
	161	+ GLSLC(1, } );
	162	+ GLSLC(0, } );
	163	+
	164	+ RET(spv->compile_shader(spv, ctx, shd, &spv_data, &spv_len, "main",
	165	+ &spv_opaque));
	166	+ RET(ff_vk_shader_create(vkctx, shd, spv_data, spv_len, "main"));
	167	+
	168	+ RET(ff_vk_init_compute_pipeline(vkctx, &s->pl, shd));
	169	+ RET(ff_vk_exec_pipeline_register(vkctx, &s->e, &s->pl));
	170	+
	171	+ s->opts.o_offset[0] = s->overlay_x;
	172	+ s->opts.o_offset[1] = s->overlay_y;
	173	+ s->opts.o_offset[2] = s->opts.o_offset[0] >> pix_desc->log2_chroma_w;
	174	+ s->opts.o_offset[3] = s->opts.o_offset[1] >> pix_desc->log2_chroma_h;
	175	+ s->opts.o_offset[4] = s->opts.o_offset[0] >> pix_desc->log2_chroma_w;
	176	+ s->opts.o_offset[5] = s->opts.o_offset[1] >> pix_desc->log2_chroma_h;
	177	+
	178	+ s->opts.o_size[0] = s->overlay_w;
	179	+ s->opts.o_size[1] = s->overlay_h;
	180	+ s->opts.o_size[2] = s->opts.o_size[0] >> pix_desc->log2_chroma_w;
	181	+ s->opts.o_size[3] = s->opts.o_size[1] >> pix_desc->log2_chroma_h;
	182	+ s->opts.o_size[4] = s->opts.o_size[0] >> pix_desc->log2_chroma_w;
	183	+ s->opts.o_size[5] = s->opts.o_size[1] >> pix_desc->log2_chroma_h;
222	184
223	185	s->initialized = 1;
224	186
225		− return 0;
226		−
227	187	fail:
228		− return err;
229		−}
230		−
231		−static int process_frames(AVFilterContext avctx, AVFrame out_f,
232		− AVFrame main_f, AVFrame overlay_f)
233		−{
234		− int err;
235		− VkCommandBuffer cmd_buf;
236		− OverlayVulkanContext *s = avctx->priv;
237		− FFVulkanContext *vkctx = &s->vkctx;
238		− FFVulkanFunctions *vk = &vkctx->vkfn;
239		− int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
240		−
241		− AVVkFrame out = (AVVkFrame )out_f->data[0];
242		− AVVkFrame main = (AVVkFrame )main_f->data[0];
243		− AVVkFrame overlay = (AVVkFrame )overlay_f->data[0];
244		−
245		− AVHWFramesContext main_fc = (AVHWFramesContext)main_f->hw_frames_ctx->data;
246		− AVHWFramesContext overlay_fc = (AVHWFramesContext)overlay_f->hw_frames_ctx->data;
247		−
248		− const VkFormat *output_formats = av_vkfmt_from_pixfmt(s->vkctx.output_format);
249		− const VkFormat *main_sw_formats = av_vkfmt_from_pixfmt(main_fc->sw_format);
250		− const VkFormat *overlay_sw_formats = av_vkfmt_from_pixfmt(overlay_fc->sw_format);
251		−
252		− /* Update descriptors and init the exec context */
253		− ff_vk_start_exec_recording(vkctx, s->exec);
254		− cmd_buf = ff_vk_get_exec_buf(s->exec);
255		−
256		− for (int i = 0; i < planes; i++) {
257		− RET(ff_vk_create_imageview(vkctx, s->exec,
258		− &s->main_images[i].imageView, main->img[i],
259		− main_sw_formats[i],
260		− ff_comp_identity_map));
261		−
262		− RET(ff_vk_create_imageview(vkctx, s->exec,
263		− &s->overlay_images[i].imageView, overlay->img[i],
264		− overlay_sw_formats[i],
265		− ff_comp_identity_map));
266		−
267		− RET(ff_vk_create_imageview(vkctx, s->exec,
268		− &s->output_images[i].imageView, out->img[i],
269		− output_formats[i],
270		− ff_comp_identity_map));
271		−
272		− s->main_images[i].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
273		− s->overlay_images[i].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
274		− s->output_images[i].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
275		− }
276		−
277		− ff_vk_update_descriptor_set(vkctx, s->pl, 0);
278		−
279		− for (int i = 0; i < planes; i++) {
280		− VkImageMemoryBarrier bar[3] = {
281		− {
282		− .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
283		− .srcAccessMask = 0,
284		− .dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
285		− .oldLayout = main->layout[i],
286		− .newLayout = s->main_images[i].imageLayout,
287		− .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
288		− .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
289		− .image = main->img[i],
290		− .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
291		− .subresourceRange.levelCount = 1,
292		− .subresourceRange.layerCount = 1,
293		− },
294		− {
295		− .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
296		− .srcAccessMask = 0,
297		− .dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
298		− .oldLayout = overlay->layout[i],
299		− .newLayout = s->overlay_images[i].imageLayout,
300		− .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
301		− .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
302		− .image = overlay->img[i],
303		− .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
304		− .subresourceRange.levelCount = 1,
305		− .subresourceRange.layerCount = 1,
306		− },
307		− {
308		− .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
309		− .srcAccessMask = 0,
310		− .dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
311		− .oldLayout = out->layout[i],
312		− .newLayout = s->output_images[i].imageLayout,
313		− .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
314		− .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
315		− .image = out->img[i],
316		− .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
317		− .subresourceRange.levelCount = 1,
318		− .subresourceRange.layerCount = 1,
319		− },
320		− };
321		−
322		− vk->CmdPipelineBarrier(cmd_buf, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
323		− VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0,
324		− 0, NULL, 0, NULL, FF_ARRAY_ELEMS(bar), bar);
325		−
326		− main->layout[i] = bar[0].newLayout;
327		− main->access[i] = bar[0].dstAccessMask;
328		−
329		− overlay->layout[i] = bar[1].newLayout;
330		− overlay->access[i] = bar[1].dstAccessMask;
331		−
332		− out->layout[i] = bar[2].newLayout;
333		− out->access[i] = bar[2].dstAccessMask;
334		− }
335		−
336		− ff_vk_bind_pipeline_exec(vkctx, s->exec, s->pl);
337		−
338		− vk->CmdDispatch(cmd_buf,
339		− FFALIGN(s->vkctx.output_width, CGROUPS[0])/CGROUPS[0],
340		− FFALIGN(s->vkctx.output_height, CGROUPS[1])/CGROUPS[1], 1);
341		−
342		− ff_vk_add_exec_dep(vkctx, s->exec, main_f, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
343		− ff_vk_add_exec_dep(vkctx, s->exec, overlay_f, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
344		− ff_vk_add_exec_dep(vkctx, s->exec, out_f, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
345		−
346		− err = ff_vk_submit_exec_queue(vkctx, s->exec);
347		− if (err)
348		− return err;
	188	+ if (spv_opaque)
	189	+ spv->free_shader(spv, &spv_opaque);
	190	+ if (spv)
	191	+ spv->uninit(&spv);
349	192
350		− ff_vk_qf_rotate(&s->qf);
351		−
352		− return err;
353		−
354		−fail:
355		− ff_vk_discard_exec_deps(s->exec);
356	193	return err;
357	194	}
358	195
@@ -394,7 +231,9 @@ static int overlay_vulkan_blend(FFFrameSync *fs) · 35 unmodified lines
394	231	goto fail;
395	232	}
396	233
397		− RET(process_frames(ctx, out, input_main, input_overlay));
	234	+ RET(ff_vk_filter_process_Nin(&s->vkctx, &s->e, &s->pl,
	235	+ out, (AVFrame *[]){ input_main, input_overlay }, 2,
	236	+ s->sampler, &s->opts, sizeof(s->opts)));
398	237
399	238	err = av_frame_copy_props(out, input_main);
400	239	if (err < 0)
@@ -443,8 +282,17 @@ static av_cold int overlay_vulkan_init(AVFilterContext *avctx) · 42 unmodified lines
443	282	static void overlay_vulkan_uninit(AVFilterContext *avctx)
444	283	{
445	284	OverlayVulkanContext *s = avctx->priv;
	285	+ FFVulkanContext *vkctx = &s->vkctx;
	286	+ FFVulkanFunctions *vk = &vkctx->vkfn;
	287	+
	288	+ ff_vk_exec_pool_free(vkctx, &s->e);
	289	+ ff_vk_pipeline_free(vkctx, &s->pl);
	290	+ ff_vk_shader_free(vkctx, &s->shd);
	291	+
	292	+ if (s->sampler)
	293	+ vk->DestroySampler(vkctx->hwctx->act_dev, s->sampler,
	294	+ vkctx->hwctx->alloc);
446	295
447		− ff_vk_free_buf(&s->vkctx, &s->params_buf);
448	296	ff_vk_uninit(&s->vkctx);
449	297	ff_framesync_uninit(&s->fs);
450	298

overlay_vulkan: port for the rewrite

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code