History

avcodec/v4l2request: Add common decode support for hwaccels

Add common support for decoding using the V4L2 Request API. Basic flow for decoding follow the kernel Memory-to-memory Stateless Video Decoder Interface > Decoding [1]. A codec hwaccel typically handle decoding as follow: In start_frame next OUTPUT buffer and its related request object is picked from a circular queue and any codec specific CONTROLs is prepared. In decode_slice the slice bitstream data is appended to the OUTPUT buffer. In end_frame a CAPTURE buffer tied to the AVFrame is queued, it will be used as the decoding target by the driver / hw decoder. The prepared codec specific CONTROLs get queued as part of the request object. Finally the request object is submitted for decoding. For slice based hw decoders only the request for the final slice of the frame is submitted in end_frame, remaining is submitted in decode_slice. [1] https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/dev-stateless-decoder.html#decoding

Jonas Karlman committed 2 years ago

Commit 5ee83d61 parent d7445b46 files · +489 −3Browse files

Showing the changes to libavcodec (5 of 6 files). Show all

Unified · Split

5 files
  1. libavcodec/Makefile +1 −1
  2. libavcodec/hwconfig.h +2 −0
  3. libavcodec/v4l2_request.c +2 −1
  4. libavcodec/v4l2_request.h +23 −0
  5. libavcodec/v4l2_request_decode.c +459 −0
libavcodec/Makefile+1−1
@@ -183,7 +183,7 @@ OBJS-$(CONFIG_VP3DSP) += vp3dsp.o · 182 unmodified lines
183OBJS-$(CONFIG_VP56DSP) += vp56dsp.o183OBJS-$(CONFIG_VP56DSP) += vp56dsp.o
184OBJS-$(CONFIG_VP8DSP) += vp8dsp.o184OBJS-$(CONFIG_VP8DSP) += vp8dsp.o
185OBJS-$(CONFIG_V4L2_M2M) += v4l2_m2m.o v4l2_context.o v4l2_buffers.o v4l2_fmt.o185OBJS-$(CONFIG_V4L2_M2M) += v4l2_m2m.o v4l2_context.o v4l2_buffers.o v4l2_fmt.o
186OBJS-$(CONFIG_V4L2_REQUEST) += v4l2_request.o v4l2_request_probe.o186OBJS-$(CONFIG_V4L2_REQUEST) += v4l2_request.o v4l2_request_probe.o v4l2_request_decode.o
187OBJS-$(CONFIG_WMA_FREQS) += wma_freqs.o187OBJS-$(CONFIG_WMA_FREQS) += wma_freqs.o
188OBJS-$(CONFIG_WMV2DSP) += wmv2dsp.o188OBJS-$(CONFIG_WMV2DSP) += wmv2dsp.o
189 189
libavcodec/hwconfig.h+2−0
@@ -79,6 +79,8 @@ void ff_hwaccel_uninit(AVCodecContext *avctx); · 78 unmodified lines
79 HW_CONFIG_HWACCEL(0, 0, 1, D3D11VA_VLD, NONE, ff_ ## codec ## _d3d11va_hwaccel)79 HW_CONFIG_HWACCEL(0, 0, 1, D3D11VA_VLD, NONE, ff_ ## codec ## _d3d11va_hwaccel)
80#define HWACCEL_D3D12VA(codec) \80#define HWACCEL_D3D12VA(codec) \
81 HW_CONFIG_HWACCEL(1, 1, 0, D3D12, D3D12VA, ff_ ## codec ## _d3d12va_hwaccel)81 HW_CONFIG_HWACCEL(1, 1, 0, D3D12, D3D12VA, ff_ ## codec ## _d3d12va_hwaccel)
82#define HWACCEL_V4L2REQUEST(codec) \
83 HW_CONFIG_HWACCEL(1, 0, 0, DRM_PRIME, V4L2REQUEST, ff_ ## codec ## _v4l2request_hwaccel)
82 84
83#define HW_CONFIG_ENCODER(device, frames, ad_hoc, format, device_type_) \85#define HW_CONFIG_ENCODER(device, frames, ad_hoc, format, device_type_) \
84 &(const AVCodecHWConfigInternal) { \86 &(const AVCodecHWConfigInternal) { \
libavcodec/v4l2_request.c+2−1
@@ -303,7 +303,8 @@ int ff_v4l2_request_uninit(AVCodecContext *avctx) · 302 unmodified lines
303 V4L2RequestContext *ctx = v4l2_request_context(avctx);303 V4L2RequestContext *ctx = v4l2_request_context(avctx);
304 304
305 if (ctx->video_fd >= 0) {305 if (ctx->video_fd >= 0) {
306 // TODO: Flush and wait on all pending requests306 // Flush and wait on all pending requests
307 ff_v4l2_request_flush(avctx);
307 308
308 // Stop output queue309 // Stop output queue
309 if (ioctl(ctx->video_fd, VIDIOC_STREAMOFF, &ctx->output_type) < 0) {310 if (ioctl(ctx->video_fd, VIDIOC_STREAMOFF, &ctx->output_type) < 0) {
libavcodec/v4l2_request.h+23−0
@@ -81,4 +81,27 @@ int ff_v4l2_request_init(AVCodecContext *avctx, · 80 unmodified lines
81 uint32_t pixelformat, uint32_t buffersize,81 uint32_t pixelformat, uint32_t buffersize,
82 struct v4l2_ext_control *control, int count);82 struct v4l2_ext_control *control, int count);
83 83
84uint64_t ff_v4l2_request_get_capture_timestamp(AVFrame *frame);
85
86int ff_v4l2_request_append_output(AVCodecContext *avctx,
87 V4L2RequestPictureContext *pic,
88 const uint8_t *data, uint32_t size);
89
90int ff_v4l2_request_decode_slice(AVCodecContext *avctx,
91 V4L2RequestPictureContext *pic,
92 struct v4l2_ext_control *control, int count,
93 bool first_slice, bool last_slice);
94
95int ff_v4l2_request_decode_frame(AVCodecContext *avctx,
96 V4L2RequestPictureContext *pic,
97 struct v4l2_ext_control *control, int count);
98
99int ff_v4l2_request_reset_picture(AVCodecContext *avctx,
100 V4L2RequestPictureContext *pic);
101
102int ff_v4l2_request_start_frame(AVCodecContext *avctx,
103 V4L2RequestPictureContext *pic, AVFrame *frame);
104
105void ff_v4l2_request_flush(AVCodecContext *avctx);
106
84#endif /* AVCODEC_V4L2_REQUEST_H */107#endif /* AVCODEC_V4L2_REQUEST_H */
libavcodec/v4l2_request_decode.c+459−0
@@ -0,0 +1,459 @@
1/*
2 * This file is part of FFmpeg.
3 *
4 * FFmpeg is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * FFmpeg is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with FFmpeg; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19#include "config.h"
20
21#include <poll.h>
22#include <sys/ioctl.h>
23
24#include "decode.h"
25#include "v4l2_request_internal.h"
26
27#define INPUT_BUFFER_PADDING_SIZE (AV_INPUT_BUFFER_PADDING_SIZE * 4)
28
29uint64_t ff_v4l2_request_get_capture_timestamp(AVFrame *frame)
30{
31 V4L2RequestFrameDescriptor *desc = v4l2_request_framedesc(frame);
32
33 /*
34 * The capture buffer index is used as a base for V4L2 frame reference.
35 * This works because frames are decoded into a capture buffer that is
36 * closely tied to an AVFrame.
37 */
38 return desc ? v4l2_timeval_to_ns(&desc->capture.buffer.timestamp) : 0;
39}
40
41static int v4l2_request_queue_buffer(V4L2RequestContext *ctx, int request_fd,
42 V4L2RequestBuffer *buf, uint32_t flags)
43{
44 struct v4l2_plane planes[1] = {};
45 struct v4l2_buffer buffer = {
46 .index = buf->index,
47 .type = buf->buffer.type,
48 .memory = buf->buffer.memory,
49 .timestamp = buf->buffer.timestamp,
50 .bytesused = buf->used,
51 .request_fd = request_fd,
52 .flags = ((request_fd >= 0) ? V4L2_BUF_FLAG_REQUEST_FD : 0) | flags,
53 };
54
55 if (V4L2_TYPE_IS_MULTIPLANAR(buffer.type)) {
56 planes[0].bytesused = buf->used;
57 buffer.bytesused = 0;
58 buffer.length = 1;
59 buffer.m.planes = planes;
60 }
61
62 // Queue the buffer
63 if (ioctl(ctx->video_fd, VIDIOC_QBUF, &buffer) < 0)
64 return AVERROR(errno);
65
66 // Mark the buffer as queued
67 if (V4L2_TYPE_IS_OUTPUT(buffer.type))
68 atomic_fetch_or(&ctx->queued_output, 1 << buffer.index);
69 else
70 atomic_fetch_or(&ctx->queued_capture, 1 << buffer.index);
71
72 return 0;
73}
74
75static int v4l2_request_dequeue_buffer(V4L2RequestContext *ctx,
76 enum v4l2_buf_type type)
77{
78 struct v4l2_plane planes[1] = {};
79 struct v4l2_buffer buffer = {
80 .type = type,
81 .memory = V4L2_MEMORY_MMAP,
82 };
83
84 if (V4L2_TYPE_IS_MULTIPLANAR(buffer.type)) {
85 buffer.length = 1;
86 buffer.m.planes = planes;
87 }
88
89 // Dequeue next completed buffer
90 if (ioctl(ctx->video_fd, VIDIOC_DQBUF, &buffer) < 0)
91 return AVERROR(errno);
92
93 // Mark the buffer as dequeued
94 if (V4L2_TYPE_IS_OUTPUT(buffer.type))
95 atomic_fetch_and(&ctx->queued_output, ~(1 << buffer.index));
96 else
97 atomic_fetch_and(&ctx->queued_capture, ~(1 << buffer.index));
98
99 return 0;
100}
101
102static inline int v4l2_request_dequeue_completed_buffers(V4L2RequestContext *ctx,
103 enum v4l2_buf_type type)
104{
105 int ret;
106
107 do {
108 ret = v4l2_request_dequeue_buffer(ctx, type);
109 } while (!ret);
110
111 return ret;
112}
113
114static int v4l2_request_wait_on_capture(V4L2RequestContext *ctx,
115 V4L2RequestBuffer *capture)
116{
117 struct pollfd pollfd = {
118 .fd = ctx->video_fd,
119 .events = POLLIN,
120 };
121
122 ff_mutex_lock(&ctx->mutex);
123
124 // Dequeue all completed capture buffers
125 if (atomic_load(&ctx->queued_capture))
126 v4l2_request_dequeue_completed_buffers(ctx, ctx->format.type);
127
128 // Wait on the specific capture buffer, when needed
129 while (atomic_load(&ctx->queued_capture) & (1 << capture->index)) {
130 int ret = poll(&pollfd, 1, 2000);
131 if (ret <= 0)
132 goto fail;
133
134 ret = v4l2_request_dequeue_buffer(ctx, ctx->format.type);
135 if (ret < 0 && ret != AVERROR(EAGAIN))
136 goto fail;
137 }
138
139 ff_mutex_unlock(&ctx->mutex);
140 return 0;
141
142fail:
143 ff_mutex_unlock(&ctx->mutex);
144 av_log(ctx, AV_LOG_ERROR, "Failed waiting on capture buffer %d\n",
145 capture->index);
146 return AVERROR(EINVAL);
147}
148
149static V4L2RequestBuffer *v4l2_request_next_output(V4L2RequestContext *ctx)
150{
151 int index;
152 V4L2RequestBuffer *output;
153 struct pollfd pollfd = {
154 .fd = ctx->video_fd,
155 .events = POLLOUT,
156 };
157
158 ff_mutex_lock(&ctx->mutex);
159
160 // Use next output buffer in the circular queue
161 index = atomic_load(&ctx->next_output);
162 output = &ctx->output[index];
163 atomic_store(&ctx->next_output, (index + 1) % FF_ARRAY_ELEMS(ctx->output));
164
165 // Dequeue all completed output buffers
166 if (atomic_load(&ctx->queued_output))
167 v4l2_request_dequeue_completed_buffers(ctx, ctx->output_type);
168
169 // Wait on the specific output buffer, when needed
170 while (atomic_load(&ctx->queued_output) & (1 << output->index)) {
171 int ret = poll(&pollfd, 1, 2000);
172 if (ret <= 0)
173 goto fail;
174
175 ret = v4l2_request_dequeue_buffer(ctx, ctx->output_type);
176 if (ret < 0 && ret != AVERROR(EAGAIN))
177 goto fail;
178 }
179
180 ff_mutex_unlock(&ctx->mutex);
181
182 // Reset used state
183 output->used = 0;
184
185 return output;
186
187fail:
188 ff_mutex_unlock(&ctx->mutex);
189 av_log(ctx, AV_LOG_ERROR, "Failed waiting on output buffer %d\n",
190 output->index);
191 return NULL;
192}
193
194static int v4l2_request_wait_on_request(V4L2RequestContext *ctx,
195 V4L2RequestBuffer *output)
196{
197 struct pollfd pollfd = {
198 .fd = output->fd,
199 .events = POLLPRI,
200 };
201
202 // Wait on the specific request to complete, when needed
203 while (atomic_load(&ctx->queued_request) & (1 << output->index)) {
204 int ret = poll(&pollfd, 1, 2000);
205 if (ret <= 0)
206 break;
207
208 // Mark request as dequeued
209 if (pollfd.revents & (POLLPRI | POLLERR)) {
210 atomic_fetch_and(&ctx->queued_request, ~(1 << output->index));
211 break;
212 }
213 }
214
215 // Reinit the request object
216 if (ioctl(output->fd, MEDIA_REQUEST_IOC_REINIT, NULL) < 0) {
217 av_log(ctx, AV_LOG_ERROR, "Failed to reinit request object %d: %s (%d)\n",
218 output->fd, strerror(errno), errno);
219 return AVERROR(errno);
220 }
221
222 // Ensure request is marked as dequeued
223 atomic_fetch_and(&ctx->queued_request, ~(1 << output->index));
224
225 return 0;
226}
227
228int ff_v4l2_request_append_output(AVCodecContext *avctx,
229 V4L2RequestPictureContext *pic,
230 const uint8_t *data, uint32_t size)
231{
232 V4L2RequestContext *ctx = v4l2_request_context(avctx);
233
234 // Append data to output buffer and ensure there is enough space for padding
235 if (pic->output->used + size + INPUT_BUFFER_PADDING_SIZE <= pic->output->size) {
236 memcpy(pic->output->addr + pic->output->used, data, size);
237 pic->output->used += size;
238 return 0;
239 } else {
240 av_log(ctx, AV_LOG_ERROR,
241 "Failed to append %u bytes data to output buffer %d (%u of %u used)\n",
242 size, pic->output->index, pic->output->used, pic->output->size);
243 return AVERROR(ENOMEM);
244 }
245}
246
247static int v4l2_request_queue_decode(AVCodecContext *avctx,
248 V4L2RequestPictureContext *pic,
249 struct v4l2_ext_control *control, int count,
250 bool first_slice, bool last_slice)
251{
252 V4L2RequestContext *ctx = v4l2_request_context(avctx);
253 uint32_t flags;
254 int ret;
255
256 if (first_slice) {
257 /*
258 * Wait on dequeue of the target capture buffer, when needed. Otherwise
259 * V4L2 decoder may use a different capture buffer than hwaccel expects.
260 *
261 * Normally decoding has already completed when a capture buffer is
262 * reused so this is more or less a no-op, however in some situations
263 * FFmpeg may reuse an AVFrame early, i.e. when no output frame was
264 * produced prior time, and a syncronization is necessary.
265 */
266 ret = v4l2_request_wait_on_capture(ctx, pic->capture);
267 if (ret < 0)
268 return ret;
269 }
270
271 ff_mutex_lock(&ctx->mutex);
272
273 /*
274 * The output buffer tied to prior use of current request object can
275 * independently be dequeued before the full decode request has been
276 * completed. This may happen when a decoder use multi stage decoding,
277 * e.g. rpivid. In such case we can start reusing the output buffer,
278 * however we must wait on the prior request to fully complete before we
279 * can reuse the request object, and a syncronization is necessary.
280 */
281 ret = v4l2_request_wait_on_request(ctx, pic->output);
282 if (ret < 0)
283 goto fail;
284
285 /*
286 * Dequeue any completed output buffers, this is strictly not necessary,
287 * however if a syncronization was necessary for the capture and/or request
288 * there is more than likely one or more output buffers that can be dequeued.
289 */
290 if (atomic_load(&ctx->queued_output))
291 v4l2_request_dequeue_completed_buffers(ctx, ctx->output_type);
292
293 // Set codec controls for current request
294 ret = ff_v4l2_request_set_request_controls(ctx, pic->output->fd, control, count);
295 if (ret < 0) {
296 av_log(ctx, AV_LOG_ERROR, "Failed to set %d control(s) for request %d: %s (%d)\n",
297 count, pic->output->fd, strerror(errno), errno);
298 goto fail;
299 }
300
301 // Ensure there is zero padding at the end of bitstream data
302 memset(pic->output->addr + pic->output->used, 0, INPUT_BUFFER_PADDING_SIZE);
303
304 // Use timestamp of the capture buffer for V4L2 frame reference
305 pic->output->buffer.timestamp = pic->capture->buffer.timestamp;
306
307 /*
308 * Queue the output buffer of current request. The capture buffer may be
309 * hold by the V4L2 decoder unless this is the last slice of a frame.
310 */
311 flags = last_slice ? 0 : V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
312 ret = v4l2_request_queue_buffer(ctx, pic->output->fd, pic->output, flags);
313 if (ret < 0) {
314 av_log(ctx, AV_LOG_ERROR, "Failed to queue output buffer %d for request %d: %s (%d)\n",
315 pic->output->index, pic->output->fd, strerror(errno), errno);
316 ret = AVERROR(errno);
317 goto fail;
318 }
319
320 if (first_slice) {
321 /*
322 * Queue the target capture buffer, hwaccel expect and depend on that
323 * this specific capture buffer will be used as decode target for
324 * current request, otherwise frames may be output in wrong order or
325 * wrong capture buffer could get used as a reference frame.
326 */
327 ret = v4l2_request_queue_buffer(ctx, -1, pic->capture, 0);
328 if (ret < 0) {
329 av_log(ctx, AV_LOG_ERROR, "Failed to queue capture buffer %d for request %d: %s (%d)\n",
330 pic->capture->index, pic->output->fd, strerror(errno), errno);
331 ret = AVERROR(errno);
332 goto fail;
333 }
334 }
335
336 // Queue current request
337 ret = ioctl(pic->output->fd, MEDIA_REQUEST_IOC_QUEUE, NULL);
338 if (ret < 0) {
339 av_log(ctx, AV_LOG_ERROR, "Failed to queue request object %d: %s (%d)\n",
340 pic->output->fd, strerror(errno), errno);
341 ret = AVERROR(errno);
342 goto fail;
343 }
344
345 // Mark current request as queued
346 atomic_fetch_or(&ctx->queued_request, 1 << pic->output->index);
347
348 ret = 0;
349fail:
350 ff_mutex_unlock(&ctx->mutex);
351 return ret;
352}
353
354int ff_v4l2_request_decode_slice(AVCodecContext *avctx,
355 V4L2RequestPictureContext *pic,
356 struct v4l2_ext_control *control, int count,
357 bool first_slice, bool last_slice)
358{
359 /*
360 * Fallback to queue each slice as a full frame when holding capture
361 * buffers is not supported by the driver.
362 */
363 if ((pic->output->capabilities & V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF) !=
364 V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF)
365 return v4l2_request_queue_decode(avctx, pic, control, count, true, true);
366
367 return v4l2_request_queue_decode(avctx, pic, control, count,
368 first_slice, last_slice);
369}
370
371int ff_v4l2_request_decode_frame(AVCodecContext *avctx,
372 V4L2RequestPictureContext *pic,
373 struct v4l2_ext_control *control, int count)
374{
375 return v4l2_request_queue_decode(avctx, pic, control, count, true, true);
376}
377
378static int v4l2_request_post_process(void *logctx, AVFrame *frame)
379{
380 V4L2RequestFrameDescriptor *desc = v4l2_request_framedesc(frame);
381 FrameDecodeData *fdd = (FrameDecodeData*)frame->private_ref->data;
382 V4L2RequestContext *ctx = fdd->hwaccel_priv;
383
384 // Wait on capture buffer before returning the frame to application
385 return v4l2_request_wait_on_capture(ctx, &desc->capture);
386}
387
388int ff_v4l2_request_reset_picture(AVCodecContext *avctx, V4L2RequestPictureContext *pic)
389{
390 V4L2RequestContext *ctx = v4l2_request_context(avctx);
391
392 // Get and wait on next output buffer from circular queue
393 pic->output = v4l2_request_next_output(ctx);
394 if (!pic->output)
395 return AVERROR(EINVAL);
396
397 return 0;
398}
399
400int ff_v4l2_request_start_frame(AVCodecContext *avctx,
401 V4L2RequestPictureContext *pic,
402 AVFrame *frame)
403{
404 V4L2RequestContext *ctx = v4l2_request_context(avctx);
405 V4L2RequestFrameDescriptor *desc = v4l2_request_framedesc(frame);
406 FrameDecodeData *fdd = (FrameDecodeData*)frame->private_ref->data;
407 int ret;
408
409 // Get next output buffer from circular queue
410 ret = ff_v4l2_request_reset_picture(avctx, pic);
411 if (ret)
412 return ret;
413
414 // Ensure capture buffer is dequeued before reuse
415 ret = v4l2_request_wait_on_capture(ctx, &desc->capture);
416 if (ret)
417 return ret;
418
419 // Wait on capture buffer in post_process() before returning to application
420 fdd->hwaccel_priv = ctx;
421 fdd->post_process = v4l2_request_post_process;
422
423 // Capture buffer used for current frame
424 pic->capture = &desc->capture;
425
426 return 0;
427}
428
429void ff_v4l2_request_flush(AVCodecContext *avctx)
430{
431 V4L2RequestContext *ctx = v4l2_request_context(avctx);
432 struct pollfd pollfd = {
433 .fd = ctx->video_fd,
434 .events = POLLOUT,
435 };
436
437 ff_mutex_lock(&ctx->mutex);
438
439 // Dequeue all completed output buffers
440 if (atomic_load(&ctx->queued_output))
441 v4l2_request_dequeue_completed_buffers(ctx, ctx->output_type);
442
443 // Wait on any remaining output buffer
444 while (atomic_load(&ctx->queued_output)) {
445 int ret = poll(&pollfd, 1, 2000);
446 if (ret <= 0)
447 break;
448
449 ret = v4l2_request_dequeue_buffer(ctx, ctx->output_type);
450 if (ret < 0 && ret != AVERROR(EAGAIN))
451 break;
452 }
453
454 // Dequeue all completed capture buffers
455 if (atomic_load(&ctx->queued_capture))
456 v4l2_request_dequeue_completed_buffers(ctx, ctx->format.type);
457
458 ff_mutex_unlock(&ctx->mutex);
459}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t