History

avcodec/v4l2request: Add common decode support for hwaccels

Add common support for decoding using the V4L2 Request API. Basic flow for decoding follow the kernel Memory-to-memory Stateless Video Decoder Interface > Decoding [1]. A codec hwaccel typically handle decoding as follow: In start_frame next OUTPUT buffer and its related request object is picked from a circular queue and any codec specific CONTROLs is prepared. In decode_slice the slice bitstream data is appended to the OUTPUT buffer. In end_frame a CAPTURE buffer tied to the AVFrame is queued, it will be used as the decoding target by the driver / hw decoder. The prepared codec specific CONTROLs get queued as part of the request object. Finally the request object is submitted for decoding. For slice based hw decoders only the request for the final slice of the frame is submitted in end_frame, remaining is submitted in decode_slice. [1] https://www.kernel.org/doc/html/latest/userspace-api/media/v4l/dev-stateless-decoder.html#decoding

Jonas Karlman committed 2 years ago

Commit 5ee83d61 parent d7445b46 files · +489 −3Browse files

Showing the changes to libavcodec (5 of 6 files). Show all

Unified · Split

5 files
  1. libavcodec/Makefile +1 −1
  2. libavcodec/hwconfig.h +2 −0
  3. libavcodec/v4l2_request.c +2 −1
  4. libavcodec/v4l2_request.h +23 −0
  5. libavcodec/v4l2_request_decode.c +459 −0
libavcodec/Makefile+1−1
@@ -183,7 +183,7 @@ OBJS-$(CONFIG_VP3DSP) += vp3dsp.o · 182 unmodified lines
183183 OBJS-$(CONFIG_VP56DSP) += vp56dsp.o
184184 OBJS-$(CONFIG_VP8DSP) += vp8dsp.o
185185 OBJS-$(CONFIG_V4L2_M2M) += v4l2_m2m.o v4l2_context.o v4l2_buffers.o v4l2_fmt.o
186−OBJS-$(CONFIG_V4L2_REQUEST) += v4l2_request.o v4l2_request_probe.o
186+OBJS-$(CONFIG_V4L2_REQUEST) += v4l2_request.o v4l2_request_probe.o v4l2_request_decode.o
187187 OBJS-$(CONFIG_WMA_FREQS) += wma_freqs.o
188188 OBJS-$(CONFIG_WMV2DSP) += wmv2dsp.o
189189
libavcodec/hwconfig.h+2−0
@@ -79,6 +79,8 @@ void ff_hwaccel_uninit(AVCodecContext *avctx); · 78 unmodified lines
7979 HW_CONFIG_HWACCEL(0, 0, 1, D3D11VA_VLD, NONE, ff_ ## codec ## _d3d11va_hwaccel)
8080 #define HWACCEL_D3D12VA(codec) \
8181 HW_CONFIG_HWACCEL(1, 1, 0, D3D12, D3D12VA, ff_ ## codec ## _d3d12va_hwaccel)
82+#define HWACCEL_V4L2REQUEST(codec) \
83+ HW_CONFIG_HWACCEL(1, 0, 0, DRM_PRIME, V4L2REQUEST, ff_ ## codec ## _v4l2request_hwaccel)
8284
8385 #define HW_CONFIG_ENCODER(device, frames, ad_hoc, format, device_type_) \
8486 &(const AVCodecHWConfigInternal) { \
libavcodec/v4l2_request.c+2−1
@@ -303,7 +303,8 @@ int ff_v4l2_request_uninit(AVCodecContext *avctx) · 302 unmodified lines
303303 V4L2RequestContext *ctx = v4l2_request_context(avctx);
304304
305305 if (ctx->video_fd >= 0) {
306− // TODO: Flush and wait on all pending requests
306+ // Flush and wait on all pending requests
307+ ff_v4l2_request_flush(avctx);
307308
308309 // Stop output queue
309310 if (ioctl(ctx->video_fd, VIDIOC_STREAMOFF, &ctx->output_type) < 0) {
libavcodec/v4l2_request.h+23−0
@@ -81,4 +81,27 @@ int ff_v4l2_request_init(AVCodecContext *avctx, · 80 unmodified lines
8181 uint32_t pixelformat, uint32_t buffersize,
8282 struct v4l2_ext_control *control, int count);
8383
84+uint64_t ff_v4l2_request_get_capture_timestamp(AVFrame *frame);
85+
86+int ff_v4l2_request_append_output(AVCodecContext *avctx,
87+ V4L2RequestPictureContext *pic,
88+ const uint8_t *data, uint32_t size);
89+
90+int ff_v4l2_request_decode_slice(AVCodecContext *avctx,
91+ V4L2RequestPictureContext *pic,
92+ struct v4l2_ext_control *control, int count,
93+ bool first_slice, bool last_slice);
94+
95+int ff_v4l2_request_decode_frame(AVCodecContext *avctx,
96+ V4L2RequestPictureContext *pic,
97+ struct v4l2_ext_control *control, int count);
98+
99+int ff_v4l2_request_reset_picture(AVCodecContext *avctx,
100+ V4L2RequestPictureContext *pic);
101+
102+int ff_v4l2_request_start_frame(AVCodecContext *avctx,
103+ V4L2RequestPictureContext *pic, AVFrame *frame);
104+
105+void ff_v4l2_request_flush(AVCodecContext *avctx);
106+
84107 #endif /* AVCODEC_V4L2_REQUEST_H */
libavcodec/v4l2_request_decode.c+459−0
@@ -0,0 +1,459 @@
1+/*
2+ * This file is part of FFmpeg.
3+ *
4+ * FFmpeg is free software; you can redistribute it and/or
5+ * modify it under the terms of the GNU Lesser General Public
6+ * License as published by the Free Software Foundation; either
7+ * version 2.1 of the License, or (at your option) any later version.
8+ *
9+ * FFmpeg is distributed in the hope that it will be useful,
10+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
11+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12+ * Lesser General Public License for more details.
13+ *
14+ * You should have received a copy of the GNU Lesser General Public
15+ * License along with FFmpeg; if not, write to the Free Software
16+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17+ */
18+
19+#include "config.h"
20+
21+#include <poll.h>
22+#include <sys/ioctl.h>
23+
24+#include "decode.h"
25+#include "v4l2_request_internal.h"
26+
27+#define INPUT_BUFFER_PADDING_SIZE (AV_INPUT_BUFFER_PADDING_SIZE * 4)
28+
29+uint64_t ff_v4l2_request_get_capture_timestamp(AVFrame *frame)
30+{
31+ V4L2RequestFrameDescriptor *desc = v4l2_request_framedesc(frame);
32+
33+ /*
34+ * The capture buffer index is used as a base for V4L2 frame reference.
35+ * This works because frames are decoded into a capture buffer that is
36+ * closely tied to an AVFrame.
37+ */
38+ return desc ? v4l2_timeval_to_ns(&desc->capture.buffer.timestamp) : 0;
39+}
40+
41+static int v4l2_request_queue_buffer(V4L2RequestContext *ctx, int request_fd,
42+ V4L2RequestBuffer *buf, uint32_t flags)
43+{
44+ struct v4l2_plane planes[1] = {};
45+ struct v4l2_buffer buffer = {
46+ .index = buf->index,
47+ .type = buf->buffer.type,
48+ .memory = buf->buffer.memory,
49+ .timestamp = buf->buffer.timestamp,
50+ .bytesused = buf->used,
51+ .request_fd = request_fd,
52+ .flags = ((request_fd >= 0) ? V4L2_BUF_FLAG_REQUEST_FD : 0) | flags,
53+ };
54+
55+ if (V4L2_TYPE_IS_MULTIPLANAR(buffer.type)) {
56+ planes[0].bytesused = buf->used;
57+ buffer.bytesused = 0;
58+ buffer.length = 1;
59+ buffer.m.planes = planes;
60+ }
61+
62+ // Queue the buffer
63+ if (ioctl(ctx->video_fd, VIDIOC_QBUF, &buffer) < 0)
64+ return AVERROR(errno);
65+
66+ // Mark the buffer as queued
67+ if (V4L2_TYPE_IS_OUTPUT(buffer.type))
68+ atomic_fetch_or(&ctx->queued_output, 1 << buffer.index);
69+ else
70+ atomic_fetch_or(&ctx->queued_capture, 1 << buffer.index);
71+
72+ return 0;
73+}
74+
75+static int v4l2_request_dequeue_buffer(V4L2RequestContext *ctx,
76+ enum v4l2_buf_type type)
77+{
78+ struct v4l2_plane planes[1] = {};
79+ struct v4l2_buffer buffer = {
80+ .type = type,
81+ .memory = V4L2_MEMORY_MMAP,
82+ };
83+
84+ if (V4L2_TYPE_IS_MULTIPLANAR(buffer.type)) {
85+ buffer.length = 1;
86+ buffer.m.planes = planes;
87+ }
88+
89+ // Dequeue next completed buffer
90+ if (ioctl(ctx->video_fd, VIDIOC_DQBUF, &buffer) < 0)
91+ return AVERROR(errno);
92+
93+ // Mark the buffer as dequeued
94+ if (V4L2_TYPE_IS_OUTPUT(buffer.type))
95+ atomic_fetch_and(&ctx->queued_output, ~(1 << buffer.index));
96+ else
97+ atomic_fetch_and(&ctx->queued_capture, ~(1 << buffer.index));
98+
99+ return 0;
100+}
101+
102+static inline int v4l2_request_dequeue_completed_buffers(V4L2RequestContext *ctx,
103+ enum v4l2_buf_type type)
104+{
105+ int ret;
106+
107+ do {
108+ ret = v4l2_request_dequeue_buffer(ctx, type);
109+ } while (!ret);
110+
111+ return ret;
112+}
113+
114+static int v4l2_request_wait_on_capture(V4L2RequestContext *ctx,
115+ V4L2RequestBuffer *capture)
116+{
117+ struct pollfd pollfd = {
118+ .fd = ctx->video_fd,
119+ .events = POLLIN,
120+ };
121+
122+ ff_mutex_lock(&ctx->mutex);
123+
124+ // Dequeue all completed capture buffers
125+ if (atomic_load(&ctx->queued_capture))
126+ v4l2_request_dequeue_completed_buffers(ctx, ctx->format.type);
127+
128+ // Wait on the specific capture buffer, when needed
129+ while (atomic_load(&ctx->queued_capture) & (1 << capture->index)) {
130+ int ret = poll(&pollfd, 1, 2000);
131+ if (ret <= 0)
132+ goto fail;
133+
134+ ret = v4l2_request_dequeue_buffer(ctx, ctx->format.type);
135+ if (ret < 0 && ret != AVERROR(EAGAIN))
136+ goto fail;
137+ }
138+
139+ ff_mutex_unlock(&ctx->mutex);
140+ return 0;
141+
142+fail:
143+ ff_mutex_unlock(&ctx->mutex);
144+ av_log(ctx, AV_LOG_ERROR, "Failed waiting on capture buffer %d\n",
145+ capture->index);
146+ return AVERROR(EINVAL);
147+}
148+
149+static V4L2RequestBuffer *v4l2_request_next_output(V4L2RequestContext *ctx)
150+{
151+ int index;
152+ V4L2RequestBuffer *output;
153+ struct pollfd pollfd = {
154+ .fd = ctx->video_fd,
155+ .events = POLLOUT,
156+ };
157+
158+ ff_mutex_lock(&ctx->mutex);
159+
160+ // Use next output buffer in the circular queue
161+ index = atomic_load(&ctx->next_output);
162+ output = &ctx->output[index];
163+ atomic_store(&ctx->next_output, (index + 1) % FF_ARRAY_ELEMS(ctx->output));
164+
165+ // Dequeue all completed output buffers
166+ if (atomic_load(&ctx->queued_output))
167+ v4l2_request_dequeue_completed_buffers(ctx, ctx->output_type);
168+
169+ // Wait on the specific output buffer, when needed
170+ while (atomic_load(&ctx->queued_output) & (1 << output->index)) {
171+ int ret = poll(&pollfd, 1, 2000);
172+ if (ret <= 0)
173+ goto fail;
174+
175+ ret = v4l2_request_dequeue_buffer(ctx, ctx->output_type);
176+ if (ret < 0 && ret != AVERROR(EAGAIN))
177+ goto fail;
178+ }
179+
180+ ff_mutex_unlock(&ctx->mutex);
181+
182+ // Reset used state
183+ output->used = 0;
184+
185+ return output;
186+
187+fail:
188+ ff_mutex_unlock(&ctx->mutex);
189+ av_log(ctx, AV_LOG_ERROR, "Failed waiting on output buffer %d\n",
190+ output->index);
191+ return NULL;
192+}
193+
194+static int v4l2_request_wait_on_request(V4L2RequestContext *ctx,
195+ V4L2RequestBuffer *output)
196+{
197+ struct pollfd pollfd = {
198+ .fd = output->fd,
199+ .events = POLLPRI,
200+ };
201+
202+ // Wait on the specific request to complete, when needed
203+ while (atomic_load(&ctx->queued_request) & (1 << output->index)) {
204+ int ret = poll(&pollfd, 1, 2000);
205+ if (ret <= 0)
206+ break;
207+
208+ // Mark request as dequeued
209+ if (pollfd.revents & (POLLPRI | POLLERR)) {
210+ atomic_fetch_and(&ctx->queued_request, ~(1 << output->index));
211+ break;
212+ }
213+ }
214+
215+ // Reinit the request object
216+ if (ioctl(output->fd, MEDIA_REQUEST_IOC_REINIT, NULL) < 0) {
217+ av_log(ctx, AV_LOG_ERROR, "Failed to reinit request object %d: %s (%d)\n",
218+ output->fd, strerror(errno), errno);
219+ return AVERROR(errno);
220+ }
221+
222+ // Ensure request is marked as dequeued
223+ atomic_fetch_and(&ctx->queued_request, ~(1 << output->index));
224+
225+ return 0;
226+}
227+
228+int ff_v4l2_request_append_output(AVCodecContext *avctx,
229+ V4L2RequestPictureContext *pic,
230+ const uint8_t *data, uint32_t size)
231+{
232+ V4L2RequestContext *ctx = v4l2_request_context(avctx);
233+
234+ // Append data to output buffer and ensure there is enough space for padding
235+ if (pic->output->used + size + INPUT_BUFFER_PADDING_SIZE <= pic->output->size) {
236+ memcpy(pic->output->addr + pic->output->used, data, size);
237+ pic->output->used += size;
238+ return 0;
239+ } else {
240+ av_log(ctx, AV_LOG_ERROR,
241+ "Failed to append %u bytes data to output buffer %d (%u of %u used)\n",
242+ size, pic->output->index, pic->output->used, pic->output->size);
243+ return AVERROR(ENOMEM);
244+ }
245+}
246+
247+static int v4l2_request_queue_decode(AVCodecContext *avctx,
248+ V4L2RequestPictureContext *pic,
249+ struct v4l2_ext_control *control, int count,
250+ bool first_slice, bool last_slice)
251+{
252+ V4L2RequestContext *ctx = v4l2_request_context(avctx);
253+ uint32_t flags;
254+ int ret;
255+
256+ if (first_slice) {
257+ /*
258+ * Wait on dequeue of the target capture buffer, when needed. Otherwise
259+ * V4L2 decoder may use a different capture buffer than hwaccel expects.
260+ *
261+ * Normally decoding has already completed when a capture buffer is
262+ * reused so this is more or less a no-op, however in some situations
263+ * FFmpeg may reuse an AVFrame early, i.e. when no output frame was
264+ * produced prior time, and a syncronization is necessary.
265+ */
266+ ret = v4l2_request_wait_on_capture(ctx, pic->capture);
267+ if (ret < 0)
268+ return ret;
269+ }
270+
271+ ff_mutex_lock(&ctx->mutex);
272+
273+ /*
274+ * The output buffer tied to prior use of current request object can
275+ * independently be dequeued before the full decode request has been
276+ * completed. This may happen when a decoder use multi stage decoding,
277+ * e.g. rpivid. In such case we can start reusing the output buffer,
278+ * however we must wait on the prior request to fully complete before we
279+ * can reuse the request object, and a syncronization is necessary.
280+ */
281+ ret = v4l2_request_wait_on_request(ctx, pic->output);
282+ if (ret < 0)
283+ goto fail;
284+
285+ /*
286+ * Dequeue any completed output buffers, this is strictly not necessary,
287+ * however if a syncronization was necessary for the capture and/or request
288+ * there is more than likely one or more output buffers that can be dequeued.
289+ */
290+ if (atomic_load(&ctx->queued_output))
291+ v4l2_request_dequeue_completed_buffers(ctx, ctx->output_type);
292+
293+ // Set codec controls for current request
294+ ret = ff_v4l2_request_set_request_controls(ctx, pic->output->fd, control, count);
295+ if (ret < 0) {
296+ av_log(ctx, AV_LOG_ERROR, "Failed to set %d control(s) for request %d: %s (%d)\n",
297+ count, pic->output->fd, strerror(errno), errno);
298+ goto fail;
299+ }
300+
301+ // Ensure there is zero padding at the end of bitstream data
302+ memset(pic->output->addr + pic->output->used, 0, INPUT_BUFFER_PADDING_SIZE);
303+
304+ // Use timestamp of the capture buffer for V4L2 frame reference
305+ pic->output->buffer.timestamp = pic->capture->buffer.timestamp;
306+
307+ /*
308+ * Queue the output buffer of current request. The capture buffer may be
309+ * hold by the V4L2 decoder unless this is the last slice of a frame.
310+ */
311+ flags = last_slice ? 0 : V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
312+ ret = v4l2_request_queue_buffer(ctx, pic->output->fd, pic->output, flags);
313+ if (ret < 0) {
314+ av_log(ctx, AV_LOG_ERROR, "Failed to queue output buffer %d for request %d: %s (%d)\n",
315+ pic->output->index, pic->output->fd, strerror(errno), errno);
316+ ret = AVERROR(errno);
317+ goto fail;
318+ }
319+
320+ if (first_slice) {
321+ /*
322+ * Queue the target capture buffer, hwaccel expect and depend on that
323+ * this specific capture buffer will be used as decode target for
324+ * current request, otherwise frames may be output in wrong order or
325+ * wrong capture buffer could get used as a reference frame.
326+ */
327+ ret = v4l2_request_queue_buffer(ctx, -1, pic->capture, 0);
328+ if (ret < 0) {
329+ av_log(ctx, AV_LOG_ERROR, "Failed to queue capture buffer %d for request %d: %s (%d)\n",
330+ pic->capture->index, pic->output->fd, strerror(errno), errno);
331+ ret = AVERROR(errno);
332+ goto fail;
333+ }
334+ }
335+
336+ // Queue current request
337+ ret = ioctl(pic->output->fd, MEDIA_REQUEST_IOC_QUEUE, NULL);
338+ if (ret < 0) {
339+ av_log(ctx, AV_LOG_ERROR, "Failed to queue request object %d: %s (%d)\n",
340+ pic->output->fd, strerror(errno), errno);
341+ ret = AVERROR(errno);
342+ goto fail;
343+ }
344+
345+ // Mark current request as queued
346+ atomic_fetch_or(&ctx->queued_request, 1 << pic->output->index);
347+
348+ ret = 0;
349+fail:
350+ ff_mutex_unlock(&ctx->mutex);
351+ return ret;
352+}
353+
354+int ff_v4l2_request_decode_slice(AVCodecContext *avctx,
355+ V4L2RequestPictureContext *pic,
356+ struct v4l2_ext_control *control, int count,
357+ bool first_slice, bool last_slice)
358+{
359+ /*
360+ * Fallback to queue each slice as a full frame when holding capture
361+ * buffers is not supported by the driver.
362+ */
363+ if ((pic->output->capabilities & V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF) !=
364+ V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF)
365+ return v4l2_request_queue_decode(avctx, pic, control, count, true, true);
366+
367+ return v4l2_request_queue_decode(avctx, pic, control, count,
368+ first_slice, last_slice);
369+}
370+
371+int ff_v4l2_request_decode_frame(AVCodecContext *avctx,
372+ V4L2RequestPictureContext *pic,
373+ struct v4l2_ext_control *control, int count)
374+{
375+ return v4l2_request_queue_decode(avctx, pic, control, count, true, true);
376+}
377+
378+static int v4l2_request_post_process(void *logctx, AVFrame *frame)
379+{
380+ V4L2RequestFrameDescriptor *desc = v4l2_request_framedesc(frame);
381+ FrameDecodeData *fdd = (FrameDecodeData*)frame->private_ref->data;
382+ V4L2RequestContext *ctx = fdd->hwaccel_priv;
383+
384+ // Wait on capture buffer before returning the frame to application
385+ return v4l2_request_wait_on_capture(ctx, &desc->capture);
386+}
387+
388+int ff_v4l2_request_reset_picture(AVCodecContext *avctx, V4L2RequestPictureContext *pic)
389+{
390+ V4L2RequestContext *ctx = v4l2_request_context(avctx);
391+
392+ // Get and wait on next output buffer from circular queue
393+ pic->output = v4l2_request_next_output(ctx);
394+ if (!pic->output)
395+ return AVERROR(EINVAL);
396+
397+ return 0;
398+}
399+
400+int ff_v4l2_request_start_frame(AVCodecContext *avctx,
401+ V4L2RequestPictureContext *pic,
402+ AVFrame *frame)
403+{
404+ V4L2RequestContext *ctx = v4l2_request_context(avctx);
405+ V4L2RequestFrameDescriptor *desc = v4l2_request_framedesc(frame);
406+ FrameDecodeData *fdd = (FrameDecodeData*)frame->private_ref->data;
407+ int ret;
408+
409+ // Get next output buffer from circular queue
410+ ret = ff_v4l2_request_reset_picture(avctx, pic);
411+ if (ret)
412+ return ret;
413+
414+ // Ensure capture buffer is dequeued before reuse
415+ ret = v4l2_request_wait_on_capture(ctx, &desc->capture);
416+ if (ret)
417+ return ret;
418+
419+ // Wait on capture buffer in post_process() before returning to application
420+ fdd->hwaccel_priv = ctx;
421+ fdd->post_process = v4l2_request_post_process;
422+
423+ // Capture buffer used for current frame
424+ pic->capture = &desc->capture;
425+
426+ return 0;
427+}
428+
429+void ff_v4l2_request_flush(AVCodecContext *avctx)
430+{
431+ V4L2RequestContext *ctx = v4l2_request_context(avctx);
432+ struct pollfd pollfd = {
433+ .fd = ctx->video_fd,
434+ .events = POLLOUT,
435+ };
436+
437+ ff_mutex_lock(&ctx->mutex);
438+
439+ // Dequeue all completed output buffers
440+ if (atomic_load(&ctx->queued_output))
441+ v4l2_request_dequeue_completed_buffers(ctx, ctx->output_type);
442+
443+ // Wait on any remaining output buffer
444+ while (atomic_load(&ctx->queued_output)) {
445+ int ret = poll(&pollfd, 1, 2000);
446+ if (ret <= 0)
447+ break;
448+
449+ ret = v4l2_request_dequeue_buffer(ctx, ctx->output_type);
450+ if (ret < 0 && ret != AVERROR(EAGAIN))
451+ break;
452+ }
453+
454+ // Dequeue all completed capture buffers
455+ if (atomic_load(&ctx->queued_capture))
456+ v4l2_request_dequeue_completed_buffers(ctx, ctx->format.type);
457+
458+ ff_mutex_unlock(&ctx->mutex);
459+}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t