lavu: add side data AV_FRAME_DATA_DETECTION_BBOXES for object detection/classification

Guo, Yejun committed 5 years ago

Commit f1bf4651 parent 8b954a77 files · +192 −1Browse files

Unified · Split

7 files

doc/APIchanges +2 −0
libavutil/Makefile +2 −0
libavutil/detection_bbox.c +73 −0
libavutil/detection_bbox.h +107 −0
libavutil/frame.c +1 −0
libavutil/frame.h +6 −0
libavutil/version.h +1 −1

@@ -14,6 +14,8 @@ libavutil: 2017-10-21 · 13 unmodified lines
14	14
15	15
16	16	API changes, most recent first:
	17	+2021-04-17 - xxxxxxxxxx - lavu 56.73.100 - frame.h detection_bbox.h
	18	+ Add AV_FRAME_DATA_DETECTION_BBOXES
17	19
18	20	2021-04-06 - xxxxxxxxxx - lavf 58.78.100 - avformat.h
19	21	Add avformat_index_get_entries_count(), avformat_index_get_entry(),

libavutil/Makefile+2−0

@@ -21,6 +21,7 @@ HEADERS = adler32.h \ · 20 unmodified lines
21	21	cpu.h \
22	22	crc.h \
23	23	des.h \
	24	+ detection_bbox.h \
24	25	dict.h \
25	26	display.h \
26	27	dovi_meta.h \
@@ -113,6 +114,7 @@ OBJS = adler32.o \ · 86 unmodified lines
113	114	cpu.o \
114	115	crc.o \
115	116	des.o \
	117	+ detection_bbox.o \
116	118	dict.o \
117	119	display.o \
118	120	dovi_meta.o \

libavutil/detection_bbox.c+73−0

@@ -0,0 +1,73 @@
	1	+/*
	2	+ * This file is part of FFmpeg.
	3	+ *
	4	+ * FFmpeg is free software; you can redistribute it and/or
	5	+ * modify it under the terms of the GNU Lesser General Public
	6	+ * License as published by the Free Software Foundation; either
	7	+ * version 2.1 of the License, or (at your option) any later version.
	8	+ *
	9	+ * FFmpeg is distributed in the hope that it will be useful,
	10	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	12	+ * Lesser General Public License for more details.
	13	+ *
	14	+ * You should have received a copy of the GNU Lesser General Public
	15	+ * License along with FFmpeg; if not, write to the Free Software
	16	+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	17	+ */
	18	+
	19	+#include "detection_bbox.h"
	20	+
	21	+AVDetectionBBoxHeader av_detection_bbox_alloc(uint32_t nb_bboxes, size_t out_size)
	22	+{
	23	+ size_t size;
	24	+ struct {
	25	+ AVDetectionBBoxHeader header;
	26	+ AVDetectionBBox boxes[1];
	27	+ } *ret;
	28	+
	29	+ size = sizeof(*ret);
	30	+ if (nb_bboxes - 1 > (SIZE_MAX - size) / sizeof(*ret->boxes))
	31	+ return NULL;
	32	+ size += sizeof(ret->boxes) (nb_bboxes - 1);
	33	+
	34	+ ret = av_mallocz(size);
	35	+ if (!ret)
	36	+ return NULL;
	37	+
	38	+ ret->header.nb_bboxes = nb_bboxes;
	39	+ ret->header.bbox_size = sizeof(*ret->boxes);
	40	+ ret->header.bboxes_offset = (char )&ret->boxes - (char )&ret->header;
	41	+
	42	+ if (out_size)
	43	+ *out_size = size;
	44	+
	45	+ return &ret->header;
	46	+}
	47	+
	48	+AVDetectionBBoxHeader av_detection_bbox_create_side_data(AVFrame frame, uint32_t nb_bboxes)
	49	+{
	50	+ AVBufferRef *buf;
	51	+ AVDetectionBBoxHeader *header;
	52	+ size_t size;
	53	+
	54	+ header = av_detection_bbox_alloc(nb_bboxes, &size);
	55	+ if (!header)
	56	+ return NULL;
	57	+ if (size > INT_MAX) {
	58	+ av_freep(&header);
	59	+ return NULL;
	60	+ }
	61	+ buf = av_buffer_create((uint8_t *)header, size, NULL, NULL, 0);
	62	+ if (!buf) {
	63	+ av_freep(&header);
	64	+ return NULL;
	65	+ }
	66	+
	67	+ if (!av_frame_new_side_data_from_buf(frame, AV_FRAME_DATA_DETECTION_BBOXES, buf)) {
	68	+ av_buffer_unref(&buf);
	69	+ return NULL;
	70	+ }
	71	+
	72	+ return header;
	73	+}

libavutil/detection_bbox.h+107−0

@@ -0,0 +1,107 @@
	1	+/*
	2	+ * This file is part of FFmpeg.
	3	+ *
	4	+ * FFmpeg is free software; you can redistribute it and/or
	5	+ * modify it under the terms of the GNU Lesser General Public
	6	+ * License as published by the Free Software Foundation; either
	7	+ * version 2.1 of the License, or (at your option) any later version.
	8	+ *
	9	+ * FFmpeg is distributed in the hope that it will be useful,
	10	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	12	+ * Lesser General Public License for more details.
	13	+ *
	14	+ * You should have received a copy of the GNU Lesser General Public
	15	+ * License along with FFmpeg; if not, write to the Free Software
	16	+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	17	+ */
	18	+
	19	+#ifndef AVUTIL_DETECTION_BBOX_H
	20	+#define AVUTIL_DETECTION_BBOX_H
	21	+
	22	+#include "rational.h"
	23	+#include "avassert.h"
	24	+#include "frame.h"
	25	+
	26	+typedef struct AVDetectionBBox {
	27	+ /**
	28	+ * Distance in pixels from the left/top edge of the frame,
	29	+ * together with width and height, defining the bounding box.
	30	+ */
	31	+ int x;
	32	+ int y;
	33	+ int w;
	34	+ int h;
	35	+
	36	+#define AV_DETECTION_BBOX_LABEL_NAME_MAX_SIZE 64
	37	+
	38	+ /**
	39	+ * Detect result with confidence
	40	+ */
	41	+ char detect_label[AV_DETECTION_BBOX_LABEL_NAME_MAX_SIZE];
	42	+ AVRational detect_confidence;
	43	+
	44	+ /**
	45	+ * At most 4 classifications based on the detected bounding box.
	46	+ * For example, we can get max 4 different attributes with 4 different
	47	+ * DNN models on one bounding box.
	48	+ * classify_count is zero if no classification.
	49	+ */
	50	+#define AV_NUM_BBOX_CLASSIFY 4
	51	+ uint32_t classify_count;
	52	+ char classify_labels[AV_NUM_BBOX_CLASSIFY][AV_DETECTION_BBOX_LABEL_NAME_MAX_SIZE];
	53	+ AVRational classify_confidences[AV_NUM_BBOX_CLASSIFY];
	54	+} AVDetectionBBox;
	55	+
	56	+typedef struct AVDetectionBBoxHeader {
	57	+ /**
	58	+ * Information about how the bounding box is generated.
	59	+ * for example, the DNN model name.
	60	+ */
	61	+ char source[256];
	62	+
	63	+ /**
	64	+ * Number of bounding boxes in the array.
	65	+ */
	66	+ uint32_t nb_bboxes;
	67	+
	68	+ /**
	69	+ * Offset in bytes from the beginning of this structure at which
	70	+ * the array of bounding boxes starts.
	71	+ */
	72	+ size_t bboxes_offset;
	73	+
	74	+ /**
	75	+ * Size of each bounding box in bytes.
	76	+ */
	77	+ size_t bbox_size;
	78	+} AVDetectionBBoxHeader;
	79	+
	80	+/*
	81	+ * Get the bounding box at the specified {@code idx}. Must be between 0 and nb_bboxes.
	82	+ */
	83	+static av_always_inline AVDetectionBBox*
	84	+av_get_detection_bbox(const AVDetectionBBoxHeader *header, unsigned int idx)
	85	+{
	86	+ av_assert0(idx < header->nb_bboxes);
	87	+ return (AVDetectionBBox )((uint8_t )header + header->bboxes_offset +
	88	+ idx * header->bbox_size);
	89	+}
	90	+
	91	+/**
	92	+ * Allocates memory for AVDetectionBBoxHeader, plus an array of {@code nb_bboxes}
	93	+ * AVDetectionBBox, and initializes the variables.
	94	+ * Can be freed with a normal av_free() call.
	95	+ *
	96	+ * @param out_size if non-NULL, the size in bytes of the resulting data array is
	97	+ * written here.
	98	+ */
	99	+AVDetectionBBoxHeader av_detection_bbox_alloc(uint32_t nb_bboxes, size_t out_size);
	100	+
	101	+/**
	102	+ * Allocates memory for AVDetectionBBoxHeader, plus an array of {@code nb_bboxes}
	103	+ * AVDetectionBBox, in the given AVFrame {@code frame} as AVFrameSideData of type
	104	+ * AV_FRAME_DATA_DETECTION_BBOXES and initializes the variables.
	105	+ */
	106	+AVDetectionBBoxHeader av_detection_bbox_create_side_data(AVFrame frame, uint32_t nb_bboxes);
	107	+#endif

libavutil/frame.c+1−0

@@ -853,6 +853,7 @@ const char *av_frame_side_data_name(enum AVFrameSideDataType type) · 852 unmodified lines
853	853	case AV_FRAME_DATA_VIDEO_ENC_PARAMS: return "Video encoding parameters";
854	854	case AV_FRAME_DATA_SEI_UNREGISTERED: return "H.26[45] User Data Unregistered SEI message";
855	855	case AV_FRAME_DATA_FILM_GRAIN_PARAMS: return "Film grain parameters";
	856	+ case AV_FRAME_DATA_DETECTION_BBOXES: return "Bounding boxes for object detection and classification";
856	857	}
857	858	return NULL;
858	859	}

libavutil/frame.h+6−0

@@ -198,6 +198,12 @@ enum AVFrameSideDataType { · 197 unmodified lines
198	198	* Must be present for every frame which should have film grain applied.
199	199	*/
200	200	AV_FRAME_DATA_FILM_GRAIN_PARAMS,
	201	+
	202	+ /**
	203	+ * Bounding boxes for object detection and classification,
	204	+ * as described by AVDetectionBBoxHeader.
	205	+ */
	206	+ AV_FRAME_DATA_DETECTION_BBOXES,
201	207	};
202	208
203	209	enum AVActiveFormatDescription {

libavutil/version.h+1−1

@@ -79,7 +79,7 @@ · 78 unmodified lines
79	79	*/
80	80
81	81	#define LIBAVUTIL_VERSION_MAJOR 56
82		−#define LIBAVUTIL_VERSION_MINOR 72
	82	+#define LIBAVUTIL_VERSION_MINOR 73
83	83	#define LIBAVUTIL_VERSION_MICRO 100
84	84
85	85	#define LIBAVUTIL_VERSION_INT AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \

lavu: add side data AV_FRAME_DATA_DETECTION_BBOXES for object detection/classification

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code