History

lavfi/dnn: add classify support with openvino backend

Guo, Yejun committed 5 years ago

Commit fc26dca1 parent a3b74656 files · +218 −19Browse files

Unified · Split

6 files
  1. libavfilter/dnn/dnn_backend_openvino.c +125 −18
  2. libavfilter/dnn/dnn_io_proc.c +60 −0
  3. libavfilter/dnn/dnn_io_proc.h +1 −0
  4. libavfilter/dnn_filter_common.c +21 −0
  5. libavfilter/dnn_filter_common.h +2 −0
  6. libavfilter/dnn_interface.h +9 −1
libavfilter/dnn/dnn_backend_openvino.c+125−18
@@ -29,6 +29,7 @@ · 28 unmodified lines
2929 #include "libavutil/avassert.h"
3030 #include "libavutil/opt.h"
3131 #include "libavutil/avstring.h"
32+#include "libavutil/detection_bbox.h"
3233 #include "../internal.h"
3334 #include "queue.h"
3435 #include "safe_queue.h"
@@ -74,6 +75,7 @@ typedef struct TaskItem { · 39 unmodified lines
7475 // one task might have multiple inferences
7576 typedef struct InferenceItem {
7677 TaskItem *task;
78+ uint32_t bbox_index;
7779 } InferenceItem;
7880
7981 // one request for one call to openvino
@@ -182,12 +184,23 @@ static DNNReturnType fill_model_input_ov(OVModel *ov_model, RequestItem *request · 102 unmodified lines
182184 request->inferences[i] = inference;
183185 request->inference_count = i + 1;
184186 task = inference->task;
185− if (task->do_ioproc) {
186− if (ov_model->model->frame_pre_proc != NULL) {
187− ov_model->model->frame_pre_proc(task->in_frame, &input, ov_model->model->filter_ctx);
188− } else {
189− ff_proc_from_frame_to_dnn(task->in_frame, &input, ov_model->model->func_type, ctx);
187+ switch (task->ov_model->model->func_type) {
188+ case DFT_PROCESS_FRAME:
189+ case DFT_ANALYTICS_DETECT:
190+ if (task->do_ioproc) {
191+ if (ov_model->model->frame_pre_proc != NULL) {
192+ ov_model->model->frame_pre_proc(task->in_frame, &input, ov_model->model->filter_ctx);
193+ } else {
194+ ff_proc_from_frame_to_dnn(task->in_frame, &input, ov_model->model->func_type, ctx);
195+ }
190196 }
197+ break;
198+ case DFT_ANALYTICS_CLASSIFY:
199+ ff_frame_to_dnn_classify(task->in_frame, &input, inference->bbox_index, ctx);
200+ break;
201+ default:
202+ av_assert0(!"should not reach here");
203+ break;
191204 }
192205 input.data = (uint8_t *)input.data
193206 + input.width * input.height * input.channels * get_datatype_size(input.dt);
@@ -276,6 +289,13 @@ static void infer_completion_callback(void *args) · 82 unmodified lines
276289 }
277290 task->ov_model->model->detect_post_proc(task->out_frame, &output, 1, task->ov_model->model->filter_ctx);
278291 break;
292+ case DFT_ANALYTICS_CLASSIFY:
293+ if (!task->ov_model->model->classify_post_proc) {
294+ av_log(ctx, AV_LOG_ERROR, "classify filter needs to provide post proc\n");
295+ return;
296+ }
297+ task->ov_model->model->classify_post_proc(task->out_frame, &output, request->inferences[i]->bbox_index, task->ov_model->model->filter_ctx);
298+ break;
279299 default:
280300 av_assert0(!"should not reach here");
281301 break;
@@ -513,7 +533,44 @@ static DNNReturnType get_input_ov(void *model, DNNData *input, const char *input · 231 unmodified lines
513533 return DNN_ERROR;
514534 }
515535
516−static DNNReturnType extract_inference_from_task(DNNFunctionType func_type, TaskItem *task, Queue *inference_queue)
536+static int contain_valid_detection_bbox(AVFrame *frame)
537+{
538+ AVFrameSideData *sd;
539+ const AVDetectionBBoxHeader *header;
540+ const AVDetectionBBox *bbox;
541+
542+ sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES);
543+ if (!sd) { // this frame has nothing detected
544+ return 0;
545+ }
546+
547+ if (!sd->size) {
548+ return 0;
549+ }
550+
551+ header = (const AVDetectionBBoxHeader *)sd->data;
552+ if (!header->nb_bboxes) {
553+ return 0;
554+ }
555+
556+ for (uint32_t i = 0; i < header->nb_bboxes; i++) {
557+ bbox = av_get_detection_bbox(header, i);
558+ if (bbox->x < 0 || bbox->w < 0 || bbox->x + bbox->w >= frame->width) {
559+ return 0;
560+ }
561+ if (bbox->y < 0 || bbox->h < 0 || bbox->y + bbox->h >= frame->width) {
562+ return 0;
563+ }
564+
565+ if (bbox->classify_count == AV_NUM_DETECTION_BBOX_CLASSIFY) {
566+ return 0;
567+ }
568+ }
569+
570+ return 1;
571+}
572+
573+static DNNReturnType extract_inference_from_task(DNNFunctionType func_type, TaskItem *task, Queue *inference_queue, DNNExecBaseParams *exec_params)
517574 {
518575 switch (func_type) {
519576 case DFT_PROCESS_FRAME:
@@ -532,6 +589,45 @@ static DNNReturnType extract_inference_from_task(DNNFunctionType func_type, Task · 12 unmodified lines
532589 }
533590 return DNN_SUCCESS;
534591 }
592+ case DFT_ANALYTICS_CLASSIFY:
593+ {
594+ const AVDetectionBBoxHeader *header;
595+ AVFrame *frame = task->in_frame;
596+ AVFrameSideData *sd;
597+ DNNExecClassificationParams *params = (DNNExecClassificationParams *)exec_params;
598+
599+ task->inference_todo = 0;
600+ task->inference_done = 0;
601+
602+ if (!contain_valid_detection_bbox(frame)) {
603+ return DNN_SUCCESS;
604+ }
605+
606+ sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES);
607+ header = (const AVDetectionBBoxHeader *)sd->data;
608+
609+ for (uint32_t i = 0; i < header->nb_bboxes; i++) {
610+ InferenceItem *inference;
611+ const AVDetectionBBox *bbox = av_get_detection_bbox(header, i);
612+
613+ if (av_strncasecmp(bbox->detect_label, params->target, sizeof(bbox->detect_label)) != 0) {
614+ continue;
615+ }
616+
617+ inference = av_malloc(sizeof(*inference));
618+ if (!inference) {
619+ return DNN_ERROR;
620+ }
621+ task->inference_todo++;
622+ inference->task = task;
623+ inference->bbox_index = i;
624+ if (ff_queue_push_back(inference_queue, inference) < 0) {
625+ av_freep(&inference);
626+ return DNN_ERROR;
627+ }
628+ }
629+ return DNN_SUCCESS;
630+ }
535631 default:
536632 av_assert0(!"should not reach here");
537633 return DNN_ERROR;
@@ -598,7 +694,7 @@ static DNNReturnType get_output_ov(void *model, const char *input_name, int inpu · 60 unmodified lines
598694 task.out_frame = out_frame;
599695 task.ov_model = ov_model;
600696
601− if (extract_inference_from_task(ov_model->model->func_type, &task, ov_model->inference_queue) != DNN_SUCCESS) {
697+ if (extract_inference_from_task(ov_model->model->func_type, &task, ov_model->inference_queue, NULL) != DNN_SUCCESS) {
602698 av_frame_free(&out_frame);
603699 av_frame_free(&in_frame);
604700 av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");
@@ -690,6 +786,14 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams * · 85 unmodified lines
690786 return DNN_ERROR;
691787 }
692788
789+ if (model->func_type == DFT_ANALYTICS_CLASSIFY) {
790+ // Once we add async support for tensorflow backend and native backend,
791+ // we'll combine the two sync/async functions in dnn_interface.h to
792+ // simplify the code in filter, and async will be an option within backends.
793+ // so, do not support now, and classify filter will not call this function.
794+ return DNN_ERROR;
795+ }
796+
693797 if (ctx->options.batch_size > 1) {
694798 avpriv_report_missing_feature(ctx, "batch mode for sync execution");
695799 return DNN_ERROR;
@@ -710,7 +814,7 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams * · 14 unmodified lines
710814 task.out_frame = exec_params->out_frame ? exec_params->out_frame : exec_params->in_frame;
711815 task.ov_model = ov_model;
712816
713− if (extract_inference_from_task(ov_model->model->func_type, &task, ov_model->inference_queue) != DNN_SUCCESS) {
817+ if (extract_inference_from_task(ov_model->model->func_type, &task, ov_model->inference_queue, exec_params) != DNN_SUCCESS) {
714818 av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");
715819 return DNN_ERROR;
716820 }
@@ -730,6 +834,7 @@ DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, DNNExecBasePa · 13 unmodified lines
730834 OVContext *ctx = &ov_model->ctx;
731835 RequestItem *request;
732836 TaskItem *task;
837+ DNNReturnType ret;
733838
734839 if (ff_check_exec_params(ctx, DNN_OV, model->func_type, exec_params) != 0) {
735840 return DNN_ERROR;
@@ -761,23 +866,25 @@ DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, DNNExecBasePa · 25 unmodified lines
761866 return DNN_ERROR;
762867 }
763868
764− if (extract_inference_from_task(ov_model->model->func_type, task, ov_model->inference_queue) != DNN_SUCCESS) {
869+ if (extract_inference_from_task(model->func_type, task, ov_model->inference_queue, exec_params) != DNN_SUCCESS) {
765870 av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");
766871 return DNN_ERROR;
767872 }
768873
769− if (ff_queue_size(ov_model->inference_queue) < ctx->options.batch_size) {
770− // not enough inference items queued for a batch
771− return DNN_SUCCESS;
772− }
874+ while (ff_queue_size(ov_model->inference_queue) >= ctx->options.batch_size) {
875+ request = ff_safe_queue_pop_front(ov_model->request_queue);
876+ if (!request) {
877+ av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
878+ return DNN_ERROR;
879+ }
773880
774− request = ff_safe_queue_pop_front(ov_model->request_queue);
775− if (!request) {
776− av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
777− return DNN_ERROR;
881+ ret = execute_model_ov(request, ov_model->inference_queue);
882+ if (ret != DNN_SUCCESS) {
883+ return ret;
884+ }
778885 }
779886
780− return execute_model_ov(request, ov_model->inference_queue);
887+ return DNN_SUCCESS;
781888 }
782889
783890 DNNAsyncStatusType ff_dnn_get_async_result_ov(const DNNModel *model, AVFrame **in, AVFrame **out)
libavfilter/dnn/dnn_io_proc.c+60−0
@@ -22,6 +22,7 @@ · 21 unmodified lines
2222 #include "libavutil/imgutils.h"
2323 #include "libswscale/swscale.h"
2424 #include "libavutil/avassert.h"
25+#include "libavutil/detection_bbox.h"
2526
2627 DNNReturnType ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
2728 {
@@ -175,6 +176,65 @@ static enum AVPixelFormat get_pixel_format(DNNData *data) · 147 unmodified lines
175176 return AV_PIX_FMT_BGR24;
176177 }
177178
179+DNNReturnType ff_frame_to_dnn_classify(AVFrame *frame, DNNData *input, uint32_t bbox_index, void *log_ctx)
180+{
181+ const AVPixFmtDescriptor *desc;
182+ int offsetx[4], offsety[4];
183+ uint8_t *bbox_data[4];
184+ struct SwsContext *sws_ctx;
185+ int linesizes[4];
186+ enum AVPixelFormat fmt;
187+ int left, top, width, height;
188+ const AVDetectionBBoxHeader *header;
189+ const AVDetectionBBox *bbox;
190+ AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DETECTION_BBOXES);
191+ av_assert0(sd);
192+
193+ header = (const AVDetectionBBoxHeader *)sd->data;
194+ bbox = av_get_detection_bbox(header, bbox_index);
195+
196+ left = bbox->x;
197+ width = bbox->w;
198+ top = bbox->y;
199+ height = bbox->h;
200+
201+ fmt = get_pixel_format(input);
202+ sws_ctx = sws_getContext(width, height, frame->format,
203+ input->width, input->height, fmt,
204+ SWS_FAST_BILINEAR, NULL, NULL, NULL);
205+ if (!sws_ctx) {
206+ av_log(log_ctx, AV_LOG_ERROR, "Failed to create scale context for the conversion "
207+ "fmt:%s s:%dx%d -> fmt:%s s:%dx%d\n",
208+ av_get_pix_fmt_name(frame->format), width, height,
209+ av_get_pix_fmt_name(fmt), input->width, input->height);
210+ return DNN_ERROR;
211+ }
212+
213+ if (av_image_fill_linesizes(linesizes, fmt, input->width) < 0) {
214+ av_log(log_ctx, AV_LOG_ERROR, "unable to get linesizes with av_image_fill_linesizes");
215+ sws_freeContext(sws_ctx);
216+ return DNN_ERROR;
217+ }
218+
219+ desc = av_pix_fmt_desc_get(frame->format);
220+ offsetx[1] = offsetx[2] = AV_CEIL_RSHIFT(left, desc->log2_chroma_w);
221+ offsetx[0] = offsetx[3] = left;
222+
223+ offsety[1] = offsety[2] = AV_CEIL_RSHIFT(top, desc->log2_chroma_h);
224+ offsety[0] = offsety[3] = top;
225+
226+ for (int k = 0; frame->data[k]; k++)
227+ bbox_data[k] = frame->data[k] + offsety[k] * frame->linesize[k] + offsetx[k];
228+
229+ sws_scale(sws_ctx, (const uint8_t *const *)&bbox_data, frame->linesize,
230+ 0, height,
231+ (uint8_t *const *)(&input->data), linesizes);
232+
233+ sws_freeContext(sws_ctx);
234+
235+ return DNN_SUCCESS;
236+}
237+
178238 static DNNReturnType proc_from_frame_to_dnn_analytics(AVFrame *frame, DNNData *input, void *log_ctx)
179239 {
180240 struct SwsContext *sws_ctx;
libavfilter/dnn/dnn_io_proc.h+1−0
@@ -32,5 +32,6 @@ · 31 unmodified lines
3232
3333 DNNReturnType ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, DNNFunctionType func_type, void *log_ctx);
3434 DNNReturnType ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx);
35+DNNReturnType ff_frame_to_dnn_classify(AVFrame *frame, DNNData *input, uint32_t bbox_index, void *log_ctx);
3536
3637 #endif
libavfilter/dnn_filter_common.c+21−0
@@ -77,6 +77,12 @@ int ff_dnn_set_detect_post_proc(DnnContext *ctx, DetectPostProc post_proc) · 76 unmodified lines
7777 return 0;
7878 }
7979
80+int ff_dnn_set_classify_post_proc(DnnContext *ctx, ClassifyPostProc post_proc)
81+{
82+ ctx->model->classify_post_proc = post_proc;
83+ return 0;
84+}
85+
8086 DNNReturnType ff_dnn_get_input(DnnContext *ctx, DNNData *input)
8187 {
8288 return ctx->model->get_input(ctx->model->model, input, ctx->model_inputname);
@@ -112,6 +118,21 @@ DNNReturnType ff_dnn_execute_model_async(DnnContext *ctx, AVFrame *in_frame, AVF · 29 unmodified lines
112118 return (ctx->dnn_module->execute_model_async)(ctx->model, &exec_params);
113119 }
114120
121+DNNReturnType ff_dnn_execute_model_classification(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, char *target)
122+{
123+ DNNExecClassificationParams class_params = {
124+ {
125+ .input_name = ctx->model_inputname,
126+ .output_names = (const char **)&ctx->model_outputname,
127+ .nb_output = 1,
128+ .in_frame = in_frame,
129+ .out_frame = out_frame,
130+ },
131+ .target = target,
132+ };
133+ return (ctx->dnn_module->execute_model_async)(ctx->model, &class_params.base);
134+}
135+
115136 DNNAsyncStatusType ff_dnn_get_async_result(DnnContext *ctx, AVFrame **in_frame, AVFrame **out_frame)
116137 {
117138 return (ctx->dnn_module->get_async_result)(ctx->model, in_frame, out_frame);
libavfilter/dnn_filter_common.h+2−0
@@ -50,10 +50,12 @@ typedef struct DnnContext { · 49 unmodified lines
5050 int ff_dnn_init(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx);
5151 int ff_dnn_set_frame_proc(DnnContext *ctx, FramePrePostProc pre_proc, FramePrePostProc post_proc);
5252 int ff_dnn_set_detect_post_proc(DnnContext *ctx, DetectPostProc post_proc);
53+int ff_dnn_set_classify_post_proc(DnnContext *ctx, ClassifyPostProc post_proc);
5354 DNNReturnType ff_dnn_get_input(DnnContext *ctx, DNNData *input);
5455 DNNReturnType ff_dnn_get_output(DnnContext *ctx, int input_width, int input_height, int *output_width, int *output_height);
5556 DNNReturnType ff_dnn_execute_model(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame);
5657 DNNReturnType ff_dnn_execute_model_async(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame);
58+DNNReturnType ff_dnn_execute_model_classification(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, char *target);
5759 DNNAsyncStatusType ff_dnn_get_async_result(DnnContext *ctx, AVFrame **in_frame, AVFrame **out_frame);
5860 DNNReturnType ff_dnn_flush(DnnContext *ctx);
5961 void ff_dnn_uninit(DnnContext *ctx);
libavfilter/dnn_interface.h+9−1
@@ -52,7 +52,7 @@ typedef enum { · 51 unmodified lines
5252 DFT_NONE,
5353 DFT_PROCESS_FRAME, // process the whole frame
5454 DFT_ANALYTICS_DETECT, // detect from the whole frame
55− // we can add more such as detect_from_crop, classify_from_bbox, etc.
55+ DFT_ANALYTICS_CLASSIFY, // classify for each bounding box
5656 }DNNFunctionType;
5757
5858 typedef struct DNNData{
@@ -71,8 +71,14 @@ typedef struct DNNExecBaseParams { · 12 unmodified lines
7171 AVFrame *out_frame;
7272 } DNNExecBaseParams;
7373
74+typedef struct DNNExecClassificationParams {
75+ DNNExecBaseParams base;
76+ const char *target;
77+} DNNExecClassificationParams;
78+
7479 typedef int (*FramePrePostProc)(AVFrame *frame, DNNData *model, AVFilterContext *filter_ctx);
7580 typedef int (*DetectPostProc)(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx);
81+typedef int (*ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx);
7682
7783 typedef struct DNNModel{
7884 // Stores model that can be different for different backends.
@@ -97,6 +103,8 @@ typedef struct DNNModel{ · 18 unmodified lines
97103 FramePrePostProc frame_post_proc;
98104 // set the post process to interpret detect result from DNNData
99105 DetectPostProc detect_post_proc;
106+ // set the post process to interpret classify result from DNNData
107+ ClassifyPostProc classify_post_proc;
100108 } DNNModel;
101109
102110 // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t