History

vf_dnn_processing: remove parameter 'fmt'

do not request AVFrame's format in vf_ddn_processing with 'fmt', but to add another filter for the format. command examples: ./ffmpeg -i input.jpg -vf format=bgr24,dnn_processing=model=halve_first_channel.model:input=dnn_in:output=dnn_out:dnn_backend=native -y out.native.png ./ffmpeg -i input.jpg -vf format=rgb24,dnn_processing=model=halve_first_channel.model:input=dnn_in:output=dnn_out:dnn_backend=native -y out.native.png

Guo, Yejun committed 6 years ago

Commit 04e6f8a1 parent 742221d2 files · +60 −52Browse files

Unified · Split

2 files
  1. doc/filters.texi +11 −6
  2. libavfilter/vf_dnn_processing.c +49 −46
doc/filters.texi+11−6
@@ -9070,8 +9070,8 @@ ffmpeg -i INPUT -f lavfi -i nullsrc=hd720,geq='r=128+80*(sin(sqrt((X-W/2)*(X-W/2 · 9069 unmodified lines
9070 9070
9071@section dnn_processing9071@section dnn_processing
9072 9072
9073Do image processing with deep neural networks. Currently only AVFrame with RGB249073Do image processing with deep neural networks. It works together with another filter
9074and BGR24 are supported, more formats will be added later.9074which converts the pixel format of the Frame to what the dnn network requires.
9075 9075
9076The filter accepts the following options:9076The filter accepts the following options:
9077 9077
@@ -9106,12 +9106,17 @@ Set the input name of the dnn network. · 28 unmodified lines
9106@item output9106@item output
9107Set the output name of the dnn network.9107Set the output name of the dnn network.
9108 9108
9109@item fmt
9110Set the pixel format for the Frame. Allowed values are @code{AV_PIX_FMT_RGB24}, and @code{AV_PIX_FMT_BGR24}.
9111Default value is @code{AV_PIX_FMT_RGB24}.
9112
9113@end table9109@end table
9114 9110
9111@itemize
9112@item
9113Halve the red channle of the frame with format rgb24:
9114@example
9115ffmpeg -i input.jpg -vf format=rgb24,dnn_processing=model=halve_first_channel.model:input=dnn_in:output=dnn_out:dnn_backend=native out.native.png
9116@end example
9117
9118@end itemize
9119
9115@section drawbox9120@section drawbox
9116 9121
9117Draw a colored box on the input image.9122Draw a colored box on the input image.
libavfilter/vf_dnn_processing.c+49−46
@@ -37,7 +37,6 @@ typedef struct DnnProcessingContext { · 36 unmodified lines
37 37
38 char *model_filename;38 char *model_filename;
39 DNNBackendType backend_type;39 DNNBackendType backend_type;
40 enum AVPixelFormat fmt;
41 char *model_inputname;40 char *model_inputname;
42 char *model_outputname;41 char *model_outputname;
43 42
@@ -60,7 +59,6 @@ static const AVOption dnn_processing_options[] = { · 16 unmodified lines
60 { "model", "path to model file", OFFSET(model_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },59 { "model", "path to model file", OFFSET(model_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
61 { "input", "input name of the model", OFFSET(model_inputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },60 { "input", "input name of the model", OFFSET(model_inputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
62 { "output", "output name of the model", OFFSET(model_outputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },61 { "output", "output name of the model", OFFSET(model_outputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
63 { "fmt", "AVPixelFormat of the frame", OFFSET(fmt), AV_OPT_TYPE_PIXEL_FMT, { .i64=AV_PIX_FMT_RGB24 }, AV_PIX_FMT_NONE, AV_PIX_FMT_NB - 1, FLAGS },
64 { NULL }62 { NULL }
65};63};
66 64
@@ -69,23 +67,6 @@ AVFILTER_DEFINE_CLASS(dnn_processing); · 2 unmodified lines
69static av_cold int init(AVFilterContext *context)67static av_cold int init(AVFilterContext *context)
70{68{
71 DnnProcessingContext *ctx = context->priv;69 DnnProcessingContext *ctx = context->priv;
72 int supported = 0;
73 // as the first step, only rgb24 and bgr24 are supported
74 const enum AVPixelFormat supported_pixel_fmts[] = {
75 AV_PIX_FMT_RGB24,
76 AV_PIX_FMT_BGR24,
77 };
78 for (int i = 0; i < sizeof(supported_pixel_fmts) / sizeof(enum AVPixelFormat); ++i) {
79 if (supported_pixel_fmts[i] == ctx->fmt) {
80 supported = 1;
81 break;
82 }
83 }
84 if (!supported) {
85 av_log(context, AV_LOG_ERROR, "pixel fmt %s not supported yet\n",
86 av_get_pix_fmt_name(ctx->fmt));
87 return AVERROR(AVERROR_INVALIDDATA);
88 }
89 70
90 if (!ctx->model_filename) {71 if (!ctx->model_filename) {
91 av_log(ctx, AV_LOG_ERROR, "model file for network is not specified\n");72 av_log(ctx, AV_LOG_ERROR, "model file for network is not specified\n");
@@ -121,14 +102,52 @@ static av_cold int init(AVFilterContext *context) · 29 unmodified lines
121 102
122static int query_formats(AVFilterContext *context)103static int query_formats(AVFilterContext *context)
123{104{
124 AVFilterFormats *formats;105 static const enum AVPixelFormat pix_fmts[] = {
125 DnnProcessingContext *ctx = context->priv;106 AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24,
126 enum AVPixelFormat pixel_fmts[2];107 AV_PIX_FMT_NONE
127 pixel_fmts[0] = ctx->fmt;108 };
128 pixel_fmts[1] = AV_PIX_FMT_NONE;109 AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts);
110 return ff_set_common_formats(context, fmts_list);
111}
112
113static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink)
114{
115 AVFilterContext *ctx = inlink->dst;
116 enum AVPixelFormat fmt = inlink->format;
117
118 // the design is to add explicit scale filter before this filter
119 if (model_input->height != -1 && model_input->height != inlink->h) {
120 av_log(ctx, AV_LOG_ERROR, "the model requires frame height %d but got %d\n",
121 model_input->height, inlink->h);
122 return AVERROR(EIO);
123 }
124 if (model_input->width != -1 && model_input->width != inlink->w) {
125 av_log(ctx, AV_LOG_ERROR, "the model requires frame width %d but got %d\n",
126 model_input->width, inlink->w);
127 return AVERROR(EIO);
128 }
129 129
130 formats = ff_make_format_list(pixel_fmts);130 switch (fmt) {
131 return ff_set_common_formats(context, formats);131 case AV_PIX_FMT_RGB24:
132 case AV_PIX_FMT_BGR24:
133 if (model_input->channels != 3) {
134 av_log(ctx, AV_LOG_ERROR, "the frame's input format %s does not match "
135 "the model input channels %d\n",
136 av_get_pix_fmt_name(fmt),
137 model_input->channels);
138 return AVERROR(EIO);
139 }
140 if (model_input->dt != DNN_FLOAT && model_input->dt != DNN_UINT8) {
141 av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type as float32 and uint8.\n");
142 return AVERROR(EIO);
143 }
144 break;
145 default:
146 av_log(ctx, AV_LOG_ERROR, "%s not supported.\n", av_get_pix_fmt_name(fmt));
147 return AVERROR(EIO);
148 }
149
150 return 0;
132}151}
133 152
134static int config_input(AVFilterLink *inlink)153static int config_input(AVFilterLink *inlink)
@@ -137,6 +156,7 @@ static int config_input(AVFilterLink *inlink) · 2 unmodified lines
137 DnnProcessingContext *ctx = context->priv;156 DnnProcessingContext *ctx = context->priv;
138 DNNReturnType result;157 DNNReturnType result;
139 DNNData model_input;158 DNNData model_input;
159 int check;
140 160
141 result = ctx->model->get_input(ctx->model->model, &model_input, ctx->model_inputname);161 result = ctx->model->get_input(ctx->model->model, &model_input, ctx->model_inputname);
142 if (result != DNN_SUCCESS) {162 if (result != DNN_SUCCESS) {
@@ -144,26 +164,9 @@ static int config_input(AVFilterLink *inlink) · 1 unmodified line
144 return AVERROR(EIO);164 return AVERROR(EIO);
145 }165 }
146 166
147 // the design is to add explicit scale filter before this filter167 check = check_modelinput_inlink(&model_input, inlink);
148 if (model_input.height != -1 && model_input.height != inlink->h) {168 if (check != 0) {
149 av_log(ctx, AV_LOG_ERROR, "the model requires frame height %d but got %d\n",169 return check;
150 model_input.height, inlink->h);
151 return AVERROR(EIO);
152 }
153 if (model_input.width != -1 && model_input.width != inlink->w) {
154 av_log(ctx, AV_LOG_ERROR, "the model requires frame width %d but got %d\n",
155 model_input.width, inlink->w);
156 return AVERROR(EIO);
157 }
158
159 if (model_input.channels != 3) {
160 av_log(ctx, AV_LOG_ERROR, "the model requires input channels %d\n",
161 model_input.channels);
162 return AVERROR(EIO);
163 }
164 if (model_input.dt != DNN_FLOAT && model_input.dt != DNN_UINT8) {
165 av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type as float32 and uint8.\n");
166 return AVERROR(EIO);
167 }170 }
168 171
169 ctx->input.width = inlink->w;172 ctx->input.width = inlink->w;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t