do not request AVFrame's format in vf_ddn_processing with 'fmt', but to add another filter for the format. command examples: ./ffmpeg -i input.jpg -vf format=bgr24,dnn_processing=model=halve_first_channel.model:input=dnn_in:output=dnn_out:dnn_backend=native -y out.native.png ./ffmpeg -i input.jpg -vf format=rgb24,dnn_processing=model=halve_first_channel.model:input=dnn_in:output=dnn_out:dnn_backend=native -y out.native.png
Showing the changes to libavfilter (1 of 2 files). Show all
Unified · Split
| @@ -37,7 +37,6 @@ typedef struct DnnProcessingContext { · 36 unmodified lines | ||
| 37 | 37 | |
| 38 | 38 | char *model_filename; |
| 39 | 39 | DNNBackendType backend_type; |
| 40 | − enum AVPixelFormat fmt; | |
| 41 | 40 | char *model_inputname; |
| 42 | 41 | char *model_outputname; |
| 43 | 42 | |
| @@ -60,7 +59,6 @@ static const AVOption dnn_processing_options[] = { · 16 unmodified lines | ||
| 60 | 59 | { "model", "path to model file", OFFSET(model_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, |
| 61 | 60 | { "input", "input name of the model", OFFSET(model_inputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, |
| 62 | 61 | { "output", "output name of the model", OFFSET(model_outputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, |
| 63 | − { "fmt", "AVPixelFormat of the frame", OFFSET(fmt), AV_OPT_TYPE_PIXEL_FMT, { .i64=AV_PIX_FMT_RGB24 }, AV_PIX_FMT_NONE, AV_PIX_FMT_NB - 1, FLAGS }, | |
| 64 | 62 | { NULL } |
| 65 | 63 | }; |
| 66 | 64 | |
| @@ -69,23 +67,6 @@ AVFILTER_DEFINE_CLASS(dnn_processing); · 2 unmodified lines | ||
| 69 | 67 | static av_cold int init(AVFilterContext *context) |
| 70 | 68 | { |
| 71 | 69 | DnnProcessingContext *ctx = context->priv; |
| 72 | − int supported = 0; | |
| 73 | − // as the first step, only rgb24 and bgr24 are supported | |
| 74 | − const enum AVPixelFormat supported_pixel_fmts[] = { | |
| 75 | − AV_PIX_FMT_RGB24, | |
| 76 | − AV_PIX_FMT_BGR24, | |
| 77 | − }; | |
| 78 | − for (int i = 0; i < sizeof(supported_pixel_fmts) / sizeof(enum AVPixelFormat); ++i) { | |
| 79 | − if (supported_pixel_fmts[i] == ctx->fmt) { | |
| 80 | − supported = 1; | |
| 81 | − break; | |
| 82 | − } | |
| 83 | − } | |
| 84 | − if (!supported) { | |
| 85 | − av_log(context, AV_LOG_ERROR, "pixel fmt %s not supported yet\n", | |
| 86 | − av_get_pix_fmt_name(ctx->fmt)); | |
| 87 | − return AVERROR(AVERROR_INVALIDDATA); | |
| 88 | − } | |
| 89 | 70 | |
| 90 | 71 | if (!ctx->model_filename) { |
| 91 | 72 | av_log(ctx, AV_LOG_ERROR, "model file for network is not specified\n"); |
| @@ -121,14 +102,52 @@ static av_cold int init(AVFilterContext *context) · 29 unmodified lines | ||
| 121 | 102 | |
| 122 | 103 | static int query_formats(AVFilterContext *context) |
| 123 | 104 | { |
| 124 | − AVFilterFormats *formats; | |
| 125 | − DnnProcessingContext *ctx = context->priv; | |
| 126 | − enum AVPixelFormat pixel_fmts[2]; | |
| 127 | − pixel_fmts[0] = ctx->fmt; | |
| 128 | − pixel_fmts[1] = AV_PIX_FMT_NONE; | |
| 105 | + static const enum AVPixelFormat pix_fmts[] = { | |
| 106 | + AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24, | |
| 107 | + AV_PIX_FMT_NONE | |
| 108 | + }; | |
| 109 | + AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts); | |
| 110 | + return ff_set_common_formats(context, fmts_list); | |
| 111 | +} | |
| 112 | + | |
| 113 | +static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink) | |
| 114 | +{ | |
| 115 | + AVFilterContext *ctx = inlink->dst; | |
| 116 | + enum AVPixelFormat fmt = inlink->format; | |
| 117 | + | |
| 118 | + // the design is to add explicit scale filter before this filter | |
| 119 | + if (model_input->height != -1 && model_input->height != inlink->h) { | |
| 120 | + av_log(ctx, AV_LOG_ERROR, "the model requires frame height %d but got %d\n", | |
| 121 | + model_input->height, inlink->h); | |
| 122 | + return AVERROR(EIO); | |
| 123 | + } | |
| 124 | + if (model_input->width != -1 && model_input->width != inlink->w) { | |
| 125 | + av_log(ctx, AV_LOG_ERROR, "the model requires frame width %d but got %d\n", | |
| 126 | + model_input->width, inlink->w); | |
| 127 | + return AVERROR(EIO); | |
| 128 | + } | |
| 129 | 129 | |
| 130 | − formats = ff_make_format_list(pixel_fmts); | |
| 131 | − return ff_set_common_formats(context, formats); | |
| 130 | + switch (fmt) { | |
| 131 | + case AV_PIX_FMT_RGB24: | |
| 132 | + case AV_PIX_FMT_BGR24: | |
| 133 | + if (model_input->channels != 3) { | |
| 134 | + av_log(ctx, AV_LOG_ERROR, "the frame's input format %s does not match " | |
| 135 | + "the model input channels %d\n", | |
| 136 | + av_get_pix_fmt_name(fmt), | |
| 137 | + model_input->channels); | |
| 138 | + return AVERROR(EIO); | |
| 139 | + } | |
| 140 | + if (model_input->dt != DNN_FLOAT && model_input->dt != DNN_UINT8) { | |
| 141 | + av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type as float32 and uint8.\n"); | |
| 142 | + return AVERROR(EIO); | |
| 143 | + } | |
| 144 | + break; | |
| 145 | + default: | |
| 146 | + av_log(ctx, AV_LOG_ERROR, "%s not supported.\n", av_get_pix_fmt_name(fmt)); | |
| 147 | + return AVERROR(EIO); | |
| 148 | + } | |
| 149 | + | |
| 150 | + return 0; | |
| 132 | 151 | } |
| 133 | 152 | |
| 134 | 153 | static int config_input(AVFilterLink *inlink) |
| @@ -137,6 +156,7 @@ static int config_input(AVFilterLink *inlink) · 2 unmodified lines | ||
| 137 | 156 | DnnProcessingContext *ctx = context->priv; |
| 138 | 157 | DNNReturnType result; |
| 139 | 158 | DNNData model_input; |
| 159 | + int check; | |
| 140 | 160 | |
| 141 | 161 | result = ctx->model->get_input(ctx->model->model, &model_input, ctx->model_inputname); |
| 142 | 162 | if (result != DNN_SUCCESS) { |
| @@ -144,26 +164,9 @@ static int config_input(AVFilterLink *inlink) · 1 unmodified line | ||
| 144 | 164 | return AVERROR(EIO); |
| 145 | 165 | } |
| 146 | 166 | |
| 147 | − // the design is to add explicit scale filter before this filter | |
| 148 | − if (model_input.height != -1 && model_input.height != inlink->h) { | |
| 149 | − av_log(ctx, AV_LOG_ERROR, "the model requires frame height %d but got %d\n", | |
| 150 | − model_input.height, inlink->h); | |
| 151 | − return AVERROR(EIO); | |
| 152 | − } | |
| 153 | − if (model_input.width != -1 && model_input.width != inlink->w) { | |
| 154 | − av_log(ctx, AV_LOG_ERROR, "the model requires frame width %d but got %d\n", | |
| 155 | − model_input.width, inlink->w); | |
| 156 | − return AVERROR(EIO); | |
| 157 | − } | |
| 158 | − | |
| 159 | − if (model_input.channels != 3) { | |
| 160 | − av_log(ctx, AV_LOG_ERROR, "the model requires input channels %d\n", | |
| 161 | − model_input.channels); | |
| 162 | − return AVERROR(EIO); | |
| 163 | − } | |
| 164 | − if (model_input.dt != DNN_FLOAT && model_input.dt != DNN_UINT8) { | |
| 165 | − av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type as float32 and uint8.\n"); | |
| 166 | − return AVERROR(EIO); | |
| 167 | + check = check_modelinput_inlink(&model_input, inlink); | |
| 168 | + if (check != 0) { | |
| 169 | + return check; | |
| 167 | 170 | } |
| 168 | 171 | |
| 169 | 172 | ctx->input.width = inlink->w; |