do not request AVFrame's format in vf_ddn_processing with 'fmt', but to add another filter for the format. command examples: ./ffmpeg -i input.jpg -vf format=bgr24,dnn_processing=model=halve_first_channel.model:input=dnn_in:output=dnn_out:dnn_backend=native -y out.native.png ./ffmpeg -i input.jpg -vf format=rgb24,dnn_processing=model=halve_first_channel.model:input=dnn_in:output=dnn_out:dnn_backend=native -y out.native.png
Showing the changes to libavfilter (1 of 2 files). Show all
Unified · Split
| @@ -37,7 +37,6 @@ typedef struct DnnProcessingContext { · 36 unmodified lines | |||
| 37 | 37 | ||
| 38 | char *model_filename; | 38 | char *model_filename; |
| 39 | DNNBackendType backend_type; | 39 | DNNBackendType backend_type; |
| 40 | enum AVPixelFormat fmt; | ||
| 41 | char *model_inputname; | 40 | char *model_inputname; |
| 42 | char *model_outputname; | 41 | char *model_outputname; |
| 43 | 42 | ||
| @@ -60,7 +59,6 @@ static const AVOption dnn_processing_options[] = { · 16 unmodified lines | |||
| 60 | { "model", "path to model file", OFFSET(model_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, | 59 | { "model", "path to model file", OFFSET(model_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, |
| 61 | { "input", "input name of the model", OFFSET(model_inputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, | 60 | { "input", "input name of the model", OFFSET(model_inputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, |
| 62 | { "output", "output name of the model", OFFSET(model_outputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, | 61 | { "output", "output name of the model", OFFSET(model_outputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, |
| 63 | { "fmt", "AVPixelFormat of the frame", OFFSET(fmt), AV_OPT_TYPE_PIXEL_FMT, { .i64=AV_PIX_FMT_RGB24 }, AV_PIX_FMT_NONE, AV_PIX_FMT_NB - 1, FLAGS }, | ||
| 64 | { NULL } | 62 | { NULL } |
| 65 | }; | 63 | }; |
| 66 | 64 | ||
| @@ -69,23 +67,6 @@ AVFILTER_DEFINE_CLASS(dnn_processing); · 2 unmodified lines | |||
| 69 | static av_cold int init(AVFilterContext *context) | 67 | static av_cold int init(AVFilterContext *context) |
| 70 | { | 68 | { |
| 71 | DnnProcessingContext *ctx = context->priv; | 69 | DnnProcessingContext *ctx = context->priv; |
| 72 | int supported = 0; | ||
| 73 | // as the first step, only rgb24 and bgr24 are supported | ||
| 74 | const enum AVPixelFormat supported_pixel_fmts[] = { | ||
| 75 | AV_PIX_FMT_RGB24, | ||
| 76 | AV_PIX_FMT_BGR24, | ||
| 77 | }; | ||
| 78 | for (int i = 0; i < sizeof(supported_pixel_fmts) / sizeof(enum AVPixelFormat); ++i) { | ||
| 79 | if (supported_pixel_fmts[i] == ctx->fmt) { | ||
| 80 | supported = 1; | ||
| 81 | break; | ||
| 82 | } | ||
| 83 | } | ||
| 84 | if (!supported) { | ||
| 85 | av_log(context, AV_LOG_ERROR, "pixel fmt %s not supported yet\n", | ||
| 86 | av_get_pix_fmt_name(ctx->fmt)); | ||
| 87 | return AVERROR(AVERROR_INVALIDDATA); | ||
| 88 | } | ||
| 89 | 70 | ||
| 90 | if (!ctx->model_filename) { | 71 | if (!ctx->model_filename) { |
| 91 | av_log(ctx, AV_LOG_ERROR, "model file for network is not specified\n"); | 72 | av_log(ctx, AV_LOG_ERROR, "model file for network is not specified\n"); |
| @@ -121,14 +102,52 @@ static av_cold int init(AVFilterContext *context) · 29 unmodified lines | |||
| 121 | 102 | ||
| 122 | static int query_formats(AVFilterContext *context) | 103 | static int query_formats(AVFilterContext *context) |
| 123 | { | 104 | { |
| 124 | AVFilterFormats *formats; | 105 | static const enum AVPixelFormat pix_fmts[] = { |
| 125 | DnnProcessingContext *ctx = context->priv; | 106 | AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24, |
| 126 | enum AVPixelFormat pixel_fmts[2]; | 107 | AV_PIX_FMT_NONE |
| 127 | pixel_fmts[0] = ctx->fmt; | 108 | }; |
| 128 | pixel_fmts[1] = AV_PIX_FMT_NONE; | 109 | AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts); |
| 110 | return ff_set_common_formats(context, fmts_list); | ||
| 111 | } | ||
| 112 | |||
| 113 | static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink) | ||
| 114 | { | ||
| 115 | AVFilterContext *ctx = inlink->dst; | ||
| 116 | enum AVPixelFormat fmt = inlink->format; | ||
| 117 | |||
| 118 | // the design is to add explicit scale filter before this filter | ||
| 119 | if (model_input->height != -1 && model_input->height != inlink->h) { | ||
| 120 | av_log(ctx, AV_LOG_ERROR, "the model requires frame height %d but got %d\n", | ||
| 121 | model_input->height, inlink->h); | ||
| 122 | return AVERROR(EIO); | ||
| 123 | } | ||
| 124 | if (model_input->width != -1 && model_input->width != inlink->w) { | ||
| 125 | av_log(ctx, AV_LOG_ERROR, "the model requires frame width %d but got %d\n", | ||
| 126 | model_input->width, inlink->w); | ||
| 127 | return AVERROR(EIO); | ||
| 128 | } | ||
| 129 | 129 | ||
| 130 | formats = ff_make_format_list(pixel_fmts); | 130 | switch (fmt) { |
| 131 | return ff_set_common_formats(context, formats); | 131 | case AV_PIX_FMT_RGB24: |
| 132 | case AV_PIX_FMT_BGR24: | ||
| 133 | if (model_input->channels != 3) { | ||
| 134 | av_log(ctx, AV_LOG_ERROR, "the frame's input format %s does not match " | ||
| 135 | "the model input channels %d\n", | ||
| 136 | av_get_pix_fmt_name(fmt), | ||
| 137 | model_input->channels); | ||
| 138 | return AVERROR(EIO); | ||
| 139 | } | ||
| 140 | if (model_input->dt != DNN_FLOAT && model_input->dt != DNN_UINT8) { | ||
| 141 | av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type as float32 and uint8.\n"); | ||
| 142 | return AVERROR(EIO); | ||
| 143 | } | ||
| 144 | break; | ||
| 145 | default: | ||
| 146 | av_log(ctx, AV_LOG_ERROR, "%s not supported.\n", av_get_pix_fmt_name(fmt)); | ||
| 147 | return AVERROR(EIO); | ||
| 148 | } | ||
| 149 | |||
| 150 | return 0; | ||
| 132 | } | 151 | } |
| 133 | 152 | ||
| 134 | static int config_input(AVFilterLink *inlink) | 153 | static int config_input(AVFilterLink *inlink) |
| @@ -137,6 +156,7 @@ static int config_input(AVFilterLink *inlink) · 2 unmodified lines | |||
| 137 | DnnProcessingContext *ctx = context->priv; | 156 | DnnProcessingContext *ctx = context->priv; |
| 138 | DNNReturnType result; | 157 | DNNReturnType result; |
| 139 | DNNData model_input; | 158 | DNNData model_input; |
| 159 | int check; | ||
| 140 | 160 | ||
| 141 | result = ctx->model->get_input(ctx->model->model, &model_input, ctx->model_inputname); | 161 | result = ctx->model->get_input(ctx->model->model, &model_input, ctx->model_inputname); |
| 142 | if (result != DNN_SUCCESS) { | 162 | if (result != DNN_SUCCESS) { |
| @@ -144,26 +164,9 @@ static int config_input(AVFilterLink *inlink) · 1 unmodified line | |||
| 144 | return AVERROR(EIO); | 164 | return AVERROR(EIO); |
| 145 | } | 165 | } |
| 146 | 166 | ||
| 147 | // the design is to add explicit scale filter before this filter | 167 | check = check_modelinput_inlink(&model_input, inlink); |
| 148 | if (model_input.height != -1 && model_input.height != inlink->h) { | 168 | if (check != 0) { |
| 149 | av_log(ctx, AV_LOG_ERROR, "the model requires frame height %d but got %d\n", | 169 | return check; |
| 150 | model_input.height, inlink->h); | ||
| 151 | return AVERROR(EIO); | ||
| 152 | } | ||
| 153 | if (model_input.width != -1 && model_input.width != inlink->w) { | ||
| 154 | av_log(ctx, AV_LOG_ERROR, "the model requires frame width %d but got %d\n", | ||
| 155 | model_input.width, inlink->w); | ||
| 156 | return AVERROR(EIO); | ||
| 157 | } | ||
| 158 | |||
| 159 | if (model_input.channels != 3) { | ||
| 160 | av_log(ctx, AV_LOG_ERROR, "the model requires input channels %d\n", | ||
| 161 | model_input.channels); | ||
| 162 | return AVERROR(EIO); | ||
| 163 | } | ||
| 164 | if (model_input.dt != DNN_FLOAT && model_input.dt != DNN_UINT8) { | ||
| 165 | av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type as float32 and uint8.\n"); | ||
| 166 | return AVERROR(EIO); | ||
| 167 | } | 170 | } |
| 168 | 171 | ||
| 169 | ctx->input.width = inlink->w; | 172 | ctx->input.width = inlink->w; |