different backend might need different options for a better performance, so, add the parameter into dnn interface, as a preparation.
Showing the changes to libavfilter (10 of 10 files). Show all
Unified · Split
| @@ -115,7 +115,7 @@ static DNNReturnType set_input_output_native(void *model, DNNData *input, const · 114 unmodified lines | |||
| 115 | // layers_num,layer_type,layer_parameterss,layer_type,layer_parameters... | 115 | // layers_num,layer_type,layer_parameterss,layer_type,layer_parameters... |
| 116 | // For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases | 116 | // For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases |
| 117 | // For DEPTH_TO_SPACE layer: block_size | 117 | // For DEPTH_TO_SPACE layer: block_size |
| 118 | DNNModel *ff_dnn_load_model_native(const char *model_filename) | 118 | DNNModel *ff_dnn_load_model_native(const char *model_filename, const char *options) |
| 119 | { | 119 | { |
| 120 | DNNModel *model = NULL; | 120 | DNNModel *model = NULL; |
| 121 | char header_expected[] = "FFMPEGDNNNATIVE"; | 121 | char header_expected[] = "FFMPEGDNNNATIVE"; |
| @@ -245,6 +245,7 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) · 123 unmodified lines | |||
| 245 | 245 | ||
| 246 | model->set_input_output = &set_input_output_native; | 246 | model->set_input_output = &set_input_output_native; |
| 247 | model->get_input = &get_input_native; | 247 | model->get_input = &get_input_native; |
| 248 | model->options = options; | ||
| 248 | 249 | ||
| 249 | return model; | 250 | return model; |
| 250 | 251 | ||
| @@ -116,7 +116,7 @@ typedef struct ConvolutionalNetwork{ · 115 unmodified lines | |||
| 116 | uint32_t nb_output; | 116 | uint32_t nb_output; |
| 117 | } ConvolutionalNetwork; | 117 | } ConvolutionalNetwork; |
| 118 | 118 | ||
| 119 | DNNModel *ff_dnn_load_model_native(const char *model_filename); | 119 | DNNModel *ff_dnn_load_model_native(const char *model_filename, const char *options); |
| 120 | 120 | ||
| 121 | DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output); | 121 | DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output); |
| 122 | 122 | ||
| @@ -155,7 +155,7 @@ err: · 154 unmodified lines | |||
| 155 | return DNN_ERROR; | 155 | return DNN_ERROR; |
| 156 | } | 156 | } |
| 157 | 157 | ||
| 158 | DNNModel *ff_dnn_load_model_ov(const char *model_filename) | 158 | DNNModel *ff_dnn_load_model_ov(const char *model_filename, const char *options) |
| 159 | { | 159 | { |
| 160 | DNNModel *model = NULL; | 160 | DNNModel *model = NULL; |
| 161 | OVModel *ov_model = NULL; | 161 | OVModel *ov_model = NULL; |
| @@ -186,6 +186,7 @@ DNNModel *ff_dnn_load_model_ov(const char *model_filename) · 24 unmodified lines | |||
| 186 | model->model = (void *)ov_model; | 186 | model->model = (void *)ov_model; |
| 187 | model->set_input_output = &set_input_output_ov; | 187 | model->set_input_output = &set_input_output_ov; |
| 188 | model->get_input = &get_input_ov; | 188 | model->get_input = &get_input_ov; |
| 189 | model->options = options; | ||
| 189 | 190 | ||
| 190 | return model; | 191 | return model; |
| 191 | 192 | ||
| @@ -29,7 +29,7 @@ · 28 unmodified lines | |||
| 29 | 29 | ||
| 30 | #include "../dnn_interface.h" | 30 | #include "../dnn_interface.h" |
| 31 | 31 | ||
| 32 | DNNModel *ff_dnn_load_model_ov(const char *model_filename); | 32 | DNNModel *ff_dnn_load_model_ov(const char *model_filename, const char *options); |
| 33 | 33 | ||
| 34 | DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNData *outputs, uint32_t nb_output); | 34 | DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNData *outputs, uint32_t nb_output); |
| 35 | 35 | ||
| @@ -572,7 +572,7 @@ static DNNReturnType load_native_model(TFModel *tf_model, const char *model_file · 571 unmodified lines | |||
| 572 | return DNN_SUCCESS; | 572 | return DNN_SUCCESS; |
| 573 | } | 573 | } |
| 574 | 574 | ||
| 575 | DNNModel *ff_dnn_load_model_tf(const char *model_filename) | 575 | DNNModel *ff_dnn_load_model_tf(const char *model_filename, const char *options) |
| 576 | { | 576 | { |
| 577 | DNNModel *model = NULL; | 577 | DNNModel *model = NULL; |
| 578 | TFModel *tf_model = NULL; | 578 | TFModel *tf_model = NULL; |
| @@ -600,6 +600,7 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename) · 21 unmodified lines | |||
| 600 | model->model = (void *)tf_model; | 600 | model->model = (void *)tf_model; |
| 601 | model->set_input_output = &set_input_output_tf; | 601 | model->set_input_output = &set_input_output_tf; |
| 602 | model->get_input = &get_input_tf; | 602 | model->get_input = &get_input_tf; |
| 603 | model->options = options; | ||
| 603 | 604 | ||
| 604 | return model; | 605 | return model; |
| 605 | } | 606 | } |
| @@ -29,7 +29,7 @@ · 28 unmodified lines | |||
| 29 | 29 | ||
| 30 | #include "../dnn_interface.h" | 30 | #include "../dnn_interface.h" |
| 31 | 31 | ||
| 32 | DNNModel *ff_dnn_load_model_tf(const char *model_filename); | 32 | DNNModel *ff_dnn_load_model_tf(const char *model_filename, const char *options); |
| 33 | 33 | ||
| 34 | DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output); | 34 | DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output); |
| 35 | 35 | ||
| @@ -43,6 +43,8 @@ typedef struct DNNData{ · 42 unmodified lines | |||
| 43 | typedef struct DNNModel{ | 43 | typedef struct DNNModel{ |
| 44 | // Stores model that can be different for different backends. | 44 | // Stores model that can be different for different backends. |
| 45 | void *model; | 45 | void *model; |
| 46 | // Stores options when the model is executed by the backend | ||
| 47 | const char *options; | ||
| 46 | // Gets model input information | 48 | // Gets model input information |
| 47 | // Just reuse struct DNNData here, actually the DNNData.data field is not needed. | 49 | // Just reuse struct DNNData here, actually the DNNData.data field is not needed. |
| 48 | DNNReturnType (*get_input)(void *model, DNNData *input, const char *input_name); | 50 | DNNReturnType (*get_input)(void *model, DNNData *input, const char *input_name); |
| @@ -54,7 +56,7 @@ typedef struct DNNModel{ · 5 unmodified lines | |||
| 54 | // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends. | 56 | // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends. |
| 55 | typedef struct DNNModule{ | 57 | typedef struct DNNModule{ |
| 56 | // Loads model and parameters from given file. Returns NULL if it is not possible. | 58 | // Loads model and parameters from given file. Returns NULL if it is not possible. |
| 57 | DNNModel *(*load_model)(const char *model_filename); | 59 | DNNModel *(*load_model)(const char *model_filename, const char *options); |
| 58 | // Executes model with specified input and output. Returns DNN_ERROR otherwise. | 60 | // Executes model with specified input and output. Returns DNN_ERROR otherwise. |
| 59 | DNNReturnType (*execute_model)(const DNNModel *model, DNNData *outputs, uint32_t nb_output); | 61 | DNNReturnType (*execute_model)(const DNNModel *model, DNNData *outputs, uint32_t nb_output); |
| 60 | // Frees memory allocated for model. | 62 | // Frees memory allocated for model. |
| @@ -161,7 +161,7 @@ static av_cold int init(AVFilterContext *ctx) · 160 unmodified lines | |||
| 161 | return AVERROR(EINVAL); | 161 | return AVERROR(EINVAL); |
| 162 | } | 162 | } |
| 163 | 163 | ||
| 164 | dr_context->model = (dr_context->dnn_module->load_model)(dr_context->model_filename); | 164 | dr_context->model = (dr_context->dnn_module->load_model)(dr_context->model_filename, NULL); |
| 165 | if (!dr_context->model) { | 165 | if (!dr_context->model) { |
| 166 | av_log(ctx, AV_LOG_ERROR, "could not load DNN model\n"); | 166 | av_log(ctx, AV_LOG_ERROR, "could not load DNN model\n"); |
| 167 | return AVERROR(EINVAL); | 167 | return AVERROR(EINVAL); |
| @@ -41,6 +41,7 @@ typedef struct DnnProcessingContext { · 40 unmodified lines | |||
| 41 | DNNBackendType backend_type; | 41 | DNNBackendType backend_type; |
| 42 | char *model_inputname; | 42 | char *model_inputname; |
| 43 | char *model_outputname; | 43 | char *model_outputname; |
| 44 | char *backend_options; | ||
| 44 | 45 | ||
| 45 | DNNModule *dnn_module; | 46 | DNNModule *dnn_module; |
| 46 | DNNModel *model; | 47 | DNNModel *model; |
| @@ -69,6 +70,7 @@ static const AVOption dnn_processing_options[] = { · 22 unmodified lines | |||
| 69 | { "model", "path to model file", OFFSET(model_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, | 70 | { "model", "path to model file", OFFSET(model_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, |
| 70 | { "input", "input name of the model", OFFSET(model_inputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, | 71 | { "input", "input name of the model", OFFSET(model_inputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, |
| 71 | { "output", "output name of the model", OFFSET(model_outputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, | 72 | { "output", "output name of the model", OFFSET(model_outputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, |
| 73 | { "options", "backend options", OFFSET(backend_options), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, | ||
| 72 | { NULL } | 74 | { NULL } |
| 73 | }; | 75 | }; |
| 74 | 76 | ||
| @@ -101,7 +103,7 @@ static av_cold int init(AVFilterContext *context) · 26 unmodified lines | |||
| 101 | return AVERROR(EINVAL); | 103 | return AVERROR(EINVAL); |
| 102 | } | 104 | } |
| 103 | 105 | ||
| 104 | ctx->model = (ctx->dnn_module->load_model)(ctx->model_filename); | 106 | ctx->model = (ctx->dnn_module->load_model)(ctx->model_filename, ctx->backend_options); |
| 105 | if (!ctx->model) { | 107 | if (!ctx->model) { |
| 106 | av_log(ctx, AV_LOG_ERROR, "could not load DNN model\n"); | 108 | av_log(ctx, AV_LOG_ERROR, "could not load DNN model\n"); |
| 107 | return AVERROR(EINVAL); | 109 | return AVERROR(EINVAL); |
| @@ -81,7 +81,7 @@ static av_cold int init(AVFilterContext *context) · 80 unmodified lines | |||
| 81 | av_log(context, AV_LOG_ERROR, "load_model for network was not specified\n"); | 81 | av_log(context, AV_LOG_ERROR, "load_model for network was not specified\n"); |
| 82 | return AVERROR(EIO); | 82 | return AVERROR(EIO); |
| 83 | } | 83 | } |
| 84 | sr_context->model = (sr_context->dnn_module->load_model)(sr_context->model_filename); | 84 | sr_context->model = (sr_context->dnn_module->load_model)(sr_context->model_filename, NULL); |
| 85 | if (!sr_context->model){ | 85 | if (!sr_context->model){ |
| 86 | av_log(context, AV_LOG_ERROR, "could not load DNN model\n"); | 86 | av_log(context, AV_LOG_ERROR, "could not load DNN model\n"); |
| 87 | return AVERROR(EIO); | 87 | return AVERROR(EIO); |