This commit uses the common TaskItem and InferenceItem typedefs for execution in TensorFlow backend.
Showing the changes to libavfilter (1 of 1 file). Show all
Unified · Split
| @@ -35,6 +35,7 @@ · 34 unmodified lines | ||
| 35 | 35 | #include "dnn_backend_native_layer_maximum.h" |
| 36 | 36 | #include "dnn_io_proc.h" |
| 37 | 37 | #include "dnn_backend_common.h" |
| 38 | +#include "queue.h" | |
| 38 | 39 | #include <tensorflow/c/c_api.h> |
| 39 | 40 | |
| 40 | 41 | typedef struct TFOptions{ |
| @@ -52,6 +53,7 @@ typedef struct TFModel{ · 11 unmodified lines | ||
| 52 | 53 | TF_Graph *graph; |
| 53 | 54 | TF_Session *session; |
| 54 | 55 | TF_Status *status; |
| 56 | + Queue *inference_queue; | |
| 55 | 57 | } TFModel; |
| 56 | 58 | |
| 57 | 59 | #define OFFSET(x) offsetof(TFContext, x) |
| @@ -63,15 +65,29 @@ static const AVOption dnn_tensorflow_options[] = { · 5 unmodified lines | ||
| 63 | 65 | |
| 64 | 66 | AVFILTER_DEFINE_CLASS(dnn_tensorflow); |
| 65 | 67 | |
| 66 | −static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_name, AVFrame *in_frame, | |
| 67 | − const char **output_names, uint32_t nb_output, AVFrame *out_frame, | |
| 68 | − int do_ioproc); | |
| 68 | +static DNNReturnType execute_model_tf(Queue *inference_queue); | |
| 69 | 69 | |
| 70 | 70 | static void free_buffer(void *data, size_t length) |
| 71 | 71 | { |
| 72 | 72 | av_freep(&data); |
| 73 | 73 | } |
| 74 | 74 | |
| 75 | +static DNNReturnType extract_inference_from_task(TaskItem *task, Queue *inference_queue) | |
| 76 | +{ | |
| 77 | + InferenceItem *inference = av_malloc(sizeof(*inference)); | |
| 78 | + if (!inference) { | |
| 79 | + return DNN_ERROR; | |
| 80 | + } | |
| 81 | + task->inference_todo = 1; | |
| 82 | + task->inference_done = 0; | |
| 83 | + inference->task = task; | |
| 84 | + if (ff_queue_push_back(inference_queue, inference) < 0) { | |
| 85 | + av_freep(&inference); | |
| 86 | + return DNN_ERROR; | |
| 87 | + } | |
| 88 | + return DNN_SUCCESS; | |
| 89 | +} | |
| 90 | + | |
| 75 | 91 | static TF_Buffer *read_graph(const char *model_filename) |
| 76 | 92 | { |
| 77 | 93 | TF_Buffer *graph_buf; |
| @@ -171,6 +187,7 @@ static DNNReturnType get_output_tf(void *model, const char *input_name, int inpu · 93 unmodified lines | ||
| 171 | 187 | TFContext *ctx = &tf_model->ctx; |
| 172 | 188 | AVFrame *in_frame = av_frame_alloc(); |
| 173 | 189 | AVFrame *out_frame = NULL; |
| 190 | + TaskItem task; | |
| 174 | 191 | |
| 175 | 192 | if (!in_frame) { |
| 176 | 193 | av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input frame\n"); |
| @@ -187,7 +204,21 @@ static DNNReturnType get_output_tf(void *model, const char *input_name, int inpu · 10 unmodified lines | ||
| 187 | 204 | in_frame->width = input_width; |
| 188 | 205 | in_frame->height = input_height; |
| 189 | 206 | |
| 190 | − ret = execute_model_tf(tf_model->model, input_name, in_frame, &output_name, 1, out_frame, 0); | |
| 207 | + task.do_ioproc = 0; | |
| 208 | + task.async = 0; | |
| 209 | + task.input_name = input_name; | |
| 210 | + task.in_frame = in_frame; | |
| 211 | + task.output_names = &output_name; | |
| 212 | + task.out_frame = out_frame; | |
| 213 | + task.model = tf_model; | |
| 214 | + task.nb_output = 1; | |
| 215 | + | |
| 216 | + if (extract_inference_from_task(&task, tf_model->inference_queue) != DNN_SUCCESS) { | |
| 217 | + av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n"); | |
| 218 | + return DNN_ERROR; | |
| 219 | + } | |
| 220 | + | |
| 221 | + ret = execute_model_tf(tf_model->inference_queue); | |
| 191 | 222 | *output_width = out_frame->width; |
| 192 | 223 | *output_height = out_frame->height; |
| 193 | 224 | |
| @@ -723,6 +754,7 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_ · 529 unmodified lines | ||
| 723 | 754 | } |
| 724 | 755 | } |
| 725 | 756 | |
| 757 | + tf_model->inference_queue = ff_queue_create(); | |
| 726 | 758 | model->model = tf_model; |
| 727 | 759 | model->get_input = &get_input_tf; |
| 728 | 760 | model->get_output = &get_output_tf; |
| @@ -733,26 +765,33 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_ · 4 unmodified lines | ||
| 733 | 765 | return model; |
| 734 | 766 | } |
| 735 | 767 | |
| 736 | −static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_name, AVFrame *in_frame, | |
| 737 | − const char **output_names, uint32_t nb_output, AVFrame *out_frame, | |
| 738 | − int do_ioproc) | |
| 768 | +static DNNReturnType execute_model_tf(Queue *inference_queue) | |
| 739 | 769 | { |
| 740 | 770 | TF_Output *tf_outputs; |
| 741 | − TFModel *tf_model = model->model; | |
| 742 | − TFContext *ctx = &tf_model->ctx; | |
| 771 | + TFModel *tf_model; | |
| 772 | + TFContext *ctx; | |
| 773 | + InferenceItem *inference; | |
| 774 | + TaskItem *task; | |
| 743 | 775 | DNNData input, *outputs; |
| 744 | 776 | TF_Tensor **output_tensors; |
| 745 | 777 | TF_Output tf_input; |
| 746 | 778 | TF_Tensor *input_tensor; |
| 747 | 779 | |
| 748 | − if (get_input_tf(tf_model, &input, input_name) != DNN_SUCCESS) | |
| 780 | + inference = ff_queue_pop_front(inference_queue); | |
| 781 | + av_assert0(inference); | |
| 782 | + task = inference->task; | |
| 783 | + tf_model = task->model; | |
| 784 | + ctx = &tf_model->ctx; | |
| 785 | + | |
| 786 | + if (get_input_tf(tf_model, &input, task->input_name) != DNN_SUCCESS) | |
| 749 | 787 | return DNN_ERROR; |
| 750 | − input.height = in_frame->height; | |
| 751 | − input.width = in_frame->width; | |
| 752 | 788 | |
| 753 | − tf_input.oper = TF_GraphOperationByName(tf_model->graph, input_name); | |
| 789 | + input.height = task->in_frame->height; | |
| 790 | + input.width = task->in_frame->width; | |
| 791 | + | |
| 792 | + tf_input.oper = TF_GraphOperationByName(tf_model->graph, task->input_name); | |
| 754 | 793 | if (!tf_input.oper){ |
| 755 | − av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name); | |
| 794 | + av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", task->input_name); | |
| 756 | 795 | return DNN_ERROR; |
| 757 | 796 | } |
| 758 | 797 | tf_input.index = 0; |
| @@ -765,30 +804,30 @@ static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_n · 6 unmodified lines | ||
| 765 | 804 | |
| 766 | 805 | switch (tf_model->model->func_type) { |
| 767 | 806 | case DFT_PROCESS_FRAME: |
| 768 | − if (do_ioproc) { | |
| 807 | + if (task->do_ioproc) { | |
| 769 | 808 | if (tf_model->model->frame_pre_proc != NULL) { |
| 770 | − tf_model->model->frame_pre_proc(in_frame, &input, tf_model->model->filter_ctx); | |
| 809 | + tf_model->model->frame_pre_proc(task->in_frame, &input, tf_model->model->filter_ctx); | |
| 771 | 810 | } else { |
| 772 | − ff_proc_from_frame_to_dnn(in_frame, &input, ctx); | |
| 811 | + ff_proc_from_frame_to_dnn(task->in_frame, &input, ctx); | |
| 773 | 812 | } |
| 774 | 813 | } |
| 775 | 814 | break; |
| 776 | 815 | case DFT_ANALYTICS_DETECT: |
| 777 | − ff_frame_to_dnn_detect(in_frame, &input, ctx); | |
| 816 | + ff_frame_to_dnn_detect(task->in_frame, &input, ctx); | |
| 778 | 817 | break; |
| 779 | 818 | default: |
| 780 | 819 | avpriv_report_missing_feature(ctx, "model function type %d", tf_model->model->func_type); |
| 781 | 820 | break; |
| 782 | 821 | } |
| 783 | 822 | |
| 784 | − tf_outputs = av_malloc_array(nb_output, sizeof(*tf_outputs)); | |
| 823 | + tf_outputs = av_malloc_array(task->nb_output, sizeof(TF_Output)); | |
| 785 | 824 | if (tf_outputs == NULL) { |
| 786 | 825 | TF_DeleteTensor(input_tensor); |
| 787 | 826 | av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *tf_outputs\n"); \ |
| 788 | 827 | return DNN_ERROR; |
| 789 | 828 | } |
| 790 | 829 | |
| 791 | − output_tensors = av_mallocz_array(nb_output, sizeof(*output_tensors)); | |
| 830 | + output_tensors = av_mallocz_array(task->nb_output, sizeof(*output_tensors)); | |
| 792 | 831 | if (!output_tensors) { |
| 793 | 832 | TF_DeleteTensor(input_tensor); |
| 794 | 833 | av_freep(&tf_outputs); |
| @@ -796,13 +835,13 @@ static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_n · 1 unmodified line | ||
| 796 | 835 | return DNN_ERROR; |
| 797 | 836 | } |
| 798 | 837 | |
| 799 | − for (int i = 0; i < nb_output; ++i) { | |
| 800 | − tf_outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]); | |
| 838 | + for (int i = 0; i < task->nb_output; ++i) { | |
| 839 | + tf_outputs[i].oper = TF_GraphOperationByName(tf_model->graph, task->output_names[i]); | |
| 801 | 840 | if (!tf_outputs[i].oper) { |
| 802 | 841 | TF_DeleteTensor(input_tensor); |
| 803 | 842 | av_freep(&tf_outputs); |
| 804 | 843 | av_freep(&output_tensors); |
| 805 | − av_log(ctx, AV_LOG_ERROR, "Could not find output \"%s\" in model\n", output_names[i]); \ | |
| 844 | + av_log(ctx, AV_LOG_ERROR, "Could not find output \"%s\" in model\n", task->output_names[i]); \ | |
| 806 | 845 | return DNN_ERROR; |
| 807 | 846 | } |
| 808 | 847 | tf_outputs[i].index = 0; |
| @@ -810,7 +849,7 @@ static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_n · 1 unmodified line | ||
| 810 | 849 | |
| 811 | 850 | TF_SessionRun(tf_model->session, NULL, |
| 812 | 851 | &tf_input, &input_tensor, 1, |
| 813 | − tf_outputs, output_tensors, nb_output, | |
| 852 | + tf_outputs, output_tensors, task->nb_output, | |
| 814 | 853 | NULL, 0, NULL, tf_model->status); |
| 815 | 854 | if (TF_GetCode(tf_model->status) != TF_OK) { |
| 816 | 855 | TF_DeleteTensor(input_tensor); |
| @@ -820,7 +859,7 @@ static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_n · 3 unmodified lines | ||
| 820 | 859 | return DNN_ERROR; |
| 821 | 860 | } |
| 822 | 861 | |
| 823 | − outputs = av_malloc_array(nb_output, sizeof(*outputs)); | |
| 862 | + outputs = av_malloc_array(task->nb_output, sizeof(*outputs)); | |
| 824 | 863 | if (!outputs) { |
| 825 | 864 | TF_DeleteTensor(input_tensor); |
| 826 | 865 | av_freep(&tf_outputs); |
| @@ -829,36 +868,36 @@ static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_n · 2 unmodified lines | ||
| 829 | 868 | return DNN_ERROR; |
| 830 | 869 | } |
| 831 | 870 | |
| 832 | − for (uint32_t i = 0; i < nb_output; ++i) { | |
| 871 | + for (uint32_t i = 0; i < task->nb_output; ++i) { | |
| 833 | 872 | outputs[i].height = TF_Dim(output_tensors[i], 1); |
| 834 | 873 | outputs[i].width = TF_Dim(output_tensors[i], 2); |
| 835 | 874 | outputs[i].channels = TF_Dim(output_tensors[i], 3); |
| 836 | 875 | outputs[i].data = TF_TensorData(output_tensors[i]); |
| 837 | 876 | outputs[i].dt = TF_TensorType(output_tensors[i]); |
| 838 | 877 | } |
| 839 | − switch (model->func_type) { | |
| 878 | + switch (tf_model->model->func_type) { | |
| 840 | 879 | case DFT_PROCESS_FRAME: |
| 841 | 880 | //it only support 1 output if it's frame in & frame out |
| 842 | − if (do_ioproc) { | |
| 881 | + if (task->do_ioproc) { | |
| 843 | 882 | if (tf_model->model->frame_post_proc != NULL) { |
| 844 | − tf_model->model->frame_post_proc(out_frame, outputs, tf_model->model->filter_ctx); | |
| 883 | + tf_model->model->frame_post_proc(task->out_frame, outputs, tf_model->model->filter_ctx); | |
| 845 | 884 | } else { |
| 846 | − ff_proc_from_dnn_to_frame(out_frame, outputs, ctx); | |
| 885 | + ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx); | |
| 847 | 886 | } |
| 848 | 887 | } else { |
| 849 | − out_frame->width = outputs[0].width; | |
| 850 | − out_frame->height = outputs[0].height; | |
| 888 | + task->out_frame->width = outputs[0].width; | |
| 889 | + task->out_frame->height = outputs[0].height; | |
| 851 | 890 | } |
| 852 | 891 | break; |
| 853 | 892 | case DFT_ANALYTICS_DETECT: |
| 854 | − if (!model->detect_post_proc) { | |
| 893 | + if (!tf_model->model->detect_post_proc) { | |
| 855 | 894 | av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post proc\n"); |
| 856 | 895 | return DNN_ERROR; |
| 857 | 896 | } |
| 858 | − model->detect_post_proc(out_frame, outputs, nb_output, model->filter_ctx); | |
| 897 | + tf_model->model->detect_post_proc(task->out_frame, outputs, task->nb_output, tf_model->model->filter_ctx); | |
| 859 | 898 | break; |
| 860 | 899 | default: |
| 861 | − for (uint32_t i = 0; i < nb_output; ++i) { | |
| 900 | + for (uint32_t i = 0; i < task->nb_output; ++i) { | |
| 862 | 901 | if (output_tensors[i]) { |
| 863 | 902 | TF_DeleteTensor(output_tensors[i]); |
| 864 | 903 | } |
| @@ -871,30 +910,39 @@ static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_n · 6 unmodified lines | ||
| 871 | 910 | av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this kind of dnn filter now\n"); |
| 872 | 911 | return DNN_ERROR; |
| 873 | 912 | } |
| 874 | − | |
| 875 | − for (uint32_t i = 0; i < nb_output; ++i) { | |
| 913 | + for (uint32_t i = 0; i < task->nb_output; ++i) { | |
| 876 | 914 | if (output_tensors[i]) { |
| 877 | 915 | TF_DeleteTensor(output_tensors[i]); |
| 878 | 916 | } |
| 879 | 917 | } |
| 918 | + task->inference_done++; | |
| 880 | 919 | TF_DeleteTensor(input_tensor); |
| 881 | 920 | av_freep(&output_tensors); |
| 882 | 921 | av_freep(&tf_outputs); |
| 883 | 922 | av_freep(&outputs); |
| 884 | 923 | return DNN_SUCCESS; |
| 924 | + return (task->inference_done == task->inference_todo) ? DNN_SUCCESS : DNN_ERROR; | |
| 885 | 925 | } |
| 886 | 926 | |
| 887 | 927 | DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params) |
| 888 | 928 | { |
| 889 | 929 | TFModel *tf_model = model->model; |
| 890 | 930 | TFContext *ctx = &tf_model->ctx; |
| 931 | + TaskItem task; | |
| 891 | 932 | |
| 892 | 933 | if (ff_check_exec_params(ctx, DNN_TF, model->func_type, exec_params) != 0) { |
| 893 | − return DNN_ERROR; | |
| 934 | + return DNN_ERROR; | |
| 894 | 935 | } |
| 895 | 936 | |
| 896 | − return execute_model_tf(model, exec_params->input_name, exec_params->in_frame, | |
| 897 | − exec_params->output_names, exec_params->nb_output, exec_params->out_frame, 1); | |
| 937 | + if (ff_dnn_fill_task(&task, exec_params, tf_model, 0, 1) != DNN_SUCCESS) { | |
| 938 | + return DNN_ERROR; | |
| 939 | + } | |
| 940 | + | |
| 941 | + if (extract_inference_from_task(&task, tf_model->inference_queue) != DNN_SUCCESS) { | |
| 942 | + av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n"); | |
| 943 | + return DNN_ERROR; | |
| 944 | + } | |
| 945 | + return execute_model_tf(tf_model->inference_queue); | |
| 898 | 946 | } |
| 899 | 947 | |
| 900 | 948 | void ff_dnn_free_model_tf(DNNModel **model) |
| @@ -903,6 +951,12 @@ void ff_dnn_free_model_tf(DNNModel **model) · 2 unmodified lines | ||
| 903 | 951 | |
| 904 | 952 | if (*model){ |
| 905 | 953 | tf_model = (*model)->model; |
| 954 | + while (ff_queue_size(tf_model->inference_queue) != 0) { | |
| 955 | + InferenceItem *item = ff_queue_pop_front(tf_model->inference_queue); | |
| 956 | + av_freep(&item); | |
| 957 | + } | |
| 958 | + ff_queue_destroy(tf_model->inference_queue); | |
| 959 | + | |
| 906 | 960 | if (tf_model->graph){ |
| 907 | 961 | TF_DeleteGraph(tf_model->graph); |
| 908 | 962 | } |