This commit uses the common TaskItem and InferenceItem typedefs for execution in TensorFlow backend.
Showing the changes to libavfilter (1 of 1 file). Show all
Unified · Split
| @@ -35,6 +35,7 @@ · 34 unmodified lines | |||
| 35 | #include "dnn_backend_native_layer_maximum.h" | 35 | #include "dnn_backend_native_layer_maximum.h" |
| 36 | #include "dnn_io_proc.h" | 36 | #include "dnn_io_proc.h" |
| 37 | #include "dnn_backend_common.h" | 37 | #include "dnn_backend_common.h" |
| 38 | #include "queue.h" | ||
| 38 | #include <tensorflow/c/c_api.h> | 39 | #include <tensorflow/c/c_api.h> |
| 39 | 40 | ||
| 40 | typedef struct TFOptions{ | 41 | typedef struct TFOptions{ |
| @@ -52,6 +53,7 @@ typedef struct TFModel{ · 11 unmodified lines | |||
| 52 | TF_Graph *graph; | 53 | TF_Graph *graph; |
| 53 | TF_Session *session; | 54 | TF_Session *session; |
| 54 | TF_Status *status; | 55 | TF_Status *status; |
| 56 | Queue *inference_queue; | ||
| 55 | } TFModel; | 57 | } TFModel; |
| 56 | 58 | ||
| 57 | #define OFFSET(x) offsetof(TFContext, x) | 59 | #define OFFSET(x) offsetof(TFContext, x) |
| @@ -63,15 +65,29 @@ static const AVOption dnn_tensorflow_options[] = { · 5 unmodified lines | |||
| 63 | 65 | ||
| 64 | AVFILTER_DEFINE_CLASS(dnn_tensorflow); | 66 | AVFILTER_DEFINE_CLASS(dnn_tensorflow); |
| 65 | 67 | ||
| 66 | static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_name, AVFrame *in_frame, | 68 | static DNNReturnType execute_model_tf(Queue *inference_queue); |
| 67 | const char **output_names, uint32_t nb_output, AVFrame *out_frame, | ||
| 68 | int do_ioproc); | ||
| 69 | 69 | ||
| 70 | static void free_buffer(void *data, size_t length) | 70 | static void free_buffer(void *data, size_t length) |
| 71 | { | 71 | { |
| 72 | av_freep(&data); | 72 | av_freep(&data); |
| 73 | } | 73 | } |
| 74 | 74 | ||
| 75 | static DNNReturnType extract_inference_from_task(TaskItem *task, Queue *inference_queue) | ||
| 76 | { | ||
| 77 | InferenceItem *inference = av_malloc(sizeof(*inference)); | ||
| 78 | if (!inference) { | ||
| 79 | return DNN_ERROR; | ||
| 80 | } | ||
| 81 | task->inference_todo = 1; | ||
| 82 | task->inference_done = 0; | ||
| 83 | inference->task = task; | ||
| 84 | if (ff_queue_push_back(inference_queue, inference) < 0) { | ||
| 85 | av_freep(&inference); | ||
| 86 | return DNN_ERROR; | ||
| 87 | } | ||
| 88 | return DNN_SUCCESS; | ||
| 89 | } | ||
| 90 | |||
| 75 | static TF_Buffer *read_graph(const char *model_filename) | 91 | static TF_Buffer *read_graph(const char *model_filename) |
| 76 | { | 92 | { |
| 77 | TF_Buffer *graph_buf; | 93 | TF_Buffer *graph_buf; |
| @@ -171,6 +187,7 @@ static DNNReturnType get_output_tf(void *model, const char *input_name, int inpu · 93 unmodified lines | |||
| 171 | TFContext *ctx = &tf_model->ctx; | 187 | TFContext *ctx = &tf_model->ctx; |
| 172 | AVFrame *in_frame = av_frame_alloc(); | 188 | AVFrame *in_frame = av_frame_alloc(); |
| 173 | AVFrame *out_frame = NULL; | 189 | AVFrame *out_frame = NULL; |
| 190 | TaskItem task; | ||
| 174 | 191 | ||
| 175 | if (!in_frame) { | 192 | if (!in_frame) { |
| 176 | av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input frame\n"); | 193 | av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input frame\n"); |
| @@ -187,7 +204,21 @@ static DNNReturnType get_output_tf(void *model, const char *input_name, int inpu · 10 unmodified lines | |||
| 187 | in_frame->width = input_width; | 204 | in_frame->width = input_width; |
| 188 | in_frame->height = input_height; | 205 | in_frame->height = input_height; |
| 189 | 206 | ||
| 190 | ret = execute_model_tf(tf_model->model, input_name, in_frame, &output_name, 1, out_frame, 0); | 207 | task.do_ioproc = 0; |
| 208 | task.async = 0; | ||
| 209 | task.input_name = input_name; | ||
| 210 | task.in_frame = in_frame; | ||
| 211 | task.output_names = &output_name; | ||
| 212 | task.out_frame = out_frame; | ||
| 213 | task.model = tf_model; | ||
| 214 | task.nb_output = 1; | ||
| 215 | |||
| 216 | if (extract_inference_from_task(&task, tf_model->inference_queue) != DNN_SUCCESS) { | ||
| 217 | av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n"); | ||
| 218 | return DNN_ERROR; | ||
| 219 | } | ||
| 220 | |||
| 221 | ret = execute_model_tf(tf_model->inference_queue); | ||
| 191 | *output_width = out_frame->width; | 222 | *output_width = out_frame->width; |
| 192 | *output_height = out_frame->height; | 223 | *output_height = out_frame->height; |
| 193 | 224 | ||
| @@ -723,6 +754,7 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_ · 529 unmodified lines | |||
| 723 | } | 754 | } |
| 724 | } | 755 | } |
| 725 | 756 | ||
| 757 | tf_model->inference_queue = ff_queue_create(); | ||
| 726 | model->model = tf_model; | 758 | model->model = tf_model; |
| 727 | model->get_input = &get_input_tf; | 759 | model->get_input = &get_input_tf; |
| 728 | model->get_output = &get_output_tf; | 760 | model->get_output = &get_output_tf; |
| @@ -733,26 +765,33 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_ · 4 unmodified lines | |||
| 733 | return model; | 765 | return model; |
| 734 | } | 766 | } |
| 735 | 767 | ||
| 736 | static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_name, AVFrame *in_frame, | 768 | static DNNReturnType execute_model_tf(Queue *inference_queue) |
| 737 | const char **output_names, uint32_t nb_output, AVFrame *out_frame, | ||
| 738 | int do_ioproc) | ||
| 739 | { | 769 | { |
| 740 | TF_Output *tf_outputs; | 770 | TF_Output *tf_outputs; |
| 741 | TFModel *tf_model = model->model; | 771 | TFModel *tf_model; |
| 742 | TFContext *ctx = &tf_model->ctx; | 772 | TFContext *ctx; |
| 773 | InferenceItem *inference; | ||
| 774 | TaskItem *task; | ||
| 743 | DNNData input, *outputs; | 775 | DNNData input, *outputs; |
| 744 | TF_Tensor **output_tensors; | 776 | TF_Tensor **output_tensors; |
| 745 | TF_Output tf_input; | 777 | TF_Output tf_input; |
| 746 | TF_Tensor *input_tensor; | 778 | TF_Tensor *input_tensor; |
| 747 | 779 | ||
| 748 | if (get_input_tf(tf_model, &input, input_name) != DNN_SUCCESS) | 780 | inference = ff_queue_pop_front(inference_queue); |
| 781 | av_assert0(inference); | ||
| 782 | task = inference->task; | ||
| 783 | tf_model = task->model; | ||
| 784 | ctx = &tf_model->ctx; | ||
| 785 | |||
| 786 | if (get_input_tf(tf_model, &input, task->input_name) != DNN_SUCCESS) | ||
| 749 | return DNN_ERROR; | 787 | return DNN_ERROR; |
| 750 | input.height = in_frame->height; | ||
| 751 | input.width = in_frame->width; | ||
| 752 | 788 | ||
| 753 | tf_input.oper = TF_GraphOperationByName(tf_model->graph, input_name); | 789 | input.height = task->in_frame->height; |
| 790 | input.width = task->in_frame->width; | ||
| 791 | |||
| 792 | tf_input.oper = TF_GraphOperationByName(tf_model->graph, task->input_name); | ||
| 754 | if (!tf_input.oper){ | 793 | if (!tf_input.oper){ |
| 755 | av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name); | 794 | av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", task->input_name); |
| 756 | return DNN_ERROR; | 795 | return DNN_ERROR; |
| 757 | } | 796 | } |
| 758 | tf_input.index = 0; | 797 | tf_input.index = 0; |
| @@ -765,30 +804,30 @@ static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_n · 6 unmodified lines | |||
| 765 | 804 | ||
| 766 | switch (tf_model->model->func_type) { | 805 | switch (tf_model->model->func_type) { |
| 767 | case DFT_PROCESS_FRAME: | 806 | case DFT_PROCESS_FRAME: |
| 768 | if (do_ioproc) { | 807 | if (task->do_ioproc) { |
| 769 | if (tf_model->model->frame_pre_proc != NULL) { | 808 | if (tf_model->model->frame_pre_proc != NULL) { |
| 770 | tf_model->model->frame_pre_proc(in_frame, &input, tf_model->model->filter_ctx); | 809 | tf_model->model->frame_pre_proc(task->in_frame, &input, tf_model->model->filter_ctx); |
| 771 | } else { | 810 | } else { |
| 772 | ff_proc_from_frame_to_dnn(in_frame, &input, ctx); | 811 | ff_proc_from_frame_to_dnn(task->in_frame, &input, ctx); |
| 773 | } | 812 | } |
| 774 | } | 813 | } |
| 775 | break; | 814 | break; |
| 776 | case DFT_ANALYTICS_DETECT: | 815 | case DFT_ANALYTICS_DETECT: |
| 777 | ff_frame_to_dnn_detect(in_frame, &input, ctx); | 816 | ff_frame_to_dnn_detect(task->in_frame, &input, ctx); |
| 778 | break; | 817 | break; |
| 779 | default: | 818 | default: |
| 780 | avpriv_report_missing_feature(ctx, "model function type %d", tf_model->model->func_type); | 819 | avpriv_report_missing_feature(ctx, "model function type %d", tf_model->model->func_type); |
| 781 | break; | 820 | break; |
| 782 | } | 821 | } |
| 783 | 822 | ||
| 784 | tf_outputs = av_malloc_array(nb_output, sizeof(*tf_outputs)); | 823 | tf_outputs = av_malloc_array(task->nb_output, sizeof(TF_Output)); |
| 785 | if (tf_outputs == NULL) { | 824 | if (tf_outputs == NULL) { |
| 786 | TF_DeleteTensor(input_tensor); | 825 | TF_DeleteTensor(input_tensor); |
| 787 | av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *tf_outputs\n"); \ | 826 | av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *tf_outputs\n"); \ |
| 788 | return DNN_ERROR; | 827 | return DNN_ERROR; |
| 789 | } | 828 | } |
| 790 | 829 | ||
| 791 | output_tensors = av_mallocz_array(nb_output, sizeof(*output_tensors)); | 830 | output_tensors = av_mallocz_array(task->nb_output, sizeof(*output_tensors)); |
| 792 | if (!output_tensors) { | 831 | if (!output_tensors) { |
| 793 | TF_DeleteTensor(input_tensor); | 832 | TF_DeleteTensor(input_tensor); |
| 794 | av_freep(&tf_outputs); | 833 | av_freep(&tf_outputs); |
| @@ -796,13 +835,13 @@ static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_n · 1 unmodified line | |||
| 796 | return DNN_ERROR; | 835 | return DNN_ERROR; |
| 797 | } | 836 | } |
| 798 | 837 | ||
| 799 | for (int i = 0; i < nb_output; ++i) { | 838 | for (int i = 0; i < task->nb_output; ++i) { |
| 800 | tf_outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]); | 839 | tf_outputs[i].oper = TF_GraphOperationByName(tf_model->graph, task->output_names[i]); |
| 801 | if (!tf_outputs[i].oper) { | 840 | if (!tf_outputs[i].oper) { |
| 802 | TF_DeleteTensor(input_tensor); | 841 | TF_DeleteTensor(input_tensor); |
| 803 | av_freep(&tf_outputs); | 842 | av_freep(&tf_outputs); |
| 804 | av_freep(&output_tensors); | 843 | av_freep(&output_tensors); |
| 805 | av_log(ctx, AV_LOG_ERROR, "Could not find output \"%s\" in model\n", output_names[i]); \ | 844 | av_log(ctx, AV_LOG_ERROR, "Could not find output \"%s\" in model\n", task->output_names[i]); \ |
| 806 | return DNN_ERROR; | 845 | return DNN_ERROR; |
| 807 | } | 846 | } |
| 808 | tf_outputs[i].index = 0; | 847 | tf_outputs[i].index = 0; |
| @@ -810,7 +849,7 @@ static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_n · 1 unmodified line | |||
| 810 | 849 | ||
| 811 | TF_SessionRun(tf_model->session, NULL, | 850 | TF_SessionRun(tf_model->session, NULL, |
| 812 | &tf_input, &input_tensor, 1, | 851 | &tf_input, &input_tensor, 1, |
| 813 | tf_outputs, output_tensors, nb_output, | 852 | tf_outputs, output_tensors, task->nb_output, |
| 814 | NULL, 0, NULL, tf_model->status); | 853 | NULL, 0, NULL, tf_model->status); |
| 815 | if (TF_GetCode(tf_model->status) != TF_OK) { | 854 | if (TF_GetCode(tf_model->status) != TF_OK) { |
| 816 | TF_DeleteTensor(input_tensor); | 855 | TF_DeleteTensor(input_tensor); |
| @@ -820,7 +859,7 @@ static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_n · 3 unmodified lines | |||
| 820 | return DNN_ERROR; | 859 | return DNN_ERROR; |
| 821 | } | 860 | } |
| 822 | 861 | ||
| 823 | outputs = av_malloc_array(nb_output, sizeof(*outputs)); | 862 | outputs = av_malloc_array(task->nb_output, sizeof(*outputs)); |
| 824 | if (!outputs) { | 863 | if (!outputs) { |
| 825 | TF_DeleteTensor(input_tensor); | 864 | TF_DeleteTensor(input_tensor); |
| 826 | av_freep(&tf_outputs); | 865 | av_freep(&tf_outputs); |
| @@ -829,36 +868,36 @@ static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_n · 2 unmodified lines | |||
| 829 | return DNN_ERROR; | 868 | return DNN_ERROR; |
| 830 | } | 869 | } |
| 831 | 870 | ||
| 832 | for (uint32_t i = 0; i < nb_output; ++i) { | 871 | for (uint32_t i = 0; i < task->nb_output; ++i) { |
| 833 | outputs[i].height = TF_Dim(output_tensors[i], 1); | 872 | outputs[i].height = TF_Dim(output_tensors[i], 1); |
| 834 | outputs[i].width = TF_Dim(output_tensors[i], 2); | 873 | outputs[i].width = TF_Dim(output_tensors[i], 2); |
| 835 | outputs[i].channels = TF_Dim(output_tensors[i], 3); | 874 | outputs[i].channels = TF_Dim(output_tensors[i], 3); |
| 836 | outputs[i].data = TF_TensorData(output_tensors[i]); | 875 | outputs[i].data = TF_TensorData(output_tensors[i]); |
| 837 | outputs[i].dt = TF_TensorType(output_tensors[i]); | 876 | outputs[i].dt = TF_TensorType(output_tensors[i]); |
| 838 | } | 877 | } |
| 839 | switch (model->func_type) { | 878 | switch (tf_model->model->func_type) { |
| 840 | case DFT_PROCESS_FRAME: | 879 | case DFT_PROCESS_FRAME: |
| 841 | //it only support 1 output if it's frame in & frame out | 880 | //it only support 1 output if it's frame in & frame out |
| 842 | if (do_ioproc) { | 881 | if (task->do_ioproc) { |
| 843 | if (tf_model->model->frame_post_proc != NULL) { | 882 | if (tf_model->model->frame_post_proc != NULL) { |
| 844 | tf_model->model->frame_post_proc(out_frame, outputs, tf_model->model->filter_ctx); | 883 | tf_model->model->frame_post_proc(task->out_frame, outputs, tf_model->model->filter_ctx); |
| 845 | } else { | 884 | } else { |
| 846 | ff_proc_from_dnn_to_frame(out_frame, outputs, ctx); | 885 | ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx); |
| 847 | } | 886 | } |
| 848 | } else { | 887 | } else { |
| 849 | out_frame->width = outputs[0].width; | 888 | task->out_frame->width = outputs[0].width; |
| 850 | out_frame->height = outputs[0].height; | 889 | task->out_frame->height = outputs[0].height; |
| 851 | } | 890 | } |
| 852 | break; | 891 | break; |
| 853 | case DFT_ANALYTICS_DETECT: | 892 | case DFT_ANALYTICS_DETECT: |
| 854 | if (!model->detect_post_proc) { | 893 | if (!tf_model->model->detect_post_proc) { |
| 855 | av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post proc\n"); | 894 | av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post proc\n"); |
| 856 | return DNN_ERROR; | 895 | return DNN_ERROR; |
| 857 | } | 896 | } |
| 858 | model->detect_post_proc(out_frame, outputs, nb_output, model->filter_ctx); | 897 | tf_model->model->detect_post_proc(task->out_frame, outputs, task->nb_output, tf_model->model->filter_ctx); |
| 859 | break; | 898 | break; |
| 860 | default: | 899 | default: |
| 861 | for (uint32_t i = 0; i < nb_output; ++i) { | 900 | for (uint32_t i = 0; i < task->nb_output; ++i) { |
| 862 | if (output_tensors[i]) { | 901 | if (output_tensors[i]) { |
| 863 | TF_DeleteTensor(output_tensors[i]); | 902 | TF_DeleteTensor(output_tensors[i]); |
| 864 | } | 903 | } |
| @@ -871,30 +910,39 @@ static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_n · 6 unmodified lines | |||
| 871 | av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this kind of dnn filter now\n"); | 910 | av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this kind of dnn filter now\n"); |
| 872 | return DNN_ERROR; | 911 | return DNN_ERROR; |
| 873 | } | 912 | } |
| 874 | 913 | for (uint32_t i = 0; i < task->nb_output; ++i) { | |
| 875 | for (uint32_t i = 0; i < nb_output; ++i) { | ||
| 876 | if (output_tensors[i]) { | 914 | if (output_tensors[i]) { |
| 877 | TF_DeleteTensor(output_tensors[i]); | 915 | TF_DeleteTensor(output_tensors[i]); |
| 878 | } | 916 | } |
| 879 | } | 917 | } |
| 918 | task->inference_done++; | ||
| 880 | TF_DeleteTensor(input_tensor); | 919 | TF_DeleteTensor(input_tensor); |
| 881 | av_freep(&output_tensors); | 920 | av_freep(&output_tensors); |
| 882 | av_freep(&tf_outputs); | 921 | av_freep(&tf_outputs); |
| 883 | av_freep(&outputs); | 922 | av_freep(&outputs); |
| 884 | return DNN_SUCCESS; | 923 | return DNN_SUCCESS; |
| 924 | return (task->inference_done == task->inference_todo) ? DNN_SUCCESS : DNN_ERROR; | ||
| 885 | } | 925 | } |
| 886 | 926 | ||
| 887 | DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params) | 927 | DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params) |
| 888 | { | 928 | { |
| 889 | TFModel *tf_model = model->model; | 929 | TFModel *tf_model = model->model; |
| 890 | TFContext *ctx = &tf_model->ctx; | 930 | TFContext *ctx = &tf_model->ctx; |
| 931 | TaskItem task; | ||
| 891 | 932 | ||
| 892 | if (ff_check_exec_params(ctx, DNN_TF, model->func_type, exec_params) != 0) { | 933 | if (ff_check_exec_params(ctx, DNN_TF, model->func_type, exec_params) != 0) { |
| 893 | return DNN_ERROR; | 934 | return DNN_ERROR; |
| 894 | } | 935 | } |
| 895 | 936 | ||
| 896 | return execute_model_tf(model, exec_params->input_name, exec_params->in_frame, | 937 | if (ff_dnn_fill_task(&task, exec_params, tf_model, 0, 1) != DNN_SUCCESS) { |
| 897 | exec_params->output_names, exec_params->nb_output, exec_params->out_frame, 1); | 938 | return DNN_ERROR; |
| 939 | } | ||
| 940 | |||
| 941 | if (extract_inference_from_task(&task, tf_model->inference_queue) != DNN_SUCCESS) { | ||
| 942 | av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n"); | ||
| 943 | return DNN_ERROR; | ||
| 944 | } | ||
| 945 | return execute_model_tf(tf_model->inference_queue); | ||
| 898 | } | 946 | } |
| 899 | 947 | ||
| 900 | void ff_dnn_free_model_tf(DNNModel **model) | 948 | void ff_dnn_free_model_tf(DNNModel **model) |
| @@ -903,6 +951,12 @@ void ff_dnn_free_model_tf(DNNModel **model) · 2 unmodified lines | |||
| 903 | 951 | ||
| 904 | if (*model){ | 952 | if (*model){ |
| 905 | tf_model = (*model)->model; | 953 | tf_model = (*model)->model; |
| 954 | while (ff_queue_size(tf_model->inference_queue) != 0) { | ||
| 955 | InferenceItem *item = ff_queue_pop_front(tf_model->inference_queue); | ||
| 956 | av_freep(&item); | ||
| 957 | } | ||
| 958 | ff_queue_destroy(tf_model->inference_queue); | ||
| 959 | |||
| 906 | if (tf_model->graph){ | 960 | if (tf_model->graph){ |
| 907 | TF_DeleteGraph(tf_model->graph); | 961 | TF_DeleteGraph(tf_model->graph); |
| 908 | } | 962 | } |