History

lavfi/dnn: Async Support for TensorFlow Backend

This commit enables async execution in the TensorFlow backend and adds function to flush extra frames. The async execution mechanism executes the TFInferRequests on a separate thread which is joined before the next execution of same TFRequestItem/while freeing the model. The following is the comparison of this mechanism with the existing sync mechanism on TensorFlow C API 2.5 CPU variant. Async Mode: 4m32.846s Sync Mode: 5m17.582s The above was performed on super resolution filter using SRCNN model.

Shubhanshu Saxena committed 5 years ago

Commit 0985e921 parent e6ae8fc3 files · +109 −18Browse files

Unified · Split

3 files
  1. libavfilter/dnn/dnn_backend_tf.c +103 −18
  2. libavfilter/dnn/dnn_backend_tf.h +3 −0
  3. libavfilter/dnn/dnn_interface.c +3 −0
libavfilter/dnn/dnn_backend_tf.c+103−18
@@ -38,7 +38,6 @@ · 37 unmodified lines
38#include "dnn_io_proc.h"38#include "dnn_io_proc.h"
39#include "dnn_backend_common.h"39#include "dnn_backend_common.h"
40#include "safe_queue.h"40#include "safe_queue.h"
41#include "queue.h"
42#include <tensorflow/c/c_api.h>41#include <tensorflow/c/c_api.h>
43 42
44typedef struct TFOptions{43typedef struct TFOptions{
@@ -59,6 +58,7 @@ typedef struct TFModel{ · 14 unmodified lines
59 TF_Status *status;58 TF_Status *status;
60 SafeQueue *request_queue;59 SafeQueue *request_queue;
61 Queue *inference_queue;60 Queue *inference_queue;
61 Queue *task_queue;
62} TFModel;62} TFModel;
63 63
64/**64/**
@@ -75,7 +75,7 @@ typedef struct TFInferRequest { · 10 unmodified lines
75typedef struct TFRequestItem {75typedef struct TFRequestItem {
76 TFInferRequest *infer_request;76 TFInferRequest *infer_request;
77 InferenceItem *inference;77 InferenceItem *inference;
78 // further properties will be added later for async78 DNNAsyncExecModule exec_module;
79} TFRequestItem;79} TFRequestItem;
80 80
81#define OFFSET(x) offsetof(TFContext, x)81#define OFFSET(x) offsetof(TFContext, x)
@@ -89,6 +89,7 @@ static const AVOption dnn_tensorflow_options[] = { · 7 unmodified lines
89AVFILTER_DEFINE_CLASS(dnn_tensorflow);89AVFILTER_DEFINE_CLASS(dnn_tensorflow);
90 90
91static DNNReturnType execute_model_tf(TFRequestItem *request, Queue *inference_queue);91static DNNReturnType execute_model_tf(TFRequestItem *request, Queue *inference_queue);
92static void infer_completion_callback(void *args);
92 93
93static void free_buffer(void *data, size_t length)94static void free_buffer(void *data, size_t length)
94{95{
@@ -886,6 +887,9 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_ · 791 unmodified lines
886 av_freep(&item);887 av_freep(&item);
887 goto err;888 goto err;
888 }889 }
890 item->exec_module.start_inference = &tf_start_inference;
891 item->exec_module.callback = &infer_completion_callback;
892 item->exec_module.args = item;
889 893
890 if (ff_safe_queue_push_back(tf_model->request_queue, item) < 0) {894 if (ff_safe_queue_push_back(tf_model->request_queue, item) < 0) {
891 av_freep(&item->infer_request);895 av_freep(&item->infer_request);
@@ -899,6 +903,11 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_ · 7 unmodified lines
899 goto err;903 goto err;
900 }904 }
901 905
906 tf_model->task_queue = ff_queue_create();
907 if (!tf_model->task_queue) {
908 goto err;
909 }
910
902 model->model = tf_model;911 model->model = tf_model;
903 model->get_input = &get_input_tf;912 model->get_input = &get_input_tf;
904 model->get_output = &get_output_tf;913 model->get_output = &get_output_tf;
@@ -1061,7 +1070,6 @@ static DNNReturnType execute_model_tf(TFRequestItem *request, Queue *inference_q · 156 unmodified lines
1061{1070{
1062 TFModel *tf_model;1071 TFModel *tf_model;
1063 TFContext *ctx;1072 TFContext *ctx;
1064 TFInferRequest *infer_request;
1065 InferenceItem *inference;1073 InferenceItem *inference;
1066 TaskItem *task;1074 TaskItem *task;
1067 1075
@@ -1074,23 +1082,14 @@ static DNNReturnType execute_model_tf(TFRequestItem *request, Queue *inference_q · 6 unmodified lines
1074 tf_model = task->model;1082 tf_model = task->model;
1075 ctx = &tf_model->ctx;1083 ctx = &tf_model->ctx;
1076 1084
1077 if (task->async) {1085 if (fill_model_input_tf(tf_model, request) != DNN_SUCCESS) {
1078 avpriv_report_missing_feature(ctx, "Async execution not supported");
1079 return DNN_ERROR;1086 return DNN_ERROR;
1080 } else {1087 }
1081 if (fill_model_input_tf(tf_model, request) != DNN_SUCCESS) {
1082 return DNN_ERROR;
1083 }
1084 1088
1085 infer_request = request->infer_request;1089 if (task->async) {
1086 TF_SessionRun(tf_model->session, NULL,1090 return ff_dnn_start_inference_async(ctx, &request->exec_module);
1087 infer_request->tf_input, &infer_request->input_tensor, 1,1091 } else {
1088 infer_request->tf_outputs, infer_request->output_tensors,1092 if (tf_start_inference(request) != DNN_SUCCESS) {
1089 task->nb_output, NULL, 0, NULL,
1090 tf_model->status);
1091 if (TF_GetCode(tf_model->status) != TF_OK) {
1092 tf_free_request(infer_request);
1093 av_log(ctx, AV_LOG_ERROR, "Failed to run session when executing model\n");
1094 return DNN_ERROR;1093 return DNN_ERROR;
1095 }1094 }
1096 infer_completion_callback(request);1095 infer_completion_callback(request);
@@ -1127,6 +1126,83 @@ DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams * · 30 unmodified lines
1127 return execute_model_tf(request, tf_model->inference_queue);1126 return execute_model_tf(request, tf_model->inference_queue);
1128}1127}
1129 1128
1129DNNReturnType ff_dnn_execute_model_async_tf(const DNNModel *model, DNNExecBaseParams *exec_params) {
1130 TFModel *tf_model = model->model;
1131 TFContext *ctx = &tf_model->ctx;
1132 TaskItem *task;
1133 TFRequestItem *request;
1134
1135 if (ff_check_exec_params(ctx, DNN_TF, model->func_type, exec_params) != 0) {
1136 return DNN_ERROR;
1137 }
1138
1139 task = av_malloc(sizeof(*task));
1140 if (!task) {
1141 av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");
1142 return DNN_ERROR;
1143 }
1144
1145 if (ff_dnn_fill_task(task, exec_params, tf_model, 1, 1) != DNN_SUCCESS) {
1146 av_freep(&task);
1147 return DNN_ERROR;
1148 }
1149
1150 if (ff_queue_push_back(tf_model->task_queue, task) < 0) {
1151 av_freep(&task);
1152 av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");
1153 return DNN_ERROR;
1154 }
1155
1156 if (extract_inference_from_task(task, tf_model->inference_queue) != DNN_SUCCESS) {
1157 av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");
1158 return DNN_ERROR;
1159 }
1160
1161 request = ff_safe_queue_pop_front(tf_model->request_queue);
1162 if (!request) {
1163 av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
1164 return DNN_ERROR;
1165 }
1166 return execute_model_tf(request, tf_model->inference_queue);
1167}
1168
1169DNNAsyncStatusType ff_dnn_get_async_result_tf(const DNNModel *model, AVFrame **in, AVFrame **out)
1170{
1171 TFModel *tf_model = model->model;
1172 return ff_dnn_get_async_result_common(tf_model->task_queue, in, out);
1173}
1174
1175DNNReturnType ff_dnn_flush_tf(const DNNModel *model)
1176{
1177 TFModel *tf_model = model->model;
1178 TFContext *ctx = &tf_model->ctx;
1179 TFRequestItem *request;
1180 DNNReturnType ret;
1181
1182 if (ff_queue_size(tf_model->inference_queue) == 0) {
1183 // no pending task need to flush
1184 return DNN_SUCCESS;
1185 }
1186
1187 request = ff_safe_queue_pop_front(tf_model->request_queue);
1188 if (!request) {
1189 av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
1190 return DNN_ERROR;
1191 }
1192
1193 ret = fill_model_input_tf(tf_model, request);
1194 if (ret != DNN_SUCCESS) {
1195 av_log(ctx, AV_LOG_ERROR, "Failed to fill model input.\n");
1196 if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
1197 av_freep(&request->infer_request);
1198 av_freep(&request);
1199 }
1200 return ret;
1201 }
1202
1203 return ff_dnn_start_inference_async(ctx, &request->exec_module);
1204}
1205
1130void ff_dnn_free_model_tf(DNNModel **model)1206void ff_dnn_free_model_tf(DNNModel **model)
1131{1207{
1132 TFModel *tf_model;1208 TFModel *tf_model;
@@ -1135,6 +1211,7 @@ void ff_dnn_free_model_tf(DNNModel **model) · 2 unmodified lines
1135 tf_model = (*model)->model;1211 tf_model = (*model)->model;
1136 while (ff_safe_queue_size(tf_model->request_queue) != 0) {1212 while (ff_safe_queue_size(tf_model->request_queue) != 0) {
1137 TFRequestItem *item = ff_safe_queue_pop_front(tf_model->request_queue);1213 TFRequestItem *item = ff_safe_queue_pop_front(tf_model->request_queue);
1214 ff_dnn_async_module_cleanup(&item->exec_module);
1138 tf_free_request(item->infer_request);1215 tf_free_request(item->infer_request);
1139 av_freep(&item->infer_request);1216 av_freep(&item->infer_request);
1140 av_freep(&item);1217 av_freep(&item);
@@ -1147,6 +1224,14 @@ void ff_dnn_free_model_tf(DNNModel **model) · 6 unmodified lines
1147 }1224 }
1148 ff_queue_destroy(tf_model->inference_queue);1225 ff_queue_destroy(tf_model->inference_queue);
1149 1226
1227 while (ff_queue_size(tf_model->task_queue) != 0) {
1228 TaskItem *item = ff_queue_pop_front(tf_model->task_queue);
1229 av_frame_free(&item->in_frame);
1230 av_frame_free(&item->out_frame);
1231 av_freep(&item);
1232 }
1233 ff_queue_destroy(tf_model->task_queue);
1234
1150 if (tf_model->graph){1235 if (tf_model->graph){
1151 TF_DeleteGraph(tf_model->graph);1236 TF_DeleteGraph(tf_model->graph);
1152 }1237 }
libavfilter/dnn/dnn_backend_tf.h+3−0
@@ -32,6 +32,9 @@ · 31 unmodified lines
32DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);32DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);
33 33
34DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params);34DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params);
35DNNReturnType ff_dnn_execute_model_async_tf(const DNNModel *model, DNNExecBaseParams *exec_params);
36DNNAsyncStatusType ff_dnn_get_async_result_tf(const DNNModel *model, AVFrame **in, AVFrame **out);
37DNNReturnType ff_dnn_flush_tf(const DNNModel *model);
35 38
36void ff_dnn_free_model_tf(DNNModel **model);39void ff_dnn_free_model_tf(DNNModel **model);
37 40
libavfilter/dnn/dnn_interface.c+3−0
@@ -48,6 +48,9 @@ DNNModule *ff_get_dnn_module(DNNBackendType backend_type) · 47 unmodified lines
48 #if (CONFIG_LIBTENSORFLOW == 1)48 #if (CONFIG_LIBTENSORFLOW == 1)
49 dnn_module->load_model = &ff_dnn_load_model_tf;49 dnn_module->load_model = &ff_dnn_load_model_tf;
50 dnn_module->execute_model = &ff_dnn_execute_model_tf;50 dnn_module->execute_model = &ff_dnn_execute_model_tf;
51 dnn_module->execute_model_async = &ff_dnn_execute_model_async_tf;
52 dnn_module->get_async_result = &ff_dnn_get_async_result_tf;
53 dnn_module->flush = &ff_dnn_flush_tf;
51 dnn_module->free_model = &ff_dnn_free_model_tf;54 dnn_module->free_model = &ff_dnn_free_model_tf;
52 #else55 #else
53 av_freep(&dnn_module);56 av_freep(&dnn_module);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t