| @@ -0,0 +1,474 @@ |
| 1 | +// Package openai provides HTTP handlers for OpenAIResponses API endpoints. |
| 2 | +// This package implements the OpenAIResponses-compatible API interface, including model listing |
| 3 | +// and chat completion functionality. It supports both streaming and non-streaming responses, |
| 4 | +// and manages a pool of clients to interact with backend services. |
| 5 | +// The handlers translate OpenAIResponses API requests to the appropriate backend format and |
| 6 | +// convert responses back to OpenAIResponses-compatible format. |
| 7 | +package openai |
| 8 | + |
| 9 | +import ( |
| 10 | + "context" |
| 11 | + "fmt" |
| 12 | + "net/http" |
| 13 | + "time" |
| 14 | + |
| 15 | + "github.com/gin-gonic/gin" |
| 16 | + "github.com/luispater/CLIProxyAPI/internal/api/handlers" |
| 17 | + . "github.com/luispater/CLIProxyAPI/internal/constant" |
| 18 | + "github.com/luispater/CLIProxyAPI/internal/interfaces" |
| 19 | + "github.com/luispater/CLIProxyAPI/internal/registry" |
| 20 | + "github.com/luispater/CLIProxyAPI/internal/util" |
| 21 | + log "github.com/sirupsen/logrus" |
| 22 | + "github.com/tidwall/gjson" |
| 23 | +) |
| 24 | + |
| 25 | +// OpenAIResponsesAPIHandler contains the handlers for OpenAIResponses API endpoints. |
| 26 | +// It holds a pool of clients to interact with the backend service. |
| 27 | +type OpenAIResponsesAPIHandler struct { |
| 28 | + *handlers.BaseAPIHandler |
| 29 | +} |
| 30 | + |
| 31 | +// NewOpenAIResponsesAPIHandler creates a new OpenAIResponses API handlers instance. |
| 32 | +// It takes an BaseAPIHandler instance as input and returns an OpenAIResponsesAPIHandler. |
| 33 | +// |
| 34 | +// Parameters: |
| 35 | +// - apiHandlers: The base API handlers instance |
| 36 | +// |
| 37 | +// Returns: |
| 38 | +// - *OpenAIResponsesAPIHandler: A new OpenAIResponses API handlers instance |
| 39 | +func NewOpenAIResponsesAPIHandler(apiHandlers *handlers.BaseAPIHandler) *OpenAIResponsesAPIHandler { |
| 40 | + return &OpenAIResponsesAPIHandler{ |
| 41 | + BaseAPIHandler: apiHandlers, |
| 42 | + } |
| 43 | +} |
| 44 | + |
| 45 | +// HandlerType returns the identifier for this handler implementation. |
| 46 | +func (h *OpenAIResponsesAPIHandler) HandlerType() string { |
| 47 | + return OPENAI_RESPONSE |
| 48 | +} |
| 49 | + |
| 50 | +// Models returns the OpenAIResponses-compatible model metadata supported by this handler. |
| 51 | +func (h *OpenAIResponsesAPIHandler) Models() []map[string]any { |
| 52 | + // Get dynamic models from the global registry |
| 53 | + modelRegistry := registry.GetGlobalRegistry() |
| 54 | + return modelRegistry.GetAvailableModels("openai") |
| 55 | +} |
| 56 | + |
| 57 | +// OpenAIResponsesModels handles the /v1/models endpoint. |
| 58 | +// It returns a list of available AI models with their capabilities |
| 59 | +// and specifications in OpenAIResponses-compatible format. |
| 60 | +func (h *OpenAIResponsesAPIHandler) OpenAIResponsesModels(c *gin.Context) { |
| 61 | + c.JSON(http.StatusOK, gin.H{ |
| 62 | + "object": "list", |
| 63 | + "data": h.Models(), |
| 64 | + }) |
| 65 | +} |
| 66 | + |
| 67 | +// ChatCompletions handles the /v1/chat/completions endpoint. |
| 68 | +// It determines whether the request is for a streaming or non-streaming response |
| 69 | +// and calls the appropriate handler based on the model provider. |
| 70 | +// |
| 71 | +// Parameters: |
| 72 | +// - c: The Gin context containing the HTTP request and response |
| 73 | +func (h *OpenAIResponsesAPIHandler) ChatCompletions(c *gin.Context) { |
| 74 | + rawJSON, err := c.GetRawData() |
| 75 | + // If data retrieval fails, return a 400 Bad Request error. |
| 76 | + if err != nil { |
| 77 | + c.JSON(http.StatusBadRequest, handlers.ErrorResponse{ |
| 78 | + Error: handlers.ErrorDetail{ |
| 79 | + Message: fmt.Sprintf("Invalid request: %v", err), |
| 80 | + Type: "invalid_request_error", |
| 81 | + }, |
| 82 | + }) |
| 83 | + return |
| 84 | + } |
| 85 | + |
| 86 | + // Check if the client requested a streaming response. |
| 87 | + streamResult := gjson.GetBytes(rawJSON, "stream") |
| 88 | + if streamResult.Type == gjson.True { |
| 89 | + h.handleStreamingResponse(c, rawJSON) |
| 90 | + } else { |
| 91 | + h.handleNonStreamingResponse(c, rawJSON) |
| 92 | + } |
| 93 | + |
| 94 | +} |
| 95 | + |
| 96 | +// Completions handles the /v1/completions endpoint. |
| 97 | +// It determines whether the request is for a streaming or non-streaming response |
| 98 | +// and calls the appropriate handler based on the model provider. |
| 99 | +// This endpoint follows the OpenAIResponses completions API specification. |
| 100 | +// |
| 101 | +// Parameters: |
| 102 | +// - c: The Gin context containing the HTTP request and response |
| 103 | +func (h *OpenAIResponsesAPIHandler) Completions(c *gin.Context) { |
| 104 | + rawJSON, err := c.GetRawData() |
| 105 | + // If data retrieval fails, return a 400 Bad Request error. |
| 106 | + if err != nil { |
| 107 | + c.JSON(http.StatusBadRequest, handlers.ErrorResponse{ |
| 108 | + Error: handlers.ErrorDetail{ |
| 109 | + Message: fmt.Sprintf("Invalid request: %v", err), |
| 110 | + Type: "invalid_request_error", |
| 111 | + }, |
| 112 | + }) |
| 113 | + return |
| 114 | + } |
| 115 | + |
| 116 | + // Check if the client requested a streaming response. |
| 117 | + streamResult := gjson.GetBytes(rawJSON, "stream") |
| 118 | + if streamResult.Type == gjson.True { |
| 119 | + h.handleCompletionsStreamingResponse(c, rawJSON) |
| 120 | + } else { |
| 121 | + h.handleCompletionsNonStreamingResponse(c, rawJSON) |
| 122 | + } |
| 123 | + |
| 124 | +} |
| 125 | + |
| 126 | +// handleNonStreamingResponse handles non-streaming chat completion responses |
| 127 | +// for Gemini models. It selects a client from the pool, sends the request, and |
| 128 | +// aggregates the response before sending it back to the client in OpenAIResponses format. |
| 129 | +// |
| 130 | +// Parameters: |
| 131 | +// - c: The Gin context containing the HTTP request and response |
| 132 | +// - rawJSON: The raw JSON bytes of the OpenAIResponses-compatible request |
| 133 | +func (h *OpenAIResponsesAPIHandler) handleNonStreamingResponse(c *gin.Context, rawJSON []byte) { |
| 134 | + c.Header("Content-Type", "application/json") |
| 135 | + |
| 136 | + modelName := gjson.GetBytes(rawJSON, "model").String() |
| 137 | + cliCtx, cliCancel := h.GetContextWithCancel(h, c, context.Background()) |
| 138 | + |
| 139 | + var cliClient interfaces.Client |
| 140 | + defer func() { |
| 141 | + if cliClient != nil { |
| 142 | + if mutex := cliClient.GetRequestMutex(); mutex != nil { |
| 143 | + mutex.Unlock() |
| 144 | + } |
| 145 | + } |
| 146 | + }() |
| 147 | + |
| 148 | + retryCount := 0 |
| 149 | + for retryCount <= h.Cfg.RequestRetry { |
| 150 | + var errorResponse *interfaces.ErrorMessage |
| 151 | + cliClient, errorResponse = h.GetClient(modelName) |
| 152 | + if errorResponse != nil { |
| 153 | + c.Status(errorResponse.StatusCode) |
| 154 | + _, _ = fmt.Fprint(c.Writer, errorResponse.Error.Error()) |
| 155 | + cliCancel() |
| 156 | + return |
| 157 | + } |
| 158 | + |
| 159 | + resp, err := cliClient.SendRawMessage(cliCtx, modelName, rawJSON, "") |
| 160 | + if err != nil { |
| 161 | + switch err.StatusCode { |
| 162 | + case 429: |
| 163 | + if h.Cfg.QuotaExceeded.SwitchProject { |
| 164 | + log.Debugf("quota exceeded, switch client") |
| 165 | + continue // Restart the client selection process |
| 166 | + } |
| 167 | + case 403, 408, 500, 502, 503, 504: |
| 168 | + log.Debugf("http status code %d, switch client", err.StatusCode) |
| 169 | + retryCount++ |
| 170 | + continue |
| 171 | + case 401: |
| 172 | + log.Debugf("unauthorized request, try to refresh token, %s", util.HideAPIKey(cliClient.GetEmail())) |
| 173 | + errRefreshTokens := cliClient.RefreshTokens(cliCtx) |
| 174 | + if errRefreshTokens != nil { |
| 175 | + log.Debugf("refresh token failed, switch client, %s", util.HideAPIKey(cliClient.GetEmail())) |
| 176 | + } |
| 177 | + retryCount++ |
| 178 | + continue |
| 179 | + default: |
| 180 | + // Forward other errors directly to the client |
| 181 | + c.Status(err.StatusCode) |
| 182 | + _, _ = c.Writer.Write([]byte(err.Error.Error())) |
| 183 | + cliCancel(err.Error) |
| 184 | + } |
| 185 | + break |
| 186 | + } else { |
| 187 | + _, _ = c.Writer.Write(resp) |
| 188 | + cliCancel(resp) |
| 189 | + break |
| 190 | + } |
| 191 | + } |
| 192 | +} |
| 193 | + |
| 194 | +// handleStreamingResponse handles streaming responses for Gemini models. |
| 195 | +// It establishes a streaming connection with the backend service and forwards |
| 196 | +// the response chunks to the client in real-time using Server-Sent Events. |
| 197 | +// |
| 198 | +// Parameters: |
| 199 | +// - c: The Gin context containing the HTTP request and response |
| 200 | +// - rawJSON: The raw JSON bytes of the OpenAIResponses-compatible request |
| 201 | +func (h *OpenAIResponsesAPIHandler) handleStreamingResponse(c *gin.Context, rawJSON []byte) { |
| 202 | + c.Header("Content-Type", "text/event-stream") |
| 203 | + c.Header("Cache-Control", "no-cache") |
| 204 | + c.Header("Connection", "keep-alive") |
| 205 | + c.Header("Access-Control-Allow-Origin", "*") |
| 206 | + |
| 207 | + // Get the http.Flusher interface to manually flush the response. |
| 208 | + flusher, ok := c.Writer.(http.Flusher) |
| 209 | + if !ok { |
| 210 | + c.JSON(http.StatusInternalServerError, handlers.ErrorResponse{ |
| 211 | + Error: handlers.ErrorDetail{ |
| 212 | + Message: "Streaming not supported", |
| 213 | + Type: "server_error", |
| 214 | + }, |
| 215 | + }) |
| 216 | + return |
| 217 | + } |
| 218 | + |
| 219 | + modelName := gjson.GetBytes(rawJSON, "model").String() |
| 220 | + cliCtx, cliCancel := h.GetContextWithCancel(h, c, context.Background()) |
| 221 | + |
| 222 | + var cliClient interfaces.Client |
| 223 | + defer func() { |
| 224 | + // Ensure the client's mutex is unlocked on function exit. |
| 225 | + if cliClient != nil { |
| 226 | + if mutex := cliClient.GetRequestMutex(); mutex != nil { |
| 227 | + mutex.Unlock() |
| 228 | + } |
| 229 | + } |
| 230 | + }() |
| 231 | + |
| 232 | + retryCount := 0 |
| 233 | +outLoop: |
| 234 | + for retryCount <= h.Cfg.RequestRetry { |
| 235 | + var errorResponse *interfaces.ErrorMessage |
| 236 | + cliClient, errorResponse = h.GetClient(modelName) |
| 237 | + if errorResponse != nil { |
| 238 | + c.Status(errorResponse.StatusCode) |
| 239 | + _, _ = fmt.Fprint(c.Writer, errorResponse.Error.Error()) |
| 240 | + flusher.Flush() |
| 241 | + cliCancel() |
| 242 | + return |
| 243 | + } |
| 244 | + |
| 245 | + // Send the message and receive response chunks and errors via channels. |
| 246 | + respChan, errChan := cliClient.SendRawMessageStream(cliCtx, modelName, rawJSON, "") |
| 247 | + |
| 248 | + for { |
| 249 | + select { |
| 250 | + // Handle client disconnection. |
| 251 | + case <-c.Request.Context().Done(): |
| 252 | + if c.Request.Context().Err().Error() == "context canceled" { |
| 253 | + log.Debugf("openai client disconnected: %v", c.Request.Context().Err()) |
| 254 | + cliCancel() // Cancel the backend request. |
| 255 | + return |
| 256 | + } |
| 257 | + // Process incoming response chunks. |
| 258 | + case chunk, okStream := <-respChan: |
| 259 | + if !okStream { |
| 260 | + // Stream is closed, send the final [DONE] message. |
| 261 | + _, _ = fmt.Fprintf(c.Writer, "data: [DONE]\n\n") |
| 262 | + flusher.Flush() |
| 263 | + cliCancel() |
| 264 | + return |
| 265 | + } |
| 266 | + |
| 267 | + _, _ = fmt.Fprintf(c.Writer, "data: %s\n\n", string(chunk)) |
| 268 | + flusher.Flush() |
| 269 | + // Handle errors from the backend. |
| 270 | + case err, okError := <-errChan: |
| 271 | + if okError { |
| 272 | + switch err.StatusCode { |
| 273 | + case 429: |
| 274 | + if h.Cfg.QuotaExceeded.SwitchProject { |
| 275 | + log.Debugf("quota exceeded, switch client") |
| 276 | + continue outLoop // Restart the client selection process |
| 277 | + } |
| 278 | + case 403, 408, 500, 502, 503, 504: |
| 279 | + log.Debugf("http status code %d, switch client", err.StatusCode) |
| 280 | + retryCount++ |
| 281 | + continue outLoop |
| 282 | + default: |
| 283 | + // Forward other errors directly to the client |
| 284 | + c.Status(err.StatusCode) |
| 285 | + _, _ = fmt.Fprint(c.Writer, err.Error.Error()) |
| 286 | + flusher.Flush() |
| 287 | + cliCancel(err.Error) |
| 288 | + } |
| 289 | + return |
| 290 | + } |
| 291 | + // Send a keep-alive signal to the client. |
| 292 | + case <-time.After(500 * time.Millisecond): |
| 293 | + } |
| 294 | + } |
| 295 | + } |
| 296 | +} |
| 297 | + |
| 298 | +// handleCompletionsNonStreamingResponse handles non-streaming completions responses. |
| 299 | +// It converts completions request to chat completions format, sends to backend, |
| 300 | +// then converts the response back to completions format before sending to client. |
| 301 | +// |
| 302 | +// Parameters: |
| 303 | +// - c: The Gin context containing the HTTP request and response |
| 304 | +// - rawJSON: The raw JSON bytes of the OpenAIResponses-compatible completions request |
| 305 | +func (h *OpenAIResponsesAPIHandler) handleCompletionsNonStreamingResponse(c *gin.Context, rawJSON []byte) { |
| 306 | + c.Header("Content-Type", "application/json") |
| 307 | + |
| 308 | + // Convert completions request to chat completions format |
| 309 | + chatCompletionsJSON := convertCompletionsRequestToChatCompletions(rawJSON) |
| 310 | + |
| 311 | + modelName := gjson.GetBytes(chatCompletionsJSON, "model").String() |
| 312 | + cliCtx, cliCancel := h.GetContextWithCancel(h, c, context.Background()) |
| 313 | + |
| 314 | + var cliClient interfaces.Client |
| 315 | + defer func() { |
| 316 | + if cliClient != nil { |
| 317 | + if mutex := cliClient.GetRequestMutex(); mutex != nil { |
| 318 | + mutex.Unlock() |
| 319 | + } |
| 320 | + } |
| 321 | + }() |
| 322 | + |
| 323 | + retryCount := 0 |
| 324 | + for retryCount <= h.Cfg.RequestRetry { |
| 325 | + var errorResponse *interfaces.ErrorMessage |
| 326 | + cliClient, errorResponse = h.GetClient(modelName) |
| 327 | + if errorResponse != nil { |
| 328 | + c.Status(errorResponse.StatusCode) |
| 329 | + _, _ = fmt.Fprint(c.Writer, errorResponse.Error.Error()) |
| 330 | + cliCancel() |
| 331 | + return |
| 332 | + } |
| 333 | + |
| 334 | + // Send the converted chat completions request |
| 335 | + resp, err := cliClient.SendRawMessage(cliCtx, modelName, chatCompletionsJSON, "") |
| 336 | + if err != nil { |
| 337 | + switch err.StatusCode { |
| 338 | + case 429: |
| 339 | + if h.Cfg.QuotaExceeded.SwitchProject { |
| 340 | + log.Debugf("quota exceeded, switch client") |
| 341 | + continue // Restart the client selection process |
| 342 | + } |
| 343 | + case 403, 408, 500, 502, 503, 504: |
| 344 | + log.Debugf("http status code %d, switch client", err.StatusCode) |
| 345 | + retryCount++ |
| 346 | + continue |
| 347 | + default: |
| 348 | + // Forward other errors directly to the client |
| 349 | + c.Status(err.StatusCode) |
| 350 | + _, _ = c.Writer.Write([]byte(err.Error.Error())) |
| 351 | + cliCancel(err.Error) |
| 352 | + } |
| 353 | + break |
| 354 | + } else { |
| 355 | + // Convert chat completions response back to completions format |
| 356 | + completionsResp := convertChatCompletionsResponseToCompletions(resp) |
| 357 | + _, _ = c.Writer.Write(completionsResp) |
| 358 | + cliCancel(completionsResp) |
| 359 | + break |
| 360 | + } |
| 361 | + } |
| 362 | +} |
| 363 | + |
| 364 | +// handleCompletionsStreamingResponse handles streaming completions responses. |
| 365 | +// It converts completions request to chat completions format, streams from backend, |
| 366 | +// then converts each response chunk back to completions format before sending to client. |
| 367 | +// |
| 368 | +// Parameters: |
| 369 | +// - c: The Gin context containing the HTTP request and response |
| 370 | +// - rawJSON: The raw JSON bytes of the OpenAIResponses-compatible completions request |
| 371 | +func (h *OpenAIResponsesAPIHandler) handleCompletionsStreamingResponse(c *gin.Context, rawJSON []byte) { |
| 372 | + c.Header("Content-Type", "text/event-stream") |
| 373 | + c.Header("Cache-Control", "no-cache") |
| 374 | + c.Header("Connection", "keep-alive") |
| 375 | + c.Header("Access-Control-Allow-Origin", "*") |
| 376 | + |
| 377 | + // Get the http.Flusher interface to manually flush the response. |
| 378 | + flusher, ok := c.Writer.(http.Flusher) |
| 379 | + if !ok { |
| 380 | + c.JSON(http.StatusInternalServerError, handlers.ErrorResponse{ |
| 381 | + Error: handlers.ErrorDetail{ |
| 382 | + Message: "Streaming not supported", |
| 383 | + Type: "server_error", |
| 384 | + }, |
| 385 | + }) |
| 386 | + return |
| 387 | + } |
| 388 | + |
| 389 | + // Convert completions request to chat completions format |
| 390 | + chatCompletionsJSON := convertCompletionsRequestToChatCompletions(rawJSON) |
| 391 | + |
| 392 | + modelName := gjson.GetBytes(chatCompletionsJSON, "model").String() |
| 393 | + cliCtx, cliCancel := h.GetContextWithCancel(h, c, context.Background()) |
| 394 | + |
| 395 | + var cliClient interfaces.Client |
| 396 | + defer func() { |
| 397 | + // Ensure the client's mutex is unlocked on function exit. |
| 398 | + if cliClient != nil { |
| 399 | + if mutex := cliClient.GetRequestMutex(); mutex != nil { |
| 400 | + mutex.Unlock() |
| 401 | + } |
| 402 | + } |
| 403 | + }() |
| 404 | + |
| 405 | + retryCount := 0 |
| 406 | +outLoop: |
| 407 | + for retryCount <= h.Cfg.RequestRetry { |
| 408 | + var errorResponse *interfaces.ErrorMessage |
| 409 | + cliClient, errorResponse = h.GetClient(modelName) |
| 410 | + if errorResponse != nil { |
| 411 | + c.Status(errorResponse.StatusCode) |
| 412 | + _, _ = fmt.Fprint(c.Writer, errorResponse.Error.Error()) |
| 413 | + flusher.Flush() |
| 414 | + cliCancel() |
| 415 | + return |
| 416 | + } |
| 417 | + |
| 418 | + // Send the converted chat completions request and receive response chunks |
| 419 | + respChan, errChan := cliClient.SendRawMessageStream(cliCtx, modelName, chatCompletionsJSON, "") |
| 420 | + |
| 421 | + for { |
| 422 | + select { |
| 423 | + // Handle client disconnection. |
| 424 | + case <-c.Request.Context().Done(): |
| 425 | + if c.Request.Context().Err().Error() == "context canceled" { |
| 426 | + log.Debugf("client disconnected: %v", c.Request.Context().Err()) |
| 427 | + cliCancel() // Cancel the backend request. |
| 428 | + return |
| 429 | + } |
| 430 | + // Process incoming response chunks. |
| 431 | + case chunk, okStream := <-respChan: |
| 432 | + if !okStream { |
| 433 | + // Stream is closed, send the final [DONE] message. |
| 434 | + _, _ = fmt.Fprintf(c.Writer, "data: [DONE]\n\n") |
| 435 | + flusher.Flush() |
| 436 | + cliCancel() |
| 437 | + return |
| 438 | + } |
| 439 | + |
| 440 | + // Convert chat completions chunk to completions chunk format |
| 441 | + completionsChunk := convertChatCompletionsStreamChunkToCompletions(chunk) |
| 442 | + // Skip this chunk if it has no meaningful content (empty text) |
| 443 | + if completionsChunk != nil { |
| 444 | + _, _ = fmt.Fprintf(c.Writer, "data: %s\n\n", string(completionsChunk)) |
| 445 | + flusher.Flush() |
| 446 | + } |
| 447 | + // Handle errors from the backend. |
| 448 | + case err, okError := <-errChan: |
| 449 | + if okError { |
| 450 | + switch err.StatusCode { |
| 451 | + case 429: |
| 452 | + if h.Cfg.QuotaExceeded.SwitchProject { |
| 453 | + log.Debugf("quota exceeded, switch client") |
| 454 | + continue outLoop // Restart the client selection process |
| 455 | + } |
| 456 | + case 403, 408, 500, 502, 503, 504: |
| 457 | + log.Debugf("http status code %d, switch client", err.StatusCode) |
| 458 | + retryCount++ |
| 459 | + continue outLoop |
| 460 | + default: |
| 461 | + // Forward other errors directly to the client |
| 462 | + c.Status(err.StatusCode) |
| 463 | + _, _ = fmt.Fprint(c.Writer, err.Error.Error()) |
| 464 | + flusher.Flush() |
| 465 | + cliCancel(err.Error) |
| 466 | + } |
| 467 | + return |
| 468 | + } |
| 469 | + // Send a keep-alive signal to the client. |
| 470 | + case <-time.After(500 * time.Millisecond): |
| 471 | + } |
| 472 | + } |
| 473 | + } |
| 474 | +} |