Richard FeldmanThought signatures (#42915)c0fadae12 months agoHistory
2595 lines · 82.9 KB · Rust
RawDownload
use super::*;use acp_thread::{AgentConnection, AgentModelGroupName, AgentModelList, UserMessageId};use agent_client_protocol::{self as acp};use agent_settings::AgentProfileId;use anyhow::Result;use client::{Client, UserStore};use cloud_llm_client::CompletionIntent;use collections::IndexMap;use context_server::{ContextServer, ContextServerCommand, ContextServerId};use fs::{FakeFs, Fs};use futures::{    StreamExt,    channel::{        mpsc::{self, UnboundedReceiver},        oneshot,    },};use gpui::{    App, AppContext, Entity, Task, TestAppContext, UpdateGlobal, http_client::FakeHttpClient,};use indoc::indoc;use language_model::{    LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelId,    LanguageModelProviderName, LanguageModelRegistry, LanguageModelRequest,    LanguageModelRequestMessage, LanguageModelToolResult, LanguageModelToolSchemaFormat,    LanguageModelToolUse, MessageContent, Role, StopReason, fake_provider::FakeLanguageModel,};use pretty_assertions::assert_eq;use project::{    Project, context_server_store::ContextServerStore, project_settings::ProjectSettings,};use prompt_store::ProjectContext;use reqwest_client::ReqwestClient;use schemars::JsonSchema;use serde::{Deserialize, Serialize};use serde_json::json;use settings::{Settings, SettingsStore};use std::{path::Path, rc::Rc, sync::Arc, time::Duration};use util::path; mod test_tools;use test_tools::*; #[gpui::test]async fn test_echo(cx: &mut TestAppContext) {    let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;    let fake_model = model.as_fake();     let events = thread        .update(cx, |thread, cx| {            thread.send(UserMessageId::new(), ["Testing: Reply with 'Hello'"], cx)        })        .unwrap();    cx.run_until_parked();    fake_model.send_last_completion_stream_text_chunk("Hello");    fake_model        .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::EndTurn));    fake_model.end_last_completion_stream();     let events = events.collect().await;    thread.update(cx, |thread, _cx| {        assert_eq!(            thread.last_message().unwrap().to_markdown(),            indoc! {"                ## Assistant                 Hello            "}        )    });    assert_eq!(stop_events(events), vec![acp::StopReason::EndTurn]);} #[gpui::test]async fn test_thinking(cx: &mut TestAppContext) {    let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;    let fake_model = model.as_fake();     let events = thread        .update(cx, |thread, cx| {            thread.send(                UserMessageId::new(),                [indoc! {"                    Testing:                     Generate a thinking step where you just think the word 'Think',                    and have your final answer be 'Hello'                "}],                cx,            )        })        .unwrap();    cx.run_until_parked();    fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::Thinking {        text: "Think".to_string(),        signature: None,    });    fake_model.send_last_completion_stream_text_chunk("Hello");    fake_model        .send_last_completion_stream_event(LanguageModelCompletionEvent::Stop(StopReason::EndTurn));    fake_model.end_last_completion_stream();     let events = events.collect().await;    thread.update(cx, |thread, _cx| {        assert_eq!(            thread.last_message().unwrap().to_markdown(),            indoc! {"                ## Assistant                 <think>Think</think>                Hello            "}        )    });    assert_eq!(stop_events(events), vec![acp::StopReason::EndTurn]);} #[gpui::test]async fn test_system_prompt(cx: &mut TestAppContext) {    let ThreadTest {        model,        thread,        project_context,        ..    } = setup(cx, TestModel::Fake).await;    let fake_model = model.as_fake();     project_context.update(cx, |project_context, _cx| {        project_context.shell = "test-shell".into()    });    thread.update(cx, |thread, _| thread.add_tool(EchoTool));    thread        .update(cx, |thread, cx| {            thread.send(UserMessageId::new(), ["abc"], cx)        })        .unwrap();    cx.run_until_parked();    let mut pending_completions = fake_model.pending_completions();    assert_eq!(        pending_completions.len(),        1,        "unexpected pending completions: {:?}",        pending_completions    );     let pending_completion = pending_completions.pop().unwrap();    assert_eq!(pending_completion.messages[0].role, Role::System);     let system_message = &pending_completion.messages[0];    let system_prompt = system_message.content[0].to_str().unwrap();    assert!(        system_prompt.contains("test-shell"),        "unexpected system message: {:?}",        system_message    );    assert!(        system_prompt.contains("## Fixing Diagnostics"),        "unexpected system message: {:?}",        system_message    );} #[gpui::test]async fn test_system_prompt_without_tools(cx: &mut TestAppContext) {    let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;    let fake_model = model.as_fake();     thread        .update(cx, |thread, cx| {            thread.send(UserMessageId::new(), ["abc"], cx)        })        .unwrap();    cx.run_until_parked();    let mut pending_completions = fake_model.pending_completions();    assert_eq!(        pending_completions.len(),        1,        "unexpected pending completions: {:?}",        pending_completions    );     let pending_completion = pending_completions.pop().unwrap();    assert_eq!(pending_completion.messages[0].role, Role::System);     let system_message = &pending_completion.messages[0];    let system_prompt = system_message.content[0].to_str().unwrap();    assert!(        !system_prompt.contains("## Tool Use"),        "unexpected system message: {:?}",        system_message    );    assert!(        !system_prompt.contains("## Fixing Diagnostics"),        "unexpected system message: {:?}",        system_message    );} #[gpui::test]async fn test_prompt_caching(cx: &mut TestAppContext) {    let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;    let fake_model = model.as_fake();     // Send initial user message and verify it's cached    thread        .update(cx, |thread, cx| {            thread.send(UserMessageId::new(), ["Message 1"], cx)        })        .unwrap();    cx.run_until_parked();     let completion = fake_model.pending_completions().pop().unwrap();    assert_eq!(        completion.messages[1..],        vec![LanguageModelRequestMessage {            role: Role::User,            content: vec!["Message 1".into()],            cache: true        }]    );    fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::Text(        "Response to Message 1".into(),    ));    fake_model.end_last_completion_stream();    cx.run_until_parked();     // Send another user message and verify only the latest is cached    thread        .update(cx, |thread, cx| {            thread.send(UserMessageId::new(), ["Message 2"], cx)        })        .unwrap();    cx.run_until_parked();     let completion = fake_model.pending_completions().pop().unwrap();    assert_eq!(        completion.messages[1..],        vec![            LanguageModelRequestMessage {                role: Role::User,                content: vec!["Message 1".into()],                cache: false            },            LanguageModelRequestMessage {                role: Role::Assistant,                content: vec!["Response to Message 1".into()],                cache: false            },            LanguageModelRequestMessage {                role: Role::User,                content: vec!["Message 2".into()],                cache: true            }        ]    );    fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::Text(        "Response to Message 2".into(),    ));    fake_model.end_last_completion_stream();    cx.run_until_parked();     // Simulate a tool call and verify that the latest tool result is cached    thread.update(cx, |thread, _| thread.add_tool(EchoTool));    thread        .update(cx, |thread, cx| {            thread.send(UserMessageId::new(), ["Use the echo tool"], cx)        })        .unwrap();    cx.run_until_parked();     let tool_use = LanguageModelToolUse {        id: "tool_1".into(),        name: EchoTool::name().into(),        raw_input: json!({"text": "test"}).to_string(),        input: json!({"text": "test"}),        is_input_complete: true,        thought_signature: None,    };    fake_model        .send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(tool_use.clone()));    fake_model.end_last_completion_stream();    cx.run_until_parked();     let completion = fake_model.pending_completions().pop().unwrap();    let tool_result = LanguageModelToolResult {        tool_use_id: "tool_1".into(),        tool_name: EchoTool::name().into(),        is_error: false,        content: "test".into(),        output: Some("test".into()),    };    assert_eq!(        completion.messages[1..],        vec![            LanguageModelRequestMessage {                role: Role::User,                content: vec!["Message 1".into()],                cache: false            },            LanguageModelRequestMessage {                role: Role::Assistant,                content: vec!["Response to Message 1".into()],                cache: false            },            LanguageModelRequestMessage {                role: Role::User,                content: vec!["Message 2".into()],                cache: false            },            LanguageModelRequestMessage {                role: Role::Assistant,                content: vec!["Response to Message 2".into()],                cache: false            },            LanguageModelRequestMessage {                role: Role::User,                content: vec!["Use the echo tool".into()],                cache: false            },            LanguageModelRequestMessage {                role: Role::Assistant,                content: vec![MessageContent::ToolUse(tool_use)],                cache: false            },            LanguageModelRequestMessage {                role: Role::User,                content: vec![MessageContent::ToolResult(tool_result)],                cache: true            }        ]    );} #[gpui::test]#[cfg_attr(not(feature = "e2e"), ignore)]async fn test_basic_tool_calls(cx: &mut TestAppContext) {    let ThreadTest { thread, .. } = setup(cx, TestModel::Sonnet4).await;     // Test a tool call that's likely to complete *before* streaming stops.    let events = thread        .update(cx, |thread, cx| {            thread.add_tool(EchoTool);            thread.send(                UserMessageId::new(),                ["Now test the echo tool with 'Hello'. Does it work? Say 'Yes' or 'No'."],                cx,            )        })        .unwrap()        .collect()        .await;    assert_eq!(stop_events(events), vec![acp::StopReason::EndTurn]);     // Test a tool calls that's likely to complete *after* streaming stops.    let events = thread        .update(cx, |thread, cx| {            thread.remove_tool(&EchoTool::name());            thread.add_tool(DelayTool);            thread.send(                UserMessageId::new(),                [                    "Now call the delay tool with 200ms.",                    "When the timer goes off, then you echo the output of the tool.",                ],                cx,            )        })        .unwrap()        .collect()        .await;    assert_eq!(stop_events(events), vec![acp::StopReason::EndTurn]);    thread.update(cx, |thread, _cx| {        assert!(            thread                .last_message()                .unwrap()                .as_agent_message()                .unwrap()                .content                .iter()                .any(|content| {                    if let AgentMessageContent::Text(text) = content {                        text.contains("Ding")                    } else {                        false                    }                }),            "{}",            thread.to_markdown()        );    });} #[gpui::test]#[cfg_attr(not(feature = "e2e"), ignore)]async fn test_streaming_tool_calls(cx: &mut TestAppContext) {    let ThreadTest { thread, .. } = setup(cx, TestModel::Sonnet4).await;     // Test a tool call that's likely to complete *before* streaming stops.    let mut events = thread        .update(cx, |thread, cx| {            thread.add_tool(WordListTool);            thread.send(UserMessageId::new(), ["Test the word_list tool."], cx)        })        .unwrap();     let mut saw_partial_tool_use = false;    while let Some(event) = events.next().await {        if let Ok(ThreadEvent::ToolCall(tool_call)) = event {            thread.update(cx, |thread, _cx| {                // Look for a tool use in the thread's last message                let message = thread.last_message().unwrap();                let agent_message = message.as_agent_message().unwrap();                let last_content = agent_message.content.last().unwrap();                if let AgentMessageContent::ToolUse(last_tool_use) = last_content {                    assert_eq!(last_tool_use.name.as_ref(), "word_list");                    if tool_call.status == acp::ToolCallStatus::Pending {                        if !last_tool_use.is_input_complete                            && last_tool_use.input.get("g").is_none()                        {                            saw_partial_tool_use = true;                        }                    } else {                        last_tool_use                            .input                            .get("a")                            .expect("'a' has streamed because input is now complete");                        last_tool_use                            .input                            .get("g")                            .expect("'g' has streamed because input is now complete");                    }                } else {                    panic!("last content should be a tool use");                }            });        }    }     assert!(        saw_partial_tool_use,        "should see at least one partially streamed tool use in the history"    );} #[gpui::test]async fn test_tool_authorization(cx: &mut TestAppContext) {    let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;    let fake_model = model.as_fake();     let mut events = thread        .update(cx, |thread, cx| {            thread.add_tool(ToolRequiringPermission);            thread.send(UserMessageId::new(), ["abc"], cx)        })        .unwrap();    cx.run_until_parked();    fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(        LanguageModelToolUse {            id: "tool_id_1".into(),            name: ToolRequiringPermission::name().into(),            raw_input: "{}".into(),            input: json!({}),            is_input_complete: true,            thought_signature: None,        },    ));    fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(        LanguageModelToolUse {            id: "tool_id_2".into(),            name: ToolRequiringPermission::name().into(),            raw_input: "{}".into(),            input: json!({}),            is_input_complete: true,            thought_signature: None,        },    ));    fake_model.end_last_completion_stream();    let tool_call_auth_1 = next_tool_call_authorization(&mut events).await;    let tool_call_auth_2 = next_tool_call_authorization(&mut events).await;     // Approve the first    tool_call_auth_1        .response        .send(tool_call_auth_1.options[1].id.clone())        .unwrap();    cx.run_until_parked();     // Reject the second    tool_call_auth_2        .response        .send(tool_call_auth_1.options[2].id.clone())        .unwrap();    cx.run_until_parked();     let completion = fake_model.pending_completions().pop().unwrap();    let message = completion.messages.last().unwrap();    assert_eq!(        message.content,        vec![

Lines 1–500 of 2595.