History

Make eval more resilient to bad input from LLM (#29703)

I saw a slice panic (for begin > end) in a debug build of the eval. This should just be a failed assertion, not a panic that takes out the whole eval run! Release Notes: - N/A

Richard Feldman committed 1 year ago

Commit afeb3d41 parent 92dd6b61 file · +51 −41Browse files

Unified · Split

crates/eval/src/examples/code_block_citations.rs+51−41
@@ -98,54 +98,64 @@ impl Example for CodeBlockCitations { · 97 unmodified lines
98 98
99 if let Some(content_len) = content_len {99 if let Some(content_len) = content_len {
100 // + 1 because there's a newline character after the citation.100 // + 1 because there's a newline character after the citation.
101 let content =101 let start_index = citation.len() + 1;
102 &text[(citation.len() + 1)..content_len - (citation.len() + 1)];102 let end_index = content_len.saturating_sub(start_index);
103
104 // deindent (trim the start of each line) because sometimes the model
105 // chooses to deindent its code snippets for the sake of readability,
106 // which in markdown is not only reasonable but usually desirable.
107 cx.assert(
108 deindent(&buffer_text)
109 .trim()
110 .contains(deindent(&content).trim()),
111 "Code block content was found in file",
112 )
113 .ok();
114
115 if let Some(range) = path_range.range {
116 let start_line_index = range.start.line.saturating_sub(1);
117 let line_count =
118 range.end.line.saturating_sub(start_line_index);
119 let mut snippet = buffer_text
120 .lines()
121 .skip(start_line_index as usize)
122 .take(line_count as usize)
123 .collect::<Vec<&str>>()
124 .join("\n");
125
126 if let Some(start_col) = range.start.col {
127 snippet = snippet[start_col as usize..].to_string();
128 }
129 103
130 if let Some(end_col) = range.end.col {104 if cx
131 let last_line = snippet.lines().last().unwrap();105 .assert(
132 snippet = snippet106 start_index <= end_index,
133 [..snippet.len() - last_line.len() + end_col as usize]107 "Code block had a valid citation",
134 .to_string();108 )
135 }109 .is_ok()
110 {
111 let content = &text[start_index..end_index];
136 112
137 // deindent (trim the start of each line) because sometimes the model113 // deindent (trim the start of each line) because sometimes the model
138 // chooses to deindent its code snippets for the sake of readability,114 // chooses to deindent its code snippets for the sake of readability,
139 // which in markdown is not only reasonable but usually desirable.115 // which in markdown is not only reasonable but usually desirable.
140 cx.assert_eq(116 cx.assert(
141 deindent(snippet.as_str()).trim(),117 deindent(&buffer_text)
142 deindent(content).trim(),118 .trim()
143 format!(119 .contains(deindent(&content).trim()),
144 "Code block was at {:?}-{:?}",120 "Code block content was found in file",
145 range.start, range.end
146 ),
147 )121 )
148 .ok();122 .ok();
123
124 if let Some(range) = path_range.range {
125 let start_line_index = range.start.line.saturating_sub(1);
126 let line_count =
127 range.end.line.saturating_sub(start_line_index);
128 let mut snippet = buffer_text
129 .lines()
130 .skip(start_line_index as usize)
131 .take(line_count as usize)
132 .collect::<Vec<&str>>()
133 .join("\n");
134
135 if let Some(start_col) = range.start.col {
136 snippet = snippet[start_col as usize..].to_string();
137 }
138
139 if let Some(end_col) = range.end.col {
140 let last_line = snippet.lines().last().unwrap();
141 snippet = snippet[..snippet.len() - last_line.len()
142 + end_col as usize]
143 .to_string();
144 }
145
146 // deindent (trim the start of each line) because sometimes the model
147 // chooses to deindent its code snippets for the sake of readability,
148 // which in markdown is not only reasonable but usually desirable.
149 cx.assert_eq(
150 deindent(snippet.as_str()).trim(),
151 deindent(content).trim(),
152 format!(
153 "Code block was at {:?}-{:?}",
154 range.start, range.end
155 ),
156 )
157 .ok();
158 }
149 }159 }
150 }160 }
151 }161 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t