History

Make eval more resilient to bad input from LLM (#29703)

I saw a slice panic (for begin > end) in a debug build of the eval. This should just be a failed assertion, not a panic that takes out the whole eval run! Release Notes: - N/A

Richard Feldman committed 1 year ago

Commit afeb3d41 parent 92dd6b61 file · +51 −41Browse files

Unified · Split

crates/eval/src/examples/code_block_citations.rs+51−41
@@ -98,54 +98,64 @@ impl Example for CodeBlockCitations { · 97 unmodified lines
9898
9999 if let Some(content_len) = content_len {
100100 // + 1 because there's a newline character after the citation.
101− let content =
102− &text[(citation.len() + 1)..content_len - (citation.len() + 1)];
103−
104− // deindent (trim the start of each line) because sometimes the model
105− // chooses to deindent its code snippets for the sake of readability,
106− // which in markdown is not only reasonable but usually desirable.
107− cx.assert(
108− deindent(&buffer_text)
109− .trim()
110− .contains(deindent(&content).trim()),
111− "Code block content was found in file",
112− )
113− .ok();
114−
115− if let Some(range) = path_range.range {
116− let start_line_index = range.start.line.saturating_sub(1);
117− let line_count =
118− range.end.line.saturating_sub(start_line_index);
119− let mut snippet = buffer_text
120− .lines()
121− .skip(start_line_index as usize)
122− .take(line_count as usize)
123− .collect::<Vec<&str>>()
124− .join("\n");
125−
126− if let Some(start_col) = range.start.col {
127− snippet = snippet[start_col as usize..].to_string();
128− }
101+ let start_index = citation.len() + 1;
102+ let end_index = content_len.saturating_sub(start_index);
129103
130− if let Some(end_col) = range.end.col {
131− let last_line = snippet.lines().last().unwrap();
132− snippet = snippet
133− [..snippet.len() - last_line.len() + end_col as usize]
134− .to_string();
135− }
104+ if cx
105+ .assert(
106+ start_index <= end_index,
107+ "Code block had a valid citation",
108+ )
109+ .is_ok()
110+ {
111+ let content = &text[start_index..end_index];
136112
137113 // deindent (trim the start of each line) because sometimes the model
138114 // chooses to deindent its code snippets for the sake of readability,
139115 // which in markdown is not only reasonable but usually desirable.
140− cx.assert_eq(
141− deindent(snippet.as_str()).trim(),
142− deindent(content).trim(),
143− format!(
144− "Code block was at {:?}-{:?}",
145− range.start, range.end
146− ),
116+ cx.assert(
117+ deindent(&buffer_text)
118+ .trim()
119+ .contains(deindent(&content).trim()),
120+ "Code block content was found in file",
147121 )
148122 .ok();
123+
124+ if let Some(range) = path_range.range {
125+ let start_line_index = range.start.line.saturating_sub(1);
126+ let line_count =
127+ range.end.line.saturating_sub(start_line_index);
128+ let mut snippet = buffer_text
129+ .lines()
130+ .skip(start_line_index as usize)
131+ .take(line_count as usize)
132+ .collect::<Vec<&str>>()
133+ .join("\n");
134+
135+ if let Some(start_col) = range.start.col {
136+ snippet = snippet[start_col as usize..].to_string();
137+ }
138+
139+ if let Some(end_col) = range.end.col {
140+ let last_line = snippet.lines().last().unwrap();
141+ snippet = snippet[..snippet.len() - last_line.len()
142+ + end_col as usize]
143+ .to_string();
144+ }
145+
146+ // deindent (trim the start of each line) because sometimes the model
147+ // chooses to deindent its code snippets for the sake of readability,
148+ // which in markdown is not only reasonable but usually desirable.
149+ cx.assert_eq(
150+ deindent(snippet.as_str()).trim(),
151+ deindent(content).trim(),
152+ format!(
153+ "Code block was at {:?}-{:?}",
154+ range.start, range.end
155+ ),
156+ )
157+ .ok();
158+ }
149159 }
150160 }
151161 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t