255 lines · 8.0 KB · TypeScript
RawDownload
/** * Example: Using Dismissed Issues for Generator-Judge Feedback Loop * * This shows how to programmatically analyze dismissed issues to improve * the issue generator and create a dialog between generator and judge. */ import { readFile } from 'fs/promises';import { join } from 'path'; interface DismissedIssue {  commentId: string;  reason: string;  dismissedAt: string;  dismissedAtIteration: number;  category: 'already-fixed' | 'not-an-issue' | 'file-unchanged' | 'false-positive' | 'duplicate';  filePath: string;  line: number | null;  commentBody: string;} interface ResolverState {  dismissedIssues?: DismissedIssue[];  // ... other state fields} /** * Load dismissed issues from state file */async function loadDismissedIssues(workdir: string): Promise<DismissedIssue[]> {  const statePath = join(workdir, '.pr-resolver-state.json');  const content = await readFile(statePath, 'utf-8');  const state: ResolverState = JSON.parse(content);  return state.dismissedIssues || [];} /** * Analyze dismissal patterns to provide feedback to generator */function analyzePatterns(dismissed: DismissedIssue[]): {  patterns: string[];  recommendations: string[];  stats: Record<string, number>;} {  const patterns: string[] = [];  const recommendations: string[] = [];  const stats: Record<string, number> = {};   // Count by category  for (const issue of dismissed) {    stats[issue.category] = (stats[issue.category] || 0) + 1;  }   // Analyze "already-fixed" issues  const alreadyFixed = dismissed.filter(i => i.category === 'already-fixed');  if (alreadyFixed.length > 0) {    patterns.push(`${alreadyFixed.length} issues were flagged as needing fixes that already exist`);     // Look for common reasons    const nullCheckReasons = alreadyFixed.filter(i =>      i.reason.toLowerCase().includes('null check') ||      i.reason.toLowerCase().includes('guard')    );     if (nullCheckReasons.length > 0) {      patterns.push(`  → ${nullCheckReasons.length} issues missed existing null checks/guards`);      recommendations.push('Improve detection of early return guards and defensive checks');    }     const typeSystemReasons = alreadyFixed.filter(i =>      i.reason.toLowerCase().includes('typescript') ||      i.reason.toLowerCase().includes('type') ||      i.reason.toLowerCase().includes('never be null')    );     if (typeSystemReasons.length > 0) {      patterns.push(`  → ${typeSystemReasons.length} issues ignored by type system guarantees`);      recommendations.push('Analyze TypeScript types to avoid flagging impossible scenarios');    }  }   // Analyze "file-unchanged" issues  const fileUnchanged = dismissed.filter(i => i.category === 'file-unchanged');  if (fileUnchanged.length > 0) {    patterns.push(`${fileUnchanged.length} issues in files that weren't modified`);    recommendations.push('Consider analyzing file change likelihood before generating issues');  }   // Analyze "false-positive" issues  const falsePositives = dismissed.filter(i => i.category === 'false-positive');  if (falsePositives.length > 0) {    patterns.push(`${falsePositives.length} false positives detected`);    recommendations.push('Review false positive patterns and adjust detection heuristics');  }   return { patterns, recommendations, stats };} /** * Generate feedback message for the generator */function generateFeedbackMessage(dismissed: DismissedIssue[]): string {  if (dismissed.length === 0) {    return '✅ All generated issues were valid! No false positives detected.';  }   const analysis = analyzePatterns(dismissed);  const lines: string[] = [];   lines.push(`\n📊 GENERATOR FEEDBACK REPORT`);  lines.push(`════════════════════════════\n`);   // Stats summary  lines.push(`Total dismissed: ${dismissed.length}`);  for (const [category, count] of Object.entries(analysis.stats)) {    lines.push(`  • ${category}: ${count}`);  }   // Patterns detected  if (analysis.patterns.length > 0) {    lines.push(`\n🔍 PATTERNS DETECTED:`);    for (const pattern of analysis.patterns) {      lines.push(pattern);    }  }   // Recommendations  if (analysis.recommendations.length > 0) {    lines.push(`\n💡 RECOMMENDATIONS FOR GENERATOR:`);    for (let i = 0; i < analysis.recommendations.length; i++) {      lines.push(`${i + 1}. ${analysis.recommendations[i]}`);    }  }   // Specific examples  lines.push(`\n📝 EXAMPLE DISMISSALS:\n`);  const examples = dismissed.slice(0, 3); // Show first 3  for (const issue of examples) {    lines.push(`Category: ${issue.category}`);    lines.push(`Location: ${issue.filePath}:${issue.line || '?'}`);    lines.push(`Generated comment: "${issue.commentBody}"`);    lines.push(`Judge's reason: "${issue.reason}"`);    lines.push(``);  }   return lines.join('\n');} /** * Generate specific learning points for the generator's prompt */function generateLearningPoints(dismissed: DismissedIssue[]): string[] {  const learnings: string[] = [];   // Analyze reasons for common keywords  const reasonTexts = dismissed.map(i => i.reason.toLowerCase()).join(' ');   if (reasonTexts.includes('null check') || reasonTexts.includes('guard clause')) {    learnings.push('Check for existing early return guards (e.g., `if (x === null) return;`)');  }   if (reasonTexts.includes('typescript') || reasonTexts.includes('type system')) {    learnings.push('Consider TypeScript type system guarantees before flagging null/undefined issues');  }   if (reasonTexts.includes('already implements') || reasonTexts.includes('already has')) {    learnings.push('Search for existing implementations before suggesting new ones');  }   // Check comment patterns  const commentPatterns = dismissed.map(i => i.commentBody.toLowerCase());   const nullSafetyComments = commentPatterns.filter(c =>    c.includes('null') || c.includes('undefined') || c.includes('check')  ).length;   if (nullSafetyComments > dismissed.length * 0.3) {    learnings.push('Null/undefined checks are frequently already present - increase scrutiny for these');  }   return learnings;} /** * Create a structured feedback object for the generator */function createFeedbackForGenerator(dismissed: DismissedIssue[]): {  timestamp: string;  dismissalRate: number;  byCategory: Record<string, number>;  learningPoints: string[];  examples: Array<{    generated: string;    judgeResponse: string;    category: string;  }>;} {  const analysis = analyzePatterns(dismissed);  const learningPoints = generateLearningPoints(dismissed);   return {    timestamp: new Date().toISOString(),    dismissalRate: dismissed.length, // Could be a percentage if we track total issues    byCategory: analysis.stats,    learningPoints,    examples: dismissed.slice(0, 5).map(issue => ({      generated: issue.commentBody,      judgeResponse: issue.reason,      category: issue.category,    })),  };} // Example usageasync function main() {  const workdir = process.argv[2] || '.';   try {    const dismissed = await loadDismissedIssues(workdir);     // Print feedback report    console.log(generateFeedbackMessage(dismissed));     // Generate structured feedback for generator    const feedback = createFeedbackForGenerator(dismissed);     console.log('\n📤 STRUCTURED FEEDBACK FOR GENERATOR:');    console.log(JSON.stringify(feedback, null, 2));     // These learning points can be added to the generator's prompt    if (feedback.learningPoints.length > 0) {      console.log('\n📚 ADD TO GENERATOR PROMPT:');      console.log('Lessons learned from previous false positives:');      for (const point of feedback.learningPoints) {        console.log(`  - ${point}`);      }    }  } catch (error) {    console.error('Error loading dismissed issues:', error);  }} // Run if executed directly (ESM compatible)import { fileURLToPath } from 'url';if (process.argv[1] === fileURLToPath(import.meta.url)) {  main();} export {  loadDismissedIssues,  analyzePatterns,  generateFeedbackMessage,  generateLearningPoints,  createFeedbackForGenerator,};