Files
review-bot/review/parser_test.go
rodin cabbb5a55a
CI / test (push) Successful in 14s
CI / review (/anthropic/v1, claude-sonnet-4-6, sonnet, anthropic, SONNET_REVIEW_TOKEN) (push) Has been skipped
CI / review (/openai/v1, gpt-4.1, gpt41, openai, GPT_REVIEW_TOKEN) (push) Has been skipped
CI / review (/openai/v1, gpt-4.1-mini, gpt41-mini, openai, GPT_REVIEW_TOKEN) (push) Has been skipped
CI / review (/openai/v1, gpt-5, gpt, openai, GPT_REVIEW_TOKEN) (push) Has been skipped
CI / review (/openai/v1, gpt-5, security, openai, SECURITY_REVIEW.md, SECURITY_REVIEW_TOKEN) (push) Has been skipped
CI / review (/openai/v1, gpt-5-mini, gpt5-mini, openai, GPT_REVIEW_TOKEN) (push) Has been skipped
Release / release (push) Successful in 34s
fix: repair unescaped quotes in LLM JSON responses (#45)
fix: repair unescaped quotes in LLM JSON responses

Add repairJSON fallback that handles unescaped quotes in LLM string
values using first-valid-candidate heuristic with structural lookahead.

Reviewed-by: sonnet-review-bot
Reviewed-by: gpt-review-bot
Reviewed-by: security-review-bot
2026-05-05 12:40:39 +00:00

225 lines
7.4 KiB
Go

package review
import (
"encoding/json"
"testing"
)
func TestParseResponse_ValidJSON(t *testing.T) {
input := `{
"verdict": "APPROVE",
"summary": "Looks good",
"findings": [
{"severity": "NIT", "file": "main.go", "line": 10, "finding": "Consider renaming"}
],
"recommendation": "Ship it"
}`
result, err := ParseResponse(input)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Verdict != "APPROVE" {
t.Errorf("expected verdict APPROVE, got %q", result.Verdict)
}
if result.Summary != "Looks good" {
t.Errorf("expected summary %q, got %q", "Looks good", result.Summary)
}
if len(result.Findings) != 1 {
t.Fatalf("expected 1 finding, got %d", len(result.Findings))
}
if result.Findings[0].Severity != "NIT" {
t.Errorf("expected severity NIT, got %q", result.Findings[0].Severity)
}
if result.Findings[0].File != "main.go" {
t.Errorf("expected file main.go, got %q", result.Findings[0].File)
}
if result.Findings[0].Line != 10 {
t.Errorf("expected line 10, got %d", result.Findings[0].Line)
}
if result.Recommendation != "Ship it" {
t.Errorf("expected recommendation %q, got %q", "Ship it", result.Recommendation)
}
}
func TestParseResponse_MarkdownFences(t *testing.T) {
input := "```json\n{\"verdict\": \"REQUEST_CHANGES\", \"summary\": \"Issues found\", \"findings\": [{\"severity\": \"MAJOR\", \"file\": \"a.go\", \"line\": 5, \"finding\": \"Bug\"}], \"recommendation\": \"Fix it\"}\n```"
result, err := ParseResponse(input)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Verdict != "REQUEST_CHANGES" {
t.Errorf("expected verdict REQUEST_CHANGES, got %q", result.Verdict)
}
if len(result.Findings) != 1 {
t.Fatalf("expected 1 finding, got %d", len(result.Findings))
}
if result.Findings[0].Severity != "MAJOR" {
t.Errorf("expected severity MAJOR, got %q", result.Findings[0].Severity)
}
}
func TestParseResponse_InvalidJSON(t *testing.T) {
_, err := ParseResponse("this is not json")
if err == nil {
t.Fatal("expected error for invalid JSON, got nil")
}
}
func TestParseResponse_InvalidVerdict(t *testing.T) {
input := `{"verdict": "MAYBE", "summary": "Hmm", "findings": [], "recommendation": "Dunno"}`
_, err := ParseResponse(input)
if err == nil {
t.Fatal("expected error for invalid verdict, got nil")
}
}
func TestParseResponse_InvalidSeverity(t *testing.T) {
input := `{"verdict": "APPROVE", "summary": "Ok", "findings": [{"severity": "CRITICAL", "file": "x.go", "line": 1, "finding": "bad"}], "recommendation": "Fix"}`
_, err := ParseResponse(input)
if err == nil {
t.Fatal("expected error for invalid severity, got nil")
}
}
func TestParseResponse_EmptyFindings(t *testing.T) {
input := `{"verdict": "APPROVE", "summary": "All good", "findings": [], "recommendation": "Merge"}`
result, err := ParseResponse(input)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(result.Findings) != 0 {
t.Errorf("expected 0 findings, got %d", len(result.Findings))
}
}
func TestParseResponse_MissingFields(t *testing.T) {
// verdict is empty string — should fail validation
input := `{"summary": "Ok", "findings": [], "recommendation": "Merge"}`
_, err := ParseResponse(input)
if err == nil {
t.Fatal("expected error for missing verdict, got nil")
}
}
func TestParseResponse_MarkdownFencesNoLang(t *testing.T) {
input := "```\n{\"verdict\": \"APPROVE\", \"summary\": \"Fine\", \"findings\": [], \"recommendation\": \"Good\"}\n```"
result, err := ParseResponse(input)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Verdict != "APPROVE" {
t.Errorf("expected APPROVE, got %q", result.Verdict)
}
}
func TestParseResponse_UnescapedQuotesInStrings(t *testing.T) {
// Real failure from CI: Sonnet puts unescaped quotes like (e.g. "28") in findings
input := `{"verdict": "APPROVE", "summary": "Clean PR", "findings": [{"severity": "NIT", "file": "ci/Dockerfile", "line": 14, "finding": "The comment says OTP_VERSION is the major version (e.g. \"28\") but it actually contains unescaped quotes like (e.g. "28") which breaks JSON"}], "recommendation": "Ship it"}`
result, err := ParseResponse(input)
if err != nil {
t.Fatalf("expected repair to handle unescaped quotes, got error: %v", err)
}
if result.Verdict != "APPROVE" {
t.Errorf("expected APPROVE, got %q", result.Verdict)
}
if len(result.Findings) != 1 {
t.Fatalf("expected 1 finding, got %d", len(result.Findings))
}
}
func TestRepairJSON_NoOpOnValid(t *testing.T) {
valid := `{"key": "value", "num": 42}`
result := repairJSON(valid)
if result != valid {
t.Errorf("repairJSON should not modify valid JSON\n got: %s\n want: %s", result, valid)
}
}
func TestRepairJSON_FixesUnescapedQuotes(t *testing.T) {
// Interior quote followed by non-structural character
input := `{"msg": "use "foo" here"}`
result := repairJSON(input)
// Should be parseable now
var m map[string]interface{}
if err := json.Unmarshal([]byte(result), &m); err != nil {
t.Fatalf("repaired JSON should parse, got: %v\nrepaired: %s", err, result)
}
}
func TestRepairJSON_InteriorQuoteBeforeComma(t *testing.T) {
// Bug reported by reviewer: interior quoted word immediately before a comma
input := `{"msg": "say "yes", and go"}`
result := repairJSON(input)
var m map[string]interface{}
if err := json.Unmarshal([]byte(result), &m); err != nil {
t.Fatalf("repaired JSON should parse, got: %v\nrepaired: %s", err, result)
}
// The full string content should be preserved
msg, ok := m["msg"].(string)
if !ok {
t.Fatal("msg field missing or not a string")
}
if msg != `say "yes", and go` {
t.Errorf("unexpected msg content: %q", msg)
}
}
func TestRepairJSON_InteriorQuoteBeforeCloseBrace(t *testing.T) {
// Bug reported by reviewer: JSON-shaped syntax inside string values
input := `{"msg": "input map {"key": "val"} caused error"}`
result := repairJSON(input)
var m map[string]interface{}
if err := json.Unmarshal([]byte(result), &m); err != nil {
t.Fatalf("repaired JSON should parse, got: %v\nrepaired: %s", err, result)
}
}
func TestRepairJSON_MultipleFields(t *testing.T) {
// Multiple string fields with unescaped quotes in different positions
input := `{"a": "hello "world"", "b": "foo"}`
result := repairJSON(input)
var m map[string]interface{}
if err := json.Unmarshal([]byte(result), &m); err != nil {
t.Fatalf("repaired JSON should parse, got: %v\nrepaired: %s", err, result)
}
if _, ok := m["b"]; !ok {
t.Error("expected 'b' field to be preserved")
}
}
func TestRepairJSON_PreservesEscapedQuotes(t *testing.T) {
// Already-escaped quotes should not be double-escaped
input := `{"msg": "already \"escaped\" here"}`
result := repairJSON(input)
if result != input {
t.Errorf("repairJSON should not modify already-escaped quotes\n got: %s\n want: %s", result, input)
}
var m map[string]interface{}
if err := json.Unmarshal([]byte(result), &m); err != nil {
t.Fatalf("repaired JSON should parse, got: %v\nrepaired: %s", err, result)
}
}
func TestRepairJSON_ComplexNestedContent(t *testing.T) {
// Combines both reviewer bugs: quoted words before commas AND JSON-like content
input := `{"verdict": "APPROVE", "findings": [{"finding": "The map {"key": "val"} and (e.g. "28") and say "yes", then stop"}]}`
result := repairJSON(input)
var parsed map[string]interface{}
if err := json.Unmarshal([]byte(result), &parsed); err != nil {
t.Fatalf("repaired JSON should parse, got: %v\nrepaired: %s", err, result)
}
if parsed["verdict"] != "APPROVE" {
t.Errorf("expected verdict APPROVE, got %v", parsed["verdict"])
}
}