Files
review-bot/review/parser_test.go
T
Rodin 80a9a7675b
CI / test (pull_request) Successful in 13s
CI / review (gpt-5, security, SECURITY_REVIEW.md, SECURITY_REVIEW_TOKEN) (pull_request) Failing after 13s
CI / review (gpt-4.1, gpt, GPT_REVIEW_TOKEN) (pull_request) Failing after 13s
CI / review (gpt-5, sonnet, SONNET_REVIEW_TOKEN) (pull_request) Failing after 12s
fix: repair unescaped quotes in LLM JSON responses
LLMs (especially Sonnet) sometimes emit JSON with unescaped double
quotes inside string values, e.g. (e.g. "28") instead of properly
escaping them. This caused parse failures in CI.

Add a repairJSON fallback that uses a character-by-character scanner
to identify interior quotes (those not followed by structural JSON
characters) and escape them before retrying the parse.

Fixes sonnet-review failures on gargoyle PR #551.
2026-05-03 09:47:22 -07:00

152 lines
5.0 KiB
Go

package review
import (
"encoding/json"
"testing"
)
func TestParseResponse_ValidJSON(t *testing.T) {
input := `{
"verdict": "APPROVE",
"summary": "Looks good",
"findings": [
{"severity": "NIT", "file": "main.go", "line": 10, "finding": "Consider renaming"}
],
"recommendation": "Ship it"
}`
result, err := ParseResponse(input)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Verdict != "APPROVE" {
t.Errorf("expected verdict APPROVE, got %q", result.Verdict)
}
if result.Summary != "Looks good" {
t.Errorf("expected summary %q, got %q", "Looks good", result.Summary)
}
if len(result.Findings) != 1 {
t.Fatalf("expected 1 finding, got %d", len(result.Findings))
}
if result.Findings[0].Severity != "NIT" {
t.Errorf("expected severity NIT, got %q", result.Findings[0].Severity)
}
if result.Findings[0].File != "main.go" {
t.Errorf("expected file main.go, got %q", result.Findings[0].File)
}
if result.Findings[0].Line != 10 {
t.Errorf("expected line 10, got %d", result.Findings[0].Line)
}
if result.Recommendation != "Ship it" {
t.Errorf("expected recommendation %q, got %q", "Ship it", result.Recommendation)
}
}
func TestParseResponse_MarkdownFences(t *testing.T) {
input := "```json\n{\"verdict\": \"REQUEST_CHANGES\", \"summary\": \"Issues found\", \"findings\": [{\"severity\": \"MAJOR\", \"file\": \"a.go\", \"line\": 5, \"finding\": \"Bug\"}], \"recommendation\": \"Fix it\"}\n```"
result, err := ParseResponse(input)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Verdict != "REQUEST_CHANGES" {
t.Errorf("expected verdict REQUEST_CHANGES, got %q", result.Verdict)
}
if len(result.Findings) != 1 {
t.Fatalf("expected 1 finding, got %d", len(result.Findings))
}
if result.Findings[0].Severity != "MAJOR" {
t.Errorf("expected severity MAJOR, got %q", result.Findings[0].Severity)
}
}
func TestParseResponse_InvalidJSON(t *testing.T) {
_, err := ParseResponse("this is not json")
if err == nil {
t.Fatal("expected error for invalid JSON, got nil")
}
}
func TestParseResponse_InvalidVerdict(t *testing.T) {
input := `{"verdict": "MAYBE", "summary": "Hmm", "findings": [], "recommendation": "Dunno"}`
_, err := ParseResponse(input)
if err == nil {
t.Fatal("expected error for invalid verdict, got nil")
}
}
func TestParseResponse_InvalidSeverity(t *testing.T) {
input := `{"verdict": "APPROVE", "summary": "Ok", "findings": [{"severity": "CRITICAL", "file": "x.go", "line": 1, "finding": "bad"}], "recommendation": "Fix"}`
_, err := ParseResponse(input)
if err == nil {
t.Fatal("expected error for invalid severity, got nil")
}
}
func TestParseResponse_EmptyFindings(t *testing.T) {
input := `{"verdict": "APPROVE", "summary": "All good", "findings": [], "recommendation": "Merge"}`
result, err := ParseResponse(input)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(result.Findings) != 0 {
t.Errorf("expected 0 findings, got %d", len(result.Findings))
}
}
func TestParseResponse_MissingFields(t *testing.T) {
// verdict is empty string — should fail validation
input := `{"summary": "Ok", "findings": [], "recommendation": "Merge"}`
_, err := ParseResponse(input)
if err == nil {
t.Fatal("expected error for missing verdict, got nil")
}
}
func TestParseResponse_MarkdownFencesNoLang(t *testing.T) {
input := "```\n{\"verdict\": \"APPROVE\", \"summary\": \"Fine\", \"findings\": [], \"recommendation\": \"Good\"}\n```"
result, err := ParseResponse(input)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.Verdict != "APPROVE" {
t.Errorf("expected APPROVE, got %q", result.Verdict)
}
}
func TestParseResponse_UnescapedQuotesInStrings(t *testing.T) {
// Real failure from CI: Sonnet puts unescaped quotes like (e.g. "28") in findings
input := `{"verdict": "APPROVE", "summary": "Clean PR", "findings": [{"severity": "NIT", "file": "ci/Dockerfile", "line": 14, "finding": "The comment says OTP_VERSION is the major version (e.g. \"28\") but it actually contains unescaped quotes like (e.g. "28") which breaks JSON"}], "recommendation": "Ship it"}`
result, err := ParseResponse(input)
if err != nil {
t.Fatalf("expected repair to handle unescaped quotes, got error: %v", err)
}
if result.Verdict != "APPROVE" {
t.Errorf("expected APPROVE, got %q", result.Verdict)
}
if len(result.Findings) != 1 {
t.Fatalf("expected 1 finding, got %d", len(result.Findings))
}
}
func TestRepairJSON_NoOpOnValid(t *testing.T) {
valid := `{"key": "value", "num": 42}`
result := repairJSON(valid)
if result != valid {
t.Errorf("repairJSON should not modify valid JSON\n got: %s\n want: %s", result, valid)
}
}
func TestRepairJSON_FixesUnescapedQuotes(t *testing.T) {
// Interior quote followed by non-structural character
input := `{"msg": "use "foo" here"}`
result := repairJSON(input)
// Should be parseable now
var m map[string]interface{}
if err := json.Unmarshal([]byte(result), &m); err != nil {
t.Fatalf("repaired JSON should parse, got: %v\nrepaired: %s", err, result)
}
}