fix: repair unescaped quotes in LLM JSON responses
CI / test (pull_request) Successful in 13s
CI / review (gpt-5, security, SECURITY_REVIEW.md, SECURITY_REVIEW_TOKEN) (pull_request) Failing after 13s
CI / review (gpt-4.1, gpt, GPT_REVIEW_TOKEN) (pull_request) Failing after 13s
CI / review (gpt-5, sonnet, SONNET_REVIEW_TOKEN) (pull_request) Failing after 12s

LLMs (especially Sonnet) sometimes emit JSON with unescaped double
quotes inside string values, e.g. (e.g. "28") instead of properly
escaping them. This caused parse failures in CI.

Add a repairJSON fallback that uses a character-by-character scanner
to identify interior quotes (those not followed by structural JSON
characters) and escape them before retrying the parse.

Fixes sonnet-review failures on gargoyle PR #551.
This commit is contained in:
Rodin
2026-05-03 09:47:22 -07:00
parent 8d8a249481
commit 80a9a7675b
2 changed files with 112 additions and 1 deletions
+37
View File
@@ -1,6 +1,7 @@
package review
import (
"encoding/json"
"testing"
)
@@ -112,3 +113,39 @@ func TestParseResponse_MarkdownFencesNoLang(t *testing.T) {
t.Errorf("expected APPROVE, got %q", result.Verdict)
}
}
func TestParseResponse_UnescapedQuotesInStrings(t *testing.T) {
// Real failure from CI: Sonnet puts unescaped quotes like (e.g. "28") in findings
input := `{"verdict": "APPROVE", "summary": "Clean PR", "findings": [{"severity": "NIT", "file": "ci/Dockerfile", "line": 14, "finding": "The comment says OTP_VERSION is the major version (e.g. \"28\") but it actually contains unescaped quotes like (e.g. "28") which breaks JSON"}], "recommendation": "Ship it"}`
result, err := ParseResponse(input)
if err != nil {
t.Fatalf("expected repair to handle unescaped quotes, got error: %v", err)
}
if result.Verdict != "APPROVE" {
t.Errorf("expected APPROVE, got %q", result.Verdict)
}
if len(result.Findings) != 1 {
t.Fatalf("expected 1 finding, got %d", len(result.Findings))
}
}
func TestRepairJSON_NoOpOnValid(t *testing.T) {
valid := `{"key": "value", "num": 42}`
result := repairJSON(valid)
if result != valid {
t.Errorf("repairJSON should not modify valid JSON\n got: %s\n want: %s", result, valid)
}
}
func TestRepairJSON_FixesUnescapedQuotes(t *testing.T) {
// Interior quote followed by non-structural character
input := `{"msg": "use "foo" here"}`
result := repairJSON(input)
// Should be parseable now
var m map[string]interface{}
if err := json.Unmarshal([]byte(result), &m); err != nil {
t.Fatalf("repaired JSON should parse, got: %v\nrepaired: %s", err, result)
}
}