80a9a7675b
CI / test (pull_request) Successful in 13s
CI / review (gpt-5, security, SECURITY_REVIEW.md, SECURITY_REVIEW_TOKEN) (pull_request) Failing after 13s
CI / review (gpt-4.1, gpt, GPT_REVIEW_TOKEN) (pull_request) Failing after 13s
CI / review (gpt-5, sonnet, SONNET_REVIEW_TOKEN) (pull_request) Failing after 12s
LLMs (especially Sonnet) sometimes emit JSON with unescaped double quotes inside string values, e.g. (e.g. "28") instead of properly escaping them. This caused parse failures in CI. Add a repairJSON fallback that uses a character-by-character scanner to identify interior quotes (those not followed by structural JSON characters) and escape them before retrying the parse. Fixes sonnet-review failures on gargoyle PR #551.
152 lines
5.0 KiB
Go
152 lines
5.0 KiB
Go
package review
|
|
|
|
import (
|
|
"encoding/json"
|
|
"testing"
|
|
)
|
|
|
|
func TestParseResponse_ValidJSON(t *testing.T) {
|
|
input := `{
|
|
"verdict": "APPROVE",
|
|
"summary": "Looks good",
|
|
"findings": [
|
|
{"severity": "NIT", "file": "main.go", "line": 10, "finding": "Consider renaming"}
|
|
],
|
|
"recommendation": "Ship it"
|
|
}`
|
|
|
|
result, err := ParseResponse(input)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if result.Verdict != "APPROVE" {
|
|
t.Errorf("expected verdict APPROVE, got %q", result.Verdict)
|
|
}
|
|
if result.Summary != "Looks good" {
|
|
t.Errorf("expected summary %q, got %q", "Looks good", result.Summary)
|
|
}
|
|
if len(result.Findings) != 1 {
|
|
t.Fatalf("expected 1 finding, got %d", len(result.Findings))
|
|
}
|
|
if result.Findings[0].Severity != "NIT" {
|
|
t.Errorf("expected severity NIT, got %q", result.Findings[0].Severity)
|
|
}
|
|
if result.Findings[0].File != "main.go" {
|
|
t.Errorf("expected file main.go, got %q", result.Findings[0].File)
|
|
}
|
|
if result.Findings[0].Line != 10 {
|
|
t.Errorf("expected line 10, got %d", result.Findings[0].Line)
|
|
}
|
|
if result.Recommendation != "Ship it" {
|
|
t.Errorf("expected recommendation %q, got %q", "Ship it", result.Recommendation)
|
|
}
|
|
}
|
|
|
|
func TestParseResponse_MarkdownFences(t *testing.T) {
|
|
input := "```json\n{\"verdict\": \"REQUEST_CHANGES\", \"summary\": \"Issues found\", \"findings\": [{\"severity\": \"MAJOR\", \"file\": \"a.go\", \"line\": 5, \"finding\": \"Bug\"}], \"recommendation\": \"Fix it\"}\n```"
|
|
|
|
result, err := ParseResponse(input)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if result.Verdict != "REQUEST_CHANGES" {
|
|
t.Errorf("expected verdict REQUEST_CHANGES, got %q", result.Verdict)
|
|
}
|
|
if len(result.Findings) != 1 {
|
|
t.Fatalf("expected 1 finding, got %d", len(result.Findings))
|
|
}
|
|
if result.Findings[0].Severity != "MAJOR" {
|
|
t.Errorf("expected severity MAJOR, got %q", result.Findings[0].Severity)
|
|
}
|
|
}
|
|
|
|
func TestParseResponse_InvalidJSON(t *testing.T) {
|
|
_, err := ParseResponse("this is not json")
|
|
if err == nil {
|
|
t.Fatal("expected error for invalid JSON, got nil")
|
|
}
|
|
}
|
|
|
|
func TestParseResponse_InvalidVerdict(t *testing.T) {
|
|
input := `{"verdict": "MAYBE", "summary": "Hmm", "findings": [], "recommendation": "Dunno"}`
|
|
_, err := ParseResponse(input)
|
|
if err == nil {
|
|
t.Fatal("expected error for invalid verdict, got nil")
|
|
}
|
|
}
|
|
|
|
func TestParseResponse_InvalidSeverity(t *testing.T) {
|
|
input := `{"verdict": "APPROVE", "summary": "Ok", "findings": [{"severity": "CRITICAL", "file": "x.go", "line": 1, "finding": "bad"}], "recommendation": "Fix"}`
|
|
_, err := ParseResponse(input)
|
|
if err == nil {
|
|
t.Fatal("expected error for invalid severity, got nil")
|
|
}
|
|
}
|
|
|
|
func TestParseResponse_EmptyFindings(t *testing.T) {
|
|
input := `{"verdict": "APPROVE", "summary": "All good", "findings": [], "recommendation": "Merge"}`
|
|
result, err := ParseResponse(input)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if len(result.Findings) != 0 {
|
|
t.Errorf("expected 0 findings, got %d", len(result.Findings))
|
|
}
|
|
}
|
|
|
|
func TestParseResponse_MissingFields(t *testing.T) {
|
|
// verdict is empty string — should fail validation
|
|
input := `{"summary": "Ok", "findings": [], "recommendation": "Merge"}`
|
|
_, err := ParseResponse(input)
|
|
if err == nil {
|
|
t.Fatal("expected error for missing verdict, got nil")
|
|
}
|
|
}
|
|
|
|
func TestParseResponse_MarkdownFencesNoLang(t *testing.T) {
|
|
input := "```\n{\"verdict\": \"APPROVE\", \"summary\": \"Fine\", \"findings\": [], \"recommendation\": \"Good\"}\n```"
|
|
result, err := ParseResponse(input)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if result.Verdict != "APPROVE" {
|
|
t.Errorf("expected APPROVE, got %q", result.Verdict)
|
|
}
|
|
}
|
|
|
|
func TestParseResponse_UnescapedQuotesInStrings(t *testing.T) {
|
|
// Real failure from CI: Sonnet puts unescaped quotes like (e.g. "28") in findings
|
|
input := `{"verdict": "APPROVE", "summary": "Clean PR", "findings": [{"severity": "NIT", "file": "ci/Dockerfile", "line": 14, "finding": "The comment says OTP_VERSION is the major version (e.g. \"28\") but it actually contains unescaped quotes like (e.g. "28") which breaks JSON"}], "recommendation": "Ship it"}`
|
|
|
|
result, err := ParseResponse(input)
|
|
if err != nil {
|
|
t.Fatalf("expected repair to handle unescaped quotes, got error: %v", err)
|
|
}
|
|
if result.Verdict != "APPROVE" {
|
|
t.Errorf("expected APPROVE, got %q", result.Verdict)
|
|
}
|
|
if len(result.Findings) != 1 {
|
|
t.Fatalf("expected 1 finding, got %d", len(result.Findings))
|
|
}
|
|
}
|
|
|
|
func TestRepairJSON_NoOpOnValid(t *testing.T) {
|
|
valid := `{"key": "value", "num": 42}`
|
|
result := repairJSON(valid)
|
|
if result != valid {
|
|
t.Errorf("repairJSON should not modify valid JSON\n got: %s\n want: %s", result, valid)
|
|
}
|
|
}
|
|
|
|
func TestRepairJSON_FixesUnescapedQuotes(t *testing.T) {
|
|
// Interior quote followed by non-structural character
|
|
input := `{"msg": "use "foo" here"}`
|
|
result := repairJSON(input)
|
|
|
|
// Should be parseable now
|
|
var m map[string]interface{}
|
|
if err := json.Unmarshal([]byte(result), &m); err != nil {
|
|
t.Fatalf("repaired JSON should parse, got: %v\nrepaired: %s", err, result)
|
|
}
|
|
}
|