Skip to content

Commit 5e31bba

Browse files
markpollackclaude
markpollack
and
claude
committed
feat: Add Phase 3c for AI-assisted validation & 2 new integration tests
- Added Phase 3c to integration testing plan for AI-powered log validation - Introduces Claude Code CLI integration for non-deterministic test validation - Extends ExampleInfo.json schema with validationMode ("regex" | "ai") - Designed to handle interactive chatbots and variable output examples - Added integration tests for 2 MCP examples: - model-context-protocol/client-starter/starter-webflux-client - model-context-protocol/web-search/brave-starter - Progress: 17/33 examples now have integration tests (52% coverage) - Phase 3b: 5/21 remaining examples converted (24% complete) 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
1 parent c92c728 commit 5e31bba

File tree

5 files changed

+381
-17
lines changed

5 files changed

+381
-17
lines changed

‎model-context-protocol/client-starter/starter-webflux-client/integration-tests/ExampleInfo.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"timeoutSec": 300,
3+
"successRegex": [
4+
">>> QUESTION: What tools are available\\?",
5+
">>> ASSISTANT:",
6+
"Brave Web Search",
7+
"Brave Local Search",
8+
"Process terminated with code 143"
9+
],
10+
"requiredEnv": [
11+
"ANTHROPIC_API_KEY",
12+
"BRAVE_API_KEY"
13+
]
14+
}

‎model-context-protocol/client-starter/starter-webflux-client/integration-tests/RunStarterWebfluxClient.java‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
///usr/bin/env jbang "$0" "$@" ; exit $?
2+
//DEPS org.zeroturnaround:zt-exec:1.12
3+
//DEPS com.fasterxml.jackson.core:jackson-databind:2.17.1
4+
//JAVA 17
5+
//FILES ExampleInfo.json
6+
7+
/*
8+
* Integration test launcher for starter-webflux-client
9+
* Generated by scaffold_integration_test.py on 2025-08-01 12:17:02
10+
*/
11+
12+
import com.fasterxml.jackson.databind.*;
13+
import org.zeroturnaround.exec.*;
14+
import java.nio.file.*;
15+
import java.util.concurrent.TimeUnit;
16+
import static java.lang.System.*;
17+
import java.util.stream.Collectors;
18+
19+
record ExampleInfo(
20+
int timeoutSec,
21+
String[] successRegex,
22+
String[] requiredEnv,
23+
String[] setupCommands,
24+
String[] cleanupCommands
25+
) {}
26+
27+
public class RunStarterWebfluxClient {
28+
29+
public static void main(String... args) throws Exception {
30+
Path configPath = Path.of("integration-tests/ExampleInfo.json");
31+
ExampleInfo cfg = new ObjectMapper().readValue(configPath.toFile(), ExampleInfo.class);
32+
33+
// Verify required environment variables
34+
for (String envVar : cfg.requiredEnv()) {
35+
if (getenv(envVar) == null) {
36+
err.println("❌ Missing required environment variable: " + envVar);
37+
exit(1);
38+
}
39+
}
40+
41+
try {
42+
43+
// Build and run the main application
44+
out.println("🏗️ Building starter-webflux-client...");
45+
runCommand(new String[]{"./mvnw", "clean", "package", "-q", "-DskipTests"}, 300);
46+
47+
out.println("🚀 Running starter-webflux-client...");
48+
49+
// Create persistent log file for debugging
50+
Path logDir = Paths.get("../../integration-testing/logs/integration-tests");
51+
Files.createDirectories(logDir);
52+
Path logFile = logDir.resolve("starter-webflux-client-spring-boot-" + System.currentTimeMillis() + ".log");
53+
54+
ProcessResult result = new ProcessExecutor()
55+
.command("./mvnw", "spring-boot:run", "-q")
56+
.timeout(cfg.timeoutSec(), TimeUnit.SECONDS)
57+
.redirectOutput(Files.newOutputStream(logFile))
58+
.redirectErrorStream(true)
59+
.execute();
60+
61+
int exitCode = result.getExitValue();
62+
63+
// Verify output patterns
64+
String output = Files.readString(logFile);
65+
out.println("✅ Verifying output patterns...");
66+
67+
int failedPatterns = 0;
68+
for (String pattern : cfg.successRegex()) {
69+
if (output.matches("(?s).*" + pattern + ".*")) {
70+
out.println(" ✓ Found: " + pattern);
71+
} else {
72+
err.println(" ❌ Missing: " + pattern);
73+
failedPatterns++;
74+
}
75+
}
76+
77+
// Display captured output for debugging
78+
out.println("\n📋 Captured Application Output:");
79+
out.println("━".repeat(80));
80+
81+
// Show MCP tool discovery response
82+
String[] lines = output.split("\n");
83+
boolean captureOutput = false;
84+
int outputLines = 0;
85+
for (String line : lines) {
86+
if (line.contains(">>> QUESTION:")) {
87+
captureOutput = true;
88+
}
89+
if (captureOutput) {
90+
out.println(line);
91+
outputLines++;
92+
if (outputLines > 50) {
93+
out.println("... (truncated, see full log)");
94+
break;
95+
}
96+
}
97+
}
98+
out.println("━".repeat(80));
99+
out.println("📁 Full Spring Boot log: " + logFile.toAbsolutePath());
100+
101+
if (exitCode != 0) {
102+
err.println("❌ Application exited with code: " + exitCode);
103+
exit(exitCode);
104+
}
105+
106+
if (failedPatterns > 0) {
107+
err.println("❌ Failed pattern verification: " + failedPatterns + " patterns missing");
108+
exit(1);
109+
}
110+
111+
out.println("🎉 Integration test completed successfully!");
112+
113+
} finally {
114+
}
115+
}
116+
117+
private static void runCommand(String[] cmd, int timeoutSec) throws Exception {
118+
ProcessResult result = new ProcessExecutor()
119+
.command(cmd)
120+
.timeout(timeoutSec, TimeUnit.SECONDS)
121+
.redirectOutput(System.out)
122+
.redirectError(System.err)
123+
.execute();
124+
125+
int exit = result.getExitValue();
126+
if (exit != 0) {
127+
throw new RuntimeException("Command failed with exit code " + exit + ": " + String.join(" ", cmd));
128+
}
129+
}
130+
}

‎model-context-protocol/web-search/brave-starter/integration-tests/ExampleInfo.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"timeoutSec": 300,
3+
"successRegex": [
4+
"QUESTION: Does Spring AI support the Model Context Protocol\\?",
5+
"ASSISTANT:",
6+
"Spring AI.*(and|Support for).*Model Context Protocol",
7+
"Yes, Spring AI does support the Model Context Protocol",
8+
"Process terminated with code 143"
9+
],
10+
"requiredEnv": [
11+
"ANTHROPIC_API_KEY",
12+
"BRAVE_API_KEY"
13+
]
14+
}

‎model-context-protocol/web-search/brave-starter/integration-tests/RunBraveStarter.java‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
///usr/bin/env jbang "$0" "$@" ; exit $?
2+
//DEPS org.zeroturnaround:zt-exec:1.12
3+
//DEPS com.fasterxml.jackson.core:jackson-databind:2.17.1
4+
//JAVA 17
5+
//FILES ExampleInfo.json
6+
7+
/*
8+
* Integration test launcher for brave-starter
9+
* Generated by scaffold_integration_test.py on 2025-08-01 12:33:39
10+
*/
11+
12+
import com.fasterxml.jackson.databind.*;
13+
import org.zeroturnaround.exec.*;
14+
import java.nio.file.*;
15+
import java.util.concurrent.TimeUnit;
16+
import static java.lang.System.*;
17+
18+
record ExampleInfo(
19+
int timeoutSec,
20+
String[] successRegex,
21+
String[] requiredEnv,
22+
String[] setupCommands,
23+
String[] cleanupCommands
24+
) {}
25+
26+
public class RunBraveStarter {
27+
28+
public static void main(String... args) throws Exception {
29+
Path configPath = Path.of("integration-tests/ExampleInfo.json");
30+
ExampleInfo cfg = new ObjectMapper().readValue(configPath.toFile(), ExampleInfo.class);
31+
32+
// Verify required environment variables
33+
for (String envVar : cfg.requiredEnv()) {
34+
if (getenv(envVar) == null) {
35+
err.println("❌ Missing required environment variable: " + envVar);
36+
exit(1);
37+
}
38+
}
39+
40+
try {
41+
42+
// Build and run the main application
43+
out.println("🏗️ Building brave-starter...");
44+
runCommand(new String[]{"./mvnw", "clean", "package", "-q", "-DskipTests"}, 300);
45+
46+
out.println("🚀 Running brave-starter...");
47+
48+
// Create persistent log file for debugging
49+
Path logDir = Paths.get("../../integration-testing/logs/integration-tests");
50+
Files.createDirectories(logDir);
51+
Path logFile = logDir.resolve("brave-starter-spring-boot-" + System.currentTimeMillis() + ".log");
52+
53+
ProcessResult result = new ProcessExecutor()
54+
.command("./mvnw", "spring-boot:run", "-q")
55+
.timeout(cfg.timeoutSec(), TimeUnit.SECONDS)
56+
.redirectOutput(Files.newOutputStream(logFile))
57+
.redirectErrorStream(true)
58+
.execute();
59+
60+
int exitCode = result.getExitValue();
61+
62+
// Verify output patterns
63+
String output = Files.readString(logFile);
64+
out.println("✅ Verifying output patterns...");
65+
66+
int failedPatterns = 0;
67+
for (String pattern : cfg.successRegex()) {
68+
if (output.matches("(?s).*" + pattern + ".*")) {
69+
out.println(" ✓ Found: " + pattern);
70+
} else {
71+
err.println(" ❌ Missing: " + pattern);
72+
failedPatterns++;
73+
}
74+
}
75+
76+
// Display captured output for debugging
77+
out.println("\n📋 Captured Application Output:");
78+
out.println("━".repeat(80));
79+
80+
// Show MCP Brave search response
81+
String[] lines = output.split("\n");
82+
boolean captureOutput = false;
83+
int outputLines = 0;
84+
for (String line : lines) {
85+
if (line.contains("QUESTION:")) {
86+
captureOutput = true;
87+
}
88+
if (captureOutput) {
89+
out.println(line);
90+
outputLines++;
91+
if (outputLines > 50) {
92+
out.println("... (truncated, see full log)");
93+
break;
94+
}
95+
}
96+
}
97+
out.println("━".repeat(80));
98+
out.println("📁 Full Spring Boot log: " + logFile.toAbsolutePath());
99+
100+
if (exitCode != 0) {
101+
err.println("❌ Application exited with code: " + exitCode);
102+
exit(exitCode);
103+
}
104+
105+
if (failedPatterns > 0) {
106+
err.println("❌ Failed pattern verification: " + failedPatterns + " patterns missing");
107+
exit(1);
108+
}
109+
110+
out.println("🎉 Integration test completed successfully!");
111+
112+
} finally {
113+
}
114+
}
115+
116+
private static void runCommand(String[] cmd, int timeoutSec) throws Exception {
117+
ProcessResult result = new ProcessExecutor()
118+
.command(cmd)
119+
.timeout(timeoutSec, TimeUnit.SECONDS)
120+
.redirectOutput(System.out)
121+
.redirectError(System.err)
122+
.execute();
123+
124+
int exit = result.getExitValue();
125+
if (exit != 0) {
126+
throw new RuntimeException("Command failed with exit code " + exit + ": " + String.join(" ", cmd));
127+
}
128+
}
129+
}

0 commit comments

Comments
 (0)