wifi-densepose/vendor/sublinear-time-solver/dist/mcp/tools/domain-validation.js

673 lines
28 KiB
JavaScript

/**
* Domain Validation MCP Tools
* Provides comprehensive validation, testing, and analysis for domains
*/
export class DomainValidationTools {
domainRegistry;
constructor(domainRegistry) {
this.domainRegistry = domainRegistry;
}
getTools() {
return [
{
name: 'domain_validate',
description: 'Validate a domain configuration without registering it',
inputSchema: {
type: 'object',
properties: {
domain_config: {
type: 'object',
description: 'Complete domain configuration to validate',
properties: {
name: { type: 'string', pattern: '^[a-z_]+$' },
version: { type: 'string', pattern: '^\\d+\\.\\d+\\.\\d+$' },
description: { type: 'string', maxLength: 500 },
keywords: {
type: 'array',
items: { type: 'string', minLength: 2 },
minItems: 3,
uniqueItems: true
},
reasoning_style: { type: 'string' },
custom_reasoning_description: { type: 'string' },
analogy_domains: { type: 'array', items: { type: 'string' } },
semantic_clusters: { type: 'array', items: { type: 'string' } },
cross_domain_mappings: { type: 'array', items: { type: 'string' } },
priority: { type: 'integer', minimum: 0, maximum: 100 },
dependencies: { type: 'array', items: { type: 'string' } }
},
required: ['name', 'version', 'description', 'keywords', 'reasoning_style']
},
validation_level: {
type: 'string',
enum: ['basic', 'comprehensive', 'strict'],
default: 'comprehensive',
description: 'Validation depth level'
},
check_conflicts: {
type: 'boolean',
default: true,
description: 'Check for conflicts with existing domains'
},
performance_test: {
type: 'boolean',
default: false,
description: 'Run performance validation tests'
}
},
required: ['domain_config']
}
},
{
name: 'domain_test',
description: 'Run comprehensive tests on a domain',
inputSchema: {
type: 'object',
properties: {
domain_name: { type: 'string', description: 'Domain to test' },
test_suite: {
type: 'array',
items: {
type: 'string',
enum: ['keyword_detection', 'reasoning_style', 'cross_domain_mapping',
'inference_rules', 'performance', 'integration']
},
default: ['keyword_detection', 'reasoning_style', 'integration'],
description: 'Test suites to run'
},
test_queries: {
type: 'array',
items: { type: 'string' },
description: 'Custom test queries for domain validation'
},
performance_iterations: {
type: 'integer',
minimum: 1,
maximum: 1000,
default: 100,
description: 'Number of performance test iterations'
}
},
required: ['domain_name']
}
},
{
name: 'domain_analyze_conflicts',
description: 'Analyze potential conflicts between domains',
inputSchema: {
type: 'object',
properties: {
domain1: { type: 'string', description: 'First domain name' },
domain2: {
type: 'string',
description: 'Second domain name (optional - analyzes against all if not provided)'
},
conflict_types: {
type: 'array',
items: {
type: 'string',
enum: ['keyword_overlap', 'reasoning_style_conflict', 'analogy_contradiction', 'inference_collision']
},
default: ['keyword_overlap', 'reasoning_style_conflict'],
description: 'Types of conflicts to analyze'
},
threshold: {
type: 'number',
minimum: 0,
maximum: 1,
default: 0.3,
description: 'Conflict threshold (0-1, higher = more sensitive)'
}
},
required: ['domain1']
}
},
{
name: 'domain_suggest_improvements',
description: 'Analyze domain and suggest improvements',
inputSchema: {
type: 'object',
properties: {
domain_name: { type: 'string', description: 'Domain to analyze' },
analysis_depth: {
type: 'string',
enum: ['basic', 'detailed', 'comprehensive'],
default: 'detailed',
description: 'Analysis depth level'
},
focus_areas: {
type: 'array',
items: {
type: 'string',
enum: ['keyword_coverage', 'reasoning_effectiveness', 'cross_domain_synergy',
'performance_optimization', 'knowledge_integration']
},
description: 'Areas to focus improvement suggestions on'
},
compare_with_similar: {
type: 'boolean',
default: true,
description: 'Compare with similar domains for benchmarking'
}
},
required: ['domain_name']
}
},
{
name: 'domain_detection_test',
description: 'Test domain detection accuracy for given queries',
inputSchema: {
type: 'object',
properties: {
test_queries: {
type: 'array',
items: { type: 'string' },
description: 'Queries to test domain detection on'
},
expected_domains: {
type: 'array',
items: {
type: 'object',
properties: {
query: { type: 'string' },
expected_domain: { type: 'string' },
confidence_threshold: { type: 'number', minimum: 0, maximum: 1, default: 0.7 }
},
required: ['query', 'expected_domain']
},
description: 'Expected domain detection results for validation'
},
include_scores: {
type: 'boolean',
default: true,
description: 'Include detection scores in results'
},
include_debug: {
type: 'boolean',
default: false,
description: 'Include debug information'
}
}
}
},
{
name: 'domain_benchmark',
description: 'Run performance benchmarks on domains',
inputSchema: {
type: 'object',
properties: {
domains: {
type: 'array',
items: { type: 'string' },
description: 'Domains to benchmark (empty for all enabled domains)'
},
benchmark_type: {
type: 'string',
enum: ['detection_speed', 'reasoning_accuracy', 'memory_usage', 'comprehensive'],
default: 'comprehensive',
description: 'Type of benchmark to run'
},
iterations: {
type: 'integer',
minimum: 10,
maximum: 10000,
default: 1000,
description: 'Number of benchmark iterations'
},
test_data_size: {
type: 'string',
enum: ['small', 'medium', 'large'],
default: 'medium',
description: 'Size of test dataset'
}
}
}
}
];
}
async handleToolCall(name, args) {
try {
switch (name) {
case 'domain_validate':
return await this.validateDomain(args);
case 'domain_test':
return await this.testDomain(args);
case 'domain_analyze_conflicts':
return await this.analyzeConflicts(args);
case 'domain_suggest_improvements':
return await this.suggestImprovements(args);
case 'domain_detection_test':
return await this.testDomainDetection(args);
case 'domain_benchmark':
return await this.benchmarkDomains(args);
default:
throw new Error(`Unknown domain validation tool: ${name}`);
}
}
catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : String(error),
timestamp: new Date().toISOString()
};
}
}
async validateDomain(args) {
const config = args.domain_config;
const level = args.validation_level || 'comprehensive';
const issues = [];
let score = 100;
// Basic schema validation
const schemaIssues = this.validateSchema(config);
issues.push(...schemaIssues);
score -= schemaIssues.filter(i => i.level === 'error').length * 20;
score -= schemaIssues.filter(i => i.level === 'warning').length * 5;
// Semantic validation
if (level === 'comprehensive' || level === 'strict') {
const semanticIssues = this.validateSemantics(config);
issues.push(...semanticIssues);
score -= semanticIssues.filter(i => i.level === 'error').length * 15;
score -= semanticIssues.filter(i => i.level === 'warning').length * 3;
}
// Conflict checking
if (args.check_conflicts) {
const conflictIssues = this.checkDomainConflicts(config);
issues.push(...conflictIssues);
score -= conflictIssues.filter(i => i.level === 'warning').length * 10;
}
// Dependency validation
const dependencyIssues = this.validateDependencies(config);
issues.push(...dependencyIssues);
score -= dependencyIssues.filter(i => i.level === 'error').length * 25;
// Performance validation
if (args.performance_test) {
const performanceIssues = await this.validatePerformance(config);
issues.push(...performanceIssues);
score -= performanceIssues.filter(i => i.level === 'warning').length * 5;
}
const result = {
valid: issues.filter(i => i.level === 'error').length === 0,
score: Math.max(0, score),
issues,
tested_at: Date.now()
};
return {
validation_result: result,
domain_name: config.name,
validation_level: level,
checks_performed: {
schema: true,
semantics: level !== 'basic',
conflicts: args.check_conflicts,
dependencies: true,
performance: args.performance_test
},
timestamp: new Date().toISOString()
};
}
async testDomain(args) {
const plugin = this.domainRegistry.getDomain(args.domain_name);
if (!plugin) {
throw new Error(`Domain '${args.domain_name}' not found`);
}
const testSuite = args.test_suite || ['keyword_detection', 'reasoning_style', 'integration'];
const testResults = [];
// Run each test
for (const testName of testSuite) {
try {
const result = await this.runIndividualTest(testName, plugin, args);
testResults.push(result);
}
catch (error) {
testResults.push({
name: testName,
passed: false,
score: 0,
details: {},
error: error instanceof Error ? error.message : String(error)
});
}
}
const overallScore = testResults.reduce((sum, r) => sum + r.score, 0) / testResults.length;
const passed = testResults.every(r => r.passed);
const suite = {
domain_name: args.domain_name,
test_results: testResults,
overall_score: overallScore,
passed,
timestamp: new Date().toISOString()
};
return {
test_suite: suite,
summary: {
total_tests: testResults.length,
passed_tests: testResults.filter(r => r.passed).length,
failed_tests: testResults.filter(r => !r.passed).length,
overall_score: overallScore,
recommendation: this.getTestRecommendation(suite)
}
};
}
async analyzeConflicts(args) {
const domain1 = this.domainRegistry.getDomain(args.domain1);
if (!domain1) {
throw new Error(`Domain '${args.domain1}' not found`);
}
const conflicts = [];
const conflictTypes = args.conflict_types || ['keyword_overlap', 'reasoning_style_conflict'];
const threshold = args.threshold || 0.3;
const domainsToCheck = args.domain2 ?
[this.domainRegistry.getDomain(args.domain2)].filter(Boolean) :
this.domainRegistry.getAllDomains().filter(d => d.config.name !== args.domain1);
for (const domain2 of domainsToCheck) {
for (const conflictType of conflictTypes) {
const conflict = this.analyzeSpecificConflict(domain1, domain2, conflictType, threshold);
if (conflict) {
conflicts.push(conflict);
}
}
}
return {
domain1: args.domain1,
domain2: args.domain2 || 'all',
conflicts,
conflict_types_checked: conflictTypes,
threshold_used: threshold,
summary: {
total_conflicts: conflicts.length,
high_severity: conflicts.filter(c => c.severity === 'high').length,
medium_severity: conflicts.filter(c => c.severity === 'medium').length,
low_severity: conflicts.filter(c => c.severity === 'low').length
},
timestamp: new Date().toISOString()
};
}
async suggestImprovements(args) {
const plugin = this.domainRegistry.getDomain(args.domain_name);
if (!plugin) {
throw new Error(`Domain '${args.domain_name}' not found`);
}
const suggestions = [];
const analysisDepth = args.analysis_depth || 'detailed';
const focusAreas = args.focus_areas || ['keyword_coverage', 'reasoning_effectiveness'];
// Analyze each focus area
for (const area of focusAreas) {
const areaSuggestions = await this.analyzeImprovementArea(plugin, area, analysisDepth);
suggestions.push(...areaSuggestions);
}
// Compare with similar domains if requested
let benchmarkComparison = null;
if (args.compare_with_similar) {
benchmarkComparison = this.compareWithSimilarDomains(plugin);
}
return {
domain_name: args.domain_name,
suggestions,
analysis_depth: analysisDepth,
focus_areas: focusAreas,
benchmark_comparison: benchmarkComparison,
priority_suggestions: suggestions
.filter(s => s.priority === 'high')
.slice(0, 5),
timestamp: new Date().toISOString()
};
}
async testDomainDetection(args) {
const results = [];
// Test with provided queries
if (args.test_queries) {
for (const query of args.test_queries) {
const detectionResult = await this.testSingleQueryDetection(query, args);
results.push(detectionResult);
}
}
// Test with expected domain mappings
if (args.expected_domains) {
for (const expected of args.expected_domains) {
const detectionResult = await this.testSingleQueryDetection(expected.query, args);
const passed = detectionResult.detected_domains.length > 0 &&
detectionResult.detected_domains[0].domain === expected.expected_domain &&
detectionResult.detected_domains[0].score >= (expected.confidence_threshold || 0.7);
results.push({
...detectionResult,
expected_domain: expected.expected_domain,
confidence_threshold: expected.confidence_threshold,
test_passed: passed
});
}
}
const accuracy = args.expected_domains ?
results.filter(r => r.test_passed).length / results.length : null;
return {
detection_results: results,
summary: {
total_queries: results.length,
accuracy: accuracy,
average_detection_time: results.reduce((sum, r) => sum + (r.detection_time_ms || 0), 0) / results.length
},
timestamp: new Date().toISOString()
};
}
async benchmarkDomains(args) {
const domains = args.domains?.length ?
args.domains.map(name => this.domainRegistry.getDomain(name)).filter(Boolean) :
this.domainRegistry.getEnabledDomains();
const benchmarkType = args.benchmark_type || 'comprehensive';
const iterations = args.iterations || 1000;
const results = [];
for (const domain of domains) {
const benchmarkResult = await this.runDomainBenchmark(domain, benchmarkType, iterations);
results.push(benchmarkResult);
}
// Sort by overall performance score
results.sort((a, b) => b.overall_score - a.overall_score);
return {
benchmark_results: results,
benchmark_type: benchmarkType,
iterations,
summary: {
best_performing: results[0]?.domain_name,
worst_performing: results[results.length - 1]?.domain_name,
average_score: results.reduce((sum, r) => sum + r.overall_score, 0) / results.length
},
timestamp: new Date().toISOString()
};
}
// Helper methods for validation
validateSchema(config) {
const issues = [];
if (!config.name?.match(/^[a-z_]+$/)) {
issues.push({
level: 'error',
message: 'Domain name must contain only lowercase letters and underscores',
field: 'name'
});
}
if (!config.version?.match(/^\d+\.\d+\.\d+$/)) {
issues.push({
level: 'error',
message: 'Version must follow semantic versioning (e.g., 1.0.0)',
field: 'version'
});
}
if (!config.keywords || config.keywords.length < 3) {
issues.push({
level: 'error',
message: 'At least 3 keywords are required for effective domain detection',
field: 'keywords'
});
}
if (config.reasoning_style === 'custom' && !config.custom_reasoning_description) {
issues.push({
level: 'error',
message: 'Custom reasoning description is required when reasoning_style is "custom"',
field: 'custom_reasoning_description'
});
}
return issues;
}
validateSemantics(config) {
const issues = [];
// Check keyword quality
const shortKeywords = config.keywords.filter(k => k.length < 3);
if (shortKeywords.length > 0) {
issues.push({
level: 'warning',
message: `Very short keywords may cause false matches: ${shortKeywords.join(', ')}`,
field: 'keywords'
});
}
// Check for overly generic keywords
const genericKeywords = ['the', 'and', 'or', 'but', 'with', 'from', 'system', 'method'];
const foundGeneric = config.keywords.filter(k => genericKeywords.includes(k.toLowerCase()));
if (foundGeneric.length > 0) {
issues.push({
level: 'warning',
message: `Generic keywords may cause incorrect detection: ${foundGeneric.join(', ')}`,
field: 'keywords',
suggestion: 'Use more specific, domain-focused keywords'
});
}
return issues;
}
checkDomainConflicts(config) {
const issues = [];
// Check for existing domain with same name
if (this.domainRegistry.getDomain(config.name)) {
issues.push({
level: 'error',
message: `Domain name '${config.name}' already exists`,
field: 'name'
});
}
// Check keyword overlap
const allDomains = this.domainRegistry.getAllDomains();
for (const existingDomain of allDomains) {
const overlap = config.keywords.filter(k => existingDomain.config.keywords.some(ek => ek.toLowerCase() === k.toLowerCase()));
if (overlap.length > 2) {
issues.push({
level: 'warning',
message: `High keyword overlap with domain '${existingDomain.config.name}': ${overlap.join(', ')}`,
field: 'keywords',
suggestion: 'Consider using more specific keywords to avoid detection conflicts'
});
}
}
return issues;
}
validateDependencies(config) {
const issues = [];
for (const dep of config.dependencies) {
if (!this.domainRegistry.getDomain(dep)) {
issues.push({
level: 'error',
message: `Dependency '${dep}' not found`,
field: 'dependencies'
});
}
}
return issues;
}
async validatePerformance(config) {
const issues = [];
// Simulate performance tests
if (config.keywords.length > 50) {
issues.push({
level: 'warning',
message: 'Large number of keywords may impact detection performance',
field: 'keywords',
suggestion: 'Consider reducing to most essential keywords'
});
}
return issues;
}
// Additional helper methods for testing and analysis would go here...
async runIndividualTest(testName, plugin, args) {
// Simplified test implementation
switch (testName) {
case 'keyword_detection':
return {
name: testName,
passed: plugin.config.keywords.length >= 3,
score: Math.min(100, plugin.config.keywords.length * 10),
details: { keyword_count: plugin.config.keywords.length }
};
default:
return {
name: testName,
passed: true,
score: 85,
details: { note: 'Test implementation pending' }
};
}
}
getTestRecommendation(suite) {
if (suite.overall_score >= 90)
return 'Excellent - domain is ready for production use';
if (suite.overall_score >= 75)
return 'Good - minor improvements recommended';
if (suite.overall_score >= 60)
return 'Fair - significant improvements needed';
return 'Poor - major issues must be addressed before use';
}
analyzeSpecificConflict(domain1, domain2, conflictType, threshold) {
// Simplified conflict analysis
if (conflictType === 'keyword_overlap') {
const overlap = domain1.config.keywords.filter(k => domain2.config.keywords.includes(k));
if (overlap.length / Math.min(domain1.config.keywords.length, domain2.config.keywords.length) >= threshold) {
return {
type: 'keyword_overlap',
domain2: domain2.config.name,
severity: 'medium',
details: { overlapping_keywords: overlap }
};
}
}
return null;
}
async analyzeImprovementArea(plugin, area, depth) {
// Simplified improvement analysis
const suggestions = [];
if (area === 'keyword_coverage' && plugin.config.keywords.length < 5) {
suggestions.push({
area,
priority: 'medium',
suggestion: 'Add more keywords to improve detection coverage',
impact: 'Better domain detection accuracy'
});
}
return suggestions;
}
compareWithSimilarDomains(plugin) {
// Simplified comparison
return {
similar_domains: [],
performance_ranking: 'Average',
recommendations: ['Improve keyword specificity']
};
}
async testSingleQueryDetection(query, args) {
// Simplified detection test
return {
query,
detected_domains: [
{ domain: 'test_domain', score: 0.8 }
],
detection_time_ms: 2.5
};
}
async runDomainBenchmark(domain, benchmarkType, iterations) {
// Simplified benchmark
return {
domain_name: domain.config.name,
benchmark_type: benchmarkType,
iterations,
overall_score: 85,
metrics: {
detection_speed_ms: 1.2,
accuracy_score: 0.9
}
};
}
}