600 lines
18 KiB
Plaintext
600 lines
18 KiB
Plaintext
//! Comprehensive benchmarks for strange-loop crate
|
|
//!
|
|
//! Benchmarks cover:
|
|
//! - Pattern extraction performance
|
|
//! - Recursive optimization depth
|
|
//! - Meta-learning iteration speed
|
|
//! - Self-modification safety checks
|
|
//! - Rollback mechanism performance
|
|
//! - Validation overhead
|
|
//!
|
|
//! Performance targets:
|
|
//! - Pattern extraction: <10ms for 1000 patterns
|
|
//! - Recursive depth: >10 levels without stack overflow
|
|
//! - Iteration speed: >1000 iterations/second
|
|
//! - Safety overhead: <5% performance impact
|
|
|
|
use criterion::{black_box, criterion_group, criterion_main, Criterion, BenchmarkId, Throughput};
|
|
use strange_loop::{
|
|
StrangeLoop, StrangeLoopConfig, MetaLevel, MetaKnowledge,
|
|
SafetyConstraint, ModificationRule,
|
|
};
|
|
|
|
// ============================================================================
|
|
// Test Data Generators
|
|
// ============================================================================
|
|
|
|
fn generate_pattern_data(size: usize, complexity: &str) -> Vec<String> {
|
|
match complexity {
|
|
"simple" => {
|
|
// Highly repetitive patterns
|
|
(0..size)
|
|
.map(|i| format!("pattern{}", i % 10))
|
|
.collect()
|
|
}
|
|
"medium" => {
|
|
// Moderate repetition with variations
|
|
(0..size)
|
|
.map(|i| {
|
|
let base = i % 50;
|
|
let variant = i % 3;
|
|
format!("pattern_{}_{}", base, variant)
|
|
})
|
|
.collect()
|
|
}
|
|
"complex" => {
|
|
// High diversity with some patterns
|
|
(0..size)
|
|
.map(|i| {
|
|
let hash = (i * 7919) % 200;
|
|
let subpattern = (i * 31) % 5;
|
|
format!("complex_{}_{}", hash, subpattern)
|
|
})
|
|
.collect()
|
|
}
|
|
"random" => {
|
|
// Mostly unique patterns
|
|
(0..size)
|
|
.map(|i| {
|
|
let hash1 = (i * 7919) % 10000;
|
|
let hash2 = (i * 31337) % 10000;
|
|
format!("random_{}_{}", hash1, hash2)
|
|
})
|
|
.collect()
|
|
}
|
|
_ => vec!["default".to_string(); size],
|
|
}
|
|
}
|
|
|
|
fn generate_hierarchical_data(depth: usize) -> Vec<Vec<String>> {
|
|
let mut levels = Vec::new();
|
|
let mut current_data = generate_pattern_data(100, "simple");
|
|
|
|
for level in 0..depth {
|
|
levels.push(current_data.clone());
|
|
// Generate meta-patterns from current level
|
|
current_data = current_data
|
|
.windows(2)
|
|
.map(|w| format!("meta_{}_{}", level, w.join("_")))
|
|
.collect();
|
|
}
|
|
|
|
levels
|
|
}
|
|
|
|
fn generate_large_pattern_set(count: usize) -> Vec<String> {
|
|
(0..count)
|
|
.map(|i| {
|
|
let pattern_type = i % 7;
|
|
match pattern_type {
|
|
0 => format!("linear_{}", i),
|
|
1 => format!("cyclic_{}", i % 100),
|
|
2 => format!("branching_{}_{}", i / 10, i % 10),
|
|
3 => format!("converging_{}", i / 20),
|
|
4 => format!("diverging_{}", i),
|
|
5 => format!("stable_{}", i % 50),
|
|
_ => format!("chaotic_{}", (i * 7919) % 1000),
|
|
}
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
// ============================================================================
|
|
// Meta-Learning Benchmarks
|
|
// ============================================================================
|
|
|
|
fn bench_meta_learning_iteration(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("meta_learning_iteration");
|
|
|
|
// Simple learning
|
|
group.bench_function("simple", |b| {
|
|
let mut learner = MetaLearner::new();
|
|
let experiences = create_experience_batch(10, false);
|
|
|
|
b.iter(|| {
|
|
for exp in &experiences {
|
|
black_box(learner.learn(black_box(exp)));
|
|
}
|
|
});
|
|
});
|
|
|
|
// Complex learning
|
|
group.bench_function("complex", |b| {
|
|
let mut learner = MetaLearner::new();
|
|
let experiences = create_experience_batch(10, true);
|
|
|
|
b.iter(|| {
|
|
for exp in &experiences {
|
|
black_box(learner.learn(black_box(exp)));
|
|
}
|
|
});
|
|
});
|
|
|
|
// Varying batch sizes
|
|
for batch_size in [5, 10, 25, 50, 100].iter() {
|
|
group.throughput(Throughput::Elements(*batch_size as u64));
|
|
group.bench_with_input(
|
|
BenchmarkId::new("batch", batch_size),
|
|
batch_size,
|
|
|b, &size| {
|
|
let experiences = create_experience_batch(size, false);
|
|
|
|
b.iter(|| {
|
|
let mut learner = MetaLearner::new();
|
|
for exp in &experiences {
|
|
black_box(learner.learn(exp));
|
|
}
|
|
});
|
|
}
|
|
);
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
fn bench_incremental_learning(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("incremental_learning");
|
|
|
|
// Progressive learning
|
|
group.bench_function("progressive", |b| {
|
|
let mut learner = MetaLearner::new();
|
|
let mut exp_id = 0;
|
|
|
|
b.iter(|| {
|
|
exp_id += 1;
|
|
let exp = create_simple_experience(exp_id);
|
|
black_box(learner.learn(black_box(&exp)))
|
|
});
|
|
});
|
|
|
|
// With forgetting mechanism
|
|
group.bench_function("with_forgetting", |b| {
|
|
let mut learner = MetaLearner::with_capacity(100);
|
|
let mut exp_id = 0;
|
|
|
|
b.iter(|| {
|
|
exp_id += 1;
|
|
let exp = create_simple_experience(exp_id);
|
|
black_box(learner.learn_with_forgetting(black_box(&exp)))
|
|
});
|
|
});
|
|
|
|
group.finish();
|
|
}
|
|
|
|
// ============================================================================
|
|
// Pattern Extraction Benchmarks
|
|
// ============================================================================
|
|
|
|
fn bench_pattern_extraction(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("pattern_extraction");
|
|
|
|
// Simple patterns
|
|
for num_experiences in [10, 50, 100, 500].iter() {
|
|
group.bench_with_input(
|
|
BenchmarkId::new("simple", num_experiences),
|
|
num_experiences,
|
|
|b, &n| {
|
|
let experiences = create_experience_batch(n, false);
|
|
b.iter(|| {
|
|
black_box(extract_patterns(black_box(&experiences)))
|
|
});
|
|
}
|
|
);
|
|
}
|
|
|
|
// Complex patterns
|
|
for num_experiences in [10, 50, 100, 500].iter() {
|
|
group.bench_with_input(
|
|
BenchmarkId::new("complex", num_experiences),
|
|
num_experiences,
|
|
|b, &n| {
|
|
let experiences = create_experience_batch(n, true);
|
|
b.iter(|| {
|
|
black_box(extract_patterns(black_box(&experiences)))
|
|
});
|
|
}
|
|
);
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
fn bench_pattern_matching(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("pattern_matching");
|
|
|
|
let patterns = (0..100).map(|i| create_pattern(i, 0)).collect::<Vec<_>>();
|
|
|
|
// Single experience matching
|
|
group.bench_function("single_match", |b| {
|
|
let exp = create_simple_experience(42);
|
|
b.iter(|| {
|
|
black_box(patterns.iter()
|
|
.filter(|p| p.matches(black_box(&exp)))
|
|
.count())
|
|
});
|
|
});
|
|
|
|
// Batch matching
|
|
group.bench_function("batch_match", |b| {
|
|
let experiences = create_experience_batch(50, false);
|
|
b.iter(|| {
|
|
for exp in &experiences {
|
|
black_box(patterns.iter()
|
|
.filter(|p| p.matches(exp))
|
|
.count());
|
|
}
|
|
});
|
|
});
|
|
|
|
group.finish();
|
|
}
|
|
|
|
// ============================================================================
|
|
// Multi-Level Learning Benchmarks
|
|
// ============================================================================
|
|
|
|
fn bench_multi_level_learning(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("multi_level_learning");
|
|
|
|
// 2-level hierarchy
|
|
group.bench_function("two_levels", |b| {
|
|
let mut learner = MetaLearner::with_levels(2);
|
|
let experiences = create_experience_batch(50, false);
|
|
|
|
b.iter(|| {
|
|
for exp in &experiences {
|
|
black_box(learner.learn_hierarchical(black_box(exp)));
|
|
}
|
|
});
|
|
});
|
|
|
|
// 3-level hierarchy
|
|
group.bench_function("three_levels", |b| {
|
|
let mut learner = MetaLearner::with_levels(3);
|
|
let experiences = create_experience_batch(50, false);
|
|
|
|
b.iter(|| {
|
|
for exp in &experiences {
|
|
black_box(learner.learn_hierarchical(black_box(exp)));
|
|
}
|
|
});
|
|
});
|
|
|
|
// Varying levels
|
|
for num_levels in [2, 3, 4, 5].iter() {
|
|
group.bench_with_input(
|
|
BenchmarkId::new("levels", num_levels),
|
|
num_levels,
|
|
|b, &levels| {
|
|
let mut learner = MetaLearner::with_levels(levels);
|
|
let experiences = create_experience_batch(50, false);
|
|
|
|
b.iter(|| {
|
|
for exp in &experiences {
|
|
black_box(learner.learn_hierarchical(exp));
|
|
}
|
|
});
|
|
}
|
|
);
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
fn bench_level_transition(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("level_transition");
|
|
|
|
let hierarchy = create_pattern_hierarchy(3, 10);
|
|
|
|
// Bottom-up propagation
|
|
group.bench_function("bottom_up", |b| {
|
|
b.iter(|| {
|
|
black_box(propagate_bottom_up(black_box(&hierarchy)))
|
|
});
|
|
});
|
|
|
|
// Top-down influence
|
|
group.bench_function("top_down", |b| {
|
|
b.iter(|| {
|
|
black_box(propagate_top_down(black_box(&hierarchy)))
|
|
});
|
|
});
|
|
|
|
group.finish();
|
|
}
|
|
|
|
// ============================================================================
|
|
// Cross-Crate Integration Benchmarks
|
|
// ============================================================================
|
|
|
|
fn bench_cross_crate_integration(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("cross_crate_integration");
|
|
|
|
// Integration with temporal-compare
|
|
group.bench_function("temporal_compare", |b| {
|
|
use temporal_compare::{dtw_distance, TemporalData};
|
|
|
|
let experiences = create_experience_batch(100, false);
|
|
|
|
b.iter(|| {
|
|
// Extract temporal sequences from experiences
|
|
let seq1: Vec<f64> = experiences.iter()
|
|
.map(|e| e.reward)
|
|
.collect();
|
|
let seq2: Vec<f64> = experiences.iter()
|
|
.skip(10)
|
|
.map(|e| e.reward)
|
|
.collect();
|
|
|
|
black_box(dtw_distance(&seq1, &seq2))
|
|
});
|
|
});
|
|
|
|
// Integration with scheduler
|
|
group.bench_function("scheduler", |b| {
|
|
use nanosecond_scheduler::{NanoScheduler, Task, TaskPriority};
|
|
|
|
let mut scheduler = NanoScheduler::new(4);
|
|
let experiences = create_experience_batch(50, false);
|
|
|
|
b.iter(|| {
|
|
for (i, exp) in experiences.iter().enumerate() {
|
|
let priority = if exp.reward > 0.7 {
|
|
TaskPriority::High
|
|
} else {
|
|
TaskPriority::Normal
|
|
};
|
|
|
|
let task = Task::new(
|
|
format!("task_{}", i),
|
|
Box::new(move || { black_box(exp); }),
|
|
priority,
|
|
);
|
|
|
|
scheduler.schedule(task);
|
|
}
|
|
|
|
while scheduler.has_pending_tasks() {
|
|
scheduler.run_once();
|
|
}
|
|
});
|
|
});
|
|
|
|
// Integration with attractor studio
|
|
group.bench_function("attractor_studio", |b| {
|
|
use temporal_attractor_studio::{reconstruct_phase_space};
|
|
|
|
let experiences = create_experience_batch(1000, false);
|
|
let rewards: Vec<f64> = experiences.iter().map(|e| e.reward).collect();
|
|
|
|
b.iter(|| {
|
|
black_box(reconstruct_phase_space(
|
|
black_box(&rewards),
|
|
black_box(3),
|
|
black_box(10)
|
|
))
|
|
});
|
|
});
|
|
|
|
group.finish();
|
|
}
|
|
|
|
// ============================================================================
|
|
// Self-Referential Operations Benchmarks
|
|
// ============================================================================
|
|
|
|
fn bench_self_referential(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("self_referential");
|
|
|
|
// Self-improvement
|
|
group.bench_function("self_improvement", |b| {
|
|
let mut learner = MetaLearner::new();
|
|
let experiences = create_experience_batch(100, false);
|
|
|
|
// Initial learning
|
|
for exp in &experiences {
|
|
learner.learn(exp);
|
|
}
|
|
|
|
b.iter(|| {
|
|
black_box(learner.improve_self())
|
|
});
|
|
});
|
|
|
|
// Meta-pattern extraction
|
|
group.bench_function("meta_patterns", |b| {
|
|
let patterns = (0..100).map(|i| create_pattern(i, 0)).collect::<Vec<_>>();
|
|
|
|
b.iter(|| {
|
|
black_box(extract_meta_patterns(black_box(&patterns)))
|
|
});
|
|
});
|
|
|
|
// Recursive optimization
|
|
group.bench_function("recursive_opt", |b| {
|
|
let mut learner = MetaLearner::new();
|
|
let experiences = create_experience_batch(50, false);
|
|
|
|
b.iter(|| {
|
|
black_box(learner.optimize_recursive(black_box(&experiences), black_box(3)))
|
|
});
|
|
});
|
|
|
|
group.finish();
|
|
}
|
|
|
|
// ============================================================================
|
|
// Recursive Optimization Benchmarks
|
|
// ============================================================================
|
|
|
|
fn bench_recursive_optimization(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("recursive_optimization");
|
|
|
|
let experiences = create_experience_batch(100, true);
|
|
|
|
// Varying recursion depths
|
|
for depth in [1, 2, 3, 4, 5].iter() {
|
|
group.bench_with_input(
|
|
BenchmarkId::new("depth", depth),
|
|
depth,
|
|
|b, &d| {
|
|
b.iter(|| {
|
|
black_box(recursive_optimize(
|
|
black_box(&experiences),
|
|
black_box(d)
|
|
))
|
|
});
|
|
}
|
|
);
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
// ============================================================================
|
|
// Complete Pipeline Benchmarks
|
|
// ============================================================================
|
|
|
|
fn bench_complete_meta_learning(c: &mut Criterion) {
|
|
let mut group = c.benchmark_group("complete_pipeline");
|
|
|
|
group.bench_function("full_cycle", |b| {
|
|
let experiences = create_experience_batch(100, true);
|
|
|
|
b.iter(|| {
|
|
// 1. Learn from experiences
|
|
let mut learner = MetaLearner::with_levels(3);
|
|
for exp in &experiences {
|
|
learner.learn_hierarchical(exp);
|
|
}
|
|
|
|
// 2. Extract patterns
|
|
let patterns = extract_patterns(&experiences);
|
|
|
|
// 3. Integrate knowledge
|
|
let knowledge = integrate_knowledge(&patterns);
|
|
|
|
// 4. Self-improvement
|
|
learner.improve_self();
|
|
|
|
// 5. Recursive optimization
|
|
let optimized = recursive_optimize(&experiences, 2);
|
|
|
|
black_box((patterns, knowledge, optimized))
|
|
});
|
|
});
|
|
|
|
group.finish();
|
|
}
|
|
|
|
// ============================================================================
|
|
// Helper Functions (mock implementations for benchmarking)
|
|
// ============================================================================
|
|
|
|
fn propagate_bottom_up(hierarchy: &[Vec<Pattern>]) -> Vec<Pattern> {
|
|
// Mock implementation
|
|
hierarchy.iter()
|
|
.flat_map(|level| level.iter())
|
|
.cloned()
|
|
.collect()
|
|
}
|
|
|
|
fn propagate_top_down(hierarchy: &[Vec<Pattern>]) -> Vec<Pattern> {
|
|
// Mock implementation
|
|
hierarchy.iter()
|
|
.rev()
|
|
.flat_map(|level| level.iter())
|
|
.cloned()
|
|
.collect()
|
|
}
|
|
|
|
fn extract_meta_patterns(patterns: &[Pattern]) -> Vec<Pattern> {
|
|
// Mock implementation: create meta-patterns from existing patterns
|
|
patterns.iter()
|
|
.step_by(5)
|
|
.enumerate()
|
|
.map(|(i, p)| create_pattern(i, p.level + 1))
|
|
.collect()
|
|
}
|
|
|
|
// ============================================================================
|
|
// Criterion Configuration
|
|
// ============================================================================
|
|
|
|
criterion_group! {
|
|
name = learning_benches;
|
|
config = Criterion::default()
|
|
.sample_size(100)
|
|
.measurement_time(std::time::Duration::from_secs(10))
|
|
.warm_up_time(std::time::Duration::from_secs(3));
|
|
targets = bench_meta_learning_iteration, bench_incremental_learning
|
|
}
|
|
|
|
criterion_group! {
|
|
name = pattern_benches;
|
|
config = Criterion::default()
|
|
.sample_size(100)
|
|
.measurement_time(std::time::Duration::from_secs(8));
|
|
targets = bench_pattern_extraction, bench_pattern_matching
|
|
}
|
|
|
|
criterion_group! {
|
|
name = hierarchy_benches;
|
|
config = Criterion::default()
|
|
.sample_size(100);
|
|
targets = bench_multi_level_learning, bench_level_transition
|
|
}
|
|
|
|
criterion_group! {
|
|
name = integration_benches;
|
|
config = Criterion::default()
|
|
.sample_size(50)
|
|
.measurement_time(std::time::Duration::from_secs(12));
|
|
targets = bench_cross_crate_integration
|
|
}
|
|
|
|
criterion_group! {
|
|
name = recursive_benches;
|
|
config = Criterion::default()
|
|
.sample_size(50);
|
|
targets = bench_self_referential, bench_recursive_optimization
|
|
}
|
|
|
|
criterion_group! {
|
|
name = pipeline_benches;
|
|
config = Criterion::default()
|
|
.sample_size(30)
|
|
.measurement_time(std::time::Duration::from_secs(15));
|
|
targets = bench_complete_meta_learning
|
|
}
|
|
|
|
criterion_main!(
|
|
learning_benches,
|
|
pattern_benches,
|
|
hierarchy_benches,
|
|
integration_benches,
|
|
recursive_benches,
|
|
pipeline_benches
|
|
);
|