/**
 * Feedback Loop System for Behavior Modification
 * Enables the system to learn from outcomes and modify behavior dynamically
 */
export class FeedbackLoopSystem {
    feedbackHistory = [];
    behaviorModifications = [];
    adaptationRules = [];
    behaviorParameters = new Map();
    performanceMetrics = new Map();
    learningCurves = new Map();
    constructor() {
        this.initializeDefaultRules();
        this.initializeDefaultParameters();
    }
    /**
     * Process feedback and trigger behavior modifications
     */
    async processFeedback(feedback) {
        // Store feedback
        this.feedbackHistory.push(feedback);
        // Update performance metrics
        this.updatePerformanceMetrics(feedback);
        // Find applicable adaptation rules
        const applicableRules = this.adaptationRules.filter(rule => rule.trigger(feedback));
        // Generate behavior modifications
        const modifications = [];
        for (const rule of applicableRules) {
            const currentState = this.getCurrentBehaviorState();
            const ruleMods = rule.modification(feedback, currentState);
            modifications.push(...ruleMods);
        }
        // Apply modifications
        for (const modification of modifications) {
            await this.applyBehaviorModification(modification);
        }
        // Learn from the feedback pattern
        await this.learnFromFeedbackPattern(feedback);
        return modifications;
    }
    /**
     * Register new adaptation rule
     */
    registerAdaptationRule(rule) {
        this.adaptationRules.push(rule);
        // Sort by priority
        this.adaptationRules.sort((a, b) => b.priority - a.priority);
    }
    /**
     * Create feedback loop for continuous improvement
     */
    createContinuousImprovementLoop(component, metric) {
        const improvementRule = {
            trigger: (feedback) => feedback.source === component,
            modification: (feedback, currentState) => {
                const currentMetric = this.getMetricTrend(metric);
                const isImproving = this.isMetricImproving(currentMetric);
                if (!isImproving) {
                    return this.generateImprovementModifications(component, feedback);
                }
                return [];
            },
            priority: 0.7,
            learningRate: 0.1,
            category: 'continuous_improvement'
        };
        this.registerAdaptationRule(improvementRule);
    }
    /**
     * Implement reinforcement learning feedback loop
     */
    createReinforcementLoop(actionSpace, rewardFunction) {
        const reinforcementRule = {
            trigger: (feedback) => actionSpace.includes(feedback.action),
            modification: (feedback, currentState) => {
                const reward = rewardFunction(feedback.outcome);
                return this.updateActionProbabilities(feedback.action, reward, actionSpace);
            },
            priority: 0.8,
            learningRate: 0.15,
            category: 'reinforcement_learning'
        };
        this.registerAdaptationRule(reinforcementRule);
    }
    /**
     * Create exploration-exploitation feedback loop
     */
    createExplorationExploitationLoop(explorationRate = 0.1) {
        const explorationRule = {
            trigger: (feedback) => feedback.type === 'unexpected' || feedback.surprise > 0.7,
            modification: (feedback, currentState) => {
                // Increase exploration if we're getting unexpected results
                if (feedback.surprise > 0.7) {
                    return [{
                            component: 'exploration_system',
                            parameter: 'exploration_rate',
                            oldValue: currentState.exploration_rate || explorationRate,
                            newValue: Math.min(1.0, (currentState.exploration_rate || explorationRate) + 0.1),
                            reason: 'High surprise level - increase exploration',
                            confidence: 0.8,
                            timestamp: Date.now(),
                            expectedImprovement: 0.2
                        }];
                }
                // Decrease exploration if we're getting predictable good results
                if (feedback.type === 'success' && feedback.surprise < 0.2) {
                    return [{
                            component: 'exploration_system',
                            parameter: 'exploration_rate',
                            oldValue: currentState.exploration_rate || explorationRate,
                            newValue: Math.max(0.01, (currentState.exploration_rate || explorationRate) - 0.05),
                            reason: 'Low surprise, high success - decrease exploration',
                            confidence: 0.7,
                            timestamp: Date.now(),
                            expectedImprovement: 0.1
                        }];
                }
                return [];
            },
            priority: 0.6,
            learningRate: 0.05,
            category: 'exploration_exploitation'
        };
        this.registerAdaptationRule(explorationRule);
    }
    /**
     * Implement meta-learning feedback loop
     */
    createMetaLearningLoop() {
        const metaLearningRule = {
            trigger: (feedback) => this.feedbackHistory.length % 50 === 0, // Every 50 feedback signals
            modification: (feedback, currentState) => {
                // Analyze learning patterns and adjust learning rates
                const learningEffectiveness = this.analyzeLearningEffectiveness();
                return this.adjustLearningParameters(learningEffectiveness);
            },
            priority: 0.9,
            learningRate: 0.02,
            category: 'meta_learning'
        };
        this.registerAdaptationRule(metaLearningRule);
    }
    /**
     * Create adaptive complexity feedback loop
     */
    createComplexityAdaptationLoop() {
        const complexityRule = {
            trigger: (feedback) => true, // Always applicable
            modification: (feedback, currentState) => {
                const performanceTrend = this.getRecentPerformanceTrend();
                const currentComplexity = currentState.reasoning_complexity || 0.5;
                // If performance is declining, try different complexity levels
                if (performanceTrend < 0.3) {
                    const newComplexity = this.adaptComplexity(currentComplexity, feedback);
                    if (newComplexity !== currentComplexity) {
                        return [{
                                component: 'reasoning_system',
                                parameter: 'reasoning_complexity',
                                oldValue: currentComplexity,
                                newValue: newComplexity,
                                reason: `Performance trend: ${performanceTrend.toFixed(2)} - adjusting complexity`,
                                confidence: 0.6,
                                timestamp: Date.now(),
                                expectedImprovement: Math.abs(newComplexity - currentComplexity) * 0.5
                            }];
                    }
                }
                return [];
            },
            priority: 0.5,
            learningRate: 0.08,
            category: 'adaptive_complexity'
        };
        this.registerAdaptationRule(complexityRule);
    }
    /**
     * Apply behavior modification to system parameters
     */
    async applyBehaviorModification(modification) {
        const key = `${modification.component}.${modification.parameter}`;
        // Store old value for potential rollback
        const oldValue = this.behaviorParameters.get(key);
        // Apply new value
        this.behaviorParameters.set(key, modification.newValue);
        // Record the modification
        this.behaviorModifications.push(modification);
        // Update performance tracking
        this.updateLearningCurve(modification.component, modification.expectedImprovement);
        console.log(`Applied behavior modification: ${modification.component}.${modification.parameter}
                ${JSON.stringify(modification.oldValue)} -> ${JSON.stringify(modification.newValue)}`);
    }
    /**
     * Learn from feedback patterns to create new adaptation rules
     */
    async learnFromFeedbackPattern(feedback) {
        // Look for patterns in recent feedback
        const recentFeedback = this.feedbackHistory.slice(-20);
        // Detect recurring failure patterns
        const failurePattern = this.detectFailurePattern(recentFeedback);
        if (failurePattern) {
            const newRule = this.createRuleFromPattern(failurePattern);
            this.registerAdaptationRule(newRule);
        }
        // Detect success patterns
        const successPattern = this.detectSuccessPattern(recentFeedback);
        if (successPattern) {
            const reinforcementRule = this.createReinforcementRule(successPattern);
            this.registerAdaptationRule(reinforcementRule);
        }
    }
    /**
     * Initialize default adaptation rules
     */
    initializeDefaultRules() {
        // Error correction rule
        this.registerAdaptationRule({
            trigger: (feedback) => feedback.type === 'failure',
            modification: (feedback, currentState) => [{
                    component: feedback.source,
                    parameter: 'error_tolerance',
                    oldValue: currentState.error_tolerance || 0.1,
                    newValue: Math.min(1.0, (currentState.error_tolerance || 0.1) + 0.05),
                    reason: 'Failure detected - increase error tolerance',
                    confidence: 0.7,
                    timestamp: Date.now(),
                    expectedImprovement: 0.1
                }],
            priority: 0.8,
            learningRate: 0.1,
            category: 'error_correction'
        });
        // Success reinforcement rule
        this.registerAdaptationRule({
            trigger: (feedback) => feedback.type === 'success' && feedback.utility > 0.8,
            modification: (feedback, currentState) => [{
                    component: feedback.source,
                    parameter: 'success_bias',
                    oldValue: currentState.success_bias || 0.5,
                    newValue: Math.min(1.0, (currentState.success_bias || 0.5) + 0.02),
                    reason: 'High utility success - reinforce successful patterns',
                    confidence: 0.9,
                    timestamp: Date.now(),
                    expectedImprovement: 0.05
                }],
            priority: 0.7,
            learningRate: 0.05,
            category: 'success_reinforcement'
        });
        // Novelty adaptation rule
        this.registerAdaptationRule({
            trigger: (feedback) => feedback.type === 'novel',
            modification: (feedback, currentState) => [{
                    component: 'novelty_system',
                    parameter: 'novelty_weight',
                    oldValue: currentState.novelty_weight || 0.3,
                    newValue: Math.min(1.0, (currentState.novelty_weight || 0.3) + 0.1),
                    reason: 'Novel outcome detected - increase novelty seeking',
                    confidence: 0.6,
                    timestamp: Date.now(),
                    expectedImprovement: 0.15
                }],
            priority: 0.5,
            learningRate: 0.08,
            category: 'novelty_adaptation'
        });
    }
    /**
     * Initialize default behavior parameters
     */
    initializeDefaultParameters() {
        this.behaviorParameters.set('reasoning_system.complexity', 0.5);
        this.behaviorParameters.set('exploration_system.exploration_rate', 0.1);
        this.behaviorParameters.set('learning_system.learning_rate', 0.1);
        this.behaviorParameters.set('novelty_system.novelty_weight', 0.3);
        this.behaviorParameters.set('error_system.error_tolerance', 0.1);
        this.behaviorParameters.set('success_system.success_bias', 0.5);
    }
    /**
     * Update performance metrics based on feedback
     */
    updatePerformanceMetrics(feedback) {
        const metricKey = `${feedback.source}_${feedback.type}`;
        const metrics = this.performanceMetrics.get(metricKey) || [];
        const score = this.calculatePerformanceScore(feedback);
        metrics.push(score);
        // Keep only recent metrics (last 100)
        if (metrics.length > 100) {
            metrics.shift();
        }
        this.performanceMetrics.set(metricKey, metrics);
    }
    /**
     * Calculate performance score from feedback
     */
    calculatePerformanceScore(feedback) {
        let score = 0.5; // Neutral baseline
        switch (feedback.type) {
            case 'success':
                score = 0.8 + feedback.utility * 0.2;
                break;
            case 'failure':
                score = 0.2 - feedback.utility * 0.2;
                break;
            case 'partial':
                score = 0.5 + feedback.utility * 0.3;
                break;
            case 'unexpected':
                score = 0.6 + feedback.surprise * 0.4;
                break;
            case 'novel':
                score = 0.7 + (feedback.utility + feedback.surprise) * 0.15;
                break;
        }
        return Math.max(0, Math.min(1, score));
    }
    /**
     * Get current behavior state
     */
    getCurrentBehaviorState() {
        const state = {};
        for (const [key, value] of this.behaviorParameters) {
            const [component, parameter] = key.split('.');
            if (!state[component])
                state[component] = {};
            state[component][parameter] = value;
            // Also add flat structure for easier access
            state[parameter] = value;
        }
        return state;
    }
    /**
     * Get metric trend for analysis
     */
    getMetricTrend(metric) {
        return this.performanceMetrics.get(metric) || [];
    }
    /**
     * Check if metric is improving
     */
    isMetricImproving(metricValues) {
        if (metricValues.length < 5)
            return true; // Not enough data
        const recent = metricValues.slice(-5);
        const older = metricValues.slice(-10, -5);
        if (older.length === 0)
            return true;
        const recentAvg = recent.reduce((a, b) => a + b, 0) / recent.length;
        const olderAvg = older.reduce((a, b) => a + b, 0) / older.length;
        return recentAvg > olderAvg;
    }
    /**
     * Generate improvement modifications
     */
    generateImprovementModifications(component, feedback) {
        const modifications = [];
        // Suggest parameter adjustments based on failure type
        if (feedback.type === 'failure') {
            modifications.push({
                component,
                parameter: 'robustness',
                oldValue: 0.5,
                newValue: 0.7,
                reason: 'Failure detected - increase robustness',
                confidence: 0.6,
                timestamp: Date.now(),
                expectedImprovement: 0.2
            });
        }
        return modifications;
    }
    /**
     * Update action probabilities based on reinforcement learning
     */
    updateActionProbabilities(action, reward, actionSpace) {
        const modifications = [];
        // Increase probability of rewarded actions
        if (reward > 0.5) {
            modifications.push({
                component: 'action_system',
                parameter: `${action}_probability`,
                oldValue: 1.0 / actionSpace.length, // Uniform prior
                newValue: Math.min(0.8, (1.0 / actionSpace.length) + reward * 0.1),
                reason: `Positive reward (${reward.toFixed(2)}) for action ${action}`,
                confidence: reward,
                timestamp: Date.now(),
                expectedImprovement: reward * 0.2
            });
        }
        return modifications;
    }
    /**
     * Analyze learning effectiveness
     */
    analyzeLearningEffectiveness() {
        const recentModifications = this.behaviorModifications.slice(-20);
        if (recentModifications.length === 0)
            return 0.5;
        const actualImprovements = recentModifications.map(mod => {
            // Compare expected vs actual improvement
            const component = mod.component;
            const metricKey = `${component}_improvement`;
            const metrics = this.performanceMetrics.get(metricKey) || [];
            if (metrics.length < 2)
                return mod.expectedImprovement;
            const beforeImprovement = metrics[metrics.length - 2] || 0;
            const afterImprovement = metrics[metrics.length - 1] || 0;
            return afterImprovement - beforeImprovement;
        });
        const avgActualImprovement = actualImprovements.reduce((a, b) => a + b, 0) / actualImprovements.length;
        const avgExpectedImprovement = recentModifications.reduce((sum, mod) => sum + mod.expectedImprovement, 0) / recentModifications.length;
        return avgExpectedImprovement > 0 ? avgActualImprovement / avgExpectedImprovement : 0.5;
    }
    /**
     * Adjust learning parameters based on effectiveness
     */
    adjustLearningParameters(effectiveness) {
        const modifications = [];
        // Adjust learning rates based on effectiveness
        for (const rule of this.adaptationRules) {
            const newLearningRate = effectiveness > 0.8 ?
                Math.min(0.5, rule.learningRate * 1.1) :
                Math.max(0.01, rule.learningRate * 0.9);
            if (Math.abs(newLearningRate - rule.learningRate) > 0.01) {
                modifications.push({
                    component: 'meta_learning',
                    parameter: `${rule.category}_learning_rate`,
                    oldValue: rule.learningRate,
                    newValue: newLearningRate,
                    reason: `Learning effectiveness: ${effectiveness.toFixed(2)} - adjust learning rate`,
                    confidence: 0.7,
                    timestamp: Date.now(),
                    expectedImprovement: Math.abs(newLearningRate - rule.learningRate) * 2
                });
                rule.learningRate = newLearningRate;
            }
        }
        return modifications;
    }
    /**
     * Get recent performance trend
     */
    getRecentPerformanceTrend() {
        const allMetrics = [];
        for (const metrics of this.performanceMetrics.values()) {
            allMetrics.push(...metrics.slice(-5)); // Recent 5 values from each metric
        }
        if (allMetrics.length === 0)
            return 0.5;
        return allMetrics.reduce((a, b) => a + b, 0) / allMetrics.length;
    }
    /**
     * Adapt complexity based on performance
     */
    adaptComplexity(currentComplexity, feedback) {
        if (feedback.type === 'failure' && feedback.utility < 0.3) {
            // Failure with low utility - try lower complexity
            return Math.max(0.1, currentComplexity - 0.1);
        }
        if (feedback.type === 'success' && feedback.surprise > 0.7) {
            // Successful but surprising - might benefit from higher complexity
            return Math.min(1.0, currentComplexity + 0.1);
        }
        return currentComplexity;
    }
    /**
     * Update learning curve for component
     */
    updateLearningCurve(component, improvement) {
        const curve = this.learningCurves.get(component) || [];
        curve.push(improvement);
        if (curve.length > 50) {
            curve.shift();
        }
        this.learningCurves.set(component, curve);
    }
    /**
     * Detect failure patterns in recent feedback
     */
    detectFailurePattern(feedback) {
        const failures = feedback.filter(f => f.type === 'failure');
        if (failures.length < 3)
            return null;
        // Look for common failure contexts
        const contexts = failures.map(f => f.context);
        const commonContext = this.findCommonElements(contexts);
        if (Object.keys(commonContext).length > 0) {
            return {
                type: 'recurring_failure',
                context: commonContext,
                frequency: failures.length / feedback.length
            };
        }
        return null;
    }
    /**
     * Detect success patterns in recent feedback
     */
    detectSuccessPattern(feedback) {
        const successes = feedback.filter(f => f.type === 'success' && f.utility > 0.7);
        if (successes.length < 2)
            return null;
        return {
            type: 'success_pattern',
            actions: successes.map(s => s.action),
            avgUtility: successes.reduce((sum, s) => sum + s.utility, 0) / successes.length
        };
    }
    /**
     * Create adaptation rule from detected pattern
     */
    createRuleFromPattern(pattern) {
        return {
            trigger: (feedback) => {
                // Check if feedback matches the pattern context
                for (const [key, value] of Object.entries(pattern.context)) {
                    if (feedback.context[key] !== value)
                        return false;
                }
                return true;
            },
            modification: (feedback, currentState) => [{
                    component: 'pattern_system',
                    parameter: 'pattern_avoidance',
                    oldValue: 0,
                    newValue: 1,
                    reason: `Avoiding detected failure pattern: ${JSON.stringify(pattern.context)}`,
                    confidence: pattern.frequency,
                    timestamp: Date.now(),
                    expectedImprovement: pattern.frequency * 0.5
                }],
            priority: 0.8,
            learningRate: 0.1,
            category: 'pattern_avoidance'
        };
    }
    /**
     * Create reinforcement rule from success pattern
     */
    createReinforcementRule(pattern) {
        return {
            trigger: (feedback) => pattern.actions.includes(feedback.action),
            modification: (feedback, currentState) => [{
                    component: 'pattern_system',
                    parameter: 'pattern_reinforcement',
                    oldValue: 0,
                    newValue: pattern.avgUtility,
                    reason: `Reinforcing successful action pattern`,
                    confidence: pattern.avgUtility,
                    timestamp: Date.now(),
                    expectedImprovement: pattern.avgUtility * 0.3
                }],
            priority: 0.7,
            learningRate: 0.08,
            category: 'pattern_reinforcement'
        };
    }
    /**
     * Find common elements across contexts
     */
    findCommonElements(contexts) {
        if (contexts.length === 0)
            return {};
        const common = {};
        const first = contexts[0] || {};
        for (const [key, value] of Object.entries(first)) {
            if (contexts.every(ctx => ctx[key] === value)) {
                common[key] = value;
            }
        }
        return common;
    }
    /**
     * Get feedback loop statistics
     */
    getStats() {
        return {
            totalFeedback: this.feedbackHistory.length,
            totalModifications: this.behaviorModifications.length,
            activeRules: this.adaptationRules.length,
            behaviorParameters: this.behaviorParameters.size,
            recentPerformance: this.getRecentPerformanceTrend(),
            learningEffectiveness: this.analyzeLearningEffectiveness(),
            mostActiveComponents: this.getMostActiveComponents(),
            adaptationCategories: this.getAdaptationCategories()
        };
    }
    getMostActiveComponents() {
        const componentCounts = new Map();
        for (const mod of this.behaviorModifications) {
            componentCounts.set(mod.component, (componentCounts.get(mod.component) || 0) + 1);
        }
        return Array.from(componentCounts.entries())
            .sort((a, b) => b[1] - a[1])
            .slice(0, 5)
            .map(entry => entry[0]);
    }
    getAdaptationCategories() {
        return [...new Set(this.adaptationRules.map(rule => rule.category))];
    }
}