fix: skip triplet JSON export for large datasets (>100K)
JSON.stringify fails on 1M+ triplets. Training succeeded (33.3% improvement) but export crashed. Now skips export when >100K triplets. Co-Authored-By: claude-flow <ruv@ruv.net>
This commit is contained in:
parent
85417b84a6
commit
828d0599d7
|
|
@ -1257,9 +1257,13 @@ async function main() {
|
|||
contrastiveResult.finalLoss = finalContrastiveLoss;
|
||||
contrastiveResult.improvement = contrastiveImprovement;
|
||||
|
||||
// Export contrastive training data
|
||||
const contrastiveOutDir = contrastiveTrainer.exportTrainingData();
|
||||
console.log(` Training data exported to: ${contrastiveOutDir}`);
|
||||
// Export contrastive training data (skip for large datasets to avoid JSON string limit)
|
||||
if (contrastiveTrainer.getTripletCount() < 100000) {
|
||||
const contrastiveOutDir = contrastiveTrainer.exportTrainingData();
|
||||
console.log(` Training data exported to: ${contrastiveOutDir}`);
|
||||
} else {
|
||||
console.log(` Skipping triplet export (${contrastiveTrainer.getTripletCount()} triplets too large for JSON)`);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Phase 2: Task head training via TrainingPipeline
|
||||
|
|
|
|||
Loading…
Reference in New Issue