From 9595b1a499c548bc4b7201b84e0e1adda1a71df4 Mon Sep 17 00:00:00 2001 From: maderix Date: Thu, 5 Mar 2026 07:41:33 -0800 Subject: [PATCH] Add tokenizer via git-lfs, fix dashboard tokenizer path - Add tokenizer.bin (434KB) to assets/models/ via git-lfs - Fix dashboard tokenizer path (was one parent too many) --- .gitattributes | 2 ++ assets/models/tokenizer.bin | 3 +++ training/dashboard.py | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .gitattributes create mode 100644 assets/models/tokenizer.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a44b889 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +assets/models/*.bin filter=lfs diff=lfs merge=lfs -text +assets/data/*.bin filter=lfs diff=lfs merge=lfs -text diff --git a/assets/models/tokenizer.bin b/assets/models/tokenizer.bin new file mode 100644 index 0000000..acfb579 --- /dev/null +++ b/assets/models/tokenizer.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50a52ef822ee9e83de5ce9d0be0a025a773d019437f58b5ff9dcafb063ece361 +size 433869 diff --git a/training/dashboard.py b/training/dashboard.py index 18203d7..01e9c1a 100644 --- a/training/dashboard.py +++ b/training/dashboard.py @@ -23,7 +23,7 @@ HD = DIM // HEADS CKPT_PATH_STATIC = 'ane_stories110M_ckpt.bin' CKPT_PATH_DYNAMIC = 'training_dynamic/ane_stories110M_dyn_ckpt.bin' CKPT_PATH = CKPT_PATH_STATIC # set in main() based on --dynamic -TOKENIZER_PATH = str(Path(__file__).resolve().parent.parent.parent / 'assets' / 'models' / 'tokenizer.bin') +TOKENIZER_PATH = str(Path(__file__).resolve().parent.parent / 'assets' / 'models' / 'tokenizer.bin') class State: