improved evaluation specifications

example/test_helm_d.sh

@@ -1,9 +1,11 @@
 lm-eval \
   --model helm_d_115M \
+  --model_args ckpt_dir=... \
   --tasks commonsense_qa,openbookqa,hellaswag \
   --num_fewshot 0
 
 lm-eval \
   --model helm_d_115M \
+  --model_args ckpt_dir=... \
   --tasks mmlu,arc_challenge \
   --num_fewshot 5

example/test_mice_120M.sh

@@ -1,9 +1,11 @@
 lm-eval \
   --model helm_mice_120M \
+  --model_args ckpt_dir=... \
   --tasks commonsense_qa,openbookqa,hellaswag \
   --num_fewshot 0
 
 lm-eval \
   --model helm_mice_120M \
+  --model_args ckpt_dir=... \
   --tasks mmlu,arc_challenge \
   --num_fewshot 5

example/test_mice_1B.sh

@@ -1,9 +1,11 @@
 lm-eval \
   --model helm_mice_1b \
+  --model_args ckpt_dir=... \
   --tasks commonsense_qa,openbookqa,hellaswag \
   --num_fewshot 0
 
 lm-eval \
   --model helm_mice_1b \
+  --model_args ckpt_dir=... \
   --tasks mmlu,arc_challenge \
   --num_fewshot 5

lm-evaluation-harness/lm_eval/models/hypercore/models/graph_retriever.py

@@ -3,7 +3,7 @@
 import torch
 from torch import Tensor
 
-from torch_geometric.nn.nlp.llm import BOS, LLM, MAX_NEW_TOKENS
+from torch_geometric.llm.models.llm import BOS, LLM, MAX_NEW_TOKENS
 from torch_geometric.utils import scatter
 from ..nn.peft import *
 

setup_training.sh

@@ -3,4 +3,7 @@ pip install torch_geometric
 pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-2.6.0+cu124.html
 pip install -r requirements.txt
 pip install scipy==1.13.1 numpy==1.26.4
-pip install llm-foundry==0.18.0
+pip install llm-foundry==0.18.0
+cd lm-evaluation-harness
+pip install -e .
+cd ..