RACE

pip install -U datasets transformers evaluate lm-eval sacrebleu rouge-score scikit-learn

from datasets import load_dataset

ds = load_dataset("ehovy/race", "all", split="test")
print(ds)            # columns: article, question, options (list of 4), answer ('A'–'D')
print(ds[0])

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL = "gpt2"                                   # or "Qwen/Qwen2.5-1.5B", "meta-llama/Llama-3.2-1B", …
tok = AutoTokenizer.from_pretrained(MODEL); model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.bfloat16).eval().to("cuda")

@torch.no_grad()
def loglik(context, continuation):
    """Sum of log-probs of `continuation` tokens given `context` — the standard multiple-choice scorer."""
    ids = tok(context + continuation, return_tensors="pt").input_ids.to(model.device)
    n_ctx = len(tok(context).input_ids)
    logp = model(ids).logits.log_softmax(-1)[0, :-1]
    return logp.gather(1, ids[0, 1:, None])[n_ctx - 1:].sum().item()

@torch.no_grad()
def generate(prompt, max_new_tokens=32, stop=None):
    ids = tok(prompt, return_tensors="pt").input_ids.to(model.device)
    out = model.generate(ids, max_new_tokens=max_new_tokens, do_sample=False, pad_token_id=tok.eos_token_id)
    text = tok.decode(out[0, ids.shape[1]:], skip_special_tokens=True)
    return text.split(stop)[0] if stop else text

acc = 0
for ex in ds:
    ctx = f"Article: {ex['article']}\n\nQuestion: {ex['question']}\nAnswer:"
    s = [loglik(ctx, " " + o) / max(1, len(o)) for o in ex["options"]]
    acc += int("ABCD"[s.index(max(s))] == ex["answer"])
print("accuracy", acc / len(ds))

lm_eval --model hf --model_args pretrained=gpt2,dtype=bfloat16 \
        --tasks race --num_fewshot 0 --batch_size auto --output_path results/