MultiRC

pip install -U datasets transformers evaluate lm-eval sacrebleu rouge-score scikit-learn

from datasets import load_dataset

ds = load_dataset("aps/super_glue", "multirc", split="validation")
print(ds)            # columns: paragraph, question, answer (candidate), label (0 false · 1 true), idx (paragraph/question
print(ds[0])

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

MODEL = "gpt2"                                   # or "Qwen/Qwen2.5-1.5B", "meta-llama/Llama-3.2-1B", …
tok = AutoTokenizer.from_pretrained(MODEL); model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.bfloat16).eval().to("cuda")

@torch.no_grad()
def loglik(context, continuation):
    """Sum of log-probs of `continuation` tokens given `context` — the standard multiple-choice scorer."""
    ids = tok(context + continuation, return_tensors="pt").input_ids.to(model.device)
    n_ctx = len(tok(context).input_ids)
    logp = model(ids).logits.log_softmax(-1)[0, :-1]
    return logp.gather(1, ids[0, 1:, None])[n_ctx - 1:].sum().item()

@torch.no_grad()
def generate(prompt, max_new_tokens=32, stop=None):
    ids = tok(prompt, return_tensors="pt").input_ids.to(model.device)
    out = model.generate(ids, max_new_tokens=max_new_tokens, do_sample=False, pad_token_id=tok.eos_token_id)
    text = tok.decode(out[0, ids.shape[1]:], skip_special_tokens=True)
    return text.split(stop)[0] if stop else text

from collections import defaultdict
by_q = defaultdict(list)                     # group rows by question for EM
for ex in ds:
    p = f"{ex['paragraph']}\nQuestion: {ex['question']}\nCandidate answer: {ex['answer']}\nIs the candidate correct? yes or no?\nAnswer:"
    pred = int(loglik(p, " yes") > loglik(p, " no"))
    by_q[(ex["idx"]["paragraph"], ex["idx"]["question"])].append((pred, ex["label"]))
em = sum(all(p == g for p, g in v) for v in by_q.values()) / len(by_q)
preds = [p for v in by_q.values() for p, _ in v]; golds = [g for v in by_q.values() for _, g in v]
from sklearn.metrics import f1_score
print("F1a", f1_score(golds, preds), "EM", em)

lm_eval --model hf --model_args pretrained=gpt2,dtype=bfloat16 \
        --tasks multirc --num_fewshot 0 --batch_size auto --output_path results/