15import math
16from typing import List
17
18import torch
19import torch.nn.functional as F
20from lm_eval import tasks, evaluator, utils
21from lm_eval.base import BaseLM
22from tokenizers import Tokenizer
23from torch import nn
24from tqdm import tqdm
25
26from labml import monit
27from labml_nn.neox.tokenizer import get_tokenizer30class EvalHarnessAdapter(BaseLM):tokenizer是 Huggingface Tokenizervocab_size是词汇量的大小（这与分词器词汇大小不同，因为neox添加了一些额外的内容来使嵌入层模型并行。）batch_size是批次大小
37 def __init__(self, tokenizer: Tokenizer, vocab_size: int, batch_size: int):45 super().__init__()
46 self.tokenizer = tokenizer
47 self._eot_token_id = self.tokenizer.token_to_id("<|endoftext|>")
48 self._vocab_size = vocab_size
49
50 self._batch_size = batch_size词汇量的大小
52 @property
53 def device(self):
54 raise RuntimeError()
55
56 @property
57 def vocab_size(self):59 return self._vocab_size文本结尾令牌
61 @property
62 def eot_token_id(self):64 return self._eot_token_id最大序列长度
66 @property
67 def max_length(self):69 return 2048要生成的令牌的最大数量
71 @property
72 def max_gen_toks(self):74 return 128批量大小
76 @property
77 def batch_size(self):81 return self._batch_size对给定文本进行编码
83 def tok_encode(self, string: str):87 return self.tokenizer.encode(string).ids解码来自令牌 ID 的文本
89 def tok_decode(self, tokens: List[int]):93 return self.tokenizer.decode(tokens)95 def _model_call(self, inps: torch.Tensor):
96 raise NotImplementedError98 def _model_generate(self, context, max_length, eos_token_id):
99 raise RuntimeError()101 def greedy_until(self, requests):
102 raise RuntimeError()104 @torch.no_grad()
105 def _loglikelihood_tokens(self, requests, disable_tqdm=False):为了结果
114 res = []按长度的降序对请求进行重新排序，以使长度相似的序列接近
118 def _collate(x):
119 toks = x[1] + x[2]
120 return -len(toks), tuple(toks)
121
122 reord = utils.Reorderer(requests, _collate)循环浏览一次包含batch_size
多个请求的请求
125 for chunk in utils.chunks(tqdm(reord.get_reordered(), disable=disable_tqdm), self.batch_size):存储批次的输入
127 inps = []该批次的延续
129 continuations = []输入序列的长度
131 inplens = []批次的填充长度
133 padded_length = None循环遍历区块中的每个请求，并将它们收集到带填充的 PyTorch 张量中
135 for _, context_enc, continuation_enc in chunk:连接上下文和延续
137 inp = context_enc + continuation_enc如果大小超过max_length
139 inp = inp[-(self.max_length + 1):]移除最终令牌
141 inp = inp[:-1]创建张量
143 inp = torch.tensor(inp, dtype=torch.long)输入长度
145 inplen = inp.shape[0]确定填充的长度。较短的序列将被填充。
149 if padded_length is None:
150 padded_length = int(math.ceil(inplen / 32)) * 32如果 padded_length 不是 padded_length 则为 padded_length 其他没有 inplen
填充
154 padding = torch.zeros(padded_length - inplen, dtype=torch.long)添加填充
157 inp = torch.cat([inp, padding], dim=0)
158
159 inps.append(inp)
160 continuations.append(continuation_enc)
161 inplens.append(inplen)获取模型日志
164 logits = self._model_call(torch.stack(inps))获取日志 softmaxes
167 multi_logits = F.log_softmax(logits, dim=-1)循环浏览批次的输入/输出对
170 for logits, inplen, cont_toks in zip(multi_logits, inplens, continuations):获取预测的代币数量
172 contlen = len(cont_toks)获取这些日志
174 logits = logits[inplen - contlen: inplen]获得概率最高的代币
176 greedy_tokens = logits.argmax(dim=-1)获取目标代币
178 cont_toks = torch.tensor(cont_toks, dtype=torch.long).to(logits.device)是否存在完全匹配
180 max_equal = (greedy_tokens == cont_toks).all()目标代币的对数可能性
182 logits = torch.gather(logits, 1, cont_toks[:, None])将总对数似然以及结果是否存在匹配项相加
184 res.append((float(logits.sum()), bool(max_equal)))重新排序并返回结果
187 return reord.get_original(res)运行给定的评估
189 @torch.no_grad()
190 def run_eval(self, name: str, eval_tasks: List[str]):运行 eleutherai/LM-Evaluation-Harnes s 评估器
196 results = evaluator.evaluate(lm=self, task_dict=tasks.get_task_dict(eval_tasks))添加配置
199 results["config"] = {
200 "name": name,
201 }204 return results207class NoeXEvalHarnessAdapter(EvalHarnessAdapter):model是模特tokenizer是 Huggingface Tokenizervocab_size是词汇量的大小（这与分词器词汇大小不同，因为neox添加了一些额外的内容来使嵌入层模型并行。）batch_size是批次大小device是该型号的设备
214 def __init__(self, model: nn.Module, tokenizer: Tokenizer, vocab_size: int, batch_size: int, device: torch.device):224 super().__init__(tokenizer, vocab_size, batch_size)
225 self.model = model
226 self._device = device给模特打电话
228 def _model_call(self, inps: torch.Tensor):232 return self.model(inps.to(self._device))使用给定模型运行评估工具
235def run_eval_harness(model: nn.Module, name: str, eval_tasks: List[str], device: torch.device, batch_size: int = 8):加载分词器
241 with monit.section('Load tokenizer'):
242 tokenizer = get_tokenizer()如果未指定任何内容，则为所有任务
245 if not eval_tasks:
246 eval_tasks = [
247 "anli_r1",
248 "anli_r2",
249 "anli_r3",
250 "hellaswag",
251 "lambada",
252 "piqa",
253 "winogrande",
254 "wsc",
255 "mathqa",
256 ]创建适配器
259 adapter = NoeXEvalHarnessAdapter(model, tokenizer, 50_432, batch_size, device)跑
262 return adapter.run_eval(name, eval_tasks)