Untitled
unknown
plain_text
a year ago
1.2 kB
11
Indexable
from factscore.factscorer import FactScorer import json def read_jsonl(filename): data = [] with open(filename, "r") as f: for line in f: data.append(json.loads(line)) return data if __name__ == "__main__": fs = FactScorer(openai_key = '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', model_name='retrieval+ChatGPT', data_dir = '/shared/data/tanayd2/CS598/eval/.cache/factscore/', cache_dir = '/shared/data/tanayd2/CS598/eval/.cache/factscore/',) filename = '../CS598AIE/outputs/full_wikibio_0.jsonl' data = read_jsonl(filename) topics = [i['misc'] for i in data][0:1] generations = [i['full_output'] for i in data][0:1] # topics: list of strings (human entities used to generate bios) # generations: list of strings (model generations) out = fs.get_score(topics, generations, gamma=10) print (out["score"]) # FActScore print (out["init_score"]) # FActScore w/o length penalty print (out["respond_ratio"]) # % of responding (not abstaining from answering) print (out["num_facts_per_response"]) # average number of atomic facts per response
Editor is loading...
Leave a Comment