Untitled

from factscore.factscorer import FactScorer
import json

def read_jsonl(filename):
    data = []
    with open(filename, "r") as f:
        for line in f:
            data.append(json.loads(line))
    return data


if __name__ == "__main__":
    fs = FactScorer(openai_key = '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',
        model_name='retrieval+ChatGPT', 
                    data_dir = '/shared/data/tanayd2/CS598/eval/.cache/factscore/',
                    cache_dir = '/shared/data/tanayd2/CS598/eval/.cache/factscore/',)

    filename = '../CS598AIE/outputs/full_wikibio_0.jsonl'
    data = read_jsonl(filename)
    topics = [i['misc'] for i in data][0:1]
    generations = [i['full_output'] for i in data][0:1]
    # topics: list of strings (human entities used to generate bios)
    # generations: list of strings (model generations)
    out = fs.get_score(topics, generations, gamma=10)
    print (out["score"]) # FActScore
    print (out["init_score"]) # FActScore w/o length penalty
    print (out["respond_ratio"]) # % of responding (not abstaining from answering)
    print (out["num_facts_per_response"]) # average number of atomic facts per response