LLMを利用したFAQ検索の評価データセットの作成

import re
from typing import List

from langchain.chains import LLMChain
from langchain.chat_models import ChatOpenAI
from langchain.output_parsers import ListOutputParser
from langchain.prompts import (
    ChatPromptTemplate,
    HumanMessagePromptTemplate,
    SystemMessagePromptTemplate,
)


class NumberedListOutputParser(ListOutputParser):
    def parse(self, output: str) -> List[str]:
        return re.findall(r"\d+\.\s+(.*?)\n", output)


paraphrase_llm = ChatOpenAI(temperature=0.5)
prompt_template = ChatPromptTemplate.from_messages(
    [
        SystemMessagePromptTemplate.from_template(
            "You are a helpful paraphrasing assistant tasked with rephrasing text."
        ),
        SystemMessagePromptTemplate.from_template("Input: <INPUT>{query}</INPUT>"),
        HumanMessagePromptTemplate.from_template(
            "What are {n_paraphrases} different ways you could paraphrase the INPUT text?"
            " Do not significantly change the meaning."
            " Respond using numbered bullets. If you cannot think of any,"
            " just say 'I don't know.'"
            " lang: ja"  # 追加
        ),
    ]
)

paraphrase_chain = LLMChain(
    llm=paraphrase_llm,
    prompt=prompt_template,
    output_parser=NumberedListOutputParser(),
)

import pandas as pd

ameba_df = pd.read_csv("{データセットPATH}")

n_paraphrases = 2  # パラフレーズを1つだけ生成したかったが、なぜか'設定値 - 1'の生成がされる傾向があった。
paraphrase_lst = []
for i, q_text in tqdm(ameba_df[["ID", "Title"]].values):
    result = await paraphrase_chain.arun(
        query=q_text, n_paraphrases=n_paraphrases
    )
    for q in result:
        paraphrase_lst.append((i, q))

paraphrase_df = pd.DataFrame(paraphrase_lst, columns=["ID", "Query"])

paraphrase_df.head(3)

'''
<OUTPUT>退会手続きに関して</OUTPUT>
のようにpromptに引きずられてタグがついてしまっている
-> 正規表現で対処
'''

import re

def remove_html_tags(text):
    clean = re.compile('<.*?>')
    return re.sub(clean, '', text)

paraphrase_df["Query"] = paraphrase_df["Query"].map(remove_html_tags)


'''
Titleと同じ文章が生成されてしまっている
-> 除外
'''

same_title_idx = []
for _, row in paraphrase_df.iterrows():
    if row["Query"] != ameba_df[ameba_df["ID"] == row["ID"]]["Title"].iloc[0]:
        same_title_idx.append(row.name)

paraphrase_df = paraphrase_df.loc[same_title_idx]


'''
Off-Target Problem: https://arxiv.org/abs/2305.10930
（ターゲットの言語は日本語だけど英語で生成されてしまっている)
-> 半角スペースの数でフィルタリング
'''

non_english_idx = [row.name for _, row in paraphrase_df.iterrows() if len(row["Query"].split()) <= 5]

paraphrase_df = paraphrase_df.loc[non_english_idx]


'''
生成フォーマットに沿わない生成になっている
例えば
Ameba Pick 利用規約
というタイトルに対して
Ameba Pick 利用規約の内容を言い換える方法が2つあります。意味を大きく変えずに、以下の番号付きの箇条書きで回答します。
という生成がされている
-> テキスト長の差でフィルタリング
'''

diff_title_idx = []
for _, row in paraphrase_df.iterrows():
    org_title = ameba_df[ameba_df["ID"] == row["ID"]]["Title"].iloc[0]
    l_diff = abs(len(row["Query"]) - len(org_title))
    if l_diff < 20:  # 決めうち
        diff_title_idx.append(row.name)
        
paraphrase_df = paraphrase_df.loc[diff_title_idx]

for _, row in paraphrase_df.sample(3).iterrows():
    org_title = ameba_df[ameba_df["ID"] == row["ID"]]["Title"].iloc[0]
    print(f"[{row.ID}]")
    print("タイトル:", org_title)
    print("生成文:", row["Query"])
    print()

num_questions_per_chunk = 1
question_gen_query = (
     "You are a Teacher/Professor. Your task is to setup"
     f" {num_questions_per_chunk} questions for an upcoming"
     " quiz/examination. The questions should be diverse in nature"
     " across the document. Restrict the questions to the"
     " context information provided."
     " lang: ja"  # 追加
)
question_gen_query

lst = []
for i, doc in tqdm(enumerate(documents), total=len(documents)):
    dataset_generator = DatasetGenerator.from_documents(
        [doc],
        question_gen_query=question_gen_query,
    )
    questions = dataset_generator.generate_questions_from_nodes(num=1)
    for q in questions:
        lst.append((ameba_df.iloc[i]["ID"], q))

gen_from_content_df = pd.DataFrame(lst, columns=["ID", "Query"])
gen_from_content_df.head(3)

print(ameba_df.query("ID=='faq_283'")["Content"].iloc[0])

input_gen_llm = ChatOpenAI(temperature=0.5)
input_gen_prompt_template = ChatPromptTemplate.from_messages(
    [
        SystemMessagePromptTemplate.from_template(
            "You are a creative assistant tasked with coming up with new inputs for an application."
        ),
        SystemMessagePromptTemplate.from_template("The following are some examples of inputs: \n{examples}"),
        HumanMessagePromptTemplate.from_template(
            "Can you generate {n_inputs} unique and plausible inputs that could be asked by different users?"
            " lang: ja"  # 追加
        ),
    ]
)

input_gen_chain = LLMChain(
    llm=input_gen_llm,
    prompt=input_gen_prompt_template,
    output_parser=NumberedListOutputParser(),
)

texts = ... # ユーザーの検索履歴や質問履歴のリスト

example_inputs_str = "\n".join([f"- {t}" for t in texts])
n_inputs = 5  # 生成数
result = await input_gen_chain.arun(
        examples=example_inputs_str, n_inputs=n_inputs
)

import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from janome.tokenizer import Tokenizer

t = Tokenizer()

corpus = ameba_df["Title"].tolist()
corpus = [" ".join(t.tokenize(c, wakati=True)) for c in corpus]

vectorizer = CountVectorizer()
X = vectorizer.fit_transform(corpus)
 
ranks = []
for _, row in paraphrase_df.iterrows():
    answer_idx = ameba_df[ameba_df["ID"] == row["ID"]].index[0]
    v = vectorizer.transform([" ".join(t.tokenize(row["Query"], wakati=True))])
    dot = np.dot(v, X.T).toarray()[0]

    rank = dot.argsort()[::-1]
    ans_rank = np.array(range(len(rank)))[rank == answer_idx][0]
    ranks.append(ans_rank+1)

p_at_1 = (np.array(ranks) <= 1).mean()
print(p_at_1)  # 0.6673773987206824