LLMで挑むTitanic生存予測: Few-Shot Leaningで表形式データはどこま解ける？

git clone https://github.com/dylan-slack/Tablet.git
cd Tablet
python3 -m pip install -e .

response = openai.ChatCompletion.create(
    model="gpt-4o-mini",
    messages=[
        {"role": "user", "content": text},
    ],
)

elif self.model == "chat-gpt":
    self.tokenizer = None
    self.model_obj = None

import pandas as pd

train_df = pd.read_csv("{解凍した場所}/train.csv")
test_df = pd.read_csv("{解凍した場所}/test.csv")

# 目的変数をテキストに変換
train_df["Survived"] = train_df["Survived"].map({0: "died", 1: "survived"})

train_x = train_df.drop(columns=["Survived"])
train_y = train_df["Survived"].values
eval_x = test_df
eval_y = np.array(["died" for _ in range(len(test_df))])  # testデータの目的変数は未知なので仮の値を置く

dtypes = train_x.dtypes
names_of_categorical_columns = dtypes[dtypes == "object"].index.tolist()
names_of_number_columns = dtypes[dtypes != "object"].index.tolist()

# カテゴリ変数の欠損値は"missing"に変換
train_x[names_of_categorical_columns] = train_x[names_of_categorical_columns].fillna("missing")
eval_x[names_of_categorical_columns] = eval_x[names_of_categorical_columns].fillna("missing")

# 数値の欠損値は-1に変換
train_x[names_of_number_columns] = train_x[names_of_number_columns].fillna(-1)
eval_x[names_of_number_columns] = eval_x[names_of_number_columns].fillna(-1)

from Tablet import create

# LLMへの指示. コンペティションサイトのOverviewより.
instructions = """The sinking of the Titanic is one of the most infamous shipwrecks in history.
On April 15, 1912, during her maiden voyage, the widely considered “unsinkable” RMS Titanic sank after colliding with an iceberg. Unfortunately, there weren’t enough lifeboats for everyone onboard, resulting in the death of 1502 out of 2224 passengers and crew.
While there was some element of luck involved in surviving, it seems some groups of people were more likely to survive than others.
In this challenge, we ask you to build a predictive model that answers the question: “what sorts of people were more likely to survive?” using passenger data (ie name, age, gender, socio-economic class, etc).

Please answer as either `survived` or `died`.
"""

create.create_task(train_x,
                   eval_x,
                   train_y,
                   eval_y,
                   name="titanic",
                   header="You must follow the instructions to predict if passengers had suvived.",
                   nl_instruction=instructions,
                   categorical_columns=names_of_categorical_columns,
                   num_gpt3_revisions=1,  # 今回は実験のため1パターンのみ生成
                   save_loc="./data/benchmark")

{
        "header": "You must follow the instructions to predict if passengers had suvived.",
        "lift_header": "",
        "class_text": "Answer with one of the following: died | survived.",
        "instructions": "The sinking of the Titanic is one of the most infamous shipwrecks in history.\nOn April 15, 1912, during her maiden voyage, the widely considered \u201cunsinkable\u201d RMS Titanic sank after colliding with an iceberg. Unfortunately, there weren\u2019t enough lifeboats for everyone onboard, resulting in the death of 1502 out of 2224 passengers and crew.\nWhile there was some element of luck involved in surviving, it seems some groups of people were more likely to survive than others.\nIn this challenge, we ask you to build a predictive model that answers the question: \u201cwhat sorts of people were more likely to survive?\u201d using passenger data (ie name, age, gender, socio-economic class, etc).\n\nPlease answer as either `survived` or `died`.",
        "label": "died",
        "serialization": "PassengerId: 109\nPclass: 3\nName: Rekic, Mr. Tido\nSex: male\nAge: 38\nSibSp: 0\nParch: 0\nTicket: 349249\nFare: 7.9\nCabin: missing\nEmbarked: S\n"
}

from Tablet import evaluate

benchmark_path = "./data/benchmark/performance/"
tasks = [
    'titanic/prototypes-synthetic-performance-0',
]

evaluator = evaluate.Evaluator(benchmark_path=benchmark_path,
                               tasks_to_run=tasks,
                               model="chat-gpt",
                               encoding_format="gpt",
                               results_file="titanic_results.txt",  # 結果の出力先
                               k_shot=1)
evaluator.run_eval(how_many=3)  # 予測を3回生成する

df = pd.read_csv("titanic_results.txt", header=None)

k = 1
sub_df = pd.read_csv("{解凍した場所}/gender_submission.csv")
for i in range(3):
    sub_df["Survived"] = [int(j == "survived") for j in eval(df[5].iloc[i])]
    sub_df.to_csv(f"tablet_gpt_k{k}_{i}.csv", index=None)