apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-05-01 20:03:28 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix hfdataset; make dirs when save notebook

Browse source
This commit is contained in:
Yizhou Chi 2024-09-10 13:51:54 +08:00
parent 294d0fe709
commit 60e8e3eab8
2 changed files with 7 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

9
expo/data/hf_data.py
View file

@ -38,18 +38,21 @@ class HFExpDataset(ExpDataset):
df = pd.read_csv(Path(raw_dir, "train.csv"))
else:
df = self.dataset["train"].to_pandas()
df.to_csv(Path(raw_dir, "train.csv"))
df.to_csv(Path(raw_dir, "train.csv"), index=False)
if os.path.exists(Path(raw_dir, "test.csv")):
test_df = pd.read_csv(Path(raw_dir, "test.csv"))
test_df = pd.read_csv(Path(raw_dir, "test.csv"), index=False)
else:
if "test" in self.dataset:
test_df = self.dataset["test"].to_pandas()
test_df.to_csv(Path(raw_dir, "test.csv"))
test_df.to_csv(Path(raw_dir, "test.csv"), index=False)
else:
test_df = None
return df, test_df
# def get_df_head(self, raw_df):
# return raw_df.head()
if __name__ == "__main__":
dataset_dir = "D:/work/automl/datasets"