diff --git a/README.md b/README.md index 1e6bf5a..e6b5bc8 100644 --- a/README.md +++ b/README.md @@ -77,15 +77,25 @@ HF_HUB_ENABLE_HF_TRANSFER=1 huggingface-cli download --repo-type dataset rajpurk gsutil -m rsync -r data/raw_datasets/self_gen gs://ctx-to-lora/data/raw_datasets/self_gen # downloading from gcp bucket to login node -gsutil -m rsync -r gs://ctx-to-lora/data/raw_datasets/self_gen data/raw_datasets/self_gen +mkdir -p data/raw_datasets/self_gen +gsutil -m rsync -r gs://ctx-to-lora/data/raw_datasets/self_gen data/raw_datasets/self_gen ``` -### Data Generation (v2) +### Upload/download checkpoints +```bash +# upload to bucket +gsutil -m rsync -r train_outputs gs://ctx-to-lora/train_outputs + +# download from bucket +gsutil -m rsync -r gs://ctx-to-lora/train_outputs train_outputs +``` + + Loading self-generated data ```python diff --git a/gcp_bucket_watcher.py b/gcp_bucket_watcher.py index 835291c..c44a231 100644 --- a/gcp_bucket_watcher.py +++ b/gcp_bucket_watcher.py @@ -128,7 +128,12 @@ def sync_cycle(args: argparse.Namespace, client: storage.Client) -> int: if should_skip(rel, includes, excludes): continue if needs_transfer(meta, remote.get(rel), tolerance=2.0): - upload_file(client, args.bucket, args.prefix, meta, args.dest) + try: + upload_file(client, args.bucket, args.prefix, meta, args.dest) + except FileNotFoundError: + # File disappeared between scan and upload attempt; skip gracefully. + print(f"[skipped missing] {rel}") + continue print(f"[uploaded] {rel} ({meta.size} bytes)") changed += 1 if changed == 0: