apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

more robust gcp watcher

Browse source
This commit is contained in:
51616 2025-08-11 16:50:42 +09:00
parent f209c8b9a1
commit 537ab4bb65
1 changed files with 19 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

22
gcp_bucket_watcher.py
View file

@ -127,27 +127,43 @@ def sync_cycle(args: argparse.Namespace, client: storage.Client) -> int:
for rel, meta in local.items():
if should_skip(rel, includes, excludes):
continue
if needs_transfer(meta, remote.get(rel), tolerance=2.0):
remote_meta = remote.get(rel)
# For uploads we consider a file up-to-date if sizes match and the remote
# object timestamp is >= local mtime (within tolerance). Using absolute
# delta caused perpetual re-uploads because GCS sets its own server time.
if (
remote_meta
and meta.size == remote_meta.size
and remote_meta.updated >= meta.updated - 2.0
):
continue
if needs_transfer(meta, remote_meta, tolerance=2.0):
print(f"[uploading ] {rel} ({meta.size} bytes)")
try:
upload_file(client, args.bucket, args.prefix, meta, args.dest)
except FileNotFoundError:
# File disappeared between scan and upload attempt; skip gracefully.
print(f"[skipped missing] {rel}")
continue
print(f"[uploaded] {rel} ({meta.size} bytes)")
print(f"[uploaded ✓] {rel} ({meta.size} bytes)")
changed += 1
if changed == 0:
print("No new or updated local files to upload.")
else:
print("All uploads complete.")
else: # download (original tolerance 1s)
for rel, meta in remote.items():
if should_skip(rel, includes, excludes):
continue
if needs_transfer(meta, local.get(rel), tolerance=1.0):
print(f"[downloading] {rel} ({meta.size} bytes)")
download_file(client, args.bucket, args.prefix, meta, args.dest)
print(f"[downloaded] {rel} ({meta.size} bytes)")
print(f"[downloaded ✓] {rel} ({meta.size} bytes)")
changed += 1
if changed == 0:
print("No new or updated objects.")
else:
print("All downloads complete.")
return changed