apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-16 23:01:06 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor: streamline TikTok and Instagram scraping logic by removing search_queries and enhancing documentation for clarity

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2026-07-13 17:11:25 -07:00
parent e8b3692b54
commit 2b018c4474
111 changed files with 1800 additions and 1580 deletions
Download patch file Download diff file Expand all files Collapse all files

26
surfsense_evals/scripts/compute_adjusted_accuracy.py
View file

@ -67,14 +67,18 @@ def classify(error: str | None, raw_text: str) -> str:
def main() -> None:
rows = [
json.loads(line) for line in RAW.read_text(encoding="utf-8").splitlines()
if line.strip()
json.loads(line) for line in RAW.read_text(encoding="utf-8").splitlines() if line.strip()
]
by_arm: dict[str, dict] = defaultdict(lambda: {
"n": 0, "correct": 0,
"transient_ssl_or_5xx": 0, "transient_empty": 0,
"intrinsic_limit": 0, "other_error": 0,
})
by_arm: dict[str, dict] = defaultdict(
lambda: {
"n": 0,
"correct": 0,
"transient_ssl_or_5xx": 0,
"transient_empty": 0,
"intrinsic_limit": 0,
"other_error": 0,
}
)
for row in rows:
arm = row["arm"]
m = by_arm[arm]
@ -86,7 +90,9 @@ def main() -> None:
if kind != "ok":
m[kind] += 1
print(f"{'arm':<25} {'raw acc%':>8} {'transient':>10} {'intrinsic':>10} {'other':>6} {'adj acc% (no transient)':>22}")
print(
f"{'arm':<25} {'raw acc%':>8} {'transient':>10} {'intrinsic':>10} {'other':>6} {'adj acc% (no transient)':>22}"
)
print("-" * 88)
for arm in sorted(by_arm):
m = by_arm[arm]
@ -96,9 +102,7 @@ def main() -> None:
other = m["other_error"]
usable = m["n"] - transient
adj = m["correct"] / usable * 100 if usable else 0
print(
f"{arm:<25} {raw:>7.1f}% {transient:>10} {intrinsic:>10} {other:>6} {adj:>21.1f}%"
)
print(f"{arm:<25} {raw:>7.1f}% {transient:>10} {intrinsic:>10} {other:>6} {adj:>21.1f}%")
print()
print("transient = SSLError / 502 / 503 / empty stream / mid-stream JSON decode (would")