apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-24 23:41:10 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix: parse indeed /viewjob detail from window._rootProps

Browse source
This commit is contained in:
CREDO23 2026-07-15 19:57:19 +02:00
parent 2a3d750cbe
commit b53f00231d
2 changed files with 47 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

32
surfsense_backend/app/proprietary/platforms/indeed_jobs/parsers.py
View file

@ -22,7 +22,10 @@ _DEFAULT_BASE = "https://www.indeed.com"
_JOBCARDS_ANCHOR = 'window.mosaic.providerData["mosaic-provider-jobcards"]='
# A /viewjob page assigns the full posting to this global.
# A /viewjob page carries the posting model in ``window._rootProps`` (JSON,
# under ``preloadedVJData``); older pages inlined it as ``window._initialData``.
_ROOT_PROPS_ANCHOR = "window._rootProps"
_ROOT_PROPS_KEY = "preloadedVJData"
_INITIAL_DATA_ANCHOR = "window._initialData"
# Indeed's extractedSalary.type -> our SalaryPeriod.
@ -82,14 +85,14 @@ def extract_jobcards_blob(html: str) -> dict | None:
return data if isinstance(data, dict) else None
def extract_initial_data(html: str) -> dict | None:
"""Decode the ``window._initialData`` assignment on a /viewjob page."""
def _decode_assignment(html: str, anchor: str) -> dict | None:
"""Decode the balanced JSON object assigned after ``anchor``, or ``None``."""
import json
idx = html.find(_INITIAL_DATA_ANCHOR)
idx = html.find(anchor)
if idx == -1:
return None
brace = html.find("{", idx + len(_INITIAL_DATA_ANCHOR))
brace = html.find("{", idx + len(anchor))
if brace == -1:
return None
blob = _brace_match(html, brace)
@ -102,6 +105,25 @@ def extract_initial_data(html: str) -> dict | None:
return data if isinstance(data, dict) else None
def extract_initial_data(html: str) -> dict | None:
"""Return a /viewjob posting model rooted at ``jobInfoWrapperModel``.
Prefers ``window._rootProps`` (JSON) unwrapped at ``preloadedVJData``; falls
back to a legacy inline ``window._initialData`` blob. ``window._initialData``
is now a JS object literal that references other globals, so it is not JSON
and is skipped when the JSON parse fails.
"""
root = _decode_assignment(html, _ROOT_PROPS_ANCHOR)
if isinstance(root, dict):
vj = root.get(_ROOT_PROPS_KEY)
if isinstance(vj, dict) and vj.get("jobInfoWrapperModel"):
return vj
legacy = _decode_assignment(html, _INITIAL_DATA_ANCHOR)
if isinstance(legacy, dict) and legacy.get("jobInfoWrapperModel"):
return legacy
return None
def job_results(blob: dict | None) -> list[dict[str, Any]]:
"""Return the raw job records from a decoded blob."""
if not isinstance(blob, dict):

20
surfsense_backend/tests/unit/platforms/indeed_jobs/test_parsers.py
View file

@ -160,6 +160,26 @@ def test_parse_job_detail_extracts_description_and_fields():
assert (sal["salaryMin"], sal["salaryMax"], sal["period"]) == (60000, 90000, "year")
def test_parse_job_detail_reads_rootprops_shape():
# Live /viewjob assigns window._rootProps (JSON) with the model under
# preloadedVJData; window._initialData is now a non-JSON JS literal that
# references other globals and must be skipped, not misparsed.
html = (
"<html><script>window._initialData = { ssr: true, "
"viewJobClientSideModel: window._rootProps.preloadedVJData };</script>"
'<script>window._rootProps = {"url":"x","preloadedVJData":'
'{"jobInfoWrapperModel":{"jobInfoModel":'
'{"sanitizedJobDescription":"<p>Full JD text</p>",'
'"jobInfoHeaderModel":{"jobTitle":"Data Analyst",'
'"companyName":"Acme"}}}}};</script></html>'
)
detail = parse_job_detail(html)
assert detail["title"] == "Data Analyst"
assert detail["company"] == "Acme"
assert detail["descriptionHtml"] == "<p>Full JD text</p>"
assert detail["descriptionText"] == "Full JD text"
def test_parse_job_detail_omits_blank_fields():
# A page with a header but no salary/description must not emit those keys,
# so a merge won't clobber listing values with blanks.