apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-05-25 19:15:18 +02:00
Code Issues Projects Releases Packages Wiki Activity

chore: removed content trunking for better UI

Browse source
This commit is contained in:
DESKTOP-RTLN3BA\$punk 2025-10-14 14:19:48 -07:00
parent d86aaea125
commit 31982cea9a
9 changed files with 42 additions and 145 deletions
Download patch file Download diff file Expand all files Collapse all files

78
surfsense_backend/app/agents/researcher/nodes.py
View file

@ -71,9 +71,7 @@ def extract_sources_from_documents(
source = { source = {
"id": doc.get("chunk_id", source_id_counter), "id": doc.get("chunk_id", source_id_counter),
"title": document_info.get("title", "Untitled Document"), "title": document_info.get("title", "Untitled Document"),
"description": doc.get("content", "")[:100] + "..." "description": doc.get("content", "").strip(),
if len(doc.get("content", "")) > 100
else doc.get("content", ""),
"url": metadata.get("url", metadata.get("page_url", "")), "url": metadata.get("url", metadata.get("page_url", "")),
} }
@ -204,11 +202,7 @@ async def fetch_documents_by_ids(
title += f" ({issue_state})" title += f" ({issue_state})"
# Create description # Create description
description = ( description = doc.content
doc.content[:100] + "..."
if len(doc.content) > 100
else doc.content
)
if comment_count: if comment_count:
description += f" | Comments: {comment_count}" description += f" | Comments: {comment_count}"
@ -229,11 +223,7 @@ async def fetch_documents_by_ids(
if message_date: if message_date:
title += f" ({message_date})" title += f" ({message_date})"
description = ( description = doc.content
doc.content[:100] + "..."
if len(doc.content) > 100
else doc.content
)
url = ( url = (
f"https://slack.com/app_redirect?channel={channel_id}" f"https://slack.com/app_redirect?channel={channel_id}"
if channel_id if channel_id
@ -246,11 +236,7 @@ async def fetch_documents_by_ids(
page_id = metadata.get("page_id", "") page_id = metadata.get("page_id", "")
title = f"Notion: {page_title}" title = f"Notion: {page_title}"
description = ( description = doc.content
doc.content[:100] + "..."
if len(doc.content) > 100
else doc.content
)
url = ( url = (
f"https://notion.so/{page_id.replace('-', '')}" f"https://notion.so/{page_id.replace('-', '')}"
if page_id if page_id
@ -261,11 +247,7 @@ async def fetch_documents_by_ids(
title = f"GitHub: {doc.title}" title = f"GitHub: {doc.title}"
description = metadata.get( description = metadata.get(
"description", "description",
( (doc.content),
doc.content[:100] + "..."
if len(doc.content) > 100
else doc.content
),
) )
url = metadata.get("url", "") url = metadata.get("url", "")
@ -281,11 +263,7 @@ async def fetch_documents_by_ids(
description = metadata.get( description = metadata.get(
"description", "description",
( (doc.content),
doc.content[:100] + "..."
if len(doc.content) > 100
else doc.content
),
) )
url = ( url = (
f"https://www.youtube.com/watch?v={video_id}" f"https://www.youtube.com/watch?v={video_id}"
@ -304,11 +282,7 @@ async def fetch_documents_by_ids(
if message_date: if message_date:
title += f" ({message_date})" title += f" ({message_date})"
description = ( description = doc.content
doc.content[:100] + "..."
if len(doc.content) > 100
else doc.content
)
if guild_id and channel_id: if guild_id and channel_id:
url = f"https://discord.com/channels/{guild_id}/{channel_id}" url = f"https://discord.com/channels/{guild_id}/{channel_id}"
@ -329,11 +303,7 @@ async def fetch_documents_by_ids(
if status: if status:
title += f" ({status})" title += f" ({status})"
description = ( description = doc.content
doc.content[:100] + "..."
if len(doc.content) > 100
else doc.content
)
if priority: if priority:
description += f" | Priority: {priority}" description += f" | Priority: {priority}"
if issue_type: if issue_type:
@ -395,11 +365,7 @@ async def fetch_documents_by_ids(
except Exception: except Exception:
pass pass
description = ( description = doc.content
doc.content[:100] + "..."
if len(doc.content) > 100
else doc.content
)
if location: if location:
description += f" | Location: {location}" description += f" | Location: {location}"
if calendar_id and calendar_id != "primary": if calendar_id and calendar_id != "primary":
@ -437,11 +403,8 @@ async def fetch_documents_by_ids(
except Exception: except Exception:
pass pass
description = ( description = doc.content
doc.content[:100] + "..."
if len(doc.content) > 100
else doc.content
)
if location_name: if location_name:
description += f" | Venue: {location_name}" description += f" | Venue: {location_name}"
elif meeting_url: elif meeting_url:
@ -466,11 +429,7 @@ async def fetch_documents_by_ids(
) )
title += f" (visited: {formatted_date})" title += f" (visited: {formatted_date})"
description = ( description = doc.content
doc.content[:100] + "..."
if len(doc.content) > 100
else doc.content
)
url = webpage_url url = webpage_url
elif doc_type == "CRAWLED_URL": elif doc_type == "CRAWLED_URL":
@ -479,22 +438,15 @@ async def fetch_documents_by_ids(
"og:description", "og:description",
metadata.get( metadata.get(
"ogDescription", "ogDescription",
( (doc.content),
doc.content[:100] + "..."
if len(doc.content) > 100
else doc.content
),
), ),
) )
url = metadata.get("url", "") url = metadata.get("url", "")
else: # FILE and other types else: # FILE and other types
title = doc.title title = doc.title
description = ( description = doc.content
doc.content[:100] + "..."
if len(doc.content) > 100
else doc.content
)
url = metadata.get("url", "") url = metadata.get("url", "")
# Create source entry # Create source entry

40
surfsense_backend/app/connectors/slack_history.py
View file

@ -379,43 +379,3 @@ class SlackHistory:
formatted["user_name"] = "Unknown" formatted["user_name"] = "Unknown"
return formatted return formatted
# Example usage (uncomment to use):
"""
if __name__ == "__main__":
# Set your token here or via environment variable
token = os.environ.get("SLACK_API_TOKEN", "xoxb-your-token-here")
slack = SlackHistory(token)
# Get all channels
try:
channels = slack.get_all_channels()
print("Available channels:")
for name, channel_id in sorted(channels.items()):
print(f"- {name}: {channel_id}")
# Example: Get history for a specific channel and date range
channel_id = channels.get("general")
if channel_id:
messages, error = slack.get_history_by_date_range(
channel_id=channel_id,
start_date="2023-01-01",
end_date="2023-01-31",
limit=500
)
if error:
print(f"Error: {error}")
else:
print(f"\nRetrieved {len(messages)} messages from #general")
# Print formatted messages
for msg in messages[:10]: # Show first 10 messages
formatted = slack.format_message(msg, include_user_info=True)
print(f"[{formatted['datetime']}] {formatted['user_name']}: {formatted['text']}")
except Exception as e:
print(f"Error: {e}")
"""

49
surfsense_backend/app/services/connector_service.py
View file

@ -115,7 +115,7 @@ class ConnectorService:
"title": document.get("title", "Untitled Document"), "title": document.get("title", "Untitled Document"),
"description": metadata.get( "description": metadata.get(
"og:description", "og:description",
metadata.get("ogDescription", chunk.get("content", "")[:100]), metadata.get("ogDescription", chunk.get("content", "")),
), ),
"url": metadata.get("url", ""), "url": metadata.get("url", ""),
} }
@ -189,7 +189,7 @@ class ConnectorService:
"title": document.get("title", "Untitled Document"), "title": document.get("title", "Untitled Document"),
"description": metadata.get( "description": metadata.get(
"og:description", "og:description",
metadata.get("ogDescription", chunk.get("content", "")[:100]), metadata.get("ogDescription", chunk.get("content", "")),
), ),
"url": metadata.get("url", ""), "url": metadata.get("url", ""),
} }
@ -330,7 +330,7 @@ class ConnectorService:
source = { source = {
"id": self.source_id_counter, "id": self.source_id_counter,
"title": result.get("title", "Tavily Result"), "title": result.get("title", "Tavily Result"),
"description": result.get("content", "")[:100], "description": result.get("content", ""),
"url": result.get("url", ""), "url": result.get("url", ""),
} }
sources_list.append(source) sources_list.append(source)
@ -520,7 +520,7 @@ class ConnectorService:
for result in searx_results: for result in searx_results:
description = result.get("content") or result.get("snippet") or "" description = result.get("content") or result.get("snippet") or ""
if len(description) > 160: if len(description) > 160:
description = f"{description[:157]}..." description = f"{description}"
source = { source = {
"id": self.source_id_counter, "id": self.source_id_counter,
@ -621,9 +621,7 @@ class ConnectorService:
title += f" ({message_date})" title += f" ({message_date})"
# Create a more descriptive description for Slack messages # Create a more descriptive description for Slack messages
description = chunk.get("content", "")[:100] description = chunk.get("content", "")
if len(description) == 100:
description += "..."
# For URL, we can use a placeholder or construct a URL to the Slack channel if available # For URL, we can use a placeholder or construct a URL to the Slack channel if available
url = "" url = ""
@ -717,7 +715,7 @@ class ConnectorService:
title += f" (indexed: {indexed_at})" title += f" (indexed: {indexed_at})"
# Create a more descriptive description for Notion pages # Create a more descriptive description for Notion pages
description = chunk.get("content", "")[:100] description = chunk.get("content", "")
if len(description) == 100: if len(description) == 100:
description += "..." description += "..."
@ -829,7 +827,7 @@ class ConnectorService:
title += f" (visited: {visit_date})" title += f" (visited: {visit_date})"
# Create a more descriptive description for extension data # Create a more descriptive description for extension data
description = chunk.get("content", "")[:100] description = chunk.get("content", "")
if len(description) == 100: if len(description) == 100:
description += "..." description += "..."
@ -936,9 +934,7 @@ class ConnectorService:
title += f" - {channel_name}" title += f" - {channel_name}"
# Create a more descriptive description for YouTube videos # Create a more descriptive description for YouTube videos
description = metadata.get( description = metadata.get("description", chunk.get("content", ""))
"description", chunk.get("content", "")[:100]
)
if len(description) == 100: if len(description) == 100:
description += "..." description += "..."
@ -1024,7 +1020,7 @@ class ConnectorService:
"title", "GitHub Document" "title", "GitHub Document"
), # Use specific title if available ), # Use specific title if available
"description": metadata.get( "description": metadata.get(
"description", chunk.get("content", "")[:100] "description", chunk.get("content", "")
), # Use description or content preview ), # Use description or content preview
"url": metadata.get("url", ""), # Use URL if available in metadata "url": metadata.get("url", ""), # Use URL if available in metadata
} }
@ -1110,7 +1106,7 @@ class ConnectorService:
title += f" ({issue_state})" title += f" ({issue_state})"
# Create a more descriptive description for Linear issues # Create a more descriptive description for Linear issues
description = chunk.get("content", "")[:100] description = chunk.get("content", "")
if len(description) == 100: if len(description) == 100:
description += "..." description += "..."
@ -1222,7 +1218,7 @@ class ConnectorService:
title += f" ({status})" title += f" ({status})"
# Create a more descriptive description for Jira issues # Create a more descriptive description for Jira issues
description = chunk.get("content", "")[:100] description = chunk.get("content", "")
if len(description) == 100: if len(description) == 100:
description += "..." description += "..."
@ -1356,9 +1352,7 @@ class ConnectorService:
title += f" ({start_time})" title += f" ({start_time})"
# Create a more descriptive description for calendar events # Create a more descriptive description for calendar events
description = chunk.get("content", "")[:100] description = chunk.get("content", "")
if len(description) == 100:
description += "..."
# Add event info to description # Add event info to description
info_parts = [] info_parts = []
@ -1573,9 +1567,7 @@ class ConnectorService:
title += f" (from {sender})" title += f" (from {sender})"
# Create a more descriptive description for Gmail messages # Create a more descriptive description for Gmail messages
description = chunk.get("content", "")[:150] description = chunk.get("content", "")
if len(description) == 150:
description += "..."
# Add message info to description # Add message info to description
info_parts = [] info_parts = []
@ -1689,9 +1681,7 @@ class ConnectorService:
title += f" ({space_key})" title += f" ({space_key})"
# Create a more descriptive description for Confluence pages # Create a more descriptive description for Confluence pages
description = chunk.get("content", "")[:100] description = chunk.get("content", "")
if len(description) == 100:
description += "..."
# For URL, we can use a placeholder or construct a URL to the Confluence page if available # For URL, we can use a placeholder or construct a URL to the Confluence page if available
url = "" # TODO: Add base_url to metadata url = "" # TODO: Add base_url to metadata
@ -1908,7 +1898,7 @@ class ConnectorService:
result.name if hasattr(result, "name") else "Linkup Result" result.name if hasattr(result, "name") else "Linkup Result"
), ),
"description": ( "description": (
result.content[:100] if hasattr(result, "content") else "" result.content if hasattr(result, "content") else ""
), ),
"url": result.url if hasattr(result, "url") else "", "url": result.url if hasattr(result, "url") else "",
} }
@ -2024,9 +2014,7 @@ class ConnectorService:
title += f" ({message_date})" title += f" ({message_date})"
# Create a more descriptive description for Discord messages # Create a more descriptive description for Discord messages
description = chunk.get("content", "")[:100] description = chunk.get("content", "")
if len(description) == 100:
description += "..."
url = "" url = ""
guild_id = metadata.get("guild_id", "") guild_id = metadata.get("guild_id", "")
@ -2143,10 +2131,7 @@ class ConnectorService:
except Exception: except Exception:
title += f" ({start_time})" title += f" ({start_time})"
# Create a more descriptive description for Luma events description = chunk.get("content", "")
description = chunk.get("content", "")[:150]
if len(description) == 150:
description += "..."
# Add event info to description # Add event info to description
info_parts = [] info_parts = []

2
surfsense_backend/app/services/streaming_service.py
View file

@ -66,7 +66,7 @@ class StreamingService:
for source in group.get("sources", []): for source in group.get("sources", []):
node = { node = {
"id": str(source.get("id", "")), "id": str(source.get("id", "")),
"text": source.get("description", ""), "text": source.get("description", "").strip(),
"url": source.get("url", ""), "url": source.get("url", ""),
"metadata": { "metadata": {
"title": source.get("title", ""), "title": source.get("title", ""),

4
surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py
View file

@ -260,8 +260,8 @@ async def index_confluence_pages(
) )
if page_content: if page_content:
# Take first 500 characters of content for summary # Take first 500 characters of content for summary
content_preview = page_content[:500] content_preview = page_content[:1000]
if len(page_content) > 500: if len(page_content) > 1000:
content_preview += "..." content_preview += "..."
summary_content += f"Content Preview: {content_preview}\n\n" summary_content += f"Content Preview: {content_preview}\n\n"
summary_content += f"Comments: {comment_count}" summary_content += f"Comments: {comment_count}"

4
surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py
View file

@ -296,8 +296,8 @@ async def index_google_calendar_events(
if location: if location:
summary_content += f"Location: {location}\n" summary_content += f"Location: {location}\n"
if description: if description:
desc_preview = description[:300] desc_preview = description[:1000]
if len(description) > 300: if len(description) > 1000:
desc_preview += "..." desc_preview += "..."
summary_content += f"Description: {desc_preview}\n" summary_content += f"Description: {desc_preview}\n"
summary_embedding = config.embedding_model_instance.embed( summary_embedding = config.embedding_model_instance.embed(

4
surfsense_backend/app/tasks/connector_indexers/linear_indexer.py
View file

@ -254,8 +254,8 @@ async def index_linear_issues(
else: else:
# Fallback to simple summary if no LLM configured # Fallback to simple summary if no LLM configured
# Truncate description if it's too long for the summary # Truncate description if it's too long for the summary
if description and len(description) > 500: if description and len(description) > 1000:
description = description[:497] + "..." description = description[:997] + "..."
summary_content = f"Linear Issue {issue_identifier}: {issue_title}\n\nStatus: {state}\n\n" summary_content = f"Linear Issue {issue_identifier}: {issue_title}\n\nStatus: {state}\n\n"
if description: if description:
summary_content += f"Description: {description}\n\n" summary_content += f"Description: {description}\n\n"

4
surfsense_backend/app/tasks/connector_indexers/luma_indexer.py
View file

@ -310,8 +310,8 @@ async def index_luma_events(
if host_names: if host_names:
summary_content += f"Hosts: {host_names}\n" summary_content += f"Hosts: {host_names}\n"
if description: if description:
desc_preview = description[:300] desc_preview = description[:1000]
if len(description) > 300: if len(description) > 1000:
desc_preview += "..." desc_preview += "..."
summary_content += f"Description: {desc_preview}\n" summary_content += f"Description: {desc_preview}\n"

2
surfsense_web/components/homepage/navbar.tsx
View file

@ -86,7 +86,7 @@ const DesktopNav = ({ navItems, isScrolled }: any) => {
className="hidden rounded-full px-3 py-2 hover:bg-gray-100 dark:hover:bg-neutral-800 transition-colors md:flex items-center gap-1.5" className="hidden rounded-full px-3 py-2 hover:bg-gray-100 dark:hover:bg-neutral-800 transition-colors md:flex items-center gap-1.5"
> >
<IconBrandGithub className="h-5 w-5 text-neutral-600 dark:text-neutral-300" /> <IconBrandGithub className="h-5 w-5 text-neutral-600 dark:text-neutral-300" />
<span className="text-sm font-medium text-neutral-600 dark:text-neutral-300">8.3k</span> <span className="text-sm font-medium text-neutral-600 dark:text-neutral-300">9.5k</span>
</Link> </Link>
<ThemeTogglerComponent /> <ThemeTogglerComponent />
<Link <Link