feat: Added Search Space System Instructions

- Added `citations_enabled` and `qna_custom_instructions` fields to the SearchSpace model for better QnA configuration.
- Updated the creation and update schemas to handle new fields with appropriate defaults.
- Refactored QnA handling in the agent to utilize the new SearchSpace fields for improved response customization.
- Adjusted UI components to include settings for managing QnA configurations.
- Enhanced onboarding process to incorporate prompt setup as an optional step.

surfsense_backend/app/agents/researcher/qna_agent/nodes.py

@@ -1,8 +1,11 @@
+import datetime
 from typing import Any
 
 from langchain_core.messages import HumanMessage, SystemMessage
 from langchain_core.runnables import RunnableConfig
+from sqlalchemy import select
 
+from app.db import SearchSpace
 from app.services.reranker_service import RerankerService
 
 from ..utils import (
@@ -12,10 +15,53 @@ from ..utils import (
     optimize_documents_for_token_limit,
 )
 from .configuration import Configuration
-from .prompts import get_qna_citation_system_prompt, get_qna_no_documents_system_prompt
+from .default_prompts import (
+    DEFAULT_QNA_BASE_PROMPT,
+    DEFAULT_QNA_CITATION_INSTRUCTIONS,
+    DEFAULT_QNA_NO_DOCUMENTS_PROMPT,
+)
 from .state import State
 
 
+def _build_language_instruction(language: str | None = None):
+    """Build language instruction for prompts."""
+    if language:
+        return f"\n\nIMPORTANT: Please respond in {language} language. All your responses, explanations, and analysis should be written in {language}."
+    return ""
+
+
+def _build_chat_history_section(chat_history: str | None = None):
+    """Build chat history section for prompts."""
+    if chat_history:
+        return f"""
+<chat_history>
+{chat_history if chat_history else "NO CHAT HISTORY PROVIDED"}
+</chat_history>
+"""
+    return """
+<chat_history>
+NO CHAT HISTORY PROVIDED
+</chat_history>
+"""
+
+
+def _format_system_prompt(
+    prompt_template: str,
+    chat_history: str | None = None,
+    language: str | None = None,
+):
+    """Format a system prompt template with dynamic values."""
+    date = datetime.datetime.now().strftime("%Y-%m-%d")
+    language_instruction = _build_language_instruction(language)
+    chat_history_section = _build_chat_history_section(chat_history)
+
+    return prompt_template.format(
+        date=date,
+        language_instruction=language_instruction,
+        chat_history_section=chat_history_section,
+    )
+
+
 async def rerank_documents(state: State, config: RunnableConfig) -> dict[str, Any]:
     """
     Rerank the documents based on relevance to the user's question.
@@ -105,6 +151,33 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any
     user_id = configuration.user_id
     search_space_id = configuration.search_space_id
     language = configuration.language
+
+    # Fetch search space to get QnA configuration
+    result = await state.db_session.execute(
+        select(SearchSpace).where(SearchSpace.id == search_space_id)
+    )
+    search_space = result.scalar_one_or_none()
+
+    if not search_space:
+        error_message = f"Search space {search_space_id} not found"
+        print(error_message)
+        raise RuntimeError(error_message)
+
+    # Get QnA configuration from search space
+    citations_enabled = search_space.citations_enabled
+    custom_instructions_text = search_space.qna_custom_instructions or ""
+
+    # Use constants for base prompt and citation instructions
+    qna_base_prompt = DEFAULT_QNA_BASE_PROMPT
+    qna_citation_instructions = (
+        DEFAULT_QNA_CITATION_INSTRUCTIONS if citations_enabled else ""
+    )
+    qna_custom_instructions = (
+        f"\n<special_important_custom_instructions>\n{custom_instructions_text}\n</special_important_custom_instructions>"
+        if custom_instructions_text
+        else ""
+    )
+
     # Get user's fast LLM
     llm = await get_user_fast_llm(state.db_session, user_id, search_space_id)
     if not llm:
@@ -117,6 +190,11 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any
     chat_history_str = langchain_chat_history_to_str(state.chat_history)
 
     if has_documents_initially:
+        # Compose the full citation prompt: base + citation instructions + custom instructions
+        full_citation_prompt_template = (
+            qna_base_prompt + qna_citation_instructions + qna_custom_instructions
+        )
+
         # Create base message template for token calculation (without documents)
         base_human_message_template = f"""
         
@@ -129,8 +207,8 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any
         """
 
         # Use initial system prompt for token calculation
-        initial_system_prompt = get_qna_citation_system_prompt(
-            chat_history_str, language
+        initial_system_prompt = _format_system_prompt(
+            full_citation_prompt_template, chat_history_str, language
         )
         base_messages = [
             SystemMessage(content=initial_system_prompt),
@@ -149,11 +227,21 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any
         has_documents = False
 
     # Choose system prompt based on final document availability
-    system_prompt = (
-        get_qna_citation_system_prompt(chat_history_str, language)
-        if has_documents
-        else get_qna_no_documents_system_prompt(chat_history_str, language)
-    )
+    # With documents: use base + citation instructions + custom instructions
+    # Without documents: use the default no-documents prompt from constants
+    if has_documents:
+        full_citation_prompt_template = (
+            qna_base_prompt + qna_citation_instructions + qna_custom_instructions
+        )
+        system_prompt = _format_system_prompt(
+            full_citation_prompt_template, chat_history_str, language
+        )
+    else:
+        system_prompt = _format_system_prompt(
+            DEFAULT_QNA_NO_DOCUMENTS_PROMPT + qna_custom_instructions,
+            chat_history_str,
+            language,
+        )
 
     # Generate documents section
     documents_text = (