diff --git a/trustgraph-flow/trustgraph/model/text_completion/azure/llm.py b/trustgraph-flow/trustgraph/model/text_completion/azure/llm.py index af3dfcaf..d2d6f1ad 100755 --- a/trustgraph-flow/trustgraph/model/text_completion/azure/llm.py +++ b/trustgraph-flow/trustgraph/model/text_completion/azure/llm.py @@ -55,7 +55,9 @@ class Processor(LlmService): self.max_output = max_output self.default_model = model - def build_prompt(self, system, content): + def build_prompt(self, system, content, temperature=None): + # Use provided temperature or fall back to default + effective_temperature = temperature if temperature is not None else self.temperature data = { "messages": [ @@ -67,7 +69,7 @@ class Processor(LlmService): } ], "max_tokens": self.max_output, - "temperature": self.temperature, + "temperature": effective_temperature, "top_p": 1 } @@ -100,18 +102,22 @@ class Processor(LlmService): return result - async def generate_content(self, system, prompt, model=None): + async def generate_content(self, system, prompt, model=None, temperature=None): # Use provided model or fall back to default model_name = model or self.default_model + # Use provided temperature or fall back to default + effective_temperature = temperature if temperature is not None else self.temperature logger.debug(f"Using model: {model_name}") + logger.debug(f"Using temperature: {effective_temperature}") try: prompt = self.build_prompt( system, - prompt + prompt, + effective_temperature ) response = self.call_llm(prompt) diff --git a/trustgraph-flow/trustgraph/model/text_completion/azure_openai/llm.py b/trustgraph-flow/trustgraph/model/text_completion/azure_openai/llm.py index f0588e5a..2442c283 100755 --- a/trustgraph-flow/trustgraph/model/text_completion/azure_openai/llm.py +++ b/trustgraph-flow/trustgraph/model/text_completion/azure_openai/llm.py @@ -62,12 +62,15 @@ class Processor(LlmService): azure_endpoint = endpoint, ) - async def generate_content(self, system, prompt, model=None): + async def generate_content(self, system, prompt, model=None, temperature=None): # Use provided model or fall back to default model_name = model or self.default_model + # Use provided temperature or fall back to default + effective_temperature = temperature if temperature is not None else self.temperature logger.debug(f"Using model: {model_name}") + logger.debug(f"Using temperature: {effective_temperature}") prompt = system + "\n\n" + prompt @@ -86,7 +89,7 @@ class Processor(LlmService): ] } ], - temperature=self.temperature, + temperature=effective_temperature, max_tokens=self.max_output, top_p=1, ) diff --git a/trustgraph-flow/trustgraph/model/text_completion/googleaistudio/llm.py b/trustgraph-flow/trustgraph/model/text_completion/googleaistudio/llm.py index c1814129..b9abcefa 100644 --- a/trustgraph-flow/trustgraph/model/text_completion/googleaistudio/llm.py +++ b/trustgraph-flow/trustgraph/model/text_completion/googleaistudio/llm.py @@ -86,12 +86,18 @@ class Processor(LlmService): logger.info("GoogleAIStudio LLM service initialized") - def _get_or_create_config(self, model_name): - """Get cached generation config or create new one""" - if model_name not in self.generation_configs: - logger.info(f"Creating generation config for '{model_name}'") - self.generation_configs[model_name] = types.GenerateContentConfig( - temperature = self.temperature, + def _get_or_create_config(self, model_name, temperature=None): + """Get or create generation config with dynamic temperature""" + # Use provided temperature or fall back to default + effective_temperature = temperature if temperature is not None else self.temperature + + # Create cache key that includes temperature to avoid conflicts + cache_key = f"{model_name}:{effective_temperature}" + + if cache_key not in self.generation_configs: + logger.info(f"Creating generation config for '{model_name}' with temperature {effective_temperature}") + self.generation_configs[cache_key] = types.GenerateContentConfig( + temperature = effective_temperature, top_p = 1, top_k = 40, max_output_tokens = self.max_output, @@ -99,16 +105,19 @@ class Processor(LlmService): safety_settings = self.safety_settings, ) - return self.generation_configs[model_name] + return self.generation_configs[cache_key] - async def generate_content(self, system, prompt, model=None): + async def generate_content(self, system, prompt, model=None, temperature=None): # Use provided model or fall back to default model_name = model or self.default_model + # Use provided temperature or fall back to default + effective_temperature = temperature if temperature is not None else self.temperature logger.debug(f"Using model: {model_name}") + logger.debug(f"Using temperature: {effective_temperature}") - generation_config = self._get_or_create_config(model_name) + generation_config = self._get_or_create_config(model_name, effective_temperature) # Set system instruction per request (can't be cached) generation_config.system_instruction = system diff --git a/trustgraph-flow/trustgraph/model/text_completion/vllm/llm.py b/trustgraph-flow/trustgraph/model/text_completion/vllm/llm.py index 71b77d74..1cf7df49 100755 --- a/trustgraph-flow/trustgraph/model/text_completion/vllm/llm.py +++ b/trustgraph-flow/trustgraph/model/text_completion/vllm/llm.py @@ -52,12 +52,15 @@ class Processor(LlmService): logger.info(f"Using vLLM service at {base_url}") logger.info("vLLM LLM service initialized") - async def generate_content(self, system, prompt, model=None): + async def generate_content(self, system, prompt, model=None, temperature=None): # Use provided model or fall back to default model_name = model or self.default_model + # Use provided temperature or fall back to default + effective_temperature = temperature if temperature is not None else self.temperature logger.debug(f"Using model: {model_name}") + logger.debug(f"Using temperature: {effective_temperature}") headers = { "Content-Type": "application/json", @@ -67,7 +70,7 @@ class Processor(LlmService): "model": model_name, "prompt": system + "\n\n" + prompt, "max_tokens": self.max_output, - "temperature": self.temperature, + "temperature": effective_temperature, } try: