diff --git a/trustgraph/model/prompt/generic/prompts.py b/trustgraph/model/prompt/generic/prompts.py index 74c80fd7..c16afc89 100644 --- a/trustgraph/model/prompt/generic/prompts.py +++ b/trustgraph/model/prompt/generic/prompts.py @@ -3,64 +3,90 @@ def to_relationships(text): prompt = f"""You are a helpful assistant that performs information extraction tasks for a provided text. -Read the provided text. You will model the text as an information network for a RDF knowledge graph. +Read the provided text. You will model the text as an information network for a RDF knowledge graph in JSON. -Information network rules: +Information Network Rules: - An information network has subjects connected by predicates to objects. -- A subject can have many predicates and objects. +- A subject is a named-entity or a conceptual topic. +- One subject can have many predicates and objects. +- An object is a property or attribute of a subject. - A subject can be connected by a predicate to another subject. -- Objects shall be either nouns or adjectives. -Here is the provided text: +Reading Instructions: +- Ignore document formatting in the provided text. +- Study the provided text carefully. + +Here is the text: {text} -Instructions: -- Obey the information network rules. -- Ignore document formatting. -- Do not provide explanations or any additional text. -- Do not use special characters. -- The key "object-entity" is true if it is a Named-Entity. -- Respond only with a well-formed JSON using the following example: +Response Instructions: +- Obey the information network rules. +- Do not return special characters. +- Respond only with well-formed JSON. +- The JSON response shall be an array of JSON objects with keys "subject", "predicate", "object", and "object-entity". +- The JSON response shall use the following structure: -JSON example: [{{"subject": string, "predicate": string, "object": string, "object-entity": boolean}}] +```json +[{{"subject": string, "predicate": string, "object": string, "object-entity": boolean}}] +``` + +- The key "object-entity" is TRUE only if the "object" is a subject. +- Do not write any additional text or explanations. """ return prompt def to_topics(text): - prompt = f"""You are a helpful assistant that performs information extraction tasks for a provided text.\nRead the provided text. You will identify topics and their definitions. + prompt = f"""You are a helpful assistant that performs information extraction tasks for a provided text.\nRead the provided text. You will identify topics and their definitions in JSON. -Here is the provided text: +Reading Instructions: +- Ignore document formatting in the provided text. +- Study the provided text carefully. + +Here is the text: {text} -Instructions: -- Ignore document formatting. -- Do not provide explanations or any additional text. -- Do not use special characters. -- Identify only topics that are unique to the provided text. -- Respond only with a well-formed JSON using the following example: +Response Instructions: +- Do not respond with special characters. +- Return only topics that are concepts and unique to the provided text. +- Respond only with well-formed JSON. +- The JSON response shall be an array of objects with keys "topic" and "definition". +- The JSON response shall use the following structure: -JSON example: [{{"topic": string, "definition": string}}] +```json +[{{"topic": string, "definition": string}}] +``` + +- Do not write any additional text or explanations. """ return prompt def to_definitions(text): - prompt = f"""You are a helpful assistant that performs information extraction tasks for a provided text.\nRead the provided text. You will identify named-entities and their definitions. + prompt = f"""You are a helpful assistant that performs information extraction tasks for a provided text.\nRead the provided text. You will identify entities and their definitions in JSON. -Here is the provided text: +Reading Instructions: +- Ignore document formatting in the provided text. +- Study the provided text carefully. + +Here is the text: {text} -Instructions: -- Ignore document formatting. -- Do not provide explanations or any additional text. -- Do not use special characters. -- Identity only entities that are named-entities. -- Respond only with a well-formed JSON using the following example: +Response Instructions: +- Do not respond with special characters. +- Return only entities that are named-entities such as: people, organizations, physical objects, locations, animals, products, commodotities, or substances. +- Respond only with well-formed JSON. +- The JSON response shall be an array of objects with keys "entity" and "definition". +- The JSON response shall use the following structure: -JSON example: [{{"entity": string, "definition": string}}]""" +```json +[{{"entity": string, "definition": string}}] +``` + +- Do not write any additional text or explanations. +""" return prompt diff --git a/trustgraph/model/prompt/generic/service.py b/trustgraph/model/prompt/generic/service.py index a6800b7e..dc75c7cf 100755 --- a/trustgraph/model/prompt/generic/service.py +++ b/trustgraph/model/prompt/generic/service.py @@ -3,6 +3,7 @@ Language service abstracts prompt engineering from LLM. """ import json +import re from .... schema import Definition, Relationship, Triple from .... schema import PromptRequest, PromptResponse, Error @@ -56,12 +57,15 @@ class Processor(ConsumerProducer): ) def parse_json(self, text): - - # Hacky, workaround temperamental JSON markdown - text = text.replace("```json", "") - text = text.replace("```", "") + json_match = re.search(r'```(?:json)?(.*?)```', text, re.DOTALL) + + if json_match: + json_str = json_match.group(1).strip() + else: + # If no delimiters, assume the entire output is JSON + json_str = text.strip() - return json.loads(text) + return json.loads(json_str) def handle(self, msg): diff --git a/trustgraph/model/prompt/template/prompts.py b/trustgraph/model/prompt/template/prompts.py index 8b25c621..e3148157 100644 --- a/trustgraph/model/prompt/template/prompts.py +++ b/trustgraph/model/prompt/template/prompts.py @@ -5,6 +5,9 @@ def to_relationships(template, text): def to_definitions(template, text): return template.format(text=text) +def to_topics(template, text): + return template.format(text=text) + def to_rows(template, schema, text): field_schema = [ diff --git a/trustgraph/model/prompt/template/service.py b/trustgraph/model/prompt/template/service.py index ce595720..fdfa2740 100755 --- a/trustgraph/model/prompt/template/service.py +++ b/trustgraph/model/prompt/template/service.py @@ -4,6 +4,7 @@ Language service abstracts prompt engineering from LLM. """ import json +import re from .... schema import Definition, Relationship, Triple from .... schema import PromptRequest, PromptResponse, Error @@ -15,7 +16,7 @@ from .... base import ConsumerProducer from .... clients.llm_client import LlmClient from . prompts import to_definitions, to_relationships, to_rows -from . prompts import to_kg_query, to_document_query +from . prompts import to_kg_query, to_document_query, to_topics module = ".".join(__name__.split(".")[1:-1]) @@ -38,6 +39,7 @@ class Processor(ConsumerProducer): ) definition_template = params.get("definition_template") relationship_template = params.get("relationship_template") + topic_template = params.get("topic_template") rows_template = params.get("rows_template") knowledge_query_template = params.get("knowledge_query_template") document_query_template = params.get("document_query_template") @@ -62,18 +64,22 @@ class Processor(ConsumerProducer): ) self.definition_template = definition_template + self.topic_template = topic_template self.relationship_template = relationship_template self.rows_template = rows_template self.knowledge_query_template = knowledge_query_template self.document_query_template = document_query_template def parse_json(self, text): - - # Hacky, workaround temperamental JSON markdown - text = text.replace("```json", "") - text = text.replace("```", "") + json_match = re.search(r'```(?:json)?(.*?)```', text, re.DOTALL) + + if json_match: + json_str = json_match.group(1).strip() + else: + # If no delimiters, assume the entire output is JSON + json_str = text.strip() - return json.loads(text) + return json.loads(json_str) def handle(self, msg): @@ -92,6 +98,11 @@ class Processor(ConsumerProducer): self.handle_extract_definitions(id, v) return + elif kind == "extract-topics": + + self.handle_extract_topics(id, v) + return + elif kind == "extract-relationships": self.handle_extract_relationships(id, v) @@ -176,6 +187,66 @@ class Processor(ConsumerProducer): self.producer.send(r, properties={"id": id}) + def handle_extract_topics(self, id, v): + + try: + + prompt = to_topics(self.topic_template, v.chunk) + + ans = self.llm.request(prompt) + + # Silently ignore JSON parse error + try: + defs = self.parse_json(ans) + except: + print("JSON parse error, ignored", flush=True) + defs = [] + + output = [] + + for defn in defs: + + try: + e = defn["topic"] + d = defn["definition"] + + if e == "": continue + if e is None: continue + if d == "": continue + if d is None: continue + + output.append( + Definition( + name=e, definition=d + ) + ) + + except: + print("definition fields missing, ignored", flush=True) + + print("Send response...", flush=True) + r = PromptResponse(topics=output, error=None) + self.producer.send(r, properties={"id": id}) + + print("Done.", flush=True) + + except Exception as e: + + print(f"Exception: {e}") + + print("Send error response...", flush=True) + + r = PromptResponse( + error=Error( + type = "llm-error", + message = str(e), + ), + response=None, + ) + + self.producer.send(r, properties={"id": id}) + + def handle_extract_relationships(self, id, v): try: @@ -415,6 +486,12 @@ class Processor(ConsumerProducer): help=f'Definition extraction template', ) + parser.add_argument( + '--topic-template', + required=True, + help=f'Topic extraction template', + ) + parser.add_argument( '--rows-template', required=True,