This commit is contained in:
Cyber MacGeddon 2024-07-17 16:55:46 +01:00
parent d0bc32892a
commit ab616b2779
5 changed files with 12 additions and 4 deletions

View file

@ -4,6 +4,9 @@ Simple decoder, accepts text documents on input, outputs chunks from the
as text as separate output objects.
"""
from langchain_text_splitters import RecursiveCharacterTextSplitter
from ... schema import TextDocument, Chunk, Source
from ... log_level import LogLevel
from ... base import ConsumerProducer

View file

@ -4,6 +4,8 @@ Simple decoder, accepts PDF documents on input, outputs pages from the
PDF document as text as separate output objects.
"""
import tempfile
import base64
from langchain_community.document_loaders import PyPDFLoader
from ... schema import Document, TextDocument, Source

View file

@ -4,6 +4,9 @@ Simple decoder, accepts vector+text chunks input, applies entity analysis to
get entity definitions which are output as graph edges.
"""
import urllib.parse
import json
from ... schema import VectorsChunk, Triple, Source, Value
from ... log_level import LogLevel
from ... llm_client import LlmClient

View file

@ -5,6 +5,8 @@ relationship analysis to get entity relationship edges which are output as
graph edges.
"""
import urllib.parse
import json
from pulsar.schema import JsonSchema
from ... schema import VectorsChunk, Triple, VectorsAssociation, Source, Value
@ -84,8 +86,6 @@ class Processor(ConsumerProducer):
chunk = v.chunk.decode("utf-8")
g = rdflib.Graph()
try:
rels = self.get_relationships(chunk)

View file

@ -37,8 +37,8 @@ class Processor(ConsumerProducer):
input_queue=input_queue,
output_queue=output_queue,
subscriber=subscriber,
input_schema=TextCompletionRequest,
output_schema=TextCompletionResponse,
input_schema=GraphRagQuery,
output_schema=GraphRagResponse,
)
self.rag = GraphRag(