Parquet split

This commit is contained in:
Cyber MacGeddon 2024-09-30 17:54:12 +01:00
parent e0866d9f92
commit 012b6f7672
21 changed files with 55 additions and 7 deletions

View file

@ -51,7 +51,6 @@ setuptools.setup(
"anthropic",
"pyyaml",
"prometheus-client",
"pyarrow",
"cohere",
"openai",
"neo4j",
@ -60,16 +59,13 @@ setuptools.setup(
scripts=[
"scripts/chunker-recursive",
"scripts/chunker-token",
"scripts/concat-parquet",
"scripts/de-query-milvus",
"scripts/de-query-qdrant",
"scripts/de-write-milvus",
"scripts/de-write-qdrant",
"scripts/document-rag",
"scripts/dump-parquet",
"scripts/embeddings-ollama",
"scripts/embeddings-vectorize",
"scripts/ge-dump-parquet",
"scripts/ge-query-milvus",
"scripts/ge-query-qdrant",
"scripts/ge-write-milvus",
@ -81,10 +77,8 @@ setuptools.setup(
"scripts/kg-extract-definitions",
"scripts/kg-extract-topics",
"scripts/kg-extract-relationships",
"scripts/load-graph-embeddings",
"scripts/load-pdf",
"scripts/load-text",
"scripts/load-triples",
"scripts/metering",
"scripts/object-extract-row",
"scripts/oe-write-milvus",
@ -103,7 +97,6 @@ setuptools.setup(
"scripts/text-completion-openai",
"scripts/tg-init-pulsar",
"scripts/tg-processor-state",
"scripts/triples-dump-parquet",
"scripts/triples-query-cassandra",
"scripts/triples-query-neo4j",
"scripts/triples-write-cassandra",