Forming embeddings-hf package

Makefile

@@ -8,6 +8,7 @@ all: pkgs
 
 pkgs:
 	pip3 wheel --no-deps --wheel-dir dist trustgraph-core/
+	pip3 wheel --no-deps --wheel-dir dist trustgraph-embeddings-hf/
 
 # container
 

trustgraph-core/scripts/embeddings-hf

@@ -1,6 +0,0 @@
-#!/usr/bin/env python3
-
-from trustgraph.core.embeddings.hf import run
-
-run()
-

trustgraph-core/setup.py

@@ -27,18 +27,13 @@ setuptools.setup(
     python_requires='>=3.8',
     download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
     install_requires=[
-        "torch",
         "urllib3",
-        "transformers",
-        "sentence-transformers",
         "rdflib",
         "pymilvus",
         "langchain",
         "langchain-core",
-        "langchain-huggingface",
         "langchain-text-splitters",
         "langchain-community",
-        "huggingface-hub",
         "requests",
         "cassandra-driver",
         "pulsar-client",
@@ -66,7 +61,6 @@ setuptools.setup(
         "scripts/de-write-qdrant",
         "scripts/document-rag",
         "scripts/dump-parquet",
-        "scripts/embeddings-hf",
         "scripts/embeddings-ollama",
         "scripts/embeddings-vectorize",
         "scripts/ge-dump-parquet",

trustgraph-embeddings-hf/scripts/embeddings-hf

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.embeddings_hf.embeddings.hf import run
+
+run()
+

trustgraph-embeddings-hf/setup.py

@@ -0,0 +1,47 @@
+import setuptools
+import os
+
+with open("../README.md", "r") as fh:
+    long_description = fh.read()
+
+version = "0.11.0"
+
+setuptools.setup(
+    name="trustgraph-embeddings-hf",
+    version=version,
+    author="trustgraph.ai",
+    author_email="security@trustgraph.ai",
+    description="HuggingFace embeddings support for TrustGraph.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/trustgraph-ai/trustgraph",
+    packages=setuptools.find_namespace_packages(
+        where='./',
+#         include=['trustgraph.core']
+    ),
+    classifiers=[ 
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+        "Operating System :: OS Independent",
+    ],
+    python_requires='>=3.8',
+    download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
+    install_requires=[
+        "trustgraph-core",
+        "torch",
+        "urllib3",
+        "transformers",
+        "sentence-transformers",
+        "langchain",
+        "langchain-core",
+        "langchain-huggingface",
+        "langchain-community",
+        "huggingface-hub",
+        "pulsar-client",
+        "pyyaml",
+        "prometheus-client",
+    ],
+    scripts=[
+        "scripts/embeddings-hf",
+    ]
+)