From 1a34a444f5f07363a9ba5328487f2f04e99e4eeb Mon Sep 17 00:00:00 2001 From: cybermaggedon Date: Thu, 3 Oct 2024 17:50:25 +0100 Subject: [PATCH 1/3] Developer guide into 0.11 branch (#101) * Add developer guide --- DEVELOPER_GUIDE.md | 78 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 DEVELOPER_GUIDE.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md new file mode 100644 index 00000000..027e13a8 --- /dev/null +++ b/DEVELOPER_GUIDE.md @@ -0,0 +1,78 @@ + +# Developer's guide + +## Release management + +To do a public release you need to... + +- Get the git directory ready for the release + +- Tag the repo e.g. + +``` + git tag -a v1.2.3 -m '' + git push --tags +``` + +- Generate the deploy templates, don't add the `v` to the version number. + +``` + templates/generate-all deploy.zip 1.2.3 +``` + +- Release + + - Go to github, on Code tab, select tags, and find the right version + - Select create release + - Select the right previous version and generate release notes + - At the bottom of the form, find the upload pad, click that and add the + deploy.zip created earlier + +- Create Python packages. You need a PyPi token with access to our repos + + - make packages + - make pypi-upload + +- Create containers. You need a docker hub token with acccess to our repos + + - make + - make push + +## Local build + +To do a local build, you need to... + +- Consider what version you want to build at, and change this in Makefile. + It doesn't really matter so long as there isn't a clash with what's in + the public repos. You could stick with the version that's there, or + change to 0.0.0 if you're paranoid about pushing something accidentally. + +- If you changed the version to generate templates with your version, or + changed deployment templates, you need to recreate launch assets to + a deploy.zip file: + +``` + templates/generate-all deploy.zip V.V.V +``` + +- Build containers + +``` + make +``` + + +- If you changed anything which affects command line stuff (which maybe + you could do if you changed schemas), then + +``` + make packages +``` + + That puts Python packages in dist/ You then need to install some or + all of those packages. Typically you only need -base and -cli to + an appropriate environment e.g. use Python `venv` to create a virtual + environment and install them there. + + + From 148092a6af06f0ef4b90043cd567753a8cd97d56 Mon Sep 17 00:00:00 2001 From: cybermaggedon Date: Fri, 4 Oct 2024 22:12:39 +0100 Subject: [PATCH 2/3] Fix/lock 0.11 version (#108) * - Locked 0.11 packages to 0.11 deps - Added 'trustgraph' uber-package which installs the rest - Added dependency to set package versions before building packages * Bump version --- Makefile | 7 +++-- trustgraph-bedrock/setup.py | 2 +- trustgraph-cli/setup.py | 2 +- trustgraph-embeddings-hf/setup.py | 4 +-- trustgraph-flow/setup.py | 2 +- trustgraph-parquet/setup.py | 2 +- trustgraph-vertexai/setup.py | 2 +- trustgraph/setup.py | 47 +++++++++++++++++++++++++++++++ 8 files changed, 59 insertions(+), 9 deletions(-) create mode 100644 trustgraph/setup.py diff --git a/Makefile b/Makefile index 5593faa7..6e78df46 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # VERSION=$(shell git describe | sed 's/^v//') -VERSION=0.11.19 +VERSION=0.11.20 DOCKER=podman @@ -8,6 +8,7 @@ all: container # Not used wheels: + pip3 wheel --no-deps --wheel-dir dist trustgraph/ pip3 wheel --no-deps --wheel-dir dist trustgraph-base/ pip3 wheel --no-deps --wheel-dir dist trustgraph-flow/ pip3 wheel --no-deps --wheel-dir dist trustgraph-vertexai/ @@ -16,8 +17,9 @@ wheels: pip3 wheel --no-deps --wheel-dir dist trustgraph-embeddings-hf/ pip3 wheel --no-deps --wheel-dir dist trustgraph-cli/ -packages: +packages: update-package-versions rm -rf dist/ + cd trustgraph && python3 setup.py sdist --dist-dir ../dist/ cd trustgraph-base && python3 setup.py sdist --dist-dir ../dist/ cd trustgraph-flow && python3 setup.py sdist --dist-dir ../dist/ cd trustgraph-vertexai && python3 setup.py sdist --dist-dir ../dist/ @@ -40,6 +42,7 @@ update-package-versions: echo __version__ = \"${VERSION}\" > trustgraph-parquet/trustgraph/parquet_version.py echo __version__ = \"${VERSION}\" > trustgraph-embeddings-hf/trustgraph/embeddings_hf_version.py echo __version__ = \"${VERSION}\" > trustgraph-cli/trustgraph/cli_version.py + echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py container: update-package-versions ${DOCKER} build -f Containerfile -t ${CONTAINER}:${VERSION} \ diff --git a/trustgraph-bedrock/setup.py b/trustgraph-bedrock/setup.py index 317f2bdf..787cbad4 100644 --- a/trustgraph-bedrock/setup.py +++ b/trustgraph-bedrock/setup.py @@ -34,7 +34,7 @@ setuptools.setup( python_requires='>=3.8', download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", install_requires=[ - "trustgraph-base", + "trustgraph-base<0.12", "pulsar-client", "prometheus-client", "boto3", diff --git a/trustgraph-cli/setup.py b/trustgraph-cli/setup.py index 68024de6..061234a6 100644 --- a/trustgraph-cli/setup.py +++ b/trustgraph-cli/setup.py @@ -34,7 +34,7 @@ setuptools.setup( python_requires='>=3.8', download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", install_requires=[ - "trustgraph-base", + "trustgraph-base<0.12", "requests", "pulsar-client", "rdflib", diff --git a/trustgraph-embeddings-hf/setup.py b/trustgraph-embeddings-hf/setup.py index f3c2869b..7f1aafa4 100644 --- a/trustgraph-embeddings-hf/setup.py +++ b/trustgraph-embeddings-hf/setup.py @@ -34,8 +34,8 @@ setuptools.setup( python_requires='>=3.8', download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", install_requires=[ - "trustgraph-base", - "trustgraph-flow", + "trustgraph-base<0.12", + "trustgraph-flow<0.12", "torch", "urllib3", "transformers", diff --git a/trustgraph-flow/setup.py b/trustgraph-flow/setup.py index a1c89797..1c86ed77 100644 --- a/trustgraph-flow/setup.py +++ b/trustgraph-flow/setup.py @@ -34,7 +34,7 @@ setuptools.setup( python_requires='>=3.8', download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", install_requires=[ - "trustgraph-base", + "trustgraph-base<0.12", "urllib3", "rdflib", "pymilvus", diff --git a/trustgraph-parquet/setup.py b/trustgraph-parquet/setup.py index 6da7d916..ee0c7ce4 100644 --- a/trustgraph-parquet/setup.py +++ b/trustgraph-parquet/setup.py @@ -34,7 +34,7 @@ setuptools.setup( python_requires='>=3.8', download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", install_requires=[ - "trustgraph-base", + "trustgraph-base<0.12", "pulsar-client", "prometheus-client", "pyarrow", diff --git a/trustgraph-vertexai/setup.py b/trustgraph-vertexai/setup.py index 73c03b20..3ef59da9 100644 --- a/trustgraph-vertexai/setup.py +++ b/trustgraph-vertexai/setup.py @@ -34,7 +34,7 @@ setuptools.setup( python_requires='>=3.8', download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", install_requires=[ - "trustgraph-base", + "trustgraph-base<0.12", "pulsar-client", "google-cloud-aiplatform", "prometheus-client", diff --git a/trustgraph/setup.py b/trustgraph/setup.py new file mode 100644 index 00000000..f840943e --- /dev/null +++ b/trustgraph/setup.py @@ -0,0 +1,47 @@ +import setuptools +import os +import importlib + +with open("README.md", "r") as fh: + long_description = fh.read() + +# Load a version number module +spec = importlib.util.spec_from_file_location( + 'version', 'trustgraph/trustgraph_version.py' +) +version_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(version_module) + +version = version_module.__version__ + +setuptools.setup( + name="trustgraph", + version=version, + author="trustgraph.ai", + author_email="security@trustgraph.ai", + description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/trustgraph-ai/trustgraph", + packages=setuptools.find_namespace_packages( + where='./', + ), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Operating System :: OS Independent", + ], + python_requires='>=3.8', + download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", + install_requires=[ + "trustgraph-base<0.12", + "trustgraph-bedrock<0.12", + "trustgraph-cli<0.12", + "trustgraph-embeddings-hf<0.12", + "trustgraph-flow<0.12", + "trustgraph-parquet<0.12", + "trustgraph-vertexai<0.12", + ], + scripts=[ + ] +) From 16aa6e0c6cd730e17733a267328ec8211b95b444 Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Fri, 4 Oct 2024 22:21:56 +0100 Subject: [PATCH 3/3] 0.11.20 on the front page --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7a587876..ffcf43ab 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,8 @@ TrustGraph releases are available [here](https://github.com/trustgraph-ai/trustg | Release Type | Release Version | | ------------ | --------------- | -| Latest | [0.11.19](https://github.com/trustgraph-ai/trustgraph/releases/download/v0.11.19/deploy.zip) | -| Stable | [0.11.19](https://github.com/trustgraph-ai/trustgraph/releases/download/v0.11.19/deploy.zip) | +| Latest | [0.11.20](https://github.com/trustgraph-ai/trustgraph/releases/download/v0.11.20/deploy.zip) | +| Stable | [0.11.20](https://github.com/trustgraph-ai/trustgraph/releases/download/v0.11.20/deploy.zip) | TrustGraph is fully containerized and is launched with a `YAML` configuration file. Unzipping the `deploy.zip` will add the `deploy` directory with the following subdirectories: