feat: adding automated tests
This commit is contained in:
parent
e484f12228
commit
29ee360082
SSH key fingerprint:
SHA256:QkkAgVoYi9TQ0UKPkiKSfnerZy2h4qhi3SVPXJmBN+M
18 changed files with 2886 additions and 4 deletions
304
test/test_api_integration.py
Normal file
304
test/test_api_integration.py
Normal file
|
|
@ -0,0 +1,304 @@
|
|||
"""
|
||||
Integration tests against the real backend at 192.168.0.50:12434.
|
||||
|
||||
Run with:
|
||||
pytest test/test_api_integration.py -v -m integration
|
||||
|
||||
All tests in this file are marked @pytest.mark.integration.
|
||||
They require the test server to be reachable and to have at least one
|
||||
chat model and one embedding model available.
|
||||
|
||||
Env vars to pin specific models:
|
||||
NOMYO_TEST_MODEL_CHAT e.g. qwen2.5:1.5b
|
||||
NOMYO_TEST_EMBED_MODEL e.g. nomic-embed-text:latest
|
||||
"""
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
# ── Health / discovery routes ─────────────────────────────────────────────────
|
||||
|
||||
class TestDiscoveryRoutes:
|
||||
async def test_version(self, integration_client):
|
||||
resp = await integration_client.get("/api/version")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "version" in data
|
||||
assert isinstance(data["version"], str)
|
||||
|
||||
async def test_tags_returns_models(self, integration_client):
|
||||
resp = await integration_client.get("/api/tags")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "models" in data
|
||||
assert isinstance(data["models"], list)
|
||||
assert len(data["models"]) > 0
|
||||
|
||||
async def test_ps_returns_list(self, integration_client):
|
||||
resp = await integration_client.get("/api/ps")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "models" in data
|
||||
assert isinstance(data["models"], list)
|
||||
|
||||
async def test_v1_models_returns_data(self, integration_client):
|
||||
resp = await integration_client.get("/v1/models")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "data" in data
|
||||
assert isinstance(data["data"], list)
|
||||
|
||||
async def test_usage_returns_counts(self, integration_client):
|
||||
resp = await integration_client.get("/api/usage")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "usage_counts" in data
|
||||
assert "token_usage_counts" in data
|
||||
|
||||
async def test_config_returns_endpoints(self, integration_client):
|
||||
resp = await integration_client.get("/api/config")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "endpoints" in data
|
||||
|
||||
async def test_hostname(self, integration_client):
|
||||
resp = await integration_client.get("/api/hostname")
|
||||
assert resp.status_code == 200
|
||||
assert "hostname" in resp.json()
|
||||
|
||||
async def test_health(self, integration_client):
|
||||
resp = await integration_client.get("/health")
|
||||
assert resp.status_code in (200, 503)
|
||||
data = resp.json()
|
||||
assert data["status"] in ("ok", "error")
|
||||
assert "endpoints" in data
|
||||
|
||||
async def test_cache_stats(self, integration_client):
|
||||
resp = await integration_client.get("/api/cache/stats")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "enabled" in data
|
||||
|
||||
|
||||
# ── /api/chat ─────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestApiChat:
|
||||
async def test_non_streaming(self, integration_client, chat_model):
|
||||
resp = await integration_client.post(
|
||||
"/api/chat",
|
||||
json={
|
||||
"model": chat_model,
|
||||
"stream": False,
|
||||
"messages": [{"role": "user", "content": "Reply with exactly: OK"}],
|
||||
"options": {"num_predict": 10},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "message" in data
|
||||
assert "content" in data["message"]
|
||||
|
||||
async def test_streaming_ndjson(self, integration_client, chat_model):
|
||||
resp = await integration_client.post(
|
||||
"/api/chat",
|
||||
json={
|
||||
"model": chat_model,
|
||||
"stream": True,
|
||||
"messages": [{"role": "user", "content": "Say hi"}],
|
||||
"options": {"num_predict": 5},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
lines = [l for l in resp.text.strip().split("\n") if l.strip()]
|
||||
assert len(lines) >= 1
|
||||
for line in lines:
|
||||
obj = json.loads(line)
|
||||
assert "model" in obj
|
||||
|
||||
async def test_non_streaming_has_token_counts(self, integration_client, chat_model):
|
||||
resp = await integration_client.post(
|
||||
"/api/chat",
|
||||
json={
|
||||
"model": chat_model,
|
||||
"stream": False,
|
||||
"messages": [{"role": "user", "content": "Count to 3"}],
|
||||
"options": {"num_predict": 20},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data.get("done") is True
|
||||
# Token counts should be present in the final chunk
|
||||
assert data.get("prompt_eval_count", 0) >= 0
|
||||
|
||||
async def test_system_message_honoured(self, integration_client, chat_model):
|
||||
resp = await integration_client.post(
|
||||
"/api/chat",
|
||||
json={
|
||||
"model": chat_model,
|
||||
"stream": False,
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a helpful assistant. Always reply with exactly: PONG"},
|
||||
{"role": "user", "content": "PING"},
|
||||
],
|
||||
"options": {"num_predict": 10},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
content = resp.json()["message"]["content"]
|
||||
assert isinstance(content, str)
|
||||
assert len(content) > 0
|
||||
|
||||
|
||||
# ── /api/generate ─────────────────────────────────────────────────────────────
|
||||
|
||||
class TestApiGenerate:
|
||||
async def test_non_streaming(self, integration_client, chat_model):
|
||||
resp = await integration_client.post(
|
||||
"/api/generate",
|
||||
json={
|
||||
"model": chat_model,
|
||||
"prompt": "Complete: The sky is",
|
||||
"stream": False,
|
||||
"options": {"num_predict": 5},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "response" in data
|
||||
|
||||
async def test_streaming(self, integration_client, chat_model):
|
||||
resp = await integration_client.post(
|
||||
"/api/generate",
|
||||
json={
|
||||
"model": chat_model,
|
||||
"prompt": "One plus one equals",
|
||||
"stream": True,
|
||||
"options": {"num_predict": 5},
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
lines = [l for l in resp.text.strip().split("\n") if l.strip()]
|
||||
assert len(lines) >= 1
|
||||
|
||||
|
||||
# ── /api/embed ────────────────────────────────────────────────────────────────
|
||||
|
||||
class TestApiEmbed:
|
||||
async def test_embed_single_string(self, integration_client, embed_model):
|
||||
resp = await integration_client.post(
|
||||
"/api/embed",
|
||||
json={"model": embed_model, "input": "The quick brown fox"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "embeddings" in data
|
||||
assert isinstance(data["embeddings"], list)
|
||||
assert len(data["embeddings"]) == 1
|
||||
assert len(data["embeddings"][0]) > 0
|
||||
|
||||
async def test_embed_multiple_inputs(self, integration_client, embed_model):
|
||||
resp = await integration_client.post(
|
||||
"/api/embed",
|
||||
json={"model": embed_model, "input": ["sentence one", "sentence two"]},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "embeddings" in data
|
||||
assert len(data["embeddings"]) == 2
|
||||
|
||||
|
||||
# ── /v1/chat/completions ──────────────────────────────────────────────────────
|
||||
|
||||
class TestOpenAIChatCompletions:
|
||||
async def test_non_streaming(self, integration_client, chat_model):
|
||||
model = chat_model.replace(":latest", "") if ":latest" in chat_model else chat_model
|
||||
resp = await integration_client.post(
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "Reply OK"}],
|
||||
"max_tokens": 10,
|
||||
"stream": False,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "choices" in data
|
||||
assert len(data["choices"]) > 0
|
||||
assert "message" in data["choices"][0]
|
||||
|
||||
async def test_streaming_sse(self, integration_client, chat_model):
|
||||
model = chat_model.replace(":latest", "") if ":latest" in chat_model else chat_model
|
||||
resp = await integration_client.post(
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "Hi"}],
|
||||
"max_tokens": 5,
|
||||
"stream": True,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
# Response should be SSE format
|
||||
assert "data:" in resp.text or "[DONE]" in resp.text
|
||||
|
||||
async def test_non_streaming_has_usage(self, integration_client, chat_model):
|
||||
model = chat_model.replace(":latest", "") if ":latest" in chat_model else chat_model
|
||||
resp = await integration_client.post(
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "Say yes"}],
|
||||
"max_tokens": 5,
|
||||
"stream": False,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
if "usage" in data and data["usage"]:
|
||||
assert data["usage"].get("prompt_tokens", 0) >= 0
|
||||
|
||||
|
||||
# ── /v1/embeddings ────────────────────────────────────────────────────────────
|
||||
|
||||
class TestOpenAIEmbeddings:
|
||||
async def test_single_input(self, integration_client, embed_model):
|
||||
model = embed_model.replace(":latest", "") if ":latest" in embed_model else embed_model
|
||||
resp = await integration_client.post(
|
||||
"/v1/embeddings",
|
||||
json={"model": model, "input": "Test sentence"},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "data" in data
|
||||
assert len(data["data"]) > 0
|
||||
embedding = data["data"][0].get("embedding")
|
||||
assert isinstance(embedding, list)
|
||||
assert len(embedding) > 0
|
||||
|
||||
|
||||
# ── Token counts (database-backed) ───────────────────────────────────────────
|
||||
|
||||
class TestTokenCounts:
|
||||
async def test_token_counts_endpoint(self, integration_client):
|
||||
resp = await integration_client.get("/api/token_counts")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "total_tokens" in data
|
||||
assert "breakdown" in data
|
||||
|
||||
|
||||
# ── ps_details (extended ps) ─────────────────────────────────────────────────
|
||||
|
||||
class TestPsDetails:
|
||||
async def test_ps_details_returns_models(self, integration_client):
|
||||
resp = await integration_client.get("/api/ps_details")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "models" in data
|
||||
assert isinstance(data["models"], list)
|
||||
Loading…
Add table
Add a link
Reference in a new issue