mirror of
https://github.com/katanemo/plano.git
synced 2026-07-11 16:12:13 +02:00
Salmanap/fix network agent demo (#153)
* staging my changes to re-based from main * adding debug statements to rust * merged with main * ready to push network agent * removed the incomplete sql example --------- Co-authored-by: Salman Paracha <salmanparacha@MacBook-Pro-261.local>
This commit is contained in:
Salman Paracha
GitHub
parent
6acfea7787
commit
b63a01fe82
41 changed files with 252 additions and 1987 deletions
19
demos/network_agent/Dockerfile
Normal file
19
demos/network_agent/Dockerfile
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
FROM python:3.10 AS base
|
||||
|
||||
FROM base AS builder
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
COPY requirements.txt /src/
|
||||
RUN pip install --prefix=/runtime --force-reinstall -r requirements.txt
|
||||
|
||||
COPY . /src
|
||||
|
||||
FROM python:3.10-slim AS output
|
||||
|
||||
COPY --from=builder /runtime /usr/local
|
||||
|
||||
COPY . /app
|
||||
WORKDIR /app
|
||||
|
||||
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "80", "--log-level", "info"]
|
||||
0
demos/network_agent/README.md
Normal file
0
demos/network_agent/README.md
Normal file
71
demos/network_agent/arch_config.yaml
Normal file
71
demos/network_agent/arch_config.yaml
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
version: v0.1
|
||||
listener:
|
||||
address: 127.0.0.1
|
||||
port: 8080 #If you configure port 443, you'll need to update the listener with tls_certificates
|
||||
message_format: huggingface
|
||||
|
||||
# Centralized way to manage LLMs, manage keys, retry logic, failover and limits in a central way
|
||||
llm_providers:
|
||||
- name: OpenAI
|
||||
provider: openai
|
||||
access_key: OPENAI_API_KEY
|
||||
model: gpt-4o
|
||||
default: true
|
||||
|
||||
# default system prompt used by all prompt targets
|
||||
system_prompt: |
|
||||
You are a network assistant that just offers facts; not advice on manufacturers or purchasing decisions.
|
||||
|
||||
prompt_targets:
|
||||
- name: reboot_devices
|
||||
description: Reboot specific devices or device groups
|
||||
endpoint:
|
||||
name: app_server
|
||||
path: /agent/device_reboot
|
||||
parameters:
|
||||
- name: device_ids
|
||||
type: list
|
||||
description: A list of device identifiers (IDs) to reboot.
|
||||
required: true
|
||||
- name: time_range
|
||||
type: int
|
||||
description: Optional time range in days for reboot operations. Defaults to 7.
|
||||
- name: network_qa
|
||||
endpoint:
|
||||
name: app_server
|
||||
path: /agent/network_summary
|
||||
description: Handle general Q/A related to networking.
|
||||
default: true
|
||||
- name: device_summary
|
||||
description: Retrieve statistics for specific devices within a time range
|
||||
endpoint:
|
||||
name: app_server
|
||||
path: /agent/device_summary
|
||||
parameters:
|
||||
- name: device_ids
|
||||
type: list
|
||||
description: A list of device identifiers (IDs) to retrieve statistics for.
|
||||
required: true # device_ids are required to get device statistics
|
||||
- name: time_range
|
||||
type: int
|
||||
description: Time range in days for which to gather device statistics. Defaults to 7.
|
||||
default: "7"
|
||||
|
||||
# Arch creates a round-robin load balancing between different endpoints, managed via the cluster subsystem.
|
||||
endpoints:
|
||||
app_server:
|
||||
# value could be ip address or a hostname with port
|
||||
# this could also be a list of endpoints for load balancing
|
||||
# for example endpoint: [ ip1:port, ip2:port ]
|
||||
endpoint: host.docker.internal:18083
|
||||
# max time to wait for a connection to be established
|
||||
connect_timeout: 0.005s
|
||||
|
||||
ratelimits:
|
||||
- model: gpt-4
|
||||
selector:
|
||||
key: selector-key
|
||||
value: selector-value
|
||||
limit:
|
||||
tokens: 1
|
||||
unit: minute
|
||||
21
demos/network_agent/docker-compose.yaml
Normal file
21
demos/network_agent/docker-compose.yaml
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
services:
|
||||
api_server:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
ports:
|
||||
- "18083:80"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl" ,"http://localhost:80/healthz"]
|
||||
interval: 5s
|
||||
retries: 20
|
||||
|
||||
chatbot_ui:
|
||||
build:
|
||||
context: ../../chatbot_ui
|
||||
dockerfile: Dockerfile
|
||||
ports:
|
||||
- "18080:8080"
|
||||
environment:
|
||||
- OPENAI_API_KEY=${OPENAI_API_KEY:?error}
|
||||
- CHAT_COMPLETION_ENDPOINT=http://host.docker.internal:10000/v1
|
||||
12
demos/network_agent/grafana/dashboard.yaml
Normal file
12
demos/network_agent/grafana/dashboard.yaml
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: "Dashboard provider"
|
||||
orgId: 1
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 10
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: true
|
||||
355
demos/network_agent/grafana/dashboards/envoy_overview.json
Normal file
355
demos/network_agent/grafana/dashboards/envoy_overview.json
Normal file
|
|
@ -0,0 +1,355 @@
|
|||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"disableTextWrap": false,
|
||||
"editorMode": "code",
|
||||
"expr": "avg(rate(envoy_cluster_internal_upstream_rq_time_sum[1m]) / rate(envoy_cluster_internal_upstream_rq_time_count[1m])) by (envoy_cluster_name)",
|
||||
"fullMetaSearch": false,
|
||||
"hide": false,
|
||||
"includeNullMetadata": true,
|
||||
"instant": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "request latency - internal (ms)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"disableTextWrap": false,
|
||||
"editorMode": "code",
|
||||
"expr": "avg(rate(envoy_cluster_external_upstream_rq_time_sum[1m]) / rate(envoy_cluster_external_upstream_rq_time_count[1m])) by (envoy_cluster_name)",
|
||||
"fullMetaSearch": false,
|
||||
"hide": false,
|
||||
"includeNullMetadata": true,
|
||||
"instant": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "request latency - external (ms)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"disableTextWrap": false,
|
||||
"editorMode": "code",
|
||||
"expr": "avg(rate(envoy_cluster_internal_upstream_rq_completed[1m])) by (envoy_cluster_name)",
|
||||
"fullMetaSearch": false,
|
||||
"includeNullMetadata": true,
|
||||
"instant": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A",
|
||||
"useBackend": false
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"disableTextWrap": false,
|
||||
"editorMode": "code",
|
||||
"expr": "avg(rate(envoy_cluster_external_upstream_rq_completed[1m])) by (envoy_cluster_name)",
|
||||
"fullMetaSearch": false,
|
||||
"hide": false,
|
||||
"includeNullMetadata": true,
|
||||
"instant": false,
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "B",
|
||||
"useBackend": false
|
||||
}
|
||||
],
|
||||
"title": "Upstream request count",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"schemaVersion": 39,
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "Intelligent Gateway Overview",
|
||||
"uid": "adt6uhx5lk8aob",
|
||||
"version": 3,
|
||||
"weekStart": ""
|
||||
}
|
||||
9
demos/network_agent/grafana/datasource.yaml
Normal file
9
demos/network_agent/grafana/datasource.yaml
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
access: proxy
|
||||
editable: true
|
||||
104
demos/network_agent/main.py
Normal file
104
demos/network_agent/main.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
from fastapi import FastAPI, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
# Define the request model
|
||||
class DeviceSummaryRequest(BaseModel):
|
||||
device_ids: List[int]
|
||||
time_range: Optional[int] = Field(default=7, description="Time range in days, defaults to 7")
|
||||
|
||||
# Define the response model
|
||||
class DeviceStatistics(BaseModel):
|
||||
device_id: int
|
||||
time_range: str
|
||||
data: str
|
||||
|
||||
class DeviceSummaryResponse(BaseModel):
|
||||
statistics: List[DeviceStatistics]
|
||||
|
||||
# Request model for device reboot
|
||||
class DeviceRebootRequest(BaseModel):
|
||||
device_ids: List[int]
|
||||
|
||||
# Response model for the device reboot
|
||||
class CoverageResponse(BaseModel):
|
||||
status: str
|
||||
summary: dict
|
||||
|
||||
@app.post("/agent/device_reboot", response_model=CoverageResponse)
|
||||
def reboot_network_device(request_data: DeviceRebootRequest):
|
||||
"""
|
||||
Endpoint to reboot network devices based on device IDs and an optional time range.
|
||||
"""
|
||||
|
||||
# Access data from the Pydantic model
|
||||
device_ids = request_data.device_ids
|
||||
|
||||
# Validate 'device_ids' (This is already validated by Pydantic, but additional logic can be added if needed)
|
||||
if not device_ids:
|
||||
raise HTTPException(status_code=400, detail="'device_ids' parameter is required")
|
||||
|
||||
# Simulate reboot operation and return the response
|
||||
statistics = []
|
||||
for device_id in device_ids:
|
||||
# Placeholder for actual data retrieval or device reboot logic
|
||||
stats = {
|
||||
"data": f"Device {device_id} has been successfully rebooted."
|
||||
}
|
||||
statistics.append(stats)
|
||||
|
||||
# Return the response with a summary
|
||||
return CoverageResponse(status="success", summary={"device_ids": device_ids})
|
||||
|
||||
# Post method for device summary
|
||||
@app.post("/agent/device_summary", response_model=DeviceSummaryResponse)
|
||||
def get_device_summary(request: DeviceSummaryRequest):
|
||||
"""
|
||||
Endpoint to retrieve device statistics based on device IDs and an optional time range.
|
||||
"""
|
||||
|
||||
# Extract 'device_ids' and 'time_range' from the request
|
||||
device_ids = request.device_ids
|
||||
time_range = request.time_range
|
||||
|
||||
# Simulate retrieving statistics for the given device IDs and time range
|
||||
statistics = []
|
||||
minutes = 1
|
||||
for device_id in device_ids:
|
||||
stats = {
|
||||
"device_id": device_id,
|
||||
"time_range": f"Last {time_range} days",
|
||||
"data": f"Device {device_id} over the last {time_range} days experienced {minutes} minutes of downtime.",
|
||||
}
|
||||
minutes += 1
|
||||
statistics.append(DeviceStatistics(**stats))
|
||||
|
||||
return DeviceSummaryResponse(statistics=statistics)
|
||||
|
||||
@app.post("/agent/network_summary")
|
||||
async def policy_qa():
|
||||
"""
|
||||
This method handles Q/A related to general issues in networks.
|
||||
It forwards the conversation to the OpenAI client via a local proxy and returns the response.
|
||||
"""
|
||||
return {
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "I am a helpful networking agent, and I can help you get status for network devices or reboot them"
|
||||
},
|
||||
"finish_reason": "completed",
|
||||
"index": 0
|
||||
}
|
||||
],
|
||||
"model": "network_agent",
|
||||
"usage": {
|
||||
"completion_tokens": 0
|
||||
}
|
||||
}
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(debug=True)
|
||||
4
demos/network_agent/requirements.txt
Normal file
4
demos/network_agent/requirements.txt
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
fastapi
|
||||
uvicorn
|
||||
pydantic
|
||||
typing
|
||||
247
demos/network_agent/utils.py
Normal file
247
demos/network_agent/utils.py
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
import pandas as pd
|
||||
import random
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import re
|
||||
import logging
|
||||
from dateparser import parse
|
||||
import sqlite3
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def load_sql():
|
||||
# Example Usage
|
||||
conn = sqlite3.connect(":memory:")
|
||||
|
||||
# create and load the devices table
|
||||
device_data = generate_device_data(conn)
|
||||
|
||||
# create and load the interface_stats table
|
||||
generate_interface_stats_data(conn, device_data)
|
||||
|
||||
# create and load the flow table
|
||||
generate_flow_data(conn, device_data)
|
||||
|
||||
return conn
|
||||
|
||||
# Function to convert natural language time expressions to "X {time} ago" format
|
||||
def convert_to_ago_format(expression):
|
||||
# Define patterns for different time units
|
||||
time_units = {
|
||||
r"seconds": "seconds",
|
||||
r"minutes": "minutes",
|
||||
r"mins": "mins",
|
||||
r"hrs": "hrs",
|
||||
r"hours": "hours",
|
||||
r"hour": "hour",
|
||||
r"hr": "hour",
|
||||
r"days": "days",
|
||||
r"day": "day",
|
||||
r"weeks": "weeks",
|
||||
r"week": "week",
|
||||
r"months": "months",
|
||||
r"month": "month",
|
||||
r"years": "years",
|
||||
r"yrs": "years",
|
||||
r"year": "year",
|
||||
r"yr": "year",
|
||||
}
|
||||
|
||||
# Iterate over each time unit and create regex for each phrase format
|
||||
for pattern, unit in time_units.items():
|
||||
# Handle "for the past X {unit}"
|
||||
match = re.search(rf"(\d+) {pattern}", expression)
|
||||
if match:
|
||||
quantity = match.group(1)
|
||||
return f"{quantity} {unit} ago"
|
||||
|
||||
# If the format is not recognized, return None or raise an error
|
||||
return None
|
||||
|
||||
|
||||
# Function to generate random MAC addresses
|
||||
def random_mac():
|
||||
return "AA:BB:CC:DD:EE:" + ":".join(
|
||||
[f"{random.randint(0, 255):02X}" for _ in range(2)]
|
||||
)
|
||||
|
||||
|
||||
# Function to generate random IP addresses
|
||||
def random_ip():
|
||||
return f"{random.randint(1, 255)}.{random.randint(1, 255)}.{random.randint(1, 255)}.{random.randint(1, 255)}"
|
||||
|
||||
|
||||
# Generate synthetic data for the device table
|
||||
def generate_device_data(
|
||||
conn,
|
||||
n=1000,
|
||||
):
|
||||
device_data = {
|
||||
"switchip": [random_ip() for _ in range(n)],
|
||||
"hwsku": [f"HW{i+1}" for i in range(n)],
|
||||
"hostname": [f"switch{i+1}" for i in range(n)],
|
||||
"osversion": [f"v{i+1}" for i in range(n)],
|
||||
"layer": ["L2" if i % 2 == 0 else "L3" for i in range(n)],
|
||||
"region": [random.choice(["US", "EU", "ASIA"]) for _ in range(n)],
|
||||
"uptime": [
|
||||
f"{random.randint(0, 10)} days {random.randint(0, 23)}:{random.randint(0, 59)}:{random.randint(0, 59)}"
|
||||
for _ in range(n)
|
||||
],
|
||||
"device_mac_address": [random_mac() for _ in range(n)],
|
||||
}
|
||||
df = pd.DataFrame(device_data)
|
||||
df.to_sql("device", conn, index=False)
|
||||
return df
|
||||
|
||||
|
||||
# Generate synthetic data for the interfacestats table
|
||||
def generate_interface_stats_data(conn, device_df, n=1000):
|
||||
interface_stats_data = []
|
||||
for _ in range(n):
|
||||
device_mac = random.choice(device_df["device_mac_address"])
|
||||
ifname = random.choice(["eth0", "eth1", "eth2", "eth3"])
|
||||
time = datetime.now(timezone.utc) - timedelta(
|
||||
minutes=random.randint(0, 1440 * 5)
|
||||
) # random timestamps in the past 5 day
|
||||
in_discards = random.randint(0, 1000)
|
||||
in_errors = random.randint(0, 500)
|
||||
out_discards = random.randint(0, 800)
|
||||
out_errors = random.randint(0, 400)
|
||||
in_octets = random.randint(1000, 100000)
|
||||
out_octets = random.randint(1000, 100000)
|
||||
|
||||
interface_stats_data.append(
|
||||
{
|
||||
"device_mac_address": device_mac,
|
||||
"ifname": ifname,
|
||||
"time": time,
|
||||
"in_discards": in_discards,
|
||||
"in_errors": in_errors,
|
||||
"out_discards": out_discards,
|
||||
"out_errors": out_errors,
|
||||
"in_octets": in_octets,
|
||||
"out_octets": out_octets,
|
||||
}
|
||||
)
|
||||
df = pd.DataFrame(interface_stats_data)
|
||||
df.to_sql("interfacestats", conn, index=False)
|
||||
return
|
||||
|
||||
|
||||
# Generate synthetic data for the ts_flow table
|
||||
def generate_flow_data(conn, device_df, n=1000):
|
||||
flow_data = []
|
||||
for _ in range(n):
|
||||
sampler_address = random.choice(device_df["switchip"])
|
||||
proto = random.choice(["TCP", "UDP"])
|
||||
src_addr = random_ip()
|
||||
dst_addr = random_ip()
|
||||
src_port = random.randint(1024, 65535)
|
||||
dst_port = random.randint(1024, 65535)
|
||||
in_if = random.randint(1, 10)
|
||||
out_if = random.randint(1, 10)
|
||||
flow_start = int(
|
||||
(datetime.now() - timedelta(days=random.randint(1, 30))).timestamp()
|
||||
)
|
||||
flow_end = int(
|
||||
(datetime.now() - timedelta(days=random.randint(1, 30))).timestamp()
|
||||
)
|
||||
bytes_transferred = random.randint(1000, 100000)
|
||||
packets = random.randint(1, 1000)
|
||||
flow_time = datetime.now(timezone.utc) - timedelta(
|
||||
minutes=random.randint(0, 1440 * 5)
|
||||
) # random flow time
|
||||
|
||||
flow_data.append(
|
||||
{
|
||||
"sampler_address": sampler_address,
|
||||
"proto": proto,
|
||||
"src_addr": src_addr,
|
||||
"dst_addr": dst_addr,
|
||||
"src_port": src_port,
|
||||
"dst_port": dst_port,
|
||||
"in_if": in_if,
|
||||
"out_if": out_if,
|
||||
"flow_start": flow_start,
|
||||
"flow_end": flow_end,
|
||||
"bytes": bytes_transferred,
|
||||
"packets": packets,
|
||||
"time": flow_time,
|
||||
}
|
||||
)
|
||||
df = pd.DataFrame(flow_data)
|
||||
df.to_sql("ts_flow", conn, index=False)
|
||||
return
|
||||
|
||||
|
||||
def load_params(req):
|
||||
# Step 1: Convert the from_time natural language string to a timestamp if provided
|
||||
if req.from_time:
|
||||
# Use `dateparser` to parse natural language timeframes
|
||||
logger.info(f"{'* ' * 50}\n\nCaptured from time: {req.from_time}\n\n")
|
||||
parsed_time = parse(req.from_time, settings={"RELATIVE_BASE": datetime.now()})
|
||||
if not parsed_time:
|
||||
conv_time = convert_to_ago_format(req.from_time)
|
||||
if conv_time:
|
||||
parsed_time = parse(
|
||||
conv_time, settings={"RELATIVE_BASE": datetime.now()}
|
||||
)
|
||||
else:
|
||||
return {
|
||||
"error": "Invalid from_time format. Please provide a valid time description such as 'past 7 days' or 'since last month'."
|
||||
}
|
||||
logger.info(f"\n\nConverted from time: {parsed_time}\n\n{'* ' * 50}\n\n")
|
||||
from_time = parsed_time
|
||||
logger.info(f"Using parsed from_time: {from_time}")
|
||||
else:
|
||||
# If no from_time is provided, use a default value (e.g., the past 7 days)
|
||||
from_time = datetime.now() - timedelta(days=7)
|
||||
logger.info(f"Using default from_time: {from_time}")
|
||||
|
||||
# Step 2: Build the dynamic SQL query based on the optional filters
|
||||
filters = []
|
||||
params = {"from_time": from_time}
|
||||
|
||||
if req.ifname:
|
||||
filters.append("i.ifname = :ifname")
|
||||
params["ifname"] = req.ifname
|
||||
|
||||
if req.region:
|
||||
filters.append("d.region = :region")
|
||||
params["region"] = req.region
|
||||
|
||||
if req.min_in_errors is not None:
|
||||
filters.append("i.in_errors >= :min_in_errors")
|
||||
params["min_in_errors"] = req.min_in_errors
|
||||
|
||||
if req.max_in_errors is not None:
|
||||
filters.append("i.in_errors <= :max_in_errors")
|
||||
params["max_in_errors"] = req.max_in_errors
|
||||
|
||||
if req.min_out_errors is not None:
|
||||
filters.append("i.out_errors >= :min_out_errors")
|
||||
params["min_out_errors"] = req.min_out_errors
|
||||
|
||||
if req.max_out_errors is not None:
|
||||
filters.append("i.out_errors <= :max_out_errors")
|
||||
params["max_out_errors"] = req.max_out_errors
|
||||
|
||||
if req.min_in_discards is not None:
|
||||
filters.append("i.in_discards >= :min_in_discards")
|
||||
params["min_in_discards"] = req.min_in_discards
|
||||
|
||||
if req.max_in_discards is not None:
|
||||
filters.append("i.in_discards <= :max_in_discards")
|
||||
params["max_in_discards"] = req.max_in_discards
|
||||
|
||||
if req.min_out_discards is not None:
|
||||
filters.append("i.out_discards >= :min_out_discards")
|
||||
params["min_out_discards"] = req.min_out_discards
|
||||
|
||||
if req.max_out_discards is not None:
|
||||
filters.append("i.out_discards <= :max_out_discards")
|
||||
params["max_out_discards"] = req.max_out_discards
|
||||
|
||||
return params, filters
|
||||
Loading…
Add table
Add a link
Reference in a new issue