From dd1208dce42a9888ac59fa3089b45c0884118d78 Mon Sep 17 00:00:00 2001 From: Akshaya Annavajhala Date: Sun, 10 Mar 2024 16:24:53 -0700 Subject: [PATCH] initial integration tests (#23) * initial integration tests * update ini path * add required env vars * fix unbound local --- .github/workflows/integration.yaml | 36 +++++ .github/workflows/k3s.yaml | 17 +++ custom_components/mindctrl/__init__.py | 26 +--- custom_components/mindctrl/config_flow.py | 21 +-- custom_components/mindctrl/entity.py | 4 +- custom_components/mindctrl/services.py | 2 +- mindctrl-addon/requirements.txt | 1 + .../rootfs/usr/bin/multiserver/config.py | 42 ++++++ .../bin/multiserver/db/models/summary_data.py | 4 - .../rootfs/usr/bin/multiserver/db/queries.py | 1 - .../multiserver/db/{config.py => setup.py} | 41 +++--- .../rootfs/usr/bin/multiserver/main.py | 61 +++++--- .../usr/bin/multiserver/mlflow_bridge.py | 9 +- .../rootfs/usr/bin/multiserver/mlmodels.py | 1 - .../rootfs/usr/bin/multiserver/mqtt.py | 36 +++-- .../rootfs/usr/bin/multiserver/rag.py | 8 +- .../test_data/state_ring_buffer.txt | 8 -- .../usr/bin/multiserver/test_multiserver.py | 97 ------------- mindctrl-addon/test-requirements.txt | 12 ++ mindctrl-addon/tests/conftest.py | 132 ++++++++++++++++++ mindctrl-addon/tests/pytest.ini | 3 + mindctrl-addon/tests/test_appsettings.py | 60 ++++++++ .../tests/test_data/state_ring_buffer.txt | 8 ++ mindctrl-addon/tests/test_multiserver.py | 71 ++++++++++ 24 files changed, 488 insertions(+), 213 deletions(-) create mode 100644 .github/workflows/integration.yaml create mode 100644 .github/workflows/k3s.yaml create mode 100644 mindctrl-addon/rootfs/usr/bin/multiserver/config.py rename mindctrl-addon/rootfs/usr/bin/multiserver/db/{config.py => setup.py} (71%) delete mode 100644 mindctrl-addon/rootfs/usr/bin/multiserver/test_data/state_ring_buffer.txt delete mode 100644 mindctrl-addon/rootfs/usr/bin/multiserver/test_multiserver.py create mode 100644 mindctrl-addon/test-requirements.txt create mode 100644 mindctrl-addon/tests/conftest.py create mode 100644 mindctrl-addon/tests/pytest.ini create mode 100644 mindctrl-addon/tests/test_appsettings.py create mode 100644 mindctrl-addon/tests/test_data/state_ring_buffer.txt create mode 100644 mindctrl-addon/tests/test_multiserver.py diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml new file mode 100644 index 0000000..7d88e2d --- /dev/null +++ b/.github/workflows/integration.yaml @@ -0,0 +1,36 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Addon Integration + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.11 + uses: actions/setup-python@v3 + with: + python-version: "3.11" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f mindctrl-addon/test-requirements.txt ]; then pip install -r mindctrl-addon/test-requirements.txt; fi + - name: Lint with Ruff + run: | + ruff check . + - name: Test with pytest + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + pytest -v -s -c mindctrl-addon/tests/pytest.ini diff --git a/.github/workflows/k3s.yaml b/.github/workflows/k3s.yaml new file mode 100644 index 0000000..3a3188e --- /dev/null +++ b/.github/workflows/k3s.yaml @@ -0,0 +1,17 @@ +name: k3s + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + some-job: + steps: + - uses: nolar/setup-k3d-k3s@v1 + with: + version: v1.21 # E.g.: v1.21, v1.21.2, v1.21.2+k3s1 + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/custom_components/mindctrl/__init__.py b/custom_components/mindctrl/__init__.py index 20c5b95..c247f96 100644 --- a/custom_components/mindctrl/__init__.py +++ b/custom_components/mindctrl/__init__.py @@ -3,35 +3,21 @@ from __future__ import annotations -from functools import partial -import logging - -import mlflow as mlflowlib - -import voluptuous as vol - from homeassistant.components import conversation as haconversation from homeassistant.components.hassio import AddonManager, AddonError, AddonState from homeassistant.config_entries import ConfigEntry -from homeassistant.const import CONF_API_KEY -from homeassistant.core import ( - HomeAssistant, - SupportsResponse, - callback -) +from homeassistant.core import HomeAssistant, callback from homeassistant.exceptions import ( ConfigEntryNotReady, ) -from homeassistant.helpers import config_validation as cv, selector +from homeassistant.helpers import config_validation as cv from homeassistant.helpers.typing import ConfigType import asyncio from .addon import get_addon_manager -from .const import ( - ADDON_NAME, CONF_URL, CONF_USE_ADDON, DOMAIN, _LOGGER -) +from .const import ADDON_NAME, CONF_URL, CONF_USE_ADDON, DOMAIN, _LOGGER from .services import MindctrlClient, async_register_services from .conversation import MLflowAgent @@ -53,11 +39,13 @@ async def async_setup(hass: HomeAssistant, config: ConfigType) -> bool: ) return True + async def update_listener(hass, entry): """Handle options update.""" # https://developers.home-assistant.io/docs/config_entries_options_flow_handler#signal-updates _LOGGER.error(f"update_listener {entry}") + async def async_setup_entry(hass: HomeAssistant, entry: ConfigEntry) -> bool: """Set up MLflow from a config entry.""" _LOGGER.error("mindctrl async_setup_entry") @@ -125,8 +113,7 @@ async def async_ensure_addon_running(hass: HomeAssistant, entry: ConfigEntry) -> addon_state = addon_info.state - addon_config = { - } + addon_config = {} if addon_state == AddonState.NOT_INSTALLED: addon_manager.async_schedule_install_setup_addon( @@ -142,7 +129,6 @@ async def async_ensure_addon_running(hass: HomeAssistant, entry: ConfigEntry) -> ) raise ConfigEntryNotReady - addon_options = addon_info.options updates = {} if updates: hass.config_entries.async_update_entry(entry, data={**entry.data, **updates}) diff --git a/custom_components/mindctrl/config_flow.py b/custom_components/mindctrl/config_flow.py index 367f135..21fbbcf 100644 --- a/custom_components/mindctrl/config_flow.py +++ b/custom_components/mindctrl/config_flow.py @@ -1,23 +1,11 @@ -from abc import ABC, abstractmethod -from typing import Any, Dict, Optional +from typing import Any from homeassistant import config_entries, exceptions from homeassistant.core import HomeAssistant, callback -from homeassistant.const import CONF_NAME, CONF_URL +from homeassistant.const import CONF_URL from homeassistant.helpers.aiohttp_client import async_get_clientsession -from homeassistant.components.hassio import ( - AddonError, - AddonInfo, - AddonManager, - AddonState, - HassioServiceInfo, - is_hassio, -) # from homeassistant.components.zeroconf import ZeroconfServiceInfo from homeassistant.data_entry_flow import ( - AbortFlow, - FlowHandler, - FlowManager, FlowResult, ) import voluptuous as vol @@ -28,15 +16,12 @@ from .const import ( ADDON_HOST, ADDON_PORT, - ADDON_SLUG, CONF_ADDON_LOG_LEVEL, CONF_INTEGRATION_CREATED_ADDON, DOMAIN, _LOGGER, CONF_USE_ADDON, - CONF_URL, ) -from .addon import get_addon_manager DEFAULT_URL = f"http://{ADDON_HOST}:{ADDON_PORT}" @@ -171,7 +156,7 @@ async def async_step_manual( errors = {} try: - version_info = await validate_input(self.hass, user_input) + _ = await validate_input(self.hass, user_input) except InvalidInput as err: errors["base"] = err.error except Exception: # pylint: disable=broad-except diff --git a/custom_components/mindctrl/entity.py b/custom_components/mindctrl/entity.py index b451bab..cd4af12 100644 --- a/custom_components/mindctrl/entity.py +++ b/custom_components/mindctrl/entity.py @@ -1,4 +1,5 @@ """AdGuard Home base entity.""" + from __future__ import annotations from abc import ABC, abstractmethod @@ -9,6 +10,7 @@ from .const import ADDON_SLUG, DOMAIN, _LOGGER from .services import MindctrlClient + # https://github.com/home-assistant/core/blob/52d27230bce239017722d8ce9dd6f5386f63aba2/homeassistant/components/adguard/entity.py class MindctrlEntity(Entity, ABC): """Defines a base Mindctrl entity.""" @@ -69,7 +71,7 @@ def device_info(self) -> DeviceInfo: manufacturer="AK", name="Mindctrl", sw_version=self.hass.data[DOMAIN][self._entry.entry_id].get( - DATA_MINDCTRL_VERSION + "version", "unknown" ), configuration_url=config_url, ) diff --git a/custom_components/mindctrl/services.py b/custom_components/mindctrl/services.py index 4c717db..d0be672 100644 --- a/custom_components/mindctrl/services.py +++ b/custom_components/mindctrl/services.py @@ -14,7 +14,7 @@ import mlflow from .const import DOMAIN, SERVICE_INVOKE_MODEL, _LOGGER, CONF_URL import voluptuous as vol -from homeassistant.helpers import config_validation as cv, selector +from homeassistant.helpers import config_validation as cv class MindctrlClient(object): diff --git a/mindctrl-addon/requirements.txt b/mindctrl-addon/requirements.txt index 17315c3..a67efc0 100644 --- a/mindctrl-addon/requirements.txt +++ b/mindctrl-addon/requirements.txt @@ -2,6 +2,7 @@ mlflow[genai]~=2.10 pydantic<3,>=1.0 +pydantic-settings~=2.2 fastapi<1 uvicorn[standard]<1 watchfiles<1 diff --git a/mindctrl-addon/rootfs/usr/bin/multiserver/config.py b/mindctrl-addon/rootfs/usr/bin/multiserver/config.py new file mode 100644 index 0000000..e3f4e72 --- /dev/null +++ b/mindctrl-addon/rootfs/usr/bin/multiserver/config.py @@ -0,0 +1,42 @@ +from typing import Optional, Union, Literal +from pydantic import BaseModel, Field, SecretStr +from pydantic_settings import BaseSettings, SettingsConfigDict + + +# this is just to make settings typing happy - I don't have another implementation yet +class UnknownEventsSettings(BaseModel): + events_type: Literal["unknown"] + +class MqttEventsSettings(BaseModel): + events_type: Literal["mqtt"] + + broker: str = "localhost" + port: int = 1883 + username: Optional[str] = None + password: Optional[SecretStr] = None + +class PostgresStoreSettings(BaseModel): + store_type: Literal["psql"] + + user: str + password: SecretStr + address: str = "localhost" + port: int = 5432 + database: str = "mindctrl" + +# Just to make typing happy for now - add dapr, sqlite, etc +class UnknownStoreSettings(BaseModel): + store_type: Literal["unknown"] + +class AppSettings(BaseSettings): + # double underscore, in case your font doesn't make it clear + model_config = SettingsConfigDict(env_nested_delimiter='__') + + store: Union[PostgresStoreSettings, UnknownStoreSettings] = Field(discriminator="store_type") + events: Union[MqttEventsSettings, UnknownEventsSettings] = Field(discriminator="events_type") + # TODO: move this into the gateway or something + openai_api_key: SecretStr + force_publish_models: bool = False + notify_fd: Optional[int] = None + include_challenger_models: bool = True + mlflow_tracking_uri: Optional[str] = None diff --git a/mindctrl-addon/rootfs/usr/bin/multiserver/db/models/summary_data.py b/mindctrl-addon/rootfs/usr/bin/multiserver/db/models/summary_data.py index 7367562..75ef4d9 100644 --- a/mindctrl-addon/rootfs/usr/bin/multiserver/db/models/summary_data.py +++ b/mindctrl-addon/rootfs/usr/bin/multiserver/db/models/summary_data.py @@ -1,8 +1,4 @@ -import datetime -from sqlalchemy import Column, Integer, String, DateTime -from sqlalchemy.dialects.postgresql.types import TIMESTAMP -from pgvector.sqlalchemy import Vector EMBEDDING_DIM = 384 diff --git a/mindctrl-addon/rootfs/usr/bin/multiserver/db/queries.py b/mindctrl-addon/rootfs/usr/bin/multiserver/db/queries.py index e2e3be1..d45b662 100644 --- a/mindctrl-addon/rootfs/usr/bin/multiserver/db/queries.py +++ b/mindctrl-addon/rootfs/usr/bin/multiserver/db/queries.py @@ -1,5 +1,4 @@ TABLE_NAME = "summary_data" -from .models.summary_data import EMBEDDING_DIM # CREATE_TABLE = """CREATE TABLE IF NOT EXISTS {table_name} # ( diff --git a/mindctrl-addon/rootfs/usr/bin/multiserver/db/config.py b/mindctrl-addon/rootfs/usr/bin/multiserver/db/setup.py similarity index 71% rename from mindctrl-addon/rootfs/usr/bin/multiserver/db/config.py rename to mindctrl-addon/rootfs/usr/bin/multiserver/db/setup.py index 2ecb732..a82a9a8 100644 --- a/mindctrl-addon/rootfs/usr/bin/multiserver/db/config.py +++ b/mindctrl-addon/rootfs/usr/bin/multiserver/db/setup.py @@ -1,42 +1,33 @@ import collections import json import logging -import os from sqlalchemy import text from sqlalchemy.ext.asyncio import ( create_async_engine, - async_sessionmaker, - AsyncSession, AsyncEngine, ) from .queries import ( - ADD_RETENTION_POLICY, CREATE_SUMMARY_TABLE, ENABLE_PGVECTOR, CONVERT_TO_HYPERTABLE, ENABLE_TIMESCALE, ) +from config import PostgresStoreSettings +from mlmodels import summarize_events -_LOGGER = logging.getLogger(__name__) +_LOGGER = logging.getLogger(__name__) -def get_connection_string(include_password: bool = False) -> str: - username = os.environ.get("POSTGRES_USER") - password = os.environ["POSTGRES_PASSWORD"] - address = os.environ.get("POSTGRES_ADDRESS", "localhost") - port = os.environ.get("POSTGRES_PORT", "5432") - database = os.environ.get("POSTGRES_DATABASE", "mindctrl") - return f"postgresql+asyncpg://{username}:{password if include_password else '****'}@{address}:{port}/{database}" +def get_connection_string( + settings: PostgresStoreSettings, include_password: bool = False +) -> str: + return f"postgresql+asyncpg://{settings.user}:{settings.password.get_secret_value() if include_password else settings.password}@{settings.address}:{settings.port}/{settings.database}" -DATABASE_URL = get_connection_string(include_password=True) -DATABASE_SAFE_URL = get_connection_string(include_password=False) -_LOGGER.info(f"Using database: {DATABASE_SAFE_URL}") -engine: AsyncEngine = create_async_engine(DATABASE_URL, future=True, echo=True) # Don't need this until we have real models # async_session: async_sessionmaker[AsyncSession] = async_sessionmaker(engine, expire_on_commit=False) # TODO: Go use real models later @@ -46,7 +37,14 @@ def get_connection_string(include_password: bool = False) -> str: # return async_session -async def setup_db() -> AsyncEngine: +async def setup_db(settings: PostgresStoreSettings) -> AsyncEngine: + connection_string = get_connection_string(settings, include_password=True) + _LOGGER.info( + f"Using database: {get_connection_string(settings, include_password=False)}" + ) + + engine: AsyncEngine = create_async_engine(connection_string, future=True, echo=True) + async with engine.begin() as conn: await conn.execute(text(ENABLE_TIMESCALE)) await conn.execute(text(ENABLE_PGVECTOR)) @@ -60,17 +58,18 @@ async def setup_db() -> AsyncEngine: # TODO: move the relevant stuff to rag interface # TODO: probably rename to mlmodels to reduce confusion with dbmodels -from mlmodels import summarize_events, embed_summary -from .models.summary_data import EMBEDDING_DIM -async def insert_summary(state_ring_buffer: collections.deque[dict]): +async def insert_summary( + engine: AsyncEngine, + include_challenger: bool, + state_ring_buffer: collections.deque[dict], +): print("Inserting summary") # TODO: do this better as a batch insert # use summarizer model to emit a LIST of summaries, each with the timestamp from relevant event events = [json.dumps(event) for event in list(state_ring_buffer)] # summarized_events = summarize_events(events) - include_challenger = bool(os.environ.get("INCLUDE_CHALLENGER", True)) champion_summary, challenger_summary = summarize_events( ["\n".join(events)], include_challenger ) diff --git a/mindctrl-addon/rootfs/usr/bin/multiserver/main.py b/mindctrl-addon/rootfs/usr/bin/multiserver/main.py index fa20155..516b2de 100644 --- a/mindctrl-addon/rootfs/usr/bin/multiserver/main.py +++ b/mindctrl-addon/rootfs/usr/bin/multiserver/main.py @@ -1,3 +1,4 @@ +from functools import lru_cache, partial import logging import os from pathlib import Path @@ -16,45 +17,64 @@ import collections -from mlmodels import log_system_models, SUMMARIZATION_PROMPT +from mlmodels import SUMMARIZER_OAI_MODEL, log_system_models, SUMMARIZATION_PROMPT from mqtt import setup_mqtt_client, listen_to_mqtt from mlflow_bridge import connect_to_mlflow, poll_registry -from db.config import setup_db, insert_summary +from db.setup import setup_db, insert_summary +from config import AppSettings _logger = logging.getLogger(__name__) -def write_healthcheck_file(): +def write_healthcheck_file(settings: AppSettings): # Write readiness: https://skarnet.org/software/s6/notifywhenup.html - notification_fd = os.environ.get("NOTIFY_FD") + notification_fd = settings.notify_fd if notification_fd: os.write(int(notification_fd), b"\n") os.close(int(notification_fd)) +@lru_cache +def get_settings(): + return AppSettings() # pyright: ignore + + @asynccontextmanager async def lifespan(app: FastAPI): + app_settings = get_settings() + print("Starting mindctrl server with settings:") + print(app_settings.model_dump()) + asyncio.create_task(poll_registry(10.0)) # The buffer should be enhanced to be token-aware state_ring_buffer: collections.deque[dict] = collections.deque(maxlen=20) print("Setting up DB") - engine = await setup_db() + # TODO: convert to ABC with a common interface + if not app_settings.store.store_type == "psql": + raise ValueError(f"unknown store type: {app_settings.store.store_type}") + engine = await setup_db(app_settings.store) + insert_summary_partial = partial( + insert_summary, engine, app_settings.include_challenger_models + ) print("Setting up MQTT") - mqtt_client = setup_mqtt_client() + if not app_settings.events.events_type == "mqtt": + raise ValueError(f"unknown events type: {app_settings.events.events_type}") + + mqtt_client = setup_mqtt_client(app_settings.events) loop = asyncio.get_event_loop() print("Starting MQTT listener") mqtt_listener_task = loop.create_task( - listen_to_mqtt(mqtt_client, state_ring_buffer, insert_summary) + listen_to_mqtt(mqtt_client, state_ring_buffer, insert_summary_partial) ) print("Logging models") - loaded_models = log_system_models(bool(os.environ.get("FORCE_PUBLISH", False))) - connect_to_mlflow() + loaded_models = log_system_models(app_settings.force_publish_models) + connect_to_mlflow(app_settings) - write_healthcheck_file() + write_healthcheck_file(app_settings) print("Finished server setup") # Make resources available to requests via .state @@ -116,8 +136,9 @@ def read_root(request: Request, response_class=HTMLResponse): print(dashboard_url) return templates.TemplateResponse( - "index.html", - { + request=request, + name="index.html", + context={ "request": request, "tracking_store": mlflow.get_tracking_uri(), "model_registry": mlflow.get_registry_uri(), @@ -145,12 +166,7 @@ def read_version(request: Request): mlflow_url = request.base_url.replace(port=5000) dashboard_url = request.base_url.replace(port=9999) - import os - - version = os.environ.get("MINDCTRL_ADDON_VERSION", "0.0.0") - print(f"Version: {version}") return { - "version": version, "tracking_store": mlflow.get_tracking_uri(), "model_registry": mlflow.get_registry_uri(), "ws_url": ws_url, @@ -201,12 +217,15 @@ def generate_state_lines(buffer: collections.deque): # TODO: when I get internet see if RAG framework already has a known technique to deal with context chunking import tiktoken + print(f"Buffer has {len(buffer)} events") + enc = tiktoken.encoding_for_model( - "gpt-3.5-turbo" + SUMMARIZER_OAI_MODEL ) # TODO: pick it up from the model meta MAX_TOKENS = 4000 # TODO: Also pick it up from the model meta and encode the slack into a smarter heuristic buffer_lines = [] - total_tokens = len(enc.encode(SUMMARIZATION_PROMPT)) + prompt_tokens = len(enc.encode(SUMMARIZATION_PROMPT)) + total_tokens = prompt_tokens for index, item in enumerate(buffer): buffer_line = f"{item}" next_tokens = len(enc.encode(buffer_line)) + 1 # \n @@ -219,7 +238,9 @@ def generate_state_lines(buffer: collections.deque): total_tokens += next_tokens state_lines = "\n".join(buffer_lines) - print(f"Generated {total_tokens} token message data:\n{state_lines}") + print( + f"Generated {total_tokens} token message, {prompt_tokens} from prompt:\n---------\n{state_lines}\n---------" + ) return state_lines diff --git a/mindctrl-addon/rootfs/usr/bin/multiserver/mlflow_bridge.py b/mindctrl-addon/rootfs/usr/bin/multiserver/mlflow_bridge.py index e525b25..50c6287 100644 --- a/mindctrl-addon/rootfs/usr/bin/multiserver/mlflow_bridge.py +++ b/mindctrl-addon/rootfs/usr/bin/multiserver/mlflow_bridge.py @@ -1,22 +1,23 @@ import mlflow from mlflow import MlflowClient -import os import logging import asyncio +from const import CHAMPION_ALIAS, CHALLENGER_ALIAS +from config import AppSettings + _logger = logging.getLogger(__name__) -def connect_to_mlflow(): - tracking_uri = os.environ.get("MLFLOW_TRACKING_URI") +def connect_to_mlflow(settings: AppSettings): + tracking_uri = settings.mlflow_tracking_uri if tracking_uri: mlflow.set_tracking_uri(tracking_uri) _logger.info(f"Tracking URI: {mlflow.get_tracking_uri()}") _logger.info(f"Model Registry URI: {mlflow.get_registry_uri()}") -from const import CHAMPION_ALIAS, CHALLENGER_ALIAS def is_deployable_alias(aliases: list[str]) -> bool: if not aliases: diff --git a/mindctrl-addon/rootfs/usr/bin/multiserver/mlmodels.py b/mindctrl-addon/rootfs/usr/bin/multiserver/mlmodels.py index 15b3e9b..9947d52 100644 --- a/mindctrl-addon/rootfs/usr/bin/multiserver/mlmodels.py +++ b/mindctrl-addon/rootfs/usr/bin/multiserver/mlmodels.py @@ -1,5 +1,4 @@ import logging -import asyncio from typing import Tuple import mlflow diff --git a/mindctrl-addon/rootfs/usr/bin/multiserver/mqtt.py b/mindctrl-addon/rootfs/usr/bin/multiserver/mqtt.py index 855cf37..6ee7c31 100644 --- a/mindctrl-addon/rootfs/usr/bin/multiserver/mqtt.py +++ b/mindctrl-addon/rootfs/usr/bin/multiserver/mqtt.py @@ -1,11 +1,12 @@ import collections import json -import os import logging from typing import Callable, Awaitable, Optional import aiomqtt import asyncio +from config import MqttEventsSettings + _logger = logging.getLogger(__name__) @@ -26,17 +27,25 @@ async def listen_to_mqtt( # This convoluted construction is to handle connection errors as the client context needs to be re-entered while True: try: - print(f"Connecting to MQTT Broker {client}...") + print( + f"Connecting as {client.identifier} to MQTT broker {client._hostname}:{client._port}..." + ) async with client: + topic = "hass_ak/#" print("Connected to MQTT Broker, subscribing to topics ...") - await client.subscribe("hass_ak/#") - print("Subscribed to topics") + await client.subscribe(topic=topic) + print(f"Subscribed to topic {topic}") async for msg in client.messages: _logger.debug(f"{msg.topic} {msg.payload}") if not isinstance(msg.payload, bytes): _logger.warning(f"Message payload is not bytes: {msg.payload}") continue - data: dict = json.loads(msg.payload.decode("utf-8")) + try: + data: dict = json.loads(msg.payload.decode("utf-8")) + except json.JSONDecodeError: + print(f"UNDECODABLE MESSAGE:\n{msg.payload.decode('utf-8')}") + continue + event_type = data.get("event_type", None) if event_type is None: _logger.warning(f"NO EVENT TYPE:\n{data}") @@ -102,15 +111,16 @@ async def listen_to_mqtt( await asyncio.sleep(interval) -def setup_mqtt_client() -> aiomqtt.Client: - broker = os.environ.get("MQTT_BROKER", "localhost") - port = int(os.environ.get("MQTT_PORT", 1883)) - username = os.environ.get("MQTT_USERNAME") - password = os.environ.get("MQTT_PASSWORD") +def setup_mqtt_client(settings: MqttEventsSettings) -> aiomqtt.Client: + user = None + password = None + if settings.username and settings.password: + user = settings.username + password = settings.password.get_secret_value() client = aiomqtt.Client( - hostname=broker, - port=port, - username=username, + hostname=settings.broker, + port=settings.port, + username=user, password=password, logger=_logger.getChild("mqtt_client"), keepalive=60, diff --git a/mindctrl-addon/rootfs/usr/bin/multiserver/rag.py b/mindctrl-addon/rootfs/usr/bin/multiserver/rag.py index ccdbd21..48450f5 100644 --- a/mindctrl-addon/rootfs/usr/bin/multiserver/rag.py +++ b/mindctrl-addon/rootfs/usr/bin/multiserver/rag.py @@ -4,6 +4,7 @@ from fastapi import Request import json from pydantic import BaseModel +from itertools import islice class EventType(Enum): @@ -51,17 +52,16 @@ def extract_timestamps(query_range_response: str) -> tuple[datetime, datetime]: # https://cookbook.openai.com/examples/embedding_long_inputs -from itertools import islice - def batched(iterable, n): """Batch data into tuples of length n. The last batch may be shorter.""" # batched('ABCDEFG', 3) --> ABC DEF G if n < 1: - raise ValueError('n must be at least one') + raise ValueError("n must be at least one") it = iter(iterable) - while (batch := tuple(islice(it, n))): + while batch := tuple(islice(it, n)): yield batch + # TODO: Need to enable this to avoid truncation for embedding # Batched tokenization makes this a bit more interesting # from sentence_transformers import SentenceTransformer diff --git a/mindctrl-addon/rootfs/usr/bin/multiserver/test_data/state_ring_buffer.txt b/mindctrl-addon/rootfs/usr/bin/multiserver/test_data/state_ring_buffer.txt deleted file mode 100644 index d083b02..0000000 --- a/mindctrl-addon/rootfs/usr/bin/multiserver/test_data/state_ring_buffer.txt +++ /dev/null @@ -1,8 +0,0 @@ -{'event_type': 'state_changed', 'event_data': {'entity_id': 'binary_sensor.ai_pro_motion', 'old_state': {'entity_id': 'binary_sensor.ai_pro_motion', 'state': 'off', 'attributes': {'attribution': 'Powered by UniFi Protect Server', 'device_class': 'motion', 'friendly_name': 'AI Pro Motion'}, 'last_changed': '2023-12-24T08:24:41.165914+00:00', 'last_updated': '2023-12-24T08:24:41.165914+00:00', 'context': {'id': '01HJDET8EDQ01CTZK3F9YJ095G', 'parent_id': None, 'user_id': None}}, 'new_state': {'entity_id': 'binary_sensor.ai_pro_motion', 'state': 'on', 'attributes': {'event_id': '6587ed3500d68303e40220a7', 'event_score': 0, 'attribution': 'Powered by UniFi Protect Server', 'device_class': 'motion', 'friendly_name': 'AI Pro Motion'}, 'last_changed': '2023-12-24T08:35:01.242443+00:00', 'last_updated': '2023-12-24T08:35:01.242443+00:00', 'context': {'id': '01HJDFD5ZTEY0RJJKNKNDARS7H', 'parent_id': None, 'user_id': None}}}} -{'event_type': 'state_changed', 'event_data': {'entity_id': 'binary_sensor.garage_door_sensor_motion', 'old_state': {'entity_id': 'binary_sensor.garage_door_sensor_motion', 'state': 'off', 'attributes': {'device_class': 'motion', 'friendly_name': 'Garage door sensor Motion'}, 'last_changed': '2023-12-24T07:52:48.516898+00:00', 'last_updated': '2023-12-24T07:52:48.516898+00:00', 'context': {'id': '01HJDCZWM4PDTZW9W983KB9E89', 'parent_id': None, 'user_id': None}}, 'new_state': {'entity_id': 'binary_sensor.garage_door_sensor_motion', 'state': 'on', 'attributes': {'device_class': 'motion', 'friendly_name': 'Garage door sensor Motion'}, 'last_changed': '2023-12-24T08:35:01.899264+00:00', 'last_updated': '2023-12-24T08:35:01.899264+00:00', 'context': {'id': '01HJDFD6MBMV4C5XEDK1096KR7', 'parent_id': None, 'user_id': None}}}} -{'event_type': 'state_changed', 'event_data': {'entity_id': 'binary_sensor.ai_pro_motion', 'old_state': {'entity_id': 'binary_sensor.ai_pro_motion', 'state': 'on', 'attributes': {'event_id': '6587ed3500d68303e40220a7', 'event_score': 0, 'attribution': 'Powered by UniFi Protect Server', 'device_class': 'motion', 'friendly_name': 'AI Pro Motion'}, 'last_changed': '2023-12-24T08:35:01.242443+00:00', 'last_updated': '2023-12-24T08:35:01.242443+00:00', 'context': {'id': '01HJDFD5ZTEY0RJJKNKNDARS7H', 'parent_id': None, 'user_id': None}}, 'new_state': {'entity_id': 'binary_sensor.ai_pro_motion', 'state': 'on', 'attributes': {'event_id': '6587ed3500d68303e40220a7', 'event_score': 100, 'attribution': 'Powered by UniFi Protect Server', 'device_class': 'motion', 'friendly_name': 'AI Pro Motion'}, 'last_changed': '2023-12-24T08:35:01.242443+00:00', 'last_updated': '2023-12-24T08:35:11.471265+00:00', 'context': {'id': '01HJDFDFZFYH0SV9ZMCC5ZQAVR', 'parent_id': None, 'user_id': None}}}} -{'event_type': 'state_changed', 'event_data': {'entity_id': 'binary_sensor.ai_pro_motion', 'old_state': {'entity_id': 'binary_sensor.ai_pro_motion', 'state': 'on', 'attributes': {'event_id': '6587ed3500d68303e40220a7', 'event_score': 100, 'attribution': 'Powered by UniFi Protect Server', 'device_class': 'motion', 'friendly_name': 'AI Pro Motion'}, 'last_changed': '2023-12-24T08:35:01.242443+00:00', 'last_updated': '2023-12-24T08:35:11.471265+00:00', 'context': {'id': '01HJDFDFZFYH0SV9ZMCC5ZQAVR', 'parent_id': None, 'user_id': None}}, 'new_state': {'entity_id': 'binary_sensor.ai_pro_motion', 'state': 'off', 'attributes': {'attribution': 'Powered by UniFi Protect Server', 'device_class': 'motion', 'friendly_name': 'AI Pro Motion'}, 'last_changed': '2023-12-24T08:35:11.781732+00:00', 'last_updated': '2023-12-24T08:35:11.781732+00:00', 'context': {'id': '01HJDFDG9558Q1Q8FD9SGNBZMW', 'parent_id': None, 'user_id': None}}}} -{'event_type': 'state_changed', 'event_data': {'entity_id': 'binary_sensor.garage_door_sensor_motion', 'old_state': {'entity_id': 'binary_sensor.garage_door_sensor_motion', 'state': 'on', 'attributes': {'device_class': 'motion', 'friendly_name': 'Garage door sensor Motion'}, 'last_changed': '2023-12-24T08:35:01.899264+00:00', 'last_updated': '2023-12-24T08:35:01.899264+00:00', 'context': {'id': '01HJDFD6MBMV4C5XEDK1096KR7', 'parent_id': None, 'user_id': None}}, 'new_state': {'entity_id': 'binary_sensor.garage_door_sensor_motion', 'state': 'off', 'attributes': {'device_class': 'motion', 'friendly_name': 'Garage door sensor Motion'}, 'last_changed': '2023-12-24T08:35:11.826021+00:00', 'last_updated': '2023-12-24T08:35:11.826021+00:00', 'context': {'id': '01HJDFDGAJDPT13FPB6FC9CPCZ', 'parent_id': None, 'user_id': None}}}} -{'event_type': 'call_service', 'event_data': {'domain': 'light', 'service': 'turn_off', 'service_data': {'entity_id': 'light.kitchen_island_pendants'}}} -{'event_type': 'call_service', 'event_data': {'domain': 'light', 'service': 'turn_off', 'service_data': {'entity_id': 'light.kitchen'}}} -{'event_type': 'call_service', 'event_data': {'domain': 'light', 'service': 'turn_off', 'service_data': {'entity_id': 'light.den_main_lights'}}} diff --git a/mindctrl-addon/rootfs/usr/bin/multiserver/test_multiserver.py b/mindctrl-addon/rootfs/usr/bin/multiserver/test_multiserver.py deleted file mode 100644 index e71cbdf..0000000 --- a/mindctrl-addon/rootfs/usr/bin/multiserver/test_multiserver.py +++ /dev/null @@ -1,97 +0,0 @@ -from fastapi.testclient import TestClient -import pandas as pd -import logging - -from .main import app - -import pytest -import time -from dotenv import load_dotenv - -import paho.mqtt.client as mqtt -import uuid -import os - -_logger = logging.getLogger(__name__) - - -@pytest.fixture(scope="session", autouse=True) -def load_env(): - load_dotenv() - - -def test_read_root(mosquitto, monkeypatch): - monkeypatch.setenv("MQTT_BROKER", "localhost") - with TestClient(app) as client: - response = client.get("/") - assert response.status_code == 200 - assert list(response.json().keys()) == ["Tracking Store", "Model Registry"] - - -def test_score_model(mosquitto, monkeypatch): - df = pd.DataFrame({"animal": ["cats", "dogs"]}) - payload = {"dataframe_split": df.to_dict(orient="split")} - monkeypatch.setenv("MQTT_BROKER", "localhost") - - with TestClient(app) as client: - response = client.post( - "/deployed-models/chat/versions/1/invocations", - json=payload, - ) - assert response.status_code == 200 - jokes = response.json() - assert len(jokes) == 2 - assert "cat" in jokes[0] - assert "dog" in jokes[1] - - -def test_summarize(mosquitto, monkeypatch): - df = pd.DataFrame( - { - "query": [ - "Has there been any motion near the garage door?", - "What rooms have activity?", - "Should any room have its lights on right now? If so, which ones and why?", - ] - } - ) - payload = {"dataframe_split": df.to_dict(orient="split")} - monkeypatch.setenv("MQTT_BROKER", "localhost") - - import paho.mqtt.publish as publish - - msgs = [] - - from pathlib import Path - - state_ring_file = Path(__file__).parent / "test_data" / "state_ring_buffer.txt" - with open(state_ring_file, "r") as f: - state_lines = f.readlines() - for event in state_lines: - msgs.append({"topic": "hass_ak", "payload": event.encode("utf-8")}) - - publish.multiple(msgs) - - with TestClient(app) as client: - # state_len = 0 - # while state_len < 20: - # time.sleep(5) - # response = client.get("/state") - # assert response.status_code == 200 - # state_len = len(response.json()) - # _logger.debug(f"Current state buffer length: {state_len}") - - response = client.post( - "/deployed-models/chat/labels/latest/invocations", - json=payload, - ) - _logger.debug(response.json()) - assert response.status_code == 200 - answers = response.json() - assert len(answers) == len(df) - assert "garage" in answers[0] - assert "bedroom" in answers[1] - assert "motion sensor" in answers[1] - assert "no room" in answers[2].lower() - for answer in answers: - print(answer) diff --git a/mindctrl-addon/test-requirements.txt b/mindctrl-addon/test-requirements.txt new file mode 100644 index 0000000..68a3afa --- /dev/null +++ b/mindctrl-addon/test-requirements.txt @@ -0,0 +1,12 @@ +# https://raw.githubusercontent.com/mlflow/mlflow/master/requirements/gateway-requirements.txt + +pytest +pytest-kubernetes +pytest-dotenv +pytest-mqtt +testcontainers-postgres +aiomqtt +pytest-asyncio +httpx +ruff +-r requirements.txt diff --git a/mindctrl-addon/tests/conftest.py b/mindctrl-addon/tests/conftest.py new file mode 100644 index 0000000..eb1c6c5 --- /dev/null +++ b/mindctrl-addon/tests/conftest.py @@ -0,0 +1,132 @@ +import logging +import time +from pydantic import SecretStr +import pytest +import multiprocessing +from uvicorn import Config, Server +import httpx +import sqlalchemy + +from config import AppSettings, MqttEventsSettings, PostgresStoreSettings + +_logger = logging.getLogger(__name__) + + +# @pytest.fixture(scope="session", autouse=True) +# def load_env(): +# load_dotenv() + + +@pytest.fixture(scope="session") +def monkeypatch_session(): + from _pytest.monkeypatch import MonkeyPatch + + m = MonkeyPatch() + yield m + m.undo() + + +@pytest.fixture(scope="session") +def postgres(): + from testcontainers.postgres import PostgresContainer + + _logger.info("Starting postgres fixture") + postgres = PostgresContainer( + image="timescale/timescaledb-ha:pg16-all-oss", + user="test-mindctrl", + password="test-password", + dbname="test-mindctrl", + ) + with postgres as p: + engine = sqlalchemy.create_engine(p.get_connection_url()) + with engine.begin() as connection: + result = connection.execute(sqlalchemy.text("select version()")) + (version,) = result.fetchone() # pyright: ignore + print(version) + yield p + + +class UvicornServer(multiprocessing.Process): + def __init__(self, config: Config): + super().__init__() + self.server = Server(config=config) + self.config = config + + def stop(self): + self.terminate() + + def run(self, *args, **kwargs): + self.server.run() + + +@pytest.fixture(scope="session") +def hosting_settings(mosquitto, postgres, monkeypatch_session): + mqtt_host, mqtt_port = mosquitto + + db_url = sqlalchemy.engine.url.make_url(postgres.get_connection_url()) + + with monkeypatch_session.context() as m: + m.setenv("STORE__STORE_TYPE", "psql") + m.setenv("STORE__USER", db_url.username) + m.setenv("STORE__PASSWORD", db_url.password) + m.setenv("STORE__ADDRESS", db_url.host) + m.setenv( + "STORE__PORT", str(db_url.port) + ) # testcontainers spins up on random ports + m.setenv("STORE__DATABASE", db_url.database) + m.setenv("EVENTS__EVENTS_TYPE", "mqtt") + m.setenv("EVENTS__BROKER", mqtt_host) + m.setenv("EVENTS__PORT", str(mqtt_port)) + # m.setenv("OPENAI_API_KEY", "key") + + # TODO: maybe just take a connection string as a setting instead of exploded + yield AppSettings( + store=PostgresStoreSettings( + user=postgres.POSTGRES_USER, + password=postgres.POSTGRES_PASSWORD, + address="localhost", + port=5432, + database=postgres.POSTGRES_DB, + store_type="psql", + ), + events=MqttEventsSettings( + events_type="mqtt", broker=mqtt_host, port=mqtt_port + ), + openai_api_key=SecretStr("key"), + ) + + +@pytest.fixture(scope="session") +def shared_server_url(hosting_settings: AppSettings): + # For typing + assert hosting_settings.store.store_type == "psql" + assert hosting_settings.events.events_type == "mqtt" + + host = "127.0.0.1" + port = 5002 + base_url = f"http://{host}:{port}" + + config = Config("main:app", host=host, port=port, log_level="debug") + server = UvicornServer(config=config) + _logger.info("Starting shared multiserver fixture") + server.start() + max_attempts = 20 + attempts = 1 + while attempts < max_attempts: + try: + httpx.get(base_url) + break + except httpx.ConnectError: + print("Waiting for shared multiserver fixture startup..") + attempts += 1 + time.sleep(2) + + yield base_url + + server.stop() + + +@pytest.fixture +async def server_client(shared_server_url): + async with httpx.AsyncClient(base_url=shared_server_url) as client: + yield client diff --git a/mindctrl-addon/tests/pytest.ini b/mindctrl-addon/tests/pytest.ini new file mode 100644 index 0000000..7de80eb --- /dev/null +++ b/mindctrl-addon/tests/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +asyncio_mode=auto +pythonpath = ../rootfs/usr/bin/multiserver diff --git a/mindctrl-addon/tests/test_appsettings.py b/mindctrl-addon/tests/test_appsettings.py new file mode 100644 index 0000000..cf4ff7a --- /dev/null +++ b/mindctrl-addon/tests/test_appsettings.py @@ -0,0 +1,60 @@ +from pydantic_core import ValidationError +from config import AppSettings + +import pytest + + +def test_basic_appsettings(monkeypatch): + monkeypatch.setenv("STORE__STORE_TYPE", "psql") + monkeypatch.setenv("STORE__USER", "user") + monkeypatch.setenv("STORE__PASSWORD", "test_password") + monkeypatch.setenv("STORE__ADDRESS", "localhost") + monkeypatch.setenv("STORE__PORT", "5432") + monkeypatch.setenv("STORE__DATABASE", "mindctrl") + monkeypatch.setenv("EVENTS__EVENTS_TYPE", "mqtt") + monkeypatch.setenv("EVENTS__BROKER", "localhost") + monkeypatch.setenv("EVENTS__PORT", "1883") + monkeypatch.setenv("EVENTS__USERNAME", "user") + monkeypatch.setenv("EVENTS__PASSWORD", "test_password") + monkeypatch.setenv("OPENAI_API_KEY", "key") + settings = AppSettings() # pyright: ignore + assert settings.store.store_type == "psql" + assert settings.store.user == "user" + assert settings.store.password.get_secret_value() == "test_password" + assert settings.store.address == "localhost" + assert settings.store.port == 5432 + assert settings.store.database == "mindctrl" + assert settings.events.events_type == "mqtt" + assert settings.events.broker == "localhost" + assert settings.events.port == 1883 + assert settings.events.username == "user" + assert settings.events.password is not None + assert settings.events.password.get_secret_value() == "test_password" + assert settings.openai_api_key.get_secret_value() == "key" + assert not settings.force_publish_models + assert settings.notify_fd is None + assert settings.include_challenger_models + assert settings.mlflow_tracking_uri is None + assert "test_password" not in f"{settings.model_dump()}" + + +def test_invalid_store(monkeypatch): + monkeypatch.setenv("STORE__STORE_TYPE", "sqlite") + monkeypatch.setenv("EVENTS__EVENTS_TYPE", "mqtt") + with pytest.raises( + ValidationError, + match="Input tag 'sqlite' found using 'store_type' does not match any of the expected tags:", + ): + settings = AppSettings() # pyright: ignore + print(settings) + + +def test_invalid_events(monkeypatch): + monkeypatch.setenv("STORE__STORE_TYPE", "psql") + monkeypatch.setenv("EVENTS__EVENTS_TYPE", "kafka") + with pytest.raises( + ValidationError, + match="Input tag 'kafka' found using 'events_type' does not match any of the expected tags:", + ): + settings = AppSettings() # pyright: ignore + print(settings) diff --git a/mindctrl-addon/tests/test_data/state_ring_buffer.txt b/mindctrl-addon/tests/test_data/state_ring_buffer.txt new file mode 100644 index 0000000..b67b54d --- /dev/null +++ b/mindctrl-addon/tests/test_data/state_ring_buffer.txt @@ -0,0 +1,8 @@ +{"event_type": "state_changed", "event_data": {"entity_id": "binary_sensor.ai_pro_motion", "old_state": {"entity_id": "binary_sensor.ai_pro_motion", "state": "off", "attributes": {"attribution": "Powered by UniFi Protect Server", "device_class": "motion", "friendly_name": "AI Pro Motion"}, "last_changed": "2023-12-24T08:24:41.165914+00:00", "last_updated": "2023-12-24T08:24:41.165914+00:00", "context": {"id": "01HJDET8EDQ01CTZK3F9YJ095G", "parent_id": null, "user_id": null}}, "new_state": {"entity_id": "binary_sensor.ai_pro_motion", "state": "on", "attributes": {"event_id": "6587ed3500d68303e40220a7", "event_score": 0, "attribution": "Powered by UniFi Protect Server", "device_class": "motion", "friendly_name": "AI Pro Motion"}, "last_changed": "2023-12-24T08:35:01.242443+00:00", "last_updated": "2023-12-24T08:35:01.242443+00:00", "context": {"id": "01HJDFD5ZTEY0RJJKNKNDARS7H", "parent_id": null, "user_id": null}}}} +{"event_type": "state_changed", "event_data": {"entity_id": "binary_sensor.garage_door_sensor_motion", "old_state": {"entity_id": "binary_sensor.garage_door_sensor_motion", "state": "off", "attributes": {"device_class": "motion", "friendly_name": "Garage door sensor Motion"}, "last_changed": "2023-12-24T07:52:48.516898+00:00", "last_updated": "2023-12-24T07:52:48.516898+00:00", "context": {"id": "01HJDCZWM4PDTZW9W983KB9E89", "parent_id": null, "user_id": null}}, "new_state": {"entity_id": "binary_sensor.garage_door_sensor_motion", "state": "on", "attributes": {"device_class": "motion", "friendly_name": "Garage door sensor Motion"}, "last_changed": "2023-12-24T08:35:01.899264+00:00", "last_updated": "2023-12-24T08:35:01.899264+00:00", "context": {"id": "01HJDFD6MBMV4C5XEDK1096KR7", "parent_id": null, "user_id": null}}}} +{"event_type": "state_changed", "event_data": {"entity_id": "binary_sensor.ai_pro_motion", "old_state": {"entity_id": "binary_sensor.ai_pro_motion", "state": "on", "attributes": {"event_id": "6587ed3500d68303e40220a7", "event_score": 0, "attribution": "Powered by UniFi Protect Server", "device_class": "motion", "friendly_name": "AI Pro Motion"}, "last_changed": "2023-12-24T08:35:01.242443+00:00", "last_updated": "2023-12-24T08:35:01.242443+00:00", "context": {"id": "01HJDFD5ZTEY0RJJKNKNDARS7H", "parent_id": null, "user_id": null}}, "new_state": {"entity_id": "binary_sensor.ai_pro_motion", "state": "on", "attributes": {"event_id": "6587ed3500d68303e40220a7", "event_score": 100, "attribution": "Powered by UniFi Protect Server", "device_class": "motion", "friendly_name": "AI Pro Motion"}, "last_changed": "2023-12-24T08:35:01.242443+00:00", "last_updated": "2023-12-24T08:35:11.471265+00:00", "context": {"id": "01HJDFDFZFYH0SV9ZMCC5ZQAVR", "parent_id": null, "user_id": null}}}} +{"event_type": "state_changed", "event_data": {"entity_id": "binary_sensor.ai_pro_motion", "old_state": {"entity_id": "binary_sensor.ai_pro_motion", "state": "on", "attributes": {"event_id": "6587ed3500d68303e40220a7", "event_score": 100, "attribution": "Powered by UniFi Protect Server", "device_class": "motion", "friendly_name": "AI Pro Motion"}, "last_changed": "2023-12-24T08:35:01.242443+00:00", "last_updated": "2023-12-24T08:35:11.471265+00:00", "context": {"id": "01HJDFDFZFYH0SV9ZMCC5ZQAVR", "parent_id": null, "user_id": null}}, "new_state": {"entity_id": "binary_sensor.ai_pro_motion", "state": "off", "attributes": {"attribution": "Powered by UniFi Protect Server", "device_class": "motion", "friendly_name": "AI Pro Motion"}, "last_changed": "2023-12-24T08:35:11.781732+00:00", "last_updated": "2023-12-24T08:35:11.781732+00:00", "context": {"id": "01HJDFDG9558Q1Q8FD9SGNBZMW", "parent_id": null, "user_id": null}}}} +{"event_type": "state_changed", "event_data": {"entity_id": "binary_sensor.garage_door_sensor_motion", "old_state": {"entity_id": "binary_sensor.garage_door_sensor_motion", "state": "on", "attributes": {"device_class": "motion", "friendly_name": "Garage door sensor Motion"}, "last_changed": "2023-12-24T08:35:01.899264+00:00", "last_updated": "2023-12-24T08:35:01.899264+00:00", "context": {"id": "01HJDFD6MBMV4C5XEDK1096KR7", "parent_id": null, "user_id": null}}, "new_state": {"entity_id": "binary_sensor.garage_door_sensor_motion", "state": "off", "attributes": {"device_class": "motion", "friendly_name": "Garage door sensor Motion"}, "last_changed": "2023-12-24T08:35:11.826021+00:00", "last_updated": "2023-12-24T08:35:11.826021+00:00", "context": {"id": "01HJDFDGAJDPT13FPB6FC9CPCZ", "parent_id": null, "user_id": null}}}} +{"event_type": "call_service", "event_data": {"domain": "light", "service": "turn_off", "service_data": {"entity_id": "light.kitchen_island_pendants"}}} +{"event_type": "call_service", "event_data": {"domain": "light", "service": "turn_off", "service_data": {"entity_id": "light.kitchen"}}} +{"event_type": "call_service", "event_data": {"domain": "light", "service": "turn_off", "service_data": {"entity_id": "light.den_main_lights"}}} diff --git a/mindctrl-addon/tests/test_multiserver.py b/mindctrl-addon/tests/test_multiserver.py new file mode 100644 index 0000000..299cc3b --- /dev/null +++ b/mindctrl-addon/tests/test_multiserver.py @@ -0,0 +1,71 @@ +import pandas as pd +import logging +from pathlib import Path +import aiomqtt + + +_logger = logging.getLogger(__name__) + + +async def test_read_root(server_client): + response = await server_client.get("/") + assert response.status_code == 200 + assert response.content is not None + assert response.text is not None + assert "mlflowContent" in response.text + + +async def test_read_version(server_client): + response = await server_client.get("/version") + assert response.status_code == 200 + assert response.content is not None + version_data: dict = response.json() + assert "tracking_store" in version_data.keys() + assert "model_registry" in version_data.keys() + assert "ws_url" in version_data.keys() + assert "chat_url" in version_data.keys() + assert "mlflow_url" in version_data.keys() + assert "dashboard_url" in version_data.keys() + + +async def test_summarize(server_client, hosting_settings): + df = pd.DataFrame( + { + "query": [ + "Has there been any motion near the garage door?", + "What rooms have activity?", + "Should any room have its lights on right now? If so, which ones and why?", + ] + } + ) + payload = {"dataframe_split": df.to_dict(orient="split")} + + state_ring_file = Path(__file__).parent / "test_data" / "state_ring_buffer.txt" + with open(state_ring_file, "r") as f: + state_lines = f.readlines() + async with aiomqtt.Client( + hostname=hosting_settings.events.broker, port=hosting_settings.events.port + ) as client: + for event in state_lines: + await client.publish(topic="hass_ak", payload=event.encode("utf-8")) + + response = await server_client.post( + "/deployed-models/chat/labels/latest/invocations", + json=payload, + timeout=120.0, # TODO: switch to streaming so we don't need silly long timeouts + ) + _logger.debug(response.json()) + assert response.status_code == 200 + answers = response.json() + assert len(answers) == len(df) + + for answer in answers: + print(answer) + + assert "Yes" in answers[0] + assert "garage" in answers[0] + + # This is super flaky + assert "garage" in answers[1].lower() + + assert "no room" in answers[2].lower()