From 387c7da6ef076fac3a6099c88a47a2f5ad38bb0d Mon Sep 17 00:00:00 2001 From: apentori Date: Mon, 11 May 2026 13:19:41 +0200 Subject: [PATCH 01/13] add refactoring plan Signed-off-by: apentori --- module-refactoring.md | 591 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 591 insertions(+) create mode 100644 module-refactoring.md diff --git a/module-refactoring.md b/module-refactoring.md new file mode 100644 index 0000000..a1e347a --- /dev/null +++ b/module-refactoring.md @@ -0,0 +1,591 @@ +# Refactoring Plan: Status Bot Modular Architecture + +## Overview + +Transform `monitor.py` from a monolithic script into a modular plugin-based system where modules are independent, configurable, and can run in periodic or event-driven mode. + +## Architecture Design + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Main Bot │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────┐ │ +│ │ Config │ │ Account │ │ Module Manager │ │ +│ │ Loader │ │ (shared) │ │ (loads/runs) │ │ +│ └──────────────┘ └──────────────┘ └──────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + │ │ │ + ▼ ▼ ▼ +┌─────────────────────────────────────────────────────────────┐ +│ Modules (plugins) │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ Monitoring │ │ Storage │ │ Custom │ ... │ +│ │ (periodic) │ │ (periodic)│ │ (event-driven) │ +│ └─────────────┘ └─────────────┘ └─────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Key Components + +### 1. Base Module Interface (`bot/modules/base.py`) + +- Abstract base class defining module contract +- Required methods: `run()`, `get_name()`, `get_config()` +- Optional: `on_start()`, `on_stop()`, `on_event()` + +### 2. Module Manager (`bot/modules/manager.py`) + +- Discovers modules from configured directory +- Loads/enables/disables based on config +- Handles lifecycle (init → start → run → stop) +- Isolates errors per module + +### 3. Configuration (`config.yaml`) + +```yaml +modules: + enabled: + - monitoring + - storage + directories: + - ./modules + settings: + monitoring: + interval: 600 # seconds + storage: + batch_size: 100 +``` + +### 4. Module Structure (`modules/`) + +- Each module is a Python file with a Module class +- Auto-discovery via naming convention or decorator + +## Module Types Support + +| Module Type | Execution | Example | +|-------------|-----------|---------| +| `periodic` | Runs on interval | Monitoring, Storage | +| `event` | Reacts to signals | Mentions, Analytics | +| `service` | Long-running | WebSocket listeners | + +## Implementation Steps + +### Phase 1: Core Framework + +#### 1. Directory Structure + +``` +bot/ +├── __init__.py +├── account.py +├── signal.py +├── logger.py +└── modules/ + ├── __init__.py + ├── base.py # BaseModule abstract class + └── manager.py # ModuleManager class +modules/ # Custom modules directory (user-created) + ├── __init__.py + └── example.py +``` + +#### 2. BaseModule Abstract Class (`bot/modules/base.py`) + +```python +from abc import ABC, abstractmethod +from typing import Any, Optional +from dataclasses import dataclass +from enum import Enum +import logging + + +class ModuleType(Enum): + """Defines how the module is executed.""" + PERIODIC = "periodic" # Runs on a fixed interval + EVENT = "event" # Reacts to signals/events + SERVICE = "service" # Long-running service + + +@dataclass +class ModuleConfig: + """Configuration for a single module.""" + name: str + enabled: bool = True + interval: int = 60 # seconds, for periodic modules + settings: dict = None # module-specific settings + + def __post_init__(self): + if self.settings is None: + self.settings = {} + + +class BaseModule(ABC): + """ + Abstract base class for all bot modules. + + Subclass this to create custom modules. Implement the required methods. + """ + + def __init__( + self, + config: ModuleConfig, + account: "Account", + logger: logging.Logger + ): + """ + Initialize the module. + + Args: + config: Module configuration from config.yaml + account: Shared Status account instance + logger: Shared logger instance + """ + self._config = config + self._account = account + self._logger = logger + self._running = False + + @property + def name(self) -> str: + """Module name, used for identification.""" + return self._config.name + + @property + def module_type(self) -> ModuleType: + """Return the type of module. Override in subclass.""" + return ModuleType.PERIODIC + + @property + def interval(self) -> int: + """Interval in seconds for periodic modules.""" + return self._config.interval + + @abstractmethod + def run(self) -> Any: + """ + Execute the module logic. + + This is called either: + - Periodically (for PERIODIC type) + - On event (for EVENT type) + - Once and keeps running (for SERVICE type) + + Returns: + Any result from the module execution + """ + pass + + def on_start(self) -> None: + """ + Called once when the module starts. + Override for initialization logic. + """ + pass + + def on_stop(self) -> None: + """ + Called once when the module stops. + Override for cleanup logic. + """ + pass + + def on_event(self, event: dict) -> Any: + """ + Handle an event signal (for EVENT type modules). + + Override to handle specific signals like 'messages.new'. + + Args: + event: The event data from Signal + + Returns: + Any result from handling the event + """ + return None + + def is_running(self) -> bool: + """Check if module is currently running.""" + return self._running + + def _set_running(self, value: bool) -> None: + """Internal: update running state.""" + self._running = value +``` + +#### 3. Module Manager (`bot/modules/manager.py`) + +```python +import os +import importlib.util +import logging +from typing import Type, Optional, List, Dict, Any +from pathlib import Path + +from .base import BaseModule, ModuleConfig, ModuleType +from bot import Account + + +class ModuleManager: + """ + Manages module discovery, loading, and lifecycle. + + Handles: + - Discovering modules from configured directories + - Loading and initializing modules + - Running modules according to their type + - Error isolation between modules + """ + + def __init__( + self, + config: dict, + account: Account, + logger: logging.Logger + ): + """ + Initialize the module manager. + + Args: + config: The full bot configuration dict + account: Shared Account instance + logger: Shared logger instance + """ + self._config = config + self._account = account + self._logger = logger + self._modules: Dict[str, BaseModule] = {} + self._module_classes: Dict[str, Type[BaseModule]] = {} + + @property + def modules(self) -> Dict[str, BaseModule]: + """Get all loaded modules.""" + return self._modules + + def discover_modules(self) -> None: + """Discover available module classes from configured directories.""" + modules_config = self._config.get("modules", {}) + directories = modules_config.get("directories", ["modules"]) + enabled = modules_config.get("enabled", []) + + for directory in directories: + self._discover_from_directory(directory) + + self._logger.info( + f"Discovered {len(self._module_classes)} module classes: " + f"{list(self._module_classes.keys())}" + ) + + def _discover_from_directory(self, directory: str) -> None: + """Discover modules from a specific directory.""" + base_path = Path(directory) + if not base_path.exists(): + self._logger.warning(f"Module directory not found: {directory}") + return + + for file_path in base_path.glob("*.py"): + if file_path.name.startswith("_"): + continue + self._load_module_from_file(file_path) + + def _load_module_from_file(self, file_path: Path) -> None: + """Load a module class from a Python file.""" + module_name = file_path.stem + + spec = importlib.util.spec_from_file_location( + f"modules.{module_name}", file_path + ) + if spec is None or spec.loader is None: + return + + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + for attr_name in dir(module): + attr = getattr(module, attr_name) + if ( + isinstance(attr, type) + and issubclass(attr, BaseModule) + and attr is not BaseModule + ): + self._module_classes[module_name] = attr + self._logger.debug(f"Found module class: {module_name}.{attr_name}") + + def load_modules(self) -> None: + """Load and initialize all enabled modules.""" + modules_config = self._config.get("modules", {}) + enabled = set(modules_config.get("enabled", [])) + settings = modules_config.get("settings", {}) + + for module_name in enabled: + if module_name not in self._module_classes: + self._logger.error( + f"Module '{module_name}' not found. " + f"Available: {list(self._module_classes.keys())}" + ) + continue + + module_class = self._module_classes[module_name] + module_settings = settings.get(module_name, {}) + + module_config = ModuleConfig( + name=module_name, + enabled=True, + interval=module_settings.get("interval", 60), + settings=module_settings + ) + + try: + module = module_class(module_config, self._account, self._logger) + self._modules[module_name] = module + self._logger.info(f"Loaded module: {module_name}") + except Exception as e: + self._logger.error(f"Failed to load module '{module_name}': {e}") + + def start_modules(self) -> None: + """Call on_start for all modules.""" + for module in self._modules.values(): + try: + module.on_start() + except Exception as e: + self._logger.error( + f"Error starting module '{module.name}': {e}" + ) + + def run_modules(self) -> None: + """ + Run all modules according to their type. + + - PERIODIC: runs in a loop with configured interval + - EVENT: registers with signal listener + - SERVICE: runs once and keeps running + """ + for module in self._modules.values(): + try: + if module.module_type == ModuleType.PERIODIC: + self._run_periodic_module(module) + elif module.module_type == ModuleType.EVENT: + self._run_event_module(module) + elif module.module_type == ModuleType.SERVICE: + self._run_service_module(module) + except Exception as e: + self._logger.error( + f"Error running module '{module.name}': {e}" + ) + + def _run_periodic_module(self, module: BaseModule) -> None: + """Run a periodic module on its interval.""" + import time + while True: + try: + module.run() + except Exception as e: + self._logger.error( + f"Error in periodic module '{module.name}': {e}" + ) + time.sleep(module.interval) + + def _run_event_module(self, module: BaseModule) -> None: + """Run an event-driven module by listening to signals.""" + for event in self._account.signal.listen("messages.new"): + try: + module.on_event(event) + except Exception as e: + self._logger.error( + f"Error handling event in module '{module.name}': {e}" + ) + + def _run_service_module(self, module: BaseModule) -> None: + """Run a service module once (blocking).""" + module.run() + + def stop_modules(self) -> None: + """Call on_stop for all modules.""" + for module in self._modules.values(): + try: + module.on_stop() + except Exception as e: + self._logger.error( + f"Error stopping module '{module.name}': {e}" + ) +``` + +#### 4. Configuration Example (`config.yaml`) + +```yaml +postgres: + schema: "status_app_monitoring" + tables: + messages: "raw_messages" + community: "raw_community_info" + +sleep: 10 + +files: + current_state: "dates.pkl" + +bot: + public_key: "0x041658626a9e1303b631f6d0fb1e047211d5603b977454f7d5d29fe583c3d6c1bd3d8e395d67f6c44b5bc659aae912040e9dd8164b5107368a29029cb53389d8b0" + compressed_key: "zQ3shNv1tnajHo5FvCvP662cWcbBfS5ZejB4TWaH9iAuFCZZe" + params: + domain: "status-backend" + port: 8080 + is_secure: false + +modules: + directories: + - ./modules + enabled: + - monitoring + - storage + settings: + monitoring: + interval: 600 + storage: + batch_size: 100 +``` + +#### 5. Example Custom Module (`modules/example.py`) + +```python +from bot.modules.base import BaseModule, ModuleConfig, ModuleType +from typing import Any + + +class ExampleModule(BaseModule): + """ + Example module that demonstrates the module interface. + + This module logs a message every interval seconds. + """ + + @property + def module_type(self) -> ModuleType: + return ModuleType.PERIODIC + + def run(self) -> Any: + """Run the module logic.""" + self._logger.info(f"Example module running for {self._account.info['display_name']}") + # Custom logic here: + # - Process messages + # - Send notifications + # - Store data + # - etc. + + def on_start(self) -> None: + """Called when module starts.""" + self._logger.info(f"Starting example module with interval {self.interval}s") + + def on_stop(self) -> None: + """Called when module stops.""" + self._logger.info("Stopping example module") +``` + +#### 6. Updated Entry Point (`monitor.py`) + +```python +import os, yaml, time +from dotenv import load_dotenv + +from bot import Account, Logger +from bot.modules.manager import ModuleManager + + +def load_config(file_path: str) -> dict: + """Load the config file and the `.env` variables.""" + with open(file_path, "r") as f: + config = yaml.safe_load(f) + + env_file_path = os.path.join(os.path.dirname(file_path), ".env") + load_dotenv(env_file_path) + + config["env_vars"] = { + key: value + for key, value in os.environ.items() + if key.startswith(("POSTGRES_", "STATUS_")) + } + + return config + + +def create_bot(config: dict) -> Account: + """Initialize a logged in bot account.""" + params = config.get("bot", {}).get("params", {}) + account = Account(**params) + + available_accounts = [acc["display_name"] for acc in account.available_accounts] + + prefix = "STATUS_" + params = { + key.replace(prefix, "").lower(): value + for key, value in config["env_vars"].items() + if key.startswith(prefix) + } + if params["display_name"] in available_accounts: + params.pop("mnemonic") + + account.login(**params) + if account.info["compressed_key"] != config["bot"]["compressed_key"]: + raise Exception("Target compressed key and logged in compressed key are different") + + account.profile_picture = os.path.join(os.path.dirname(__file__), "assets", "profile.jpg") + account.logger.info( + f"Account Information:\n" + f"Compressed Key: {account.info['compressed_key']}\n" + f"Public Key: {account.info['public_key']}\n" + f"URL: {account.info['url']}" + ) + return account + + +if __name__ == "__main__": + folder = os.path.dirname(__file__) + config = load_config(os.path.join(folder, "config.yaml")) + logger = Logger() + account = create_bot(config) + + # Initialize module manager + manager = ModuleManager(config, account, logger) + manager.discover_modules() + manager.load_modules() + manager.start_modules() + + try: + manager.run_modules() + except KeyboardInterrupt: + logger.info("Shutting down...") + manager.stop_modules() +``` + +### Phase 2: Main Refactor + +1. Rewrite `monitor.py` as entry point +2. Move `download()` → `MonitoringModule` +3. Move `store()` → `StorageModule` + +### Phase 3: Module System + +1. Implement module discovery from directory +2. Add event-driven support via Signal class +3. Add error isolation (try/except per module) + +### Phase 4: Testing & Docs + +1. Create example custom module +2. Add module API documentation + +## Backward Compatibility + +- Keep `config.yaml` structure compatible (add `modules` section) +- Current functionality preserved as built-in modules +- Existing `.env` works unchanged + +## Requirements from User + +1. **Module flexibility**: Anyone should be able to create their own module (store messages, react to mentions, analytics, automated messages) +2. **Configuration**: Modules configured in config.yaml but loaded from directory +3. **Execution**: Independent modules (not sequential pipeline) +4. **Execution pattern**: Both periodic and event-driven support +5. **Interface**: BaseModule class with required methods +6. **Error handling**: Isolated (one module failure doesn't crash others) +7. **Shared resources**: Passed through constructor (Account, config, logger) \ No newline at end of file From f95224e3848a6559cd6361820252adff8cfe799a Mon Sep 17 00:00:00 2001 From: apentori Date: Wed, 13 May 2026 12:29:50 +0200 Subject: [PATCH 02/13] refacto v1 Signed-off-by: apentori --- Dockerfile | 5 +- bot/modules/__init__.py | 4 + bot/modules/base.py | 73 +++ bot/modules/manager.py | 201 +++++++ bot/signal.py | 9 +- config.yaml | 11 + docs/refactoring-module-plan.md | 929 ++++++++++++++++++++++++++++++++ main.py | 176 ++++++ module-refactoring.md | 20 +- modules/__init__.py | 0 requirements.txt | 1 + 11 files changed, 1415 insertions(+), 14 deletions(-) create mode 100644 bot/modules/__init__.py create mode 100644 bot/modules/base.py create mode 100644 bot/modules/manager.py create mode 100644 docs/refactoring-module-plan.md create mode 100644 main.py create mode 100644 modules/__init__.py diff --git a/Dockerfile b/Dockerfile index c406581..bd21436 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,9 +5,10 @@ COPY requirements.txt . COPY bot/requirements.txt bot/requirements.txt RUN pip install --no-cache-dir -r requirements.txt \ - && if [ -f bot/requirements.txt ]; then pip install --no-cache-dir -r bot/requirements.txt; fi + && if [ -f bot/requirements.txt ]; then pip install --no-cache-dir -r bot/requirements.txt; fi \ + && mkdir -p modules COPY . . -ENTRYPOINT ["python", "monitor.py"] +ENTRYPOINT ["python", "main.py"] CMD [] diff --git a/bot/modules/__init__.py b/bot/modules/__init__.py new file mode 100644 index 0000000..aa7ce1d --- /dev/null +++ b/bot/modules/__init__.py @@ -0,0 +1,4 @@ +from .base import BaseModule, ModuleType, ModuleConfig, ModuleContext +from .manager import ModuleManager + +__all__ = ["BaseModule", "ModuleType", "ModuleConfig", "ModuleContext", "ModuleManager"] diff --git a/bot/modules/base.py b/bot/modules/base.py new file mode 100644 index 0000000..2ee68f5 --- /dev/null +++ b/bot/modules/base.py @@ -0,0 +1,73 @@ +from abc import ABC, abstractmethod +from typing import Any, Optional +from dataclasses import dataclass, field +from enum import Enum +import threading +import logging + + +class ModuleType(Enum): + PERIODIC = "periodic" + EVENT = "event" + SERVICE = "service" + + +@dataclass +class ModuleConfig: + name: str + enabled: bool = True + interval: int = 60 + max_retries: int = 3 + backoff_seconds: int = 30 + settings: dict = None + + def __post_init__(self): + if self.settings is None: + self.settings = {} + + +@dataclass +class ModuleContext: + account: Any + config: ModuleConfig + logger: logging.Logger + db: Optional[Any] = None + shared_state: dict = field(default_factory=dict) + stop_event: Optional[threading.Event] = None + + +class BaseModule(ABC): + + def __init__(self, ctx: ModuleContext): + self._ctx = ctx + self._running = False + + @property + def ctx(self) -> ModuleContext: + return self._ctx + + @property + @abstractmethod + def module_type(self) -> ModuleType: + ... + + @property + def name(self) -> str: + return self._ctx.config.name + + @abstractmethod + def execute(self) -> Any: + ... + + def on_start(self) -> None: + pass + + def on_stop(self) -> None: + pass + + def on_event(self, event: dict) -> Any: + return None + + @property + def is_running(self) -> bool: + return self._running diff --git a/bot/modules/manager.py b/bot/modules/manager.py new file mode 100644 index 0000000..8d47e4d --- /dev/null +++ b/bot/modules/manager.py @@ -0,0 +1,201 @@ +import os +import importlib.util +import threading +import logging +from typing import Optional, Type +from pathlib import Path + +from .base import BaseModule, ModuleConfig, ModuleContext, ModuleType + + +class ModuleManager: + + def __init__(self, config: dict, account, db, logger: logging.Logger): + self._config = config + self._account = account + self._db = db + self._logger = logger + self._modules: dict[str, BaseModule] = {} + self._module_classes: dict[str, Type[BaseModule]] = {} + self._threads: dict[str, threading.Thread] = {} + self._stop_event = threading.Event() + + @property + def modules(self) -> dict[str, BaseModule]: + return self._modules + + @property + def module_names(self) -> list[str]: + return list(self._modules.keys()) + + def discover_modules(self) -> None: + modules_config = self._config.get("modules", {}) + directories = modules_config.get("directories", ["modules"]) + + for directory in directories: + self._discover_from_directory(directory) + + self._logger.info( + f"Discovered {len(self._module_classes)} module class(es): " + f"{list(self._module_classes.keys())}" + ) + + def _discover_from_directory(self, directory: str) -> None: + base_path = Path(directory) + if not base_path.exists(): + self._logger.warning(f"Module directory not found: {directory}") + return + + for file_path in sorted(base_path.glob("*.py")): + if file_path.name.startswith("_"): + continue + self._load_module_from_file(file_path) + + def _load_module_from_file(self, file_path: Path) -> None: + module_name = file_path.stem + + spec = importlib.util.spec_from_file_location( + f"modules.{module_name}", file_path + ) + if spec is None or spec.loader is None: + return + + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + for attr_name in dir(module): + attr = getattr(module, attr_name) + if ( + isinstance(attr, type) + and issubclass(attr, BaseModule) + and attr is not BaseModule + ): + self._module_classes[module_name] = attr + self._logger.debug(f"Found module class: {module_name}.{attr_name}") + + def load_modules(self) -> None: + modules_config = self._config.get("modules", {}) + enabled = set(modules_config.get("enabled", [])) + settings = modules_config.get("settings", {}) + + if not enabled: + self._logger.info("No modules enabled in config") + return + + for module_name in enabled: + if module_name not in self._module_classes: + self._logger.error( + f"Module '{module_name}' not found. " + f"Available: {list(self._module_classes.keys())}" + ) + continue + + module_class = self._module_classes[module_name] + module_settings = settings.get(module_name, {}) + + module_config = ModuleConfig( + name=module_name, + enabled=True, + interval=module_settings.get("interval", 60), + max_retries=module_settings.get("max_retries", 3), + backoff_seconds=module_settings.get("backoff_seconds", 30), + settings=module_settings, + ) + + ctx = ModuleContext( + account=self._account, + config=module_config, + logger=self._logger, + db=self._db, + stop_event=self._stop_event, + ) + + try: + module = module_class(ctx) + self._modules[module_name] = module + self._logger.info(f"Loaded module: {module_name}") + except Exception as e: + self._logger.error(f"Failed to load module '{module_name}': {e}") + + def start_all(self) -> None: + for name, module in self._modules.items(): + t = threading.Thread( + target=self._run_module_wrapper, + args=(module,), + daemon=True, + name=f"module-{name}", + ) + self._threads[name] = t + t.start() + self._logger.info(f"Started module thread: {name}") + + def stop_all(self) -> None: + self._logger.info("Stopping all modules...") + self._stop_event.set() + for name, t in self._threads.items(): + self._logger.debug(f"Waiting for module '{name}' to stop...") + t.join(timeout=5) + if t.is_alive(): + self._logger.warning(f"Module '{name}' did not stop in time") + self._logger.info("All modules stopped") + + def _run_module_wrapper(self, module: BaseModule) -> None: + retries = 0 + max_retries = module.ctx.config.max_retries + backoff = module.ctx.config.backoff_seconds + + while retries <= max_retries and not self._stop_event.is_set(): + try: + module._running = True + module.on_start() + + if module.module_type == ModuleType.PERIODIC: + self._run_periodic(module) + elif module.module_type == ModuleType.EVENT: + self._run_event(module) + elif module.module_type == ModuleType.SERVICE: + module.execute() + + break + + except Exception as e: + retries += 1 + self._logger.error( + f"Module '{module.name}' failed ({retries}/{max_retries}): {e}", + exc_info=True, + ) + if retries <= max_retries: + wait = backoff * (2 ** (retries - 1)) + self._logger.info(f"Restarting '{module.name}' in {wait}s...") + self._stop_event.wait(wait) + else: + self._logger.error( + f"Module '{module.name}' permanently failed after {max_retries} retries" + ) + finally: + module._running = False + try: + module.on_stop() + except Exception: + pass + + def _run_periodic(self, module: BaseModule) -> None: + interval = module.ctx.config.interval + while not self._stop_event.is_set(): + module.execute() + self._stop_event.wait(interval) + + def _run_event(self, module: BaseModule) -> None: + for event in self._account.signal.listen("messages.new", stop_event=self._stop_event): + if self._stop_event.is_set(): + break + try: + module.on_event(event) + except Exception as e: + self._logger.error( + f"Error in event module '{module.name}': {e}", + exc_info=True, + ) + + def has_alive_modules(self) -> bool: + return any(t.is_alive() for t in self._threads.values()) diff --git a/bot/signal.py b/bot/signal.py index 46b070b..558a56b 100644 --- a/bot/signal.py +++ b/bot/signal.py @@ -120,12 +120,13 @@ def __close_thread(self): if self.__thread.is_alive(): self.__thread.join(1) - def listen(self, signal_type: str): + def listen(self, signal_type: str, stop_event: Optional[threading.Event] = None): """ Listen for a specific Signal forever Parameters: - `signal_type` - the "type" as it is in Status Backend + - `stop_event` - optional threading.Event for graceful shutdown """ self.__signal_type = signal_type ws = websocket.WebSocketApp( @@ -139,7 +140,11 @@ def listen(self, signal_type: str): self.__thread.start() while True: try: - data = self.__queue.get() + data = self.__queue.get(timeout=1) + except queue.Empty: + if stop_event and stop_event.is_set(): + break + continue except KeyboardInterrupt: break if self.__error_message: diff --git a/config.yaml b/config.yaml index 35f763b..2675e0b 100644 --- a/config.yaml +++ b/config.yaml @@ -21,3 +21,14 @@ bot: domain: "status-backend" port: 8080 is_secure: false + +modules: + directories: + - ./modules + enabled: [] + settings: {} + +prometheus: + enabled: false + host: "0.0.0.0" + port: 8000 diff --git a/docs/refactoring-module-plan.md b/docs/refactoring-module-plan.md new file mode 100644 index 0000000..af560ac --- /dev/null +++ b/docs/refactoring-module-plan.md @@ -0,0 +1,929 @@ +# Refactoring Plan: Status Bot Modular Architecture + +> This document captures the full context, design decisions, and step-by-step +> implementation plan for transforming `monitor.py` into a modular plugin-based +> system. It is designed to be self-contained so the refactoring can span +> multiple sessions. + +--- + +## Table of Contents + +1. [Current Architecture](#1-current-architecture) +2. [Issues with the Current Codebase](#2-issues-with-the-current-codebase) +3. [Target Architecture](#3-target-architecture) +4. [Module System Design](#4-module-system-design) + - 4.1 ModuleType + - 4.2 ModuleConfig + - 4.3 ModuleContext + - 4.4 BaseModule (ABC) + - 4.5 ModuleManager +5. [Threading & Lifecycle](#5-threading--lifecycle) +6. [Error Handling & Retry](#6-error-handling--retry) +7. [Configuration Shape](#7-configuration-shape) +8. [Prometheus Metrics](#8-prometheus-metrics) +9. [Signal.listen() Graceful Shutdown](#9-signallisten-graceful-shutdown) +10. [Implementation Steps](#10-implementation-steps) + - [Step 1: Core Framework + Entry Point](#step-1-core-framework--entry-point) + - [Step 2: Refactor monitor.py → MonitoringModule](#step-2-refactor-monitorpy--monitoringmodule) + - [Step 3: Connection Pool + Retry Logic](#step-3-connection-pool--retry-logic) + - [Step 4: Example Event-Driven Module](#step-4-example-event-driven-module) + - [Step 5: Tests + Documentation](#step-5-tests--documentation) +11. [File-by-File Summary](#11-file-by-file-summary) + +--- + +## 1. Current Architecture + +``` +/repos/status-bot/ +├── bot/ +│ ├── __init__.py # Exports Account and Logger +│ ├── account.py # Account class — Status Backend API wrapper (1112 lines) +│ ├── signal.py # Signal class — WebSocket handler for Status signals +│ ├── logger.py # Logger singleton +│ └── requirements.txt +├── modules/ # (does not exist yet) +├── accounts/ +├── assets/ +├── data-dir/ # Status Backend runtime data +├── docs/ +├── tests/ # Empty +├── monitor.py # Main entry point — 316-line monolithic script +├── postgres.py # Postgres connector +├── config.yaml +├── Dockerfile +├── docker-compose.yaml +├── module-refactoring.md # Initial draft (superseded by this document) +└── README.md +``` + +### Current Flow + +`monitor.py` does everything in one file: +1. Config loading (`load_config`) +2. Bot creation (`create_bot`) +3. Community data extraction (`download`) +4. Data storage / upload (`store`) +5. Main loop (while True: download → store → sleep) + +### SDK Layer (`bot/`) + +The `bot/` package is well-structured and reusable: +- **`Account`** — wraps Status Backend HTTP / RPC / WebSocket APIs +- **`Signal`** — WebSocket event handling (single-get + streaming) +- **`Logger`** — singleton logging + +These should remain largely unchanged. + +--- + +## 2. Issues with the Current Codebase + +| Issue | Description | +|-------|-------------| +| **Monolithic** | `monitor.py` combines config, bot init, extraction, storage, and cleanup in one file | +| **Hardcoded pipeline** | `download()` then `store()` — no way to run independently or add middleware | +| **Tight coupling** | `store()` knows about pickle format from `download()`; `latest_dates` is shared implicitly | +| **No error isolation** | One failure can corrupt state (files deleted mid-failure) | +| **Untestable** | Global functions reference filesystem directly; no mocking boundary | +| **No concurrency** | Single-threaded loop — if monitor sleeps 10 min, auto-reply can't react | +| **SQL injection risk** | `postgres.py` uses f-strings for schema/table/column identifiers | +| **No module system** | Adding a new feature means either bloating `monitor.py` or duplicating init code | +| **No graceful shutdown** | No SIGTERM handler; `Account.__del__` is unreliable in Docker | + +--- + +## 3. Target Architecture + +``` +main.py ────────────────────────────────────────────────────────┐ + │ - Load config.yaml & .env │ + │ - Create Account (login) │ + │ - Create Postgres (connection pool) │ + │ - Init ModuleManager(config, account, db, logger) │ + │ - manager.start_all() → returns immediately │ + │ - Start Prometheus HTTP endpoint (/metrics) │ + │ - signal.signal(SIGTERM) → manager.stop_all() │ + │ - Wait forever (or until all modules die) │ + └───────────────────────────────────────────────────────────────┘ + │ + ┌─────────────┴─────────────┐ + │ ModuleManager │ + │ - discover_modules() │ + │ - load_modules() │ + │ - start_all() → threads │ + │ - stop_all() → join │ + └─────────────┬──────────────┘ + │ + ┌──────────────────┼──────────────────┐ + ▼ ▼ ▼ + MonitoringModule AutoReplyModule (user modules) + (PERIODIC, thread) (EVENT, thread) ... + │ │ + │ while running: │ for event in + │ download() │ signal.listen(): + │ store() │ on_event() + │ sleep(N) │ +``` + +### Directory Layout After Refactoring + +``` +status-bot/ +├── bot/ # SDK layer (minimal changes) +│ ├── __init__.py +│ ├── account.py +│ ├── signal.py # ← modified: listen() accepts stop_event +│ ├── logger.py +│ └── modules/ # ← NEW: module framework +│ ├── __init__.py +│ ├── base.py # BaseModule, ModuleType, ModuleConfig, ModuleContext +│ ├── manager.py # ModuleManager +│ └── utils.py # Shared helpers (to_sha256_hash, etc.) +│ +├── modules/ # ← NEW: user-space modules +│ ├── __init__.py +│ ├── monitoring.py # MonitoringModule (extracted from monitor.py) +│ └── auto_reply.py # Example event-driven module +│ +├── main.py # ← NEW: entry point +├── postgres.py # ← modified: connection pool +├── config.yaml # ← modified: + modules: + prometheus: sections +├── Dockerfile # ← modified: ENTRYPOINT → main.py +│ +├── tests/ +│ ├── test_base.py +│ ├── test_manager.py +│ └── test_monitoring_module.py +│ +└── docs/ + └── refactoring-module-plan.md # This document +``` + +--- + +## 4. Module System Design + +### 4.1 ModuleType (Enum) + +Defines how a module is executed by the manager. + +```python +class ModuleType(Enum): + PERIODIC = "periodic" # Runs execute() on a fixed interval in its own thread + EVENT = "event" # Reacts to Signal events via on_event() callback + SERVICE = "service" # Runs once and keeps running (blocking) +``` + +| Type | Execution | Example | +|------|-----------|---------| +| `PERIODIC` | Thread loop: `execute()` → sleep(interval) | Monitoring, Storage | +| `EVENT` | Thread: `for event in signal.listen(stop_event): on_event(event)` | Auto-reply, Mentions | +| `SERVICE` | Thread: `execute()` runs forever | WebSocket listener | + +### 4.2 ModuleConfig (Dataclass) + +Configuration for a single module. Each module receives only its own config, not the full config.yaml. + +```python +@dataclass +class ModuleConfig: + name: str # Module identifier (matches config key) + enabled: bool = True + interval: int = 60 # For PERIODIC modules (seconds) + max_retries: int = 3 # Max restart attempts before permanent failure + backoff_seconds: int = 30 # Wait between restarts (doubles each attempt) + settings: dict = None # Module-specific arbitrary settings + + def __post_init__(self): + if self.settings is None: + self.settings = {} +``` + +### 4.3 ModuleContext (Dataclass) + +Shared dependencies injected into every module. Wrapping in a single dataclass makes it easy to add new shared resources later without changing module constructors. + +```python +@dataclass +class ModuleContext: + account: "Account" # Logged-in Status bot account + config: ModuleConfig # This module's config only + logger: logging.Logger # Shared logger + db: Optional["Postgres"] = None # Database connection (optional) + shared_state: dict = field(default_factory=dict) # Cross-module state + stop_event: threading.Event = None # Signal for graceful shutdown +``` + +### 4.4 BaseModule (ABC) + +```python +class BaseModule(ABC): + + def __init__(self, ctx: ModuleContext): + self._ctx = ctx + self._running = False + + # --- Properties --- + + @property + def ctx(self) -> ModuleContext: + return self._ctx + + @property + @abstractmethod + def module_type(self) -> ModuleType: + """Return the execution type of this module.""" + ... + + @property + def name(self) -> str: + return self._ctx.config.name + + # --- Lifecycle --- + + @abstractmethod + def execute(self) -> Any: + """ + Main logic. Called periodically (PERIODIC), once (SERVICE), + or not at all for EVENT modules. + """ + ... + + def on_start(self) -> None: + """Called once when the module starts. Override for init logic.""" + + def on_stop(self) -> None: + """Called once when the module stops. Override for cleanup.""" + + def on_event(self, event: dict) -> Any: + """Handle a Signal event (for EVENT type modules).""" + + # --- Internal --- + + @property + def is_running(self) -> bool: + return self._running +``` + +### 4.5 ModuleManager + +```python +class ModuleManager: + + def __init__(self, config: dict, account: Account, db: Optional[Postgres], logger: Logger): + self._config = config # Full config (for module settings extraction) + self._account = account + self._db = db + self._logger = logger + self._modules: dict[str, BaseModule] = {} + self._module_classes: dict[str, Type[BaseModule]] = {} + self._threads: dict[str, Thread] = {} + self._stop_event = threading.Event() + + def discover_modules(self) -> None: + """Scan configured directories for BaseModule subclasses.""" + for directory in self._config["modules"]["directories"]: + for py_file in Path(directory).glob("*.py"): + if py_file.name.startswith("_"): + continue + # importlib: spec → module → find BaseModule subclasses + # Store in self._module_classes[file_stem] = class + + def load_modules(self) -> None: + """Instantiate only the enabled modules.""" + for module_name in self._config["modules"]["enabled"]: + cls = self._module_classes.get(module_name) + if not cls: + self._logger.error(f"Module '{module_name}' not found") + continue + settings = self._config["modules"]["settings"].get(module_name, {}) + module_config = ModuleConfig( + name=module_name, + interval=settings.get("interval", 60), + max_retries=settings.get("max_retries", 3), + backoff_seconds=settings.get("backoff_seconds", 30), + settings=settings, + ) + ctx = ModuleContext( + account=self._account, + config=module_config, + logger=self._logger, + db=self._db, + stop_event=self._stop_event, + ) + self._modules[module_name] = cls(ctx) + + def start_all(self) -> None: + """Start all modules in their own threads. Returns immediately.""" + for name, module in self._modules.items(): + t = Thread(target=self._run_module_wrapper, args=(module,), daemon=True) + self._threads[name] = t + t.start() + + def stop_all(self) -> None: + """Trigger graceful shutdown of all modules.""" + self._stop_event.set() + for name, t in self._threads.items(): + t.join(timeout=5) + + def _run_module_wrapper(self, module: BaseModule) -> None: + """Run a single module with retry/backoff logic (see Section 6).""" + ... +``` + +--- + +## 5. Threading & Lifecycle + +### Design Decisions + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Concurrency | `threading.Thread` | Simple, works with blocking I/O, no asyncio refactor needed | +| Shutdown signal | `threading.Event` | Thread-safe, all loops check it | +| SIGTERM handling | `signal.signal(SIGTERM, handler)` | Required for Docker `docker stop` | +| Manager return | Returns immediately | `main.py` owns the wait / signal handling | +| Daemon threads | `daemon=True` | Prevents hanging if stop_all() hangs | + +### Lifecycle Diagram + +``` +main.py + │ + ├── load_config() + ├── Account().login() + ├── Postgres() + ├── ModuleManager(config, account, db, logger) + ├── manager.discover_modules() + ├── manager.load_modules() + │ + ├── manager.start_all() + │ ├── Thread(MONITORING).start() + │ ├── Thread(AUTO_REPLY).start() + │ └── returns immediately + │ + ├── start Prometheus HTTP server (thread or daemon) + │ + ├── signal.signal(SIGTERM, handler) + │ └── handler → manager.stop_all() + │ + └── wait (threading.Event().wait() or while True: sleep) +``` + +### Thread-per-Module: PERIODIC + +```python +def _run_module_wrapper(self, module: BaseModule): + try: + module.on_start() + except Exception: + ... # handle + + if module.module_type == ModuleType.PERIODIC: + while not self._stop_event.is_set(): + try: + module.execute() + except Exception: + ... # retry/backoff (see Section 6) + self._stop_event.wait(module.interval) +``` + +### Thread-per-Module: EVENT + +```python + elif module.module_type == ModuleType.EVENT: + for event in self._account.signal.listen("messages.new", stop_event=self._stop_event): + if self._stop_event.is_set(): + break + try: + module.on_event(event) + except Exception: + ... # log, don't crash +``` + +### Thread-per-Module: SERVICE + +```python + elif module.module_type == ModuleType.SERVICE: + try: + module.execute() # blocking — runs until stop + except Exception: + ... # retry/backoff +``` + +--- + +## 6. Error Handling & Retry + +### Principles + +1. **Error isolation**: One module crash never brings down another module or the bot. +2. **Retry with backoff**: Failed modules are restarted with exponential backoff. +3. **Permanent failure**: After `max_retries` consecutive failures, the module is marked dead and not restarted. +4. **Bot survival**: The bot stops only if: + - It cannot log in to the Status account (fatal). + - No module started successfully (all modules are permanently dead). + - A module can be dead while others continue running. + +### Retry Logic in `_run_module_wrapper` + +```python +def _run_module_wrapper(self, module: BaseModule): + retries = 0 + max_retries = module.ctx.config.max_retries + backoff = module.ctx.config.backoff_seconds + + while retries <= max_retries and not self._stop_event.is_set(): + try: + module._running = True + module.on_start() + + if module.module_type == ModuleType.PERIODIC: + while not self._stop_event.is_set(): + module.execute() + self._stop_event.wait(module.interval) + elif module.module_type == ModuleType.EVENT: + for event in self._account.signal.listen("messages.new", stop_event=self._stop_event): + if self._stop_event.is_set(): + break + module.on_event(event) + elif module.module_type == ModuleType.SERVICE: + module.execute() + + # If we get here without exception, module exited cleanly + break + + except Exception as e: + retries += 1 + self._logger.error( + f"Module '{module.name}' failed ({retries}/{max_retries}): {e}" + ) + if retries <= max_retries: + wait = backoff * (2 ** (retries - 1)) # exponential backoff + self._logger.info(f"Restarting '{module.name}' in {wait}s...") + self._stop_event.wait(wait) + else: + self._logger.error(f"Module '{module.name}' permanently failed.") + finally: + module._running = False + try: + module.on_stop() + except Exception: + pass +``` + +### config.yaml settings for retry + +```yaml +modules: + settings: + monitoring: + interval: 600 + max_retries: 3 + backoff_seconds: 30 + auto_reply: + max_retries: 3 + backoff_seconds: 10 +``` + +--- + +## 7. Configuration Shape + +### Guiding Principles + +- **Do not modify existing keys** — `postgres:`, `sleep:`, `files:`, `bot:` stay exactly as they are. +- **Add new sections only**: `modules:` and `prometheus:`. +- Each module gets its own sub-section under `modules.settings.`. +- Each module receives only its own `ModuleConfig` (not the full config). + +### Final `config.yaml` + +```yaml +postgres: + schema: "status_app_monitoring" + tables: + messages: "raw_messages" + community: "raw_community_info" +sleep: 10 +files: + current_state: "dates.pkl" + +bot: + public_key: "0x..." + compressed_key: "zQ3..." + params: + domain: "status-backend" + port: 8080 + is_secure: false + +# ===== NEW SECTIONS BELOW ===== + +modules: + directories: + - ./modules + enabled: + - monitoring + settings: + monitoring: + interval: 600 # seconds between runs + max_retries: 3 + backoff_seconds: 30 + auto_reply: + max_retries: 3 + backoff_seconds: 10 + commands: + help: "!help" + status: "!status" + +prometheus: + enabled: true + host: "0.0.0.0" + port: 8000 +``` + +### How `main.py` extracts module configs + +```python +modules_config = config.get("modules", {}) +enabled = modules_config.get("enabled", []) +settings = modules_config.get("settings", {}) + +for module_name in enabled: + module_settings = settings.get(module_name, {}) + module_config = ModuleConfig( + name=module_name, + interval=module_settings.get("interval", 60), + max_retries=module_settings.get("max_retries", 3), + backoff_seconds=module_settings.get("backoff_seconds", 30), + settings=module_settings, + ) +``` + +--- + +## 8. Prometheus Metrics + +### Endpoint + +A simple HTTP server on a configurable `host:port` (default `0.0.0.0:8000`) +serving `/metrics` in Prometheus text format using the `prometheus_client` library. + +### Metrics + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `status_bot_health` | Gauge | — | 1 if running, 0 if stopped | +| `status_bot_version` | Gauge | `version` | Version info (constant 1 with label) | +| `status_bot_module_loaded` | Gauge | `module` | 1 for each loaded module | +| `status_bot_module_errors_total` | Counter | `module` | Total error count per module | +| `status_bot_module_restarts_total` | Counter | `module` | Total restart count per module | + +### Implementation in `main.py` + +```python +from prometheus_client import start_http_server, Gauge, Counter + +def start_prometheus_server(config: dict, manager: ModuleManager): + prom_config = config.get("prometheus", {}) + if not prom_config.get("enabled", False): + return + + health = Gauge("status_bot_health", "Bot health status") + version = Gauge("status_bot_version", "Bot version", ["version"]) + module_loaded = Gauge("status_bot_module_loaded", "Module loaded", ["module"]) + module_errors = Counter("status_bot_module_errors_total", "Module errors", ["module"]) + module_restarts = Counter("status_bot_module_restarts_total", "Module restarts", ["module"]) + + health.set(1) + version.labels(version="0.1.0").set(1) + for module_name in manager.modules: + module_loaded.labels(module=module_name).set(1) + + host = prom_config.get("host", "0.0.0.0") + port = prom_config.get("port", 8000) + start_http_server(port, host) +``` + +### Dependency + +Add `prometheus-client` to `requirements.txt`. + +--- + +## 9. Signal.listen() Graceful Shutdown + +### Problem + +The current `Signal.listen()` blocks forever on `queue.get()` with no timeout: + +```python +# Current (blocks forever): +while True: + data = self.__queue.get() # ← blocks indefinitely + yield data +``` + +This means an EVENT-type module thread can never be stopped cleanly — +`stop_event.set()` is never checked. + +### Solution + +Add an optional `stop_event: threading.Event` parameter to `listen()`. +Replace `queue.get()` with `queue.get(timeout=1)` in a loop that checks +the event: + +```python +# Modified: +def listen(self, signal_type: str, stop_event: Optional[threading.Event] = None): + self.__signal_type = signal_type + ws = websocket.WebSocketApp(...) + self.__thread = threading.Thread(target=ws.run_forever, daemon=True) + self.__thread.start() + + while True: + try: + data = self.__queue.get(timeout=1) # ← non-blocking with timeout + except queue.Empty: + if stop_event and stop_event.is_set(): + break + continue + except KeyboardInterrupt: + break + if self.__error_message: + raise Exception(self.__error_message) + yield data +``` + +### Changes to `bot/signal.py` + +- Add `import threading` (already imported) +- Change `listen(self, signal_type: str)` → `listen(self, signal_type: str, stop_event: Optional[threading.Event] = None)` +- Replace `self.__queue.get()` with the timeout loop above + +--- + +## 10. Implementation Steps + +### Step 1: Core Framework + Entry Point + +**Goal**: Create the module framework and a working `main.py` entry point. +No existing monitoring functionality is moved yet — `modules.enabled` would be +empty, and the bot starts with no modules (or a placeholder). + +**Files to create:** + +| File | Content | +|------|---------| +| `bot/modules/__init__.py` | Empty package marker | +| `bot/modules/base.py` | `ModuleType` Enum, `ModuleConfig` dataclass, `ModuleContext` dataclass, `BaseModule` ABC | +| `bot/modules/manager.py` | `ModuleManager` class with `discover_modules()`, `load_modules()`, `start_all()` (threaded), `stop_all()`, `_run_module_wrapper()` with retry/backoff | +| `main.py` | New entry point — config loading, bot creation, DB init, manager init, Prometheus server, signal handlers | + +**Files to modify:** + +| File | Change | +|------|--------| +| `bot/signal.py` | Add `stop_event` parameter to `listen()` (see [Section 9](#9-signallisten-graceful-shutdown)) | +| `config.yaml` | Add `modules:` and `prometheus:` sections | +| `Dockerfile` | Change `ENTRYPOINT ["python", "main.py"]` | +| `requirements.txt` | Add `prometheus-client` | + +**Files to delete:** + +| File | Reason | +|------|--------| +| `module-refactoring.md` | Superseded by this document | + +**Verification:** + +```bash +# Without any modules enabled, the bot should: +# 1. Load config +# 2. Log in +# 3. Start Prometheus endpoint +# 4. Wait for SIGTERM +python main.py +``` + +--- + +### Step 2: Refactor monitor.py → MonitoringModule + +**Goal**: Extract the community monitoring logic from `monitor.py` into a +proper `MonitoringModule` under `./modules/`. Delete the old `monitor.py`. + +**Files to create:** + +| File | Content | +|------|---------| +| `modules/__init__.py` | Empty package marker | +| `modules/monitoring.py` | `MonitoringModule(BaseModule)` — PERIODIC type, calls `execute()` which runs `_download()` + `_store()` | +| `bot/modules/utils.py` | Shared helpers extracted from `monitor.py`: `to_sha256_hash()`, `to_midnight()`, `save_file()`, `extract_community_channels()` | + +**What goes where in `modules/monitoring.py`:** + +| Original function in `monitor.py` | New home | +|-----------------------------------|----------| +| `load_config()` | Stays in `main.py` | +| `create_bot()` | Stays in `main.py` | +| `to_sha256_hash()` | `bot/modules/utils.py` | +| `to_midnight()` | `bot/modules/utils.py` | +| `save_file()` | `bot/modules/utils.py` | +| `extract_community_channels()` | `bot/modules/utils.py` or private to MonitoringModule | +| `download()` | `MonitoringModule._download()` | +| `store()` | `MonitoringModule._store()` | +| `if __name__ == "__main__":` block | Deleted (replaced by `main.py`) | + +**Behavior**: The module receives `Postgres` via `ModuleContext.db`. It reads +`monitoring`-specific settings from `ModuleContext.config.settings`. + +**Verification:** + +```bash +# With monitoring enabled in config.yaml: +# The bot should download community data and store to Postgres. +python main.py +``` + +--- + +### Step 3: Connection Pool + Retry Logic + +**Goal**: Make `Postgres` thread-safe with a connection pool, and verify the +retry/backoff logic in `ModuleManager` works. + +**Files to modify:** + +| File | Change | +|------|--------| +| `postgres.py` | Refactor to use `sqlalchemy.create_engine` with connection pooling (pool_size=5, max_overflow=10). Ensure `pandas.to_sql()` reuses the same engine. | +| `bot/modules/manager.py` | Retry/backoff logic should already be implemented in Step 1. Verify and test edge cases. | + +**Connection pool approach:** + +```python +class Postgres: + def __init__(self, ...): + self._engine = create_engine( + self.__url, + pool_size=5, + max_overflow=10, + pool_pre_ping=True, # verify connections before use + ) + + def insert(self, data, table_name, schema, json_columns=None): + # Use self._engine.begin() for transaction management + ... + + def to_pandas(self, query): + return pd.read_sql(query, self._engine) +``` + +**Verification:** + +```bash +# Run with multiple modules enabled; verify concurrent DB access works. +python main.py +``` + +--- + +### Step 4: Example Event-Driven Module + +**Goal**: Create an `AutoReplyModule` to demonstrate the event-driven pattern +for future module developers. + +**Files to create:** + +| File | Content | +|------|---------| +| `modules/auto_reply.py` | `AutoReplyModule(BaseModule)` — EVENT type, listens for `messages.new`, checks for `!help` / `!status` commands, replies with configured responses | + +**`modules/auto_reply.py` structure:** + +```python +class AutoReplyModule(BaseModule): + + @property + def module_type(self) -> ModuleType: + return ModuleType.EVENT + + def on_start(self): + self._commands = self._ctx.config.settings.get("commands", {}) + self._logger.info(f"Auto-reply loaded with commands: {list(self._commands.keys())}") + + def on_event(self, event: dict): + messages = event.get("event", {}).get("messages", []) + for msg in messages: + text = msg.get("text", "") + for cmd_name, cmd_text in self._commands.items(): + if text.strip().lower() == cmd_text.lower(): + response = self._build_response(cmd_name, msg) + self._account.send_message(msg["chatId"], response) + + def _build_response(self, command_name, message): + # Module-specific reply logic + ... +``` + +**Verification:** + +```bash +# Enable auto_reply in config.yaml, send a message with "!help" to the bot. +``` + +--- + +### Step 5: Tests + Documentation + +**Goal**: Ensure the framework is reliable and documented. + +**Files to create:** + +| File | Content | +|------|---------| +| `tests/test_base.py` | Test `ModuleConfig`, `ModuleContext`, `BaseModule` subclass contract | +| `tests/test_manager.py` | Mock `Account` and test discovery, loading, thread start/stop | +| `tests/test_monitoring_module.py` | Test `MonitoringModule` with mocked account and DB | + +**Files to modify:** + +| File | Change | +|------|--------| +| `README.md` | Document module API, how to write a module, directory structure | + +**Test approach:** + +```python +# test_base.py +def test_module_config_defaults(): + cfg = ModuleConfig(name="test") + assert cfg.enabled == True + assert cfg.interval == 60 + assert cfg.max_retries == 3 + +# test_manager.py +def test_discover_modules(tmp_path): + # Create a mock module file + module_file = tmp_path / "test_mod.py" + module_file.write_text(""" +from bot.modules.base import BaseModule, ModuleType +class TestModule(BaseModule): + @property + def module_type(self): return ModuleType.PERIODIC + def execute(self): pass +""") + manager = ModuleManager({"modules": {"directories": [str(tmp_path)], "enabled": []}}, mock_account, None, logger) + manager.discover_modules() + assert "test_mod" in manager._module_classes +``` + +--- + +## 11. File-by-File Summary + +### New Files (8) + +| # | Path | Step | +|---|------|------| +| 1 | `bot/modules/__init__.py` | 1 | +| 2 | `bot/modules/base.py` | 1 | +| 3 | `bot/modules/manager.py` | 1 | +| 4 | `main.py` | 1 | +| 5 | `modules/__init__.py` | 2 | +| 6 | `bot/modules/utils.py` | 2 | +| 7 | `modules/monitoring.py` | 2 | +| 8 | `modules/auto_reply.py` | 4 | +| 9 | `tests/test_base.py` | 5 | +| 10 | `tests/test_manager.py` | 5 | +| 11 | `tests/test_monitoring_module.py` | 5 | + +### Modified Files (5) + +| # | Path | Change | +|---|------|--------| +| 1 | `bot/signal.py` | Step 1: `listen()` accepts `stop_event` | +| 2 | `config.yaml` | Step 1: add `modules:` + `prometheus:` sections | +| 3 | `Dockerfile` | Step 1: change `ENTRYPOINT` to `main.py` | +| 4 | `requirements.txt` | Step 1: add `prometheus-client` | +| 5 | `README.md` | Step 5: update documentation | + +### Deleted Files (2) + +| # | Path | Reason | +|---|------|--------| +| 1 | `module-refactoring.md` | Superseded by this document | +| 2 | `monitor.py` | Step 2: logic moved to `modules/monitoring.py` | + +--- + +## Appendix: Key References + +- **`bot/account.py`** — ~1112 lines, wraps Status Backend HTTP/RPC API +- **`bot/signal.py`** — WebSocket handler; `listen()` will be modified for `stop_event` +- **`bot/logger.py`** — Singleton logger, 24 lines +- **`postgres.py`** — Postgres connector, 149 lines; needs connection pool refactor +- **`monitor.py`** — 316 lines, to be deleted after Step 2 diff --git a/main.py b/main.py new file mode 100644 index 0000000..3cf68e2 --- /dev/null +++ b/main.py @@ -0,0 +1,176 @@ +import os +import sys +import signal +import time +from pathlib import Path + +import yaml +from dotenv import load_dotenv + +from bot import Account, Logger +from bot.modules.manager import ModuleManager +from postgres import Postgres + + +def load_config(file_path: str) -> dict: + with open(file_path, "r") as f: + config: dict = yaml.safe_load(f) + + env_file_path = os.path.join(os.path.dirname(file_path), ".env") + load_dotenv(env_file_path) + + config["env_vars"] = { + key: value + for key, value in os.environ.items() + if key.startswith(("POSTGRES_", "STATUS_")) + } + + return config + + +def create_bot(config: dict) -> Account: + params = config.get("bot", {}).get("params", {}) + account = Account(**params) + available_accounts = [acc["display_name"] for acc in account.available_accounts] + + prefix = "STATUS_" + params = { + key.replace(prefix, "").lower(): value + for key, value in config["env_vars"].items() + if key.startswith(prefix) + } + if params["display_name"] in available_accounts: + params.pop("mnemonic") + + account.login(**params) + if account.info["compressed_key"] != config["bot"]["compressed_key"]: + raise Exception("Target compressed key and logged in compressed key are different") + + account.profile_picture = os.path.join(os.path.dirname(__file__), "assets", "profile.jpg") + account.logger.info( + f"Account Information:\nCompressed Key: {account.info['compressed_key']}\n" + f"Public Key: {account.info['public_key']}\nURL: {account.info['url']}" + ) + return account + + +def init_postgres(config: dict, logger) -> Postgres: + prefix = "POSTGRES_" + params = { + key.replace(prefix, "").lower(): value + for key, value in config["env_vars"].items() + if key.startswith(prefix) + } + return Postgres(**params) + + +def start_prometheus(config: dict, manager: ModuleManager, logger): + prom_config = config.get("prometheus", {}) + if not prom_config.get("enabled", False): + logger.info("Prometheus metrics disabled") + return + + try: + from prometheus_client import start_http_server, Gauge, Counter + + health = Gauge("status_bot_health", "Bot health status") + version = Gauge("status_bot_version", "Bot version", ["version"]) + module_loaded = Gauge( + "status_bot_module_loaded", "Module loaded", ["module"] + ) + module_errors = Counter( + "status_bot_module_errors_total", "Module errors", ["module"] + ) + module_restarts = Counter( + "status_bot_module_restarts_total", "Module restarts", ["module"] + ) + + health.set(1) + version.labels(version="0.1.0").set(1) + + for module_name in manager.module_names: + module_loaded.labels(module=module_name).set(1) + + host = prom_config.get("host", "0.0.0.0") + port = prom_config.get("port", 8000) + start_http_server(port, host) + logger.info(f"Prometheus metrics server started on {host}:{port}") + except ImportError: + logger.warning( + "prometheus-client not installed. Install with: pip install prometheus-client" + ) + + +def main(): + folder = os.path.dirname(os.path.abspath(__file__)) + config_path = os.path.join(folder, "config.yaml") + + logger = Logger() + + try: + config = load_config(config_path) + except Exception as e: + logger.error(f"Failed to load config: {e}") + sys.exit(1) + + logger.info("Status Bot starting...") + + try: + account = create_bot(config) + except Exception as e: + logger.error(f"Failed to create bot account: {e}") + sys.exit(1) + + db = None + has_postgres = any( + key.startswith("POSTGRES_") for key in config.get("env_vars", {}) + ) + if has_postgres: + try: + db = init_postgres(config, logger) + logger.info("Postgres connection established") + except Exception as e: + logger.warning(f"Failed to connect to Postgres: {e}") + logger.warning("Continuing without database connection") + else: + logger.info("No Postgres configuration found, running without database") + + manager = ModuleManager(config, account, db, logger) + manager.discover_modules() + manager.load_modules() + + start_prometheus(config, manager, logger) + + stop_event = manager._stop_event + + def handle_sigterm(signum, frame): + logger.info("Received SIGTERM, shutting down...") + stop_event.set() + + signal.signal(signal.SIGTERM, handle_sigterm) + signal.signal(signal.SIGINT, handle_sigterm) + + if manager.module_names: + logger.info(f"Starting {len(manager.module_names)} module(s): {manager.module_names}") + manager.start_all() + + try: + while manager.has_alive_modules(): + stop_event.wait(1) + except KeyboardInterrupt: + logger.info("Received keyboard interrupt, shutting down...") + else: + logger.info("No modules enabled. Bot running without modules.") + logger.info("Press Ctrl+C to stop.") + try: + while True: + stop_event.wait(1) + except KeyboardInterrupt: + logger.info("Received keyboard interrupt, shutting down...") + + manager.stop_all() + logger.info("Status Bot stopped") + + +if __name__ == "__main__": + main() diff --git a/module-refactoring.md b/module-refactoring.md index a1e347a..07016cf 100644 --- a/module-refactoring.md +++ b/module-refactoring.md @@ -9,19 +9,19 @@ Transform `monitor.py` from a monolithic script into a modular plugin-based syst ``` ┌─────────────────────────────────────────────────────────────┐ │ Main Bot │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────┐ │ -│ │ Config │ │ Account │ │ Module Manager │ │ -│ │ Loader │ │ (shared) │ │ (loads/runs) │ │ -│ └──────────────┘ └──────────────┘ └──────────────────┘ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────┐ │ +│ │ Config │ │ Account │ │ Module Manager │ │ +│ │ Loader │ │ (shared) │ │ (loads/runs) │ │ +│ └──────────────┘ └──────────────┘ └──────────────────┘ │ └─────────────────────────────────────────────────────────────┘ │ │ │ ▼ ▼ ▼ ┌─────────────────────────────────────────────────────────────┐ -│ Modules (plugins) │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ Monitoring │ │ Storage │ │ Custom │ ... │ -│ │ (periodic) │ │ (periodic)│ │ (event-driven) │ -│ └─────────────┘ └─────────────┘ └─────────────┘ │ +│ Modules (plugins) │ +│ ┌─────────────┐ ┌─────────────┐ ┌───────────────┐ │ +│ │ Monitoring │ │ Storage │ │ Custom │ ... │ +│ │ (periodic) │ │ (periodic) │ │(event-driven) │ │ +│ └─────────────┘ └─────────────┘ └───────────────┘ │ └─────────────────────────────────────────────────────────────┘ ``` @@ -588,4 +588,4 @@ if __name__ == "__main__": 4. **Execution pattern**: Both periodic and event-driven support 5. **Interface**: BaseModule class with required methods 6. **Error handling**: Isolated (one module failure doesn't crash others) -7. **Shared resources**: Passed through constructor (Account, config, logger) \ No newline at end of file +7. **Shared resources**: Passed through constructor (Account, config, logger) diff --git a/modules/__init__.py b/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt index 3307083..e6727f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ pyyaml python-dotenv psycopg2-binary sqlalchemy +prometheus-client From 7f045c9708affb221a8fbf5451219c02b9c1a53e Mon Sep 17 00:00:00 2001 From: apentori Date: Mon, 18 May 2026 18:03:10 +0200 Subject: [PATCH 03/13] fix code Signed-off-by: apentori --- config.yaml | 6 +++--- docker-compose.yaml | 5 ++++- main.py | 3 +++ modules/__init__.py | 0 4 files changed, 10 insertions(+), 4 deletions(-) delete mode 100644 modules/__init__.py diff --git a/config.yaml b/config.yaml index 2675e0b..1cd979c 100644 --- a/config.yaml +++ b/config.yaml @@ -11,8 +11,8 @@ files: bot: # Public information for the bot - public_key: "0x041658626a9e1303b631f6d0fb1e047211d5603b977454f7d5d29fe583c3d6c1bd3d8e395d67f6c44b5bc659aae912040e9dd8164b5107368a29029cb53389d8b0" - compressed_key: "zQ3shNv1tnajHo5FvCvP662cWcbBfS5ZejB4TWaH9iAuFCZZe" + public_key: "0x04e06d0fea33b0df9d6617353ee8e78ca766a366b924049444ec163bdbb02939640df273e7ac3578e9810519f65cbb9594be1a7e43d89373807a7268fb1d90daa2" #"0x041658626a9e1303b631f6d0fb1e047211d5603b977454f7d5d29fe583c3d6c1bd3d8e395d67f6c44b5bc659aae912040e9dd8164b5107368a29029cb53389d8b0" + compressed_key: "zQ3shcWrY3jZSRUKX33uwXxvGJciKSLphUdVMttC5xiir6Qks" # Parameters based on Account class params: # localhost - to run code locally with Status Backend Dockerfile @@ -29,6 +29,6 @@ modules: settings: {} prometheus: - enabled: false + enabled: true host: "0.0.0.0" port: 8000 diff --git a/docker-compose.yaml b/docker-compose.yaml index b30370b..4291c16 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -24,14 +24,17 @@ services: build: context: . dockerfile: Dockerfile - container_name: status-monitor + container_name: status-bot depends_on: - backend - database volumes: - ./backups:/backups + - ./config.yaml:/app/config.yaml env_file: - .env + ports: + - 8000:8000 networks: - status-bridge diff --git a/main.py b/main.py index 3cf68e2..ca41ce6 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,4 @@ +from logging import log import os import sys import signal @@ -43,6 +44,7 @@ def create_bot(config: dict) -> Account: params.pop("mnemonic") account.login(**params) + account.logger.info(f"Account Info {account.info}") if account.info["compressed_key"] != config["bot"]["compressed_key"]: raise Exception("Target compressed key and logged in compressed key are different") @@ -108,6 +110,7 @@ def main(): logger = Logger() try: + logger.info("Loading the configuration") config = load_config(config_path) except Exception as e: logger.error(f"Failed to load config: {e}") diff --git a/modules/__init__.py b/modules/__init__.py deleted file mode 100644 index e69de29..0000000 From 541de4df77ad7b39a7a6a96251b0ca91df5c6835 Mon Sep 17 00:00:00 2001 From: apentori Date: Mon, 18 May 2026 22:12:14 +0200 Subject: [PATCH 04/13] refacto: extract config and metrics Signed-off-by: apentori --- bot/config.py | 60 ++++++++++++++++++++++++++++ bot/metrics.py | 40 +++++++++++++++++++ main.py | 106 +++---------------------------------------------- 3 files changed, 105 insertions(+), 101 deletions(-) create mode 100644 bot/config.py create mode 100644 bot/metrics.py diff --git a/bot/config.py b/bot/config.py new file mode 100644 index 0000000..048ae23 --- /dev/null +++ b/bot/config.py @@ -0,0 +1,60 @@ +import os + +import yaml +from dotenv import load_dotenv + +from bot import Account +from postgres import Postgres + + +def load_config(file_path: str) -> dict: + with open(file_path, "r") as f: + config: dict = yaml.safe_load(f) + + env_file_path = os.path.join(os.path.dirname(file_path), ".env") + load_dotenv(env_file_path) + + config["env_vars"] = { + key: value + for key, value in os.environ.items() + if key.startswith(("POSTGRES_", "STATUS_")) + } + + return config + + +def create_bot(config: dict, project_root: str) -> Account: + params = config.get("bot", {}).get("params", {}) + account = Account(**params) + available_accounts = [acc["display_name"] for acc in account.available_accounts] + + prefix = "STATUS_" + params = { + key.replace(prefix, "").lower(): value + for key, value in config["env_vars"].items() + if key.startswith(prefix) + } + if params["display_name"] in available_accounts: + params.pop("mnemonic") + + account.login(**params) + account.logger.info(f"Account Info {account.info}") + if account.info["compressed_key"] != config["bot"]["compressed_key"]: + raise Exception("Target compressed key and logged in compressed key are different") + + account.profile_picture = os.path.join(project_root, "assets", "profile.jpg") + account.logger.info( + f"Account Information:\nCompressed Key: {account.info['compressed_key']}\n" + f"Public Key: {account.info['public_key']}\nURL: {account.info['url']}" + ) + return account + + +def init_postgres(config: dict) -> Postgres: + prefix = "POSTGRES_" + params = { + key.replace(prefix, "").lower(): value + for key, value in config["env_vars"].items() + if key.startswith(prefix) + } + return Postgres(**params) diff --git a/bot/metrics.py b/bot/metrics.py new file mode 100644 index 0000000..11d38b9 --- /dev/null +++ b/bot/metrics.py @@ -0,0 +1,40 @@ +import logging + +from bot.modules.manager import ModuleManager + + +def start_prometheus(config: dict, manager: ModuleManager, logger: logging.Logger): + prom_config = config.get("prometheus", {}) + if not prom_config.get("enabled", False): + logger.info("Prometheus metrics disabled") + return + + try: + from prometheus_client import start_http_server, Gauge, Counter + + health = Gauge("status_bot_health", "Bot health status") + version = Gauge("status_bot_version", "Bot version", ["version"]) + module_loaded = Gauge( + "status_bot_module_loaded", "Module loaded", ["module"] + ) + module_errors = Counter( + "status_bot_module_errors_total", "Module errors", ["module"] + ) + module_restarts = Counter( + "status_bot_module_restarts_total", "Module restarts", ["module"] + ) + + health.set(1) + version.labels(version="0.1.0").set(1) + + for module_name in manager.module_names: + module_loaded.labels(module=module_name).set(1) + + host = prom_config.get("host", "0.0.0.0") + port = prom_config.get("port", 8000) + start_http_server(port, host) + logger.info(f"Prometheus metrics server started on {host}:{port}") + except ImportError: + logger.warning( + "prometheus-client not installed. Install with: pip install prometheus-client" + ) diff --git a/main.py b/main.py index ca41ce6..d290a9b 100644 --- a/main.py +++ b/main.py @@ -1,106 +1,11 @@ -from logging import log import os import sys import signal -import time -from pathlib import Path -import yaml -from dotenv import load_dotenv - -from bot import Account, Logger +from bot import Logger +from bot.config import load_config, create_bot, init_postgres +from bot.metrics import start_prometheus from bot.modules.manager import ModuleManager -from postgres import Postgres - - -def load_config(file_path: str) -> dict: - with open(file_path, "r") as f: - config: dict = yaml.safe_load(f) - - env_file_path = os.path.join(os.path.dirname(file_path), ".env") - load_dotenv(env_file_path) - - config["env_vars"] = { - key: value - for key, value in os.environ.items() - if key.startswith(("POSTGRES_", "STATUS_")) - } - - return config - - -def create_bot(config: dict) -> Account: - params = config.get("bot", {}).get("params", {}) - account = Account(**params) - available_accounts = [acc["display_name"] for acc in account.available_accounts] - - prefix = "STATUS_" - params = { - key.replace(prefix, "").lower(): value - for key, value in config["env_vars"].items() - if key.startswith(prefix) - } - if params["display_name"] in available_accounts: - params.pop("mnemonic") - - account.login(**params) - account.logger.info(f"Account Info {account.info}") - if account.info["compressed_key"] != config["bot"]["compressed_key"]: - raise Exception("Target compressed key and logged in compressed key are different") - - account.profile_picture = os.path.join(os.path.dirname(__file__), "assets", "profile.jpg") - account.logger.info( - f"Account Information:\nCompressed Key: {account.info['compressed_key']}\n" - f"Public Key: {account.info['public_key']}\nURL: {account.info['url']}" - ) - return account - - -def init_postgres(config: dict, logger) -> Postgres: - prefix = "POSTGRES_" - params = { - key.replace(prefix, "").lower(): value - for key, value in config["env_vars"].items() - if key.startswith(prefix) - } - return Postgres(**params) - - -def start_prometheus(config: dict, manager: ModuleManager, logger): - prom_config = config.get("prometheus", {}) - if not prom_config.get("enabled", False): - logger.info("Prometheus metrics disabled") - return - - try: - from prometheus_client import start_http_server, Gauge, Counter - - health = Gauge("status_bot_health", "Bot health status") - version = Gauge("status_bot_version", "Bot version", ["version"]) - module_loaded = Gauge( - "status_bot_module_loaded", "Module loaded", ["module"] - ) - module_errors = Counter( - "status_bot_module_errors_total", "Module errors", ["module"] - ) - module_restarts = Counter( - "status_bot_module_restarts_total", "Module restarts", ["module"] - ) - - health.set(1) - version.labels(version="0.1.0").set(1) - - for module_name in manager.module_names: - module_loaded.labels(module=module_name).set(1) - - host = prom_config.get("host", "0.0.0.0") - port = prom_config.get("port", 8000) - start_http_server(port, host) - logger.info(f"Prometheus metrics server started on {host}:{port}") - except ImportError: - logger.warning( - "prometheus-client not installed. Install with: pip install prometheus-client" - ) def main(): @@ -108,7 +13,6 @@ def main(): config_path = os.path.join(folder, "config.yaml") logger = Logger() - try: logger.info("Loading the configuration") config = load_config(config_path) @@ -119,7 +23,7 @@ def main(): logger.info("Status Bot starting...") try: - account = create_bot(config) + account = create_bot(config, folder) except Exception as e: logger.error(f"Failed to create bot account: {e}") sys.exit(1) @@ -130,7 +34,7 @@ def main(): ) if has_postgres: try: - db = init_postgres(config, logger) + db = init_postgres(config) logger.info("Postgres connection established") except Exception as e: logger.warning(f"Failed to connect to Postgres: {e}") From 9e7cb0f4c0d1c03e5542b3c0336980ecc9e48f5c Mon Sep 17 00:00:00 2001 From: apentori Date: Tue, 19 May 2026 17:56:11 +0200 Subject: [PATCH 05/13] fix: improve config Signed-off-by: apentori --- bot/account.py | 4 +- bot/config.py | 108 ++++++++++++++++++++++++----------------- bot/metrics.py | 10 ++-- bot/modules/manager.py | 15 +++--- config.yaml | 53 +++++++++++--------- main.py | 102 +++++++++++++++++++++++++++++++++----- postgres.py | 6 +-- requirements.txt | 2 + 8 files changed, 200 insertions(+), 100 deletions(-) diff --git a/bot/account.py b/bot/account.py index 8a1fd8a..17833e4 100644 --- a/bot/account.py +++ b/bot/account.py @@ -990,8 +990,8 @@ def __load_backup(self): self.__signal.get("messages.new") self.logger.info(f"Successfully loaded file!") break - - self.logger.warning(error) + else: + self.logger.warning(f"Error while loading the backup: {error}") def __call_rpc(self, prefix: str, method_name: str, params: Optional[Union[list, dict]] = None) -> dict: """ diff --git a/bot/config.py b/bot/config.py index 048ae23..fc41f50 100644 --- a/bot/config.py +++ b/bot/config.py @@ -1,60 +1,78 @@ -import os +from pydantic import BaseModel +from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic_settings.sources import YamlConfigSettingsSource -import yaml -from dotenv import load_dotenv -from bot import Account -from postgres import Postgres +class BotParams(BaseModel): + domain: str = "localhost" + port: int = 8080 + is_secure: bool = False -def load_config(file_path: str) -> dict: - with open(file_path, "r") as f: - config: dict = yaml.safe_load(f) +class BotConfig(BaseModel): + display_name: str = "" + public_key: str = "" + password: str = "" + mnemonic_phrase: str = "" + init_account: bool = False + compressed_key: str = "" + infura_token: str = "" + coingecko_api_key: str = "" + params: BotParams = BotParams() - env_file_path = os.path.join(os.path.dirname(file_path), ".env") - load_dotenv(env_file_path) - config["env_vars"] = { - key: value - for key, value in os.environ.items() - if key.startswith(("POSTGRES_", "STATUS_")) - } +class PostgresConfig(BaseModel): + host: str = "database" + port: int = 5432 + user: str = "" + password: str = "" + name: str = "" + schema: str = "public" + tables: dict = {} - return config +class PrometheusConfig(BaseModel): + enabled: bool = False + host: str = "0.0.0.0" + port: int = 8000 -def create_bot(config: dict, project_root: str) -> Account: - params = config.get("bot", {}).get("params", {}) - account = Account(**params) - available_accounts = [acc["display_name"] for acc in account.available_accounts] - prefix = "STATUS_" - params = { - key.replace(prefix, "").lower(): value - for key, value in config["env_vars"].items() - if key.startswith(prefix) - } - if params["display_name"] in available_accounts: - params.pop("mnemonic") +class FilesConfig(BaseModel): + current_state: str = "dates.pkl" - account.login(**params) - account.logger.info(f"Account Info {account.info}") - if account.info["compressed_key"] != config["bot"]["compressed_key"]: - raise Exception("Target compressed key and logged in compressed key are different") - account.profile_picture = os.path.join(project_root, "assets", "profile.jpg") - account.logger.info( - f"Account Information:\nCompressed Key: {account.info['compressed_key']}\n" - f"Public Key: {account.info['public_key']}\nURL: {account.info['url']}" +class ModulesConfig(BaseModel): + directories: list[str] = ["./modules"] + enabled: list[str] = [] + settings: dict = {} + + +class Config(BaseSettings): + model_config = SettingsConfigDict( + env_nested_delimiter="_", + extra="ignore", ) - return account + sleep: int = 10 + files: FilesConfig = FilesConfig() + bot: BotConfig = BotConfig() + modules: ModulesConfig = ModulesConfig() + prometheus: PrometheusConfig = PrometheusConfig() + postgres: PostgresConfig = PostgresConfig() + + _yaml_file = "./config.yaml" -def init_postgres(config: dict) -> Postgres: - prefix = "POSTGRES_" - params = { - key.replace(prefix, "").lower(): value - for key, value in config["env_vars"].items() - if key.startswith(prefix) - } - return Postgres(**params) + @classmethod + def settings_customise_sources( + cls, + settings_cls, + init_settings, + env_settings, + dotenv_settings, + file_secret_settings, + ): + return ( + YamlConfigSettingsSource(settings_cls, yaml_file=cls._yaml_file), + env_settings, + dotenv_settings, + ) diff --git a/bot/metrics.py b/bot/metrics.py index 11d38b9..1b8fbe7 100644 --- a/bot/metrics.py +++ b/bot/metrics.py @@ -1,11 +1,11 @@ import logging +from bot.config import PrometheusConfig from bot.modules.manager import ModuleManager -def start_prometheus(config: dict, manager: ModuleManager, logger: logging.Logger): - prom_config = config.get("prometheus", {}) - if not prom_config.get("enabled", False): +def start_prometheus(prometheus_config: PrometheusConfig, manager: ModuleManager, logger: logging.Logger): + if not prometheus_config.enabled: logger.info("Prometheus metrics disabled") return @@ -30,8 +30,8 @@ def start_prometheus(config: dict, manager: ModuleManager, logger: logging.Logge for module_name in manager.module_names: module_loaded.labels(module=module_name).set(1) - host = prom_config.get("host", "0.0.0.0") - port = prom_config.get("port", 8000) + host = prometheus_config.host + port = prometheus_config.port start_http_server(port, host) logger.info(f"Prometheus metrics server started on {host}:{port}") except ImportError: diff --git a/bot/modules/manager.py b/bot/modules/manager.py index 8d47e4d..23c14dd 100644 --- a/bot/modules/manager.py +++ b/bot/modules/manager.py @@ -6,12 +6,13 @@ from pathlib import Path from .base import BaseModule, ModuleConfig, ModuleContext, ModuleType +from bot.config import ModulesConfig class ModuleManager: - def __init__(self, config: dict, account, db, logger: logging.Logger): - self._config = config + def __init__(self, modules_config: ModulesConfig, account, db, logger: logging.Logger): + self._modules_config = modules_config self._account = account self._db = db self._logger = logger @@ -29,10 +30,7 @@ def module_names(self) -> list[str]: return list(self._modules.keys()) def discover_modules(self) -> None: - modules_config = self._config.get("modules", {}) - directories = modules_config.get("directories", ["modules"]) - - for directory in directories: + for directory in self._modules_config.directories: self._discover_from_directory(directory) self._logger.info( @@ -74,9 +72,8 @@ def _load_module_from_file(self, file_path: Path) -> None: self._logger.debug(f"Found module class: {module_name}.{attr_name}") def load_modules(self) -> None: - modules_config = self._config.get("modules", {}) - enabled = set(modules_config.get("enabled", [])) - settings = modules_config.get("settings", {}) + enabled = set(self._modules_config.enabled) + settings = self._modules_config.settings if not enabled: self._logger.info("No modules enabled in config") diff --git a/config.yaml b/config.yaml index 1cd979c..c0b36af 100644 --- a/config.yaml +++ b/config.yaml @@ -1,34 +1,39 @@ postgres: - schema: "status_app_monitoring" - tables: - messages: "raw_messages" - community: "raw_community_info" + host: "database" + port: 5432 + schema: "status_app_monitoring" + name: "status-bot" + tables: + messages: "raw_messages" + community: "raw_community_info" # Value must be in MINUTES sleep: 10 files: - # Get the latest community chat dates for next run - current_state: "dates.pkl" + # Get the latest community chat dates for next run + current_state: "dates.pkl" bot: - # Public information for the bot - public_key: "0x04e06d0fea33b0df9d6617353ee8e78ca766a366b924049444ec163bdbb02939640df273e7ac3578e9810519f65cbb9594be1a7e43d89373807a7268fb1d90daa2" #"0x041658626a9e1303b631f6d0fb1e047211d5603b977454f7d5d29fe583c3d6c1bd3d8e395d67f6c44b5bc659aae912040e9dd8164b5107368a29029cb53389d8b0" - compressed_key: "zQ3shcWrY3jZSRUKX33uwXxvGJciKSLphUdVMttC5xiir6Qks" - # Parameters based on Account class - params: - # localhost - to run code locally with Status Backend Dockerfile - # status-backend - to run code with docker-compose.yaml - # domain: "status-backend" - domain: "status-backend" - port: 8080 - is_secure: false + display_name: 'e-raccoon' + public_key: "0xExample" + compressed_key: "example-compressed-key" + password: 'ChangeMe' + mnemonic_phrase: 'some random word ...' + init_account: true + infura_token: "" + coingecko_api_key: "" + # Parameters based on Account class + params: + domain: "status-backend" + port: 8080 + is_secure: false modules: - directories: - - ./modules - enabled: [] - settings: {} + directories: + - ./modules + enabled: [] + settings: {} prometheus: - enabled: true - host: "0.0.0.0" - port: 8000 + enabled: true + host: "0.0.0.0" + port: 8000 diff --git a/main.py b/main.py index d290a9b..a788d6c 100644 --- a/main.py +++ b/main.py @@ -1,37 +1,112 @@ import os import sys import signal +import argparse -from bot import Logger -from bot.config import load_config, create_bot, init_postgres +from bot import Account, Logger +from bot.config import Config from bot.metrics import start_prometheus from bot.modules.manager import ModuleManager +from postgres import Postgres + + +def create_bot(config: Config, project_root: str) -> Account: + account = Account(**config.bot.params.model_dump()) + available_accounts = [a["display_name"] for a in account.available_accounts] + + display_name = config.bot.display_name + password = config.bot.password + + if display_name in available_accounts: + account.logger.info(f"Logging in with display name: {display_name}") + account.login(display_name=display_name, password=password) + elif config.bot.init_account: + mnemonic = config.bot.mnemonic_phrase + if not mnemonic: + raise ValueError( + "init_account is true but no mnemonic_phrase provided" + ) + account.logger.info(f"Creating/restoring account: {display_name}") + account.login( + display_name=display_name, + password=password, + mnemonic=mnemonic, + ) + else: + raise ValueError( + f"Account '{display_name}' not found and init_account is false. " + f"Available accounts: {[a['display_name'] for a in available_accounts]}" + ) + + account.logger.info(f"Account Info {account.info}") + + if account.info["compressed_key"] != config.bot.compressed_key: + raise Exception( + "Target compressed key and logged in compressed key are different" + ) + + profile_path = os.path.join(project_root, "assets", "profile.jpg") + account.profile_picture = profile_path + account.logger.info( + f"Account Information:\n" + f"Compressed Key: {account.info['compressed_key']}\n" + f"Public Key: {account.info['public_key']}\n" + f"URL: {account.info['url']}" + ) + return account + + +def init_postgres(config: Config) -> Postgres: + return Postgres( + host=config.postgres.host, + port=config.postgres.port, + user=config.postgres.user, + password=config.postgres.password, + database=config.postgres.name, + ) def main(): - folder = os.path.dirname(os.path.abspath(__file__)) - config_path = os.path.join(folder, "config.yaml") + parser = argparse.ArgumentParser( + description="Status Bot - Modular monitoring framework" + ) + parser.add_argument( + "--config", + default="./config.yaml", + help="Path to configuration file (default: ./config.yaml)", + ) + args = parser.parse_args() + + config_path = args.config logger = Logger() + try: logger.info("Loading the configuration") - config = load_config(config_path) + Config._yaml_file = config_path + config = Config() except Exception as e: logger.error(f"Failed to load config: {e}") sys.exit(1) logger.info("Status Bot starting...") + project_root = os.path.dirname(os.path.abspath(__file__)) + try: - account = create_bot(config, folder) + account = create_bot(config, project_root) except Exception as e: logger.error(f"Failed to create bot account: {e}") sys.exit(1) db = None - has_postgres = any( - key.startswith("POSTGRES_") for key in config.get("env_vars", {}) - ) + has_postgres = all([ + config.postgres.host, + config.postgres.user, + config.postgres.password, + config.postgres.name, + ]) + logger.info(f"Postgres configuration {config.postgres}") if has_postgres: try: db = init_postgres(config) @@ -42,11 +117,11 @@ def main(): else: logger.info("No Postgres configuration found, running without database") - manager = ModuleManager(config, account, db, logger) + manager = ModuleManager(config.modules, account, db, logger) manager.discover_modules() manager.load_modules() - start_prometheus(config, manager, logger) + start_prometheus(config.prometheus, manager, logger) stop_event = manager._stop_event @@ -58,7 +133,10 @@ def handle_sigterm(signum, frame): signal.signal(signal.SIGINT, handle_sigterm) if manager.module_names: - logger.info(f"Starting {len(manager.module_names)} module(s): {manager.module_names}") + logger.info( + f"Starting {len(manager.module_names)} module(s): " + f"{manager.module_names}" + ) manager.start_all() try: diff --git a/postgres.py b/postgres.py index 6a4512e..755f9e8 100644 --- a/postgres.py +++ b/postgres.py @@ -11,20 +11,20 @@ class Postgres: - def __init__(self, username: str, password: str, port: Union[int, str], database: str, host: str): + def __init__(self, user: str, password: str, port: Union[int, str], database: str, host: str): if isinstance(port, str): port = int(port) self.__params = { "host": host, - "user": username, + "user": user, "password": password, "port": port, "database": database } - self.__url = f"postgresql://{username}:{password}@{host}:{port}/{database}" + self.__url = f"postgresql://{user}:{password}@{host}:{port}/{database}" self.__conn: psycopg2.extensions.connection = psycopg2.connect(**self.__params) self.__cursor: psycopg2.extensions.cursor = self.__conn.cursor() diff --git a/requirements.txt b/requirements.txt index e6727f9..91a7e40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,5 @@ python-dotenv psycopg2-binary sqlalchemy prometheus-client +pydantic>=2.0 +pydantic-settings>=2.0 \ No newline at end of file From ba3400683e5a734d46972e6add1df88102a692e8 Mon Sep 17 00:00:00 2001 From: apentori Date: Wed, 20 May 2026 17:35:04 +0200 Subject: [PATCH 06/13] tmp: adding monitoring as module Signed-off-by: apentori --- bot/account.py | 7 ++ bot/config.py | 4 +- bot/metrics.py | 4 +- bot/modules/manager.py | 17 ++- bot/modules/monitoring.py | 214 ++++++++++++++++++++++++++++++++++++++ bot/modules/utils.py | 28 +++++ docker-compose.yaml | 6 +- main.py | 27 +++-- 8 files changed, 287 insertions(+), 20 deletions(-) create mode 100644 bot/modules/monitoring.py create mode 100644 bot/modules/utils.py diff --git a/bot/account.py b/bot/account.py index 17833e4..b0367bf 100644 --- a/bot/account.py +++ b/bot/account.py @@ -157,10 +157,17 @@ def login(self, password: str, key_uid: Optional[str] = None, display_name: Opti # Wallet usage if infura_token: + self.logger.info("Adding Infura token") params["infuraToken"] = infura_token + else: + self.logger.info("No infura token") if coingecko_api_key: + self.logger.info("Adding CoinGechko token") params["coingeckoApiKey"] = coingecko_api_key + else: + self.logger.info("No coingecko token") + if infura_token and coingecko_api_key: self.__is_wallet_set = True diff --git a/bot/config.py b/bot/config.py index fc41f50..88bf323 100644 --- a/bot/config.py +++ b/bot/config.py @@ -42,14 +42,14 @@ class FilesConfig(BaseModel): class ModulesConfig(BaseModel): - directories: list[str] = ["./modules"] + directories: list[str] = ["./modules", "bot/modules"] enabled: list[str] = [] settings: dict = {} class Config(BaseSettings): model_config = SettingsConfigDict( - env_nested_delimiter="_", + env_nested_delimiter="__", extra="ignore", ) diff --git a/bot/metrics.py b/bot/metrics.py index 1b8fbe7..dc4b817 100644 --- a/bot/metrics.py +++ b/bot/metrics.py @@ -2,6 +2,8 @@ from bot.config import PrometheusConfig from bot.modules.manager import ModuleManager +from prometheus_client import start_http_server, Gauge, Counter + def start_prometheus(prometheus_config: PrometheusConfig, manager: ModuleManager, logger: logging.Logger): @@ -10,8 +12,6 @@ def start_prometheus(prometheus_config: PrometheusConfig, manager: ModuleManager return try: - from prometheus_client import start_http_server, Gauge, Counter - health = Gauge("status_bot_health", "Bot health status") version = Gauge("status_bot_version", "Bot version", ["version"]) module_loaded = Gauge( diff --git a/bot/modules/manager.py b/bot/modules/manager.py index 23c14dd..055b7eb 100644 --- a/bot/modules/manager.py +++ b/bot/modules/manager.py @@ -11,7 +11,7 @@ class ModuleManager: - def __init__(self, modules_config: ModulesConfig, account, db, logger: logging.Logger): + def __init__(self, modules_config: ModulesConfig, account, db, logger: logging.Logger, shared_state: dict = None): self._modules_config = modules_config self._account = account self._db = db @@ -20,6 +20,7 @@ def __init__(self, modules_config: ModulesConfig, account, db, logger: logging.L self._module_classes: dict[str, Type[BaseModule]] = {} self._threads: dict[str, threading.Thread] = {} self._stop_event = threading.Event() + self._shared_state = shared_state or {} @property def modules(self) -> dict[str, BaseModule]: @@ -52,9 +53,16 @@ def _discover_from_directory(self, directory: str) -> None: def _load_module_from_file(self, file_path: Path) -> None: module_name = file_path.stem - spec = importlib.util.spec_from_file_location( - f"modules.{module_name}", file_path - ) + if module_name in ("base", "manager", "utils"): + return + + bot_modules_dir = Path(__file__).parent.resolve() + if file_path.parent.resolve() == bot_modules_dir: + pkg_name = f"bot.modules.{module_name}" + else: + pkg_name = f"modules.{module_name}" + + spec = importlib.util.spec_from_file_location(pkg_name, file_path) if spec is None or spec.loader is None: return @@ -104,6 +112,7 @@ def load_modules(self) -> None: config=module_config, logger=self._logger, db=self._db, + shared_state=self._shared_state, stop_event=self._stop_event, ) diff --git a/bot/modules/monitoring.py b/bot/modules/monitoring.py new file mode 100644 index 0000000..5d3452a --- /dev/null +++ b/bot/modules/monitoring.py @@ -0,0 +1,214 @@ +import datetime +import os +from pathlib import Path + +import pandas as pd + +from bot.modules.base import BaseModule, ModuleType +from bot.modules.utils import save_file, to_midnight, to_sha256_hash + + +def extract_community_channels(account, community: dict, latest_dates: dict[str, pd.Timestamp]) -> pd.DataFrame: + bridge_key = "bridge_message" + columns = { + "id": True, + "whisper_timestamp": False, + "from": True, + "seen": False, + "chat_id": False, + "community_id": False, + "message_type": False, + "response_to": True, + "timestamp": False, + "deleted": False, + "extracted_timestamp": False, + } + + final = [] + for channel in community["channels"]: + now = datetime.datetime.now() + start_timestamp = latest_dates.get(channel["chat_id"]) + if start_timestamp: + start_timestamp += datetime.timedelta(seconds=1) + else: + start_timestamp = to_midnight(now - datetime.timedelta(days=30)) + + account.logger.info( + f"Starting message extraction for # {channel['name']} [{start_timestamp} - {now}]" + ) + messages = account.get_messages(channel["chat_id"], start_timestamp, now) + messages = pd.DataFrame(messages) + if len(messages) == 0: + account.logger.info("No messages found") + continue + + account.logger.info(f"Extracted {len(messages)} message(s)") + messages = messages.assign( + community_id=community["id"], + extracted_timestamp=now, + ) + final.append(messages) + + extracted_data = pd.concat(final, ignore_index=True) if final else pd.DataFrame() + if len(extracted_data) == 0: + return extracted_data + + existing_columns = extracted_data.columns.to_list() + for column, should_hash in columns.items(): + if column not in existing_columns: + loc = len(extracted_data.columns.to_list()) + extracted_data.insert(loc, column, None) + continue + + if should_hash: + extracted_data[column] = extracted_data[column].astype(str).apply(to_sha256_hash) + + if bridge_key in extracted_data.columns: + extracted_data["source"] = extracted_data[bridge_key].apply( + lambda value: value["bridgeName"] if not pd.isna(value) else "status" + ) + else: + extracted_data["source"] = "status" + + extracted_data = extracted_data[list(columns.keys()) + ["source"]].assign( + deleted=extracted_data["deleted"].fillna(False), + seen=extracted_data["seen"].fillna(False), + ) + account.logger.info("Sensitive data has been hashed") + + return extracted_data + + +class MonitoringModule(BaseModule): + + @property + def module_type(self) -> ModuleType: + return ModuleType.PERIODIC + + def on_start(self) -> None: + account = self.ctx.account + balance = account["GBP"] + query = ( + (balance["symbol"] == "SNT") + & (balance["fiat_value"] > 0) + & (balance["chain_id"] == 1) + ) + if query.sum() != 1: + raise RuntimeError("Wallet balance check failed — Infura or Coingecko issue") + + def execute(self) -> None: + config = self.ctx.shared_state.get("config") + if config is None: + self.ctx.logger.error("MonitoringModule: config not found in shared_state") + return + + project_root = self.ctx.shared_state.get("project_root") + if not project_root: + project_root = os.path.dirname(os.path.abspath(__file__)) + + account = self.ctx.account + logger = self.ctx.logger + + upload_folder = self.ctx.config.settings.get("upload_folder", "uploads") + upload_path = os.path.join(project_root, upload_folder) + current_state_path = os.path.join(project_root, config.files.current_state) + + self._download(account, upload_path, current_state_path, config) + + if self.ctx.db is not None: + self._store(upload_path, current_state_path, config, logger) + else: + logger.info("No database configured, skipping store step") + + def _download(self, account, upload_path: str, current_state_path: str, config) -> None: + latest_dates: dict[str, pd.Timestamp] = ( + pd.read_pickle(current_state_path) if os.path.exists(current_state_path) else {} + ) + + get_file_name = lambda: str(to_midnight(datetime.datetime.now()).timestamp()).replace(".", "") + communities = account.communities + if not communities: + account.logger.warning("No communities found...") + return + + for community in communities: + if not community["is_member"]: + continue + + community_folder_name = community["name"].replace(" ", "-") + messages_folder = os.path.join(upload_path, "messages", community_folder_name) + community_info_folder = os.path.join(upload_path, "community", community_folder_name) + + account.logger.info(f"Extracting data for {community['name']}") + community["extracted_timestamp"] = datetime.datetime.now() + + file_path = os.path.join(community_info_folder, get_file_name() + ".pkl") + if not os.path.exists(file_path): + save_file(file_path, community) + account.logger.info(f"Created {file_path}") + + file_path = os.path.join(messages_folder, get_file_name() + ".csv") + if not os.path.exists(file_path): + messages = extract_community_channels(account, community, latest_dates) + if len(messages) > 0: + save_file(file_path, messages) + account.logger.info(f"Created {file_path}") + + def _store(self, upload_path: str, current_state_path: str, config, logger) -> None: + path = Path(upload_path) + table_name_mapping: dict[str, str] = config.postgres.tables + table_schema = config.postgres.schema + + upload: dict[str, list] = {} + latest_dates: dict[str, pd.Timestamp] = ( + pd.read_pickle(current_state_path) if os.path.exists(current_state_path) else {} + ) + completed = [] + + files = list(path.rglob("*.pkl")) + list(path.rglob("*.csv")) + logger.info(f"There are {len(files)} file(s) to upload") + for file_path in files: + table_name = table_name_mapping.get(file_path.parent.parent.name) + if not table_name: + continue + + file_name = str(file_path.name) + data = pd.read_pickle(file_path) if file_name.endswith(".pkl") else pd.read_csv(file_path) + if isinstance(data, dict): + data = pd.DataFrame([data]) + + for column in data.columns: + if "timestamp" not in column: + continue + data[column] = pd.to_datetime(data[column]) + + if table_name not in upload: + upload[table_name] = [] + + if "timestamp" in data.columns: + latest_dates.update(data.groupby("chat_id")["timestamp"].max().to_dict()) + + upload[table_name].append(data) + completed.append(str(file_path)) + + save_file(current_state_path, latest_dates) + logger.info(f"Updated {current_state_path}") + + connector = self.ctx.db + for table_name, data in upload.items(): + if len(data) == 0: + continue + + df = pd.concat(data, ignore_index=True).assign(batch_timestamp=datetime.datetime.now()) + json_columns = [ + column + for column in df.columns + if len(df[column].dropna()) > 0 + and isinstance(df[column].dropna().reset_index(drop=True).iloc[0], (dict, list)) + ] + connector.insert(df, table_name, table_schema, json_columns) + logger.info(f"Uploaded {len(df)} record(s) to {table_schema}.{table_name}") + + for file_path in completed: + os.remove(file_path) + logger.info(f"Deleted {file_path}") diff --git a/bot/modules/utils.py b/bot/modules/utils.py new file mode 100644 index 0000000..efd9c9d --- /dev/null +++ b/bot/modules/utils.py @@ -0,0 +1,28 @@ +import datetime +import os +import pickle +from hashlib import sha256 + +import pandas as pd +from typing import Any + + +def to_sha256_hash(value: str) -> str: + return sha256(value.encode()).hexdigest() + + +def to_midnight(timestamp: datetime.datetime) -> datetime.datetime: + return timestamp.replace(minute=0, second=0, hour=0, microsecond=0) + + +def save_file(file_path: str, data: Any): + folder = os.path.dirname(file_path) + if len(folder) > 0: + os.makedirs(folder, exist_ok=True) + + if isinstance(data, pd.DataFrame): + data.to_csv(file_path, index=False) + return + + with open(file_path, "wb") as f: + pickle.dump(data, f) diff --git a/docker-compose.yaml b/docker-compose.yaml index 4291c16..d27faf6 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -41,8 +41,10 @@ services: database: image: postgres:15 container_name: database - env_file: - - .env + environment: + POSTGRES_DB: 'status-bot' + POSTGRES_USER: 'status' + POSTGRES_PASSWORD: 'ChangeMeIfYouCare' ports: - 5432:5432 volumes: diff --git a/main.py b/main.py index a788d6c..6c08352 100644 --- a/main.py +++ b/main.py @@ -4,6 +4,7 @@ import argparse from bot import Account, Logger +import bot from bot.config import Config from bot.metrics import start_prometheus from bot.modules.manager import ModuleManager @@ -19,7 +20,12 @@ def create_bot(config: Config, project_root: str) -> Account: if display_name in available_accounts: account.logger.info(f"Logging in with display name: {display_name}") - account.login(display_name=display_name, password=password) + account.login( + display_name=display_name, + password=password, + infura_token=config.bot.infura_token, + coingecko_api_key=config.bot.coingecko_api_key + ) elif config.bot.init_account: mnemonic = config.bot.mnemonic_phrase if not mnemonic: @@ -31,6 +37,8 @@ def create_bot(config: Config, project_root: str) -> Account: display_name=display_name, password=password, mnemonic=mnemonic, + infura_token=config.bot.infura_token, + coingecko_api_key=config.bot.coingecko_api_key ) else: raise ValueError( @@ -38,20 +46,18 @@ def create_bot(config: Config, project_root: str) -> Account: f"Available accounts: {[a['display_name'] for a in available_accounts]}" ) - account.logger.info(f"Account Info {account.info}") - if account.info["compressed_key"] != config.bot.compressed_key: raise Exception( - "Target compressed key and logged in compressed key are different" + "Target compressed key and logged in compressed key are different." ) profile_path = os.path.join(project_root, "assets", "profile.jpg") account.profile_picture = profile_path account.logger.info( - f"Account Information:\n" - f"Compressed Key: {account.info['compressed_key']}\n" - f"Public Key: {account.info['public_key']}\n" - f"URL: {account.info['url']}" + f"Account Information: {account.info['display_name']}\n" + f"\tCompressed Key: {account.info['compressed_key']}\n" + f"\tPublic Key: {account.info['public_key']}\n" + f"\tURL: {account.info['url']}" ) return account @@ -106,7 +112,7 @@ def main(): config.postgres.password, config.postgres.name, ]) - logger.info(f"Postgres configuration {config.postgres}") + if has_postgres: try: db = init_postgres(config) @@ -117,7 +123,8 @@ def main(): else: logger.info("No Postgres configuration found, running without database") - manager = ModuleManager(config.modules, account, db, logger) + shared_state = {"config": config, "project_root": project_root} + manager = ModuleManager(config.modules, account, db, logger, shared_state=shared_state) manager.discover_modules() manager.load_modules() From 82226ff53175b0975534e1aa9a814c8c34f78f17 Mon Sep 17 00:00:00 2001 From: apentori Date: Wed, 20 May 2026 19:24:50 +0200 Subject: [PATCH 07/13] fix module loading Signed-off-by: apentori --- bot/account.py | 3 +-- bot/metrics.py | 1 - bot/modules/manager.py | 5 +++++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bot/account.py b/bot/account.py index b0367bf..1b7ca62 100644 --- a/bot/account.py +++ b/bot/account.py @@ -157,13 +157,11 @@ def login(self, password: str, key_uid: Optional[str] = None, display_name: Opti # Wallet usage if infura_token: - self.logger.info("Adding Infura token") params["infuraToken"] = infura_token else: self.logger.info("No infura token") if coingecko_api_key: - self.logger.info("Adding CoinGechko token") params["coingeckoApiKey"] = coingecko_api_key else: self.logger.info("No coingecko token") @@ -530,6 +528,7 @@ def balance(self) -> pd.DataFrame: params = [[self.info["wallet_address"]], True] results = self.__call_rpc("wallets", "fetchOrGetCachedWalletBalances", params).get("result", {}).get(self.info["wallet_address"].lower(), []) if not results: + self.logger.warning(f"No result in RPC response") return empty.copy() balance = pd.DataFrame(results) diff --git a/bot/metrics.py b/bot/metrics.py index dc4b817..8a887fa 100644 --- a/bot/metrics.py +++ b/bot/metrics.py @@ -5,7 +5,6 @@ from prometheus_client import start_http_server, Gauge, Counter - def start_prometheus(prometheus_config: PrometheusConfig, manager: ModuleManager, logger: logging.Logger): if not prometheus_config.enabled: logger.info("Prometheus metrics disabled") diff --git a/bot/modules/manager.py b/bot/modules/manager.py index 055b7eb..6454ade 100644 --- a/bot/modules/manager.py +++ b/bot/modules/manager.py @@ -31,7 +31,12 @@ def module_names(self) -> list[str]: return list(self._modules.keys()) def discover_modules(self) -> None: + builtin_dir = str(Path(__file__).parent.resolve()) + self._discover_from_directory(builtin_dir) + for directory in self._modules_config.directories: + if directory == builtin_dir: + continue self._discover_from_directory(directory) self._logger.info( From 3ff1971be2b3be06a1e2172d3bae33a7cce3d6d9 Mon Sep 17 00:00:00 2001 From: apentori Date: Wed, 20 May 2026 19:27:38 +0200 Subject: [PATCH 08/13] docs: small update Signed-off-by: apentori --- .dockerignore | 4 + README.md | 127 ++++++------------- docs/SUMMARY.md | 11 +- docs/{ => development}/account.md | 0 docs/development/modules.md | 0 docs/usage/configuration.md | 194 ++++++++++++++++++++++++++++++ docs/usage/monitoring.md | 58 +++++++++ 7 files changed, 306 insertions(+), 88 deletions(-) rename docs/{ => development}/account.md (100%) create mode 100644 docs/development/modules.md create mode 100644 docs/usage/configuration.md create mode 100644 docs/usage/monitoring.md diff --git a/.dockerignore b/.dockerignore index 256108d..8898e7a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -12,3 +12,7 @@ bot/docs config.yaml .env + +accounts/ +backups/ +data-dir/ diff --git a/README.md b/README.md index efbcd9c..b05569c 100644 --- a/README.md +++ b/README.md @@ -1,91 +1,48 @@ -# [Status App Community Monitoring](https://status.app/) - -Monitoring tool for Status App communities. **No personal data is collected from users.** - - -| Field | Hashed | Description | -|:----------------------|:---------|:------------------------------------------------------------| -| **id** | **Yes** | The message's ID | -| **whisper_timestamp** | No | The whisper timestamp of the message | -| **from** | **Yes** | The public key of the user | -| **message_type** | No | The message type | -| **seen** | No | True if the message has been seen otherwise False | -| **chat_id** | No | The chat ID is a combination of community ID and channel ID | -| **community_id** | No | The ID of the community | -| **response_to** | **Yes** | Ithe public key of the user who the response is for | -| **timestamp** | No | The timestamp of the message | -| **deleted** | No | True if the message was deleted otherwise False | - -Status Bot account information can be found in [`config.yaml`](./config.yaml). - -## How it works - -```mermaid -graph LR - subgraph Communities[Status App] - subgraph Status[Status Community] - StatusMessages[Messages] - StatusInfo[Information] - end - subgraph Logos[Logos Community] - LogosMessages[Messages] - LogosInfo[Information] - end - - end - - subgraph Bot[Docker Container] - RawDataLocal[(Raw Data)] - Script[monitor.py] - end - - subgraph IFT[IFT Infrastructure] - RawDataIFT[(Raw Data)] - ProcessedDataIFT[(Processed Data)] - - end - - Communities <--> |class Account| Script - Script --> |SHA256| RawDataLocal - RawDataLocal --> |Airbyte| RawDataIFT - RawDataIFT --> |dbt| ProcessedDataIFT -``` +# Status Bot + +## Description + +This repository allow to run a Bot for the [Status App](https://status.app). The bot can be used for multiple reason: +- Sending messaging (WIP) +- [Monitoring Community](./docs/usage/monitoring.md) + + +## Setup -# Setup +### Configuration -## Environment Variables +The bot is configured via `config.yaml` and environment variables. +See the [full configuration reference](./docs/usage/configuration.md) for all options. -- `POSTGRES_USERNAME` - Postgres username. -- `POSTGRES_PASSWORD` - Postgres password. -- `POSTGRES_DATABASE` - The database name in the Postgres connection. -- `POSTGRES_HOST` - The Postgres host name that will be remotely connected to. -- `POSTGRES_PORT` - The Postgres port that will be remotely connected to. -- `STATUS_DISPLAY_NAME` - The Status display name that will be used to create an account. -- `STATUS_PASSWORD` - The Status password that will be used to create an account. -- `STATUS_MNEMONIC` - The mnemonic used to recover the account. If passed a `.bkp` file will be loaded as well. Use this when you want to login to a bot account via Status App, join a community / leave community and export the `.bkp` file. -- `STATUS_INFURA_TOKEN` - [Infura token](https://www.infura.io/) is required for **token gated communities** -- `STATUS_COINGECKO_API_KEY` - [Coingecko API Key](https://www.coingecko.com/) is required for **token gated communities** +Required settings: +- **Bot account**: `bot.display_name`, `bot.password`, and `bot.compressed_key` in `config.yaml` +- **Postgres** (optional): `POSTGRES_HOST`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_NAME` env vars -## Docker deployement +### Docker deployment -You can use the `docker-compose.yaml` to run the project. +Use the provided `docker-compose.yaml` to run the bot: -Example of `.env` file to use +```bash +docker-compose up ``` -# Status Backend connection -STATUS_DISPLAY_NAME = "bot-status" -STATUS_PASSWORD = "ChangeThisPassword" -STATUS_MNEMONIC= "test test test test test test test test test test test test" -# Necessary for communities that have tokens -STATUS_INFURA_TOKEN = "Token from https://www.infura.io/" -STATUS_COINGECKO_API_KEY = "Token from https://www.coingecko.com/" +Minimal `.env` file: + +``` +# Bot account (required) +BOT_DISPLAY_NAME=bot-status +BOT_PASSWORD=ChangeThisPassword +BOT_MNEMONIC_PHRASE=test test test test test test test test test test test test -# Database config +# Wallet API for token-gated communities (optional) +BOT_PARAMS_INFURA_TOKEN=your-infura-token +BOT_PARAMS_COINGECKO_API_KEY=your-coingecko-key + +# Database (optional) POSTGRES_HOST=database POSTGRES_PORT=5432 -POSTGRES_DATABASE=status-bot -POSTGRES_USERNAME=status +POSTGRES_NAME=status-bot +POSTGRES_USER=status POSTGRES_PASSWORD=ChangeThisOneAlso ``` @@ -98,14 +55,10 @@ conda create -n status-monitoring python=3.12 **Note**: Code has been tested with **Python 3.12**. -2. Install `monitor.py` and `bot` requirements +2. Install requirements ```bash -# To run Status bot -pip install -r ./bot/requirements.txt - -# To run monitor.py -pip install -r ./requirements.txt +pip install -r requirements.txt ``` **Note**: If you are on Windows, you will have to install `psycopg2` instead of `psycopg2-binary`. @@ -114,13 +67,13 @@ pip install -r ./requirements.txt If you have already created a Status account and want to use it with it's current data, please make sure you export the `.bkp` file and put it in folder **backups** and have the following `.env` variables: -- `STATUS_DISPLAY_NAME` -- `STATUS_PASSWORD` -- `STATUS_MNEMONIC` +- `BOT_DISPLAY_NAME` +- `BOT_PASSWORD` +- `BOT_MNEMONIC_PHRASE` ## Files -- `monitor.py` - Status community message monitoring. It will download and upload messages in parallel. +- `main.py` - Entry point for running the bot with modules (monitoring, etc.) # Guidelines diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index b2d1e9f..2f1f63e 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -1,3 +1,12 @@ # Summary -[Account](./account.md) +## Development + +- [Account](./development//account.md) +- [Modules](./development/modules.md) + + +## Usage + +- [Configuration](./usage/configuration.md) +- [Community Monitoring](./usage/monitoring.md) diff --git a/docs/account.md b/docs/development/account.md similarity index 100% rename from docs/account.md rename to docs/development/account.md diff --git a/docs/development/modules.md b/docs/development/modules.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/usage/configuration.md b/docs/usage/configuration.md new file mode 100644 index 0000000..cf52efb --- /dev/null +++ b/docs/usage/configuration.md @@ -0,0 +1,194 @@ +# Configuration + +The bot is configured through a YAML file (`config.yaml`) and environment variables. +Environment variables override values from the YAML file. + +## Loading order + +1. `config.yaml` — base configuration +2. Shell environment variables — override YAML values +3. `.env` file — override YAML values (loaded from the same directory as `config.yaml`) + +## Usage + +```bash +python main.py --config /path/to/config.yaml +``` + +The `--config` argument defaults to `./config.yaml`. + +--- + +## Reference + +### `sleep` + +| Type | Default | Description | +|------|---------|-------------| +| `int` | `10` | Sleep interval in minutes between monitoring cycles | + +### `files` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `current_state` | `str` | `"dates.pkl"` | File path for tracking latest message timestamps per chat | + +```yaml +files: + current_state: "dates.pkl" +``` + +--- + +### `bot` + +| Field | Type | Default | Env var | Description | +|-------|------|---------|---------|-------------| +| `display_name` | `str` | `""` | `BOT_DISPLAY_NAME` | Status display name used to log in | +| `public_key` | `str` | `""` | — | Expected public key for verification | +| `password` | `str` | `""` | `BOT_PASSWORD` | Status account password | +| `mnemonic_phrase` | `str` | `""` | `BOT_MNEMONIC_PHRASE` | 12-word recovery phrase (used when `init_account: true`) | +| `init_account` | `bool` | `false` | `BOT_INIT_ACCOUNT` | If `false`, the account must already exist. If `true`, creates or restores the account using `mnemonic_phrase` | +| `compressed_key` | `str` | `""` | — | Expected compressed key for verification after login | +| `infura_token` | `str` | `""` | `BOT_INFURA_TOKEN` | [Infura token](https://www.infura.io/) required for token-gated communities | +| `coingecko_api_key` | `str` | `""` | `BOT_COINGECKO_API_KEY` | [CoinGecko API key](https://www.coingecko.com/) required for token-gated communities | + + +```yaml +bot: + display_name: 'my-bot' + public_key: '0x...' + password: 'ChangeMe' + mnemonic_phrase: 'word1 word2 ... word12' + init_account: false + compressed_key: 'zQ3...' +``` + +#### `bot.params` + +Parameters passed to the `Account()` constructor. + +| Field | Type | Default | Env var | Description | +|-------|------|---------|---------|-------------| +| `domain` | `str` | `"localhost"` | `BOT_PARAMS_DOMAIN` | Status Backend hostname (`localhost` for local, `status-backend` for Docker) | +| `port` | `int` | `8080` | `BOT_PARAMS_PORT` | Status Backend API port | +| `is_secure` | `bool` | `false` | `BOT_PARAMS_IS_SECURE` | Use HTTPS instead of HTTP | +```yaml +bot: + params: + domain: "status-backend" + port: 8080 + is_secure: false + infura_token: "" + coingecko_api_key: "" +``` + +--- + +### `postgres` + +| Field | Type | Default | Env var | Description | +|-------|------|---------|---------|-------------| +| `host` | `str` | `"database"` | `POSTGRES_HOST` | Postgres server hostname | +| `port` | `int` | `5432` | `POSTGRES_PORT` | Postgres server port | +| `user` | `str` | `""` | `POSTGRES_USER` | Postgres username | +| `password` | `str` | `""` | `POSTGRES_PASSWORD` | Postgres password | +| `name` | `str` | `""` | `POSTGRES_NAME` | Postgres database name | +| `schema` | `str` | `"public"` | `POSTGRES_SCHEMA` | Database schema for storing data | +| `tables` | `dict` | `{}` | — | Mapping of data type to table name | + +```yaml +postgres: + host: database + port: 5432 + user: 'myuser' + password: 'ChangeMe' + name: 'status-bot' + schema: "status_app_monitoring" + tables: + messages: "raw_messages" + community: "raw_community_info" +``` + +--- + +### `modules` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `directories` | `list[str]` | `["./modules"]` | Directories to scan for module `.py` files | +| `enabled` | `list[str]` | `[]` | List of module names to enable | +| `settings` | `dict` | `{}` | Per-module settings (each module defines its own schema) | + +```yaml +modules: + directories: + - ./modules + enabled: + - monitoring + settings: + monitoring: + interval: 600 + max_retries: 3 + backoff_seconds: 30 +``` + +--- + +### `prometheus` + +| Field | Type | Default | Env var | Description | +|-------|------|---------|---------|-------------| +| `enabled` | `bool` | `false` | `PROMETHEUS_ENABLED` | Enable Prometheus metrics HTTP server | +| `host` | `str` | `"0.0.0.0"` | `PROMETHEUS_HOST` | Prometheus HTTP server bind address | +| `port` | `int` | `8000` | `PROMETHEUS_PORT` | Prometheus HTTP server port | + +```yaml +prometheus: + enabled: true + host: "0.0.0.0" + port: 8000 +``` + +--- + +## Environment variables + +All configuration fields can be set via environment variables using the `_` separator for nested fields. + +| Env var | Config path | +|---------|-------------| +| `BOT_DISPLAY_NAME` | `bot.display_name` | +| `BOT_PASSWORD` | `bot.password` | +| `BOT_MNEMONIC_PHRASE` | `bot.mnemonic_phrase` | +| `BOT_INIT_ACCOUNT` | `bot.init_account` | +| `BOT_PARAMS_DOMAIN` | `bot.params.domain` | +| `BOT_PARAMS_PORT` | `bot.params.port` | +| `BOT_PARAMS_IS_SECURE` | `bot.params.is_secure` | +| `BOT_PARAMS_INFURA_TOKEN` | `bot.params.infura_token` | +| `BOT_PARAMS_COINGECKO_API_KEY` | `bot.params.coingecko_api_key` | +| `POSTGRES_HOST` | `postgres.host` | +| `POSTGRES_PORT` | `postgres.port` | +| `POSTGRES_USER` | `postgres.user` | +| `POSTGRES_PASSWORD` | `postgres.password` | +| `POSTGRES_NAME` | `postgres.name` | +| `POSTGRES_SCHEMA` | `postgres.schema` | +| `PROMETHEUS_ENABLED` | `prometheus.enabled` | +| `PROMETHEUS_HOST` | `prometheus.host` | +| `PROMETHEUS_PORT` | `prometheus.port` | + +Example `.env` file: + +``` +# Bot account +BOT_DISPLAY_NAME=my-bot +BOT_PASSWORD=ChangeThisPassword +BOT_MNEMONIC_PHRASE=test test test test test test test test test test test test + +# Postgres +POSTGRES_HOST=database +POSTGRES_PORT=5432 +POSTGRES_NAME=status-bot +POSTGRES_USER=status +POSTGRES_PASSWORD=ChangeThisOneAlso +``` diff --git a/docs/usage/monitoring.md b/docs/usage/monitoring.md new file mode 100644 index 0000000..6398f98 --- /dev/null +++ b/docs/usage/monitoring.md @@ -0,0 +1,58 @@ +# Community Monitoring + +The Status Bot can be used to monitore activity in community + +## Fetch Data + + **No personal data is collected from users.** + + +| Field | Hashed | Description | +|:----------------------|:---------|:------------------------------------------------------------| +| **id** | **Yes** | The message's ID | +| **whisper_timestamp** | No | The whisper timestamp of the message | +| **from** | **Yes** | The public key of the user | +| **message_type** | No | The message type | +| **seen** | No | True if the message has been seen otherwise False | +| **chat_id** | No | The chat ID is a combination of community ID and channel ID | +| **community_id** | No | The ID of the community | +| **response_to** | **Yes** | Ithe public key of the user who the response is for | +| **timestamp** | No | The timestamp of the message | +| **deleted** | No | True if the message was deleted otherwise False | + +Status Bot account information can be found in [`config.yaml`](./config.yaml). + +## How it works + +```mermaid +graph LR + subgraph Communities[Status App] + subgraph Status[Status Community] + StatusMessages[Messages] + StatusInfo[Information] + end + subgraph Logos[Logos Community] + LogosMessages[Messages] + LogosInfo[Information] + end + + end + + subgraph Bot[Docker Container] + RawDataLocal[(Raw Data)] + Script[monitor.py] + end + + subgraph IFT[IFT Infrastructure] + RawDataIFT[(Raw Data)] + ProcessedDataIFT[(Processed Data)] + + end + + Communities <--> |class Account| Script + Script --> |SHA256| RawDataLocal + RawDataLocal --> |Airbyte| RawDataIFT + RawDataIFT --> |dbt| ProcessedDataIFT +``` + + From 47a561e38ea492ea11a5efe63117f8dd0defc86c Mon Sep 17 00:00:00 2001 From: apentori Date: Wed, 20 May 2026 20:22:05 +0200 Subject: [PATCH 09/13] feat: api server Signed-off-by: apentori --- bot/account.py | 2 - bot/config.py | 12 +++-- bot/modules/api_server.py | 43 +++++++++++++++++ bot/modules/messaging.py | 99 +++++++++++++++++++++++++++++++++++++++ config.yaml | 26 +++++----- docker-compose.yaml | 1 + main.py | 12 ++++- requirements.txt | 4 +- 8 files changed, 178 insertions(+), 21 deletions(-) create mode 100644 bot/modules/api_server.py create mode 100644 bot/modules/messaging.py diff --git a/bot/account.py b/bot/account.py index 1b7ca62..12270e0 100644 --- a/bot/account.py +++ b/bot/account.py @@ -186,7 +186,6 @@ def login(self, password: str, key_uid: Optional[str] = None, display_name: Opti self.__info = { "public_key": event["public-key"], "url": None, - "emojis": event["emojiHash"], "key_uid": event["key-uid"], "compressed_key": event["compressedKey"], "mnemonic": event.get("mnemonic", mnemonic), @@ -393,7 +392,6 @@ def contacts(self) -> dict[str, dict]: "url": self.__call_rpc("urls", "shareUserURLWithData", [contact["id"]]).get("result"), "chat_id": contact["id"], "key_uid": contact["compressedKey"], - "emojis": contact["emojiHash"], "contact_state": self.__mappings["contact_request"][contact["contactRequestState"]], "external_contact_state": self.__mappings["contact_request"][contact["contactRequestRemoteState"]], "has_added_us": contact["hasAddedUs"], diff --git a/bot/config.py b/bot/config.py index 88bf323..d2366d6 100644 --- a/bot/config.py +++ b/bot/config.py @@ -3,7 +3,7 @@ from pydantic_settings.sources import YamlConfigSettingsSource -class BotParams(BaseModel): +class BackendConfig(BaseModel): domain: str = "localhost" port: int = 8080 is_secure: bool = False @@ -18,8 +18,6 @@ class BotConfig(BaseModel): compressed_key: str = "" infura_token: str = "" coingecko_api_key: str = "" - params: BotParams = BotParams() - class PostgresConfig(BaseModel): host: str = "database" @@ -31,6 +29,12 @@ class PostgresConfig(BaseModel): tables: dict = {} +class ApiConfig(BaseModel): + enable: bool = True + host: str = "0.0.0.0" + port: int = 8081 + + class PrometheusConfig(BaseModel): enabled: bool = False host: str = "0.0.0.0" @@ -56,6 +60,8 @@ class Config(BaseSettings): sleep: int = 10 files: FilesConfig = FilesConfig() bot: BotConfig = BotConfig() + backend: BackendConfig = BackendConfig() + api: ApiConfig = ApiConfig() modules: ModulesConfig = ModulesConfig() prometheus: PrometheusConfig = PrometheusConfig() postgres: PostgresConfig = PostgresConfig() diff --git a/bot/modules/api_server.py b/bot/modules/api_server.py new file mode 100644 index 0000000..aa3a567 --- /dev/null +++ b/bot/modules/api_server.py @@ -0,0 +1,43 @@ +import threading + +import uvicorn +from fastapi import FastAPI + +from bot.modules.base import BaseModule, ModuleType + + +class APIServerModule(BaseModule): + + @property + def module_type(self) -> ModuleType: + return ModuleType.SERVICE + + def on_start(self): + api_config = self.ctx.shared_state["config"].api + self._host = api_config.host + self._port = api_config.port + self._app: FastAPI = self.ctx.shared_state["fastapi_app"] + self._server = None + + def execute(self): + if not self.ctx.shared_state["config"].api.enable: + self.ctx.logger.info("API server is disabled, skipping startup") + return + + config = uvicorn.Config( + self._app, + host=self._host, + port=self._port, + log_level="info", + ) + server = uvicorn.Server(config) + self._server = server + thread = threading.Thread(target=server.run, daemon=True) + thread.start() + self.ctx.stop_event.wait() + server.should_exit = True + thread.join(timeout=10) + + def on_stop(self): + if self._server: + self._server.should_exit = True diff --git a/bot/modules/messaging.py b/bot/modules/messaging.py new file mode 100644 index 0000000..0a53f74 --- /dev/null +++ b/bot/modules/messaging.py @@ -0,0 +1,99 @@ +from datetime import datetime +from typing import Optional + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel + +from bot.modules.base import BaseModule, ModuleType + + +class AddContactRequest(BaseModel): + public_key: str + display_name: Optional[str] = None + + +class SendMessageRequest(BaseModel): + text: str + + +class MessagingModule(BaseModule): + + @property + def module_type(self) -> ModuleType: + return ModuleType.SERVICE + + def on_start(self): + app: FastAPI = self.ctx.shared_state["fastapi_app"] + self._setup_routes(app) + + def _setup_routes(self, app: FastAPI): + account = self.ctx.account + + @app.get("/health") + def health(): + return {"status": "healthy"} + + @app.get("/api/v1/contacts") + def get_contacts(): + return account.contacts + + @app.post("/api/v1/contacts", status_code=201) + def add_contact(payload: AddContactRequest): + try: + account.add_contact(payload.public_key, payload.display_name) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + return {"status": "ok"} + + @app.delete("/api/v1/contacts/{public_key}") + def remove_contact(public_key: str): + if not account.remove_contact(public_key): + raise HTTPException( + status_code=404, + detail="Contact not found or already removed", + ) + return {"status": "ok"} + + @app.get("/api/v1/chats") + def get_chats(): + return account.chats + + @app.get("/api/v1/chats/{chat_id}/messages") + def get_messages( + chat_id: str, + start_timestamp: Optional[str] = None, + end_timestamp: Optional[str] = None, + ): + start = None + end = None + if start_timestamp: + try: + start = datetime.fromisoformat(start_timestamp) + except ValueError: + raise HTTPException( + status_code=400, + detail="Invalid start_timestamp, use ISO format", + ) + if end_timestamp: + try: + end = datetime.fromisoformat(end_timestamp) + except ValueError: + raise HTTPException( + status_code=400, + detail="Invalid end_timestamp, use ISO format", + ) + return account.get_messages(chat_id, start, end) + + @app.post("/api/v1/chats/{chat_id}/messages", status_code=201) + def send_message(chat_id: str, payload: SendMessageRequest): + if not payload.text: + raise HTTPException(status_code=400, detail="Message text is required") + account.send_message(chat_id, payload.text) + return {"status": "ok"} + + @app.get("/api/v1/communities") + def get_communities(): + return account.communities + + def execute(self): + self.ctx.stop_event.wait() diff --git a/config.yaml b/config.yaml index c0b36af..c764035 100644 --- a/config.yaml +++ b/config.yaml @@ -13,24 +13,24 @@ files: current_state: "dates.pkl" bot: + init_account: false display_name: 'e-raccoon' public_key: "0xExample" compressed_key: "example-compressed-key" - password: 'ChangeMe' - mnemonic_phrase: 'some random word ...' - init_account: true - infura_token: "" - coingecko_api_key: "" - # Parameters based on Account class - params: - domain: "status-backend" - port: 8080 - is_secure: false + +backend: + domain: "status-backend" + port: 8080 + is_secure: false + +api: + enable: true + host: "0.0.0.0" + port: 8081 modules: - directories: - - ./modules - enabled: [] + directories: [] + enabled: ["messaging"] settings: {} prometheus: diff --git a/docker-compose.yaml b/docker-compose.yaml index d27faf6..39825dd 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -34,6 +34,7 @@ services: env_file: - .env ports: + - 8081:8081 - 8000:8000 networks: - status-bridge diff --git a/main.py b/main.py index 6c08352..9b145d7 100644 --- a/main.py +++ b/main.py @@ -3,6 +3,8 @@ import signal import argparse +from fastapi import FastAPI + from bot import Account, Logger import bot from bot.config import Config @@ -12,7 +14,7 @@ def create_bot(config: Config, project_root: str) -> Account: - account = Account(**config.bot.params.model_dump()) + account = Account(**config.backend.model_dump()) available_accounts = [a["display_name"] for a in account.available_accounts] display_name = config.bot.display_name @@ -100,6 +102,7 @@ def main(): project_root = os.path.dirname(os.path.abspath(__file__)) try: + logger.info(f"{config}") account = create_bot(config, project_root) except Exception as e: logger.error(f"Failed to create bot account: {e}") @@ -123,7 +126,12 @@ def main(): else: logger.info("No Postgres configuration found, running without database") - shared_state = {"config": config, "project_root": project_root} + fastapi_app = FastAPI(title="Status Bot API") + shared_state = {"config": config, "project_root": project_root, "fastapi_app": fastapi_app} + + if config.api.enable and "api_server" not in config.modules.enabled: + config.modules.enabled.append("api_server") + manager = ModuleManager(config.modules, account, db, logger, shared_state=shared_state) manager.discover_modules() manager.load_modules() diff --git a/requirements.txt b/requirements.txt index 91a7e40..e5d2708 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,6 @@ psycopg2-binary sqlalchemy prometheus-client pydantic>=2.0 -pydantic-settings>=2.0 \ No newline at end of file +pydantic-settings>=2.0 +fastapi +uvicorn \ No newline at end of file From 19091a14d023b3d6b6cad2aad6fa951e29523b0c Mon Sep 17 00:00:00 2001 From: apentori Date: Wed, 20 May 2026 22:17:09 +0200 Subject: [PATCH 10/13] docs: update Documentation Renaming configuration for lisibility Signed-off-by: apentori --- README.md | 45 +- bot/__init__.py | 2 + bot/config.py | 8 +- bot/metrics.py | 51 +- postgres.py => bot/postgres.py | 0 docs/SUMMARY.md | 2 + docs/development/modules.md | 139 +++++ docs/refactoring-module-plan.md | 929 -------------------------------- docs/usage/configuration.md | 100 ++-- docs/usage/messaging.md | 162 ++++++ docs/usage/metrics.md | 66 +++ main.py | 25 +- module-refactoring.md | 591 -------------------- monitor.py | 325 ----------- 14 files changed, 457 insertions(+), 1988 deletions(-) rename postgres.py => bot/postgres.py (100%) delete mode 100644 docs/refactoring-module-plan.md create mode 100644 docs/usage/messaging.md create mode 100644 docs/usage/metrics.md delete mode 100644 module-refactoring.md delete mode 100644 monitor.py diff --git a/README.md b/README.md index b05569c..07557e4 100644 --- a/README.md +++ b/README.md @@ -3,50 +3,31 @@ ## Description This repository allow to run a Bot for the [Status App](https://status.app). The bot can be used for multiple reason: -- Sending messaging (WIP) +- [Interracting with Status Chats](./docs/usage/messaging.md) - [Monitoring Community](./docs/usage/monitoring.md) ## Setup -### Configuration - -The bot is configured via `config.yaml` and environment variables. -See the [full configuration reference](./docs/usage/configuration.md) for all options. - -Required settings: -- **Bot account**: `bot.display_name`, `bot.password`, and `bot.compressed_key` in `config.yaml` -- **Postgres** (optional): `POSTGRES_HOST`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_NAME` env vars +The recommanded deployement option is with docker compose. ### Docker deployment -Use the provided `docker-compose.yaml` to run the bot: +The [docker compose](./docker-compose.yaml) deployement will spin 3 containers: +- Status Backend: to interract with the Logos Delivery network used by Status App. +- Status Bot: the instance of the bot +- A Postgres database + +You can run it with: ```bash -docker-compose up +docker-compose up -d --build ``` +it will require a `.env` file for secrets and `config.yaml`. The configuration is detailed in [the documentation](./docs/usage//configuration.md). -Minimal `.env` file: - -``` -# Bot account (required) -BOT_DISPLAY_NAME=bot-status -BOT_PASSWORD=ChangeThisPassword -BOT_MNEMONIC_PHRASE=test test test test test test test test test test test test - -# Wallet API for token-gated communities (optional) -BOT_PARAMS_INFURA_TOKEN=your-infura-token -BOT_PARAMS_COINGECKO_API_KEY=your-coingecko-key - -# Database (optional) -POSTGRES_HOST=database -POSTGRES_PORT=5432 -POSTGRES_NAME=status-bot -POSTGRES_USER=status -POSTGRES_PASSWORD=ChangeThisOneAlso -``` +### Python -## Python +You can also run the bot with python, it will require a Status Backend instance available. 1. Setup environment. [Conda](https://www.anaconda.com/) example: ```bash @@ -71,9 +52,7 @@ If you have already created a Status account and want to use it with it's curren - `BOT_PASSWORD` - `BOT_MNEMONIC_PHRASE` -## Files -- `main.py` - Entry point for running the bot with modules (monitoring, etc.) # Guidelines diff --git a/bot/__init__.py b/bot/__init__.py index d607e0e..70e9bf3 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -1,2 +1,4 @@ from .account import Account from .logger import Logger +from .postgres import Postgres +from .config import Config diff --git a/bot/config.py b/bot/config.py index d2366d6..4155768 100644 --- a/bot/config.py +++ b/bot/config.py @@ -19,7 +19,7 @@ class BotConfig(BaseModel): infura_token: str = "" coingecko_api_key: str = "" -class PostgresConfig(BaseModel): +class DatabaseConfig(BaseModel): host: str = "database" port: int = 5432 user: str = "" @@ -35,7 +35,7 @@ class ApiConfig(BaseModel): port: int = 8081 -class PrometheusConfig(BaseModel): +class MetricsConfig(BaseModel): enabled: bool = False host: str = "0.0.0.0" port: int = 8000 @@ -63,8 +63,8 @@ class Config(BaseSettings): backend: BackendConfig = BackendConfig() api: ApiConfig = ApiConfig() modules: ModulesConfig = ModulesConfig() - prometheus: PrometheusConfig = PrometheusConfig() - postgres: PostgresConfig = PostgresConfig() + metrics: MetricsConfig = MetricsConfig() + database: DatabaseConfig = DatabaseConfig() _yaml_file = "./config.yaml" diff --git a/bot/metrics.py b/bot/metrics.py index 8a887fa..5147ddc 100644 --- a/bot/metrics.py +++ b/bot/metrics.py @@ -1,39 +1,34 @@ import logging -from bot.config import PrometheusConfig +from bot.config import MetricsConfig from bot.modules.manager import ModuleManager from prometheus_client import start_http_server, Gauge, Counter -def start_prometheus(prometheus_config: PrometheusConfig, manager: ModuleManager, logger: logging.Logger): - if not prometheus_config.enabled: - logger.info("Prometheus metrics disabled") +def start_prometheus(metrics_config: MetricsConfig, manager: ModuleManager, logger: logging.Logger): + if not metrics_config.enabled: + logger.info("Prometheus metrics exporter disabled") return - try: - health = Gauge("status_bot_health", "Bot health status") - version = Gauge("status_bot_version", "Bot version", ["version"]) - module_loaded = Gauge( - "status_bot_module_loaded", "Module loaded", ["module"] - ) - module_errors = Counter( - "status_bot_module_errors_total", "Module errors", ["module"] - ) - module_restarts = Counter( - "status_bot_module_restarts_total", "Module restarts", ["module"] - ) + health = Gauge("status_bot_health", "Bot health status") + version = Gauge("status_bot_version", "Bot version", ["version"]) + module_loaded = Gauge( + "status_bot_module_loaded", "Module loaded", ["module"] + ) + module_errors = Counter( + "status_bot_module_errors_total", "Module errors", ["module"] + ) + module_restarts = Counter( + "status_bot_module_restarts_total", "Module restarts", ["module"] + ) - health.set(1) - version.labels(version="0.1.0").set(1) + health.set(1) + version.labels(version="0.1.0").set(1) - for module_name in manager.module_names: - module_loaded.labels(module=module_name).set(1) + for module_name in manager.module_names: + module_loaded.labels(module=module_name).set(1) - host = prometheus_config.host - port = prometheus_config.port - start_http_server(port, host) - logger.info(f"Prometheus metrics server started on {host}:{port}") - except ImportError: - logger.warning( - "prometheus-client not installed. Install with: pip install prometheus-client" - ) + host = metrics_config.host + port = metrics_config.port + start_http_server(port, host) + logger.info(f"Prometheus metrics exporter server started on {host}:{port}") diff --git a/postgres.py b/bot/postgres.py similarity index 100% rename from postgres.py rename to bot/postgres.py diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 2f1f63e..115a185 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -9,4 +9,6 @@ ## Usage - [Configuration](./usage/configuration.md) +- [API](./usage/api.md) - [Community Monitoring](./usage/monitoring.md) +- [Metrics](./usage/metrics.md) diff --git a/docs/development/modules.md b/docs/development/modules.md index e69de29..13a7136 100644 --- a/docs/development/modules.md +++ b/docs/development/modules.md @@ -0,0 +1,139 @@ +# Modules + +Modules are plugins that extend the bot with custom logic. Each module runs in its own daemon thread and follows a defined lifecycle. + +--- + +## Module types + +| Type | Behaviour | Use case | +|------|-----------|----------| +| `PERIODIC` | Calls `execute()` in a loop, sleeping `interval` seconds between runs | Scheduled data extraction, polling | +| `EVENT` | Iterates over WebSocket signal events, calling `on_event()` for each one | Real-time message reactions, auto-reply | +| `SERVICE` | Calls `execute()` once. Expected to block until shutdown (e.g. runs a server) | HTTP API server, long-running workers | + +--- + +## BaseModule API + +```python +from bot.modules.base import BaseModule, ModuleType + +class MyModule(BaseModule): + + @property + def module_type(self) -> ModuleType: + return ModuleType.PERIODIC + + def on_start(self): + ... # called once when the module starts + + def execute(self): + ... # main logic (called periodically or once) + + def on_stop(self): + ... # called once when the module stops + + def on_event(self, event: dict): + ... # handle a signal event (EVENT type only) +``` + +### ModuleContext + +Every module receives a `ModuleContext` via the constructor, accessible as `self.ctx`: + +| Field | Type | Description | +|-------|------|-------------| +| `account` | `Account` | Logged-in Status account | +| `config` | `ModuleConfig` | This module's configuration | +| `logger` | `Logger` | Shared logger | +| `db` | `Postgres / None` | Optional Postgres connection | +| `shared_state` | `dict` | Cross-module shared data | +| `stop_event` | `threading.Event` | Set when the bot is shutting down | + +### ModuleConfig + +Each module receives only its own settings from `config.yaml`: + +```python +self.ctx.config.name # module identifier +self.ctx.config.interval # seconds between PERIODIC runs +self.ctx.config.max_retries # restart attempts before permanent failure +self.ctx.config.settings # dict of module-specific settings +``` + +--- + +## Adding API routes + +Modules that want to expose HTTP endpoints can register routes on the shared FastAPI app instead of starting their own server. + +The central `api_server` module owns the uvicorn lifecycle. Other modules register routes during `on_start()`: + +```python +class MyAPIModule(BaseModule): + + @property + def module_type(self) -> ModuleType: + return ModuleType.SERVICE + + def on_start(self): + app = self.ctx.shared_state["fastapi_app"] + self._setup_routes(app) + + def _setup_routes(self, app): + @app.get("/api/v1/my-endpoint") + def my_handler(): + return {"hello": "world"} + + def execute(self): + self.ctx.stop_event.wait() # block until shutdown +``` + +The `api_server` module is auto-loaded whenever any API module is enabled and `api.enable` is `true`. + +--- + +## Signals and EVENT modules + +EVENT modules react to Status WebSocket signals: + +```python +class AutoReplyModule(BaseModule): + + @property + def module_type(self) -> ModuleType: + return ModuleType.EVENT + + def on_start(self): + self._commands = self.ctx.config.settings.get("commands", {}) + + def on_event(self, event: dict): + messages = event.get("event", {}).get("messages", []) + for msg in messages: + self.ctx.account.send_message(msg["chatId"], "Message received") +``` + +The signal listener respects `stop_event` for graceful shutdown. + +--- + +## Utility functions + +`bot/modules/utils.py` provides shared helpers: + +| Function | Description | +|----------|-------------| +| `to_sha256_hash(value)` | Returns the SHA-256 hex digest of a string | +| `to_midnight(timestamp)` | Truncates a datetime to the start of its day | +| `save_file(file_path, data)` | Saves a DataFrame as CSV or pickles any other object | + +--- + +## Best practices + +- **Error isolation**: One module crash never affects others. The `ModuleManager` restarts failed modules with exponential backoff. +- **Graceful shutdown**: Always check `self.ctx.stop_event.is_set()` or use `self.ctx.stop_event.wait()` in long-running loops. +- **No blocking in EVENT modules**: EVENT handlers process one event at a time — keep `on_event()` fast. +- **Thread safety**: Modules run in separate threads. Use `shared_state` with caution for shared mutable data. +- **Configuration**: Read module-specific settings from `self.ctx.config.settings` — each module gets its own config namespace. diff --git a/docs/refactoring-module-plan.md b/docs/refactoring-module-plan.md deleted file mode 100644 index af560ac..0000000 --- a/docs/refactoring-module-plan.md +++ /dev/null @@ -1,929 +0,0 @@ -# Refactoring Plan: Status Bot Modular Architecture - -> This document captures the full context, design decisions, and step-by-step -> implementation plan for transforming `monitor.py` into a modular plugin-based -> system. It is designed to be self-contained so the refactoring can span -> multiple sessions. - ---- - -## Table of Contents - -1. [Current Architecture](#1-current-architecture) -2. [Issues with the Current Codebase](#2-issues-with-the-current-codebase) -3. [Target Architecture](#3-target-architecture) -4. [Module System Design](#4-module-system-design) - - 4.1 ModuleType - - 4.2 ModuleConfig - - 4.3 ModuleContext - - 4.4 BaseModule (ABC) - - 4.5 ModuleManager -5. [Threading & Lifecycle](#5-threading--lifecycle) -6. [Error Handling & Retry](#6-error-handling--retry) -7. [Configuration Shape](#7-configuration-shape) -8. [Prometheus Metrics](#8-prometheus-metrics) -9. [Signal.listen() Graceful Shutdown](#9-signallisten-graceful-shutdown) -10. [Implementation Steps](#10-implementation-steps) - - [Step 1: Core Framework + Entry Point](#step-1-core-framework--entry-point) - - [Step 2: Refactor monitor.py → MonitoringModule](#step-2-refactor-monitorpy--monitoringmodule) - - [Step 3: Connection Pool + Retry Logic](#step-3-connection-pool--retry-logic) - - [Step 4: Example Event-Driven Module](#step-4-example-event-driven-module) - - [Step 5: Tests + Documentation](#step-5-tests--documentation) -11. [File-by-File Summary](#11-file-by-file-summary) - ---- - -## 1. Current Architecture - -``` -/repos/status-bot/ -├── bot/ -│ ├── __init__.py # Exports Account and Logger -│ ├── account.py # Account class — Status Backend API wrapper (1112 lines) -│ ├── signal.py # Signal class — WebSocket handler for Status signals -│ ├── logger.py # Logger singleton -│ └── requirements.txt -├── modules/ # (does not exist yet) -├── accounts/ -├── assets/ -├── data-dir/ # Status Backend runtime data -├── docs/ -├── tests/ # Empty -├── monitor.py # Main entry point — 316-line monolithic script -├── postgres.py # Postgres connector -├── config.yaml -├── Dockerfile -├── docker-compose.yaml -├── module-refactoring.md # Initial draft (superseded by this document) -└── README.md -``` - -### Current Flow - -`monitor.py` does everything in one file: -1. Config loading (`load_config`) -2. Bot creation (`create_bot`) -3. Community data extraction (`download`) -4. Data storage / upload (`store`) -5. Main loop (while True: download → store → sleep) - -### SDK Layer (`bot/`) - -The `bot/` package is well-structured and reusable: -- **`Account`** — wraps Status Backend HTTP / RPC / WebSocket APIs -- **`Signal`** — WebSocket event handling (single-get + streaming) -- **`Logger`** — singleton logging - -These should remain largely unchanged. - ---- - -## 2. Issues with the Current Codebase - -| Issue | Description | -|-------|-------------| -| **Monolithic** | `monitor.py` combines config, bot init, extraction, storage, and cleanup in one file | -| **Hardcoded pipeline** | `download()` then `store()` — no way to run independently or add middleware | -| **Tight coupling** | `store()` knows about pickle format from `download()`; `latest_dates` is shared implicitly | -| **No error isolation** | One failure can corrupt state (files deleted mid-failure) | -| **Untestable** | Global functions reference filesystem directly; no mocking boundary | -| **No concurrency** | Single-threaded loop — if monitor sleeps 10 min, auto-reply can't react | -| **SQL injection risk** | `postgres.py` uses f-strings for schema/table/column identifiers | -| **No module system** | Adding a new feature means either bloating `monitor.py` or duplicating init code | -| **No graceful shutdown** | No SIGTERM handler; `Account.__del__` is unreliable in Docker | - ---- - -## 3. Target Architecture - -``` -main.py ────────────────────────────────────────────────────────┐ - │ - Load config.yaml & .env │ - │ - Create Account (login) │ - │ - Create Postgres (connection pool) │ - │ - Init ModuleManager(config, account, db, logger) │ - │ - manager.start_all() → returns immediately │ - │ - Start Prometheus HTTP endpoint (/metrics) │ - │ - signal.signal(SIGTERM) → manager.stop_all() │ - │ - Wait forever (or until all modules die) │ - └───────────────────────────────────────────────────────────────┘ - │ - ┌─────────────┴─────────────┐ - │ ModuleManager │ - │ - discover_modules() │ - │ - load_modules() │ - │ - start_all() → threads │ - │ - stop_all() → join │ - └─────────────┬──────────────┘ - │ - ┌──────────────────┼──────────────────┐ - ▼ ▼ ▼ - MonitoringModule AutoReplyModule (user modules) - (PERIODIC, thread) (EVENT, thread) ... - │ │ - │ while running: │ for event in - │ download() │ signal.listen(): - │ store() │ on_event() - │ sleep(N) │ -``` - -### Directory Layout After Refactoring - -``` -status-bot/ -├── bot/ # SDK layer (minimal changes) -│ ├── __init__.py -│ ├── account.py -│ ├── signal.py # ← modified: listen() accepts stop_event -│ ├── logger.py -│ └── modules/ # ← NEW: module framework -│ ├── __init__.py -│ ├── base.py # BaseModule, ModuleType, ModuleConfig, ModuleContext -│ ├── manager.py # ModuleManager -│ └── utils.py # Shared helpers (to_sha256_hash, etc.) -│ -├── modules/ # ← NEW: user-space modules -│ ├── __init__.py -│ ├── monitoring.py # MonitoringModule (extracted from monitor.py) -│ └── auto_reply.py # Example event-driven module -│ -├── main.py # ← NEW: entry point -├── postgres.py # ← modified: connection pool -├── config.yaml # ← modified: + modules: + prometheus: sections -├── Dockerfile # ← modified: ENTRYPOINT → main.py -│ -├── tests/ -│ ├── test_base.py -│ ├── test_manager.py -│ └── test_monitoring_module.py -│ -└── docs/ - └── refactoring-module-plan.md # This document -``` - ---- - -## 4. Module System Design - -### 4.1 ModuleType (Enum) - -Defines how a module is executed by the manager. - -```python -class ModuleType(Enum): - PERIODIC = "periodic" # Runs execute() on a fixed interval in its own thread - EVENT = "event" # Reacts to Signal events via on_event() callback - SERVICE = "service" # Runs once and keeps running (blocking) -``` - -| Type | Execution | Example | -|------|-----------|---------| -| `PERIODIC` | Thread loop: `execute()` → sleep(interval) | Monitoring, Storage | -| `EVENT` | Thread: `for event in signal.listen(stop_event): on_event(event)` | Auto-reply, Mentions | -| `SERVICE` | Thread: `execute()` runs forever | WebSocket listener | - -### 4.2 ModuleConfig (Dataclass) - -Configuration for a single module. Each module receives only its own config, not the full config.yaml. - -```python -@dataclass -class ModuleConfig: - name: str # Module identifier (matches config key) - enabled: bool = True - interval: int = 60 # For PERIODIC modules (seconds) - max_retries: int = 3 # Max restart attempts before permanent failure - backoff_seconds: int = 30 # Wait between restarts (doubles each attempt) - settings: dict = None # Module-specific arbitrary settings - - def __post_init__(self): - if self.settings is None: - self.settings = {} -``` - -### 4.3 ModuleContext (Dataclass) - -Shared dependencies injected into every module. Wrapping in a single dataclass makes it easy to add new shared resources later without changing module constructors. - -```python -@dataclass -class ModuleContext: - account: "Account" # Logged-in Status bot account - config: ModuleConfig # This module's config only - logger: logging.Logger # Shared logger - db: Optional["Postgres"] = None # Database connection (optional) - shared_state: dict = field(default_factory=dict) # Cross-module state - stop_event: threading.Event = None # Signal for graceful shutdown -``` - -### 4.4 BaseModule (ABC) - -```python -class BaseModule(ABC): - - def __init__(self, ctx: ModuleContext): - self._ctx = ctx - self._running = False - - # --- Properties --- - - @property - def ctx(self) -> ModuleContext: - return self._ctx - - @property - @abstractmethod - def module_type(self) -> ModuleType: - """Return the execution type of this module.""" - ... - - @property - def name(self) -> str: - return self._ctx.config.name - - # --- Lifecycle --- - - @abstractmethod - def execute(self) -> Any: - """ - Main logic. Called periodically (PERIODIC), once (SERVICE), - or not at all for EVENT modules. - """ - ... - - def on_start(self) -> None: - """Called once when the module starts. Override for init logic.""" - - def on_stop(self) -> None: - """Called once when the module stops. Override for cleanup.""" - - def on_event(self, event: dict) -> Any: - """Handle a Signal event (for EVENT type modules).""" - - # --- Internal --- - - @property - def is_running(self) -> bool: - return self._running -``` - -### 4.5 ModuleManager - -```python -class ModuleManager: - - def __init__(self, config: dict, account: Account, db: Optional[Postgres], logger: Logger): - self._config = config # Full config (for module settings extraction) - self._account = account - self._db = db - self._logger = logger - self._modules: dict[str, BaseModule] = {} - self._module_classes: dict[str, Type[BaseModule]] = {} - self._threads: dict[str, Thread] = {} - self._stop_event = threading.Event() - - def discover_modules(self) -> None: - """Scan configured directories for BaseModule subclasses.""" - for directory in self._config["modules"]["directories"]: - for py_file in Path(directory).glob("*.py"): - if py_file.name.startswith("_"): - continue - # importlib: spec → module → find BaseModule subclasses - # Store in self._module_classes[file_stem] = class - - def load_modules(self) -> None: - """Instantiate only the enabled modules.""" - for module_name in self._config["modules"]["enabled"]: - cls = self._module_classes.get(module_name) - if not cls: - self._logger.error(f"Module '{module_name}' not found") - continue - settings = self._config["modules"]["settings"].get(module_name, {}) - module_config = ModuleConfig( - name=module_name, - interval=settings.get("interval", 60), - max_retries=settings.get("max_retries", 3), - backoff_seconds=settings.get("backoff_seconds", 30), - settings=settings, - ) - ctx = ModuleContext( - account=self._account, - config=module_config, - logger=self._logger, - db=self._db, - stop_event=self._stop_event, - ) - self._modules[module_name] = cls(ctx) - - def start_all(self) -> None: - """Start all modules in their own threads. Returns immediately.""" - for name, module in self._modules.items(): - t = Thread(target=self._run_module_wrapper, args=(module,), daemon=True) - self._threads[name] = t - t.start() - - def stop_all(self) -> None: - """Trigger graceful shutdown of all modules.""" - self._stop_event.set() - for name, t in self._threads.items(): - t.join(timeout=5) - - def _run_module_wrapper(self, module: BaseModule) -> None: - """Run a single module with retry/backoff logic (see Section 6).""" - ... -``` - ---- - -## 5. Threading & Lifecycle - -### Design Decisions - -| Decision | Choice | Rationale | -|----------|--------|-----------| -| Concurrency | `threading.Thread` | Simple, works with blocking I/O, no asyncio refactor needed | -| Shutdown signal | `threading.Event` | Thread-safe, all loops check it | -| SIGTERM handling | `signal.signal(SIGTERM, handler)` | Required for Docker `docker stop` | -| Manager return | Returns immediately | `main.py` owns the wait / signal handling | -| Daemon threads | `daemon=True` | Prevents hanging if stop_all() hangs | - -### Lifecycle Diagram - -``` -main.py - │ - ├── load_config() - ├── Account().login() - ├── Postgres() - ├── ModuleManager(config, account, db, logger) - ├── manager.discover_modules() - ├── manager.load_modules() - │ - ├── manager.start_all() - │ ├── Thread(MONITORING).start() - │ ├── Thread(AUTO_REPLY).start() - │ └── returns immediately - │ - ├── start Prometheus HTTP server (thread or daemon) - │ - ├── signal.signal(SIGTERM, handler) - │ └── handler → manager.stop_all() - │ - └── wait (threading.Event().wait() or while True: sleep) -``` - -### Thread-per-Module: PERIODIC - -```python -def _run_module_wrapper(self, module: BaseModule): - try: - module.on_start() - except Exception: - ... # handle - - if module.module_type == ModuleType.PERIODIC: - while not self._stop_event.is_set(): - try: - module.execute() - except Exception: - ... # retry/backoff (see Section 6) - self._stop_event.wait(module.interval) -``` - -### Thread-per-Module: EVENT - -```python - elif module.module_type == ModuleType.EVENT: - for event in self._account.signal.listen("messages.new", stop_event=self._stop_event): - if self._stop_event.is_set(): - break - try: - module.on_event(event) - except Exception: - ... # log, don't crash -``` - -### Thread-per-Module: SERVICE - -```python - elif module.module_type == ModuleType.SERVICE: - try: - module.execute() # blocking — runs until stop - except Exception: - ... # retry/backoff -``` - ---- - -## 6. Error Handling & Retry - -### Principles - -1. **Error isolation**: One module crash never brings down another module or the bot. -2. **Retry with backoff**: Failed modules are restarted with exponential backoff. -3. **Permanent failure**: After `max_retries` consecutive failures, the module is marked dead and not restarted. -4. **Bot survival**: The bot stops only if: - - It cannot log in to the Status account (fatal). - - No module started successfully (all modules are permanently dead). - - A module can be dead while others continue running. - -### Retry Logic in `_run_module_wrapper` - -```python -def _run_module_wrapper(self, module: BaseModule): - retries = 0 - max_retries = module.ctx.config.max_retries - backoff = module.ctx.config.backoff_seconds - - while retries <= max_retries and not self._stop_event.is_set(): - try: - module._running = True - module.on_start() - - if module.module_type == ModuleType.PERIODIC: - while not self._stop_event.is_set(): - module.execute() - self._stop_event.wait(module.interval) - elif module.module_type == ModuleType.EVENT: - for event in self._account.signal.listen("messages.new", stop_event=self._stop_event): - if self._stop_event.is_set(): - break - module.on_event(event) - elif module.module_type == ModuleType.SERVICE: - module.execute() - - # If we get here without exception, module exited cleanly - break - - except Exception as e: - retries += 1 - self._logger.error( - f"Module '{module.name}' failed ({retries}/{max_retries}): {e}" - ) - if retries <= max_retries: - wait = backoff * (2 ** (retries - 1)) # exponential backoff - self._logger.info(f"Restarting '{module.name}' in {wait}s...") - self._stop_event.wait(wait) - else: - self._logger.error(f"Module '{module.name}' permanently failed.") - finally: - module._running = False - try: - module.on_stop() - except Exception: - pass -``` - -### config.yaml settings for retry - -```yaml -modules: - settings: - monitoring: - interval: 600 - max_retries: 3 - backoff_seconds: 30 - auto_reply: - max_retries: 3 - backoff_seconds: 10 -``` - ---- - -## 7. Configuration Shape - -### Guiding Principles - -- **Do not modify existing keys** — `postgres:`, `sleep:`, `files:`, `bot:` stay exactly as they are. -- **Add new sections only**: `modules:` and `prometheus:`. -- Each module gets its own sub-section under `modules.settings.`. -- Each module receives only its own `ModuleConfig` (not the full config). - -### Final `config.yaml` - -```yaml -postgres: - schema: "status_app_monitoring" - tables: - messages: "raw_messages" - community: "raw_community_info" -sleep: 10 -files: - current_state: "dates.pkl" - -bot: - public_key: "0x..." - compressed_key: "zQ3..." - params: - domain: "status-backend" - port: 8080 - is_secure: false - -# ===== NEW SECTIONS BELOW ===== - -modules: - directories: - - ./modules - enabled: - - monitoring - settings: - monitoring: - interval: 600 # seconds between runs - max_retries: 3 - backoff_seconds: 30 - auto_reply: - max_retries: 3 - backoff_seconds: 10 - commands: - help: "!help" - status: "!status" - -prometheus: - enabled: true - host: "0.0.0.0" - port: 8000 -``` - -### How `main.py` extracts module configs - -```python -modules_config = config.get("modules", {}) -enabled = modules_config.get("enabled", []) -settings = modules_config.get("settings", {}) - -for module_name in enabled: - module_settings = settings.get(module_name, {}) - module_config = ModuleConfig( - name=module_name, - interval=module_settings.get("interval", 60), - max_retries=module_settings.get("max_retries", 3), - backoff_seconds=module_settings.get("backoff_seconds", 30), - settings=module_settings, - ) -``` - ---- - -## 8. Prometheus Metrics - -### Endpoint - -A simple HTTP server on a configurable `host:port` (default `0.0.0.0:8000`) -serving `/metrics` in Prometheus text format using the `prometheus_client` library. - -### Metrics - -| Metric | Type | Labels | Description | -|--------|------|--------|-------------| -| `status_bot_health` | Gauge | — | 1 if running, 0 if stopped | -| `status_bot_version` | Gauge | `version` | Version info (constant 1 with label) | -| `status_bot_module_loaded` | Gauge | `module` | 1 for each loaded module | -| `status_bot_module_errors_total` | Counter | `module` | Total error count per module | -| `status_bot_module_restarts_total` | Counter | `module` | Total restart count per module | - -### Implementation in `main.py` - -```python -from prometheus_client import start_http_server, Gauge, Counter - -def start_prometheus_server(config: dict, manager: ModuleManager): - prom_config = config.get("prometheus", {}) - if not prom_config.get("enabled", False): - return - - health = Gauge("status_bot_health", "Bot health status") - version = Gauge("status_bot_version", "Bot version", ["version"]) - module_loaded = Gauge("status_bot_module_loaded", "Module loaded", ["module"]) - module_errors = Counter("status_bot_module_errors_total", "Module errors", ["module"]) - module_restarts = Counter("status_bot_module_restarts_total", "Module restarts", ["module"]) - - health.set(1) - version.labels(version="0.1.0").set(1) - for module_name in manager.modules: - module_loaded.labels(module=module_name).set(1) - - host = prom_config.get("host", "0.0.0.0") - port = prom_config.get("port", 8000) - start_http_server(port, host) -``` - -### Dependency - -Add `prometheus-client` to `requirements.txt`. - ---- - -## 9. Signal.listen() Graceful Shutdown - -### Problem - -The current `Signal.listen()` blocks forever on `queue.get()` with no timeout: - -```python -# Current (blocks forever): -while True: - data = self.__queue.get() # ← blocks indefinitely - yield data -``` - -This means an EVENT-type module thread can never be stopped cleanly — -`stop_event.set()` is never checked. - -### Solution - -Add an optional `stop_event: threading.Event` parameter to `listen()`. -Replace `queue.get()` with `queue.get(timeout=1)` in a loop that checks -the event: - -```python -# Modified: -def listen(self, signal_type: str, stop_event: Optional[threading.Event] = None): - self.__signal_type = signal_type - ws = websocket.WebSocketApp(...) - self.__thread = threading.Thread(target=ws.run_forever, daemon=True) - self.__thread.start() - - while True: - try: - data = self.__queue.get(timeout=1) # ← non-blocking with timeout - except queue.Empty: - if stop_event and stop_event.is_set(): - break - continue - except KeyboardInterrupt: - break - if self.__error_message: - raise Exception(self.__error_message) - yield data -``` - -### Changes to `bot/signal.py` - -- Add `import threading` (already imported) -- Change `listen(self, signal_type: str)` → `listen(self, signal_type: str, stop_event: Optional[threading.Event] = None)` -- Replace `self.__queue.get()` with the timeout loop above - ---- - -## 10. Implementation Steps - -### Step 1: Core Framework + Entry Point - -**Goal**: Create the module framework and a working `main.py` entry point. -No existing monitoring functionality is moved yet — `modules.enabled` would be -empty, and the bot starts with no modules (or a placeholder). - -**Files to create:** - -| File | Content | -|------|---------| -| `bot/modules/__init__.py` | Empty package marker | -| `bot/modules/base.py` | `ModuleType` Enum, `ModuleConfig` dataclass, `ModuleContext` dataclass, `BaseModule` ABC | -| `bot/modules/manager.py` | `ModuleManager` class with `discover_modules()`, `load_modules()`, `start_all()` (threaded), `stop_all()`, `_run_module_wrapper()` with retry/backoff | -| `main.py` | New entry point — config loading, bot creation, DB init, manager init, Prometheus server, signal handlers | - -**Files to modify:** - -| File | Change | -|------|--------| -| `bot/signal.py` | Add `stop_event` parameter to `listen()` (see [Section 9](#9-signallisten-graceful-shutdown)) | -| `config.yaml` | Add `modules:` and `prometheus:` sections | -| `Dockerfile` | Change `ENTRYPOINT ["python", "main.py"]` | -| `requirements.txt` | Add `prometheus-client` | - -**Files to delete:** - -| File | Reason | -|------|--------| -| `module-refactoring.md` | Superseded by this document | - -**Verification:** - -```bash -# Without any modules enabled, the bot should: -# 1. Load config -# 2. Log in -# 3. Start Prometheus endpoint -# 4. Wait for SIGTERM -python main.py -``` - ---- - -### Step 2: Refactor monitor.py → MonitoringModule - -**Goal**: Extract the community monitoring logic from `monitor.py` into a -proper `MonitoringModule` under `./modules/`. Delete the old `monitor.py`. - -**Files to create:** - -| File | Content | -|------|---------| -| `modules/__init__.py` | Empty package marker | -| `modules/monitoring.py` | `MonitoringModule(BaseModule)` — PERIODIC type, calls `execute()` which runs `_download()` + `_store()` | -| `bot/modules/utils.py` | Shared helpers extracted from `monitor.py`: `to_sha256_hash()`, `to_midnight()`, `save_file()`, `extract_community_channels()` | - -**What goes where in `modules/monitoring.py`:** - -| Original function in `monitor.py` | New home | -|-----------------------------------|----------| -| `load_config()` | Stays in `main.py` | -| `create_bot()` | Stays in `main.py` | -| `to_sha256_hash()` | `bot/modules/utils.py` | -| `to_midnight()` | `bot/modules/utils.py` | -| `save_file()` | `bot/modules/utils.py` | -| `extract_community_channels()` | `bot/modules/utils.py` or private to MonitoringModule | -| `download()` | `MonitoringModule._download()` | -| `store()` | `MonitoringModule._store()` | -| `if __name__ == "__main__":` block | Deleted (replaced by `main.py`) | - -**Behavior**: The module receives `Postgres` via `ModuleContext.db`. It reads -`monitoring`-specific settings from `ModuleContext.config.settings`. - -**Verification:** - -```bash -# With monitoring enabled in config.yaml: -# The bot should download community data and store to Postgres. -python main.py -``` - ---- - -### Step 3: Connection Pool + Retry Logic - -**Goal**: Make `Postgres` thread-safe with a connection pool, and verify the -retry/backoff logic in `ModuleManager` works. - -**Files to modify:** - -| File | Change | -|------|--------| -| `postgres.py` | Refactor to use `sqlalchemy.create_engine` with connection pooling (pool_size=5, max_overflow=10). Ensure `pandas.to_sql()` reuses the same engine. | -| `bot/modules/manager.py` | Retry/backoff logic should already be implemented in Step 1. Verify and test edge cases. | - -**Connection pool approach:** - -```python -class Postgres: - def __init__(self, ...): - self._engine = create_engine( - self.__url, - pool_size=5, - max_overflow=10, - pool_pre_ping=True, # verify connections before use - ) - - def insert(self, data, table_name, schema, json_columns=None): - # Use self._engine.begin() for transaction management - ... - - def to_pandas(self, query): - return pd.read_sql(query, self._engine) -``` - -**Verification:** - -```bash -# Run with multiple modules enabled; verify concurrent DB access works. -python main.py -``` - ---- - -### Step 4: Example Event-Driven Module - -**Goal**: Create an `AutoReplyModule` to demonstrate the event-driven pattern -for future module developers. - -**Files to create:** - -| File | Content | -|------|---------| -| `modules/auto_reply.py` | `AutoReplyModule(BaseModule)` — EVENT type, listens for `messages.new`, checks for `!help` / `!status` commands, replies with configured responses | - -**`modules/auto_reply.py` structure:** - -```python -class AutoReplyModule(BaseModule): - - @property - def module_type(self) -> ModuleType: - return ModuleType.EVENT - - def on_start(self): - self._commands = self._ctx.config.settings.get("commands", {}) - self._logger.info(f"Auto-reply loaded with commands: {list(self._commands.keys())}") - - def on_event(self, event: dict): - messages = event.get("event", {}).get("messages", []) - for msg in messages: - text = msg.get("text", "") - for cmd_name, cmd_text in self._commands.items(): - if text.strip().lower() == cmd_text.lower(): - response = self._build_response(cmd_name, msg) - self._account.send_message(msg["chatId"], response) - - def _build_response(self, command_name, message): - # Module-specific reply logic - ... -``` - -**Verification:** - -```bash -# Enable auto_reply in config.yaml, send a message with "!help" to the bot. -``` - ---- - -### Step 5: Tests + Documentation - -**Goal**: Ensure the framework is reliable and documented. - -**Files to create:** - -| File | Content | -|------|---------| -| `tests/test_base.py` | Test `ModuleConfig`, `ModuleContext`, `BaseModule` subclass contract | -| `tests/test_manager.py` | Mock `Account` and test discovery, loading, thread start/stop | -| `tests/test_monitoring_module.py` | Test `MonitoringModule` with mocked account and DB | - -**Files to modify:** - -| File | Change | -|------|--------| -| `README.md` | Document module API, how to write a module, directory structure | - -**Test approach:** - -```python -# test_base.py -def test_module_config_defaults(): - cfg = ModuleConfig(name="test") - assert cfg.enabled == True - assert cfg.interval == 60 - assert cfg.max_retries == 3 - -# test_manager.py -def test_discover_modules(tmp_path): - # Create a mock module file - module_file = tmp_path / "test_mod.py" - module_file.write_text(""" -from bot.modules.base import BaseModule, ModuleType -class TestModule(BaseModule): - @property - def module_type(self): return ModuleType.PERIODIC - def execute(self): pass -""") - manager = ModuleManager({"modules": {"directories": [str(tmp_path)], "enabled": []}}, mock_account, None, logger) - manager.discover_modules() - assert "test_mod" in manager._module_classes -``` - ---- - -## 11. File-by-File Summary - -### New Files (8) - -| # | Path | Step | -|---|------|------| -| 1 | `bot/modules/__init__.py` | 1 | -| 2 | `bot/modules/base.py` | 1 | -| 3 | `bot/modules/manager.py` | 1 | -| 4 | `main.py` | 1 | -| 5 | `modules/__init__.py` | 2 | -| 6 | `bot/modules/utils.py` | 2 | -| 7 | `modules/monitoring.py` | 2 | -| 8 | `modules/auto_reply.py` | 4 | -| 9 | `tests/test_base.py` | 5 | -| 10 | `tests/test_manager.py` | 5 | -| 11 | `tests/test_monitoring_module.py` | 5 | - -### Modified Files (5) - -| # | Path | Change | -|---|------|--------| -| 1 | `bot/signal.py` | Step 1: `listen()` accepts `stop_event` | -| 2 | `config.yaml` | Step 1: add `modules:` + `prometheus:` sections | -| 3 | `Dockerfile` | Step 1: change `ENTRYPOINT` to `main.py` | -| 4 | `requirements.txt` | Step 1: add `prometheus-client` | -| 5 | `README.md` | Step 5: update documentation | - -### Deleted Files (2) - -| # | Path | Reason | -|---|------|--------| -| 1 | `module-refactoring.md` | Superseded by this document | -| 2 | `monitor.py` | Step 2: logic moved to `modules/monitoring.py` | - ---- - -## Appendix: Key References - -- **`bot/account.py`** — ~1112 lines, wraps Status Backend HTTP/RPC API -- **`bot/signal.py`** — WebSocket handler; `listen()` will be modified for `stop_event` -- **`bot/logger.py`** — Singleton logger, 24 lines -- **`postgres.py`** — Postgres connector, 149 lines; needs connection pool refactor -- **`monitor.py`** — 316 lines, to be deleted after Step 2 diff --git a/docs/usage/configuration.md b/docs/usage/configuration.md index cf52efb..1b81808 100644 --- a/docs/usage/configuration.md +++ b/docs/usage/configuration.md @@ -64,9 +64,9 @@ bot: compressed_key: 'zQ3...' ``` -#### `bot.params` +### `backend` -Parameters passed to the `Account()` constructor. +Parameter to connect to the Status Backend instance. | Field | Type | Default | Env var | Description | |-------|------|---------|---------|-------------| @@ -74,18 +74,34 @@ Parameters passed to the `Account()` constructor. | `port` | `int` | `8080` | `BOT_PARAMS_PORT` | Status Backend API port | | `is_secure` | `bool` | `false` | `BOT_PARAMS_IS_SECURE` | Use HTTPS instead of HTTP | ```yaml -bot: - params: - domain: "status-backend" - port: 8080 - is_secure: false - infura_token: "" - coingecko_api_key: "" +backend: + domain: "status-backend" + port: 8080 + is_secure: false ``` --- -### `postgres` +### `api` + +Configuration for the WebServer avaialable to the modules. + +| Field | Type | Default | Env var | Description | +|-------|------|---------|---------|-------------| +| `enable` | `bool` | `true` | `API_ENABLE` | Enable the REST API server | +| `host` | `str` | `"0.0.0.0"` | `API_HOST` | API server bind address | +| `port` | `int` | `8081` | `API_PORT` | API server port | + +```yaml +api: + enable: true + host: "0.0.0.0" + port: 8081 +``` + +--- + +### `database` | Field | Type | Default | Env var | Description | |-------|------|---------|---------|-------------| @@ -98,7 +114,7 @@ bot: | `tables` | `dict` | `{}` | — | Mapping of data type to table name | ```yaml -postgres: +database: host: database port: 5432 user: 'myuser' @@ -120,75 +136,31 @@ postgres: | `enabled` | `list[str]` | `[]` | List of module names to enable | | `settings` | `dict` | `{}` | Per-module settings (each module defines its own schema) | +The `api_server` module is auto-loaded whenever `api.enable` is `true` — it does not need to be listed in `enabled`. + ```yaml modules: directories: - ./modules enabled: - - monitoring + - messaging settings: - monitoring: - interval: 600 - max_retries: 3 - backoff_seconds: 30 + messaging: {} ``` --- -### `prometheus` +### `metrics` | Field | Type | Default | Env var | Description | |-------|------|---------|---------|-------------| -| `enabled` | `bool` | `false` | `PROMETHEUS_ENABLED` | Enable Prometheus metrics HTTP server | -| `host` | `str` | `"0.0.0.0"` | `PROMETHEUS_HOST` | Prometheus HTTP server bind address | -| `port` | `int` | `8000` | `PROMETHEUS_PORT` | Prometheus HTTP server port | +| `enabled` | `bool` | `false` | `METRICS_ENABLED` | Enable Prometheus metrics HTTP server | +| `host` | `str` | `"0.0.0.0"` | `METRICS_HOST` | Prometheus HTTP server bind address | +| `port` | `int` | `8000` | `METRICS_PORT` | Prometheus HTTP server port | ```yaml -prometheus: +metrics: enabled: true host: "0.0.0.0" port: 8000 ``` - ---- - -## Environment variables - -All configuration fields can be set via environment variables using the `_` separator for nested fields. - -| Env var | Config path | -|---------|-------------| -| `BOT_DISPLAY_NAME` | `bot.display_name` | -| `BOT_PASSWORD` | `bot.password` | -| `BOT_MNEMONIC_PHRASE` | `bot.mnemonic_phrase` | -| `BOT_INIT_ACCOUNT` | `bot.init_account` | -| `BOT_PARAMS_DOMAIN` | `bot.params.domain` | -| `BOT_PARAMS_PORT` | `bot.params.port` | -| `BOT_PARAMS_IS_SECURE` | `bot.params.is_secure` | -| `BOT_PARAMS_INFURA_TOKEN` | `bot.params.infura_token` | -| `BOT_PARAMS_COINGECKO_API_KEY` | `bot.params.coingecko_api_key` | -| `POSTGRES_HOST` | `postgres.host` | -| `POSTGRES_PORT` | `postgres.port` | -| `POSTGRES_USER` | `postgres.user` | -| `POSTGRES_PASSWORD` | `postgres.password` | -| `POSTGRES_NAME` | `postgres.name` | -| `POSTGRES_SCHEMA` | `postgres.schema` | -| `PROMETHEUS_ENABLED` | `prometheus.enabled` | -| `PROMETHEUS_HOST` | `prometheus.host` | -| `PROMETHEUS_PORT` | `prometheus.port` | - -Example `.env` file: - -``` -# Bot account -BOT_DISPLAY_NAME=my-bot -BOT_PASSWORD=ChangeThisPassword -BOT_MNEMONIC_PHRASE=test test test test test test test test test test test test - -# Postgres -POSTGRES_HOST=database -POSTGRES_PORT=5432 -POSTGRES_NAME=status-bot -POSTGRES_USER=status -POSTGRES_PASSWORD=ChangeThisOneAlso -``` diff --git a/docs/usage/messaging.md b/docs/usage/messaging.md new file mode 100644 index 0000000..fb6cb0c --- /dev/null +++ b/docs/usage/messaging.md @@ -0,0 +1,162 @@ +# Messaging API + +The bot exposes a REST endpoint for sending messages, managing contacts, and querying chats. + +## Interactive docs + +| Tool | URL | +|------|-----| +| Swagger UI | `http://localhost:8081/docs` | +| ReDoc | `http://localhost:8081/redoc` | +| OpenAPI JSON | `http://localhost:8081/openapi.json` | + +No authentication is required. + +--- + +## Endpoints + +### `GET /health` + +Health check. + +```bash +curl http://localhost:8081/health +``` + +Response `200`: +```json +{"status": "healthy"} +``` + +--- + +### `POST /api/v1/contacts` + +Add a contact (send friend request). + +```bash +curl -X POST http://localhost:8081/api/v1/contacts \ + -H "Content-Type: application/json" \ + -d '{"public_key": "0x...", "display_name": "Alice"}' +``` + +Request body: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `public_key` | `string` | Yes | Contact's public key | +| `display_name` | `string` | No | Display name (required if not already a contact) | + +Response `201`: +```json +{"status": "ok"} +``` + +Error `400` — missing or invalid fields. + +--- + +### `DELETE /api/v1/contacts/{public_key}` + +Remove a contact. + +```bash +curl -X DELETE http://localhost:8081/api/v1/contacts/0x... +``` + +Response `200`: +```json +{"status": "ok"} +``` + +Error `404` — contact not found or already removed. + +--- + +### `GET /api/v1/chats` + +List all available chats (contacts, community channels, and group chats). + +```bash +curl http://localhost:8081/api/v1/chats +``` + +Response `200`: +```json +[ + {"type": "contact", "id": "0x...", "name": "Alice"}, + {"type": "channel", "id": "0x...", "name": "Community #general"}, + {"type": "group_chat", "id": "0x...", "name": "Group Chat Name"} +] +``` + +Each chat has: +| Field | Type | Description | +|-------|------|-------------| +| `type` | `string` | `contact`, `channel`, or `group_chat` | +| `id` | `string` | Chat ID (used in message endpoints) | +| `name` | `string` | Human-readable name | + +--- + +### `GET /api/v1/chats/{chat_id}/messages` + +Get messages from a chat. Messages are returned newest-first. + +```bash +curl "http://localhost:8081/api/v1/chats/0x.../messages" +``` + +Optional query parameters: + +| Parameter | Type | Description | +|-----------|------|-------------| +| `start_timestamp` | ISO 8601 | Only messages after this timestamp | +| `end_timestamp` | ISO 8601 | Only messages before this timestamp | + +```bash +curl "http://localhost:8081/api/v1/chats/0x.../messages?start_timestamp=2025-01-01T00:00:00&end_timestamp=2025-06-01T00:00:00" +``` + +Response `200` — array of message objects. + +--- + +### `POST /api/v1/chats/{chat_id}/messages` + +Send a text message to a chat. + +```bash +curl -X POST http://localhost:8081/api/v1/chats/0x.../messages \ + -H "Content-Type: application/json" \ + -d '{"text": "Hello from the bot!"}' +``` + +Request body: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `text` | `string` | Yes | Message content | + +Response `201`: +```json +{"status": "ok"} +``` + +--- + +## Error handling + +All errors return a JSON body with a `detail` field: + +```json +{"detail": "Contact not found or already removed"} +``` + +| Status | Meaning | +|--------|---------| +| `400` | Bad request (invalid input, missing fields, wrong format) | +| `404` | Resource not found (contact, chat) | +| `422` | Validation error (malformed request body) | +| `500` | Internal server error | diff --git a/docs/usage/metrics.md b/docs/usage/metrics.md new file mode 100644 index 0000000..5904fbb --- /dev/null +++ b/docs/usage/metrics.md @@ -0,0 +1,66 @@ +# Metrics + +The bot exposes Prometheus metrics for health monitoring and observability. + +--- + +## Endpoint + +``` +http://:/metrics +``` + +Default: `http://0.0.0.0:8000/metrics` + +Configured via the `metrics` section in `config.yaml` (see [Configuration](./configuration.md#metrics)). + +--- + +## Available metrics + +### `status_bot_health` + +| Type | Labels | Description | +|------|--------|-------------| +| Gauge | — | `1` if the bot is running, `0` if stopped | + +### `status_bot_version` + +| Type | Labels | Description | +|------|--------|-------------| +| Gauge | `version` | Constant `1` with the bot version as a label value | + +### `status_bot_module_loaded` + +| Type | Labels | Description | +|------|--------|-------------| +| Gauge | `module` | `1` for each loaded module | + +Example: +``` +status_bot_module_loaded{module="messaging"} 1 +status_bot_module_loaded{module="api_server"} 1 +``` + +### `status_bot_module_errors_total` + +| Type | Labels | Description | +|------|--------|-------------| +| Counter | `module` | Total number of errors encountered by a module | + +### `status_bot_module_restarts_total` + +| Type | Labels | Description | +|------|--------|-------------| +| Counter | `module` | Total number of times a module has been restarted | + +--- + +## Example Prometheus scrape config + +```yaml +scrape_configs: + - job_name: "status-bot" + static_configs: + - targets: ["localhost:8000"] +``` diff --git a/main.py b/main.py index 9b145d7..3777478 100644 --- a/main.py +++ b/main.py @@ -5,12 +5,9 @@ from fastapi import FastAPI -from bot import Account, Logger -import bot -from bot.config import Config +from bot import Account, Logger, Config, Postgres from bot.metrics import start_prometheus from bot.modules.manager import ModuleManager -from postgres import Postgres def create_bot(config: Config, project_root: str) -> Account: @@ -66,11 +63,11 @@ def create_bot(config: Config, project_root: str) -> Account: def init_postgres(config: Config) -> Postgres: return Postgres( - host=config.postgres.host, - port=config.postgres.port, - user=config.postgres.user, - password=config.postgres.password, - database=config.postgres.name, + host=config.database.host, + port=config.database.port, + user=config.database.user, + password=config.database.password, + database=config.database.name, ) @@ -110,10 +107,10 @@ def main(): db = None has_postgres = all([ - config.postgres.host, - config.postgres.user, - config.postgres.password, - config.postgres.name, + config.database.host, + config.database.user, + config.database.password, + config.database.name, ]) if has_postgres: @@ -136,7 +133,7 @@ def main(): manager.discover_modules() manager.load_modules() - start_prometheus(config.prometheus, manager, logger) + start_prometheus(config.metrics, manager, logger) stop_event = manager._stop_event diff --git a/module-refactoring.md b/module-refactoring.md deleted file mode 100644 index 07016cf..0000000 --- a/module-refactoring.md +++ /dev/null @@ -1,591 +0,0 @@ -# Refactoring Plan: Status Bot Modular Architecture - -## Overview - -Transform `monitor.py` from a monolithic script into a modular plugin-based system where modules are independent, configurable, and can run in periodic or event-driven mode. - -## Architecture Design - -``` -┌─────────────────────────────────────────────────────────────┐ -│ Main Bot │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────┐ │ -│ │ Config │ │ Account │ │ Module Manager │ │ -│ │ Loader │ │ (shared) │ │ (loads/runs) │ │ -│ └──────────────┘ └──────────────┘ └──────────────────┘ │ -└─────────────────────────────────────────────────────────────┘ - │ │ │ - ▼ ▼ ▼ -┌─────────────────────────────────────────────────────────────┐ -│ Modules (plugins) │ -│ ┌─────────────┐ ┌─────────────┐ ┌───────────────┐ │ -│ │ Monitoring │ │ Storage │ │ Custom │ ... │ -│ │ (periodic) │ │ (periodic) │ │(event-driven) │ │ -│ └─────────────┘ └─────────────┘ └───────────────┘ │ -└─────────────────────────────────────────────────────────────┘ -``` - -## Key Components - -### 1. Base Module Interface (`bot/modules/base.py`) - -- Abstract base class defining module contract -- Required methods: `run()`, `get_name()`, `get_config()` -- Optional: `on_start()`, `on_stop()`, `on_event()` - -### 2. Module Manager (`bot/modules/manager.py`) - -- Discovers modules from configured directory -- Loads/enables/disables based on config -- Handles lifecycle (init → start → run → stop) -- Isolates errors per module - -### 3. Configuration (`config.yaml`) - -```yaml -modules: - enabled: - - monitoring - - storage - directories: - - ./modules - settings: - monitoring: - interval: 600 # seconds - storage: - batch_size: 100 -``` - -### 4. Module Structure (`modules/`) - -- Each module is a Python file with a Module class -- Auto-discovery via naming convention or decorator - -## Module Types Support - -| Module Type | Execution | Example | -|-------------|-----------|---------| -| `periodic` | Runs on interval | Monitoring, Storage | -| `event` | Reacts to signals | Mentions, Analytics | -| `service` | Long-running | WebSocket listeners | - -## Implementation Steps - -### Phase 1: Core Framework - -#### 1. Directory Structure - -``` -bot/ -├── __init__.py -├── account.py -├── signal.py -├── logger.py -└── modules/ - ├── __init__.py - ├── base.py # BaseModule abstract class - └── manager.py # ModuleManager class -modules/ # Custom modules directory (user-created) - ├── __init__.py - └── example.py -``` - -#### 2. BaseModule Abstract Class (`bot/modules/base.py`) - -```python -from abc import ABC, abstractmethod -from typing import Any, Optional -from dataclasses import dataclass -from enum import Enum -import logging - - -class ModuleType(Enum): - """Defines how the module is executed.""" - PERIODIC = "periodic" # Runs on a fixed interval - EVENT = "event" # Reacts to signals/events - SERVICE = "service" # Long-running service - - -@dataclass -class ModuleConfig: - """Configuration for a single module.""" - name: str - enabled: bool = True - interval: int = 60 # seconds, for periodic modules - settings: dict = None # module-specific settings - - def __post_init__(self): - if self.settings is None: - self.settings = {} - - -class BaseModule(ABC): - """ - Abstract base class for all bot modules. - - Subclass this to create custom modules. Implement the required methods. - """ - - def __init__( - self, - config: ModuleConfig, - account: "Account", - logger: logging.Logger - ): - """ - Initialize the module. - - Args: - config: Module configuration from config.yaml - account: Shared Status account instance - logger: Shared logger instance - """ - self._config = config - self._account = account - self._logger = logger - self._running = False - - @property - def name(self) -> str: - """Module name, used for identification.""" - return self._config.name - - @property - def module_type(self) -> ModuleType: - """Return the type of module. Override in subclass.""" - return ModuleType.PERIODIC - - @property - def interval(self) -> int: - """Interval in seconds for periodic modules.""" - return self._config.interval - - @abstractmethod - def run(self) -> Any: - """ - Execute the module logic. - - This is called either: - - Periodically (for PERIODIC type) - - On event (for EVENT type) - - Once and keeps running (for SERVICE type) - - Returns: - Any result from the module execution - """ - pass - - def on_start(self) -> None: - """ - Called once when the module starts. - Override for initialization logic. - """ - pass - - def on_stop(self) -> None: - """ - Called once when the module stops. - Override for cleanup logic. - """ - pass - - def on_event(self, event: dict) -> Any: - """ - Handle an event signal (for EVENT type modules). - - Override to handle specific signals like 'messages.new'. - - Args: - event: The event data from Signal - - Returns: - Any result from handling the event - """ - return None - - def is_running(self) -> bool: - """Check if module is currently running.""" - return self._running - - def _set_running(self, value: bool) -> None: - """Internal: update running state.""" - self._running = value -``` - -#### 3. Module Manager (`bot/modules/manager.py`) - -```python -import os -import importlib.util -import logging -from typing import Type, Optional, List, Dict, Any -from pathlib import Path - -from .base import BaseModule, ModuleConfig, ModuleType -from bot import Account - - -class ModuleManager: - """ - Manages module discovery, loading, and lifecycle. - - Handles: - - Discovering modules from configured directories - - Loading and initializing modules - - Running modules according to their type - - Error isolation between modules - """ - - def __init__( - self, - config: dict, - account: Account, - logger: logging.Logger - ): - """ - Initialize the module manager. - - Args: - config: The full bot configuration dict - account: Shared Account instance - logger: Shared logger instance - """ - self._config = config - self._account = account - self._logger = logger - self._modules: Dict[str, BaseModule] = {} - self._module_classes: Dict[str, Type[BaseModule]] = {} - - @property - def modules(self) -> Dict[str, BaseModule]: - """Get all loaded modules.""" - return self._modules - - def discover_modules(self) -> None: - """Discover available module classes from configured directories.""" - modules_config = self._config.get("modules", {}) - directories = modules_config.get("directories", ["modules"]) - enabled = modules_config.get("enabled", []) - - for directory in directories: - self._discover_from_directory(directory) - - self._logger.info( - f"Discovered {len(self._module_classes)} module classes: " - f"{list(self._module_classes.keys())}" - ) - - def _discover_from_directory(self, directory: str) -> None: - """Discover modules from a specific directory.""" - base_path = Path(directory) - if not base_path.exists(): - self._logger.warning(f"Module directory not found: {directory}") - return - - for file_path in base_path.glob("*.py"): - if file_path.name.startswith("_"): - continue - self._load_module_from_file(file_path) - - def _load_module_from_file(self, file_path: Path) -> None: - """Load a module class from a Python file.""" - module_name = file_path.stem - - spec = importlib.util.spec_from_file_location( - f"modules.{module_name}", file_path - ) - if spec is None or spec.loader is None: - return - - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - for attr_name in dir(module): - attr = getattr(module, attr_name) - if ( - isinstance(attr, type) - and issubclass(attr, BaseModule) - and attr is not BaseModule - ): - self._module_classes[module_name] = attr - self._logger.debug(f"Found module class: {module_name}.{attr_name}") - - def load_modules(self) -> None: - """Load and initialize all enabled modules.""" - modules_config = self._config.get("modules", {}) - enabled = set(modules_config.get("enabled", [])) - settings = modules_config.get("settings", {}) - - for module_name in enabled: - if module_name not in self._module_classes: - self._logger.error( - f"Module '{module_name}' not found. " - f"Available: {list(self._module_classes.keys())}" - ) - continue - - module_class = self._module_classes[module_name] - module_settings = settings.get(module_name, {}) - - module_config = ModuleConfig( - name=module_name, - enabled=True, - interval=module_settings.get("interval", 60), - settings=module_settings - ) - - try: - module = module_class(module_config, self._account, self._logger) - self._modules[module_name] = module - self._logger.info(f"Loaded module: {module_name}") - except Exception as e: - self._logger.error(f"Failed to load module '{module_name}': {e}") - - def start_modules(self) -> None: - """Call on_start for all modules.""" - for module in self._modules.values(): - try: - module.on_start() - except Exception as e: - self._logger.error( - f"Error starting module '{module.name}': {e}" - ) - - def run_modules(self) -> None: - """ - Run all modules according to their type. - - - PERIODIC: runs in a loop with configured interval - - EVENT: registers with signal listener - - SERVICE: runs once and keeps running - """ - for module in self._modules.values(): - try: - if module.module_type == ModuleType.PERIODIC: - self._run_periodic_module(module) - elif module.module_type == ModuleType.EVENT: - self._run_event_module(module) - elif module.module_type == ModuleType.SERVICE: - self._run_service_module(module) - except Exception as e: - self._logger.error( - f"Error running module '{module.name}': {e}" - ) - - def _run_periodic_module(self, module: BaseModule) -> None: - """Run a periodic module on its interval.""" - import time - while True: - try: - module.run() - except Exception as e: - self._logger.error( - f"Error in periodic module '{module.name}': {e}" - ) - time.sleep(module.interval) - - def _run_event_module(self, module: BaseModule) -> None: - """Run an event-driven module by listening to signals.""" - for event in self._account.signal.listen("messages.new"): - try: - module.on_event(event) - except Exception as e: - self._logger.error( - f"Error handling event in module '{module.name}': {e}" - ) - - def _run_service_module(self, module: BaseModule) -> None: - """Run a service module once (blocking).""" - module.run() - - def stop_modules(self) -> None: - """Call on_stop for all modules.""" - for module in self._modules.values(): - try: - module.on_stop() - except Exception as e: - self._logger.error( - f"Error stopping module '{module.name}': {e}" - ) -``` - -#### 4. Configuration Example (`config.yaml`) - -```yaml -postgres: - schema: "status_app_monitoring" - tables: - messages: "raw_messages" - community: "raw_community_info" - -sleep: 10 - -files: - current_state: "dates.pkl" - -bot: - public_key: "0x041658626a9e1303b631f6d0fb1e047211d5603b977454f7d5d29fe583c3d6c1bd3d8e395d67f6c44b5bc659aae912040e9dd8164b5107368a29029cb53389d8b0" - compressed_key: "zQ3shNv1tnajHo5FvCvP662cWcbBfS5ZejB4TWaH9iAuFCZZe" - params: - domain: "status-backend" - port: 8080 - is_secure: false - -modules: - directories: - - ./modules - enabled: - - monitoring - - storage - settings: - monitoring: - interval: 600 - storage: - batch_size: 100 -``` - -#### 5. Example Custom Module (`modules/example.py`) - -```python -from bot.modules.base import BaseModule, ModuleConfig, ModuleType -from typing import Any - - -class ExampleModule(BaseModule): - """ - Example module that demonstrates the module interface. - - This module logs a message every interval seconds. - """ - - @property - def module_type(self) -> ModuleType: - return ModuleType.PERIODIC - - def run(self) -> Any: - """Run the module logic.""" - self._logger.info(f"Example module running for {self._account.info['display_name']}") - # Custom logic here: - # - Process messages - # - Send notifications - # - Store data - # - etc. - - def on_start(self) -> None: - """Called when module starts.""" - self._logger.info(f"Starting example module with interval {self.interval}s") - - def on_stop(self) -> None: - """Called when module stops.""" - self._logger.info("Stopping example module") -``` - -#### 6. Updated Entry Point (`monitor.py`) - -```python -import os, yaml, time -from dotenv import load_dotenv - -from bot import Account, Logger -from bot.modules.manager import ModuleManager - - -def load_config(file_path: str) -> dict: - """Load the config file and the `.env` variables.""" - with open(file_path, "r") as f: - config = yaml.safe_load(f) - - env_file_path = os.path.join(os.path.dirname(file_path), ".env") - load_dotenv(env_file_path) - - config["env_vars"] = { - key: value - for key, value in os.environ.items() - if key.startswith(("POSTGRES_", "STATUS_")) - } - - return config - - -def create_bot(config: dict) -> Account: - """Initialize a logged in bot account.""" - params = config.get("bot", {}).get("params", {}) - account = Account(**params) - - available_accounts = [acc["display_name"] for acc in account.available_accounts] - - prefix = "STATUS_" - params = { - key.replace(prefix, "").lower(): value - for key, value in config["env_vars"].items() - if key.startswith(prefix) - } - if params["display_name"] in available_accounts: - params.pop("mnemonic") - - account.login(**params) - if account.info["compressed_key"] != config["bot"]["compressed_key"]: - raise Exception("Target compressed key and logged in compressed key are different") - - account.profile_picture = os.path.join(os.path.dirname(__file__), "assets", "profile.jpg") - account.logger.info( - f"Account Information:\n" - f"Compressed Key: {account.info['compressed_key']}\n" - f"Public Key: {account.info['public_key']}\n" - f"URL: {account.info['url']}" - ) - return account - - -if __name__ == "__main__": - folder = os.path.dirname(__file__) - config = load_config(os.path.join(folder, "config.yaml")) - logger = Logger() - account = create_bot(config) - - # Initialize module manager - manager = ModuleManager(config, account, logger) - manager.discover_modules() - manager.load_modules() - manager.start_modules() - - try: - manager.run_modules() - except KeyboardInterrupt: - logger.info("Shutting down...") - manager.stop_modules() -``` - -### Phase 2: Main Refactor - -1. Rewrite `monitor.py` as entry point -2. Move `download()` → `MonitoringModule` -3. Move `store()` → `StorageModule` - -### Phase 3: Module System - -1. Implement module discovery from directory -2. Add event-driven support via Signal class -3. Add error isolation (try/except per module) - -### Phase 4: Testing & Docs - -1. Create example custom module -2. Add module API documentation - -## Backward Compatibility - -- Keep `config.yaml` structure compatible (add `modules` section) -- Current functionality preserved as built-in modules -- Existing `.env` works unchanged - -## Requirements from User - -1. **Module flexibility**: Anyone should be able to create their own module (store messages, react to mentions, analytics, automated messages) -2. **Configuration**: Modules configured in config.yaml but loaded from directory -3. **Execution**: Independent modules (not sequential pipeline) -4. **Execution pattern**: Both periodic and event-driven support -5. **Interface**: BaseModule class with required methods -6. **Error handling**: Isolated (one module failure doesn't crash others) -7. **Shared resources**: Passed through constructor (Account, config, logger) diff --git a/monitor.py b/monitor.py deleted file mode 100644 index 3cc105d..0000000 --- a/monitor.py +++ /dev/null @@ -1,325 +0,0 @@ -import datetime, os, pickle, yaml, time -import pandas as pd -from typing import Any -from pathlib import Path -from dotenv import load_dotenv -from hashlib import sha256 -# Manual file imports -from bot import Account, Logger -from postgres import Postgres - -def to_sha256_hash(value: str) -> str: - """ - Hash personal information before it is put in the database. - - Parameters: - - `value` - personal information - - Output: - - sha256 hashed value - """ - return sha256(value.encode()).hexdigest() - -def to_midnight(timestamp: datetime.datetime) -> datetime.datetime: - """ - Convert the given timestamp to midnight - - Parameters: - - `timestamp` - current timestap - - Output: - - `timestamp` at midnight - """ - return timestamp.replace(minute=0, second=0, hour=0, microsecond=0) - -def load_config(file_path: str) -> dict: - """ - Load the config file and the `.env` variables - - Parameter: - - `file_path` - the file path of the config yaml file. The `.env` variable must be in the same folder - - Output: - - The config variables and secret from `.env` - """ - with open(file_path, "r") as f: - config: dict = yaml.safe_load(f) - - env_file_path = os.path.join(os.path.dirname(file_path), ".env") - load_dotenv(env_file_path) - - config["env_vars"] = { - key: value - for key, value in os.environ.items() - if key.startswith(("POSTGRES_", "STATUS_")) - } - - return config - -def extract_community_channels(account: Account, community: dict, latest_dates: dict[str, pd.Timestamp]) -> pd.DataFrame: - """ - Extract the community channel messages. - - Parameters: - - `account` - logged in Status Bot account - - `community` - the current community from `account` - - `start_timestamp` - start timestamp for message fetching - - `end_timestamp` - end timestamp for message fetching - - Output: - - DataFrame with all of the community messages for the given start and end timestamps - """ - # Column name -> True if data should be hashed - bridge_key = "bridge_message" - columns = { - "id": True, - "whisper_timestamp": False, - "from": True, - "seen": False, - "chat_id": False, - "community_id": False, - "message_type": False, - "response_to": True, - "timestamp": False, - "deleted": False, - "extracted_timestamp": False, - } - - final = [] - for channel in community["channels"]: - - now = datetime.datetime.now() - start_timestamp = latest_dates.get(channel["chat_id"]) - if start_timestamp: - start_timestamp += datetime.timedelta(seconds=1) - else: - # Node will only return known / fetched messages for this channel. - # Without enabling community archives feature the node can only fetch last 30 days (from store nodes). - start_timestamp = to_midnight(now - datetime.timedelta(days=30)) - - account.logger.info(f"Starting message extraction for # {channel['name']} [{start_timestamp} - {now}]") - messages = account.get_messages(channel["chat_id"], start_timestamp, now) - messages = pd.DataFrame(messages) - if len(messages) == 0: - account.logger.info(f"No messages found") - continue - - account.logger.info(f"Extracted {len(messages)} message(s)") - messages = messages.assign( - community_id = community["id"], - extracted_timestamp = now - ) - final.append(messages) - - extracted_data = pd.concat(final, ignore_index=True) if final else pd.DataFrame() - if len(extracted_data) == 0: - return extracted_data - - existing_columns = extracted_data.columns.to_list() - for column, should_hash in columns.items(): - if column not in existing_columns: - loc = len(extracted_data.columns.to_list()) - extracted_data.insert(loc, column, None) - continue - - if should_hash: - extracted_data[column] = extracted_data[column].astype(str).apply(to_sha256_hash) - - if bridge_key in extracted_data.columns: - extracted_data["source"] = extracted_data[bridge_key].apply(lambda value: value["bridgeName"] if not pd.isna(value) else "status") - else: - extracted_data["source"] = "status" - - extracted_data = extracted_data[list(columns.keys()) + ["source"]].assign( - deleted = extracted_data["deleted"].fillna(False), - seen = extracted_data["seen"].fillna(False) - ) - account.logger.info(f"Sensitive data has been hashed") - - return extracted_data - -def save_file(file_path: str, data: Any): - """ - Save data to a pickle file. Creates directories if they don't exist. - - Parameters: - - `file_path` - Full pikle path - - `data` - Python object to be saved - """ - folder = os.path.dirname(file_path) - if len(folder) > 0: - os.makedirs(folder, exist_ok=True) - - if isinstance(data, pd.DataFrame): - data.to_csv(file_path, index=False) - return - - with open(file_path, "wb") as f: - pickle.dump(data, f) - -def create_bot(config: dict) -> Account: - """ - Initialized a logged in bot account that will monitor the communities. - - Parameters: - - `config` - the `load_config` configuration - - Output: - - Logged in Bot account - """ - params = config.get("bot", {}).get("params", {}) - account = Account(**params) - available_accounts = [acc["display_name"] for acc in account.available_accounts] - - prefix = "STATUS_" - params = { - key.replace(prefix, "").lower(): value - for key, value in config["env_vars"].items() - if key.startswith(prefix) - } - if params["display_name"] in available_accounts: - params.pop("mnemonic") - - account.login(**params) - if account.info["compressed_key"] != config["bot"]["compressed_key"]: - raise Exception("Target compressed key and logged in compressed key are different...") - else: - account.logger.info("[SUCCESS] Logged in with correct account") - - balance = account["GBP"] - query = (balance["symbol"] == "SNT") & (balance["fiat_value"] > 0) & (balance["chain_id"] == 1) - if query.sum() != 1: - raise Exception("There were issues with Infura Token and Coingecko initialization...") - else: - account.logger.info("[SUCCESS] Wallet balance is available") - - account.profile_picture = os.path.join(os.path.dirname(__file__), "assets", "profile.jpg") - account.logger.info(f"Account Information:\nCompressed Key: {account.info['compressed_key']}\nPublic Key: {account.info['public_key']}\nURL: {account.info['url']}") - return account - -def download(account: Account, folder: str, config: dict): - """ - Download Status App messages / info from communities and store them in pickle files. - - Parameters: - - `folder` - the folder where the files will be created. Sub folders are automatically created - - `config` - the `load_config` configuration - """ - file_path = os.path.join(os.path.dirname(__file__), config["files"]["current_state"]) - latest_dates: dict[str, pd.Timestamp] = pd.read_pickle(file_path) if os.path.exists(file_path) else {} - - get_file_name = lambda: str(to_midnight(datetime.datetime.now()).timestamp()).replace(".", "") - communities = account.communities - if not communities: - account.logger.warning("No communities found...") - - for community in communities: - - if not community["is_member"]: - continue - - community_folder_name = community["name"].replace(" ", "-") - messages_folder = os.path.join(folder, "messages", community_folder_name) - community_info_folder = os.path.join(folder, "community", community_folder_name) - - account.logger.info(f"Extracting data for {community['name']}") - community["extracted_timestamp"] = datetime.datetime.now() - - file_path = os.path.join(community_info_folder, get_file_name() + ".pkl") - if not os.path.exists(file_path): - save_file(file_path, community) - account.logger.info(f"Created {file_path}") - - file_path = os.path.join(messages_folder, get_file_name() + ".csv") - if not os.path.exists(file_path): - messages = extract_community_channels(account, community, latest_dates) - if len(messages) > 0: - save_file(file_path, messages) - account.logger.info(f"Created {file_path}") - -def store(folder: str, config: dict, logger: Logger): - """ - Upload Status App `download` file to Postgres. - NOTE: The Postgres schema must already exist - - Parameters: - - `folder` - the folder where the files will be created. Sub folders are automatically created - - `config` - the `load_config` configuration - """ - path = Path(folder) - table_name_mapping: dict[str, str] = config["postgres"]["tables"] - table_schema = config["postgres"]["schema"] - - upload: dict[str, list] = {} - - file_path = os.path.join(os.path.dirname(__file__), config["files"]["current_state"]) - latest_dates: dict[str, pd.Timestamp] = pd.read_pickle(file_path) if os.path.exists(file_path) else {} - - completed = [] - - files = list(path.rglob("*.pkl")) + list(path.rglob("*.csv")) - logger.info(f"There are {len(files)} file(s) to upload") - for file_path in files: - - table_name = table_name_mapping.get(file_path.parent.parent.name) - if not table_name: - continue - - file_name = str(file_path.name) - data = pd.read_pickle(file_path) if file_name.endswith(".pkl") else pd.read_csv(file_path) - if isinstance(data, dict): - data = pd.DataFrame([data]) - - for column in data.columns: - if "timestamp" not in column: - continue - data[column] = pd.to_datetime(data[column]) - - if table_name not in upload: - upload[table_name] = [] - - if "timestamp" in data.columns: - latest_dates.update(data.groupby("chat_id")["timestamp"].max().to_dict()) - - upload[table_name].append(data) - completed.append(str(file_path)) - - save_file(config["files"]["current_state"], latest_dates) - logger.info(f"Updated {config['files']['current_state']}") - - prefix = "POSTGRES_" - params = { - key.replace(prefix, "").lower(): value - for key, value in config["env_vars"].items() - if key.startswith(prefix) - } - connector = Postgres(**params) - for table_name, data in upload.items(): - if len(data) == 0: - continue - - df = pd.concat(data, ignore_index=True).assign(batch_timestamp = datetime.datetime.now()) - json_columns = [ - column - for column in df.columns - if len(df[column].dropna()) > 0 and isinstance(df[column].dropna().reset_index(drop=True).iloc[0], (dict, list)) - ] - connector.insert(df, table_name, table_schema, json_columns) - logger.info(f"Uploaded {len(df)} record(s) to {table_schema}.{table_name}") - - for file_path in completed: - os.remove(file_path) - logger.info(f"Deleted {file_path}") - -if __name__ == "__main__": - folder = os.path.dirname(__file__) - config = load_config(os.path.join(folder, "config.yaml")) - upload_folder = os.path.join(os.path.dirname(__file__), "uploads") - logger = Logger() - account = create_bot(config) - - while True: - download(account, upload_folder, config) - store(upload_folder, config, logger) - logger.info(f"Sleeping for {config['sleep']} minute(s)") - time.sleep(config["sleep"] * 60) From c28a596ccddac71ce380e75ed035074728f0345b Mon Sep 17 00:00:00 2001 From: apentori Date: Wed, 27 May 2026 19:10:09 +0200 Subject: [PATCH 11/13] api: adding optional API KEY Signed-off-by: apentori --- bot/config.py | 1 + bot/modules/api_server.py | 21 ++++++++++++++++++++- config.yaml | 1 + docs/SUMMARY.md | 2 +- docs/usage/configuration.md | 2 ++ docs/usage/messaging.md | 14 +++++++++++++- 6 files changed, 38 insertions(+), 3 deletions(-) diff --git a/bot/config.py b/bot/config.py index 4155768..dca6175 100644 --- a/bot/config.py +++ b/bot/config.py @@ -33,6 +33,7 @@ class ApiConfig(BaseModel): enable: bool = True host: str = "0.0.0.0" port: int = 8081 + api_key: str = "" class MetricsConfig(BaseModel): diff --git a/bot/modules/api_server.py b/bot/modules/api_server.py index aa3a567..a091455 100644 --- a/bot/modules/api_server.py +++ b/bot/modules/api_server.py @@ -1,7 +1,8 @@ import threading import uvicorn -from fastapi import FastAPI +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse from bot.modules.base import BaseModule, ModuleType @@ -18,6 +19,24 @@ def on_start(self): self._port = api_config.port self._app: FastAPI = self.ctx.shared_state["fastapi_app"] self._server = None + self._add_auth_middleware(api_config.api_key) + + def _add_auth_middleware(self, api_key: str): + if not api_key: + return + + exempt = {"/health", "/docs", "/redoc", "/openapi.json"} + + @self._app.middleware("http") + async def require_api_key(request: Request, call_next): + if request.url.path in exempt: + return await call_next(request) + if request.headers.get("X-API-Key") != api_key: + return JSONResponse( + status_code=401, + content={"detail": "Invalid or missing API key"}, + ) + return await call_next(request) def execute(self): if not self.ctx.shared_state["config"].api.enable: diff --git a/config.yaml b/config.yaml index c764035..c3d0c14 100644 --- a/config.yaml +++ b/config.yaml @@ -27,6 +27,7 @@ api: enable: true host: "0.0.0.0" port: 8081 + # api_key: "your-secret-key" modules: directories: [] diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 115a185..e2a3633 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -9,6 +9,6 @@ ## Usage - [Configuration](./usage/configuration.md) -- [API](./usage/api.md) +- [Messaging API](./usage/messaging.md) - [Community Monitoring](./usage/monitoring.md) - [Metrics](./usage/metrics.md) diff --git a/docs/usage/configuration.md b/docs/usage/configuration.md index 1b81808..578514a 100644 --- a/docs/usage/configuration.md +++ b/docs/usage/configuration.md @@ -91,12 +91,14 @@ Configuration for the WebServer avaialable to the modules. | `enable` | `bool` | `true` | `API_ENABLE` | Enable the REST API server | | `host` | `str` | `"0.0.0.0"` | `API_HOST` | API server bind address | | `port` | `int` | `8081` | `API_PORT` | API server port | +| `api_key` | `str` | `""` | `API_API_KEY` | API key for request authentication (empty = disabled) | ```yaml api: enable: true host: "0.0.0.0" port: 8081 + # api_key: "your-secret-key" ``` --- diff --git a/docs/usage/messaging.md b/docs/usage/messaging.md index fb6cb0c..a4dea73 100644 --- a/docs/usage/messaging.md +++ b/docs/usage/messaging.md @@ -10,7 +10,19 @@ The bot exposes a REST endpoint for sending messages, managing contacts, and que | ReDoc | `http://localhost:8081/redoc` | | OpenAPI JSON | `http://localhost:8081/openapi.json` | -No authentication is required. +## Authentication + +If `api.api_key` is configured in `config.yaml`, all requests (except `/health`, `/docs`, `/redoc`, and `/openapi.json`) must include the `X-API-Key` header: + +```bash +curl -H "X-API-Key: your-secret-key" http://localhost:8081/api/v1/chats +``` + +Requests without a valid key return `401 Unauthorized`: + +```json +{"detail": "Invalid or missing API key"} +``` --- From c5afc4d6b7897f2bf8565d200a9cf7629fde0be2 Mon Sep 17 00:00:00 2001 From: apentori Date: Wed, 27 May 2026 19:51:41 +0200 Subject: [PATCH 12/13] api: add wehbook Signed-off-by: apentori --- bot/modules/messaging.py | 11 +++++++++++ docs/usage/messaging.md | 23 +++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/bot/modules/messaging.py b/bot/modules/messaging.py index 0a53f74..546aa6c 100644 --- a/bot/modules/messaging.py +++ b/bot/modules/messaging.py @@ -16,6 +16,10 @@ class SendMessageRequest(BaseModel): text: str +class WebhookMessage(BaseModel): + content: str + + class MessagingModule(BaseModule): @property @@ -91,6 +95,13 @@ def send_message(chat_id: str, payload: SendMessageRequest): account.send_message(chat_id, payload.text) return {"status": "ok"} + @app.post("/api/v1/webhook/messages/{chat_id}") + def webhook_send_message(chat_id: str, payload: WebhookMessage): + if not payload.content: + raise HTTPException(status_code=400, detail="Content is required") + account.send_message(chat_id, payload.content) + return {"status": "ok"} + @app.get("/api/v1/communities") def get_communities(): return account.communities diff --git a/docs/usage/messaging.md b/docs/usage/messaging.md index a4dea73..45b1adf 100644 --- a/docs/usage/messaging.md +++ b/docs/usage/messaging.md @@ -158,6 +158,29 @@ Response `201`: --- +### `POST /api/v1/webhook/messages/{chat_id}` + +Send a text message to a chat. Designed for external services to trigger messages without extra client logic. + +```bash +curl -X POST http://localhost:8081/api/v1/webhook/messages/0x... \ + -H "Content-Type: application/json" \ + -d '{"content": "Notification from external service"}' +``` + +Request body: + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `content` | `string` | Yes | Message content | + +Response `200`: +```json +{"status": "ok"} +``` + +--- + ## Error handling All errors return a JSON body with a `detail` field: From ffc525f604d1874706097cabc393c0883a2715a5 Mon Sep 17 00:00:00 2001 From: apentori Date: Wed, 27 May 2026 23:24:39 +0200 Subject: [PATCH 13/13] refactor database and add receiver modules Signed-off-by: apentori --- bot/__init__.py | 2 +- bot/config.py | 1 + bot/database.py | 82 ++++++++++++++++++++ bot/models/__init__.py | 3 + bot/models/base.py | 5 ++ bot/models/chat.py | 12 +++ bot/models/message.py | 33 ++++++++ bot/modules/monitoring.py | 7 +- bot/modules/receiver.py | 76 ++++++++++++++++++ bot/postgres.py | 149 ------------------------------------ config.yaml | 12 ++- docs/usage/configuration.md | 24 ++++-- main.py | 23 +++--- 13 files changed, 255 insertions(+), 174 deletions(-) create mode 100644 bot/database.py create mode 100644 bot/models/__init__.py create mode 100644 bot/models/base.py create mode 100644 bot/models/chat.py create mode 100644 bot/models/message.py create mode 100644 bot/modules/receiver.py delete mode 100644 bot/postgres.py diff --git a/bot/__init__.py b/bot/__init__.py index 70e9bf3..548b641 100644 --- a/bot/__init__.py +++ b/bot/__init__.py @@ -1,4 +1,4 @@ from .account import Account from .logger import Logger -from .postgres import Postgres +from .database import Database from .config import Config diff --git a/bot/config.py b/bot/config.py index dca6175..31724e9 100644 --- a/bot/config.py +++ b/bot/config.py @@ -20,6 +20,7 @@ class BotConfig(BaseModel): coingecko_api_key: str = "" class DatabaseConfig(BaseModel): + type: str = "postgres" host: str = "database" port: int = 5432 user: str = "" diff --git a/bot/database.py b/bot/database.py new file mode 100644 index 0000000..8d015c2 --- /dev/null +++ b/bot/database.py @@ -0,0 +1,82 @@ +import logging + +import pandas as pd +from typing import Optional +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.exc import IntegrityError, NoSuchTableError + +from .models import Base + + +class Database: + + def __init__(self, db_type: str, host: str, port: int, user: str, password: str, name: str, schema: str): + self._type = db_type + self._schema = schema + self._url = self._build_url(db_type, host, port, user, password, name) + self._engine = create_engine(self._url) + self._logger = logging.getLogger("status_bot.database") + + def init_tables(self): + Base.metadata.create_all(self._engine) + self._logger.info("Database tables initialized") + + def _build_url(self, db_type: str, host: str, port: int, user: str, password: str, name: str) -> str: + if db_type == "postgres": + return f"postgresql://{user}:{password}@{host}:{port}/{name}" + elif db_type == "sqlite": + return f"sqlite:///{name}" + raise ValueError(f"Unsupported database type: {db_type}") + + def insert(self, data: pd.DataFrame, table_name: str, json_columns: Optional[list] = None): + if len(data) == 0: + return + + if self._type == "postgres": + with self._engine.begin() as conn: + conn.execute(text(f"CREATE SCHEMA IF NOT EXISTS {self._schema}")) + + data.columns = [column.lower() for column in data.columns] + + params = { + "name": table_name, + "con": self._engine, + "schema": self._schema, + "if_exists": "append", + "index": False, + } + if json_columns and self._type == "postgres": + params["dtype"] = {col: JSONB for col in json_columns} + + existing_columns = self.get_columns(table_name) + if existing_columns: + for column in data.columns: + if column not in existing_columns: + with self._engine.begin() as conn: + conn.execute( + text(f"ALTER TABLE {self._schema}.{table_name} ADD COLUMN {column} TEXT") + ) + + try: + data.to_sql(**params) + except IntegrityError: + self._logger.warning(f"Duplicate rows skipped in {table_name}") + + def execute(self, query: str): + with self._engine.begin() as conn: + conn.execute(text(query)) + + def to_pandas(self, query: str) -> pd.DataFrame: + return pd.read_sql(query, self._engine) + + def get_columns(self, table_name: str) -> list[str]: + insp = inspect(self._engine) + try: + columns = insp.get_columns(table_name, schema=self._schema) + except NoSuchTableError: + return [] + return [col["name"] for col in columns] + + def close(self): + self._engine.dispose() diff --git a/bot/models/__init__.py b/bot/models/__init__.py new file mode 100644 index 0000000..483df24 --- /dev/null +++ b/bot/models/__init__.py @@ -0,0 +1,3 @@ +from .base import Base +from .message import ReceivedMessage +from .chat import ReceivedChat diff --git a/bot/models/base.py b/bot/models/base.py new file mode 100644 index 0000000..fa2b68a --- /dev/null +++ b/bot/models/base.py @@ -0,0 +1,5 @@ +from sqlalchemy.orm import DeclarativeBase + + +class Base(DeclarativeBase): + pass diff --git a/bot/models/chat.py b/bot/models/chat.py new file mode 100644 index 0000000..70aeb74 --- /dev/null +++ b/bot/models/chat.py @@ -0,0 +1,12 @@ +from sqlalchemy import Column, DateTime, String + +from .base import Base + + +class ReceivedChat(Base): + __tablename__ = "received_chats" + + id = Column(String, primary_key=True) + type = Column(String, nullable=True) + name = Column(String, nullable=True) + received_timestamp = Column(DateTime, nullable=True) diff --git a/bot/models/message.py b/bot/models/message.py new file mode 100644 index 0000000..5a2ccaa --- /dev/null +++ b/bot/models/message.py @@ -0,0 +1,33 @@ +from sqlalchemy import Boolean, Column, DateTime, Integer, JSON, String, BigInteger + +from .base import Base + + +class ReceivedMessage(Base): + __tablename__ = "received_messages" + + id = Column(String, primary_key=True) + whisper_timestamp = Column(DateTime, nullable=True) + from_ = Column("from", String, nullable=True) + alias = Column(String, nullable=True) + seen = Column(Boolean, nullable=True) + quoted_message = Column(JSON, nullable=True) + rtl = Column(Boolean, nullable=True) + parsed_text = Column(JSON, nullable=True) + line_count = Column(Integer, nullable=True) + text = Column(String, nullable=True) + chat_id = Column(String, nullable=True) + local_chat_id = Column(String, nullable=True) + clock = Column(BigInteger, nullable=True) + replace = Column(String, nullable=True) + response_to = Column(String, nullable=True) + ens_name = Column(String, nullable=True) + display_name = Column(String, nullable=True) + gap_parameters = Column(JSON, nullable=True) + timestamp = Column(DateTime, nullable=True) + content_type = Column(Integer, nullable=True) + message_type = Column(Integer, nullable=True) + contact_request_state = Column(Integer, nullable=True) + compressed_key = Column(String, nullable=True) + emoji_hash = Column(JSON, nullable=True) + received_timestamp = Column(DateTime, nullable=True) diff --git a/bot/modules/monitoring.py b/bot/modules/monitoring.py index 5d3452a..310be8c 100644 --- a/bot/modules/monitoring.py +++ b/bot/modules/monitoring.py @@ -156,8 +156,7 @@ def _download(self, account, upload_path: str, current_state_path: str, config) def _store(self, upload_path: str, current_state_path: str, config, logger) -> None: path = Path(upload_path) - table_name_mapping: dict[str, str] = config.postgres.tables - table_schema = config.postgres.schema + table_name_mapping: dict[str, str] = config.database.tables upload: dict[str, list] = {} latest_dates: dict[str, pd.Timestamp] = ( @@ -206,8 +205,8 @@ def _store(self, upload_path: str, current_state_path: str, config, logger) -> N if len(df[column].dropna()) > 0 and isinstance(df[column].dropna().reset_index(drop=True).iloc[0], (dict, list)) ] - connector.insert(df, table_name, table_schema, json_columns) - logger.info(f"Uploaded {len(df)} record(s) to {table_schema}.{table_name}") + connector.insert(df, table_name, json_columns) + logger.info(f"Uploaded {len(df)} record(s) to {table_name}") for file_path in completed: os.remove(file_path) diff --git a/bot/modules/receiver.py b/bot/modules/receiver.py new file mode 100644 index 0000000..3f84ed4 --- /dev/null +++ b/bot/modules/receiver.py @@ -0,0 +1,76 @@ +import datetime + +import pandas as pd + +from bot.modules.base import BaseModule, ModuleType +from bot.modules.utils import to_sha256_hash + + +class ReceiverModule(BaseModule): + + @property + def module_type(self) -> ModuleType: + return ModuleType.EVENT + + def on_start(self): + settings = self.ctx.config.settings + self._messages_table = settings.get("messages_table", "received_messages") + self._chats_table = settings.get("chats_table", "received_chats") + self._hash_columns = {"id", "from", "response_to"} + + if self.ctx.db is None: + self.ctx.logger.warning("Receiver: no database configured, disabling") + self._disabled = True + return + + self._disabled = False + + def execute(self): + pass + + def on_event(self, event: dict): + if self._disabled: + return + + event_data = event.get("event", {}) + + messages = event_data.get("messages", []) + if messages: + + self.ctx.logger.info(f"message received {messages}") + self._process_and_insert(messages, self._messages_table) + + chats = event_data.get("chats", []) + if chats: + self._process_and_insert(chats, self._chats_table) + + def _process_and_insert(self, raw_data: list[dict], table_name: str): + if not raw_data: + return + + df = pd.DataFrame(raw_data) + df.columns = [col.lower() for col in df.columns] + + for col in self._hash_columns: + if col in df.columns: + df[col] = df[col].astype(str).apply(to_sha256_hash) + + df["received_timestamp"] = datetime.datetime.now() + + json_columns = self._detect_json_columns(df) + + self.ctx.db.insert(df, table_name, json_columns) + self.ctx.logger.info( + f"Receiver: stored {len(df)} record(s) in {table_name}" + ) + + @staticmethod + def _detect_json_columns(df: pd.DataFrame) -> list[str]: + json_columns = [] + for col in df.columns: + non_null = df[col].dropna() + if len(non_null) > 0: + first = non_null.reset_index(drop=True).iloc[0] + if isinstance(first, (dict, list)): + json_columns.append(col) + return json_columns diff --git a/bot/postgres.py b/bot/postgres.py deleted file mode 100644 index 755f9e8..0000000 --- a/bot/postgres.py +++ /dev/null @@ -1,149 +0,0 @@ -""" -Minimum code to upload data taken from: -https://github.com/status-im/ift-data-py/blob/master/ift_data/clients/postgres.py -""" - -import psycopg2 -import pandas as pd -from typing import Optional, Union -from sqlalchemy import create_engine -from sqlalchemy.dialects.postgresql import JSONB - -class Postgres: - - def __init__(self, user: str, password: str, port: Union[int, str], database: str, host: str): - - if isinstance(port, str): - port = int(port) - - self.__params = { - "host": host, - "user": user, - "password": password, - "port": port, - "database": database - } - - self.__url = f"postgresql://{user}:{password}@{host}:{port}/{database}" - self.__conn: psycopg2.extensions.connection = psycopg2.connect(**self.__params) - self.__cursor: psycopg2.extensions.cursor = self.__conn.cursor() - - def insert(self, data: pd.DataFrame, table_name: str, schema: str, json_columns: Optional[list] = None): - """ - Insert the DataFrame in the specified schema > table. - If the schema / table name does not exist, it will be created. - - Parameters: - - `data` - the data to be inserted in Postgres - - `table_name` - the name of the table - - `schema` - the name of the schema - - `json_columns` - when creating the table, `dict` columns will be turned into JSON objects in Postgres - """ - self.execute(f"CREATE SCHEMA IF NOT EXISTS {schema}") - engine = create_engine(self.__url) - - data.columns = [column.lower() for column in data.columns] - - params = { - "name": table_name, - "con": engine, - "schema": schema, - "if_exists": "append", - "index": False - } - if json_columns: - params["dtype"] = { - json_column: JSONB - for json_column in json_columns - } - - # Add new columns as they come - existing_columns = self.get_columns(schema, table_name) - - if existing_columns: - for column in data.columns: - if column in existing_columns: - continue - # NOTE: New values will have to be transformed - self.execute(f"ALTER TABLE {schema}.{table_name} ADD COLUMN {column} TEXT") - - data.to_sql(**params) - - def execute(self, query: str): - """ - Execute queries such as INSERT, UPDATE, DELETE etc. - - Parameters: - - `query` - the PostgreSQL query - """ - self.__execute(query) - self.__conn.commit() - - def to_pandas(self, query: str, batch_size: int = 50_000, uppercase: bool = True) -> pd.DataFrame: - """ - Create a DataFrame from the given query - - Parameters: - - `query` - the PostgreSQL query - - `batch_size` - how many rows will be fetched at once - - `uppercase` - if `True` then the columns will be uppercase. If `False` the columns will be lowercase - Output: - - DataFrame for the executed query - """ - self.__execute(query) - columns = [column.name.upper() if uppercase else column.name.lower() for column in self.__cursor.description] - chunks = [] - - while True: - rows = self.__cursor.fetchmany(batch_size) - if not rows: - break - chunks.append(pd.DataFrame(rows, columns=columns)) - - return pd.concat(chunks, ignore_index=True) if chunks else pd.DataFrame(columns=columns) - - def close(self): - self.__cursor.close() - self.__conn.close() - - def __del__(self): - self.close() - - def __execute(self, query: str): - - failed = False - is_closed = bool(self.__conn.closed) - - if is_closed: - self.__conn: psycopg2.extensions.connection = psycopg2.connect(**self.__params) - self.__cursor: psycopg2.extensions.cursor = self.__conn.cursor() - - try: - self.__cursor.execute(query) - except psycopg2.errors.InFailedSqlTransaction: - self.__conn.rollback() - failed = True - - if failed: - self.__cursor.execute(query) - - - def get_columns(self, schema: str, table_name: str) -> list[str]: - """ - Get the column names in the correct order for the given table. - - Parameters: - - `table_name` - the name of the table - - `schema` - the name of the schema - - Output: - - the table's columns in the correct order - """ - query = f""" - SELECT column_name - FROM information_schema.columns - WHERE table_name = '{table_name}' - AND table_schema = '{schema}' - ORDER BY ordinal_position ASC - """ - return self.to_pandas(query)["COLUMN_NAME"].to_list() diff --git a/config.yaml b/config.yaml index c3d0c14..b547f5a 100644 --- a/config.yaml +++ b/config.yaml @@ -1,4 +1,5 @@ -postgres: +database: + type: postgres host: "database" port: 5432 schema: "status_app_monitoring" @@ -13,7 +14,7 @@ files: current_state: "dates.pkl" bot: - init_account: false + init_account: true display_name: 'e-raccoon' public_key: "0xExample" compressed_key: "example-compressed-key" @@ -31,8 +32,11 @@ api: modules: directories: [] - enabled: ["messaging"] - settings: {} + enabled: ["messaging", "receiver"] + settings: + receiver: + messages_table: "received_messages" + chats_table: "received_chats" prometheus: enabled: true diff --git a/docs/usage/configuration.md b/docs/usage/configuration.md index 578514a..2b4a87c 100644 --- a/docs/usage/configuration.md +++ b/docs/usage/configuration.md @@ -105,18 +105,23 @@ api: ### `database` +Supports **Postgres** and **SQLite** (SQLite via `type: sqlite`, where `name` is the file path). + | Field | Type | Default | Env var | Description | |-------|------|---------|---------|-------------| -| `host` | `str` | `"database"` | `POSTGRES_HOST` | Postgres server hostname | -| `port` | `int` | `5432` | `POSTGRES_PORT` | Postgres server port | -| `user` | `str` | `""` | `POSTGRES_USER` | Postgres username | -| `password` | `str` | `""` | `POSTGRES_PASSWORD` | Postgres password | -| `name` | `str` | `""` | `POSTGRES_NAME` | Postgres database name | -| `schema` | `str` | `"public"` | `POSTGRES_SCHEMA` | Database schema for storing data | +| `type` | `str` | `"postgres"` | `DATABASE_TYPE` | Database engine (`postgres` or `sqlite`) | +| `host` | `str` | `"database"` | `DATABASE_HOST` | Database server hostname (Postgres) | +| `port` | `int` | `5432` | `DATABASE_PORT` | Database server port (Postgres) | +| `user` | `str` | `""` | `DATABASE_USER` | Database username (Postgres) | +| `password` | `str` | `""` | `DATABASE_PASSWORD` | Database password (Postgres) | +| `name` | `str` | `""` | `DATABASE_NAME` | Database name (Postgres) or file path (SQLite) | +| `schema` | `str` | `"public"` | `DATABASE_SCHEMA` | Database schema (Postgres) | | `tables` | `dict` | `{}` | — | Mapping of data type to table name | +Postgres example: ```yaml database: + type: postgres host: database port: 5432 user: 'myuser' @@ -128,6 +133,13 @@ database: community: "raw_community_info" ``` +SQLite example: +```yaml +database: + type: sqlite + name: "/data/status-bot.db" +``` + --- ### `modules` diff --git a/main.py b/main.py index 3777478..e0299de 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ from fastapi import FastAPI -from bot import Account, Logger, Config, Postgres +from bot import Account, Logger, Config, Database from bot.metrics import start_prometheus from bot.modules.manager import ModuleManager @@ -61,13 +61,15 @@ def create_bot(config: Config, project_root: str) -> Account: return account -def init_postgres(config: Config) -> Postgres: - return Postgres( +def init_database(config: Config) -> Database: + return Database( + db_type=config.database.type, host=config.database.host, port=config.database.port, user=config.database.user, password=config.database.password, - database=config.database.name, + name=config.database.name, + schema=config.database.schema, ) @@ -106,22 +108,23 @@ def main(): sys.exit(1) db = None - has_postgres = all([ + has_database = all([ config.database.host, config.database.user, config.database.password, config.database.name, ]) - if has_postgres: + if has_database: try: - db = init_postgres(config) - logger.info("Postgres connection established") + db = init_database(config) + db.init_tables() + logger.info(f"Database connection established ({config.database.type})") except Exception as e: - logger.warning(f"Failed to connect to Postgres: {e}") + logger.warning(f"Failed to connect to database: {e}") logger.warning("Continuing without database connection") else: - logger.info("No Postgres configuration found, running without database") + logger.info("No database configuration found, running without database") fastapi_app = FastAPI(title="Status Bot API") shared_state = {"config": config, "project_root": project_root, "fastapi_app": fastapi_app}