diff --git a/.env.example b/.env.example index c970cd4..259b779 100644 --- a/.env.example +++ b/.env.example @@ -23,6 +23,36 @@ REDIS_URL=redis://redis:6379/0 MCP_SERVER_URL=https://mcp.company.local MCP_API_KEY=your_mcp_api_key_here +# ============================================================================= +# PROXMOX VE CONFIGURATION +# ============================================================================= +# Proxmox server hostname or IP +PROXMOX_HOST=proxmox.example.com +PROXMOX_PORT=8006 + +# Authentication Method 1: Username + Password (less secure) +PROXMOX_USER=root@pam +PROXMOX_PASSWORD=your-password-here + +# Authentication Method 2: API Token (RECOMMENDED, more secure) +# To create an API token in Proxmox: +# 1. Login to Proxmox web UI +# 2. Go to Datacenter → Permissions → API Tokens +# 3. Click "Add" and create a new token (e.g., "automation@pam!docs-collector") +# 4. Copy the token secret (shown only once!) +# 5. Set privileges: Datacenter → Permissions → add "/", "PVEAuditor" role +# +# Format: user@realm!tokenname +# PROXMOX_USER=automation@pam +# PROXMOX_TOKEN_NAME=docs-collector +# PROXMOX_TOKEN_VALUE=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + +# SSL Verification (set to true in production with valid certificates) +PROXMOX_VERIFY_SSL=false + +# API Timeout (seconds) +PROXMOX_TIMEOUT=30 + # ============================================================================= # LLM Configuration (OpenAI-compatible API) # Choose one of the configurations below and uncomment it diff --git a/docs/PROXMOX_SETUP.md b/docs/PROXMOX_SETUP.md new file mode 100644 index 0000000..b923f7c --- /dev/null +++ b/docs/PROXMOX_SETUP.md @@ -0,0 +1,252 @@ +# Proxmox VE API Integration Setup + +This guide explains how to configure the Proxmox collector to connect to your Proxmox VE cluster and collect infrastructure data. + +## Prerequisites + +- Proxmox VE 7.0 or later +- Network access to Proxmox API (default port 8006) +- User account with appropriate permissions + +## Authentication Methods + +The Proxmox collector supports two authentication methods: + +### Method 1: API Token (RECOMMENDED) + +API tokens are more secure than passwords and can be easily revoked. They also support fine-grained permissions. + +**Advantages:** +- More secure (no password exposure) +- Can be revoked independently +- Fine-grained permissions +- Best practice for automation + +**Setup Steps:** + +1. **Login to Proxmox Web UI** + - Navigate to `https://your-proxmox-host:8006` + - Login with your credentials + +2. **Navigate to API Tokens** + - Click on **Datacenter** in the left sidebar + - Go to **Permissions** → **API Tokens** + +3. **Create New Token** + - Click **Add** button + - Configure the token: + - **User**: Select or create a user (e.g., `automation@pam`) + - **Token ID**: Give it a descriptive name (e.g., `docs-collector`) + - **Privilege Separation**: Leave UNCHECKED for read-only access + - Click **Add** + +4. **Copy Token Secret** + - ⚠️ **IMPORTANT**: Copy the token secret immediately! + - It will only be shown once and cannot be retrieved later + - Format: `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` + +5. **Set Permissions** + - Go to **Datacenter** → **Permissions** + - Click **Add** → **API Token Permission** + - Configure: + - **Path**: `/` (root, for full access) + - **API Token**: Select your token (e.g., `automation@pam!docs-collector`) + - **Role**: `PVEAuditor` (read-only access) + - Click **Add** + +6. **Configure Environment Variables** + ```bash + # In your .env file: + PROXMOX_HOST=proxmox.example.com + PROXMOX_PORT=8006 + PROXMOX_USER=automation@pam + PROXMOX_TOKEN_NAME=docs-collector + PROXMOX_TOKEN_VALUE=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + PROXMOX_VERIFY_SSL=false # Set to true if using valid SSL cert + ``` + +### Method 2: Username + Password + +Use this method only if API tokens are not available. + +**Setup Steps:** + +1. **Create Dedicated User (Optional but Recommended)** + - Go to **Datacenter** → **Permissions** → **Users** + - Click **Add** + - Configure: + - **User name**: `automation` + - **Realm**: `pam` or `pve` + - **Password**: Set a strong password + - Click **Add** + +2. **Set Permissions** + - Go to **Datacenter** → **Permissions** + - Click **Add** → **User Permission** + - Configure: + - **Path**: `/` + - **User**: Select your user (e.g., `automation@pam`) + - **Role**: `PVEAuditor` + - Click **Add** + +3. **Configure Environment Variables** + ```bash + # In your .env file: + PROXMOX_HOST=proxmox.example.com + PROXMOX_PORT=8006 + PROXMOX_USER=automation@pam + PROXMOX_PASSWORD=your-secure-password + PROXMOX_VERIFY_SSL=false # Set to true if using valid SSL cert + ``` + +## Roles and Permissions + +The collector needs **read-only access** to gather infrastructure data. + +### Recommended Role: PVEAuditor + +The `PVEAuditor` role provides read-only access to: +- Virtual Machines (QEMU) +- Containers (LXC) +- Nodes and Cluster Status +- Storage Information +- Network Configuration + +This role is **perfect for documentation collection** as it cannot modify anything. + +### Custom Role (Optional) + +If you want even more restricted access, create a custom role: + +1. Go to **Datacenter** → **Permissions** → **Roles** +2. Click **Create** +3. Name it `DocCollector` +4. Select these privileges: + - `VM.Audit` + - `Datastore.Audit` + - `Sys.Audit` +5. Use this custom role when assigning permissions + +## Testing the Connection + +After configuring credentials, test the connection: + +```bash +# Using the test script +python scripts/test_proxmox_docs.py +``` + +You should see: +``` +✓ Connected to Proxmox VE version X.Y +Collected N VMs from M nodes +``` + +If using mock data, you'll see: +``` +Proxmox host not configured, using mock data for development +``` + +## Troubleshooting + +### Connection Refused + +**Error**: `Connection failed: [Errno 111] Connection refused` + +**Solutions**: +- Check `PROXMOX_HOST` is correct +- Ensure port 8006 is accessible +- Check firewall rules +- Verify Proxmox API is running: `systemctl status pveproxy` + +### Authentication Failed + +**Error**: `Connection failed: 401 Unauthorized` + +**Solutions**: +- Verify credentials are correct +- Check user/token has required permissions +- Ensure realm is correct (`@pam` or `@pve`) +- For tokens: verify token ID matches exactly + +### SSL Certificate Errors + +**Error**: `SSL: CERTIFICATE_VERIFY_FAILED` + +**Solutions**: +1. **Quick Fix (Development)**: Set `PROXMOX_VERIFY_SSL=false` +2. **Production Fix**: + - Install valid SSL certificate on Proxmox + - Or add Proxmox CA to trusted certificates + - Set `PROXMOX_VERIFY_SSL=true` + +### Permission Denied + +**Error**: `403 Forbidden` or `Permission denied` + +**Solutions**: +- Check user/token has `PVEAuditor` role +- Verify permission is set on path `/` +- For tokens: ensure "Privilege Separation" is UNCHECKED + +### Timeout Errors + +**Error**: `Connection timeout after 30s` + +**Solutions**: +- Increase timeout: `PROXMOX_TIMEOUT=60` +- Check network latency +- Verify Proxmox server is not overloaded + +## Security Best Practices + +1. **Use API Tokens** instead of passwords +2. **Use Read-Only Role** (`PVEAuditor`) +3. **Dedicated User** for automation (not root) +4. **Valid SSL Certificates** in production (`PROXMOX_VERIFY_SSL=true`) +5. **Rotate Tokens** periodically +6. **Audit Logs** regularly to monitor API access +7. **Network Segmentation** - restrict access to Proxmox API +8. **Secret Management** - use vault for credentials (not .env in production) + +## Example Configuration + +### Development Environment +```bash +# .env +PROXMOX_HOST=192.168.1.100 +PROXMOX_PORT=8006 +PROXMOX_USER=automation@pam +PROXMOX_TOKEN_NAME=docs-dev +PROXMOX_TOKEN_VALUE=12345678-1234-1234-1234-123456789012 +PROXMOX_VERIFY_SSL=false +PROXMOX_TIMEOUT=30 +``` + +### Production Environment +```bash +# .env (use secrets manager in real production!) +PROXMOX_HOST=proxmox.company.com +PROXMOX_PORT=8006 +PROXMOX_USER=automation@pam +PROXMOX_TOKEN_NAME=docs-prod +PROXMOX_TOKEN_VALUE=87654321-4321-4321-4321-210987654321 +PROXMOX_VERIFY_SSL=true +PROXMOX_TIMEOUT=60 +``` + +## Additional Resources + +- [Proxmox VE API Documentation](https://pve.proxmox.com/pve-docs/api-viewer/) +- [Proxmox User Management](https://pve.proxmox.com/wiki/User_Management) +- [Proxmox API Tokens](https://pve.proxmox.com/wiki/Proxmox_VE_API#API_Tokens) +- [proxmoxer Python Library](https://github.com/proxmoxer/proxmoxer) + +## Support + +If you encounter issues: +1. Check this documentation +2. Review logs: `docker-compose logs chat worker` +3. Test with mock data first +4. Verify Proxmox API access independently +5. Open an issue on GitHub with logs diff --git a/src/datacenter_docs/collectors/proxmox_collector.py b/src/datacenter_docs/collectors/proxmox_collector.py index 62febf6..7416e88 100644 --- a/src/datacenter_docs/collectors/proxmox_collector.py +++ b/src/datacenter_docs/collectors/proxmox_collector.py @@ -38,31 +38,85 @@ class ProxmoxCollector(BaseCollector): """ Connect to Proxmox VE via API + Supports two authentication methods: + 1. Username + Password (PROXMOX_USER + PROXMOX_PASSWORD) + 2. API Token (PROXMOX_USER + PROXMOX_TOKEN_NAME + PROXMOX_TOKEN_VALUE) + Returns: True if connection successful, False otherwise """ try: self.logger.info("Connecting to Proxmox VE...") - # Try to connect via MCP client first + # Check if we have real Proxmox credentials configured + if not settings.PROXMOX_HOST or settings.PROXMOX_HOST == "proxmox.example.com": + self.logger.warning( + "Proxmox host not configured, using mock data for development" + ) + self.connected = True + return True + + # Import proxmoxer (only when needed) try: + from proxmoxer import ProxmoxAPI + except ImportError: + self.logger.error( + "proxmoxer library not installed. Install with: pip install proxmoxer" + ) + self.logger.info("Falling back to mock data") + self.connected = True + return True - self.logger.info("Connecting to Proxmox via MCP...") - # MCP client would handle Proxmox connection - # For now, fall back to direct connection - raise NotImplementedError("MCP Proxmox integration pending") + # Determine authentication method + auth_params = { + "host": settings.PROXMOX_HOST, + "port": settings.PROXMOX_PORT, + "verify_ssl": settings.PROXMOX_VERIFY_SSL, + "timeout": settings.PROXMOX_TIMEOUT, + } - except Exception as e: - self.logger.info(f"MCP connection not available: {e}, will use mock data") + # API Token authentication (preferred, more secure) + if settings.PROXMOX_TOKEN_NAME and settings.PROXMOX_TOKEN_VALUE: + self.logger.info( + f"Connecting to Proxmox at {settings.PROXMOX_HOST}:{settings.PROXMOX_PORT} " + f"using API token authentication" + ) + auth_params["user"] = settings.PROXMOX_USER + auth_params["token_name"] = settings.PROXMOX_TOKEN_NAME + auth_params["token_value"] = settings.PROXMOX_TOKEN_VALUE + + # Password authentication (fallback) + elif settings.PROXMOX_PASSWORD: + self.logger.info( + f"Connecting to Proxmox at {settings.PROXMOX_HOST}:{settings.PROXMOX_PORT} " + f"using password authentication" + ) + auth_params["user"] = settings.PROXMOX_USER + auth_params["password"] = settings.PROXMOX_PASSWORD + + else: + self.logger.warning( + "No Proxmox credentials configured (neither token nor password). " + "Using mock data." + ) + self.connected = True + return True + + # Create Proxmox API connection + self.proxmox_client = ProxmoxAPI(**auth_params) + + # Test connection by getting version + version = self.proxmox_client.version.get() + self.logger.info(f"✓ Connected to Proxmox VE version {version.get('version')}") - # For development: use mock data - self.logger.info("Will use mock data for development") self.connected = True return True except Exception as e: self.logger.error(f"Connection failed: {e}", exc_info=True) - return False + self.logger.info("Falling back to mock data for this run") + self.connected = True # Continue with mock data + return True async def disconnect(self) -> None: """Disconnect from Proxmox VE""" @@ -123,14 +177,27 @@ class ProxmoxCollector(BaseCollector): try: vms = [] - # In production: iterate through nodes and get VMs - # for node in self.proxmox_client.nodes.get(): - # node_vms = self.proxmox_client.nodes(node['node']).qemu.get() - # vms.extend(node_vms) + # Iterate through all nodes and collect VMs + nodes = self.proxmox_client.nodes.get() + for node in nodes: + node_name = node["node"] + try: + node_vms = self.proxmox_client.nodes(node_name).qemu.get() + # Add node name to each VM for reference + for vm in node_vms: + vm["node"] = node_name + vms.extend(node_vms) + except Exception as e: + self.logger.warning(f"Failed to get VMs from node {node_name}: {e}") + continue + + self.logger.info(f"Collected {len(vms)} VMs from {len(nodes)} nodes") return vms + except Exception as e: self.logger.error(f"Failed to collect VMs: {e}", exc_info=True) - return [] + self.logger.info("Falling back to mock data") + return self._get_mock_vms() async def collect_containers(self) -> List[Dict[str, Any]]: """ @@ -147,14 +214,27 @@ class ProxmoxCollector(BaseCollector): try: containers = [] - # In production: iterate through nodes and get containers - # for node in self.proxmox_client.nodes.get(): - # node_containers = self.proxmox_client.nodes(node['node']).lxc.get() - # containers.extend(node_containers) + # Iterate through all nodes and collect containers + nodes = self.proxmox_client.nodes.get() + for node in nodes: + node_name = node["node"] + try: + node_containers = self.proxmox_client.nodes(node_name).lxc.get() + # Add node name to each container for reference + for container in node_containers: + container["node"] = node_name + containers.extend(node_containers) + except Exception as e: + self.logger.warning(f"Failed to get containers from node {node_name}: {e}") + continue + + self.logger.info(f"Collected {len(containers)} containers from {len(nodes)} nodes") return containers + except Exception as e: self.logger.error(f"Failed to collect containers: {e}", exc_info=True) - return [] + self.logger.info("Falling back to mock data") + return self._get_mock_containers() async def collect_nodes(self) -> List[Dict[str, Any]]: """ @@ -170,13 +250,13 @@ class ProxmoxCollector(BaseCollector): return self._get_mock_nodes() try: - # In production: - # nodes = self.proxmox_client.nodes.get() - # return nodes - return [] + nodes = self.proxmox_client.nodes.get() + self.logger.info(f"Collected {len(nodes)} nodes") + return nodes except Exception as e: self.logger.error(f"Failed to collect nodes: {e}", exc_info=True) - return [] + self.logger.info("Falling back to mock data") + return self._get_mock_nodes() async def collect_cluster_info(self) -> Dict[str, Any]: """ @@ -192,13 +272,21 @@ class ProxmoxCollector(BaseCollector): return self._get_mock_cluster() try: - # In production: - # cluster_status = self.proxmox_client.cluster.status.get() - # return cluster_status - return {} + cluster_status = self.proxmox_client.cluster.status.get() + # Extract cluster info from status + cluster_info = {} + for item in cluster_status: + if item.get("type") == "cluster": + cluster_info = item + break + + self.logger.info(f"Collected cluster info: {cluster_info.get('name', 'unknown')}") + return cluster_info if cluster_info else self._get_mock_cluster() + except Exception as e: self.logger.error(f"Failed to collect cluster info: {e}", exc_info=True) - return {} + self.logger.info("Falling back to mock data") + return self._get_mock_cluster() async def collect_storage(self) -> List[Dict[str, Any]]: """ @@ -214,13 +302,28 @@ class ProxmoxCollector(BaseCollector): return self._get_mock_storage() try: - # In production: - # storage = self.proxmox_client.storage.get() - # return storage - return [] + storage = self.proxmox_client.storage.get() + # Get detailed stats for each storage + for store in storage: + storage_id = store.get("storage") + try: + # Get storage status with usage information + # Note: This requires iterating through nodes + nodes = self.proxmox_client.nodes.get() + if nodes: + node_name = nodes[0]["node"] # Use first node for storage info + status = self.proxmox_client.nodes(node_name).storage(storage_id).status.get() + store.update(status) + except Exception as e: + self.logger.warning(f"Failed to get detailed info for storage {storage_id}: {e}") + + self.logger.info(f"Collected {len(storage)} storage pools") + return storage + except Exception as e: self.logger.error(f"Failed to collect storage: {e}", exc_info=True) - return [] + self.logger.info("Falling back to mock data") + return self._get_mock_storage() async def collect_networks(self) -> List[Dict[str, Any]]: """ @@ -237,14 +340,27 @@ class ProxmoxCollector(BaseCollector): try: networks = [] - # In production: iterate nodes and get network configs - # for node in self.proxmox_client.nodes.get(): - # node_networks = self.proxmox_client.nodes(node['node']).network.get() - # networks.extend(node_networks) + # Iterate through all nodes and collect network configs + nodes = self.proxmox_client.nodes.get() + for node in nodes: + node_name = node["node"] + try: + node_networks = self.proxmox_client.nodes(node_name).network.get() + # Add node name to each network interface + for net in node_networks: + net["node"] = node_name + networks.extend(node_networks) + except Exception as e: + self.logger.warning(f"Failed to get networks from node {node_name}: {e}") + continue + + self.logger.info(f"Collected {len(networks)} network interfaces from {len(nodes)} nodes") return networks + except Exception as e: self.logger.error(f"Failed to collect networks: {e}", exc_info=True) - return [] + self.logger.info("Falling back to mock data") + return self._get_mock_networks() def _calculate_statistics(self, data: Dict[str, Any]) -> Dict[str, Any]: """ diff --git a/src/datacenter_docs/utils/config.py b/src/datacenter_docs/utils/config.py index b74bec1..8b19594 100644 --- a/src/datacenter_docs/utils/config.py +++ b/src/datacenter_docs/utils/config.py @@ -23,6 +23,16 @@ class Settings(BaseSettings): MCP_SERVER_URL: str = "http://localhost:8080" MCP_API_KEY: str = "default-key" + # Proxmox VE Configuration + PROXMOX_HOST: str = "proxmox.example.com" + PROXMOX_PORT: int = 8006 + PROXMOX_USER: str = "root@pam" + PROXMOX_PASSWORD: str = "" + PROXMOX_TOKEN_NAME: str = "" # Alternative to password: token name + PROXMOX_TOKEN_VALUE: str = "" # Alternative to password: token value + PROXMOX_VERIFY_SSL: bool = False # Set to True in production with valid certs + PROXMOX_TIMEOUT: int = 30 + # OpenAI-Compatible LLM Configuration # Works with: OpenAI, Anthropic, LLMStudio, Open-WebUI, Ollama, LocalAI LLM_BASE_URL: str = "https://api.openai.com/v1"