feat: Implement CLI tool, Celery workers, and VMware collector

Complete implementation of core MVP components: CLI Tool (src/datacenter_docs/cli.py): - 11 commands for system management (serve, worker, init-db, generate, etc.) - Auto-remediation policy management (enable/disable/status) - System statistics and monitoring - Rich formatted output with tables and panels Celery Workers (src/datacenter_docs/workers/): - celery_app.py with 4 specialized queues (documentation, auto_remediation, data_collection, maintenance) - tasks.py with 8 async tasks integrated with MongoDB/Beanie - Celery Beat scheduling (6h docs, 1h data collection, 15m metrics, 2am cleanup) - Rate limiting (10 auto-remediation/h) and timeout configuration - Task lifecycle signals and comprehensive logging VMware Collector (src/datacenter_docs/collectors/): - BaseCollector abstract class with full workflow (connect/collect/validate/store/disconnect) - VMwareCollector for vSphere infrastructure data collection - Collects VMs, ESXi hosts, clusters, datastores, networks with statistics - MCP client integration with mock data fallback for development - MongoDB storage via AuditLog and data validation Documentation & Configuration: - Updated README.md with CLI commands and Workers sections - Updated TODO.md with project status (55% completion) - Added CLAUDE.md with comprehensive project instructions - Added Docker compose setup for development environment Project Status: - Completion: 50% -> 55% - MVP Milestone: 80% complete (only Infrastructure Generator remaining) - Estimated time to MVP: 1-2 days 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-10-19 22:29:59 +02:00
parent 541222ad68
commit 52655e9eee
34 changed files with 5246 additions and 456 deletions
--- a/deploy/docker/Dockerfile.api
+++ b/deploy/docker/Dockerfile.api
@@ -0,0 +1,64 @@
+# Dockerfile for FastAPI API Service
+FROM python:3.12-slim as builder
+
+WORKDIR /build
+
+# Install Poetry
+RUN pip install --no-cache-dir poetry==1.8.0
+
+# Copy dependency files
+COPY pyproject.toml poetry.lock ./
+
+# Export dependencies
+RUN poetry config virtualenvs.create false \
+    && poetry export -f requirements.txt --output requirements.txt --without-hashes
+
+# Runtime stage
+FROM python:3.12-slim
+
+LABEL maintainer="automation-team@company.com"
+LABEL description="Datacenter Documentation API Server"
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    gcc \
+    libpq-dev \
+    openssh-client \
+    snmp \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy requirements from builder
+COPY --from=builder /build/requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code and package definition
+COPY src/ /app/src/
+COPY config/ /app/config/
+COPY pyproject.toml README.md /app/
+
+# Install the package in editable mode
+RUN pip install --no-cache-dir -e /app
+
+# Create necessary directories
+RUN mkdir -p /app/logs /app/output
+
+# Create non-root user
+RUN useradd -m -u 1000 appuser && \
+    chown -R appuser:appuser /app
+
+USER appuser
+
+# Expose API port
+EXPOSE 8000
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:8000/health || exit 1
+
+# Run the API server
+CMD ["python", "-m", "uvicorn", "datacenter_docs.api.main:app", "--host", "0.0.0.0", "--port", "8000"]
--- a/deploy/docker/Dockerfile.chat
+++ b/deploy/docker/Dockerfile.chat
@@ -0,0 +1,60 @@
+# Dockerfile for Chat Service
+FROM python:3.12-slim as builder
+
+WORKDIR /build
+
+# Install Poetry
+RUN pip install --no-cache-dir poetry==1.8.0
+
+# Copy dependency files
+COPY pyproject.toml poetry.lock ./
+
+# Export dependencies
+RUN poetry config virtualenvs.create false \
+    && poetry export -f requirements.txt --output requirements.txt --without-hashes
+
+# Runtime stage
+FROM python:3.12-slim
+
+LABEL maintainer="automation-team@company.com"
+LABEL description="Datacenter Documentation Chat Server"
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy requirements from builder
+COPY --from=builder /build/requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code and package definition
+COPY src/ /app/src/
+COPY config/ /app/config/
+COPY pyproject.toml README.md /app/
+
+# Install the package in editable mode
+RUN pip install --no-cache-dir -e /app
+
+# Create necessary directories
+RUN mkdir -p /app/logs
+
+# Create non-root user
+RUN useradd -m -u 1000 appuser && \
+    chown -R appuser:appuser /app
+
+USER appuser
+
+# Expose chat port
+EXPOSE 8001
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:8001/health || exit 1
+
+# Run the chat server
+CMD ["python", "-m", "datacenter_docs.chat.main"]
--- a/deploy/docker/Dockerfile.frontend
+++ b/deploy/docker/Dockerfile.frontend
@@ -0,0 +1,41 @@
+# Dockerfile for React Frontend
+# Build stage
+FROM node:20-alpine as builder
+
+WORKDIR /build
+
+# Copy package files
+COPY frontend/package*.json ./
+
+# Install dependencies
+RUN npm install
+
+# Copy frontend source code
+COPY frontend/src ./src
+COPY frontend/index.html ./
+COPY frontend/vite.config.js ./
+
+# Build the frontend
+RUN npm run build
+
+# Production stage with nginx
+FROM nginx:alpine
+
+LABEL maintainer="automation-team@company.com"
+LABEL description="Datacenter Documentation Frontend"
+
+# Copy built assets from builder
+COPY --from=builder /build/dist /usr/share/nginx/html
+
+# Copy nginx configuration
+COPY deploy/docker/nginx.conf /etc/nginx/conf.d/default.conf
+
+# Expose port
+EXPOSE 80
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD wget --no-verbose --tries=1 --spider http://localhost/health || exit 1
+
+# Run nginx
+CMD ["nginx", "-g", "daemon off;"]
--- a/deploy/docker/Dockerfile.worker
+++ b/deploy/docker/Dockerfile.worker
@@ -0,0 +1,57 @@
+# Dockerfile for Celery Worker Service
+FROM python:3.12-slim as builder
+
+WORKDIR /build
+
+# Install Poetry
+RUN pip install --no-cache-dir poetry==1.8.0
+
+# Copy dependency files
+COPY pyproject.toml poetry.lock ./
+
+# Export dependencies
+RUN poetry config virtualenvs.create false \
+    && poetry export -f requirements.txt --output requirements.txt --without-hashes
+
+# Runtime stage
+FROM python:3.12-slim
+
+LABEL maintainer="automation-team@company.com"
+LABEL description="Datacenter Documentation Background Worker"
+
+# Install system dependencies for network automation
+RUN apt-get update && apt-get install -y \
+    gcc \
+    libpq-dev \
+    openssh-client \
+    snmp \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy requirements from builder
+COPY --from=builder /build/requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code and package definition
+COPY src/ /app/src/
+COPY config/ /app/config/
+COPY pyproject.toml README.md /app/
+
+# Install the package in editable mode
+RUN pip install --no-cache-dir -e /app
+
+# Create necessary directories
+RUN mkdir -p /app/logs /app/output
+
+# Create non-root user
+RUN useradd -m -u 1000 appuser && \
+    chown -R appuser:appuser /app
+
+USER appuser
+
+# Run the Celery worker
+CMD ["celery", "-A", "datacenter_docs.workers.celery_app", "worker", "--loglevel=info", "--concurrency=4"]
--- a/deploy/docker/README.md
+++ b/deploy/docker/README.md
@@ -0,0 +1,121 @@
+# Docker Development Environment
+
+This directory contains Docker configurations for running the Datacenter Documentation System in development mode.
+
+## Prerequisites
+
+- Docker Engine 20.10+
+- Docker Compose V2
+- At least 4GB RAM available for Docker
+
+## Quick Start
+
+```bash
+# Start all services
+cd deploy/docker
+docker-compose -f docker-compose.dev.yml up -d
+
+# View logs
+docker-compose -f docker-compose.dev.yml logs -f
+
+# Stop all services
+docker-compose -f docker-compose.dev.yml down
+```
+
+## Environment Variables
+
+Create a `.env` file in the project root with:
+
+```env
+ANTHROPIC_API_KEY=your_api_key_here
+MCP_SERVER_URL=http://localhost:8001
+```
+
+## Services
+
+### Running Services
+
+| Service | Port | Description | Status |
+|---------|------|-------------|--------|
+| **API** | 8000 | FastAPI documentation server | ✅ Healthy |
+| **MongoDB** | 27017 | Database | ✅ Healthy |
+| **Redis** | 6379 | Cache & message broker | ✅ Healthy |
+| **Frontend** | 80 | React web interface | ⚠️ Running |
+| **Flower** | 5555 | Celery monitoring | ✅ Running |
+
+### Not Implemented Yet
+
+- **Chat Service** (port 8001) - WebSocket chat interface
+- **Worker Service** - Celery background workers
+
+These services are commented out in docker-compose.dev.yml and will be enabled when implemented.
+
+## Access Points
+
+- **API Documentation**: http://localhost:8000/docs
+- **API Health**: http://localhost:8000/health
+- **Frontend**: http://localhost
+- **Flower (Celery Monitor)**: http://localhost:5555
+- **MongoDB**: `mongodb://admin:admin123@localhost:27017`
+- **Redis**: `localhost:6379`
+
+## Build Individual Services
+
+```bash
+# Rebuild a specific service
+docker-compose -f docker-compose.dev.yml up --build -d api
+
+# View logs for a specific service
+docker-compose -f docker-compose.dev.yml logs -f api
+```
+
+## Troubleshooting
+
+### API not starting
+
+Check logs:
+```bash
+docker-compose -f docker-compose.dev.yml logs api
+```
+
+### MongoDB connection issues
+
+Ensure MongoDB is healthy:
+```bash
+docker-compose -f docker-compose.dev.yml ps mongodb
+```
+
+### Clear volumes and restart
+
+```bash
+docker-compose -f docker-compose.dev.yml down -v
+docker-compose -f docker-compose.dev.yml up --build -d
+```
+
+## Development Workflow
+
+1. **Code changes** are mounted as volumes, so changes to `src/` are reflected immediately
+2. **Restart services** after dependency changes:
+   ```bash
+   docker-compose -f docker-compose.dev.yml restart api
+   ```
+3. **Rebuild** after pyproject.toml changes:
+   ```bash
+   docker-compose -f docker-compose.dev.yml up --build -d api
+   ```
+
+## Files
+
+- `Dockerfile.api` - FastAPI service
+- `Dockerfile.chat` - Chat service (not yet implemented)
+- `Dockerfile.worker` - Celery worker (not yet implemented)
+- `Dockerfile.frontend` - React frontend with Nginx
+- `docker-compose.dev.yml` - Development orchestration
+- `nginx.conf` - Nginx configuration for frontend
+
+## Notes
+
+- Python version: 3.12
+- Black formatter uses Python 3.12 target
+- Services use Poetry for dependency management
+- Frontend uses Vite for building
--- a/deploy/docker/docker-compose.dev.yml
+++ b/deploy/docker/docker-compose.dev.yml
@@ -0,0 +1,177 @@
+version: '3.8'
+
+services:
+  # MongoDB Database
+  mongodb:
+    image: mongo:7-jammy
+    container_name: datacenter-docs-mongodb-dev
+    ports:
+      - "27017:27017"
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: admin
+      MONGO_INITDB_ROOT_PASSWORD: admin123
+      MONGO_INITDB_DATABASE: datacenter_docs
+    volumes:
+      - mongodb-data:/data/db
+      - mongodb-config:/data/configdb
+    networks:
+      - datacenter-network
+    healthcheck:
+      test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # Redis Cache & Message Broker
+  redis:
+    image: redis:7-alpine
+    container_name: datacenter-docs-redis-dev
+    ports:
+      - "6379:6379"
+    command: redis-server --appendonly yes
+    volumes:
+      - redis-data:/data
+    networks:
+      - datacenter-network
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # FastAPI API Service
+  api:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.api
+    container_name: datacenter-docs-api-dev
+    ports:
+      - "8000:8000"
+    environment:
+      - MONGODB_URL=mongodb://admin:admin123@mongodb:27017
+      - MONGODB_DATABASE=datacenter_docs
+      - REDIS_URL=redis://redis:6379
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
+      - MCP_SERVER_URL=${MCP_SERVER_URL:-http://localhost:8001}
+      - LOG_LEVEL=DEBUG
+    volumes:
+      - ../../src:/app/src
+      - ../../config:/app/config
+      - api-logs:/app/logs
+      - api-output:/app/output
+    depends_on:
+      mongodb:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    networks:
+      - datacenter-network
+    restart: unless-stopped
+
+  # Chat Service
+  chat:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.chat
+    container_name: datacenter-docs-chat-dev
+    ports:
+      - "8001:8001"
+    environment:
+      - MONGODB_URL=mongodb://admin:admin123@mongodb:27017
+      - MONGODB_DATABASE=datacenter_docs
+      - REDIS_URL=redis://redis:6379
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
+      - LOG_LEVEL=DEBUG
+    volumes:
+      - ../../src:/app/src
+      - ../../config:/app/config
+      - chat-logs:/app/logs
+    depends_on:
+      mongodb:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    networks:
+      - datacenter-network
+    restart: unless-stopped
+
+  # Celery Worker
+  worker:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.worker
+    container_name: datacenter-docs-worker-dev
+    environment:
+      - MONGODB_URL=mongodb://admin:admin123@mongodb:27017
+      - MONGODB_DATABASE=datacenter_docs
+      - REDIS_URL=redis://redis:6379
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
+      - LOG_LEVEL=DEBUG
+    volumes:
+      - ../../src:/app/src
+      - ../../config:/app/config
+      - worker-logs:/app/logs
+      - worker-output:/app/output
+    depends_on:
+      mongodb:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    networks:
+      - datacenter-network
+    restart: unless-stopped
+
+  # Flower - Celery Monitoring
+  flower:
+    image: mher/flower:2.0
+    container_name: datacenter-docs-flower-dev
+    ports:
+      - "5555:5555"
+    environment:
+      - CELERY_BROKER_URL=redis://redis:6379
+      - CELERY_RESULT_BACKEND=redis://redis:6379
+      - FLOWER_PORT=5555
+    depends_on:
+      - redis
+      - worker
+    networks:
+      - datacenter-network
+    restart: unless-stopped
+
+  # Frontend
+  frontend:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.frontend
+    container_name: datacenter-docs-frontend-dev
+    ports:
+      - "80:80"
+    depends_on:
+      - api
+      - chat
+    networks:
+      - datacenter-network
+    restart: unless-stopped
+
+volumes:
+  mongodb-data:
+    name: datacenter-docs-mongodb-data-dev
+  mongodb-config:
+    name: datacenter-docs-mongodb-config-dev
+  redis-data:
+    name: datacenter-docs-redis-data-dev
+  api-logs:
+    name: datacenter-docs-api-logs-dev
+  api-output:
+    name: datacenter-docs-api-output-dev
+  chat-logs:
+    name: datacenter-docs-chat-logs-dev
+  worker-logs:
+    name: datacenter-docs-worker-logs-dev
+  worker-output:
+    name: datacenter-docs-worker-output-dev
+
+networks:
+  datacenter-network:
+    name: datacenter-docs-network-dev
+    driver: bridge
--- a/deploy/docker/nginx.conf
+++ b/deploy/docker/nginx.conf
@@ -0,0 +1,61 @@
+server {
+    listen 80;
+    server_name _;
+
+    root /usr/share/nginx/html;
+    index index.html;
+
+    # Gzip compression
+    gzip on;
+    gzip_vary on;
+    gzip_min_length 1024;
+    gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/xml+rss application/json;
+
+    # Security headers
+    add_header X-Frame-Options "SAMEORIGIN" always;
+    add_header X-Content-Type-Options "nosniff" always;
+    add_header X-XSS-Protection "1; mode=block" always;
+
+    # Health check endpoint
+    location /health {
+        access_log off;
+        return 200 "OK\n";
+        add_header Content-Type text/plain;
+    }
+
+    # API proxy
+    location /api/ {
+        proxy_pass http://api:8000/;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection 'upgrade';
+        proxy_set_header Host $host;
+        proxy_cache_bypass $http_upgrade;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    # WebSocket for chat
+    location /ws/ {
+        proxy_pass http://chat:8001/;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+
+    # React app - all routes go to index.html
+    location / {
+        try_files $uri $uri/ /index.html;
+    }
+
+    # Cache static assets
+    location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ {
+        expires 1y;
+        add_header Cache-Control "public, immutable";
+    }
+}