268214d971
- Migrated from HIVE branding to WHOOSH across all components - Enhanced backend API with new services: AI models, BZZZ integration, templates, members - Added comprehensive testing suite with security, performance, and integration tests - Improved frontend with new components for project setup, AI models, and team management - Updated MCP server implementation with WHOOSH-specific tools and resources - Enhanced deployment configurations with production-ready Docker setups - Added comprehensive documentation and setup guides - Implemented age encryption service and UCXL integration 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
1165 lines
36 KiB
Python
1165 lines
36 KiB
Python
"""
|
|
Project Template Service for WHOOSH - Advanced project template management with starter files.
|
|
"""
|
|
import os
|
|
import json
|
|
import shutil
|
|
import tempfile
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Any
|
|
from datetime import datetime
|
|
import zipfile
|
|
import yaml
|
|
|
|
from app.services.gitea_service import GiteaService
|
|
|
|
|
|
class ProjectTemplateService:
|
|
"""
|
|
Advanced project template service for WHOOSH.
|
|
Manages project templates, starter files, and automated project setup.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.gitea_service = GiteaService()
|
|
self.templates_path = Path("/home/tony/chorus/project-queues/active/WHOOSH/backend/templates")
|
|
self.templates_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Initialize built-in templates
|
|
self._init_builtin_templates()
|
|
|
|
def _init_builtin_templates(self):
|
|
"""Initialize built-in project templates with comprehensive configurations."""
|
|
|
|
# Full-Stack Web Application Template
|
|
self._create_fullstack_template()
|
|
|
|
# Simple React + FastAPI Template
|
|
self._create_react_fastapi_template()
|
|
|
|
# TODO: Enable other templates as their implementations are completed
|
|
# self._create_ai_research_template()
|
|
# self._create_microservices_template()
|
|
# self._create_devops_template()
|
|
# self._create_docs_template()
|
|
# self._create_mobile_template()
|
|
# self._create_blockchain_template()
|
|
|
|
def _create_fullstack_template(self):
|
|
"""Create comprehensive full-stack web application template."""
|
|
template_id = "fullstack-web-app"
|
|
template_dir = self.templates_path / template_id
|
|
template_dir.mkdir(exist_ok=True)
|
|
|
|
# Template metadata
|
|
metadata = {
|
|
"template_id": template_id,
|
|
"name": "Full-Stack Web Application",
|
|
"description": "Complete web application with React frontend, Node.js/FastAPI backend, PostgreSQL database, and Docker deployment",
|
|
"icon": "🌐",
|
|
"category": "web-development",
|
|
"tags": ["react", "nodejs", "fastapi", "postgresql", "docker", "typescript"],
|
|
"difficulty": "intermediate",
|
|
"estimated_setup_time": "15-30 minutes",
|
|
"features": [
|
|
"React 18 with TypeScript",
|
|
"Node.js/Express or Python/FastAPI backend options",
|
|
"PostgreSQL database with migrations",
|
|
"Docker containerization",
|
|
"CI/CD with GitHub Actions",
|
|
"Authentication & authorization",
|
|
"API documentation with OpenAPI/Swagger",
|
|
"Testing setup (Jest, Pytest)",
|
|
"ESLint & Prettier configuration",
|
|
"Environment management"
|
|
],
|
|
"tech_stack": {
|
|
"frontend": ["React", "TypeScript", "Tailwind CSS", "React Query"],
|
|
"backend": ["Node.js/Express", "Python/FastAPI"],
|
|
"database": ["PostgreSQL", "Redis"],
|
|
"deployment": ["Docker", "Docker Compose"],
|
|
"testing": ["Jest", "Pytest", "Cypress"],
|
|
"ci_cd": ["GitHub Actions", "Docker Hub"]
|
|
},
|
|
"requirements": {
|
|
"nodejs": ">=18.0.0",
|
|
"python": ">=3.9.0",
|
|
"docker": ">=20.0.0",
|
|
"postgresql": ">=13.0"
|
|
}
|
|
}
|
|
|
|
# Starter files structure
|
|
starter_files = {
|
|
# Root configuration files
|
|
".gitignore": self._get_fullstack_gitignore(),
|
|
"README.md": self._get_fullstack_readme(),
|
|
"docker-compose.yml": self._get_fullstack_docker_compose(),
|
|
"docker-compose.prod.yml": self._get_fullstack_docker_compose_prod(),
|
|
".env.example": self._get_fullstack_env_example(),
|
|
|
|
# Frontend structure
|
|
"frontend/package.json": self._get_react_package_json(),
|
|
"frontend/tsconfig.json": self._get_react_tsconfig(),
|
|
"frontend/tailwind.config.js": self._get_tailwind_config(),
|
|
"frontend/src/App.tsx": self._get_react_app_tsx(),
|
|
"frontend/src/index.tsx": self._get_react_index_tsx(),
|
|
"frontend/src/components/Layout.tsx": self._get_react_layout(),
|
|
"frontend/src/pages/Home.tsx": self._get_react_home_page(),
|
|
"frontend/src/services/api.ts": self._get_react_api_service(),
|
|
"frontend/src/hooks/useAuth.ts": self._get_react_auth_hook(),
|
|
"frontend/Dockerfile": self._get_react_dockerfile(),
|
|
|
|
# Backend structure (FastAPI)
|
|
"backend/requirements.txt": self._get_fastapi_requirements(),
|
|
"backend/pyproject.toml": self._get_fastapi_pyproject(),
|
|
"backend/app/main.py": self._get_fastapi_main(),
|
|
"backend/app/core/config.py": self._get_fastapi_config(),
|
|
"backend/app/core/database.py": self._get_fastapi_database(),
|
|
"backend/app/api/auth.py": self._get_fastapi_auth(),
|
|
"backend/app/api/users.py": self._get_fastapi_users(),
|
|
"backend/app/models/user.py": self._get_fastapi_user_model(),
|
|
"backend/app/schemas/user.py": self._get_fastapi_user_schema(),
|
|
"backend/Dockerfile": self._get_fastapi_dockerfile(),
|
|
"backend/alembic.ini": self._get_alembic_config(),
|
|
"backend/alembic/env.py": self._get_alembic_env(),
|
|
|
|
# Database migrations
|
|
"database/init.sql": self._get_postgres_init_sql(),
|
|
|
|
# CI/CD
|
|
".github/workflows/ci.yml": self._get_github_actions_ci(),
|
|
".github/workflows/deploy.yml": self._get_github_actions_deploy(),
|
|
|
|
# Testing
|
|
"frontend/src/__tests__/App.test.tsx": self._get_react_test(),
|
|
"backend/tests/test_main.py": self._get_fastapi_test(),
|
|
|
|
# Documentation
|
|
"docs/SETUP.md": self._get_setup_documentation(),
|
|
"docs/API.md": self._get_api_documentation(),
|
|
"docs/DEPLOYMENT.md": self._get_deployment_documentation()
|
|
}
|
|
|
|
# Save template
|
|
self._save_template(template_id, metadata, starter_files)
|
|
|
|
def _create_ai_research_template(self):
|
|
"""Create AI/ML research project template."""
|
|
template_id = "ai-ml-research"
|
|
template_dir = self.templates_path / template_id
|
|
template_dir.mkdir(exist_ok=True)
|
|
|
|
metadata = {
|
|
"template_id": template_id,
|
|
"name": "AI/ML Research Project",
|
|
"description": "Machine learning research environment with Jupyter notebooks, experiment tracking, and model deployment",
|
|
"icon": "🤖",
|
|
"category": "data-science",
|
|
"tags": ["python", "jupyter", "pytorch", "tensorflow", "mlflow", "docker"],
|
|
"difficulty": "advanced",
|
|
"estimated_setup_time": "10-20 minutes",
|
|
"features": [
|
|
"Jupyter Lab environment",
|
|
"PyTorch & TensorFlow support",
|
|
"MLflow experiment tracking",
|
|
"DVC for data versioning",
|
|
"Model serving with FastAPI",
|
|
"GPU support configuration",
|
|
"Weights & Biases integration",
|
|
"Data pipeline automation",
|
|
"Model evaluation frameworks",
|
|
"Reproducible research setup"
|
|
],
|
|
"tech_stack": {
|
|
"ml_frameworks": ["PyTorch", "TensorFlow", "Scikit-learn"],
|
|
"experiment_tracking": ["MLflow", "Weights & Biases"],
|
|
"data_tools": ["Pandas", "NumPy", "DVC"],
|
|
"visualization": ["Matplotlib", "Plotly", "Seaborn"],
|
|
"deployment": ["FastAPI", "Docker", "Kubernetes"],
|
|
"notebooks": ["Jupyter Lab", "Papermill"]
|
|
}
|
|
}
|
|
|
|
starter_files = {
|
|
".gitignore": self._get_ml_gitignore(),
|
|
"README.md": self._get_ml_readme(),
|
|
"requirements.txt": self._get_ml_requirements(),
|
|
"environment.yml": self._get_conda_environment(),
|
|
"pyproject.toml": self._get_ml_pyproject(),
|
|
"docker-compose.yml": self._get_ml_docker_compose(),
|
|
"Dockerfile": self._get_ml_dockerfile(),
|
|
"dvc.yaml": self._get_dvc_pipeline(),
|
|
".dvcignore": self._get_dvc_ignore(),
|
|
|
|
# Notebook structure
|
|
"notebooks/01_data_exploration.ipynb": self._get_exploration_notebook(),
|
|
"notebooks/02_data_preprocessing.ipynb": self._get_preprocessing_notebook(),
|
|
"notebooks/03_model_training.ipynb": self._get_training_notebook(),
|
|
"notebooks/04_model_evaluation.ipynb": self._get_evaluation_notebook(),
|
|
|
|
# Source code structure
|
|
"src/data/loader.py": self._get_data_loader(),
|
|
"src/models/base_model.py": self._get_base_model(),
|
|
"src/training/trainer.py": self._get_model_trainer(),
|
|
"src/evaluation/metrics.py": self._get_evaluation_metrics(),
|
|
"src/api/model_server.py": self._get_model_server(),
|
|
|
|
# Configuration
|
|
"config/model_config.yaml": self._get_model_config(),
|
|
"config/data_config.yaml": self._get_data_config(),
|
|
|
|
# Scripts
|
|
"scripts/download_data.py": self._get_data_download_script(),
|
|
"scripts/train_model.py": self._get_training_script(),
|
|
"scripts/evaluate_model.py": self._get_evaluation_script(),
|
|
|
|
# MLflow setup
|
|
"mlflow/MLproject": self._get_mlflow_project(),
|
|
"mlflow/conda.yaml": self._get_mlflow_conda(),
|
|
|
|
# Documentation
|
|
"docs/RESEARCH.md": self._get_research_documentation(),
|
|
"docs/MODEL_CARDS.md": self._get_model_cards_template()
|
|
}
|
|
|
|
self._save_template(template_id, metadata, starter_files)
|
|
|
|
def _create_microservices_template(self):
|
|
"""Create microservices architecture template."""
|
|
template_id = "microservices-architecture"
|
|
|
|
metadata = {
|
|
"template_id": template_id,
|
|
"name": "Microservices Architecture",
|
|
"description": "Distributed microservices system with API Gateway, service discovery, and monitoring",
|
|
"icon": "🔧",
|
|
"category": "architecture",
|
|
"tags": ["microservices", "docker", "kubernetes", "api-gateway", "monitoring"],
|
|
"difficulty": "advanced",
|
|
"estimated_setup_time": "30-45 minutes",
|
|
"features": [
|
|
"Multiple service templates",
|
|
"API Gateway with Kong/Nginx",
|
|
"Service discovery with Consul",
|
|
"Monitoring with Prometheus & Grafana",
|
|
"Distributed logging with ELK stack",
|
|
"Circuit breaker patterns",
|
|
"Health checks and metrics",
|
|
"Inter-service communication",
|
|
"Database per service pattern",
|
|
"Event-driven architecture"
|
|
]
|
|
}
|
|
|
|
starter_files = {
|
|
"README.md": self._get_microservices_readme(),
|
|
"docker-compose.yml": self._get_microservices_docker_compose(),
|
|
"kubernetes/namespace.yaml": self._get_k8s_namespace(),
|
|
"api-gateway/kong.yml": self._get_kong_config(),
|
|
"monitoring/prometheus.yml": self._get_prometheus_config(),
|
|
"monitoring/grafana/dashboards/services.json": self._get_grafana_dashboard(),
|
|
|
|
# User Service
|
|
"services/user-service/Dockerfile": self._get_service_dockerfile("user"),
|
|
"services/user-service/main.py": self._get_service_main("user"),
|
|
"services/user-service/requirements.txt": self._get_service_requirements(),
|
|
|
|
# Product Service
|
|
"services/product-service/Dockerfile": self._get_service_dockerfile("product"),
|
|
"services/product-service/main.py": self._get_service_main("product"),
|
|
"services/product-service/requirements.txt": self._get_service_requirements(),
|
|
|
|
# Order Service
|
|
"services/order-service/Dockerfile": self._get_service_dockerfile("order"),
|
|
"services/order-service/main.py": self._get_service_main("order"),
|
|
"services/order-service/requirements.txt": self._get_service_requirements(),
|
|
|
|
# Shared libraries
|
|
"shared/auth/auth_middleware.py": self._get_auth_middleware(),
|
|
"shared/monitoring/health_check.py": self._get_health_check(),
|
|
"shared/database/base.py": self._get_database_base()
|
|
}
|
|
|
|
self._save_template(template_id, metadata, starter_files)
|
|
|
|
def _create_react_fastapi_template(self):
|
|
"""Create React + FastAPI specific template."""
|
|
template_id = "react-fastapi"
|
|
|
|
metadata = {
|
|
"template_id": template_id,
|
|
"name": "React + FastAPI",
|
|
"description": "Modern web application with React frontend and FastAPI backend",
|
|
"icon": "⚛️",
|
|
"category": "web-development",
|
|
"tags": ["react", "fastapi", "typescript", "python"],
|
|
"difficulty": "beginner",
|
|
"estimated_setup_time": "10-15 minutes",
|
|
"features": [
|
|
"React 18 with TypeScript",
|
|
"FastAPI with automatic OpenAPI docs",
|
|
"JWT authentication",
|
|
"Real-time updates with WebSockets",
|
|
"Database integration with SQLAlchemy",
|
|
"Testing with Jest and Pytest",
|
|
"Docker development environment"
|
|
]
|
|
}
|
|
|
|
# Similar to fullstack but more focused
|
|
starter_files = {
|
|
"README.md": self._get_react_fastapi_readme(),
|
|
"docker-compose.yml": self._get_simple_docker_compose(),
|
|
# ... simplified structure
|
|
}
|
|
|
|
self._save_template(template_id, metadata, starter_files)
|
|
|
|
def _save_template(self, template_id: str, metadata: Dict[str, Any], starter_files: Dict[str, str]):
|
|
"""Save template metadata and starter files."""
|
|
template_dir = self.templates_path / template_id
|
|
template_dir.mkdir(exist_ok=True)
|
|
|
|
# Save metadata
|
|
metadata_file = template_dir / "template.json"
|
|
metadata_file.write_text(json.dumps(metadata, indent=2))
|
|
|
|
# Save starter files
|
|
for file_path, content in starter_files.items():
|
|
full_path = template_dir / "files" / file_path
|
|
full_path.parent.mkdir(parents=True, exist_ok=True)
|
|
full_path.write_text(content)
|
|
|
|
print(f"✅ Template '{template_id}' saved successfully")
|
|
|
|
# File content generators (a selection of key files)
|
|
|
|
def _get_fullstack_gitignore(self) -> str:
|
|
return """# Dependencies
|
|
node_modules/
|
|
__pycache__/
|
|
*.pyc
|
|
venv/
|
|
.venv/
|
|
|
|
# Environment files
|
|
.env
|
|
.env.local
|
|
.env.production
|
|
|
|
# Build outputs
|
|
build/
|
|
dist/
|
|
*.egg-info/
|
|
|
|
# Database
|
|
*.db
|
|
*.sqlite
|
|
|
|
# IDE
|
|
.vscode/
|
|
.idea/
|
|
*.swp
|
|
*.swo
|
|
|
|
# OS
|
|
.DS_Store
|
|
Thumbs.db
|
|
|
|
# Docker
|
|
.dockerignore
|
|
|
|
# Logs
|
|
*.log
|
|
logs/
|
|
|
|
# Test coverage
|
|
coverage/
|
|
.coverage
|
|
.pytest_cache/
|
|
"""
|
|
|
|
def _get_fullstack_readme(self) -> str:
|
|
return """# Full-Stack Web Application
|
|
|
|
A modern full-stack web application built with React, FastAPI, and PostgreSQL.
|
|
|
|
## Features
|
|
|
|
- 🎯 **React 18** with TypeScript for the frontend
|
|
- 🚀 **FastAPI** for high-performance backend API
|
|
- 🗄️ **PostgreSQL** database with SQLAlchemy ORM
|
|
- 🐳 **Docker** containerization for development and production
|
|
- 🔐 **JWT Authentication** and authorization
|
|
- 📚 **Automatic API documentation** with OpenAPI/Swagger
|
|
- ✅ **Comprehensive testing** setup
|
|
- 🎨 **Tailwind CSS** for beautiful, responsive UI
|
|
- 📱 **Mobile-first** responsive design
|
|
|
|
## Quick Start
|
|
|
|
### Prerequisites
|
|
|
|
- Docker and Docker Compose
|
|
- Node.js 18+ (for local development)
|
|
- Python 3.9+ (for local development)
|
|
|
|
### Development Setup
|
|
|
|
1. **Clone and setup environment:**
|
|
```bash
|
|
cp .env.example .env
|
|
# Edit .env with your configuration
|
|
```
|
|
|
|
2. **Start development environment:**
|
|
```bash
|
|
docker-compose up -d
|
|
```
|
|
|
|
3. **Access the application:**
|
|
- Frontend: http://localhost:3000
|
|
- Backend API: http://localhost:8000
|
|
- API Documentation: http://localhost:8000/docs
|
|
- Database: localhost:5432
|
|
|
|
### Local Development
|
|
|
|
**Frontend:**
|
|
```bash
|
|
cd frontend
|
|
npm install
|
|
npm start
|
|
```
|
|
|
|
**Backend:**
|
|
```bash
|
|
cd backend
|
|
python -m venv venv
|
|
source venv/bin/activate
|
|
pip install -r requirements.txt
|
|
uvicorn app.main:app --reload
|
|
```
|
|
|
|
## Project Structure
|
|
|
|
```
|
|
├── frontend/ # React TypeScript frontend
|
|
│ ├── src/
|
|
│ │ ├── components/
|
|
│ │ ├── pages/
|
|
│ │ ├── services/
|
|
│ │ └── hooks/
|
|
│ └── package.json
|
|
├── backend/ # FastAPI backend
|
|
│ ├── app/
|
|
│ │ ├── api/
|
|
│ │ ├── core/
|
|
│ │ ├── models/
|
|
│ │ └── schemas/
|
|
│ └── requirements.txt
|
|
├── database/ # Database initialization
|
|
├── docs/ # Documentation
|
|
└── docker-compose.yml
|
|
```
|
|
|
|
## API Documentation
|
|
|
|
The API is automatically documented using OpenAPI/Swagger. Access the interactive documentation at:
|
|
- **Swagger UI:** http://localhost:8000/docs
|
|
- **ReDoc:** http://localhost:8000/redoc
|
|
|
|
## Testing
|
|
|
|
**Frontend tests:**
|
|
```bash
|
|
cd frontend
|
|
npm test
|
|
```
|
|
|
|
**Backend tests:**
|
|
```bash
|
|
cd backend
|
|
pytest
|
|
```
|
|
|
|
## Deployment
|
|
|
|
### Production Deployment
|
|
|
|
1. **Build production images:**
|
|
```bash
|
|
docker-compose -f docker-compose.prod.yml build
|
|
```
|
|
|
|
2. **Deploy to production:**
|
|
```bash
|
|
docker-compose -f docker-compose.prod.yml up -d
|
|
```
|
|
|
|
### Environment Variables
|
|
|
|
Key environment variables (see `.env.example`):
|
|
|
|
- `DATABASE_URL`: PostgreSQL connection string
|
|
- `SECRET_KEY`: JWT secret key
|
|
- `CORS_ORIGINS`: Allowed CORS origins
|
|
- `ENVIRONMENT`: Development/production environment
|
|
|
|
## Contributing
|
|
|
|
1. Fork the repository
|
|
2. Create a feature branch
|
|
3. Make your changes
|
|
4. Add tests for new functionality
|
|
5. Submit a pull request
|
|
|
|
## License
|
|
|
|
This project is licensed under the MIT License - see the LICENSE file for details.
|
|
"""
|
|
|
|
def _get_fullstack_docker_compose(self) -> str:
|
|
return """version: '3.8'
|
|
|
|
services:
|
|
frontend:
|
|
build:
|
|
context: ./frontend
|
|
dockerfile: Dockerfile
|
|
ports:
|
|
- "3000:3000"
|
|
environment:
|
|
- REACT_APP_API_URL=http://localhost:8000
|
|
volumes:
|
|
- ./frontend:/app
|
|
- /app/node_modules
|
|
depends_on:
|
|
- backend
|
|
|
|
backend:
|
|
build:
|
|
context: ./backend
|
|
dockerfile: Dockerfile
|
|
ports:
|
|
- "8000:8000"
|
|
environment:
|
|
- DATABASE_URL=postgresql://whoosh:password@postgres:5432/whoosh_db
|
|
- SECRET_KEY=your-secret-key-change-in-production
|
|
- CORS_ORIGINS=http://localhost:3000
|
|
volumes:
|
|
- ./backend:/app
|
|
depends_on:
|
|
- postgres
|
|
- redis
|
|
|
|
postgres:
|
|
image: postgres:15
|
|
environment:
|
|
- POSTGRES_USER=whoosh
|
|
- POSTGRES_PASSWORD=password
|
|
- POSTGRES_DB=whoosh_db
|
|
ports:
|
|
- "5432:5432"
|
|
volumes:
|
|
- postgres_data:/var/lib/postgresql/data
|
|
- ./database/init.sql:/docker-entrypoint-initdb.d/init.sql
|
|
|
|
redis:
|
|
image: redis:7-alpine
|
|
ports:
|
|
- "6379:6379"
|
|
volumes:
|
|
- redis_data:/data
|
|
|
|
nginx:
|
|
image: nginx:alpine
|
|
ports:
|
|
- "80:80"
|
|
volumes:
|
|
- ./nginx/nginx.conf:/etc/nginx/nginx.conf
|
|
depends_on:
|
|
- frontend
|
|
- backend
|
|
|
|
volumes:
|
|
postgres_data:
|
|
redis_data:
|
|
"""
|
|
|
|
def _get_react_package_json(self) -> str:
|
|
return """{
|
|
"name": "whoosh-frontend",
|
|
"version": "1.0.0",
|
|
"private": true,
|
|
"dependencies": {
|
|
"@types/node": "^20.0.0",
|
|
"@types/react": "^18.2.0",
|
|
"@types/react-dom": "^18.2.0",
|
|
"react": "^18.2.0",
|
|
"react-dom": "^18.2.0",
|
|
"react-router-dom": "^6.8.0",
|
|
"react-query": "^3.39.0",
|
|
"axios": "^1.3.0",
|
|
"typescript": "^5.0.0",
|
|
"@headlessui/react": "^1.7.0",
|
|
"@heroicons/react": "^2.0.0",
|
|
"tailwindcss": "^3.2.0",
|
|
"autoprefixer": "^10.4.0",
|
|
"postcss": "^8.4.0"
|
|
},
|
|
"scripts": {
|
|
"start": "react-scripts start",
|
|
"build": "react-scripts build",
|
|
"test": "react-scripts test",
|
|
"eject": "react-scripts eject"
|
|
},
|
|
"eslintConfig": {
|
|
"extends": [
|
|
"react-app",
|
|
"react-app/jest"
|
|
]
|
|
},
|
|
"browserslist": {
|
|
"production": [
|
|
">0.2%",
|
|
"not dead",
|
|
"not op_mini all"
|
|
],
|
|
"development": [
|
|
"last 1 chrome version",
|
|
"last 1 firefox version",
|
|
"last 1 safari version"
|
|
]
|
|
},
|
|
"devDependencies": {
|
|
"@testing-library/jest-dom": "^5.16.0",
|
|
"@testing-library/react": "^14.0.0",
|
|
"@testing-library/user-event": "^14.4.0",
|
|
"react-scripts": "5.0.1"
|
|
}
|
|
}"""
|
|
|
|
def _get_fastapi_main(self) -> str:
|
|
return """from fastapi import FastAPI, Depends, HTTPException
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from sqlalchemy.orm import Session
|
|
import os
|
|
|
|
from app.core.config import settings
|
|
from app.core.database import engine, get_db
|
|
from app.api import auth, users
|
|
from app.models import user
|
|
|
|
# Create database tables
|
|
user.Base.metadata.create_all(bind=engine)
|
|
|
|
app = FastAPI(
|
|
title="WHOOSH API",
|
|
description="Full-stack application backend API",
|
|
version="1.0.0",
|
|
docs_url="/docs",
|
|
redoc_url="/redoc"
|
|
)
|
|
|
|
# Add CORS middleware
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=settings.CORS_ORIGINS,
|
|
allow_credentials=True,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
# Include routers
|
|
app.include_router(auth.router, prefix="/auth", tags=["authentication"])
|
|
app.include_router(users.router, prefix="/users", tags=["users"])
|
|
|
|
@app.get("/")
|
|
async def root():
|
|
return {
|
|
"message": "Welcome to WHOOSH API",
|
|
"version": "1.0.0",
|
|
"docs": "/docs"
|
|
}
|
|
|
|
@app.get("/health")
|
|
async def health_check(db: Session = Depends(get_db)):
|
|
return {
|
|
"status": "healthy",
|
|
"database": "connected"
|
|
}
|
|
"""
|
|
|
|
def list_templates(self) -> List[Dict[str, Any]]:
|
|
"""List all available project templates."""
|
|
templates = []
|
|
|
|
for template_dir in self.templates_path.iterdir():
|
|
if template_dir.is_dir():
|
|
metadata_file = template_dir / "template.json"
|
|
if metadata_file.exists():
|
|
try:
|
|
metadata = json.loads(metadata_file.read_text())
|
|
templates.append(metadata)
|
|
except Exception as e:
|
|
print(f"Error loading template {template_dir.name}: {e}")
|
|
|
|
return sorted(templates, key=lambda x: x.get("name", ""))
|
|
|
|
def get_template(self, template_id: str) -> Optional[Dict[str, Any]]:
|
|
"""Get specific template metadata and files."""
|
|
template_dir = self.templates_path / template_id
|
|
metadata_file = template_dir / "template.json"
|
|
|
|
if not metadata_file.exists():
|
|
return None
|
|
|
|
try:
|
|
metadata = json.loads(metadata_file.read_text())
|
|
|
|
# Load starter files
|
|
files_dir = template_dir / "files"
|
|
starter_files = {}
|
|
|
|
if files_dir.exists():
|
|
for file_path in files_dir.rglob("*"):
|
|
if file_path.is_file():
|
|
relative_path = file_path.relative_to(files_dir)
|
|
starter_files[str(relative_path)] = file_path.read_text()
|
|
|
|
return {
|
|
"metadata": metadata,
|
|
"starter_files": starter_files
|
|
}
|
|
|
|
except Exception as e:
|
|
print(f"Error loading template {template_id}: {e}")
|
|
return None
|
|
|
|
def create_project_from_template(self, template_id: str, project_data: Dict[str, Any],
|
|
target_directory: str) -> Dict[str, Any]:
|
|
"""Create a new project from a template."""
|
|
template = self.get_template(template_id)
|
|
if not template:
|
|
raise ValueError(f"Template {template_id} not found")
|
|
|
|
target_path = Path(target_directory)
|
|
target_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Process template variables
|
|
variables = {
|
|
"project_name": project_data.get("name", "my-project"),
|
|
"project_description": project_data.get("description", ""),
|
|
"author_name": project_data.get("author", "WHOOSH User"),
|
|
"current_year": str(datetime.now().year)
|
|
}
|
|
|
|
created_files = []
|
|
|
|
# Create files from template
|
|
for file_path, content in template["starter_files"].items():
|
|
# Process template variables in content
|
|
processed_content = self._process_template_variables(content, variables)
|
|
|
|
# Create file
|
|
full_path = target_path / file_path
|
|
full_path.parent.mkdir(parents=True, exist_ok=True)
|
|
full_path.write_text(processed_content)
|
|
created_files.append(str(file_path))
|
|
|
|
return {
|
|
"template_id": template_id,
|
|
"project_path": str(target_path),
|
|
"files_created": created_files,
|
|
"next_steps": template["metadata"].get("next_steps", [])
|
|
}
|
|
|
|
def _process_template_variables(self, content: str, variables: Dict[str, str]) -> str:
|
|
"""Process template variables in file content."""
|
|
for key, value in variables.items():
|
|
content = content.replace(f"{{{{ {key} }}}}", value)
|
|
content = content.replace(f"{{{{WHOOSH_{key.upper()}}}}}", value)
|
|
|
|
return content
|
|
|
|
# Additional file generators for other templates...
|
|
def _get_ml_gitignore(self) -> str:
|
|
return """# Data files
|
|
*.csv
|
|
*.h5
|
|
*.pkl
|
|
*.npz
|
|
data/raw/
|
|
data/processed/
|
|
models/trained/
|
|
|
|
# ML artifacts
|
|
mlruns/
|
|
wandb/
|
|
.mlflow/
|
|
|
|
# Jupyter
|
|
.ipynb_checkpoints/
|
|
*.ipynb
|
|
|
|
# Python
|
|
__pycache__/
|
|
*.pyc
|
|
*.pyo
|
|
*.pyd
|
|
.Python
|
|
env/
|
|
venv/
|
|
.venv/
|
|
|
|
# IDE
|
|
.vscode/
|
|
.idea/
|
|
|
|
# OS
|
|
.DS_Store
|
|
Thumbs.db
|
|
"""
|
|
|
|
def _get_ml_requirements(self) -> str:
|
|
return """# Core ML libraries
|
|
torch>=2.0.0
|
|
tensorflow>=2.12.0
|
|
scikit-learn>=1.3.0
|
|
numpy>=1.24.0
|
|
pandas>=2.0.0
|
|
|
|
# Data processing
|
|
scipy>=1.10.0
|
|
matplotlib>=3.7.0
|
|
seaborn>=0.12.0
|
|
plotly>=5.14.0
|
|
|
|
# Experiment tracking
|
|
mlflow>=2.3.0
|
|
wandb>=0.15.0
|
|
|
|
# Jupyter and notebook tools
|
|
jupyterlab>=4.0.0
|
|
ipywidgets>=8.0.0
|
|
papermill>=2.4.0
|
|
|
|
# Development tools
|
|
pytest>=7.3.0
|
|
black>=23.3.0
|
|
flake8>=6.0.0
|
|
isort>=5.12.0
|
|
|
|
# API serving
|
|
fastapi>=0.95.0
|
|
uvicorn>=0.22.0
|
|
pydantic>=1.10.0
|
|
|
|
# Data versioning
|
|
dvc>=3.0.0
|
|
dvc[s3]>=3.0.0
|
|
|
|
# GPU acceleration (optional)
|
|
# torch-audio>=2.0.0
|
|
# torch-vision>=0.15.0
|
|
"""
|
|
|
|
def _get_exploration_notebook(self) -> str:
|
|
return """{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Data Exploration\\n",
|
|
"\\n",
|
|
"This notebook contains exploratory data analysis for the project."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import pandas as pd\\n",
|
|
"import numpy as np\\n",
|
|
"import matplotlib.pyplot as plt\\n",
|
|
"import seaborn as sns\\n",
|
|
"\\n",
|
|
"# Set style\\n",
|
|
"plt.style.use('seaborn-v0_8')\\n",
|
|
"sns.set_palette('husl')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Load Data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Load your dataset here\\n",
|
|
"# df = pd.read_csv('data/raw/dataset.csv')\\n",
|
|
"# print(f'Dataset shape: {df.shape}')"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"name": "python",
|
|
"version": "3.9.0"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 4
|
|
}"""
|
|
|
|
# ML/AI Research template file generators
|
|
def _get_ml_readme(self) -> str:
|
|
return """# AI/ML Research Project
|
|
|
|
A comprehensive machine learning research environment with experiment tracking and model deployment.
|
|
|
|
## Features
|
|
|
|
- 🧠 **PyTorch & TensorFlow** support for deep learning
|
|
- 📊 **MLflow** experiment tracking and model registry
|
|
- 📈 **Weights & Biases** integration for advanced monitoring
|
|
- 🔄 **DVC** for data versioning and pipeline management
|
|
- 📓 **Jupyter Lab** for interactive development
|
|
- 🚀 **FastAPI** model serving
|
|
- 🐳 **Docker** containerization with GPU support
|
|
- 📋 **Model cards** and research documentation
|
|
|
|
## Quick Start
|
|
|
|
### Prerequisites
|
|
|
|
- Docker and Docker Compose
|
|
- Python 3.9+
|
|
- CUDA drivers (for GPU support)
|
|
|
|
### Setup
|
|
|
|
1. **Clone and setup environment:**
|
|
```bash
|
|
conda env create -f environment.yml
|
|
conda activate ml-research
|
|
# OR
|
|
pip install -r requirements.txt
|
|
```
|
|
|
|
2. **Start development environment:**
|
|
```bash
|
|
docker-compose up -d
|
|
```
|
|
|
|
3. **Launch Jupyter Lab:**
|
|
```bash
|
|
jupyter lab
|
|
```
|
|
|
|
4. **Access services:**
|
|
- Jupyter Lab: http://localhost:8888
|
|
- MLflow UI: http://localhost:5000
|
|
- Model API: http://localhost:8080
|
|
|
|
## Project Structure
|
|
|
|
```
|
|
├── notebooks/ # Jupyter notebooks for exploration and analysis
|
|
│ ├── 01_data_exploration.ipynb
|
|
│ ├── 02_data_preprocessing.ipynb
|
|
│ ├── 03_model_training.ipynb
|
|
│ └── 04_model_evaluation.ipynb
|
|
├── src/ # Source code modules
|
|
│ ├── data/ # Data loading and processing
|
|
│ ├── models/ # Model definitions and training
|
|
│ ├── evaluation/ # Metrics and evaluation
|
|
│ └── api/ # Model serving API
|
|
├── config/ # Configuration files
|
|
├── scripts/ # Automation scripts
|
|
├── data/ # Data storage (gitignored)
|
|
├── models/ # Trained models (gitignored)
|
|
└── mlruns/ # MLflow tracking (gitignored)
|
|
```
|
|
|
|
## Experiment Tracking
|
|
|
|
This project uses MLflow for experiment tracking:
|
|
|
|
```python
|
|
import mlflow
|
|
import mlflow.pytorch
|
|
|
|
# Start a new run
|
|
with mlflow.start_run():
|
|
# Log parameters
|
|
mlflow.log_param("learning_rate", 0.001)
|
|
mlflow.log_param("batch_size", 32)
|
|
|
|
# Train your model
|
|
model = train_model(lr=0.001, batch_size=32)
|
|
|
|
# Log metrics
|
|
mlflow.log_metric("accuracy", accuracy)
|
|
mlflow.log_metric("loss", loss)
|
|
|
|
# Log model
|
|
mlflow.pytorch.log_model(model, "model")
|
|
```
|
|
|
|
## Data Versioning with DVC
|
|
|
|
Track data and model versions:
|
|
|
|
```bash
|
|
# Add data to DVC
|
|
dvc add data/raw/dataset.csv
|
|
|
|
# Create data pipeline
|
|
dvc run -d data/raw -o data/processed \\
|
|
python src/data/preprocess.py
|
|
|
|
# Reproduce pipeline
|
|
dvc repro
|
|
```
|
|
|
|
## Model Serving
|
|
|
|
Deploy models with FastAPI:
|
|
|
|
```bash
|
|
# Start model server
|
|
python src/api/model_server.py
|
|
|
|
# Make predictions
|
|
curl -X POST "http://localhost:8080/predict" \\
|
|
-H "Content-Type: application/json" \\
|
|
-d '{"features": [1.0, 2.0, 3.0]}'
|
|
```
|
|
|
|
## GPU Support
|
|
|
|
To enable GPU support in Docker:
|
|
|
|
1. Install NVIDIA Docker runtime
|
|
2. Use `docker-compose.gpu.yml`:
|
|
```bash
|
|
docker-compose -f docker-compose.gpu.yml up
|
|
```
|
|
|
|
## Research Workflow
|
|
|
|
1. **Data Exploration** - Use `01_data_exploration.ipynb`
|
|
2. **Data Preprocessing** - Use `02_data_preprocessing.ipynb`
|
|
3. **Model Training** - Use `03_model_training.ipynb`
|
|
4. **Model Evaluation** - Use `04_model_evaluation.ipynb`
|
|
5. **Experiment Tracking** - Monitor in MLflow UI
|
|
6. **Model Deployment** - Deploy via FastAPI
|
|
|
|
## Contributing
|
|
|
|
1. Create feature branches for experiments
|
|
2. Document findings in notebooks
|
|
3. Track experiments with MLflow
|
|
4. Update model cards for significant models
|
|
5. Follow code style guidelines
|
|
|
|
## License
|
|
|
|
This project is licensed under the MIT License.
|
|
"""
|
|
|
|
# Placeholder implementations for other template methods
|
|
def _create_devops_template(self): pass
|
|
def _create_docs_template(self): pass
|
|
def _create_mobile_template(self): pass
|
|
def _create_blockchain_template(self): pass
|
|
def _get_fullstack_docker_compose_prod(self): return "# Production docker-compose configuration"
|
|
def _get_fullstack_env_example(self): return "# Environment variables example"
|
|
def _get_react_tsconfig(self): return "// TypeScript configuration"
|
|
def _get_tailwind_config(self): return "// Tailwind CSS configuration"
|
|
def _get_react_app_tsx(self): return "// React App component"
|
|
def _get_react_index_tsx(self): return "// React index file"
|
|
def _get_react_layout(self): return "// React layout component"
|
|
def _get_react_home_page(self): return "// React home page"
|
|
def _get_react_api_service(self): return "// API service for React"
|
|
def _get_react_auth_hook(self): return "// React authentication hook"
|
|
def _get_react_dockerfile(self): return "# React Dockerfile"
|
|
def _get_fastapi_requirements(self): return "# FastAPI requirements"
|
|
def _get_fastapi_pyproject(self): return "# FastAPI pyproject.toml"
|
|
def _get_fastapi_config(self): return "# FastAPI configuration"
|
|
def _get_fastapi_database(self): return "# FastAPI database configuration"
|
|
def _get_fastapi_auth(self): return "# FastAPI authentication"
|
|
def _get_fastapi_users(self): return "# FastAPI users API"
|
|
def _get_fastapi_user_model(self): return "# FastAPI user model"
|
|
def _get_fastapi_user_schema(self): return "# FastAPI user schema"
|
|
def _get_fastapi_dockerfile(self): return "# FastAPI Dockerfile"
|
|
def _get_alembic_config(self): return "# Alembic configuration"
|
|
def _get_alembic_env(self): return "# Alembic environment"
|
|
def _get_postgres_init_sql(self): return "-- PostgreSQL initialization"
|
|
def _get_github_actions_ci(self): return "# GitHub Actions CI"
|
|
def _get_github_actions_deploy(self): return "# GitHub Actions deployment"
|
|
def _get_react_test(self): return "// React test file"
|
|
def _get_fastapi_test(self): return "# FastAPI test file"
|
|
def _get_setup_documentation(self): return "# Setup documentation"
|
|
def _get_api_documentation(self): return "# API documentation"
|
|
def _get_deployment_documentation(self): return "# Deployment documentation"
|
|
|
|
# ML template methods (simplified)
|
|
def _get_conda_environment(self): return "# Conda environment"
|
|
def _get_ml_pyproject(self): return "# ML pyproject.toml"
|
|
def _get_ml_docker_compose(self): return "# ML docker-compose"
|
|
def _get_ml_dockerfile(self): return "# ML Dockerfile"
|
|
def _get_dvc_pipeline(self): return "# DVC pipeline"
|
|
def _get_dvc_ignore(self): return "# DVC ignore"
|
|
def _get_preprocessing_notebook(self): return "# Preprocessing notebook"
|
|
def _get_training_notebook(self): return "# Training notebook"
|
|
def _get_evaluation_notebook(self): return "# Evaluation notebook"
|
|
def _get_data_loader(self): return "# Data loader"
|
|
def _get_base_model(self): return "# Base model"
|
|
def _get_model_trainer(self): return "# Model trainer"
|
|
def _get_evaluation_metrics(self): return "# Evaluation metrics"
|
|
def _get_model_server(self): return "# Model server"
|
|
def _get_model_config(self): return "# Model configuration"
|
|
def _get_data_config(self): return "# Data configuration"
|
|
def _get_data_download_script(self): return "# Data download script"
|
|
def _get_training_script(self): return "# Training script"
|
|
def _get_evaluation_script(self): return "# Evaluation script"
|
|
def _get_mlflow_project(self): return "# MLflow project"
|
|
def _get_mlflow_conda(self): return "# MLflow conda"
|
|
def _get_research_documentation(self): return "# Research documentation"
|
|
def _get_model_cards_template(self): return "# Model cards template"
|
|
|
|
# Microservices template methods
|
|
def _get_microservices_readme(self): return "# Microservices README"
|
|
def _get_microservices_docker_compose(self): return "# Microservices docker-compose"
|
|
def _get_k8s_namespace(self): return "# Kubernetes namespace"
|
|
def _get_kong_config(self): return "# Kong configuration"
|
|
def _get_prometheus_config(self): return "# Prometheus configuration"
|
|
def _get_grafana_dashboard(self): return "# Grafana dashboard"
|
|
def _get_service_dockerfile(self, service): return f"# {service} service Dockerfile"
|
|
def _get_service_main(self, service): return f"# {service} service main"
|
|
def _get_service_requirements(self): return "# Service requirements"
|
|
def _get_auth_middleware(self): return "# Auth middleware"
|
|
def _get_health_check(self): return "# Health check"
|
|
def _get_database_base(self): return "# Database base"
|
|
|
|
# React FastAPI template methods
|
|
def _get_react_fastapi_readme(self): return "# React FastAPI README"
|
|
def _get_simple_docker_compose(self): return "# Simple docker-compose" |