diff --git a/.env.example b/.env.example index 9f32a785..4ddc2e53 100644 --- a/.env.example +++ b/.env.example @@ -1,69 +1,128 @@ -# STUPA PDF API Configuration Example -# Copy this file to .env and update with your values +# ======================================== +# STUPA PDF API Configuration +# ======================================== +# Application Settings +APP_NAME="STUPA PDF API" +APP_VERSION="3.0.0" +ENVIRONMENT="development" +DEBUG=true +LOG_LEVEL="INFO" +TIMEZONE="Europe/Berlin" +FRONTEND_URL="http://localhost:3001" + +# API Settings +API_PREFIX="/api" +DOCS_URL="/docs" +REDOC_URL="/redoc" +OPENAPI_URL="/openapi.json" + +# Feature Flags +ENABLE_METRICS=false +ENABLE_TRACING=false +ENABLE_CACHE=true +CACHE_TTL=300 +ENABLE_FORM_DESIGNER=true +ENABLE_PDF_UPLOAD=true +ENABLE_WORKFLOW=true + +# ======================================== # Database Configuration +# ======================================== MYSQL_HOST=db MYSQL_PORT=3306 MYSQL_DB=stupa MYSQL_USER=stupa -MYSQL_PASSWORD=your_secure_password_here -MYSQL_ROOT_PASSWORD=your_secure_root_password_here +MYSQL_PASSWORD=secret +DB_POOL_SIZE=10 +DB_MAX_OVERFLOW=20 +DB_POOL_PRE_PING=true +DB_ECHO=false -# Authentication -# Master key for admin access - keep this secure! -MASTER_KEY=your_secure_master_key_here +# ======================================== +# Security Settings +# ======================================== +MASTER_KEY="change_me_to_secure_key" +JWT_SECRET_KEY="change_me_to_secure_jwt_secret" +JWT_ALGORITHM="HS256" +ACCESS_TOKEN_EXPIRE_MINUTES=30 +REFRESH_TOKEN_EXPIRE_DAYS=7 +ENCRYPTION_KEY="change_me_to_32_byte_encryption_key" +API_KEY_HEADER="X-API-Key" +# CORS Settings +CORS_ORIGINS="http://localhost:3001,http://localhost:3000" +CORS_CREDENTIALS=true +CORS_METHODS="*" +CORS_HEADERS="*" + +# ======================================== +# OIDC/OAuth2 Settings (Nextcloud) +# ======================================== +OIDC_ENABLED=true +OIDC_ISSUER="https://nextcloud.example.com" +OIDC_CLIENT_ID="your_client_id" +OIDC_CLIENT_SECRET="your_client_secret" +OIDC_REDIRECT_URI="http://localhost:3001/auth/callback" +OIDC_SCOPE="openid profile email groups" +OIDC_AUTO_CREATE_USERS=true + +# OIDC Group Mappings (comma-separated) +OIDC_ADMIN_GROUPS="admin,administrators" +OIDC_BUDGET_REVIEWER_GROUPS="haushaltsbeauftragte,budget_reviewers" +OIDC_FINANCE_REVIEWER_GROUPS="finanzreferent,finance_reviewers" +OIDC_ASTA_GROUPS="asta,asta_members" + +# ======================================== +# Email Settings +# ======================================== +EMAIL_ENABLED=true +SMTP_HOST="localhost" +SMTP_PORT=587 +SMTP_TLS=true +SMTP_SSL=false +SMTP_USERNAME="" +SMTP_PASSWORD="" +EMAIL_FROM="noreply@example.com" +EMAIL_FROM_NAME="STUPA System" + +# Email Templates +EMAIL_VERIFICATION_SUBJECT="Verifizieren Sie Ihre E-Mail-Adresse" +EMAIL_MAGIC_LINK_SUBJECT="Anmelden bei STUPA" +EMAIL_APP_NOTIFICATION_SUBJECT="Status-Update zu Ihrer Bewerbung" + +# ======================================== # Rate Limiting -# Requests per minute per IP address +# ======================================== +RATE_LIMIT_ENABLED=true RATE_IP_PER_MIN=60 -# Requests per minute per application key RATE_KEY_PER_MIN=30 +RATE_GLOBAL_PER_MIN=1000 +RATE_BURST_SIZE=10 -# Application Settings -# Timezone for the application -TZ=Europe/Berlin +# ======================================== +# Storage Settings +# ======================================== +UPLOAD_DIR="/app/uploads" +TEMPLATE_DIR="/app/templates" +MAX_FILE_SIZE=10485760 # 10MB +ALLOWED_EXTENSIONS="pdf,json,jpg,jpeg,png" +TEMP_DIR="/tmp" +ATTACHMENT_STORAGE="filesystem" # database or filesystem +FILESYSTEM_PATH="/app/attachments" -# PDF Templates (paths inside container - don't change unless you know what you're doing) -QSM_TEMPLATE=/app/assets/qsm.pdf -VSM_TEMPLATE=/app/assets/vsm.pdf +# ======================================== +# Workflow Settings +# ======================================== +WORKFLOW_REQUIRED_VOTES=5 +WORKFLOW_APPROVAL_THRESHOLD=50.0 # Percentage +WORKFLOW_REVIEW_TIMEOUT_DAYS=14 +WORKFLOW_VOTING_TIMEOUT_DAYS=7 +WORKFLOW_ALLOW_MANUAL_STATUS_CHANGE=true +WORKFLOW_AUTO_LOCK_ON_SUBMISSION=true -# Frontend Configuration -NODE_ENV=production - -# Optional: CORS Configuration (for production) -# CORS_ORIGINS=["https://your-domain.com"] - -# Optional: Debug Mode (never enable in production) -# DEBUG=false - -# Optional: Database Connection Pool -# DB_POOL_SIZE=10 -# DB_MAX_OVERFLOW=20 - -# Optional: File Upload Limits -# MAX_UPLOAD_SIZE=104857600 # 100MB in bytes -# MAX_ATTACHMENTS_PER_APP=30 - -# Optional: Session Configuration -# SESSION_TIMEOUT=3600 # 1 hour in seconds - -# Optional: Email Configuration (for future notifications) -# SMTP_HOST=smtp.example.com -# SMTP_PORT=587 -# SMTP_USER=notifications@example.com -# SMTP_PASSWORD=smtp_password_here -# SMTP_FROM=noreply@example.com - -# Optional: Logging Configuration -# LOG_LEVEL=INFO -# LOG_FILE=/var/log/stupa-api.log - -# Optional: Backup Configuration -# BACKUP_ENABLED=true -# BACKUP_SCHEDULE="0 2 * * *" # Daily at 2 AM -# BACKUP_RETENTION_DAYS=30 - -# Production Security Headers (uncomment for production) -# SECURE_HEADERS=true -# HSTS_ENABLED=true -# CSP_ENABLED=true +# ======================================== +# Docker Compose Specific +# ======================================== +MYSQL_ROOT_PASSWORD=rootsecret +TZ="Europe/Berlin" diff --git a/README_NEW_ARCHITECTURE.md b/README_NEW_ARCHITECTURE.md new file mode 100644 index 00000000..b866e625 --- /dev/null +++ b/README_NEW_ARCHITECTURE.md @@ -0,0 +1,310 @@ +# STUPA PDF API - New Architecture (v3.0) + +## Overview + +The STUPA PDF API has been completely redesigned to support modern authentication, flexible PDF form handling, and a sophisticated workflow system. This document describes the new architecture and features. + +## Key Changes from Previous Version + +### 1. **Removed LaTeX Compilation** +- No more embedded LaTeX compilation +- Users upload fillable PDF templates instead +- More flexible and maintainable approach + +### 2. **OIDC/OAuth2 Authentication** +- Full integration with Nextcloud via OIDC +- Automatic role mapping from OIDC groups +- Single Sign-On (SSO) support + +### 3. **Dual Authentication System** +- **OIDC**: Primary authentication via Nextcloud +- **Email**: Magic link authentication as fallback +- Seamless switching between auth methods + +### 4. **Role-Based Access Control (RBAC)** +- Dynamic role assignment from OIDC claims +- Configurable permissions per role +- Special roles for workflow participants + +### 5. **PDF Template Management** +- Upload any fillable PDF as a template +- Graphical field mapping interface +- Support for multiple form types + +### 6. **Visual Form Designer** +- Drag-and-drop form builder +- Custom styling and themes +- Responsive form layouts + +### 7. **Advanced Workflow System** +- Multi-stage approval process +- Voting mechanism for AStA members +- Automatic status transitions + +## Architecture Components + +### Backend Services + +#### Authentication Services +- `auth_oidc.py`: OIDC/OAuth2 authentication with Nextcloud +- `auth_email.py`: Email-based authentication with magic links + +#### PDF Services +- `pdf_template.py`: Template upload and field mapping +- `pdf_processor.py`: PDF filling and generation + +#### Workflow Services +- `workflow_engine.py`: Application workflow management +- `voting_service.py`: Voting and approval handling + +### Database Schema + +#### Core Tables +- `users`: User accounts with OIDC integration +- `roles`: Role definitions with permissions +- `sessions`: Active user sessions + +#### Form Management +- `form_templates`: PDF template definitions +- `field_mappings`: PDF field to data mappings +- `form_designs`: Visual form configurations + +#### Application Management +- `applications`: Main application records +- `application_votes`: Voting records +- `application_history`: Audit trail +- `application_attachments`: File attachments + +## Workflow Process + +The new application workflow follows this state machine: + +``` +DRAFT → BEANTRAGT → BEARBEITUNG_GESPERRT → ZU_PRUEFEN → ZUR_ABSTIMMUNG → GENEHMIGT/ABGELEHNT +``` + +### States Explained + +1. **DRAFT**: Initial creation, user can edit +2. **BEANTRAGT**: Submitted by user, awaiting processing +3. **BEARBEITUNG_GESPERRT**: Processing locked, under review +4. **ZU_PRUEFEN**: To be reviewed by Haushaltsbeauftragte & Finanzreferent +5. **ZUR_ABSTIMMUNG**: Open for voting by AStA members (requires 5 votes) +6. **GENEHMIGT**: Approved +7. **ABGELEHNT**: Rejected (can be manually reset to BEANTRAGT) + +## Configuration + +### Environment Variables + +#### OIDC Configuration +```env +OIDC_ENABLED=true +OIDC_ISSUER=https://nextcloud.example.com +OIDC_CLIENT_ID=your_client_id +OIDC_CLIENT_SECRET=your_client_secret +OIDC_REDIRECT_URI=http://localhost:3001/auth/callback +``` + +#### Role Mapping from OIDC Groups +```env +OIDC_ADMIN_GROUPS=admin,administrators +OIDC_BUDGET_REVIEWER_GROUPS=haushaltsbeauftragte +OIDC_FINANCE_REVIEWER_GROUPS=finanzreferent +OIDC_ASTA_GROUPS=asta,asta_members +``` + +#### Email Configuration +```env +EMAIL_ENABLED=true +SMTP_HOST=localhost +SMTP_PORT=587 +EMAIL_FROM=noreply@example.com +``` + +#### Workflow Configuration +```env +WORKFLOW_REQUIRED_VOTES=5 +WORKFLOW_APPROVAL_THRESHOLD=50.0 +WORKFLOW_REVIEW_TIMEOUT_DAYS=14 +WORKFLOW_VOTING_TIMEOUT_DAYS=7 +``` + +## API Endpoints + +### Authentication +- `POST /api/auth/oidc/authorize` - Initiate OIDC flow +- `POST /api/auth/oidc/callback` - OIDC callback handler +- `POST /api/auth/email/register` - Email registration +- `POST /api/auth/email/verify` - Verify email token +- `POST /api/auth/email/login` - Request magic link +- `GET /api/auth/me` - Get current user + +### Form Templates +- `GET /api/templates` - List available templates +- `POST /api/templates` - Upload new template (admin) +- `GET /api/templates/{id}` - Get template details +- `PUT /api/templates/{id}/mappings` - Update field mappings +- `POST /api/templates/{id}/test` - Test field mappings + +### Applications +- `GET /api/applications` - List user's applications +- `POST /api/applications` - Create new application +- `GET /api/applications/{id}` - Get application details +- `PUT /api/applications/{id}` - Update application +- `POST /api/applications/{id}/submit` - Submit for review +- `POST /api/applications/{id}/review` - Review application +- `POST /api/applications/{id}/vote` - Vote on application + +### Admin +- `GET /api/admin/users` - List all users +- `PUT /api/admin/users/{id}/roles` - Update user roles +- `GET /api/admin/roles` - List all roles +- `POST /api/admin/roles/{id}/mappings` - Configure OIDC mappings + +## Frontend Components + +### User Dashboard +- View all applications +- Track application status +- Access application history + +### Form Designer (Admin) +- Visual form builder interface +- Drag-drop field arrangement +- Field property configuration +- Conditional logic builder + +### PDF Field Mapper (Admin) +- Upload PDF templates +- Auto-detect form fields +- Map fields to data model +- Configure validation rules + +### Voting Interface (AStA Members) +- Review pending applications +- Cast votes with comments +- View voting history + +## Security Features + +### Authentication +- JWT-based session management +- Refresh token rotation +- Automatic session expiry + +### Authorization +- Role-based permissions +- Resource-level access control +- API rate limiting + +### Data Protection +- Encrypted token storage +- Secure password hashing +- CSRF protection +- XSS prevention + +## Deployment + +### Docker Compose + +```bash +# Development +docker-compose up -d + +# Production +docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d +``` + +### Required Services +- MySQL 8.0+ +- Redis 7.0+ +- SMTP server (for email auth) +- Nextcloud instance (for OIDC) + +### Volumes +- `/app/uploads` - Uploaded files +- `/app/templates` - PDF templates +- `/app/attachments` - Application attachments + +## Migration from v2 + +1. **Backup existing data** + ```bash + docker-compose exec db mysqldump -u root -p stupa > backup.sql + ``` + +2. **Run migrations** + ```bash + docker-compose exec api python -m alembic upgrade head + ``` + +3. **Configure OIDC** + - Register application in Nextcloud + - Configure redirect URIs + - Map groups to roles + +4. **Upload PDF templates** + - Convert existing LaTeX templates to fillable PDFs + - Upload via admin interface + - Configure field mappings + +## Development + +### Setup Development Environment + +```bash +# Clone repository +git clone https://github.com/your-org/stupa-pdf-api.git +cd stupa-pdf-api + +# Checkout new architecture branch +git checkout feature/oidc-pdf-upload-redesign + +# Copy environment file +cp .env.example .env + +# Start services +docker-compose --profile dev up +``` + +### Testing + +```bash +# Run backend tests +docker-compose exec api pytest + +# Run frontend tests +docker-compose exec frontend npm test + +# Test OIDC integration +docker-compose exec api python -m tests.test_oidc +``` + +## Troubleshooting + +### OIDC Connection Issues +- Verify Nextcloud OAuth2 app is configured +- Check redirect URI matches configuration +- Ensure client ID/secret are correct + +### PDF Processing Issues +- Verify PDF has fillable form fields +- Check field names don't contain special characters +- Ensure PDF is not password protected + +### Email Delivery Issues +- Check SMTP configuration +- Verify firewall allows SMTP port +- Test with local mail server (MailHog) + +## Support + +For issues or questions about the new architecture: +1. Check the documentation +2. Review the migration guide +3. Contact the development team + +## License + +This project is licensed under the MIT License - see the LICENSE file for details. \ No newline at end of file diff --git a/backend/ARCHITECTURE.md b/backend/ARCHITECTURE.md new file mode 100644 index 00000000..73f196bb --- /dev/null +++ b/backend/ARCHITECTURE.md @@ -0,0 +1,420 @@ +# Backend Architecture Documentation + +## Overview + +The backend has been refactored from a monolithic structure into a modular, service-oriented architecture that emphasizes: +- **Separation of Concerns**: Clear boundaries between layers (API, Service, Repository, Model) +- **Dependency Injection**: Dynamic service resolution and configuration +- **Extensibility**: Plugin-based system for PDF variants and providers +- **Maintainability**: Organized code structure with single responsibility principle +- **Scalability**: Stateless services with proper connection pooling + +## Directory Structure + +``` +backend/ +├── src/ +│ ├── api/ # API Layer +│ │ ├── routes/ # FastAPI routers +│ │ ├── middleware/ # Custom middleware +│ │ └── dependencies/ # Dependency injection helpers +│ │ +│ ├── services/ # Business Logic Layer +│ │ ├── base.py # Base service classes +│ │ ├── application.py # Application business logic +│ │ ├── pdf.py # PDF processing service +│ │ └── auth.py # Authentication service +│ │ +│ ├── repositories/ # Data Access Layer +│ │ ├── base.py # Base repository pattern +│ │ ├── application.py # Application repository +│ │ └── attachment.py # Attachment repository +│ │ +│ ├── models/ # Database Models +│ │ ├── base.py # Base model with mixins +│ │ └── application.py # Application entities +│ │ +│ ├── providers/ # Dynamic Providers +│ │ ├── pdf_qsm.py # QSM PDF variant provider +│ │ └── pdf_vsm.py # VSM PDF variant provider +│ │ +│ ├── config/ # Configuration Management +│ │ └── settings.py # Centralized settings with Pydantic +│ │ +│ ├── core/ # Core Infrastructure +│ │ ├── container.py # Dependency injection container +│ │ └── database.py # Database management +│ │ +│ └── utils/ # Utility Functions +│ └── helpers.py # Common utilities +``` + +## Architecture Layers + +### 1. API Layer (`api/`) +**Responsibility**: HTTP request/response handling, validation, routing + +- **Routes**: Modular FastAPI routers for different domains +- **Middleware**: Cross-cutting concerns (rate limiting, logging, error handling) +- **Dependencies**: FastAPI dependency injection functions + +```python +# Example: api/routes/applications.py +@router.post("/", response_model=ApplicationResponse) +async def create_application( + data: ApplicationCreate, + service: ApplicationService = Depends(get_application_service) +): + return await service.create(data.dict()) +``` + +### 2. Service Layer (`services/`) +**Responsibility**: Business logic, orchestration, validation rules + +- Encapsulates all business rules and workflows +- Coordinates between repositories and external services +- Handles complex validations and transformations +- Stateless and testable + +```python +# Example: services/application.py +class ApplicationService(CRUDService[Application]): + def submit_application(self, id: int) -> Application: + # Business logic for submission + app = self.repository.get_or_404(id) + self._validate_submission(app) + app.status = ApplicationStatus.SUBMITTED + return self.repository.update(app) +``` + +### 3. Repository Layer (`repositories/`) +**Responsibility**: Data access abstraction, CRUD operations + +- Implements repository pattern for database access +- Provides clean abstraction over SQLAlchemy +- Handles query building and optimization +- Transaction management + +```python +# Example: repositories/application.py +class ApplicationRepository(BaseRepository[Application]): + def find_by_status(self, status: ApplicationStatus) -> List[Application]: + return self.query().filter( + Application.status == status + ).all() +``` + +### 4. Model Layer (`models/`) +**Responsibility**: Data structure definition, ORM mapping + +- SQLAlchemy models with proper relationships +- Base classes with common functionality (timestamps, soft delete) +- Model mixins for reusable behavior +- Business entity representation + +```python +# Example: models/application.py +class Application(ExtendedBaseModel): + __tablename__ = "applications" + + pa_id = Column(String(64), unique=True, index=True) + status = Column(SQLEnum(ApplicationStatus)) + payload = Column(JSON) +``` + +## Key Components + +### Dependency Injection Container + +The system uses a custom dependency injection container for managing service lifecycles: + +```python +# core/container.py +class Container: + def register_service(self, name: str, service_class: Type[BaseService]): + # Register service with automatic dependency resolution + + def get_service(self, name: str) -> BaseService: + # Retrieve service instance with dependencies injected +``` + +**Benefits:** +- Loose coupling between components +- Easy testing with mock services +- Dynamic service configuration +- Singleton pattern support + +### Configuration Management + +Centralized configuration using Pydantic Settings: + +```python +# config/settings.py +class Settings(BaseSettings): + database: DatabaseSettings + security: SecuritySettings + rate_limit: RateLimitSettings + storage: StorageSettings + pdf: PDFSettings + app: ApplicationSettings +``` + +**Features:** +- Environment variable support +- Type validation +- Default values +- Configuration file support (JSON/YAML) +- Dynamic override capability + +### Provider Pattern for PDF Variants + +Extensible system for handling different PDF types: + +```python +# providers/pdf_qsm.py +class QSMProvider(PDFVariantProvider): + def parse_pdf_fields(self, fields: Dict) -> Dict: + # QSM-specific parsing logic + + def map_payload_to_fields(self, payload: Dict) -> Dict: + # QSM-specific field mapping +``` + +**Advantages:** +- Easy to add new PDF variants +- Variant-specific validation rules +- Dynamic provider registration +- Clean separation of variant logic + +## Database Architecture + +### Base Model Classes + +```python +# models/base.py +class BaseModel: + # Common fields and methods + +class TimestampMixin: + created_at = Column(DateTime) + updated_at = Column(DateTime) + +class SoftDeleteMixin: + is_deleted = Column(Boolean) + deleted_at = Column(DateTime) + +class AuditMixin: + created_by = Column(String) + updated_by = Column(String) +``` + +### Connection Management + +- Connection pooling with configurable size +- Automatic retry on connection failure +- Session scoping for transaction management +- Health check utilities + +## Service Patterns + +### CRUD Service Base + +```python +class CRUDService(BaseService): + def create(self, data: Dict) -> T + def update(self, id: Any, data: Dict) -> T + def delete(self, id: Any, soft: bool = True) -> bool + def get(self, id: Any) -> Optional[T] + def list(self, filters: Dict, page: int, page_size: int) -> Dict +``` + +### Error Handling + +Hierarchical exception system: + +```python +ServiceException +├── ValidationError +├── BusinessRuleViolation +├── ResourceNotFoundError +└── ResourceConflictError +``` + +### Transaction Management + +```python +with service.handle_errors("operation"): + with repository.transaction(): + # Perform multiple operations + # Automatic rollback on error +``` + +## API Design + +### RESTful Endpoints + +``` +POST /api/applications # Create application +GET /api/applications # List applications +GET /api/applications/{id} # Get application +PUT /api/applications/{id} # Update application +DELETE /api/applications/{id} # Delete application + +POST /api/applications/{id}/submit # Submit application +POST /api/applications/{id}/review # Review application +GET /api/applications/{id}/pdf # Generate PDF +``` + +### Request/Response Models + +Using Pydantic for validation: + +```python +class ApplicationCreate(BaseModel): + variant: ApplicationType + payload: Dict[str, Any] + +class ApplicationResponse(BaseModel): + id: int + pa_id: str + status: ApplicationStatus + created_at: datetime +``` + +## Middleware Stack + +1. **CORS Middleware**: Cross-origin resource sharing +2. **Rate Limit Middleware**: Request throttling +3. **Logging Middleware**: Request/response logging +4. **Error Handler Middleware**: Global error handling +5. **Authentication Middleware**: JWT/API key validation + +## Security Features + +- JWT-based authentication +- API key support +- Rate limiting per IP/key +- SQL injection prevention via ORM +- Input sanitization +- Audit logging + +## Performance Optimizations + +- Database connection pooling +- Lazy loading relationships +- Query optimization with indexes +- Caching support (Redis) +- Async request handling +- PDF generation caching + +## Testing Strategy + +### Unit Tests +- Service logic testing +- Repository method testing +- Model validation testing + +### Integration Tests +- API endpoint testing +- Database transaction testing +- PDF processing testing + +### End-to-End Tests +- Complete workflow testing +- Multi-service interaction testing + +## Deployment Considerations + +### Environment Variables + +```env +# Database +MYSQL_HOST=localhost +MYSQL_PORT=3306 +MYSQL_DB=stupa +MYSQL_USER=user +MYSQL_PASSWORD=password + +# Security +MASTER_KEY=secret_key +JWT_SECRET_KEY=jwt_secret + +# Rate Limiting +RATE_IP_PER_MIN=60 +RATE_KEY_PER_MIN=30 + +# PDF Templates +QSM_TEMPLATE=assets/qsm.pdf +VSM_TEMPLATE=assets/vsm.pdf +``` + +### Docker Support + +```dockerfile +FROM python:3.11-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install -r requirements.txt +COPY . . +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000"] +``` + +### Scaling Considerations + +- Stateless services for horizontal scaling +- Database read replicas support +- Cache layer for frequently accessed data +- Async processing for heavy operations +- Message queue integration ready + +## Migration Path + +### From Old to New Architecture + +1. **Phase 1**: Setup new structure alongside old code +2. **Phase 2**: Migrate database models +3. **Phase 3**: Implement service layer +4. **Phase 4**: Create API routes +5. **Phase 5**: Migrate business logic +6. **Phase 6**: Remove old code + +### Database Migrations + +Using Alembic for version control: + +```bash +alembic init migrations +alembic revision --autogenerate -m "Initial migration" +alembic upgrade head +``` + +## Monitoring & Observability + +- Structured logging with context +- Prometheus metrics integration +- Health check endpoints +- Performance profiling hooks +- Error tracking integration ready + +## Future Enhancements + +1. **GraphQL Support**: Alternative API interface +2. **WebSocket Support**: Real-time updates +3. **Event Sourcing**: Audit trail and history +4. **Microservices**: Service decomposition +5. **API Gateway**: Advanced routing and auth +6. **Message Queue**: Async task processing +7. **Search Engine**: Elasticsearch integration +8. **Machine Learning**: PDF field prediction + +## Conclusion + +This refactored architecture provides: +- **Maintainability**: Clear structure and separation +- **Scalability**: Ready for growth +- **Testability**: Isolated components +- **Extensibility**: Plugin-based design +- **Performance**: Optimized patterns +- **Security**: Built-in best practices + +The modular design allows teams to work independently on different components while maintaining system integrity through well-defined interfaces. \ No newline at end of file diff --git a/backend/Dockerfile b/backend/Dockerfile index 15dd6722..988c0c96 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,71 +1,61 @@ -# ---------- LaTeX Builder Stage ---------- -FROM texlive/texlive:latest AS latex-builder - -# Install additional dependencies for LaTeX -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - inkscape \ - make \ - fonts-liberation \ - fonts-dejavu-core \ - && rm -rf /var/lib/apt/lists/* \ - && inkscape --version - -WORKDIR /latex - -# Copy the LaTeX source files from local worktrees -COPY latex-qsm /latex/qsm -COPY latex-vsm /latex/vsm - -# Build QSM PDF -WORKDIR /latex/qsm -RUN latexmk -xelatex -interaction=nonstopmode -halt-on-error -shell-escape Main.tex && \ - cp Main.pdf /latex/qsm.pdf && \ - latexmk -c - -# Build VSM PDF -WORKDIR /latex/vsm -RUN latexmk -xelatex -interaction=nonstopmode -shell-escape Main.tex && \ - cp Main.pdf /latex/vsm.pdf && \ - latexmk -c - -# ---------- Base Python Stage ---------- -FROM python:3.11-slim AS base - -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 - -# System deps -RUN apt-get update && apt-get install -y --no-install-recommends \ - tzdata ca-certificates \ - qpdf \ - pdftk-java \ - libmupdf-dev \ - mupdf-tools \ - && rm -rf /var/lib/apt/lists/* +FROM python:3.11-slim +# Set working directory WORKDIR /app -# ---------- Dependencies ---------- -COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir -r /app/requirements.txt +# Install system dependencies +RUN apt-get update && apt-get install -y \ + gcc \ + libmariadb-dev \ + pkg-config \ + wget \ + curl \ + # PDF processing tools + poppler-utils \ + # Clean up + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements first for better caching +COPY requirements.txt . + +# Install Python dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Install additional PDF processing libraries +RUN pip install --no-cache-dir \ + PyMuPDF \ + pypdf \ + pillow \ + python-multipart \ + httpx \ + redis \ + python-jose[cryptography] \ + passlib \ + bcrypt \ + emails \ + jinja2 -# ---------- App ---------- # Copy application code -COPY src/ /app/src/ +COPY src/ ./src/ +COPY assets/ ./assets/ -# Copy pre-built PDFs from latex-builder stage -COPY --from=latex-builder /latex/qsm.pdf /app/assets/qsm.pdf -COPY --from=latex-builder /latex/vsm.pdf /app/assets/vsm.pdf +# Create necessary directories +RUN mkdir -p /app/uploads \ + /app/templates \ + /app/attachments \ + /app/pdf_forms \ + /app/logs -# Set Python path -ENV PYTHONPATH=/app/src +# Set permissions +RUN chmod -R 755 /app -# Configure PDF template paths -ENV QSM_TEMPLATE=/app/assets/qsm.pdf \ - VSM_TEMPLATE=/app/assets/vsm.pdf +# Health check +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')" || exit 1 +# Expose port EXPOSE 8000 -# ---------- Run ---------- -CMD ["uvicorn", "service_api:app", "--host", "0.0.0.0", "--port", "8000"] +# Run the application +CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] diff --git a/backend/latex-qsm b/backend/latex-qsm deleted file mode 160000 index 4dca8b58..00000000 --- a/backend/latex-qsm +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4dca8b58bc350b5cd1835499ed6119b7e2220794 diff --git a/backend/latex-vsm b/backend/latex-vsm deleted file mode 160000 index c5aa64c4..00000000 --- a/backend/latex-vsm +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c5aa64c41e25a4a938928c266460f0a020ab7b14 diff --git a/backend/requirements.txt b/backend/requirements.txt index 76bafc89..2b5e6f10 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,20 +1,89 @@ -# Core API & HTTP -fastapi>=0.110 -uvicorn[standard]>=0.27 +# Core Framework +fastapi==0.109.0 +uvicorn[standard]==0.27.0 +pydantic==2.5.3 +pydantic-settings==2.1.0 +python-multipart==0.0.6 -# Data parsing / validation -pydantic>=2.6 +# Database +sqlalchemy==2.0.25 +pymysql==1.1.0 +alembic==1.13.1 +cryptography==41.0.7 -# PDF handling -PyPDF2>=3.0.1 -PyMuPDF>=1.23.0 +# Authentication & Security +python-jose[cryptography]==3.3.0 +passlib[bcrypt]==1.7.4 +bcrypt==4.1.2 +authlib==1.3.0 +httpx==0.26.0 +requests==2.31.0 -# DB (MySQL via SQLAlchemy + PyMySQL) -SQLAlchemy>=2.0 -PyMySQL>=1.1 +# OIDC/OAuth2 +oauthlib==3.2.2 +requests-oauthlib==1.3.1 -# Env handling -python-dotenv>=1.0 +# PDF Processing +pypdf==3.17.4 +PyMuPDF==1.23.16 +reportlab==4.0.8 +pillow==10.2.0 +pypdfium2==4.25.0 -# File uploads (FastAPI Form/File) -python-multipart>=0.0.9 +# Email +emails==0.6 +aiosmtplib==3.0.1 +email-validator==2.1.0.post1 + +# Caching & Sessions +redis==5.0.1 +python-redis==0.6.0 + +# Template Processing +jinja2==3.1.3 +markupsafe==2.1.4 + +# Utilities +python-dotenv==1.0.0 +pytz==2023.3.post1 +croniter==2.0.1 +python-dateutil==2.8.2 + +# File Processing +python-magic==0.4.27 +filetype==1.2.0 + +# API Documentation +openapi-schema-pydantic==1.2.4 + +# Testing (optional, for development) +pytest==7.4.4 +pytest-asyncio==0.23.3 +pytest-cov==4.1.0 +httpx-mock==0.4.0 +faker==22.0.0 + +# Development Tools +black==23.12.1 +flake8==7.0.0 +mypy==1.8.0 +pre-commit==3.6.0 + +# Logging & Monitoring +python-json-logger==2.0.7 +sentry-sdk[fastapi]==1.39.2 + +# Data Validation & Serialization +marshmallow==3.20.2 +pyyaml==6.0.1 + +# Background Tasks (optional) +celery==5.3.6 +kombu==5.3.5 +flower==2.0.1 + +# Rate Limiting +slowapi==0.1.9 + +# CORS & Security Headers +secure==0.3.0 diff --git a/backend/scripts/make_fields_readonly.py b/backend/scripts/make_fields_readonly.py deleted file mode 100644 index 5b907695..00000000 --- a/backend/scripts/make_fields_readonly.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python3 -""" -Make all form fields in LaTeX documents readonly by adding readonly=true attribute. -This script carefully parses LaTeX commands to avoid breaking the syntax. -""" - -import re -import sys -from pathlib import Path - - -def add_readonly_to_textfield(content): - """Add readonly=true to CustomTextFieldDefault commands.""" - # Pattern to match CustomTextFieldDefault{...}{...}{...}{params} - pattern = r'(\\CustomTextFieldDefault\{[^}]*\}\{[^}]*\}\{[^}]*\}\{)([^}]*)\}' - - def replacer(match): - prefix = match.group(1) - params = match.group(2) - - # Check if readonly is already present - if 'readonly=' in params: - return match.group(0) - - # Add readonly=true to the parameters - if params.strip(): - new_params = params + ',readonly=true' - else: - new_params = 'readonly=true' - - return prefix + new_params + '}' - - return re.sub(pattern, replacer, content) - - -def add_readonly_to_choicemenu(content): - """Add readonly=true to CustomChoiceMenuDefault commands.""" - # Pattern to match CustomChoiceMenuDefault{...}{...}{params}{...} - pattern = r'(\\CustomChoiceMenuDefault\{[^}]*\}\{[^}]*\}\{)([^}]*)\}(\{[^}]*\})' - - def replacer(match): - prefix = match.group(1) - params = match.group(2) - suffix = match.group(3) - - # Check if readonly is already present - if 'readonly=' in params: - return match.group(0) - - # Add readonly=true to the parameters - if params.strip(): - new_params = params + ',readonly=true' - else: - new_params = 'readonly=true' - - return prefix + new_params + '}' + suffix - - return re.sub(pattern, replacer, content) - - -def add_readonly_to_checkbox(content): - """Add readonly=true to CheckBox commands.""" - # Pattern to match CheckBox[params] - pattern = r'(\\CheckBox\[)([^\]]*)\]' - - def replacer(match): - prefix = match.group(1) - params = match.group(2) - - # Check if readonly is already present - if 'readonly=' in params: - return match.group(0) - - # Add readonly=true to the parameters - params_lines = params.split('\n') - - # Find a good place to insert readonly=true (after first parameter) - for i, line in enumerate(params_lines): - if line.strip() and not line.strip().startswith('%'): - # Insert after this line - params_lines.insert(i + 1, '\t\t\t\treadonly=true,') - break - - new_params = '\n'.join(params_lines) - return prefix + new_params + ']' - - return re.sub(pattern, replacer, content, flags=re.MULTILINE | re.DOTALL) - - -def add_readonly_to_textfield_multiline(content): - """Add readonly=true to TextField commands (multiline text fields).""" - # Pattern to match TextField[params] for multiline fields - pattern = r'(\\TextField\[)([^\]]*name=pa-project-description[^\]]*)\]' - - def replacer(match): - prefix = match.group(1) - params = match.group(2) - - # Check if readonly is already present - if 'readonly=' in params: - return match.group(0) - - # Add readonly=true to the parameters - params_lines = params.split('\n') - - # Find a good place to insert readonly=true (after multiline parameter) - for i, line in enumerate(params_lines): - if 'multiline' in line: - # Insert after this line - params_lines.insert(i + 1, '\t\t\t\treadonly=true,') - break - - new_params = '\n'.join(params_lines) - return prefix + new_params + ']' - - return re.sub(pattern, replacer, content, flags=re.MULTILINE | re.DOTALL) - - -def process_file(filepath): - """Process a single LaTeX file to make all fields readonly.""" - print(f"Processing {filepath}...") - - with open(filepath, 'r', encoding='utf-8') as f: - content = f.read() - - # Apply transformations - content = add_readonly_to_textfield(content) - content = add_readonly_to_choicemenu(content) - content = add_readonly_to_checkbox(content) - content = add_readonly_to_textfield_multiline(content) - - # Write back - with open(filepath, 'w', encoding='utf-8') as f: - f.write(content) - - print(f"✓ Processed {filepath}") - - -def main(): - """Main function to process QSM and VSM LaTeX files.""" - base_dir = Path(__file__).parent.parent - - files_to_process = [ - base_dir / "latex-qsm" / "Content" / "01_content.tex", - base_dir / "latex-vsm" / "Content" / "01_content.tex", - ] - - for filepath in files_to_process: - if filepath.exists(): - process_file(filepath) - else: - print(f"✗ File not found: {filepath}") - - -if __name__ == "__main__": - main() diff --git a/backend/src/api/v1/auth.py b/backend/src/api/v1/auth.py new file mode 100644 index 00000000..2bb54bf1 --- /dev/null +++ b/backend/src/api/v1/auth.py @@ -0,0 +1,391 @@ +""" +Authentication API Routes + +This module provides API endpoints for OIDC and email authentication. +""" + +from datetime import datetime, timedelta +from typing import Dict, Any, Optional +from fastapi import APIRouter, Depends, HTTPException, status, Request, Response +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from pydantic import BaseModel, EmailStr, Field +from sqlalchemy.orm import Session + +from ...core.database import get_db +from ...services.auth_oidc import OIDCAuthService +from ...services.auth_email import EmailAuthService +from ...models.user import User +from ...config.settings import get_settings + +router = APIRouter(prefix="/auth", tags=["Authentication"]) +security = HTTPBearer() +settings = get_settings() + + +class EmailRegisterRequest(BaseModel): + """Email registration request""" + email: EmailStr + name: Optional[str] = None + + +class EmailLoginRequest(BaseModel): + """Email login request""" + email: EmailStr + + +class TokenVerifyRequest(BaseModel): + """Token verification request""" + token: str + + +class OIDCCallbackRequest(BaseModel): + """OIDC callback request""" + code: str + state: str + + +class RefreshTokenRequest(BaseModel): + """Refresh token request""" + refresh_token: str + + +class UserProfileUpdate(BaseModel): + """User profile update request""" + given_name: Optional[str] = None + family_name: Optional[str] = None + display_name: Optional[str] = None + + +class TokenResponse(BaseModel): + """Token response model""" + access_token: str + refresh_token: Optional[str] = None + token_type: str = "Bearer" + expires_in: int + user: Dict[str, Any] + + +async def get_current_user( + credentials: HTTPAuthorizationCredentials = Depends(security), + db: Session = Depends(get_db) +) -> User: + """Get current authenticated user from JWT token""" + + token = credentials.credentials + email_service = EmailAuthService(db, settings) + + try: + user = await email_service.get_current_user(token) + return user + except HTTPException: + raise + except Exception as e: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid authentication credentials" + ) + + +@router.get("/oidc/authorize") +async def oidc_authorize( + request: Request, + db: Session = Depends(get_db) +) -> Dict[str, str]: + """Initiate OIDC authorization flow""" + + if not settings.oidc.enabled: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="OIDC authentication is not enabled" + ) + + async with OIDCAuthService(db, settings) as service: + state = service.generate_state_token() + nonce = service.generate_nonce() + + # Store state and nonce in session (or cache) + request.session["oidc_state"] = state + request.session["oidc_nonce"] = nonce + + authorization_url = service.get_authorization_url(state, nonce) + + return { + "authorization_url": authorization_url, + "state": state + } + + +@router.post("/oidc/callback", response_model=TokenResponse) +async def oidc_callback( + request: Request, + callback_data: OIDCCallbackRequest, + db: Session = Depends(get_db) +) -> TokenResponse: + """Handle OIDC callback""" + + if not settings.oidc.enabled: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="OIDC authentication is not enabled" + ) + + # Retrieve stored state and nonce + stored_state = request.session.get("oidc_state") + stored_nonce = request.session.get("oidc_nonce") + + if not stored_state: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid session state" + ) + + async with OIDCAuthService(db, settings) as service: + user, token_data = await service.authenticate_user( + code=callback_data.code, + state=callback_data.state, + stored_state=stored_state, + nonce=stored_nonce + ) + + # Clear session state + request.session.pop("oidc_state", None) + request.session.pop("oidc_nonce", None) + + return TokenResponse(**token_data) + + +@router.post("/email/register", response_model=Dict[str, str]) +async def email_register( + registration: EmailRegisterRequest, + db: Session = Depends(get_db) +) -> Dict[str, str]: + """Register with email""" + + if not settings.email.enabled: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Email authentication is not enabled" + ) + + service = EmailAuthService(db, settings) + + user = await service.register_user( + email=registration.email, + name=registration.name + ) + + return { + "message": "Verification email sent", + "email": user.email + } + + +@router.post("/email/verify", response_model=TokenResponse) +async def email_verify( + verification: TokenVerifyRequest, + db: Session = Depends(get_db) +) -> TokenResponse: + """Verify email with token""" + + if not settings.email.enabled: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Email authentication is not enabled" + ) + + service = EmailAuthService(db, settings) + + user, token_data = await service.verify_email(verification.token) + + return TokenResponse(**token_data) + + +@router.post("/email/login", response_model=Dict[str, str]) +async def email_login( + login: EmailLoginRequest, + db: Session = Depends(get_db) +) -> Dict[str, str]: + """Request magic login link""" + + if not settings.email.enabled: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Email authentication is not enabled" + ) + + service = EmailAuthService(db, settings) + + await service.login_with_magic_link(login.email) + + return { + "message": "If the email exists, a login link has been sent", + "email": login.email + } + + +@router.post("/email/magic-link", response_model=TokenResponse) +async def email_magic_link( + verification: TokenVerifyRequest, + db: Session = Depends(get_db) +) -> TokenResponse: + """Login with magic link token""" + + if not settings.email.enabled: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Email authentication is not enabled" + ) + + service = EmailAuthService(db, settings) + + user, token_data = await service.verify_magic_link(verification.token) + + return TokenResponse(**token_data) + + +@router.post("/refresh", response_model=TokenResponse) +async def refresh_token( + refresh_request: RefreshTokenRequest, + db: Session = Depends(get_db) +) -> TokenResponse: + """Refresh access token""" + + # Try OIDC service first + if settings.oidc.enabled: + async with OIDCAuthService(db, settings) as service: + try: + token_data = await service.refresh_access_token(refresh_request.refresh_token) + return TokenResponse(**token_data) + except HTTPException: + pass + + # Try email service + if settings.email.enabled: + service = EmailAuthService(db, settings) + try: + # Email service doesn't have refresh implementation yet, + # but we can add it if needed + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid refresh token" + ) + except HTTPException: + pass + + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid refresh token" + ) + + +@router.post("/logout") +async def logout( + user: User = Depends(get_current_user), + credentials: HTTPAuthorizationCredentials = Depends(security), + db: Session = Depends(get_db) +) -> Dict[str, str]: + """Logout current user""" + + token = credentials.credentials + + # Logout based on user's auth provider + if user.auth_provider.value == "oidc" and settings.oidc.enabled: + async with OIDCAuthService(db, settings) as service: + await service.logout(user, token) + elif user.auth_provider.value == "email" and settings.email.enabled: + # Email service can implement logout if needed + pass + + return {"message": "Logged out successfully"} + + +@router.get("/me", response_model=Dict[str, Any]) +async def get_current_user_profile( + user: User = Depends(get_current_user) +) -> Dict[str, Any]: + """Get current user profile""" + + return user.to_dict(include_sensitive=False) + + +@router.put("/me", response_model=Dict[str, Any]) +async def update_current_user_profile( + profile_update: UserProfileUpdate, + user: User = Depends(get_current_user), + db: Session = Depends(get_db) +) -> Dict[str, Any]: + """Update current user profile""" + + # Update based on auth provider + if user.auth_provider.value == "email" and settings.email.enabled: + service = EmailAuthService(db, settings) + updated_user = await service.update_user_profile( + user=user, + given_name=profile_update.given_name, + family_name=profile_update.family_name, + display_name=profile_update.display_name + ) + return updated_user.to_dict(include_sensitive=False) + + # OIDC users might have restricted profile updates + if user.auth_provider.value == "oidc": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Profile updates are managed through your identity provider" + ) + + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Profile updates not available" + ) + + +@router.post("/resend-verification", response_model=Dict[str, str]) +async def resend_verification( + email_request: EmailLoginRequest, + db: Session = Depends(get_db) +) -> Dict[str, str]: + """Resend verification email""" + + if not settings.email.enabled: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Email authentication is not enabled" + ) + + service = EmailAuthService(db, settings) + + await service.resend_verification(email_request.email) + + return { + "message": "If the email exists and is unverified, a verification email has been sent", + "email": email_request.email + } + + +@router.get("/providers") +async def get_auth_providers() -> Dict[str, Any]: + """Get available authentication providers""" + + providers = [] + + if settings.oidc.enabled: + providers.append({ + "type": "oidc", + "name": "Nextcloud", + "enabled": True, + "issuer": settings.oidc.issuer + }) + + if settings.email.enabled: + providers.append({ + "type": "email", + "name": "Email", + "enabled": True, + "features": ["magic_link", "verification"] + }) + + return { + "providers": providers, + "default": "oidc" if settings.oidc.enabled else "email" + } diff --git a/backend/src/config/settings.py b/backend/src/config/settings.py new file mode 100644 index 00000000..f0dc5e65 --- /dev/null +++ b/backend/src/config/settings.py @@ -0,0 +1,347 @@ +""" +Application Settings and Configuration Management + +This module provides centralized configuration management using Pydantic Settings. +All environment variables and application settings are defined here with proper +validation and type hints. +""" + +from typing import Optional, Dict, Any, List +from functools import lru_cache +from pydantic import Field, field_validator, ConfigDict +from pydantic_settings import BaseSettings +from pathlib import Path + + +class DatabaseSettings(BaseSettings): + """Database connection settings""" + + host: str = Field(default="127.0.0.1", env="MYSQL_HOST") + port: int = Field(default=3306, env="MYSQL_PORT") + database: str = Field(default="stupa", env="MYSQL_DB") + username: str = Field(default="stupa", env="MYSQL_USER") + password: str = Field(default="secret", env="MYSQL_PASSWORD") + pool_size: int = Field(default=10, env="DB_POOL_SIZE") + max_overflow: int = Field(default=20, env="DB_MAX_OVERFLOW") + pool_pre_ping: bool = Field(default=True, env="DB_POOL_PRE_PING") + echo: bool = Field(default=False, env="DB_ECHO") + + @property + def dsn(self) -> str: + """Generate database connection string""" + return ( + f"mysql+pymysql://{self.username}:{self.password}@" + f"{self.host}:{self.port}/{self.database}?charset=utf8mb4" + ) + + model_config = ConfigDict(env_prefix="") + + +class SecuritySettings(BaseSettings): + """Security-related settings""" + + master_key: str = Field(default="", env="MASTER_KEY") + jwt_secret_key: Optional[str] = Field(default=None, env="JWT_SECRET_KEY") + jwt_algorithm: str = Field(default="HS256", env="JWT_ALGORITHM") + access_token_expire_minutes: int = Field(default=30, env="ACCESS_TOKEN_EXPIRE_MINUTES") + refresh_token_expire_days: int = Field(default=7, env="REFRESH_TOKEN_EXPIRE_DAYS") + api_key_header: str = Field(default="X-API-Key", env="API_KEY_HEADER") + cors_origins: List[str] = Field(default=["*"], env="CORS_ORIGINS") + cors_credentials: bool = Field(default=True, env="CORS_CREDENTIALS") + cors_methods: List[str] = Field(default=["*"], env="CORS_METHODS") + cors_headers: List[str] = Field(default=["*"], env="CORS_HEADERS") + encryption_key: str = Field(default="", env="ENCRYPTION_KEY") + + @field_validator("cors_origins", "cors_methods", "cors_headers", mode="before") + @classmethod + def parse_cors_list(cls, v): + if isinstance(v, str): + return [item.strip() for item in v.split(",")] + return v + + model_config = ConfigDict(env_prefix="") + + +class OIDCSettings(BaseSettings): + """OIDC/OAuth2 settings for Nextcloud integration""" + + enabled: bool = Field(default=False, env="OIDC_ENABLED") + issuer: str = Field(default="", env="OIDC_ISSUER") # e.g., https://nextcloud.example.com + client_id: str = Field(default="", env="OIDC_CLIENT_ID") + client_secret: str = Field(default="", env="OIDC_CLIENT_SECRET") + redirect_uri: str = Field(default="", env="OIDC_REDIRECT_URI") + scope: str = Field(default="openid profile email groups", env="OIDC_SCOPE") + + # Role mapping from OIDC groups + admin_groups: List[str] = Field(default=[], env="OIDC_ADMIN_GROUPS") + budget_reviewer_groups: List[str] = Field(default=[], env="OIDC_BUDGET_REVIEWER_GROUPS") + finance_reviewer_groups: List[str] = Field(default=[], env="OIDC_FINANCE_REVIEWER_GROUPS") + asta_groups: List[str] = Field(default=[], env="OIDC_ASTA_GROUPS") + + # Auto-create users from OIDC + auto_create_users: bool = Field(default=True, env="OIDC_AUTO_CREATE_USERS") + + @field_validator("admin_groups", "budget_reviewer_groups", "finance_reviewer_groups", "asta_groups", mode="before") + @classmethod + def parse_groups(cls, v): + if isinstance(v, str): + return [item.strip() for item in v.split(",") if item.strip()] + return v + + model_config = ConfigDict(env_prefix="") + + +class EmailSettings(BaseSettings): + """Email configuration settings""" + + enabled: bool = Field(default=False, env="EMAIL_ENABLED") + smtp_host: str = Field(default="localhost", env="SMTP_HOST") + smtp_port: int = Field(default=587, env="SMTP_PORT") + smtp_tls: bool = Field(default=True, env="SMTP_TLS") + smtp_ssl: bool = Field(default=False, env="SMTP_SSL") + smtp_username: str = Field(default="", env="SMTP_USERNAME") + smtp_password: str = Field(default="", env="SMTP_PASSWORD") + from_email: str = Field(default="noreply@example.com", env="EMAIL_FROM") + from_name: str = Field(default="STUPA System", env="EMAIL_FROM_NAME") + + # Email templates + verification_subject: str = Field(default="Verify your email", env="EMAIL_VERIFICATION_SUBJECT") + magic_link_subject: str = Field(default="Login to STUPA", env="EMAIL_MAGIC_LINK_SUBJECT") + application_notification_subject: str = Field(default="Application Status Update", env="EMAIL_APP_NOTIFICATION_SUBJECT") + + model_config = ConfigDict(env_prefix="") + + +class RateLimitSettings(BaseSettings): + """Rate limiting settings""" + + enabled: bool = Field(default=True, env="RATE_LIMIT_ENABLED") + ip_per_minute: int = Field(default=60, env="RATE_IP_PER_MIN") + key_per_minute: int = Field(default=30, env="RATE_KEY_PER_MIN") + global_per_minute: int = Field(default=1000, env="RATE_GLOBAL_PER_MIN") + burst_size: int = Field(default=10, env="RATE_BURST_SIZE") + + model_config = ConfigDict(env_prefix="") + + +class StorageSettings(BaseSettings): + """File storage settings""" + + upload_dir: Path = Field(default=Path("/tmp/uploads"), env="UPLOAD_DIR") + template_dir: Path = Field(default=Path("/tmp/templates"), env="TEMPLATE_DIR") + max_file_size: int = Field(default=10 * 1024 * 1024, env="MAX_FILE_SIZE") # 10MB + allowed_extensions: List[str] = Field( + default=["pdf", "json", "jpg", "jpeg", "png"], + env="ALLOWED_EXTENSIONS" + ) + temp_dir: Path = Field(default=Path("/tmp"), env="TEMP_DIR") + attachment_storage: str = Field(default="database", env="ATTACHMENT_STORAGE") # database or filesystem + filesystem_path: Optional[Path] = Field(default=None, env="FILESYSTEM_PATH") + + @field_validator("allowed_extensions", mode="before") + @classmethod + def parse_extensions(cls, v): + if isinstance(v, str): + return [ext.strip().lower() for ext in v.split(",")] + return v + + def validate_paths(self): + """Create directories if they don't exist""" + self.upload_dir.mkdir(parents=True, exist_ok=True) + self.template_dir.mkdir(parents=True, exist_ok=True) + self.temp_dir.mkdir(parents=True, exist_ok=True) + if self.filesystem_path: + self.filesystem_path.mkdir(parents=True, exist_ok=True) + + model_config = ConfigDict(env_prefix="") + + +class WorkflowSettings(BaseSettings): + """Workflow configuration settings""" + + # Required number of votes for approval + required_votes: int = Field(default=5, env="WORKFLOW_REQUIRED_VOTES") + + # Vote thresholds (percentage) + approval_threshold: float = Field(default=50.0, env="WORKFLOW_APPROVAL_THRESHOLD") + + # Timeouts (in days) + review_timeout_days: int = Field(default=14, env="WORKFLOW_REVIEW_TIMEOUT_DAYS") + voting_timeout_days: int = Field(default=7, env="WORKFLOW_VOTING_TIMEOUT_DAYS") + + # Allow manual status changes + allow_manual_status_change: bool = Field(default=True, env="WORKFLOW_ALLOW_MANUAL_STATUS_CHANGE") + + # Auto-lock on submission + auto_lock_on_submission: bool = Field(default=True, env="WORKFLOW_AUTO_LOCK_ON_SUBMISSION") + + model_config = ConfigDict(env_prefix="") + + +class ApplicationSettings(BaseSettings): + """General application settings""" + + app_name: str = Field(default="STUPA PDF API", env="APP_NAME") + app_version: str = Field(default="3.0.0", env="APP_VERSION") + debug: bool = Field(default=False, env="DEBUG") + environment: str = Field(default="production", env="ENVIRONMENT") + log_level: str = Field(default="INFO", env="LOG_LEVEL") + timezone: str = Field(default="Europe/Berlin", env="TIMEZONE") + + # Frontend URL for emails and redirects + frontend_url: str = Field(default="http://localhost:3001", env="FRONTEND_URL") + + # API settings + api_prefix: str = Field(default="/api", env="API_PREFIX") + docs_url: Optional[str] = Field(default="/docs", env="DOCS_URL") + redoc_url: Optional[str] = Field(default="/redoc", env="REDOC_URL") + openapi_url: Optional[str] = Field(default="/openapi.json", env="OPENAPI_URL") + + # Feature flags + enable_metrics: bool = Field(default=False, env="ENABLE_METRICS") + enable_tracing: bool = Field(default=False, env="ENABLE_TRACING") + enable_cache: bool = Field(default=True, env="ENABLE_CACHE") + cache_ttl: int = Field(default=300, env="CACHE_TTL") # seconds + + # New feature flags + enable_form_designer: bool = Field(default=True, env="ENABLE_FORM_DESIGNER") + enable_pdf_upload: bool = Field(default=True, env="ENABLE_PDF_UPLOAD") + enable_workflow: bool = Field(default=True, env="ENABLE_WORKFLOW") + + @field_validator("environment") + @classmethod + def validate_environment(cls, v): + allowed = ["development", "staging", "production", "testing"] + if v.lower() not in allowed: + raise ValueError(f"environment must be one of {allowed}") + return v.lower() + + @field_validator("log_level") + @classmethod + def validate_log_level(cls, v): + allowed = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] + if v.upper() not in allowed: + raise ValueError(f"log_level must be one of {allowed}") + return v.upper() + + model_config = ConfigDict(env_prefix="") + + +class Settings(BaseSettings): + """Main settings class that aggregates all setting groups""" + + database: DatabaseSettings = Field(default_factory=DatabaseSettings) + security: SecuritySettings = Field(default_factory=SecuritySettings) + oidc: OIDCSettings = Field(default_factory=OIDCSettings) + email: EmailSettings = Field(default_factory=EmailSettings) + rate_limit: RateLimitSettings = Field(default_factory=RateLimitSettings) + storage: StorageSettings = Field(default_factory=StorageSettings) + workflow: WorkflowSettings = Field(default_factory=WorkflowSettings) + app: ApplicationSettings = Field(default_factory=ApplicationSettings) + + # Dynamic configuration support + config_file: Optional[Path] = Field(default=None, env="CONFIG_FILE") + config_overrides: Dict[str, Any] = Field(default_factory=dict) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + # Validate and create necessary directories + self.storage.validate_paths() + + # Load configuration from file if specified + if self.config_file and self.config_file.exists(): + self._load_config_file() + + def _load_config_file(self): + """Load configuration from JSON or YAML file""" + import json + try: + import yaml + has_yaml = True + except ImportError: + has_yaml = False + + if self.config_file.suffix == ".json": + with open(self.config_file) as f: + config = json.load(f) + elif self.config_file.suffix in [".yaml", ".yml"] and has_yaml: + with open(self.config_file) as f: + config = yaml.safe_load(f) + else: + return + + # Apply overrides from config file + self._apply_overrides(config) + + def _apply_overrides(self, overrides: Dict[str, Any]): + """Apply configuration overrides""" + for key, value in overrides.items(): + if "." in key: + # Nested configuration + parts = key.split(".") + obj = self + for part in parts[:-1]: + obj = getattr(obj, part) + setattr(obj, parts[-1], value) + else: + if hasattr(self, key): + setattr(self, key, value) + + def to_dict(self) -> Dict[str, Any]: + """Export settings as dictionary""" + return { + "database": { + "host": self.database.host, + "port": self.database.port, + "database": self.database.database, + }, + "security": { + "cors_origins": self.security.cors_origins, + "api_key_header": self.security.api_key_header, + }, + "oidc": { + "enabled": self.oidc.enabled, + "issuer": self.oidc.issuer, + }, + "email": { + "enabled": self.email.enabled, + "from_email": self.email.from_email, + }, + "rate_limit": { + "enabled": self.rate_limit.enabled, + "ip_per_minute": self.rate_limit.ip_per_minute, + }, + "storage": { + "upload_dir": str(self.storage.upload_dir), + "template_dir": str(self.storage.template_dir), + "max_file_size": self.storage.max_file_size, + }, + "workflow": { + "required_votes": self.workflow.required_votes, + "approval_threshold": self.workflow.approval_threshold, + }, + "app": { + "name": self.app.app_name, + "version": self.app.app_version, + "environment": self.app.environment, + "enable_form_designer": self.app.enable_form_designer, + "enable_pdf_upload": self.app.enable_pdf_upload, + } + } + + model_config = ConfigDict( + env_file=".env", + env_file_encoding="utf-8", + case_sensitive=False, + extra="allow" + ) + + +@lru_cache() +def get_settings() -> Settings: + """Get cached settings instance""" + return Settings() + + +# Convenience function for getting settings +settings = get_settings() diff --git a/backend/src/core/container.py b/backend/src/core/container.py new file mode 100644 index 00000000..d75673bb --- /dev/null +++ b/backend/src/core/container.py @@ -0,0 +1,401 @@ +""" +Dependency Injection Container + +This module provides a dependency injection container for managing +services, repositories, and other dependencies throughout the application. +""" + +from typing import Dict, Any, Type, Optional, Callable, TypeVar, Generic +from functools import lru_cache +import inspect +import logging +from contextlib import contextmanager + +from sqlalchemy.orm import Session, sessionmaker + +from ..config.settings import Settings, get_settings +from ..repositories.base import BaseRepository +from ..services.base import BaseService + +logger = logging.getLogger(__name__) + +T = TypeVar('T') + + +class ServiceRegistry: + """Registry for storing service instances and factories""" + + def __init__(self): + self._services: Dict[str, Any] = {} + self._factories: Dict[str, Callable] = {} + self._singletons: Dict[str, Any] = {} + + def register( + self, + name: str, + factory: Optional[Callable] = None, + instance: Optional[Any] = None, + singleton: bool = False + ): + """ + Register a service. + + Args: + name: Service name + factory: Factory function to create service instances + instance: Pre-created instance + singleton: Whether to create only one instance + """ + if instance is not None: + self._services[name] = instance + elif factory is not None: + if singleton: + self._singletons[name] = None + self._factories[name] = factory + else: + raise ValueError("Either factory or instance must be provided") + + def get(self, name: str) -> Any: + """Get a service instance""" + # Check if we have a direct instance + if name in self._services: + return self._services[name] + + # Check if it's a singleton that's already created + if name in self._singletons and self._singletons[name] is not None: + return self._singletons[name] + + # Check if we have a factory + if name in self._factories: + instance = self._factories[name]() + + # Store singleton if needed + if name in self._singletons: + self._singletons[name] = instance + + return instance + + raise KeyError(f"Service '{name}' not found") + + def has(self, name: str) -> bool: + """Check if a service is registered""" + return name in self._services or name in self._factories + + def clear(self): + """Clear all registered services""" + self._services.clear() + self._factories.clear() + self._singletons.clear() + + +class Container: + """ + Main dependency injection container. + + This container manages the lifecycle of all services, repositories, + and other dependencies in the application. + """ + + def __init__(self, settings: Optional[Settings] = None): + self.settings = settings or get_settings() + self._registry = ServiceRegistry() + self._session_factory: Optional[sessionmaker] = None + self._current_session: Optional[Session] = None + self._initialize() + + def _initialize(self): + """Initialize the container with default services""" + # Register settings as a singleton + self._registry.register("settings", instance=self.settings) + + # Register session factory + self._registry.register( + "session_factory", + factory=self._create_session_factory, + singleton=True + ) + + def _create_session_factory(self) -> sessionmaker: + """Create SQLAlchemy session factory""" + if self._session_factory is None: + from sqlalchemy import create_engine + + engine = create_engine( + self.settings.database.dsn, + pool_size=self.settings.database.pool_size, + max_overflow=self.settings.database.max_overflow, + pool_pre_ping=self.settings.database.pool_pre_ping, + echo=self.settings.database.echo + ) + + self._session_factory = sessionmaker( + bind=engine, + autoflush=False, + autocommit=False + ) + + return self._session_factory + + @contextmanager + def session_scope(self): + """ + Provide a transactional scope for database operations. + + Usage: + with container.session_scope() as session: + # Use session + pass + """ + session_factory = self._registry.get("session_factory") + session = session_factory() + + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + def get_session(self) -> Session: + """Get current database session""" + if self._current_session is None: + session_factory = self._registry.get("session_factory") + self._current_session = session_factory() + return self._current_session + + def close_session(self): + """Close current database session""" + if self._current_session: + self._current_session.close() + self._current_session = None + + def register_repository( + self, + name: str, + repository_class: Type[BaseRepository], + singleton: bool = False + ): + """ + Register a repository class. + + Args: + name: Repository name + repository_class: Repository class + singleton: Whether to create only one instance + """ + def factory(): + session = self.get_session() + return repository_class(session) + + self._registry.register(name, factory=factory, singleton=singleton) + + def register_service( + self, + name: str, + service_class: Type[BaseService], + dependencies: Optional[Dict[str, str]] = None, + singleton: bool = False + ): + """ + Register a service class with dependency injection. + + Args: + name: Service name + service_class: Service class + dependencies: Dictionary mapping parameter names to service names + singleton: Whether to create only one instance + """ + def factory(): + # Get constructor signature + sig = inspect.signature(service_class.__init__) + kwargs = {} + + # Resolve dependencies + for param_name, param in sig.parameters.items(): + if param_name == "self": + continue + + # Check if we have a mapping for this parameter + if dependencies and param_name in dependencies: + service_name = dependencies[param_name] + if self._registry.has(service_name): + kwargs[param_name] = self._registry.get(service_name) + # Try to auto-resolve by parameter name + elif self._registry.has(param_name): + kwargs[param_name] = self._registry.get(param_name) + # Check if it's the container itself + elif param.annotation == Container or param_name == "container": + kwargs[param_name] = self + # Check if it's settings + elif param.annotation == Settings or param_name == "settings": + kwargs[param_name] = self.settings + # Check if it's a session + elif param.annotation == Session or param_name == "session": + kwargs[param_name] = self.get_session() + # Skip if it has a default value + elif param.default != inspect.Parameter.empty: + continue + else: + logger.warning( + f"Could not resolve dependency '{param_name}' for service '{name}'" + ) + + return service_class(**kwargs) + + self._registry.register(name, factory=factory, singleton=singleton) + + def get(self, name: str) -> Any: + """Get a service or repository instance""" + return self._registry.get(name) + + def get_repository(self, name: str) -> BaseRepository: + """Get a repository instance""" + repo = self._registry.get(name) + if not isinstance(repo, BaseRepository): + raise TypeError(f"'{name}' is not a repository") + return repo + + def get_service(self, name: str) -> BaseService: + """Get a service instance""" + service = self._registry.get(name) + if not isinstance(service, BaseService): + raise TypeError(f"'{name}' is not a service") + return service + + def resolve(self, service_class: Type[T]) -> T: + """ + Resolve a service class to an instance. + + This method attempts to find a registered service by class type. + """ + # Search for registered service by class + for name in ["settings", "session_factory"]: + try: + instance = self._registry.get(name) + if isinstance(instance, service_class): + return instance + except KeyError: + continue + + # Try to create a new instance with dependency injection + return self._auto_wire(service_class) + + def _auto_wire(self, service_class: Type[T]) -> T: + """ + Automatically wire dependencies for a service class. + + Args: + service_class: Service class to instantiate + + Returns: + Service instance with dependencies injected + """ + sig = inspect.signature(service_class.__init__) + kwargs = {} + + for param_name, param in sig.parameters.items(): + if param_name == "self": + continue + + # Try to resolve by type annotation + if param.annotation != inspect.Parameter.empty: + if param.annotation == Container: + kwargs[param_name] = self + elif param.annotation == Settings: + kwargs[param_name] = self.settings + elif param.annotation == Session: + kwargs[param_name] = self.get_session() + elif self._registry.has(param.annotation.__name__.lower()): + kwargs[param_name] = self._registry.get(param.annotation.__name__.lower()) + + # Try to resolve by parameter name + if param_name not in kwargs and self._registry.has(param_name): + kwargs[param_name] = self._registry.get(param_name) + + # Skip if it has a default value + if param_name not in kwargs and param.default == inspect.Parameter.empty: + raise ValueError( + f"Cannot resolve dependency '{param_name}' for {service_class.__name__}" + ) + + return service_class(**kwargs) + + def register_provider(self, provider: 'BaseProvider'): + """ + Register all services from a provider. + + Args: + provider: Provider instance + """ + provider.register(self) + + def clear(self): + """Clear all registered services and close session""" + self.close_session() + self._registry.clear() + + +class BaseProvider: + """Base class for service providers""" + + def register(self, container: Container): + """ + Register services with the container. + + Args: + container: Container instance + """ + raise NotImplementedError + + +# Global container instance +_container: Optional[Container] = None + + +def get_container() -> Container: + """Get the global container instance""" + global _container + if _container is None: + _container = Container() + return _container + + +def set_container(container: Container): + """Set the global container instance""" + global _container + _container = container + + +@lru_cache() +def create_container(settings: Optional[Settings] = None) -> Container: + """Create and configure a new container instance""" + container = Container(settings) + + # Register default repositories + from ..repositories.application import ApplicationRepository + from ..repositories.attachment import AttachmentRepository + + container.register_repository("application_repository", ApplicationRepository) + container.register_repository("attachment_repository", AttachmentRepository) + + # Register default services + from ..services.application import ApplicationService + from ..services.pdf import PDFService + from ..services.auth import AuthService + + container.register_service( + "application_service", + ApplicationService, + dependencies={ + "repository": "application_repository", + "pdf_service": "pdf_service" + }, + singleton=True + ) + + container.register_service("pdf_service", PDFService, singleton=True) + container.register_service("auth_service", AuthService, singleton=True) + + return container diff --git a/backend/src/core/database.py b/backend/src/core/database.py new file mode 100644 index 00000000..42bbaa66 --- /dev/null +++ b/backend/src/core/database.py @@ -0,0 +1,402 @@ +""" +Database Initialization and Management + +This module provides database initialization, connection management, +and migration support for the application. +""" + +from typing import Optional, Generator, Any +from contextlib import contextmanager +import logging + +from sqlalchemy import create_engine, event, text +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session, sessionmaker, scoped_session +from sqlalchemy.pool import QueuePool +from sqlalchemy.exc import SQLAlchemyError + +from ..config.settings import Settings, get_settings +from ..models.base import Base +from ..models.application import ( + Application, + ApplicationAttachment, + Attachment, + ComparisonOffer, + CostPositionJustification, + Counter +) + +logger = logging.getLogger(__name__) + + +class DatabaseManager: + """Manages database connections and operations""" + + def __init__(self, settings: Optional[Settings] = None): + """ + Initialize database manager. + + Args: + settings: Application settings + """ + self.settings = settings or get_settings() + self._engine: Optional[Engine] = None + self._session_factory: Optional[sessionmaker] = None + self._scoped_session: Optional[scoped_session] = None + + @property + def engine(self) -> Engine: + """Get or create database engine""" + if self._engine is None: + self._engine = self._create_engine() + return self._engine + + @property + def session_factory(self) -> sessionmaker: + """Get or create session factory""" + if self._session_factory is None: + self._session_factory = sessionmaker( + bind=self.engine, + autoflush=False, + autocommit=False, + expire_on_commit=False + ) + return self._session_factory + + @property + def scoped_session(self) -> scoped_session: + """Get or create scoped session""" + if self._scoped_session is None: + self._scoped_session = scoped_session(self.session_factory) + return self._scoped_session + + def _create_engine(self) -> Engine: + """Create SQLAlchemy engine with configuration""" + engine = create_engine( + self.settings.database.dsn, + poolclass=QueuePool, + pool_size=self.settings.database.pool_size, + max_overflow=self.settings.database.max_overflow, + pool_pre_ping=self.settings.database.pool_pre_ping, + echo=self.settings.database.echo, + future=True, + connect_args={ + "connect_timeout": 10, + "charset": "utf8mb4" + } + ) + + # Add event listeners + self._setup_event_listeners(engine) + + return engine + + def _setup_event_listeners(self, engine: Engine): + """Setup database event listeners""" + + @event.listens_for(engine, "connect") + def set_sqlite_pragma(dbapi_conn, connection_record): + """Set SQLite pragmas for better performance""" + if "sqlite" in self.settings.database.dsn: + cursor = dbapi_conn.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.execute("PRAGMA journal_mode=WAL") + cursor.close() + + @event.listens_for(engine, "checkout") + def ping_connection(dbapi_conn, connection_record, connection_proxy): + """Ping connection to verify it's still valid""" + if self.settings.database.pool_pre_ping: + try: + dbapi_conn.ping(False) + except Exception: + # Connection is dead, raise DisconnectionError + raise SQLAlchemyError("Connection failed ping test") + + def create_session(self) -> Session: + """Create a new database session""" + return self.session_factory() + + @contextmanager + def session_scope(self) -> Generator[Session, None, None]: + """ + Provide a transactional scope for database operations. + + Yields: + Database session + + Usage: + with db_manager.session_scope() as session: + # Perform database operations + pass + """ + session = self.create_session() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + def init_database(self, drop_existing: bool = False): + """ + Initialize database tables. + + Args: + drop_existing: Whether to drop existing tables first + """ + try: + if drop_existing: + logger.warning("Dropping existing database tables...") + Base.metadata.drop_all(bind=self.engine) + + logger.info("Creating database tables...") + Base.metadata.create_all(bind=self.engine) + + # Initialize default data + self._init_default_data() + + logger.info("Database initialization complete") + + except SQLAlchemyError as e: + logger.error(f"Database initialization failed: {e}") + raise + + def _init_default_data(self): + """Initialize default data in the database""" + with self.session_scope() as session: + # Initialize counters + counters = [ + { + "key": "application_id", + "value": 0, + "prefix": "PA", + "format_string": "{prefix}{value:06d}" + }, + { + "key": "attachment_id", + "value": 0, + "prefix": "ATT", + "format_string": "{prefix}{value:08d}" + } + ] + + for counter_data in counters: + existing = session.query(Counter).filter_by( + key=counter_data["key"] + ).first() + + if not existing: + counter = Counter(**counter_data) + session.add(counter) + logger.info(f"Created counter: {counter_data['key']}") + + def verify_connection(self) -> bool: + """ + Verify database connection is working. + + Returns: + True if connection is successful + """ + try: + with self.engine.connect() as conn: + result = conn.execute(text("SELECT 1")) + return result.scalar() == 1 + except SQLAlchemyError as e: + logger.error(f"Database connection verification failed: {e}") + return False + + def get_table_stats(self) -> Dict[str, int]: + """ + Get row counts for all tables. + + Returns: + Dictionary mapping table names to row counts + """ + stats = {} + with self.session_scope() as session: + for table in Base.metadata.sorted_tables: + try: + count = session.query(table).count() + stats[table.name] = count + except SQLAlchemyError: + stats[table.name] = -1 + return stats + + def execute_raw_sql(self, sql: str, params: Optional[Dict[str, Any]] = None) -> Any: + """ + Execute raw SQL query. + + Args: + sql: SQL query string + params: Query parameters + + Returns: + Query result + + Warning: + Use with caution - prefer ORM queries when possible + """ + with self.engine.connect() as conn: + result = conn.execute(text(sql), params or {}) + conn.commit() + return result + + def backup_database(self, backup_path: str): + """ + Create database backup. + + Args: + backup_path: Path to save backup file + + Note: + Implementation depends on database type + """ + if "sqlite" in self.settings.database.dsn: + import shutil + import re + + # Extract database file path from DSN + match = re.search(r'sqlite:///(.+)', self.settings.database.dsn) + if match: + db_path = match.group(1) + shutil.copy2(db_path, backup_path) + logger.info(f"Database backed up to {backup_path}") + else: + # For MySQL, would need to use mysqldump + logger.warning("Backup not implemented for this database type") + + def close(self): + """Close database connections""" + if self._scoped_session: + self._scoped_session.remove() + if self._engine: + self._engine.dispose() + self._engine = None + self._session_factory = None + self._scoped_session = None + + +# Global database manager instance +_db_manager: Optional[DatabaseManager] = None + + +def get_db_manager() -> DatabaseManager: + """Get global database manager instance""" + global _db_manager + if _db_manager is None: + _db_manager = DatabaseManager() + return _db_manager + + +def init_database(settings: Optional[Settings] = None, drop_existing: bool = False): + """ + Initialize the database. + + Args: + settings: Application settings + drop_existing: Whether to drop existing tables + """ + db_manager = DatabaseManager(settings) + db_manager.init_database(drop_existing) + return db_manager + + +def get_db_session() -> Generator[Session, None, None]: + """ + Get database session for FastAPI dependency injection. + + Yields: + Database session + """ + db_manager = get_db_manager() + session = db_manager.create_session() + try: + yield session + finally: + session.close() + + +class DatabaseHealthCheck: + """Database health check utilities""" + + @staticmethod + def check_connection(db_manager: Optional[DatabaseManager] = None) -> Dict[str, Any]: + """ + Check database connection health. + + Returns: + Health check results + """ + manager = db_manager or get_db_manager() + + try: + # Test connection + is_connected = manager.verify_connection() + + if is_connected: + # Get table statistics + stats = manager.get_table_stats() + total_rows = sum(v for v in stats.values() if v >= 0) + + return { + "status": "healthy", + "connected": True, + "tables": len(stats), + "total_rows": total_rows, + "details": stats + } + else: + return { + "status": "unhealthy", + "connected": False, + "error": "Connection test failed" + } + + except Exception as e: + return { + "status": "unhealthy", + "connected": False, + "error": str(e) + } + + @staticmethod + def check_migrations() -> Dict[str, Any]: + """ + Check database migration status. + + Returns: + Migration status information + """ + # This would integrate with Alembic or similar migration tool + # For now, just check if tables exist + manager = get_db_manager() + + required_tables = [ + "applications", + "attachments", + "application_attachments", + "comparison_offers", + "cost_position_justifications", + "counters" + ] + + with manager.session_scope() as session: + existing_tables = [] + for table_name in required_tables: + try: + session.execute(text(f"SELECT 1 FROM {table_name} LIMIT 1")) + existing_tables.append(table_name) + except SQLAlchemyError: + pass + + missing_tables = set(required_tables) - set(existing_tables) + + return { + "status": "up_to_date" if not missing_tables else "pending", + "required_tables": required_tables, + "existing_tables": existing_tables, + "missing_tables": list(missing_tables) + } diff --git a/backend/src/main.py b/backend/src/main.py new file mode 100644 index 00000000..10e74776 --- /dev/null +++ b/backend/src/main.py @@ -0,0 +1,207 @@ +""" +Main FastAPI Application + +This module creates and configures the main FastAPI application with +modular routers, middleware, and dependency injection. +""" + +from contextlib import asynccontextmanager +from typing import Dict, Any + +from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.trustedhost import TrustedHostMiddleware +from fastapi.responses import JSONResponse +import logging + +from .config.settings import get_settings +from .core.container import create_container, get_container +from .core.database import init_database +from .api.middleware.rate_limit import RateLimitMiddleware +from .api.middleware.logging import LoggingMiddleware +from .api.middleware.error_handler import ErrorHandlerMiddleware +from .api.routes import ( + application_router, + attachment_router, + pdf_router, + auth_router, + health_router +) +from .providers.pdf_qsm import QSMProvider +from .providers.pdf_vsm import VSMProvider +from .services.pdf import PDFService + +# Configure logging +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger(__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """ + Application lifespan manager. + Handles startup and shutdown events. + """ + # Startup + logger.info("Starting up application...") + + # Initialize settings + settings = get_settings() + + # Initialize database + init_database(settings) + + # Initialize dependency injection container + container = create_container(settings) + + # Register PDF providers + pdf_service = container.get_service("pdf_service") + pdf_service.register_provider(QSMProvider(settings)) + pdf_service.register_provider(VSMProvider(settings)) + + # Store container in app state + app.state.container = container + app.state.settings = settings + + logger.info(f"Application started in {settings.app.environment} mode") + + yield + + # Shutdown + logger.info("Shutting down application...") + container.clear() + logger.info("Application shutdown complete") + + +def create_app() -> FastAPI: + """ + Create and configure the FastAPI application. + + Returns: + Configured FastAPI application instance + """ + settings = get_settings() + + # Create FastAPI app + app = FastAPI( + title=settings.app.app_name, + version=settings.app.app_version, + description="PDF Processing API for STUPA Applications", + docs_url=settings.app.docs_url, + redoc_url=settings.app.redoc_url, + openapi_url=settings.app.openapi_url, + lifespan=lifespan + ) + + # Add CORS middleware + app.add_middleware( + CORSMiddleware, + allow_origins=settings.security.cors_origins, + allow_credentials=settings.security.cors_credentials, + allow_methods=settings.security.cors_methods, + allow_headers=settings.security.cors_headers, + ) + + # Add trusted host middleware + if settings.app.environment == "production": + app.add_middleware( + TrustedHostMiddleware, + allowed_hosts=["*"] # Configure based on your needs + ) + + # Add custom middleware + app.add_middleware(ErrorHandlerMiddleware) + app.add_middleware(LoggingMiddleware) + + if settings.rate_limit.enabled: + app.add_middleware( + RateLimitMiddleware, + settings=settings.rate_limit + ) + + # Include routers + app.include_router( + health_router, + prefix="/health", + tags=["health"] + ) + + app.include_router( + auth_router, + prefix=f"{settings.app.api_prefix}/auth", + tags=["authentication"] + ) + + app.include_router( + application_router, + prefix=f"{settings.app.api_prefix}/applications", + tags=["applications"] + ) + + app.include_router( + attachment_router, + prefix=f"{settings.app.api_prefix}/attachments", + tags=["attachments"] + ) + + app.include_router( + pdf_router, + prefix=f"{settings.app.api_prefix}/pdf", + tags=["pdf"] + ) + + # Root endpoint + @app.get("/", tags=["root"]) + async def root() -> Dict[str, Any]: + """Root endpoint returning API information""" + return { + "name": settings.app.app_name, + "version": settings.app.app_version, + "environment": settings.app.environment, + "docs": settings.app.docs_url, + "health": "/health" + } + + # Global exception handler + @app.exception_handler(Exception) + async def global_exception_handler(request: Request, exc: Exception): + """Handle uncaught exceptions""" + logger.error(f"Unhandled exception: {exc}", exc_info=True) + + if settings.app.debug: + return JSONResponse( + status_code=500, + content={ + "error": "Internal server error", + "detail": str(exc), + "type": type(exc).__name__ + } + ) + else: + return JSONResponse( + status_code=500, + content={"error": "Internal server error"} + ) + + return app + + +# Create app instance +app = create_app() + +# For development server +if __name__ == "__main__": + import uvicorn + + settings = get_settings() + + uvicorn.run( + "main:app", + host="0.0.0.0", + port=8000, + reload=settings.app.environment == "development", + log_level=settings.app.log_level.lower() + ) diff --git a/backend/src/migrations/001_add_oidc_and_templates.py b/backend/src/migrations/001_add_oidc_and_templates.py new file mode 100644 index 00000000..55bc6bd6 --- /dev/null +++ b/backend/src/migrations/001_add_oidc_and_templates.py @@ -0,0 +1,379 @@ +""" +Database migration for OIDC authentication, user management, and form templates + +This migration adds support for: +- User authentication with OIDC and email +- Role-based access control +- PDF template management +- Enhanced application workflow +- Voting and review system +""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql +from datetime import datetime + +# Revision identifiers +revision = '001_add_oidc_and_templates' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + """Apply migration changes""" + + # Create users table + op.create_table( + 'users', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('email', sa.String(255), nullable=False), + sa.Column('auth_provider', sa.Enum('LOCAL', 'OIDC', 'EMAIL', name='authprovider'), nullable=False), + sa.Column('oidc_sub', sa.String(255), nullable=True), + sa.Column('oidc_issuer', sa.String(255), nullable=True), + sa.Column('given_name', sa.String(255), nullable=True), + sa.Column('family_name', sa.String(255), nullable=True), + sa.Column('preferred_username', sa.String(255), nullable=True), + sa.Column('display_name', sa.String(255), nullable=True), + sa.Column('picture_url', sa.Text(), nullable=True), + sa.Column('verification_status', sa.Enum('UNVERIFIED', 'EMAIL_VERIFIED', 'OIDC_VERIFIED', 'FULLY_VERIFIED', name='verificationstatus'), nullable=False), + sa.Column('email_verified', sa.Boolean(), nullable=True, default=False), + sa.Column('email_verification_token', sa.String(255), nullable=True), + sa.Column('email_verification_sent_at', sa.DateTime(), nullable=True), + sa.Column('email_verified_at', sa.DateTime(), nullable=True), + sa.Column('last_login_at', sa.DateTime(), nullable=True), + sa.Column('last_activity_at', sa.DateTime(), nullable=True), + sa.Column('oidc_access_token', sa.Text(), nullable=True), + sa.Column('oidc_refresh_token', sa.Text(), nullable=True), + sa.Column('oidc_token_expires_at', sa.DateTime(), nullable=True), + sa.Column('oidc_claims', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False, default=datetime.utcnow), + sa.Column('updated_at', sa.DateTime(), nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id'), + comment='User accounts with OIDC support' + ) + op.create_index('idx_user_email', 'users', ['email']) + op.create_index('idx_user_oidc_sub', 'users', ['oidc_sub']) + op.create_index('idx_user_email_provider', 'users', ['email', 'auth_provider']) + op.create_index('idx_user_verification', 'users', ['verification_status', 'email_verified']) + + # Create roles table + op.create_table( + 'roles', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(100), nullable=False), + sa.Column('display_name', sa.String(255), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('oidc_role_claim', sa.String(255), nullable=True), + sa.Column('permissions', sa.JSON(), nullable=False, default=list), + sa.Column('is_system', sa.Boolean(), nullable=True, default=False), + sa.Column('is_admin', sa.Boolean(), nullable=True, default=False), + sa.Column('can_review_budget', sa.Boolean(), nullable=True, default=False), + sa.Column('can_review_finance', sa.Boolean(), nullable=True, default=False), + sa.Column('can_vote', sa.Boolean(), nullable=True, default=False), + sa.Column('priority', sa.Integer(), nullable=True, default=0), + sa.Column('created_at', sa.DateTime(), nullable=False, default=datetime.utcnow), + sa.Column('updated_at', sa.DateTime(), nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name'), + sa.UniqueConstraint('oidc_role_claim'), + comment='User roles for permission management' + ) + op.create_index('idx_role_name', 'roles', ['name']) + op.create_index('idx_role_oidc_claim', 'roles', ['oidc_role_claim']) + + # Create user_roles association table + op.create_table( + 'user_roles', + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('role_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['role_id'], ['roles.id'], ondelete='CASCADE'), + sa.UniqueConstraint('user_id', 'role_id', name='uq_user_role'), + comment='User-role associations' + ) + + # Create sessions table + op.create_table( + 'sessions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('session_token', sa.String(255), nullable=False), + sa.Column('refresh_token', sa.String(255), nullable=True), + sa.Column('ip_address', sa.String(45), nullable=True), + sa.Column('user_agent', sa.Text(), nullable=True), + sa.Column('expires_at', sa.DateTime(), nullable=False), + sa.Column('last_activity_at', sa.DateTime(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False, default=datetime.utcnow), + sa.Column('updated_at', sa.DateTime(), nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('session_token'), + sa.UniqueConstraint('refresh_token'), + comment='User sessions for tracking active logins' + ) + op.create_index('idx_session_user', 'sessions', ['user_id']) + op.create_index('idx_session_token', 'sessions', ['session_token']) + op.create_index('idx_session_expires', 'sessions', ['expires_at']) + + # Create form_templates table + op.create_table( + 'form_templates', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(255), nullable=False), + sa.Column('display_name', sa.String(255), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('form_type', sa.Enum('QSM', 'VSM', 'CUSTOM', name='formtype'), nullable=False), + sa.Column('pdf_file_path', sa.String(500), nullable=True), + sa.Column('pdf_file_name', sa.String(255), nullable=True), + sa.Column('pdf_file_size', sa.Integer(), nullable=True), + sa.Column('pdf_file_hash', sa.String(64), nullable=True), + sa.Column('is_active', sa.Boolean(), nullable=True, default=True), + sa.Column('is_public', sa.Boolean(), nullable=True, default=True), + sa.Column('requires_verification', sa.Boolean(), nullable=True, default=True), + sa.Column('allowed_roles', sa.JSON(), nullable=True), + sa.Column('form_design', sa.JSON(), nullable=True), + sa.Column('workflow_config', sa.JSON(), nullable=True), + sa.Column('usage_count', sa.Integer(), nullable=True, default=0), + sa.Column('version', sa.String(20), nullable=True, default='1.0.0'), + sa.Column('parent_template_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False, default=datetime.utcnow), + sa.Column('updated_at', sa.DateTime(), nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['parent_template_id'], ['form_templates.id']), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name'), + comment='Form templates for configurable PDF forms' + ) + op.create_index('idx_template_name', 'form_templates', ['name']) + op.create_index('idx_template_active_public', 'form_templates', ['is_active', 'is_public']) + op.create_index('idx_template_type_active', 'form_templates', ['form_type', 'is_active']) + + # Create field_mappings table + op.create_table( + 'field_mappings', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('template_id', sa.Integer(), nullable=False), + sa.Column('pdf_field_name', sa.String(255), nullable=False), + sa.Column('pdf_field_type', sa.String(50), nullable=True), + sa.Column('field_key', sa.String(255), nullable=False), + sa.Column('field_label', sa.String(255), nullable=False), + sa.Column('field_type', sa.Enum('TEXT', 'NUMBER', 'DATE', 'EMAIL', 'PHONE', 'CHECKBOX', 'RADIO', 'SELECT', 'TEXTAREA', 'FILE', 'SIGNATURE', 'CURRENCY', name='fieldtype'), nullable=False), + sa.Column('field_order', sa.Integer(), nullable=True, default=0), + sa.Column('is_required', sa.Boolean(), nullable=True, default=False), + sa.Column('is_readonly', sa.Boolean(), nullable=True, default=False), + sa.Column('is_hidden', sa.Boolean(), nullable=True, default=False), + sa.Column('is_email_field', sa.Boolean(), nullable=True, default=False), + sa.Column('is_name_field', sa.Boolean(), nullable=True, default=False), + sa.Column('validation_rules', sa.JSON(), nullable=True), + sa.Column('field_options', sa.JSON(), nullable=True), + sa.Column('default_value', sa.Text(), nullable=True), + sa.Column('placeholder', sa.String(500), nullable=True), + sa.Column('help_text', sa.Text(), nullable=True), + sa.Column('display_conditions', sa.JSON(), nullable=True), + sa.Column('transform_rules', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False, default=datetime.utcnow), + sa.Column('updated_at', sa.DateTime(), nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['template_id'], ['form_templates.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('template_id', 'pdf_field_name', name='uq_template_pdf_field'), + sa.UniqueConstraint('template_id', 'field_key', name='uq_template_field_key'), + comment='Field mappings for PDF form fields' + ) + op.create_index('idx_field_template', 'field_mappings', ['template_id']) + op.create_index('idx_field_template_order', 'field_mappings', ['template_id', 'field_order']) + + # Create form_designs table + op.create_table( + 'form_designs', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('template_id', sa.Integer(), nullable=False), + sa.Column('layout_type', sa.String(50), nullable=True, default='single-column'), + sa.Column('sections', sa.JSON(), nullable=False), + sa.Column('theme', sa.JSON(), nullable=True), + sa.Column('custom_css', sa.Text(), nullable=True), + sa.Column('components', sa.JSON(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False, default=datetime.utcnow), + sa.Column('updated_at', sa.DateTime(), nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['template_id'], ['form_templates.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('template_id'), + comment='Visual form designer configuration' + ) + + # Modify existing applications table + if op.get_bind().dialect.has_table(op.get_bind(), 'applications'): + # Add new columns to applications table + op.add_column('applications', sa.Column('user_id', sa.Integer(), nullable=True)) + op.add_column('applications', sa.Column('template_id', sa.Integer(), nullable=True)) + op.add_column('applications', sa.Column('locked_at', sa.DateTime(), nullable=True)) + op.add_column('applications', sa.Column('locked_by', sa.Integer(), nullable=True)) + op.add_column('applications', sa.Column('budget_reviewed_by', sa.Integer(), nullable=True)) + op.add_column('applications', sa.Column('budget_reviewed_at', sa.DateTime(), nullable=True)) + op.add_column('applications', sa.Column('budget_review_status', sa.String(50), nullable=True)) + op.add_column('applications', sa.Column('budget_review_comment', sa.Text(), nullable=True)) + op.add_column('applications', sa.Column('finance_reviewed_by', sa.Integer(), nullable=True)) + op.add_column('applications', sa.Column('finance_reviewed_at', sa.DateTime(), nullable=True)) + op.add_column('applications', sa.Column('finance_review_status', sa.String(50), nullable=True)) + op.add_column('applications', sa.Column('finance_review_comment', sa.Text(), nullable=True)) + op.add_column('applications', sa.Column('voting_opened_at', sa.DateTime(), nullable=True)) + op.add_column('applications', sa.Column('voting_closed_at', sa.DateTime(), nullable=True)) + op.add_column('applications', sa.Column('voting_result', sa.String(50), nullable=True)) + op.add_column('applications', sa.Column('votes_for', sa.Integer(), nullable=True, default=0)) + op.add_column('applications', sa.Column('votes_against', sa.Integer(), nullable=True, default=0)) + op.add_column('applications', sa.Column('votes_abstain', sa.Integer(), nullable=True, default=0)) + + # Create foreign key constraints + op.create_foreign_key('fk_app_user', 'applications', 'users', ['user_id'], ['id']) + op.create_foreign_key('fk_app_template', 'applications', 'form_templates', ['template_id'], ['id']) + op.create_foreign_key('fk_app_locker', 'applications', 'users', ['locked_by'], ['id']) + op.create_foreign_key('fk_app_budget_reviewer', 'applications', 'users', ['budget_reviewed_by'], ['id']) + op.create_foreign_key('fk_app_finance_reviewer', 'applications', 'users', ['finance_reviewed_by'], ['id']) + + # Create indexes + op.create_index('idx_app_user', 'applications', ['user_id']) + op.create_index('idx_app_template', 'applications', ['template_id']) + + # Create application_votes table + op.create_table( + 'application_votes', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('application_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('vote', sa.String(20), nullable=False), + sa.Column('comment', sa.Text(), nullable=True), + sa.Column('voted_at', sa.DateTime(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False, default=datetime.utcnow), + sa.Column('updated_at', sa.DateTime(), nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['application_id'], ['applications.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('application_id', 'user_id', name='uq_application_user_vote'), + comment='Application voting records' + ) + op.create_index('idx_vote_application', 'application_votes', ['application_id']) + op.create_index('idx_vote_user', 'application_votes', ['user_id']) + + # Create application_history table + op.create_table( + 'application_history', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('application_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('action', sa.String(100), nullable=False), + sa.Column('old_status', sa.String(50), nullable=True), + sa.Column('new_status', sa.String(50), nullable=True), + sa.Column('changes', sa.JSON(), nullable=True), + sa.Column('comment', sa.Text(), nullable=True), + sa.Column('timestamp', sa.DateTime(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False, default=datetime.utcnow), + sa.Column('updated_at', sa.DateTime(), nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['application_id'], ['applications.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['user_id'], ['users.id']), + sa.PrimaryKeyConstraint('id'), + comment='Application history tracking' + ) + op.create_index('idx_history_application', 'application_history', ['application_id']) + op.create_index('idx_history_user', 'application_history', ['user_id']) + op.create_index('idx_history_timestamp', 'application_history', ['timestamp']) + + # Create application_attachments table + op.create_table( + 'application_attachments', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('application_id', sa.Integer(), nullable=False), + sa.Column('file_name', sa.String(255), nullable=False), + sa.Column('file_path', sa.String(500), nullable=False), + sa.Column('file_size', sa.Integer(), nullable=False), + sa.Column('file_type', sa.String(100), nullable=True), + sa.Column('file_hash', sa.String(64), nullable=True), + sa.Column('uploaded_by', sa.Integer(), nullable=True), + sa.Column('uploaded_at', sa.DateTime(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False, default=datetime.utcnow), + sa.Column('updated_at', sa.DateTime(), nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['application_id'], ['applications.id'], ondelete='CASCADE'), + sa.ForeignKeyConstraint(['uploaded_by'], ['users.id']), + sa.PrimaryKeyConstraint('id'), + comment='Application attachment files' + ) + op.create_index('idx_attachment_application', 'application_attachments', ['application_id']) + + # Insert default roles + op.execute(""" + INSERT INTO roles (name, display_name, description, is_system, permissions) + VALUES + ('admin', 'Administrator', 'Full system access', TRUE, '["*"]'), + ('user', 'User', 'Basic user access', TRUE, '["read:own", "write:own"]'), + ('haushaltsbeauftragte', 'Haushaltsbeauftragte(r)', 'Budget reviewer', FALSE, '["review:budget", "read:applications"]'), + ('finanzreferent', 'Finanzreferent', 'Finance reviewer', FALSE, '["review:finance", "read:applications"]'), + ('asta', 'AStA Member', 'Can vote on applications', FALSE, '["vote:applications", "read:applications"]') + """) + + # Update application status enum + op.execute(""" + ALTER TABLE applications + MODIFY COLUMN status ENUM('draft', 'beantragt', 'bearbeitung_gesperrt', 'zu_pruefen', 'zur_abstimmung', 'genehmigt', 'abgelehnt', 'cancelled') + """) + + +def downgrade(): + """Revert migration changes""" + + # Drop new tables + op.drop_table('application_attachments') + op.drop_table('application_history') + op.drop_table('application_votes') + op.drop_table('form_designs') + op.drop_table('field_mappings') + op.drop_table('form_templates') + op.drop_table('sessions') + op.drop_table('user_roles') + op.drop_table('roles') + op.drop_table('users') + + # Remove foreign keys from applications table + if op.get_bind().dialect.has_table(op.get_bind(), 'applications'): + op.drop_constraint('fk_app_user', 'applications', type_='foreignkey') + op.drop_constraint('fk_app_template', 'applications', type_='foreignkey') + op.drop_constraint('fk_app_locker', 'applications', type_='foreignkey') + op.drop_constraint('fk_app_budget_reviewer', 'applications', type_='foreignkey') + op.drop_constraint('fk_app_finance_reviewer', 'applications', type_='foreignkey') + + # Drop indexes + op.drop_index('idx_app_user', 'applications') + op.drop_index('idx_app_template', 'applications') + + # Drop columns + op.drop_column('applications', 'user_id') + op.drop_column('applications', 'template_id') + op.drop_column('applications', 'locked_at') + op.drop_column('applications', 'locked_by') + op.drop_column('applications', 'budget_reviewed_by') + op.drop_column('applications', 'budget_reviewed_at') + op.drop_column('applications', 'budget_review_status') + op.drop_column('applications', 'budget_review_comment') + op.drop_column('applications', 'finance_reviewed_by') + op.drop_column('applications', 'finance_reviewed_at') + op.drop_column('applications', 'finance_review_status') + op.drop_column('applications', 'finance_review_comment') + op.drop_column('applications', 'voting_opened_at') + op.drop_column('applications', 'voting_closed_at') + op.drop_column('applications', 'voting_result') + op.drop_column('applications', 'votes_for') + op.drop_column('applications', 'votes_against') + op.drop_column('applications', 'votes_abstain') + + # Revert status enum + op.execute(""" + ALTER TABLE applications + MODIFY COLUMN status ENUM('draft', 'submitted', 'in_review', 'approved', 'rejected', 'cancelled', 'completed') + """) diff --git a/backend/src/models/application.py b/backend/src/models/application.py new file mode 100644 index 00000000..9ca8778c --- /dev/null +++ b/backend/src/models/application.py @@ -0,0 +1,620 @@ +""" +Application Database Models + +This module defines the database models for the application system. +""" + +from sqlalchemy import ( + Column, Integer, String, Text, DateTime, JSON, Boolean, + ForeignKey, UniqueConstraint, Index, Float, Enum as SQLEnum +) +from sqlalchemy.orm import relationship, backref +from sqlalchemy.dialects.mysql import LONGTEXT +import enum +from typing import Optional, Dict, Any +from datetime import datetime + +from .base import ExtendedBaseModel, BaseModel, TimestampMixin + + +class ApplicationStatus(enum.Enum): + """Application status enumeration""" + DRAFT = "draft" + BEANTRAGT = "beantragt" # Submitted + BEARBEITUNG_GESPERRT = "bearbeitung_gesperrt" # Processing locked + ZU_PRUEFEN = "zu_pruefen" # To be reviewed + ZUR_ABSTIMMUNG = "zur_abstimmung" # For voting + GENEHMIGT = "genehmigt" # Approved + ABGELEHNT = "abgelehnt" # Rejected + CANCELLED = "cancelled" + + +class ApplicationType(enum.Enum): + """Application type enumeration""" + QSM = "QSM" + VSM = "VSM" + + +class InstitutionType(enum.Enum): + """Institution type enumeration""" + STUDENT_FS = "stud-fs" + STUDENT_RF = "stud-rf" + STUDENT_HG = "stud-hg" + FACULTY = "faculty" + HS_INSTITUTION = "hs-institution" + + +class Application(ExtendedBaseModel): + """Main application model""" + + __tablename__ = "applications" + + # Core fields + pa_id = Column( + String(64), + unique=True, + nullable=False, + index=True, + comment="Public application ID" + ) + pa_key = Column( + String(255), + nullable=False, + comment="Application access key (hashed)" + ) + + # User relationship + user_id = Column( + Integer, + ForeignKey('users.id'), + nullable=True, + index=True, + comment="User who created the application" + ) + + # Template relationship + template_id = Column( + Integer, + ForeignKey('form_templates.id'), + nullable=True, + index=True, + comment="Form template used" + ) + + variant = Column( + SQLEnum(ApplicationType), + nullable=False, + default=ApplicationType.QSM, + comment="Application variant (QSM or VSM)" + ) + status = Column( + SQLEnum(ApplicationStatus), + nullable=False, + default=ApplicationStatus.DRAFT, + index=True, + comment="Application status" + ) + + # JSON payload containing all form data + payload = Column( + JSON, + nullable=False, + default=dict, + comment="Complete application payload" + ) + + # Searchable extracted fields for quick queries + institution_name = Column(String(255), index=True) + institution_type = Column(SQLEnum(InstitutionType), index=True) + applicant_first_name = Column(String(255), index=True) + applicant_last_name = Column(String(255), index=True) + applicant_email = Column(String(255), index=True) + project_name = Column(String(500), index=True) + project_start_date = Column(String(50)) + project_end_date = Column(String(50)) + total_amount = Column(Float, default=0.0) + + +class ApplicationVote(ExtendedBaseModel): + """Application voting model""" + + __tablename__ = "application_votes" + + application_id = Column( + Integer, + ForeignKey('applications.id', ondelete='CASCADE'), + nullable=False, + index=True + ) + + user_id = Column( + Integer, + ForeignKey('users.id', ondelete='CASCADE'), + nullable=False, + index=True + ) + + vote = Column( + String(20), + nullable=False, + comment="Vote: for, against, abstain" + ) + + comment = Column( + Text, + nullable=True, + comment="Optional vote comment" + ) + + voted_at = Column( + DateTime, + nullable=False, + default=datetime.utcnow + ) + + # Relationships + application = relationship("Application", back_populates="votes") + user = relationship("User") + + # Unique constraint + __table_args__ = ( + UniqueConstraint('application_id', 'user_id', name='uq_application_user_vote'), + ) + + +class ApplicationHistory(ExtendedBaseModel): + """Application history tracking""" + + __tablename__ = "application_history" + + application_id = Column( + Integer, + ForeignKey('applications.id', ondelete='CASCADE'), + nullable=False, + index=True + ) + + user_id = Column( + Integer, + ForeignKey('users.id'), + nullable=True, + index=True + ) + + action = Column( + String(100), + nullable=False, + comment="Action performed" + ) + + old_status = Column( + String(50), + nullable=True, + comment="Previous status" + ) + + new_status = Column( + String(50), + nullable=True, + comment="New status" + ) + + changes = Column( + JSON, + nullable=True, + default=dict, + comment="Field changes" + ) + + comment = Column( + Text, + nullable=True, + comment="History comment" + ) + + timestamp = Column( + DateTime, + nullable=False, + default=datetime.utcnow, + index=True + ) + + # Relationships + application = relationship("Application", back_populates="history") + user = relationship("User") + + +class ApplicationAttachment(ExtendedBaseModel): + """Application attachment model""" + + __tablename__ = "application_attachments" + + application_id = Column( + Integer, + ForeignKey('applications.id', ondelete='CASCADE'), + nullable=False, + index=True + ) + + file_name = Column( + String(255), + nullable=False, + comment="Original file name" + ) + + file_path = Column( + String(500), + nullable=False, + comment="Storage path" + ) + + file_size = Column( + Integer, + nullable=False, + comment="File size in bytes" + ) + + file_type = Column( + String(100), + nullable=True, + comment="MIME type" + ) + + file_hash = Column( + String(64), + nullable=True, + comment="File SHA256 hash" + ) + + uploaded_by = Column( + Integer, + ForeignKey('users.id'), + nullable=True + ) + + uploaded_at = Column( + DateTime, + nullable=False, + default=datetime.utcnow + ) + + # Relationships + application = relationship("Application", back_populates="attachments") + uploader = relationship("User") + + # Metadata + submitted_at = Column(DateTime, nullable=True) + reviewed_at = Column(DateTime, nullable=True) + completed_at = Column(DateTime, nullable=True) + + # Workflow fields + locked_at = Column(DateTime, nullable=True, comment="When processing was locked") + locked_by = Column(Integer, ForeignKey('users.id'), nullable=True, comment="Who locked the processing") + + # Budget review + budget_reviewed_by = Column(Integer, ForeignKey('users.id'), nullable=True, comment="Haushaltsbeauftragte") + budget_reviewed_at = Column(DateTime, nullable=True) + budget_review_status = Column(String(50), nullable=True) # approved, rejected, pending + budget_review_comment = Column(Text, nullable=True) + + # Finance review + finance_reviewed_by = Column(Integer, ForeignKey('users.id'), nullable=True, comment="Finanzreferent") + finance_reviewed_at = Column(DateTime, nullable=True) + finance_review_status = Column(String(50), nullable=True) # approved, rejected, pending + finance_review_comment = Column(Text, nullable=True) + + # Voting + voting_opened_at = Column(DateTime, nullable=True) + voting_closed_at = Column(DateTime, nullable=True) + voting_result = Column(String(50), nullable=True) # approved, rejected + votes_for = Column(Integer, default=0) + votes_against = Column(Integer, default=0) + votes_abstain = Column(Integer, default=0) + + # Relationships + user = relationship("User", foreign_keys=[user_id], back_populates="applications") + template = relationship("FormTemplate", back_populates="applications") + locker = relationship("User", foreign_keys=[locked_by]) + budget_reviewer = relationship("User", foreign_keys=[budget_reviewed_by]) + finance_reviewer = relationship("User", foreign_keys=[finance_reviewed_by]) + votes = relationship("ApplicationVote", back_populates="application", cascade="all, delete-orphan") + attachments = relationship("ApplicationAttachment", back_populates="application", cascade="all, delete-orphan") + history = relationship("ApplicationHistory", back_populates="application", cascade="all, delete-orphan") + reviewed_by = Column(String(255), nullable=True) + + # PDF storage + pdf_data = Column( + LONGTEXT, + nullable=True, + comment="Base64 encoded PDF data" + ) + pdf_generated_at = Column(DateTime, nullable=True) + + # Relationships + attachments = relationship( + "ApplicationAttachment", + back_populates="application", + cascade="all, delete-orphan", + lazy="dynamic" + ) + comparison_offers = relationship( + "ComparisonOffer", + back_populates="application", + cascade="all, delete-orphan", + lazy="dynamic" + ) + cost_justifications = relationship( + "CostPositionJustification", + back_populates="application", + cascade="all, delete-orphan", + lazy="dynamic" + ) + + # Indexes + __table_args__ = ( + Index("idx_app_status_created", "status", "created_at"), + Index("idx_app_email_status", "applicant_email", "status"), + Index("idx_app_institution", "institution_type", "institution_name"), + Index("idx_app_dates", "project_start_date", "project_end_date"), + ) + + def update_from_payload(self): + """Update searchable fields from payload""" + if not self.payload: + return + + pa = self.payload.get("pa", {}) + + # Extract applicant info + applicant = pa.get("applicant", {}) + self.applicant_first_name = applicant.get("name", {}).get("first") + self.applicant_last_name = applicant.get("name", {}).get("last") + self.applicant_email = applicant.get("contact", {}).get("email") + + # Extract institution info + institution = applicant.get("institution", {}) + self.institution_name = institution.get("name") + inst_type = institution.get("type") + if inst_type and inst_type != "-": + try: + self.institution_type = InstitutionType(inst_type) + except ValueError: + pass + + # Extract project info + project = pa.get("project", {}) + self.project_name = project.get("name") + + dates = project.get("dates", {}) + self.project_start_date = dates.get("start") + self.project_end_date = dates.get("end") + + # Calculate total amount + costs = project.get("costs", []) + total = 0.0 + for cost in costs: + amount = cost.get("amountEur", 0) + if amount: + total += float(amount) + self.total_amount = total + + def to_dict(self, exclude: Optional[set] = None, include_pdf: bool = False) -> Dict[str, Any]: + """Convert to dictionary with optional PDF exclusion""" + exclude = exclude or set() + if not include_pdf: + exclude.add("pdf_data") + + data = super().to_dict(exclude=exclude) + + # Convert enums to strings + if "status" in data and data["status"]: + data["status"] = data["status"].value if hasattr(data["status"], "value") else data["status"] + if "variant" in data and data["variant"]: + data["variant"] = data["variant"].value if hasattr(data["variant"], "value") else data["variant"] + if "institution_type" in data and data["institution_type"]: + data["institution_type"] = data["institution_type"].value if hasattr(data["institution_type"], "value") else data["institution_type"] + + return data + + +class Attachment(BaseModel, TimestampMixin): + """Base attachment model""" + + __tablename__ = "attachments" + + id = Column(Integer, primary_key=True) + filename = Column(String(255), nullable=False) + content_type = Column(String(100), nullable=False) + size = Column(Integer, nullable=False) + checksum = Column(String(64), nullable=True) + storage_type = Column( + String(50), + default="database", + comment="Storage type: database or filesystem" + ) + storage_path = Column( + String(500), + nullable=True, + comment="Path if stored in filesystem" + ) + data = Column( + LONGTEXT, + nullable=True, + comment="Base64 encoded data if stored in database" + ) + + # Indexes + __table_args__ = ( + Index("idx_attachment_checksum", "checksum"), + ) + + +class ApplicationAttachment(BaseModel): + """Junction table for application attachments with additional metadata""" + + __tablename__ = "application_attachments" + + id = Column(Integer, primary_key=True) + application_id = Column( + Integer, + ForeignKey("applications.id", ondelete="CASCADE"), + nullable=False, + index=True + ) + attachment_id = Column( + Integer, + ForeignKey("attachments.id", ondelete="CASCADE"), + nullable=False, + index=True + ) + category = Column( + String(50), + nullable=True, + comment="Attachment category (e.g., invoice, receipt, etc.)" + ) + description = Column(Text, nullable=True) + uploaded_at = Column(DateTime, default=datetime.utcnow) + uploaded_by = Column(String(255), nullable=True) + + # Relationships + application = relationship("Application", back_populates="attachments") + attachment = relationship("Attachment", backref="applications", cascade="all, delete") + + # Constraints + __table_args__ = ( + UniqueConstraint("application_id", "attachment_id", name="uq_app_attachment"), + Index("idx_app_attach_category", "application_id", "category"), + ) + + +class ComparisonOffer(ExtendedBaseModel): + """Comparison offers for cost positions""" + + __tablename__ = "comparison_offers" + + application_id = Column( + Integer, + ForeignKey("applications.id", ondelete="CASCADE"), + nullable=False, + index=True + ) + cost_position_idx = Column( + Integer, + nullable=False, + comment="Index of cost position in application" + ) + supplier_name = Column(String(255), nullable=False) + amount_eur = Column(Float, nullable=False) + description = Column(Text, nullable=True) + is_preferred = Column( + Boolean, + default=False, + comment="Whether this is the preferred offer" + ) + attachment_id = Column( + Integer, + ForeignKey("attachments.id", ondelete="SET NULL"), + nullable=True + ) + + # Relationships + application = relationship("Application", back_populates="comparison_offers") + attachment = relationship("Attachment", backref="comparison_offers") + + # Indexes + __table_args__ = ( + Index("idx_comp_offer_app_pos", "application_id", "cost_position_idx"), + Index("idx_comp_offer_preferred", "application_id", "is_preferred"), + ) + + def to_dict(self, exclude: Optional[set] = None) -> Dict[str, Any]: + """Convert to dictionary""" + data = super().to_dict(exclude=exclude) + + # Include attachment info if available + if self.attachment: + data["attachment_info"] = { + "id": self.attachment.id, + "filename": self.attachment.filename, + "size": self.attachment.size, + "content_type": self.attachment.content_type + } + + return data + + +class CostPositionJustification(ExtendedBaseModel): + """Justifications for cost positions""" + + __tablename__ = "cost_position_justifications" + + application_id = Column( + Integer, + ForeignKey("applications.id", ondelete="CASCADE"), + nullable=False, + index=True + ) + cost_position_idx = Column( + Integer, + nullable=False, + comment="Index of cost position in application" + ) + justification = Column( + Text, + nullable=False, + comment="Justification text" + ) + justification_type = Column( + String(50), + default="standard", + comment="Type of justification" + ) + + # Relationships + application = relationship("Application", back_populates="cost_justifications") + + # Constraints + __table_args__ = ( + UniqueConstraint( + "application_id", "cost_position_idx", + name="uq_app_cost_justification" + ), + Index("idx_cost_just_type", "justification_type"), + ) + + +class Counter(BaseModel): + """Counter for generating sequential IDs""" + + __tablename__ = "counters" + + key = Column(String(50), unique=True, nullable=False) + value = Column(Integer, default=0, nullable=False) + prefix = Column(String(20), nullable=True) + suffix = Column(String(20), nullable=True) + format_string = Column( + String(100), + default="{prefix}{value:06d}{suffix}", + comment="Python format string for ID generation" + ) + + @classmethod + def get_next_value(cls, session, key: str, increment: int = 1) -> int: + """Get next counter value with atomic increment""" + counter = session.query(cls).filter_by(key=key).with_for_update().first() + if not counter: + counter = cls(key=key, value=0) + session.add(counter) + + counter.value += increment + session.flush() + return counter.value + + def format_id(self, value: Optional[int] = None) -> str: + """Format counter value as ID string""" + val = value if value is not None else self.value + return self.format_string.format( + prefix=self.prefix or "", + suffix=self.suffix or "", + value=val + ) diff --git a/backend/src/models/base.py b/backend/src/models/base.py new file mode 100644 index 00000000..c90ebebc --- /dev/null +++ b/backend/src/models/base.py @@ -0,0 +1,257 @@ +""" +Base Database Models + +This module provides base classes and mixins for SQLAlchemy models. +All database models should inherit from these base classes. +""" + +from datetime import datetime +from typing import Any, Dict, Optional +from sqlalchemy import Column, Integer, DateTime, String, Boolean, func +from sqlalchemy.orm import declarative_base, declared_attr +from sqlalchemy.ext.declarative import DeclarativeMeta +import json + + +# Create base class for all models +Base = declarative_base() + + +class TimestampMixin: + """Mixin that adds timestamp fields to models""" + + @declared_attr + def created_at(cls): + return Column( + DateTime, + default=func.now(), + nullable=False, + comment="Record creation timestamp" + ) + + @declared_attr + def updated_at(cls): + return Column( + DateTime, + default=func.now(), + onupdate=func.now(), + nullable=False, + comment="Record last update timestamp" + ) + + +class SoftDeleteMixin: + """Mixin that adds soft delete functionality""" + + @declared_attr + def is_deleted(cls): + return Column( + Boolean, + default=False, + nullable=False, + index=True, + comment="Soft delete flag" + ) + + @declared_attr + def deleted_at(cls): + return Column( + DateTime, + nullable=True, + comment="Soft delete timestamp" + ) + + def soft_delete(self): + """Mark record as deleted""" + self.is_deleted = True + self.deleted_at = datetime.utcnow() + + def restore(self): + """Restore soft deleted record""" + self.is_deleted = False + self.deleted_at = None + + +class AuditMixin: + """Mixin that adds audit fields""" + + @declared_attr + def created_by(cls): + return Column( + String(255), + nullable=True, + comment="User who created the record" + ) + + @declared_attr + def updated_by(cls): + return Column( + String(255), + nullable=True, + comment="User who last updated the record" + ) + + @declared_attr + def version(cls): + return Column( + Integer, + default=1, + nullable=False, + comment="Record version for optimistic locking" + ) + + +class BaseModel(Base): + """ + Base model class that includes common fields and methods. + All database models should inherit from this class. + """ + + __abstract__ = True + + id = Column( + Integer, + primary_key=True, + autoincrement=True, + comment="Primary key" + ) + + def to_dict(self, exclude: Optional[set] = None) -> Dict[str, Any]: + """ + Convert model instance to dictionary. + + Args: + exclude: Set of field names to exclude from the result + + Returns: + Dictionary representation of the model + """ + exclude = exclude or set() + result = {} + + for column in self.__table__.columns: + if column.name not in exclude: + value = getattr(self, column.name) + + # Handle datetime objects + if isinstance(value, datetime): + value = value.isoformat() + # Handle other non-serializable types + elif hasattr(value, "__dict__") and not isinstance(value, (str, int, float, bool, list, dict)): + continue + + result[column.name] = value + + return result + + def from_dict(self, data: Dict[str, Any], exclude: Optional[set] = None) -> 'BaseModel': + """ + Update model instance from dictionary. + + Args: + data: Dictionary with field values + exclude: Set of field names to exclude from update + + Returns: + Self for method chaining + """ + exclude = exclude or set() + + for key, value in data.items(): + if key not in exclude and hasattr(self, key): + # Skip relationships and computed properties + if not hasattr(self.__class__.__dict__.get(key), "property"): + setattr(self, key, value) + + return self + + def __repr__(self): + """String representation of the model""" + return f"<{self.__class__.__name__}(id={self.id})>" + + @classmethod + def get_columns(cls) -> list: + """Get list of column names""" + return [column.name for column in cls.__table__.columns] + + @classmethod + def get_relationships(cls) -> list: + """Get list of relationship names""" + return [ + rel for rel in dir(cls) + if not rel.startswith("_") and + hasattr(getattr(cls, rel), "property") and + hasattr(getattr(cls, rel).property, "mapper") + ] + + +class ExtendedBaseModel(BaseModel, TimestampMixin, SoftDeleteMixin, AuditMixin): + """ + Extended base model that includes all mixins. + Use this for models that need full audit trail and soft delete support. + """ + + __abstract__ = True + + def to_dict(self, exclude: Optional[set] = None, include_deleted: bool = False) -> Optional[Dict[str, Any]]: + """ + Convert model instance to dictionary with soft delete awareness. + + Args: + exclude: Set of field names to exclude + include_deleted: Whether to include soft deleted records + + Returns: + Dictionary representation or None if deleted and not including deleted + """ + if not include_deleted and self.is_deleted: + return None + + return super().to_dict(exclude=exclude) + + def to_json(self, exclude: Optional[set] = None, include_deleted: bool = False) -> str: + """ + Convert model instance to JSON string. + + Args: + exclude: Set of field names to exclude + include_deleted: Whether to include soft deleted records + + Returns: + JSON string representation + """ + data = self.to_dict(exclude=exclude, include_deleted=include_deleted) + if data is None: + return "null" + return json.dumps(data, ensure_ascii=False, default=str) + + +class JSONEncodedDict(dict): + """Custom type for JSON columns that automatically handles encoding/decoding""" + + def __init__(self, data=None): + if isinstance(data, str): + super().__init__(json.loads(data) if data else {}) + elif data is None: + super().__init__() + else: + super().__init__(data) + + +def create_model_registry(): + """ + Create a registry of all models for dynamic access. + + Returns: + Dictionary mapping model names to model classes + """ + registry = {} + + for mapper in Base.registry.mappers: + model = mapper.class_ + registry[model.__name__] = model + # Also register by table name + if hasattr(model, "__tablename__"): + registry[model.__tablename__] = model + + return registry diff --git a/backend/src/models/form_template.py b/backend/src/models/form_template.py new file mode 100644 index 00000000..3fbf121f --- /dev/null +++ b/backend/src/models/form_template.py @@ -0,0 +1,458 @@ +""" +Form Template Database Models + +This module defines the database models for form templates and PDF field mappings. +""" + +from sqlalchemy import ( + Column, Integer, String, Text, DateTime, JSON, Boolean, + ForeignKey, UniqueConstraint, Index, Enum as SQLEnum +) +from sqlalchemy.orm import relationship, backref +import enum +from typing import Optional, Dict, Any, List +from datetime import datetime + +from .base import ExtendedBaseModel, BaseModel, TimestampMixin + + +class FormType(enum.Enum): + """Form type enumeration""" + QSM = "QSM" + VSM = "VSM" + CUSTOM = "CUSTOM" + + +class FieldType(enum.Enum): + """Field type enumeration""" + TEXT = "text" + NUMBER = "number" + DATE = "date" + EMAIL = "email" + PHONE = "phone" + CHECKBOX = "checkbox" + RADIO = "radio" + SELECT = "select" + TEXTAREA = "textarea" + FILE = "file" + SIGNATURE = "signature" + CURRENCY = "currency" + + +class FormTemplate(ExtendedBaseModel): + """Form template model for configurable PDF forms""" + + __tablename__ = "form_templates" + + # Core fields + name = Column( + String(255), + unique=True, + nullable=False, + index=True, + comment="Template name" + ) + + display_name = Column( + String(255), + nullable=False, + comment="Display name for UI" + ) + + description = Column( + Text, + nullable=True, + comment="Template description" + ) + + form_type = Column( + SQLEnum(FormType), + nullable=False, + default=FormType.CUSTOM, + index=True, + comment="Form type" + ) + + # PDF template file + pdf_file_path = Column( + String(500), + nullable=True, + comment="Path to uploaded PDF template file" + ) + + pdf_file_name = Column( + String(255), + nullable=True, + comment="Original PDF file name" + ) + + pdf_file_size = Column( + Integer, + nullable=True, + comment="PDF file size in bytes" + ) + + pdf_file_hash = Column( + String(64), + nullable=True, + comment="PDF file SHA256 hash" + ) + + # Form configuration + is_active = Column( + Boolean, + default=True, + index=True, + comment="Whether template is active" + ) + + is_public = Column( + Boolean, + default=True, + comment="Whether template is publicly accessible" + ) + + requires_verification = Column( + Boolean, + default=True, + comment="Whether user verification is required" + ) + + # Access control + allowed_roles = Column( + JSON, + nullable=True, + default=list, + comment="List of role names allowed to use this template" + ) + + # Form designer configuration + form_design = Column( + JSON, + nullable=True, + default=dict, + comment="Visual form designer configuration" + ) + + # Workflow configuration + workflow_config = Column( + JSON, + nullable=True, + default=dict, + comment="Workflow configuration for approval process" + ) + + # Statistics + usage_count = Column( + Integer, + default=0, + comment="Number of times template has been used" + ) + + # Version management + version = Column( + String(20), + default="1.0.0", + comment="Template version" + ) + + parent_template_id = Column( + Integer, + ForeignKey('form_templates.id'), + nullable=True, + comment="Parent template for versioning" + ) + + # Relationships + field_mappings = relationship( + "FieldMapping", + back_populates="template", + cascade="all, delete-orphan", + order_by="FieldMapping.field_order" + ) + + applications = relationship( + "Application", + back_populates="template" + ) + + parent_template = relationship( + "FormTemplate", + remote_side="FormTemplate.id", + backref=backref("versions", lazy="dynamic") + ) + + # Indexes + __table_args__ = ( + Index('idx_template_active_public', 'is_active', 'is_public'), + Index('idx_template_type_active', 'form_type', 'is_active'), + ) + + def to_dict(self, include_mappings: bool = False) -> Dict[str, Any]: + """Convert to dictionary representation""" + data = { + "id": self.id, + "name": self.name, + "display_name": self.display_name, + "description": self.description, + "form_type": self.form_type.value if self.form_type else None, + "is_active": self.is_active, + "is_public": self.is_public, + "requires_verification": self.requires_verification, + "allowed_roles": self.allowed_roles or [], + "version": self.version, + "usage_count": self.usage_count, + "pdf_file_name": self.pdf_file_name, + "pdf_file_size": self.pdf_file_size, + "created_at": self.created_at.isoformat() if self.created_at else None, + "updated_at": self.updated_at.isoformat() if self.updated_at else None, + } + + if include_mappings: + data["field_mappings"] = [ + mapping.to_dict() for mapping in self.field_mappings + ] + + return data + + +class FieldMapping(ExtendedBaseModel): + """Field mapping model for PDF form fields""" + + __tablename__ = "field_mappings" + + template_id = Column( + Integer, + ForeignKey('form_templates.id', ondelete='CASCADE'), + nullable=False, + index=True + ) + + # PDF field information + pdf_field_name = Column( + String(255), + nullable=False, + comment="Field name in PDF" + ) + + pdf_field_type = Column( + String(50), + nullable=True, + comment="Original PDF field type" + ) + + # Mapping configuration + field_key = Column( + String(255), + nullable=False, + comment="Internal field key for data storage" + ) + + field_label = Column( + String(255), + nullable=False, + comment="Display label for field" + ) + + field_type = Column( + SQLEnum(FieldType), + nullable=False, + default=FieldType.TEXT, + comment="Mapped field type" + ) + + field_order = Column( + Integer, + default=0, + comment="Field display order" + ) + + # Field configuration + is_required = Column( + Boolean, + default=False, + comment="Whether field is required" + ) + + is_readonly = Column( + Boolean, + default=False, + comment="Whether field is read-only" + ) + + is_hidden = Column( + Boolean, + default=False, + comment="Whether field is hidden" + ) + + # Special field flags + is_email_field = Column( + Boolean, + default=False, + comment="Whether this is the email field (auto-filled from user)" + ) + + is_name_field = Column( + Boolean, + default=False, + comment="Whether this is a name field (auto-filled from OIDC)" + ) + + # Validation rules + validation_rules = Column( + JSON, + nullable=True, + default=dict, + comment="Field validation rules" + ) + + # Options for select/radio fields + field_options = Column( + JSON, + nullable=True, + default=list, + comment="Options for select/radio/checkbox fields" + ) + + # Default value + default_value = Column( + Text, + nullable=True, + comment="Default field value" + ) + + # Placeholder and help text + placeholder = Column( + String(500), + nullable=True, + comment="Field placeholder text" + ) + + help_text = Column( + Text, + nullable=True, + comment="Field help text" + ) + + # Conditional display + display_conditions = Column( + JSON, + nullable=True, + default=dict, + comment="Conditions for displaying field" + ) + + # Data transformation + transform_rules = Column( + JSON, + nullable=True, + default=dict, + comment="Rules for transforming field data" + ) + + # Relationships + template = relationship( + "FormTemplate", + back_populates="field_mappings" + ) + + # Indexes + __table_args__ = ( + UniqueConstraint('template_id', 'pdf_field_name', name='uq_template_pdf_field'), + UniqueConstraint('template_id', 'field_key', name='uq_template_field_key'), + Index('idx_field_template_order', 'template_id', 'field_order'), + ) + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary representation""" + return { + "id": self.id, + "pdf_field_name": self.pdf_field_name, + "pdf_field_type": self.pdf_field_type, + "field_key": self.field_key, + "field_label": self.field_label, + "field_type": self.field_type.value if self.field_type else None, + "field_order": self.field_order, + "is_required": self.is_required, + "is_readonly": self.is_readonly, + "is_hidden": self.is_hidden, + "is_email_field": self.is_email_field, + "is_name_field": self.is_name_field, + "validation_rules": self.validation_rules or {}, + "field_options": self.field_options or [], + "default_value": self.default_value, + "placeholder": self.placeholder, + "help_text": self.help_text, + "display_conditions": self.display_conditions or {}, + "transform_rules": self.transform_rules or {}, + } + + +class FormDesign(ExtendedBaseModel): + """Visual form designer configuration""" + + __tablename__ = "form_designs" + + template_id = Column( + Integer, + ForeignKey('form_templates.id', ondelete='CASCADE'), + unique=True, + nullable=False, + index=True + ) + + # Designer layout + layout_type = Column( + String(50), + default="single-column", + comment="Layout type (single-column, two-column, custom)" + ) + + # Sections configuration + sections = Column( + JSON, + nullable=False, + default=list, + comment="Form sections configuration" + ) + + # Styling + theme = Column( + JSON, + nullable=True, + default=dict, + comment="Theme configuration" + ) + + custom_css = Column( + Text, + nullable=True, + comment="Custom CSS styles" + ) + + # Components configuration + components = Column( + JSON, + nullable=True, + default=list, + comment="Custom components configuration" + ) + + # Relationships + template = relationship( + "FormTemplate", + backref=backref("design", uselist=False, cascade="all, delete-orphan") + ) + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary representation""" + return { + "id": self.id, + "template_id": self.template_id, + "layout_type": self.layout_type, + "sections": self.sections or [], + "theme": self.theme or {}, + "custom_css": self.custom_css, + "components": self.components or [], + } diff --git a/backend/src/models/user.py b/backend/src/models/user.py new file mode 100644 index 00000000..a7454b72 --- /dev/null +++ b/backend/src/models/user.py @@ -0,0 +1,363 @@ +""" +User Database Models with OIDC Support + +This module defines the database models for users with OIDC/OAuth2 integration. +""" + +from sqlalchemy import ( + Column, Integer, String, Text, DateTime, JSON, Boolean, + ForeignKey, UniqueConstraint, Index, Table, Enum as SQLEnum +) +from sqlalchemy.orm import relationship, backref +import enum +from typing import Optional, Dict, Any, List +from datetime import datetime + +from .base import ExtendedBaseModel, BaseModel, TimestampMixin + + +class AuthProvider(enum.Enum): + """Authentication provider enumeration""" + LOCAL = "local" + OIDC = "oidc" + EMAIL = "email" + + +class VerificationStatus(enum.Enum): + """Verification status enumeration""" + UNVERIFIED = "unverified" + EMAIL_VERIFIED = "email_verified" + OIDC_VERIFIED = "oidc_verified" + FULLY_VERIFIED = "fully_verified" + + +# Association table for user roles +user_roles = Table( + 'user_roles', + BaseModel.metadata, + Column('user_id', Integer, ForeignKey('users.id', ondelete='CASCADE')), + Column('role_id', Integer, ForeignKey('roles.id', ondelete='CASCADE')), + UniqueConstraint('user_id', 'role_id', name='uq_user_role') +) + + +class User(ExtendedBaseModel): + """User model with OIDC support""" + + __tablename__ = "users" + + # Core fields + email = Column( + String(255), + unique=True, + nullable=False, + index=True, + comment="User email address" + ) + + # Authentication fields + auth_provider = Column( + SQLEnum(AuthProvider), + nullable=False, + default=AuthProvider.LOCAL, + comment="Authentication provider" + ) + + oidc_sub = Column( + String(255), + nullable=True, + unique=True, + index=True, + comment="OIDC subject identifier" + ) + + oidc_issuer = Column( + String(255), + nullable=True, + comment="OIDC issuer URL" + ) + + # Profile fields from OIDC + given_name = Column(String(255), nullable=True, comment="Given name from OIDC") + family_name = Column(String(255), nullable=True, comment="Family name from OIDC") + preferred_username = Column(String(255), nullable=True, comment="Preferred username from OIDC") + display_name = Column(String(255), nullable=True, comment="Display name") + picture_url = Column(Text, nullable=True, comment="Profile picture URL") + + # Verification + verification_status = Column( + SQLEnum(VerificationStatus), + nullable=False, + default=VerificationStatus.UNVERIFIED, + index=True, + comment="User verification status" + ) + + email_verified = Column(Boolean, default=False, comment="Email verification flag") + email_verification_token = Column(String(255), nullable=True, comment="Email verification token") + email_verification_sent_at = Column(DateTime, nullable=True, comment="When verification email was sent") + email_verified_at = Column(DateTime, nullable=True, comment="When email was verified") + + # Session management + last_login_at = Column(DateTime, nullable=True, comment="Last login timestamp") + last_activity_at = Column(DateTime, nullable=True, comment="Last activity timestamp") + + # OIDC tokens (encrypted) + oidc_access_token = Column(Text, nullable=True, comment="Encrypted OIDC access token") + oidc_refresh_token = Column(Text, nullable=True, comment="Encrypted OIDC refresh token") + oidc_token_expires_at = Column(DateTime, nullable=True, comment="OIDC token expiration") + + # Additional OIDC claims + oidc_claims = Column( + JSON, + nullable=True, + default=dict, + comment="Additional OIDC claims" + ) + + # Relationships + roles = relationship( + "Role", + secondary=user_roles, + back_populates="users", + lazy="joined" + ) + + applications = relationship( + "Application", + back_populates="user", + cascade="all, delete-orphan" + ) + + # Indexes + __table_args__ = ( + Index('idx_user_email_provider', 'email', 'auth_provider'), + Index('idx_user_verification', 'verification_status', 'email_verified'), + ) + + def has_role(self, role_name: str) -> bool: + """Check if user has a specific role""" + return any(role.name == role_name for role in self.roles) + + def has_any_role(self, role_names: List[str]) -> bool: + """Check if user has any of the specified roles""" + user_roles = {role.name for role in self.roles} + return bool(user_roles.intersection(role_names)) + + def get_display_name(self) -> str: + """Get the best available display name""" + if self.display_name: + return self.display_name + if self.given_name and self.family_name: + return f"{self.given_name} {self.family_name}" + if self.preferred_username: + return self.preferred_username + return self.email.split('@')[0] + + def to_dict(self, include_sensitive: bool = False) -> Dict[str, Any]: + """Convert to dictionary representation""" + data = { + "id": self.id, + "email": self.email, + "auth_provider": self.auth_provider.value if self.auth_provider else None, + "given_name": self.given_name, + "family_name": self.family_name, + "display_name": self.get_display_name(), + "picture_url": self.picture_url, + "verification_status": self.verification_status.value if self.verification_status else None, + "email_verified": self.email_verified, + "roles": [role.to_dict() for role in self.roles], + "last_login_at": self.last_login_at.isoformat() if self.last_login_at else None, + "created_at": self.created_at.isoformat() if self.created_at else None, + "updated_at": self.updated_at.isoformat() if self.updated_at else None, + } + + if include_sensitive: + data.update({ + "oidc_sub": self.oidc_sub, + "oidc_issuer": self.oidc_issuer, + "oidc_claims": self.oidc_claims, + }) + + return data + + +class Role(ExtendedBaseModel): + """Role model for permission management""" + + __tablename__ = "roles" + + name = Column( + String(100), + unique=True, + nullable=False, + index=True, + comment="Role name" + ) + + display_name = Column( + String(255), + nullable=False, + comment="Display name for UI" + ) + + description = Column( + Text, + nullable=True, + comment="Role description" + ) + + # OIDC role mapping + oidc_role_claim = Column( + String(255), + nullable=True, + unique=True, + index=True, + comment="OIDC role claim value to map to this role" + ) + + # Permissions as JSON + permissions = Column( + JSON, + nullable=False, + default=list, + comment="List of permission strings" + ) + + # System role flags + is_system = Column( + Boolean, + default=False, + comment="System role that cannot be deleted" + ) + + is_admin = Column( + Boolean, + default=False, + comment="Admin role with full access" + ) + + # Special role flags for application workflow + can_review_budget = Column( + Boolean, + default=False, + comment="Can review budget (Haushaltsbeauftragte)" + ) + + can_review_finance = Column( + Boolean, + default=False, + comment="Can review finance (Finanzreferent)" + ) + + can_vote = Column( + Boolean, + default=False, + comment="Can vote on applications (AStA member)" + ) + + # Priority for role assignment (higher = more important) + priority = Column( + Integer, + default=0, + comment="Role priority for conflicts" + ) + + # Relationships + users = relationship( + "User", + secondary=user_roles, + back_populates="roles" + ) + + def has_permission(self, permission: str) -> bool: + """Check if role has a specific permission""" + if self.is_admin: + return True + return permission in (self.permissions or []) + + def to_dict(self) -> Dict[str, Any]: + """Convert to dictionary representation""" + return { + "id": self.id, + "name": self.name, + "display_name": self.display_name, + "description": self.description, + "is_admin": self.is_admin, + "can_review_budget": self.can_review_budget, + "can_review_finance": self.can_review_finance, + "can_vote": self.can_vote, + "permissions": self.permissions or [], + } + + +class Session(ExtendedBaseModel): + """User session model for tracking active sessions""" + + __tablename__ = "sessions" + + user_id = Column( + Integer, + ForeignKey('users.id', ondelete='CASCADE'), + nullable=False, + index=True + ) + + session_token = Column( + String(255), + unique=True, + nullable=False, + index=True, + comment="Session token" + ) + + refresh_token = Column( + String(255), + unique=True, + nullable=True, + index=True, + comment="Refresh token" + ) + + ip_address = Column( + String(45), + nullable=True, + comment="Client IP address" + ) + + user_agent = Column( + Text, + nullable=True, + comment="Client user agent" + ) + + expires_at = Column( + DateTime, + nullable=False, + index=True, + comment="Session expiration" + ) + + last_activity_at = Column( + DateTime, + nullable=False, + default=datetime.utcnow, + comment="Last activity timestamp" + ) + + # Relationships + user = relationship("User", backref=backref("sessions", cascade="all, delete-orphan")) + + @property + def is_expired(self) -> bool: + """Check if session is expired""" + return datetime.utcnow() > self.expires_at + + @property + def is_active(self) -> bool: + """Check if session is active""" + if self.is_expired: + return False + # Consider session inactive if no activity for 30 minutes + inactive_threshold = datetime.utcnow() - timedelta(minutes=30) + return self.last_activity_at > inactive_threshold diff --git a/backend/src/providers/pdf_qsm.py b/backend/src/providers/pdf_qsm.py new file mode 100644 index 00000000..d4941a77 --- /dev/null +++ b/backend/src/providers/pdf_qsm.py @@ -0,0 +1,400 @@ +""" +QSM PDF Variant Provider + +This module provides the PDF variant provider for QSM (Qualitätssicherungsmittel) forms. +""" + +from typing import Dict, Any, List, Optional +from pathlib import Path +import re +import logging + +from ..services.pdf import PDFVariantProvider +from ..config.settings import Settings, get_settings + +logger = logging.getLogger(__name__) + + +class QSMProvider(PDFVariantProvider): + """Provider for QSM PDF variant""" + + def __init__(self, settings: Optional[Settings] = None): + """Initialize QSM provider""" + self.settings = settings or get_settings() + self._field_mapping = self._initialize_field_mapping() + + def get_variant_name(self) -> str: + """Get the name of this variant""" + return "QSM" + + def get_template_path(self) -> Path: + """Get the path to the PDF template for this variant""" + return self.settings.pdf.qsm_template + + def get_variant_indicators(self) -> List[str]: + """Get list of field names that indicate this variant""" + return [ + "pa-qsm-financing", + "pa-qsm-vwv-3-2-1-1", + "pa-qsm-vwv-3-2-1-2" + ] + + def _initialize_field_mapping(self) -> Dict[str, Any]: + """Initialize field mapping configuration""" + return { + # Meta fields + "pa-id": { + "target": "pa.meta.id", + "type": "str", + "required": True + }, + "pa-key": { + "target": "pa.meta.key", + "type": "str", + "required": True + }, + + # Applicant fields + "pa-institution-type": { + "target": "pa.applicant.institution.type", + "type": "enum", + "values": { + "stud-fs": "Fachschaft", + "stud-rf": "STUPA-Referat", + "stud-hg": "Studentische Hochschulgruppe", + "faculty": "Fakultät", + "hs-institution": "Hochschuleinrichtung" + } + }, + "pa-institution": { + "target": "pa.applicant.institution.name", + "type": "str", + "required": True + }, + "pa-first-name": { + "target": "pa.applicant.name.first", + "type": "str", + "required": True + }, + "pa-last-name": { + "target": "pa.applicant.name.last", + "type": "str", + "required": True + }, + "pa-email": { + "target": "pa.applicant.contact.email", + "type": "str", + "required": True + }, + "pa-phone": { + "target": "pa.applicant.contact.phone", + "type": "str" + }, + "pa-course": { + "target": "pa.applicant.course", + "type": "enum", + "values": ["INF", "ESB", "LS", "TEC", "TEX", "NXT"] + }, + "pa-role": { + "target": "pa.applicant.role", + "type": "enum", + "values": [ + "Student", "Professor", "Mitarbeiter", + "ASTA", "Referatsleitung", "Fachschaftsvorstand" + ] + }, + + # Project fields + "pa-project-name": { + "target": "pa.project.name", + "type": "str", + "required": True + }, + "pa-project-description": { + "target": "pa.project.description", + "type": "str", + "required": True + }, + "pa-start-date": { + "target": "pa.project.dates.start", + "type": "str", + "required": True + }, + "pa-end-date": { + "target": "pa.project.dates.end", + "type": "str" + }, + "pa-participants": { + "target": "pa.project.participants", + "type": "int" + }, + + # QSM-specific fields + "pa-qsm-financing": { + "target": "pa.project.financing.qsm.code", + "type": "enum", + "values": { + "vwv-3-2-1-1": "Finanzierung zusätzlicher Lehr- und Seminarangebote", + "vwv-3-2-1-2": "Fachspezifische Studienprojekte", + "vwv-3-2-1-3": "Hochschuldidaktische Fort- und Weiterbildungsmaßnahmen", + "vwv-3-2-1-4": "Studentische Tutorien und Arbeitsgruppen", + "vwv-3-2-1-5": "Exkursionen", + "vwv-3-2-1-6": "Sonstige Maßnahmen" + }, + "required": True + } + } + + def get_field_mapping(self) -> Dict[str, Any]: + """Get the field mapping configuration for this variant""" + return self._field_mapping + + def parse_pdf_fields(self, pdf_fields: Dict[str, Any]) -> Dict[str, Any]: + """Parse PDF form fields into a structured payload""" + payload = { + "pa": { + "meta": {}, + "applicant": { + "name": {}, + "contact": {}, + "institution": {} + }, + "project": { + "dates": {}, + "costs": [], + "participation": {"faculties": {}}, + "financing": {"qsm": {}} + } + } + } + + # Process each field according to mapping + for field_name, field_value in pdf_fields.items(): + if field_value is None or field_value == "": + continue + + # Handle cost fields with wildcards + cost_match = re.match(r"pa-cost-(\d+)-(name|amount-euro)", field_name) + if cost_match: + idx = int(cost_match.group(1)) - 1 + field_type = cost_match.group(2) + + # Ensure costs array is large enough + while len(payload["pa"]["project"]["costs"]) <= idx: + payload["pa"]["project"]["costs"].append({}) + + if field_type == "name": + payload["pa"]["project"]["costs"][idx]["name"] = field_value + elif field_type == "amount-euro": + payload["pa"]["project"]["costs"][idx]["amountEur"] = self._parse_amount(field_value) + continue + + # Handle participation checkboxes + if field_name.startswith("pa-participating-faculties-"): + faculty = field_name.replace("pa-participating-faculties-", "") + payload["pa"]["project"]["participation"]["faculties"][faculty] = self._parse_bool(field_value) + continue + + # Handle QSM-specific checkboxes + if field_name.startswith("pa-qsm-vwv-"): + key = field_name.replace("pa-qsm-", "") + if "flags" not in payload["pa"]["project"]["financing"]["qsm"]: + payload["pa"]["project"]["financing"]["qsm"]["flags"] = {} + payload["pa"]["project"]["financing"]["qsm"]["flags"][key] = self._parse_bool(field_value) + continue + + # Process regular mapped fields + if field_name in self._field_mapping: + mapping = self._field_mapping[field_name] + target_path = mapping["target"] + field_type = mapping.get("type", "str") + + # Transform value based on type + transformed_value = self.transform_value(field_value, field_type) + + # Handle enum values + if field_type == "enum" and "values" in mapping: + if isinstance(mapping["values"], dict): + # Map to display value if available + transformed_value = mapping["values"].get(field_value, field_value) + + # Set value in payload using target path + self._set_nested_value(payload, target_path, transformed_value) + + # Clean up empty costs + if payload["pa"]["project"]["costs"]: + payload["pa"]["project"]["costs"] = [ + cost for cost in payload["pa"]["project"]["costs"] + if cost.get("name") or cost.get("amountEur") + ] + + return payload + + def map_payload_to_fields(self, payload: Dict[str, Any]) -> Dict[str, Any]: + """Map a structured payload to PDF form fields""" + pdf_fields = {} + + # Process regular fields + for field_name, mapping in self._field_mapping.items(): + target_path = mapping["target"] + value = self._get_nested_value(payload, target_path) + + if value is not None: + field_type = mapping.get("type", "str") + + # Handle enum reverse mapping + if field_type == "enum" and "values" in mapping and isinstance(mapping["values"], dict): + # Find key by value + for key, display_value in mapping["values"].items(): + if display_value == value or key == value: + value = key + break + + # Format value for PDF + if field_type == "float": + value = self._format_amount(value) + elif field_type == "bool": + value = "Ja" if value else "Nein" + + pdf_fields[field_name] = str(value) if value is not None else "" + + # Process costs array + pa = payload.get("pa", {}) + project = pa.get("project", {}) + costs = project.get("costs", []) + + for i, cost in enumerate(costs[:24], 1): # Limit to 24 cost positions + if "name" in cost: + pdf_fields[f"pa-cost-{i}-name"] = cost["name"] + if "amountEur" in cost: + pdf_fields[f"pa-cost-{i}-amount-euro"] = self._format_amount(cost["amountEur"]) + + # Process participation faculties + participation = project.get("participation", {}) + faculties = participation.get("faculties", {}) + + for faculty, is_participating in faculties.items(): + pdf_fields[f"pa-participating-faculties-{faculty}"] = "Ja" if is_participating else "Nein" + + # Process QSM flags + financing = project.get("financing", {}) + qsm = financing.get("qsm", {}) + flags = qsm.get("flags", {}) + + for flag_key, flag_value in flags.items(): + pdf_fields[f"pa-qsm-{flag_key}"] = "Ja" if flag_value else "Nein" + + return pdf_fields + + def validate_payload(self, payload: Dict[str, Any]) -> List[str]: + """Validate a payload for this variant""" + errors = [] + + # Check required fields + for field_name, mapping in self._field_mapping.items(): + if mapping.get("required", False): + target_path = mapping["target"] + value = self._get_nested_value(payload, target_path) + + if value is None or (isinstance(value, str) and not value.strip()): + errors.append(f"Required field missing: {target_path}") + + # Validate QSM-specific requirements + pa = payload.get("pa", {}) + project = pa.get("project", {}) + + # Check if financing code is valid + financing = project.get("financing", {}) + qsm = financing.get("qsm", {}) + financing_code = qsm.get("code") + + if not financing_code: + errors.append("QSM financing code is required") + elif financing_code not in self._field_mapping["pa-qsm-financing"]["values"]: + errors.append(f"Invalid QSM financing code: {financing_code}") + + # Validate costs + costs = project.get("costs", []) + if not costs: + errors.append("At least one cost position is required") + else: + for i, cost in enumerate(costs): + if not cost.get("name"): + errors.append(f"Cost position {i+1}: name is required") + if cost.get("amountEur") is not None: + try: + amount = float(cost["amountEur"]) + if amount < 0: + errors.append(f"Cost position {i+1}: amount cannot be negative") + except (TypeError, ValueError): + errors.append(f"Cost position {i+1}: invalid amount") + + return errors + + def detect_variant(self, pdf_fields: Dict[str, Any]) -> bool: + """Check if the given PDF fields match this variant""" + # Check for QSM-specific fields + qsm_indicators = ["pa-qsm-financing"] + + for indicator in qsm_indicators: + if indicator in pdf_fields and pdf_fields[indicator]: + return True + + # Check for QSM flags + for field_name in pdf_fields: + if field_name.startswith("pa-qsm-vwv-"): + return True + + return False + + # Helper methods + + def _set_nested_value(self, obj: Dict[str, Any], path: str, value: Any): + """Set a value in a nested dictionary using dot notation""" + keys = path.split(".") + current = obj + + for key in keys[:-1]: + if key not in current: + current[key] = {} + current = current[key] + + current[keys[-1]] = value + + def _get_nested_value(self, obj: Dict[str, Any], path: str) -> Any: + """Get a value from a nested dictionary using dot notation""" + keys = path.split(".") + current = obj + + for key in keys: + if isinstance(current, dict) and key in current: + current = current[key] + else: + return None + + return current + + def _parse_amount(self, value: str) -> float: + """Parse amount from German format (1.234,56) to float""" + if not value: + return 0.0 + + # Remove thousands separator and replace comma with dot + cleaned = value.replace(".", "").replace(",", ".") + + try: + return float(cleaned) + except (TypeError, ValueError): + return 0.0 + + def _format_amount(self, value: float) -> str: + """Format amount to German format (1.234,56)""" + return f"{value:,.2f}".replace(",", "X").replace(".", ",").replace("X", ".") + + def _parse_bool(self, value: Any) -> bool: + """Parse boolean value from various formats""" + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.lower() in ["true", "yes", "1", "on", "ja", "/Yes"] + return bool(value) diff --git a/backend/src/repositories/application.py b/backend/src/repositories/application.py new file mode 100644 index 00000000..1a780a18 --- /dev/null +++ b/backend/src/repositories/application.py @@ -0,0 +1,457 @@ +""" +Application Repository + +This module provides the repository for application database operations. +""" + +from typing import Optional, List, Dict, Any, Tuple +from datetime import datetime, timedelta +from sqlalchemy import and_, or_, func, desc, asc +from sqlalchemy.orm import Session, joinedload + +from .base import BaseRepository, RepositoryException +from ..models.application import ( + Application, + ApplicationStatus, + ApplicationType, + InstitutionType, + ApplicationAttachment, + ComparisonOffer, + CostPositionJustification, + Counter +) + + +class ApplicationRepository(BaseRepository[Application]): + """Repository for application database operations""" + + def __init__(self, session: Session): + """Initialize application repository""" + super().__init__(session, Application) + + def get_by_pa_id(self, pa_id: str) -> Optional[Application]: + """ + Get application by PA ID. + + Args: + pa_id: Public application ID + + Returns: + Application or None if not found + """ + return self.get_by(pa_id=pa_id) + + def find_by_status( + self, + status: ApplicationStatus, + limit: Optional[int] = None, + offset: Optional[int] = None + ) -> List[Application]: + """ + Find applications by status. + + Args: + status: Application status + limit: Maximum number of results + offset: Number of results to skip + + Returns: + List of applications + """ + query = self.query().filter(Application.status == status) + + if offset: + query = query.offset(offset) + if limit: + query = query.limit(limit) + + return query.all() + + def find_by_institution( + self, + institution_type: Optional[InstitutionType] = None, + institution_name: Optional[str] = None, + limit: Optional[int] = None, + offset: Optional[int] = None + ) -> List[Application]: + """ + Find applications by institution. + + Args: + institution_type: Type of institution + institution_name: Name of institution + limit: Maximum number of results + offset: Number of results to skip + + Returns: + List of applications + """ + query = self.query() + + if institution_type: + query = query.filter(Application.institution_type == institution_type) + if institution_name: + query = query.filter(Application.institution_name.ilike(f"%{institution_name}%")) + + if offset: + query = query.offset(offset) + if limit: + query = query.limit(limit) + + return query.all() + + def find_by_applicant( + self, + email: Optional[str] = None, + first_name: Optional[str] = None, + last_name: Optional[str] = None, + limit: Optional[int] = None, + offset: Optional[int] = None + ) -> List[Application]: + """ + Find applications by applicant information. + + Args: + email: Applicant email + first_name: Applicant first name + last_name: Applicant last name + limit: Maximum number of results + offset: Number of results to skip + + Returns: + List of applications + """ + query = self.query() + + if email: + query = query.filter(Application.applicant_email.ilike(f"%{email}%")) + if first_name: + query = query.filter(Application.applicant_first_name.ilike(f"%{first_name}%")) + if last_name: + query = query.filter(Application.applicant_last_name.ilike(f"%{last_name}%")) + + if offset: + query = query.offset(offset) + if limit: + query = query.limit(limit) + + return query.all() + + def find_by_date_range( + self, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + date_field: str = "created_at", + limit: Optional[int] = None, + offset: Optional[int] = None + ) -> List[Application]: + """ + Find applications within a date range. + + Args: + start_date: Start date (ISO format) + end_date: End date (ISO format) + date_field: Field to filter by (created_at, submitted_at, etc.) + limit: Maximum number of results + offset: Number of results to skip + + Returns: + List of applications + """ + query = self.query() + + if hasattr(Application, date_field): + field = getattr(Application, date_field) + if start_date: + query = query.filter(field >= start_date) + if end_date: + query = query.filter(field <= end_date) + + if offset: + query = query.offset(offset) + if limit: + query = query.limit(limit) + + return query.all() + + def search( + self, + search_query: str, + filters: Optional[Dict[str, Any]] = None, + order_by: str = "created_at", + order_desc: bool = True, + limit: int = 20, + offset: int = 0 + ) -> Tuple[List[Application], int]: + """ + Search applications with full-text search and filters. + + Args: + search_query: Search query string + filters: Additional filters + order_by: Field to order by + order_desc: Whether to order descending + limit: Maximum number of results + offset: Number of results to skip + + Returns: + Tuple of (applications, total_count) + """ + query = self.query() + + # Apply search query + if search_query: + search_term = f"%{search_query}%" + query = query.filter( + or_( + Application.pa_id.ilike(search_term), + Application.project_name.ilike(search_term), + Application.institution_name.ilike(search_term), + Application.applicant_email.ilike(search_term), + Application.applicant_first_name.ilike(search_term), + Application.applicant_last_name.ilike(search_term) + ) + ) + + # Apply filters + if filters: + if "status" in filters: + query = query.filter(Application.status == filters["status"]) + if "variant" in filters: + query = query.filter(Application.variant == filters["variant"]) + if "institution_type" in filters: + query = query.filter(Application.institution_type == filters["institution_type"]) + if "min_amount" in filters: + query = query.filter(Application.total_amount >= filters["min_amount"]) + if "max_amount" in filters: + query = query.filter(Application.total_amount <= filters["max_amount"]) + if "created_after" in filters: + query = query.filter(Application.created_at >= filters["created_after"]) + if "created_before" in filters: + query = query.filter(Application.created_at <= filters["created_before"]) + if "is_deleted" in filters: + query = query.filter(Application.is_deleted == filters["is_deleted"]) + + # Get total count + total_count = query.count() + + # Apply ordering + if hasattr(Application, order_by): + field = getattr(Application, order_by) + query = query.order_by(desc(field) if order_desc else asc(field)) + + # Apply pagination + query = query.offset(offset).limit(limit) + + return query.all(), total_count + + def get_with_attachments(self, id: int) -> Optional[Application]: + """ + Get application with attachments loaded. + + Args: + id: Application ID + + Returns: + Application with attachments or None + """ + return self.query().options( + joinedload(Application.attachments).joinedload(ApplicationAttachment.attachment) + ).filter(Application.id == id).first() + + def get_with_offers(self, id: int) -> Optional[Application]: + """ + Get application with comparison offers loaded. + + Args: + id: Application ID + + Returns: + Application with offers or None + """ + return self.query().options( + joinedload(Application.comparison_offers).joinedload(ComparisonOffer.attachment) + ).filter(Application.id == id).first() + + def get_statistics( + self, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None + ) -> Dict[str, Any]: + """ + Get application statistics. + + Args: + start_date: Start date for statistics + end_date: End date for statistics + + Returns: + Dictionary with statistics + """ + query = self.session.query(Application) + + if start_date: + query = query.filter(Application.created_at >= start_date) + if end_date: + query = query.filter(Application.created_at <= end_date) + + # Count by status + status_counts = {} + for status in ApplicationStatus: + count = query.filter(Application.status == status).count() + status_counts[status.value] = count + + # Count by variant + variant_counts = {} + for variant in ApplicationType: + count = query.filter(Application.variant == variant).count() + variant_counts[variant.value] = count + + # Count by institution type + institution_counts = {} + for inst_type in InstitutionType: + count = query.filter(Application.institution_type == inst_type).count() + institution_counts[inst_type.value] = count + + # Calculate totals + total_applications = query.count() + total_amount = self.session.query( + func.sum(Application.total_amount) + ).scalar() or 0.0 + + # Average processing time + completed_apps = query.filter( + Application.status == ApplicationStatus.COMPLETED, + Application.completed_at.isnot(None) + ).all() + + avg_processing_time = None + if completed_apps: + processing_times = [ + (app.completed_at - app.created_at).total_seconds() / 86400 + for app in completed_apps + if app.completed_at + ] + if processing_times: + avg_processing_time = sum(processing_times) / len(processing_times) + + return { + "total_applications": total_applications, + "total_amount": float(total_amount), + "status_distribution": status_counts, + "variant_distribution": variant_counts, + "institution_distribution": institution_counts, + "average_processing_days": avg_processing_time, + "date_range": { + "start": start_date.isoformat() if start_date else None, + "end": end_date.isoformat() if end_date else None + } + } + + def bulk_update_status( + self, + application_ids: List[int], + new_status: ApplicationStatus, + user: Optional[str] = None + ) -> int: + """ + Bulk update application status. + + Args: + application_ids: List of application IDs + new_status: New status to set + user: User performing the update + + Returns: + Number of updated applications + """ + try: + now = datetime.utcnow() + update_data = { + "status": new_status, + "updated_at": now + } + + if user: + update_data["updated_by"] = user + + if new_status == ApplicationStatus.SUBMITTED: + update_data["submitted_at"] = now + elif new_status in [ApplicationStatus.APPROVED, ApplicationStatus.REJECTED]: + update_data["reviewed_at"] = now + update_data["reviewed_by"] = user + elif new_status == ApplicationStatus.COMPLETED: + update_data["completed_at"] = now + + count = self.session.query(Application).filter( + Application.id.in_(application_ids) + ).update(update_data, synchronize_session=False) + + self.session.commit() + return count + + except Exception as e: + self.session.rollback() + raise RepositoryException(f"Failed to bulk update status: {str(e)}") + + def generate_next_pa_id(self, prefix: str = "PA") -> str: + """ + Generate next sequential PA ID. + + Args: + prefix: Prefix for the ID + + Returns: + Generated PA ID + """ + try: + # Get or create counter + counter = self.session.query(Counter).filter_by( + key="application_id" + ).with_for_update().first() + + if not counter: + counter = Counter( + key="application_id", + value=0, + prefix=prefix, + suffix="", + format_string="{prefix}{value:06d}" + ) + self.session.add(counter) + + # Increment counter + counter.value += 1 + pa_id = counter.format_id() + + self.session.flush() + return pa_id + + except Exception as e: + self.session.rollback() + raise RepositoryException(f"Failed to generate PA ID: {str(e)}") + + def cleanup_old_drafts(self, days: int = 30) -> int: + """ + Clean up old draft applications. + + Args: + days: Number of days to keep drafts + + Returns: + Number of deleted applications + """ + try: + cutoff_date = datetime.utcnow() - timedelta(days=days) + + count = self.session.query(Application).filter( + Application.status == ApplicationStatus.DRAFT, + Application.created_at < cutoff_date + ).delete(synchronize_session=False) + + self.session.commit() + return count + + except Exception as e: + self.session.rollback() + raise RepositoryException(f"Failed to cleanup old drafts: {str(e)}") diff --git a/backend/src/repositories/base.py b/backend/src/repositories/base.py new file mode 100644 index 00000000..db8723bb --- /dev/null +++ b/backend/src/repositories/base.py @@ -0,0 +1,466 @@ +""" +Base Repository Pattern + +This module provides base repository classes for database operations. +All repositories should inherit from these base classes. +""" + +from typing import Generic, TypeVar, Type, Optional, List, Dict, Any, Union +from sqlalchemy.orm import Session, Query +from sqlalchemy import and_, or_, desc, asc, func +from sqlalchemy.exc import IntegrityError, SQLAlchemyError +from contextlib import contextmanager +import logging + +from ..models.base import BaseModel + +logger = logging.getLogger(__name__) + +T = TypeVar('T', bound=BaseModel) + + +class RepositoryException(Exception): + """Base exception for repository errors""" + pass + + +class NotFoundError(RepositoryException): + """Raised when an entity is not found""" + pass + + +class DuplicateError(RepositoryException): + """Raised when trying to create a duplicate entity""" + pass + + +class BaseRepository(Generic[T]): + """ + Base repository class providing common CRUD operations. + + This class implements the repository pattern for database access, + providing a clean abstraction over SQLAlchemy operations. + """ + + def __init__(self, session: Session, model_class: Type[T]): + """ + Initialize repository. + + Args: + session: SQLAlchemy session + model_class: Model class this repository manages + """ + self.session = session + self.model_class = model_class + + @contextmanager + def transaction(self): + """ + Context manager for handling transactions. + + Usage: + with repository.transaction(): + repository.create(entity) + repository.update(another_entity) + """ + try: + yield self.session + self.session.commit() + except SQLAlchemyError as e: + self.session.rollback() + logger.error(f"Transaction failed: {e}") + raise RepositoryException(f"Transaction failed: {str(e)}") + + def query(self) -> Query: + """Get base query for the model""" + return self.session.query(self.model_class) + + def get(self, id: Union[int, str]) -> Optional[T]: + """ + Get entity by ID. + + Args: + id: Entity ID + + Returns: + Entity instance or None if not found + """ + try: + return self.query().filter(self.model_class.id == id).first() + except SQLAlchemyError as e: + logger.error(f"Error getting entity {id}: {e}") + raise RepositoryException(f"Failed to get entity: {str(e)}") + + def get_or_404(self, id: Union[int, str]) -> T: + """ + Get entity by ID or raise NotFoundError. + + Args: + id: Entity ID + + Returns: + Entity instance + + Raises: + NotFoundError: If entity not found + """ + entity = self.get(id) + if entity is None: + raise NotFoundError(f"{self.model_class.__name__} with id {id} not found") + return entity + + def get_by(self, **kwargs) -> Optional[T]: + """ + Get single entity by field values. + + Args: + **kwargs: Field names and values to filter by + + Returns: + Entity instance or None if not found + """ + try: + query = self.query() + for key, value in kwargs.items(): + if hasattr(self.model_class, key): + query = query.filter(getattr(self.model_class, key) == value) + return query.first() + except SQLAlchemyError as e: + logger.error(f"Error getting entity by {kwargs}: {e}") + raise RepositoryException(f"Failed to get entity: {str(e)}") + + def find(self, **kwargs) -> List[T]: + """ + Find entities by field values. + + Args: + **kwargs: Field names and values to filter by + + Returns: + List of matching entities + """ + try: + query = self.query() + for key, value in kwargs.items(): + if hasattr(self.model_class, key): + if value is None: + query = query.filter(getattr(self.model_class, key).is_(None)) + else: + query = query.filter(getattr(self.model_class, key) == value) + return query.all() + except SQLAlchemyError as e: + logger.error(f"Error finding entities by {kwargs}: {e}") + raise RepositoryException(f"Failed to find entities: {str(e)}") + + def find_all( + self, + filters: Optional[Dict[str, Any]] = None, + order_by: Optional[str] = None, + order_desc: bool = False, + limit: Optional[int] = None, + offset: Optional[int] = None + ) -> List[T]: + """ + Find all entities with optional filtering and pagination. + + Args: + filters: Dictionary of field names and values to filter by + order_by: Field name to order by + order_desc: Whether to order descending + limit: Maximum number of results + offset: Number of results to skip + + Returns: + List of entities + """ + try: + query = self.query() + + # Apply filters + if filters: + for key, value in filters.items(): + if hasattr(self.model_class, key): + field = getattr(self.model_class, key) + if value is None: + query = query.filter(field.is_(None)) + elif isinstance(value, list): + query = query.filter(field.in_(value)) + elif isinstance(value, dict): + # Support operators like {'gte': 100, 'lt': 200} + for op, val in value.items(): + if op == 'gte': + query = query.filter(field >= val) + elif op == 'gt': + query = query.filter(field > val) + elif op == 'lte': + query = query.filter(field <= val) + elif op == 'lt': + query = query.filter(field < val) + elif op == 'ne': + query = query.filter(field != val) + elif op == 'like': + query = query.filter(field.like(f"%{val}%")) + elif op == 'ilike': + query = query.filter(field.ilike(f"%{val}%")) + else: + query = query.filter(field == value) + + # Apply ordering + if order_by and hasattr(self.model_class, order_by): + field = getattr(self.model_class, order_by) + query = query.order_by(desc(field) if order_desc else asc(field)) + + # Apply pagination + if offset: + query = query.offset(offset) + if limit: + query = query.limit(limit) + + return query.all() + except SQLAlchemyError as e: + logger.error(f"Error finding all entities: {e}") + raise RepositoryException(f"Failed to find entities: {str(e)}") + + def count(self, **kwargs) -> int: + """ + Count entities matching criteria. + + Args: + **kwargs: Field names and values to filter by + + Returns: + Number of matching entities + """ + try: + query = self.session.query(func.count(self.model_class.id)) + for key, value in kwargs.items(): + if hasattr(self.model_class, key): + query = query.filter(getattr(self.model_class, key) == value) + return query.scalar() + except SQLAlchemyError as e: + logger.error(f"Error counting entities: {e}") + raise RepositoryException(f"Failed to count entities: {str(e)}") + + def exists(self, **kwargs) -> bool: + """ + Check if entity exists. + + Args: + **kwargs: Field names and values to filter by + + Returns: + True if entity exists, False otherwise + """ + return self.count(**kwargs) > 0 + + def create(self, entity: T, commit: bool = True) -> T: + """ + Create new entity. + + Args: + entity: Entity instance to create + commit: Whether to commit immediately + + Returns: + Created entity + + Raises: + DuplicateError: If entity violates unique constraint + """ + try: + self.session.add(entity) + if commit: + self.session.commit() + self.session.refresh(entity) + else: + self.session.flush() + return entity + except IntegrityError as e: + self.session.rollback() + logger.error(f"Integrity error creating entity: {e}") + raise DuplicateError(f"Entity already exists or violates constraint: {str(e)}") + except SQLAlchemyError as e: + self.session.rollback() + logger.error(f"Error creating entity: {e}") + raise RepositoryException(f"Failed to create entity: {str(e)}") + + def create_many(self, entities: List[T], commit: bool = True) -> List[T]: + """ + Create multiple entities. + + Args: + entities: List of entity instances to create + commit: Whether to commit immediately + + Returns: + List of created entities + """ + try: + self.session.add_all(entities) + if commit: + self.session.commit() + for entity in entities: + self.session.refresh(entity) + else: + self.session.flush() + return entities + except IntegrityError as e: + self.session.rollback() + logger.error(f"Integrity error creating entities: {e}") + raise DuplicateError(f"One or more entities already exist: {str(e)}") + except SQLAlchemyError as e: + self.session.rollback() + logger.error(f"Error creating entities: {e}") + raise RepositoryException(f"Failed to create entities: {str(e)}") + + def update(self, entity: T, commit: bool = True) -> T: + """ + Update entity. + + Args: + entity: Entity instance with updated values + commit: Whether to commit immediately + + Returns: + Updated entity + """ + try: + if commit: + self.session.commit() + self.session.refresh(entity) + else: + self.session.flush() + return entity + except SQLAlchemyError as e: + self.session.rollback() + logger.error(f"Error updating entity: {e}") + raise RepositoryException(f"Failed to update entity: {str(e)}") + + def update_by_id(self, id: Union[int, str], data: Dict[str, Any], commit: bool = True) -> Optional[T]: + """ + Update entity by ID. + + Args: + id: Entity ID + data: Dictionary of fields to update + commit: Whether to commit immediately + + Returns: + Updated entity or None if not found + """ + entity = self.get(id) + if entity: + for key, value in data.items(): + if hasattr(entity, key): + setattr(entity, key, value) + return self.update(entity, commit) + return None + + def delete(self, entity: T, commit: bool = True) -> bool: + """ + Delete entity. + + Args: + entity: Entity instance to delete + commit: Whether to commit immediately + + Returns: + True if deleted successfully + """ + try: + self.session.delete(entity) + if commit: + self.session.commit() + else: + self.session.flush() + return True + except SQLAlchemyError as e: + self.session.rollback() + logger.error(f"Error deleting entity: {e}") + raise RepositoryException(f"Failed to delete entity: {str(e)}") + + def delete_by_id(self, id: Union[int, str], commit: bool = True) -> bool: + """ + Delete entity by ID. + + Args: + id: Entity ID + commit: Whether to commit immediately + + Returns: + True if deleted, False if not found + """ + entity = self.get(id) + if entity: + return self.delete(entity, commit) + return False + + def delete_many(self, filters: Dict[str, Any], commit: bool = True) -> int: + """ + Delete multiple entities matching criteria. + + Args: + filters: Dictionary of field names and values to filter by + commit: Whether to commit immediately + + Returns: + Number of deleted entities + """ + try: + query = self.query() + for key, value in filters.items(): + if hasattr(self.model_class, key): + query = query.filter(getattr(self.model_class, key) == value) + + count = query.count() + query.delete(synchronize_session=False) + + if commit: + self.session.commit() + else: + self.session.flush() + + return count + except SQLAlchemyError as e: + self.session.rollback() + logger.error(f"Error deleting entities: {e}") + raise RepositoryException(f"Failed to delete entities: {str(e)}") + + def refresh(self, entity: T) -> T: + """ + Refresh entity from database. + + Args: + entity: Entity instance to refresh + + Returns: + Refreshed entity + """ + try: + self.session.refresh(entity) + return entity + except SQLAlchemyError as e: + logger.error(f"Error refreshing entity: {e}") + raise RepositoryException(f"Failed to refresh entity: {str(e)}") + + def commit(self): + """Commit current transaction""" + try: + self.session.commit() + except SQLAlchemyError as e: + self.session.rollback() + logger.error(f"Error committing transaction: {e}") + raise RepositoryException(f"Failed to commit transaction: {str(e)}") + + def rollback(self): + """Rollback current transaction""" + self.session.rollback() + + def flush(self): + """Flush pending changes without committing""" + try: + self.session.flush() + except SQLAlchemyError as e: + self.session.rollback() + logger.error(f"Error flushing changes: {e}") + raise RepositoryException(f"Failed to flush changes: {str(e)}") diff --git a/backend/src/services/application.py b/backend/src/services/application.py new file mode 100644 index 00000000..79894144 --- /dev/null +++ b/backend/src/services/application.py @@ -0,0 +1,563 @@ +""" +Application Service + +This module provides the business logic for application management. +""" + +from typing import Optional, Dict, Any, List, Tuple +from datetime import datetime +import hashlib +import secrets +import base64 +import json + +from .base import CRUDService, ValidationError, BusinessRuleViolation, ResourceNotFoundError +from ..repositories.application import ApplicationRepository +from ..models.application import ( + Application, + ApplicationStatus, + ApplicationType, + InstitutionType +) +from ..config.settings import Settings + + +class ApplicationService(CRUDService[Application]): + """Service for application business logic""" + + def __init__( + self, + repository: ApplicationRepository, + pdf_service: Optional['PDFService'] = None, + settings: Optional[Settings] = None + ): + """ + Initialize application service. + + Args: + repository: Application repository + pdf_service: PDF processing service + settings: Application settings + """ + super().__init__(repository, settings) + self.repository: ApplicationRepository = repository + self.pdf_service = pdf_service + + def validate_create(self, data: Dict[str, Any]): + """Validate data for application creation""" + # Required fields + self.validate_required_fields(data, ["variant", "payload"]) + + # Validate variant + variant = data.get("variant", "").upper() + if variant not in ["QSM", "VSM"]: + raise ValidationError("Invalid variant", {"variant": "Must be QSM or VSM"}) + + # Validate payload structure + payload = data.get("payload", {}) + if not isinstance(payload, dict): + raise ValidationError("Invalid payload", {"payload": "Must be a dictionary"}) + + # Validate payload content + self._validate_payload(payload, variant) + + def validate_update(self, data: Dict[str, Any], entity: Application, partial: bool = True): + """Validate data for application update""" + # Check if application can be updated + if entity.status in [ApplicationStatus.APPROVED, ApplicationStatus.COMPLETED]: + raise BusinessRuleViolation( + f"Cannot update application in {entity.status.value} status" + ) + + # If updating payload, validate it + if "payload" in data: + payload = data["payload"] + if not isinstance(payload, dict): + raise ValidationError("Invalid payload", {"payload": "Must be a dictionary"}) + + variant = data.get("variant", entity.variant.value) + self._validate_payload(payload, variant, partial=partial) + + # If updating status, validate transition + if "status" in data: + new_status = data["status"] + if isinstance(new_status, str): + try: + new_status = ApplicationStatus(new_status) + except ValueError: + raise ValidationError( + "Invalid status", + {"status": f"Invalid status value: {new_status}"} + ) + + self._validate_status_transition(entity.status, new_status) + + def validate_delete(self, entity: Application): + """Validate application deletion""" + if entity.status in [ApplicationStatus.APPROVED, ApplicationStatus.IN_REVIEW]: + raise BusinessRuleViolation( + f"Cannot delete application in {entity.status.value} status" + ) + + def _validate_payload(self, payload: Dict[str, Any], variant: str, partial: bool = False): + """ + Validate application payload. + + Args: + payload: Application payload + variant: Application variant (QSM or VSM) + partial: Whether this is a partial update + """ + errors = {} + + # Check for required top-level structure + if not partial and "pa" not in payload: + errors["pa"] = "Application data is required" + + if "pa" in payload: + pa = payload["pa"] + + # Validate applicant information + if not partial and "applicant" not in pa: + errors["applicant"] = "Applicant information is required" + + if "applicant" in pa: + applicant = pa["applicant"] + + # Validate required applicant fields + if not partial: + required_applicant_fields = ["name", "contact", "institution"] + for field in required_applicant_fields: + if field not in applicant: + errors[f"applicant.{field}"] = f"{field} is required" + + # Validate email format + if "contact" in applicant and "email" in applicant["contact"]: + email = applicant["contact"]["email"] + if email and "@" not in email: + errors["applicant.contact.email"] = "Invalid email format" + + # Validate institution type + if "institution" in applicant and "type" in applicant["institution"]: + inst_type = applicant["institution"]["type"] + if inst_type and inst_type != "-": + valid_types = [e.value for e in InstitutionType] + if inst_type not in valid_types: + errors["applicant.institution.type"] = f"Invalid institution type: {inst_type}" + + # Validate project information + if not partial and "project" not in pa: + errors["project"] = "Project information is required" + + if "project" in pa: + project = pa["project"] + + # Validate required project fields + if not partial: + required_project_fields = ["name", "description", "dates"] + for field in required_project_fields: + if field not in project: + errors[f"project.{field}"] = f"{field} is required" + + # Validate dates + if "dates" in project: + dates = project["dates"] + if "start" in dates and dates["start"]: + # Validate date format (basic check) + if not self._is_valid_date_format(dates["start"]): + errors["project.dates.start"] = "Invalid date format" + + if "end" in dates and dates["end"]: + if not self._is_valid_date_format(dates["end"]): + errors["project.dates.end"] = "Invalid date format" + + # Validate costs + if "costs" in project: + costs = project["costs"] + if not isinstance(costs, list): + errors["project.costs"] = "Costs must be a list" + else: + for i, cost in enumerate(costs): + if not isinstance(cost, dict): + errors[f"project.costs[{i}]"] = "Cost must be a dictionary" + elif "amountEur" in cost: + try: + float(cost["amountEur"]) + except (TypeError, ValueError): + errors[f"project.costs[{i}].amountEur"] = "Amount must be a number" + + if errors: + raise ValidationError("Payload validation failed", errors) + + def _validate_status_transition(self, current_status: ApplicationStatus, new_status: ApplicationStatus): + """ + Validate status transition. + + Args: + current_status: Current application status + new_status: New application status + + Raises: + BusinessRuleViolation: If transition is not allowed + """ + allowed_transitions = { + ApplicationStatus.DRAFT: [ + ApplicationStatus.SUBMITTED, + ApplicationStatus.CANCELLED + ], + ApplicationStatus.SUBMITTED: [ + ApplicationStatus.IN_REVIEW, + ApplicationStatus.CANCELLED + ], + ApplicationStatus.IN_REVIEW: [ + ApplicationStatus.APPROVED, + ApplicationStatus.REJECTED, + ApplicationStatus.SUBMITTED # Send back for revision + ], + ApplicationStatus.APPROVED: [ + ApplicationStatus.COMPLETED, + ApplicationStatus.CANCELLED + ], + ApplicationStatus.REJECTED: [], # Terminal state + ApplicationStatus.CANCELLED: [], # Terminal state + ApplicationStatus.COMPLETED: [] # Terminal state + } + + if new_status not in allowed_transitions.get(current_status, []): + raise BusinessRuleViolation( + f"Cannot transition from {current_status.value} to {new_status.value}" + ) + + def _is_valid_date_format(self, date_str: str) -> bool: + """Check if date string has valid format (YYYY-MM-DD or YYYY-MM)""" + if not date_str: + return False + + # Basic format check + parts = date_str.split("-") + if len(parts) not in [2, 3]: + return False + + try: + year = int(parts[0]) + month = int(parts[1]) + if year < 1900 or year > 2100: + return False + if month < 1 or month > 12: + return False + if len(parts) == 3: + day = int(parts[2]) + if day < 1 or day > 31: + return False + return True + except (ValueError, IndexError): + return False + + def _create_entity_from_data(self, data: Dict[str, Any]) -> Application: + """Create application entity from data""" + # Generate PA ID if not provided + if "pa_id" not in data: + data["pa_id"] = self.repository.generate_next_pa_id() + + # Generate access key + if "pa_key" not in data: + raw_key = secrets.token_urlsafe(32) + data["pa_key_hash"] = self._hash_key(raw_key) + data["pa_key_raw"] = raw_key # Store temporarily for response + else: + # Hash provided key + data["pa_key_hash"] = self._hash_key(data["pa_key"]) + data["pa_key_raw"] = data["pa_key"] + + # Create application instance + app = Application( + pa_id=data["pa_id"], + pa_key=data["pa_key_hash"], + variant=ApplicationType(data["variant"].upper()), + status=ApplicationStatus(data.get("status", ApplicationStatus.DRAFT.value)), + payload=data["payload"] + ) + + # Update searchable fields from payload + app.update_from_payload() + + # Set timestamps based on status + if app.status == ApplicationStatus.SUBMITTED: + app.submitted_at = datetime.utcnow() + + return app + + def _update_entity_from_data(self, entity: Application, data: Dict[str, Any]): + """Update application entity from data""" + # Update basic fields + if "variant" in data: + entity.variant = ApplicationType(data["variant"].upper()) + + if "payload" in data: + entity.payload = data["payload"] + entity.update_from_payload() + + if "status" in data: + new_status = ApplicationStatus(data["status"]) + entity.status = new_status + + # Update timestamps based on status + now = datetime.utcnow() + if new_status == ApplicationStatus.SUBMITTED and not entity.submitted_at: + entity.submitted_at = now + elif new_status in [ApplicationStatus.APPROVED, ApplicationStatus.REJECTED]: + entity.reviewed_at = now + if "reviewed_by" in data: + entity.reviewed_by = data["reviewed_by"] + elif new_status == ApplicationStatus.COMPLETED: + entity.completed_at = now + + def _hash_key(self, key: str) -> str: + """Hash an access key""" + salt = self.settings.security.master_key or "default_salt" + return hashlib.sha256(f"{salt}:{key}".encode()).hexdigest() + + def verify_key(self, pa_id: str, key: str) -> bool: + """ + Verify access key for an application. + + Args: + pa_id: Public application ID + key: Access key to verify + + Returns: + True if key is valid, False otherwise + """ + app = self.repository.get_by_pa_id(pa_id) + if not app: + return False + + hashed_key = self._hash_key(key) + return app.pa_key == hashed_key + + def create_from_pdf( + self, + pdf_data: bytes, + variant: Optional[str] = None, + user: Optional[str] = None + ) -> Application: + """ + Create application from PDF data. + + Args: + pdf_data: PDF file data + variant: PDF variant (QSM or VSM) + user: User creating the application + + Returns: + Created application + + Raises: + ValidationError: If PDF parsing fails + ServiceException: If creation fails + """ + if not self.pdf_service: + raise BusinessRuleViolation("PDF service not available") + + # Parse PDF to payload + try: + payload = self.pdf_service.parse_pdf(pdf_data, variant) + except Exception as e: + raise ValidationError(f"Failed to parse PDF: {str(e)}") + + # Detect variant if not provided + if not variant: + variant = self.pdf_service.detect_variant(payload) + + # Create application data + data = { + "variant": variant, + "payload": payload, + "status": ApplicationStatus.DRAFT.value + } + + # Create application + return self.create(data, user=user) + + def generate_pdf(self, id: int, flatten: bool = True) -> bytes: + """ + Generate PDF for an application. + + Args: + id: Application ID + flatten: Whether to flatten the PDF + + Returns: + PDF data + + Raises: + ResourceNotFoundError: If application not found + ServiceException: If PDF generation fails + """ + if not self.pdf_service: + raise BusinessRuleViolation("PDF service not available") + + # Get application + app = self.get_or_404(id) + + # Generate PDF + try: + pdf_data = self.pdf_service.fill_pdf( + payload=app.payload, + variant=app.variant.value, + flatten=flatten + ) + + # Store PDF data + app.pdf_data = base64.b64encode(pdf_data).decode('utf-8') + app.pdf_generated_at = datetime.utcnow() + self.repository.update(app) + + return pdf_data + + except Exception as e: + self.log_error(f"Failed to generate PDF for application {id}", e) + raise ServiceException(f"Failed to generate PDF: {str(e)}") + + def submit_application(self, id: int, user: Optional[str] = None) -> Application: + """ + Submit an application for review. + + Args: + id: Application ID + user: User submitting the application + + Returns: + Updated application + + Raises: + BusinessRuleViolation: If application cannot be submitted + """ + app = self.get_or_404(id) + + # Validate status transition + if app.status != ApplicationStatus.DRAFT: + raise BusinessRuleViolation( + f"Can only submit applications in DRAFT status, current status: {app.status.value}" + ) + + # Validate payload completeness + self._validate_payload(app.payload, app.variant.value, partial=False) + + # Update status + update_data = { + "status": ApplicationStatus.SUBMITTED.value + } + + return self.update(id, update_data, user=user) + + def review_application( + self, + id: int, + approved: bool, + comments: Optional[str] = None, + reviewer: Optional[str] = None + ) -> Application: + """ + Review an application. + + Args: + id: Application ID + approved: Whether to approve the application + comments: Review comments + reviewer: Reviewer identifier + + Returns: + Updated application + + Raises: + BusinessRuleViolation: If application cannot be reviewed + """ + app = self.get_or_404(id) + + # Validate status + if app.status not in [ApplicationStatus.SUBMITTED, ApplicationStatus.IN_REVIEW]: + raise BusinessRuleViolation( + f"Can only review applications in SUBMITTED or IN_REVIEW status" + ) + + # Update status + new_status = ApplicationStatus.APPROVED if approved else ApplicationStatus.REJECTED + update_data = { + "status": new_status.value, + "reviewed_by": reviewer + } + + # Add comments to payload if provided + if comments: + payload = app.payload.copy() + if "review" not in payload: + payload["review"] = {} + payload["review"]["comments"] = comments + payload["review"]["decision"] = "approved" if approved else "rejected" + payload["review"]["reviewer"] = reviewer + payload["review"]["date"] = datetime.utcnow().isoformat() + update_data["payload"] = payload + + return self.update(id, update_data, user=reviewer) + + def get_statistics( + self, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None + ) -> Dict[str, Any]: + """ + Get application statistics. + + Args: + start_date: Start date for statistics + end_date: End date for statistics + + Returns: + Statistics dictionary + """ + return self.repository.get_statistics(start_date, end_date) + + def bulk_update_status( + self, + application_ids: List[int], + new_status: ApplicationStatus, + user: Optional[str] = None + ) -> int: + """ + Bulk update application status. + + Args: + application_ids: List of application IDs + new_status: New status to set + user: User performing the update + + Returns: + Number of updated applications + """ + # Validate each application can transition to new status + apps = [self.get(app_id) for app_id in application_ids] + for app in apps: + if app: + try: + self._validate_status_transition(app.status, new_status) + except BusinessRuleViolation as e: + self.log_warning(f"Cannot update application {app.id}: {str(e)}") + application_ids.remove(app.id) + + # Perform bulk update + return self.repository.bulk_update_status(application_ids, new_status, user) + + def cleanup_old_drafts(self, days: int = 30) -> int: + """ + Clean up old draft applications. + + Args: + days: Number of days to keep drafts + + Returns: + Number of deleted applications + """ + count = self.repository.cleanup_old_drafts(days) + self.log_info(f"Cleaned up {count} old draft applications older than {days} days") + return count diff --git a/backend/src/services/auth_email.py b/backend/src/services/auth_email.py new file mode 100644 index 00000000..6910bf5e --- /dev/null +++ b/backend/src/services/auth_email.py @@ -0,0 +1,380 @@ +""" +Email Authentication and Verification Service + +This module provides email-based authentication and verification. +""" + +import secrets +import hashlib +from datetime import datetime, timedelta +from typing import Dict, Any, Optional, Tuple +import logging + +from sqlalchemy.orm import Session +from fastapi import HTTPException, status +from jose import JWTError, jwt as jose_jwt + +from ..models.user import User, Role, AuthProvider, VerificationStatus +from ..config.settings import Settings +from ..repositories.user import UserRepository +from ..repositories.role import RoleRepository +from ..utils.email import EmailService + +logger = logging.getLogger(__name__) + + +class EmailAuthService: + """Service for handling email-based authentication""" + + def __init__(self, db_session: Session, settings: Settings): + self.db = db_session + self.settings = settings + self.user_repo = UserRepository(db_session) + self.role_repo = RoleRepository(db_session) + self.email_service = EmailService(settings) + + def generate_verification_token(self) -> str: + """Generate a secure verification token""" + return secrets.token_urlsafe(32) + + def hash_token(self, token: str) -> str: + """Hash a token for storage""" + return hashlib.sha256(token.encode()).hexdigest() + + async def register_user(self, email: str, name: Optional[str] = None) -> User: + """Register a new user with email verification""" + + # Check if user already exists + existing_user = self.user_repo.get_by_email(email) + + if existing_user: + if existing_user.auth_provider != AuthProvider.EMAIL: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="User already exists with different authentication method" + ) + + if existing_user.email_verified: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Email already verified" + ) + + # Resend verification email + return await self.send_verification_email(existing_user) + + # Parse name if provided + given_name = None + family_name = None + if name: + name_parts = name.split(" ", 1) + given_name = name_parts[0] + family_name = name_parts[1] if len(name_parts) > 1 else None + + # Create new user + user = User( + email=email, + auth_provider=AuthProvider.EMAIL, + given_name=given_name, + family_name=family_name, + display_name=name, + email_verified=False, + verification_status=VerificationStatus.UNVERIFIED, + ) + + self.db.add(user) + self.db.commit() + self.db.refresh(user) + + # Send verification email + await self.send_verification_email(user) + + return user + + async def send_verification_email(self, user: User) -> User: + """Send verification email to user""" + + # Check rate limiting + if user.email_verification_sent_at: + time_since_last = datetime.utcnow() - user.email_verification_sent_at + if time_since_last < timedelta(minutes=5): + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail="Please wait before requesting another verification email" + ) + + # Generate verification token + verification_token = self.generate_verification_token() + user.email_verification_token = self.hash_token(verification_token) + user.email_verification_sent_at = datetime.utcnow() + + self.db.commit() + + # Create verification URL + verification_url = f"{self.settings.app.frontend_url}/verify-email?token={verification_token}" + + # Send email + await self.email_service.send_verification_email( + to_email=user.email, + user_name=user.get_display_name(), + verification_url=verification_url + ) + + return user + + async def verify_email(self, token: str) -> Tuple[User, Dict[str, Any]]: + """Verify email with token""" + + # Hash the token to find user + token_hash = self.hash_token(token) + + user = self.user_repo.get_by_verification_token(token_hash) + + if not user: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid verification token" + ) + + # Check if token has expired (24 hours) + if user.email_verification_sent_at: + time_since_sent = datetime.utcnow() - user.email_verification_sent_at + if time_since_sent > timedelta(hours=24): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Verification token has expired" + ) + + # Verify email + user.email_verified = True + user.email_verified_at = datetime.utcnow() + user.email_verification_token = None + user.verification_status = VerificationStatus.EMAIL_VERIFIED + user.last_login_at = datetime.utcnow() + + # Add user role + user_role = self.role_repo.get_by_name("user") + if user_role and user_role not in user.roles: + user.roles.append(user_role) + + self.db.commit() + self.db.refresh(user) + + # Create session token + access_token = self.create_session_token(user) + refresh_token = self.create_refresh_token(user) + + return user, { + "access_token": access_token, + "refresh_token": refresh_token, + "token_type": "Bearer", + "expires_in": self.settings.security.access_token_expire_minutes * 60, + "user": user.to_dict() + } + + async def login_with_magic_link(self, email: str) -> None: + """Send magic login link to email""" + + user = self.user_repo.get_by_email(email) + + if not user: + # Don't reveal if user exists + return + + if user.auth_provider != AuthProvider.EMAIL: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Please use your configured authentication method" + ) + + # Generate login token + login_token = self.generate_verification_token() + user.email_verification_token = self.hash_token(login_token) + user.email_verification_sent_at = datetime.utcnow() + + self.db.commit() + + # Create login URL + login_url = f"{self.settings.app.frontend_url}/login-email?token={login_token}" + + # Send email + await self.email_service.send_magic_link_email( + to_email=user.email, + user_name=user.get_display_name(), + login_url=login_url + ) + + async def verify_magic_link(self, token: str) -> Tuple[User, Dict[str, Any]]: + """Verify magic link token and login user""" + + # Hash the token to find user + token_hash = self.hash_token(token) + + user = self.user_repo.get_by_verification_token(token_hash) + + if not user: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid login token" + ) + + # Check if token has expired (15 minutes for login) + if user.email_verification_sent_at: + time_since_sent = datetime.utcnow() - user.email_verification_sent_at + if time_since_sent > timedelta(minutes=15): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Login token has expired" + ) + + # Clear token and update login time + user.email_verification_token = None + user.last_login_at = datetime.utcnow() + + # If not yet verified, verify now + if not user.email_verified: + user.email_verified = True + user.email_verified_at = datetime.utcnow() + user.verification_status = VerificationStatus.EMAIL_VERIFIED + + # Add user role + user_role = self.role_repo.get_by_name("user") + if user_role and user_role not in user.roles: + user.roles.append(user_role) + + self.db.commit() + self.db.refresh(user) + + # Create session tokens + access_token = self.create_session_token(user) + refresh_token = self.create_refresh_token(user) + + return user, { + "access_token": access_token, + "refresh_token": refresh_token, + "token_type": "Bearer", + "expires_in": self.settings.security.access_token_expire_minutes * 60, + "user": user.to_dict() + } + + def create_session_token(self, user: User) -> str: + """Create JWT session token""" + + expire = datetime.utcnow() + timedelta(minutes=self.settings.security.access_token_expire_minutes) + + payload = { + "sub": str(user.id), + "email": user.email, + "roles": [role.name for role in user.roles], + "exp": expire, + "iat": datetime.utcnow(), + "type": "access" + } + + return jose_jwt.encode( + payload, + self.settings.security.jwt_secret_key, + algorithm=self.settings.security.jwt_algorithm + ) + + def create_refresh_token(self, user: User) -> str: + """Create JWT refresh token""" + + expire = datetime.utcnow() + timedelta(days=self.settings.security.refresh_token_expire_days) + + payload = { + "sub": str(user.id), + "exp": expire, + "iat": datetime.utcnow(), + "type": "refresh" + } + + return jose_jwt.encode( + payload, + self.settings.security.jwt_secret_key, + algorithm=self.settings.security.jwt_algorithm + ) + + async def resend_verification(self, email: str) -> None: + """Resend verification email""" + + user = self.user_repo.get_by_email(email) + + if not user: + # Don't reveal if user exists + return + + if user.auth_provider != AuthProvider.EMAIL: + return + + if user.email_verified: + return + + await self.send_verification_email(user) + + def verify_token(self, token: str) -> Dict[str, Any]: + """Verify JWT token and return payload""" + + try: + payload = jose_jwt.decode( + token, + self.settings.security.jwt_secret_key, + algorithms=[self.settings.security.jwt_algorithm] + ) + + return payload + + except JWTError: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid token" + ) + + async def get_current_user(self, token: str) -> User: + """Get current user from JWT token""" + + payload = self.verify_token(token) + + if payload.get("type") != "access": + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid token type" + ) + + user_id = payload.get("sub") + user = self.user_repo.get_by_id(int(user_id)) + + if not user: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="User not found" + ) + + # Update last activity + user.last_activity_at = datetime.utcnow() + self.db.commit() + + return user + + async def update_user_profile( + self, + user: User, + given_name: Optional[str] = None, + family_name: Optional[str] = None, + display_name: Optional[str] = None + ) -> User: + """Update user profile information""" + + if given_name is not None: + user.given_name = given_name + + if family_name is not None: + user.family_name = family_name + + if display_name is not None: + user.display_name = display_name + + self.db.commit() + self.db.refresh(user) + + return user diff --git a/backend/src/services/auth_oidc.py b/backend/src/services/auth_oidc.py new file mode 100644 index 00000000..81fba933 --- /dev/null +++ b/backend/src/services/auth_oidc.py @@ -0,0 +1,454 @@ +""" +OIDC/OAuth2 Authentication Service + +This module provides OIDC/OAuth2 authentication with Nextcloud integration. +""" + +import secrets +import hashlib +import jwt +from datetime import datetime, timedelta +from typing import Dict, Any, Optional, List, Tuple +from urllib.parse import urlencode, quote +import httpx +import logging + +from sqlalchemy.orm import Session +from fastapi import HTTPException, status +from jose import JWTError, jwt as jose_jwt + +from ..models.user import User, Role, AuthProvider, VerificationStatus +from ..models.user import Session as UserSession +from ..config.settings import Settings +from ..repositories.user import UserRepository +from ..repositories.role import RoleRepository +from ..utils.crypto import encrypt_token, decrypt_token + +logger = logging.getLogger(__name__) + + +class OIDCConfig: + """OIDC configuration for Nextcloud""" + + def __init__(self, settings: Settings): + self.settings = settings + self.issuer = settings.oidc.issuer # e.g., "https://nextcloud.example.com" + self.client_id = settings.oidc.client_id + self.client_secret = settings.oidc.client_secret + self.redirect_uri = settings.oidc.redirect_uri + self.scope = settings.oidc.scope or "openid profile email" + + # OIDC endpoints (Nextcloud specific) + self.authorization_endpoint = f"{self.issuer}/index.php/apps/oauth2/authorize" + self.token_endpoint = f"{self.issuer}/index.php/apps/oauth2/api/v1/token" + self.userinfo_endpoint = f"{self.issuer}/ocs/v2.php/cloud/user" + self.jwks_uri = f"{self.issuer}/index.php/apps/oauth2/jwks" + + # Alternative standard OIDC discovery + self.discovery_endpoint = f"{self.issuer}/.well-known/openid-configuration" + + +class OIDCAuthService: + """Service for handling OIDC/OAuth2 authentication""" + + def __init__(self, db_session: Session, settings: Settings): + self.db = db_session + self.settings = settings + self.config = OIDCConfig(settings) + self.user_repo = UserRepository(db_session) + self.role_repo = RoleRepository(db_session) + self.http_client = httpx.AsyncClient() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.http_client.aclose() + + def generate_state_token(self) -> str: + """Generate a secure state token for OIDC flow""" + return secrets.token_urlsafe(32) + + def generate_nonce(self) -> str: + """Generate a secure nonce for OIDC flow""" + return secrets.token_urlsafe(32) + + def get_authorization_url(self, state: str, nonce: str) -> str: + """Get the OIDC authorization URL""" + params = { + "client_id": self.config.client_id, + "response_type": "code", + "scope": self.config.scope, + "redirect_uri": self.config.redirect_uri, + "state": state, + "nonce": nonce, + } + + return f"{self.config.authorization_endpoint}?{urlencode(params)}" + + async def exchange_code_for_tokens(self, code: str) -> Dict[str, Any]: + """Exchange authorization code for tokens""" + data = { + "grant_type": "authorization_code", + "code": code, + "redirect_uri": self.config.redirect_uri, + "client_id": self.config.client_id, + "client_secret": self.config.client_secret, + } + + try: + response = await self.http_client.post( + self.config.token_endpoint, + data=data, + headers={"Content-Type": "application/x-www-form-urlencoded"} + ) + response.raise_for_status() + return response.json() + except httpx.HTTPError as e: + logger.error(f"Failed to exchange code for tokens: {e}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Failed to exchange authorization code" + ) + + async def get_userinfo(self, access_token: str) -> Dict[str, Any]: + """Get user information from OIDC provider""" + headers = {"Authorization": f"Bearer {access_token}"} + + try: + # Try standard OIDC userinfo endpoint + response = await self.http_client.get( + self.config.userinfo_endpoint, + headers=headers + ) + + if response.status_code == 200: + return response.json() + + # Fallback to Nextcloud OCS API + ocs_headers = { + **headers, + "OCS-APIRequest": "true", + "Accept": "application/json" + } + + response = await self.http_client.get( + f"{self.config.issuer}/ocs/v2.php/cloud/user?format=json", + headers=ocs_headers + ) + response.raise_for_status() + + ocs_data = response.json() + if "ocs" in ocs_data and "data" in ocs_data["ocs"]: + user_data = ocs_data["ocs"]["data"] + + # Map Nextcloud data to OIDC claims + return { + "sub": user_data.get("id"), + "email": user_data.get("email"), + "email_verified": bool(user_data.get("email")), + "name": user_data.get("display-name", user_data.get("displayname")), + "preferred_username": user_data.get("id"), + "groups": user_data.get("groups", []), + "quota": user_data.get("quota"), + "language": user_data.get("language"), + "locale": user_data.get("locale"), + } + + return response.json() + + except httpx.HTTPError as e: + logger.error(f"Failed to get user info: {e}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Failed to get user information" + ) + + async def authenticate_user( + self, + code: str, + state: str, + stored_state: str, + nonce: Optional[str] = None + ) -> Tuple[User, Dict[str, Any]]: + """Authenticate user with OIDC""" + + # Verify state + if state != stored_state: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid state parameter" + ) + + # Exchange code for tokens + token_response = await self.exchange_code_for_tokens(code) + + access_token = token_response.get("access_token") + refresh_token = token_response.get("refresh_token") + id_token = token_response.get("id_token") + expires_in = token_response.get("expires_in", 3600) + + # Get user info + userinfo = await self.get_userinfo(access_token) + + # Validate nonce if ID token is present + if id_token and nonce: + try: + # Decode without verification first to get claims + # In production, should verify against JWKS + claims = jose_jwt.decode(id_token, options={"verify_signature": False}) + if claims.get("nonce") != nonce: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Invalid nonce" + ) + except JWTError: + logger.warning("Failed to decode ID token") + + # Create or update user + user = await self.create_or_update_user(userinfo, access_token, refresh_token, expires_in) + + # Create session + session_token = self.create_session_token(user) + + return user, { + "access_token": session_token, + "token_type": "Bearer", + "expires_in": self.settings.security.access_token_expire_minutes * 60, + "user": user.to_dict() + } + + async def create_or_update_user( + self, + userinfo: Dict[str, Any], + access_token: str, + refresh_token: Optional[str], + expires_in: int + ) -> User: + """Create or update user from OIDC claims""" + + sub = userinfo.get("sub") + email = userinfo.get("email") + + if not sub or not email: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Missing required user information" + ) + + # Try to find existing user + user = self.user_repo.get_by_oidc_sub(sub, self.config.issuer) + + if not user: + # Try to find by email + user = self.user_repo.get_by_email(email) + + if user and user.auth_provider != AuthProvider.OIDC: + # User exists with different auth provider + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="User already exists with different authentication method" + ) + + # Parse name fields + name = userinfo.get("name", "") + given_name = userinfo.get("given_name", "") + family_name = userinfo.get("family_name", "") + + if name and not (given_name or family_name): + # Try to split name + name_parts = name.split(" ", 1) + given_name = name_parts[0] + family_name = name_parts[1] if len(name_parts) > 1 else "" + + # Calculate token expiration + token_expires_at = datetime.utcnow() + timedelta(seconds=expires_in) + + if user: + # Update existing user + user.oidc_sub = sub + user.oidc_issuer = self.config.issuer + user.given_name = given_name or user.given_name + user.family_name = family_name or user.family_name + user.preferred_username = userinfo.get("preferred_username", user.preferred_username) + user.display_name = name or user.display_name + user.picture_url = userinfo.get("picture", user.picture_url) + user.email_verified = userinfo.get("email_verified", False) + user.oidc_access_token = encrypt_token(access_token, self.settings.security.encryption_key) + user.oidc_refresh_token = encrypt_token(refresh_token, self.settings.security.encryption_key) if refresh_token else None + user.oidc_token_expires_at = token_expires_at + user.oidc_claims = userinfo + user.last_login_at = datetime.utcnow() + + if user.email_verified: + user.verification_status = VerificationStatus.OIDC_VERIFIED + + else: + # Create new user + user = User( + email=email, + auth_provider=AuthProvider.OIDC, + oidc_sub=sub, + oidc_issuer=self.config.issuer, + given_name=given_name, + family_name=family_name, + preferred_username=userinfo.get("preferred_username"), + display_name=name, + picture_url=userinfo.get("picture"), + email_verified=userinfo.get("email_verified", False), + verification_status=VerificationStatus.OIDC_VERIFIED if userinfo.get("email_verified") else VerificationStatus.UNVERIFIED, + oidc_access_token=encrypt_token(access_token, self.settings.security.encryption_key), + oidc_refresh_token=encrypt_token(refresh_token, self.settings.security.encryption_key) if refresh_token else None, + oidc_token_expires_at=token_expires_at, + oidc_claims=userinfo, + last_login_at=datetime.utcnow() + ) + self.db.add(user) + + # Map OIDC roles to application roles + await self.sync_user_roles(user, userinfo.get("groups", [])) + + self.db.commit() + self.db.refresh(user) + + return user + + async def sync_user_roles(self, user: User, oidc_groups: List[str]): + """Sync user roles based on OIDC groups""" + + # Get role mappings + role_mappings = self.role_repo.get_oidc_role_mappings() + + # Clear existing non-system roles + user.roles = [role for role in user.roles if role.is_system] + + # Add roles based on OIDC groups + for group in oidc_groups: + if group in role_mappings: + role = role_mappings[group] + if role not in user.roles: + user.roles.append(role) + + # Special handling for admin groups + admin_groups = self.settings.oidc.admin_groups or [] + if any(group in admin_groups for group in oidc_groups): + admin_role = self.role_repo.get_by_name("admin") + if admin_role and admin_role not in user.roles: + user.roles.append(admin_role) + + # Special handling for specific roles + role_group_mapping = { + "haushaltsbeauftragte": self.settings.oidc.budget_reviewer_groups or [], + "finanzreferent": self.settings.oidc.finance_reviewer_groups or [], + "asta": self.settings.oidc.asta_groups or [], + } + + for role_name, groups in role_group_mapping.items(): + if any(group in groups for group in oidc_groups): + role = self.role_repo.get_by_name(role_name) + if role and role not in user.roles: + user.roles.append(role) + + # Ensure every verified user has at least the user role + if user.verification_status != VerificationStatus.UNVERIFIED: + user_role = self.role_repo.get_by_name("user") + if user_role and user_role not in user.roles: + user.roles.append(user_role) + + def create_session_token(self, user: User) -> str: + """Create JWT session token""" + + expire = datetime.utcnow() + timedelta(minutes=self.settings.security.access_token_expire_minutes) + + payload = { + "sub": str(user.id), + "email": user.email, + "roles": [role.name for role in user.roles], + "exp": expire, + "iat": datetime.utcnow(), + "type": "access" + } + + return jose_jwt.encode( + payload, + self.settings.security.jwt_secret_key, + algorithm=self.settings.security.jwt_algorithm + ) + + def create_refresh_token(self, user: User) -> str: + """Create JWT refresh token""" + + expire = datetime.utcnow() + timedelta(days=self.settings.security.refresh_token_expire_days) + + payload = { + "sub": str(user.id), + "exp": expire, + "iat": datetime.utcnow(), + "type": "refresh" + } + + return jose_jwt.encode( + payload, + self.settings.security.jwt_secret_key, + algorithm=self.settings.security.jwt_algorithm + ) + + async def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]: + """Refresh access token using refresh token""" + + try: + payload = jose_jwt.decode( + refresh_token, + self.settings.security.jwt_secret_key, + algorithms=[self.settings.security.jwt_algorithm] + ) + + if payload.get("type") != "refresh": + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid token type" + ) + + user_id = payload.get("sub") + user = self.user_repo.get_by_id(int(user_id)) + + if not user: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="User not found" + ) + + # Create new access token + access_token = self.create_session_token(user) + + return { + "access_token": access_token, + "token_type": "Bearer", + "expires_in": self.settings.security.access_token_expire_minutes * 60 + } + + except JWTError: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid refresh token" + ) + + async def logout(self, user: User, session_token: Optional[str] = None): + """Logout user and invalidate session""" + + if session_token: + # Invalidate specific session + session = self.db.query(UserSession).filter( + UserSession.session_token == session_token, + UserSession.user_id == user.id + ).first() + + if session: + self.db.delete(session) + else: + # Invalidate all user sessions + self.db.query(UserSession).filter( + UserSession.user_id == user.id + ).delete() + + self.db.commit() diff --git a/backend/src/services/base.py b/backend/src/services/base.py new file mode 100644 index 00000000..1f750562 --- /dev/null +++ b/backend/src/services/base.py @@ -0,0 +1,545 @@ +""" +Base Service Layer + +This module provides base service classes for implementing business logic. +All services should inherit from these base classes. +""" + +from typing import Optional, Dict, Any, List, TypeVar, Generic +from abc import ABC, abstractmethod +import logging +from datetime import datetime +from contextlib import contextmanager + +from ..config.settings import Settings, get_settings +from ..repositories.base import BaseRepository, RepositoryException + +logger = logging.getLogger(__name__) + +T = TypeVar('T') + + +class ServiceException(Exception): + """Base exception for service layer errors""" + pass + + +class ValidationError(ServiceException): + """Raised when validation fails""" + def __init__(self, message: str, errors: Optional[Dict[str, Any]] = None): + super().__init__(message) + self.errors = errors or {} + + +class BusinessRuleViolation(ServiceException): + """Raised when a business rule is violated""" + pass + + +class ResourceNotFoundError(ServiceException): + """Raised when a requested resource is not found""" + pass + + +class ResourceConflictError(ServiceException): + """Raised when there's a conflict with existing resources""" + pass + + +class BaseService(ABC): + """ + Base service class providing common functionality for all services. + + Services encapsulate business logic and coordinate between repositories, + external services, and other components. + """ + + def __init__(self, settings: Optional[Settings] = None): + """ + Initialize service. + + Args: + settings: Application settings + """ + self.settings = settings or get_settings() + self._logger = logging.getLogger(self.__class__.__name__) + + @property + def logger(self) -> logging.Logger: + """Get logger for this service""" + return self._logger + + def log_info(self, message: str, **kwargs): + """Log info message with context""" + self._logger.info(message, extra=kwargs) + + def log_error(self, message: str, error: Optional[Exception] = None, **kwargs): + """Log error message with context""" + if error: + self._logger.error(f"{message}: {str(error)}", exc_info=True, extra=kwargs) + else: + self._logger.error(message, extra=kwargs) + + def log_warning(self, message: str, **kwargs): + """Log warning message with context""" + self._logger.warning(message, extra=kwargs) + + def log_debug(self, message: str, **kwargs): + """Log debug message with context""" + self._logger.debug(message, extra=kwargs) + + @contextmanager + def handle_errors(self, operation: str = "operation"): + """ + Context manager for handling service errors. + + Args: + operation: Description of the operation being performed + + Usage: + with self.handle_errors("creating user"): + # Service logic here + pass + """ + try: + yield + except ValidationError: + raise + except BusinessRuleViolation: + raise + except ResourceNotFoundError: + raise + except ResourceConflictError: + raise + except RepositoryException as e: + self.log_error(f"Repository error during {operation}", e) + raise ServiceException(f"Database error during {operation}: {str(e)}") + except Exception as e: + self.log_error(f"Unexpected error during {operation}", e) + raise ServiceException(f"Unexpected error during {operation}: {str(e)}") + + def validate_required_fields(self, data: Dict[str, Any], required_fields: List[str]): + """ + Validate that required fields are present and not empty. + + Args: + data: Data dictionary to validate + required_fields: List of required field names + + Raises: + ValidationError: If validation fails + """ + errors = {} + for field in required_fields: + if field not in data or data[field] is None: + errors[field] = f"{field} is required" + elif isinstance(data[field], str) and not data[field].strip(): + errors[field] = f"{field} cannot be empty" + + if errors: + raise ValidationError("Validation failed", errors) + + def validate_field_types(self, data: Dict[str, Any], field_types: Dict[str, type]): + """ + Validate field types. + + Args: + data: Data dictionary to validate + field_types: Dictionary mapping field names to expected types + + Raises: + ValidationError: If validation fails + """ + errors = {} + for field, expected_type in field_types.items(): + if field in data and data[field] is not None: + if not isinstance(data[field], expected_type): + errors[field] = f"{field} must be of type {expected_type.__name__}" + + if errors: + raise ValidationError("Type validation failed", errors) + + def validate_field_values(self, data: Dict[str, Any], field_validators: Dict[str, callable]): + """ + Validate field values using custom validators. + + Args: + data: Data dictionary to validate + field_validators: Dictionary mapping field names to validator functions + + Raises: + ValidationError: If validation fails + """ + errors = {} + for field, validator in field_validators.items(): + if field in data and data[field] is not None: + try: + if not validator(data[field]): + errors[field] = f"{field} validation failed" + except Exception as e: + errors[field] = str(e) + + if errors: + raise ValidationError("Value validation failed", errors) + + def sanitize_input(self, data: Dict[str, Any]) -> Dict[str, Any]: + """ + Sanitize input data. + + Args: + data: Input data to sanitize + + Returns: + Sanitized data + """ + sanitized = {} + for key, value in data.items(): + if isinstance(value, str): + # Strip whitespace + value = value.strip() + # Remove null bytes + value = value.replace('\x00', '') + sanitized[key] = value + return sanitized + + def audit_log( + self, + action: str, + entity_type: str, + entity_id: Optional[Any] = None, + user: Optional[str] = None, + details: Optional[Dict[str, Any]] = None + ): + """ + Create an audit log entry. + + Args: + action: Action performed (e.g., "create", "update", "delete") + entity_type: Type of entity affected + entity_id: ID of the affected entity + user: User who performed the action + details: Additional details about the action + """ + log_entry = { + "timestamp": datetime.utcnow().isoformat(), + "action": action, + "entity_type": entity_type, + "entity_id": entity_id, + "user": user, + "details": details + } + self.log_info(f"Audit: {action} {entity_type}", **log_entry) + + +class CRUDService(BaseService, Generic[T]): + """ + Base service class for CRUD operations. + + This class provides standard CRUD operations with business logic validation. + """ + + def __init__( + self, + repository: BaseRepository[T], + settings: Optional[Settings] = None + ): + """ + Initialize CRUD service. + + Args: + repository: Repository for data access + settings: Application settings + """ + super().__init__(settings) + self.repository = repository + + def create( + self, + data: Dict[str, Any], + user: Optional[str] = None, + validate: bool = True + ) -> T: + """ + Create a new entity. + + Args: + data: Entity data + user: User performing the action + validate: Whether to validate data + + Returns: + Created entity + + Raises: + ValidationError: If validation fails + ServiceException: If creation fails + """ + with self.handle_errors("creating entity"): + if validate: + self.validate_create(data) + + # Sanitize input + data = self.sanitize_input(data) + + # Create entity + entity = self._create_entity_from_data(data) + entity = self.repository.create(entity) + + # Audit log + self.audit_log( + action="create", + entity_type=entity.__class__.__name__, + entity_id=entity.id, + user=user, + details={"data": data} + ) + + return entity + + def update( + self, + id: Any, + data: Dict[str, Any], + user: Optional[str] = None, + validate: bool = True, + partial: bool = True + ) -> T: + """ + Update an existing entity. + + Args: + id: Entity ID + data: Update data + user: User performing the action + validate: Whether to validate data + partial: Whether this is a partial update + + Returns: + Updated entity + + Raises: + ResourceNotFoundError: If entity not found + ValidationError: If validation fails + ServiceException: If update fails + """ + with self.handle_errors("updating entity"): + # Get existing entity + entity = self.repository.get(id) + if not entity: + raise ResourceNotFoundError(f"Entity with id {id} not found") + + if validate: + self.validate_update(data, entity, partial) + + # Sanitize input + data = self.sanitize_input(data) + + # Update entity + self._update_entity_from_data(entity, data) + entity = self.repository.update(entity) + + # Audit log + self.audit_log( + action="update", + entity_type=entity.__class__.__name__, + entity_id=entity.id, + user=user, + details={"data": data} + ) + + return entity + + def delete( + self, + id: Any, + user: Optional[str] = None, + soft: bool = True + ) -> bool: + """ + Delete an entity. + + Args: + id: Entity ID + user: User performing the action + soft: Whether to perform soft delete + + Returns: + True if deleted successfully + + Raises: + ResourceNotFoundError: If entity not found + ServiceException: If deletion fails + """ + with self.handle_errors("deleting entity"): + # Get existing entity + entity = self.repository.get(id) + if not entity: + raise ResourceNotFoundError(f"Entity with id {id} not found") + + # Validate deletion + self.validate_delete(entity) + + # Delete entity + if soft and hasattr(entity, 'soft_delete'): + entity.soft_delete() + self.repository.update(entity) + action = "soft_delete" + else: + self.repository.delete(entity) + action = "delete" + + # Audit log + self.audit_log( + action=action, + entity_type=entity.__class__.__name__, + entity_id=id, + user=user + ) + + return True + + def get(self, id: Any) -> Optional[T]: + """ + Get an entity by ID. + + Args: + id: Entity ID + + Returns: + Entity or None if not found + """ + with self.handle_errors("getting entity"): + return self.repository.get(id) + + def get_or_404(self, id: Any) -> T: + """ + Get an entity by ID or raise error. + + Args: + id: Entity ID + + Returns: + Entity + + Raises: + ResourceNotFoundError: If entity not found + """ + entity = self.get(id) + if not entity: + raise ResourceNotFoundError(f"Entity with id {id} not found") + return entity + + def list( + self, + filters: Optional[Dict[str, Any]] = None, + page: int = 1, + page_size: int = 20, + order_by: Optional[str] = None, + order_desc: bool = False + ) -> Dict[str, Any]: + """ + List entities with pagination. + + Args: + filters: Filter criteria + page: Page number (1-based) + page_size: Number of items per page + order_by: Field to order by + order_desc: Whether to order descending + + Returns: + Dictionary with items and pagination info + """ + with self.handle_errors("listing entities"): + # Calculate offset + offset = (page - 1) * page_size + + # Get total count + total = self.repository.count(**(filters or {})) + + # Get items + items = self.repository.find_all( + filters=filters, + limit=page_size, + offset=offset, + order_by=order_by, + order_desc=order_desc + ) + + # Calculate pagination info + total_pages = (total + page_size - 1) // page_size + + return { + "items": items, + "pagination": { + "page": page, + "page_size": page_size, + "total": total, + "total_pages": total_pages, + "has_next": page < total_pages, + "has_prev": page > 1 + } + } + + def search( + self, + query: str, + fields: List[str], + filters: Optional[Dict[str, Any]] = None, + page: int = 1, + page_size: int = 20 + ) -> Dict[str, Any]: + """ + Search entities. + + Args: + query: Search query + fields: Fields to search in + filters: Additional filters + page: Page number + page_size: Number of items per page + + Returns: + Search results with pagination + """ + # This is a basic implementation - override in specific services + # for more sophisticated search functionality + combined_filters = filters or {} + + # Add search to filters (basic implementation) + if query and fields: + # This would need to be implemented based on your database + # For now, we'll just use the first field + combined_filters[fields[0]] = {"ilike": query} + + return self.list( + filters=combined_filters, + page=page, + page_size=page_size + ) + + # Abstract methods to be implemented by subclasses + + @abstractmethod + def validate_create(self, data: Dict[str, Any]): + """Validate data for entity creation""" + pass + + @abstractmethod + def validate_update(self, data: Dict[str, Any], entity: T, partial: bool = True): + """Validate data for entity update""" + pass + + def validate_delete(self, entity: T): + """Validate entity deletion - override if needed""" + pass + + @abstractmethod + def _create_entity_from_data(self, data: Dict[str, Any]) -> T: + """Create entity instance from data""" + pass + + @abstractmethod + def _update_entity_from_data(self, entity: T, data: Dict[str, Any]): + """Update entity instance from data""" + pass diff --git a/backend/src/services/pdf.py b/backend/src/services/pdf.py new file mode 100644 index 00000000..9ac5cbc8 --- /dev/null +++ b/backend/src/services/pdf.py @@ -0,0 +1,522 @@ +""" +PDF Service + +This module provides PDF processing services with dynamic variant support. +""" + +from typing import Optional, Dict, Any, List, Type +from abc import ABC, abstractmethod +import base64 +import io +import logging +from pathlib import Path + +import PyPDF2 +from PyPDF2.errors import PdfReadError + +from .base import BaseService, ServiceException, ValidationError +from ..config.settings import Settings + +logger = logging.getLogger(__name__) + + +class PDFVariantProvider(ABC): + """Abstract base class for PDF variant providers""" + + @abstractmethod + def get_variant_name(self) -> str: + """Get the name of this variant (e.g., 'QSM', 'VSM')""" + pass + + @abstractmethod + def get_template_path(self) -> Path: + """Get the path to the PDF template for this variant""" + pass + + @abstractmethod + def get_field_mapping(self) -> Dict[str, Any]: + """Get the field mapping configuration for this variant""" + pass + + @abstractmethod + def parse_pdf_fields(self, pdf_fields: Dict[str, Any]) -> Dict[str, Any]: + """ + Parse PDF form fields into a structured payload. + + Args: + pdf_fields: Raw PDF form fields + + Returns: + Structured payload + """ + pass + + @abstractmethod + def map_payload_to_fields(self, payload: Dict[str, Any]) -> Dict[str, Any]: + """ + Map a structured payload to PDF form fields. + + Args: + payload: Structured payload + + Returns: + PDF form fields + """ + pass + + @abstractmethod + def validate_payload(self, payload: Dict[str, Any]) -> List[str]: + """ + Validate a payload for this variant. + + Args: + payload: Payload to validate + + Returns: + List of validation errors (empty if valid) + """ + pass + + def detect_variant(self, pdf_fields: Dict[str, Any]) -> bool: + """ + Check if the given PDF fields match this variant. + + Args: + pdf_fields: PDF form fields + + Returns: + True if this variant matches + """ + # Default implementation - check for variant-specific fields + variant_indicators = self.get_variant_indicators() + for field in variant_indicators: + if field not in pdf_fields: + return False + return True + + def get_variant_indicators(self) -> List[str]: + """Get list of field names that indicate this variant""" + return [] + + def transform_value(self, value: Any, field_type: str) -> Any: + """ + Transform a value based on field type. + + Args: + value: Value to transform + field_type: Type of the field + + Returns: + Transformed value + """ + if field_type == "float": + try: + return float(value) if value else 0.0 + except (TypeError, ValueError): + return 0.0 + elif field_type == "int": + try: + return int(value) if value else 0 + except (TypeError, ValueError): + return 0 + elif field_type == "bool": + if isinstance(value, str): + return value.lower() in ["true", "yes", "1", "on", "ja"] + return bool(value) + elif field_type == "str": + return str(value) if value else "" + else: + return value + + +class PDFService(BaseService): + """Service for PDF processing operations""" + + def __init__(self, settings: Optional[Settings] = None): + """ + Initialize PDF service. + + Args: + settings: Application settings + """ + super().__init__(settings) + self.providers: Dict[str, PDFVariantProvider] = {} + self._flattening_enabled = settings.pdf.enable_flattening if settings else True + self._flattening_method = settings.pdf.flattening_method if settings else "pymupdf" + + # Register default providers + self._register_default_providers() + + def _register_default_providers(self): + """Register default PDF variant providers""" + # Import default providers if they exist + try: + from ..providers.pdf_qsm import QSMProvider + from ..providers.pdf_vsm import VSMProvider + + self.register_provider(QSMProvider()) + self.register_provider(VSMProvider()) + except ImportError: + logger.warning("Default PDF providers not found") + + def register_provider(self, provider: PDFVariantProvider): + """ + Register a PDF variant provider. + + Args: + provider: Provider instance to register + """ + variant_name = provider.get_variant_name() + self.providers[variant_name.upper()] = provider + logger.info(f"Registered PDF provider for variant: {variant_name}") + + def get_provider(self, variant: str) -> PDFVariantProvider: + """ + Get provider for a specific variant. + + Args: + variant: Variant name + + Returns: + Provider instance + + Raises: + ValidationError: If variant not found + """ + variant = variant.upper() + if variant not in self.providers: + raise ValidationError(f"Unknown PDF variant: {variant}") + return self.providers[variant] + + def parse_pdf(self, pdf_data: bytes, variant: Optional[str] = None) -> Dict[str, Any]: + """ + Parse PDF data into a structured payload. + + Args: + pdf_data: PDF file data + variant: PDF variant (if known) + + Returns: + Structured payload + + Raises: + ValidationError: If PDF parsing fails + """ + try: + # Read PDF fields + with io.BytesIO(pdf_data) as bio: + reader = PyPDF2.PdfReader(bio) + pdf_fields = reader.get_fields() or {} + pdf_fields = {k: (v or {}) for k, v in pdf_fields.items()} + + # Extract values from fields + field_values = {} + for field_name, field_data in pdf_fields.items(): + if isinstance(field_data, dict) and "/V" in field_data: + value = field_data["/V"] + # Handle PyPDF2 name objects + if hasattr(value, "startswith") and value.startswith("/"): + value = value[1:] + field_values[field_name] = value + + # Detect variant if not provided + if not variant: + variant = self.detect_variant(field_values) + + # Get provider and parse + provider = self.get_provider(variant) + payload = provider.parse_pdf_fields(field_values) + + # Validate payload + errors = provider.validate_payload(payload) + if errors: + raise ValidationError("PDF validation failed", {"errors": errors}) + + return payload + + except PdfReadError as e: + raise ValidationError(f"Invalid PDF file: {str(e)}") + except Exception as e: + logger.error(f"Failed to parse PDF: {e}", exc_info=True) + raise ServiceException(f"Failed to parse PDF: {str(e)}") + + def fill_pdf( + self, + payload: Dict[str, Any], + variant: str, + flatten: bool = True + ) -> bytes: + """ + Fill a PDF template with data from a payload. + + Args: + payload: Data payload + variant: PDF variant + flatten: Whether to flatten the PDF + + Returns: + Filled PDF data + + Raises: + ServiceException: If PDF filling fails + """ + try: + # Get provider + provider = self.get_provider(variant) + + # Validate payload + errors = provider.validate_payload(payload) + if errors: + logger.warning(f"Payload validation warnings: {errors}") + + # Map payload to PDF fields + field_values = provider.map_payload_to_fields(payload) + + # Get template path + template_path = provider.get_template_path() + if not template_path.exists(): + raise ServiceException(f"Template not found: {template_path}") + + # Read template + with open(template_path, "rb") as f: + reader = PyPDF2.PdfReader(f) + writer = PyPDF2.PdfWriter() + + # Copy pages and update form fields + for page in reader.pages: + writer.add_page(page) + + # Update form fields + if reader.get_fields(): + writer.update_page_form_field_values( + writer.pages[0], + field_values + ) + + # Generate output + output = io.BytesIO() + writer.write(output) + pdf_data = output.getvalue() + + # Flatten if requested + if flatten and self._flattening_enabled: + pdf_data = self._flatten_pdf(pdf_data) + + return pdf_data + + except Exception as e: + logger.error(f"Failed to fill PDF: {e}", exc_info=True) + raise ServiceException(f"Failed to fill PDF: {str(e)}") + + def _flatten_pdf(self, pdf_data: bytes) -> bytes: + """ + Flatten a PDF (make form fields non-editable). + + Args: + pdf_data: PDF data to flatten + + Returns: + Flattened PDF data + """ + if self._flattening_method == "pymupdf": + return self._flatten_with_pymupdf(pdf_data) + elif self._flattening_method == "pdftk": + return self._flatten_with_pdftk(pdf_data) + else: + logger.warning(f"Unknown flattening method: {self._flattening_method}") + return pdf_data + + def _flatten_with_pymupdf(self, pdf_data: bytes) -> bytes: + """Flatten PDF using PyMuPDF""" + try: + import fitz # PyMuPDF + + # Open PDF + doc = fitz.open(stream=pdf_data, filetype="pdf") + + # Create new PDF without forms + output = io.BytesIO() + new_doc = fitz.open() + + for page in doc: + # Render page as image and add to new document + pix = page.get_pixmap(matrix=fitz.Matrix(2, 2)) + img_data = pix.tobytes("pdf") + img_doc = fitz.open(stream=img_data, filetype="pdf") + new_doc.insert_pdf(img_doc) + + new_doc.save(output) + new_doc.close() + doc.close() + + return output.getvalue() + + except ImportError: + logger.warning("PyMuPDF not available, returning unflattened PDF") + return pdf_data + except Exception as e: + logger.error(f"Failed to flatten PDF with PyMuPDF: {e}") + return pdf_data + + def _flatten_with_pdftk(self, pdf_data: bytes) -> bytes: + """Flatten PDF using pdftk command line tool""" + import subprocess + import tempfile + + try: + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as input_file: + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as output_file: + # Write input PDF + input_file.write(pdf_data) + input_file.flush() + + # Run pdftk + cmd = [ + self.settings.pdf.pdftk_path, + input_file.name, + "output", + output_file.name, + "flatten" + ] + + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=30 + ) + + if result.returncode != 0: + logger.error(f"pdftk failed: {result.stderr}") + return pdf_data + + # Read flattened PDF + with open(output_file.name, "rb") as f: + return f.read() + + except subprocess.TimeoutExpired: + logger.error("pdftk timeout") + return pdf_data + except Exception as e: + logger.error(f"Failed to flatten PDF with pdftk: {e}") + return pdf_data + + def detect_variant(self, field_values: Dict[str, Any]) -> str: + """ + Detect PDF variant from field values. + + Args: + field_values: PDF field values + + Returns: + Detected variant name + + Raises: + ValidationError: If variant cannot be detected + """ + for variant_name, provider in self.providers.items(): + if provider.detect_variant(field_values): + logger.info(f"Detected PDF variant: {variant_name}") + return variant_name + + # Default fallback + default_variant = self.settings.pdf.default_variant if self.settings else "QSM" + logger.warning(f"Could not detect variant, using default: {default_variant}") + return default_variant + + def merge_pdfs(self, pdf_list: List[bytes]) -> bytes: + """ + Merge multiple PDFs into one. + + Args: + pdf_list: List of PDF data bytes + + Returns: + Merged PDF data + + Raises: + ServiceException: If merge fails + """ + try: + writer = PyPDF2.PdfWriter() + + for pdf_data in pdf_list: + with io.BytesIO(pdf_data) as bio: + reader = PyPDF2.PdfReader(bio) + for page in reader.pages: + writer.add_page(page) + + output = io.BytesIO() + writer.write(output) + return output.getvalue() + + except Exception as e: + logger.error(f"Failed to merge PDFs: {e}") + raise ServiceException(f"Failed to merge PDFs: {str(e)}") + + def extract_text(self, pdf_data: bytes) -> str: + """ + Extract text content from PDF. + + Args: + pdf_data: PDF data + + Returns: + Extracted text + + Raises: + ServiceException: If extraction fails + """ + try: + text_parts = [] + + with io.BytesIO(pdf_data) as bio: + reader = PyPDF2.PdfReader(bio) + for page in reader.pages: + text = page.extract_text() + if text: + text_parts.append(text) + + return "\n".join(text_parts) + + except Exception as e: + logger.error(f"Failed to extract text from PDF: {e}") + raise ServiceException(f"Failed to extract text: {str(e)}") + + def get_pdf_info(self, pdf_data: bytes) -> Dict[str, Any]: + """ + Get metadata and information about a PDF. + + Args: + pdf_data: PDF data + + Returns: + PDF information dictionary + """ + try: + with io.BytesIO(pdf_data) as bio: + reader = PyPDF2.PdfReader(bio) + + info = { + "num_pages": len(reader.pages), + "has_forms": bool(reader.get_fields()), + "is_encrypted": reader.is_encrypted, + "metadata": {} + } + + # Extract metadata + if reader.metadata: + for key, value in reader.metadata.items(): + if hasattr(value, "startswith"): + info["metadata"][key] = str(value) + + # Get form field names if present + if info["has_forms"]: + fields = reader.get_fields() or {} + info["form_fields"] = list(fields.keys()) + + return info + + except Exception as e: + logger.error(f"Failed to get PDF info: {e}") + return {"error": str(e)} diff --git a/backend/src/services/pdf_template.py b/backend/src/services/pdf_template.py new file mode 100644 index 00000000..6f8c16f7 --- /dev/null +++ b/backend/src/services/pdf_template.py @@ -0,0 +1,562 @@ +""" +PDF Template and Field Mapping Service + +This module provides services for uploading PDF templates and managing field mappings. +""" + +import os +import hashlib +import json +from pathlib import Path +from typing import Dict, Any, List, Optional, Tuple, BinaryIO +from datetime import datetime +import logging + +from sqlalchemy.orm import Session +from fastapi import HTTPException, status, UploadFile +import pypdf +from pypdf.generic import NameObject, DictionaryObject, ArrayObject +import fitz # PyMuPDF for better PDF field extraction + +from ..models.form_template import FormTemplate, FieldMapping, FormDesign, FormType, FieldType +from ..config.settings import Settings +from ..repositories.form_template import FormTemplateRepository +from ..utils.file_storage import FileStorageService + +logger = logging.getLogger(__name__) + + +class PDFTemplateService: + """Service for managing PDF templates and field mappings""" + + def __init__(self, db_session: Session, settings: Settings): + self.db = db_session + self.settings = settings + self.template_repo = FormTemplateRepository(db_session) + self.storage = FileStorageService(settings.storage.upload_dir) + self.template_dir = Path(settings.storage.template_dir) + self.template_dir.mkdir(parents=True, exist_ok=True) + + async def upload_pdf_template( + self, + file: UploadFile, + name: str, + display_name: str, + description: Optional[str] = None, + form_type: FormType = FormType.CUSTOM, + allowed_roles: Optional[List[str]] = None + ) -> FormTemplate: + """Upload a new PDF template and extract fields""" + + # Validate file type + if not file.content_type or 'pdf' not in file.content_type.lower(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="File must be a PDF" + ) + + # Check if template name already exists + existing = self.template_repo.get_by_name(name) + if existing: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Template with this name already exists" + ) + + # Read file content + content = await file.read() + + # Calculate file hash + file_hash = hashlib.sha256(content).hexdigest() + + # Check if this PDF is already uploaded + existing_hash = self.template_repo.get_by_hash(file_hash) + if existing_hash: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="This PDF file has already been uploaded" + ) + + # Save file to storage + file_path = self.template_dir / f"{name}_{file_hash[:8]}.pdf" + with open(file_path, 'wb') as f: + f.write(content) + + # Extract form fields + fields = await self.extract_pdf_fields(content) + + # Create template + template = FormTemplate( + name=name, + display_name=display_name, + description=description, + form_type=form_type, + pdf_file_path=str(file_path), + pdf_file_name=file.filename, + pdf_file_size=len(content), + pdf_file_hash=file_hash, + allowed_roles=allowed_roles or [], + is_active=True, + is_public=True, + requires_verification=True + ) + + self.db.add(template) + self.db.flush() + + # Create field mappings + for idx, field_info in enumerate(fields): + mapping = FieldMapping( + template_id=template.id, + pdf_field_name=field_info['name'], + pdf_field_type=field_info['type'], + field_key=self.generate_field_key(field_info['name']), + field_label=self.generate_field_label(field_info['name']), + field_type=self.map_pdf_field_type(field_info['type']), + field_order=idx, + is_required=False, + is_readonly=field_info.get('readonly', False), + field_options=field_info.get('options', []) + ) + self.db.add(mapping) + + self.db.commit() + self.db.refresh(template) + + return template + + async def extract_pdf_fields(self, pdf_content: bytes) -> List[Dict[str, Any]]: + """Extract form fields from PDF content""" + + fields = [] + + try: + # Try with PyMuPDF first (better field extraction) + doc = fitz.open(stream=pdf_content, filetype="pdf") + + for page_num in range(doc.page_count): + page = doc[page_num] + widgets = page.widgets() + + for widget in widgets: + field_info = { + 'name': widget.field_name, + 'type': widget.field_type_string, + 'page': page_num, + 'rect': list(widget.rect), + 'flags': widget.field_flags, + 'value': widget.field_value, + 'default': widget.field_default_value, + 'readonly': bool(widget.field_flags & (1 << 0)), # Check readonly flag + 'required': bool(widget.field_flags & (1 << 1)), # Check required flag + } + + # Extract options for choice fields + if widget.field_type in [fitz.PDF_WIDGET_TYPE_COMBOBOX, fitz.PDF_WIDGET_TYPE_LISTBOX]: + field_info['options'] = widget.choice_values + + # Extract checkbox states + if widget.field_type == fitz.PDF_WIDGET_TYPE_CHECKBOX: + field_info['states'] = ['Off', 'Yes'] # Standard checkbox states + + fields.append(field_info) + + doc.close() + + except Exception as e: + logger.warning(f"PyMuPDF extraction failed, falling back to pypdf: {e}") + + # Fallback to pypdf + try: + pdf = pypdf.PdfReader(content=pdf_content) + + if '/AcroForm' in pdf.trailer['/Root']: + form = pdf.trailer['/Root']['/AcroForm'] + + if '/Fields' in form: + for field_ref in form['/Fields']: + field = field_ref.get_object() + field_info = self.extract_field_info(field) + if field_info: + fields.append(field_info) + + except Exception as e: + logger.error(f"Failed to extract PDF fields: {e}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Failed to extract form fields from PDF" + ) + + return fields + + def extract_field_info(self, field: DictionaryObject) -> Optional[Dict[str, Any]]: + """Extract field information from PDF field object""" + + try: + field_info = { + 'name': field.get('/T', ''), + 'type': self.get_field_type(field), + 'value': field.get('/V', ''), + 'default': field.get('/DV', ''), + 'flags': field.get('/Ff', 0), + 'readonly': bool(field.get('/Ff', 0) & 1), + 'required': bool(field.get('/Ff', 0) & 2), + } + + # Extract options for choice fields + if '/Opt' in field: + options = field['/Opt'] + if isinstance(options, ArrayObject): + field_info['options'] = [str(opt) for opt in options] + + # Extract additional properties + if '/TU' in field: # Tooltip + field_info['tooltip'] = field['/TU'] + + if '/Q' in field: # Text alignment + field_info['alignment'] = field['/Q'] + + if '/MaxLen' in field: # Maximum length + field_info['maxlength'] = field['/MaxLen'] + + return field_info + + except Exception as e: + logger.warning(f"Failed to extract field info: {e}") + return None + + def get_field_type(self, field: DictionaryObject) -> str: + """Determine field type from PDF field object""" + + ft = field.get('/FT') + if ft == '/Tx': + # Text field + if field.get('/Ff', 0) & (1 << 12): # Multiline flag + return 'textarea' + return 'text' + elif ft == '/Btn': + # Button field + if field.get('/Ff', 0) & (1 << 16): # Pushbutton flag + return 'button' + elif field.get('/Ff', 0) & (1 << 15): # Radio flag + return 'radio' + else: + return 'checkbox' + elif ft == '/Ch': + # Choice field + if field.get('/Ff', 0) & (1 << 17): # Combo flag + return 'combobox' + else: + return 'listbox' + elif ft == '/Sig': + return 'signature' + else: + return 'unknown' + + def map_pdf_field_type(self, pdf_type: str) -> FieldType: + """Map PDF field type to internal field type""" + + mapping = { + 'text': FieldType.TEXT, + 'textarea': FieldType.TEXTAREA, + 'checkbox': FieldType.CHECKBOX, + 'radio': FieldType.RADIO, + 'combobox': FieldType.SELECT, + 'listbox': FieldType.SELECT, + 'signature': FieldType.SIGNATURE, + 'button': FieldType.TEXT, # Fallback for buttons + } + + return mapping.get(pdf_type.lower(), FieldType.TEXT) + + def generate_field_key(self, pdf_field_name: str) -> str: + """Generate internal field key from PDF field name""" + + # Clean up the field name + key = pdf_field_name.lower() + key = key.replace(' ', '_') + key = key.replace('-', '_') + key = ''.join(c if c.isalnum() or c == '_' else '' for c in key) + + # Remove duplicate underscores + while '__' in key: + key = key.replace('__', '_') + + return key.strip('_') + + def generate_field_label(self, pdf_field_name: str) -> str: + """Generate human-readable label from PDF field name""" + + # Clean up the field name + label = pdf_field_name.replace('_', ' ').replace('-', ' ') + + # Convert camelCase to spaces + import re + label = re.sub(r'([a-z])([A-Z])', r'\1 \2', label) + + # Capitalize words + label = ' '.join(word.capitalize() for word in label.split()) + + return label + + async def update_field_mapping( + self, + template_id: int, + field_mappings: List[Dict[str, Any]] + ) -> FormTemplate: + """Update field mappings for a template""" + + template = self.template_repo.get_by_id(template_id) + if not template: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Template not found" + ) + + # Delete existing mappings + self.db.query(FieldMapping).filter( + FieldMapping.template_id == template_id + ).delete() + + # Create new mappings + for idx, mapping_data in enumerate(field_mappings): + mapping = FieldMapping( + template_id=template_id, + pdf_field_name=mapping_data['pdf_field_name'], + pdf_field_type=mapping_data.get('pdf_field_type'), + field_key=mapping_data['field_key'], + field_label=mapping_data['field_label'], + field_type=FieldType[mapping_data['field_type']], + field_order=mapping_data.get('field_order', idx), + is_required=mapping_data.get('is_required', False), + is_readonly=mapping_data.get('is_readonly', False), + is_hidden=mapping_data.get('is_hidden', False), + is_email_field=mapping_data.get('is_email_field', False), + is_name_field=mapping_data.get('is_name_field', False), + validation_rules=mapping_data.get('validation_rules', {}), + field_options=mapping_data.get('field_options', []), + default_value=mapping_data.get('default_value'), + placeholder=mapping_data.get('placeholder'), + help_text=mapping_data.get('help_text'), + display_conditions=mapping_data.get('display_conditions', {}), + transform_rules=mapping_data.get('transform_rules', {}) + ) + self.db.add(mapping) + + self.db.commit() + self.db.refresh(template) + + return template + + async def get_field_mappings(self, template_id: int) -> List[Dict[str, Any]]: + """Get field mappings for a template""" + + template = self.template_repo.get_by_id(template_id) + if not template: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Template not found" + ) + + return [mapping.to_dict() for mapping in template.field_mappings] + + async def test_field_mapping( + self, + template_id: int, + test_data: Dict[str, Any] + ) -> Dict[str, Any]: + """Test field mapping with sample data""" + + template = self.template_repo.get_by_id(template_id) + if not template: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Template not found" + ) + + # Map test data to PDF fields + pdf_fields = {} + validation_errors = [] + + for mapping in template.field_mappings: + field_value = test_data.get(mapping.field_key) + + # Apply transformations + if field_value is not None and mapping.transform_rules: + field_value = self.apply_transformations(field_value, mapping.transform_rules) + + # Validate field + if mapping.is_required and not field_value: + validation_errors.append(f"Field '{mapping.field_label}' is required") + + if field_value and mapping.validation_rules: + errors = self.validate_field(field_value, mapping.validation_rules) + if errors: + validation_errors.extend([f"{mapping.field_label}: {error}" for error in errors]) + + # Map to PDF field + if field_value is not None: + pdf_fields[mapping.pdf_field_name] = field_value + + return { + 'pdf_fields': pdf_fields, + 'validation_errors': validation_errors, + 'is_valid': len(validation_errors) == 0 + } + + def apply_transformations(self, value: Any, rules: Dict[str, Any]) -> Any: + """Apply transformation rules to a field value""" + + if 'uppercase' in rules and rules['uppercase']: + value = str(value).upper() + + if 'lowercase' in rules and rules['lowercase']: + value = str(value).lower() + + if 'trim' in rules and rules['trim']: + value = str(value).strip() + + if 'format' in rules: + format_pattern = rules['format'] + # Apply format pattern (e.g., phone number formatting) + # This would need more sophisticated implementation + + return value + + def validate_field(self, value: Any, rules: Dict[str, Any]) -> List[str]: + """Validate field value against rules""" + + errors = [] + + if 'min_length' in rules and len(str(value)) < rules['min_length']: + errors.append(f"Minimum length is {rules['min_length']}") + + if 'max_length' in rules and len(str(value)) > rules['max_length']: + errors.append(f"Maximum length is {rules['max_length']}") + + if 'pattern' in rules: + import re + if not re.match(rules['pattern'], str(value)): + errors.append("Value does not match required pattern") + + if 'min_value' in rules and float(value) < rules['min_value']: + errors.append(f"Minimum value is {rules['min_value']}") + + if 'max_value' in rules and float(value) > rules['max_value']: + errors.append(f"Maximum value is {rules['max_value']}") + + return errors + + async def delete_template(self, template_id: int) -> bool: + """Delete a template and its associated files""" + + template = self.template_repo.get_by_id(template_id) + if not template: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Template not found" + ) + + # Check if template is in use + if template.usage_count > 0: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Cannot delete template that has been used in applications" + ) + + # Delete PDF file + if template.pdf_file_path and os.path.exists(template.pdf_file_path): + try: + os.remove(template.pdf_file_path) + except Exception as e: + logger.error(f"Failed to delete PDF file: {e}") + + # Delete from database + self.db.delete(template) + self.db.commit() + + return True + + async def duplicate_template( + self, + template_id: int, + new_name: str, + new_display_name: str + ) -> FormTemplate: + """Duplicate an existing template""" + + source_template = self.template_repo.get_by_id(template_id) + if not source_template: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Source template not found" + ) + + # Check if new name already exists + existing = self.template_repo.get_by_name(new_name) + if existing: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Template with this name already exists" + ) + + # Copy PDF file + if source_template.pdf_file_path and os.path.exists(source_template.pdf_file_path): + new_file_path = self.template_dir / f"{new_name}_{source_template.pdf_file_hash[:8]}.pdf" + import shutil + shutil.copy2(source_template.pdf_file_path, new_file_path) + else: + new_file_path = None + + # Create new template + new_template = FormTemplate( + name=new_name, + display_name=new_display_name, + description=source_template.description, + form_type=source_template.form_type, + pdf_file_path=str(new_file_path) if new_file_path else None, + pdf_file_name=source_template.pdf_file_name, + pdf_file_size=source_template.pdf_file_size, + pdf_file_hash=source_template.pdf_file_hash, + allowed_roles=source_template.allowed_roles, + is_active=True, + is_public=source_template.is_public, + requires_verification=source_template.requires_verification, + form_design=source_template.form_design, + workflow_config=source_template.workflow_config, + parent_template_id=source_template.id, + version="1.0.0" + ) + + self.db.add(new_template) + self.db.flush() + + # Copy field mappings + for mapping in source_template.field_mappings: + new_mapping = FieldMapping( + template_id=new_template.id, + pdf_field_name=mapping.pdf_field_name, + pdf_field_type=mapping.pdf_field_type, + field_key=mapping.field_key, + field_label=mapping.field_label, + field_type=mapping.field_type, + field_order=mapping.field_order, + is_required=mapping.is_required, + is_readonly=mapping.is_readonly, + is_hidden=mapping.is_hidden, + is_email_field=mapping.is_email_field, + is_name_field=mapping.is_name_field, + validation_rules=mapping.validation_rules, + field_options=mapping.field_options, + default_value=mapping.default_value, + placeholder=mapping.placeholder, + help_text=mapping.help_text, + display_conditions=mapping.display_conditions, + transform_rules=mapping.transform_rules + ) + self.db.add(new_mapping) + + self.db.commit() + self.db.refresh(new_template) + + return new_template diff --git a/docker-compose.yml b/docker-compose.yml index 6a1c81c4..9e5e0ff6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,25 @@ services: - "3306:3306" volumes: - db_data:/var/lib/mysql + networks: + - stupa_network + + redis: + image: redis:7-alpine + container_name: stupa_redis + restart: unless-stopped + command: redis-server --appendonly yes + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + ports: + - "6379:6379" + volumes: + - redis_data:/data + networks: + - stupa_network api: build: @@ -39,36 +58,88 @@ services: depends_on: db: condition: service_healthy + redis: + condition: service_healthy environment: - # DB + # Database MYSQL_HOST: db MYSQL_PORT: 3306 MYSQL_DB: ${MYSQL_DB:-stupa} MYSQL_USER: ${MYSQL_USER:-stupa} MYSQL_PASSWORD: ${MYSQL_PASSWORD:-secret} - # Auth / Limits + + # Redis + REDIS_HOST: redis + REDIS_PORT: 6379 + + # Security MASTER_KEY: ${MASTER_KEY:-change_me} + JWT_SECRET_KEY: ${JWT_SECRET_KEY:-change_me_jwt} + ENCRYPTION_KEY: ${ENCRYPTION_KEY:-change_me_encryption} + + # OIDC Settings + OIDC_ENABLED: ${OIDC_ENABLED:-false} + OIDC_ISSUER: ${OIDC_ISSUER:-} + OIDC_CLIENT_ID: ${OIDC_CLIENT_ID:-} + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:-} + OIDC_REDIRECT_URI: ${OIDC_REDIRECT_URI:-http://localhost:3001/auth/callback} + OIDC_ADMIN_GROUPS: ${OIDC_ADMIN_GROUPS:-admin} + OIDC_BUDGET_REVIEWER_GROUPS: ${OIDC_BUDGET_REVIEWER_GROUPS:-haushaltsbeauftragte} + OIDC_FINANCE_REVIEWER_GROUPS: ${OIDC_FINANCE_REVIEWER_GROUPS:-finanzreferent} + OIDC_ASTA_GROUPS: ${OIDC_ASTA_GROUPS:-asta} + + # Email Settings + EMAIL_ENABLED: ${EMAIL_ENABLED:-false} + SMTP_HOST: ${SMTP_HOST:-localhost} + SMTP_PORT: ${SMTP_PORT:-587} + SMTP_USERNAME: ${SMTP_USERNAME:-} + SMTP_PASSWORD: ${SMTP_PASSWORD:-} + EMAIL_FROM: ${EMAIL_FROM:-noreply@example.com} + EMAIL_FROM_NAME: ${EMAIL_FROM_NAME:-STUPA System} + + # Rate Limiting RATE_IP_PER_MIN: ${RATE_IP_PER_MIN:-60} RATE_KEY_PER_MIN: ${RATE_KEY_PER_MIN:-30} - # PDF-Templates (liegen im Image in /app/assets) - QSM_TEMPLATE: /app/assets/qsm.pdf - VSM_TEMPLATE: /app/assets/vsm.pdf - # Optional: TZ + + # Storage + UPLOAD_DIR: /app/uploads + TEMPLATE_DIR: /app/templates + ATTACHMENT_STORAGE: ${ATTACHMENT_STORAGE:-filesystem} + FILESYSTEM_PATH: /app/attachments + + # Workflow + WORKFLOW_REQUIRED_VOTES: ${WORKFLOW_REQUIRED_VOTES:-5} + WORKFLOW_APPROVAL_THRESHOLD: ${WORKFLOW_APPROVAL_THRESHOLD:-50.0} + + # Application + FRONTEND_URL: ${FRONTEND_URL:-http://localhost:3001} + ENVIRONMENT: ${ENVIRONMENT:-production} + DEBUG: ${DEBUG:-false} TZ: ${TZ:-Europe/Berlin} ports: - "8000:8000" - # Healthcheck: ping FastAPI root + volumes: + - ./backend/uploads:/app/uploads + - ./backend/templates:/app/templates + - ./backend/attachments:/app/attachments + - pdf_forms:/app/pdf_forms healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8000/ || exit 1"] + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8000/health || exit 1"] interval: 10s timeout: 5s retries: 6 + networks: + - stupa_network frontend: build: context: ./frontend dockerfile: Dockerfile network: host + args: + - VITE_API_URL=${VITE_API_URL:-http://localhost:8000} + - VITE_OIDC_ENABLED=${OIDC_ENABLED:-false} + - VITE_EMAIL_ENABLED=${EMAIL_ENABLED:-false} container_name: stupa_frontend restart: unless-stopped depends_on: @@ -82,6 +153,29 @@ services: interval: 10s timeout: 5s retries: 6 + networks: + - stupa_network + + form_designer: + image: node:18-alpine + container_name: stupa_form_designer + restart: unless-stopped + working_dir: /app + command: npm run dev + depends_on: + - api + ports: + - "3002:3000" + volumes: + - ./form-designer:/app + - /app/node_modules + environment: + - NODE_ENV=development + - VITE_API_URL=http://localhost:8000 + networks: + - stupa_network + profiles: + - dev adminer: image: adminer:4 @@ -92,8 +186,32 @@ services: condition: service_healthy environment: ADMINER_DEFAULT_SERVER: db + ADMINER_DESIGN: pepa-linha-dark ports: - "8080:8080" + networks: + - stupa_network + + mailhog: + image: mailhog/mailhog:latest + container_name: stupa_mailhog + restart: unless-stopped + ports: + - "1025:1025" # SMTP server + - "8025:8025" # Web UI + networks: + - stupa_network + profiles: + - dev volumes: db_data: + driver: local + redis_data: + driver: local + pdf_forms: + driver: local + +networks: + stupa_network: + driver: bridge