#!/usr/bin/env python3 """ Production Deployment Script for RateRight Handles complete production deployment with health checks and rollback capabilities """ import os import sys import subprocess import time import requests import yaml import json from datetime import datetime from pathlib import Path import argparse import logging # Setup logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', handlers=[ logging.FileHandler('deployment.log'), logging.StreamHandler() ] ) logger = logging.getLogger(__name__) class ProductionDeployment: """Production deployment manager with comprehensive checks""" def __init__(self, environment='production'): self.environment = environment self.project_root = Path(__file__).parent.parent self.deployment_config = self.load_deployment_config() self.current_version = self.get_current_version() self.new_version = self.generate_version() def load_deployment_config(self): """Load deployment configuration""" config_file = self.project_root / 'deployment' / f'{self.environment}.yml' if config_file.exists(): with open(config_file, 'r') as f: return yaml.safe_load(f) return self.get_default_config() def get_default_config(self): """Default deployment configuration""" return { 'app_name': 'rateright', 'replicas': 3, 'max_unavailable': 1, 'health_check_timeout': 300, 'rollback_on_failure': True, 'pre_deployment_checks': True, 'post_deployment_verification': True, 'monitoring_enabled': True, 'backup_before_deploy': True } def get_current_version(self): """Get current deployed version""" try: result = subprocess.run( ['docker', 'image', 'ls', '--format', '{{.Tag}}', 'rateright:latest'], capture_output=True, text=True ) return result.stdout.strip() or 'unknown' except Exception: return 'unknown' def generate_version(self): """Generate new version tag""" timestamp = datetime.now().strftime('%Y%m%d-%H%M%S') try: # Get git commit hash result = subprocess.run(['git', 'rev-parse', '--short', 'HEAD'], capture_output=True, text=True) commit_hash = result.stdout.strip() return f"v{timestamp}-{commit_hash}" except Exception: return f"v{timestamp}" def run_pre_deployment_checks(self): """Run comprehensive pre-deployment checks""" logger.info("Running pre-deployment checks...") checks = [ self.check_environment_variables, self.check_database_connection, self.check_redis_connection, self.run_security_scan, self.check_resource_availability, self.validate_configuration, self.run_integration_tests, self.check_external_dependencies ] for check in checks: try: check_name = check.__name__.replace('_', ' ').title() logger.info(f"Running {check_name}...") result = check() if not result: raise Exception(f"{check_name} failed") logger.info(f"✅ {check_name} passed") except Exception as e: logger.error(f"❌ {check.__name__} failed: {e}") return False logger.info("✅ All pre-deployment checks passed") return True def check_environment_variables(self): """Check required environment variables""" required_vars = [ 'DATABASE_URL', 'REDIS_URL', 'SECRET_KEY', 'JWT_SECRET_KEY', 'STRIPE_SECRET_KEY', 'EMAIL_USERNAME', 'EMAIL_PASSWORD' ] missing_vars = [] for var in required_vars: if not os.getenv(var): missing_vars.append(var) if missing_vars: logger.error(f"Missing environment variables: {missing_vars}") return False return True def check_database_connection(self): """Test database connectivity""" try: # Use the app's database connection test result = subprocess.run([ 'python', '-c', 'from app import create_app; from app.extensions import db; ' 'app = create_app(); ' 'with app.app_context(): db.engine.execute("SELECT 1")' ], capture_output=True, text=True, cwd=self.project_root) return result.returncode == 0 except Exception as e: logger.error(f"Database connection failed: {e}") return False def check_redis_connection(self): """Test Redis connectivity""" try: import redis redis_url = os.getenv('REDIS_URL', 'redis://localhost:6379/0') client = redis.from_url(redis_url) client.ping() return True except Exception as e: logger.error(f"Redis connection failed: {e}") return False def run_security_scan(self): """Run security vulnerability scan""" try: # Run safety check on Python dependencies result = subprocess.run(['safety', 'check'], capture_output=True, text=True) if result.returncode != 0: logger.warning(f"Security scan found issues: {result.stdout}") # Don't fail deployment for security warnings, just log them # Check for secrets in codebase result = subprocess.run(['git-secrets', '--scan'], capture_output=True, text=True) if result.returncode != 0: logger.error("Secrets found in codebase!") return False return True except FileNotFoundError: logger.warning("Security scanning tools not available, skipping...") return True except Exception as e: logger.error(f"Security scan failed: {e}") return False def check_resource_availability(self): """Check system resources""" import psutil # Check available memory memory = psutil.virtual_memory() if memory.percent > 90: logger.error(f"Low memory available: {memory.percent}% used") return False # Check available disk space disk = psutil.disk_usage('/') if disk.percent > 90: logger.error(f"Low disk space: {disk.percent}% used") return False return True def validate_configuration(self): """Validate configuration files""" config_files = [ 'docker-compose.production.yml', 'nginx/nginx.conf', 'monitoring/prometheus.yml' ] for config_file in config_files: file_path = self.project_root / config_file if not file_path.exists(): logger.error(f"Configuration file missing: {config_file}") return False # Validate docker-compose syntax try: result = subprocess.run([ 'docker-compose', '-f', 'docker-compose.production.yml', 'config' ], capture_output=True, text=True, cwd=self.project_root) if result.returncode != 0: logger.error(f"Docker compose validation failed: {result.stderr}") return False except Exception as e: logger.error(f"Configuration validation failed: {e}") return False return True def run_integration_tests(self): """Run integration tests""" try: logger.info("Running integration test suite...") result = subprocess.run([ 'python', '-m', 'pytest', 'tests/test_integration_enhanced.py', '-v' ], capture_output=True, text=True, cwd=self.project_root) if result.returncode != 0: logger.error(f"Integration tests failed: {result.stdout}") return False return True except Exception as e: logger.error(f"Integration test execution failed: {e}") return False def check_external_dependencies(self): """Check external service dependencies""" dependencies = [ {'name': 'Stripe API', 'url': 'https://api.stripe.com/v1'}, {'name': 'Email Service', 'url': 'https://smtp.gmail.com', 'port': 587}, ] for dep in dependencies: try: if 'port' in dep: # Check port connectivity import socket sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(5) result = sock.connect_ex((dep['url'].replace('https://', '').replace('http://', ''), dep['port'])) sock.close() if result != 0: logger.error(f"{dep['name']} port check failed") return False else: # Check HTTP connectivity response = requests.get(dep['url'], timeout=10) if response.status_code >= 400: logger.error(f"{dep['name']} returned status {response.status_code}") return False logger.info(f"✅ {dep['name']} connectivity confirmed") except Exception as e: logger.error(f"{dep['name']} check failed: {e}") return False return True def create_backup(self): """Create backup before deployment""" logger.info("Creating pre-deployment backup...") timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') backup_name = f"rateright_backup_{timestamp}" try: # Database backup db_backup_cmd = [ 'pg_dump', os.getenv('DATABASE_URL'), '-f', f'/tmp/{backup_name}_database.sql' ] subprocess.run(db_backup_cmd, check=True) # Application files backup app_backup_cmd = [ 'tar', '-czf', f'/tmp/{backup_name}_app.tar.gz', str(self.project_root) ] subprocess.run(app_backup_cmd, check=True) logger.info(f"✅ Backup created: {backup_name}") return backup_name except Exception as e: logger.error(f"Backup creation failed: {e}") return None def build_and_push_image(self): """Build and push Docker image""" logger.info(f"Building Docker image version {self.new_version}...") try: # Build image build_cmd = [ 'docker', 'build', '-f', 'Dockerfile.production', '-t', f'rateright:{self.new_version}', '-t', 'rateright:latest', '.' ] result = subprocess.run(build_cmd, cwd=self.project_root) if result.returncode != 0: raise Exception("Docker build failed") # Tag for registry if configured registry = os.getenv('DOCKER_REGISTRY') if registry: tag_cmd = ['docker', 'tag', f'rateright:{self.new_version}', f'{registry}/rateright:{self.new_version}'] subprocess.run(tag_cmd, check=True) push_cmd = ['docker', 'push', f'{registry}/rateright:{self.new_version}'] subprocess.run(push_cmd, check=True) logger.info(f"✅ Image pushed to registry: {registry}/rateright:{self.new_version}") logger.info(f"✅ Docker image built: rateright:{self.new_version}") return True except Exception as e: logger.error(f"Docker build/push failed: {e}") return False def deploy_application(self): """Deploy the application using Docker Compose""" logger.info("Deploying application...") try: # Set environment variable for image version os.environ['RATERIGHT_VERSION'] = self.new_version # Deploy with rolling update deploy_cmd = [ 'docker-compose', '-f', 'docker-compose.production.yml', 'up', '-d', '--remove-orphans' ] result = subprocess.run(deploy_cmd, cwd=self.project_root) if result.returncode != 0: raise Exception("Docker compose deployment failed") logger.info("✅ Application deployed successfully") return True except Exception as e: logger.error(f"Deployment failed: {e}") return False def run_health_checks(self): """Run post-deployment health checks""" logger.info("Running health checks...") health_endpoints = [ '/api/health', '/api/health/database', '/api/health/redis', '/api/health/external' ] base_url = os.getenv('APP_URL', 'http://localhost:8080') # Wait for application to start logger.info("Waiting for application to start...") time.sleep(30) for endpoint in health_endpoints: url = f"{base_url}{endpoint}" for attempt in range(10): # 10 attempts with 30 second intervals try: response = requests.get(url, timeout=10) if response.status_code == 200: logger.info(f"✅ Health check passed: {endpoint}") break else: logger.warning(f"Health check failed: {endpoint} returned {response.status_code}") except Exception as e: logger.warning(f"Health check attempt {attempt + 1} failed for {endpoint}: {e}") if attempt < 9: time.sleep(30) else: logger.error(f"❌ Health check failed after 10 attempts: {endpoint}") return False # Run smoke tests return self.run_smoke_tests() def run_smoke_tests(self): """Run smoke tests to verify core functionality""" logger.info("Running smoke tests...") try: result = subprocess.run([ 'python', '-m', 'pytest', 'tests/test_core_functionality.py', '--tb=short', '-v' ], capture_output=True, text=True, cwd=self.project_root) if result.returncode != 0: logger.error(f"Smoke tests failed: {result.stdout}") return False logger.info("✅ Smoke tests passed") return True except Exception as e: logger.error(f"Smoke test execution failed: {e}") return False def rollback_deployment(self, backup_name=None): """Rollback to previous version""" logger.info("Rolling back deployment...") try: if backup_name: # Restore from backup restore_cmd = [ 'docker-compose', '-f', 'docker-compose.production.yml', 'down' ] subprocess.run(restore_cmd, cwd=self.project_root) # Restore database restore_db_cmd = [ 'psql', os.getenv('DATABASE_URL'), '-f', f'/tmp/{backup_name}_database.sql' ] subprocess.run(restore_db_cmd) logger.info(f"✅ Rolled back to backup: {backup_name}") else: # Simple rollback to previous image os.environ['RATERIGHT_VERSION'] = self.current_version rollback_cmd = [ 'docker-compose', '-f', 'docker-compose.production.yml', 'up', '-d' ] subprocess.run(rollback_cmd, cwd=self.project_root) logger.info(f"✅ Rolled back to version: {self.current_version}") return True except Exception as e: logger.error(f"Rollback failed: {e}") return False def setup_monitoring(self): """Setup monitoring and alerting""" logger.info("Setting up monitoring...") try: # Start monitoring services monitoring_cmd = [ 'docker-compose', '-f', 'docker-compose.production.yml', 'up', '-d', 'prometheus', 'grafana' ] subprocess.run(monitoring_cmd, cwd=self.project_root) # Wait for services to start time.sleep(15) # Configure Grafana dashboards grafana_url = 'http://localhost:3000' for attempt in range(5): try: response = requests.get(f"{grafana_url}/api/health", timeout=10) if response.status_code == 200: logger.info("✅ Monitoring services are running") return True except Exception: time.sleep(10) logger.warning("Monitoring services may not be fully ready") return True except Exception as e: logger.error(f"Monitoring setup failed: {e}") return False def deploy(self, skip_tests=False, skip_backup=False): """Main deployment orchestration""" logger.info(f"Starting deployment of version {self.new_version}") start_time = time.time() backup_name = None try: # Pre-deployment checks if not skip_tests and not self.run_pre_deployment_checks(): return False # Create backup if not skip_backup: backup_name = self.create_backup() if not backup_name and self.deployment_config.get('backup_before_deploy', True): logger.error("Backup creation failed and is required") return False # Build and push image if not self.build_and_push_image(): return False # Deploy application if not self.deploy_application(): if self.deployment_config.get('rollback_on_failure', True): self.rollback_deployment(backup_name) return False # Health checks if not self.run_health_checks(): if self.deployment_config.get('rollback_on_failure', True): self.rollback_deployment(backup_name) return False # Setup monitoring if self.deployment_config.get('monitoring_enabled', True): self.setup_monitoring() # Log successful deployment duration = time.time() - start_time logger.info(f"🎉 Deployment successful! Version {self.new_version} deployed in {duration:.2f} seconds") # Create deployment record self.record_deployment(self.new_version, True, duration) return True except Exception as e: logger.error(f"Deployment failed: {e}") if self.deployment_config.get('rollback_on_failure', True): self.rollback_deployment(backup_name) duration = time.time() - start_time self.record_deployment(self.new_version, False, duration, str(e)) return False def record_deployment(self, version, success, duration, error=None): """Record deployment in deployment history""" deployment_record = { 'version': version, 'timestamp': datetime.now().isoformat(), 'success': success, 'duration': duration, 'environment': self.environment, 'error': error } # Save to deployment history file history_file = self.project_root / 'deployment_history.json' history = [] if history_file.exists(): with open(history_file, 'r') as f: history = json.load(f) history.append(deployment_record) # Keep only last 50 deployments history = history[-50:] with open(history_file, 'w') as f: json.dump(history, f, indent=2) def main(): parser = argparse.ArgumentParser(description='RateRight Production Deployment') parser.add_argument('--environment', default='production', help='Deployment environment (default: production)') parser.add_argument('--skip-tests', action='store_true', help='Skip pre-deployment tests') parser.add_argument('--skip-backup', action='store_true', help='Skip backup creation') parser.add_argument('--rollback', metavar='VERSION', help='Rollback to specific version') args = parser.parse_args() deployment = ProductionDeployment(args.environment) if args.rollback: logger.info(f"Rolling back to version: {args.rollback}") success = deployment.rollback_deployment() else: success = deployment.deploy( skip_tests=args.skip_tests, skip_backup=args.skip_backup ) sys.exit(0 if success else 1) if __name__ == '__main__': main()