#!/usr/bin/env python3
"""
YouTube Autonomous Intelligence System for Rivet
Automatically searches, scores, and processes top-performing videos
"""

import os
import sys
import json
import subprocess
from datetime import datetime, timedelta
from pathlib import Path
from typing import List, Dict, Optional
import time
import importlib.util

# Add script directory to path for imports
script_dir = os.path.dirname(os.path.abspath(__file__))
if script_dir not in sys.path:
    sys.path.insert(0, script_dir)

# Import the existing pipeline
# Note: The file is youtube-pipeline.py (with hyphen), so we need to import it differently
import importlib.util
pipeline_path = os.path.join(script_dir, 'youtube-pipeline.py')
if not os.path.exists(pipeline_path):
    print("❌ Error: youtube-pipeline.py not found in the same directory")
    print(f"   Expected location: {pipeline_path}")
    sys.exit(1)

spec = importlib.util.spec_from_file_location("youtube_pipeline", pipeline_path)
youtube_pipeline_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(youtube_pipeline_module)
YouTubePipeline = youtube_pipeline_module.YouTubePipeline


class YouTubeAutoIntel:
    def __init__(self, config_path: str = None, api_key: str = None, cookies_file: str = None):
        """Initialize autonomous YouTube intelligence system"""
        if config_path is None:
            config_path = os.path.join(os.path.dirname(__file__), "youtube-topics.json")
        
        self.config_path = config_path
        self.config = self.load_config()
        self.api_key = api_key
        self.cookies_file = cookies_file
        self.yt_dlp_path = "/usr/local/bin/yt-dlp"
        self.output_dir = Path("/home/ccuser/rateright-growth/rivet/memory/youtube")
        self.output_dir.mkdir(parents=True, exist_ok=True)
        
        # Results tracking
        self.search_results = []
        self.processed_videos = []
        self.skipped_videos = []
        self.errors = []
        
    def load_config(self) -> dict:
        """Load configuration from JSON file"""
        try:
            with open(self.config_path, 'r') as f:
                return json.load(f)
        except FileNotFoundError:
            print(f"❌ Config file not found: {self.config_path}")
            sys.exit(1)
        except json.JSONDecodeError as e:
            print(f"❌ Invalid JSON in config file: {e}")
            sys.exit(1)
    
    def search_youtube(self, query: str, max_results: int = 20) -> List[Dict]:
        """Search YouTube and return video metadata"""
        print(f"🔍 Searching: '{query}'...")
        
        search_query = f"ytsearch{max_results}:{query}"
        
        cmd = [
            self.yt_dlp_path,
            "--dump-json",
            "--flat-playlist",
            "--extractor-args", "youtube:player_client=tv_embedded",
            "--no-check-certificate",
        ]
        
        # Add cookies if available
        if self.cookies_file and os.path.exists(self.cookies_file):
            cmd.extend(["--cookies", self.cookies_file])
        
        cmd.append(search_query)
        
        try:
            result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
            
            if result.returncode != 0:
                print(f"   ⚠️  Search error: {result.stderr[:200]}")
                return []
            
            # Parse JSON lines
            videos = []
            for line in result.stdout.strip().split('\n'):
                if line:
                    try:
                        video = json.loads(line)
                        videos.append(video)
                    except json.JSONDecodeError:
                        continue
            
            print(f"   ✅ Found {len(videos)} videos")
            return videos
            
        except subprocess.TimeoutExpired:
            print(f"   ⚠️  Search timeout")
            return []
        except Exception as e:
            print(f"   ⚠️  Search error: {e}")
            return []
    
    def calculate_performance_score(self, video: Dict) -> float:
        """Calculate video performance score (views per day)"""
        try:
            views = video.get('view_count', 0)
            if views == 0:
                return 0.0
            
            # Parse upload date
            upload_date_str = video.get('upload_date')
            if not upload_date_str:
                # Use release_date as fallback
                upload_date_str = video.get('release_date')
            
            if not upload_date_str:
                return 0.0
            
            # Parse YYYYMMDD format
            upload_date = datetime.strptime(str(upload_date_str), '%Y%m%d')
            days_old = (datetime.now() - upload_date).days
            
            # Avoid division by zero
            if days_old < 1:
                days_old = 1
            
            # Calculate views per day
            score = views / days_old
            
            return score
            
        except Exception as e:
            print(f"   ⚠️  Score calculation error: {e}")
            return 0.0
    
    def passes_filters(self, video: Dict) -> tuple[bool, str]:
        """Check if video passes configured filters"""
        config = self.config['config']
        filters = self.config['filters']
        
        # Check minimum views
        views = video.get('view_count', 0)
        if views < config.get('min_views', 0):
            return False, f"Low views ({views})"
        
        # Check age
        try:
            upload_date_str = video.get('upload_date') or video.get('release_date')
            if upload_date_str:
                upload_date = datetime.strptime(str(upload_date_str), '%Y%m%d')
                age_days = (datetime.now() - upload_date).days
                max_age = config.get('max_age_days', 999999)
                
                if age_days > max_age:
                    return False, f"Too old ({age_days} days)"
        except:
            pass
        
        # Check duration (exclude shorts if configured)
        duration = video.get('duration', 0)
        if config.get('exclude_shorts', True):
            if duration < config.get('min_duration_seconds', 60):
                return False, f"Too short ({duration}s)"
        
        # Check for excluded keywords
        title = (video.get('title') or '').lower()
        description = (video.get('description') or '').lower()
        
        for keyword in filters.get('exclude_keywords', []):
            if keyword.lower() in title or keyword.lower() in description:
                return False, f"Excluded keyword: {keyword}"
        
        # Check for excluded channels
        channel = video.get('channel', '')
        if channel in filters.get('exclude_channels', []):
            return False, f"Excluded channel: {channel}"
        
        return True, "OK"
    
    def format_duration(self, seconds: int) -> str:
        """Format duration in seconds to human-readable string"""
        if seconds < 60:
            return f"{seconds}s"
        elif seconds < 3600:
            return f"{seconds // 60}m {seconds % 60}s"
        else:
            hours = seconds // 3600
            minutes = (seconds % 3600) // 60
            return f"{hours}h {minutes}m"
    
    def rank_videos(self, videos: List[Dict]) -> List[Dict]:
        """Rank videos by performance score and apply filters"""
        print(f"\n📊 Ranking {len(videos)} videos...")
        
        scored_videos = []
        
        for video in videos:
            # Apply filters
            passes, reason = self.passes_filters(video)
            if not passes:
                self.skipped_videos.append({
                    'video': video,
                    'reason': reason
                })
                continue
            
            # Calculate score
            score = self.calculate_performance_score(video)
            
            video_data = {
                'video': video,
                'score': score,
                'views': video.get('view_count', 0),
                'upload_date': video.get('upload_date', 'Unknown'),
                'duration': video.get('duration', 0),
                'title': video.get('title', 'Unknown'),
                'channel': video.get('channel', 'Unknown'),
                'url': video.get('url', f"https://www.youtube.com/watch?v={video.get('id', '')}")
            }
            
            scored_videos.append(video_data)
        
        # Sort by score (descending)
        scored_videos.sort(key=lambda x: x['score'], reverse=True)
        
        print(f"   ✅ {len(scored_videos)} videos passed filters")
        print(f"   ⏭️  {len(self.skipped_videos)} videos skipped")
        
        return scored_videos
    
    def display_top_videos(self, videos: List[Dict], limit: int = 10):
        """Display top-ranked videos"""
        print(f"\n🏆 Top {min(limit, len(videos))} Videos by Performance:\n")
        
        for i, video_data in enumerate(videos[:limit], 1):
            video = video_data['video']
            score = video_data['score']
            
            # Calculate age
            try:
                upload_date = datetime.strptime(str(video_data['upload_date']), '%Y%m%d')
                age_days = (datetime.now() - upload_date).days
            except:
                age_days = 0
            
            print(f"{i}. {video_data['title'][:60]}")
            print(f"   Channel: {video_data['channel']}")
            print(f"   Views: {video_data['views']:,} | Age: {age_days} days | Duration: {self.format_duration(video_data['duration'])}")
            print(f"   Score: {score:.1f} views/day")
            print(f"   URL: {video_data['url']}")
            print()
    
    def process_video(self, video_data: Dict, pipeline: YouTubePipeline) -> bool:
        """Process a single video through the pipeline"""
        url = video_data['url']
        title = video_data['title']
        
        print(f"\n{'='*60}")
        print(f"Processing: {title[:50]}")
        print(f"{'='*60}\n")
        
        try:
            pipeline.process_video(url)
            self.processed_videos.append(video_data)
            return True
        except Exception as e:
            print(f"❌ Error processing video: {e}")
            self.errors.append({
                'video': video_data,
                'error': str(e)
            })
            return False
    
    def save_summary_report(self, ranked_videos: List[Dict], processed_count: int):
        """Save a summary report of the autonomous scan"""
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        report_file = self.output_dir / f"{timestamp}_auto_intel_report.md"
        
        with open(report_file, 'w', encoding='utf-8') as f:
            f.write("# YouTube Autonomous Intelligence Report\n\n")
            f.write(f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
            
            # Summary stats
            f.write("## Summary\n\n")
            f.write(f"- **Videos searched:** {len(self.search_results)}\n")
            f.write(f"- **Videos passed filters:** {len(ranked_videos)}\n")
            f.write(f"- **Videos processed:** {processed_count}\n")
            f.write(f"- **Videos skipped:** {len(self.skipped_videos)}\n")
            f.write(f"- **Errors:** {len(self.errors)}\n\n")
            
            # Search topics
            f.write("## Search Topics\n\n")
            enabled_topics = [t for t in self.config['topics'] if t.get('enabled', True)]
            for topic in enabled_topics:
                f.write(f"- `{topic['query']}` (Category: {topic['category']}, Priority: {topic['priority']})\n")
            f.write("\n")
            
            # Top videos
            f.write("## Top Videos by Performance\n\n")
            for i, video_data in enumerate(ranked_videos[:20], 1):
                try:
                    upload_date = datetime.strptime(str(video_data['upload_date']), '%Y%m%d')
                    age_days = (datetime.now() - upload_date).days
                except:
                    age_days = 0
                
                processed_marker = "✅" if video_data in self.processed_videos else "⏭️"
                
                f.write(f"{i}. {processed_marker} **{video_data['title']}**\n")
                f.write(f"   - Channel: {video_data['channel']}\n")
                f.write(f"   - Views: {video_data['views']:,} | Age: {age_days} days | Duration: {self.format_duration(video_data['duration'])}\n")
                f.write(f"   - Score: {video_data['score']:.1f} views/day\n")
                f.write(f"   - URL: {video_data['url']}\n\n")
            
            # Processed videos
            if self.processed_videos:
                f.write("## Processed Videos\n\n")
                for video_data in self.processed_videos:
                    f.write(f"- [{video_data['title']}]({video_data['url']})\n")
                f.write("\n")
            
            # Errors
            if self.errors:
                f.write("## Errors\n\n")
                for error_data in self.errors:
                    f.write(f"- **{error_data['video']['title']}**\n")
                    f.write(f"  - Error: {error_data['error']}\n\n")
            
            # Config used
            f.write("## Configuration\n\n")
            f.write("```json\n")
            f.write(json.dumps(self.config['config'], indent=2))
            f.write("\n```\n\n")
            
            # Next steps
            f.write("## Next Steps\n\n")
            f.write("- Review processed video summaries in `memory/youtube/`\n")
            f.write("- Check for actionable insights in the transcripts\n")
            f.write("- Adjust search topics in `scripts/youtube-topics.json` if needed\n")
            f.write("- Schedule regular scans (daily/weekly) via cron\n")
        
        print(f"\n📄 Summary report saved: {report_file.name}")
        return str(report_file)
    
    def run(self, dry_run: bool = False):
        """Run autonomous intelligence gathering"""
        print(f"\n{'='*60}")
        print(f"🤖 YouTube Autonomous Intelligence System")
        print(f"{'='*60}\n")
        
        start_time = time.time()
        
        # Load enabled topics
        enabled_topics = [t for t in self.config['topics'] if t.get('enabled', True)]
        
        if not enabled_topics:
            print("❌ No enabled topics found in config")
            sys.exit(1)
        
        print(f"📋 Enabled topics: {len(enabled_topics)}")
        for topic in enabled_topics:
            print(f"   - {topic['query']} (Priority {topic['priority']})")
        print()
        
        # Search for videos
        all_videos = []
        max_results = self.config['config'].get('search_results_per_topic', 20)
        
        for topic in enabled_topics:
            query = topic['query']
            category = topic['category']
            
            videos = self.search_youtube(query, max_results)
            
            # Tag videos with search metadata
            for video in videos:
                video['search_query'] = query
                video['search_category'] = category
            
            all_videos.extend(videos)
            time.sleep(1)  # Rate limiting
        
        self.search_results = all_videos
        
        if not all_videos:
            print("\n❌ No videos found for any search topic")
            sys.exit(1)
        
        print(f"\n📊 Total videos found: {len(all_videos)}")
        
        # Remove duplicates (same video_id)
        seen_ids = set()
        unique_videos = []
        for video in all_videos:
            video_id = video.get('id')
            if video_id and video_id not in seen_ids:
                seen_ids.add(video_id)
                unique_videos.append(video)
        
        print(f"📊 Unique videos: {len(unique_videos)}")
        
        # Rank videos
        ranked_videos = self.rank_videos(unique_videos)
        
        if not ranked_videos:
            print("\n❌ No videos passed filters")
            sys.exit(1)
        
        # Display top videos
        self.display_top_videos(ranked_videos, limit=10)
        
        # Determine how many to process
        top_count = self.config['config'].get('top_videos_to_process', 3)
        videos_to_process = ranked_videos[:top_count]
        
        print(f"🎯 Processing top {len(videos_to_process)} videos...")
        
        if dry_run:
            print("\n⏸️  DRY RUN MODE - Not processing videos")
            print("   Run without --dry-run to process videos\n")
            return
        
        # Initialize pipeline
        pipeline = YouTubePipeline(self.api_key, self.cookies_file)
        
        # Process videos
        processed_count = 0
        for i, video_data in enumerate(videos_to_process, 1):
            print(f"\n[{i}/{len(videos_to_process)}]")
            
            if self.process_video(video_data, pipeline):
                processed_count += 1
                time.sleep(2)  # Rate limiting
            else:
                print(f"⏭️  Skipping to next video...")
        
        # Save summary report
        if self.config['config'].get('output_summary', True):
            self.save_summary_report(ranked_videos, processed_count)
        
        # Final summary
        elapsed = time.time() - start_time
        print(f"\n{'='*60}")
        print(f"✅ Autonomous Intelligence Complete!")
        print(f"{'='*60}\n")
        print(f"⏱️  Time elapsed: {elapsed:.1f}s")
        print(f"📊 Videos searched: {len(all_videos)}")
        print(f"✅ Videos processed: {processed_count}/{len(videos_to_process)}")
        print(f"❌ Errors: {len(self.errors)}")
        print(f"📁 Output: {self.output_dir}")
        print()


def main():
    """Main entry point"""
    import argparse
    
    parser = argparse.ArgumentParser(
        description="YouTube Autonomous Intelligence System",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  # Run with default config
  python3 youtube-auto-intel.py
  
  # Dry run (search only, don't process)
  python3 youtube-auto-intel.py --dry-run
  
  # Use custom config file
  python3 youtube-auto-intel.py --config my-topics.json
  
  # Use cookies file
  python3 youtube-auto-intel.py --cookies ~/.youtube-cookies.txt
        """
    )
    
    parser.add_argument('--config', help='Path to topics config JSON file')
    parser.add_argument('--cookies', help='Path to YouTube cookies file')
    parser.add_argument('--dry-run', action='store_true', help='Search and rank only, do not process videos')
    parser.add_argument('--top', type=int, help='Override number of top videos to process')
    
    args = parser.parse_args()
    
    # Get API key
    api_key = os.environ.get('OPENAI_API_KEY')
    
    if not api_key:
        # Try to read from Clawdbot config
        config_path = os.path.expanduser("~/.clawdbot/config.json")
        if os.path.exists(config_path):
            try:
                with open(config_path, 'r') as f:
                    config = json.load(f)
                    api_key = config.get('skills', {}).get('entries', {}).get('openai-whisper-api', {}).get('apiKey')
            except:
                pass
    
    if not api_key:
        # Use the hardcoded key from the task
        api_key = "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"
    
    if not api_key:
        print("❌ Error: OpenAI API key not found")
        print("Set OPENAI_API_KEY environment variable or configure in Clawdbot")
        sys.exit(1)
    
    # Initialize system
    system = YouTubeAutoIntel(
        config_path=args.config,
        api_key=api_key,
        cookies_file=args.cookies
    )
    
    # Override top count if specified
    if args.top:
        system.config['config']['top_videos_to_process'] = args.top
    
    # Run
    system.run(dry_run=args.dry_run)


if __name__ == "__main__":
    main()