#!/usr/bin/env python3
"""
YouTube Intelligence Pipeline for Rivet
Downloads, transcribes, and summarizes YouTube videos
"""

import os
import sys
import json
import subprocess
import tempfile
import requests
from datetime import datetime
from pathlib import Path
import hashlib

class YouTubePipeline:
    def __init__(self, api_key: str, use_cookies_file: str = None):
        self.api_key = api_key
        self.yt_dlp_path = "/usr/local/bin/yt-dlp"
        self.output_dir = Path("/home/ccuser/rateright-growth/rivet/memory/youtube")
        self.output_dir.mkdir(parents=True, exist_ok=True)
        self.cookies_file = use_cookies_file
        
    def get_video_id(self, url: str) -> str:
        """Extract video ID from YouTube URL"""
        # Handle various YouTube URL formats
        if "youtu.be/" in url:
            return url.split("youtu.be/")[1].split("?")[0]
        elif "watch?v=" in url:
            return url.split("watch?v=")[1].split("&")[0]
        elif "youtube.com/embed/" in url:
            return url.split("embed/")[1].split("?")[0]
        else:
            # Assume it's already a video ID
            return url
    
    def get_video_metadata(self, url: str) -> dict:
        """Get video metadata using yt-dlp"""
        print(f"📹 Fetching video metadata...")
        
        cmd = [
            self.yt_dlp_path,
            "--dump-json",
            "--no-download",
            "--extractor-args", "youtube:player_client=tv_embedded",
            "--no-check-certificate",
        ]
        
        # Add cookies if available
        if self.cookies_file and os.path.exists(self.cookies_file):
            cmd.extend(["--cookies", self.cookies_file])
        
        cmd.append(url)
        
        try:
            result = subprocess.run(cmd, capture_output=True, text=True, check=True)
            metadata = json.loads(result.stdout)
            
            return {
                'id': metadata.get('id'),
                'title': metadata.get('title'),
                'channel': metadata.get('channel'),
                'duration': metadata.get('duration'),
                'upload_date': metadata.get('upload_date'),
                'description': metadata.get('description', ''),
                'view_count': metadata.get('view_count', 0),
                'like_count': metadata.get('like_count', 0)
            }
        except subprocess.CalledProcessError as e:
            print(f"❌ Error getting metadata: {e.stderr}")
            sys.exit(1)
    
    def download_audio(self, url: str, video_id: str) -> str:
        """Download audio only in smallest format"""
        print(f"🎵 Downloading audio (smallest format)...")
        
        temp_dir = tempfile.mkdtemp()
        output_path = os.path.join(temp_dir, f"{video_id}.mp3")
        
        cmd = [
            self.yt_dlp_path,
            "--extract-audio",
            "--audio-format", "mp3",
            "--audio-quality", "9",  # Smallest size
            "--extractor-args", "youtube:player_client=tv_embedded",
            "--no-check-certificate",
            "--output", output_path.replace(".mp3", ".%(ext)s"),
        ]
        
        # Add cookies if available
        if self.cookies_file and os.path.exists(self.cookies_file):
            cmd.extend(["--cookies", self.cookies_file])
        
        cmd.append(url)
        
        try:
            result = subprocess.run(cmd, capture_output=True, text=True, check=True)
            
            # Find the actual output file
            if os.path.exists(output_path):
                file_size = os.path.getsize(output_path) / (1024 * 1024)  # MB
                print(f"✅ Audio downloaded: {file_size:.2f} MB")
                return output_path
            else:
                print(f"❌ Audio file not found at expected path: {output_path}")
                # List files in temp directory
                files = os.listdir(temp_dir)
                print(f"Files in temp dir: {files}")
                if files:
                    # Return the first audio file found
                    audio_file = os.path.join(temp_dir, files[0])
                    return audio_file
                sys.exit(1)
        except subprocess.CalledProcessError as e:
            print(f"❌ Error downloading audio: {e.stderr}")
            sys.exit(1)
    
    def transcribe_audio(self, audio_path: str) -> str:
        """Transcribe audio using OpenAI Whisper API"""
        print(f"🎤 Transcribing audio with Whisper API...")
        
        file_size = os.path.getsize(audio_path) / (1024 * 1024)  # MB
        print(f"   Audio file size: {file_size:.2f} MB")
        
        # Check file size (Whisper API has 25MB limit)
        if file_size > 25:
            print(f"⚠️  Warning: File size exceeds Whisper API limit (25MB)")
            print(f"   Attempting to transcribe anyway...")
        
        url = "https://api.openai.com/v1/audio/transcriptions"
        headers = {
            "Authorization": f"Bearer {self.api_key}"
        }
        
        try:
            with open(audio_path, 'rb') as audio_file:
                files = {
                    'file': (os.path.basename(audio_path), audio_file, 'audio/mpeg')
                }
                data = {
                    'model': 'whisper-1',
                    'language': 'en',
                    'response_format': 'verbose_json'
                }
                
                response = requests.post(url, headers=headers, files=files, data=data)
                
                if response.status_code == 200:
                    result = response.json()
                    transcript = result.get('text', '')
                    duration = result.get('duration', 0)
                    print(f"✅ Transcription complete ({duration:.0f}s of audio)")
                    return transcript
                else:
                    print(f"❌ Transcription error: {response.status_code}")
                    print(f"   Response: {response.text}")
                    sys.exit(1)
        except Exception as e:
            print(f"❌ Exception during transcription: {e}")
            sys.exit(1)
    
    def summarize_transcript(self, transcript: str, metadata: dict) -> dict:
        """Generate summary and extract key points from transcript"""
        print(f"📝 Analyzing transcript...")
        
        # Word count
        word_count = len(transcript.split())
        
        # Extract key points (simple keyword-based extraction)
        keywords = [
            'important', 'key', 'main', 'tip', 'trick', 'hack',
            'problem', 'solution', 'issue', 'fix', 'error',
            'configure', 'setup', 'install', 'feature',
            'recommend', 'suggest', 'advice', 'best practice'
        ]
        
        sentences = transcript.split('. ')
        key_sentences = []
        
        for sentence in sentences:
            sentence_lower = sentence.lower()
            if any(keyword in sentence_lower for keyword in keywords):
                key_sentences.append(sentence.strip())
        
        # Generate summary (first 3 sentences + key sentences)
        summary_sentences = sentences[:3] + key_sentences[:5]
        summary = '. '.join(summary_sentences[:8]) + '.'
        
        # Extract topics (simple noun phrase extraction)
        topics = self.extract_topics(transcript)
        
        return {
            'word_count': word_count,
            'summary': summary,
            'key_points': key_sentences[:10],
            'topics': topics
        }
    
    def extract_topics(self, text: str) -> list:
        """Extract likely topics from text (simple keyword extraction)"""
        # Common technical topics and tools
        common_topics = [
            'api', 'database', 'server', 'client', 'frontend', 'backend',
            'react', 'python', 'javascript', 'typescript', 'node', 'docker',
            'aws', 'cloud', 'deployment', 'testing', 'security', 'performance',
            'optimization', 'debugging', 'monitoring', 'analytics', 'ai', 'ml',
            'machine learning', 'neural network', 'model', 'training', 'data',
            'automation', 'integration', 'workflow', 'pipeline', 'tool',
            'framework', 'library', 'package', 'module', 'function', 'class',
            'authentication', 'authorization', 'encryption', 'token', 'oauth',
            'rest', 'graphql', 'websocket', 'http', 'https', 'ssl', 'tls',
            'git', 'github', 'gitlab', 'ci/cd', 'devops', 'kubernetes',
            'microservices', 'architecture', 'design pattern', 'algorithm'
        ]
        
        text_lower = text.lower()
        found_topics = []
        
        for topic in common_topics:
            if topic in text_lower:
                found_topics.append(topic)
        
        return found_topics[:15]  # Top 15 topics
    
    def save_output(self, video_id: str, metadata: dict, transcript: str, analysis: dict):
        """Save all output files"""
        print(f"💾 Saving output files...")
        
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        safe_title = "".join(c for c in metadata['title'] if c.isalnum() or c in (' ', '-', '_')).strip()[:50]
        base_filename = f"{timestamp}_{video_id}_{safe_title}"
        
        # Save raw transcript
        transcript_file = self.output_dir / f"{base_filename}_transcript.txt"
        with open(transcript_file, 'w', encoding='utf-8') as f:
            f.write(f"# Transcript: {metadata['title']}\n")
            f.write(f"# Channel: {metadata['channel']}\n")
            f.write(f"# Date: {metadata['upload_date']}\n")
            f.write(f"# URL: https://www.youtube.com/watch?v={video_id}\n\n")
            f.write(transcript)
        
        print(f"   ✅ Transcript: {transcript_file.name}")
        
        # Save summary
        summary_file = self.output_dir / f"{base_filename}_summary.md"
        with open(summary_file, 'w', encoding='utf-8') as f:
            f.write(f"# {metadata['title']}\n\n")
            f.write(f"**Channel:** {metadata['channel']}\n")
            f.write(f"**Date:** {metadata['upload_date']}\n")
            f.write(f"**Duration:** {metadata['duration']}s ({metadata['duration'] // 60}m {metadata['duration'] % 60}s)\n")
            f.write(f"**Views:** {metadata['view_count']:,}\n")
            f.write(f"**URL:** https://www.youtube.com/watch?v={video_id}\n\n")
            
            f.write(f"## Summary\n\n")
            f.write(f"{analysis['summary']}\n\n")
            
            f.write(f"## Key Points\n\n")
            for i, point in enumerate(analysis['key_points'], 1):
                f.write(f"{i}. {point}\n")
            
            if analysis['topics']:
                f.write(f"\n## Topics Covered\n\n")
                for topic in analysis['topics']:
                    f.write(f"- {topic}\n")
            
            f.write(f"\n## Metadata\n\n")
            f.write(f"- Word count: {analysis['word_count']}\n")
            f.write(f"- Processed: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
        
        print(f"   ✅ Summary: {summary_file.name}")
        
        # Save JSON with full data
        json_file = self.output_dir / f"{base_filename}_data.json"
        full_data = {
            'metadata': metadata,
            'transcript': transcript,
            'analysis': analysis,
            'processed_at': datetime.now().isoformat()
        }
        
        with open(json_file, 'w', encoding='utf-8') as f:
            json.dump(full_data, f, indent=2, ensure_ascii=False)
        
        print(f"   ✅ JSON data: {json_file.name}")
        
        return {
            'transcript_file': str(transcript_file),
            'summary_file': str(summary_file),
            'json_file': str(json_file)
        }
    
    def process_local_audio(self, audio_path: str, title: str = None):
        """Process a local audio file (for testing or direct audio processing)"""
        print(f"\n{'='*60}")
        print(f"🎬 YouTube Intelligence Pipeline (Local Audio Mode)")
        print(f"{'='*60}\n")
        print(f"Audio file: {audio_path}\n")
        
        if not os.path.exists(audio_path):
            print(f"❌ Error: Audio file not found: {audio_path}")
            sys.exit(1)
        
        # Create mock metadata
        video_id = hashlib.md5(audio_path.encode()).hexdigest()[:11]
        metadata = {
            'id': video_id,
            'title': title or os.path.basename(audio_path),
            'channel': 'Local File',
            'duration': 0,  # Unknown
            'upload_date': datetime.now().strftime('%Y%m%d'),
            'description': '',
            'view_count': 0,
            'like_count': 0
        }
        
        print(f"Title: {metadata['title']}\n")
        
        # Transcribe
        transcript = self.transcribe_audio(audio_path)
        
        # Analyze and summarize
        analysis = self.summarize_transcript(transcript, metadata)
        
        # Save everything
        output_files = self.save_output(video_id, metadata, transcript, analysis)
        
        print(f"\n{'='*60}")
        print(f"✅ Pipeline complete!")
        print(f"{'='*60}\n")
        print(f"Output directory: {self.output_dir}")
        print(f"\nFiles created:")
        for file_type, file_path in output_files.items():
            print(f"  - {file_path}")
        print()
    
    def process_video(self, url: str):
        """Main pipeline: download, transcribe, summarize, save"""
        print(f"\n{'='*60}")
        print(f"🎬 YouTube Intelligence Pipeline")
        print(f"{'='*60}\n")
        print(f"URL: {url}\n")
        
        # Get video ID and metadata
        video_id = self.get_video_id(url)
        metadata = self.get_video_metadata(url)
        
        print(f"Title: {metadata['title']}")
        print(f"Channel: {metadata['channel']}")
        print(f"Duration: {metadata['duration']}s ({metadata['duration'] // 60}m {metadata['duration'] % 60}s)\n")
        
        # Download audio
        audio_path = self.download_audio(url, video_id)
        
        try:
            # Transcribe
            transcript = self.transcribe_audio(audio_path)
            
            # Analyze and summarize
            analysis = self.summarize_transcript(transcript, metadata)
            
            # Save everything
            output_files = self.save_output(video_id, metadata, transcript, analysis)
            
            print(f"\n{'='*60}")
            print(f"✅ Pipeline complete!")
            print(f"{'='*60}\n")
            print(f"Output directory: {self.output_dir}")
            print(f"\nFiles created:")
            for file_type, file_path in output_files.items():
                print(f"  - {file_path}")
            print()
            
        finally:
            # Cleanup temp audio file
            if os.path.exists(audio_path):
                os.remove(audio_path)
                temp_dir = os.path.dirname(audio_path)
                if os.path.exists(temp_dir):
                    os.rmdir(temp_dir)

def main():
    """Main entry point"""
    if len(sys.argv) < 2:
        print("Usage: youtube-pipeline.py <youtube_url_or_audio_file> [options]")
        print("\nExamples:")
        print("  youtube-pipeline.py https://www.youtube.com/watch?v=dQw4w9WgXcQ")
        print("  youtube-pipeline.py /path/to/audio.mp3 --title 'My Audio'")
        print("  youtube-pipeline.py https://youtu.be/... --cookies cookies.txt")
        print("\nOptions:")
        print("  --title TITLE       Set title for local audio files")
        print("  --cookies FILE      Path to cookies.txt file for YouTube downloads")
        sys.exit(1)
    
    url_or_path = sys.argv[1]
    
    # Parse optional arguments
    title = None
    cookies_file = None
    
    i = 2
    while i < len(sys.argv):
        if sys.argv[i] == '--title' and i + 1 < len(sys.argv):
            title = sys.argv[i + 1]
            i += 2
        elif sys.argv[i] == '--cookies' and i + 1 < len(sys.argv):
            cookies_file = sys.argv[i + 1]
            i += 2
        else:
            i += 1
    
    # Get API key from environment or config
    api_key = os.environ.get('OPENAI_API_KEY')
    
    if not api_key:
        # Try to read from Clawdbot config
        config_path = os.path.expanduser("~/.clawdbot/config.json")
        if os.path.exists(config_path):
            try:
                with open(config_path, 'r') as f:
                    config = json.load(f)
                    api_key = config.get('skills', {}).get('entries', {}).get('openai-whisper-api', {}).get('apiKey')
            except:
                pass
    
    if not api_key:
        # Use the hardcoded key from the task
        api_key = "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"
    
    if not api_key:
        print("❌ Error: OpenAI API key not found")
        print("Set OPENAI_API_KEY environment variable or configure in Clawdbot")
        sys.exit(1)
    
    # Create pipeline
    pipeline = YouTubePipeline(api_key, cookies_file)
    
    # Determine if input is a local file or URL
    if os.path.exists(url_or_path):
        # Local audio file
        pipeline.process_local_audio(url_or_path, title)
    else:
        # YouTube URL
        pipeline.process_video(url_or_path)

if __name__ == "__main__":
    main()