#!/usr/bin/env python3
"""
YouTube Clawdbot Intelligence Pipeline
Searches for and extracts tips/tricks/guides about Clawdbot from YouTube videos
"""

import os
import re
import json
import subprocess
import tempfile
from datetime import datetime
from typing import List, Dict, Optional
import requests
from urllib.parse import quote_plus

class YouTubeClawdbotIntel:
    def __init__(self, output_file: str = "/home/ccuser/rateright-growth/rivet/memory/plans/clawdbot-youtube-intel.md"):
        self.output_file = output_file
        self.yt_dlp_path = "/usr/local/bin/yt-dlp"
        self.search_queries = [
            "clawdbot tutorial",
            "clawdbot setup",
            "clawdbot tips",
            "clawdbot autonomous",
            "clawdbot agents",
            "Alex Finn Official clawdbot"
        ]
        self.processed_videos = set()
        self.intel_data = []
        
    def search_youtube_videos(self, query: str, max_results: int = 10) -> List[Dict]:
        """Search YouTube for videos related to the query"""
        print(f"Searching for: {query}")
        
        # Use yt-dlp to search
        search_cmd = [
            self.yt_dlp_path,
            "--default-search", "ytsearch",
            "--flat-playlist",
            "--print", "%(id)s\t%(title)s\t%(channel)s\t%(duration_string)s\t%(view_count)s\t%(upload_date)s",
            f"ytsearch{max_results}:{query}"
        ]
        
        try:
            result = subprocess.run(search_cmd, capture_output=True, text=True)
            if result.returncode != 0:
                print(f"Error searching YouTube: {result.stderr}")
                return []
                
            videos = []
            for line in result.stdout.strip().split('\n'):
                if line and '\t' in line:
                    parts = line.split('\t')
                    if len(parts) >= 6:
                        video = {
                            'id': parts[0],
                            'title': parts[1],
                            'channel': parts[2],
                            'duration': parts[3],
                            'views': parts[4],
                            'upload_date': parts[5],
                            'url': f"https://www.youtube.com/watch?v={parts[0]}"
                        }
                        videos.append(video)
            
            return videos
            
        except Exception as e:
            print(f"Exception during search: {e}")
            return []
    
    def download_transcript(self, video_id: str) -> Optional[str]:
        """Download transcript/subtitles for a video"""
        print(f"Downloading transcript for video: {video_id}")
        
        with tempfile.TemporaryDirectory() as temp_dir:
            cmd = [
                self.yt_dlp_path,
                "--write-sub",
                "--write-auto-sub",
                "--sub-lang", "en",
                "--skip-download",
                "--output", f"{temp_dir}/%(id)s.%(ext)s",
                f"https://www.youtube.com/watch?v={video_id}"
            ]
            
            try:
                subprocess.run(cmd, capture_output=True, text=True)
                
                # Look for subtitle files
                for file in os.listdir(temp_dir):
                    if file.endswith(('.vtt', '.srt', '.ass')):
                        with open(os.path.join(temp_dir, file), 'r', encoding='utf-8') as f:
                            return f.read()
                            
                print(f"No subtitles found for {video_id}")
                return None
                
            except Exception as e:
                print(f"Error downloading transcript: {e}")
                return None
    
    def extract_tips_from_transcript(self, transcript: str, video_title: str) -> Dict:
        """Extract tips, tricks, and configuration advice from transcript"""
        # Clean transcript by removing timestamps and formatting
        lines = transcript.split('\n')
        text_content = []
        
        for line in lines:
            # Skip timestamp lines (various formats)
            if re.match(r'^\d{2}:\d{2}:\d{2}', line) or re.match(r'^\d{2}:\d{2}', line):
                continue
            # Skip empty lines
            if not line.strip():
                continue
            # Skip WEBVTT headers
            if 'WEBVTT' in line or 'Kind:' in line or 'Language:' in line:
                continue
                
            text_content.append(line.strip())
        
        full_text = ' '.join(text_content)
        
        # Extract key information
        tips = []
        
        # Look for tip patterns
        tip_patterns = [
            r"(?:tip|trick|hack|pro tip|protip|advice|recommendation|best practice)\s*:?\s*([^.!?]+[.!?])",
            r"you should\s+([^.!?]+[.!?])",
            r"make sure to\s+([^.!?]+[.!?])",
            r"don't forget to\s+([^.!?]+[.!?])",
            r"important to\s+([^.!?]+[.!?])",
            r"key is to\s+([^.!?]+[.!?])",
            r"configure\s+([^.!?]+[.!?])",
            r"setup\s+([^.!?]+[.!?])",
            r"setting\s+([^.!?]+[.!?])",
            r"parameter\s+([^.!?]+[.!?])",
            r"option\s+([^.!?]+[.!?])",
            r"feature\s+([^.!?]+[.!?])",
            r"plugin\s+([^.!?]+[.!?])",
            r"skill\s+([^.!?]+[.!?])",
            r"mistake\s+([^.!?]+[.!?])",
            r"error\s+([^.!?]+[.!?])",
            r"issue\s+([^.!?]+[.!?])",
            r"problem\s+([^.!?]+[.!?])",
            r"gotcha\s+([^.!?]+[.!?])"
        ]
        
        for pattern in tip_patterns:
            matches = re.findall(pattern, full_text, re.IGNORECASE)
            for match in matches:
                tip = match.strip()
                if len(tip) > 10 and len(tip) < 200:  # Filter reasonable length tips
                    tips.append(tip)
        
        # Look for configuration mentions
        config_patterns = [
            r"(?:config|configuration|setting|parameter)\s+([^\s]+)\s*=\s*([^\s]+)",
            r"(?:yaml|yml|json|toml)\s+file\s+([^.!?]+[.!?])",
            r"\.clawdbot[^\s]*\s+([^.!?]+[.!?])",
            r"environment variable\s+([^.!?]+[.!?])",
            r"export\s+([^\s]+)\s*="
        ]
        
        configs = []
        for pattern in config_patterns:
            matches = re.findall(pattern, full_text, re.IGNORECASE)
            configs.extend(matches)
        
        # Generate summary
        sentences = full_text.split('. ')
        summary = '. '.join(sentences[:3]) + '.' if len(sentences) > 3 else full_text[:300] + '...'
        
        return {
            'summary': summary,
            'tips': list(set(tips))[:10],  # Top 10 unique tips
            'configs': configs,
            'features_mentioned': self.extract_features(full_text),
            'mistakes_mentioned': self.extract_mistakes(full_text)
        }
    
    def extract_features(self, text: str) -> List[str]:
        """Extract mentioned features/skills/plugins"""
        features = []
        
        # Common Clawdbot features/skills
        clawdbot_features = [
            'heartbeat', 'cron', 'memory', 'agents', 'skills', 'plugins',
            'telegram', 'discord', 'voice', 'tts', 'browser', 'web search',
            'canvas', 'nodes', 'message', 'email', 'calendar', 'github',
            'file operations', 'image analysis', 'code execution',
            'autonomous mode', 'thinking', 'reasoning', 'tools'
        ]
        
        for feature in clawdbot_features:
            if feature.lower() in text.lower():
                features.append(feature)
        
        return features
    
    def extract_mistakes(self, text: str) -> List[str]:
        """Extract common mistakes/gotchas mentioned"""
        mistakes = []
        
        # Look for mistake patterns
        mistake_patterns = [
            r"(?:common mistake|gotcha|pitfall|don't|never|avoid|wrong)\s+([^.,!?]{10,})",
            r"(?:issue|problem|trouble|difficulty)\s+([^.,!?]{10,})",
            r"(?:fails?|broken|doesn't work|error)\s+([^.,!?]{10,})"
        ]
        
        for pattern in mistake_patterns:
            matches = re.findall(pattern, text, re.IGNORECASE)
            mistakes.extend([m.strip() for m in matches if len(m.strip()) > 10])
        
        return mistakes[:5]  # Top 5 mistakes
    
    def analyze_video(self, video: Dict) -> Optional[Dict]:
        """Analyze a single video for Clawdbot intelligence"""
        video_id = video['id']
        
        if video_id in self.processed_videos:
            return None
            
        print(f"\nAnalyzing video: {video['title']}")
        
        # Download transcript
        transcript = self.download_transcript(video_id)
        if not transcript:
            print(f"No transcript available for: {video['title']}")
            return None
        
        # Extract tips and information
        analysis = self.extract_tips_from_transcript(transcript, video['title'])
        
        # Check if applicable to us
        applicable = self.check_applicability(analysis)
        
        intel = {
            'title': video['title'],
            'channel': video['channel'],
            'url': video['url'],
            'date': video['upload_date'],
            'summary': analysis['summary'],
            'tips': analysis['tips'],
            'features_mentioned': analysis['features_mentioned'],
            'mistakes_mentioned': analysis['mistakes_mentioned'],
            'applicable': applicable
        }
        
        self.processed_videos.add(video_id)
        return intel
    
    def check_applicability(self, analysis: Dict) -> str:
        """Check if the tips/features are applicable to our setup"""
        if not analysis['tips'] and not analysis['features_mentioned']:
            return "No - No relevant tips or features found"
        
        # Check if we already use mentioned features
        our_features = ['telegram', 'voice', 'tts', 'browser', 'web search', 'canvas', 'nodes', 'message', 'file operations', 'image analysis', 'code execution']
        
        new_features = [f for f in analysis['features_mentioned'] if f.lower() not in our_features]
        
        if new_features:
            return f"Yes - New features mentioned: {', '.join(new_features)}"
        elif analysis['tips']:
            return "Yes - Contains actionable tips"
        else:
            return "Maybe - Review for potential improvements"
    
    def run_intelligence_pipeline(self):
        """Run the complete intelligence pipeline"""
        print("Starting YouTube Clawdbot Intelligence Pipeline...")
        
        # Search for videos
        all_videos = []
        for query in self.search_queries:
            videos = self.search_youtube_videos(query, max_results=10)
            all_videos.extend(videos)
            print(f"Found {len(videos)} videos for query: {query}")
        
        # Remove duplicates
        unique_videos = {v['id']: v for v in all_videos}.values()
        print(f"Total unique videos found: {len(unique_videos)}")
        
        # Analyze each video
        for video in unique_videos:
            intel = self.analyze_video(video)
            if intel:
                self.intel_data.append(intel)
        
        # Save results
        self.save_results()
        
        print(f"\nIntelligence pipeline complete!")
        print(f"Analyzed {len(self.intel_data)} videos")
        print(f"Results saved to: {self.output_file}")
    
    def save_results(self):
        """Save intelligence results to markdown file"""
        os.makedirs(os.path.dirname(self.output_file), exist_ok=True)
        
        with open(self.output_file, 'w', encoding='utf-8') as f:
            f.write("# Clawdbot YouTube Intelligence Report\n\n")
            f.write(f"Generated on: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
            f.write(f"Total videos analyzed: {len(self.intel_data)}\n\n")
            
            # Summary section
            f.write("## Summary\n\n")
            applicable_count = sum(1 for intel in self.intel_data if intel['applicable'].startswith('Yes'))
            f.write(f"- Videos with applicable tips: {applicable_count}\n")
            f.write(f"- Total tips extracted: {sum(len(intel['tips']) for intel in self.intel_data)}\n")
            f.write(f"- Unique features mentioned: {len(set(f for intel in self.intel_data for f in intel['features_mentioned']))}\n\n")
            
            # Top tips section
            f.write("## Top 5 Most Useful Tips\n\n")
            all_tips = []
            for intel in self.intel_data:
                for tip in intel['tips']:
                    all_tips.append({
                        'tip': tip,
                        'video': intel['title'],
                        'channel': intel['channel']
                    })
            
            # Display first 5 tips (in a real implementation, you might rank them)
            for i, tip_data in enumerate(all_tips[:5], 1):
                f.write(f"{i}. **{tip_data['tip']}**\n")
                f.write(f"   - From: {tip_data['video']} ({tip_data['channel']})\n\n")
            
            # Detailed video analysis
            f.write("## Detailed Video Analysis\n\n")
            f.write("---\n\n")
            
            for intel in self.intel_data:
                f.write(f"## {intel['title']}\n")
                f.write(f"- Channel: {intel['channel']}\n")
                f.write(f"- URL: {intel['url']}\n")
                f.write(f"- Date: {intel['date']}\n")
                f.write(f"- Summary: {intel['summary']}\n")
                
                if intel['tips']:
                    f.write("- Key Tips:\n")
                    for tip in intel['tips']:
                        f.write(f"  1. {tip}\n")
                
                f.write(f"- Applicable to us: {intel['applicable']}\n")
                
                if intel['mistakes_mentioned']:
                    f.write("- Common Mistakes/Gotchas:\n")
                    for mistake in intel['mistakes_mentioned']:
                        f.write(f"  - {mistake}\n")
                
                f.write("\n---\n\n")
            
            # Features not used section
            f.write("## Features/Skills Mentioned That We're Not Using\n\n")
            our_features = {'telegram', 'voice', 'tts', 'browser', 'web search', 'canvas', 'nodes', 'message', 'file operations', 'image analysis', 'code execution'}
            
            unused_features = set()
            for intel in self.intel_data:
                for feature in intel['features_mentioned']:
                    if feature.lower() not in our_features:
                        unused_features.add(feature)
            
            if unused_features:
                for feature in sorted(unused_features):
                    f.write(f"- {feature}\n")
            else:
                f.write("- No new features mentioned\n")
            
            f.write("\n---\n\n")
            f.write("## Configuration Tips That Could Improve Our Setup\n\n")
            
            # Extract configuration tips
            config_tips = []
            for intel in self.intel_data:
                for tip in intel['tips']:
                    if any(word in tip.lower() for word in ['config', 'setting', 'parameter', 'yaml', 'json', 'environment']):
                        config_tips.append(tip)
            
            if config_tips:
                for tip in config_tips:
                    f.write(f"- {tip}\n")
            else:
                f.write("- No specific configuration tips found\n")

def main():
    """Main entry point"""
    intel = YouTubeClawdbotIntel()
    intel.run_intelligence_pipeline()

if __name__ == "__main__":
    main()