
    ^,iD                         d Z ddlZddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
Z
 G d d      Zd Zedk(  r e        yy)	z_
YouTube Intelligence Pipeline for Rivet
Downloads, transcribes, and summarizes YouTube videos
    N)datetime)Pathc                       e Zd ZddedefdZdedefdZdedefdZded	edefd
ZdedefdZ	dededefdZ
dedefdZd	edededefdZddedefdZdefdZy)YouTubePipelineNapi_keyuse_cookies_filec                     || _         d| _        t        d      | _        | j                  j	                  dd       || _        y )Nz/usr/local/bin/yt-dlpz2/home/ccuser/rateright-growth/rivet/memory/youtubeT)parentsexist_ok)r   yt_dlp_pathr   
output_dirmkdircookies_file)selfr   r   s      ?/home/ccuser/rateright-growth/rivet/scripts/youtube-pipeline.py__init__zYouTubePipeline.__init__   s=    2STdT:,    urlreturnc                    d|v r&|j                  d      d   j                  d      d   S d|v r&|j                  d      d   j                  d      d   S d|v r&|j                  d      d   j                  d      d   S |S )	z!Extract video ID from YouTube URLz	youtu.be/   ?r   zwatch?v=&zyoutube.com/embed/zembed/)split)r   r   s     r   get_video_idzYouTubePipeline.get_video_id   s     #99[)!,2237::399Z(+11#6q99!S(99X&q)//4Q77 Jr   c                    t        d       | j                  dddddg}| j                  rFt        j                  j                  | j                        r|j                  d| j                  g       |j                  |       	 t        j                  |ddd	      }t        j                  |j                        }|j                  d
      |j                  d      |j                  d      |j                  d      |j                  d      |j                  dd      |j                  dd      |j                  dd      dS # t        j                  $ r7}t        d|j                          t!        j"                  d       Y d}~yd}~ww xY w)zGet video metadata using yt-dlpu   📹 Fetching video metadata...z--dump-jsonz--no-download--extractor-args!youtube:player_client=tv_embedded--no-check-certificate	--cookiesTcapture_outputtextcheckidtitlechanneldurationupload_datedescription 
view_countr   
like_countr%   r&   r'   r(   r)   r*   r,   r-   u   ❌ Error getting metadata: r   N)printr   r   ospathexistsextendappend
subprocessrunjsonloadsstdoutgetCalledProcessErrorstderrsysexit)r   r   cmdresultmetadataes         r   get_video_metadataz"YouTubePipeline.get_video_metadata&   sD   /1  C$
 0A0A!BJJT%6%678

3	^^C4tTFzz&--0H ll4(!g.#<<	2$LL4'||M:'||M2>&ll<;&ll<;	 	 ,, 	0
;<HHQKK	s   B=E   F
-FF
video_idc                    t        d       t        j                         }t        j                  j                  || d      }| j                  ddddddd	d
d|j                  dd      g}| j                  rFt        j                  j                  | j                        r|j                  d| j                  g       |j                  |       	 t        j                  |ddd      }t        j                  j                  |      r4t        j                  j                  |      dz  }t        d|dd       |S t        d|        t        j                  |      }t        d|        |r%t        j                  j                  ||d         }	|	S t!        j"                  d       y# t        j$                  $ r7}
t        d|
j&                          t!        j"                  d       Y d}
~
yd}
~
ww xY w)z&Download audio only in smallest formatu+   🎵 Downloading audio (smallest format)...z.mp3z--extract-audioz--audio-formatmp3z--audio-quality9r   r   r   z--outputz.%(ext)sr    Tr!      u   ✅ Audio downloaded: .2f MBu+   ❌ Audio file not found at expected path: zFiles in temp dir: r   r   u   ❌ Error downloading audio: N)r/   tempfilemkdtempr0   r1   joinr   replacer   r2   r3   r4   r5   r6   getsizelistdirr=   r>   r;   r<   )r   r   rD   temp_diroutput_pathr?   r@   	file_sizefiles
audio_filerB   s              r   download_audiozYouTubePipeline.download_audioJ   s   ;=##%ggll8z->? es C$++FJ?
 0A0A!BJJT%6%678

3	^^C4tTF ww~~k*GGOOK8KH	.yoSAB""CK=QR

8,+E734!#ha!AJ%%,, 	1!((<=HHQKK	s&   A+F( :AF( F( (G2;-G--G2
audio_pathc                 ^   t        d       t        j                  j                  |      dz  }t        d|dd       |dkD  rt        d       t        d       d	}d
d| j                   i}	 t        |d      5 }dt        j                  j                  |      |dfi}dddd}t        j                  ||||      }|j                  dk(  rO|j                         }	|	j                  dd      }
|	j                  dd      }t        d|dd       |
cddd       S t        d|j                          t        d|j                          t        j                  d       	 ddd       y# 1 sw Y   yxY w# t        $ r-}t        d |        t        j                  d       Y d}~yd}~ww xY w)!z)Transcribe audio using OpenAI Whisper APIu+   🎤 Transcribing audio with Whisper API...rH   z   Audio file size: rI   rJ      u;   ⚠️  Warning: File size exceeds Whisper API limit (25MB)z%   Attempting to transcribe anyway...z.https://api.openai.com/v1/audio/transcriptionsAuthorizationzBearer rbfilez
audio/mpegz	whisper-1enverbose_json)modellanguageresponse_format)headersrT   data   r#   r+   r(   r   u   ✅ Transcription complete (z.0fzs of audio)Nu   ❌ Transcription error: z   Response: r   u$   ❌ Exception during transcription: )r/   r0   r1   rO   r   openbasenamerequestspoststatus_coder7   r:   r#   r=   r>   	Exception)r   rW   rS   r   rb   rU   rT   rc   responser@   
transcriptr(   rB   s                r   transcribe_audioz YouTubePipeline.transcribe_audiow   s   ;=GGOOJ/;?	$YsO378 r>OQ9;>wt||n5
	j$'  :RWW--j9:|T ) $'5 $==gUQUV''3.%]]_F!'FB!7J%zz*a8H8#kRS%#   & 5h6J6J5KLMM(--9:HHQK+     ,  	8<=HHQKK	sD   -E6 9BE*	E6 AE*!E6 *E3/E6 3E6 6	F,?#F''F,rl   rA   c                    t        d       t        |j                               }g d}|j                  d      }g }|D ]F  }|j                         t	        fd|D              s(|j                  |j                                H |dd |dd z   }dj                  |dd       d	z   }	| j                  |      }
||	|dd
 |
dS )z7Generate summary and extract key points from transcriptu   📝 Analyzing transcript...)	importantkeymaintiptrickhackproblemsolutionissuefixerror	configuresetupinstallfeature	recommendsuggestadvicezbest practice. c              3   &   K   | ]  }|v  
 y wN ).0keywordsentence_lowers     r   	<genexpr>z7YouTubePipeline.summarize_transcript.<locals>.<genexpr>   s     E7n,Es   N         .
   )
word_countsummary
key_pointstopics)	r/   lenr   loweranyr4   striprM   extract_topics)r   rl   rA   r   keywords	sentenceskey_sentencessentencesummary_sentencesr   r   r   s              @r   summarize_transcriptz$YouTubePipeline.summarize_transcript   s    ,. ))+,

 $$T*	! 	7H%^^-NEHEE$$X^^%56	7 &bqMM"1,==))-bq12S8 $$Z0 %',	
 	
r   r#   c                 r    g d}|j                         }g }|D ]  }||v s|j                  |        |dd S )z;Extract likely topics from text (simple keyword extraction))>apidatabaseserverclientfrontendbackendreactpython
javascript
typescriptnodedockerawscloud
deploymenttestingsecurityperformanceoptimization	debugging
monitoring	analyticsaimlzmachine learningzneural networkr_   trainingrc   
automationintegrationworkflowpipelinetool	frameworklibrarypackagemodulefunctionclassauthenticationauthorization
encryptiontokenoauthrestgraphql	websockethttphttpsssltlsgitgithubgitlabzci/cddevops
kubernetesmicroservicesarchitecturezdesign pattern	algorithmN   )r   r4   )r   r#   common_topics
text_lowerfound_topicstopics         r   r   zYouTubePipeline.extract_topics   sO    
 ZZ\
" 	+E
"##E*	+ CR  r   analysisc           
      n   t        d       t        j                         j                  d      }dj	                  d |d   D              j                         dd }| d| d| }| j                  | d	z  }t        |d
d      5 }	|	j                  d|d    d       |	j                  d|d    d       |	j                  d|d    d       |	j                  d| d       |	j                  |       ddd       t        d|j                          | j                  | dz  }
t        |
d
d      5 }	|	j                  d|d    d       |	j                  d|d    d       |	j                  d|d    d       |	j                  d|d    d|d   dz   d|d   dz   d       |	j                  d |d!   d"d       |	j                  d#| d       |	j                  d$       |	j                  |d%    d       |	j                  d&       t        |d'   d(      D ]  \  }}|	j                  | d)| d        |d*   r0|	j                  d+       |d*   D ]  }|	j                  d,| d        |	j                  d-       |	j                  d.|d/    d       |	j                  d0t        j                         j                  d1       d       ddd       t        d2|
j                          | j                  | d3z  }|||t        j                         j                         d4}t        |d
d      5 }	t        j                  ||	d5d67       ddd       t        d8|j                          t        |      t        |
      t        |      d9S # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   [xY w):zSave all output filesu   💾 Saving output files...z%Y%m%d_%H%M%Sr+   c              3   J   K   | ]  }|j                         s|d v s|  yw)) -_N)isalnum)r   cs     r   r   z.YouTubePipeline.save_output.<locals>.<genexpr>   s      a1QYY[AQ`L`Qas   ##r&   N2   r   z_transcript.txtwzutf-8)encodingz# Transcript: 
z# Channel: r'   z# Date: r)   z'# URL: https://www.youtube.com/watch?v=z

u      ✅ Transcript: z_summary.mdz# z**Channel:** z
**Date:** z**Duration:** r(   s (<   m s)
z**Views:** r,   ,z)**URL:** https://www.youtube.com/watch?v=z## Summary

r   z## Key Points

r   r   r   r   z
## Topics Covered

z- z
## Metadata

z- Word count: r   z- Processed: z%Y-%m-%d %H:%M:%Su      ✅ Summary: z
_data.json)rA   rl   r   processed_at   F)indentensure_asciiu      ✅ JSON data: )transcript_filesummary_file	json_file)r/   r   nowstrftimerM   r   r   re   writename	enumerate	isoformatr7   dumpstr)r   rD   rA   rl   r   	timestamp
safe_titlebase_filenamer   fr   ipointr   r   	full_datas                   r   save_outputzYouTubePipeline.save_output   s   +-LLN++O<	WWa(9aaggijmkmn
$+Qxj*> //}o_,MM/39 	 QGGnXg%6$7r:;GGk(9"5!6b9:GGhx67r:;GG=hZtLMGGJ	  	#O$8$8#9:; M?+)FF,g6 	V!GGb'*+401GGmHY$7#8;<GGj-!8 9<=GGnXj%9$:#hz>RVX>X=YY[\deo\psu\u[vvz{|GGk(<"8!;2>?GG?zNOGGn&GGx	*+401GG')%h|&<a@ +51#Rwb)*+ !13%h/ ,EGGbrN+, GG')GGnXl%;$<B?@GGmHLLN$;$;<O$P#QQSTU-	V0 	 !2!2 345 OOz&BB	 $ $LLN446	
	 )S73 	BqIIi15A	B 	"9>>"234  #?3-Y
 	
g	  	 	V 	VF	B 	Bs&   A/N4FN7N+NN(+N4r&   c           	      |   t        dd        t        d       t        d d       t        d| d       t        j                  j                  |      s#t        d|        t	        j
                  d       t        j                  |j                               j                         dd }||xs t        j                  j                  |      d	d
t        j                         j                  d      dd
d
d}t        d|d    d       | j                  |      }| j                  ||      }| j!                  ||||      }t        dd        t        d       t        d d       t        d| j"                          t        d       |j%                         D ]  \  }}	t        d|	         t                y)zCProcess a local audio file (for testing or direct audio processing)r   <============================================================u5   🎬 YouTube Intelligence Pipeline (Local Audio Mode)zAudio file: u!   ❌ Error: Audio file not found: r   N   z
Local Filer   z%Y%m%dr+   r.   Title: r&      ✅ Pipeline complete!Output directory: 
Files created:  - )r/   r0   r1   r2   r=   r>   hashlibmd5encode	hexdigestrf   r   r   r   rm   r   r  r   items)
r   rW   r&   rD   rA   rl   r   output_files	file_type	file_paths
             r   process_local_audioz#YouTubePipeline.process_local_audio%  s   6(mEGmZL+,ww~~j)5j\BCHHQK ;;z0023==?D:bgg..z:##<<>228<	
 	)*"-. **:6
 ,,ZB ''(JQ6(m&(m"4??"345 "$0$6$6$8 	& IyD$%	&r   c           	         t        dd        t        d       t        d d       t        d| d       | j                  |      }| j                  |      }t        d|d           t        d|d           t        d	|d
    d|d
   dz   d|d
   dz   d       | j                  ||      }	 | j	                  |      }| j                  ||      }| j                  ||||      }t        dd        t        d       t        d d       t        d| j                          t        d       |j                         D ]  \  }}	t        d|	         t                t        j                  j                  |      rjt        j                  |       t        j                  j                  |      }
t        j                  j                  |
      rt        j                  |
       yyy# t        j                  j                  |      rjt        j                  |       t        j                  j                  |      }
t        j                  j                  |
      rt        j                  |
       w w w xY w)z4Main pipeline: download, transcribe, summarize, saver   r  u"   🎬 YouTube Intelligence PipelinezURL: r  r&   z	Channel: r'   z
Duration: r(   r   r   r   r   r  r  r	  r
  N)r/   r   rC   rV   rm   r   r  r   r  r0   r1   r2   removedirnamermdir)r   r   rD   rA   rW   rl   r   r  r  r  rQ   s              r   process_videozYouTubePipeline.process_videoQ  s4   6(m24mcU"o $$S)**3/)*+,	(9-./0
8J/0HZ4HB4N3OrRZ[eRfikRkQllpqr ((h7
	'..z:J 00XFH  ++Hh
HULBvh- *,VHB- &t&789$&(4(:(:(< *$	9YK()*G ww~~j)		*%77??:677>>(+HHX& , *rww~~j)		*%77??:677>>(+HHX& , *s   2B1G- -BI8r   )__name__
__module____qualname__r   r   r   dictrC   rV   rm   r   listr   r  r  r  r   r   r   r   r      s    - -s -  "c "d "H+# + + +Z*3 *3 *X#
s #
d #
t #
J!3 !4 !4A
C A
4 A
S A
TX A
F*c *# *X+' +'r   r   c                     t        t        j                        dk  rmt        d       t        d       t        d       t        d       t        d       t        d       t        d       t        d	       t        j                  d
       t        j                  d
   } d}d}d}|t        t        j                        k  rt        j                  |   dk(  r;|d
z   t        t        j                        k  rt        j                  |d
z      }|dz  }nVt        j                  |   dk(  r;|d
z   t        t        j                        k  rt        j                  |d
z      }|dz  }n|d
z  }|t        t        j                        k  rt
        j                  j                  d      }|st
        j                  j                  d      }t
        j                  j                  |      rl	 t        |d      5 }t        j                  |      }|j                  di       j                  di       j                  di       j                  d      }ddd       |sd}|s+t        d       t        d       t        j                  d
       t        ||      }t
        j                  j                  |       r|j                  | |       y|j!                  |        y# 1 sw Y   xY w#  Y xY w)zMain entry pointr   z@Usage: youtube-pipeline.py <youtube_url_or_audio_file> [options]z

Examples:zA  youtube-pipeline.py https://www.youtube.com/watch?v=dQw4w9WgXcQz;  youtube-pipeline.py /path/to/audio.mp3 --title 'My Audio'z@  youtube-pipeline.py https://youtu.be/... --cookies cookies.txtz	
Options:z5  --title TITLE       Set title for local audio fileszD  --cookies FILE      Path to cookies.txt file for YouTube downloadsr   Nz--titler    OPENAI_API_KEYz~/.clawdbot/config.jsonrskillsentrieszopenai-whisper-apiapiKeyzsk-proj-cXZKVl2oAo3cVqn5C1tAp1LKQ3eT_mKCI0PMcSeM1ZTSWx-DMiZlTEgXd5vjlRtydTSuIqZDVZT3BlbkFJO0t-94fBChPZ4TunA_MkcPLHb5AR2CXa9QBwbOpoQEQgOVu-mIlIvmBOKw8K1jyBpGAfApH6wAu#   ❌ Error: OpenAI API key not foundz@Set OPENAI_API_KEY environment variable or configure in Clawdbot)r   r=   argvr/   r>   r0   environr:   r1   
expanduserr2   re   r7   loadr   r  r  )	url_or_pathr&   r   r   r   config_pathr   configr   s	            r   rq   rq   ~  sH   
388}qPQmQRKLPQlEFTU((1+K EL	A
c#((m
88A;)#ACHH(=HHQUOEFAXXa[K'AECM,A88AE?LFAFA c#((m
 jjnn-.Ggg(()BC77>>+&+s+ vq!YYq\F$jj26::9bIMMNbdfgkkltuGv  y34PQ w5H 
ww~~k"$$[%8 	{+1v vs%   #K /AKK KK K__main__)__doc__r0   r=   r7   r5   rK   rg   r   pathlibr   r  r   rq   r  r   r   r   <module>r.     sN   
 
 
       k' k'Z>,@ zF r   