import os import time import datetime import pickle import threading import queue from concurrent.futures import ThreadPoolExecutor from googleapiclient.discovery import build import subprocess from google.auth.transport.requests import Request import sys from auth import get_token, get_token_non_interactive, get_service_account_token # Constants OUTPUT_DIR = "/media/media/YouTube" # Change this to your Emby folder DAYS_BACK = 5 # Videos published in the last 5 days DAYS_TO_KEEP = 10 # Delete videos older than 10 days TOKEN_FILE = "youtube_token.pickle" # Token file from the OAuth script EXIT_TIME = datetime.time(6, 0) # Exit the script at 06:00 POPULAR_VIDEOS_MAX_RESULTS = 10 # Number of popular videos to fetch (you can change this) REGION_CODES = ["US", "GB", "LU", "LK"] # Regions to fetch popular videos from # Conversion Queue Settings MAX_CONCURRENT_CONVERSIONS = 2 # Maximum number of simultaneous video conversions CONVERSION_QUEUE_SIZE = 50 # Maximum items in conversion queue # Global conversion queue and executor conversion_queue = queue.Queue(maxsize=CONVERSION_QUEUE_SIZE) conversion_executor = ThreadPoolExecutor(max_workers=MAX_CONCURRENT_CONVERSIONS, thread_name_prefix="VideoConverter") def update_yt_dlp(): """Update yt-dlp to the latest version.""" print("Updating yt-dlp to the latest version...") subprocess.run(["pip", "install", "--upgrade", "yt-dlp"], check=True) print("yt-dlp updated successfully!") def authenticate_youtube_api(): """Authenticate using our enhanced auth system (tries non-interactive first).""" print("Attempting non-interactive authentication...") # Try non-interactive authentication first creds = get_token_non_interactive() if not creds: print("Non-interactive authentication failed.") print("You need to run: python3 auth.py --interactive") print("Or set up service account authentication.") raise Exception("Authentication failed. No valid credentials available.") print("Authentication successful!") # Build the YouTube API client youtube = build("youtube", "v3", credentials=creds) return youtube def authenticate_with_token(): """Legacy function - kept for compatibility but redirects to new auth system.""" return authenticate_youtube_api() def get_subscribed_channels(youtube): """Fetch the list of subscribed channels.""" channels = [] request = youtube.subscriptions().list( part="snippet", mine=True, maxResults=50 ) while request: response = request.execute() for item in response["items"]: channels.append(item["snippet"]["resourceId"]["channelId"]) request = youtube.subscriptions().list_next(request, response) return channels def get_recent_videos(youtube, channel_id): """Fetch videos from a channel published in the last `DAYS_BACK` days.""" recent_videos = [] published_after = (datetime.datetime.now() - datetime.timedelta(days=DAYS_BACK)).isoformat("T") + "Z" request = youtube.search().list( part="snippet", channelId=channel_id, publishedAfter=published_after, maxResults=50, type="video", order="date" ) response = request.execute() for item in response["items"]: recent_videos.append({ "video_id": item["id"]["videoId"], "title": item["snippet"]["title"], "description": item["snippet"]["description"], "published_at": item["snippet"]["publishedAt"], "channel_title": item["snippet"]["channelTitle"] }) return recent_videos def get_popular_videos(youtube, region_code): """Fetch the most popular videos for a given region.""" popular_videos = [] request = youtube.videos().list( part="snippet,statistics", chart="mostPopular", regionCode=region_code, # Region code passed here maxResults=POPULAR_VIDEOS_MAX_RESULTS ) response = request.execute() for item in response["items"]: popular_videos.append({ "video_id": item["id"], "title": item["snippet"]["title"], "description": item["snippet"]["description"], "published_at": item["snippet"]["publishedAt"], "channel_title": item["snippet"]["channelTitle"] }) return popular_videos def download_video(video_id, output_path): """Download a YouTube video with quality priority - downloads 4K VP9/AV1 if needed for conversion.""" try: # Quality-priority format selection: # 1. First try: Apple TV compatible codecs at any resolution # 2. Fallback: Download highest quality available (including 4K VP9/AV1 for conversion) # 3. Last resort: Best available format command = [ "yt-dlp", "-f", "(bv[vcodec~='^(avc|h264|hevc|h265)'] / bv[height>=2160] / bv[height>=1080] / best) + (ba[acodec~='^(aac|mp4a)'] / ba)", "--merge-output-format", "mp4", "--no-mtime", f"https://www.youtube.com/watch?v={video_id}", "-o", os.path.join(output_path, "%(title)s.%(ext)s") ] subprocess.run(command, check=True) print(f"Successfully downloaded video {video_id} (Quality priority - will convert VP9/AV1 if needed)") except subprocess.CalledProcessError as e: print(f"Error downloading video {video_id}: {e}") return False return True def add_to_conversion_queue(input_file, priority=1): """Add a video file to the conversion queue with priority.""" try: # Higher number = higher priority (4K gets priority 2, others get 1) conversion_queue.put((priority, input_file), timeout=1) print(f"Added to conversion queue: {os.path.basename(input_file)} (priority: {priority})") return True except queue.Full: print(f"āš ļø Conversion queue is full! Skipping: {os.path.basename(input_file)}") return False def get_queue_status(): """Get current conversion queue status.""" queue_size = conversion_queue.qsize() active_conversions = conversion_executor._threads # Active thread count return { 'queued': queue_size, 'active': len(active_conversions) if active_conversions else 0, 'max_concurrent': MAX_CONCURRENT_CONVERSIONS } def print_queue_status(): """Print current queue status.""" status = get_queue_status() print(f"šŸŽ¬ Conversion Status: {status['active']}/{status['max_concurrent']} active, {status['queued']} queued") def process_conversion_queue(): """Background worker to process the conversion queue.""" while True: try: # Get item from queue (blocks until available) priority, input_file = conversion_queue.get(timeout=30) # Check if file still exists if not os.path.exists(input_file): print(f"āš ļø File no longer exists, skipping conversion: {input_file}") conversion_queue.task_done() continue print(f"šŸ”„ Starting conversion (priority {priority}): {os.path.basename(input_file)}") # Perform the conversion success = convert_to_apple_tv_compatible(input_file) if success: print(f"āœ… Conversion completed: {os.path.basename(input_file)}") else: print(f"āŒ Conversion failed: {os.path.basename(input_file)}") # Mark task as done conversion_queue.task_done() except queue.Empty: # No items in queue for 30 seconds, continue waiting continue except Exception as e: print(f"āŒ Conversion queue error: {e}") conversion_queue.task_done() def convert_to_apple_tv_compatible(input_file): """Convert a video to Apple TV 4K compatible format, optimized by resolution.""" base_name = os.path.splitext(input_file)[0] mp4_file = base_name + ".mp4" print(f"Converting {input_file} to Apple TV compatible format...") # Detect video properties to optimize conversion try: import json probe_cmd = [ "ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", input_file ] probe_result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True) video_info = json.loads(probe_result.stdout) # Find video stream video_stream = next((s for s in video_info['streams'] if s['codec_type'] == 'video'), None) is_4k = video_stream and int(video_stream.get('height', 0)) >= 2160 is_high_res = video_stream and int(video_stream.get('height', 0)) >= 1080 print(f"Video resolution: {video_stream.get('height', 'unknown')}p") except (subprocess.CalledProcessError, json.JSONDecodeError, Exception) as e: print(f"Could not probe video properties: {e}, using default settings") is_4k = False is_high_res = True # Apple TV 4K optimized encoding based on source quality: if is_4k: # 4K content: Use H.265 for better compression at high resolution codec_args = [ "-c:v", "libx265", # H.265 for 4K (better compression) "-preset", "medium", # Better compression for 4K "-crf", "20", # Higher quality for 4K content "-level", "5.1", # Level 5.1 supports 4K "-pix_fmt", "yuv420p", # Ensure compatibility ] print("Using H.265 encoding for 4K content") elif is_high_res: # 1080p content: Use H.264 for maximum compatibility codec_args = [ "-c:v", "libx264", # H.264 for 1080p (max compatibility) "-profile:v", "high", # High profile for better quality "-preset", "fast", # Faster encoding for 1080p "-crf", "21", # Good quality for 1080p "-level", "4.1", # Level 4.1 supports 1080p at high bitrates ] print("Using H.264 encoding for 1080p content") else: # Lower resolution: Use H.264 with faster settings codec_args = [ "-c:v", "libx264", # H.264 for lower res "-profile:v", "main", # Main profile for lower res "-preset", "fast", # Fast encoding "-crf", "23", # Standard quality "-level", "3.1", # Level 3.1 for lower res ] print("Using H.264 encoding for lower resolution content") command = [ "ffmpeg", "-i", input_file, *codec_args, "-c:a", "aac", # AAC audio codec "-b:a", "192k", # Higher audio bitrate for better quality "-movflags", "+faststart", # Optimize for streaming/quick playback "-y", # Overwrite output file if exists mp4_file ] try: print("Starting conversion... This may take a while for 4K content.") subprocess.run(command, check=True) print(f"Conversion completed: {mp4_file}") if os.path.exists(mp4_file): os.remove(input_file) print(f"Original file deleted: {input_file}") return True except subprocess.CalledProcessError as e: print(f"Conversion failed for {input_file}: {e}") return False # Temporarily commented out video conversion functionality # def convert_to_mp4(input_file): # """Convert a video to mp4 and delete the original file.""" # mp4_file = os.path.splitext(input_file)[0] + ".mp4" # print(f"Converting {input_file} to {mp4_file}...") # command = ["ffmpeg", "-i", input_file, "-c:v", "libx264", "-preset", "fast", "-c:a", "aac", mp4_file] # subprocess.run(command, check=True) # print(f"Conversion completed: {mp4_file}") # if os.path.exists(mp4_file): # os.remove(input_file) # print(f"Original file deleted: {input_file}") def create_nfo_file(video_data, file_path): """Create an Emby-compatible NFO file.""" nfo_content = f""" {video_data['title']} {video_data['description']} {video_data['published_at'][:4]} {video_data['published_at']} {video_data['channel_title']} """ with open(file_path, "w", encoding="utf-8") as nfo_file: nfo_file.write(nfo_content) def delete_old_videos(folder, days=5): """Delete video files and their folders older than `days`.""" now = time.time() cutoff = now - (days * 86400) # Convert days to seconds for root, dirs, files in os.walk(folder, topdown=False): # Delete files for file in files: file_path = os.path.join(root, file) if os.path.isfile(file_path): if os.path.getmtime(file_path) < cutoff: print(f"Deleting file: {file_path}") os.remove(file_path) # Remove empty directories for dir in dirs: dir_path = os.path.join(root, dir) if not os.listdir(dir_path): # Check if the directory is empty print(f"Deleting folder: {dir_path}") os.rmdir(dir_path) def video_exists(output_path, video_id): """Check if a video already exists in the directory.""" return os.path.exists(os.path.join(output_path, video_id)) def check_exit_time(): """Check if the current time is past the exit time.""" now = datetime.datetime.now().time() if now >= EXIT_TIME: print("Current time exceeds exit time. Terminating script.") sys.exit(0) def main(): # Start the conversion queue worker print("šŸŽ¬ Starting video conversion queue system...") print(f"šŸ“Š Max concurrent conversions: {MAX_CONCURRENT_CONVERSIONS}") # Start background conversion worker conversion_worker = threading.Thread(target=process_conversion_queue, daemon=True, name="ConversionQueueWorker") conversion_worker.start() # Update yt-dlp update_yt_dlp() # Authenticate YouTube API print("Authenticating with YouTube API...") youtube = authenticate_youtube_api() if not os.path.exists(OUTPUT_DIR): os.makedirs(OUTPUT_DIR) # Clean up old videos print(f"Deleting videos older than {DAYS_TO_KEEP} days in {OUTPUT_DIR}...") delete_old_videos(OUTPUT_DIR, days=DAYS_TO_KEEP) # Fetch subscribed channels print("Fetching subscribed channels...") channels = get_subscribed_channels(youtube) # Fetch and process recent videos from subscribed channels for channel_id in channels: print(f"Fetching recent videos for channel: {channel_id}") videos = get_recent_videos(youtube, channel_id) for video in videos: # Check the exit time before processing each video check_exit_time() print(f"Processing video: {video['title']}") video_dir = os.path.join(OUTPUT_DIR, video["video_id"]) # Check if the video already exists if os.path.exists(video_dir): print(f"Skipping download, already exists: {video['title']} ({video['video_id']})") continue os.makedirs(video_dir, exist_ok=True) # Download the video if not download_video(video["video_id"], video_dir): print(f"Skipping video {video['video_id']} due to download error.") continue # Skip this video if download failed # Check if conversion is needed (VP9/AV1 codecs or non-MP4 files) needs_conversion = False conversion_file = None for file in os.listdir(video_dir): if file.endswith(('.mp4', '.mkv', '.webm', '.m4v')): file_path = os.path.join(video_dir, file) try: # Quick check for problematic codecs probe_cmd = ["ffprobe", "-v", "quiet", "-select_streams", "v:0", "-show_entries", "stream=codec_name", "-of", "csv=p=0", file_path] result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True) codec = result.stdout.strip().lower() if codec in ['vp9', 'av01', 'av1'] or not file.endswith('.mp4'): needs_conversion = True conversion_file = file_path print(f"Found {codec} codec - will convert for Apple TV compatibility") break except subprocess.CalledProcessError: # If we can't probe, assume it might need conversion if not MP4 if not file.endswith('.mp4'): needs_conversion = True conversion_file = file_path break if needs_conversion and conversion_file: # Determine priority based on video resolution try: probe_cmd = ["ffprobe", "-v", "quiet", "-select_streams", "v:0", "-show_entries", "stream=height", "-of", "csv=p=0", conversion_file] result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True) height = int(result.stdout.strip()) if result.stdout.strip().isdigit() else 1080 priority = 2 if height >= 2160 else 1 # 4K gets higher priority except: priority = 1 # Default priority add_to_conversion_queue(conversion_file, priority) print_queue_status() # Create the NFO file nfo_file_path = os.path.join(video_dir, "movie.nfo") create_nfo_file(video, nfo_file_path) # Fetch and process most popular videos worldwide (from multiple regions) for region_code in REGION_CODES: print(f"Fetching the most popular videos from {region_code}...") popular_videos = get_popular_videos(youtube, region_code) for video in popular_videos: # Check the exit time before processing each video check_exit_time() print(f"Processing popular video: {video['title']} ({region_code})") video_dir = os.path.join(OUTPUT_DIR, video["video_id"]) # Check if the video already exists if os.path.exists(video_dir): print(f"Skipping download, already exists: {video['title']} ({video['video_id']})") continue os.makedirs(video_dir, exist_ok=True) # Download the popular video if not download_video(video["video_id"], video_dir): print(f"Skipping popular video {video['video_id']} due to download error.") continue # Skip this video if download failed # Check if conversion is needed (VP9/AV1 codecs or non-MP4 files) needs_conversion = False conversion_file = None for file in os.listdir(video_dir): if file.endswith(('.mp4', '.mkv', '.webm', '.m4v')): file_path = os.path.join(video_dir, file) try: # Quick check for problematic codecs probe_cmd = ["ffprobe", "-v", "quiet", "-select_streams", "v:0", "-show_entries", "stream=codec_name", "-of", "csv=p=0", file_path] result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True) codec = result.stdout.strip().lower() if codec in ['vp9', 'av01', 'av1'] or not file.endswith('.mp4'): needs_conversion = True conversion_file = file_path print(f"Found {codec} codec - will convert for Apple TV compatibility") break except subprocess.CalledProcessError: # If we can't probe, assume it might need conversion if not MP4 if not file.endswith('.mp4'): needs_conversion = True conversion_file = file_path break if needs_conversion and conversion_file: # Determine priority based on video resolution try: probe_cmd = ["ffprobe", "-v", "quiet", "-select_streams", "v:0", "-show_entries", "stream=height", "-of", "csv=p=0", conversion_file] result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True) height = int(result.stdout.strip()) if result.stdout.strip().isdigit() else 1080 priority = 2 if height >= 2160 else 1 # 4K gets higher priority except: priority = 1 # Default priority add_to_conversion_queue(conversion_file, priority) print_queue_status() # Create the NFO file nfo_file_path = os.path.join(video_dir, "movie.nfo") create_nfo_file(video, nfo_file_path) print("All videos processed successfully!") # Final conversion queue status print("\nšŸŽ¬ Final Conversion Queue Status:") print_queue_status() queued_items = conversion_queue.qsize() if queued_items > 0: print(f"ā³ {queued_items} videos still in conversion queue") print("šŸ’” Conversions will continue in the background") print("šŸ’” You can monitor progress by checking the output folder") else: print("āœ… All conversions completed!") if __name__ == "__main__": main()