YouTube_predownloader/main.py

import os
import time
import datetime
import pickle
import threading
from googleapiclient.discovery import build
import subprocess
from google.auth.transport.requests import Request
import sys
from auth import get_token, get_token_non_interactive, get_service_account_token

# Constants
OUTPUT_DIR = "/media/media/YouTube"  # Change this to your Emby folder
DAYS_BACK = 5  # Videos published in the last 5 days
DAYS_TO_KEEP = 10  # Delete videos older than 10 days
TOKEN_FILE = "youtube_token.pickle"  # Token file from the OAuth script
EXIT_TIME = datetime.time(6, 0)  # Exit the script at 06:00
POPULAR_VIDEOS_MAX_RESULTS = 10  # Number of popular videos to fetch (you can change this)
REGION_CODES = ["US", "GB", "LU", "LK"]  # Regions to fetch popular videos from

def update_yt_dlp():
    """Update yt-dlp to the latest version."""
    print("Updating yt-dlp to the latest version...")
    subprocess.run(["pip", "install", "--upgrade", "yt-dlp"], check=True)
    print("yt-dlp updated successfully!")

def authenticate_youtube_api():
    """Authenticate using our enhanced auth system (tries non-interactive first)."""
    print("Attempting non-interactive authentication...")
    
    # Try non-interactive authentication first
    creds = get_token_non_interactive()
    
    if not creds:
        print("Non-interactive authentication failed.")
        print("You need to run: python3 auth.py --interactive")
        print("Or set up service account authentication.")
        raise Exception("Authentication failed. No valid credentials available.")
    
    print("Authentication successful!")
    
    # Build the YouTube API client
    youtube = build("youtube", "v3", credentials=creds)
    return youtube

def authenticate_with_token():
    """Legacy function - kept for compatibility but redirects to new auth system."""
    return authenticate_youtube_api()

def get_subscribed_channels(youtube):
    """Fetch the list of subscribed channels."""
    channels = []
    request = youtube.subscriptions().list(
        part="snippet",
        mine=True,
        maxResults=50
    )
    while request:
        response = request.execute()
        for item in response["items"]:
            channels.append(item["snippet"]["resourceId"]["channelId"])
        request = youtube.subscriptions().list_next(request, response)
    return channels

def get_recent_videos(youtube, channel_id):
    """Fetch videos from a channel published in the last `DAYS_BACK` days."""
    recent_videos = []
    published_after = (datetime.datetime.now() - datetime.timedelta(days=DAYS_BACK)).isoformat("T") + "Z"
    
    request = youtube.search().list(
        part="snippet",
        channelId=channel_id,
        publishedAfter=published_after,
        maxResults=50,
        type="video",
        order="date"
    )
    response = request.execute()
    for item in response["items"]:
        recent_videos.append({
            "video_id": item["id"]["videoId"],
            "title": item["snippet"]["title"],
            "description": item["snippet"]["description"],
            "published_at": item["snippet"]["publishedAt"],
            "channel_title": item["snippet"]["channelTitle"]
        })
    return recent_videos

def get_popular_videos(youtube, region_code):
    """Fetch the most popular videos for a given region."""
    popular_videos = []
    
    request = youtube.videos().list(
        part="snippet,statistics",
        chart="mostPopular",
        regionCode=region_code,  # Region code passed here
        maxResults=POPULAR_VIDEOS_MAX_RESULTS
    )
    response = request.execute()
    for item in response["items"]:
        popular_videos.append({
            "video_id": item["id"],
            "title": item["snippet"]["title"],
            "description": item["snippet"]["description"],
            "published_at": item["snippet"]["publishedAt"],
            "channel_title": item["snippet"]["channelTitle"]
        })
    return popular_videos

def download_video(video_id, output_path):
    """Download a YouTube video with quality priority - downloads 4K VP9/AV1 if needed for conversion."""
    try:
        # Quality-priority format selection:
        # 1. First try: Apple TV compatible codecs at any resolution
        # 2. Fallback: Download highest quality available (including 4K VP9/AV1 for conversion)
        # 3. Last resort: Best available format
        command = [
            "yt-dlp",
            "-f", "(bv[vcodec~='^(avc|h264|hevc|h265)'] / bv[height>=2160] / bv[height>=1080] / best) + (ba[acodec~='^(aac|mp4a)'] / ba)",
            "--merge-output-format", "mp4",
            "--no-mtime",
            f"https://www.youtube.com/watch?v={video_id}",
            "-o", os.path.join(output_path, "%(title)s.%(ext)s")
        ]
        subprocess.run(command, check=True)
        print(f"Successfully downloaded video {video_id} (Quality priority - will convert VP9/AV1 if needed)")
    except subprocess.CalledProcessError as e:
        print(f"Error downloading video {video_id}: {e}")
        return False
    return True

def convert_to_apple_tv_compatible(input_file):
    """Convert a video to Apple TV 4K compatible format, optimized by resolution."""
    base_name = os.path.splitext(input_file)[0]
    mp4_file = base_name + ".mp4"
    
    print(f"Converting {input_file} to Apple TV compatible format...")
    
    # Detect video properties to optimize conversion
    try:
        import json
        probe_cmd = [
            "ffprobe", "-v", "quiet", "-print_format", "json", 
            "-show_format", "-show_streams", input_file
        ]
        probe_result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True)
        video_info = json.loads(probe_result.stdout)
        
        # Find video stream
        video_stream = next((s for s in video_info['streams'] if s['codec_type'] == 'video'), None)
        is_4k = video_stream and int(video_stream.get('height', 0)) >= 2160
        is_high_res = video_stream and int(video_stream.get('height', 0)) >= 1080
        
        print(f"Video resolution: {video_stream.get('height', 'unknown')}p")
        
    except (subprocess.CalledProcessError, json.JSONDecodeError, Exception) as e:
        print(f"Could not probe video properties: {e}, using default settings")
        is_4k = False
        is_high_res = True
    
    # Apple TV 4K optimized encoding based on source quality:
    if is_4k:
        # 4K content: Use H.265 for better compression at high resolution
        codec_args = [
            "-c:v", "libx265",          # H.265 for 4K (better compression)
            "-preset", "medium",        # Better compression for 4K
            "-crf", "20",               # Higher quality for 4K content
            "-level", "5.1",            # Level 5.1 supports 4K
            "-pix_fmt", "yuv420p",      # Ensure compatibility
        ]
        print("Using H.265 encoding for 4K content")
    elif is_high_res:
        # 1080p content: Use H.264 for maximum compatibility
        codec_args = [
            "-c:v", "libx264",          # H.264 for 1080p (max compatibility)
            "-profile:v", "high",       # High profile for better quality
            "-preset", "fast",          # Faster encoding for 1080p
            "-crf", "21",               # Good quality for 1080p
            "-level", "4.1",            # Level 4.1 supports 1080p at high bitrates
        ]
        print("Using H.264 encoding for 1080p content")
    else:
        # Lower resolution: Use H.264 with faster settings
        codec_args = [
            "-c:v", "libx264",          # H.264 for lower res
            "-profile:v", "main",       # Main profile for lower res
            "-preset", "fast",          # Fast encoding
            "-crf", "23",               # Standard quality
            "-level", "3.1",            # Level 3.1 for lower res
        ]
        print("Using H.264 encoding for lower resolution content")
    
    command = [
        "ffmpeg", "-i", input_file,
        *codec_args,
        "-c:a", "aac",              # AAC audio codec
        "-b:a", "192k",             # Higher audio bitrate for better quality
        "-movflags", "+faststart",  # Optimize for streaming/quick playback
        "-y",                       # Overwrite output file if exists
        mp4_file
    ]
    
    try:
        print("Starting conversion... This may take a while for 4K content.")
        subprocess.run(command, check=True)
        print(f"Conversion completed: {mp4_file}")
        if os.path.exists(mp4_file):
            os.remove(input_file)
            print(f"Original file deleted: {input_file}")
        return True
    except subprocess.CalledProcessError as e:
        print(f"Conversion failed for {input_file}: {e}")
        return False

# Temporarily commented out video conversion functionality
# def convert_to_mp4(input_file):
#     """Convert a video to mp4 and delete the original file."""
#     mp4_file = os.path.splitext(input_file)[0] + ".mp4"
#     print(f"Converting {input_file} to {mp4_file}...")
#     command = ["ffmpeg", "-i", input_file, "-c:v", "libx264", "-preset", "fast", "-c:a", "aac", mp4_file]
#     subprocess.run(command, check=True)
#     print(f"Conversion completed: {mp4_file}")
#     if os.path.exists(mp4_file):
#         os.remove(input_file)
#         print(f"Original file deleted: {input_file}")

def create_nfo_file(video_data, file_path):
    """Create an Emby-compatible NFO file."""
    nfo_content = f"""<?xml version="1.0" encoding="utf-8" standalone="yes"?>
<movie>
  <title>{video_data['title']}</title>
  <plot>{video_data['description']}</plot>
  <year>{video_data['published_at'][:4]}</year>
  <premiered>{video_data['published_at']}</premiered>
  <studio>{video_data['channel_title']}</studio>
</movie>
"""
    with open(file_path, "w", encoding="utf-8") as nfo_file:
        nfo_file.write(nfo_content)

def delete_old_videos(folder, days=5):
    """Delete video files and their folders older than `days`."""
    now = time.time()
    cutoff = now - (days * 86400)  # Convert days to seconds

    for root, dirs, files in os.walk(folder, topdown=False):
        # Delete files
        for file in files:
            file_path = os.path.join(root, file)
            if os.path.isfile(file_path):
                if os.path.getmtime(file_path) < cutoff:
                    print(f"Deleting file: {file_path}")
                    os.remove(file_path)
        
        # Remove empty directories
        for dir in dirs:
            dir_path = os.path.join(root, dir)
            if not os.listdir(dir_path):  # Check if the directory is empty
                print(f"Deleting folder: {dir_path}")
                os.rmdir(dir_path)

def video_exists(output_path, video_id):
    """Check if a video already exists in the directory."""
    return os.path.exists(os.path.join(output_path, video_id))

def check_exit_time():
    """Check if the current time is past the exit time."""
    now = datetime.datetime.now().time()
    if now >= EXIT_TIME:
        print("Current time exceeds exit time. Terminating script.")
        sys.exit(0)

def main():
    # Update yt-dlp
    update_yt_dlp()

    # Authenticate YouTube API
    print("Authenticating with YouTube API...")
    youtube = authenticate_youtube_api()

    if not os.path.exists(OUTPUT_DIR):
        os.makedirs(OUTPUT_DIR)

    # Clean up old videos
    print(f"Deleting videos older than {DAYS_TO_KEEP} days in {OUTPUT_DIR}...")
    delete_old_videos(OUTPUT_DIR, days=DAYS_TO_KEEP)
    
    # Fetch subscribed channels
    print("Fetching subscribed channels...")
    channels = get_subscribed_channels(youtube)
    
    # Fetch and process recent videos from subscribed channels
    for channel_id in channels:
        print(f"Fetching recent videos for channel: {channel_id}")
        videos = get_recent_videos(youtube, channel_id)
        
        for video in videos:
            # Check the exit time before processing each video
            check_exit_time()
            
            print(f"Processing video: {video['title']}")
            video_dir = os.path.join(OUTPUT_DIR, video["video_id"])
            
            # Check if the video already exists
            if os.path.exists(video_dir):
                print(f"Skipping download, already exists: {video['title']} ({video['video_id']})")
                continue
            
            os.makedirs(video_dir, exist_ok=True)
            
            # Download the video
            if not download_video(video["video_id"], video_dir):
                print(f"Skipping video {video['video_id']} due to download error.")
                continue  # Skip this video if download failed
            
            # Check if conversion is needed (VP9/AV1 codecs or non-MP4 files)
            needs_conversion = False
            conversion_file = None
            
            for file in os.listdir(video_dir):
                if file.endswith(('.mp4', '.mkv', '.webm', '.m4v')):
                    file_path = os.path.join(video_dir, file)
                    try:
                        # Quick check for problematic codecs
                        probe_cmd = ["ffprobe", "-v", "quiet", "-select_streams", "v:0", 
                                   "-show_entries", "stream=codec_name", "-of", "csv=p=0", file_path]
                        result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True)
                        codec = result.stdout.strip().lower()
                        
                        if codec in ['vp9', 'av01', 'av1'] or not file.endswith('.mp4'):
                            needs_conversion = True
                            conversion_file = file_path
                            print(f"Found {codec} codec - will convert for Apple TV compatibility")
                            break
                    except subprocess.CalledProcessError:
                        # If we can't probe, assume it might need conversion if not MP4
                        if not file.endswith('.mp4'):
                            needs_conversion = True
                            conversion_file = file_path
                            break
            
            if needs_conversion and conversion_file:
                threading.Thread(target=convert_to_apple_tv_compatible, args=(conversion_file,)).start()
            
            # Create the NFO file
            nfo_file_path = os.path.join(video_dir, "movie.nfo")
            create_nfo_file(video, nfo_file_path)

    # Fetch and process most popular videos worldwide (from multiple regions)
    for region_code in REGION_CODES:
        print(f"Fetching the most popular videos from {region_code}...")
        popular_videos = get_popular_videos(youtube, region_code)

        for video in popular_videos:
            # Check the exit time before processing each video
            check_exit_time()
            
            print(f"Processing popular video: {video['title']} ({region_code})")
            video_dir = os.path.join(OUTPUT_DIR, video["video_id"])
            
            # Check if the video already exists
            if os.path.exists(video_dir):
                print(f"Skipping download, already exists: {video['title']} ({video['video_id']})")
                continue
            
            os.makedirs(video_dir, exist_ok=True)
            
            # Download the popular video
            if not download_video(video["video_id"], video_dir):
                print(f"Skipping popular video {video['video_id']} due to download error.")
                continue  # Skip this video if download failed
            
            # Check if conversion is needed (VP9/AV1 codecs or non-MP4 files)
            needs_conversion = False
            conversion_file = None
            
            for file in os.listdir(video_dir):
                if file.endswith(('.mp4', '.mkv', '.webm', '.m4v')):
                    file_path = os.path.join(video_dir, file)
                    try:
                        # Quick check for problematic codecs
                        probe_cmd = ["ffprobe", "-v", "quiet", "-select_streams", "v:0", 
                                   "-show_entries", "stream=codec_name", "-of", "csv=p=0", file_path]
                        result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True)
                        codec = result.stdout.strip().lower()
                        
                        if codec in ['vp9', 'av01', 'av1'] or not file.endswith('.mp4'):
                            needs_conversion = True
                            conversion_file = file_path
                            print(f"Found {codec} codec - will convert for Apple TV compatibility")
                            break
                    except subprocess.CalledProcessError:
                        # If we can't probe, assume it might need conversion if not MP4
                        if not file.endswith('.mp4'):
                            needs_conversion = True
                            conversion_file = file_path
                            break
            
            if needs_conversion and conversion_file:
                threading.Thread(target=convert_to_apple_tv_compatible, args=(conversion_file,)).start()
            
            # Create the NFO file
            nfo_file_path = os.path.join(video_dir, "movie.nfo")
            create_nfo_file(video, nfo_file_path)
    
    print("All videos processed successfully!")

if __name__ == "__main__":
    main()