cookie

auth.py

@@ -4,159 +4,118 @@ from google.oauth2 import service_account
 import os
 import pickle
 import json
+import sqlite3
 import subprocess
+import shutil
 import tempfile
-import time
 from pathlib import Path
+from datetime import datetime
+import logging
 
 # OAuth 2.0 scopes
 SCOPES = ["https://www.googleapis.com/auth/youtube.readonly"]
 
-def extract_cookies_from_browser():
-    """Extract cookies from browsers to bypass YouTube bot detection."""
-    print("🍪 Attempting to extract cookies from browsers...")
-    
-    # Try different browsers in order of preference
-    browsers = ['chrome', 'firefox', 'edge', 'safari', 'opera', 'brave']
-    cookies_file = "youtube_cookies.txt"
-    
-    for browser in browsers:
+# Configure logging
+logger = logging.getLogger(__name__)
+
+def extract_chrome_cookies(output_file="youtube_cookies.txt"):
+    """Extract YouTube cookies from Chrome browser."""
+    try:
+        # Common Chrome cookie database locations
+        chrome_paths = [
+            Path.home() / ".config/google-chrome/Default/Cookies",
+            Path.home() / ".config/google-chrome/Profile 1/Cookies",
+            Path.home() / ".config/chromium/Default/Cookies",
+            Path.home() / "snap/chromium/common/chromium/Default/Cookies",
+            Path.home() / ".var/app/com.google.Chrome/config/google-chrome/Default/Cookies",
+        ]
+        
+        # Find existing Chrome cookies database
+        chrome_cookies_db = None
+        for path in chrome_paths:
+            if path.exists():
+                chrome_cookies_db = path
+                logger.info(f"Found Chrome cookies database: {chrome_cookies_db}")
+                break
+        
+        if not chrome_cookies_db:
+            logger.warning("Chrome cookies database not found")
+            return False
+        
+        # Use yt-dlp to extract cookies directly from browser
         try:
-            print(f"  Trying {browser}...")
-            
-            # Use yt-dlp's built-in cookie extraction
-            cmd = [
-                'yt-dlp', 
-                '--cookies-from-browser', browser,
-                '--cookies', cookies_file,
-                '--simulate',
-                '--no-warnings',
-                'https://www.youtube.com/watch?v=dQw4w9WgXcQ'  # Test video
-            ]
-            
+            cmd = ['yt-dlp', '--cookies-from-browser', 'chrome', '--print-to-file', 'cookies', output_file, '--no-download', 'https://www.youtube.com']
             result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
             
-            if result.returncode == 0 and os.path.exists(cookies_file):
-                print(f"  ✅ Successfully extracted cookies from {browser}")
-                return cookies_file
+            if result.returncode == 0:
+                logger.info(f"Successfully extracted Chrome cookies to {output_file}")
+                return True
             else:
-                print(f"  ❌ Failed to extract from {browser}")
-                # Clean up any partial cookie file
-                if os.path.exists(cookies_file):
-                    os.remove(cookies_file)
-                    
+                logger.error(f"yt-dlp cookie extraction failed: {result.stderr}")
+                return False
+                
         except subprocess.TimeoutExpired:
-            print(f"  ⏰ Timeout extracting from {browser}")
+            logger.error("Cookie extraction timed out")
+            return False
         except Exception as e:
-            print(f"  ❌ Error with {browser}: {e}")
-    
-    print("  ⚠️ Could not extract cookies from any browser")
-    return None
-
-def generate_youtube_cookies():
-    """Generate or update YouTube cookies for yt-dlp."""
-    cookies_file = "youtube_cookies.txt"
-    
-    # Check if cookies file already exists and is recent
-    if os.path.exists(cookies_file):
-        file_age = time.time() - os.path.getmtime(cookies_file)
-        if file_age < 3600:  # Less than 1 hour old
-            print(f"✅ Using existing cookies file (age: {file_age/60:.1f} minutes)")
-            return cookies_file
-        else:
-            print(f"🔄 Cookies file is {file_age/3600:.1f} hours old, refreshing...")
-    
-    # Try to extract fresh cookies
-    result = extract_cookies_from_browser()
-    if result:
-        return result
-    
-    # If extraction failed, try manual cookie creation
-    print("🔧 Attempting manual cookie setup...")
-    return create_manual_cookies()
-
-def create_manual_cookies():
-    """Create a basic cookie file for YouTube."""
-    cookies_file = "youtube_cookies.txt"
-    
-    # Create a basic Netscape-format cookie file
-    # This won't have real session cookies but may help with some requests
-    cookie_content = """# Netscape HTTP Cookie File
-# This is a generated file! Do not edit.
-
-.youtube.com	TRUE	/	FALSE	0	PREF	f1=50000000
-.youtube.com	TRUE	/	FALSE	0	VISITOR_INFO1_LIVE	
-.youtube.com	TRUE	/	TRUE	0	YSC	
-.google.com	TRUE	/	FALSE	0	NID	
-"""
-    
-    try:
-        with open(cookies_file, 'w') as f:
-            f.write(cookie_content)
-        print(f"📝 Created basic cookie file: {cookies_file}")
-        return cookies_file
+            logger.error(f"Error running yt-dlp cookie extraction: {e}")
+            return False
+            
     except Exception as e:
-        print(f"❌ Failed to create cookie file: {e}")
-        return None
+        logger.error(f"Error extracting Chrome cookies: {e}")
+        return False
 
-def test_youtube_access():
-    """Test if YouTube access works with current authentication."""
-    print("🧪 Testing YouTube access...")
-    
-    cookies_file = "youtube_cookies.txt"
-    test_commands = [
-        # Test with cookies
-        {
-            'name': 'With Cookies',
-            'cmd': ['yt-dlp', '--cookies', cookies_file, '--simulate', '--print', 'title', 'https://www.youtube.com/watch?v=dQw4w9WgXcQ']
-        },
-        # Test with browser extraction
-        {
-            'name': 'Browser Extraction', 
-            'cmd': ['yt-dlp', '--cookies-from-browser', 'chrome', '--simulate', '--print', 'title', 'https://www.youtube.com/watch?v=dQw4w9WgXcQ']
-        },
-        # Test with user agent
-        {
-            'name': 'Custom User Agent',
-            'cmd': ['yt-dlp', '--user-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', '--simulate', '--print', 'title', 'https://www.youtube.com/watch?v=dQw4w9WgXcQ']
-        }
-    ]
+def check_browser_cookies():
+    """Check which browsers have accessible cookies for YouTube."""
+    browsers = ['chrome', 'chromium', 'firefox', 'brave', 'edge']
+    available_browsers = []
     
-    for test in test_commands:
+    for browser in browsers:
         try:
-            print(f"  Testing: {test['name']}")
-            result = subprocess.run(test['cmd'], capture_output=True, text=True, timeout=20)
-            
+            # Test if browser cookies are accessible
+            cmd = ['yt-dlp', '--cookies-from-browser', browser, '--simulate', '--quiet', 'https://www.youtube.com/watch?v=dQw4w9WgXcQ']
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
             if result.returncode == 0:
-                print(f"    ✅ Success: {result.stdout.strip()}")
-                return test['name']
+                available_browsers.append(browser)
+                logger.info(f"Browser {browser} has accessible cookies")
             else:
-                print(f"    ❌ Failed: {result.stderr[:100]}...")
-                
-        except subprocess.TimeoutExpired:
-            print(f"    ⏰ Timeout")
-        except Exception as e:
-            print(f"    ❌ Error: {e}")
+                logger.debug(f"Browser {browser} cookies not accessible: {result.stderr}")
+        except (subprocess.TimeoutExpired, subprocess.SubprocessError) as e:
+            logger.debug(f"Error checking {browser} cookies: {e}")
+            continue
     
-    print("  ⚠️ All YouTube access tests failed")
-    return None
+    return available_browsers
 
-def setup_youtube_access():
-    """Setup YouTube access with cookie authentication."""
-    print("🔧 Setting up YouTube access with cookie authentication...")
-    
-    # Generate cookies
-    cookies_file = generate_youtube_cookies()
-    
-    if cookies_file:
-        # Test the access
-        working_method = test_youtube_access()
-        if working_method:
-            print(f"✅ YouTube access working with: {working_method}")
-            return cookies_file
-    
-    print("⚠️ YouTube access setup incomplete - may encounter bot detection")
-    return None
+def update_cookies_from_browser(browser='chrome', output_file="youtube_cookies.txt"):
+    """Update cookies file from specified browser."""
+    try:
+        logger.info(f"Updating cookies from {browser} browser...")
+        
+        # Use yt-dlp to extract cookies from browser
+        cmd = ['yt-dlp', '--cookies-from-browser', browser, '--print-to-file', 'cookies', output_file, '--no-download', 'https://www.youtube.com']
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+        
+        if result.returncode == 0:
+            logger.info(f"Successfully updated cookies from {browser} to {output_file}")
+            
+            # Verify the cookies file was created and has content
+            cookies_path = Path(output_file)
+            if cookies_path.exists() and cookies_path.stat().st_size > 0:
+                logger.info(f"Cookies file created: {cookies_path} ({cookies_path.stat().st_size} bytes)")
+                return True
+            else:
+                logger.warning("Cookies file was not created or is empty")
+                return False
+        else:
+            logger.error(f"Failed to extract cookies from {browser}: {result.stderr}")
+            return False
+            
+    except subprocess.TimeoutExpired:
+        logger.error("Cookie extraction timed out")
+        return False
+    except Exception as e:
+        logger.error(f"Error updating cookies from browser: {e}")
+        return False
 
 def get_service_account_token():
     """Authenticate using service account (no user interaction needed)."""
@@ -255,37 +214,53 @@ def get_token():
 if __name__ == "__main__":
     import sys
     
-    if "--interactive" in sys.argv:
-        creds = get_token_interactive()
-    elif "--cookies" in sys.argv:
-        print("🍪 Setting up YouTube cookies...")
-        cookies_file = setup_youtube_access()
-        if cookies_file:
-            print(f"✅ Cookie setup complete: {cookies_file}")
+    # Set up logging
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+    
+    if "--extract-cookies" in sys.argv:
+        # Extract cookies from Chrome
+        browser = 'chrome'
+        if "--browser" in sys.argv:
+            browser_idx = sys.argv.index("--browser") + 1
+            if browser_idx < len(sys.argv):
+                browser = sys.argv[browser_idx]
+        
+        output_file = "youtube_cookies.txt"
+        if "--output" in sys.argv:
+            output_idx = sys.argv.index("--output") + 1
+            if output_idx < len(sys.argv):
+                output_file = sys.argv[output_idx]
+        
+        print(f"Extracting cookies from {browser} browser...")
+        success = update_cookies_from_browser(browser, output_file)
+        if success:
+            print(f"Cookies successfully extracted to {output_file}")
         else:
-            print("❌ Cookie setup failed")
-        sys.exit(0)
-    elif "--test" in sys.argv:
-        print("🧪 Testing YouTube access...")
-        working_method = test_youtube_access()
-        if working_method:
-            print(f"✅ YouTube access working with: {working_method}")
+            print("Cookie extraction failed")
+            sys.exit(1)
+    
+    elif "--check-browsers" in sys.argv:
+        # Check available browsers
+        print("Checking available browsers with YouTube cookies...")
+        browsers = check_browser_cookies()
+        if browsers:
+            print(f"Available browsers: {', '.join(browsers)}")
         else:
-            print("❌ YouTube access not working")
-        sys.exit(0)
+            print("No browsers with accessible YouTube cookies found")
+    
+    elif "--interactive" in sys.argv:
+        creds = get_token_interactive()
     else:
         creds = get_token_non_interactive()
         if not creds:
             print("\nTo authenticate interactively, run: python3 auth.py --interactive")
-            print("To setup YouTube cookies, run: python3 auth.py --cookies")
-            print("To test YouTube access, run: python3 auth.py --test")
+            print("To extract cookies from browser, run: python3 auth.py --extract-cookies")
+            print("To check available browsers, run: python3 auth.py --check-browsers")
     
-    if creds:
-        print("Authentication successful!")
-        # Also setup cookies when authenticating
-        print("\n🍪 Setting up YouTube cookies for bot detection bypass...")
-        setup_youtube_access()
-    else:
-        print("Authentication failed!")
-        sys.exit(1)
+    if "--extract-cookies" not in sys.argv and "--check-browsers" not in sys.argv:
+        if creds:
+            print("Authentication successful!")
+        else:
+            print("Authentication failed!")
+            sys.exit(1)
 

main.py

@@ -14,7 +14,7 @@ from googleapiclient.discovery import build
 import subprocess
 from google.auth.transport.requests import Request
 import sys
-from auth import get_token, get_token_non_interactive, get_service_account_token, setup_youtube_access
+from auth import get_token, get_token_non_interactive, get_service_account_token
 
 # Home Assistant Integration
 try:
@@ -49,10 +49,30 @@ OUTPUT_DIR = "/media/media/YouTube"  # Change this to your Emby folder
 DAYS_BACK = 5  # Videos published in the last 5 days
 DAYS_TO_KEEP = 10  # Delete videos older than 10 days
 TOKEN_FILE = "youtube_token.pickle"  # Token file from the OAuth script
+COOKIES_FILE = "youtube_cookies.txt"  # YouTube cookies file for authentication
 EXIT_TIME = dt_time(6, 0)  # Exit the script at 06:00
 POPULAR_VIDEOS_MAX_RESULTS = 10  # Number of popular videos to fetch (you can change this)
 REGION_CODES = ["US", "GB", "LU", "LK"]  # Regions to fetch popular videos from
 
+# Helper function to build yt-dlp commands with cookies
+def build_yt_dlp_command(base_args, url=None):
+    """Build a yt-dlp command with cookies if available."""
+    command = ["yt-dlp"] + base_args
+    
+    # Add cookies if file exists
+    if os.path.exists(COOKIES_FILE):
+        command.extend(["--cookies", COOKIES_FILE])
+        # Log cookies usage at debug level to avoid spam
+        logging.getLogger(__name__).debug(f"Using cookies file: {COOKIES_FILE}")
+    else:
+        logging.getLogger(__name__).debug(f"Cookies file not found: {COOKIES_FILE}")
+    
+    # Add URL if provided
+    if url:
+        command.append(url)
+    
+    return command
+
 # Logging Configuration
 LOG_DIR = "logs"
 LOG_FILE = "youtube_predownloader.log"
@@ -235,63 +255,6 @@ def update_yt_dlp():
         logger.debug(f"yt-dlp update error: {e.stderr}")
         raise
 
-def get_youtube_cookie_args():
-    """Get YouTube cookie arguments for yt-dlp to bypass bot detection."""
-    logger = logging.getLogger(__name__)
-    
-    cookies_file = "youtube_cookies.txt"
-    cookie_args = []
-    
-    # Check if cookies file exists and is recent (less than 24 hours)
-    if os.path.exists(cookies_file):
-        file_age = time.time() - os.path.getmtime(cookies_file)
-        if file_age < 86400:  # 24 hours
-            logger.debug(f"Using existing cookies file (age: {file_age/3600:.1f} hours)")
-            cookie_args = ["--cookies", cookies_file]
-        else:
-            logger.debug("Cookies file is old, will try to refresh")
-    
-    # If no cookie file or it's old, try to setup fresh cookies
-    if not cookie_args:
-        try:
-            logger.debug("Setting up YouTube cookie authentication...")
-            fresh_cookies = setup_youtube_access()
-            if fresh_cookies and os.path.exists(fresh_cookies):
-                cookie_args = ["--cookies", fresh_cookies]
-                logger.debug("✅ Fresh cookies obtained")
-            else:
-                logger.debug("⚠️ Could not obtain fresh cookies")
-        except Exception as e:
-            logger.debug(f"Cookie setup failed: {e}")
-    
-    # Fallback: try browser extraction as last resort
-    if not cookie_args:
-        logger.debug("Trying browser cookie extraction as fallback...")
-        browser_options = ['chrome', 'firefox', 'edge']
-        for browser in browser_options:
-            try:
-                # Test if browser cookies work
-                test_cmd = ['yt-dlp', '--cookies-from-browser', browser, '--simulate', '--no-warnings', 'https://www.youtube.com/watch?v=dQw4w9WgXcQ']
-                result = subprocess.run(test_cmd, capture_output=True, text=True, timeout=15)
-                if result.returncode == 0:
-                    cookie_args = ["--cookies-from-browser", browser]
-                    logger.debug(f"✅ Using {browser} browser cookies")
-                    break
-            except:
-                continue
-    
-    # Add additional anti-bot arguments
-    anti_bot_args = [
-        "--user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-        "--sleep-interval", "1",
-        "--max-sleep-interval", "3", 
-        "--retry-sleep", "linear=2:5",
-        "--fragment-retries", "3",
-        "--socket-timeout", "30"
-    ]
-    
-    return cookie_args + anti_bot_args
-
 def authenticate_youtube_api():
     """Authenticate using our enhanced auth system (tries non-interactive first)."""
     logger = logging.getLogger(__name__)
@@ -421,15 +384,7 @@ def list_available_formats(video_id):
     logger = logging.getLogger(__name__)
     
     try:
-        # Get cookie arguments for bot detection bypass
-        cookie_args = get_youtube_cookie_args()
-        
-        command = [
-            "yt-dlp",
-            "--list-formats",
-        ] + cookie_args + [
-            f"https://www.youtube.com/watch?v={video_id}"
-        ]
+        command = build_yt_dlp_command(["--list-formats"], f"https://www.youtube.com/watch?v={video_id}")
         
         result = subprocess.run(command, capture_output=True, text=True, timeout=30)
         if result.returncode == 0:
@@ -458,15 +413,10 @@ def download_video(video_id, output_path, video_title="Unknown"):
         # First, check what format would be downloaded to determine compatibility
         logger.debug(f"Checking video format compatibility for: {video_title}")
         
-        # Get cookie arguments for bot detection bypass
-        cookie_args = get_youtube_cookie_args()
-        
-        simulate_command = [
-            "yt-dlp", "--simulate", "--print", "%(format_id)s|%(vcodec)s|%(height)s|%(ext)s",
-            "-f", "(bestvideo[height>=2160][vcodec~='^(avc|h264|hevc|h265)']+bestaudio)/(bestvideo[height>=2160]+bestaudio)/(bestvideo[height>=1440]+bestaudio)/(bestvideo[height>=1080]+bestaudio)/best",
-        ] + cookie_args + [
-            f"https://www.youtube.com/watch?v={video_id}"
-        ]
+        simulate_command = build_yt_dlp_command([
+            "--simulate", "--print", "%(format_id)s|%(vcodec)s|%(height)s|%(ext)s",
+            "-f", "(bestvideo[height>=2160][vcodec~='^(avc|h264|hevc|h265)']+bestaudio)/(bestvideo[height>=2160]+bestaudio)/(bestvideo[height>=1440]+bestaudio)/(bestvideo[height>=1080]+bestaudio)/best"
+        ], f"https://www.youtube.com/watch?v={video_id}")
         
         will_be_compatible = True
         predicted_codec = "unknown"
@@ -517,22 +467,15 @@ def download_video(video_id, output_path, video_title="Unknown"):
         )
         
         # 4K-prioritized approach: Try Apple TV compatible 4K first, then any 4K, then lower resolutions
-        base_command = [
-            "yt-dlp",
+        command = build_yt_dlp_command([
             "-f", "(bestvideo[height>=2160][vcodec~='^(avc|h264|hevc|h265)']+bestaudio)/(bestvideo[height>=2160]+bestaudio)/(bestvideo[height>=1440]+bestaudio)/(bestvideo[height>=1080]+bestaudio)/best",
             "--merge-output-format", "mp4",
             "--no-mtime",
             "--newline",  # Better for parsing progress
             "--verbose" if logger.isEnabledFor(logging.DEBUG) else "--no-warnings",
             "--write-info-json",  # Write metadata for analysis
-        ]
-        
-        # Add cookie authentication to bypass bot detection
-        cookie_args = get_youtube_cookie_args()
-        command = base_command + cookie_args + [
-            f"https://www.youtube.com/watch?v={video_id}",
             "-o", output_template
-        ]
+        ], f"https://www.youtube.com/watch?v={video_id}")
         
         logger.debug(f"yt-dlp command: {' '.join(command)}")
         
@@ -1619,12 +1562,36 @@ def check_exit_time():
         print("Current time exceeds exit time. Terminating script.")
         sys.exit(0)
 
+def check_cookies_status():
+    """Check and log the status of the YouTube cookies file."""
+    logger = logging.getLogger(__name__)
+    
+    if os.path.exists(COOKIES_FILE):
+        try:
+            # Check if cookies file is readable and has content
+            with open(COOKIES_FILE, 'r') as f:
+                content = f.read().strip()
+                if content:
+                    logger.info(f"✅ YouTube cookies file found and loaded: {COOKIES_FILE}")
+                else:
+                    logger.warning(f"⚠️ YouTube cookies file is empty: {COOKIES_FILE}")
+        except Exception as e:
+            logger.error(f"❌ Error reading cookies file {COOKIES_FILE}: {e}")
+    else:
+        logger.warning(f"⚠️ YouTube cookies file not found: {COOKIES_FILE}")
+        logger.info("   To avoid bot detection, create cookies file using:")
+        logger.info("   1. Export cookies from browser using a browser extension")
+        logger.info("   2. Save as 'youtube_cookies.txt' in the application directory")
+
 def main():
     global download_stats
     logger = logging.getLogger(__name__)
     
     logger.info("Starting main download process")
     
+    # Check cookies status
+    check_cookies_status()
+    
     # Initialize Home Assistant integration
     if HOMEASSISTANT_ENABLED:
         logger.info("Initializing Home Assistant integration...")