bugfixes

main.py

@@ -1,6 +1,7 @@
 import os
 import time
 import datetime
+from datetime import datetime, time as dt_time
 import pickle
 import threading
 import queue
@@ -11,12 +12,40 @@ from google.auth.transport.requests import Request
 import sys
 from auth import get_token, get_token_non_interactive, get_service_account_token
 
+# Home Assistant Integration
+try:
+    from homeassistant_integration import (
+        init_homeassistant_integration, 
+        update_homeassistant_stats, 
+        stop_homeassistant_integration,
+        set_service_waiting,
+        set_service_running,
+        set_service_completed,
+        set_service_error
+    )
+    HOMEASSISTANT_ENABLED = True
+    print("🏠 Home Assistant integration available")
+except ImportError:
+    HOMEASSISTANT_ENABLED = False
+    print("⚠️ Home Assistant integration not available (install paho-mqtt and psutil)")
+    
+    # Dummy functions if Home Assistant is not available
+    def set_service_waiting(next_run): pass
+    def set_service_running(): pass
+    def set_service_completed(): pass
+    def set_service_error(): pass
+    
+    # Dummy functions if not available
+    def init_homeassistant_integration(*args, **kwargs): return None
+    def update_homeassistant_stats(*args, **kwargs): pass
+    def stop_homeassistant_integration(*args, **kwargs): pass
+
 # Constants
 OUTPUT_DIR = "/media/media/YouTube"  # Change this to your Emby folder
 DAYS_BACK = 5  # Videos published in the last 5 days
 DAYS_TO_KEEP = 10  # Delete videos older than 10 days
 TOKEN_FILE = "youtube_token.pickle"  # Token file from the OAuth script
-EXIT_TIME = datetime.time(6, 0)  # Exit the script at 06:00
+EXIT_TIME = dt_time(6, 0)  # Exit the script at 06:00
 POPULAR_VIDEOS_MAX_RESULTS = 10  # Number of popular videos to fetch (you can change this)
 REGION_CODES = ["US", "GB", "LU", "LK"]  # Regions to fetch popular videos from
 
@@ -24,10 +53,30 @@ REGION_CODES = ["US", "GB", "LU", "LK"]  # Regions to fetch popular videos from
 MAX_CONCURRENT_CONVERSIONS = 2  # Maximum number of simultaneous video conversions
 CONVERSION_QUEUE_SIZE = 50       # Maximum items in conversion queue
 
+# FFmpeg/FFprobe paths
+FFPROBE_PATH = "/usr/bin/ffprobe"
+FFMPEG_PATH = "/usr/bin/ffmpeg"
+
 # Global conversion queue and executor
 conversion_queue = queue.Queue(maxsize=CONVERSION_QUEUE_SIZE)
 conversion_executor = ThreadPoolExecutor(max_workers=MAX_CONCURRENT_CONVERSIONS, thread_name_prefix="VideoConverter")
 
+# Progress tracking
+download_stats = {
+    'total_videos': 0,
+    'completed_videos': 0,
+    'current_video': '',
+    'download_speed': 0,
+    'active_downloads': 0
+}
+
+conversion_stats = {
+    'active_conversions': 0,
+    'completed_conversions': 0,
+    'current_conversion': '',
+    'conversion_progress': 0
+}
+
 def update_yt_dlp():
     """Update yt-dlp to the latest version."""
     print("Updating yt-dlp to the latest version...")
@@ -117,27 +166,56 @@ def get_popular_videos(youtube, region_code):
         })
     return popular_videos
 
-def download_video(video_id, output_path):
+def download_video(video_id, output_path, video_title="Unknown"):
     """Download a YouTube video with quality priority - downloads 4K VP9/AV1 if needed for conversion."""
+    global download_stats
+    
     try:
+        # Update stats
+        download_stats['active_downloads'] += 1
+        download_stats['current_video'] = video_title[:50] + "..." if len(video_title) > 50 else video_title
+        
+        # Update Home Assistant
+        update_homeassistant_stats(
+            download_queue=download_stats['total_videos'] - download_stats['completed_videos'],
+            conversion_queue=conversion_queue.qsize(),
+            download_progress=round((download_stats['completed_videos'] / max(download_stats['total_videos'], 1)) * 100, 1),
+            active_downloads=download_stats['active_downloads'],
+            active_conversions=conversion_stats['active_conversions']
+        )
+        
         # Quality-priority format selection:
-        # 1. First try: Apple TV compatible codecs at any resolution
-        # 2. Fallback: Download highest quality available (including 4K VP9/AV1 for conversion)
-        # 3. Last resort: Best available format
         command = [
             "yt-dlp",
             "-f", "(bv[vcodec~='^(avc|h264|hevc|h265)'] / bv[height>=2160] / bv[height>=1080] / best) + (ba[acodec~='^(aac|mp4a)'] / ba)",
             "--merge-output-format", "mp4",
             "--no-mtime",
+            "--newline",  # Better for parsing progress
             f"https://www.youtube.com/watch?v={video_id}",
             "-o", os.path.join(output_path, "%(title)s.%(ext)s")
         ]
-        subprocess.run(command, check=True)
+        
+        result = subprocess.run(command, check=True, capture_output=True, text=True)
+        
+        # Update completed count
+        download_stats['completed_videos'] += 1
+        download_stats['active_downloads'] -= 1
+        
         print(f"Successfully downloaded video {video_id} (Quality priority - will convert VP9/AV1 if needed)")
+        
+        # Update Home Assistant with completion
+        update_homeassistant_stats(
+            download_queue=download_stats['total_videos'] - download_stats['completed_videos'],
+            download_progress=round((download_stats['completed_videos'] / max(download_stats['total_videos'], 1)) * 100, 1),
+            active_downloads=download_stats['active_downloads']
+        )
+        
+        return True
+        
     except subprocess.CalledProcessError as e:
+        download_stats['active_downloads'] -= 1
         print(f"Error downloading video {video_id}: {e}")
         return False
-    return True
 
 def add_to_conversion_queue(input_file, priority=1):
     """Add a video file to the conversion queue with priority."""
@@ -200,16 +278,28 @@ def process_conversion_queue():
 
 def convert_to_apple_tv_compatible(input_file):
     """Convert a video to Apple TV 4K compatible format, optimized by resolution."""
+    global conversion_stats
+    
     base_name = os.path.splitext(input_file)[0]
     mp4_file = base_name + ".mp4"
     
+    # Update conversion stats
+    conversion_stats['active_conversions'] += 1
+    conversion_stats['current_conversion'] = os.path.basename(input_file)
+    
     print(f"Converting {input_file} to Apple TV compatible format...")
     
+    # Update Home Assistant
+    update_homeassistant_stats(
+        conversion_queue=conversion_queue.qsize(),
+        active_conversions=conversion_stats['active_conversions']
+    )
+    
     # Detect video properties to optimize conversion
     try:
         import json
         probe_cmd = [
-            "ffprobe", "-v", "quiet", "-print_format", "json", 
+            FFPROBE_PATH, "-v", "quiet", "-print_format", "json", 
             "-show_format", "-show_streams", input_file
         ]
         probe_result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True)
@@ -260,7 +350,7 @@ def convert_to_apple_tv_compatible(input_file):
         print("Using H.264 encoding for lower resolution content")
     
     command = [
-        "ffmpeg", "-i", input_file,
+        FFMPEG_PATH, "-i", input_file,
         *codec_args,
         "-c:a", "aac",              # AAC audio codec
         "-b:a", "192k",             # Higher audio bitrate for better quality
@@ -273,12 +363,31 @@ def convert_to_apple_tv_compatible(input_file):
         print("Starting conversion... This may take a while for 4K content.")
         subprocess.run(command, check=True)
         print(f"Conversion completed: {mp4_file}")
+        
+        # Update completion stats
+        conversion_stats['active_conversions'] -= 1
+        conversion_stats['completed_conversions'] += 1
+        
+        # Update Home Assistant
+        update_homeassistant_stats(
+            conversion_queue=conversion_queue.qsize(),
+            active_conversions=conversion_stats['active_conversions']
+        )
+        
         if os.path.exists(mp4_file):
             os.remove(input_file)
             print(f"Original file deleted: {input_file}")
         return True
     except subprocess.CalledProcessError as e:
+        conversion_stats['active_conversions'] -= 1
         print(f"Conversion failed for {input_file}: {e}")
+        
+        # Update Home Assistant
+        update_homeassistant_stats(
+            conversion_queue=conversion_queue.qsize(),
+            active_conversions=conversion_stats['active_conversions']
+        )
+        
         return False
 
 # Temporarily commented out video conversion functionality
@@ -334,12 +443,23 @@ def video_exists(output_path, video_id):
 
 def check_exit_time():
     """Check if the current time is past the exit time."""
-    now = datetime.datetime.now().time()
+    now = datetime.now().time()
     if now >= EXIT_TIME:
         print("Current time exceeds exit time. Terminating script.")
         sys.exit(0)
 
 def main():
+    global download_stats
+    
+    # Initialize Home Assistant integration
+    if HOMEASSISTANT_ENABLED:
+        print("🏠 Initializing Home Assistant integration...")
+        ha_integration = init_homeassistant_integration()
+        if ha_integration:
+            print("✅ Home Assistant integration started")
+        else:
+            print("⚠️ Home Assistant integration failed to start")
+    
     # Start the conversion queue worker
     print("🎬 Starting video conversion queue system...")
     print(f"📊 Max concurrent conversions: {MAX_CONCURRENT_CONVERSIONS}")
@@ -366,75 +486,191 @@ def main():
     print("Fetching subscribed channels...")
     channels = get_subscribed_channels(youtube)
     
-    # Fetch and process recent videos from subscribed channels
+    # Count total videos for progress tracking
+    print("📊 Counting total videos to download...")
+    total_videos = 0
+    all_videos = []
+    
+    # Collect all videos first for counting
     for channel_id in channels:
-        print(f"Fetching recent videos for channel: {channel_id}")
         videos = get_recent_videos(youtube, channel_id)
-        
         for video in videos:
-            # Check the exit time before processing each video
-            check_exit_time()
-            
-            print(f"Processing video: {video['title']}")
             video_dir = os.path.join(OUTPUT_DIR, video["video_id"])
-            
-            # Check if the video already exists
-            if os.path.exists(video_dir):
-                print(f"Skipping download, already exists: {video['title']} ({video['video_id']})")
-                continue
-            
-            os.makedirs(video_dir, exist_ok=True)
-            
-            # Download the video
-            if not download_video(video["video_id"], video_dir):
-                print(f"Skipping video {video['video_id']} due to download error.")
-                continue  # Skip this video if download failed
-            
-            # Check if conversion is needed (VP9/AV1 codecs or non-MP4 files)
-            needs_conversion = False
-            conversion_file = None
-            
-            for file in os.listdir(video_dir):
-                if file.endswith(('.mp4', '.mkv', '.webm', '.m4v')):
-                    file_path = os.path.join(video_dir, file)
-                    try:
-                        # Quick check for problematic codecs
-                        probe_cmd = ["ffprobe", "-v", "quiet", "-select_streams", "v:0", 
-                                   "-show_entries", "stream=codec_name", "-of", "csv=p=0", file_path]
-                        result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True)
-                        codec = result.stdout.strip().lower()
-                        
-                        if codec in ['vp9', 'av01', 'av1'] or not file.endswith('.mp4'):
-                            needs_conversion = True
-                            conversion_file = file_path
-                            print(f"Found {codec} codec - will convert for Apple TV compatibility")
-                            break
-                    except subprocess.CalledProcessError:
-                        # If we can't probe, assume it might need conversion if not MP4
-                        if not file.endswith('.mp4'):
-                            needs_conversion = True
-                            conversion_file = file_path
-                            break
-            
-            if needs_conversion and conversion_file:
-                # Determine priority based on video resolution
+            if not os.path.exists(video_dir):
+                all_videos.append(('subscription', video))
+                total_videos += 1
+    
+    # Add popular videos
+    for region_code in REGION_CODES:
+        popular_videos = get_popular_videos(youtube, region_code)
+        for video in popular_videos:
+            video_dir = os.path.join(OUTPUT_DIR, video["video_id"])
+            if not os.path.exists(video_dir):
+                all_videos.append(('popular', video))
+                total_videos += 1
+    
+    # Initialize download stats
+    download_stats['total_videos'] = total_videos
+    download_stats['completed_videos'] = 0
+    
+    print(f"📈 Total videos to download: {total_videos}")
+    
+    # Update initial Home Assistant stats
+    update_homeassistant_stats(
+        download_queue=total_videos,
+        conversion_queue=0,
+        download_progress=0,
+        active_downloads=0,
+        active_conversions=0
+    )
+    
+    # Process all videos
+    for video_type, video in all_videos:
+        # Check the exit time before processing each video
+        check_exit_time()
+        
+        print(f"Processing {video_type} video: {video['title']}")
+        video_dir = os.path.join(OUTPUT_DIR, video["video_id"])
+        
+        # Create directory
+        os.makedirs(video_dir, exist_ok=True)
+        
+        # Download the video
+        if not download_video(video["video_id"], video_dir, video["title"]):
+            print(f"Skipping video {video['video_id']} due to download error.")
+            continue  # Skip this video if download failed
+        
+        # Check if conversion is needed (VP9/AV1 codecs or non-MP4 files)
+        needs_conversion = False
+        conversion_file = None
+        
+        for file in os.listdir(video_dir):
+            if file.endswith(('.mp4', '.mkv', '.webm', '.m4v')):
+                file_path = os.path.join(video_dir, file)
                 try:
-                    probe_cmd = ["ffprobe", "-v", "quiet", "-select_streams", "v:0", 
-                               "-show_entries", "stream=height", "-of", "csv=p=0", conversion_file]
+                    # Quick check for problematic codecs
+                    probe_cmd = [FFPROBE_PATH, "-v", "quiet", "-select_streams", "v:0", 
+                               "-show_entries", "stream=codec_name", "-of", "csv=p=0", file_path]
                     result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True)
-                    height = int(result.stdout.strip()) if result.stdout.strip().isdigit() else 1080
-                    priority = 2 if height >= 2160 else 1  # 4K gets higher priority
-                except:
-                    priority = 1  # Default priority
+                    codec = result.stdout.strip().lower()
+                    
+                    if codec in ['vp9', 'av01', 'av1'] or not file.endswith('.mp4'):
+                        needs_conversion = True
+                        conversion_file = file_path
+                        print(f"Found {codec} codec - will convert for Apple TV compatibility")
+                        break
+                except subprocess.CalledProcessError:
+                    # If we can't probe, assume it might need conversion if not MP4
+                    if not file.endswith('.mp4'):
+                        needs_conversion = True
+                        conversion_file = file_path
+                        break
+        
+        if needs_conversion and conversion_file:
+            # Determine priority based on video resolution
+            try:
+                probe_cmd = [FFPROBE_PATH, "-v", "quiet", "-select_streams", "v:0", 
+                           "-show_entries", "stream=height", "-of", "csv=p=0", conversion_file]
+                result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True)
+                height = int(result.stdout.strip()) if result.stdout.strip().isdigit() else 1080
+                priority = 2 if height >= 2160 else 1  # 4K gets higher priority
+            except:
+                priority = 1  # Default priority
+            
+            add_to_conversion_queue(conversion_file, priority)
+            print_queue_status()
+        
+        # Create the NFO file
+        nfo_file_path = os.path.join(video_dir, "movie.nfo")
+        create_nfo_file(video, nfo_file_path)
+        
+        # Update completed videos count
+        download_stats['completed_videos'] += 1
+        progress = (download_stats['completed_videos'] / download_stats['total_videos']) * 100
+        
+        # Update Home Assistant stats
+        update_homeassistant_stats(
+            download_queue=max(0, download_stats['total_videos'] - download_stats['completed_videos']),
+            download_progress=round(progress, 1)
+        )
+
+    print("✅ All videos processed!")
+    print(f"📊 Final stats: {download_stats['completed_videos']}/{download_stats['total_videos']} videos completed")
+    
+    # Wait for all conversions to complete
+    print("⏳ Waiting for all conversions to complete...")
+    conversion_executor.shutdown(wait=True)
+    print("✅ All conversions completed!")
+    
+    # Final Home Assistant update
+    update_homeassistant_stats(
+        download_queue=0,
+        conversion_queue=0,
+        download_progress=100,
+        active_downloads=0,
+        active_conversions=0
+    )
+
+
+def schedule_downloads():
+    """Schedule downloads to run at specified times"""
+    # Configuration for scheduling
+    START_TIME = dt_time(0, 0)   # 00:00
+    END_TIME = dt_time(7, 0)     # 07:00
+    
+    while True:
+        current_time = datetime.now().time()
+        
+        # Check if we're in the allowed time window
+        if START_TIME <= current_time <= END_TIME:
+            try:
+                set_service_running()
+                print(f"🚀 Starting download session at {datetime.now().strftime('%H:%M:%S')}")
                 
-                add_to_conversion_queue(conversion_file, priority)
-                print_queue_status()
+                # Run the main download process
+                main()
+                
+                set_service_completed()
+                print(f"✅ Download session completed at {datetime.now().strftime('%H:%M:%S')}")
+                
+                # Calculate next run time (next day at START_TIME)
+                next_run = datetime.now().replace(hour=START_TIME.hour, minute=START_TIME.minute, second=0, microsecond=0)
+                if current_time > START_TIME:
+                    next_run = next_run.replace(day=next_run.day + 1)
+                
+                set_service_waiting(next_run.isoformat())
+                
+                # Sleep until next day
+                sleep_seconds = (next_run - datetime.now()).total_seconds()
+                print(f"😴 Sleeping until next run at {next_run.strftime('%Y-%m-%d %H:%M:%S')}")
+                time.sleep(sleep_seconds)
+                
+            except Exception as e:
+                print(f"❌ Error during download session: {e}")
+                set_service_error()
+                # Sleep for 1 hour before retrying
+                time.sleep(3600)
+        else:
+            # We're outside the allowed time window
+            if current_time < START_TIME:
+                # Before start time today
+                next_run = datetime.now().replace(hour=START_TIME.hour, minute=START_TIME.minute, second=0, microsecond=0)
+            else:
+                # After end time today, schedule for tomorrow
+                next_run = datetime.now().replace(hour=START_TIME.hour, minute=START_TIME.minute, second=0, microsecond=0)
+                next_run = next_run.replace(day=next_run.day + 1)
             
-            # Create the NFO file
-            nfo_file_path = os.path.join(video_dir, "movie.nfo")
-            create_nfo_file(video, nfo_file_path)
+            set_service_waiting(next_run.isoformat())
+            
+            # Sleep until start time
+            sleep_seconds = (next_run - datetime.now()).total_seconds()
+            print(f"⏰ Waiting until {next_run.strftime('%Y-%m-%d %H:%M:%S')} to start downloads")
+            time.sleep(min(sleep_seconds, 3600))  # Check every hour maximum
 
-    # Fetch and process most popular videos worldwide (from multiple regions)
+
+# Fetch and process most popular videos worldwide (from multiple regions)
+def fetch_popular_videos_old():
+    """Old function kept for reference - this logic is now integrated into main()"""
     for region_code in REGION_CODES:
         print(f"Fetching the most popular videos from {region_code}...")
         popular_videos = get_popular_videos(youtube, region_code)
@@ -467,7 +703,7 @@ def main():
                     file_path = os.path.join(video_dir, file)
                     try:
                         # Quick check for problematic codecs
-                        probe_cmd = ["ffprobe", "-v", "quiet", "-select_streams", "v:0", 
+                        probe_cmd = [FFPROBE_PATH, "-v", "quiet", "-select_streams", "v:0", 
                                    "-show_entries", "stream=codec_name", "-of", "csv=p=0", file_path]
                         result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True)
                         codec = result.stdout.strip().lower()
@@ -487,7 +723,7 @@ def main():
             if needs_conversion and conversion_file:
                 # Determine priority based on video resolution
                 try:
-                    probe_cmd = ["ffprobe", "-v", "quiet", "-select_streams", "v:0", 
+                    probe_cmd = [FFPROBE_PATH, "-v", "quiet", "-select_streams", "v:0", 
                                "-show_entries", "stream=height", "-of", "csv=p=0", conversion_file]
                     result = subprocess.run(probe_cmd, capture_output=True, text=True, check=True)
                     height = int(result.stdout.strip()) if result.stdout.strip().isdigit() else 1080
@@ -508,13 +744,33 @@ def main():
     print("\n🎬 Final Conversion Queue Status:")
     print_queue_status()
     
+    # Final Home Assistant update
+    update_homeassistant_stats(
+        download_queue=0,
+        conversion_queue=conversion_queue.qsize(),
+        download_progress=100,
+        active_downloads=0,
+        active_conversions=conversion_stats['active_conversions']
+    )
+    
     queued_items = conversion_queue.qsize()
     if queued_items > 0:
         print(f"⏳ {queued_items} videos still in conversion queue")
         print("💡 Conversions will continue in the background")
-        print("💡 You can monitor progress by checking the output folder")
+        print("💡 You can monitor progress in Home Assistant")
     else:
         print("✅ All conversions completed!")
+    
+    print("🏠 Home Assistant integration will continue running in background")
+
 
 if __name__ == "__main__":
-    main()
+    import sys
+    
+    # Check if we should run as a service with scheduling
+    if len(sys.argv) > 1 and sys.argv[1] == "--service":
+        print("🔄 Starting YouTube Predownloader Service with scheduling")
+        schedule_downloads()
+    else:
+        print("▶️ Running YouTube Predownloader once")
+        main()