feat(player): add natural language search with subtitle index

src/main.rs

@@ -12,6 +12,7 @@ use std::path::Path;
 mod config;
 mod overlay;
 mod player;
+mod search;
 mod web;
 
 use config::Config;
@@ -19,6 +20,7 @@ use overlay::{AsrLoader, ChunkLoader, YoloLoader};
 use player::audio::AudioPlayer;
 use player::ffmpeg::FFmpegDecoder;
 use player::state::{PlaybackState, PlayerState};
+use search::{create_searcher, SearchResult, VectorSearcher};
 
 fn main() -> Result<()> {
     env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
@@ -68,6 +70,10 @@ fn run(config: &Config) -> Result<()> {
     let mut audio_player: Option<AudioPlayer> = None;
     let mut is_fullscreen = false;
     let mut is_dragging = false;
+    let mut searcher: Option<VectorSearcher> = None;
+    let mut show_search = false;
+    let mut search_query = String::new();
+    let mut search_results: Vec<SearchResult> = Vec::new();
 
     if let Some(ref video_path) = config.video {
         info!("Loading video: {:?}", video_path);
@@ -99,6 +105,12 @@ fn run(config: &Config) -> Result<()> {
             Ok(loader) => {
                 info!("ASR loaded: {} segments", loader.segment_count());
                 asr = Some(loader);
+
+                let mut search = create_searcher();
+                if search.load_asr(Path::new(asr_path)).is_ok() {
+                    searcher = Some(search);
+                    info!("Search index ready");
+                }
             }
             Err(e) => {
                 error!("Failed to load ASR: {}", e);
@@ -155,6 +167,9 @@ fn run(config: &Config) -> Result<()> {
                 sdl2::event::Event::Quit { .. } => {
                     running = false;
                 }
+                sdl2::event::Event::TextInput { text, .. } if show_search => {
+                    search_query.push_str(&text);
+                }
                 sdl2::event::Event::KeyDown {
                     keycode, keymod, ..
                 } => {
@@ -162,7 +177,77 @@ fn run(config: &Config) -> Result<()> {
                         let shift = keymod.intersects(sdl2::keyboard::Mod::LSHIFTMOD)
                             || keymod.intersects(sdl2::keyboard::Mod::RSHIFTMOD);
                         match key {
-                            sdl2::keyboard::Keycode::Escape => running = false,
+                            sdl2::keyboard::Keycode::Escape => {
+                                if show_search {
+                                    show_search = false;
+                                    search_query.clear();
+                                    search_results.clear();
+                                } else {
+                                    running = false;
+                                }
+                            }
+                            sdl2::keyboard::Keycode::Slash => {
+                                if searcher.is_some() {
+                                    show_search = true;
+                                    search_query.clear();
+                                    search_results.clear();
+                                    info!("Search mode enabled");
+                                }
+                            }
+                            sdl2::keyboard::Keycode::Return | sdl2::keyboard::Keycode::KpEnter => {
+                                if show_search && !search_query.is_empty() {
+                                    if let Some(ref search) = searcher {
+                                        search_results = search.search(&search_query);
+                                        info!("Found {} results", search_results.len());
+                                    }
+                                }
+                            }
+                            sdl2::keyboard::Keycode::Backspace => {
+                                if show_search && !search_query.is_empty() {
+                                    search_query.pop();
+                                    if search_query.is_empty() {
+                                        search_results.clear();
+                                    }
+                                }
+                            }
+                            sdl2::keyboard::Keycode::Num1
+                            | sdl2::keyboard::Keycode::Num2
+                            | sdl2::keyboard::Keycode::Num3
+                            | sdl2::keyboard::Keycode::Num4
+                            | sdl2::keyboard::Keycode::Num5
+                            | sdl2::keyboard::Keycode::Num6 => {
+                                if show_search && !search_results.is_empty() {
+                                    let idx = match key {
+                                        sdl2::keyboard::Keycode::Num1 => 0,
+                                        sdl2::keyboard::Keycode::Num2 => 1,
+                                        sdl2::keyboard::Keycode::Num3 => 2,
+                                        sdl2::keyboard::Keycode::Num4 => 3,
+                                        sdl2::keyboard::Keycode::Num5 => 4,
+                                        sdl2::keyboard::Keycode::Num6 => 5,
+                                        _ => 0,
+                                    };
+                                    if idx < search_results.len() {
+                                        let result = search_results[idx].clone();
+                                        if let Some(ref mut dec) = decoder {
+                                            if dec.seek(result.time_ms).is_ok() {
+                                                player_state.current_frame = result.frame;
+                                                player_state.current_time_ms = result.time_ms;
+                                                player_state.playback = PlaybackState::Paused;
+                                                show_search = false;
+                                                search_query.clear();
+                                                search_results.clear();
+                                                sync_audio(
+                                                    &mut audio_player,
+                                                    &config.video,
+                                                    result.time_ms,
+                                                    false,
+                                                );
+                                                info!("Jumped to: {}", result.text);
+                                            }
+                                        }
+                                    }
+                                }
+                            }
                             sdl2::keyboard::Keycode::Space => {
                                 let was_playing = player_state.playback == PlaybackState::Playing;
                                 player_state.playback = if was_playing {
@@ -658,6 +743,87 @@ fn run(config: &Config) -> Result<()> {
             }
         }
 
+        if show_search {
+            if let Some(ref font) = font {
+                let search_bg_h = 250u32;
+                let search_bg_y = (config.height as i32 - search_bg_h as i32) / 2;
+                canvas.set_draw_color(sdl2::pixels::Color::RGBA(20, 20, 30, 230));
+                let _ = canvas.fill_rect(Rect::new(
+                    config.width as i32 / 2 - 300,
+                    search_bg_y,
+                    600,
+                    search_bg_h,
+                ));
+                canvas.set_draw_color(sdl2::pixels::Color::RGBA(60, 60, 80, 255));
+                let _ = canvas.draw_rect(Rect::new(
+                    config.width as i32 / 2 - 300,
+                    search_bg_y,
+                    600,
+                    search_bg_h,
+                ));
+
+                let input_text = if search_query.is_empty() {
+                    "Type to search... (Enter to search, Esc to close)".to_string()
+                } else {
+                    format!("Search: {}", search_query)
+                };
+                if let Ok(surface) = font
+                    .render(&input_text)
+                    .solid(sdl2::pixels::Color::RGB(200, 200, 200))
+                {
+                    if let Ok(tex) = texture_creator.create_texture_from_surface(&surface) {
+                        let rect = Rect::new(
+                            config.width as i32 / 2 - 290,
+                            search_bg_y + 10,
+                            surface.width(),
+                            surface.height(),
+                        );
+                        canvas.copy(&tex, None, Some(rect)).ok();
+                    }
+                }
+
+                if !search_results.is_empty() {
+                    let mut y_offset = search_bg_y + 40;
+                    for (i, result) in search_results.iter().take(6).enumerate() {
+                        let result_text = format!("[{}] {}", i + 1, result.text);
+                        let color = if i == 0 {
+                            sdl2::pixels::Color::RGB(100, 200, 255)
+                        } else {
+                            sdl2::pixels::Color::RGB(180, 180, 180)
+                        };
+                        if let Ok(surface) = font.render(&result_text).solid(color) {
+                            if let Ok(tex) = texture_creator.create_texture_from_surface(&surface) {
+                                let rect = Rect::new(
+                                    config.width as i32 / 2 - 290,
+                                    y_offset,
+                                    surface.width().min(580),
+                                    surface.height(),
+                                );
+                                canvas.copy(&tex, None, Some(rect)).ok();
+                            }
+                        }
+                        y_offset += 25;
+                    }
+                } else if !search_query.is_empty() {
+                    let hint = "Press Enter to search...";
+                    if let Ok(surface) = font
+                        .render(hint)
+                        .solid(sdl2::pixels::Color::RGB(150, 150, 150))
+                    {
+                        if let Ok(tex) = texture_creator.create_texture_from_surface(&surface) {
+                            let rect = Rect::new(
+                                config.width as i32 / 2 - 290,
+                                search_bg_y + 40,
+                                surface.width(),
+                                surface.height(),
+                            );
+                            canvas.copy(&tex, None, Some(rect)).ok();
+                        }
+                    }
+                }
+            }
+        }
+
         canvas.present();
         std::thread::sleep(std::time::Duration::from_millis(16));
     }

src/search/mod.rs

@@ -0,0 +1,96 @@
+use anyhow::{Context, Result};
+use log::{error, info, warn};
+use serde::{Deserialize, Serialize};
+use std::path::Path;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SearchResult {
+    pub frame: u64,
+    pub time_ms: u64,
+    pub text: String,
+    pub score: f32,
+}
+
+#[derive(Debug)]
+pub struct VectorSearcher {
+    qdrant_url: String,
+    collection: String,
+    asr_loader: Option<super::overlay::AsrLoader>,
+}
+
+impl VectorSearcher {
+    pub fn new(qdrant_url: &str, collection: &str) -> Self {
+        Self {
+            qdrant_url: qdrant_url.to_string(),
+            collection: collection.to_string(),
+            asr_loader: None,
+        }
+    }
+
+    pub fn load_asr(&mut self, path: &Path) -> Result<()> {
+        let loader =
+            super::overlay::AsrLoader::load(path).context("Failed to load ASR for search")?;
+        info!(
+            "Loaded ASR with {} segments for search",
+            loader.segment_count()
+        );
+        self.asr_loader = Some(loader);
+        Ok(())
+    }
+
+    pub fn search(&self, query: &str) -> Vec<SearchResult> {
+        info!("Searching for: {}", query);
+
+        if let Some(ref asr) = self.asr_loader {
+            let query_lower = query.to_lowercase();
+            let mut results: Vec<SearchResult> = Vec::new();
+
+            for segment in asr.get_all_segments() {
+                let text_lower = segment.text.to_lowercase();
+                if text_lower.contains(&query_lower) {
+                    let score = self.calculate_score(&query_lower, &text_lower);
+                    results.push(SearchResult {
+                        frame: (segment.start * 60.0) as u64,
+                        time_ms: (segment.start * 1000.0) as u64,
+                        text: segment.text.clone(),
+                        score,
+                    });
+                }
+            }
+
+            results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap());
+            results.truncate(10);
+
+            info!("Found {} results", results.len());
+            return results;
+        }
+
+        warn!("No ASR loaded for search");
+        Vec::new()
+    }
+
+    fn calculate_score(&self, query: &str, text: &str) -> f32 {
+        let query_words: Vec<&str> = query.split_whitespace().collect();
+        let text_words: Vec<&str> = text.split_whitespace().collect();
+
+        let mut matches = 0;
+        for qw in &query_words {
+            for tw in &text_words {
+                if tw.contains(qw) {
+                    matches += 1;
+                    break;
+                }
+            }
+        }
+
+        (matches as f32) / (query_words.len() as f32)
+    }
+
+    pub fn is_available(&self) -> bool {
+        true
+    }
+}
+
+pub fn create_searcher() -> VectorSearcher {
+    VectorSearcher::new("http://localhost:6333", "AccusysDB")
+}

target/release/deps/momentry-d0e10ec51fcac34b.d

@@ -1,6 +1,6 @@
-/Users/accusys/momentry_playground/target/release/deps/momentry-d0e10ec51fcac34b.d: src/main.rs src/config.rs src/overlay/mod.rs src/overlay/asr.rs src/overlay/chunk.rs src/overlay/yolo.rs src/player/mod.rs src/player/audio.rs src/player/ffmpeg.rs src/player/renderer.rs src/player/state.rs src/player/video.rs src/web/mod.rs src/web/bridge.rs
+/Users/accusys/momentry_playground/target/release/deps/momentry-d0e10ec51fcac34b.d: src/main.rs src/config.rs src/overlay/mod.rs src/overlay/asr.rs src/overlay/chunk.rs src/overlay/yolo.rs src/player/mod.rs src/player/audio.rs src/player/ffmpeg.rs src/player/renderer.rs src/player/state.rs src/player/video.rs src/search/mod.rs src/web/mod.rs src/web/bridge.rs
 
-/Users/accusys/momentry_playground/target/release/deps/momentry-d0e10ec51fcac34b: src/main.rs src/config.rs src/overlay/mod.rs src/overlay/asr.rs src/overlay/chunk.rs src/overlay/yolo.rs src/player/mod.rs src/player/audio.rs src/player/ffmpeg.rs src/player/renderer.rs src/player/state.rs src/player/video.rs src/web/mod.rs src/web/bridge.rs
+/Users/accusys/momentry_playground/target/release/deps/momentry-d0e10ec51fcac34b: src/main.rs src/config.rs src/overlay/mod.rs src/overlay/asr.rs src/overlay/chunk.rs src/overlay/yolo.rs src/player/mod.rs src/player/audio.rs src/player/ffmpeg.rs src/player/renderer.rs src/player/state.rs src/player/video.rs src/search/mod.rs src/web/mod.rs src/web/bridge.rs
 
 src/main.rs:
 src/config.rs:
@@ -14,5 +14,6 @@ src/player/ffmpeg.rs:
 src/player/renderer.rs:
 src/player/state.rs:
 src/player/video.rs:
+src/search/mod.rs:
 src/web/mod.rs:
 src/web/bridge.rs:

target/release/momentry.d

@@ -1 +1 @@
-/Users/accusys/momentry_playground/target/release/momentry: /Users/accusys/momentry_playground/src/config.rs /Users/accusys/momentry_playground/src/lib.rs /Users/accusys/momentry_playground/src/main.rs /Users/accusys/momentry_playground/src/overlay/asr.rs /Users/accusys/momentry_playground/src/overlay/chunk.rs /Users/accusys/momentry_playground/src/overlay/mod.rs /Users/accusys/momentry_playground/src/overlay/yolo.rs /Users/accusys/momentry_playground/src/player/audio.rs /Users/accusys/momentry_playground/src/player/ffmpeg.rs /Users/accusys/momentry_playground/src/player/mod.rs /Users/accusys/momentry_playground/src/player/renderer.rs /Users/accusys/momentry_playground/src/player/state.rs /Users/accusys/momentry_playground/src/player/video.rs /Users/accusys/momentry_playground/src/web/bridge.rs /Users/accusys/momentry_playground/src/web/mod.rs
+/Users/accusys/momentry_playground/target/release/momentry: /Users/accusys/momentry_playground/src/config.rs /Users/accusys/momentry_playground/src/lib.rs /Users/accusys/momentry_playground/src/main.rs /Users/accusys/momentry_playground/src/overlay/asr.rs /Users/accusys/momentry_playground/src/overlay/chunk.rs /Users/accusys/momentry_playground/src/overlay/mod.rs /Users/accusys/momentry_playground/src/overlay/yolo.rs /Users/accusys/momentry_playground/src/player/audio.rs /Users/accusys/momentry_playground/src/player/ffmpeg.rs /Users/accusys/momentry_playground/src/player/mod.rs /Users/accusys/momentry_playground/src/player/renderer.rs /Users/accusys/momentry_playground/src/player/state.rs /Users/accusys/momentry_playground/src/player/video.rs /Users/accusys/momentry_playground/src/search/mod.rs /Users/accusys/momentry_playground/src/web/bridge.rs /Users/accusys/momentry_playground/src/web/mod.rs