feat(player): add natural language search with subtitle index
This commit is contained in:
accusys
168
src/main.rs
168
src/main.rs
@@ -12,6 +12,7 @@ use std::path::Path;
|
||||
mod config;
|
||||
mod overlay;
|
||||
mod player;
|
||||
mod search;
|
||||
mod web;
|
||||
|
||||
use config::Config;
|
||||
@@ -19,6 +20,7 @@ use overlay::{AsrLoader, ChunkLoader, YoloLoader};
|
||||
use player::audio::AudioPlayer;
|
||||
use player::ffmpeg::FFmpegDecoder;
|
||||
use player::state::{PlaybackState, PlayerState};
|
||||
use search::{create_searcher, SearchResult, VectorSearcher};
|
||||
|
||||
fn main() -> Result<()> {
|
||||
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
|
||||
@@ -68,6 +70,10 @@ fn run(config: &Config) -> Result<()> {
|
||||
let mut audio_player: Option<AudioPlayer> = None;
|
||||
let mut is_fullscreen = false;
|
||||
let mut is_dragging = false;
|
||||
let mut searcher: Option<VectorSearcher> = None;
|
||||
let mut show_search = false;
|
||||
let mut search_query = String::new();
|
||||
let mut search_results: Vec<SearchResult> = Vec::new();
|
||||
|
||||
if let Some(ref video_path) = config.video {
|
||||
info!("Loading video: {:?}", video_path);
|
||||
@@ -99,6 +105,12 @@ fn run(config: &Config) -> Result<()> {
|
||||
Ok(loader) => {
|
||||
info!("ASR loaded: {} segments", loader.segment_count());
|
||||
asr = Some(loader);
|
||||
|
||||
let mut search = create_searcher();
|
||||
if search.load_asr(Path::new(asr_path)).is_ok() {
|
||||
searcher = Some(search);
|
||||
info!("Search index ready");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to load ASR: {}", e);
|
||||
@@ -155,6 +167,9 @@ fn run(config: &Config) -> Result<()> {
|
||||
sdl2::event::Event::Quit { .. } => {
|
||||
running = false;
|
||||
}
|
||||
sdl2::event::Event::TextInput { text, .. } if show_search => {
|
||||
search_query.push_str(&text);
|
||||
}
|
||||
sdl2::event::Event::KeyDown {
|
||||
keycode, keymod, ..
|
||||
} => {
|
||||
@@ -162,7 +177,77 @@ fn run(config: &Config) -> Result<()> {
|
||||
let shift = keymod.intersects(sdl2::keyboard::Mod::LSHIFTMOD)
|
||||
|| keymod.intersects(sdl2::keyboard::Mod::RSHIFTMOD);
|
||||
match key {
|
||||
sdl2::keyboard::Keycode::Escape => running = false,
|
||||
sdl2::keyboard::Keycode::Escape => {
|
||||
if show_search {
|
||||
show_search = false;
|
||||
search_query.clear();
|
||||
search_results.clear();
|
||||
} else {
|
||||
running = false;
|
||||
}
|
||||
}
|
||||
sdl2::keyboard::Keycode::Slash => {
|
||||
if searcher.is_some() {
|
||||
show_search = true;
|
||||
search_query.clear();
|
||||
search_results.clear();
|
||||
info!("Search mode enabled");
|
||||
}
|
||||
}
|
||||
sdl2::keyboard::Keycode::Return | sdl2::keyboard::Keycode::KpEnter => {
|
||||
if show_search && !search_query.is_empty() {
|
||||
if let Some(ref search) = searcher {
|
||||
search_results = search.search(&search_query);
|
||||
info!("Found {} results", search_results.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
sdl2::keyboard::Keycode::Backspace => {
|
||||
if show_search && !search_query.is_empty() {
|
||||
search_query.pop();
|
||||
if search_query.is_empty() {
|
||||
search_results.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
sdl2::keyboard::Keycode::Num1
|
||||
| sdl2::keyboard::Keycode::Num2
|
||||
| sdl2::keyboard::Keycode::Num3
|
||||
| sdl2::keyboard::Keycode::Num4
|
||||
| sdl2::keyboard::Keycode::Num5
|
||||
| sdl2::keyboard::Keycode::Num6 => {
|
||||
if show_search && !search_results.is_empty() {
|
||||
let idx = match key {
|
||||
sdl2::keyboard::Keycode::Num1 => 0,
|
||||
sdl2::keyboard::Keycode::Num2 => 1,
|
||||
sdl2::keyboard::Keycode::Num3 => 2,
|
||||
sdl2::keyboard::Keycode::Num4 => 3,
|
||||
sdl2::keyboard::Keycode::Num5 => 4,
|
||||
sdl2::keyboard::Keycode::Num6 => 5,
|
||||
_ => 0,
|
||||
};
|
||||
if idx < search_results.len() {
|
||||
let result = search_results[idx].clone();
|
||||
if let Some(ref mut dec) = decoder {
|
||||
if dec.seek(result.time_ms).is_ok() {
|
||||
player_state.current_frame = result.frame;
|
||||
player_state.current_time_ms = result.time_ms;
|
||||
player_state.playback = PlaybackState::Paused;
|
||||
show_search = false;
|
||||
search_query.clear();
|
||||
search_results.clear();
|
||||
sync_audio(
|
||||
&mut audio_player,
|
||||
&config.video,
|
||||
result.time_ms,
|
||||
false,
|
||||
);
|
||||
info!("Jumped to: {}", result.text);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
sdl2::keyboard::Keycode::Space => {
|
||||
let was_playing = player_state.playback == PlaybackState::Playing;
|
||||
player_state.playback = if was_playing {
|
||||
@@ -658,6 +743,87 @@ fn run(config: &Config) -> Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
if show_search {
|
||||
if let Some(ref font) = font {
|
||||
let search_bg_h = 250u32;
|
||||
let search_bg_y = (config.height as i32 - search_bg_h as i32) / 2;
|
||||
canvas.set_draw_color(sdl2::pixels::Color::RGBA(20, 20, 30, 230));
|
||||
let _ = canvas.fill_rect(Rect::new(
|
||||
config.width as i32 / 2 - 300,
|
||||
search_bg_y,
|
||||
600,
|
||||
search_bg_h,
|
||||
));
|
||||
canvas.set_draw_color(sdl2::pixels::Color::RGBA(60, 60, 80, 255));
|
||||
let _ = canvas.draw_rect(Rect::new(
|
||||
config.width as i32 / 2 - 300,
|
||||
search_bg_y,
|
||||
600,
|
||||
search_bg_h,
|
||||
));
|
||||
|
||||
let input_text = if search_query.is_empty() {
|
||||
"Type to search... (Enter to search, Esc to close)".to_string()
|
||||
} else {
|
||||
format!("Search: {}", search_query)
|
||||
};
|
||||
if let Ok(surface) = font
|
||||
.render(&input_text)
|
||||
.solid(sdl2::pixels::Color::RGB(200, 200, 200))
|
||||
{
|
||||
if let Ok(tex) = texture_creator.create_texture_from_surface(&surface) {
|
||||
let rect = Rect::new(
|
||||
config.width as i32 / 2 - 290,
|
||||
search_bg_y + 10,
|
||||
surface.width(),
|
||||
surface.height(),
|
||||
);
|
||||
canvas.copy(&tex, None, Some(rect)).ok();
|
||||
}
|
||||
}
|
||||
|
||||
if !search_results.is_empty() {
|
||||
let mut y_offset = search_bg_y + 40;
|
||||
for (i, result) in search_results.iter().take(6).enumerate() {
|
||||
let result_text = format!("[{}] {}", i + 1, result.text);
|
||||
let color = if i == 0 {
|
||||
sdl2::pixels::Color::RGB(100, 200, 255)
|
||||
} else {
|
||||
sdl2::pixels::Color::RGB(180, 180, 180)
|
||||
};
|
||||
if let Ok(surface) = font.render(&result_text).solid(color) {
|
||||
if let Ok(tex) = texture_creator.create_texture_from_surface(&surface) {
|
||||
let rect = Rect::new(
|
||||
config.width as i32 / 2 - 290,
|
||||
y_offset,
|
||||
surface.width().min(580),
|
||||
surface.height(),
|
||||
);
|
||||
canvas.copy(&tex, None, Some(rect)).ok();
|
||||
}
|
||||
}
|
||||
y_offset += 25;
|
||||
}
|
||||
} else if !search_query.is_empty() {
|
||||
let hint = "Press Enter to search...";
|
||||
if let Ok(surface) = font
|
||||
.render(hint)
|
||||
.solid(sdl2::pixels::Color::RGB(150, 150, 150))
|
||||
{
|
||||
if let Ok(tex) = texture_creator.create_texture_from_surface(&surface) {
|
||||
let rect = Rect::new(
|
||||
config.width as i32 / 2 - 290,
|
||||
search_bg_y + 40,
|
||||
surface.width(),
|
||||
surface.height(),
|
||||
);
|
||||
canvas.copy(&tex, None, Some(rect)).ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
canvas.present();
|
||||
std::thread::sleep(std::time::Duration::from_millis(16));
|
||||
}
|
||||
|
||||
96
src/search/mod.rs
Normal file
96
src/search/mod.rs
Normal file
@@ -0,0 +1,96 @@
|
||||
use anyhow::{Context, Result};
|
||||
use log::{error, info, warn};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SearchResult {
|
||||
pub frame: u64,
|
||||
pub time_ms: u64,
|
||||
pub text: String,
|
||||
pub score: f32,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct VectorSearcher {
|
||||
qdrant_url: String,
|
||||
collection: String,
|
||||
asr_loader: Option<super::overlay::AsrLoader>,
|
||||
}
|
||||
|
||||
impl VectorSearcher {
|
||||
pub fn new(qdrant_url: &str, collection: &str) -> Self {
|
||||
Self {
|
||||
qdrant_url: qdrant_url.to_string(),
|
||||
collection: collection.to_string(),
|
||||
asr_loader: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load_asr(&mut self, path: &Path) -> Result<()> {
|
||||
let loader =
|
||||
super::overlay::AsrLoader::load(path).context("Failed to load ASR for search")?;
|
||||
info!(
|
||||
"Loaded ASR with {} segments for search",
|
||||
loader.segment_count()
|
||||
);
|
||||
self.asr_loader = Some(loader);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn search(&self, query: &str) -> Vec<SearchResult> {
|
||||
info!("Searching for: {}", query);
|
||||
|
||||
if let Some(ref asr) = self.asr_loader {
|
||||
let query_lower = query.to_lowercase();
|
||||
let mut results: Vec<SearchResult> = Vec::new();
|
||||
|
||||
for segment in asr.get_all_segments() {
|
||||
let text_lower = segment.text.to_lowercase();
|
||||
if text_lower.contains(&query_lower) {
|
||||
let score = self.calculate_score(&query_lower, &text_lower);
|
||||
results.push(SearchResult {
|
||||
frame: (segment.start * 60.0) as u64,
|
||||
time_ms: (segment.start * 1000.0) as u64,
|
||||
text: segment.text.clone(),
|
||||
score,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap());
|
||||
results.truncate(10);
|
||||
|
||||
info!("Found {} results", results.len());
|
||||
return results;
|
||||
}
|
||||
|
||||
warn!("No ASR loaded for search");
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
fn calculate_score(&self, query: &str, text: &str) -> f32 {
|
||||
let query_words: Vec<&str> = query.split_whitespace().collect();
|
||||
let text_words: Vec<&str> = text.split_whitespace().collect();
|
||||
|
||||
let mut matches = 0;
|
||||
for qw in &query_words {
|
||||
for tw in &text_words {
|
||||
if tw.contains(qw) {
|
||||
matches += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(matches as f32) / (query_words.len() as f32)
|
||||
}
|
||||
|
||||
pub fn is_available(&self) -> bool {
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
pub fn create_searcher() -> VectorSearcher {
|
||||
VectorSearcher::new("http://localhost:6333", "AccusysDB")
|
||||
}
|
||||
Reference in New Issue
Block a user