def combined_search(query: str, limit: int = 20, 
                   weights: Dict[str, float] = None) -> Tuple[List[Dict], str]:
    """Perform fusion search across all modalities."""
    # Default fusion weights
    if weights is None:
        weights = {
            'clip': 0.35,     # CLIP visual similarity 
            'filename': 0.3,  # Filename fuzzy match
            'semantic': 0.25, # AI captions/labels
            'text': 0.1       # OCR text  
        }
    
    all_results = {}  # file_id -> result_data
    methods_used = []
    
    # Run all search methods in parallel
    for method in ['filename', 'semantic', 'text']:
        search_func = {
            'filename': search_filename,
            'semantic': search_semantic, 
            'text': search_ocr_text
        }[method]
        
        results = search_func(query, limit * 2)
        if results:
            methods_used.append(method)
            for result in results:
                file_id = result['id']
                if file_id not in all_results:
                    all_results[file_id] = result.copy()
                    all_results[file_id]['scores'] = {}
                
                # Normalize scores to 0-1 range
                score = result.get('confidence', 0) / 100.0
                all_results[file_id]['scores'][method] = score

    # CLIP embedding search
    query_embedding = extract_text_embedding(query)
    if query_embedding is not None:
        clip_results = search_by_embedding(query_embedding, limit * 2)
        if clip_results:
            methods_used.append('clip')
            for result in clip_results:
                file_id = result['file_id']
                if file_id not in all_results:
                    file_data = get_file_by_id(file_id)
                    if file_data:
                        all_results[file_id] = file_data
                        all_results[file_id]['scores'] = {}
                
                # CLIP similarity is already 0-1 normalized
                all_results[file_id]['scores']['clip'] = result['similarity']

    # Calculate weighted fusion scores
    final_results = []
    for file_id, result in all_results.items():
        total_score = 0.0
        active_weight = 0.0
        
        for method, score in result.get('scores', {}).items():
            if method in weights and score > 0:
                total_score += weights[method] * score
                active_weight += weights[method]
        
        if active_weight > 0:
            result['fusion_score'] = total_score / active_weight
            final_results.append(result)
    
    # Sort by fusion score and return top results
    final_results.sort(key=lambda x: x['fusion_score'], reverse=True)
    return final_results[:limit], f"fusion ({', '.join(methods_used)})"

# Intelligent search routing based on query analysis
def route_search(query: str, limit: int = 20) -> Tuple[List[Dict], str]:
    """Route search query to most appropriate method."""
    query = query.strip()
    
    # Try filename search first (often most relevant)
    filename_results = search_filename(query, limit)
    if filename_results:
        for result in filename_results:
            result['search_method'] = 'filename'
        return filename_results, "filename (fuzzy)"
    
    # Try semantic search (captions + labels)
    semantic_results = search_semantic(query, limit)
    if semantic_results:
        for result in semantic_results:
            result['search_method'] = 'semantic'
        return semantic_results, "semantic (AI captions + labels)"
    
    # Fall back to OCR text search
    ocr_results = search_ocr_text(query, limit)
    for result in ocr_results:
        result['search_method'] = 'text'
    
    return ocr_results, "text (OCR)"

# Robust database corruption handling
@handle_db_corruption
def handle_db_corruption(func):
    """Decorator to handle database corruption gracefully."""
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except sqlite3.DatabaseError as e:
            if ("malformed" in str(e).lower() or 
                "corrupt" in str(e).lower() or 
                "invalid fts5 file format" in str(e).lower()):
                logger.warning(f"Database corruption detected: {e}")
                logger.warning("Rebuilding database...")
                rebuild_database()
                # Retry the operation once after rebuild
                return func(*args, **kwargs)
            else:
                raise
    return wrapper

def encode_image_base64(image_path: Path) -> str:
    """Encode and optimize image for Ollama processing."""
    with Image.open(image_path) as img:
        # Convert to RGB and resize for efficiency
        if img.mode != 'RGB':
            img = img.convert('RGB')
        
        # Resize if too large (max 1024x1024 for most models)
        max_size = 1024
        if img.width > max_size or img.height > max_size:
            img.thumbnail((max_size, max_size), Image.Resampling.LANCZOS)
        
        # Convert to base64
        buffer = io.BytesIO()
        img.save(buffer, format='JPEG', quality=85)
        image_bytes = buffer.getvalue()
        
    return base64.b64encode(image_bytes).decode('utf-8')

def generate_caption_ollama(image_path: Path, model: str = "qwen2.5vl") -> Tuple[str, float]:
    """Generate AI caption using Ollama vision models."""
    try:
        # Load data extraction prompt from external file
        prompt = load_prompt_template("data_extraction")
        image_b64 = encode_image_base64(image_path)
        
        response = ollama.generate(
            model=model,
            prompt=prompt,
            images=[image_b64],
            options={'temperature': 0.1}  # Low temp for consistency
        )
        
        caption = response['response'].strip()
        
        # Try to parse as JSON for structured data extraction
        try:
            data = json.loads(caption)
            # Convert structured data back to searchable text
            caption = f"{data.get('description', '')} {' '.join(data.get('text_elements', []))}"
        except json.JSONDecodeError:
            # Fallback to raw caption
            pass
            
        return caption, 0.8  # High confidence for local models
        
    except Exception as e:
        logger.error(f"Ollama captioning failed: {e}")
        return "", 0.0

def get_file_stats(file_path: Path) -> Tuple[int, float, str]:
    """Get file size, modification time, and hash for change detection."""
    stat = file_path.stat()
    size = stat.st_size
    mtime = stat.st_mtime
    
    # Calculate hash for duplicate detection
    hasher = hashlib.sha256()
    with open(file_path, 'rb') as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hasher.update(chunk)
    
    return size, mtime, hasher.hexdigest()

@handle_db_corruption
def should_process_file(file_path: Path) -> bool:
    """Check if file needs processing based on hash and mtime."""
    size, mtime, new_hash = get_file_stats(file_path)
    
    existing_file = get_file_by_path(str(file_path))
    if not existing_file:
        return True  # New file
        
    # Check if file changed
    return (existing_file['hash'] != new_hash or 
            existing_file['mtime'] != mtime)

def parallel_extract_features(image_path: Path, config: dict) -> dict:
    """Extract all features in parallel for better performance."""
    with ThreadPoolExecutor(max_workers=4) as executor:
        # Submit all extraction tasks
        futures = {}
        
        if config.get('enable_ocr', True):
            futures['ocr'] = executor.submit(extract_text, image_path)
            
        if config.get('enable_color', True):
            futures['color'] = executor.submit(extract_color_histogram, image_path)
            
        if config.get('enable_ai', True):
            futures['caption'] = executor.submit(generate_caption, image_path)
            
        if config.get('enable_clip', True):
            futures['embedding'] = executor.submit(extract_image_embedding, image_path)
        
        # Collect results as they complete
        results = {}
        for feature, future in futures.items():
            try:
                results[feature] = future.result(timeout=30)
            except Exception as e:
                logger.warning(f"Feature extraction failed for {feature}: {e}")
                results[feature] = None
                
    return results

# Global model cache to avoid repeated loading
_clip_model = None
_clip_preprocess = None

def load_clip_model(model_name: str = "ViT-B-32") -> Tuple[Any, Any]:
    """Load CLIP model with caching and device optimization."""
    global _clip_model, _clip_preprocess
    
    if _clip_model is not None and _clip_preprocess is not None:
        return _clip_model, _clip_preprocess
    
    import open_clip
    import torch
    
    # Load model and preprocessing
    model, _, preprocess = open_clip.create_model_and_transforms(
        model_name, 
        pretrained="openai"
    )
    
    # Set to evaluation mode
    model.eval()
    
    # Move to best available device
    device = "mps" if torch.backends.mps.is_available() else "cpu"
    model = model.to(device)
    
    # Cache globally for reuse
    _clip_model = model
    _clip_preprocess = preprocess
    
    return model, preprocess

def extract_image_embedding(image_path: Path) -> Optional[np.ndarray]:
    """Extract CLIP embedding with cached model."""
    model, preprocess = load_clip_model()  # Uses cached model
    
    with torch.no_grad():
        image = preprocess(Image.open(image_path)).unsqueeze(0)
        features = model.encode_image(image)
        # L2 normalize for cosine similarity
        features = features / features.norm(dim=-1, keepdim=True)
    
    return features.cpu().numpy().flatten()

def extract_color_histogram(image_path: Path) -> Tuple[np.ndarray, str]:
    """Extract HSV color histogram for color-based search."""
    img = cv2.imread(str(image_path))
    if img is None:
        return np.zeros(27), "#000000"  # 3x3x3 HSV bins
    
    # Convert BGR to HSV for perceptually uniform color space
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    
    # Calculate 3D histogram (3x3x3 bins for H,S,V)
    hist = cv2.calcHist(
        [hsv], 
        [0, 1, 2],  # H, S, V channels
        None, 
        [3, 3, 3],  # 3 bins each for H, S, V
        [0, 180, 0, 256, 0, 256]  # ranges
    )
    
    # Flatten and normalize to probability distribution
    hist_vector = hist.flatten()
    hist_vector = hist_vector / (hist_vector.sum() + 1e-7)
    
    # Extract dominant color using K-means
    dominant_color = get_dominant_color_kmeans(img)
    
    return hist_vector, dominant_color

def detect_color_words(query: str) -> List[str]:
    """Detect color terms in search queries."""
    color_patterns = {
        'red': ['red', 'crimson', 'scarlet'],
        'blue': ['blue', 'navy', 'azure', 'cyan'],
        'green': ['green', 'lime', 'forest', 'mint'],
        'yellow': ['yellow', 'gold', 'amber'],
        'orange': ['orange', 'coral', 'peach'],
        'purple': ['purple', 'violet', 'magenta'],
        'black': ['black', 'dark'],
        'white': ['white', 'light'],
        'gray': ['gray', 'grey', 'silver']
    }
    
    detected_colors = []
    query_lower = query.lower()
    
    for base_color, variants in color_patterns.items():
        if any(variant in query_lower for variant in variants):
            detected_colors.append(base_color)
    
    return detected_colors