import gradio as gr from qdrant_client import QdrantClient from qdrant_client.models import Distance, VectorParams, PointStruct from sentence_transformers import SentenceTransformer from datasets import load_dataset import pandas as pd import os import tqdm import uuid # --- Configurations --- DATASET_ID = "stevenbucaille/semantic-transformers" COLLECTION_NAME = "transformers_code" # We'll use a local directory for Qdrant storage within the Space QDRANT_PATH = "./qdrant_data" # Model must match the one used for embedding generation. # Ideally we read this from the dataset metadata, but let's default to the popular one # or make it configurable if we want. For now, let's assume valid model is available or use a default one # that matches what was likely used (Snowflake/snowflake-arctic-embed-m). # NOTE: The dataset should have "embedding" column. If so, we don't need the model for indexing, ONLY for query. DEFAULT_MODEL = "Snowflake/snowflake-arctic-embed-m" print("Initializing Qdrant Client...") client = QdrantClient(path=QDRANT_PATH) print("Loading Embedding Model for queries...") model = SentenceTransformer(DEFAULT_MODEL, trust_remote_code=True) def initialize_index(progress=gr.Progress()): """ Checks if collection exists. If not, loads dataset and indexes it. """ collections = client.get_collections().collections exists = any(c.name == COLLECTION_NAME for c in collections) if exists: count = client.count(COLLECTION_NAME).count if count > 0: return f"Index already exists with {count} vectors. Ready." # Needs indexing progress(0.1, desc=f"Pulling dataset {DATASET_ID}...") try: ds = load_dataset(DATASET_ID, split="train") df = ds.to_pandas() except Exception as e: return f"Error loading dataset: {e}" if "embedding" not in df.columns: return "Error: Dataset does not contain 'embedding' column." # Remove rows with None embeddings df = df.dropna(subset=["embedding"]) total_vectors = len(df) # Create Collection # Determine vector size from first element sample_vec = df.iloc[0]["embedding"] vec_size = len(sample_vec) client.recreate_collection( collection_name=COLLECTION_NAME, vectors_config=VectorParams(size=vec_size, distance=Distance.COSINE), ) # Upload in batches BATCH_SIZE = 500 points = [] progress(0.2, desc="Indexing vectors...") for idx, row in tqdm.tqdm(df.iterrows(), total=total_vectors): # Create metadata dict (exclude embedding) payload = row.drop("embedding").to_dict() # Point ID: use a UUID based on index or just simple integer index if unique point_id = idx points.append( PointStruct(id=point_id, vector=row["embedding"], payload=payload) ) if len(points) >= BATCH_SIZE: client.upsert(collection_name=COLLECTION_NAME, points=points) points = [] if idx % 5000 == 0: progress( 0.2 + 0.8 * (idx / total_vectors), desc=f"Indexed {idx}/{total_vectors}...", ) # Final batch if points: client.upsert(collection_name=COLLECTION_NAME, points=points) return f"Successfully indexed {total_vectors} chunks." def search_code(query, limit=5): """ Embeds query and searches Qdrant. """ if not query.strip(): return "Please enter a query." # Embed query query_vector = model.encode(query) hits = client.search( collection_name=COLLECTION_NAME, query_vector=query_vector, limit=limit ) results = [] for hit in hits: score = hit.score payload = hit.payload file_path = payload.get("file_path", "Unknown") name = payload.get("name", "Unknown") lines = f"{payload.get('start_line')}-{payload.get('end_line')}" code = payload.get("content", "") results.append((score, file_path, name, lines, code)) return results def format_search_results(results): if isinstance(results, str): return results # Error message html = "" for score, fpath, name, lines, code in results: html += f"""
{code}
{base["code"]}{m["code"]}