RAG 100% local e privado: LanceDB, Ollama, Qdrant local

# 1. Instalar Ollama + modelo
brew install ollama
ollama pull qwen2.5:14b           # ou llama3.3:70b se tiver VRAM
ollama pull bge-m3                # embedding model

# 2. Python env
uv venv && source .venv/bin/activate
uv pip install docling lancedb sentence-transformers FlagEmbedding

from docling import DocumentConverter
from sentence_transformers import SentenceTransformer
import lancedb

# 1. Parse PDF mantendo estrutura
converter = DocumentConverter()
doc = converter.convert('contrato.pdf')
chunks = doc.export_to_markdown().split('\n\n')  # simplificado

# 2. Embed local (sem API)
embedder = SentenceTransformer('BAAI/bge-m3')
vectors = embedder.encode(chunks)

# 3. Index local em LanceDB
db = lancedb.connect('./vault')
table = db.create_table('docs', data=[
    {'text': c, 'vector': v.tolist()} for c, v in zip(chunks, vectors)
])

# 4. Query
query_vec = embedder.encode('cláusula de rescisão')
results = table.search(query_vec).limit(5).to_list()

# 5. Generate com Ollama local
import ollama
ctx = '\n\n'.join(r['text'] for r in results)
response = ollama.chat(
    model='qwen2.5:14b',
    messages=[{
        'role': 'user',
        'content': f'Contexto:\n{ctx}\n\nPergunta: O que diz a cláusula de rescisão?'
    }]
)
print(response['message']['content'])

# LanceDB suporta hybrid nativamente com FTS + vector
import lancedb
from lancedb.rerankers import LinearCombinationReranker

db = lancedb.connect('./vault')
table = db.open_table('docs')

# Cria índice FTS (BM25-like) sobre 'text'
table.create_fts_index('text')

# Hybrid search
reranker = LinearCombinationReranker(weight=0.7)
results = (
    table.search('cláusula rescisão contratual', query_type='hybrid')
    .vector(embedder.encode('cláusula rescisão contratual').tolist())
    .rerank(reranker=reranker)
    .limit(10)
    .to_list()
)

from FlagEmbedding import FlagReranker

# Roda local — sem API
reranker = FlagReranker('BAAI/bge-reranker-v2-m3', use_fp16=True)

# Pair-wise scoring
pairs = [['cláusula rescisão', candidate['text']] for candidate in results[:50]]
scores = reranker.compute_score(pairs, normalize=True)

# Reorder
reranked = sorted(zip(scores, results), reverse=True)[:5]