第五篇 RAG高级篇 (LlamaIndex)#

# 安装高级组件依赖
pip install llama-index-retrievers-bm25
pip install llama-index-postprocessor-cohere-rerank
pip install llama-index-graph-stores-neo4j
pip install llama-parse

from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, StorageContext
from llama_index.retrievers.bm25 import BM25Retriever
from llama_index.core.retrievers import QueryFusionRetriever
from llama_index.core import Settings

# 1. 准备数据与向量索引
documents = SimpleDirectoryReader("./data").load_data()
vector_index = VectorStoreIndex.from_documents(documents)

# 2. 创建 BM25 检索器 (基于关键词)
# 注意：BM25 需要 docstore 来构建倒排索引
bm25_retriever = BM25Retriever.from_defaults(
    docstore=vector_index.docstore,
    similarity_top_k=5
)

# 3. 创建 Vector 检索器 (基于语义)
vector_retriever = vector_index.as_retriever(similarity_top_k=5)

# 4. 创建融合检索器 (Hybrid)
hybrid_retriever = QueryFusionRetriever(
    retrievers=[vector_retriever, bm25_retriever],
    num_queries=1,  # 不生成扩展查询，仅融合当前结果
    mode="reciprocal_rerank",  # 使用 RRF (倒数排名融合) 算法
    similarity_top_k=5,
    use_async=True
)

# 5. 测试检索
nodes = hybrid_retriever.retrieve("LlamaIndex 的自动合并检索原理是什么？")
for node in nodes:
    print(f"得分: {node.score:.4f} | 内容: {node.text[:50]}...")

from llama_index.core.tools import QueryEngineTool, ToolMetadata
from llama_index.core.query_engine import RouterQueryEngine
from llama_index.core.selectors import LLMSingleSelector
from llama_index.core import SummaryIndex

# 1. 构建两个索引
summary_index = SummaryIndex.from_documents(documents)
vector_index = VectorStoreIndex.from_documents(documents)

# 2. 封装为工具
summary_tool = QueryEngineTool.from_defaults(
    query_engine=summary_index.as_query_engine(response_mode="tree_summarize"),
    description="用于回答关于文档整体摘要、大纲、总结类的问题"
)

vector_tool = QueryEngineTool.from_defaults(
    query_engine=vector_index.as_query_engine(),
    description="用于回答关于文档中具体事实、细节、定义的精确问题"
)

# 3. 构建 Router (大脑)
router_query_engine = RouterQueryEngine(
    selector=LLMSingleSelector.from_defaults(),
    query_engine_tools=[summary_tool, vector_tool],
    verbose=True
)

# 4. 测试自适应路由
response = router_query_engine.query("这篇文章主要讲了什么？")
print(f"Used Tool: {response.metadata['selector_result']}")

from llama_index.core.indices.query.query_transform import HyDEQueryTransform
from llama_index.core.query_engine import TransformQueryEngine

# 1. 定义 HyDE 变换
hyde = HyDEQueryTransform(include_original=True)

# 2. 包装原始查询引擎
vector_query_engine = vector_index.as_query_engine()
hyde_query_engine = TransformQueryEngine(vector_query_engine, query_transform=hyde)

# 3. 查询
response = hyde_query_engine.query("如何提高检索的准确率？")

from llama_index.postprocessor.cohere_rerank import CohereRerank

# 1.配置 API Key
# os.environ["COHERE_API_KEY"] = "your_key"

# 2. 定义 Reranker
cohere_rerank = CohereRerank(
    model="rerank-english-v3.0",
    top_n=3  # 最终只保留前3名
)

# 3. 注入到查询引擎 (作为 Postprocessor)
query_engine = vector_index.as_query_engine(
    similarity_top_k=20,  # 第一步：先宽泛召回20个
    node_postprocessors=[cohere_rerank]  # 第二步：精排选出3个
)

response = query_engine.query("LlamaIndex 和 LangChain 的区别？")

# 查看重排后的得分
for node in response.source_nodes:
    print(f"Rerank Score: {node.score:.4f} - {node.node.get_text()[:30]}...")

from llama_index.core.memory import ChatMemoryBuffer

# 1. 设置记忆缓冲区 (Token 限制)
memory = ChatMemoryBuffer.from_defaults(token_limit=3000)

# 2. 创建聊天引擎
chat_engine = vector_index.as_chat_engine(
    chat_mode="condense_plus_context",
    memory=memory,
    similarity_top_k=3,
    verbose=True  # 开启后可以看到"重写后的查询"是什么
)

# 3. 多轮对话
response = chat_engine.chat("LlamaIndex 有哪些索引类型？")
print(response)

# 下一个问题隐含了上下文 ("它们")
response = chat_engine.chat("它们之间有什么区别？")
# 内部会重写为: "LlamaIndex 的不同索引类型之间有什么区别？"
print(response)

from llama_index.core import PropertyGraphIndex
from llama_index.core.indices.property_graph import (
    ImplicitPathExtractor,
    SimpleLLMPathExtractor
)

# 1. 定义提取器 (如何从文本变图)
# 自动提取实体和关系，如 (Steve Jobs)-[FOUNDED]->(Apple)
kg_extractor = SimpleLLMPathExtractor(
    llm=Settings.llm,
    max_paths_per_chunk=10,
    num_workers=4
)

# 2. 构建图索引 (同时生成 Embedding)
graph_index = PropertyGraphIndex.from_documents(
    documents,
    kg_extractors=[kg_extractor],
    embed_kg_nodes=True,  # 关键：开启向量化，支持混合检索
    show_progress=True
)

# 3. 混合检索查询 (Graph + Vector)
# 既能匹配关键词实体，通过图游走找到关系，也能通过向量匹配语义
query_engine = graph_index.as_query_engine(
    include_text=True,
    similarity_top_k=5
)

response = query_engine.query("Steve Jobs 和 Pixar 是什么关系？")

from llama_index.core.tools import QueryEngineTool, ToolMetadata
from llama_index.core.agent import FunctionCallingAgentWorker, AgentRunner

# 1. 将 QueryEngine 包装为 Tool
rag_tool = QueryEngineTool(
    query_engine=vector_index.as_query_engine(),
    metadata=ToolMetadata(
        name="company_wiki",
        description="用于查询公司内部规章制度、报销流程、假期政策等。"
    )
)

# 2. 创建 Agent
agent_worker = FunctionCallingAgentWorker.from_tools(
    tools=[rag_tool],
    llm=Settings.llm,
    verbose=True
)
agent = AgentRunner(agent_worker)

# 3. 复杂任务 (ReAct 模式)
# 用户问："我下周想请年假，流程是什么？如果我还有5天年假，够吗？"
# Agent 会：
# 1. 调用 company_wiki 查"请假流程"
# 2. 这里演示的是单步，如果接了数据库工具，它还能去查"剩余年假"
# 3. 综合回答
response = agent.chat("请假流程怎么走？")
print(response)