第六篇 文档处理与数据清洗：从非结构化到结构化#

from langchain_community.document_loaders import WebBaseLoader

# 1. 加载单个网页
loader = WebBaseLoader("https://python.langchain.com/docs/get_started/introduction")
docs = loader.load()

print(f"Loaded {len(docs)} docs")
print(f"Content preview: {docs[0].page_content[:200]}")
print(f"Metadata: {docs[0].metadata}")

# 2. 并行加载多个网页
loader_multi = WebBaseLoader([
    "https://www.google.com",
    "https://www.baidu.com"
])
loader_multi.requests_per_second = 2  # 限流
docs_multi = loader_multi.aload() # 异步加载

from langchain_community.document_loaders import DirectoryLoader
from langchain_community.document_loaders import TextLoader

# 加载指定目录下所有的 .md 文件
loader = DirectoryLoader(
    './knowledge_base',
    glob="**/*.md",
    loader_cls=TextLoader,
    show_progress=True
)
docs = loader.load()

from llama_index.core import SimpleDirectoryReader

# 1. 基础用法：加载目录
reader = SimpleDirectoryReader(
    input_dir="./data",
    recursive=True,         # 递归子目录
    required_exts=[".pdf", ".docx"], # 指定后缀
    filename_as_id=True     # 使用文件名做ID
)
documents = reader.load_data()

print(f"Loaded {len(documents)} docs")

# 2. 自定义特定的加载器 (例如对 .pdf 使用特殊的解析逻辑)
from llama_index.readers.file import PDFReader
file_extractor = {".pdf": PDFReader()}

reader_custom = SimpleDirectoryReader(
    "./data",
    file_extractor=file_extractor
)

# 例如：加载 Notion 数据
from llama_index.readers.notion import NotionPageReader

# 需要先在 Notion 申请 Integration Token
reader = NotionPageReader(integration_token="secret_xxx")
documents = reader.load_data(page_ids=["page_id_1", "page_id_2"])

from langchain_text_splitters import RecursiveCharacterTextSplitter

text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,      # 块大小
    chunk_overlap=200,    # 重叠部分，防止上下文丢失
    separators=["\n\n", "\n", " ", ""] # 优先级从左到右
)

splits = text_splitter.split_documents(docs)

from llama_index.core.node_parser import SentenceSplitter

# 默认分块器
splitter = SentenceSplitter(
    chunk_size=1024,
    chunk_overlap=20

)

nodes = splitter.get_nodes_from_documents(documents)

# 查看生成的 Node
print(f"生成了 {len(nodes)} 个节点")
print(nodes[0].get_content()) # 内容
print(nodes[0].relationships) # 关系（上一块、下一块的ID）

from llama_index.core.node_parser import HierarchicalNodeParser

node_parser = HierarchicalNodeParser.from_defaults(
    chunk_sizes=[2048, 512, 128] # 三层切分
)
nodes = node_parser.get_nodes_from_documents(documents)
# 检索时，如果命中 128 的块，可以自动回溯到 2048 的父块内容

from langchain_core.pydantic_v1 import BaseModel, Field
from langchain_openai import ChatOpenAI

# 1. 定义元数据结构
class DocumentMetadata(BaseModel):
    title: str = Field(..., description="The title of the document")
    summary: str = Field(..., description="A one-sentence summary")
    tags: list[str] = Field(..., description="Keywords for categorization")
    sentiment: str = Field(..., description="Sentiment: positive, negative, or neutral")

# 2. 配置 LLM
llm = ChatOpenAI(model="gpt-4", temperature=0)
structured_llm = llm.with_structured_output(DocumentMetadata)

# 3. 提取函数
def extract_metadata(text_chunk):
    return structured_llm.invoke(text_chunk)

# 测试
sample_text = """
LlamaIndex v0.10 发布了！这次更新带来了重大的架构变革，
将库拆分为 llama-index-core 和各种插件包。用户现在可以按需安装依赖，
大大减小了包体积。社区对此反应热烈。
"""

metadata = extract_metadata(sample_text)
print(metadata.json(indent=2))
# 输出:
# {
#   "title": "LlamaIndex v0.10 Release",
#   "summary": "LlamaIndex v0.10 introduces architectural changes splitting the core library from plugins for optimized dependency management.",
#   "tags": ["LlamaIndex", "Release", "Python", "Architecture"],
#   "sentiment": "positive"
# }

from llama_index.core.ingestion import IngestionPipeline
from llama_index.core.extractors import TitleExtractor, SummaryExtractor

pipeline = IngestionPipeline(
    transformations=[
        SentenceSplitter(chunk_size=1024, chunk_overlap=200),
        TitleExtractor(nodes=5),    #利用前5个节点生成标题
        SummaryExtractor(summaries=["prev", "self", "next"]), # 生成摘要
    ]
)

nodes = pipeline.run(documents=documents)
# nodes[0].metadata 现在会自动包含 'document_title', 'section_summary' 等字段

# pip install llama-parse
import os
from llama_parse import LlamaParse
from llama_index.core import SimpleDirectoryReader

# 配置 API Key (https://cloud.llamaindex.ai/)
os.environ["LLAMA_CLOUD_API_KEY"] = "llx-..."

# 1. 配置解析器
parser = LlamaParse(
    result_type="markdown",  # 输出 Markdown
    verbose=True,
    language="zh",           # 支持中文
    gpt4o_mode=True          # 开启高级多模态识别 (消耗更多 credit)
)

# 2. 结合 SimpleDirectoryReader 使用
file_extractor = {".pdf": parser}
reader = SimpleDirectoryReader(
    "./complex_docs",
    file_extractor=file_extractor
)
documents = reader.load_data()

# 3. 检查结果
print(documents[0].text[:500])
# 你会发现表格被转换为了 Markdown Table 格式：
# | 季度 | 营收 | 利润 |
# |-----|------|-----|
# | Q1  | 100  | 20  |

# 假设已在本地安装 magic-pdf
# pip install magic-pdf[full]

import os

# MinerU 通常通过命令行使用，但也可以封装为 Python 函数
def process_pdf_with_mineru(pdf_path):
    output_dir = os.path.dirname(pdf_path)
    # 调用 CLI (实际生产环境建议使用 API 服务模式)
    os.system(f"magic-pdf -p {pdf_path} -o {output_dir}")

    # MinerU 会生成 .md 文件
    md_path = pdf_path.replace(".pdf", ".md")
    if os.path.exists(md_path):
        with open(md_path, "r") as f:
            return f.read()
    return None

# 读取后的 Markdown 内容可以直接喂给 LangChain/LlamaIndex
markdown_content = process_process_pdf_with_mineru("./paper.pdf")

# 这是一个概念性示例
def process_image(image_bytes):
    from langchain_openai import ChatOpenAI
    from langchain_core.messages import HumanMessage

    chat = ChatOpenAI(model="gpt-4o")

    msg = HumanMessage(content=[
        {"type": "text", "text": "请详细描述这张图片中的图表数据或关键信息。"},
        {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}}
    ])

    response = chat.invoke([msg])
    return response.content