传统CV              多模态基础           视觉大模型              未来方向
  |                    |                    |                      |
ImageNet          CLIP(2021)         Florence-2(2024)      视觉AGI
ResNet            BLIP(2022)         GPT-4V(2023)          具身智能
Detection    -->  LLaVA(2023)   -->  Gemini(2024)    -->   世界模型
Segmentation      BLIP-2(2023)       GPT-4o(2024)          多模态推理

# 核心库
pip install transformers>=4.35.0 torch torchvision
pip install accelerate bitsandbytes  # 量化加速
pip install pillow requests datasets

# API客户端
pip install openai>=1.0.0
pip install google-generativeai
pip install anthropic

# 3D/视频
pip install open3d trimesh
pip install opencv-python decord

Image Input                Text Input
     |                          |
Image Encoder              Text Encoder
(ViT-L/14)                (Transformer)
     |                          |
  Image                      Text
Embedding                 Embedding
     |                          |
     +--------Cosine Sim--------+
              (Similarity Score)

from transformers import CLIPProcessor, CLIPModel
import torch
from PIL import Image

# 加载模型(自动下载)
model = CLIPModel.from_pretrained("openai/clip-vit-large-patch14")
processor = CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14")

# 准备输入
image = Image.open("image.jpg")
text_candidates = ["a photo of a cat", "a photo of a dog", "a photo of a bird"]

# 编码
inputs = processor(
    text=text_candidates,
    images=image,
    return_tensors="pt",
    padding=True
)

# 推理
with torch.no_grad():
    outputs = model(**inputs)
    logits_per_image = outputs.logits_per_image  # (1, 3)
    probs = logits_per_image.softmax(dim=1)  # (1, 3)

# 结果
print(f"概率分布: {probs[0].tolist()}")
predicted_label = text_candidates[probs.argmax()]
print(f"预测类别: {predicted_label}")

Frozen Image       Q-Former        Frozen LLM
Encoder          (Learnable)      (OPT-2.7B)
   |                 |                 |
 ViT-g      32个Query Tokens      Language
(1.4B)           (762M)            Model
   |                 |                 |
   +-------Cross Attention---------+
                     |
                Text Output

from transformers import Blip2Processor, Blip2ForConditionalGeneration
import torch

# 加载模型(推荐float16)
processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-2.7b")
model = Blip2ForConditionalGeneration.from_pretrained(
    "Salesforce/blip2-opt-2.7b",
    torch_dtype=torch.float16,
    device_map="auto"
)

# 生成描述
image = Image.open("image.jpg")
inputs = processor(image, return_tensors="pt").to("cuda", torch.float16)

generated_ids = model.generate(**inputs, max_new_tokens=50)
caption = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
print(f"图像描述: {caption}")

# 提问
question = "What is the color of the car?"
inputs = processor(image, question, return_tensors="pt").to("cuda", torch.float16)

# 生成答案
generated_ids = model.generate(**inputs, max_new_tokens=20)
answer = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
print(f"回答: {answer}")

model = Blip2ForConditionalGeneration.from_pretrained(
    "Salesforce/blip2-opt-2.7b",
    load_in_8bit=True,
    device_map="auto"
)
# 显存: 14GB -> 7GB

from transformers import BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.float16
)

model = Blip2ForConditionalGeneration.from_pretrained(
    "Salesforce/blip2-opt-2.7b",
    quantization_config=quantization_config,
    device_map="auto"
)
# 显存: 14GB -> 3.5GB

Image Input
    |
Vision Encoder ────────> Projection Layer ────> LLM
(CLIP ViT-L/14)         (Linear Layer)      (Vicuna-7B/13B)
    |                         |                    |
768D Embedding  ────>  4096D Embedding  ──> Text Generation

from transformers import AutoProcessor, LlavaForConditionalGeneration
import torch

# 加载模型
model_id = "llava-hf/llava-1.5-7b-hf"
model = LlavaForConditionalGeneration.from_pretrained(
    model_id,
    torch_dtype=torch.float16,
    device_map="auto"
)
processor = AutoProcessor.from_pretrained(model_id)

# 构建对话
conversation = [
    {
        "role": "user",
        "content": [
            {"type": "image"},
            {"type": "text", "text": "描述这张图片中的内容。"}
        ]
    }
]

# 生成回复
prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
inputs = processor(images=image, text=prompt, return_tensors="pt").to("cuda", torch.float16)

output = model.generate(**inputs, max_new_tokens=200)
response = processor.decode(output[0], skip_special_tokens=True)
print(response)

from transformers import BitsAndBytesConfig

# 配置4-bit量化
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.float16
)

# 加载量化模型
model = LlavaForConditionalGeneration.from_pretrained(
    "llava-hf/llava-1.5-7b-hf",
    quantization_config=bnb_config,
    device_map="auto"
)
# 显存: 28GB -> 5GB (减少82%)

# 1. 使用CLIP构建图像索引
image_features = model.get_image_features(pixel_values=images)
# 存入向量数据库(FAISS/Milvus)

# 2. 查询时编码并检索
query_features = model.get_image_features(pixel_values=query_image)
similar_indices = faiss_index.search(query_features, k=10)

# 3. BLIP-2生成商品描述
description = blip2_model.generate(query_image)

# 使用LLaVA处理多模态输入
conversation = [
    {
        "role": "user",
        "content": [
            {"type": "image"},
            {"type": "text", "text": "这个商品如何使用?"}
        ]
    }
]
response = llava_model.chat(conversation)

# 1. CLIP快速初筛(零样本)
labels = ["正常内容", "暴力内容", "色情内容", "政治敏感"]
probs = clip_classify(image, labels)

# 2. 高置信度直接通过/拒绝
if probs.max() > 0.95:
    return probs.argmax()

# 3. 低置信度用LLaVA详细分析
analysis = llava_analyze(image, "请详细分析这张图片是否包含不适内容。")

# CLIP批量推理(提升10x)
inputs = processor(text=texts, images=images, return_tensors="pt", padding=True)
outputs = model(**inputs)

# 预计算文本特征(类别固定时)
with torch.no_grad():
    text_features = model.get_text_features(**text_inputs)
    # 缓存,后续只需计算图像特征

# 使用torch.autocast加速
with torch.autocast(device_type='cuda', dtype=torch.float16):
    outputs = model(**inputs)

from transformers import AutoProcessor, AutoModelForCausalLM
import torch
from PIL import Image

# 加载模型(推荐使用large-ft版本)
model_id = "microsoft/Florence-2-large-ft"
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.float16,
    trust_remote_code=True
).to("cuda")

processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)

def run_florence(image, task_prompt, text_input=None):
    """通用Florence-2推理函数"""
    if text_input:
        prompt = task_prompt + text_input
    else:
        prompt = task_prompt

    inputs = processor(text=prompt, images=image, return_tensors="pt").to("cuda", torch.float16)

    generated_ids = model.generate(
        input_ids=inputs["input_ids"],
        pixel_values=inputs["pixel_values"],
        max_new_tokens=1024,
        num_beams=3
    )

    generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
    parsed_result = processor.post_process_generation(
        generated_text,
        task=task_prompt,
        image_size=(image.width, image.height)
    )
    return parsed_result

image = Image.open("example.jpg")

# 1. 图像描述
caption = run_florence(image, "<CAPTION>")
print(f"基础描述: {caption}")
# 输出: {'<CAPTION>': 'A cat sitting on a red sofa in a living room.'}

detailed = run_florence(image, "<DETAILED_CAPTION>")
print(f"详细描述: {detailed}")

# 2. 目标检测
detection = run_florence(image, "<OD>")
print(f"检测结果: {detection}")
# 输出: {'<OD>': {'bboxes': [[x1, y1, x2, y2], ...], 'labels': ['cat', 'sofa', ...]}}

# 3. OCR文字识别
ocr_result = run_florence(image, "<OCR>")
print(f"识别文字: {ocr_result}")

# 4. OCR带位置
ocr_with_region = run_florence(image, "<OCR_WITH_REGION>")
print(f"文字+位置: {ocr_with_region}")

# 5. 短语定位(Grounding)
grounding = run_florence(image, "<CAPTION_TO_PHRASE_GROUNDING>", "a cat")
print(f"定位结果: {grounding}")

# 6. 开放词汇检测
open_det = run_florence(image, "<OPEN_VOCABULARY_DETECTION>", "cat, dog, person")
print(f"开放检测: {open_det}")

# 7. 区域级描述
dense_caption = run_florence(image, "<DENSE_REGION_CAPTION>")
print(f"区域描述: {dense_caption}")

import matplotlib.pyplot as plt
import matplotlib.patches as patches

def visualize_detection(image, result, task="<OD>"):
    """可视化检测结果"""
    fig, ax = plt.subplots(1, figsize=(12, 8))
    ax.imshow(image)

    data = result[task]
    bboxes = data.get('bboxes', [])
    labels = data.get('labels', [])

    colors = plt.cm.Set3(range(len(bboxes)))

    for bbox, label, color in zip(bboxes, labels, colors):
        x1, y1, x2, y2 = bbox
        rect = patches.Rectangle(
            (x1, y1), x2-x1, y2-y1,
            linewidth=2, edgecolor=color, facecolor='none'
        )
        ax.add_patch(rect)
        ax.text(x1, y1-5, label, fontsize=10, color='white',
                bbox=dict(boxstyle='round', facecolor=color, alpha=0.8))

    ax.axis('off')
    plt.tight_layout()
    plt.savefig('detection_result.png', dpi=150, bbox_inches='tight')
    plt.show()

# 使用
detection_result = run_florence(image, "<OD>")
visualize_detection(image, detection_result)

def process_document(image_path):
    """文档智能处理流程"""
    image = Image.open(image_path)

    # OCR识别
    text = run_florence(image, "<OCR>")

    # 带位置的OCR(用于表格等)
    ocr_regions = run_florence(image, "<OCR_WITH_REGION>")

    # 图表/图像检测
    objects = run_florence(image, "<OD>")

    return {
        "text": text,
        "regions": ocr_regions,
        "objects": objects
    }

def analyze_product_image(image_path):
    """电商商品图分析"""
    image = Image.open(image_path)

    # 商品描述
    description = run_florence(image, "<DETAILED_CAPTION>")

    # 检测商品主体
    detection = run_florence(image, "<OD>")

    # 提取商品上的文字(品牌、规格等)
    text_info = run_florence(image, "<OCR>")

    return {
        "description": description,
        "objects": detection,
        "text": text_info
    }

from openai import OpenAI
import os

client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))

def analyze_image_url(image_url, question):
    """使用URL分析图像"""
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": question},
                    {
                        "type": "image_url",
                        "image_url": {"url": image_url}
                    }
                ]
            }
        ],
        max_tokens=500
    )
    return response.choices[0].message.content

# 使用示例
result = analyze_image_url(
    "https://example.com/image.jpg",
    "请详细描述这张图片中的内容"
)
print(result)

import base64

def encode_image(image_path):
    """将本地图像编码为base64"""
    with open(image_path, "rb") as f:
        return base64.b64encode(f.read()).decode("utf-8")

def analyze_local_image(image_path, question):
    """分析本地图像"""
    base64_image = encode_image(image_path)

    # 自动检测图像格式
    if image_path.lower().endswith('.png'):
        media_type = "image/png"
    elif image_path.lower().endswith('.gif'):
        media_type = "image/gif"
    elif image_path.lower().endswith('.webp'):
        media_type = "image/webp"
    else:
        media_type = "image/jpeg"

    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": question},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": f"data:{media_type};base64,{base64_image}"
                        }
                    }
                ]
            }
        ],
        max_tokens=1000
    )
    return response.choices[0].message.content

# 使用
result = analyze_local_image("product.jpg", "这个商品的主要特点是什么?")

def compare_images(image_paths, question):
    """多图像对比分析"""
    content = [{"type": "text", "text": question}]

    for path in image_paths:
        base64_image = encode_image(path)
        content.append({
            "type": "image_url",
            "image_url": {
                "url": f"data:image/jpeg;base64,{base64_image}"
            }
        })

    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{"role": "user", "content": content}],
        max_tokens=1500
    )
    return response.choices[0].message.content

# 对比两张图片
result = compare_images(
    ["before.jpg", "after.jpg"],
    "请对比这两张图片的差异,描述发生了什么变化"
)

def analyze_with_detail(image_path, question, detail="auto"):
    """
    控制图像分析精度
    detail参数:
    - "low": 512x512固定,65 tokens,快速便宜
    - "high": 最高2048x2048,详细分析,更多tokens
    - "auto": 自动选择(默认)
    """
    base64_image = encode_image(image_path)

    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": question},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": f"data:image/jpeg;base64,{base64_image}",
                            "detail": detail  # "low", "high", "auto"
                        }
                    }
                ]
            }
        ],
        max_tokens=1000
    )
    return response.choices[0].message.content

# 快速预览(省钱)
quick_result = analyze_with_detail("doc.jpg", "这是什么文档?", detail="low")

# 详细分析(精确)
detailed_result = analyze_with_detail("doc.jpg", "请提取文档中的所有文字", detail="high")

import json

def extract_structured_info(image_path, schema_description):
    """提取结构化信息"""
    base64_image = encode_image(image_path)

    prompt = f"""分析这张图片,按照以下格式返回JSON:
{schema_description}

只返回JSON,不要其他内容。"""

    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": prompt},
                    {
                        "type": "image_url",
                        "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}
                    }
                ]
            }
        ],
        max_tokens=1000,
        response_format={"type": "json_object"}  # 强制JSON输出
    )

    return json.loads(response.choices[0].message.content)

# 提取商品信息
schema = """
{
    "product_name": "商品名称",
    "brand": "品牌",
    "price": "价格(如果可见)",
    "features": ["特点1", "特点2"],
    "category": "类别"
}
"""
product_info = extract_structured_info("product.jpg", schema)

class GPT4VisionOptimizer:
    """GPT-4o Vision成本优化器"""

    def __init__(self):
        self.client = OpenAI()

    def preprocess_image(self, image_path, max_size=1024):
        """预处理图像以减少tokens"""
        from PIL import Image

        img = Image.open(image_path)

        # 调整大小
        if max(img.size) > max_size:
            ratio = max_size / max(img.size)
            new_size = (int(img.width * ratio), int(img.height * ratio))
            img = img.resize(new_size, Image.LANCZOS)

        # 转换为JPEG(通常更小)
        import io
        buffer = io.BytesIO()
        img.convert('RGB').save(buffer, format='JPEG', quality=85)
        return base64.b64encode(buffer.getvalue()).decode('utf-8')

    def smart_analyze(self, image_path, question, use_mini=True):
        """智能分析,根据任务选择模型"""
        # 简单任务用mini
        model = "gpt-4o-mini" if use_mini else "gpt-4o"
        base64_image = self.preprocess_image(image_path)

        response = self.client.chat.completions.create(
            model=model,
            messages=[
                {
                    "role": "user",
                    "content": [
                        {"type": "text", "text": question},
                        {
                            "type": "image_url",
                            "image_url": {
                                "url": f"data:image/jpeg;base64,{base64_image}",
                                "detail": "low"  # 先用低精度
                            }
                        }
                    ]
                }
            ],
            max_tokens=300
        )
        return response.choices[0].message.content

# 使用
optimizer = GPT4VisionOptimizer()
result = optimizer.smart_analyze("image.jpg", "图片中有什么?", use_mini=True)

pip install google-generativeai

from google import genai
from google.genai import types
import os

# 初始化客户端
client = genai.Client(api_key=os.environ.get("GOOGLE_API_KEY"))

def analyze_image_gemini(image_path, prompt):
    """使用Gemini分析图像"""
    with open(image_path, 'rb') as f:
        image_bytes = f.read()

    # 检测MIME类型
    if image_path.lower().endswith('.png'):
        mime_type = 'image/png'
    elif image_path.lower().endswith('.webp'):
        mime_type = 'image/webp'
    else:
        mime_type = 'image/jpeg'

    response = client.models.generate_content(
        model='gemini-2.5-flash',
        contents=[
            types.Part.from_bytes(data=image_bytes, mime_type=mime_type),
            prompt
        ]
    )
    return response.text

# 使用
result = analyze_image_gemini("photo.jpg", "描述这张照片中的场景")
print(result)

def analyze_large_image(image_path, prompt):
    """使用File API处理大文件(推荐)"""
    # 上传文件
    uploaded_file = client.files.upload(file=image_path)

    # 等待处理完成
    import time
    while uploaded_file.state.name == "PROCESSING":
        time.sleep(1)
        uploaded_file = client.files.get(name=uploaded_file.name)

    # 生成内容
    response = client.models.generate_content(
        model="gemini-2.5-flash",
        contents=[uploaded_file, prompt]
    )

    # 可选:删除上传的文件
    # client.files.delete(name=uploaded_file.name)

    return response.text

# 使用
result = analyze_large_image("high_res_image.jpg", "详细分析这张图片")

def compare_images_gemini(image_paths, prompt):
    """多图像对比分析"""
    contents = []

    for path in image_paths:
        with open(path, 'rb') as f:
            image_bytes = f.read()
        contents.append(types.Part.from_bytes(data=image_bytes, mime_type='image/jpeg'))

    contents.append(prompt)

    response = client.models.generate_content(
        model='gemini-2.5-flash',
        contents=contents
    )
    return response.text

# 使用
result = compare_images_gemini(
    ["img1.jpg", "img2.jpg", "img3.jpg"],
    "比较这三张图片的异同点"
)

def analyze_video(video_path, prompt):
    """分析视频内容"""
    # 上传视频
    video_file = client.files.upload(file=video_path)

    # 等待处理
    import time
    while video_file.state.name == "PROCESSING":
        time.sleep(2)
        video_file = client.files.get(name=video_file.name)

    if video_file.state.name == "FAILED":
        raise ValueError("视频处理失败")

    # 分析视频
    response = client.models.generate_content(
        model="gemini-2.5-flash",
        contents=[video_file, prompt]
    )
    return response.text

# 使用
result = analyze_video("demo.mp4", "总结这个视频的主要内容")
print(result)

def query_video_timestamp(video_path, question):
    """查询视频特定时间点的内容"""
    video_file = client.files.upload(file=video_path)

    # 等待处理
    import time
    while video_file.state.name == "PROCESSING":
        time.sleep(2)
        video_file = client.files.get(name=video_file.name)

    prompt = f"""观看这个视频并回答问题。
如果问题涉及特定场景,请指出大概的时间点。

问题: {question}"""

    response = client.models.generate_content(
        model="gemini-2.5-flash",
        contents=[video_file, prompt]
    )
    return response.text

# 使用
result = query_video_timestamp("lecture.mp4", "讲师什么时候开始讲解神经网络?")

def detect_objects_gemini(image_path, objects_to_detect):
    """使用Gemini进行目标检测"""
    with open(image_path, 'rb') as f:
        image_bytes = f.read()

    prompt = f"""检测图片中的以下物体: {objects_to_detect}

返回每个检测到的物体的边界框坐标,格式为:
物体名称: [x_min, y_min, x_max, y_max] (归一化到0-1000)"""

    response = client.models.generate_content(
        model='gemini-2.5-flash',
        contents=[
            types.Part.from_bytes(data=image_bytes, mime_type='image/jpeg'),
            prompt
        ]
    )
    return response.text

# 使用
result = detect_objects_gemini("street.jpg", "人, 车, 红绿灯")

from abc import ABC, abstractmethod
from PIL import Image
import base64
import io

class VLMInterface(ABC):
    """VLM统一接口"""

    @abstractmethod
    def analyze(self, image_path: str, prompt: str) -> str:
        pass

    def _encode_image(self, image_path: str) -> str:
        with open(image_path, 'rb') as f:
            return base64.b64encode(f.read()).decode('utf-8')

class GPT4oVLM(VLMInterface):
    def __init__(self, api_key: str):
        from openai import OpenAI
        self.client = OpenAI(api_key=api_key)

    def analyze(self, image_path: str, prompt: str) -> str:
        base64_image = self._encode_image(image_path)
        response = self.client.chat.completions.create(
            model="gpt-4o",
            messages=[{
                "role": "user",
                "content": [
                    {"type": "text", "text": prompt},
                    {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}}
                ]
            }],
            max_tokens=1000
        )
        return response.choices[0].message.content

class GeminiVLM(VLMInterface):
    def __init__(self, api_key: str):
        from google import genai
        self.client = genai.Client(api_key=api_key)

    def analyze(self, image_path: str, prompt: str) -> str:
        from google.genai import types
        with open(image_path, 'rb') as f:
            image_bytes = f.read()
        response = self.client.models.generate_content(
            model='gemini-2.5-flash',
            contents=[types.Part.from_bytes(data=image_bytes, mime_type='image/jpeg'), prompt]
        )
        return response.text

class Florence2VLM(VLMInterface):
    def __init__(self):
        from transformers import AutoProcessor, AutoModelForCausalLM
        import torch
        self.model = AutoModelForCausalLM.from_pretrained(
            "microsoft/Florence-2-large-ft",
            torch_dtype=torch.float16,
            trust_remote_code=True
        ).to("cuda")
        self.processor = AutoProcessor.from_pretrained(
            "microsoft/Florence-2-large-ft",
            trust_remote_code=True
        )

    def analyze(self, image_path: str, prompt: str) -> str:
        image = Image.open(image_path)
        inputs = self.processor(text=prompt, images=image, return_tensors="pt").to("cuda")
        output = self.model.generate(**inputs, max_new_tokens=1024)
        return self.processor.batch_decode(output, skip_special_tokens=True)[0]

# 使用示例
def analyze_with_fallback(image_path, prompt, vlm_list):
    """带fallback的分析"""
    for vlm in vlm_list:
        try:
            return vlm.analyze(image_path, prompt)
        except Exception as e:
            print(f"{vlm.__class__.__name__} 失败: {e}")
            continue
    raise RuntimeError("所有VLM都失败了")

class DocumentAnalyzer:
    """文档智能分析器"""

    def __init__(self, vlm: VLMInterface):
        self.vlm = vlm

    def extract_text(self, image_path: str) -> str:
        """提取文档中的文字"""
        prompt = "提取图片中的所有文字,保持原有格式和布局"
        return self.vlm.analyze(image_path, prompt)

    def analyze_table(self, image_path: str) -> dict:
        """分析表格内容"""
        prompt = """分析图片中的表格,返回JSON格式:
{
    "headers": ["列1", "列2", ...],
    "rows": [["数据1", "数据2", ...], ...]
}
只返回JSON,不要其他内容。"""
        result = self.vlm.analyze(image_path, prompt)
        import json
        return json.loads(result)

    def summarize_document(self, image_path: str) -> dict:
        """文档摘要"""
        prompt = """分析这份文档,返回JSON格式:
{
    "type": "文档类型(如:发票/合同/报告)",
    "title": "文档标题",
    "date": "日期(如果有)",
    "summary": "主要内容摘要(100字以内)",
    "key_info": ["关键信息1", "关键信息2", ...]
}"""
        result = self.vlm.analyze(image_path, prompt)
        import json
        return json.loads(result)

# 使用
vlm = GPT4oVLM(api_key="your-key")
analyzer = DocumentAnalyzer(vlm)

# 分析发票
invoice_info = analyzer.summarize_document("invoice.jpg")
print(invoice_info)

class VideoAnalyzer:
    """视频智能分析器(仅支持Gemini)"""

    def __init__(self, api_key: str):
        from google import genai
        self.client = genai.Client(api_key=api_key)

    def _upload_video(self, video_path: str):
        """上传并等待视频处理完成"""
        import time
        video_file = self.client.files.upload(file=video_path)
        while video_file.state.name == "PROCESSING":
            time.sleep(2)
            video_file = self.client.files.get(name=video_file.name)
        return video_file

    def generate_summary(self, video_path: str) -> str:
        """生成视频摘要"""
        video_file = self._upload_video(video_path)
        response = self.client.models.generate_content(
            model="gemini-2.5-flash",
            contents=[video_file, "生成这个视频的详细摘要,包括主要内容、关键场景和结论"]
        )
        return response.text

    def extract_key_frames(self, video_path: str) -> list:
        """提取关键帧描述"""
        video_file = self._upload_video(video_path)
        prompt = """分析视频中的关键场景,返回JSON格式:
[
    {"timestamp": "MM:SS", "description": "场景描述"},
    ...
]
只返回JSON列表。"""
        response = self.client.models.generate_content(
            model="gemini-2.5-flash",
            contents=[video_file, prompt]
        )
        import json
        return json.loads(response.text)

    def answer_question(self, video_path: str, question: str) -> str:
        """视频问答"""
        video_file = self._upload_video(video_path)
        response = self.client.models.generate_content(
            model="gemini-2.5-flash",
            contents=[video_file, f"观看视频并回答问题: {question}"]
        )
        return response.text

# 使用
analyzer = VideoAnalyzer(api_key="your-google-key")
summary = analyzer.generate_summary("lecture.mp4")
print(summary)

import time

def benchmark_vlms(image_path: str, prompt: str, vlms: dict) -> dict:
    """VLM性能基准测试"""
    results = {}

    for name, vlm in vlms.items():
        try:
            start = time.time()
            response = vlm.analyze(image_path, prompt)
            latency = time.time() - start

            results[name] = {
                "success": True,
                "latency": round(latency, 2),
                "response_length": len(response),
                "response_preview": response[:200] + "..."
            }
        except Exception as e:
            results[name] = {
                "success": False,
                "error": str(e)
            }

    return results

# 运行基准测试
vlms = {
    "GPT-4o": GPT4oVLM(api_key="..."),
    "Gemini": GeminiVLM(api_key="..."),
    "Florence-2": Florence2VLM()
}

results = benchmark_vlms("test_image.jpg", "描述这张图片", vlms)
for name, result in results.items():
    print(f"{name}: {result}")

输入: (x, y, z, θ, φ) - 空间位置 + 观察方向
       ↓
    MLP网络
       ↓
输出: (r, g, b, σ) - 颜色 + 密度

C(r) = ∫ T(t) · σ(r(t)) · c(r(t), d) dt

其中:
- T(t): 透射率,光线到达t点的概率
- σ: 体积密度
- c: 颜色
- d: 视角方向

import torch
import torch.nn as nn

class NeRF(nn.Module):
    """简化版NeRF网络"""

    def __init__(self, D=8, W=256, input_ch=63, input_ch_views=27):
        """
        Args:
            D: 网络深度
            W: 隐藏层宽度
            input_ch: 位置编码后的位置维度
            input_ch_views: 位置编码后的方向维度
        """
        super().__init__()
        self.D = D
        self.W = W
        self.input_ch = input_ch
        self.input_ch_views = input_ch_views

        # 位置编码后的位置输入 -> 特征
        self.pts_linears = nn.ModuleList(
            [nn.Linear(input_ch, W)] +
            [nn.Linear(W, W) if i != 4 else nn.Linear(W + input_ch, W)
             for i in range(D - 1)]
        )

        # 方向相关的颜色预测
        self.views_linears = nn.ModuleList([nn.Linear(input_ch_views + W, W // 2)])

        # 输出层
        self.feature_linear = nn.Linear(W, W)
        self.alpha_linear = nn.Linear(W, 1)  # 密度
        self.rgb_linear = nn.Linear(W // 2, 3)  # 颜色

    def forward(self, x):
        # 分离位置和方向
        input_pts, input_views = torch.split(
            x, [self.input_ch, self.input_ch_views], dim=-1
        )

        h = input_pts
        for i, layer in enumerate(self.pts_linears):
            h = layer(h)
            h = torch.relu(h)
            if i == 4:
                h = torch.cat([input_pts, h], -1)

        # 密度输出(与视角无关)
        alpha = self.alpha_linear(h)

        # 颜色输出(与视角相关)
        feature = self.feature_linear(h)
        h = torch.cat([feature, input_views], -1)

        for layer in self.views_linears:
            h = layer(h)
            h = torch.relu(h)

        rgb = torch.sigmoid(self.rgb_linear(h))

        return torch.cat([rgb, alpha], -1)

class PositionalEncoding:
    """位置编码:将低维输入映射到高维空间"""

    def __init__(self, L=10):
        """
        Args:
            L: 编码频率数量
        """
        self.L = L
        self.freq_bands = 2.0 ** torch.linspace(0, L - 1, L)

    def encode(self, x):
        """
        Args:
            x: 输入坐标 [..., C]
        Returns:
            编码后的坐标 [..., C * (2L + 1)]
        """
        out = [x]
        for freq in self.freq_bands:
            out.append(torch.sin(freq * x))
            out.append(torch.cos(freq * x))
        return torch.cat(out, dim=-1)

# 使用示例
pos_encoder = PositionalEncoding(L=10)  # 位置用L=10
dir_encoder = PositionalEncoding(L=4)   # 方向用L=4

# 3D位置: 3 -> 3*(2*10+1) = 63
# 2D方向: 3 -> 3*(2*4+1) = 27