2014-2019: GAN时代
├── 2014: GAN提出 (Goodfellow)
├── 2015: DCGAN - 稳定训练的GAN
├── 2018: StyleGAN - 高质量人脸生成
└── 2019: StyleGAN2 - 生成质量巅峰

2020-至今: Diffusion崛起
├── 2020: DDPM提出 (Ho et al.)
├── 2021: DALL-E (OpenAI)
├── 2022: Stable Diffusion开源
├── 2023: ControlNet、SDXL
├── 2024: Stable Diffusion 3、FLUX.1
└── 2025: 扩散模型持续迭代

# Python >= 3.10
python --version

# GPU要求(强烈建议)
# - VRAM >= 8GB (Stable Diffusion 1.5)
# - VRAM >= 12GB (Stable Diffusion XL)
# - VRAM >= 16GB (FLUX.1)

# 查看GPU信息
nvidia-smi

# GAN相关(第14章)
pip install torch torchvision
pip install matplotlib pillow
pip install tqdm

# Diffusion相关(第15章)
pip install diffusers==0.35.1  # 最新版本(2024-11)
pip install transformers accelerate
pip install safetensors
pip install controlnet-aux  # ControlNet预处理

# 可选:模型下载加速
pip install huggingface-hub
export HF_ENDPOINT=https://hf-mirror.com  # 国内镜像

import torch
import diffusers
from diffusers import DiffusionPipeline

print(f"PyTorch: {torch.__version__}")
print(f"Diffusers: {diffusers.__version__}")
print(f"CUDA可用: {torch.cuda.is_available()}")

if torch.cuda.is_available():
    print(f"GPU: {torch.cuda.get_device_name(0)}")
    print(f"VRAM: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB")

# 低质量提示词
"a cat"

# 高质量提示词
"a fluffy orange cat sitting on a wooden table,
soft natural lighting, shallow depth of field,
bokeh background, photorealistic, 8k uhd,
professional photography"

第一步:跑通Stable Diffusion基础代码
   ↓
第二步:尝试不同的提示词和参数
   ↓
第三步:学习ControlNet可控生成
   ↓
第四步:探索高级技巧(LoRA、IP-Adapter等)
   ↓
第五步:构建自己的图像生成应用

# 使用HuggingFace镜像
export HF_ENDPOINT=https://hf-mirror.com

# 或使用ModelScope
pip install modelscope
# 代码中使用modelscope下载

第五篇:图像分割
       ↓
  (判别式模型的最后一篇)
       ↓
第六篇:生成模型 ← 当前篇
       ↓
  (生成式模型:GAN → Diffusion)
       ↓
第七篇:视觉大模型
       ↓
  (多模态模型:CLIP、BLIP、LLaVA等)

part6_generation/
├── README.md                    # 本文件
├── chapter14/                   # GAN
│   ├── README.md
│   └── code/
│       └── dcgan_generation.py  # DCGAN完整实现
└── chapter15/                   # Diffusion
    ├── README.md
    └── code/
        ├── stable_diffusion_demo.py    # SD基础
        ├── controlnet_demo.py          # ControlNet
        └── utils/
            ├── prompt_templates.py     # 提示词模板
            └── image_utils.py          # 图像处理工具

"""
模块文档字符串:说明功能
"""
import torch
from diffusers import DiffusionPipeline
from typing import List, Optional

def generate_image(
    prompt: str,
    negative_prompt: Optional[str] = None,
    num_inference_steps: int = 50,
    guidance_scale: float = 7.5,
    seed: Optional[int] = None
) -> torch.Tensor:
    """
    生成图像的核心函数

    Args:
        prompt: 正向提示词
        negative_prompt: 负向提示词
        num_inference_steps: 采样步数(越大越慢但质量越好)
        guidance_scale: 引导强度(7.5是常用值)
        seed: 随机种子(可复现结果)

    Returns:
        生成的图像Tensor
    """
    # 实现代码...
    pass

[主体描述] + [细节修饰] + [艺术风格] + [技术参数] + [质量词]

示例:
"a majestic lion standing on a cliff,
golden hour lighting, dramatic clouds,
digital painting, trending on artstation,
highly detailed, 8k uhd, masterpiece"

第1-2天: 第14章 GAN快速回顾
    ├── 理解GAN原理
    ├── 跑通DCGAN代码
    └── 了解GAN的局限性

第3-7天: 第15章 Diffusion深入学习
    ├── Day 3: 理解扩散过程数学原理
    ├── Day 4: Stable Diffusion架构与使用
    ├── Day 5: 提示词工程与实战
    ├── Day 6: ControlNet可控生成
    └── Day 7: FLUX等前沿模型探索

第8天: 综合实战项目
    └── 构建自己的图像生成应用

生成器G: 学习生成逼真的假图像,试图欺骗判别器
判别器D: 学习区分真实图像和生成图像

训练过程:
1. G生成假图像
2. D学习区分真假(真实数据标记为1,假数据标记为0)
3. G根据D的反馈改进,生成更逼真的图像
4. D也在改进,提高鉴别能力
5. 最终达到纳什均衡:G生成的图像足够逼真,D无法区分

min_G max_D V(D, G) = E[log D(x)] + E[log(1 - D(G(z)))]

其中:
- x: 真实数据
- z: 随机噪声(通常是高斯噪声)
- G(z): 生成器根据噪声z生成的假图像
- D(x): 判别器对真实图像的判断(接近1表示认为是真的)
- D(G(z)): 判别器对假图像的判断(接近0表示认为是假的)

for epoch in range(num_epochs):
    for real_images, _ in dataloader:
        # ===== 训练判别器 =====
        # 1. 真实图像送入判别器,标签为1
        real_labels = torch.ones(batch_size, 1)
        real_outputs = discriminator(real_images)
        d_loss_real = criterion(real_outputs, real_labels)

        # 2. 生成假图像,标签为0
        noise = torch.randn(batch_size, latent_dim)
        fake_images = generator(noise)
        fake_labels = torch.zeros(batch_size, 1)
        fake_outputs = discriminator(fake_images.detach())
        d_loss_fake = criterion(fake_outputs, fake_labels)

        # 3. 判别器总损失
        d_loss = d_loss_real + d_loss_fake
        # 反向传播更新判别器
        optimizer_D.zero_grad()
        d_loss.backward()
        optimizer_D.step()

        # ===== 训练生成器 =====
        # 生成器希望判别器认为假图像是真的(标签为1)
        noise = torch.randn(batch_size, latent_dim)
        fake_images = generator(noise)
        outputs = discriminator(fake_images)
        g_loss = criterion(outputs, torch.ones(batch_size, 1))

        # 反向传播更新生成器
        optimizer_G.zero_grad()
        g_loss.backward()
        optimizer_G.step()

class Generator(nn.Module):
    def __init__(self, latent_dim=100, channels=3):
        super().__init__()
        self.model = nn.Sequential(
            # 输入: (batch, latent_dim, 1, 1)
            # 第一层:转置卷积 100 -> 512x4x4
            nn.ConvTranspose2d(latent_dim, 512, 4, 1, 0, bias=False),
            nn.BatchNorm2d(512),
            nn.ReLU(True),
            # 512x4x4 -> 256x8x8
            nn.ConvTranspose2d(512, 256, 4, 2, 1, bias=False),
            nn.BatchNorm2d(256),
            nn.ReLU(True),
            # 256x8x8 -> 128x16x16
            nn.ConvTranspose2d(256, 128, 4, 2, 1, bias=False),
            nn.BatchNorm2d(128),
            nn.ReLU(True),
            # 128x16x16 -> 64x32x32
            nn.ConvTranspose2d(128, 64, 4, 2, 1, bias=False),
            nn.BatchNorm2d(64),
            nn.ReLU(True),
            # 64x32x32 -> 3x64x64
            nn.ConvTranspose2d(64, channels, 4, 2, 1, bias=False),
            nn.Tanh()  # 输出范围[-1, 1]
        )

    def forward(self, z):
        # z: (batch, latent_dim, 1, 1)
        return self.model(z)

class Discriminator(nn.Module):
    def __init__(self, channels=3):
        super().__init__()
        self.model = nn.Sequential(
            # 输入: 3x64x64
            # 3x64x64 -> 64x32x32
            nn.Conv2d(channels, 64, 4, 2, 1, bias=False),
            nn.LeakyReLU(0.2, inplace=True),
            # 64x32x32 -> 128x16x16
            nn.Conv2d(64, 128, 4, 2, 1, bias=False),
            nn.BatchNorm2d(128),
            nn.LeakyReLU(0.2, inplace=True),
            # 128x16x16 -> 256x8x8
            nn.Conv2d(128, 256, 4, 2, 1, bias=False),
            nn.BatchNorm2d(256),
            nn.LeakyReLU(0.2, inplace=True),
            # 256x8x8 -> 512x4x4
            nn.Conv2d(256, 512, 4, 2, 1, bias=False),
            nn.BatchNorm2d(512),
            nn.LeakyReLU(0.2, inplace=True),
            # 512x4x4 -> 1x1x1
            nn.Conv2d(512, 1, 4, 1, 0, bias=False),
            nn.Sigmoid()  # 输出范围[0, 1]
        )

    def forward(self, img):
        return self.model(img).view(-1, 1)

def weights_init(m):
    """
    从均值0、标准差0.02的正态分布初始化权重
    """
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        nn.init.normal_(m.weight.data, 0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        nn.init.normal_(m.weight.data, 1.0, 0.02)
        nn.init.constant_(m.bias.data, 0)

# 应用初始化
generator.apply(weights_init)
discriminator.apply(weights_init)

# 生成器和判别器都接受条件输入
class ConditionalGenerator(nn.Module):
    def __init__(self, latent_dim, num_classes):
        super().__init__()
        self.label_emb = nn.Embedding(num_classes, num_classes)
        # 输入是噪声+条件拼接
        self.model = nn.Sequential(...)

    def forward(self, noise, labels):
        # 将标签嵌入与噪声拼接
        gen_input = torch.cat([noise, self.label_emb(labels)], -1)
        return self.model(gen_input)

理想情况: 数据分布有10个模式,生成器都能覆盖
模式崩溃: 生成器只生成其中2-3个模式

from torchvision import datasets, transforms

transform = transforms.Compose([
    transforms.Resize(64),
    transforms.CenterCrop(64),
    transforms.ToTensor(),
    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])  # 归一化到[-1, 1]
])

dataset = datasets.CelebA(
    root='./data',
    split='train',
    transform=transform,
    download=True
)

dataloader = torch.utils.data.DataLoader(
    dataset,
    batch_size=128,
    shuffle=True,
    num_workers=4
)

15.1 扩散模型基础
  ↓ 理解前向加噪和反向去噪
15.2 Stable Diffusion架构
  ↓ 掌握Latent Diffusion的优势
15.3 ControlNet可控生成
  ↓ 学习条件控制技术
15.4 FLUX最新模型
  ↓ 了解2024前沿进展
15.5 提示词工程
  ↓ 编写高质量提示词
实战项目
  ↓ 文生图、图生图、ControlNet

原始图像 → +噪声 → +噪声 → ... → 纯噪声  (前向过程,固定的)
          ↓        ↓        ↓        ↓
纯噪声 → -噪声 → -噪声 → ... → 生成图像  (反向过程,需要学习)

x₀ → x₁ → x₂ → ... → x_T

每一步:
x_t = √(1-β_t) · x_{t-1} + √β_t · ε

其中:
- β_t: 第t步的噪声系数(通常从0.0001到0.02)
- ε ~ N(0, I): 标准高斯噪声
- T: 总步数(通常1000步)

x_t = √(ᾱ_t) · x₀ + √(1-ᾱ_t) · ε

其中:
ᾱ_t = ∏(1-β_i)  (累乘)

x_T → x_{T-1} → x_{T-2} → ... → x₀

每一步:
x_{t-1} = (1/√α_t) · (x_t - (β_t/√(1-ᾱ_t)) · ε_θ(x_t, t))

损失函数:
L = E[||ε - ε_θ(x_t, t)||²]

其中:
- ε: 真实噪声
- ε_θ(x_t, t): 模型预测的噪声

for epoch in range(num_epochs):
    for x0 in dataloader:  # 原始图像
        # 1. 随机选择时间步 t ∈ [1, T]
        t = random.randint(1, T)

        # 2. 生成随机噪声
        epsilon = torch.randn_like(x0)

        # 3. 根据公式计算 x_t
        xt = sqrt(alpha_bar_t) * x0 + sqrt(1 - alpha_bar_t) * epsilon

        # 4. 模型预测噪声
        epsilon_pred = model(xt, t)

        # 5. 计算损失(简单的MSE)
        loss = (epsilon - epsilon_pred).pow(2).mean()

        # 6. 反向传播
        loss.backward()
        optimizer.step()

# 从纯噪声开始
x_T = torch.randn(1, 3, H, W)

# 逐步去噪
for t in reversed(range(1, T+1)):
    # 预测噪声
    epsilon_pred = model(x_t, t)

    # 去噪一步
    x_{t-1} = denoise_step(x_t, epsilon_pred, t)

# 最终得到 x_0,即生成的图像
generated_image = x_0

像素空间(512x512x3) → VAE编码 → 潜空间(64x64x4)
                                    ↓
                              在这里进行扩散
                                    ↓
潜空间(64x64x4) → VAE解码 → 像素空间(512x512x3)

# 编码器:图像 → 潜空间
latent = vae.encode(image)  # (3, 512, 512) → (4, 64, 64)

# 解码器:潜空间 → 图像
image = vae.decode(latent)  # (4, 64, 64) → (3, 512, 512)

输入: 噪声潜空间 z_t + 时间步 t + 条件 c(文本嵌入)
        ↓
    Encoder (下采样)
        ↓
    Bottleneck (Attention层处理)
        ↓
    Decoder (上采样,带Skip Connection)
        ↓
输出: 预测的噪声

# 文本 → 向量
prompt = "a beautiful sunset over the ocean"
text_embedding = clip_text_encoder(prompt)  # (1, 77, 768)

# 1. 文本编码
text_embedding = clip_text_encoder(prompt)

# 2. 初始化随机噪声(在潜空间)
latent = torch.randn(1, 4, 64, 64)  # 对应512x512图像

# 3. 迭代去噪
for t in scheduler.timesteps:  # 通常50步
    # 预测噪声
    noise_pred = unet(latent, t, text_embedding)

    # 去噪一步
    latent = scheduler.step(noise_pred, t, latent).prev_sample

# 4. 解码为图像
image = vae.decode(latent)

最终噪声预测 = 无条件预测 + guidance_scale × (有条件预测 - 无条件预测)

# guidance_scale = 1.0:完全忽略提示词
# guidance_scale = 7.5:标准值
# guidance_scale = 15.0:强烈遵循提示词

原始U-Net(冻结)
    ↓
  复制
    ↓
ControlNet分支(可训练)
    ↑
视觉条件(Canny边缘、深度图等)

from controlnet_aux import CannyDetector

canny = CannyDetector()
edge_image = canny(input_image)  # 提取边缘

from controlnet_aux import DepthEstimator

depth_estimator = DepthEstimator.from_pretrained("Intel/dpt-hybrid-midas")
depth_map = depth_estimator(input_image)

from controlnet_aux import OpenposeDetector

openpose = OpenposeDetector.from_pretrained("lllyasviel/ControlNet")
pose_image = openpose(input_image)

from diffusers import StableDiffusionControlNetPipeline, ControlNetModel

# 加载多个ControlNet
controlnet_canny = ControlNetModel.from_pretrained("lllyasviel/control_v11p_sd15_canny")
controlnet_depth = ControlNetModel.from_pretrained("lllyasviel/control_v11f1p_sd15_depth")

# 创建pipeline
pipe = StableDiffusionControlNetPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    controlnet=[controlnet_canny, controlnet_depth]
)

# 生成(同时使用边缘和深度控制)
image = pipe(
    prompt="...",
    image=[canny_image, depth_image],
    controlnet_conditioning_scale=[0.5, 0.8]  # 分别控制权重
).images[0]

from diffusers import FluxPipeline

# 加载FLUX.1-schnell(快速版本)
pipe = FluxPipeline.from_pretrained(
    "black-forest-labs/FLUX.1-schnell",
    torch_dtype=torch.bfloat16
).to("cuda")

# 生成(只需4步!)
image = pipe(
    prompt="a cat holding a sign that says 'hello world'",
    num_inference_steps=4,
    guidance_scale=0.0  # schnell版本不需要guidance
).images[0]

"a cat"

"a fluffy orange tabby cat with green eyes,
sitting on a vintage wooden table,
warm golden hour lighting,
shallow depth of field,
bokeh background,
professional pet photography,
8k uhd, sharp focus,
highly detailed fur texture"

[主体描述] + [细节修饰] + [环境/背景] + [光照] +
[艺术风格] + [技术参数] + [质量词]

a majestic lion with a flowing golden mane and piercing amber eyes,
standing on a rocky cliff at sunset,
dramatic golden hour lighting with rim light,
digital painting, cinematic composition,
8k uhd, sharp focus, volumetric lighting,
masterpiece, award winning photography

低质量相关:
lowres, bad anatomy, bad hands, text, error,
missing fingers, extra digit, fewer digits,
cropped, worst quality, low quality,
normal quality, jpeg artifacts, signature,
watermark, username, blurry

不需要的元素:
ugly, duplicate, morbid, mutilated,
extra fingers, mutated hands, poorly drawn hands,
poorly drawn face, mutation, deformed

image = pipe(
    prompt="a beautiful landscape...",
    negative_prompt="blurry, low quality, distorted, watermark",
    num_inference_steps=50,
    guidance_scale=7.5
).images[0]

(keyword)     # 权重1.1
((keyword))   # 权重1.1 * 1.1 = 1.21
(keyword:1.5) # 权重1.5
[keyword]     # 权重0.9

"a ((highly detailed)) portrait,
(red hair:1.3),
[background]"

解释:
- highly detailed: 权重1.21(强调)
- red hair: 权重1.3(非常强调红色头发)
- background: 权重0.9(弱化背景)

portrait photography:
- "professional portrait photography"
- "studio lighting, soft box"
- "85mm lens, f/1.4"
- "bokeh background"

landscape photography:
- "landscape photography"
- "golden hour, dramatic clouds"
- "wide angle lens"
- "high dynamic range"

digital art:
- "digital painting"
- "concept art"
- "trending on artstation"

anime:
- "anime style"
- "studio ghibli"
- "makoto shinkai"

oil painting:
- "oil painting"
- "impressionism"
- "visible brush strokes"

通用质量词:
- "masterpiece"
- "best quality"
- "highly detailed"
- "8k uhd"
- "professional"
- "award winning"

清晰度:
- "sharp focus"
- "high resolution"
- "ultra detailed"

from diffusers import StableDiffusionPipeline
import torch

pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    torch_dtype=torch.float16
).to("cuda")

prompt = """
a serene Japanese garden with a wooden bridge over a koi pond,
cherry blossoms falling, soft morning light,
traditional architecture in background,
photorealistic, 8k uhd, highly detailed
"""

negative_prompt = "blurry, low quality, distorted, watermark"

image = pipe(
    prompt=prompt,
    negative_prompt=negative_prompt,
    num_inference_steps=50,
    guidance_scale=7.5,
    height=512,
    width=512
).images[0]

image.save("output.png")

from diffusers import StableDiffusionImg2ImgPipeline
from PIL import Image

pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    torch_dtype=torch.float16
).to("cuda")

# 加载参考图
init_image = Image.open("input.jpg").resize((512, 512))

prompt = "turn this photo into an oil painting, impressionism style"

image = pipe(
    prompt=prompt,
    image=init_image,
    strength=0.75,  # 0.0-1.0,越高变化越大
    guidance_scale=7.5
).images[0]

from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
from controlnet_aux import CannyDetector
from PIL import Image

# 提取边缘
canny = CannyDetector()
input_image = Image.open("input.jpg")
canny_image = canny(input_image)

# 加载ControlNet
controlnet = ControlNetModel.from_pretrained(
    "lllyasviel/control_v11p_sd15_canny"
)

pipe = StableDiffusionControlNetPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    controlnet=controlnet,
    torch_dtype=torch.float16
).to("cuda")

# 生成
image = pipe(
    prompt="a modern architectural rendering, glass and steel",
    image=canny_image,
    controlnet_conditioning_scale=0.8
).images[0]

def batch_generate(prompts: list, output_dir: str):
    """批量生成图像"""
    pipe = StableDiffusionPipeline.from_pretrained(...)

    for i, prompt in enumerate(prompts):
        image = pipe(prompt=prompt).images[0]
        image.save(f"{output_dir}/image_{i:03d}.png")
        print(f"生成 {i+1}/{len(prompts)}")

# 使用
prompts = [
    "a cat in a garden",
    "a dog on a beach",
    "a bird in the sky"
]
batch_generate(prompts, "./outputs")

from diffusers import StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(...)

# 加载LoRA权重
pipe.load_lora_weights("path/to/lora.safetensors")

# 调整LoRA强度
pipe.set_adapters("default", adapter_weights=[0.8])

pipe.load_textual_inversion("path/to/embedding.pt", token="<my-concept>")

prompt = "a photo of <my-concept> in a forest"

from diffusers import StableDiffusionInpaintPipeline

pipe = StableDiffusionInpaintPipeline.from_pretrained(...)

image = pipe(
    prompt="a red car",
    image=original_image,
    mask_image=mask,  # 白色区域会被重绘
).images[0]

from diffusers import StableDiffusionUpscalePipeline

pipe = StableDiffusionUpscalePipeline.from_pretrained(
    "stabilityai/stable-diffusion-x4-upscaler"
)

upscaled = pipe(
    prompt="high quality, detailed",
    image=low_res_image
).images[0]

pipe.enable_attention_slicing()  # 减少显存占用
pipe.enable_vae_slicing()  # VAE分块处理
pipe.enable_xformers_memory_efficient_attention()  # 使用xFormers

# 使用torch.compile(PyTorch 2.0+)
pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True)

# 使用更快的采样器
from diffusers import DPMSolverMultistepScheduler
pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)

pipe.enable_sequential_cpu_offload()  # 组件轮流使用GPU
pipe.enable_model_cpu_offload()  # 更智能的offload