目标检测：这里有一辆车 [矩形框]
语义分割：这些像素是车 [像素级mask]
实例分割：这是第1辆车，那是第2辆车 [区分个体]

IoU = 交集 / 并集 = (预测 ∩ 真实) / (预测 ∪ 真实)

Dice = 2 × 交集 / (预测 + 真实) = 2 × |A ∩ B| / (|A| + |B|)

mIoU = (1/N) × Σ IoU_i

PA = 正确分类的像素数 / 总像素数

2015    FCN           全卷积网络，开创像素级预测
         ↓
2015    U-Net         编码器-解码器，医学图像分割经典
         ↓
2017    DeepLab v3    空洞卷积 + ASPP
         ↓
2017    Mask R-CNN    实例分割里程碑
         ↓
2019    YOLACT        实时实例分割
         ↓
2023    YOLOv8-Seg    YOLO统一框架
         ↓
2023    SAM           零样本分割，Prompt驱动
         ↓
2024    SAM 2         扩展到视频分割

import torch
from models import UNet

# 加载U-Net模型
model = UNet(in_channels=1, num_classes=2)
model.load_state_dict(torch.load('unet.pth'))

# 推理
image = load_medical_image('ct_scan.nii')
with torch.no_grad():
    mask = model(image)
    prediction = torch.argmax(mask, dim=1)

from ultralytics import YOLO

# 加载YOLOv8-Seg
model = YOLO('yolo11n-seg.pt')

# 推理
results = model('street.jpg')
masks = results[0].masks.data  # 实例掩码
boxes = results[0].boxes.data  # 边界框

from transformers import SamModel, SamProcessor

# 加载SAM
model = SamModel.from_pretrained("facebook/sam-vit-huge")
processor = SamProcessor.from_pretrained("facebook/sam-vit-huge")

# 点击式分割
input_points = [[[450, 600]]]  # 点击坐标
inputs = processor(image, input_points=input_points, return_tensors="pt")
outputs = model(**inputs)
masks = processor.post_process_masks(outputs.pred_masks)

输入：街景图像 (512×512×3)
输出：
  - 像素(100, 200) → 类别2 (人)
  - 像素(300, 150) → 类别7 (汽车)
  - 像素(450, 400) → 类别0 (背景)

输入图像 (224×224×3)
    ↓
卷积层 (提取特征)
    ↓
全连接层 (分类)
    ↓
输出：单一类别标签

输入图像 (任意尺寸 H×W×3)
    ↓
全卷积层 (无全连接层)
    ↓
上采样 (恢复分辨率)
    ↓
输出：像素级预测 (H×W×C)

# 传统分类网络
fc6 = nn.Linear(512 * 7 * 7, 4096)  # 展平后全连接

# FCN改造
fc6 = nn.Conv2d(512, 4096, kernel_size=7)  # 7×7卷积
fc7 = nn.Conv2d(4096, 4096, kernel_size=1)  # 1×1卷积
score = nn.Conv2d(4096, num_classes, kernel_size=1)  # 分类层

upsampled = F.interpolate(x, scale_factor=2, mode='bilinear')

# 也称为反卷积（Deconvolution）
upsample = nn.ConvTranspose2d(
    in_channels=512,
    out_channels=num_classes,
    kernel_size=4,
    stride=2,
    padding=1
)

输入图像 (H×W)
    ↓
conv1 → pool1 (H/2×W/2)  ←─┐
    ↓                      │ Skip
conv2 → pool2 (H/4×W/4)  ←─┼─┐
    ↓                      │ │ Skip
conv3 → pool3 (H/8×W/8)  ←─┼─┼─┐
    ↓                      │ │ │
[卷积层]                   │ │ │
    ↓                      │ │ │
上采样2倍 + 融合pool3 ──────┘ │ │
    ↓                          │ │
上采样2倍 + 融合pool2 ──────────┘ │
    ↓                            │
上采样2倍 + 融合pool1 ────────────┘
    ↓
输出 (H×W×C)

import torch
import torch.nn as nn
import torch.nn.functional as F

class FCN8s(nn.Module):
    def __init__(self, num_classes=21):
        super(FCN8s, self).__init__()

        # VGG16的卷积层
        # Block 1
        self.conv1_1 = nn.Conv2d(3, 64, 3, padding=1)
        self.relu1_1 = nn.ReLU(inplace=True)
        self.conv1_2 = nn.Conv2d(64, 64, 3, padding=1)
        self.relu1_2 = nn.ReLU(inplace=True)
        self.pool1 = nn.MaxPool2d(2, stride=2, ceil_mode=True)

        # Block 2
        self.conv2_1 = nn.Conv2d(64, 128, 3, padding=1)
        self.relu2_1 = nn.ReLU(inplace=True)
        self.conv2_2 = nn.Conv2d(128, 128, 3, padding=1)
        self.relu2_2 = nn.ReLU(inplace=True)
        self.pool2 = nn.MaxPool2d(2, stride=2, ceil_mode=True)

        # Block 3
        self.conv3_1 = nn.Conv2d(128, 256, 3, padding=1)
        self.relu3_1 = nn.ReLU(inplace=True)
        self.conv3_2 = nn.Conv2d(256, 256, 3, padding=1)
        self.relu3_2 = nn.ReLU(inplace=True)
        self.conv3_3 = nn.Conv2d(256, 256, 3, padding=1)
        self.relu3_3 = nn.ReLU(inplace=True)
        self.pool3 = nn.MaxPool2d(2, stride=2, ceil_mode=True)

        # Block 4
        self.conv4_1 = nn.Conv2d(256, 512, 3, padding=1)
        self.relu4_1 = nn.ReLU(inplace=True)
        self.conv4_2 = nn.Conv2d(512, 512, 3, padding=1)
        self.relu4_2 = nn.ReLU(inplace=True)
        self.conv4_3 = nn.Conv2d(512, 512, 3, padding=1)
        self.relu4_3 = nn.ReLU(inplace=True)
        self.pool4 = nn.MaxPool2d(2, stride=2, ceil_mode=True)

        # Block 5
        self.conv5_1 = nn.Conv2d(512, 512, 3, padding=1)
        self.relu5_1 = nn.ReLU(inplace=True)
        self.conv5_2 = nn.Conv2d(512, 512, 3, padding=1)
        self.relu5_2 = nn.ReLU(inplace=True)
        self.conv5_3 = nn.Conv2d(512, 512, 3, padding=1)
        self.relu5_3 = nn.ReLU(inplace=True)
        self.pool5 = nn.MaxPool2d(2, stride=2, ceil_mode=True)

        # FC6-7替换为卷积
        self.fc6 = nn.Conv2d(512, 4096, 7)
        self.relu6 = nn.ReLU(inplace=True)
        self.drop6 = nn.Dropout2d()

        self.fc7 = nn.Conv2d(4096, 4096, 1)
        self.relu7 = nn.ReLU(inplace=True)
        self.drop7 = nn.Dropout2d()

        # 分类层
        self.score_fr = nn.Conv2d(4096, num_classes, 1)
        self.score_pool3 = nn.Conv2d(256, num_classes, 1)
        self.score_pool4 = nn.Conv2d(512, num_classes, 1)

        # 上采样层
        self.upscore2 = nn.ConvTranspose2d(
            num_classes, num_classes, 4, stride=2, bias=False)
        self.upscore8 = nn.ConvTranspose2d(
            num_classes, num_classes, 16, stride=8, bias=False)
        self.upscore_pool4 = nn.ConvTranspose2d(
            num_classes, num_classes, 4, stride=2, bias=False)

    def forward(self, x):
        h = x

        # VGG卷积
        h = self.relu1_1(self.conv1_1(h))
        h = self.relu1_2(self.conv1_2(h))
        h = self.pool1(h)

        h = self.relu2_1(self.conv2_1(h))
        h = self.relu2_2(self.conv2_2(h))
        h = self.pool2(h)

        h = self.relu3_1(self.conv3_1(h))
        h = self.relu3_2(self.conv3_2(h))
        h = self.relu3_3(self.conv3_3(h))
        h = self.pool3(h)
        pool3 = h  # 保存pool3用于跳跃连接

        h = self.relu4_1(self.conv4_1(h))
        h = self.relu4_2(self.conv4_2(h))
        h = self.relu4_3(self.conv4_3(h))
        h = self.pool4(h)
        pool4 = h  # 保存pool4

        h = self.relu5_1(self.conv5_1(h))
        h = self.relu5_2(self.conv5_2(h))
        h = self.relu5_3(self.conv5_3(h))
        h = self.pool5(h)

        # FC层
        h = self.relu6(self.fc6(h))
        h = self.drop6(h)

        h = self.relu7(self.fc7(h))
        h = self.drop7(h)

        h = self.score_fr(h)
        h = self.upscore2(h)
        upscore2 = h  # 2倍上采样

        # 融合pool4
        h = self.score_pool4(pool4)
        h = h[:, :, 5:5+upscore2.size()[2], 5:5+upscore2.size()[3]]
        score_pool4c = h

        h = upscore2 + score_pool4c
        h = self.upscore_pool4(h)
        upscore_pool4 = h

        # 融合pool3
        h = self.score_pool3(pool3)
        h = h[:, :, 9:9+upscore_pool4.size()[2], 9:9+upscore_pool4.size()[3]]
        score_pool3c = h

        h = upscore_pool4 + score_pool3c
        h = self.upscore8(h)
        h = h[:, :, 31:31+x.size()[2], 31:31+x.size()[3]].contiguous()

        return h

criterion = nn.CrossEntropyLoss(ignore_index=255)
loss = criterion(output, target)

# 计算类别权重
class_weights = compute_class_weight('balanced', classes, labels)
criterion = nn.CrossEntropyLoss(weight=torch.FloatTensor(class_weights))

def compute_iou(pred, target, num_classes):
    ious = []
    pred = pred.view(-1)
    target = target.view(-1)

    for cls in range(num_classes):
        pred_cls = (pred == cls)
        target_cls = (target == cls)

        intersection = (pred_cls & target_cls).sum().float()
        union = (pred_cls | target_cls).sum().float()

        if union == 0:
            ious.append(float('nan'))
        else:
            ious.append((intersection / union).item())

    return ious

# mIoU (mean IoU)
mean_iou = np.nanmean(ious)

输入图像 (572×572×1)
    ↓
【编码器 Encoder】下采样路径
    conv 3×3, ReLU
    conv 3×3, ReLU
    max pool 2×2  → (280×280×64)  ─────┐
    ↓                                   │
    conv 3×3, ReLU                      │
    conv 3×3, ReLU                      │
    max pool 2×2  → (136×136×128) ─────┼───┐
    ↓                                   │   │
    conv 3×3, ReLU                      │   │
    conv 3×3, ReLU                      │   │
    max pool 2×2  → (64×64×256)   ─────┼───┼───┐
    ↓                                   │   │   │
    conv 3×3, ReLU                      │   │   │
    conv 3×3, ReLU                      │   │   │
    max pool 2×2  → (28×28×512)   ─────┼───┼───┼───┐
    ↓                                   │   │   │   │
【Bottleneck】                          │   │   │   │
    conv 3×3, ReLU                      │   │   │   │
    conv 3×3, ReLU  → (24×24×1024)      │   │   │   │
    ↓                                   │   │   │   │
【解码器 Decoder】上采样路径             │   │   │   │
    up-conv 2×2 → (48×48×512)           │   │   │   │
    concat ←────────────────────────────┘   │   │   │
    conv 3×3, ReLU                          │   │   │
    conv 3×3, ReLU                          │   │   │
    ↓                                       │   │   │
    up-conv 2×2 → (96×96×256)               │   │   │
    concat ←────────────────────────────────┘   │   │
    conv 3×3, ReLU                              │   │
    conv 3×3, ReLU                              │   │
    ↓                                           │   │
    up-conv 2×2 → (192×192×128)                 │   │
    concat ←────────────────────────────────────┘   │
    conv 3×3, ReLU                                  │
    conv 3×3, ReLU                                  │
    ↓                                               │
    up-conv 2×2 → (384×384×64)                      │
    concat ←────────────────────────────────────────┘
    conv 3×3, ReLU
    conv 3×3, ReLU
    ↓
    conv 1×1 → (388×388×2)  输出

def down_block(in_channels, out_channels):
    return nn.Sequential(
        nn.Conv2d(in_channels, out_channels, 3, padding=1),
        nn.ReLU(inplace=True),
        nn.Conv2d(out_channels, out_channels, 3, padding=1),
        nn.ReLU(inplace=True)
    )

def up_block(in_channels, out_channels):
    return nn.Sequential(
        nn.ConvTranspose2d(in_channels, out_channels, 2, stride=2),
        nn.Conv2d(out_channels, out_channels, 3, padding=1),
        nn.ReLU(inplace=True),
        nn.Conv2d(out_channels, out_channels, 3, padding=1),
        nn.ReLU(inplace=True)
    )

# 编码器特征
encoder_feature = encoder(x)  # (B, C, H, W)

# 解码器上采样
decoder_upsampled = upsample(decoder_input)  # (B, C, H, W)

# 拼接
combined = torch.cat([encoder_feature, decoder_upsampled], dim=1)  # (B, 2C, H, W)

# 继续卷积
output = conv(combined)

import albumentations as A

train_transform = A.Compose([
    A.HorizontalFlip(p=0.5),
    A.VerticalFlip(p=0.5),
    A.RandomRotate90(p=0.5),
    A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=45, p=0.5),
    A.ElasticTransform(alpha=1, sigma=50, p=0.5),  # 弹性变形
    A.GridDistortion(p=0.5),  # 网格扭曲
    A.RandomBrightnessContrast(p=0.3),
    A.Normalize(mean=(0.485,), std=(0.229,))
])

class WeightedBCELoss(nn.Module):
    def __init__(self, weight_map):
        super().__init__()
        self.weight_map = weight_map

    def forward(self, pred, target):
        bce = F.binary_cross_entropy_with_logits(pred, target, reduction='none')
        weighted_bce = bce * self.weight_map
        return weighted_bce.mean()

# 计算权重图
def compute_weight_map(mask):
    # 边界权重更高
    from scipy.ndimage import distance_transform_edt

    distances = distance_transform_edt(mask == 0)
    weights = np.exp(-distances ** 2 / (2 * sigma ** 2))
    return weights

class DiceLoss(nn.Module):
    def __init__(self, smooth=1.0):
        super().__init__()
        self.smooth = smooth

    def forward(self, pred, target):
        pred = torch.sigmoid(pred)

        intersection = (pred * target).sum(dim=(2, 3))
        union = pred.sum(dim=(2, 3)) + target.sum(dim=(2, 3))

        dice = (2. * intersection + self.smooth) / (union + self.smooth)
        return 1 - dice.mean()

# 组合损失
class CombinedLoss(nn.Module):
    def __init__(self, alpha=0.5):
        super().__init__()
        self.alpha = alpha
        self.bce = nn.BCEWithLogitsLoss()
        self.dice = DiceLoss()

    def forward(self, pred, target):
        return self.alpha * self.bce(pred, target) + (1 - self.alpha) * self.dice(pred, target)

class AttentionBlock(nn.Module):
    def __init__(self, F_g, F_l, F_int):
        super().__init__()
        self.W_g = nn.Conv2d(F_g, F_int, 1, stride=1, padding=0)
        self.W_x = nn.Conv2d(F_l, F_int, 1, stride=1, padding=0)
        self.psi = nn.Conv2d(F_int, 1, 1, stride=1, padding=0)
        self.relu = nn.ReLU(inplace=True)
        self.sigmoid = nn.Sigmoid()

    def forward(self, g, x):
        g1 = self.W_g(g)
        x1 = self.W_x(x)
        psi = self.relu(g1 + x1)
        psi = self.sigmoid(self.psi(psi))
        return x * psi  # 加权特征

class ResidualBlock(nn.Module):
    def __init__(self, in_channels, out_channels):
        super().__init__()
        self.conv1 = nn.Conv2d(in_channels, out_channels, 3, padding=1)
        self.bn1 = nn.BatchNorm2d(out_channels)
        self.conv2 = nn.Conv2d(out_channels, out_channels, 3, padding=1)
        self.bn2 = nn.BatchNorm2d(out_channels)

        self.shortcut = nn.Sequential()
        if in_channels != out_channels:
            self.shortcut = nn.Conv2d(in_channels, out_channels, 1)

    def forward(self, x):
        residual = self.shortcut(x)

        out = F.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))
        out += residual
        out = F.relu(out)

        return out

X^0,0 → X^0,1 → X^0,2 → X^0,3 → X^0,4
  ↓ ↘     ↓ ↘     ↓ ↘     ↓ ↘
X^1,0 → X^1,1 → X^1,2 → X^1,3
  ↓ ↘     ↓ ↘     ↓ ↘
X^2,0 → X^2,1 → X^2,2
  ↓ ↘     ↓ ↘
X^3,0 → X^3,1
  ↓ ↘
X^4,0

× × ×
× × ×
× × ×

×  ·  ×  ·  ×
·  ·  ·  ·  ·
×  ·  ×  ·  ×
·  ·  ·  ·  ·
×  ·  ×  ·  ×

×  ·  ·  ·  ×  ·  ·  ·  ×
·  ·  ·  ·  ·  ·  ·  ·  ·
·  ·  ·  ·  ·  ·  ·  ·  ·
·  ·  ·  ·  ·  ·  ·  ·  ·
×  ·  ·  ·  ×  ·  ·  ·  ×

# 标准卷积：感受野3×3
conv = nn.Conv2d(64, 64, kernel_size=3, padding=1, dilation=1)

# 空洞卷积：感受野5×5，但参数量相同
atrous_conv = nn.Conv2d(64, 64, kernel_size=3, padding=2, dilation=2)

# 空洞卷积：感受野9×9
atrous_conv2 = nn.Conv2d(64, 64, kernel_size=3, padding=4, dilation=4)

receptive_field = (kernel_size - 1) * dilation + 1

kernel=3, dilation=1 → RF = 3
kernel=3, dilation=2 → RF = 5
kernel=3, dilation=4 → RF = 9
kernel=3, dilation=8 → RF = 17

输入特征图
    ├── rate=6  空洞卷积  ──┐
    ├── rate=12 空洞卷积  ──┤
    ├── rate=18 空洞卷积  ──┤ Concatenate
    └── rate=24 空洞卷积  ──┘
            ↓
       1×1卷积融合
            ↓
        输出特征

class ASPP(nn.Module):
    def __init__(self, in_channels, out_channels=256):
        super().__init__()

        # 1×1卷积
        self.conv1 = nn.Sequential(
            nn.Conv2d(in_channels, out_channels, 1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True)
        )

        # 3×3空洞卷积，rate=6
        self.conv2 = nn.Sequential(
            nn.Conv2d(in_channels, out_channels, 3, padding=6, dilation=6, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True)
        )

        # 3×3空洞卷积，rate=12
        self.conv3 = nn.Sequential(
            nn.Conv2d(in_channels, out_channels, 3, padding=12, dilation=12, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True)
        )

        # 3×3空洞卷积，rate=18
        self.conv4 = nn.Sequential(
            nn.Conv2d(in_channels, out_channels, 3, padding=18, dilation=18, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True)
        )

        # 全局平均池化
        self.global_avg_pool = nn.Sequential(
            nn.AdaptiveAvgPool2d((1, 1)),
            nn.Conv2d(in_channels, out_channels, 1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True)
        )

        # 融合
        self.project = nn.Sequential(
            nn.Conv2d(out_channels * 5, out_channels, 1, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU(inplace=True),
            nn.Dropout(0.5)
        )

    def forward(self, x):
        size = x.shape[-2:]

        # 并行分支
        feat1 = self.conv1(x)
        feat2 = self.conv2(x)
        feat3 = self.conv3(x)
        feat4 = self.conv4(x)
        feat5 = F.interpolate(self.global_avg_pool(x), size=size, mode='bilinear', align_corners=True)

        # 拼接
        x = torch.cat([feat1, feat2, feat3, feat4, feat5], dim=1)

        # 融合
        x = self.project(x)
        return x

输入图像
    ↓
【Encoder】
Xception Backbone
    ├── 低层特征 (1/4) ──────────────┐
    ↓                              │
ASPP模块                           │
    ↓                              │
1×1卷积 (降维到256)                 │
    ↓                              │
上采样4倍                           │
    ↓                              │
【Decoder】                         │
    ├─ 拼接 ←──────────────────────┘
    ↓
3×3卷积
    ↓
3×3卷积
    ↓
上采样4倍
    ↓
输出 (原始分辨率)

import torchvision.models.segmentation as segmentation

# 加载预训练DeepLab v3
model = segmentation.deeplabv3_resnet50(pretrained=True, num_classes=21)

# 或使用MobileNetV3作为backbone（更轻量）
model = segmentation.deeplabv3_mobilenet_v3_large(pretrained=True, num_classes=21)

# 推理
model.eval()
with torch.no_grad():
    output = model(image)['out']  # 注意返回字典
    prediction = torch.argmax(output, dim=1)

import nibabel as nib  # 读取医学图像格式
import numpy as np
from torch.utils.data import Dataset

class LungCTDataset(Dataset):
    def __init__(self, image_paths, mask_paths, transform=None):
        self.image_paths = image_paths
        self.mask_paths = mask_paths
        self.transform = transform

    def __len__(self):
        return len(self.image_paths)

    def __getitem__(self, idx):
        # 读取NIfTI格式
        image = nib.load(self.image_paths[idx]).get_fdata()
        mask = nib.load(self.mask_paths[idx]).get_fdata()

        # 归一化
        image = (image - image.min()) / (image.max() - image.min())

        # 转换为tensor
        image = torch.FloatTensor(image).unsqueeze(0)  # (1, H, W)
        mask = torch.LongTensor(mask)

        if self.transform:
            # 应用数据增强
            augmented = self.transform(image=image.numpy(), mask=mask.numpy())
            image = torch.FloatTensor(augmented['image'])
            mask = torch.LongTensor(augmented['mask'])

        return image, mask

def compute_dice_score(pred, target):
    """Dice系数"""
    smooth = 1.0
    pred_flat = pred.view(-1)
    target_flat = target.view(-1)

    intersection = (pred_flat * target_flat).sum()
    dice = (2. * intersection + smooth) / (pred_flat.sum() + target_flat.sum() + smooth)

    return dice.item()

def compute_hausdorff_distance(pred, target):
    """Hausdorff距离（边界精度）"""
    from scipy.spatial.distance import directed_hausdorff

    pred_points = np.argwhere(pred)
    target_points = np.argwhere(target)

    forward = directed_hausdorff(pred_points, target_points)[0]
    backward = directed_hausdorff(target_points, pred_points)[0]

    return max(forward, backward)

场景：街道上有3辆汽车

【语义分割】输出：
  像素分类图：
  - 像素(100,200) → 类别: 汽车
  - 像素(300,150) → 类别: 汽车
  - 像素(500,400) → 类别: 汽车
  所有汽车像素标记为同一类别

【实例分割】输出：
  实例掩码:
  - 汽车1的mask
  - 汽车2的mask
  - 汽车3的mask
  每辆汽车有独立的mask

输入图像
    ↓
Backbone (ResNet/FPN)
    ↓
RPN (Region Proposal Network)
    ↓
RoI Pooling
    ↓
分类 + 边界框回归

输入图像
    ↓
Backbone + FPN
    ↓
RPN
    ↓
RoI Align (替代RoI Pooling)
    ├── 分类分支 → 类别
    ├── 边界框分支 → 坐标
    └── Mask分支 → 像素级掩码

原始RoI: [12.4, 8.7, 45.2, 31.9]
量化后:   [12,   8,   45,   31  ]  ← 丢失小数部分

# RoI Align伪代码
def roi_align(feature_map, roi, output_size):
    """
    feature_map: 特征图
    roi: [x1, y1, x2, y2]（保留小数）
    output_size: 输出尺寸 (7, 7)
    """
    # 计算每个bin的精确位置（不量化）
    bin_height = (roi[3] - roi[1]) / output_size[0]
    bin_width = (roi[2] - roi[0]) / output_size[1]

    output = []
    for i in range(output_size[0]):
        for j in range(output_size[1]):
            # 计算bin的精确边界
            y_start = roi[1] + i * bin_height
            x_start = roi[0] + j * bin_width

            # 在bin内采样4个点（或更多）
            sampling_points = [
                (y_start + 0.25 * bin_height, x_start + 0.25 * bin_width),
                (y_start + 0.25 * bin_height, x_start + 0.75 * bin_width),
                (y_start + 0.75 * bin_height, x_start + 0.25 * bin_width),
                (y_start + 0.75 * bin_height, x_start + 0.75 * bin_width),
            ]

            # 对每个采样点使用双线性插值
            values = [bilinear_interpolate(feature_map, y, x) for y, x in sampling_points]

            # 聚合（max pooling或average）
            output.append(max(values))

    return output

class MaskHead(nn.Module):
    def __init__(self, in_channels=256, num_classes=80):
        super().__init__()

        # 4个3×3卷积（保持分辨率）
        self.conv1 = nn.Conv2d(in_channels, 256, 3, padding=1)
        self.conv2 = nn.Conv2d(256, 256, 3, padding=1)
        self.conv3 = nn.Conv2d(256, 256, 3, padding=1)
        self.conv4 = nn.Conv2d(256, 256, 3, padding=1)

        # 转置卷积（上采样2倍）
        self.deconv = nn.ConvTranspose2d(256, 256, 2, stride=2)

        # 每个类别一个mask
        self.predictor = nn.Conv2d(256, num_classes, 1)

    def forward(self, x):
        # x: RoI Align输出 (N, 256, 14, 14)

        x = F.relu(self.conv1(x))
        x = F.relu(self.conv2(x))
        x = F.relu(self.conv3(x))
        x = F.relu(self.conv4(x))

        x = F.relu(self.deconv(x))  # (N, 256, 28, 28)

        mask = self.predictor(x)  # (N, num_classes, 28, 28)

        return mask

L = L_cls + L_box + L_mask

其中：
  L_cls: 分类损失（交叉熵）
  L_box: 边界框回归损失（Smooth L1）
  L_mask: Mask损失（二元交叉熵）

def mask_loss(mask_pred, mask_target, labels):
    """
    mask_pred: (N, num_classes, H, W)
    mask_target: (N, H, W)
    labels: (N,) - 每个RoI的真实类别
    """
    N = mask_pred.size(0)
    loss = 0

    for i in range(N):
        # 只计算真实类别的mask损失
        cls = labels[i]
        pred = mask_pred[i, cls]  # (H, W)
        target = mask_target[i]   # (H, W)

        # 二元交叉熵
        loss += F.binary_cross_entropy_with_logits(pred, target)

    return loss / N

from detectron2.engine import DefaultPredictor
from detectron2.config import get_cfg
from detectron2 import model_zoo
import cv2

# 配置
cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml"))
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # 置信度阈值
cfg.MODEL.DEVICE = "cuda"

# 创建预测器
predictor = DefaultPredictor(cfg)

# 推理
image = cv2.imread("test.jpg")
outputs = predictor(image)

# 提取结果
instances = outputs["instances"].to("cpu")
boxes = instances.pred_boxes.tensor.numpy()  # 边界框
masks = instances.pred_masks.numpy()  # 实例掩码 (N, H, W)
classes = instances.pred_classes.numpy()  # 类别
scores = instances.scores.numpy()  # 置信度

# 可视化
from detectron2.utils.visualizer import Visualizer
from detectron2.data import MetadataCatalog

v = Visualizer(image[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)
out = v.draw_instance_predictions(instances)
cv2.imwrite("result.jpg", out.get_image()[:, :, ::-1])

输入图像
    ↓
RPN
    ↓
Stage 1: IoU threshold=0.5
    ↓
Stage 2: IoU threshold=0.6
    ↓
Stage 3: IoU threshold=0.7
    ↓
最终预测

粗糙mask (28×28)
    ↓
上采样到原始分辨率
    ↓
选择不确定的点（边界附近）
    ↓
对这些点进行高分辨率推理
    ↓
精细mask

输入图像
    ├── Protonet分支 → 原型masks (k个基础mask)
    └── 检测分支 → 边界框 + 类别 + mask系数
                    ↓
            线性组合原型masks
                    ↓
            实例masks

M = σ(P × C^T)

其中：
  M: 最终mask (H×W)
  P: 原型masks (H×W×k)
  C: mask系数 (k×1)
  σ: sigmoid激活

class Protonet(nn.Module):
    def __init__(self, in_channels=256, num_protos=32):
        super().__init__()

        self.conv1 = nn.Conv2d(in_channels, 256, 3, padding=1)
        self.conv2 = nn.Conv2d(256, 256, 3, padding=1)
        self.conv3 = nn.Conv2d(256, 256, 3, padding=1)

        # 上采样
        self.upsample = nn.Upsample(scale_factor=2, mode='bilinear')

        # 生成原型
        self.final_conv = nn.Conv2d(256, num_protos, 1)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = F.relu(self.conv2(x))
        x = F.relu(self.conv3(x))

        x = self.upsample(x)

        protos = self.final_conv(x)  # (B, k, H, W)
        return protos

class PredictionHead(nn.Module):
    def __init__(self, in_channels=256, num_classes=80, num_protos=32):
        super().__init__()

        # 类别预测
        self.class_conv = nn.Conv2d(in_channels, num_classes, 3, padding=1)

        # 边界框预测
        self.box_conv = nn.Conv2d(in_channels, 4, 3, padding=1)

        # Mask系数预测
        self.coef_conv = nn.Conv2d(in_channels, num_protos, 3, padding=1)

    def forward(self, x):
        classes = self.class_conv(x)
        boxes = self.box_conv(x)
        coefs = self.coef_conv(x)  # (B, k, H, W)

        return classes, boxes, coefs

def assemble_masks(protos, coefs, boxes):
    """
    protos: (H, W, k) - 原型masks
    coefs: (n, k) - n个实例的系数
    boxes: (n, 4) - 边界框
    """
    # 线性组合
    masks = torch.matmul(protos, coefs.t())  # (H, W, n)
    masks = torch.sigmoid(masks)

    # 裁剪到边界框内
    masks = crop_masks(masks, boxes)

    return masks

输入图像
    ↓
【Backbone】CSPDarknet + C2f
    ↓
【Neck】PAN-FPN
    ├── 检测分支
    │   ├── 类别预测
    │   └── 边界框预测
    └── 分割分支
        ├── Prototype生成（类似YOLACT）
        └── Mask系数预测

class_id x1 y1 x2 y2 x3 y3 ... xn yn

# person.txt
0 0.1 0.2 0.3 0.2 0.3 0.8 0.1 0.8  # 人的轮廓多边形

# custom_seg.yaml
path: /path/to/dataset
train: images/train
val: images/val

nc: 2  # 类别数
names: ['person', 'car']

from ultralytics import YOLO

# 加载预训练模型
model = YOLO('yolo11n-seg.pt')

# 训练
results = model.train(
    data='custom_seg.yaml',
    epochs=100,
    imgsz=640,
    batch=16,
    device=0,
    workers=8,

    # 分割特定参数
    overlap_mask=True,  # mask重叠
    mask_ratio=4,  # mask下采样比例

    # 数据增强
    degrees=10.0,
    translate=0.1,
    scale=0.5,
    mosaic=1.0,
    mixup=0.1,
)

def compute_mask_iou(mask1, mask2):
    """计算两个mask的IoU"""
    intersection = np.logical_and(mask1, mask2).sum()
    union = np.logical_or(mask1, mask2).sum()

    if union == 0:
        return 0.0
    return intersection / union

# COCO评估
from pycocotools.cocoeval import COCOeval

coco_eval = COCOeval(coco_gt, coco_dt, 'segm')
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()

def boundary_iou(pred_mask, gt_mask, dilation_ratio=0.02):
    """计算边界IoU"""
    from scipy.ndimage import binary_dilation

    # 提取边界
    pred_boundary = binary_dilation(pred_mask) ^ pred_mask
    gt_boundary = binary_dilation(gt_mask) ^ gt_mask

    # 计算IoU
    intersection = (pred_boundary & gt_boundary).sum()
    union = (pred_boundary | gt_boundary).sum()

    return intersection / union if union > 0 else 0

from scipy.ndimage import gaussian_filter

def smooth_mask(mask, sigma=1.0):
    """高斯平滑mask边界"""
    smoothed = gaussian_filter(mask.astype(float), sigma=sigma)
    return (smoothed > 0.5).astype(np.uint8)

def remove_small_regions(mask, min_area=100):
    """移除小区域"""
    from skimage.measure import label, regionprops

    labeled = label(mask)
    for region in regionprops(labeled):
        if region.area < min_area:
            mask[labeled == region.label] = 0

    return mask

def merge_overlapping_masks(masks, scores, iou_threshold=0.5):
    """合并重叠的mask"""
    keep = []

    # 按置信度排序
    order = np.argsort(scores)[::-1]

    while order.size > 0:
        i = order[0]
        keep.append(i)

        # 计算与其他mask的IoU
        ious = [compute_mask_iou(masks[i], masks[j]) for j in order[1:]]

        # 保留IoU低的mask
        inds = np.where(np.array(ious) <= iou_threshold)[0]
        order = order[inds + 1]

    return keep