PyTorch_Note

PyTorch

1. PyTorch_tutorial

• Pytorch_60min.md60PyTorch
• Pytorch_Basic.py
• Pytorch_linearRegression.py
• Pytorch_logisticRegression.pyMINIST+
• Pytorch_NNdemo.pyMINIST+
• Pytorch_CNNMINST+
• pytorch_cuda.ipynbPyTorchcuda
• LeNet.ipynbPyTorchLeNet
• ResNet.ipynbPyTorchResNet
• Pytorch_PyTorch7
• DenseNet_PyTorchDenseNetPyTorch
• PyTorchload
• PyTorch-torch.utils.datasubset
• PyTorch
• OpenCV-
• 60PyTorchPyTorch
• PyTorchgpugpu
• PyTorch_trickPyTorch
• lrPolypoly
• PyTorchBroadcastingtorchtensorBroadcasting
• PyTorchcontiguous()
• PyTorchnn.DataParalleltorch.distributedtorch.multiprocessingapexhorovod
• torch.mean()

2. PyTorch__1

• unet:

RuntimeError: Given groups=1, weight of size 64 3 3 3, expected input[4, 64, 158, 158] to have 3 channels, but got 64 channels instead

• 3 channels64

• size:[4,3,160,160],

class DoubleConv(nn.Module):
	def __init__(self, in_channels, out_channels):
		super.__init__()
		# 
		self.double_conv = nn.Sequential(
			nn.Conv2d(in_channels,out_channels,kernel_size=3),
			nn.BatchNorm2d(out_channels),
			nn.ReLU(inplace=True),
			nn.Conv2d(in_channels,out_channels,kernel_size=3),
			nn.BatchNorm2d(out_channels),
			nn.ReLU(inplace=True)
		)
	def forward(self,x):
		return self.double_conv(x)

• 25in_channelsout_channels,

class DoubleConv(nn.Module):
	def __init__(self, in_channels, out_channels):
		super.__init__()
		# 
		self.double_conv = nn.Sequential(
			nn.Conv2d(in_channels,out_channels,kernel_size=3),
			nn.BatchNorm2d(out_channels),
			nn.ReLU(inplace=True),
			nn.Conv2d(out_channels,out_channels,kernel_size=3),
			nn.BatchNorm2d(out_channels),
			nn.ReLU(inplace=True)
		)
	def forward(self,x):
		return self.double_conv(x)

3. PyTorch_cv2_Tensor

• cv2imageshape[H, W]

• torchvision.ToTensor()shape[C, H, W],[0,1.0]torch.FloatTensor

import cv2
import torch
import torchvision.transforms

image_name = 'Dataset/2d_images/ID_0000_Z_0142.tif'
image = cv2.imread(image_name, 0)
print(image.shape) # (512,512)
image_tensor = torchvision.transforms.ToTensor()(image)
print(image_tensor.shape) # torch.Size([1, 512, 512])

• shaperesizereshapetensor

import torch
import torchvision.transforms
import cv2

image_name = 'Dataset/2d_images/ID_0000_Z_0142.tif'
image = cv2.imread(image_name, 0)
print(image.shape)
image = cv2.resize(image, (160, 160))
image_new = image.reshape((1, 160, 160))
image_tensor = torch.FloatTensor(image_new)
print(image_new.shape)
print(image_tensor.shape)

# 
(512, 512)
(1, 160, 160)
torch.Size([1, 160, 160])

• cv2.resize(img, (width, height)):

• class torchvision.transforms.ToTensor:

  • [0,255]``PIL.Image``shape``(H,W,C)``numpy.ndarray``[C,H,W]``[0,1.0]``torch.FloatTensor

• class torchvision.transforms.Normalize(mean, std):

  • (R,G,B) RGB``Tensor``Normalized_image=(image-mean)/std

• cv2.imread(img, 1)type: numpy.ndarray

4. PyTorch

4.1 Module

• MLPModule__init__forward

import torch
from torch import nn

class MLP(nn.Module):
    # 
    def __init__(self, **kwargs):
        # MLPModule
        # params
        super(MLP, self).__init__(**kwargs)
        self.hidden = nn.Linear(784, 256) # 
        self.act = nn.ReLU()
        self.output = nn.Linear(256, 10)  # 


    # x
    def forward(self, x):
        a = self.act(self.hidden(x))
        return self.output(a)

4.2 Sequential

• OrderedDictModule

import torch
from torch import nn
class LeNet(nn.Module):
    def __init__(self):
        super(LeNet, self).__init__()
        self.conv = nn.Sequential(
            nn.Conv2d(1, 6, 5), # in_channels, out_channels, kernel_size
            nn.Sigmoid(),
            nn.MaxPool2d(2, 2), # kernel_size, stride
            nn.Conv2d(6, 16, 5),# in_channels, out_channels, kernel_size
            nn.Sigmoid(),
            nn.MaxPool2d(2, 2)
        )
        self.fc = nn.Sequential(
            nn.Linear(16*4*4, 120),
            nn.Sigmoid(),
            nn.Linear(120, 84),
            nn.Sigmoid(),
            nn.Linear(84, 10)
        )
    
    def forward(self, img):
        feature = self.conv(img)
        # viewreshapeimg.shape[0]batch_size-1H*W*Channels
        output = self.fc(feature.view(img.shape[0], -1))
        return output

4.3 ModuleList

• ModuleListListappendextend:

net = nn.ModuleList([nn.Linear(784, 256), nn.ReLU()])
net.append(nn.Linear(256, 10)) # # Listappend
print(net[-1])  # List
print(net)
# net(torch.zeros(1, 784)) # NotImplementedError
# 
Linear(in_features=256, out_features=10, bias=True)
ModuleList(
  (0): Linear(in_features=784, out_features=256, bias=True)
  (1): ReLU()
  (2): Linear(in_features=256, out_features=10, bias=True)
)

• SequentialModuleList
• ModuleListforwardnet(torch.zeros(1, 784))NotImplementedError
• Sequentialforward

4.4 ModuleDict

• ModuleDict, :

net = nn.ModuleDict({
    'linear': nn.Linear(784, 256),
    'act': nn.ReLU(),
})
net['output'] = nn.Linear(256, 10) # 
print(net['linear']) # 
print(net.output)
print(net)
# net(torch.zeros(1, 784)) # NotImplementedError
# 
Linear(in_features=784, out_features=256, bias=True)
Linear(in_features=256, out_features=10, bias=True)
ModuleDict(
  (act): ReLU()
  (linear): Linear(in_features=784, out_features=256, bias=True)
  (output): Linear(in_features=256, out_features=10, bias=True)
)

• ModuleListModuleDictforward
• ModuleDictPythonDictModuleDict

5. PyTorchCrossEntropyLoss

• labelonehot(4,4,640,640)loss

criterion = nn.CrossEntropyLoss().to(device)
loss = criterion(output, label)

• target34

• softmax****target0C-1label0-34
  
• lossshape4input(N,C,H,W)target(N, H, W)target[i]0-C-1
• onehotlabel

6. TensorboardX

pip install tensorboardX

• : writer.add_scalar(x)

from tensorboardX import SummaryWriter
writer.add_scalar('train_loss', train_loss/len(train_dataloader), epo)

7.

• Image
• deeplabv3-resnetresnetdeeplabv3asppFC

model_urls = {
    'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
    'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
    'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
}

# ResNetdeeplabv3
class ResNet(nn.Module):
    def __init__(self, block, block_num, num_classes, num_groups=None, weight_std=False, beta=False, pretrained=False):
        self.inplanes = 64 #  planes:
        # nn.BatchNorm2dnn.GroupNorm
        self.norm = nn.BatchNorm2d
        self.conv = Conv2d if weight_std else nn.Conv2d
        super(ResNet, self).__init__()

        if not beta:
            # ResNetconv
            self.conv1 = self.conv(3, 64, kernel_size=7, stride=2, padding=3,
                                   bias=False)
        else:
            # conv
            self.conv1 = nn.Sequential(
                self.conv(3, 64, 3, stride=2, padding=1, bias=False),
                self.conv(64, 64, 3, stride=1, padding=1, bias=False),
                self.conv(64, 64, 3, stride=1, padding=1, bias=False))
        self.bn1 = self.norm(64)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)

        # 
        self.layer1 = self._make_layer(block, 64,  block_num[0])
        self.layer2 = self._make_layer(block, 128, block_num[1], stride=2)
        self.layer3 = self._make_layer(block, 256, block_num[2], stride=2)
        # block4dilation
        self.layer4 = self._make_layer(block, 512, block_num[3], stride=1, dilation=2)
        # aspp,512 * block.expansion
        self.aspp = ASPP(512 * block.expansion, 256, num_classes, conv=self.conv, norm=self.norm)
        # 
        for m in self.modules():
            if isinstance(m, self.conv):        #isinstancem     conv
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))  #
            elif isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.GroupNorm): #batchnorm
                m.weight.data.fill_(1)          #weight1
                m.bias.data.zero_()             #bias0

        if pretrained:
            self._load_pretrained_model()

    def _make_layer(self, block, planes, blocks, stride=1, dilation=1):
        downsample = None
        # stride!=1 stride=2stride
        if stride != 1 or dilation != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                self.conv(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, dilation=max(1, dilation/2), bias=False),
                self.norm(planes * block.expansion),
            )
        # laysers 
        layers = []
        # downsampleblock
        layers.append(block(self.inplanes, planes, stride, downsample, dilation=max(1, dilation/2), conv=self.conv, norm=self.norm))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes, dilation=dilation, conv=self.conv, norm=self.norm))

        return nn.Sequential(*layers)


    def forward(self, x):
        # x.shape:[batch_size, channels, H, w]
        size = (x.shape[2], x.shape[3])
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)

        # ASPP
        x = self.aspp(x)
        #x = x.reshape(-1, x.shape[1])
        x = nn.Upsample(size, mode='bilinear', align_corners=True)(x)
        return x
    #     
    def _load_pretrained_model(self):
        pretrain_dict = model_zoo.load_url(model_urls['resnet152'])
        model_dict = {}
        state_dict = self.state_dict()
        for k, v in pretrain_dict.items():
            if k in state_dict:
                model_dict[k] = v
        state_dict.update(model_dict)
        self.load_state_dict(state_dict)

def resnet152(pretrained=True, **kwargs):
    """Constructs a ResNet-152 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(Bottleneck, [3, 8, 36, 3], num_classes=4, pretrained=pretrained, **kwargs)
    return model

8. batch_size

• 320x320192batchRuntime Error
• batch
• padresize320label

9. cuda

• cuda:3"cuda:3"4
• RuntimeError: Attempting to deserialize object on CUDA device 3 but torch.cuda.device_count() is 2. Please use torch.load with map_location to map your storages to an existing device.
• :

device = torch.device("cuda:0")
model = torch.load(PATH, map_location=device)