爬虫链接含有中文导致无法访问

解决代码：

1
2
3
4
5
6
7
8
9
import urllib 
from urllib.request import urlopen

link = "含有中文的超链接"  

# 需要用 urllib.parse.quote 转换
link = urllib.parse.quote(link)
html = urlopen(link).read() #就可以访问了，不会报异常

当爬取到的页面源代码出现乱码时

通常这种情况要确定你的编码格式是否和网站的一致

1
2
3
4
# 当网站编码格式为utf-8时，这个在页面源代码中可以查到charset=
strs = requests.get(Url)
strs.encoding = "utf-8"
strs = strs.text

xpath 基本套路

代码如下：

1
2
3
4
5
6
7
8
9
10
11
import requests
from lxml import etree

url = '我是要被处理的网址'
resp = requests.get(url)

tree = etree.HTML(resp.text)

tmp = tree.xpath('直接去F12里面复制它不香吗')

# 我个人特别喜欢用这个，比re和beautiful更方便。但是对于指定元素还是be好用

selenium 相关问题

selenium 不自动关闭浏览器

代码如下：

1
2
3
4
5
6
7
8
9
10
from selenium import webdriver
from selenium.webdriver.common.by import By #当你要使用xpath时

# 不自动关闭浏览器
option = webdriver.EdgeOptions()
option.add_experimental_option("detach", True)
option.add_argument("--disable-blink-features=AutomationControlled")#解决自动化检测
# 将option作为参数添加到edge中
driver = webdriver.Edge(options=option)
driver.get('http:\\www.baidu.com')

selenium 禁止加载图片相关设置

代码如下：

1
2
3
4
5
6
7
# 设置禁用图片的相关设置
    prefs = {
        'profile.default_content_setting_values': {
            'images': 2,  # 屏蔽图片
        }
    }
    option.add_experimental_option("prefs", prefs)

对于简单的ts文件的合成方法

代码如下：

1
2
3
with open(名字 +'.ts',mode='rb') as f1:
    with open(最后视频合成的名字+ '.mp4',mode='ab') as f2:
        f2.write(f1.read())

处理简单的登录情况

1
2
3
4
5
6
7
8
9
10
11
12
import requests

session = requests.session()
data ={
    'loginName' : '',
    'password' : ''
}
#登录
url = ''
resp = session.get(url,data=data)
#另一种方法
resp = requests.get(url,headers={'cookie':'.......'})

音视频文件合成

典型例子b站
代码如下：

1
2
3
4
5
6
7
8
9
import subprocess
merge_command = f'ffmpeg -loglevel quiet -i .\\{name}\\{title}.mp4 -i .\\{name}\\{title}.mp3 -c:v copy -c:a aac -strict experimental .\\{name}\\{title}output.mp4'
subprocess.run(merge_command, shell=True)
print('音视频合并完成')

# 当你发现合成出现问题的时候使用下面这个代码去检测问题所在
merge_command = f'ffmpeg -i .\\{name}\\{title}.mp4 -i .\\{name}\\{title}.mp3 -c:v copy -c:a aac -strict experimental .\\{name}\\{title}output.mp4'
subprocess.run(merge_command, shell=True)
print('音视频合并完成')

多线程模板

代码如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
from threading import Thread
from queue import Queue

class Spider():
    def __init__(self):
        self.qurl = Queue()
        self.thread_num = 10  # 多线程处理数量

    def produce_url(self):
        for url in 你要多线程处理的url列表:
            self.qurl.put(url) # 生成URL存入队列，等待其他线程提取

    def get_info(self):
        while not self.qurl.empty(): # 保证url遍历结束后能退出线程
            url = self.qurl.get() # 从队列中获取URL
            print(f'{url} ing')
            具体多线程处理函数(url)  #这个就是你要多线程处理的主体了

    def run(self):
        self.produce_url()
        for _ in range(self.thread_num):
            th = Thread(target=self.get_info)
            th.start()
        print('finished.')

Spider().run() #多线程！启动！

re表达式

findall

举一个栗子：

1
2
3
4
5
6
7
8
9
#finditer: 匹配字符串的所有内容|返回的是迭代器|要从迭代器拿到内容要用.group()

list = re.findall(r'\d+', '我的电话号10086，其他人的为10010')
print(list)

it = re.finditer(r'\d+','我的电话号10086，其他人的为10010')
for i in it:
    print(i.group())

search ,match

举一个栗子：

1
2
3
4
5
#search返回的是match对象，也要.group()|只会返回的一个数据
s = re.search(r'\d+','我的电话号10086，其他人的为10010')
print(s.group())

#match是从头开始匹配，这个很重要,也要用到.group()

预加载正则表达式

代码如下：

1
2
3
4
obj = re.compile(r'\d+')
ret = obj.finditer('我的电话号10086，其他人的为10010')
for i in ret:
    print(i.group())

给每一个匹配命名分组

代码如下：

1
2
3
4
5
6
7
8
9
10
11
12

s = '''
<div class='jay'><span id='1'>中国移动</span></div>
<div class='jfsd'><span id='2'>中国联通</span></div>
<div class='asds'><span id='3'>中国电信</span></div>
'''

obj2 = re.compile(r"<div class='.*?'><span id='\d+'>(?P<yyy>.*?)</span></div>",re.S)
result = obj2.finditer(s)
for i in result:
    print(i.group('yyy'))

常用模板

1
2
3
chinese_pattern = re.compile(r'[\u4e00-\u9fa5]')  # 匹配中文字符
alphanumeric_pattern = re.compile(r'[a-zA-Z0-9]')  # 匹配字母数字
patten = re.compile(r'[\u4e00-\u9fa5a-zA-Z0-9]')

pandas 之 to_csv 保存数据出现中文乱码问题及解决办法

代码如下:

1
data.to_csv('data.csv',encoding='utf_8_sig')

conda常用指令

  1. 查看安装了哪些包
1
conda list
  1. 查看当前存在哪些虚拟环境
1
2
conda env list 
conda info -e
  1. 检查更新当前conda
1
conda update conda
  1. Python创建虚拟环境
    anaconda命令创建python版本为x.x，名字为your_env_name的虚拟环境。your_env_name文件可以在Anaconda安装目录envs文件下找到。
1
conda create -n your_env_name python=x.x
  1. 激活或者切换虚拟环境

打开命令行，输入python --version检查当前 python 版本。

1
2
Linux:  source activate your_env_nam
Windows: activate your_env_name
  1. 对虚拟环境中安装额外的包
1
conda install -n your_env_name [package]
  1. 关闭虚拟环境(即从当前环境退出返回使用PATH环境中的默认python版本)
1
2
3
deactivate env_name
或者`activate root`切回root环境
Linux下：source deactivate
  1. 删除虚拟环境
1
conda remove -n your_env_name --all
  1. 删除环境钟的某个包
1
conda remove --name $your_env_name  $package_name

Cifar10图像分类(CNN模板)

任务介绍：

  1. 数据准备：
    下载并加载 CIFAR-10 数据集。
    对数据集进行预处理，包括归一化、图像增强等操作。
  2. 模型构建：
    使用 PyTorch 构建卷积神经网络模型。
    设计合适的网络结构，包括卷积层、池化层、全连接层等。
  3. 模型训练：
    定义损失函数和优化器。
    将数据集分为训练集和验证集。
    在训练集上训练模型，通过验证集调整模型参数，避免过拟合。
  4. 模型评估：
    使用测试集评估模型性能，计算准确率等指标。
  5. 可视化展示：
    通过表格展示准确率等实验结果。
    绘制准确率和损失函数随训练轮次变化的曲线图。
    随机选取部分图像，展示模型的预测结果和真实标签。

模板如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
import numpy as np
import pandas as pd
import torch
import torchvision
import torchvision.transforms as transforms
from matplotlib import pyplot as plt
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

# 定义网络结构
class CNN_NET(torch.nn.Module):
    def __init__(self):
        super(CNN_NET, self).__init__()
        self.conv1 = torch.nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, stride=1, padding=1)
        self.conv1_bn = torch.nn.BatchNorm2d(32)
        self.conv2 = torch.nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, stride=1, padding=1)
        self.conv2_bn = torch.nn.BatchNorm2d(64)
        self.pool = torch.nn.MaxPool2d(kernel_size=2, stride=2)
        self.conv3 = torch.nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=1, padding=1)
        self.conv3_bn = torch.nn.BatchNorm2d(128)
        self.fc1 = torch.nn.Linear(128 * 4 * 4, 512)
        self.fc2 = torch.nn.Linear(512, 256)
        self.fc3 = torch.nn.Linear(256, 10)
        self.dropout = torch.nn.Dropout(0.5)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1_bn(self.conv1(x))))
        x = self.pool(F.relu(self.conv2_bn(self.conv2(x))))
        x = self.pool(F.relu(self.conv3_bn(self.conv3(x))))
        x = x.view(-1, 128 * 4 * 4)
        x = F.relu(self.fc1(x))
        x = self.dropout(x)
        x = F.relu(self.fc2(x))
        x = self.dropout(x)
        x = self.fc3(x)
        return x

if __name__ == '__main__':
    BATCH_SIZE = 4  # 批处理
    EPOCH = 15  # 训练周期
    classer = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']

    # 图像预处理
    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

    # 加载数据集
    trainset = torchvision.datasets.CIFAR10(root='./data', train=True, download=False, transform=transform)
    trainloader = torch.utils.data.DataLoader(trainset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)

    testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=False, transform=transform)
    testloader = torch.utils.data.DataLoader(testset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)

    net = CNN_NET()  # 初始化网络

    criterion = nn.CrossEntropyLoss()  # 损失函数
    optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)  # 优化器

    '''# 替换优化器为 Adam
    optimizer = optim.Adam(net.parameters(), lr=0.001)
    '''

    # 训练网络
    losses = []
    accuracies = []
    for epoch in range(EPOCH):
        running_loss = 0.0
        correct = 0
        total = 0
        for i, data in enumerate(trainloader, 0):
            inputs, labels = data
            optimizer.zero_grad()  # 梯度清零
            outputs = net(inputs)  # 前向传播
            loss = criterion(outputs, labels)  # 计算损失
            loss.backward()  # 反向传播
            optimizer.step()  # 更新参数
            running_loss += loss.item()
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
            if i % 2000 == 1999:  # 每 2000 个 mini-batches 输出一次
                print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 2000))
                losses.append(running_loss / 2000)
                accuracies.append(100 * correct / total)
                running_loss = 0.0

    print('Finished Training')

    # 可视化训练损失和准确率
    fig, ax1 = plt.subplots()

    color = 'tab:red'
    ax1.set_xlabel('Iterations')
    ax1.set_ylabel('Loss', color=color)
    ax1.plot(losses, color=color)
    ax1.tick_params(axis='y', labelcolor=color)

    ax2 = ax1.twinx()
    color = 'tab:blue'
    ax2.set_ylabel('Accuracy (%)', color=color)
    ax2.plot(accuracies, color=color)
    ax2.tick_params(axis='y', labelcolor=color)

    fig.tight_layout()
    plt.title('Training Loss and Accuracy')
    plt.show()

    # 在测试集上测试模型
    correct = 0
    total = 0
    with torch.no_grad():
        for data in testloader:
            images, labels = data
            outputs = net(images)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

    print('Accuracy of the network on the 10000 test images: %d %%' % (100 * correct / total))

MLP代码示范

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from torch.utils.data import DataLoader, TensorDataset, Dataset
import matplotlib.pyplot as plt

# 加载鸢尾花数据集
iris = load_iris()
X = iris.data
y = iris.target

# 数据标准化
scaler = StandardScaler()
X = scaler.fit_transform(X)

# 划分数据集为训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 转换为PyTorch张量
X_train_tensor = torch.tensor(X_train, dtype=torch.float32)
y_train_tensor = torch.tensor(y_train, dtype=torch.long)
X_test_tensor = torch.tensor(X_test, dtype=torch.float32)
y_test_tensor = torch.tensor(y_test, dtype=torch.long)


####################### 定义数据集类为数据分批训练做准备#######################

class Mydata(Dataset):
    def __init__(self, instances, labels):
        self.instances = instances
        self.labels = labels

    def __len__(self):
        return self.instances.shape[0]

    def __getitem__(self, index):
        return self.instances[index], self.labels[index]


# 定义多层感知机模型
class MLP(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(MLP, self).__init__()
        self.fc1 = nn.Linear(input_size, hidden_size)
        self.relu = nn.ReLU()
        self.fc2 = nn.Linear(hidden_size, output_size)

    def forward(self, x):
        x = self.fc1(x)
        x = self.relu(x)
        x = self.fc2(x)
        return x


# 定义模型参数
input_size = X_train.shape[1]
hidden_size = 64
output_size = len(set(y_train))

# 初始化模型、损失函数和优化器
model = MLP(input_size, hidden_size, output_size)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

# 将数据转换为DataLoader
# train_dataset = TensorDataset(X_train_tensor, y_train_tensor)

train_dataset = Mydata(X_train_tensor, y_train_tensor)
train_loader = DataLoader(train_dataset, batch_size=10, shuffle=True)

# 训练模型
loss_epoch = []
epochs = 200
for epoch in range(epochs):
    running_loss = 0.0
    for i, (inputs, labels) in enumerate(train_loader):
        # for inputs, labels in train_loader:
        optimizer.zero_grad()
        outputs = model(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        # if (i+1) % 10 == 0:  
        #     print(f'Epoch [{epoch+1}/{epochs}], Step [{i+1}/{len(train_loader)}], Loss: {loss.item()}')  
        running_loss += loss.item()
    print(f"Epoch {epoch + 1}/{epochs}, Loss: {running_loss}")
    loss_epoch.append(running_loss)

plt.plot(loss_epoch)
plt.xlabel(xlabel='epoch')
plt.ylabel(ylabel='loss')

# 模型评估
model.eval()
with torch.no_grad():
    outputs = model(X_test_tensor)
    _, predicted = torch.max(outputs, 1)
    accuracy = (predicted == y_test_tensor).sum().item() / len(y_test)
    print(f"Test Accuracy: {accuracy}")

保留最优模型

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
import numpy as np
import pandas as pd
import torch
import torchvision.transforms as transforms
from matplotlib import pyplot as plt
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision.datasets import ImageFolder
from torch.utils.data import DataLoader

# 定义网络结构
class CNN_NET(torch.nn.Module):
    """人马是二分类问题
    """

    def __init__(self):
        super(CNN_NET, self).__init__()
        self.conv1 = torch.nn.Conv2d(in_channels=3, out_channels=128, kernel_size=4, stride=2, padding=1)
        self.conv1_bn = torch.nn.BatchNorm2d(128)
        self.conv2 = torch.nn.Conv2d(in_channels=128, out_channels=64, kernel_size=3, stride=1, padding=1)
        self.conv2_bn = torch.nn.BatchNorm2d(64)
        self.pool = torch.nn.MaxPool2d(kernel_size=2, stride=2)
        self.conv3 = torch.nn.Conv2d(in_channels=64, out_channels=32, kernel_size=3, stride=1, padding=1)
        self.conv3_bn = torch.nn.BatchNorm2d(32)
        self.conv4 = torch.nn.Conv2d(in_channels=32, out_channels=16, kernel_size=2, stride=1, padding=1)
        self.conv4_bn = torch.nn.BatchNorm2d(16)
        self.fc1 = torch.nn.Linear(9 * 9 * 16, 1024)
        self.fc2 = torch.nn.Linear(1024, 512)
        self.fc3 = torch.nn.Linear(512, 256)
        self.fc4 = torch.nn.Linear(256, 128)
        self.fc5 = torch.nn.Linear(128, 64)
        self.fc6 = torch.nn.Linear(64, 2)
        self.dropout = torch.nn.Dropout(0.5)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1_bn(self.conv1(x))))
        x = self.pool(F.relu(self.conv2_bn(self.conv2(x))))
        x = self.pool(F.relu(self.conv3_bn(self.conv3(x))))
        x = self.pool(F.relu(self.conv4_bn(self.conv4(x))))
        x = x.view(-1, 9 * 9 * 16)
        x = F.relu(self.fc1(x))
        x = self.dropout(x)
        x = F.relu(self.fc2(x))
        x = self.dropout(x)
        x = F.relu(self.fc3(x))
        x = self.dropout(x)
        x = F.relu(self.fc4(x))
        x = self.dropout(x)
        x = F.relu(self.fc5(x))
        x = self.dropout(x)
        x = self.fc6(x)
        return x

if __name__ == '__main__':
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    print(device)
    EPOCH = 8
    
    # 定义数据预处理
    transform = transforms.Compose([
        transforms.Resize((300, 300)),
        transforms.ToTensor(),  # 将图像转换为张量
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))  # 标准化图像数据
    ])

    # 加载数据集
    train_dataset = ImageFolder(root='data/horse-or-human/horse-or-human', transform=transform)
    test_dataset = ImageFolder(root='data/horse-or-human/validation-horse-or-human', transform=transform)

    # 创建数据加载器
    train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
    test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)

    net = CNN_NET().to(device)  # 初始化网络并移到GPU

    criterion = nn.CrossEntropyLoss()  # 损失函数
    optimizer = optim.Adam(net.parameters(), lr=0.001)  # 优化器
    
    best_accuracy = 0.0
    best_model_path = 'best_model.pth'

    # 训练网络
    losses = []
    accuracies = []
    for epoch in range(EPOCH):
        running_loss = 0.0
        correct = 0
        total = 0
        
        for i, data in enumerate(train_loader, 0):
            inputs, labels = data
            inputs, labels = inputs.to(device), labels.to(device)  # 将数据移到GPU
            optimizer.zero_grad()  # 梯度清零
            outputs = net(inputs)  # 前向传播
            loss = criterion(outputs, labels)  # 计算损失
            loss.backward()  # 反向传播
            optimizer.step()  # 更新参数
            running_loss += loss.item()
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
        
        # 计算验证集上的准确率
        net.eval()
        with torch.no_grad():
            correct_val = 0
            total_val = 0
            for data in test_loader:
                images, labels = data
                images, labels = images.to(device), labels.to(device)
                outputs = net(images)
                _, predicted = torch.max(outputs.data, 1)
                total_val += labels.size(0)
                correct_val += (predicted == labels).sum().item()
            val_accuracy = correct_val / total_val
            
            # 如果当前模型性能优于最佳模型，则保存当前模型
            if val_accuracy > best_accuracy:
                torch.save(net.state_dict(), best_model_path)
                best_accuracy = val_accuracy
        
        net.train()  # 恢复模型为训练模式
        
        # 打印训练信息
        print(f'Epoch [{epoch + 1}/{EPOCH}], Loss: {running_loss / len(train_loader):.4f}, Accuracy: {100 * correct / total:.2f}%, Validation Accuracy: {100 * correct_val / total_val:.2f}%')

    print('Finished Training')

    # 可视化训练损失和准确率
    fig, ax1 = plt.subplots()

    color = 'tab:red'
    ax1.set_xlabel('Iterations')
    ax1.set_ylabel('Loss', color=color)
    ax1.plot(losses, color=color)
    ax1.tick_params(axis='y', labelcolor=color)

    ax2 = ax1.twinx()
    color = 'tab:blue'
    ax2.set_ylabel('Accuracy (%)', color=color)
    ax2.plot(accuracies, color=color)
    ax2.tick_params(axis='y', labelcolor=color)

    fig.tight_layout()
    plt.title('Training Loss and Accuracy')
    plt.show()

    # 加载最佳模型并在测试集上测试
    best_model = CNN_NET().to(device)
    best_model.load_state_dict(torch.load(best_model_path))
    best_model.eval()
    
    correct = 0
    total = 0
    with torch.no_grad():
        for data in test_loader:
            images, labels = data
            images, labels = images.to(device), labels.to(device)  # 将数据移到GPU
            outputs = best_model(images)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

    print('Accuracy of the best model on the test images: %d %%' % (100 * correct / total))

下面是不保留：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
import numpy as np
import pandas as pd
import torch
import torchvision.transforms as transforms
from matplotlib import pyplot as plt
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision.datasets import ImageFolder
from torch.utils.data import DataLoader



# 定义网络结构
class CNN_NET(torch.nn.Module):
    """人马是二分类问题
    """

    def __init__(self):
        super(CNN_NET, self).__init__()
        self.conv1 = torch.nn.Conv2d(in_channels=3, out_channels=128, kernel_size=4, stride=2, padding=1)
        self.conv1_bn = torch.nn.BatchNorm2d(128)

        self.conv2 = torch.nn.Conv2d(in_channels=128, out_channels=64, kernel_size=3, stride=1, padding=1)
        self.conv2_bn = torch.nn.BatchNorm2d(64)

        self.pool = torch.nn.MaxPool2d(kernel_size=2, stride=2)

        self.conv3 = torch.nn.Conv2d(in_channels=64, out_channels=32, kernel_size=3, stride=1, padding=1)
        self.conv3_bn = torch.nn.BatchNorm2d(32)

        self.conv4 = torch.nn.Conv2d(in_channels=32, out_channels=16, kernel_size=2, stride=1, padding=1)
        self.conv4_bn = torch.nn.BatchNorm2d(16)

        self.fc1 = torch.nn.Linear(9 * 9 * 16, 1024)

        self.fc2 = torch.nn.Linear(1024, 512)

        self.fc3 = torch.nn.Linear(512, 256)

        self.fc4 = torch.nn.Linear(256, 128)

        self.fc5 = torch.nn.Linear(128, 64)

        self.fc6 = torch.nn.Linear(64, 2)

        self.dropout = torch.nn.Dropout(0.5)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1_bn(self.conv1(x))))
        x = self.pool(F.relu(self.conv2_bn(self.conv2(x))))
        x = self.pool(F.relu(self.conv3_bn(self.conv3(x))))
        x = self.pool(F.relu(self.conv4_bn(self.conv4(x))))

        x = x.view(-1, 9 * 9 * 16)
        x = F.relu(self.fc1(x))
        x = self.dropout(x)
        x = F.relu(self.fc2(x))
        x = self.dropout(x)
        x = F.relu(self.fc3(x))
        x = self.dropout(x)
        x = F.relu(self.fc4(x))
        x = self.dropout(x)
        x = F.relu(self.fc5(x))
        x = self.dropout(x)
        x = self.fc6(x)
        return x

if __name__ == '__main__':
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    print(device)
    EPOCH = 8
    # 定义数据预处理
    transform = transforms.Compose([
        transforms.Resize((300, 300)),
        transforms.ToTensor(),  # 将图像转换为张量
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))  # 标准化图像数据
    ])

    # 加载数据集
    train_dataset = ImageFolder(root='data/horse-or-human/horse-or-human', transform=transform)
    test_dataset = ImageFolder(root='data/horse-or-human/validation-horse-or-human', transform=transform)

    # 创建数据加载器
    train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
    test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)

    net = CNN_NET().to(device)  # 初始化网络并移到GPU

    criterion = nn.CrossEntropyLoss()  # 损失函数
    optimizer = optim.Adam(net.parameters(), lr=0.001)  # 优化器

    # 训练网络
    losses = []
    accuracies = []
    for epoch in range(EPOCH):
        running_loss = 0.0
        correct = 0

        total = 0
        for i, data in enumerate(train_loader, 0):
            inputs, labels = data
            inputs, labels = inputs.to(device), labels.to(device)  # 将数据移到GPU
            optimizer.zero_grad()  # 梯度清零
            outputs = net(inputs)  # 前向传播
            loss = criterion(outputs, labels)  # 计算损失
            loss.backward()  # 反向传播
            optimizer.step()  # 更新参数
            running_loss += loss.item()
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
            if i % 10 == 9:  # 每10个批次打印一次信息
                print(f'[Epoch {epoch + 1}, Batch {i + 1}] loss: {running_loss / 10:.3f}')
                losses.append(running_loss / 10)
                accuracies.append(100 * correct / total)
                running_loss = 0.0

    print('Finished Training')

    # 可视化训练损失和准确率
    fig, ax1 = plt.subplots()

    color = 'tab:red'
    ax1.set_xlabel('Iterations')
    ax1.set_ylabel('Loss', color=color)
    ax1.plot(losses, color=color)
    ax1.tick_params(axis='y', labelcolor=color)

    ax2 = ax1.twinx()
    color = 'tab:blue'
    ax2.set_ylabel('Accuracy (%)', color=color)
    ax2.plot(accuracies, color=color)
    ax2.tick_params(axis='y', labelcolor=color)

    fig.tight_layout()
    plt.title('Training Loss and Accuracy')
    plt.show()

    # 在测试集上测试模型
    correct = 0
    total = 0
    with torch.no_grad():
        for data in test_loader:
            images, labels = data
            images, labels = images.to(device), labels.to(device)  # 将数据移到GPU
            outputs = net(images)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

    print('Accuracy of the network on the 10000 test images: %d %%' % (100 * correct / total))