MNIST Plus++竞赛(更加困难的mnist数据集竞赛)！ |
- 本文章是 TensorFlow2.0学习笔记 系列，欢迎关注该，专栏链接: TensorFlow2.0学习笔记，文章会持续更细，多希望大家点赞收藏加转发！
- 文章总目录链接：TensorFlow2.0学习笔记总目录!
文章目录
- 一. 数据集描述
- 1.1. 训练集(15,670个jpg文件)
- 1.2. 测试集(54,330个jpg文件)
- 二. 数据集转csv文件
- 2.1. 训练集及标签转csv文件
- 2.2. 测试集转csv文件
- 三. Inception网络结构
- 3.1. 训练模型
- 3.2. 测试模型
- 3.2. 最终预测结果result.csv
一. 数据集描述
- 首先提供数据集百度云链接：提取码：1eji
MNIST Plus++，一个困难的mnsit竞赛数据集！
- 对mnist数据集的一个困难扩充
- 为了检验模型对分类任务的推广能力和特征抓取能力
1.1. 训练集(15,670个jpg文件)
文件格式如下(命名方式)
- format("%d_%05d.jpg", label, digit_index)
- label为图像文件对应的真值（0-9）
- index为图像文件的索引（每个类别0到类别数）
1.2. 测试集(54,330个jpg文件)
文件格式如下(命名方式)
- format("%05d.jpg", index)
- index为文件编号（0-54329）
二. 数据集转csv文件
2.1. 训练集及标签转csv文件
- 训练集及标签转csv文件
#coding:utf-8
import os
import csv
import random
import numpy as np
def create_csv(dirname):
path = './'+ dirname +'/' # 修改为你自己的图片路径即可
name = os.listdir(path)
images = []
for n in name:
if n[-4:] == '.jpg':
label = n[0] # 标签；
src_label=[path+n, label]
images.append(src_label)
else:
pass
# 随机打散图片
random.shuffle(images)
random.shuffle(images)
random.shuffle(images)
with open(dirname+'.csv', mode='w', newline='') as f:
writer = csv.writer(f)
for img_label in images:
label = img_label[-1]
img = img_label[:-1][0]
writer.writerow([img, label])
# 加载数据，分为训练和测试部分。
def load_data(path, mode='train'):
# read from csv file
images, labels = [], []
with open(path+'.csv') as f:
reader = csv.reader(f)
for row in reader:
img, label = row
label = int(label)
images.append(img)
labels.append(label)
# 图片切割成，训练70%，验证15%，测试15%。
if mode == 'train': # 70% 训练集
images = images[:int(0.99 * len(images))]
labels = labels[:int(0.99 * len(labels))]
# elif mode == 'val': # 15% = 70%->85% 验证集
# images = images[int(0.70 * len(images)):int(0.98 * len(images))]
# labels = labels[int(0.70 * len(labels)):int(0.98* len(labels))]
else: # 15% = 70%->85% 测试集
images = images[int(0.99 * len(images)):]
labels = labels[int(0.99 * len(labels)):]
return images, labels
if __name__ == "__main__":
# create_csv('train')
img, label = load_data('./train', 'train')
print(len(label))
img, label = load_data('./train', 'test')
print(len(label))
# create_csv('test')
2.2. 测试集转csv文件
- 测试集转csv文件(这就是最终需要预测并且上传的结果！)
#coding:utf-8
import os
import csv
import random
import numpy as np
import tensorflow as tf
# 1. 路径变图片
def cover_to_image(x):
x = tf.io.read_file(x)
x = tf.image.decode_jpeg(x, channels=3) # RGBA
x = tf.image.resize(x, [224, 224])
return x
# 2. 计算均值和方差
#1. 0~10000: [113.85617, 108.830154, 100.53466] [67.78309, 66.85325, 69.263535]
#2. 10000~20000: [113.80364, 108.86746, 101.23807] [68.67979, 68.00103, 70.159836]
#3. 20000~30000: [113.1947, 108.1745, 100.01061] [68.5435, 67.56627, 69.880646]
#4. 30000~40000: [112.90792, 108.4116, 100.10866] [68.22623, 67.380264, 69.61325]
#5. 40000~50000: [112.88327, 108.186104, 99.96623] [67.59057, 66.67929, 69.02802]
#6. 50000~60000: [114.23421, 109.06324, 100.649376] [67.503265, 66.92577, 69.64176]
#7. 60000~70000: [114.297066, 109.30525, 101.281685] [68.341255, 67.64873, 69.79894]
######################################
# 均值： [113.59671086, 108.69118686, 100.54132729]
# 方差： [68.09538571, 67.29351486, 69.62656957]
def compute_mean_var(image):
# image.shape: [image_num, w, h, c]
mean = []
var = []
for c in range(image.shape[-1]):
mean.append(np.mean(image[..., c]))
var.append(np.std(image[..., c]))
return mean, var
# 3. 归一化图片
def norm_images(image):
# image.shape: [image_num, w, h, c]
image = image.astype('float32')
mean, var = compute_mean_var(image)
image[..., 0] = (image[..., 0] - mean[0]) / var[0]
image[..., 1] = (image[..., 1] - mean[1]) / var[1]
image[..., 2] = (image[..., 2] - mean[2]) / var[2]
return image
def create_csv(dirname):
path = './'+ dirname +'/' # 修改为你自己的图片路径即可
name = os.listdir(path)
with open (dirname + '.csv','w') as csvfile:
writer = csv.writer(csvfile)
for n in name:
if n[-4:] == '.jpg':
writer.writerow(['./'+str(dirname) +'/' + str(n)])
else:
pass
# 加载应用的数据。
def load_data(path):
# read from csv file
images = []
with open(path+'.csv') as f:
reader = csv.reader(f)
for row in reader:
img = row[0]
images.append(img)
return images
if __name__ == "__main__":
# create_csv('all_data')
images_src = load_data('./all_data')[60000:70000]
images = []
for i in range(len(images_src)):
print(i)
img_i = cover_to_image(images_src[i])
images.append(img_i.numpy())
images = np.array(images)
print(images.shape)
mean, std = compute_mean_var(images)
print(mean, std)
三. Inception网络结构
3.1. 训练模型
- 训练部分代码，使用了预训练模型！
import os
import tensorflow as tf
import numpy as np
from tensorflow import keras
from tensorflow.python.keras.api._v2.keras import layers, optimizers, losses
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from loadmnist_Inception import load_data
tf.random.set_seed(22)
np.random.seed(22)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
assert tf.__version__.startswith('2.') # 判断tf的版本是否是以‘2.’开头，如果是，则返回True，否则返回False
########################################################################################################################
img_mean = tf.constant([0.485, 0.456, 0.406])
img_std = tf.constant([0.229, 0.224, 0.225])
def normalize(x, mean=img_mean, std=img_std):
# x shape: [224, 224, 3]
# mean：shape为1；这里用到了广播机制。我们安装好右边对齐的原则，可以得到如下；
# mean : [1, 1, 3], std: [3] 先插入1
# mean : [224, 224, 3], std: [3] 再变为224
x = (x - mean)/std
return x
# 预处理的函数，复制过来。
def preprocess(x,y):
# x: 图片的路径，y：图片的数字编码
x = tf.io.read_file(x)
x = tf.image.decode_jpeg(x, channels=3) # RGBA
x = tf.image.resize(x, [224, 224])
# x = tf.pad(x, [[4, 4], [4, 4], [0, 0]]) # 上下(0轴)填充4个0，左右(1轴)填充4个0，3轴不填充。最终变为[40, 40, 3]
# x = tf.image.random_flip_left_right(x)
# x = tf.image.random_flip_up_down(x)
# x = tf.image.random_crop(x, [224,224,3])
# x: [0,255]=> -1~1
x = 2 * (tf.cast(x, dtype=tf.float32) / 255. - 0.5)
x = normalize(x)
y = tf.convert_to_tensor(y)
y = tf.one_hot(y, depth=10)
return x, y
########################################################################################################################
batchsz = 64
# creat train db 一般训练的时候需要shuffle。其它是不需要的。
images, labels = load_data('./train', 'train')
db_train = tf.data.Dataset.from_tensor_slices((images, labels)) # 变成个Dataset对象。
db_train = db_train.shuffle(1000).map(preprocess).batch(batchsz) # map函数图片路径变为内容。
# # crate validation db
# images2, labels2 = load_data('./train', 'val')
# db_val = tf.data.Dataset.from_tensor_slices((images2, labels2))
# db_val = db_val.map(preprocess).batch(batchsz)
# create test db
images3, labels3 = load_data('./train', 'test')
db_test = tf.data.Dataset.from_tensor_slices((images3, labels3))
db_test = db_test.map(preprocess).batch(batchsz)
##########################################################################################
# 导入别的已经训练好的网络和参数, 这部分工作在keras网络中提供了一些经典的网络以及经典网络训练好的参数。
# 这里使用Vgg19,还把他的权值导入进来。imagenet训练的1000类，我们就把输出层去掉。
net = keras.applications.InceptionV3(weights='imagenet', include_top=False, pooling='max')
# for i in range(len(net.layers)-16): # print(len(model.layers))=23
# net.layers[i].trainable = False
model = keras.Sequential([net, layers.Dense(10)])
model.build(input_shape=(None, 224, 224, 3))
model.summary()
##########################################################################################
# early_stopping：monitor监听器,当验证集损失值，连续增加小于0时，持续10个epoch，则终止训练。
# early_stopping = EarlyStopping(monitor='val_accuracy', min_delta=0.00001, patience=10, mode='max')
# reduce_lr：当评价指标不在提升时，减少学习率，每次减少10%，当验证损失值，持续3次未减少时，则终止训练。
reduce_lr = ReduceLROnPlateau(monitor='val_accuracy', factor=0.2, patience=10, min_lr=0.000003, mode='max')
##########################################################################################
model.compile(optimizer=optimizers.Adam(lr=1e-4),
loss=losses.CategoricalCrossentropy(from_logits=True),
metrics=['accuracy']) # 损失函数
model.fit(db_train, epochs=50, initial_epoch=0) # 1个epoch验证1次
model.evaluate(db_test)
model.save('./my_transfer_InceptionV3.h5')
3.2. 测试模型
- 保存预测结果到result.csv文件
import os
import tensorflow as tf
import numpy as np
import csv
from tensorflow import keras
from tensorflow.keras import layers, optimizers, losses
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from predict_test import load_data
tf.random.set_seed(22)
np.random.seed(22)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
assert tf.__version__.startswith('2.') # 判断tf的版本是否是以‘2.’开头，如果是，则返回True，否则返回False
##########################################################################################
img_mean = tf.constant([0.485, 0.456, 0.406])
img_std = tf.constant([0.229, 0.224, 0.225])
def normalize(x, mean=img_mean, std=img_std):
# x shape: [224, 224, 3]
# mean：shape为1；这里用到了广播机制。我们安装好右边对齐的原则，可以得到如下；
# mean : [1, 1, 3], std: [3] 先插入1
# mean : [224, 224, 3], std: [3] 再变为224
x = (x - mean)/std
return x
# 预处理的函数，复制过来。
def preprocess(x):
# x: 图片的路径，y：图片的数字编码
x = tf.io.read_file(x)
x = tf.image.decode_jpeg(x, channels=3) # RGBA
x = tf.image.resize(x, [224, 224])
# x = tf.pad(x, [[4, 4], [4, 4], [0, 0]]) # 上下(0轴)填充4个0，左右(1轴)填充4个0，3轴不填充。最终变为[40, 40, 3]
# x = tf.image.random_flip_left_right(x)
# x = tf.image.random_flip_up_down(x)
# x = tf.image.random_crop(x, [224,224,3])
# x: [0,255]=> -1~1
x = 2 * (tf.cast(x, dtype=tf.float32) / 255. - 0.5)
x = normalize(x)
return x
##########################################################################################
batchsz = 64
imgs = load_data('./test')
db_test = tf.data.Dataset.from_tensor_slices(imgs)
db_test = db_test.map(preprocess).batch(batchsz)
##########################################################################################
net = keras.applications.InceptionV3(weights='imagenet', include_top=False, pooling='max')
# for i in range(len(net.layers)-16): # print(len(model.layers))=23
# net.layers[i].trainable = False
model = keras.Sequential([net, layers.Dense(10)])
model.build(input_shape=(None, 224, 224, 3))
model.summary()
model.load_weights('./my_transfer_InceptionV3.h5')
############################################## 测试结果保存在一个list中; ####################
result_all = []
i = 1
for x in db_test:
preds = model.predict(x)
pred_result = np.argmax(preds, axis=1)
result_all.extend(pred_result) # 结果合并到一个list中;
print('epoch:', i)
i= i + 1
print(len(result_all))
############################################## 保存预测结果到result.csv文件 #################
csvfile = open("result.csv", "w+")
for i, result in enumerate(result_all):
img_num1 = imgs[i].split('/') # imgs[i]是 ./test/01677.jpg
img_num2 = img_num1[2].split('.')
img_num = img_num2[0] # 01677
result_i = result_all[i]
print('图像数:', i+1, img_num, result_i)
# 保存预测结果到result.csv文件
writer = csv.writer(csvfile)
writer.writerow((img_num, result_i))
3.2. 最终预测结果result.csv
- 选取了部分代码，第一列是图片的编号，第二列是图片预测的数字！
01677,5
17028,2
20216,2
51817,6
52902,9
11626,6
51851,4
11020,0
54059,5
45142,0
13129,6
13544,7
21336,2
04468,8
45685,6
02609,3
03635,0
06451,1
26061,0
22283,2
30184,8
09008,2
04459,6
01228,2
21062,0
39979,9
37160,8
27048,8
04408,4
51317,7
19533,0
42744,1
25122,3
35339,0
15835,1
08527,2
08047,0
52603,4
26161,3
20064,9
- 最终取得的名次！
