13。数据集处理

　　视频：13.数据集处理import os import numpy as np import matplotlib.pyplot as plt import matplotlib.image as mpimg  #查看显示图像 datasetdir=＂catdog＂ filepath=＂catdog/cat/cat.1.jpg＂ img=mpimg.imread(filepath) plt.imshow(img) plt.show()  classname=[] #classname=os.listdir(datasetdir) for fpath in os.listdir(datasetdir):     if os.path.isdir(os.path.join(datasetdir,fpath)):         classname.append(fpath)  print(classname)  with open(＂labels.txt＂,＂w＂) as f:     for v in classname:         f.write(v+＂ ＂)  datalst=[] for i,fdir in enumerate(classname):     for fname in os.listdir(os.path.join(datasetdir,fdir)):         datalst.append([os.path.join(fdir,fname),i]) print(datalst)  datanp=np.array(datalst) print(datanp.shape) np.random.shuffle(datanp) print(datanp)  #train:val:test=7:2:1 ratiotrain=0.7 ratioval=0.2 ratiotest=0.1 train=datanp[:int(datanp.shape[0]*ratiotrain)] val=datanp[int(datanp.shape[0]*ratiotrain):int(datanp.shape[0]*(ratiotrain+ratioval))] test=datanp[int(datanp.shape[0]*(ratiotrain+ratioval)):] print(train.shape,val.shape,test.shape)  with open(＂train_list.txt＂,＂w＂) as f:     for v in train:         f.write(v[0]+＂	＂+v[1]+＂ ＂)  with open(＂val_list.txt＂,＂w＂) as f:     for v in val:         f.write(v[0]+＂	＂+v[1]+＂ ＂)  with open(＂test_list.txt＂,＂w＂) as f:     for v in test:         f.write(v[0]+＂	＂+v[1]+＂ ＂)  import cv2 def transform_img(img):     img=cv2.resize(img,(224,224))     img=np.transpose(img,(2,0,1)) #[H,W,C]=>[C,H,W]     img=img.astype(np.float32)     img=img/127.5-1.0 #[-1.0,1.0]             return img  datalst=[] with open(＂train_list.txt＂) as f:     data=f.readlines() for v in data:     s=v.split(＂	＂)     imgpath=s[0]     label=s[1][:-1]     datalst.append([imgpath,label]) datanp=np.array(datalst) print(datanp.shape,datanp[0])  datalst=[] def data_loader(data=datanp,batch_size=10,mode=＂train＂):             def reader():         if mode==＂train＂:             np.random.shuffle(data)         batch_imgs=[]         batch_labels=[]         for v in data:             img=cv2.imread(os.path.join(datasetdir,v[0]))             img=transform_img(img)             batch_imgs.append(img)             batch_labels.append(v[1])             if len(batch_imgs)==batch_size:                 imgs_array=np.array(batch_imgs).astype(np.float32)                 labels_array=np.array(batch_labels).astype(np.int64)                 yield imgs_array,labels_array                 batch_imgs=[]                 batch_labels=[]                          if len(batch_imgs)>0:             imgs_array=np.array(batch_imgs).astype(np.float32)             labels_array=np.array(batch_labels).astype(np.int64)             yield imgs_array,labels_array             batch_imgs=[]             batch_labels=[]              return reader  for i,batch in enumerate(data_loader(datanp,batch_size=10,mode=＂train＂)()):     print(i)     print(batch[1].shape,batch[0].shape)     break