有些時候，我們的輸入不是標準的圖像，而是其它一些格式，比如：頻譜圖、特征向量等等，這種情況下LMDB、Leveldb以及ImageData layer等就不好使了，這時候我們就需要一個新的輸入接口——HDF5Data.

本文主要介紹一種應用python hdf5軟件庫制作hdf5數據源的方法.

代碼示例

import h5py
import os
import cv2
import math
import numpy as np
import random
import re

root_path = "/home/tyd/caffe_case/HDF5/image" # 圖片存放的路經

with open("/home/tyd/caffe_case/HDF5/hdf5.txt", 'r') as f: # txt文件中存放的圖片路經和相應回歸數據
    lines = f.readlines()

num = len(lines)
random.shuffle(lines)


imgAccu = 0
imgs = np.zeros([num, 3, 224, 224]) # 用于存放圖片數據
labels = np.zeros([num, 10]) # 存放標簽
for i in range(num):
    line = lines[i]
    segments = re.split('\s+', line)[:-1]
    print segments[0]
    img = cv2.imread(os.path.join(root_path, segments[0]))
    img = cv2.resize(img, (224, 224))
    img = img.transpose(2,0,1)
    imgs[i,:,:,:] = img.astype(np.float32)
    for j in range(10):
        labels[i,j] = float(segments[j+1])*224/256

batchSize = 1  # 每一個hdf5文件中存放圖片的數量,一般不超過8000個
batchNum = int(math.ceil(1.0*num/batchSize))

imgsMean = np.mean(imgs, axis=0)
# imgs = (imgs - imgsMean)/255.0  # 數據中心化,標準化
labelsMean = np.mean(labels, axis=0)    # 標簽中心化,標準化(預測時需要加上mean值)
labels = (labels - labelsMean)/10

if os.path.exists('trainlist.txt'):
    os.remove('trainlist.txt')
if os.path.exists('testlist.txt'):
    os.remove('testlist.txt')
comp_kwargs = {'compression': 'gzip', 'compression_opts': 1}

## 將數據寫入hdf5文件中
for i in range(batchNum):
    start = i*batchSize
    end = min((i+1)*batchSize, num)
    if i < batchNum-1:
        filename = '/home/tyd/caffe_case/HDF5/h5/train{0}.h5'.format(i)
    else:
        filename = '/home/tyd/caffe_case/HDF5/h5/test{0}.h5'.format(i-batchNum+1)
    print filename
    with h5py.File(filename, 'w') as f:
        f.create_dataset('data', data = np.array((imgs[start:end]-imgsMean)/255.0).astype(np.float32), **comp_kwargs)
        f.create_dataset('label', data = np.array(labels[start:end]).astype(np.float32), **comp_kwargs)

    if i < batchNum-1:
        with open('/home/tyd/caffe_case/HDF5/h5/trainlist.txt', 'a') as f:
            f.write(os.path.join(os.getcwd(), 'train{0}.h5').format(i) + '\n')
    else:
        with open('/home/tyd/caffe_case/HDF5/h5/testlist.txt', 'a') as f:
            f.write(os.path.join(os.getcwd(), 'test{0}.h5').format(i-batchNum+1) + '\n')

imgsMean = np.mean(imgsMean, axis=(1,2))
with open('mean.txt', 'w') as f:
    f.write(str(imgsMean[0]) + '\n' + str(imgsMean[1]) + '\n' + str(imgsMean[2]))

根據中文注釋的步驟我們可以了解到hdf5制作的流程.