IMDB數(shù)據(jù)集下載和探索

根據(jù)TensorFlow官方教程實(shí)現(xiàn)：

# -*- coding: utf-8 -*-

import tensorflow as tf
from tensorflow import keras

import numpy as np

# 查看tensorflow版本
print(tf.__version__)

# 下載imdb數(shù)據(jù)集
imdb = keras.datasets.imdb
# 參數(shù)num_words=10000保留訓(xùn)練數(shù)據(jù)中出現(xiàn)頻率最高的10,000個(gè)單詞
(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)

# 探索數(shù)據(jù)
print("Training entries: {}, labels: {}".format(len(train_data), len(train_labels)))
print(train_data[0])
# 每篇文本長(zhǎng)度不同
print(len(train_data[0]), len(train_data[1]))

# 將arry從整數(shù)轉(zhuǎn)為單詞

word_index = imdb.get_word_index()

reverse_word_index = {value:key for key, value in word_index.items()}
content = []
for text in train_data:
    text_words = []
    content.append(' '.join([reverse_word_index[num] for num in text]))
    

# 將數(shù)據(jù)轉(zhuǎn)化成張量

train_data = keras.preprocessing.sequence.pad_sequences(train_data, 
                                                       padding='post',
                                                       maxlen=256)

test_data = keras.preprocessing.sequence.pad_sequences(test_data, 
                                                       padding='post',
                                                       maxlen=256)
print(train_data[0])

# 建立模型
vocab_size = 10000

model = keras.Sequential()
# Embedding層將正整數(shù)轉(zhuǎn)換為具有固定大小的向量
model.add(keras.layers.Embedding(vocab_size, 16))
# GlobalAveragePooling1D對(duì)序列維數(shù)進(jìn)行平均，輸出為一個(gè)1*1*D的張量。
model.add(keras.layers.GlobalAveragePooling1D())
# 16個(gè)隱藏單元的全連接(密集)層
model.add(keras.layers.Dense(16, activation=tf.nn.relu))
model.add(keras.layers.Dense(1, activation=tf.nn.sigmoid))

model.summary()
# 二分類問題，選擇binary_crossentropy作為損失函數(shù)
model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['acc'])

# 構(gòu)建數(shù)據(jù)集 取前10000條數(shù)據(jù)作為驗(yàn)證集
x_val = train_data[:10000]
partial_x_train = train_data[10000:]

y_val = train_labels[:10000]
partial_y_train = train_labels[10000:]

history = model.fit(partial_x_train,
                    partial_y_train,
                    epochs=40,
                    batch_size=512,
                    validation_data=(x_val, y_val),
                    verbose=1)

# 評(píng)價(jià)模型
result = model.evaluate(test_data, test_labels)
print(result)

THUCNews數(shù)據(jù)集下載和探索

根據(jù)githut進(jìn)行復(fù)現(xiàn)

# -*- coding: utf-8 -*-
"""
Created on Sun May 12 16:07:05 2019

@author: pc
"""

import tensorflow as tf
from tensorflow import keras

import numpy as np
import pandas as pd
from collections import Counter

TRAIN_PATH = 'E:/task1/cnews.train.txt'
VAL_PATH = 'E:/task1/cnews.val.txt'
TEST_PATH = 'E:/task1/cnews.test.txt'
VOCAB_SIZE = 5000
MAX_LEN = 600
BATCH_SIZE = 64

def read_file(file_name):
    '''
        讀文件
    '''
    file_path = {'train': TRAIN_PATH, 'val': VAL_PATH, 'test': TEST_PATH}
    contents = []
    labels = []
    with open(file_path[file_name], 'r', encoding='utf-8') as f:
        for line in f:
            try:
                labels.append(line.strip().split('\t')[0])
                contents.append(line.strip().split('\t')[1])
            except:
                pass
    data = pd.DataFrame()
    data['text'] = contents
    data['label'] = labels
    return data


def build_vocab(data):
    '''
        構(gòu)建詞匯表，
        使用字符級(jí)的表示
    '''
    all_content = []
    for _, text in data.iterrows():
        all_content.extend(text['text'])
    counter = Counter(all_content)
    count_pairs = counter.most_common(VOCAB_SIZE - 1)
    words = [i[0] for i in count_pairs]
    words = ['<PAD>'] + list(words)
    
    return words
        

def read_vocab(words):
    words_id = dict(zip(words, range(len(words))))
    return words_id


def read_category(data):
    '''
       將分類目錄固定，轉(zhuǎn)換為{類別: id}表示 
    '''
    category = list(data['label'].drop_duplicates())
    return dict(zip(category, range(len(category))))
    
def to_words(content, words):
    return ' '.join(words[i] for i in content)

def preocess_file(data, words_id, category_id):
    """
        將文件轉(zhuǎn)換為id表示
    """
    content = data['text']
    labels = data['label']
    content_id = []
    label_id = []
    for text, label in zip(content, labels):
        content_id.append([words_id[i] for i in text if i in words_id])
        label_id.append(category_id[label])
    
    # 使用keras提供的pad_sequences來將文本pad為固定長(zhǎng)度
    x_pad = keras.preprocessing.sequence.pad_sequences(content_id, MAX_LEN)
    y_pad = keras.utils.to_categorical(label_id, num_classes=len(category_id))
    return x_pad, y_pad
    

def batch_iter(x, y):
    '''
        為神經(jīng)網(wǎng)絡(luò)的訓(xùn)練準(zhǔn)備經(jīng)過shuffle的批次的數(shù)據(jù)
    '''
    num_batch = int((len(x) - 1) / BATCH_SIZE) + 1
    indices = np.random.permutation(np.arange(len(x)))
    
    x_shuffle = x[indices]
    y_shuffle = y[indices]
    for i in range(num_batch):
        start_id = i * BATCH_SIZE
        end_id = min((i + 1) * BATCH_SIZE, len(x))
        yield x_shuffle[start_id:end_id], y_shuffle[start_id:end_id]   
    

train = read_file('train')
# 查看label類別
print(train['label'].drop_duplicates())
words = build_vocab(train)
words_id = read_vocab(words)
category_id = read_category(train)
x_pad, y_pad = preocess_file(train, words_id, category_id)
batch_iter(x_pad, y_pad)
test = read_file('test')
val = read_file('val')

對(duì)于函數(shù)batch_iter(x, y)的使用還存在疑惑，還有待學(xué)習(xí)