# coding = UTF-8

# ++++++++++++++++++++++++++++++++++++++++++++++++++++
# machine_classfied_ldmwp.py
# @簡介：用scikit-learn估計器分類
# @作者：Glen
# @日期：2016.8.13
# @資料來源：Python數據挖掘入門與實踐
# +++++++++++++++++++++++++++++++++++++++++++++++++++++

# --------------------------------------------------
# 用Python編寫的scikit-learn庫，實現了一系列數據挖掘算法，
# 提供通用編程接口、標準化的測試和調參工具。
# 基本概念：
# - 估計器(Estimator)：用于分類、聚類和回歸處理
# - 轉換器(Transformer)：用于數據預處理和數據轉換
# - 流水線(Pipeline)：組合數據挖掘流程，便于再次使用
# --------------------------------------------------

# ------------------------------------------
# scikit-learn估計器
# 估計器用于分類任務，主要包括以下兩個函數。
# - fit()：訓練算法，設置內部參數
# - predict()：參數為測試集。
# ------------------------------------------

# ----------------------------------------
# 近鄰算法
# 計算近鄰的重要之處在于對距離的度量
# 常用的度量方法有：歐式距離、曼哈頓距離和余弦距離
# 距離的選擇方式對結果可能會產生重要的影響
# -----------------------------------------

import sys
import pickle
import numpy as np
import pandas as pd
from sklearn.neighbors import KNeighborsClassifier
from sklearn.cross_validation import train_test_split, cross_val_score
from sklearn.preprocessing import MinMaxScaler
from sklearn.pipeline import Pipeline
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.grid_search import GridSearchCV
from collections import defaultdict
from operator import itemgetter

# 導入數據
F = open(r'E:\github\workrobot\workrobot\data\ceic\random_datasets.pkl', 'rb')
datasets = pickle.load(F)

dataset_choosed = datasets[0]

# 變量的中英文轉換
columns_chinese = dataset_choosed.columns
columns_english = ['_'.join(['v',str(item)]) for item in range(len(columns_chinese))]
columns_mapping = dict(zip(columns_english, columns_chinese))

# 設定數據框的新變量
dataset_choosed.columns = columns_english

# 分類數據
y = dataset_choosed['v_0'].gt(dataset_choosed['v_0'].mean())
x = dataset_choosed.iloc[:,range(1,len(columns_english))]

# 劃分數據集
x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=14)

# K近鄰分類器
estimator = KNeighborsClassifier()
# 估計
estimator.fit(x_train, y_train)

# 預測
y_predicted = estimator.predict(x_test)

# 打印結果
accuracy = np.mean(y_test == y_predicted) * 100
print('The accuracy is {0:.1f}%.'.format(accuracy))

# 交叉檢驗
scores = cross_val_score(estimator,x,y,scoring='accuracy')
average_accuracy = np.mean(scores) * 100
print('The average accuracy is {0:.1f}%.'.format(average_accuracy))

# 設置參數
avg_scores = []
all_scores = []
parameter_values = list(range(1,21))
for n_neighbors in parameter_values:
    estimator = KNeighborsClassifier(n_neighbors=n_neighbors)
    scores = cross_val_score(estimator, x, y, scoring='accuracy')
    avg_scores.append(np.mean(scores))
    all_scores.append(scores)

# 標準預處理
x_transformed = MinMaxScaler().fit_transform(x)
estimator = KNeighborsClassifier()
transformed_scores = cross_val_score(estimator, x_transformed, y, scoring='accuracy')
print('The average accuracy is {0:.1f}%.'.format(np.mean(transformed_scores) * 100))

# 流水線
scaling_pipeline = Pipeline([('scale', MinMaxScaler()), ('predict', KNeighborsClassifier())])
scores = cross_val_score(scaling_pipeline, x, y, scoring='accuracy')
print('The pipeline scored an average accuracy is {0:.1f}%.'.format(np.mean(scores) * 100))

# ---------------------------------------
# 分類算法 —— 決策樹
# @簡介：一種有監督的機器學習算法
# @方法：scikit-learn庫實現了分類回歸樹(CART)
# ----------------------------------------

print('-'*50)
print('CART')

# 創建對象
clf = DecisionTreeClassifier(random_state=14)
transformed_scores = cross_val_score(clf, x_transformed, y, scoring='accuracy')
print('The average accuracy is {0:.1f}%.'.format(np.mean(transformed_scores) * 100))

# 隨機森林
clf = RandomForestClassifier(random_state=14)
transformed_scores = cross_val_score(clf, x_transformed, y, scoring='accuracy')
print('The average accuracy is {0:.1f}%.'.format(np.mean(transformed_scores) * 100))

'''
# 搜索最佳參數
parameter_space = {'max_features':[2, 40, 'auto'],
                   'n_estimators': [100, ],
                   'criterion': ['gini', 'entropy'],
                   'min_samples_leaf': [2, 4, 6]}
clf = RandomForestClassifier(random_state=14)
grid = GridSearchCV(clf, parameter_space)
grid.fit(x, y)
print('Accuracy: {0:.1f}%.'.format(grid.best_score_ * 100))
print(grid.best_estimator_)'''

# --------------------------
# 親和性分析 —— 電影推薦
# @算法：Apriori算法
# __________________________

print('\n---------------Apriori-------------')

# 導入數據
ratings_filename = r'E:\data\bigdata\movies\u.data'
all_ratings = pd.read_csv(ratings_filename, delimiter='\t', header=None,
                          names = ['UserID', 'MovieID', 'Rating', 'Datetime'])
all_ratings['Datetime'] = pd.to_datetime(all_ratings['Datetime'], unit='s')
print(all_ratings[:5])

all_ratings['Favorable'] = all_ratings['Rating'] > 3
print(all_ratings[10:15])

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# 變量all_ratings
#     UserID  MovieID  Rating            Datetime Favorable
# 10      62      257       2 1997-11-12 22:07:14     False
# 11     286     1014       5 1997-11-17 15:38:45      True
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# 選擇前200名用戶的打分數據用作訓練集
ratings = all_ratings[all_ratings['UserID'].isin(range(200))]

# 只包含用戶喜歡某部電影的數據行
favorable_ratings = ratings[ratings["Favorable"]]

# 字典，key是用戶id，value是該用戶喜歡的電影的id集合
favorable_reviews_by_users = dict((k, frozenset(v.values)) for k, v in favorable_ratings.groupby("UserID")["MovieID"])
for k, v in favorable_ratings.groupby("UserID")['MovieID']:
    print(k,' --> ',list(v))

# 每部電影影迷的數量
num_favorable_by_movie = ratings[["MovieID", "Favorable"]].groupby("MovieID").sum()
print(num_favorable_by_movie.sort_values("Favorable", ascending=False)[:5])

print('----------------------Start---------------------')

def find_frequent_itemsets(favorable_reviews_by_users, k_1_itemsets, min_support):
    counts = defaultdict(int)
    for user, reviews in favorable_reviews_by_users.items():
        for itemset in k_1_itemsets:
            if itemset.issubset(reviews):
                for other_reviewed_movie in reviews - itemset:
                    current_superset = itemset | frozenset((other_reviewed_movie,))
                    counts[current_superset] += 1
    return dict([(itemset, frequency) for itemset, frequency in counts.items() if frequency >= min_support])


# 頻繁項集
frequent_itemsets = {}  # itemsets are sorted by length
# 最小支持度
min_support = 50

#生成初始的頻繁項集
frequent_itemsets[1] = dict((frozenset((movie_id,)), row["Favorable"])
                                for movie_id, row in num_favorable_by_movie.iterrows()
                                if row["Favorable"] > min_support)
print(frequent_itemsets)

print("There are {} movies with more than {} favorable reviews".format(len(frequent_itemsets[1]), min_support))
sys.stdout.flush()

# 遍歷生成頻繁項集
for k in range(2, 20):
    # Generate candidates of length k, using the frequent itemsets of length k-1
    # Only store the frequent itemsets
    cur_frequent_itemsets = find_frequent_itemsets(favorable_reviews_by_users, frequent_itemsets[k-1],
                                                   min_support)
    if len(cur_frequent_itemsets) == 0:
        print("Did not find any frequent itemsets of length {}".format(k))
        sys.stdout.flush()
        break
    else:
        print("I found {} frequent itemsets of length {}".format(len(cur_frequent_itemsets), k))
        #print(cur_frequent_itemsets)
        sys.stdout.flush()
        frequent_itemsets[k] = cur_frequent_itemsets
# We aren't interested in the itemsets of length 1, so remove those
del frequent_itemsets[1]

# 頻繁項集生成完畢，現在需要生成一些統計量
# Now we create the association rules. First, they are candidates until the confidence has been tested
candidate_rules = []
for itemset_length, itemset_counts in frequent_itemsets.items():
    for itemset in itemset_counts.keys():
        for conclusion in itemset:
            premise = itemset - set((conclusion,))
            candidate_rules.append((premise, conclusion))
print("There are {} candidate rules".format(len(candidate_rules)))

# candidate_rules變量是字典，key是前提，value是結論
print(candidate_rules[:5])

# 計算每條規則的置信度
# Now, we compute the confidence of each of these rules. This is very similar to what we did in chapter 1
correct_counts = defaultdict(int)
incorrect_counts = defaultdict(int)
for user, reviews in favorable_reviews_by_users.items():
    for candidate_rule in candidate_rules:
        premise, conclusion = candidate_rule
        if premise.issubset(reviews):
            if conclusion in reviews:
                correct_counts[candidate_rule] += 1
            else:
                incorrect_counts[candidate_rule] += 1
rule_confidence = {candidate_rule: correct_counts[candidate_rule] / float(correct_counts[candidate_rule] + incorrect_counts[candidate_rule])
              for candidate_rule in candidate_rules}

# 根據置信度排序
sorted_confidence = sorted(rule_confidence.items(), key=itemgetter(1), reverse=True)
for index in range(5):
    print("Rule #{0}".format(index + 1))
    (premise, conclusion) = sorted_confidence[index][0]
    print("Rule: If a person recommends {0} they will also recommend {1}".format(premise, conclusion))
    print(" - Confidence: {0:.3f}".format(rule_confidence[(premise, conclusion)]))
    print("")

# ---------------------------------------
# 用轉換器抽取特征
# ----------------------------------------

# 模型就是用來簡化世界，特征抽取也是一樣。
# 降低復雜性有好處，但也有不足，簡化會忽略很多細節。

# 這里的例子用adult數據集，預測一個人是否年收入多于五萬美元

adult_filename = r'E:\data\bigdata\adult\adult.data'
adult = pd.read_csv(adult_filename, header=None, names=["Age", "Work-Class", "fnlwgt", "Education",
                                                        "Education-Num", "Marital-Status", "Occupation",
                                                        "Relationship", "Race", "Sex", "Capital-gain",
                                                        "Capital-loss", "Hours-per-week", "Native-Country",
                                                        "Earnings-Raw"])
print(adult.head)