昨天工作：
dssm網絡效果調優（40w數據量）。現在的效果是在網絡訓練過程中（acc:0.966， auc：0.884），測試階段（acc:0.834， auc：0.884），訓練結果良好，但是測試的結果還有問題，正在尋找出現該問題的原因。
今天計劃：
測試階段acc過低問題的定位和解決。使用dssm網絡進行inference效果測試。
問題解決：
batch normalization的使用有問題，在訓練階段需要自動更新它的均值方差，在test階段才能使用。具體細節參考下面的第0條。
遇到的問題：
https://www.gitmemory.com/issue/tensorflow/hub/44/494272642
batch normalization在test的時候的使用問題：
https://blog.csdn.net/huitailangyz/article/details/85015611batch normalization 問題的解決：
pai -name tensorflow140 -Dscript="file:///home/hengsong/origin_deep_cluster_odps_8.tar.gz" -DentryFile="train_v4.py" -Dcluster='{"worker":{"count":10, "cpu":200, "memory":4000}, "ps":{"count":3, "cpu":200, "memory":5000}}' -Dtables="odps://graph_embedding/tables/hs_train_data_dssm_3,odps://graph_embedding/tables/hs_test_data_dssm_3" -DcheckpointDir="oss://bucket-automl/hengsong/?role_arn=acs:ram::1293303983251548:role/graph2018&host=cn-hangzhou.oss-internal.aliyun-inc.com" -DuserDefinedParameters="--learning_rate=3e-4 --batch_size=1024 --is_save_model=True --attention_type=1 --num_epochs=1000 --ckpt=hs_ugc_video_40w.ckpt" -DuseSparseClusterSchema=True;
pai -name tensorflow140 -Dscript="file:///home/hengsong/origin_deep_cluster_odps_8.tar.gz" -DentryFile="train_v4.py" -Dcluster='{"worker":{"count":30, "cpu":200, "memory":4000}, "ps":{"count":10, "cpu":200, "memory":5000}}' -Dtables="odps://graph_embedding/tables/hs_train_data_dssm_2,odps://graph_embedding/tables/hs_test_data_dssm_2" -DcheckpointDir="oss://bucket-automl/hengsong/?role_arn=acs:ram::1293303983251548:role/graph2018&host=cn-hangzhou.oss-internal.aliyun-inc.com" -DuserDefinedParameters="--learning_rate=3e-4 --batch_size=1024 --is_save_model=True --attention_type=1 --num_epochs=100 --ckpt=hs_ugc_video.ckpt" -DuseSparseClusterSchema=True;
without update:訓練正常，測試全零
with update:訓練測試正常
修改后：使用40w數據進行訓練測試
使用70y數據進行訓練測試：
注：test的precision和f1_score低是因為/step->/step_test
inference 指令：
pai -name tensorflow140 -Dscript="file:///home/hengsong/origin_deep_cluster_odps_8.tar.gz" -DentryFile="inference_v4.py" -Dcluster='{"worker":{"count":10, "cpu":200, "memory":4000}, "ps":{"count":3, "cpu":200, "memory":5000}}' -Dtables="odps://graph_embedding/tables/hs_tmp_54" -Doutputs="odps://graph_embedding/tables/hs_tmp_56" -DcheckpointDir="oss://bucket-automl/hengsong/?role_arn=acs:ram::1293303983251548:role/graph2018&host=cn-hangzhou.oss-internal.aliyun-inc.com" -DuserDefinedParameters="--learning_rate=3e-4 --batch_size=1024 --is_save_model=True --attention_type=1 --num_epochs=100 --ckpt=hs_ugc_video.ckpt-1" -DuseSparseClusterSchema=True;
之己給的參考：
pai -name tensorflow140 -Dscript="file:///Users/xuejinbao/zhiji/graph/query_co_video/xhs_dssm.tar.gz" -DentryFile="inference.py" -Dcluster='{"worker":{"count":10, "cpu":200, "memory":4000}, "ps":{"count":10, "cpu":200, "memory":5000}}' -Dtables="odps://graph_embedding/tables/zj_xhs_video_topic_infos_" -Doutputs="odps://graph_embedding/tables/zj_xhs_video_topic_emb_" -DcheckpointDir="oss://bucket-automl/xhs_video/?role_arn=acs:ram::1293303983251548:role/graph2018&host=cn-hangzhou.oss-internal.aliyun-inc.com" -DuserDefinedParameters="--learning_rate=1e-2 --batch_size=1024 --attention_type=1 --ckpt=xhs_video.ckpt-1" -DuseSparseClusterSchema=True;
- 準備inference數據
得到最新分區的數據
create table hs_tmp_57 as
select coalesce(get_json_object(body, '.entities.k0.item_id/l'),get_json_object(body, '
.entities.k3.item_id/l') ,get_json_object(body, '
.entities.k5.item_id/l'), get_json_object(body, '
.entities.k7.item_id/l'))as id, coalesce(get_json_object(body, '
.entities.k1.title/s'), get_json_object(body, '
.entities.k3.title/s'), get_json_object(body, '
.entities.k5.title/s'), get_json_object(body, '
.entities.k7.title/s')) as words from graph_embedding.jl_jingyan_query_related_video_pool where ds=max_pt('graph_embedding.jl_jingyan_query_related_video_pool') and type_biz=2;
create table hs_tmp_58 as
select row_number()over() as id, query as words from
graph_embedding.jl_jingyan_query_related_top_query where ds=max_pt('graph_embedding.jl_jingyan_query_related_top_query');
使用主搜進行分詞
create table if not exists hs_tmp_59 LIFECYCLE 20 as select id, words, search_kg:alinlp_segment(words, "MAINSE", "0", "1") as words_mainse_ws from hs_tmp_57;
create table if not exists hs_tmp_60 LIFECYCLE 20 as select id, words, search_kg:alinlp_segment(words, "MAINSE", "0", "1") as words_mainse_ws from hs_tmp_58;
去除低頻詞:hs_tmp_dssm_inf_titles; hs_tmp_dssm_inf_querys
PAI -name FilterNoise -project algo_public
-DinputTableName=graph_embedding.hs_tmp_59
-DnoiseTableName=graph_embedding.hs_dirty_words_info_
-DoutputTableName=graph_embedding.hs_tmp_dssm_inf_titles
-DselectedColNames="words_mainse_ws"
-Dlifecycle=30;
PAI -name FilterNoise -project algo_public
-DinputTableName=graph_embedding.hs_tmp_60
-DnoiseTableName=graph_embedding.hs_dirty_words_info_
-DoutputTableName=graph_embedding.hs_tmp_dssm_inf_querys
-DselectedColNames="words_mainse_ws"
-Dlifecycle=30;
inference階段 hs_dssm_result_query_0; hs_dssm_result_title_0
truncate table hs_dssm_result_query_0;
pai -name tensorflow140 -Dscript="file:///home/hengsong/origin_deep_cluster_odps_8.tar.gz" -DentryFile="inference_v4.py" -Dcluster='{"worker":{"count":10, "cpu":200, "memory":4000}, "ps":{"count":3, "cpu":200, "memory":5000}}' -Dtables="odps://graph_embedding/tables/hs_tmp_dssm_inf_querys" -Doutputs="odps://graph_embedding/tables/hs_dssm_result_query_0" -DcheckpointDir="oss://bucket-automl/hengsong/?role_arn=acs:ram::1293303983251548:role/graph2018&host=cn-hangzhou.oss-internal.aliyun-inc.com" -DuserDefinedParameters="--learning_rate=3e-4 --batch_size=1024 --is_save_model=True --attention_type=1 --num_epochs=100 --ckpt=hs_ugc_video.ckpt-1" -DuseSparseClusterSchema=True;
- knn求得最終結果
PAI -name am_vsearch_nearest_neighbor_014 -project algo_market
-Dcluster="{"worker":{"count":1,"gpu":100}}"
-Ddim=100
-Did_col="index"
-Dvector_col="words_mainse_emb"
-Dinput_slice=1
-Dtopk=50
-Dnprob=1024
-Dmetric="l2"
-Dinput="odps://graph_embedding/tables/hs_tmp_64"
-Dquery="odps://graph_embedding/tables/hs_dssm_result_query_1"
-Doutputs="odps://graph_embedding/tables/hs_dssm_result_0"
-DenableDynamicCluster=true -DmaxTrainingTimeInHour=60;
create table hs_dssm_result_query_1 as select distinct * from hs_dssm_result_query_0;
create table hs_tmp_61 as select bi_udf:bi_split_value(query, title, ",") as (query_id, item_id) from hs_dssm_result_1;
- 看一下出來的結果有沒有什么問題
drop table hs_tmp_62;
yes
create table hs_tmp_62 as select bi_udf:bi_split_value(id, words_mainse_emb, ",") as (index, query_word) from hs_dssm_result_query_1;
drop table hs_tmp_63;
yes
create table hs_tmp_63 as select index, count(*) as freq from hs_tmp_62 group by index order by freq desc;
對結果進行修改之后，應該沒有問題了，但是不知道為什么還是報錯：
看來還是使用網絡直接得到分數更合適啊。。。
- 根據group結果取前k大：
SELECT
*
FROM
yourtable
WHERE
id IN (SELECT
SUBSTRING_INDEX(GROUP_CONCAT(id
ORDER BY rate DESC),
',',
1) id
FROM
yourtable
GROUP BY year)
ORDER BY rate DESC;
