-- coding: utf-8 --

import os
import datetime
import pymysql
import matplotlib
import matplotlib.pyplot as plt
from pylab import mpl
import time
import xlwt
import xlrd
import threading
import pandas as pd
import glob
from matplotlib.font_manager import FontProperties
import sys
import platform
from shutil import copy
import shutil

1、設置可視化繪圖參數

def parse_datetime(s):
try:
if s.find(' ') != -1:
return datetime.datetime.strptime(s, "%Y-%m-%d %H:%M:%S")
else:
return datetime.datetime.strptime(s, "%Y-%m-%d")
except:
print ("string '%s' is not a valid date or datetime")
def datetime_toString(dt):
return dt.strftime("%Y-%m-%d %H:%M:%S")

def datetime_toTimestamp(dt):
return time.mktime(dt.timetuple())

def timestamp_toString(sp):
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(sp))

def execute_sql(sql_cmd):
db = pymysql.connect()
cursor = db.cursor()
cursor.execute(sql_cmd)
results = cursor.fetchall()
db.close()
return results

tmp_path = 'D:/tmp'

if not os.path.exists(tmp_path):

os.mkdir(tmp_path)

if dest_path is None:

dest_path = os.path.join(path, '%s_%s' % (from_date.date(), to_date.date()))

通過日期區間和漁船設備號查找數據

def get_signal_info_by_device_no(from_date, to_date,device_no):
print ("Partition ship data by device_no")
process_day = from_date
signal_change_points= []
data_time_temp1 = 0
data_time_temp2 = 0
signal_change_point_temp1 = 0
signal_change_point_temp2 = 0
pre_temp_dts =[]
temp_dts =[]
GPRS_TIME = 0
LBS_TIME = 0
GPRS_loss_num = 0
GPRS_loss_point = []
GPRS_fact_num = 0
LBS_fact_num = 0
LBS_loss_num =0
LBS_loss_point = []
flag = 0 #用于日期遞增
flag1 = 0 #判斷信號頻率轉換的標志位
flag2 = 1 #判斷GPRS轉為北斗信號后的頻率為2分鐘一次還是5分鐘一次的標志位
flag3 = 1 #判斷第12標志位為1時出現10分鐘一次的GPRS信號
while process_day <= to_date:
table_name = process_day.strftime("t_acq_data_%Y%m%d")
get_ship_data_by_date_cmd = ("select ALARM_STATUS,ACQ_TIME,DATA_TYPE from %s where DEVICE_NO = %s"
% (table_name,device_no))
pre_temp_dts = execute_sql(get_ship_data_by_date_cmd)
if pre_temp_dts == ():
print('--------------device_no為'+device_no+'在'+str(process_day.date())+'這天沒有數據')
else:
for pre_temp_dt in pre_temp_dts:
temp_dts.append({'ALARM_STATUS': str(pre_temp_dt[0]), 'ACQ_TIME': pre_temp_dt[1], 'DATA_TYPE': pre_temp_dt[2]})
flag +=1
process_day += datetime.timedelta(1)
if (temp_dts) == []:
print('--------------device_no為'+device_no+'在這幾天都沒有數據')
else:
if (int(temp_dts[0].get('ALARM_STATUS'),2) & int(8192)):
data_type_begin = 2
flag1 = 4
else:
data_type_begin = 1
data_time_temp1 = datetime_toTimestamp(temp_dts[0].get('ACQ_TIME'))
signal_change_point_temp1 = data_time_temp1
signal_change_points.append(timestamp_toString(data_time_temp1))
for temp_dt in temp_dts:
data_time_temp2 = datetime_toTimestamp(temp_dt.get('ACQ_TIME'))#下一個點的時間以秒數計算
if (2 if int(temp_dt.get('ALARM_STATUS'),2) & int(8192) else 1) == 1:#判斷是否GPRS信號
GPRS_fact_num += 1
if flag1 == 4:
flag2 = 1
signal_change_points.append(timestamp_toString(data_time_temp2))
signal_change_point_temp2 = data_time_temp2
LBS_TIME += signal_change_point_temp2 - signal_change_point_temp1
signal_change_point_temp1 = signal_change_point_temp2
if (int(temp_dt.get('ALARM_STATUS'),2) & int(32)):#5分鐘一次
flag1 =1
elif (int(temp_dt.get('ALARM_STATUS'),2) & int(2048)):#10分鐘一次
flag1 =2
else:#2分鐘一次
flag1 =3
elif (int(temp_dt.get('ALARM_STATUS'),2) & int(32)):#5分鐘一次
if (flag1 == 1) and (data_time_temp2-data_time_temp1) > (560+60):
GPRS_loss_num += (round((data_time_temp2-data_time_temp1)/60/5)-1)
GPRS_loss_point.append(datetime_toString(temp_dt.get('ACQ_TIME')))
flag1 =1
elif (int(temp_dt.get('ALARM_STATUS'),2) & int(2048)):#10分鐘一次
if (flag1 == 2) and (flag3 == 1) and (data_time_temp2-data_time_temp1) < (1060+60) and (data_time_temp2-data_time_temp1) > (1060-60):
print('--------------------警告：設備號為'+device_no+'第12標志位為1時出現10分鐘一次的GPRS信號----------------------------')
flag3 = 2
if (flag1 == 2) and (data_time_temp2-data_time_temp1) > (1060+120):
GPRS_loss_num += (round((data_time_temp2-data_time_temp1)/60/10)-1)
GPRS_loss_point.append(datetime_toString(temp_dt.get('ACQ_TIME')))
flag1 =2
else:#2分鐘一次
if (flag1 == 3) and (data_time_temp2-data_time_temp1) > (260+40):
GPRS_loss_num += (round((data_time_temp2-data_time_temp1)/60/2)-1)
GPRS_loss_point.append(datetime_toString(temp_dt.get('ACQ_TIME')))
flag1 =3
elif (2 if int(temp_dt.get('ALARM_STATUS'),2) & int(8192) else 1) == 2:#判斷是否北斗信號
LBS_fact_num += 1
if not (flag1 == 4):
signal_change_points.append(timestamp_toString(data_time_temp2))
signal_change_point_temp2 = data_time_temp2
GPRS_TIME += (signal_change_point_temp2 - signal_change_point_temp1)
signal_change_point_temp1 = signal_change_point_temp2
else:
if flag2 == 1:
if (data_time_temp2-data_time_temp1) > (560-40) and (data_time_temp2-data_time_temp1) < (560+40):
flag2 = 2#標志2分鐘一次變為5分鐘一次
else:
if (data_time_temp2-data_time_temp1) > (260+40):
LBS_loss_num += (round((data_time_temp2-data_time_temp1)/60/2)-1)
LBS_loss_point.append(datetime_toString(temp_dt.get('ACQ_TIME')))
elif flag2 == 2 and (data_time_temp2-data_time_temp1) > (5*60+60):
LBS_loss_num += (round((data_time_temp2-data_time_temp1)/60/5)-1)
LBS_loss_point.append(datetime_toString(temp_dt.get('ACQ_TIME')))
flag1 = 4
elif int(temp_dt.get('ALARM_STATUS'),2) & int(49152):
print('-------------------存在報警信息-------------------------------')
data_time_temp1=data_time_temp2
signal_change_point_temp2 = datetime_toTimestamp(temp_dts[-1].get('ACQ_TIME'))
signal_change_points.append(timestamp_toString(signal_change_point_temp2))
if not flag1 == 4:
GPRS_TIME += signal_change_point_temp2-signal_change_point_temp1
else:
LBS_TIME += signal_change_point_temp2-signal_change_point_temp1

print('GPRS_fact_num:'+ str(GPRS_fact_num))
print('GPRS_loss_num:'+ str(GPRS_loss_num))
print('LBS_fact_num:'+ str(LBS_fact_num))
print('LBS_loss_num:'+ str(LBS_loss_num))
print('GPRS_TIME:'+ str(GPRS_TIME))
print('LBS_TIME:'+ str(LBS_TIME))
print('signal_change_points:'+ str(signal_change_points))
print('GPRS_loss_point:'+ str(GPRS_loss_point))
print('LBS_loss_point:'+ str(LBS_loss_point))
if GPRS_fact_num == 0:
    gprs_success = None
else:
    gprs_success =  GPRS_fact_num/(GPRS_fact_num + GPRS_loss_num)
if LBS_fact_num == 0:
    lbs_success = None
else:
    lbs_success =  LBS_fact_num/(LBS_fact_num + LBS_loss_num)
if GPRS_TIME + LBS_TIME == 0:
    gprs_time_percent = None
    lbs_time_percent = None
else:
    gprs_time_percent = GPRS_TIME/(GPRS_TIME + LBS_TIME)
    lbs_time_percent = LBS_TIME/(GPRS_TIME + LBS_TIME)

return   gprs_success,lbs_success,gprs_time_percent,lbs_time_percent  

def get_device_nos(dev_type):
get_device_nos_by_date_cmd = ("select distinct(DEVICE_NO) from t_device where DEVICE_TYPE = %s" % (dev_type))
temp_device_nos = execute_sql(get_device_nos_by_date_cmd)
device_nos=[]
for temp_device_no in temp_device_nos:
device_nos.append(temp_device_no[0])
return device_nos

def chunks(device_nos, n):
for i in range(0, len(device_nos), n):
yield device_nos[i:i + n]

def get_excel(device_nos,num):
flag_excel = 1
workbook = xlwt.Workbook(encoding='utf-8')
booksheet = workbook.add_sheet('Sheet 1', cell_overwrite_ok=True)
c = ['設備NO', '統計數據時間區間','GPRS成功率', 'GPRS占比', '北斗成功率','北斗占比','設備代數']
#用于初步處理后的數據存儲
data_toexcel = []
for i in range(7):
booksheet.write(0, i, c[i])
for device_no in device_nos:
print('---------------------------------------------------------------------------------------------------------')
print('ship which is now handled is device_no:'+ str(device_no))
gprs_success,lbs_success,gprs_time_percent,lbs_time_percent = get_signal_info_by_device_no(from_date, to_date,device_no)
data_toexcel.append({"DEVICE_NO": device_no, "COUNT_TIME_INTERVAL": '2018-08-26~2018-08-28', "GPRS_SUCCESS": gprs_success, "GPRS_TIME_PERCENT": gprs_time_percent,
"LBS_SUCCESS": lbs_success,"LBS_TIME_PERCENT": lbs_time_percent, "DEVICE_TYPE": '二代'})
booksheet.write(flag_excel, 0, data_toexcel[flag_excel-1].get('DEVICE_NO'))
booksheet.write(flag_excel, 1, data_toexcel[flag_excel-1].get('COUNT_TIME_INTERVAL'))
booksheet.write(flag_excel, 2, data_toexcel[flag_excel-1].get('GPRS_SUCCESS'))
booksheet.write(flag_excel, 3, data_toexcel[flag_excel-1].get('GPRS_TIME_PERCENT'))
booksheet.write(flag_excel, 4, data_toexcel[flag_excel-1].get('LBS_SUCCESS'))
booksheet.write(flag_excel, 5, data_toexcel[flag_excel-1].get('LBS_TIME_PERCENT'))
booksheet.write(flag_excel, 6, data_toexcel[flag_excel-1].get('DEVICE_TYPE'))
flag_excel += 1
path = '.'
os.chdir(path)
tmp_path = './tmp'
if not os.path.exists(tmp_path):
os.mkdir(tmp_path)
outexcel_path = os.path.join(tmp_path, '設備數據報表%s.xls' % (num))
workbook.save(outexcel_path)

def excels_toOne_excel():
filearray=[]
filelocation="D:/temp/"
for filename in glob.glob(filelocation+"*.xls"):
filearray.append(filename)
res=pd.read_excel(filearray[0])
for i in range(1,len(filearray)):
A=pd.read_excel(filearray[i])
res=pd.concat([res,A],ignore_index=True)
print(res.index)
writer = pd.ExcelWriter('D:/excel/output2.xlsx')
res.to_excel(writer,'sheet1')
writer.save()
"""
@summary:
根據平臺類型處理字體庫
linux: 復制字體庫到系統字體目錄，并手動載入使用
windows: 設置matplot的字體參數即可
"""
def prepare_png_font():
font = None
if platform.system() == 'Windows':
mpl.rcParams['font.sans-serif'] = ['FangSong'] # 指定默認字體
else:
fontFile = '/usr/share/fonts/SimHei.ttf'
if not os.path.exists(fontFile) and os.path.exists('./SimHei.ttf'):
copy('./SimHei.ttf', fontFile)
font = FontProperties(fname=fontFile,size=17)

return font

def draw_signal(dev_type, from_date, to_date,dest_path=None,path='.'):
fname = 'D:/excel/output.xlsx'
book = xlrd.open_workbook(fname)
try:
sheet=book.sheet_by_name("sheet1")
except:
print ("在文件%s中沒有找到sheet1，讀取文件數據失敗,要不你換換表格的名字？" %fname)
nrows = sheet.nrows

gprs_success_count_100=0
gprs_success_count_90=0
gprs_success_count_80=0
gprs_success_count_70=0
gprs_success_count_60=0
gprs_success_count_50=0
gprs_success_count_40=0
gprs_success_count_30=0
gprs_success_count_20=0
gprs_success_count_10=0 
gprs_success_count_0=0 
gprs_success = 0.0
gprs_time_count_100=0
gprs_time_count_90=0
gprs_time_count_80=0
gprs_time_count_70=0
gprs_time_count_60=0
gprs_time_count_50=0
gprs_time_count_40=0
gprs_time_count_30=0
gprs_time_count_20=0
gprs_time_count_10=0 
gprs_time_count_0=0 
gprs_time_percent = 0.0

for j in range(1,nrows-1):
    if not sheet.cell(j,2).value == '':
        gprs_success = sheet.cell(j,2).value
    if not gprs_success == '':
        if(gprs_success == 1.0):
            gprs_success_count_100+=1
        if(gprs_success < 1.0 and gprs_success >= 0.9):
            gprs_success_count_90+=1
        if(gprs_success < 0.9 and gprs_success>=0.8):
            gprs_success_count_80+=1
        if(gprs_success< 0.8 and gprs_success>=0.7):
            gprs_success_count_70+=1
        if(gprs_success<0.7 and gprs_success>=0.6):
            gprs_success_count_60+=1
        if(gprs_success< 0.6 and gprs_success>=0.5):
            gprs_success_count_50+=1
        if(gprs_success<0.5 and gprs_success>= 0.4):
            gprs_success_count_40+=1
        if(gprs_success<0.4 and gprs_success>=0.3):
            gprs_success_count_30+=1
        if(gprs_success<0.3 and gprs_success>=0.2):
            gprs_success_count_20+=1
        if(gprs_success<0.2 and gprs_success>=0.1):
            gprs_success_count_10+=1
        if(gprs_success<0.1):
            gprs_success_count_0+=1
    if not sheet.cell(j,3).value == '':
        gprs_time_percent = sheet.cell(j,3).value
    if not gprs_time_percent == '':
        if(gprs_time_percent == 1.0):
            gprs_time_count_100+=1
        if(gprs_time_percent < 1.0 and gprs_time_percent >= 0.9):
            gprs_time_count_90+=1
        if(gprs_time_percent < 0.9 and gprs_time_percent>=0.8):
            gprs_time_count_80+=1
        if(gprs_time_percent< 0.8 and gprs_time_percent>=0.7):
            gprs_time_count_70+=1
        if(gprs_time_percent<0.7 and gprs_time_percent>=0.6):
            gprs_time_count_60+=1
        if(gprs_time_percent< 0.6 and gprs_time_percent>=0.5):
            gprs_time_count_50+=1
        if(gprs_time_percent<0.5 and gprs_time_percent>= 0.4):
            gprs_time_count_40+=1
        if(gprs_time_percent<0.4 and gprs_time_percent>=0.3):
            gprs_time_count_30+=1
        if(gprs_time_percent<0.3 and gprs_time_percent>=0.2):
            gprs_time_count_20+=1
        if(gprs_time_percent<0.2 and gprs_time_percent>=0.1):
            gprs_time_count_10+=1
        if(gprs_time_percent<0.1):
            gprs_time_count_0+=1
gprs_name_list = ['100%','90%','80%','70%','60%','50%','40%','30%','20%','10%','<10%']
gprs_num_list = [gprs_success_count_100,gprs_success_count_90,gprs_success_count_80,gprs_success_count_70,gprs_success_count_60,gprs_success_count_50,gprs_success_count_40,gprs_success_count_30,gprs_success_count_20,gprs_success_count_10,gprs_success_count_0]
gprs_num_list1 = [gprs_time_count_100,gprs_time_count_90,gprs_time_count_80,gprs_time_count_70,gprs_time_count_60,gprs_time_count_50,gprs_time_count_40,gprs_time_count_30,gprs_time_count_20,gprs_time_count_10,gprs_time_count_0]
x =list(range(len(gprs_num_list)))
total_width, n = 0.8, 2
width = total_width / n 

myfont = matplotlib.font_manager.FontProperties(fname=r'C:/Windows/Fonts/simsun.ttc',size=14) # 這一行

mpl.rcParams['font.sans-serif'] = ['FangSong'] # 指定默認字體

kwargs = {}
font = prepare_png_font()
if font:
    kwargs['fontproperties'] = font
#指定輸出圖片的尺寸
plt.figure(figsize=(14, 7))
# 設置標題
plt.title(u"設備類型：%s 時間段：%s ~ %s" % (dev_type,from_date.date(), to_date.date()), **kwargs)
b1 = plt.bar(x, gprs_num_list, width=width,label='成功率',tick_label = gprs_name_list,fc = 'lightskyblue')
for i in range(len(x)):    
    x[i] = x[i] + width
b2 = plt.bar(x, gprs_num_list1, width=width, label=u'占比',fc = 'yellowgreen')
plt.legend()
plt.xlabel('百分比')
plt.ylabel('船只個數')
for rect in b1+b2:
    h = rect.get_height()
    plt.text(rect.get_x()+rect.get_width()/2,h,'%d'%int (h),ha ='center',va='bottom')
# change to work path
os.chdir(path)
if dest_path is None:
    dest_path = os.path.join(path, '%s_%s' % (from_date.date(), to_date.date()))
if not os.path.exists(path):
    os.mkdir(path)
if not os.path.exists(dest_path):
    os.mkdir(dest_path)
outfn1 = os.path.join(dest_path, u"%s_%s漁船GPRS信號統計.png" % (from_date.date(), to_date.date()))
plt.savefig(outfn1,dpi=300)

plt.show()

plt.close(0)

lbs_success_count_100=0
lbs_success_count_90=0
lbs_success_count_80=0
lbs_success_count_70=0
lbs_success_count_60=0
lbs_success_count_50=0
lbs_success_count_40=0
lbs_success_count_30=0
lbs_success_count_20=0
lbs_success_count_10=0 
lbs_success_count_0=0 
lbs_success = 0.0
lbs_time_count_100=0
lbs_time_count_90=0
lbs_time_count_80=0
lbs_time_count_70=0
lbs_time_count_60=0
lbs_time_count_50=0
lbs_time_count_40=0
lbs_time_count_30=0
lbs_time_count_20=0
lbs_time_count_10=0 
lbs_time_count_0=0 
lbs_time_percent = 0.0

for j in range(1,nrows-1):
    if not sheet.cell(j,4).value == '':
        lbs_success = sheet.cell(j,4).value
    if not lbs_success == '':
        if(lbs_success == 1.0):
            lbs_success_count_100+=1
        if(lbs_success < 1.0 and lbs_success >= 0.9):
            lbs_success_count_90+=1
        if(lbs_success < 0.9 and lbs_success>=0.8):
            lbs_success_count_80+=1
        if(lbs_success< 0.8 and lbs_success>=0.7):
            lbs_success_count_70+=1
        if(lbs_success<0.7 and lbs_success>=0.6):
            lbs_success_count_60+=1
        if(lbs_success< 0.6 and lbs_success>=0.5):
            lbs_success_count_50+=1
        if(lbs_success<0.5 and lbs_success>= 0.4):
            lbs_success_count_40+=1
        if(lbs_success<0.4 and lbs_success>=0.3):
            lbs_success_count_30+=1
        if(lbs_success<0.3 and lbs_success>=0.2):
            lbs_success_count_20+=1
        if(lbs_success<0.2 and lbs_success>=0.1):
            lbs_success_count_10+=1
        if(lbs_success<0.1):
            lbs_success_count_0+=1
    if not sheet.cell(j,5).value == '':
        lbs_time_percent = sheet.cell(j,5).value
    if not lbs_time_percent == '':
        if(lbs_time_percent == 1.0):
            lbs_time_count_100+=1
        if(lbs_time_percent < 1.0 and lbs_time_percent >= 0.9):
            lbs_time_count_90+=1
        if(lbs_time_percent < 0.9 and lbs_time_percent>=0.8):
            lbs_time_count_80+=1
        if(lbs_time_percent< 0.8 and lbs_time_percent>=0.7):
            lbs_time_count_70+=1
        if(lbs_time_percent<0.7 and lbs_time_percent>=0.6):
            lbs_time_count_60+=1
        if(lbs_time_percent< 0.6 and lbs_time_percent>=0.5):
            lbs_time_count_50+=1
        if(lbs_time_percent<0.5 and lbs_time_percent>= 0.4):
            lbs_time_count_40+=1
        if(lbs_time_percent<0.4 and lbs_time_percent>=0.3):
            lbs_time_count_30+=1
        if(lbs_time_percent<0.3 and lbs_time_percent>=0.2):
            lbs_time_count_20+=1
        if(lbs_time_percent<0.2 and lbs_time_percent>=0.1):
            lbs_time_count_10+=1
        if(lbs_time_percent<0.1):
            lbs_time_count_0+=1
lbs_name_list = ['100%','90%','80%','70%','60%','50%','40%','30%','20%','10%','<10%']
lbs_num_list = [lbs_success_count_100,lbs_success_count_90,lbs_success_count_80,lbs_success_count_70,lbs_success_count_60,lbs_success_count_50,lbs_success_count_40,lbs_success_count_30,lbs_success_count_20,lbs_success_count_10,lbs_success_count_0]
lbs_num_list1 = [lbs_time_count_100,lbs_time_count_90,lbs_time_count_80,lbs_time_count_70,lbs_time_count_60,lbs_time_count_50,lbs_time_count_40,lbs_time_count_30,lbs_time_count_20,lbs_time_count_10,lbs_time_count_0]
x =list(range(len(lbs_num_list)))
total_width, n = 0.8, 2
width = total_width / n 
#指定輸出圖片的尺寸
plt.figure(figsize=(14, 7))
# 設置標題
plt.title(u"設備類型：%s 時間段：%s ~ %s" % (dev_type,from_date.date(), to_date.date()), **kwargs)
b1 = plt.bar(x, lbs_num_list, width=width,label='成功率',tick_label = lbs_name_list,fc = 'lightskyblue')
for i in range(len(x)):    
    x[i] = x[i] + width
b2 = plt.bar(x, lbs_num_list1, width=width, label=u'占比',fc = 'yellowgreen')
plt.legend()
plt.xlabel('百分比')
plt.ylabel('船只個數')
for rect in b1+b2:
    h = rect.get_height()
    plt.text(rect.get_x()+rect.get_width()/2,h,'%d'%int (h),ha ='center',va='bottom')
# change to work path
outfn2 = os.path.join(dest_path, u"%s_%s漁船北斗信號統計.png" % (from_date.date(), to_date.date()))
plt.savefig(outfn2,dpi=300)

plt.show()

plt.close(0)

def get_parser():
from optparse import OptionParser
parser = OptionParser("Usage: %prog [options] from_date to_date ")
parser.add_option("-l", "--use-local", dest="use_local", default=False,
action="store_true", help="whether to use local data for processing")
parser.add_option("-p", "--working-path", dest="path", default='.',
help="working path to save or load data, default: ./")
parser.add_option("-i", "--device-no", dest="device_nos", default=[], action='append',
help="ship identities, seperated by comma, string like 211,222,232")
parser.add_option("-r", "--replace", dest="replace", default=False,
action="store_true", help="to replace any result already done")
return parser

if name == 'main':

sys.argv = ['', '2018-09-16', '2018-09-20'] 
parser = get_parser()
(options, argv) = parser.parse_args(sys.argv)

from_date = parse_datetime(argv[1])
to_date = parse_datetime(argv[2])   

dev_type = 2

device_nos = get_device_nos(dev_type)

print(device_nos)

threads=[]

num = 1

for i in list(chunks(device_nos, 60)):

threads.append(threading.Thread(target=get_excel,args=(i,num) ))

num += 1

for thread in threads:

thread.start()

draw_signal(dev_type, from_date, to_date)