最近有個任務需要給一個MySQL數據庫的一個表插入 1億條數據。之前并沒有處理過這么大的數據量，所以用之前的 for 循環一條條插入實在是太慢了。
解決辦法：批量插入。
思路：每次生成1w條數據，然后批量插入,重復循環1w次。
直接上代碼
# -*- coding: utf-8 -*-
"""
@Time: 1/5/2019 7:30 PM
@Author: hejing
@Email: 906099915@qq.com
"""
import pymysql
import random
import time
# 批量插的次數
loop_count = 10000
# 每次批量查的數據量
batch_size = 10000
success_count = 0
fails_count = 0
conn = pymysql.connect(host='localhost', user='root', password='123456', database='test', charset='utf8')
def random_generate_datetime():
a1 = (2018, 1, 1, 0, 0, 0, 0, 0, 0) # 設置開始日期時間元組（1976-01-01 00：00：00）
a2 = (2018, 12, 31, 23, 59, 59, 0, 0, 0) # 設置結束日期時間元組（1990-12-31 23：59：59）
start = time.mktime(a1) # 生成開始時間戳
end = time.mktime(a2) # 生成結束時間戳
t = random.randint(start, end) # 在開始和結束時間戳中隨機取出一個
date_touple = time.localtime(t) # 將時間戳生成時間元組
datetime = time.strftime("%Y-%m-%d %H:%M:%S", date_touple) # 將時間元組轉成格式化字符串（1976-05-21）
# print(datetime)
return datetime
def random_generate_search_controller_data(base_cid, max_cid, base_light_pole_id):
def _random_generate_search_controller_data():
# get LightPole_id -- Controller_id
Controller_id = random.randint(base_cid, max_cid)
LightPole_id = Controller_id - base_cid + base_light_pole_id
# Read_time
Read_time = random_generate_datetime()
# Electric_quantity 1--43
Electric_quantity = random.randint(1, 43)
# Electric_voltage 220~ 245
Electric_voltage = random.randint(220, 245)
# Electric_current 0 ~ 0.189
Electric_current = random.randint(0, 189) / 1000
# Power 0 ~ 46
Power = random.randint(0, 46)
return (
LightPole_id,
Controller_id,
Read_time,
'測試服務器',
'T8-501',
Electric_quantity,
Electric_voltage,
Electric_current,
Power,
'1',
'0',
'0',
'1',
'0',
'100'
)
return _random_generate_search_controller_data
def execute_many(insert_sql, batch_data):
global success_count, fails_count
cursor = conn.cursor()
try:
cursor.executemany(insert_sql, batch_data)
except Exception as e:
conn.rollback()
fails_count = fails_count + len(batch_data)
print(e)
raise
else:
conn.commit()
success_count = success_count + len(batch_data)
print(str(success_count) + " commit")
finally:
cursor.close()
def start(base_cid, max_cid, base_light_pole_id):
try:
insert_sql = "INSERT INTO `search_controller` (`s_Controller_id`, `LightPole_id`, `Controller_id`, `Read_time`, `Concentrator_name`, `Concentrator_address`, `Electric_quantity`, `Electric_voltage`, `Electric_current`, `Power`, `Power_factor`, `OneSwitch_status`, `TwoSwitch_status`, `Fault_status`, `LightOn_duration`, `brightness`) VALUES (NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
# batch_count = 0
begin_time = time.time()
batch_data = []
for x in range(loop_count):
gen_fun = random_generate_search_controller_data(base_cid, max_cid, base_light_pole_id)
batch_data = [gen_fun() for x in range(batch_size)]
execute_many(insert_sql, batch_data)
end_time = time.time()
total_sec = end_time - begin_time
qps = success_count / total_sec
print("總共生成數據： " + str(success_count))
print("總共耗時(s): " + str(total_sec))
print("QPS: " + str(qps))
except Exception as e:
print(e)
raise
else:
pass
finally:
pass
if __name__ == '__main__':
base_cid = 1000
max_cid = 10000 + base_cid - 1
base_light_pole_id = 53
start(base_cid, max_cid, base_light_pole_id)
測試結果：
...
99980000 commit
10000
99990000 commit
10000
100000000 commit
總共生成數據： 100000000
總共耗時(s): 12926.742953062057
QPS: 7735.900710883419
本機測試平均每秒插入7735條記錄，耗時約3.5個小時。
