Python多线程批量插入数据小结(2)

for i in range(int(deal_num)):
        orderData=""
        orderData = "%s,'%s','%s','%s'" % (order_base, Ftde_id, Fbank_list, Fbatchno)
        orderDataList.append(orderData)
        Ftde_id = int(Ftde_id) + 1
        Flistid = int(Flistid) + 1
        Fbank_list = int(Fbank_list) + 1

writeDownSqlData(resultDir+filename, "\n".join(orderDataList) + "\n")

loadDataIntoDB(resultDir+filename,tablename,order_columns)

def loadDataIntoDB(filename,tableName,order_columns):
    mysqlCmd = r"LOAD DATA LOCAL INFILE '%s' into table %s FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\''  LINES TERMINATED BY '\n' (%s"%(filename,tableName,order_columns)+")"

mysqlConCmd = r"mysql -u%s -p%s -h%s"%(dbUser,dbPasswd,dbHost)
    runCmd = mysqlConCmd+' -e "'+mysqlCmd+'"'
    result = call(runCmd,shell=True)
    if result != 0:
        print "load local data into database failed,exit "
        sys.exit()

if __name__ == "__main__":
    start = time.time()

begin_Flistid ='110180809100012153304210311120'
    begin_Ftde_id ="1"
    begin_Fbank_seq ="2018100800000110734321790770100"
    total_num  =20000000
    threadpool_num = 20
    func_var=[]
    seperate =2000

pool = threadpool.ThreadPool(threadpool_num)

for i  in range(seperate):
        list_temp =[]
        list_temp =[str(total_num/seperate),begin_Flistid,begin_Ftde_id,begin_Fbank_seq]
        func_var.append((list_temp,None))
        begin_Flistid = str(int(begin_Flistid)+ total_num/threadpool_num)
        begin_Ftde_id = str(int(begin_Ftde_id)+ total_num/threadpool_num)
        begin_Fbank_seq = str(int(begin_Fbank_seq)+ total_num/threadpool_num)

pool = threadpool.ThreadPool(threadpool_num)
    requests = threadpool.makeRequests(make_t_tcpay_list, func_var)
    for req in requests:
        pool.putRequest(req)
    pool.wait()

end = time.time()
    print end - start

Linux公社的RSS地址：https://www.linuxidc.com/rssFeed.aspx