上周末,幫朋友處理了一個關於大數據的查詢與導出問題,整理一下,在此記錄一下用以備忘,同時也為有類似需要的朋友提供一個參考.
背景:
改造步驟:
/// <summary>
/// 執行導出操作
/// </summary>
/// <param name="p"></param>
private void ExecExport(string fileName)
{
plProcessStatus.Visible = true;
SetControlStatus(true);
IExport rpter = new ExcelExporter();
var formater = BuildExportFormater();
bool isQueryEnd = false; //當前是否查詢結束
var templateFieName = Path.Combine(Application.StartupPath, "Template", "Rpt_CustomerList.xls");
//創建供導出的隊列
Queue<List<OrderDetail>> exportQueue = new Queue<List<OrderDetail>>();
#region 查詢線程
//處理後台查詢
Thread thQuery = new Thread(() =>
{
int tempTotal = 0;
int tempPgIndex = 0;
int queryPageSize = 3000; //每次查詢3k
var tempList = ExecQuery(tempPgIndex, queryPageSize, out tempTotal);
if (tempList != null && tempList.Count > 0)
{
lock (locker)
{
exportQueue.Enqueue(tempList);
Monitor.PulseAll(locker);
}
tempPgIndex += 1;
//循環查詢直至查詢結束
while (tempPgIndex * _pageSize < tempTotal)
{
var temp_tempList = ExecQuery(tempPgIndex, queryPageSize, out tempTotal);
if (temp_tempList != null && temp_tempList.Count > 0)
{
lock (locker)
{
exportQueue.Enqueue(temp_tempList); //將查詢結果加入到隊列
Monitor.PulseAll(locker);
}
}
tempPgIndex += 1;
}
}
isQueryEnd = true;
});
#endregion
#region 導出excel線程
//處理將查詢的結果寫入到文件中
Thread thExport = new Thread(() =>
{
rpter.Export(templateFieName, fileName, formater);//讀取模板,並創建新文件,
int tempRowIndex = 0;
while (!isQueryEnd || exportQueue.Count > 0) //未查詢結束及隊列不為空,執行導出
{
if (exportQueue.Count > 0)
{
List<OrderDetail> tempExpotLst = null;
lock (locker)
{
tempExpotLst = exportQueue.Dequeue(); //取隊列數據,導出excel操作
}
if (tempExpotLst != null && tempExpotLst.Count > 0)
{
formater.DetailRowBeginIndex += tempRowIndex;
rpter.ExportByAppend(fileName, formater, tempExpotLst); //執行導出操作(追加形式)
tempRowIndex = tempExpotLst.Count;
}
}
else
{
Thread.Sleep(200);
}
}
//導出貼圖片
var imgRow = formater.DetailRowBeginIndex + tempRowIndex + 8;
formater.ImageCellFormaters.Add(new ImageCellFormater(imgRow, 2, Resources.ywz));
rpter.ExportByAppend(fileName, formater, null);
//導出結束 恢復按鈕可用狀態
btnExport.Invoke(new Action(() =>
{
plProcessStatus.Visible = false; //隱藏進度欄
SetControlStatus(false);
if (MessageBox.Show(this, "數據已成功導出至[" + fileName + "],是否立即打開導出文件?",
"提示",
MessageBoxButtons.YesNo,
MessageBoxIcon.Question) == DialogResult.Yes)
{
Process.Start(fileName);
}
}));
});
#endregion
thQuery.IsBackground = true;
thExport.IsBackground = true;
thQuery.Start(); //開始查詢線程
thExport.Start(); //開始導出線程
}
/// <summary>
/// 獲取物流明細記錄
/// </summary>
/// <param name="queryParam">查詢條件</param>
/// <param name="total">返回符合條件的總記錄數量</param>
/// <returns></returns>
public List<OrderDetail> GetOrderDetailList(EFQueryParam<OrderDetail> queryParam, out int total)
{
total = 0;
var lst = GetRepository<OrderDetail, Int64>().Get(queryParam, out total).ToList();
//組織其他冗余數據
if (lst != null && lst.Count > 0)
{
//冗余公司信息,供前台UI使用
var companyList = GetCompanyList();
if (companyList != null && companyList.Count > 0)
{
var companyDic = companyList.ToDictionary(p => p.Id); //轉化為字典,提高效率
var tempbgIndex = queryParam.PageIndex * queryParam.PageSize + 1; //生成排序
lst.ForEach(t =>
{
t.Index = tempbgIndex;
//寄件公司
if (companyDic.ContainsKey(t.FromCompanyId))
{
t.FromComoany = companyDic[t.FromCompanyId];
}
//收件公司
if (companyDic.ContainsKey(t.ToCompanyId))
{
t.ToCompany = companyDic[t.ToCompanyId];
}
//付款公司
if (companyDic.ContainsKey(t.PaymentCompanyId))
{
t.PaymentCompany = companyDic[t.PaymentCompanyId];
}
tempbgIndex += 1;
});
}
}
return lst;
}
/// <summary>
/// 獲取公司信息
/// </summary>
/// <returns></returns>
public List<Company> GetCompanyList()
{
//從緩存中獲取
var lst = ApplicationRuntime.Instance.CurrentCache.Get<List<Company>>(KYEConsts.CachesKey.Company,
() => this.GetRepository<Company>().Get().ToList());
return lst;
}
結語: 經過這一系列改造後, 性能上大大改進了,查詢響應耗時<=1s, 導出<=8s, 每次符合條件的數據大概在2W條左右.
至於本文中提到的
1. EF構建查詢條件表達式,及查詢數據庫方式,可參見之前文章.http://www.cnblogs.com/xie-zhonglai/archive/2012/04/07/2435903.html
2. 導出Excel.本文使用了NPOI 這個組件,詳情可參見文章: http://www.cnblogs.com/xie-zhonglai/p/3979771.html