libcurl採用curl_multi_perform() + curl_multi_wait()方式實現非同步高效能l傳送資料的方法
阿新 • • 發佈:2019-01-27
前兩篇文章
講述了採用libcurl傳送資料的基礎方法和高效能方法,基礎方法較為容易但效能一般,高階方法的效能卓越但比較難理解,這裡再給出一個保證效能的同時又相對較容易理解的方法,該方法最初是由facebook貢獻,實現如下:
#define CURL_MULTI_NUM 5 #define CURL_MAX_WAIT_MSECS 30*1000 //將單條資料新增到總 static void add_multi_curl(CURLM* cm, Task* task) { struct curl_slist* headers = NULL; std::string tmp_str; //新增頭資訊 tmp_str = "User-Agent: "; tmp_str += task->user_agent; headers = curl_slist_append(headers, tmp_str.c_str()); tmp_str = "Accept-Language: "; tmp_str += task->language; headers = curl_slist_append(headers, tmp_str.c_str()); tmp_str = "X-FORWORDED-FOR: "; tmp_str += task->x_forwarded_for; headers = curl_slist_append(headers, tmp_str.c_str()); CURL* curl = curl_easy_init(); //設定curl引數 curl_easy_setopt(curl, CURLOPT_DNS_CACHE_TIMEOUT, 60 * 60 * 72); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5); curl_easy_setopt(curl, CURLOPT_HEADER, 0L); curl_easy_setopt(curl, CURLOPT_URL, task->uri.c_str()); curl_easy_setopt(curl, CURLOPT_PRIVATE, task->uri.c_str()); curl_easy_setopt(curl, CURLOPT_VERBOSE, 0L); //新增單條curl資訊到總資訊中 curl_multi_add_handle(cm, curl); } //傳送總資料 bool send_multi_data(CURLM* cm) { CURL* eh = NULL; CURLMsg* msg = NULL; CURLcode return_code = CURLE_OK; int32_t still_running = 0; int32_t msgs_left = 0; int32_t http_status_code; const char* szUrl; //傳送資料並用still_running監測資料是否傳送完成,如果未傳送完成則持續傳送 curl_multi_perform(cm, &still_running); do { int numfds = 0; int res = curl_multi_wait(cm, NULL, 0, CURL_MAX_WAIT_MSECS, &numfds); if (res != CURLM_OK) { LOG(WARNING) << "error: curl_multi_wait() returned" << res; return false; } curl_multi_perform(cm, &still_running); } while (still_running); //讀取傳送返回結果資訊 while ((msg = curl_multi_info_read(cm, &msgs_left))) { //是否讀取成功 if (msg->msg == CURLMSG_DONE) { eh = msg->easy_handle; return_code = msg->data.result; //檢測資料傳送結果 if (return_code != CURLE_OK) { LOG(WARNING) << "CURL error code: " << msg->data.result; continue; } //檢測HTTP狀態 http_status_code = 0; szUrl = NULL; curl_easy_getinfo(eh, CURLINFO_RESPONSE_CODE, &http_status_code); curl_easy_getinfo(eh, CURLINFO_PRIVATE, &szUrl); if (http_status_code != 200) { LOG(WARNING) << "GET of" << szUrl << "returned http status code " << http_status_code; } curl_multi_remove_handle(cm, eh); curl_easy_cleanup(eh); } else { LOG(WARNING) << "error: after curl_multi_info_read(), CURLMsg= " << msg->msg; } } return true; } int main() { int loop() { CURLM* cm = NULL; cm = curl_multi_init(); int add_num = 0; while (!IsStoped()) { Task* task = GetTaskItem(); //獲取待發送資料,採取訊息佇列的方式 if (task == nullptr) { //如果獲取不到資料,則將當前已獲取的資料做一次性發送後等待 if (add_num == 0) { //迴圈等待訊息佇列 WaitTask(); continue; } else { send_multi_data(cm); add_num = 0; } } else { //資料量積累到CURL_MULTI_NUM後做一次性發送 add_multi_curl(cm, task); add_num += 1; if (add_num >= CURL_MULTI_NUM) { send_multi_data(cm); add_num = 0; } else { continue; } } } curl_multi_cleanup(cm); return true; }; };