1. 程式人生 > >libcurl採用curl_multi_perform() + curl_multi_wait()方式實現非同步高效能l傳送資料的方法

libcurl採用curl_multi_perform() + curl_multi_wait()方式實現非同步高效能l傳送資料的方法

前兩篇文章

講述了採用libcurl傳送資料的基礎方法和高效能方法,基礎方法較為容易但效能一般,高階方法的效能卓越但比較難理解,這裡再給出一個保證效能的同時又相對較容易理解的方法,該方法最初是由facebook貢獻,實現如下:

#define CURL_MULTI_NUM 5
#define CURL_MAX_WAIT_MSECS 30*1000

//將單條資料新增到總
static void add_multi_curl(CURLM* cm, Task* task) {
  struct curl_slist* headers = NULL;
  std::string tmp_str;

  //新增頭資訊
  tmp_str = "User-Agent: ";
  tmp_str += task->user_agent;
  headers = curl_slist_append(headers, tmp_str.c_str());
  tmp_str = "Accept-Language: ";
  tmp_str += task->language;
  headers = curl_slist_append(headers, tmp_str.c_str());
  tmp_str = "X-FORWORDED-FOR: ";
  tmp_str += task->x_forwarded_for;
  headers = curl_slist_append(headers, tmp_str.c_str());

  
  CURL* curl = curl_easy_init();
  //設定curl引數
  curl_easy_setopt(curl, CURLOPT_DNS_CACHE_TIMEOUT, 60 * 60 * 72);
  curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
  curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5);
  curl_easy_setopt(curl, CURLOPT_HEADER, 0L);
  curl_easy_setopt(curl, CURLOPT_URL, task->uri.c_str());
  curl_easy_setopt(curl, CURLOPT_PRIVATE, task->uri.c_str());
  curl_easy_setopt(curl, CURLOPT_VERBOSE, 0L);

  //新增單條curl資訊到總資訊中
  curl_multi_add_handle(cm, curl);
}

//傳送總資料
bool send_multi_data(CURLM* cm) {
  CURL* eh = NULL;
  CURLMsg* msg = NULL;
  CURLcode return_code = CURLE_OK;
  int32_t still_running = 0;
  int32_t msgs_left = 0;
  int32_t http_status_code;
  const char* szUrl;

  //傳送資料並用still_running監測資料是否傳送完成,如果未傳送完成則持續傳送
  curl_multi_perform(cm, &still_running);
  do {
    int numfds = 0;
    int res = curl_multi_wait(cm, NULL, 0, CURL_MAX_WAIT_MSECS, &numfds);
    if (res != CURLM_OK) {
      LOG(WARNING) << "error: curl_multi_wait() returned" << res;
      return false;
    }
    curl_multi_perform(cm, &still_running);
  } while (still_running);

  //讀取傳送返回結果資訊
  while ((msg = curl_multi_info_read(cm, &msgs_left))) {
	//是否讀取成功
    if (msg->msg == CURLMSG_DONE) {
      eh = msg->easy_handle;

      return_code = msg->data.result;
	  //檢測資料傳送結果
      if (return_code != CURLE_OK) {
        LOG(WARNING) << "CURL error code: " << msg->data.result;
        continue;
      }

      //檢測HTTP狀態
      http_status_code = 0;
      szUrl = NULL;

      curl_easy_getinfo(eh, CURLINFO_RESPONSE_CODE, &http_status_code);
      curl_easy_getinfo(eh, CURLINFO_PRIVATE, &szUrl);

      if (http_status_code != 200) {
        LOG(WARNING) << "GET of" << szUrl << "returned http status code " << http_status_code;
      }

      curl_multi_remove_handle(cm, eh);
      curl_easy_cleanup(eh);
    } else {
      LOG(WARNING) << "error: after curl_multi_info_read(), CURLMsg= " << msg->msg;
    }
  }

  return true;
}

int main()
{
	
  int loop() {
    CURLM* cm = NULL;
    cm = curl_multi_init();
    int add_num = 0;
    while (!IsStoped()) {
      Task* task = GetTaskItem(); //獲取待發送資料,採取訊息佇列的方式
      if (task == nullptr) {
		//如果獲取不到資料,則將當前已獲取的資料做一次性發送後等待
        if (add_num == 0) {
		  //迴圈等待訊息佇列
          WaitTask();
          continue;
        } else {
          send_multi_data(cm);
          add_num = 0;
        }
      } else {
		//資料量積累到CURL_MULTI_NUM後做一次性發送
        add_multi_curl(cm, task);
        add_num += 1;
        if (add_num >= CURL_MULTI_NUM) {
          send_multi_data(cm);
          add_num = 0;
        } else {
          continue;
        }
      }
    }
    curl_multi_cleanup(cm);
    return true;
  };
};