php curl批處理--可控併發非同步【轉】
阿新 • • 發佈：2020-07-01
通常情況下 PHP 中的 cURL 是阻塞執行的，就是說建立一個 cURL 請求以後必須等它執行成功或者超時才會執行下一個請求:API介面訪問一般會首選CURL
在實際專案或者自己編寫小工具(比如新聞聚合,商品價格監控,比價)的過程中, 通常需要從第3方網站或者API介面獲取資料, 在需要處理1個URL佇列時, 為了提高效能, 可以採用cURL提供的curl_multi_*族函式實現簡單的併發.
<?php include 'curl.class.php'; function callback($response, $info, $error, $request) { echo 'response:<br>'; print_r($response); echo '<br>' . date("Y-m-d H:i:s") . ' <br>'; echo '<br>' . str_repeat("-", 100) . '<br>'; } $USER_COOKIE = (!empty($_REQUEST['cookie'])) ? $_REQUEST['cookie'] : file_get_contents("cookie.txt"); $curl = new Curl ("callback"); $data = array( array( 'url' => 'http://dyactive2.vip.xunlei.com/com_sign/?game=qmr&type=rec_gametime&referfrom=&rt=0.42521539455332336', //秦美人 'method' => 'POST', 'post_data' => '', 'header' => null, 'options' => array( CURLOPT_REFERER => "http://niu.xunlei.com/entergame/?gameNo=qmr&fenQuNum=3", CURLOPT_COOKIE => $USER_COOKIE, ) ), array( 'url' => 'http://dyactive2.vip.xunlei.com/com_sign/?game=sq&type=rec_gametime&referfrom=&rt=0.42521539455332336', //神曲 'method' => 'POST', 'post_data' => '', 'header' => null, 'options' => array( CURLOPT_REFERER => "http://niu.xunlei.com/entergame/?gameNo=sq&fenQuNum=41", CURLOPT_COOKIE => $USER_COOKIE, ) ), array( 'url' => 'http://dyactive2.vip.xunlei.com/com_sign/?game=frxz&type=rec_gametime&referfrom=&rt=0.42521539455332336', //凡人修真 'method' => 'POST', 'post_data' => '', 'header' => null, 'options' => array( CURLOPT_REFERER => "http://niu.xunlei.com/entergame/?gameNo=frxz&fenQuNum=3", CURLOPT_COOKIE => $USER_COOKIE, ) ), array( 'url' => 'http://dyactive2.vip.xunlei.com/com_sign/?game=smxj&type=rec_gametime&referfrom=&rt=0.42521539455332336', //神魔仙界 'method' => 'POST', 'post_data' => '', 'header' => null, 'options' => array( CURLOPT_REFERER => "http://niu.xunlei.com/entergame/?gameNo=smxj&fenQuNum=2", CURLOPT_COOKIE => $USER_COOKIE, ) ), array( 'url' => 'http://dyactive2.vip.xunlei.com/com_sign/?game=qsqy&type=rec_gametime&referfrom=&rt=0.42521539455332336', //傾世情緣 'method' => 'POST', 'post_data' => '', 'header' => null, 'options' => array( CURLOPT_REFERER => "http://niu.xunlei.com/entergame/?gameNo=qsqy&fenQuNum=11", CURLOPT_COOKIE => $USER_COOKIE, ) ), ); foreach ($data as $val) { $request = new Curl_request ($val ['url'], $val ['method'], $val ['post_data'], $val ['header'], $val ['options']); $curl->add($request); } $curl->execute(); echo $curl->display_errors();
使用下來效果很好，沒有副作用，併發數可控，應用之處多多，自己發揮想象吧
<?php /** * cURL批量處理 工具類 * * @since Version 1.0 * @author Justmepzy <[email protected]> * @link http://t.qq.com/JustPzy */ /** *單一的請求物件 */ class Curl_request { public $url = ''; public $method = 'GET'; public $post_data = null; public $headers = null; public $options = null; /** * * @param string $url * @param string $method * @param string $post_data * @param string $headers * @param array $options * @return void */ public function __construct($url, $method = 'GET', $post_data = null, $headers = null, $options = null) { $this->url = $url; $this->method = strtoupper( $method ); $this->post_data = $post_data; $this->headers = $headers; $this->options = $options; } /** * @return void */ public function __destruct() { unset ( $this->url, $this->method, $this->post_data, $this->headers, $this->options ); } } /** * 包含請求列隊處理 */ class Curl { /** * 請求url個數 * @var int */ private $size = 5; /** * 等待所有cURL批處理中的活動連線等待響應時間 * @var int */ private $timeout = 5; /** * 完成請求回撥函式 * @var string */ private $callback = null; /** * cRUL配置 * @var array */ private $options = array (CURLOPT_SSL_VERIFYPEER => 0,CURLOPT_RETURNTRANSFER => 1,CURLOPT_CONNECTTIMEOUT => 30 ); /** * 請求頭 * @var array */ private $headers = array (); /** * 請求列隊 * @var array */ private $requests = array (); /** * 請求列隊索引 * @var array */ private $request_map = array (); /** * 錯誤 * @var array */ private $errors = array (); /** * @access public * @param string $callback 回撥函式 * 該函式有4個引數($response,$info,$error,$request) * $response url返回的body * $info cURL連線資源控制代碼的資訊 * $error 錯誤 * $request 請求物件 */ public function __construct($callback = null) { $this->callback = $callback; } /** * 新增一個請求物件到列隊 * @access public * @param object $request * @return boolean */ public function add($request) { $this->requests [] = $request; return TRUE; } /** * 建立一個請求物件並新增到列隊 * @access public * @param string $url * @param string $method * @param string $post_data * @param string $headers * @param array $options * @return boolean */ public function request($url, $method = 'GET', $post_data = null, $headers = null, $options = null) { $this->requests [] = new Curl_request ( $url, $method, $post_data, $headers, $options ); return TRUE; } /** * 建立GET請求物件 * @access public * @param string $url * @param string $headers * @param array $options * @return boolean */ public function get($url, $headers = null, $options = null) { return $this->request ( $url, "GET", null, $headers, $options ); } /** * 建立一個POST請求物件 * @access public * @param string $url * @param string $post_data * @param string $headers * @param array $options * @return boolean */ public function post($url, $post_data = null, $headers = null, $options = null) { return $this->request ( $url, "POST", $post_data, $headers, $options ); } /** * 執行cURL * @access public * @param int $size 最大連線數 * @return Ambigous <boolean, mixed>|boolean */ public function execute($size = null) { if (sizeof ( $this->requests ) == 1) { return $this->single_curl (); } else { return $this->rolling_curl ( $size ); } } /** * 單個url請求 * @access private * @return mixed|boolean */ private function single_curl() { $ch = curl_init (); $request = array_shift ( $this->requests ); $options = $this->get_options ( $request ); curl_setopt_array ( $ch, $options ); $output = curl_exec ( $ch ); $info = curl_getinfo ( $ch ); // it's not neccesary to set a callback for one-off requests if ($this->callback) { $callback = $this->callback; if (is_callable ( $this->callback )) { call_user_func ( $callback, $output, $info, $request ); } } else return $output; return true; } /** * 多個url請求 * @access private * @param int $size 最大連線數 * @return boolean */ private function rolling_curl($size = null) { if ($size) $this->size = $size; else $this->size = count($this->requests); if (sizeof ( $this->requests ) < $this->size) $this->size = sizeof ( $this->requests ); if ($this->size < 2) $this->set_error ( 'size must be greater than 1' ); $master = curl_multi_init (); //新增cURL連線資源控制代碼到map索引 for($i = 0; $i < $this->size; $i ++) { $ch = curl_init (); $options = $this->get_options ( $this->requests [$i] ); curl_setopt_array ( $ch, $options ); curl_multi_add_handle ( $master, $ch ); $key = ( string ) $ch; $this->request_map [$key] = $i; } $active = $done = null; do { while ( ($execrun = curl_multi_exec ( $master, $active )) == CURLM_CALL_MULTI_PERFORM ) ; if ($execrun != CURLM_OK) break; //有一個請求完成則回撥 while ( $done = curl_multi_info_read ( $master ) ) { //$done 完成的請求控制代碼 $info = curl_getinfo ( $done ['handle'] );// $output = curl_multi_getcontent ( $done ['handle'] );// $error = curl_error ( $done ['handle'] );// $this->set_error ( $error ); //呼叫回撥函式,如果存在的話 $callback = $this->callback; if (is_callable ( $callback )) { $key = ( string ) $done ['handle']; $request = $this->requests [$this->request_map [$key]]; unset ( $this->request_map [$key] ); call_user_func ( $callback, $output, $info, $error, $request ); } curl_close ( $done ['handle'] ); //從列隊中移除已經完成的request curl_multi_remove_handle ( $master, $done ['handle'] ); } //等待所有cURL批處理中的活動連線 if ($active) curl_multi_select ( $master, $this->timeout ); } while ( $active ); //完成關閉 curl_multi_close ( $master ); return true; } /** * 獲取沒得請求物件的cURL配置 * @access private * @param object $request * @return array */ private function get_options($request) { $options = $this->__get ( 'options' ); if (ini_get ( 'safe_mode' ) == 'Off' || ! ini_get ( 'safe_mode' )) { $options [CURLOPT_FOLLOWLOCATION] = 1; $options [CURLOPT_MAXREDIRS] = 5; } $headers = $this->__get ( 'headers' ); if ($request->options) { $options = $request->options + $options; } if ($request->headers) { $headers = $request->headers + $headers; } $options [CURLOPT_URL] = $request->url; if ($request->post_data && strtolower($request->method) == 'post' ) { $options [CURLOPT_POST] = 1; $options [CURLOPT_POSTFIELDS] = $request->post_data; } if ($headers) { $options [CURLOPT_HEADER] = 0; $options [CURLOPT_HTTPHEADER] = $headers; } return $options; } /** * 設定錯誤資訊 * @access public * @param string $msg */ public function set_error($msg) { if (! empty ( $msg )) $this->errors [] = $msg; } /** * 獲取錯誤資訊 * @access public * @param string $open * @param string $close * @return string */ public function display_errors($open = '<p>', $close = '</p>') { $str = ''; foreach ( $this->errors as $val ) { $str .= $open . $val . $close; } return $str; } /** * @access public * @param string $name * @param string $value * @return boolean */ public function __set($name, $value) { if ($name == 'options' || $name == 'headers') { $this->{$name} = $value + $this->{$name}; } else { $this->{$name} = $value; } return TRUE; } /** * * @param string $name * @return mixed * @access public */ public function __get($name) { return (isset ( $this->{$name} )) ? $this->{$name} : null; } /** * @return void * @access public */ public function __destruct() { unset ( $this->size, $this->timeout, $this->callback, $this->options, $this->headers, $this->requests, $this->request_map, $this->errors ); } } // END Curl Class /* End of file curl.class.php */
參考連結：https://www.cnblogs.com/chunguang/p/5895195.html
原文連結：https://www.iteye.com/blog/hudeyong926-1635386