Redis一致性hash(php版)
阿新 • • 發佈:2018-12-25
一致性hash的使用在PHP中有三種選擇分別是原生的memcache擴充套件,memcached擴充套件,還有一個是網上比較流行的flexihash類。前兩者都適用於memcache但不適合Redis。
php一致性hash類下載地址:http://code.google.com/p/flexihash/
我們根據flexihash的應該改寫了一遍Redis的應用。
下面給出測試原始碼:
flexihash.php
<?php /** * Flexihash - A simple consistent hashing implementation for PHP. * * The MIT License * * Copyright (c) 2008 Paul Annesley * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * @author Paul Annesley * @link http://paul.annesley.cc/ * @copyright Paul Annesley, 2008 * @comment by MyZ (http://blog.csdn.net/mayongzhan) */ /** * A simple consistent hashing implementation with pluggable hash algorithms. * * @author Paul Annesley * @package Flexihash * @licence http://www.opensource.org/licenses/mit-license.php */ class flexihash { /** * The number of positions to hash each target to. * * @var int * @comment 虛擬節點數,解決節點分佈不均的問題 */ private $_replicas = 64; /** * The hash algorithm, encapsulated in a Flexihash_Hasher implementation. * @var object Flexihash_Hasher * @comment 使用的hash方法 : md5,crc32 */ private $_hasher; /** * Internal counter for current number of targets. * @var int * @comment 節點記數器 */ private $_targetCount = 0; /** * Internal map of positions (hash outputs) to targets * @var array { position => target, ... } * @comment 位置對應節點,用於lookup中根據位置確定要訪問的節點 */ private $_positionToTarget = array(); /** * Internal map of targets to lists of positions that target is hashed to. * @var array { target => [ position, position, ... ], ... } * @comment 節點對應位置,用於刪除節點 */ private $_targetToPositions = array(); /** * Whether the internal map of positions to targets is already sorted. * @var boolean * @comment 是否已排序 */ private $_positionToTargetSorted = false; /** * Constructor * @param object $hasher Flexihash_Hasher * @param int $replicas Amount of positions to hash each target to. * @comment 建構函式,確定要使用的hash方法和需擬節點數,虛擬節點數越多,分佈越均勻,但程式的分散式運算越慢 */ public function __construct(Flexihash_Hasher $hasher = null, $replicas = null) { $this->_hasher = $hasher ? $hasher : new Flexihash_Crc32Hasher(); if (!empty($replicas)) $this->_replicas = $replicas; } /** * Add a target. * @param string $target * @chainable * @comment 新增節點,根據虛擬節點數,將節點分佈到多個虛擬位置上 */ public function addTarget($target) { if (isset($this->_targetToPositions[$target])) { throw new Flexihash_Exception("Target '$target' already exists."); } $this->_targetToPositions[$target] = array(); // hash the target into multiple positions for ($i = 0; $i < $this->_replicas; $i++) { $position = $this->_hasher->hash($target . $i); $this->_positionToTarget[$position] = $target; // lookup $this->_targetToPositions[$target] []= $position; // target removal } $this->_positionToTargetSorted = false; $this->_targetCount++; return $this; } /** * Add a list of targets. * @param array $targets * @chainable */ public function addTargets($targets) { foreach ($targets as $target) { $this->addTarget($target); } return $this; } /** * Remove a target. * @param string $target * @chainable */ public function removeTarget($target) { if (!isset($this->_targetToPositions[$target])) { throw new Flexihash_Exception("Target '$target' does not exist."); } foreach ($this->_targetToPositions[$target] as $position) { unset($this->_positionToTarget[$position]); } unset($this->_targetToPositions[$target]); $this->_targetCount--; return $this; } /** * A list of all potential targets * @return array */ public function getAllTargets() { return array_keys($this->_targetToPositions); } /** * Looks up the target for the given resource. * @param string $resource * @return string */ public function lookup($resource) { $targets = $this->lookupList($resource, 1); if (empty($targets)) throw new Flexihash_Exception('No targets exist'); return $targets[0]; } /** * Get a list of targets for the resource, in order of precedence. * Up to $requestedCount targets are returned, less if there are fewer in total. * * @param string $resource * @param int $requestedCount The length of the list to return * @return array List of targets * @comment 查詢當前的資源對應的節點, * 節點為空則返回空,節點只有一個則返回該節點, * 對當前資源進行hash,對所有的位置進行排序,在有序的位置列上尋找當前資源的位置 * 當全部沒有找到的時候,將資源的位置確定為有序位置的第一個(形成一個環) * 返回所找到的節點 */ public function lookupList($resource, $requestedCount) { if (!$requestedCount) throw new Flexihash_Exception('Invalid count requested'); // handle no targets if (empty($this->_positionToTarget)) return array(); // optimize single target if ($this->_targetCount == 1) return array_unique(array_values($this->_positionToTarget)); // hash resource to a position $resourcePosition = $this->_hasher->hash($resource); $results = array(); $collect = false; $this->_sortPositionTargets(); // search values above the resourcePosition foreach ($this->_positionToTarget as $key => $value) { // start collecting targets after passing resource position if (!$collect && $key > $resourcePosition) { $collect = true; } // only collect the first instance of any target if ($collect && !in_array($value, $results)) { $results []= $value; } // return when enough results, or list exhausted if (count($results) == $requestedCount || count($results) == $this->_targetCount) { return $results; } } // loop to start - search values below the resourcePosition foreach ($this->_positionToTarget as $key => $value) { if (!in_array($value, $results)) { $results []= $value; } // return when enough results, or list exhausted if (count($results) == $requestedCount || count($results) == $this->_targetCount) { return $results; } } // return results after iterating through both "parts" return $results; } public function __toString() { return sprintf( '%s{targets:[%s]}', get_class($this), implode(',', $this->getAllTargets()) ); } // ---------------------------------------- // private methods /** * Sorts the internal mapping (positions to targets) by position */ private function _sortPositionTargets() { // sort by key (position) if not already if (!$this->_positionToTargetSorted) { ksort($this->_positionToTarget, SORT_REGULAR); $this->_positionToTargetSorted = true; } } } /** * Hashes given values into a sortable fixed size address space. * * @author Paul Annesley * @package Flexihash * @licence http://www.opensource.org/licenses/mit-license.php */ interface Flexihash_Hasher { /** * Hashes the given string into a 32bit address space. * * Note that the output may be more than 32bits of raw data, for example * hexidecimal characters representing a 32bit value. * * The data must have 0xFFFFFFFF possible values, and be sortable by * PHP sort functions using SORT_REGULAR. * * @param string * @return mixed A sortable format with 0xFFFFFFFF possible values */ public function hash($string); } /** * Uses CRC32 to hash a value into a signed 32bit int address space. * Under 32bit PHP this (safely) overflows into negatives ints. * * @author Paul Annesley * @package Flexihash * @licence http://www.opensource.org/licenses/mit-license.php */ class Flexihash_Crc32Hasher implements Flexihash_Hasher { /* (non-phpdoc) * @see Flexihash_Hasher::hash() */ public function hash($string) { return crc32($string); } } /** * Uses CRC32 to hash a value into a 32bit binary string data address space. * * @author Paul Annesley * @package Flexihash * @licence http://www.opensource.org/licenses/mit-license.php */ class Flexihash_Md5Hasher implements Flexihash_Hasher { /* (non-phpdoc) * @see Flexihash_Hasher::hash() */ public function hash($string) { return substr(md5($string), 0, 8); // 8 hexits = 32bit // 4 bytes of binary md5 data could also be used, but // performance seems to be the same. } } /** * An exception thrown by Flexihash. * * @author Paul Annesley * @package Flexihash * @licence http://www.opensource.org/licenses/mit-license.php */ class Flexihash_Exception extends Exception { }
flexihash_redis.php
<?php require 'flexihash.php'; Class FRedis { public $hash = null; public $memcache = null; public $connectPool = null; public $timeOut = 3; public $auth = ""; public function __construct() { $this->hash = new Flexihash(); } public function addServers( $servers ) { foreach ($servers as $server) { $node = $server['host'] . ':' . $server['port']; $this->connectPool[$node] = false; $targets[] = $node; } $this->hash->addTargets( $targets ); } public function hset($hashkey, $valuekey, $value ) { $node = $this->hash->lookup( $hashkey, count( $this->connectPool ) ); if (!$this->connectPool[$node]) { $server = explode( ':', $node ); $redis = new Redis(); $redis->connect($server[0], $server[1], $this->timeOut); if(!empty($this->auth)){ $redis->auth($this->auth); } $this->connectPool[$node] = $redis; } if ($this->connectPool[$node]) { $redis = $this->connectPool[$node]; if($redis->hset($hashkey,$valuekey,$value)){ return true; } } return false; } public function hget( $hashkey, $valuekey ) { $node = $this->hash->lookup( $hashkey, count( $this->connectPool ) ); if (!$this->connectPool[$node]) { $server = explode( ':', $node ); $redis = new Redis(); $redis->connect($server[0], $server[1], $this->timeOut); if(!empty($this->auth)){ $redis->auth($this->auth); } $this->connectPool[$node] = $redis; } if ($this->connectPool[$node]) { $redis = $this->connectPool[$node]; if($value = $redis->hget($hashkey,$valuekey)){ return $value; } } return false; } }
FRedis_test.php
<?php require_once 'flexihash_redis.php'; $st = microtime( true ); $fredis = new FRedis(); $tt_server_pool = array( '0' => array( 'host' => '192.168.75.128', 'port' => '6379', ), '1' => array( 'host' => '192.168.75.130', 'port' => '6379', ), ); $fredis->addServers( $tt_server_pool ); for($i=0;$i<10;$i++){ for($k=0;$k<10;$k++){ $fredis->hset("hashkey:".$i,"valuekey".$k,md5($i*$k)); } } $et = microtime( true ) - $st; echo "time used:" . $et."<br/>"; echo $fredis->hget("hashkey:0","valuekey9");
redis版雜湊函式類:
<?php
class UniqueHashService{
private static $len; //64位
private static $loop = 1; //迴圈次數
private $prefix = '';
public function __construct() {
if(empty(self::$len)){
self::$len = abs(1 << 0x3f); //左移63位
}
}
/**
* 獲取字串的雜湊值
*/
public function hashing($key){
return abs((('0x'.hash('crc32',$key).hash('crc32b',$key)) + 0)) % self::$len;
}
/**
* 獲取到redis hash 的鍵名
* @param type $str
*/
public function getRedisKey($str, $prefix){
//字首
$this->prefix = $prefix;
$hstr = $this->hashing($str); //獲取一個字串的雜湊值
//如果字串長度小於6,往字串左側填充0,填充至6位長度。
if(strlen($hstr) < 6){
$hstr = str_pad($hstr, 6, '0', STR_PAD_LEFT);
}
$kbin = $this->str2Bin($hstr); //計算hash key的二進位制,$hstr的二進位制
$karr = explode(' ', $kbin);
//陣列長度
$kcount = count($karr);
$kcount_mid = ceil($kcount/2);
$redis_key = $this->prefix; //排重表字首
//迴圈次數
$i = 0;
while(1){
if($i > self::$loop){
break;
}
//組裝redis鍵
$redis_key .= ':'.$this->returnHashKey($karr[$i]);
$redis_key .= ':'.$this->returnHashKey($karr[$kcount_mid+$i]);
$redis_key .= ':'.$this->returnHashKey($karr[$kcount-1-$i]);
++$i;
}
return $redis_key;
}
/**
* 擷取二進位制字串中的一部分
* @param type $sbin
* @return type
*/
private function returnHashKey($sbin){
return substr($sbin, -2, 2);
}
/**
* 把給定的字串中的每個字元轉為二進位制,並以空格連線每個二進位制字串並返回。
* @param string $hstr
* @return type
*/
private function str2Bin($str){
//把字串拆分成陣列,每個字元為陣列中的一個元素。
$arr = str_split($str);
//把陣列中的每個字元轉成二進位制
foreach($arr as &$v){
$tmp = unpack('H*', $v); //把字元解包為16進位制
$v = base_convert($tmp[1],16,2);//把16進位制轉為2進位制
unset($tmp);
}
//把二進位制字串陣列以空格連線在一起並返回
$bin = join(' ',$arr);
unset($arr);
return $bin;
}
/**
* 把$this->str2Bin 處理過的字串還原
* @param string $bin 二進位制字串
* @return type
*/
private function bin2Str($bin){
//把字串拆分成陣列,以空拆分。
$arr = explode(' ', $bin);
foreach($arr as &$v){
$tmp = base_convert($v, 2, 16); //二進位制轉換成16進位制
$v = pack('H'.strlen($tmp), $tmp); //把16進位制打包成二進位制字串
unset($tmp);
}
return join('', $arr);
}
}