1. 程式人生 > >Redis一致性hash(php版)

Redis一致性hash(php版)


一致性hash的使用在PHP中有三種選擇分別是原生的memcache擴充套件,memcached擴充套件,還有一個是網上比較流行的flexihash類。前兩者都適用於memcache但不適合Redis。

php一致性hash類下載地址:http://code.google.com/p/flexihash/

我們根據flexihash的應該改寫了一遍Redis的應用。

下面給出測試原始碼:

flexihash.php
<?php
/**
 * Flexihash - A simple consistent hashing implementation for PHP.
 *
 * The MIT License
 *
 * Copyright (c) 2008 Paul Annesley
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * @author Paul Annesley
 * @link http://paul.annesley.cc/
 * @copyright Paul Annesley, 2008
 * @comment by MyZ (http://blog.csdn.net/mayongzhan)
 */

/**
 * A simple consistent hashing implementation with pluggable hash algorithms.
 *
 * @author Paul Annesley
 * @package Flexihash
 * @licence http://www.opensource.org/licenses/mit-license.php
 */
class flexihash
{

    /**
     * The number of positions to hash each target to.
     *
     * @var int
     * @comment 虛擬節點數,解決節點分佈不均的問題
     */
    private $_replicas = 64;

    /**
     * The hash algorithm, encapsulated in a Flexihash_Hasher implementation.
     * @var object Flexihash_Hasher
     * @comment 使用的hash方法 : md5,crc32
     */
    private $_hasher;

    /**
     * Internal counter for current number of targets.
     * @var int
     * @comment 節點記數器
     */
    private $_targetCount = 0;

    /**
     * Internal map of positions (hash outputs) to targets
     * @var array { position => target, ... }
     * @comment 位置對應節點,用於lookup中根據位置確定要訪問的節點
     */
    private $_positionToTarget = array();

    /**
     * Internal map of targets to lists of positions that target is hashed to.
     * @var array { target => [ position, position, ... ], ... }
     * @comment 節點對應位置,用於刪除節點
     */
    private $_targetToPositions = array();

    /**
     * Whether the internal map of positions to targets is already sorted.
     * @var boolean
     * @comment 是否已排序
     */
    private $_positionToTargetSorted = false;

    /**
     * Constructor
     * @param object $hasher Flexihash_Hasher
     * @param int $replicas Amount of positions to hash each target to.
     * @comment 建構函式,確定要使用的hash方法和需擬節點數,虛擬節點數越多,分佈越均勻,但程式的分散式運算越慢
     */
    public function __construct(Flexihash_Hasher $hasher = null, $replicas = null)
    {
        $this->_hasher = $hasher ? $hasher : new Flexihash_Crc32Hasher();
        if (!empty($replicas)) $this->_replicas = $replicas;
    }

    /**
     * Add a target.
     * @param string $target
     * @chainable
     * @comment 新增節點,根據虛擬節點數,將節點分佈到多個虛擬位置上
     */
    public function addTarget($target)
    {
        if (isset($this->_targetToPositions[$target]))
        {
            throw new Flexihash_Exception("Target '$target' already exists.");
        }

        $this->_targetToPositions[$target] = array();

        // hash the target into multiple positions
        for ($i = 0; $i < $this->_replicas; $i++)
        {
            $position = $this->_hasher->hash($target . $i);
            $this->_positionToTarget[$position] = $target; // lookup
            $this->_targetToPositions[$target] []= $position; // target removal
        }

        $this->_positionToTargetSorted = false;
        $this->_targetCount++;

        return $this;
    }

    /**
     * Add a list of targets.
     * @param array $targets
     * @chainable
     */
    public function addTargets($targets)
    {
        foreach ($targets as $target)
        {
            $this->addTarget($target);
        }

        return $this;
    }

    /**
     * Remove a target.
     * @param string $target
     * @chainable
     */
    public function removeTarget($target)
    {
        if (!isset($this->_targetToPositions[$target]))
        {
            throw new Flexihash_Exception("Target '$target' does not exist.");
        }

        foreach ($this->_targetToPositions[$target] as $position)
        {
            unset($this->_positionToTarget[$position]);
        }

        unset($this->_targetToPositions[$target]);

        $this->_targetCount--;

        return $this;
    }

    /**
     * A list of all potential targets
     * @return array
     */
    public function getAllTargets()
    {
        return array_keys($this->_targetToPositions);
    }

    /**
     * Looks up the target for the given resource.
     * @param string $resource
     * @return string
     */
    public function lookup($resource)
    {
        $targets = $this->lookupList($resource, 1);
        if (empty($targets)) throw new Flexihash_Exception('No targets exist');
        return $targets[0];
    }

    /**
     * Get a list of targets for the resource, in order of precedence.
     * Up to $requestedCount targets are returned, less if there are fewer in total.
     *
     * @param string $resource
     * @param int $requestedCount The length of the list to return
     * @return array List of targets
     * @comment 查詢當前的資源對應的節點,
     *          節點為空則返回空,節點只有一個則返回該節點,
     *          對當前資源進行hash,對所有的位置進行排序,在有序的位置列上尋找當前資源的位置
     *          當全部沒有找到的時候,將資源的位置確定為有序位置的第一個(形成一個環)
     *          返回所找到的節點
     */
    public function lookupList($resource, $requestedCount)
    {
        if (!$requestedCount)
            throw new Flexihash_Exception('Invalid count requested');

        // handle no targets
        if (empty($this->_positionToTarget))
            return array();

        // optimize single target
        if ($this->_targetCount == 1)
            return array_unique(array_values($this->_positionToTarget));

        // hash resource to a position
        $resourcePosition = $this->_hasher->hash($resource);

        $results = array();
        $collect = false;

        $this->_sortPositionTargets();

        // search values above the resourcePosition
        foreach ($this->_positionToTarget as $key => $value)
        {
            // start collecting targets after passing resource position
            if (!$collect && $key > $resourcePosition)
            {
                $collect = true;
            }

            // only collect the first instance of any target
            if ($collect && !in_array($value, $results))
            {
                $results []= $value;
            }

            // return when enough results, or list exhausted
            if (count($results) == $requestedCount || count($results) == $this->_targetCount)
            {
                return $results;
            }
        }

        // loop to start - search values below the resourcePosition
        foreach ($this->_positionToTarget as $key => $value)
        {
            if (!in_array($value, $results))
            {
                $results []= $value;
            }

            // return when enough results, or list exhausted
            if (count($results) == $requestedCount || count($results) == $this->_targetCount)
            {
                return $results;
            }
        }

        // return results after iterating through both "parts"
        return $results;
    }

    public function __toString()
    {
        return sprintf(
            '%s{targets:[%s]}',
            get_class($this),
            implode(',', $this->getAllTargets())
        );
    }

    // ----------------------------------------
    // private methods

    /**
     * Sorts the internal mapping (positions to targets) by position
     */
    private function _sortPositionTargets()
    {
        // sort by key (position) if not already
        if (!$this->_positionToTargetSorted)
        {
            ksort($this->_positionToTarget, SORT_REGULAR);
            $this->_positionToTargetSorted = true;
        }
    }

}


/**
 * Hashes given values into a sortable fixed size address space.
 *
 * @author Paul Annesley
 * @package Flexihash
 * @licence http://www.opensource.org/licenses/mit-license.php
 */
interface Flexihash_Hasher
{

    /**
     * Hashes the given string into a 32bit address space.
     *
     * Note that the output may be more than 32bits of raw data, for example
     * hexidecimal characters representing a 32bit value.
     *
     * The data must have 0xFFFFFFFF possible values, and be sortable by
     * PHP sort functions using SORT_REGULAR.
     *
     * @param string
     * @return mixed A sortable format with 0xFFFFFFFF possible values
     */
    public function hash($string);

}


/**
 * Uses CRC32 to hash a value into a signed 32bit int address space.
 * Under 32bit PHP this (safely) overflows into negatives ints.
 *
 * @author Paul Annesley
 * @package Flexihash
 * @licence http://www.opensource.org/licenses/mit-license.php
 */
class Flexihash_Crc32Hasher
    implements Flexihash_Hasher
{

    /* (non-phpdoc)
     * @see Flexihash_Hasher::hash()
     */
    public function hash($string)
    {
        return crc32($string);
    }

}


/**
 * Uses CRC32 to hash a value into a 32bit binary string data address space.
 *
 * @author Paul Annesley
 * @package Flexihash
 * @licence http://www.opensource.org/licenses/mit-license.php
 */
class Flexihash_Md5Hasher
    implements Flexihash_Hasher
{

    /* (non-phpdoc)
     * @see Flexihash_Hasher::hash()
     */
    public function hash($string)
    {
        return substr(md5($string), 0, 8); // 8 hexits = 32bit

        // 4 bytes of binary md5 data could also be used, but
        // performance seems to be the same.
    }

}


/**
 * An exception thrown by Flexihash.
 *
 * @author Paul Annesley
 * @package Flexihash
 * @licence http://www.opensource.org/licenses/mit-license.php
 */
class Flexihash_Exception extends Exception
{
}


flexihash_redis.php
<?php
require 'flexihash.php';
Class FRedis
{
    public $hash = null;
    public $memcache = null;
    public $connectPool = null;
    public $timeOut = 3;
    public $auth = "";
 
    public function __construct()
    {
        $this->hash = new Flexihash();
    }
 
    public function addServers( $servers )
    {
        foreach ($servers as $server)
        {
            $node = $server['host'] . ':' . $server['port'];
            $this->connectPool[$node] = false;
            $targets[] = $node;
        }
        $this->hash->addTargets( $targets );
        
    }
 
    public function hset($hashkey, $valuekey, $value )
    {
        $node = $this->hash->lookup( $hashkey, count( $this->connectPool ) );
        if (!$this->connectPool[$node])
        {
            $server = explode( ':', $node );
            $redis = new Redis();
            $redis->connect($server[0], $server[1], $this->timeOut);
            if(!empty($this->auth)){
                $redis->auth($this->auth);
            }
            $this->connectPool[$node] = $redis;

        }
        if ($this->connectPool[$node])
        {
            $redis = $this->connectPool[$node];
            if($redis->hset($hashkey,$valuekey,$value)){
                return true;
            }
        }
        return false;
    }
 
    public function hget( $hashkey, $valuekey )
    {
        $node = $this->hash->lookup( $hashkey, count( $this->connectPool ) );
        
        if (!$this->connectPool[$node])
        {
            $server = explode( ':', $node );
            $redis = new Redis();
            $redis->connect($server[0], $server[1], $this->timeOut);
            if(!empty($this->auth)){
                $redis->auth($this->auth);
            }
            $this->connectPool[$node] = $redis;
        
        }
        if ($this->connectPool[$node])
        {
            $redis = $this->connectPool[$node];
            if($value = $redis->hget($hashkey,$valuekey)){
                return $value;
            }
        }
        return false;
    }
}

FRedis_test.php
<?php
require_once 'flexihash_redis.php';
$st = microtime( true );
$fredis = new FRedis();
$tt_server_pool = array(
    '0' => array(
        'host' => '192.168.75.128',
        'port' => '6379',
    ),
    '1' => array(
        'host' => '192.168.75.130',
        'port' => '6379',
    ),
);
$fredis->addServers( $tt_server_pool );

for($i=0;$i<10;$i++){
    for($k=0;$k<10;$k++){
        $fredis->hset("hashkey:".$i,"valuekey".$k,md5($i*$k));
    }
}

$et = microtime( true ) - $st;
echo "time used:" . $et."<br/>";
echo $fredis->hget("hashkey:0","valuekey9");

redis版雜湊函式類:

<?php
class UniqueHashService{
    private static $len; //64位
    private static $loop = 1; //迴圈次數
    private $prefix = '';
    
    public function __construct() {
        if(empty(self::$len)){
            self::$len = abs(1 << 0x3f); //左移63位
        }
        
    }
    
    /**
     * 獲取字串的雜湊值
     */
    public function hashing($key){
        return abs((('0x'.hash('crc32',$key).hash('crc32b',$key)) + 0)) % self::$len;
    }
    
    /**
     * 獲取到redis hash 的鍵名
     * @param type $str
     */
    public function getRedisKey($str, $prefix){
        //字首
        $this->prefix = $prefix;
        
        $hstr = $this->hashing($str);    //獲取一個字串的雜湊值
        
        //如果字串長度小於6,往字串左側填充0,填充至6位長度。
        if(strlen($hstr) < 6){
            $hstr = str_pad($hstr, 6, '0', STR_PAD_LEFT);
        }
        $kbin = $this->str2Bin($hstr);  //計算hash key的二進位制,$hstr的二進位制
        
        $karr = explode(' ', $kbin);
        
        //陣列長度
        $kcount = count($karr);
        
        $kcount_mid = ceil($kcount/2);
        
        $redis_key = $this->prefix; //排重表字首
        
        //迴圈次數
        $i = 0;
        while(1){
            if($i > self::$loop){
                break;
            }
            
            //組裝redis鍵
            $redis_key .= ':'.$this->returnHashKey($karr[$i]);
            $redis_key .= ':'.$this->returnHashKey($karr[$kcount_mid+$i]);
            $redis_key .= ':'.$this->returnHashKey($karr[$kcount-1-$i]);
            
            ++$i;
        }
        
        return $redis_key;
    }
    
    /**
     * 擷取二進位制字串中的一部分
     * @param type $sbin
     * @return type
     */
    private function returnHashKey($sbin){
        return substr($sbin, -2, 2);
    }
    
    /**
     * 把給定的字串中的每個字元轉為二進位制,並以空格連線每個二進位制字串並返回。
     * @param string $hstr
     * @return type
     */
    private function str2Bin($str){
        //把字串拆分成陣列,每個字元為陣列中的一個元素。
        $arr = str_split($str);
        
        //把陣列中的每個字元轉成二進位制
        foreach($arr as &$v){
            $tmp = unpack('H*', $v);        //把字元解包為16進位制
            $v = base_convert($tmp[1],16,2);//把16進位制轉為2進位制
            unset($tmp);
        }
        
        //把二進位制字串陣列以空格連線在一起並返回
        $bin = join(' ',$arr);
        unset($arr);
        return $bin;
    }
    
    /**
     * 把$this->str2Bin 處理過的字串還原
     * @param string $bin 二進位制字串
     * @return type
     */
    private function bin2Str($bin){
        //把字串拆分成陣列,以空拆分。
        $arr = explode(' ', $bin);
        
        foreach($arr as &$v){
            $tmp = base_convert($v, 2, 16);     //二進位制轉換成16進位制
            $v = pack('H'.strlen($tmp), $tmp);  //把16進位制打包成二進位制字串
            unset($tmp);
        }
        
        return join('', $arr);
    }
}