第一步，模擬生成ip地址

def generateRandom(rangeFrom, rangeTo):
    import random
    return random.randint(rangeFrom,rangeTo)

def generageMassiveIPAddr(fileLocation,numberOfLines):
    IP = []
    file_handler = open(fileLocation, 'a+')
    for i in range(numberOfLines):
        IP.append('10.197.' + str(generateRandom(0,255))+'.'+ str(generateRandom(0,255)) + '\n')

    file_handler.writelines(IP)
    file_handler.close()

if __name__ == '__main__':
    from time import ctime
    print ctime()
    for i in range(10):
        print '  ' + str(i) + ": " + ctime()
        generageMassiveIPAddr('/opt/work/iplogProcess/logs', 10000)
    print ctime()
 

import os

def mkFile(file_path):
    if(os.path.exists(file_path)):
        return 1
    else:
        os.mknod(file_path)
        return 1
    return 0

def splitFile(inputFile, splitNumbers):
    print "begin to split files........"
    file_handler = open(inputFile, 'r+')
    for IP in file_handler:
        IPtest = IP.strip("\n")

        hashvalue = hash(IPtest) %splitNumbers
        file_path = "/opt/work/iplogProcess" + "/logs_" + str(hashvalue)
        if mkFile(file_path):

            filePoint = open(file_path, "a+")

            filePoint.writelines(IP)
            filePoint.close()
    file_handler.close()
 

from time import ctime
import os
import re
def ipProcess(dirPath):
    i = 0


    maxvalue = 0
    Result = {}
    resultdir = {}

    for root, dirs,files in os.walk(dirPath):
        print "get file"
    print "begin to statcstic............."

    #using re to match files 
    for item in next(os.walk('/opt/work/iplogProcess'))[2]:
        match = re.search(r'logs_\d+',item)
        if match is None:
                continue
        #path = dirPath + "/" + files[i]  
        #maxvalue = 0

        dire = {}
        # open a new file to statistic
        path = dirPath + "/" + item
        filePoint = open(path, 'r')
        #print file names
        #print filePoint  
        for cloums in filePoint:
            #print cloums
            if cloums in dire:
                dire[cloums] = dire[cloums] + 1

            else:
                dire[cloums] = 1
            #print cloums,dire[cloums]  

        #sort the dictionary dire
        dire = sorted(dire.items(), key=lambda d: d[1], reverse=True)

        #for every file, find top 10
        i = 0
        for k in dire:
            if i < 10:
                Result[dire[i][0]] = dire[i][1]
                #print dire[i][0]
            else:
                break
            i = i+1
        print '--------------------------to read next file----------------------------------------'
    Result = sorted(Result.items(), key=lambda d: d[1], reverse=True)
    i = 0
    for keys in Result:
        if i < 10:
            print keys
        else:
            break
        i = i+1

if __name__ == '__main__':
    ipProcess("/opt/work/iplogProcess/")

 

import os
from time import ctime

def findIPAtOnce(targetFile):
    print "Started At: " + ctime()
    Result = {}
    file_handler = open(targetFile, 'r')

    for line in file_handler:
        if line in Result:
            Result[line] = Result[line] + 1
        else:
            Result[line] = 1
    print "Write to Dic Finished At: " + ctime()

    file_handler.close()

    Result = sorted(Result.items(), key=lambda d: d[1])
    print "Sorting Finished At: " + ctime()

    print 'Result:'
    for i in range(10):
        print '  ' + str(Result.pop())

if __name__ == '__main__':
    findIPAtOnce("/opt/work/iplogProcess/logs")