1. 程式人生 > >flume 1.7.0-taildirSource 支援 windows系統

flume 1.7.0-taildirSource 支援 windows系統

Flume-ng 1.7.0 中增加了TaildirSource,可以監控目錄中檔案的變化自動讀取檔案內容。
不過實際應用時發現幾個問題:

1,不支援windows系統。
2,windows下會影響 log4j 日誌檔案的切分,會使log4j日誌不切分一直增大,flume停了才會 切分日誌。

不支援 windows 系統的問題是因為 taildirSource 的原始碼ReliableTaildirEventReader類的getInode(File file) 方法中的程式碼依賴linux系統。

 private long getInode(File file) throws IOException {
    long
inode = (long) Files.getAttribute(file.toPath(), "unix:ino"); return inode; }

window系統下的JDK不支援:Files.getAttribute(file.toPath(), “unix:ino”)。不支援咋辦?當然是想辦法實現一個啊。Linux系統下可以使用inode跟蹤一個檔案,檔案被重新命名inode不會改變。Windows的NTFS系統有類似的東東,具體實現方式如下:

1.新增依賴

          <dependency>
            <groupId
>
net.java.dev.jna</groupId> <artifactId>jna</artifactId> <version>4.2.2</version> </dependency> <dependency> <groupId>net.java.dev.jna</groupId> <artifactId>jna-platform</artifactId
>
<version>4.2.2</version> </dependency>

2.新建Kernel32 類


import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.sun.jna.Library;
import com.sun.jna.Native;
import com.sun.jna.Structure;
import com.sun.jna.platform.win32.WinBase.FILETIME;
import com.sun.jna.platform.win32.WinDef.DWORD;
import com.sun.jna.platform.win32.WinNT.HANDLE;
import com.sun.jna.win32.StdCallLibrary;
import com.sun.jna.win32.W32APIFunctionMapper;
import com.sun.jna.win32.W32APITypeMapper;

/**
 * Created by yangyibo on 17/3/14.
 */
public interface Kernel32 extends StdCallLibrary {
    final static Map WIN32API_OPTIONS = new HashMap() {
        private static final long serialVersionUID = 1L;

        {
            put(Library.OPTION_FUNCTION_MAPPER, W32APIFunctionMapper.UNICODE);
            put(Library.OPTION_TYPE_MAPPER, W32APITypeMapper.UNICODE);
        }
    };

    Kernel32 INSTANCE = (Kernel32) Native.loadLibrary("Kernel32",
            Kernel32.class, WIN32API_OPTIONS);

    int GetLastError();

    class BY_HANDLE_FILE_INFORMATION extends Structure {
        public DWORD dwFileAttributes;
        public FILETIME ftCreationTime;
        public FILETIME ftLastAccessTime;
        public FILETIME ftLastWriteTime;
        public DWORD dwVolumeSerialNumber;
        public DWORD nFileSizeHigh;
        public DWORD nFileSizeLow;
        public DWORD nNumberOfLinks;
        public DWORD nFileIndexHigh;
        public DWORD nFileIndexLow;

        public static class ByReference extends BY_HANDLE_FILE_INFORMATION
                implements Structure.ByReference {

        }

        ;

        public static class ByValue extends BY_HANDLE_FILE_INFORMATION
                implements Structure.ByValue {

        }

        @Override
        protected List getFieldOrder() {
            List fields = new ArrayList();
            fields.addAll(Arrays.asList(new String[]{"dwFileAttributes",
                    "ftCreationTime", "ftLastAccessTime", "ftLastWriteTime",
                    "dwVolumeSerialNumber", "nFileSizeHigh", "nFileSizeLow",
                    "nNumberOfLinks", "nFileIndexHigh", "nFileIndexLow"}));
            return fields;

        }

        ;
    }

    ;

    boolean GetFileInformationByHandle(HANDLE hFile,
                                       BY_HANDLE_FILE_INFORMATION lpFileInformation);
}

3.新建WinFileUtil 類

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.sun.jna.platform.win32.Kernel32;
import com.sun.jna.platform.win32.WinBase;
import com.sun.jna.platform.win32.WinNT.HANDLE;

import java.io.File;
import java.nio.file.Files;

/**
 * Created by yangyibo on 17/3/14.
 */
public class WinFileUtil {

    public  static WinFileUtil getWinFile(){
        return  new WinFileUtil();
    }
    private static Logger logger = LoggerFactory.getLogger(WinFileUtil.class);

    public static String getFileId(String filepath) {

        final int FILE_SHARE_READ = (0x00000001);
        final int OPEN_EXISTING = (3);
        final int GENERIC_READ = (0x80000000);
        final int FILE_ATTRIBUTE_ARCHIVE = (0x20);

        WinBase.SECURITY_ATTRIBUTES attr = null;
        com.wh.example.Kernel32.BY_HANDLE_FILE_INFORMATION lpFileInformation = new com.wh.example.Kernel32.BY_HANDLE_FILE_INFORMATION();
        HANDLE hFile = null;

        hFile = Kernel32.INSTANCE.CreateFile(filepath, 0,
                FILE_SHARE_READ, attr, OPEN_EXISTING, FILE_ATTRIBUTE_ARCHIVE,
                null);
        String ret = "0";
        if (Kernel32.INSTANCE.GetLastError() == 0) {

            com.wh.example.Kernel32.INSTANCE
                    .GetFileInformationByHandle(hFile, lpFileInformation);

            ret = lpFileInformation.dwVolumeSerialNumber.toString()
                    + lpFileInformation.nFileIndexLow.toString();

            Kernel32.INSTANCE.CloseHandle(hFile);

            if (Kernel32.INSTANCE.GetLastError() == 0) {
                logger.debug("inode:" + ret);
                return ret;
            } else {
                logger.error("關閉檔案發生錯誤:{}", filepath);
                throw new RuntimeException("關閉檔案發生錯誤:" + filepath);
            }
        } else {
            if (hFile != null) {
                Kernel32.INSTANCE.CloseHandle(hFile);
            }
            logger.error("開啟檔案發生錯誤:{}", filepath);
            throw new RuntimeException("開啟檔案發生錯誤:" + filepath);
        }

    }

    public static void main(String[] args) throws Exception {
        File f=new File("/Users/yangyibo/Idea/fileInfoTest/target/logs/fileInfo.log");
        System.out.println(f.toPath());
        System.out.println(Long.valueOf("1232188718728378"));
        System.out.println("file ino: "+Files.getAttribute(f.toPath(), "unix:ino"));
    }
}

4.修改原始碼

    public static final String OS_NAME = System.getProperty("os.name").toLowerCase();

 private long getInode(File file) throws IOException {
        long inode;
        if (OS_NAME.contains("windows")) {
            inode = Long.parseLong(WinFileUtil.getFileId(file.toPath().toString()));
        } else {
            inode = (long) Files.getAttribute(file.toPath(), "unix:ino");
        }
        return inode;
    }

這樣就解決了windows 的第一個問題,關於第二個問題沒有很好的解決方案。如果有好的解決方法,私聊我哦。