ELK 使用者行為日誌獲取mangoDB
我的後臺小夥伴 已經把 日誌寫到了mangoDB裡瞭然後我需要先把它們從資料庫裡都給取出來寫成檔案,然後在匯入到ELK 裡這是我的思路也不知道對不對
下面是個java程式碼 是springboot 工程
我提前在D盤下建立了 這個資料夾
資料庫配置檔案改成自己的
POM 檔案 核心的地方貼出來
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>1.5.6.RELEASE</version>
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<!-- springboot-web -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- spring-boot-starter-data-mongodb -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.4</version>
</dependency>
</dependencies>
-------------------------------------------------------------------
Contorller 檔案
@RestController
@EnableAutoConfiguration
public class Controller {
//一次讀取幾條
private int COUNT = 2000;
//檔案
private File file;
//寫入流
private BufferedWriter output ;
//寫入路徑
private String path;
//臨時路徑防止同一個檔案存入過多資料
private int tempP;
//控制單獨檔案條數閾值
private int count = 2000;
private Gson gson=new Gson();
@Autowired
private MongoTemplate mongoTemplate;
@GetMapping("/userlog")
public List<ClientLog> test(HttpServletRequest request) throws Exception {
String start = null;
String zone = "logs";
Query queryI = new Query();
file = new File("D:\\export\\"+zone);
String split = readDirectiory();
start = split.length() == 0 ? "0" : split;
System.out.println("起始偏移量 = " + start);
List<ClientLog> res = null;
try {
Integer pageNo = 1;
Integer pageSize = 10;
queryI.skip((pageNo - 1) * pageSize).limit(pageSize);
res = mongoTemplate.find(queryI,ClientLog.class);
int i =0;
for (ClientLog clientLog : res) {
i++;
writeOffset(Integer.parseInt(start) + i + "",clientLog);
//一個檔案做多存 固定條數,防止單個檔案過大
if (i % count == 0 && i != 0) {
output.close();
output = null;
path = "D:\\export\\" + zone +"\\"+ (++tempP) + ".json";
}
}
} catch (Exception e) {
}
return res;
}
//取出資料夾中對應的檔案的最後一行
public String readDirectiory() throws IOException {
String[] filelist = file.list();
if (filelist.length > 0) {
int fileName = sort(filelist);
tempP = fileName + 1;
String p = file + "\\";
file = new File(file + "\\" + fileName + ".json");
path = p + (++fileName) + ".json";
return getEndLine();
}else {
System.out.println("檔案不存在");
file = new File(file + "\\1.json");
path = file + "";
file.createNewFile();// 不存在則建立
return "0";
}
}
//找出最大(最後一個儲存的)的檔案
private int sort(String[] arr) {
int temp = 0;
for (int i = 0;i < arr.length ;i++ ) {
int m = Integer.parseInt(arr[i].substring(0, arr[i].length() - 5));
temp = temp > m ? temp : m;
}
return temp;
}
//寫入第插入第幾行
private void writeOffset(String num ,ClientLog log) throws IOException {
if (output == null) {
file = new File(path);
output = new BufferedWriter(new FileWriter(file,true));//true,則追加寫入text文字
}
System.out.println("寫入第 " + num + "條");
output.write("{\"index\":{\"_id\":\""+log.getId()+"\"}}");
output.write("\r\n");//換行
output.write(gson.toJson(log));
output.write("\r\n");//換行
output.flush();
}
//獲取指定檔案最後一行
private String getEndLine() throws IOException {
System.out.println("檔案存在");
InputStreamReader read = new InputStreamReader(new FileInputStream(file));
BufferedReader bufferedReader = new BufferedReader(read);
String lineTxt = "";
String end = "";
while((lineTxt = bufferedReader.readLine()) != null){
if (lineTxt.length() > 0) {
end = lineTxt;
}
}
read.close();
return end;
}
--------------------------------------------------------
ClientLog 是個 實體類 就是那些屬性就不貼了 實現個Serializable 就好
--------------------------------------------
跑起來以後直接生成的檔案 1.json 是這個樣子的 一定要注意 不是純json,我試過純json ELK 不解析
然後在上傳到伺服器上解析
curl -H 'Content-Type: application/x-ndjson' -XPOST 'localhost:9201/logs/user/_bulk?pretty' --data-binary @1.json
然後在Kibana裡測試
GET /bank/_search
{
"query": {
"match": {
"firstname": "Amber"
}
}
}
噔噔
