Java網路爬蟲crawler4j學習筆記 RobotstxtParser類
阿新 • • 發佈:2018-12-26
原始碼
package edu.uci.ics.crawler4j.robotstxt;
import java.util.StringTokenizer;
// 根據網站的robot.txt文字,構建allows和disallow集合
public class RobotstxtParser {
// 當使用String.matches方法呼叫時,"?i"表示忽略大小寫
private static final String PATTERNS_USERAGENT = "(?i)^User-agent:.*";
private static final String PATTERNS_DISALLOW = "(?i)Disallow:.*" ;
private static final String PATTERNS_ALLOW = "(?i)Allow:.*";
// "User-agent:"長度為11
private static final int PATTERNS_USERAGENT_LENGTH = 11;
private static final int PATTERNS_DISALLOW_LENGTH = 9;
private static final int PATTERNS_ALLOW_LENGTH = 6;
public static HostDirectives parse(String content, String myUserAgent) {
HostDirectives directives = null ;
boolean inMatchingUserAgent = false;
// 一次提取robot.txt的每一行
StringTokenizer st = new StringTokenizer(content, "\n\r");
while (st.hasMoreTokens()) {
String line = st.nextToken();
// #號之後的都是註釋
int commentIndex = line.indexOf("#");
if (commentIndex > -1) {
line = line.substring(0 , commentIndex);
}
// remove any html markup
line = line.replaceAll("<[^>]+>", ""); // "<[除了右括號的字元]+>"
line = line.trim();
if (line.length() == 0) {
continue;
}
if (line.matches(PATTERNS_USERAGENT)) { // User-agenet行的內容
String ua = line.substring(PATTERNS_USERAGENT_LENGTH).trim().toLowerCase();
// user-agent是否是針對當前爬蟲的
if (ua.equals("*") || ua.contains(myUserAgent)) {
inMatchingUserAgent = true;
} else {
inMatchingUserAgent = false;
}
} else if (line.matches(PATTERNS_DISALLOW)) { // disallow行的內容
if (!inMatchingUserAgent) {
continue;
}
String path = line.substring(PATTERNS_DISALLOW_LENGTH).trim();
if (path.endsWith("*")) {
// 獲取星號之前的path路徑
path = path.substring(0, path.length() - 1);
}
path = path.trim();
if (path.length() > 0) {
if (directives == null) {
directives = new HostDirectives();
}
// 增加disallow規則
directives.addDisallow(path);
}
} else if (line.matches(PATTERNS_ALLOW)) { // allow行的內容
if (!inMatchingUserAgent) {
continue;
}
String path = line.substring(PATTERNS_ALLOW_LENGTH).trim();
// 獲取星號之前的Path路徑
if (path.endsWith("*")) {
path = path.substring(0, path.length() - 1);
}
path = path.trim();
if (directives == null) {
directives = new HostDirectives();
}
// 增加allow規則
directives.addAllow(path);
}
}
return directives;
}
}