源代码
package edu.uci.ics.crawler4j.robotstxt;
public class HostDirectives {
private static final long EXPIRATION_DELAY = 24 * 60 * 1000 L;
private RuleSet disallows = new RuleSet();
private RuleSet allows = new RuleSet();
private long timeFetched;
private long timeLastAccessed;
public HostDirectives () {
timeFetched = System.currentTimeMillis();
}
public boolean needsRefetch () {
return (System.currentTimeMillis() - timeFetched > EXPIRATION_DELAY);
}
public boolean allows (String path) {
timeLastAccessed = System.currentTimeMillis();
return !disallows.containsPrefixOf(path) || allows.containsPrefixOf(path);
}
public void addDisallow (String path) {
disallows.add(path);
}
public void addAllow (String path) {
allows.add(path);
}
public long getLastAccessTime () {
return timeLastAccessed;
}
}