nutch2.3.1 nutch-site.xml配置

<configuration>
<property>
    <name>storage.data.store.class</name>
    <value>org.apache.gora.mongodb.store.MongoStore</value>
</property>
<property>
    <name>http.agent.name</name>
    <value>User-AgentMozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko</value>
</property>
<property>
  <name>plugin.includes</name>
  <value>protocol-httpclient|urlfilter-regex|index-(basic|more)|query-(basic|site|url|lang)|indexer-solr|nutch-extensionpoints|protocol-httpclient|urlfilter-regex|parse-(text|html|msexcel|msword|mspowerpoint|pdf)|summary-basic|scoring-opic|urlnormalizer-(pass|regex|basic)protocol-http|urlfilter-regex|parse-(html|tika|metatags)|index-(basic|anchor|more|metadata)</value>
</property>
<property>
  <name>plugin.folders</name>
  <value>plugins</value>
</property>
<property>
  <name>http.accept.language</name>
  <value>zh-CN,zh,ja-jp,en-us,en-gb,en;q=0.7,*;q=0.3</value>
</property>
<property>
  <name>parser.character.encoding.default</name>
  <value>utf-8</value>
</property>
<property>
  <name>http.content.limit</name>
  <value>-1</value>
</property>
<property>
    <name>file.content.limit</name>
    <value>-1</value>
</property>
<property>
  <name>parser.skip.truncated</name>
  <value>false</value>
</property>
<property>
    <name>anchorIndexingFilter.deduplicate</name>
    <value>true</value>
</property>
<property>
  <name>db.max.outlinks.per.page</name>
  <value>-1</value>
</property>
<property>
  <name>fetcher.server.delay</name>
  <value>0.9</value>
</property>
<property> 
  <name>fetcher.server.min.delay</name>
  <value>0.8</value>
</property>
<property>
  <name>fetcher.threads.fetch</name>
  <value>25</value>
</property
<property>
  <name>fetcher.threads.per.queue</name>
  <value>3</value>
</property>
<property>
  <name>generate.max.distance</name>
  <value>4</value>
</property>
<property>
  <name>generate.max.count</name>
  <value>2000</value>
  <description>url count per host at one fetch</description>
</property>
</configuration>