<configuration>
<property>
<name>storage.data.store.class</name>
<value>org.apache.gora.mongodb.store.MongoStore</value>
</property>
<property>
<name>http.agent.name</name>
<value>User-AgentMozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko</value>
</property>
<property>
<name>plugin.includes</name>
<value>protocol-httpclient|urlfilter-regex|index-(basic|more)|query-(basic|site|url|lang)|indexer-solr|nutch-extensionpoints|protocol-httpclient|urlfilter-regex|parse-(text|html|msexcel|msword|mspowerpoint|pdf)|summary-basic|scoring-opic|urlnormalizer-(pass|regex|basic)protocol-http|urlfilter-regex|parse-(html|tika|metatags)|index-(basic|anchor|more|metadata)</value>
</property>
<property>
<name>plugin.folders</name>
<value>plugins</value>
</property>
<property>
<name>http.accept.language</name>
<value>zh-CN,zh,ja-jp,en-us,en-gb,en;q=0.7,*;q=0.3</value>
</property>
<property>
<name>parser.character.encoding.default</name>
<value>utf-8</value>
</property>
<property>
<name>http.content.limit</name>
<value>-1</value>
</property>
<property>
<name>file.content.limit</name>
<value>-1</value>
</property>
<property>
<name>parser.skip.truncated</name>
<value>false</value>
</property>
<property>
<name>anchorIndexingFilter.deduplicate</name>
<value>true</value>
</property>
<property>
<name>db.max.outlinks.per.page</name>
<value>-1</value>
</property>
<property>
<name>fetcher.server.delay</name>
<value>0.9</value>
</property>
<property>
<name>fetcher.server.min.delay</name>
<value>0.8</value>
</property>
<property>
<name>fetcher.threads.fetch</name>
<value>25</value>
</property
<property>
<name>fetcher.threads.per.queue</name>
<value>3</value>
</property>
<property>
<name>generate.max.distance</name>
<value>4</value>
</property>
<property>
<name>generate.max.count</name>
<value>2000</value>
<description>url count per host at one fetch</description>
</property>
</configuration>
nutch2.3.1 nutch-site.xml配置
最新推荐文章于 2018-12-28 18:32:35 发布