1: 在Windows下调用nutch的脚本,可实现自动运行,这样做可以不用crywin来模拟linux,下面式win xp调用nutch的脚本
nutch.bat
@cmd /V:on /c %~dp0nutch1.bat %*
nutch1.bat
@echo on
rem *********************************************************************
rem * A script to launch nutch on Windows 2000/XP System.
rem *
rem * Written by babatu
rem * babatu@gmail.com blog: blog.babatu.com
rem *
rem * Because delayed environment is used, cmd /V:on should be used to
rem * run this script.
rem *****************************************************************
if "%OS%"=="Windows_NT" @setlocal
if "%OS%"=="WINNT" @setlocal
if "%1" == "" goto :msg
goto :begin
:msg
echo "Usage: nutch COMMAND"
echo "where COMMAND is one of:"
echo " crawl one-step crawler for intranets"
echo " readdb read / dump crawl db"
echo " readlinkdb read / dump link db"
echo " inject inject new urls into the database"
echo " generate generate new segments to fetch"
echo " fetch fetch a segment's pages"
echo " parse parse a segment's pages"
echo " segread read / dump segment data"
echo " updatedb update crawl db from segments after fetching"
echo " invertlinks create a linkdb from parsed segments"
echo " index run the indexer on parsed segments and linkdb"
echo " merge merge several segment indexes"
echo " dedup remove duplicates from a set of segment indexes"
echo " plugin load a plugin and run one of its classes main()"
echo " server run a search server"
echo " or"
echo " CLASSNAME run the class named CLASSNAME"
echo "Most commands print help when invoked w/o parameters."
pause
goto :end
:begin
rem %~dp0 这个脚本的扩展path ( expanded pathname of the current script under NT)
set DEFAULT_NUTCH_HOME=%~dp0..
rem set DEFAULT_NUTCH_HOME=..
if "%NUTCH_HOME%"=="" set NUTCH_HOME=%DEFAULT_NUTCH_HOME%
set DEFAULT_NUTCH_HOME=""
rem 设置默认DEFAULT_NUTCH_HOME
echo %NUTCH_HOME%
rem set _USE_CLASSPATH=yes
if "%CLASSPATH%"=="" ( set CLASSPATH=%JAVA_HOME%\lib\tools.jar) ELSE set
CLASSPATH=%CLASSPATH%;%JAVA_HOME%\lib\tools.jar
set CLASSPATH=%CLASSPATH%;%NUTCH_HOME%\conf;
echo %CLASSPATH%
echo before other
rem for developers, add plugins, job & test code to CLASSPATH
if exist %NUTCH_HOME%\build\plugins set
CLASSPATH=%CLASSPATH%;%NUTCH_HOME%\build
for /R %NUTCH_HOME%\build %%i in (nutch*.job) do set
CLASSPATH=!CLASSPATH!;%%i
if exist %NUTCH_HOME%\build\test\classes set
CLASSPATH=%CLASSPATH%;%NUTCH_HOME%\build\test\classes
rem for releases, add Nutch job to CLASSPATH
for /R %NUTCH_HOME% %%i in (nutch*.job) do set CLASSPATH=!CLASSPATH!;%%i
rem add plugins to classpath
if exist %NUTCH_HOME%\plugins set CLASSPATH=%CLASSPATH%;%NUTCH_HOME%
rem add libs to CLASSPATH
for /R %NUTCH_HOME%\lib %%f in (*.jar) do set CLASSPATH=!CLASSPATH!;%%f
echo %CLASSPATH%
rem translate command
if "%1"=="crawl" set CLASS=org.apache.nutch.crawl.Crawl
if "%1"=="inject" set CLASS=org.apache.nutch.crawl.Injector
if "%1"=="generate" set CLASS=org.apache.nutch.crawl.Generator
if "%1"=="fetch" set CLASS=org.apache.nutch.fetcher.Fetcher
if "%1"=="parse" set CLASS=org.apache.nutch.parse.ParseSegment
if "%1"=="readdb" set CLASS=org.apache.nutch.crawl.CrawlDbReader
if "%1"=="readlinkdb" set CLASS=org.apache.nutch.crawl.LinkDbReader
if "%1"=="segread" set CLASS=org.apache.nutch.segment.SegmentReader
if "%1"=="updatedb" set CLASS=org.apache.nutch.crawl.CrawlDb
if "%1"=="invertlinks" set CLASS=org.apache.nutch.crawl.LinkDb
if "%1"=="index" set CLASS=org.apache.nutch.indexer.Indexer
if "%1"=="dedup" set CLASS=org.apache.nutch.indexer .DeleteDuplicates
if "%1"=="merge" set CLASS=org.apache.nutch.indexer.IndexMerger
if "%1"=="plugin" set CLASS=org.apache.nutch.plugin.PluginRepository
if "%1"=="server" set CLASS='
org.apache.nutch.searcher.DistributedSearch$Server'
if "%CLASS%"=="" set CLASS=%1
%JAVA_HOME%\bin\java -cp %CLASSPATH% %CLASS% %*
if "%OS%"=="Windows_NT" @endlocal
if "%OS%"=="WINNT" @endlocal
:end
2:
写一个维护脚本,定时运行
#!/bin/bash
# Set JAVA_HOME to reflect your systems java configuration
export JAVA_HOME=/usr/lib/j2sdk1.5-sun
# Start index updation,只查找最热门的前1000条记录,由此创建新的segment
bin/nutch generate crawl.mydomain/db crawl.mydomain/segments -topN 1000
#得到最新的segment目录名
s=`ls -d crawl.virtusa/segments/2* | tail -1`
echo Segment is $s
bin/nutch fetch $s
bin/nutch updatedb crawl.mydomain /db $s
bin/nutch analyze crawl.mydomain /db 5
bin/nutch index $s
#删除重复记录
bin/nutch dedup crawl.mydomain /segments crawl.mydomain/tmpfile
# Merge segments to prevent too many open files exception in Lucene
#合并成一个新的segment
bin/nutch mergesegs -dir crawl.mydomain/segments -i -ds
s=`ls -d crawl.mydomain/segments/2* | tail -1`
echo Merged Segment is $s
rm -rf crawl.mydomain/index
以上是在urls文件内容没有变化的时候采用的办法,如果我加入的新的URL在urls文件里,那么在运行generate以前,要执行下面一命令:
#bin/nutch inject crawl.mydomain/db -urlfile urls
在generate的时候,如果不加topN参数,那么crawl只会去处理新加的或原来由于其它原因没有fetch的url或page,所以我感觉,脚本1和用2修改的脚本交替运行,会有很好的效果。
摘录自http://www.lvban.com/blog/computer/index.html
3:使用shell scripts实现nutch的自动运行需要shell。这里还有一种使用快速
python的nutch脚本
import os, sys, glob # The Nutch command script # # Environment Variables # # NUTCH_JAVA_HOME The java implementation to use. Overrides JAVA_HOME. # # NUTCH_HEAPSIZE The maximum amount of heap to use, in MB. # Default is 1000. # # NUTCH_OPTS Extra Java runtime options. # # ported to python by Ben Ogle (ogle dot ben [at] gmail) #does not handle links. thisdir = os.getcwd() cpsep = ":" if( os.name == "nt" ): cpsep = ";" if( len(sys.argv) == 1 ): print "Usage: python nutch.py COMMAND" print "where COMMAND is one of:" print " crawl one-step crawler for intranets" print " readdb read / dump crawl db" print " mergedb merge crawldb-s, with optional filtering" print " readlinkdb read / dump link db" print " inject inject new urls into the database" print " generate generate new segments to fetch" print " fetch fetch a segment's pages" print " parse parse a segment's pages" print " segread read / dump segment data" print " mergesegs merge several segments, with optional filtering and slicing" print " updatedb update crawl db from segments after fetching" print " invertlinks create a linkdb from parsed segments" print " mergelinkdb merge linkdb-s, with optional filtering" print " index run the indexer on parsed segments and linkdb" print " merge merge several segment indexes" print " dedup remove duplicates from a set of segment indexes" print " plugin load a plugin and run one of its classes main()" print " server run a search server" print " or" print " CLASSNAME run the class named CLASSNAME" print "Most commands print help when invoked w/o parameters." sys.exit(1) command = sys.argv[1] #print "COMMAND: " + command nutch_home = thisdir + "/.." java_home = os.getenv("NUTCH_JAVA_HOME") if(java_home != None): os.setenv("JAVA_HOME", java_home) print java_home java_home = os.getenv("JAVA_HOME") if(java_home == None): print "Error: JAVA_HOME is not set." exit(1) java = java_home + "/bin/java.exe" java_heap_max = "-Xmx1000m" nutch_heap_sz = os.getenv("NUTCH_HEAPSIZE") if(nutch_heap_sz != None): java_heap_max = "-Xmx"+ nutch_heap_sz +"m" #print java_heap_max classpath = nutch_home + "/conf" classpath = classpath + cpsep + nutch_home + "/lib/tools.jar" # for developers, add plugins, job & test code to CLASSPATH if( os.path.exists( nutch_home + "/build/plugins" ) ): classpath = classpath + cpsep + nutch_home + "/build/plugins" flist = glob.glob(nutch_home + "/build/nutch-*.job") for l in flist: classpath = classpath + cpsep + l if( os.path.exists( nutch_home + "/build/test/classes" ) ): classpath = classpath + cpsep + nutch_home + "/build/test/classes" flist = glob.glob(nutch_home + "/nutch-*.job") for l in flist: classpath = classpath + cpsep + l if( os.path.exists( nutch_home + "/plugins" ) ): classpath = classpath + cpsep + nutch_home + "/plugins" flist = glob.glob(nutch_home + "/lib/*.jar") for l in flist: classpath = classpath + cpsep + l flist = glob.glob(nutch_home + "/lib/jetty-ext/*.jar") for l in flist: classpath = classpath + cpsep + l #print classpath nutch_log_dir = os.getenv("NUTCH_LOG_DIR") if(nutch_log_dir == None): nutch_log_dir = nutch_home + "/logs" nutch_log_file = os.getenv("NUTCH_LOGFILE") if(nutch_log_file == None): nutch_log_file = "hadoop.log" nutch_opts = os.getenv("NUTCH_OPTS") if( nutch_opts == None ): nutch_opts = "" nutch_opts = nutch_opts + " -Dhadoop.log.dir=" + nutch_log_dir nutch_opts = nutch_opts + " -Dhadoop.log.file=" + nutch_log_file # figure out which class to run theclass = command if ( command == "crawl" ): theclass="org.apache.nutch.crawl.Crawl" elif ( command == "inject" ): theclass="org.apache.nutch.crawl.Injector" elif ( command == "generate" ): theclass="org.apache.nutch.crawl.Generator" elif ( command == "fetch" ): theclass="org.apache.nutch.fetcher.Fetcher" elif ( command == "parse" ): theclass="org.apache.nutch.parse.ParseSegment" elif ( command == "readdb" ): theclass="org.apache.nutch.crawl.CrawlDbReader" elif ( command == "mergedb" ): theclass="org.apache.nutch.crawl.CrawlDbMerger" elif ( command == "readlinkdb" ): theclass="org.apache.nutch.crawl.LinkDbReader" elif ( command == "segread" ): theclass="org.apache.nutch.segment.SegmentReader" elif ( command == "mergesegs" ): theclass="org.apache.nutch.segment.SegmentMerger" elif ( command == "updatedb" ): theclass="org.apache.nutch.crawl.CrawlDb" elif ( command == "invertlinks" ): theclass="org.apache.nutch.crawl.LinkDb" elif ( command == "mergelinkdb" ): theclass="org.apache.nutch.crawl.LinkDbMerger" elif ( command == "index" ): theclass="org.apache.nutch.indexer.Indexer" elif ( command == "dedup" ): theclass="org.apache.nutch.indexer.DeleteDuplicates" elif ( command == "merge" ): theclass="org.apache.nutch.indexer.IndexMerger" elif ( command == "plugin" ): theclass="org.apache.nutch.plugin.PluginRepository" elif ( command == "server" ): #what goes in place of the $Server? theclass="org.apache.nutch.searcher.DistributedSearch$Server" args = "" for i in range(2, len(sys.argv)): args = args + " " + sys.argv[i] #windows doesnt like this even though there are quotes around it... #"\"" + java +"\" " cmdtorun = "java " + java_heap_max + " " + nutch_opts + " -classpath \"" + classpath + "\" " + theclass + args #print cmdtorun os.system(cmdtorun)