Nutch recrawl script

For Apache Nutch 1.8 or latter.

For Apache Nutch, after a time intervial, it will begin to recrawl crawled urls.

But the original "crawl" command did not have full function for recrawl.

So, we need a script to do this kind of work. More information of recrawl: http://wiki.apache.org/nutch/Recrawl

It support hadoop environment.

#! /usr/bin/env bash
# recrawl script to run the Nutch bot for crawling and re-crawling.
# Usage: bin/recrawl
# Author: Jerome(base on Joey and Susam Pal's work. please refer http://blog.csdn.net/iterjpnic/article/details/7644407 and http://wiki.apache.org/nutch/Crawl)

#if [ $# != 3 ]
#then
#  echo "Usage: recrawl DATADIR URLDIR SOLRSERVERADDRESS"
#  echo "where "
#  echo "        DATADIR is the parent directory where crawled data will be stored "
#  echo "        URLDIR is the parent directory of the injected url files "
#  echo "        SOLRSERVERADDRESS is the address of solr index server "
#  echo "eg: recrawl hdfs://localhost:9000/user/root/mydir \
#  hdfs://localhost:9000/user/root/urls http://master:8983/solr/collection1_shard1_replica1"
#  exit 1
#fi

# DATADIR is the parent directory where crawled data will be stored
DATADIR="mydir"
# URLDIR is the parent directory of the injected url files
URLDIR="urls"
#URLDIR="seed.txt"
# SOLRADDRESS is the address of solr index server
SOLRADDRESS="http://slave1:8983/solr/collection1"
# set the number of slaves nodes
numSlaves=8
# set the timeinterval which page need to re-fetch
addDays=0
# and the total number of available tasks
# set Hadoop parameter "mapred.reduce.tasks"
numTasks=`expr $numSlaves \* 5`
# number of urls to fethc in one iteration
sizeFetchlist=`expr $numSlaves \* 4000`
#sizeFetchlist=1
# time limit for feching
timeLimitFetch=180
# num threads for fetching
numThreads=50
# iterations of re-crawl
depth=10

# Temp segments dir in Hadoop DFS
TEMPSEGMENTSDIR="tempsegments"
BACKUPSEGDIR="backdir/segments"

# Arguments for hadoop dfsshell ls/cp/rm/mv command
LSCOMMAND="hadoop fs -ls"
RMCOMMAND="hadoop fs -rm -r"
CPCOMMAND="hadoop fs -cp"
MVCOMMAND="hadoop fs -mv"

if [ -z "$NUTCH_HOME" ]
then
  NUTCH_HOME="/usr/nutch/apache-nutch/runtime/deploy/bin"
  echo recrawl: $0 could not find environment variable NUTCH_HOME
  echo recrawl: NUTCH_HOME=$NUTCH_HOME has been set by the script
else
  echo recrawl: $0 found environment variable NUTCH_HOME=$NUTCH_HOME
fi

# note that some of the options listed here could be set in the
# corresponding hadoop site xml param file
commonOptions="-D mapreduce.job.reduces=$numTasks -D mapred.child.java.opts=-Xmx1000m -D mapred.reduce.tasks.speculative.execution=false -D mapred.map.tasks.speculative.execution=false -D mapred.compress.map.output=true"

for((j=0;j>-1;j++))
do
  # "----- See if it will go on to crawl -----"
  for switch in 'cat $NUTCH_HOME/recrawlswitcher'
  do
    break
  done
  if [$switch == "off"]
  then
      echo "--- Shut down the recrawl due to recrawl switcher is off ---"
      break
  fi

  echo "--- Beginning at count `expr $j + 1` ---"
  steps=6
  echo "----- Inject (Step 1 of $steps) -----"
  $NUTCH_HOME/nutch inject $DATADIR/crawldb $URLDIR
  if [ $? -ne 0 ]
  then exit $?
  fi

  echo "----- Generate, Fetch, Parese, Update (Step 2 of $steps) -----"
  for((i=0;i<$depth;i++))
  do
    echo "--- Beginning crawl at depth `expr $i + 1` of $depth ---"
    $NUTCH_HOME/nutch generate $commomOptions $DATADIR/crawldb $DATADIR/segments -topN $sizeFetchlist -numFetchers $numSlaves -adddays $addDays
    if [ $? -ne 0 ]
    then
        echo "recrawl: Stopping at depth $depth. No more URLs to fetch."
        break
    fi
    segment=`$LSCOMMAND $DATADIR/segments/ | tail -1 | awk '{print $8}'`

    echo "--- fetch into segment:"$segment" ---"
    $NUTCH_HOME/nutch fetch $commonOptions -D fetcher.timelimit.mins=$timeLimitFetch $segment -noParsing -threads $numThreads
    if [ $? -ne 0 ]
    then
        echo "recrawl: fetch $segment at depth `expr $i + 1` failed."
        echo "recrawl: Deleting segment $segment."
        $RMCOMMAND $segment
        continue
    fi

    echo "--- Beginning parsing ---"
    skipRecordsOptions="-D mapred.skip.attempts.to.start.skipping=2 -D mapred.skip.map.max.skip.records=1"
    $NUTCH_HOME/nutch parse $commomOptions $skipRecordsOptions $segment
    if [ $? -ne 0 ]
    then exit $?
    fi

    echo "--- Begin updatedb ---"
    $NUTCH_HOME/nutch updatedb $commonOptions $DATADIR/crawldb $segment #-filter
    if [ $? -ne 0 ]
    then exit $?
    fi

    echo "--- Begin copy segment dir to temp segments dir ---"
    $CPCOMMAND $segment $TEMPSEGMENTSDIR/

    echo "--- Merge Segments (Step 3 of $steps) ---"
    $NUTCH_HOME/nutch mergesegs $DATADIR/MERGEDsegments -dir $DATADIR/segments #-filter
    if [ $? -ne 0 ]
    then exit $?
    fi

    echo "--- Backup old segments dir, delete backups when this-count crawl finished ---"
    $MVCOMMAND $DATADIR/segments $DATADIR/segmentsBackup
    if [ $? -ne 0 ]
    then
      echo "recrawl: Failed to backup current segments, so exit in case of data loss"
      exit 1
    fi

    echo "--- Move the MERGEDsegments to segments dir ---"
    $MVCOMMAND $DATADIR/MERGEDsegments $DATADIR/segments
    if [ $? -ne 0 ]
    then
        echo "recrawl: Failed to move MERGEDsegments to segments, so exit in case of data loss"
        exit 1
    fi

    echo "----- Invert Links (Step 4 of $steps) -----"
    $NUTCH_HOME/nutch invertlinks $DATADIR/linkdb -dir $TEMPSEGMENTSDIR
    if [ $? -ne 0 ]
    then exit $?
    fi

    echo "----- Delete Duplicates (Step 5 of $steps)  -----"
    $NUTCH_HOME/nutch dedup $DATADIR/crawldb
    if [ $? -ne 0 ]
    then exit $?
    fi

    echo "----- Index (Step 6 of $steps) -----"
    $NUTCH_HOME/nutch index -D solr.server.url=$SOLRADDRESS $DATADIR/crawldb -linkdb $DATADIR/linkdb -dir $TEMPSEGMENTSDIR
    if [ $? -ne 0 ]
    then exit $?
    fi


    echo "--- The main recrawl process is done, now gona delete the temp segments dir ---"
    $MVCOMMAND $TEMPSEGMENTSDIR/* $BACKUPSEGDIR/

    echo "--- Delete the temp old segments backups ---"
    $RMCOMMAND $DATADIR/segmentsBackup
    $RMCOMMAND .Trash
    echo "recrawl: FINISHED: Crawl `expr $i + 1` -th completed!"

  done
  echo "----- Clean Index -----"
  $NUTCH_HOME/nutch clean -D solr.server.url=$SOLRADDRESS $DATADIR/crawldb
  if [ $? -ne 0 ]
  then exit $?
  fi
  break
done
echo "All FINISHED with `expr $j + 1` count..."