实现Heritrix的完全代码

零零散散的讲了这么多，下面的代码是一个很好的总结：

1 /* Copyright (C) 2003 Internet Archive.
2   *
3   * This file is part of the Heritrix web crawler (crawler.archive.org).
4   *
5   * Heritrix is free software; you can redistribute it and/or modify
6   * it under the terms of the GNU Lesser Public License as published by
7   * the Free Software Foundation; either version 2.1 of the License, or
8   * any later version.
9   *
10  * Heritrix is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU Lesser Public License for more details.
14  *
15  * You should have received a copy of the GNU Lesser Public License
16  * along with Heritrix; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
18  *
19  * ToeThread.java
20  * Created on May 14, 2003
21  *
22  * $Header$
23  */
24  package org.archive.crawler.framework;
25 
26  import java.io.PrintWriter;
27  import java.util.HashMap;
28  import java.util.logging.Level;
29  import java.util.logging.Logger;
30 
31  import org.archive.crawler.datamodel.CoreAttributeConstants;
32  import org.archive.crawler.datamodel.CrawlOrder;
33  import org.archive.crawler.datamodel.CrawlURI;
34  import org.archive.crawler.datamodel.FetchStatusCodes;
35  import org.archive.crawler.datamodel.InstancePerThread;
36  import org.archive.crawler.framework.exceptions.EndedException;
37  import org.archive.util.ArchiveUtils;
38  import org.archive.util.DevUtils;
39  import org.archive.util.HttpRecorder;
40  import org.archive.util.HttpRecorderMarker;
41  import org.archive.util.ProgressStatisticsReporter;
42  import org.archive.util.Reporter;
43 
44  import com.sleepycat.util.RuntimeExceptionWrapper;
45 
46  /***
47  * One "worker thread"; asks for CrawlURIs, processes them,
48  * repeats unless told otherwise.
49  *
50  * @author Gordon Mohr
51  */
52  public class ToeThread extends Thread
53  implements CoreAttributeConstants, FetchStatusCodes, HttpRecorderMarker,
54  Reporter, ProgressStatisticsReporter {
55      private static final String STEP_NASCENT = "NASCENT";
56      private static final String STEP_ABOUT_TO_GET_URI = "ABOUT_TO_GET_URI";
57      private static final String STEP_FINISHED = "FINISHED";
58      private static final String STEP_ABOUT_TO_BEGIN_CHAIN =
59          "ABOUT_TO_BEGIN_CHAIN";
60      private static final String STEP_ABOUT_TO_BEGIN_PROCESSOR =
61          "ABOUT_TO_BEGIN_PROCESSOR";
62      private static final String STEP_DONE_WITH_PROCESSORS =
63          "DONE_WITH_PROCESSORS";
64      private static final String STEP_HANDLING_RUNTIME_EXCEPTION =
65          "HANDLING_RUNTIME_EXCEPTION";
66      private static final String STEP_ABOUT_TO_RETURN_URI =
67          "ABOUT_TO_RETURN_URI";
68      private static final String STEP_FINISHING_PROCESS = "FINISHING_PROCESS";
69 
70      private static Logger logger =
71          Logger.getLogger("org.archive.crawler.framework.ToeThread");
72 
73      private CrawlController controller;
74      private int serialNumber;
75     
76      /***
77       * Each ToeThead has an instance of HttpRecord that gets used
78       * over and over by each request.
79       *
80       * @see org.archive.util.HttpRecorderMarker
81       */
82      private HttpRecorder httpRecorder = null;
83     
84      private HashMap<String,Processor> localProcessors
85       = new HashMap<String,Processor>();
86      private String currentProcessorName = "";
87 
88      private String coreName;
89      private CrawlURI currentCuri;
90      private long lastStartTime;
91      private long lastFinishTime;
92 
93      // activity monitoring, debugging, and problem detection
94      private String step = STEP_NASCENT;
95      private long atStepSince;
96     
97      // default priority; may not be meaningful in recent JVMs
98      private static final int DEFAULT_PRIORITY = Thread.NORM_PRIORITY-2;
99     
100     // indicator that a thread is now surplus based on current desired
101     // count; it should wrap up cleanly
102     private volatile boolean shouldRetire = false;
103    
104     /***
105      * Create a ToeThread
106      *
107      * @param g ToeThreadGroup
108      * @param sn serial number
109      */
110     public ToeThread(ToePool g, int sn) {
111         // TODO: add crawl name?
112         super(g,"ToeThread #" + sn);
113         coreName="ToeThread #" + sn + ": ";
114         controller = g.getController();
115         serialNumber = sn;
116         setPriority(DEFAULT_PRIORITY);
117         int outBufferSize = ((Integer) controller
118                 .getOrder()
119                 .getUncheckedAttribute(null,CrawlOrder.ATTR_RECORDER_OUT_BUFFER))
120                         .intValue();
121         int inBufferSize = ((Integer) controller
122                 .getOrder()
123                 .getUncheckedAttribute(null, CrawlOrder.ATTR_RECORDER_IN_BUFFER))
124                 .intValue(); 
125         httpRecorder = new HttpRecorder(controller.getScratchDisk(),
126             "tt" + sn + "http", outBufferSize, inBufferSize);
127         lastFinishTime = System.currentTimeMillis();
128     }
129
130     /*** (non-Javadoc)
131      * @see java.lang.Thread#run()
132      */
133     public void run() {
134         String name = controller.getOrder().getCrawlOrderName();
135         logger.fine(getName()+" started for order '"+name+"'");
136
137         try {
138             while ( true ) {
139                 // TODO check for thread-abort? or is waiting for interrupt enough?
140                 continueCheck();
141                
142                 setStep(STEP_ABOUT_TO_GET_URI);
143                
144                 CrawlURI curi = controller.getFrontier().next();
145                
146                 synchronized(this) {
147                     continueCheck();
148                     setCurrentCuri(curi);
149                 }
150                
151                 processCrawlUri();
152                
153                 setStep(STEP_ABOUT_TO_RETURN_URI);
154                 continueCheck();
155
156                 synchronized(this) {
157                     controller.getFrontier().finished(currentCuri);
158                     setCurrentCuri(null);
159                 }
160                
161                 setStep(STEP_FINISHING_PROCESS);
162                 lastFinishTime = System.currentTimeMillis();
163                 controller.releaseContinuePermission();
164                 if(shouldRetire) {
165                     break; // from while(true)
166                 }
167             }
168         } catch (EndedException e) {
169             // crawl ended (or thread was retired), so allow thread to end
170         } catch (Exception e) {
171             // everything else (including interruption)
172             logger.log(Level.SEVERE,"Fatal exception in "+getName(),e);
173         } catch (OutOfMemoryError err) {
174             seriousError(err);
175         } finally {
176             controller.releaseContinuePermission();
177         }
178         setCurrentCuri(null);
179         // Do cleanup so that objects can be GC.
180         this.httpRecorder.closeRecorders();
181         this.httpRecorder = null;
182         localProcessors = null;
183
184         logger.fine(getName()+" finished for order '"+name+"'");
185         setStep(STEP_FINISHED);
186         controller.toeEnded();
187         controller = null;
188     }
189
190     /***
191      * Set currentCuri, updating thread name as appropriate
192      * @param curi
193      */
194     private void setCurrentCuri(CrawlURI curi) {
195         if(curi==null) {
196             setName(coreName);
197         } else {
198             setName(coreName+curi);
199         }
200         currentCuri = curi;
201     }
202
203     /***
204      * @param s
205      */
206     private void setStep(String s) {
207         step=s;
208         atStepSince = System.currentTimeMillis();
209     }
210
211 private void seriousError(Error err) {
212     // try to prevent timeslicing until we have a chance to deal with OOM
213         // TODO: recognize that new JVM priority indifference may make this
214         // priority-jumbling pointless
215         setPriority(DEFAULT_PRIORITY+1); 
216         if (controller!=null) {
217             // hold all ToeThreads from proceeding to next processor
218             controller.singleThreadMode();
219             // TODO: consider if SoftReferences would be a better way to
220             // engineer a soft-landing for low-memory conditions
221             controller.freeReserveMemory();
222             controller.requestCrawlPause();
223             if (controller.getFrontier().getFrontierJournal() != null) {
224 controller.getFrontier().getFrontierJournal().seriousError( 225                     getName() + err.getMessage());
226             }
227         }
228        
229         // OutOfMemory etc.
230         String extraInfo = DevUtils.extraInfo();
231         System.err.println("<<<");
232         System.err.println(ArchiveUtils.getLog17Date());
233         System.err.println(err);
234         System.err.println(extraInfo);
235         err.printStackTrace(System.err);
236        
237         if (controller!=null) {
238             PrintWriter pw = new PrintWriter(System.err);
239             controller.getToePool().compactReportTo(pw);
240             pw.flush();
241         }
242         System.err.println(">>>");
243 //        DevUtils.sigquitSelf();
244        
245         String context = "unknown";
246 if(currentCuri!=null) {
247             // update fetch-status, saving original as annotation
248             currentCuri.addAnnotation("err="+err.getClass().getName());
249             currentCuri.addAnnotation("os"+currentCuri.getFetchStatus());
250 currentCuri.setFetchStatus(S_SERIOUS_ERROR);
251             context = currentCuri.singleLineReport() + " in " + currentProcessorName;
252 }
253         String message = "Serious error occured trying " +
254             "to process '" + context + "'/n" + extraInfo;
255         logger.log(Level.SEVERE, message.toString(), err);
256         setPriority(DEFAULT_PRIORITY);
257 }
258
259 /***
260      * Perform checks as to whether normal execution should proceed.
261      *
262      * If an external interrupt is detected, throw an interrupted exception.
263      * Used before anything that should not be attempted by a 'zombie' thread
264      * that the Frontier/Crawl has given up on.
265      *
266      * Otherwise, if the controller's memoryGate has been closed,
267      * hold until it is opened. (Provides a better chance of
268      * being able to complete some tasks after an OutOfMemoryError.)
269      *
270      * @throws InterruptedException
271      */
272     private void continueCheck() throws InterruptedException {
273         if(Thread.interrupted()) {
274             throw new InterruptedException("die request detected");
275         }
276         controller.acquireContinuePermission();
277     }
278
279     /***
280      * Pass the CrawlURI to all appropriate processors
281      *
282      * @throws InterruptedException
283      */
284     private void processCrawlUri() throws InterruptedException {
285         currentCuri.setThreadNumber(this.serialNumber);
286         currentCuri.setNextProcessorChain(controller.getFirstProcessorChain());
287         lastStartTime = System.currentTimeMillis();
288 //        System.out.println(currentCuri);
289         try {
290             while (currentCuri.nextProcessorChain() != null) {
291                 setStep(STEP_ABOUT_TO_BEGIN_CHAIN);
292                 // Starting on a new processor chain.
293                 currentCuri.setNextProcessor(currentCuri.nextProcessorChain().getFirstProcessor());
294                 currentCuri.setNextProcessorChain(currentCuri.nextProcessorChain().getNextProcessorChain());
295
296                 while (currentCuri.nextProcessor() != null) {
297                     setStep(STEP_ABOUT_TO_BEGIN_PROCESSOR);
298                     Processor currentProcessor = getProcessor(currentCuri.nextProcessor());
299                     currentProcessorName = currentProcessor.getName();
300                     continueCheck();
301 //                    long memBefore = (Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory())/1024;
302                     currentProcessor.process(currentCuri);
303 //                    long memAfter = (Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory())/1024;
304 //                    System.out.println((memAfter-memBefore)+"K in "+currentProcessorName);
305                 }
306             }
307             setStep(STEP_DONE_WITH_PROCESSORS);
308             currentProcessorName = "";
309         } catch (RuntimeExceptionWrapper e) {
310             // Workaround to get cause from BDB
311             if(e.getCause() == null) {
312                 e.initCause(e.getCause());
313             }
314             recoverableProblem(e);
315         } catch (AssertionError ae) {
316             // This risks leaving crawl in fatally inconsistent state,
317             // but is often reasonable for per-Processor assertion problems
318             recoverableProblem(ae);
319         } catch (RuntimeException e) {
320             recoverableProblem(e);
321         } catch (StackOverflowError err) {
322             recoverableProblem(err);
323         } catch (Error err) {
324             // OutOfMemory and any others
325             seriousError(err);
326         }
327     }
328
329
330     /***
331      * Handling for exceptions and errors that are possibly recoverable.
332      *
333      * @param e
334      */
335     private void recoverableProblem(Throwable e) {
336         Object previousStep = step;
337         setStep(STEP_HANDLING_RUNTIME_EXCEPTION);
338         e.printStackTrace(System.err);
339         currentCuri.setFetchStatus(S_RUNTIME_EXCEPTION);
340         // store exception temporarily for logging
341         currentCuri.addAnnotation("err="+e.getClass().getName());
342         currentCuri.putObject(A_RUNTIME_EXCEPTION, e);
343         String message = "Problem " + e +
344                 " occured when trying to process '"
345                 + currentCuri.toString()
346                 + "' at step " + previousStep
347                 + " in " + currentProcessorName +"/n";
348         logger.log(Level.SEVERE, message.toString(), e);
349     }
350
351     private Processor getProcessor(Processor processor) {
352         if(!(processor instanceof InstancePerThread)) {
353             // just use the shared Processor
354              return processor;
355         }
356         // must use local copy of processor
357         Processor localProcessor = (Processor) localProcessors.get(
358                     processor.getClass().getName());
359         if (localProcessor == null) {
360             localProcessor = processor.spawn(this.getSerialNumber());
361             localProcessors.put(processor.getClass().getName(),localProcessor);
362         }
363         return localProcessor;
364     }
365
366     /***
367      * @return Return toe thread serial number.
368      */
369     public int getSerialNumber() {
370         return this.serialNumber;
371     }
372
373     /***
374      * Used to get current threads HttpRecorder instance.
375      * Implementation of the HttpRecorderMarker interface.
376      * @return Returns instance of HttpRecorder carried by this thread.
377      * @see org.archive.util.HttpRecorderMarker#getHttpRecorder()
378      */
379     public HttpRecorder getHttpRecorder() {
380         return this.httpRecorder;
381     }
382    
383     /*** Get the CrawlController acossiated with this thread.
384      *
385      * @return Returns the CrawlController.
386      */
387     public CrawlController getController() {
388         return controller;
389     }
390
391     /***
392      * Terminates a thread.
393      *
394      * <p> Calling this method will ensure that the current thread will stop
395      * processing as soon as possible (note: this may be never). Meant to
396      * 'short circuit' hung threads.
397      *
398      * <p> Current crawl uri will have its fetch status set accordingly and
399      * will be immediately returned to the frontier.
400      *
401      * <p> As noted before, this does not ensure that the thread will stop
402      * running (ever). But once evoked it will not try and communicate with
403      * other parts of crawler and will terminate as soon as control is
404      * established.
405      */
406     protected void kill(){
407         this.interrupt();
408         synchronized(this) {
409             if (currentCuri!=null) {
410                 currentCuri.setFetchStatus(S_PROCESSING_THREAD_KILLED);
411                 controller.getFrontier().finished(currentCuri);
412              }
413         }
414     }
415
416 /***
417 * @return Current step (For debugging/reporting, give abstract step
418      * where this thread is).
419 */
420 public Object getStep() {
421 return step;
422 }
423
424     /***
425      * Is this thread validly processing a URI, not paused, waiting for
426      * a URI, or interrupted?
427      * @return whether thread is actively processing a URI
428      */
429     public boolean isActive() {
430         // if alive and not waiting in/for frontier.next(), we're 'active'
431         return this.isAlive() && (currentCuri != null) && !isInterrupted();
432     }
433    
434     /***
435      * Request that this thread retire (exit cleanly) at the earliest
436      * opportunity.
437      */
438     public void retire() {
439         shouldRetire = true;
440     }
441
442     /***
443      * Whether this thread should cleanly retire at the earliest
444      * opportunity.
445      *
446      * @return True if should retire.
447      */
448     public boolean shouldRetire() {
449         return shouldRetire;
450     }
451
452     //
453     // Reporter implementation
454     //
455    
456     /***
457      * Compiles and returns a report on its status.
458      * @param name Report name.
459      * @param pw Where to print.
460      */
461     public void reportTo(String name, PrintWriter pw) {
462         // name is ignored for now: only one kind of report
463        
464         pw.print("[");
465         pw.println(getName());
466
467         // Make a local copy of the currentCuri reference in case it gets
468         // nulled while we're using it.  We're doing this because
469         // alternative is synchronizing and we don't want to do this --
470         // it causes hang ups as controller waits on a lock for this thread,
471         // something it gets easily enough on old threading model but something
472         // it can wait interminably for on NPTL threading model.
473         // See [ 994946 ] Pause/Terminate ignored on 2.6 kernel 1.5 JVM.
474         CrawlURI c = currentCuri;
475         if(c != null) {
476             pw.print(" ");
477             c.singleLineReportTo(pw);
478             pw.print("    ");
479             pw.print(c.getFetchAttempts());
480             pw.print(" attempts");
481             pw.println();
482             pw.print("    ");
483             pw.print("in processor: ");
484             pw.print(currentProcessorName);
485         } else {
486             pw.print(" -no CrawlURI- ");
487         }
488         pw.println();
489
490         long now = System.currentTimeMillis();
491         long time = 0;
492
493         pw.print("    ");
494         if(lastFinishTime > lastStartTime) {
495             // That means we finished something after we last started something
496             // or in other words we are not working on anything.
497             pw.print("WAITING for ");
498             time = now - lastFinishTime;
499         } else if(lastStartTime > 0) {
500             // We are working on something
501             pw.print("ACTIVE for ");
502             time = now-lastStartTime;
503         }
504         pw.print(ArchiveUtils.formatMillisecondsToConventional(time));
505         pw.println();
506
507         pw.print("    ");
508         pw.print("step: ");
509         pw.print(step);
510         pw.print(" for ");
511         pw.print(ArchiveUtils.formatMillisecondsToConventional(System.currentTimeMillis()-atStepSince));
512         pw.println();
513
514         StackTraceElement[] ste = this.getStackTrace();
515         for(int i=0;i<ste.length;i++) {
516             pw.print("    ");
517             pw.print(ste[i].toString());
518             pw.println();
519         }
520         pw.print("]");
521         pw.println();
522        
523         pw.flush();
524     }
525
526     /***
527      * @param w PrintWriter to write to.
528      */
529     public void singleLineReportTo(PrintWriter w)
530     {
531         w.print("#");
532         w.print(this.serialNumber);
533
534         // Make a local copy of the currentCuri reference in case it gets
535         // nulled while we're using it.  We're doing this because
536         // alternative is synchronizing and we don't want to do this --
537         // it causes hang ups as controller waits on a lock for this thread,
538         // something it gets easily enough on old threading model but something
539         // it can wait interminably for on NPTL threading model.
540         // See [ 994946 ] Pause/Terminate ignored on 2.6 kernel 1.5 JVM.
541         CrawlURI c = currentCuri;
542         if(c != null) {
543             w.print(" ");
544             w.print(currentProcessorName);
545             w.print(" ");
546             w.print(c.toString());
547             w.print(" (");
548             w.print(c.getFetchAttempts());
549             w.print(") ");
550         } else {
551             w.print(" [no CrawlURI] ");
552         }
553        
554         long now = System.currentTimeMillis();
555         long time = 0;
556
557         if(lastFinishTime > lastStartTime) {
558             // That means we finished something after we last started something
559             // or in other words we are not working on anything.
560             w.print("WAITING for ");
561             time = now - lastFinishTime;
562         } else if(lastStartTime > 0) {
563             // We are working on something
564             w.print("ACTIVE for ");
565             time = now-lastStartTime;
566         }
567         w.print(ArchiveUtils.formatMillisecondsToConventional(time));
568         w.print(" at ");
569         w.print(step);
570         w.print(" for ");
571         w.print(ArchiveUtils.formatMillisecondsToConventional(now-atStepSince));
572         w.print("/n");
573         w.flush();
574     }
575
576     /* (non-Javadoc)
577      * @see org.archive.util.Reporter#singleLineLegend()
578      */
579     public String singleLineLegend() {
580         return "#serialNumber processorName currentUri (fetchAttempts) threadState threadStep";
581     }
582    
583     /* (non-Javadoc)
584      * @see org.archive.util.Reporter#getReports()
585      */
586     public String[] getReports() {
587         // for now none but the default
588         return new String[] {};
589     }
590
591     public void reportTo(PrintWriter writer) {
592         reportTo(null, writer);
593     }
594
595     /* (non-Javadoc)
596      * @see org.archive.util.Reporter#singleLineReport()
597      */
598     public String singleLineReport() {
599         return ArchiveUtils.singleLineReport(this);
600     }
601
602     public void progressStatisticsLine(PrintWriter writer) {
603         writer.print(getController().getStatistics()
604             .getProgressStatisticsLine());
605         writer.print("/n");
606     }
607
608     public void progressStatisticsLegend(PrintWriter writer) {
609         writer.print(getController().getStatistics()
610             .progressStatisticsLegend());
611         writer.print("/n");
612     }
613    
614     public String getCurrentProcessorName() {
615         return currentProcessorName;
616     }
617 }