JavaScript 网页信息采集Demo

     目前可以从淘宝的商品列表的起始页开始自动翻页直到最后一页，然后将信息保存到Excel文件中。但对页面显示格式有要求，以后可以继续改进，只做尝试！用到了Thread , Jquery  Ajax

 

如:

女装:http://list.taobao.com/browse/16/n-1----------------------12---------yes---------------------2-------b-y-40-grid-commend-0-all-16.htm

 

 

CODE:

 

js

 

//Author:JackYin //Date:2008.11 //taobao crawl dataList //---------------------------------------------------------------------------------- var savePath; var threadSleep=10; //------------------------------------------------------- function setConfig() { if($("#config").css("display")=='none'){ $("#config").slideDown("slow"); }else{ $("#config").slideUp("slow"); } } function trim(s) { return s.replace( /^/s*/, "" ).replace( //s*$/, "" ); } function createFile(filename,folder,content) { var fso = new ActiveXObject("Scripting.FileSystemObject"); var stream = fso.CreateTextFile("crawl//data//"+folder+"//"+filename,true); stream.Write(content); stream.Close(); } function createFolder(foldername) { var fso = new ActiveXObject("Scripting.FileSystemObject"); fso.CreateFolder ("crawl//data//"+foldername); } function curDate(){ var today = new Date(); var day = today.getDate(); var month = today.getMonth() + 1; var year = today.getYear(); var hour=today.getHours() ; var minute=today.getMinutes(); var second=today.getSeconds() ; var date = year + "" + month + "" + day+"_"+hour+""+minute+""+second; return date; } function ajax(sendurl,handle){ $.ajax({url: sendurl, type:"get", cache: false, success:function(msg){ handle(msg); } }); } function go(){ if($('#enter').val()!=""){ Concurrent.Thread.create(ajax,$('#enter').val(),handle);//create thread Concurrent.Thread.sleep(threadSleep); }else{ alert("请输入入口地址!"); } } var num=1; var foldername; var classname; function handle(msg){ $("#tmp").html(msg); var pageEnd=eval($("#lastPage").val()); $("#status").val($("span.page-info").html()); if(pageEnd==0){ var next=eval($("#nextPage").val()); } if(num==1){ foldername=eval($("#foldername").val());//class name savePath=trim($("#savePath").val()); classname=$("select[@name='cat']:first option[selected]").text();//classname if(foldername.indexOf("/")!=-1){ foldername=foldername.replace("/","-"); } createFolder(foldername); } createFile(foldername+num+".html",foldername,msg); getData(next,pageEnd); } //----------------------------------------------------------------------------------------------------------- var sheetCount=0; var i , j; var excel,workbook,sheet; function getData(next,pageEnd){ var htag=eval($("#titleZone").val()); var photo=eval($("#picZone").val()); var price=eval($("#priceZone").val()); var data=""; // alert("标题:"+htag.get(0).firstChild.innerHTML+"/n"+"价格:"+price.get(0).lastChild.innerHTML+"/n"+"图片:"+photo.get(0).firstChild.firstChild.src+"/n"+"fromUrl:"+htag.get(0).firstChild.href); //----------------------------------------------------------------------------------------------------- if(num==1){ excel = new ActiveXObject("Excel.Application"); //创建AX对象excel excel.visible =true; //设置excel可见属性 workbook = excel.Workbooks.Add; sheet = workbook.Worksheets(num); //创建sheet sheet.Application.Visible = true; sheet.Cells(1,1).Value="标题"; sheet.Cells(1,2).Value="分类"; sheet.Cells(1,3).Value="价格"; sheet.Cells(1,4).Value="图片"; sheet.Cells(1,5).Value="地址"; } for( i=0, j=sheetCount;j<htag.length+sheetCount,i<htag.length;++i,++j) { //data+=htag.get(i).innerHTML+" "+price.get(i).innerHTML+" "+photo.get(i).innerHTML+"<br>"; //------------------------------------------------------------------------------ sheet.Cells(j+2,1).Value=eval($("#title").val()); sheet.Cells(j+2,2).Value=classname; sheet.Cells(j+2,3).Value=eval($("#price").val()); sheet.Cells(j+2,4).Value=eval($("#pic").val()); sheet.Cells(j+2,5).Value=eval($("#fromUrl").val()); //------------------------------------------------------------------------------ } sheetCount+=i; num++; // $("#list").html(data); $("#tmp").html(""); //clear tmp if(pageEnd==0){ Concurrent.Thread.create(ajax,next,handle);//create thread Concurrent.Thread.sleep(threadSleep); } else{ try { sheet.SaveAs(savePath+foldername+"_"+curDate()+".xls"); sheet.Application.Quit(); Concurrent.Thread.stop(); } catch ( e ) { //error alert("exception!save already crewled..."); sheet.SaveAs(savePath+foldername+"_"+curDate()+".xls"); sheet.Application.Quit(); Concurrent.Thread.stop(); } } }

 

html:

 

<html> <head> <title>crawl-demon--@author:JackYin</title> <mce:style><!-- *{ font-size:12pt; font-family:Arial; } img{ border-style:none; } #enter{ heigth:120px; background-color:#cccccc; } #config{ display:none; border:1px dashed #cccccc; width:500px; } .showConfig{ width:50px; height:8px; cursor:pointer; border:1px dotted #cccccc; font-size:12pt; font-family:Arial; } --></mce:style><style mce_bogus="1">*{ font-size:12pt; font-family:Arial; } img{ border-style:none; } #enter{ heigth:120px; background-color:#cccccc; } #config{ display:none; border:1px dashed #cccccc; width:500px; } .showConfig{ width:50px; height:8px; cursor:pointer; border:1px dotted #cccccc; font-size:12pt; font-family:Arial; }</style> <mce:script type="text/javascript" src="lib/jquery.js" mce_src="lib/jquery.js"></mce:script> <mce:script type="text/javascript" src="lib/Concurrent.Thread.js" mce_src="lib/Concurrent.Thread.js"></mce:script> <mce:script type="text/javascript" src="lib/core.js" mce_src="lib/core.js"></mce:script> </head> <body> <h3>商品信息采集-Demo</h3> 入口:<input type="text" size="80" id="enter" value="" /> <input type="button" value="Start" οnclick="go()" />  Status:<input type="text" name="status" id="status" size="6" readonly> <hr> <span class="showConfig" οnclick="setConfig()">配 置</span> <div id="config"> <ul> <li>数据保存目录:<input type="text" name="savePath" id="savePath" value="D://Data//"></li> <li>文件夹:<input type="text" name="foldername" id="foldername" value="$('h1.list-heading').html()"></li> <li>标签设置: <ul> <li>最后页:<input type="text" name="lastPage" id="lastPage" value="$('span.page-end').length"></li> <li>下一页:<input type="text" name="nextPage" id="nextPage" value="$('a.page-next').attr('href').split('#')[0]"></li> <li>标题域:<input type="text" name="titleZone" id="titleZone" value="$('h3.summary')"></li> <li>图片域:<input type="text" name="picZone" id="picZone" value="$('div.photo')"></li> <li>价格域:<input type="text" name="priceZone" id="priceZone" value="$('div.price')"></li> <li>标题值:<input type="text" name="title" id="title" value="htag.get(i).firstChild.innerHTML"></li> <li>图片值:<input type="text" name="pic" id="pic" value="photo.get(i).firstChild.firstChild.src"></li> <li>价格值:<input type="text" name="price" id="price" value="price.get(i).lastChild.innerHTML"></li> <li>地址值:<input type="text" name="fromUrl" id="fromUrl" value="htag.get(i).firstChild.href"></li> </ul> </li> </ul> </div> <div id="tmp" style="display:none;height:0px"> </div> <div id="list"> </div> </body> </html>

 

代码里可以修改成只保存采集的页面到指定目录，同时可以保存到Excel文件。当然还有不少问题，只想说明

Javascript 太强大了。

 

小弟学习中，可能会有不少问题，欢迎指教！