功能:爬xxx.com网站列表页面;然后得到连接访问文章标题和详细内容;然后插入WordPress6版本的cms管理系统wp_posts表中 (post_title,post_content)
package spider
import (
"strings"
"github.com/gocolly/colly/v2"
)
type Spider struct{}
// 插入WordPress的sql语句
const sqlbase = "insert into wp_posts(post_author,post_date,post_date_gmt,post_content,post_title,post_status,comment_status,ping_status,post_modified,post_modified_gmt,post_parent,menu_order,post_type,post_excerpt,to_ping,pinged,post_content_filtered) VALUE(1,now(),DATE_ADD(now(),INTERVAL '-8' HOUR),?,?,'publish','open','open',now(),DATE_ADD(now(),INTERVAL '-8' HOUR),0,0,'post','','','','')"
// 爬第三方网站的内容
func (Spider) ExecSpider() {
c := colly.NewCollector(
// 限制网站范围,防止去抓类似友情链接的外网等等
colly.AllowedDomains("yy.xxx.com", "aa.xxx.com"),
)
// 搜索文章列