colly框架学习（三）

colly框架学习（三）

将数据持久化

1、mysql

• 安装mysql

  go get -u github.com/go-sql-driver/mysql
  

• mysql连接

  import (
      "database/sql"
      _ "github.com/go-sql-driver/mysql"
  )
  
  dsn := "username:password@/database" // 本地mysql
  db, err := sql.Open("mysql", dsn)
  if err != nil {
      fmt.Printf("error:%s", err)
  }
  defer db.Close()
  
  
  //回调
  stmt, err := db.Prepare("INSERT INTO table (字段1, 字段2, 字段3) VALUES (?, ?, ?)")
  if err != nil {
      fmt.Println(err)
  }
  defer stmt.Close()
  
  // 执行插入操作
  if _, err := stmt.Exec("值1", "值2", "值3"); err != nil {
      fmt.Println(err)
  }
  

2、mongodb

• 安装mongodb

  go get go.mongodb.org/mongo-driver/mongo
  

• mongo连接

  import (
      "context"
      "time"
      "go.mongodb.org/mongo-driver/bson"
      "go.mongodb.org/mongo-driver/mongo"
      "go.mongodb.org/mongo-driver/mongo/options"
  )
  ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
  defer cancel()
  client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
  if err != nil {
      fmt.Printf("client-->err: %s\n", err)
  }
  if err = client.Ping(context.Background(), nil); err != nil {
      fmt.Printf("client ping --> error: %s\n" ,err)
  }
  defer client.Disconnect(ctx)
  collection := client.Database("数据库名").Collection("集合名")
  
  doc := bson.D{
              {"字段1", "值1"},
              {"字段2", "值2"},
              {"字段3", "值3"},
          }
  insertResult, err := collection.InsertMany(ctx, doc)
  if err != nil {
      fmt.Printf("insert --> error: %s\n", err)
  }
  

3、csv

• 存入csv文件

  import (
      "encoding/csv"
      "os"
  )
  
  file, err := os.Create("output.csv")
  if err != nil {
      fmt.Printf("file create err:%s", err)
  }
  writer := csv.NewWriter(file)
  defer writer.Flush()
  if err := writer.Write([]string{"字段1", "字段2", "字段3"}); err != nil {
      fmt.Printf("writer err:%s", err)
  }
  
  if err := writer.Write([]string{"值1", "值2", "值3"}); err != nil {
          fmt.Printf("writer err:%s", err)
      }