丢失的数据不能小于校验快的数据,否则没法复原数据
可用容量看,4+2纠删码的利用率是66%(4/(4+2)),但3副本只有33%(4/(4*3)),两者差了2倍,8+2纠删码的利用率更可以做到80%,由此可见纠删码的可用容量对比多副本要高很多
rs.txt 内容
AAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDSDG
分片和解码
package main
// rs 纠删码
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"github.com/klauspost/reedsolomon"
)
const (
dataShards = 2 // 数据分片数
parShards = 1 // 校验分片数
)
func main() {
//encoder()
decode()
}
func encoder() {
fname := "rs.txt"
// Create encoding matrix.
enc, err := reedsolomon.New(dataShards, parShards)
checkErr(err)
fmt.Println("Opening", fname)
b, err := ioutil.ReadFile(fname)
checkErr(err)
// Split the file into equally sized shards.
// 此时分成了3等片
shards, err := enc.Split(b)
checkErr(err)
fmt.Printf("File split into %d data+parity shards with %d bytes/shard.\n", len(shards), len(shards[0]))
// Encode parity
// 分片 encode 下
err = enc.Encode(shards)
checkErr(err)
// Write out the resulting files.
_, file := filepath.Split(fname)
dir := "./tmp/encoder/"
for i, shard := range shards {
outfn := fmt.Sprintf("%s.%d", file, i)
fmt.Println("Writing to", outfn)
// 分割的分片写入各个文件中
err = ioutil.WriteFile(filepath.Join(dir, outfn), shard, os.ModePerm)
checkErr(err)
}
}
func checkErr(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "Error: %s", err.Error())
os.Exit(2)
}
}
// ------------------------- decode -----------------------
func decode() {
// Create matrix
enc, err := reedsolomon.New(dataShards, parShards)
checkErr(err)
fname := "./tmp/encoder/rs.txt"
// Create shards and load the data.
shards := make([][]byte, dataShards+parShards)
var iNum []int
for i := range shards {
infn := fmt.Sprintf("%s.%d", fname, i)
fmt.Println("Opening", infn)
shards[i], err = ioutil.ReadFile(infn)
if err != nil {
fmt.Println("Error reading file", err)
shards[i] = nil
iNum = append(iNum, i)
}
}
// Verify the shards
ok, err := enc.Verify(shards)
if ok {
fmt.Println("No reconstruction needed")
} else {
fmt.Println("Verification failed. Reconstructing data")
// 恢复切片
err = enc.Reconstruct(shards)
if err != nil {
fmt.Println("Reconstruct failed -", err)
os.Exit(1)
}
// 再次验证分片
ok, err = enc.Verify(shards)
if !ok {
fmt.Println("Verification failed after reconstruction, data likely corrupted.")
os.Exit(1)
}
checkErr(err)
// 丢失的分片数据恢复
for _, i2 := range iNum {
outfn := fname + "." + strconv.Itoa(i2)
err = ioutil.WriteFile(outfn, shards[i2], os.ModePerm)
}
}
outfn := "./tmp/decoder/rs.txt"
fmt.Println("Writing data to", outfn)
f, err := os.Create(outfn)
checkErr(err)
// We don't know the exact filesize.
// 还原数据
err = enc.Join(f, shards, len(shards[0])*dataShards)
checkErr(err)
}