package main
import (
"context"
"fmt"
"io"
"log"
"net/http"
"strconv"
"strings"
"sync"
"time"
)
var mu sync.Mutex // 创建一个互斥锁
var (
httpClient *http.Client
httpClientOnce sync.Once
)
func getHTTPClient() *http.Client {
httpClientOnce.Do(func() {
httpClient = &http.Client{
Transport: &http.Transport{
MaxIdleConns: 100, // 最大空闲连接数
MaxIdleConnsPerHost: 100, // 每个主机的最大空闲连接数
},
}
})
return httpClient
}
const (
videoURL = "https://cn-beijing-data.aliyundrive.net/5fc0c8b4f1f50595b5094497b75fea8afe7b715e%2F5fc0c8b405587a69f8254396abeec229a33223ac?callback=eyJjYWxsYmFja1VybCI6Imh0dHA6Ly9wZHNhcGkuYWxpeXVuZHJpdmUuY29tL3YyL2ZpbGUvZG93bmxvYWRfY2FsbGJhY2siLCJjYWxsYmFja0JvZHkiOiJodHRwSGVhZGVyLnJhbmdlPSR7aHR0cEhlYWRlci5yYW5nZX1cdTAwMjZidWNrZXQ9JHtidWNrZXR9XHUwMDI2b2JqZWN0PSR7b2JqZWN0fVx1MDAyNmRvbWFpbl9pZD0ke3g6ZG9tYWluX2lkfVx1MDAyNnVzZXJfaWQ9JHt4OnVzZXJfaWR9XHUwMDI2ZHJpdmVfaWQ9JHt4OmRyaXZlX2lkfVx1MDAyNmZpbGVfaWQ9JHt4OmZpbGVfaWR9IiwiY2FsbGJhY2tCb2R5VHlwZSI6ImFwcGxpY2F0aW9uL3gtd3d3LWZvcm0tdXJsZW5jb2RlZCIsImNhbGxiYWNrU3RhZ2UiOiJiZWZvcmUtZXhlY3V0ZSIsImNhbGxiYWNrRmFpbHVyZUFjdGlvbiI6Imlnbm9yZSJ9&callback-var=eyJ4OmRvbWFpbl9pZCI6ImJqMjkiLCJ4OnVzZXJfaWQiOiI3NGQ2NzQ4M2M5MWU0NzViYmJhZjc0YmQyMmUyNTUxZSIsIng6ZHJpdmVfaWQiOiIzNDYwOTcwMiIsIng6ZmlsZV9pZCI6IjY0MDM2MjNhMTFjZWJmOWQ0MzAxNDJjZjk5ZTQ4OThkNDZlYzc3ZWYifQ%3D%3D&di=bj29&dr=34609702&f=6403623a11cebf9d430142cf99e4898d46ec77ef&response-content-disposition=attachment%3B%20filename%2A%3DUTF-8%27%27dolby-atmos-trailer_leaf_1080.mp4&security-token=CAIS%2BgF1q6Ft5B2yfSjIr5fHA9zioaVOg4efbE%2FLsHcNXOFmm4%2Fnkjz2IHFPeHJrBeAYt%2FoxmW1X5vwSlq5rR4QAXlDfNXHEIlWOqFHPWZHInuDox55m4cTXNAr%2BIhr%2F29CoEIedZdjBe%2FCrRknZnytou9XTfimjWFrXWv%2Fgy%2BQQDLItUxK%2FcCBNCfpPOwJms7V6D3bKMuu3OROY6Qi5TmgQ41Uh1jgjtPzkkpfFtkGF1GeXkLFF%2B97DRbG%2FdNRpMZtFVNO44fd7bKKp0lQLukMWr%2Fwq3PIdp2ma447NWQlLnzyCMvvJ9OVDFyN0aKEnH7J%2Bq%2FzxhTPrMnpkSlacGoABYuNFSMqzzPFumJqvBlk4UjgCsRBBo%2FQdlhXy8ZRP0Px2BnomjBt1okB2AKE8jEy9DZ3weSVc3pq7xE%2FxA8XI7bNxKce2p2y63bbzIv56zQ9DNBg%2FwrLKoewHidty0PedVVGg6o5uHW3QaVY8TUZltxuS07mDI2BFgIm70QAkzTcgAA%3D%3D&u=74d67483c91e475bbbaf74bd22e2551e&x-oss-access-key-id=STS.NTrHfVLzo4EtnizTwXPnJtJLp&x-oss-additional-headers=referer&x-oss-expires=1696755901&x-oss-signature=JG2Agz%2B0XBxDpXwbA3fAzD%2F7b6GubB42pl4o5xeENKo%3D&x-oss-signature-version=OSS2"
contentType = "video/mp4"
defaultRanges = "bytes=0-"
chunkSize = 128 * 1024 // 每个线程下载的块大小（128KB）
numThreads = 32
)
func main() {
// 初始化 BlockingQueue 或在需要的地方进行初始化
handler := func(w http.ResponseWriter, r *http.Request) {
// 获取请求的 Range 头信息
var status int
rangeHeader := r.Header.Get("Range")
fmt.Printf("请求开始 rangeHeader", rangeHeader)
if rangeHeader == "" {
rangeHeader = defaultRanges
status = http.StatusOK
} else {
status = http.StatusPartialContent
}
// 解析 Range 头信息
rangeParts := strings.Split(rangeHeader, "=")
if len(rangeParts) != 2 {
http.Error(w, "无效的 Range 头信息", http.StatusBadRequest)
return
}
rangeValues := strings.Split(rangeParts[1], "-")
if len(rangeValues) != 2 {
http.Error(w, "无效的 Range 头信息", http.StatusBadRequest)
return
}
startByte, err := strconv.Atoi(rangeValues[0])
if err != nil {
http.Error(w, "无效的 Range 头信息", http.StatusBadRequest)
return
}
// 设置 endByte 为文件大小
endByte := getFileSize(w, videoURL) - 1
// 设置响应头
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", startByte, endByte, endByte+1))
w.Header().Set("Content-Length", fmt.Sprintf("%d", endByte-startByte+1))
w.WriteHeader(status)
processChunkRequests(w, startByte, endByte)
}
fmt.Println("服务器启动，监听端口 8080")
// 创建一个中间件函数，用于全局异常处理
middleware := func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
try := func() {
next.ServeHTTP(w, r)
}
defer func() {
if err := recover(); err != nil {
log.Printf("捕获到异常了：%v", err)
http.Error(w, "内部服务器错误", http.StatusInternalServerError)
}
}()
try()
})
}
// 使用中间件包装处理器函数
wrappedHandler := middleware(http.HandlerFunc(handler))
// 启动HTTP服务器并使用包装后的处理器处理请求
http.ListenAndServe("0.0.0.0:8080", wrappedHandler)
}
func getFileSize(w http.ResponseWriter, url string) int {
// 构造请求
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatal(err)
}
req.Header.Add("Referer", "https://www.aliyundrive.com/")
// 发送请求
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
// 复制响应头部信息到响应对象 w
copyResponseHeaders(resp.Header, w.Header())
// 从响应头中获取文件大小
fileSize, err := strconv.Atoi(resp.Header.Get("Content-Length"))
if err != nil {
log.Fatal(err)
}
return fileSize
}
func copyResponseHeaders(src, dst http.Header) {
contentType := src.Get("Content-Type")
contentDisposition := src.Get("Content-Disposition")
if contentDisposition != "" {
if strings.HasSuffix(contentDisposition, ".mp4") {
contentType = "video/mp4"
} else if strings.HasSuffix(contentDisposition, ".webm") {
contentType = "video/webm"
} else if strings.HasSuffix(contentDisposition, ".avi") {
contentType = "video/x-msvideo"
} else if strings.HasSuffix(contentDisposition, ".wmv") {
contentType = "video/x-ms-wmv"
} else if strings.HasSuffix(contentDisposition, ".flv") {
contentType = "video/x-flv"
} else if strings.HasSuffix(contentDisposition, ".mov") {
contentType = "video/quicktime"
} else if strings.HasSuffix(contentDisposition, ".mkv") {
contentType = "video/x-matroska"
} else if strings.HasSuffix(contentDisposition, ".mpeg") {
contentType = "video/mpeg"
} else if strings.HasSuffix(contentDisposition, ".3gp") {
contentType = "video/3gpp"
}
}
dst.Add("Content-Type", contentType)
for key, values := range src {
for _, value := range values {
if strings.EqualFold(key, "Content-Length") || strings.EqualFold(key, "Content-Type") {
continue
}
dst.Add(key, value)
}
}
}
func processChunkRequests(w http.ResponseWriter, startByte int, endByte int) {
//18963693568
var currentStart int
currentStart = startByte
running := true
ctx := context.Background()
dataMap := NewBlockingMap[[]byte](ctx, 500)
currentNumThreads := (endByte - startByte + 1) / chunkSize
if (endByte-startByte+1)%chunkSize != 0 {
currentNumThreads++
}
// mu.Lock() // 在写入 map 之前锁定互斥锁
// defer mu.Unlock() // 在函数返回前释放互斥锁
// 按顺序取出块数据并写入响应流
go func() {
//time.Sleep(2 * time.Second)
for i := startByte; i <= startByte+(currentNumThreads-1)*chunkSize; i += chunkSize {
data, ok := dataMap.TryPoll(i, 10*time.Second)
if ok {
fmt.Printf("获取元素 %d 成功\n", i)
_, err := w.Write(data)
if err != nil {
running = false
log.Println(err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
} else {
running = false
log.Printf("Missing data for startByte %d", i)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
fmt.Printf("获取元素 %d 失败\n", i)
return
}
// data, ok := dataMap[i]
// if !ok {
// running = false
// log.Printf("Missing data for startByte %d", i)
// http.Error(w, "Internal Server Error", http.StatusInternalServerError)
// return
// }
// _, err := w.Write(data)
// dataMap.Delete(i)
// if err != nil {
// running = false
// log.Println(err)
// http.Error(w, "Internal Server Error", http.StatusInternalServerError)
// return
// }
}
}()
for currentStart < endByte {
if !running {
fmt.Println("running 状态", running)
break
}
fmt.Println("请求的 currentStart", currentStart)
dataStart, err := getChunkData(dataMap, w, currentStart, endByte)
if err != nil {
running = false
// 错误处理，可以记录日志或者返回特定错误信息给客户端
log.Printf("管道可能断开: %v", err)
//close(chunkReq.DataChan) // 关闭通道，终止数据传输
return
}
// 更新 currentStart
currentStart = dataStart
}
time.Sleep(2 * time.Second)
}
func getChunkData(dataMap *BlockingMap[[]byte], w http.ResponseWriter, startByte, endByte int) (int, error) {
// 循环下载 32 个 128K 数据块
currentNumThreads := (endByte - startByte + 1) / chunkSize
if (endByte-startByte+1)%chunkSize != 0 {
currentNumThreads++
}
if currentNumThreads > numThreads {
currentNumThreads = numThreads
}
fmt.Println("currentNumThreads : ", currentNumThreads)
var wg sync.WaitGroup // 创建一个等待组
for i := 0; i < currentNumThreads; i++ {
wg.Add(1) // 每个协程开始时增加等待组计数
go func(i int) {
defer wg.Done() // 协程完成时减少等待组计数
// 构造请求
req, err := http.NewRequest("GET", videoURL, nil)
if err != nil {
fmt.Errorf(err.Error())
}
// 计算当前数据块的起始和结束字节
currentStart := startByte + i*chunkSize
currentEnd := currentStart + chunkSize - 1
if currentEnd > endByte {
currentEnd = endByte
}
//fmt.Println("请求的 Range", currentStart, currentEnd)
req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", currentStart, currentEnd))
req.Header.Add("Referer", "https://www.aliyundrive.com/")
// 发送请求
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
fmt.Errorf(err.Error())
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusPartialContent {
fmt.Errorf("下载失败 (HTTP %d): %s", resp.StatusCode, resp.Status)
}
// 将下载的数据保存在内存中
data, err := io.ReadAll(resp.Body)
if err != nil {
fmt.Errorf(err.Error())
}
//mu.Lock() // 在写入 map 之前锁定互斥锁
if dataMap.TryPush(currentStart, data, 10*time.Second) {
fmt.Printf("添加元素 %d 成功\n", currentStart)
} else {
fmt.Printf("添加元素 %d 失败\n", currentStart)
}
//dataMap[currentStart] = data
//mu.Unlock() // 写入完成后释放互斥锁
}(i)
}
// 等待所有协程完成
wg.Wait()
// 所有协程已完成，可以继续处理数据或执行下一步操作
fmt.Println("所有协程已完成任务")
return startByte + currentNumThreads*chunkSize, nil
}
同步阻塞map BlockingMap
package main
import (
"context"
"fmt"
"sync"
"time"
)
// RejectHandler Reject push into queue if return false.
type RejectHandler func(ctx context.Context) bool
type Queue[T any] interface {
Push(value T)
TryPush(key T, value T, timeout time.Duration) bool
Poll() T
TryPoll(key T, timeout time.Duration) (T, bool)
}
type BlockingMap[T any] struct {
q sync.Map // 阻塞队列使用 sync.Map 表示
limit int // 阻塞队列的大小
ctx context.Context
//handler RejectHandler
}
func NewBlockingMap[T any](ctx context.Context, queueSize int) *BlockingMap[T] {
return &BlockingMap[T]{
q: sync.Map{},
limit: queueSize,
ctx: ctx,
}
}
// SetRejectHandler Set a reject handler.
//
// func (q *BlockingMap[T]) SetRejectHandler(handler RejectHandler) {
// q.handler = handler
// }
//
// 设置拒绝处理程序
//
// dataMap.SetRejectHandler(func(ctx context.Context) bool {
// fmt.Println("拒绝添加元素，等待超时...")
// select {
// case <-ctx.Done():
// fmt.Println("超时，继续添加元素")
// return true
// }
// })
func (q *BlockingMap[T]) Push(value T) {
q.q.Store(value, struct{}{})
}
func (q *BlockingMap[T]) TryPush(key int, value T, timeout time.Duration) bool {
ctx, cancel := context.WithTimeout(q.ctx, timeout)
defer cancel()
// if q.handler != nil {
// if q.size() < q.limit {
// q.q.Store(key, value)
// return true
// } else {
// ok := q.handler(ctx)
// if ok {
// q.q.Store(key, value)
// }
// return ok
// }
// } else {
// q.q.Store(key, value)
// return true
// }
for {
select {
case <-ctx.Done():
fmt.Printf("超时 %d \n", key)
return false // 超时
default:
// 使用 Load 方法获取值
//fmt.Printf("q.size() %d \n", q.size())
if q.size() < q.limit {
q.q.Store(key, value)
return true
}
time.Sleep(100 * time.Millisecond)
}
}
}
func (q *BlockingMap[T]) TryPoll(key int, timeout time.Duration) (T, bool) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
for {
var value T
select {
case <-ctx.Done():
return value, false // 超时
default:
// 使用 Load 方法获取值
v, ok := q.q.Load(key)
if ok {
value = v.(T)
// 只在值存在时才删除
q.q.Delete(key) // 从 map 中删除该 key
return value, true
}
time.Sleep(100 * time.Millisecond)
}
}
}
func (q *BlockingMap[T]) size() int {
count := 0
q.q.Range(func(key, value interface{}) bool {
count++
return true
})
return count
}
// Delete 从队列中删除指定 key 的元素
func (q *BlockingMap[T]) Delete(key int) {
q.q.Delete(key)
}
同步阻塞Queue BlockingQueue
package main
import (
"context"
"time"
)
/**
@author: nizhenxian
@date: 2022/3/11 10:07:12
**/
// RejectHandler Reject push into queue if return false.
type RejectHandler func(ctx context.Context) bool
type Queue[T any] interface {
Push(value T)
TryPush(value T, timeout time.Duration) bool
Poll() T
TryPoll(timeout time.Duration) (T, bool)
}
type BlockingQueue[T any] struct {
q chan T // 阻塞队列使用通道表示
limit int // 阻塞队列的大小
ctx context.Context
handler RejectHandler
}
func NewBlockingQueue[T any](ctx context.Context, queueSize int) *BlockingQueue[T] {
return &BlockingQueue[T]{
q: make(chan T, queueSize),
limit: queueSize,
ctx: ctx,
}
}
// SetRejectHandler Set a reject handler.
func (q *BlockingQueue[T]) SetRejectHandler(handler RejectHandler) {
q.handler = handler
}
func (q *BlockingQueue[T]) Push(value T) {
ok := true
if q.handler != nil {
select {
case q.q <- value:
return
default:
ok = q.handler(q.ctx)
}
}
if ok {
q.q <- value
}
}
func (q *BlockingQueue[T]) TryPush(value T, timeout time.Duration) bool {
ctx, cancel := context.WithTimeout(q.ctx, timeout)
defer cancel()
select {
case q.q <- value:
return true
case <-ctx.Done():
}
return false
}
func (q *BlockingQueue[T]) Poll() T {
ret := <-q.q
return ret
}
func (q *BlockingQueue[T]) TryPoll(timeout time.Duration) (ret T, ok bool) {
ctx, cancel := context.WithTimeout(q.ctx, timeout)
defer cancel()
select {
case ret = <-q.q:
return ret, true
case <-ctx.Done():
}
return ret, false
}
func (q *BlockingQueue[T]) size() int {
return len(q.q)
}
1986
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
