rust reqwest异步并发处理

use futures;
use futures::future;
use futures::stream::{self, StreamExt};
#[tokio::main]
async fn main(){

    let paths = vec![
        "https://www.baidu.com".to_string(),
        "https://www.baidu.com".to_string(),
        "https://www.baidu.com".to_string(),
        "https://www.baidu.com".to_string(),
        "https://www.baidu.com".to_string(),
        "https://www.baidu.com".to_string(),
        "https://www.littley.top".to_string(),
        "https://www.littley.top".to_string(),
        "https://www.littley.top".to_string(),
        "https://www.littley.top".to_string(),
        "https://www.littley.top".to_string(),
        "https://www.littley.top".to_string(),
        "https://www.littley.top".to_string(),
        "https://blog.csdn.net/".to_string(),
        "https://blog.csdn.net/".to_string(),
        "https://blog.csdn.net/".to_string(),
        "https://blog.csdn.net/".to_string(),
        "https://blog.csdn.net/".to_string(),
        "https://blog.csdn.net/".to_string(),
    ];
    let fetches = futures::stream::iter(
        paths.into_iter().map(|path| {
            async move {
                match reqwest::get(&path).await {
                    Ok(resp) => {
                        match resp.text().await {
                            Ok(text) => {
                                println!("RESPONSE: {} bytes from {}", text.len(), path);
                            }
                            Err(_) => println!("ERROR reading {}", path),
                        }
                    }
                    Err(_) => println!("ERROR downloading {}", path),
                }
            }
        })
    ).buffer_unordered(8).collect::<Vec<()>>();
    println!("Waiting...");
    fetches.await;
}

Cargo.toml

[dependencies]
reqwest = { version = "0.10.1", features = ["json"] }
tokio = { version = "0.2.11", features = ["full"] }
futures = "0.3.4"

其他例子

use futures::{stream, StreamExt}; // 0.3.1
use reqwest::Client; // 0.10.0
use tokio; // 0.2.4, features = ["macros"]

const PARALLEL_REQUESTS: usize = 2; //控制并行请求数量，太大容易内存不足

#[tokio::main]
async fn main() {
    let client = Client::new();

    let urls = vec!["https://api.ipify.org", "https://api.ipify.org"];

    let bodies = stream::iter(urls)
        .map(|url| {
            let client = &client;
            async move {
                let resp = client.get(url).send().await?;
                resp.bytes().await
            }
        })
        .buffer_unordered(PARALLEL_REQUESTS);

    bodies
        .for_each(|b| {
            async {
                match b {
                    Ok(b) => println!("Got {} bytes", b.len()),
                    Err(e) => eprintln!("Got an error: {}", e),
                }
            }
        })
        .await;
}

另一个例子不控制并行数量

use futures::future; // 0.3.4
use reqwest::Client; // 0.10.1
use tokio; // 0.2.11

#[tokio::main]
async fn main() {
    let client = Client::new();

    let urls = vec!["https://api.ipify.org", "https://api.ipify.org"];

    let bodies = future::join_all(urls.into_iter().map(|url| {
        let client = &client;
        async move {
            let resp = client.get(url).send().await?;
            resp.bytes().await
        }
    }))
    .await;

    for b in bodies {
        match b {
            Ok(b) => println!("Got {} bytes", b.len()),
            Err(e) => eprintln!("Got an error: {}", e),
        }
    }
}