多进程跑Rij代码

import json
import sys
import time
from multiprocessing import Process, Manager
import requests


def nlists_ndatas(process_num, data_num, origin_long_dataset, server_num):
    """
    共会处理 process_num * data_num 个数据
    server_num是当前用的是第几个服务器，用于自动给整个originlist划分section
    返回的list是 有process_num个列表组成的 process_num * data_num 个数据
    """
    items_list = list(origin_long_dataset.items())
    list_of_list = []
    origin_section = items_list[
                     (process_num * data_num) * (server_num - 1):(process_num * data_num) * server_num]
    for v in range(process_num):
        list_temp = origin_section[v * data_num:(v + 1) * data_num]
        list_of_list.append(list_temp)
    return list_of_list


def get_road_distance(cbg_lonlat_dict_section,cbg_lonlat_dict):
    with lock:
        for a in cbg_lonlat_dict_section:
            point1_lon = cbg_lonlat_dict_section[a][0]
            point1_lat = cbg_lonlat_dict_section[a][1]
            list1 = []
            for b in cbg_lonlat_dict:
                point2_lon = cbg_lonlat_dict[b][0]
                point2_lat = cbg_lonlat_dict[b][1]
                if (point1_lon == point2_lon) and (point1_lat == point2_lat):
                    list1.append(0)
                else:
                    url = f'http://localhost:7070/otp/routers/default/plan?fromPlace={point1_lat},{point1_lon}&' \
                          f'toPlace={point2_lat},{point2_lon}&time=1:02pm&date=11-14-2017&mode=CAR&arriveBy=false&numItineraries=1'
                    response = requests.request("GET", url, verify=False)
                    drive_data = response.json()
                    try:
                        distance = round(drive_data['plan']['itineraries'][0]['legs'][0]['distance'])
                        list1.append(distance)
                    except KeyError:
                        list1.append(0)
                        print("KeyError: The key does not exist in the dictionary.")
            dict0[a] = list1


def nprocess(process_num):
    """
    :param process_num: 进程数量
    :param list_of_list:一个服务器要处理的所有数据
    :return:进程池
    """
    processes = []
    for i in range(process_num):
        lonlat_dict = dict(list_of_list[i])
        p = Process(target=get_road_distance, args=(lonlat_dict,cbg_lonlat_dict,))
        processes.append(p)
    return processes


if __name__ == '__main__':
    with open('cbg_lonlat_dict.json', 'r', encoding='utf-8') as f:
        cbg_lonlat_dict = json.load(f)

    list_of_list = nlists_ndatas(45, 10, cbg_lonlat_dict, 1)

    with Manager() as manager:
        dict0 = manager.dict()
        list1 = manager.list()
        lock = manager.Lock()
        process_list = nprocess(45)

        start = time.perf_counter()
        for process in process_list:
            process.start()
        for process in process_list:
            process.join()

        end = time.perf_counter()
        print('程序运行时间：' + format(end - start))

        dict0 = dict(dict0)

    print(dict0)
    print(len(dict0))

    with open('server1_cbg_road_distance.json', 'w', encoding='utf-8') as f:
        json.dump(dict0, f, indent=4, ensure_ascii=False)

代码将总1726个cbg的经纬度输入到otp本地服务器上不断爬取两个经纬度之间的道路距离数据。

代码中用到了lock锁便于保证共享数据写入时不会发生错乱，但是具体是否加上锁就一定能保证按照想要的去写入还不确定，还需要进一步证实数据写入方式是否会引起错乱、对不齐的情况。

函数get_road_distance中有list1=[ ]，可能是导致数据写入保持有序的原因。

从这个代码来看的话写多进程也不算太难，主要就是代码同时运行的时候，如何保证数据的写入方式是我们想要的那样才是重点。