#!/usr/bin/env python # -*- coding: utf-8 -*- # # Copyright 2019 The FATE Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import json import os import subprocess import sys import time # from examples.load_file.load_file import load_file home_dir = os.path.split(os.path.realpath(__file__))[0] config_path = home_dir + '/conf' data_path = home_dir + '/../data' load_file_program = home_dir + '/../load_file/load_file.py' # data_set = 'breast' data_set = 'default_credit' # data_set = 'give_credit' mode = 'fit' # mode = 'transform' def make_config_file(work_mode, job_id, role, guest_partyid, host_partyid): with open(config_path + '/{}_runtime_conf.json'.format(role), 'r', encoding='utf-8') as load_f: role_config = json.load(load_f) if role == 'guest': role_config['local']['party_id'] = guest_partyid data_suffix = 'b' else: role_config['local']['party_id'] = host_partyid data_suffix = 'a' role_config['role']['host'][0] = host_partyid role_config['role']['guest'][0] = guest_partyid role_config['WorkFlowParam']['work_mode'] = int(work_mode) # role_config['FeatureSelectionParam']['method'] = mode role_config['WorkFlowParam']['train_input_table'] = "{}_{}_{}".format(data_set, role, job_id) # guest_config_path = config_path + '/guest_runtime_conf.json_' + str(job_id) role_config_path = "{}/{}_runtime_conf.json_{}".format( config_path, role, job_id ) with open(role_config_path, 'w', encoding='utf-8') as json_file: json.dump(role_config, json_file, ensure_ascii=False) with open(config_path + '/load_file.json', 'r', encoding='utf-8') as load_f: load_config = json.load(load_f) load_config['work_mode'] = work_mode # load_config['file'] = data_path + '/' + data_set + '_b.csv' load_config['file'] = "{}/{}_{}.csv".format(data_path, data_set, data_suffix) load_config['table_name'] = "{}_{}_{}".format(data_set, role, job_id) # load_file_role = config_path + '/load_file.json_guest_' + str(job_id) load_file_path = "{}/load_file.json_{}_{}".format(config_path, role, job_id) with open(load_file_path, 'w', encoding='utf-8') as json_file: json.dump(load_config, json_file, ensure_ascii=False) return role_config_path, load_file_path def attach_all_result_log(running_process, job_id, role): while True: poll = running_process.poll() if poll is None: print('Workflow is still running') time.sleep(5) else: break log_dir = home_dir + '/../../logs/{}'.format(job_id) role = role.lower() if role == 'guest': role_list = ['Guest', 'guest', 'GUEST'] else: role_list = ['Host', 'host', 'HOST'] result_log_list = [] for log_file in os.listdir(log_dir): # print('current log_file: {}'.format(log_file)) with open(log_dir + '/' + log_file, 'r', encoding='utf-8') as f: for log_line in f: if 'Result' in log_line: for r in role_list: if r in log_line: result_log_list.append(log_line) break log_list = sorted(result_log_list, key=get_time) for log_line in log_list: print(log_line) def get_time(log_line): time_str = log_line.split(' - ')[0] time_str = time_str[1:] time_str.strip() time_sec, time_mili_sec = time_str.split(',') timestamp = time.mktime(time.strptime(time_sec, '%Y-%m-%d %H:%M:%S')) timestamp += float(time_mili_sec) / 1000 return timestamp def load_file(load_file_path): load_process = subprocess.Popen(["python", load_file_program, "-c", load_file_path, ]) # load_process.communicate() returncode = load_process.wait() print("Load file return code : {}".format(returncode)) if __name__ == '__main__': work_mode = int(sys.argv[1]) jobid = sys.argv[2] role = sys.argv[3] guest_partyid = int(sys.argv[4]) host_partyid = int(sys.argv[5]) role_config_path, load_file_path = make_config_file(work_mode, jobid, role, guest_partyid, host_partyid) load_file(load_file_path) work_path = home_dir + '/../../workflow/hetero_feature_selection_workflow/' \ 'hetero_feature_selection_{}_workflow.py'.format(role) p = subprocess.Popen(["python", work_path, "-c", role_config_path, "-j", jobid ]) attach_all_result_log(p, jobid, role)