def keypoint_error(
df_error: pd.DataFrame,
df_error_p_cutoff: pd.DataFrame,
train_indices: List[int],
test_indices: List[int],
) -> pd.DataFrame:
df_error = df_error.copy()
df_error_p_cutoff = df_error_p_cutoff.copy()
error_rows = []
for row_name, df in [
("Train error (px)", df_error.iloc[train_indices, :]),
("Test error (px)", df_error.iloc[test_indices, :]),
("Train error (px) with p-cutoff", df_error_p_cutoff.iloc[train_indices, :]),
("Test error (px) with p-cutoff", df_error_p_cutoff.iloc[test_indices, :]),
]:
df_flat = df.copy()
if isinstance(df.columns, pd.MultiIndex):
# MA projects have column indices "scorer", "individuals" and "bodyparts"
# Drop the scorer level, and put individuals in rows
df_flat = df.droplevel("scorer", axis=1).stack(level="individuals").copy()
bodypart_error = df_flat.mean()
bodypart_error["Error Type"] = row_name
error_rows.append(bodypart_error)
# The error rows are series; stack in axis 1 and pivot to get DF
keypoint_error_df = pd.concat(error_rows, axis=1)
return keypoint_error_df.T.set_index("Error Type")
def evaluate_network(
config,
Shuffles=[1],
trainingsetindex=0,
plotting=False,
show_errors=True,
comparisonbodyparts="all",
gputouse=None,
rescale=False,
modelprefix="",
per_keypoint_evaluation: bool = False,
):
if plotting not in (True, False, "bodypart", "individual"):
raise ValueError(f"Unknown value for `plotting`={plotting}")
import os
start_path = os.getcwd()
from deeplabcut.utils import auxiliaryfunctions
cfg = auxiliaryfunctions.read_config(config)
if cfg.get("multianimalproject", False):
from .evaluate_multianimal import evaluate_multianimal_full
# TODO: Make this code not so redundant!
evaluate_multianimal_full(
config=config,
Shuffles=Shuffles,
trainingsetindex=trainingsetindex,
plotting=plotting,
comparisonbodyparts=comparisonbodyparts,
gputouse=gputouse,
modelprefix=modelprefix,
per_keypoint_evaluation=per_keypoint_evaluation,
)
else:
from deeplabcut.utils.auxfun_videos import imread, imresize
from deeplabcut.pose_estimation_tensorflow.core import predict
from deeplabcut.pose_estimation_tensorflow.config import load_config
from deeplabcut.pose_estimation_tensorflow.datasets.utils import data_to_input
from deeplabcut.utils import auxiliaryfunctions, conversioncode
import tensorflow as tf
# If a string was passed in, auto-convert to True for backward compatibility
plotting = bool(plotting)
if "TF_CUDNN_USE_AUTOTUNE" in os.environ:
del os.environ[
"TF_CUDNN_USE_AUTOTUNE"
] # was potentially set during training
tf.compat.v1.reset_default_graph()
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" #
# tf.logging.set_verbosity(tf.logging.WARN)
start_path = os.getcwd()
# Read file path for pose_config file. >> pass it on
cfg = auxiliaryfunctions.read_config(config)
if gputouse is not None: # gpu selectinon
os.environ["CUDA_VISIBLE_DEVICES"] = str(gputouse)
if trainingsetindex == "all":
TrainingFractions = cfg["TrainingFraction"]
else:
if 0 <= trainingsetindex < len(cfg["TrainingFraction"]):
TrainingFractions = [cfg["TrainingFraction"][int(trainingsetindex)]]
else:
raise Exception(
"Please check the trainingsetindex! ",
trainingsetindex,
" should be an integer from 0 .. ",
int(len(cfg["TrainingFraction"]) - 1),
)
# Loading human annotatated data
trainingsetfolder = auxiliaryfunctions.get_training_set_folder(cfg)
Data = pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
)
)
# Get list of body parts to evaluate network for
comparisonbodyparts = (
auxiliaryfunctions.intersection_of_body_parts_and_ones_given_by_user(
cfg, comparisonbodyparts
)
)
# Make folder for evaluation
auxiliaryfunctions.attempt_to_make_folder(
str(cfg["project_path"] + "/evaluation-results/")
)
for shuffle in Shuffles:
for trainFraction in TrainingFractions:
##################################################
# Load and setup CNN part detector
##################################################
modelfolder_rel_path = auxiliaryfunctions.get_model_folder(
trainFraction, shuffle, cfg, modelprefix=modelprefix
)
modelfolder = Path(cfg["project_path"]) / modelfolder_rel_path
# TODO: Unlike using create_training_dataset() If create_training_model_comparison() is used there won't
# necessarily be training fractions for every shuffle which will raise the FileNotFoundError..
# Not sure if this should throw an exception or just be a warning...
if not modelfolder.exists():
raise FileNotFoundError(
f"Model with shuffle {shuffle} and trainFraction {trainFraction} does not exist."
)
if trainingsetindex == "all":
train_frac_idx = cfg["TrainingFraction"].index(trainFraction)
else:
train_frac_idx = trainingsetindex
path_train_config, path_test_config, _ = return_train_network_path(
config=config,
shuffle=shuffle,
trainingsetindex=train_frac_idx,
modelprefix=modelprefix,
)
test_pose_cfg = load_config(str(path_test_config))
train_pose_cfg = load_config(str(path_train_config))
# Load meta data
_, trainIndices, testIndices, _ = auxiliaryfunctions.load_metadata(
Path(cfg["project_path"], train_pose_cfg["metadataset"])
)
# change batch size, if it was edited during analysis!
test_pose_cfg["batch_size"] = 1 # in case this was edited for analysis.
# Create folder structure to store results.
evaluationfolder = os.path.join(
cfg["project_path"],
str(
auxiliaryfunctions.get_evaluation_folder(
trainFraction, shuffle, cfg, modelprefix=modelprefix
)
),
)
auxiliaryfunctions.attempt_to_make_folder(
evaluationfolder, recursive=True
)
# Check which snapshots are available and sort them by # iterations
Snapshots = np.array(
[
fn.split(".")[0]
for fn in os.listdir(os.path.join(str(modelfolder), "train"))
if "index" in fn
]
)
try: # check if any where found?
Snapshots[0]
except IndexError:
raise FileNotFoundError(
"Snapshots not found! It seems the dataset for shuffle %s and trainFraction %s is not trained.\nPlease train it before evaluating.\nUse the function 'train_network' to do so."
% (shuffle, trainFraction)
)
increasing_indices = np.argsort(
[int(m.split("-")[1]) for m in Snapshots]
)
Snapshots = Snapshots[increasing_indices]
if cfg["snapshotindex"] == -1:
snapindices = [-1]
elif cfg["snapshotindex"] == "all":
snapindices = range(len(Snapshots))
elif cfg["snapshotindex"] < len(Snapshots):
snapindices = [cfg["snapshotindex"]]
else:
raise ValueError(
"Invalid choice, only -1 (last), any integer up to last, or all (as string)!"
)
final_result = []
########################### RESCALING (to global scale)
if rescale:
scale = test_pose_cfg["global_scale"]
Data = (
pd.read_hdf(
os.path.join(
cfg["project_path"],
str(trainingsetfolder),
"CollectedData_" + cfg["scorer"] + ".h5",
)
)
* scale
)
else:
scale = 1
conversioncode.guarantee_multiindex_rows(Data)
##################################################
# Compute predictions over images
##################################################
for snapindex in snapindices:
test_pose_cfg["init_weights"] = os.path.join(
str(modelfolder), "train", Snapshots[snapindex]
) # setting weights to corresponding snapshot.
trainingsiterations = (
test_pose_cfg["init_weights"].split(os.sep)[-1]
).split("-")[
-1
] # read how many training siterations that corresponds to.
# Name for deeplabcut net (based on its parameters)
DLCscorer, DLCscorerlegacy = auxiliaryfunctions.get_scorer_name(
cfg,
shuffle,
trainFraction,
trainingsiterations,
modelprefix=modelprefix,
)
print(
"Running ",
DLCscorer,
" with # of training iterations:",
trainingsiterations,
)
(
notanalyzed,
resultsfilename,
DLCscorer,
) = auxiliaryfunctions.check_if_not_evaluated(
str(evaluationfolder),
DLCscorer,
DLCscorerlegacy,
Snapshots[snapindex],
)
if notanalyzed:
# Specifying state of model (snapshot / training state)
sess, inputs, outputs = predict.setup_pose_prediction(
test_pose_cfg
)
Numimages = len(Data.index)
PredicteData = np.zeros(
(Numimages, 3 * len(test_pose_cfg["all_joints_names"]))
)
print("Running evaluation ...")
for imageindex, imagename in tqdm(enumerate(Data.index)):
image = imread(
os.path.join(cfg["project_path"], *imagename),
mode="skimage",
)
if scale != 1:
image = imresize(image, scale)
image_batch = data_to_input(image)
# Compute prediction with the CNN
outputs_np = sess.run(
outputs, feed_dict={inputs: image_batch}
)
scmap, locref = predict.extract_cnn_output(
outputs_np, test_pose_cfg
)
# Extract maximum scoring location from the heatmap, assume 1 person
pose = predict.argmax_pose_predict(
scmap, locref, test_pose_cfg["stride"]
)
PredicteData[
imageindex, :
] = (
pose.flatten()
) # NOTE: thereby cfg_test['all_joints_names'] should be same order as bodyparts!
sess.close() # closes the current tf session
index = pd.MultiIndex.from_product(
[
[DLCscorer],
test_pose_cfg["all_joints_names"],
["x", "y", "likelihood"],
],
names=["scorer", "bodyparts", "coords"],
)
# Saving results
DataMachine = pd.DataFrame(
PredicteData, columns=index, index=Data.index
)
DataMachine.to_hdf(resultsfilename, "df_with_missing")
print(
"Analysis is done and the results are stored (see evaluation-results) for snapshot: ",
Snapshots[snapindex],
)
DataCombined = pd.concat(
[Data.T, DataMachine.T], axis=0, sort=False
).T
RMSE, RMSEpcutoff = pairwisedistances(
DataCombined,
cfg["scorer"],
DLCscorer,
cfg["pcutoff"],
comparisonbodyparts,
)
testerror = np.nanmean(RMSE.iloc[testIndices].values.flatten())
trainerror = np.nanmean(
RMSE.iloc[trainIndices].values.flatten()
)
testerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[testIndices].values.flatten()
)
trainerrorpcutoff = np.nanmean(
RMSEpcutoff.iloc[trainIndices].values.flatten()
)
results = [
trainingsiterations,
int(100 * trainFraction),
shuffle,
np.round(trainerror, 2),
np.round(testerror, 2),
cfg["pcutoff"],
np.round(trainerrorpcutoff, 2),
np.round(testerrorpcutoff, 2),
]
final_result.append(results)
if per_keypoint_evaluation:
df_keypoint_error = keypoint_error(
RMSE, RMSEpcutoff, trainIndices, testIndices
)
kpt_filename = DLCscorer + "-keypoint-results.csv"
df_keypoint_error.to_csv(
Path(evaluationfolder) / kpt_filename
)
if show_errors:
print(
"Results for",
trainingsiterations,
" training iterations:",
int(100 * trainFraction),
shuffle,
"train error:",
np.round(trainerror, 2),
"pixels. Test error:",
np.round(testerror, 2),
" pixels.",
)
print(
"With pcutoff of",
cfg["pcutoff"],
" train error:",
np.round(trainerrorpcutoff, 2),
"pixels. Test error:",
np.round(testerrorpcutoff, 2),
"pixels",
)
if scale != 1:
print(
"The predictions have been calculated for rescaled images (and rescaled ground truth). Scale:",
scale,
)
print(
"Thereby, the errors are given by the average distances between the labels by DLC and the scorer."
)
if plotting:
print("Plotting...")
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_"
+ DLCscorer
+ "_"
+ Snapshots[snapindex],
)
auxiliaryfunctions.attempt_to_make_folder(foldername)
Plotting(
cfg,
comparisonbodyparts,
DLCscorer,
trainIndices,
DataCombined * 1.0 / scale,
foldername,
) # Rescaling coordinates to have figure in original size!
tf.compat.v1.reset_default_graph()
# print(final_result)
else:
DataMachine = pd.read_hdf(resultsfilename)
conversioncode.guarantee_multiindex_rows(DataMachine)
if plotting:
DataCombined = pd.concat(
[Data.T, DataMachine.T], axis=0, sort=False
).T
foldername = os.path.join(
str(evaluationfolder),
"LabeledImages_"
+ DLCscorer
+ "_"
+ Snapshots[snapindex],
)
if not os.path.exists(foldername):
print(
"Plotting...(attention scale might be inconsistent in comparison to when data was analyzed; i.e. if you used rescale)"
)
auxiliaryfunctions.attempt_to_make_folder(foldername)
Plotting(
cfg,
comparisonbodyparts,
DLCscorer,
trainIndices,
DataCombined * 1.0 / scale,
foldername,
)
else:
print(
"Plots already exist for this snapshot... Skipping to the next one."
)
if len(final_result) > 0: # Only append if results were calculated
make_results_file(final_result, evaluationfolder, DLCscorer)
print(
"The network is evaluated and the results are stored in the subdirectory 'evaluation_results'."
)
print(
"Please check the results, then choose the best model (snapshot) for prediction. You can update the config.yaml file with the appropriate index for the 'snapshotindex'.\nUse the function 'analyze_video' to make predictions on new videos."
)
print(
"Otherwise, consider adding more labeled-data and retraining the network (see DeepLabCut workflow Fig 2, Nath 2019)"
)
# returning to initial folder
os.chdir(str(start_path))
1万+
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
