defines a function for chunking data for multiprocessing
# function that splits a list into n chunks for multiprocessing
def chunk(file_list, n_chunks):
# make chunks of files to be distributed across processes
chunks = []
chunk_size = math.ceil(float(len(file_list))/n_chunks)
for i in range(0, n_chunks-1):
chunks.append(file_list[i*chunk_size:(i+1)*chunk_size])
chunks.append(file_list[(n_chunks-1)*chunk_size:])
return chunks
从网上下载资源文件
来自facebook fvcore download.py
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
import logging
import os
import shutil
from typing import Callable, List, Optional
from urllib import request
def download(
url: str, dir: str, *, filename: Optional[str] = None, progress: bool = True
) -> str:
"""
Download a file from a given URL to a directory. If file exists, will not
overwrite the existing file.
Args:
url (str):
dir (str): the directory to download the file
filename (str or None): the basename to save the file.
Will use the name in the URL if not given.
progress (bool): whether to use tqdm to draw a progress bar.
Returns:
str: the path to the downloaded file or the existing one.
"""
os.makedirs(dir, exist_ok=True)
if filename is None:
filename = url.split("/")[-1]
assert len(filename), "Cannot obtain filename from url {}".format(url)
fpath = os.path.join(dir, filename)
logger = logging.getLogger(__name__)
if os.path.isfile(fpath):
logger.info("File {} exists! Skipping download.".format(filename))
return fpath
tmp = fpath + ".tmp" # download to a tmp file first, to be more atomic.
try:
logger.info("Downloading from {} ...".format(url))
if progress:
import tqdm
def hook(t: tqdm.tqdm) -> Callable[[int, int, Optional[int]], None]:
last_b: List[int] = [0]
def inner(b: int, bsize: int, tsize: Optional[int] = None) -> None:
if tsize is not None:
t.total = tsize
t.update((b - last_b[0]) * bsize) # type: ignore
last_b[0] = b
return inner
with tqdm.tqdm( # type: ignore
unit="B", unit_scale=True, miniters=1, desc=filename, leave=True
) as t:
tmp, _ = request.urlretrieve(url, filename=tmp, reporthook=hook(t))
else:
tmp, _ = request.urlretrieve(url, filename=tmp)
statinfo = os.stat(tmp)
size = statinfo.st_size
if size == 0:
raise IOError("Downloaded an empty file from {}!".format(url))
# download to tmp first and move to fpath, to make this function more
# atomic.
shutil.move(tmp, fpath)
except IOError:
logger.error("Failed to download {}".format(url))
raise
finally:
try:
os.unlink(tmp)
except IOError:
pass
logger.info("Successfully downloaded " + fpath + ". " + str(size) + " bytes.")
return fpath
if __name__ == '__main__':
download(
url = pdf_url,
dir=self.pdf_path,
filename=None,
progress=False
)
Labelme转voc(8个关键点)
- 转换
#! /usr/bin/python import os, sys import glob import json import cv2 import numpy as np from PIL import Image if __name__ == '__main__': # source files json_files_dir = "datasets/json" anno_files_dir = "datasets/annotations" img_files_dir = "datasets/images" img_lists = glob.glob('datasets/images/*.jpg') img_basenames = [] for item in img_lists: img_basenames.append(os.path.basename(item)) img_names = [] for item in img_basenames: temp1, temp2 = os.path.splitext(item) img_names.append(temp1) for img in img_names: im = cv2.imread(img_files_dir + '/' + img + '.jpg') height, width = im.shape[:2] # open the crospronding txt file json_file = json_files_dir + '/' + img + '.json' with open(json_file, "r") as f: load_dict = json.load(f) imagePath = load_dict['imagePath'] imageWidth = load_dict['imageWidth'] imageHeight = load_dict['imageHeight'] shapes = load_dict['shapes'] # write in xml file xml_file = open((anno_files_dir + '/' + img + '.xml'), 'w') xml_file.write('<annotation>\n') xml_file.write('\t<folder>images</folder>\n') xml_file.write('\t<filename>' + str(img) + '.jpg' + '</filename>\n') xml_file.write('\t<path>' + '../images/' + str(img) + '.jpg' + '</path>\n') xml_file.write('\t<source>\n') xml_file.write('\t\t<database>' + 'OCRDatabase' + '</database>\n') xml_file.write('\t</source>\n') xml_file.write('\t<size>\n') xml_file.write('\t\t<width>' + str(imageWidth) + '</width>\n') xml_file.write('\t\t<height>' + str(imageHeight) + '</height>\n') xml_file.write('\t\t<depth>3</depth>\n') xml_file.write('\t</size>\n') xml_file.write('\t<segmented>0</segmented>\n') # write the region of text on xml file for shape in shapes: spt = shape['points'] # bounding box rect = list(np.array(spt).reshape(1, -1)[0]) xmin = int(min(rect[::2])) ymin = int(min(rect[1::2])) xmax = int(max(rect[::2])) ymax = int(max(rect[1::2])) xml_file.write('\t<object>\n') xml_file.write('\t\t<name>text</name>\n') xml_file.write('\t\t<pose>Unspecified</pose>\n') xml_file.write('\t\t<truncated>0</truncated>\n') xml_file.write('\t\t<difficult>0</difficult>\n') #bndbox xml_file.write('\t\t<bndbox>\n') xml_file.write('\t\t\t<xmin>' + str(xmin) + '</xmin>\n') xml_file.write('\t\t\t<ymin>' + str(ymin) + '</ymin>\n') xml_file.write('\t\t\t<xmax>' + str(xmax) + '</xmax>\n') xml_file.write('\t\t\t<ymax>' + str(ymax) + '</ymax>\n') xml_file.write('\t\t</bndbox>\n') # keypoint xml_file.write('\t\t<keypoints>\n') xml_file.write('\t\t\t<pt0>\n') xml_file.write('\t\t\t\t<x>' + str(int(spt[0][0])) + '</x>\n') xml_file.write('\t\t\t\t<y>' + str(int(spt[0][1])) + '</y>\n') xml_file.write('\t\t\t</pt0>\n') xml_file.write('\t\t\t<pt1>\n') xml_file.write('\t\t\t\t<x>' + str(int(spt[1][0])) + '</x>\n') xml_file.write('\t\t\t\t<y>' + str(int(spt[1][1])) + '</y>\n') xml_file.write('\t\t\t</pt1>\n') xml_file.write('\t\t\t<pt2>\n') xml_file.write('\t\t\t\t<x>' + str(int(spt[2][0])) + '</x>\n') xml_file.write('\t\t\t\t<y>' + str(int(spt[2][1])) + '</y>\n') xml_file.write('\t\t\t</pt2>\n') xml_file.write('\t\t\t<pt3>\n') xml_file.write('\t\t\t\t<x>' + str(int(spt[3][0])) + '</x>\n') xml_file.write('\t\t\t\t<y>' + str(int(spt[3][1])) + '</y>\n') xml_file.write('\t\t\t</pt3>\n') xml_file.write('\t\t\t<pt4>\n') xml_file.write('\t\t\t\t<x>' + str(int(spt[4][0])) + '</x>\n') xml_file.write('\t\t\t\t<y>' + str(int(spt[4][1])) + '</y>\n') xml_file.write('\t\t\t</pt4>\n') xml_file.write('\t\t\t<pt5>\n') xml_file.write('\t\t\t\t<x>' + str(int(spt[5][0])) + '</x>\n') xml_file.write('\t\t\t\t<y>' + str(int(spt[5][1])) + '</y>\n') xml_file.write('\t\t\t</pt5>\n') xml_file.write('\t\t\t<pt6>\n') xml_file.write('\t\t\t\t<x>' + str(int(spt[6][0])) + '</x>\n') xml_file.write('\t\t\t\t<y>' + str(int(spt[6][1])) + '</y>\n') xml_file.write('\t\t\t</pt6>\n') xml_file.write('\t\t\t<pt7>\n') xml_file.write('\t\t\t\t<x>' + str(int(spt[7][0])) + '</x>\n') xml_file.write('\t\t\t\t<y>' + str(int(spt[7][1])) + '</y>\n') xml_file.write('\t\t\t</pt7>\n') xml_file.write('\t\t</keypoints>\n') xml_file.write('\t</object>\n') xml_file.write('</annotation>') xml_file.close() print(f"=========== convert done ===========") - 验证
#! /usr/bin/python import os, sys import glob import json import cv2 import numpy as np import xml.etree.ElementTree as ET colors = [ (0, 0, 255), (0,255, 0), (255, 0, 0), (0, 255, 255), (255, 0, 255), (255, 255, 0), (0, 255, 255), (0, 255, 255) ] def get_bndboxs_and_keypoints(xml_file): tree = ET.parse(xml_file) root = tree.getroot() bndboxs = [] keypoints = [] for obj in root.findall('object'): bndbox = obj.find('bndbox') xmin = int(bndbox[0].text) ymin = int(bndbox[1].text) xmax = int(bndbox[2].text) ymax = int(bndbox[3].text) bndboxs.append([xmin, ymin, xmax, ymax]) p = [] points = obj.find('keypoints') for point in points: x = int(point[0].text) y = int(point[1].text) p.append([x, y]) keypoints.append(p) return bndboxs, keypoints if __name__ == '__main__': # source files json_files_dir = "datasets/json" img_files_dir = "datasets/images" anno_files_dir = "datasets/annotations" img_lists = glob.glob('datasets/images/*.jpg') img_basenames = [] for item in img_lists: img_basenames.append(os.path.basename(item)) img_names = [] for item in img_basenames: temp1, temp2 = os.path.splitext(item) img_names.append(temp1) for i in range(len(img_names)): img_file = img_files_dir + '/' + img_names[i] + '.jpg' xml_file = anno_files_dir + '/' + img_names[i] + '.xml' print(img_file) bndboxs, keypoints = get_bndboxs_and_keypoints(xml_file) print(bndboxs) print(keypoints) img = cv2.imread(img_file) print(f"img.shape:{img.shape}") for bbox in bndboxs: cv2.rectangle(img, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 0, 255), 1) for points in keypoints: for ii in range(len(points)): cv2.circle(img, (points[i][0], points[i][1]), 5, colors[ii], -1) cv2.imwrite("ddd.jpg", img) break print(f"=========== convert done ===========")
2707
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
