对比实验系列：Efficientdet环境配置及训练个人数据集

一、源码下载

可以通过下方链接下载Efficientdet源码

GitHub - zylo117/Yet-Another-EfficientDet-Pytorch: The pytorch re-implement of the official efficientdet with SOTA performance in real time and pretrained weights.The pytorch re-implement of the official efficientdet with SOTA performance in real time and pretrained weights. - zylo117/Yet-Another-EfficientDet-Pytorchhttps://github.com/zylo117/Yet-Another-EfficientDet-Pytorch/tree/master

二、环境配置

1、使用anaconda创建环境

conda create -n efficient python==3.8 -y

2、进入环境

conda activate efficient

3、安装依赖包

pip install pycocotools-windows numpy opencv-python tqdm tensorboard tensorboardX pyyaml webcolors

4、安装torch和torchvision

pip install torch==1.9.1+cu111 torchvision==0.10.1+cu111 -f https://download.pytorch.org/whl/torch_stable.html

5、测试配置

5.1预训练权重下载

在源码下载的地方将预训练权重下载下来，这里选择的是Efficientdet-D1版本的预训练权重

在根目录下创建weights文件夹，将预训练权重放在文件夹内

5.2修改参数

在efficientdet_test.py中修改测试图片路径和efficientdet的版本，这里我下的版本是D1，所以就要将compound_coef改为1，如果你下的是其他版本，就将compound_coef改成对应的版本数字。

测试图片官方已经放在test\img.png，修改完之后运行efficientdet_test.py，检测结果保存在test文件夹内。

至此，efficientdet的所有环境配置结束。

三、自己的数据集制作

由于efficientdet使用的是coco格式的数据集，需要将标签转为coco格式，如果使用labelme进行标注，可以直接在labelme中切换成json格式的标签，最后用脚本进行合并，本文介绍的是使用VOC格式（xml）如何转为coco格式数据集。

1、划分验证集和训练集

使用下面的脚本将训练集进行划分：

# 将标签格式为xml的数据集按照8：2的比例划分为训练集和验证集import os
import shutil
import random
from tqdm import tqdmdef split_img(img_path, label_path, split_list):try:  # 创建数据集文件夹Data = 'yourdatasetsname'os.mkdir(Data)train_img_dir = Data + '/images/train'val_img_dir = Data + '/images/val'# test_img_dir = Data + '/images/test'train_label_dir = Data + '/labels/train'val_label_dir = Data + '/labels/val'# test_label_dir = Data + '/labels/test'# 创建文件夹os.makedirs(train_img_dir)os.makedirs(train_label_dir)os.makedirs(val_img_dir)os.makedirs(val_label_dir)# os.makedirs(test_img_dir)# os.makedirs(test_label_dir)except:print('文件目录已存在')train, val = split_listall_img = os.listdir(img_path)all_img_path = [os.path.join(img_path, img) for img in all_img]# all_label = os.listdir(label_path)# all_label_path = [os.path.join(label_path, label) for label in all_label]train_img = random.sample(all_img_path, int(train * len(all_img_path)))train_img_copy = [os.path.join(train_img_dir, img.split('\\')[-1]) for img in train_img]train_label = [toLabelPath(img, label_path) for img in train_img]train_label_copy = [os.path.join(train_label_dir, label.split('\\')[-1]) for label in train_label]for i in tqdm(range(len(train_img)), desc='train2007 ', ncols=80, unit='img'):_copy(train_img[i], train_img_dir)_copy(train_label[i], train_label_dir)all_img_path.remove(train_img[i])val_img = all_img_pathval_label = [toLabelPath(img, label_path) for img in val_img]for i in tqdm(range(len(val_img)), desc='test2007 ', ncols=80, unit='img'):_copy(val_img[i], val_img_dir)_copy(val_label[i], val_label_dir)def _copy(from_path, to_path):shutil.copy(from_path, to_path)def toLabelPath(img_path, label_path):img = img_path.split('\\')[-1]label = img.split('.jpg')[0] + '.xml'return os.path.join(label_path, label)def main():img_path = '/path/to/your/image/folder'label_path = '/path/to/your/xml/folder'split_list = [0.8, 0.2]  # 数据集划分比例[train2007:test2007]split_img(img_path, label_path, split_list)if __name__ == '__main__':main()

2、移动图片

在数据集根目录下创建datasets文件夹，并创建coco文件夹，在coco文件夹内创建annotations、train2017和val2017文件夹。将上一步划分好的训练集图片放入trian2017文件夹内，验证集图片放入val2017文件夹内。

3、生成json文件

使用下方的脚本分别对验证集和训练集的所有xml文件转为json文件，结果会生成两个json文件。

import xml.etree.ElementTree as ET
import os
import jsoncoco = dict()
coco['images'] = []
coco['type'] = 'instances'
coco['annotations'] = []
coco['categories'] = []category_set = dict()
image_set = set()# category_item_id = -1
# VOC数据集的类别id与coco数据集一样都是从1开始，如果初始设为-1，那么转出来的coco的json文件中category_id和类别id会从0开始，不符合coco标准，在调coco.py的时候会报错list越界
category_item_id = 0
image_id = 20180000000
annotation_id = 0def addCatItem(name):global category_item_idcategory_item = dict()category_item['supercategory'] = 'none'category_item_id += 1category_item['id'] = category_item_idcategory_item['name'] = namecoco['categories'].append(category_item)category_set[name] = category_item_idreturn category_item_iddef addImgItem(file_name, size):global image_idif file_name is None:raise Exception('Could not find filename tag in xml file.')if size['width'] is None:raise Exception('Could not find width tag in xml file.')if size['height'] is None:raise Exception('Could not find height tag in xml file.')image_id += 1image_item = dict()image_item['id'] = image_idimage_item['file_name'] = file_nameimage_item['width'] = size['width']image_item['height'] = size['height']coco['images'].append(image_item)image_set.add(file_name)return image_iddef addAnnoItem(object_name, image_id, category_id, bbox):global annotation_idannotation_item = dict()annotation_item['segmentation'] = []seg = []# bbox[] is x,y,w,h# left_topseg.append(bbox[0])seg.append(bbox[1])# left_bottomseg.append(bbox[0])seg.append(bbox[1] + bbox[3])# right_bottomseg.append(bbox[0] + bbox[2])seg.append(bbox[1] + bbox[3])# right_topseg.append(bbox[0] + bbox[2])seg.append(bbox[1])annotation_item['segmentation'].append(seg)annotation_item['area'] = bbox[2] * bbox[3]annotation_item['iscrowd'] = 0annotation_item['ignore'] = 0annotation_item['image_id'] = image_idannotation_item['bbox'] = bboxannotation_item['category_id'] = category_idannotation_id += 1annotation_item['id'] = annotation_idcoco['annotations'].append(annotation_item)def _read_image_ids(image_sets_file):ids = []with open(image_sets_file) as f:for line in f:ids.append(line.rstrip())return ids"""通过txt文件生成"""
#split ='train' 'va' 'trainval' 'test'
def parseXmlFiles_by_txt(data_dir,json_save_path,split='train'):print("hello")labelfile=split+".txt"image_sets_file = data_dir + "/ImageSets/Main/"+labelfileids=_read_image_ids(image_sets_file)for _id in ids:xml_file=data_dir + f"/Annotations/{_id}.xml"bndbox = dict()size = dict()current_image_id = Nonecurrent_category_id = Nonefile_name = Nonesize['width'] = Nonesize['height'] = Nonesize['depth'] = Nonetree = ET.parse(xml_file)root = tree.getroot()if root.tag != 'annotation':raise Exception('pascal voc xml root element should be annotation, rather than {}'.format(root.tag))# elem is <folder>, <filename>, <size>, <object>for elem in root:current_parent = elem.tagcurrent_sub = Noneobject_name = Noneif elem.tag == 'folder':continueif elem.tag == 'filename':file_name = elem.textif file_name in category_set:raise Exception('file_name duplicated')# add img item only after parse <size> tagelif current_image_id is None and file_name is not None and size['width'] is not None:if file_name not in image_set:current_image_id = addImgItem(file_name, size)print('add image with {} and {}'.format(file_name, size))else:raise Exception('duplicated image: {}'.format(file_name))# subelem is <width>, <height>, <depth>, <name>, <bndbox>for subelem in elem:bndbox['xmin'] = Nonebndbox['xmax'] = Nonebndbox['ymin'] = Nonebndbox['ymax'] = Nonecurrent_sub = subelem.tagif current_parent == 'object' and subelem.tag == 'name':object_name = subelem.textif object_name not in category_set:current_category_id = addCatItem(object_name)else:current_category_id = category_set[object_name]elif current_parent == 'size':if size[subelem.tag] is not None:raise Exception('xml structure broken at size tag.')size[subelem.tag] = int(subelem.text)# option is <xmin>, <ymin>, <xmax>, <ymax>, when subelem is <bndbox>for option in subelem:if current_sub == 'bndbox':if bndbox[option.tag] is not None:raise Exception('xml structure corrupted at bndbox tag.')bndbox[option.tag] = int(option.text)# only after parse the <object> tagif bndbox['xmin'] is not None:if object_name is None:raise Exception('xml structure broken at bndbox tag')if current_image_id is None:raise Exception('xml structure broken at bndbox tag')if current_category_id is None:raise Exception('xml structure broken at bndbox tag')bbox = []# xbbox.append(bndbox['xmin'])# ybbox.append(bndbox['ymin'])# wbbox.append(bndbox['xmax'] - bndbox['xmin'])# hbbox.append(bndbox['ymax'] - bndbox['ymin'])print('add annotation with {},{},{},{}'.format(object_name, current_image_id, current_category_id,bbox))addAnnoItem(object_name, current_image_id, current_category_id, bbox)json.dump(coco, open(json_save_path, 'w'))"""直接从xml文件夹中生成"""
def parseXmlFiles(xml_path,json_save_path):for f in os.listdir(xml_path):if not f.endswith('.xml'):continue# print(path)bndbox = dict()size = dict()current_image_id = Nonecurrent_category_id = Nonefile_name = Nonesize['width'] = Nonesize['height'] = Nonesize['depth'] = Nonexml_file = os.path.join(xml_path, f)print(xml_file)tree = ET.parse(xml_file)root = tree.getroot()if root.tag != 'annotation':raise Exception('pascal voc xml root element should be annotation, rather than {}'.format(root.tag))# elem is <folder>, <filename>, <size>, <object>for elem in root:current_parent = elem.tagcurrent_sub = Noneobject_name = Noneif elem.tag == 'folder':continueif elem.tag == 'filename':# 改成与xml文件同名的图片文件file_name = f.split(".")[0] + ".jpg"# file_name = elem.text# if file_name in category_set:#     raise Exception('file_name duplicated')# add img item only after parse <size> tagelif current_image_id is None and file_name is not None and size['width'] is not None:if file_name not in image_set:current_image_id = addImgItem(file_name, size)print('add image with {} and {}'.format(file_name, size))else:raise Exception('duplicated image: {}'.format(file_name))# subelem is <width>, <height>, <depth>, <name>, <bndbox>for subelem in elem:bndbox['xmin'] = Nonebndbox['xmax'] = Nonebndbox['ymin'] = Nonebndbox['ymax'] = Nonecurrent_sub = subelem.tagif current_parent == 'object' and subelem.tag == 'name':object_name = subelem.textif object_name not in category_set:current_category_id = addCatItem(object_name)else:current_category_id = category_set[object_name]elif current_parent == 'size':if size[subelem.tag] is not None:raise Exception('xml structure broken at size tag.')size[subelem.tag] = int(subelem.text)# option is <xmin>, <ymin>, <xmax>, <ymax>, when subelem is <bndbox>for option in subelem:if current_sub == 'bndbox':if bndbox[option.tag] is not None:raise Exception('xml structure corrupted at bndbox tag.')bndbox[option.tag] = int(option.text)# only after parse the <object> tagif bndbox['xmin'] is not None:if object_name is None:raise Exception('xml structure broken at bndbox tag')if current_image_id is None:raise Exception('xml structure broken at bndbox tag')if current_category_id is None:raise Exception('xml structure broken at bndbox tag')bbox = []# xbbox.append(bndbox['xmin'])# ybbox.append(bndbox['ymin'])# wbbox.append(bndbox['xmax'] - bndbox['xmin'])# hbbox.append(bndbox['ymax'] - bndbox['ymin'])print('add annotation with {},{},{},{}'.format(object_name, current_image_id, current_category_id,bbox))addAnnoItem(object_name, current_image_id, current_category_id, bbox)json.dump(coco, open(json_save_path, 'w'))if __name__ == '__main__':#通过txt文件生成# voc_data_dir="E:/VOCdevkit/VOC2007"# json_save_path="E:/VOCdevkit/voc2007trainval.json"# parseXmlFiles_by_txt(voc_data_dir,json_save_path,"trainval")#通过文件夹生成ann_path='/path/to/your/xml/folder'json_save_path="instances_val.json"parseXmlFiles(ann_path,json_save_path)