无其他依赖,本地GPU,本地模型
多本地测试数据集支持
3090 用时6分钟
创建环境
conda env list
conda create --name opencompass python=3.10
conda activate opencompass
克隆代码
git clone -b 0.2.4 https://github.com/open-compass/opencompass
cd opencompass
拉取数据
opencompass目录下
wget https://github.com/open-compass/opencompass/releases/download/0.1.8.rc1/OpenCompassData-core-20231110.zip
解压
unzip OpenCompassData-core-20231110.zip
安装依赖
apt-get update
apt-get install cmake
pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
pip install protobuf -i https://pypi.tuna.tsinghua.edu.cn/simple
pip install -e .
下载测评模型
export HF_ENDPOINT=https://hf-mirror.com
huggingface-cli download --resume-download internlm/internlm2-chat-1_8b
huggingface-cli download --resume-download Qwen/Qwen2-7B-Instruct
执行测评
export MKL_SERVICE_FORCE_INTEL=1
python run.py --datasets ceval_gen --models hf_internlm2_chat_1_8b --debug
测试时间
显卡:3090
模型:internlm/internlm2-chat-1_8b
时间:6分钟左右
测试结果
dataset version metric mode internlm2-chat-1.8b-hf
---------------------------------------------- --------- ------------- ------ ------------------------
ceval-computer_network db9ce2 accuracy gen 31.58
ceval-operating_system 1c2571 accuracy gen 47.37
ceval-computer_architecture a74dad accuracy gen 19.05
ceval-college_programming 4ca32a accuracy gen 35.14
ceval-college_physics 963fa8 accuracy gen 31.58
ceval-college_chemistry e78857 accuracy gen 37.5
ceval-advanced_mathematics ce03e2 accuracy gen 26.32
ceval-probability_and_statistics 65e812 accuracy gen 44.44
ceval-discrete_mathematics e894ae accuracy gen 37.5
ceval-electrical_engineer ae42b9 accuracy gen 32.43
ceval-metrology_engineer ee34ea accuracy gen 58.33
ceval-high_school_mathematics 1dc5bf accuracy gen 11.11
ceval-high_school_physics adf25f accuracy gen 42.11
ceval-high_school_chemistry 2ed27f accuracy gen 52.63
ceval-high_school_biology 8e2b9a accuracy gen 31.58
ceval-middle_school_mathematics bee8d5 accuracy gen 26.32
ceval-middle_school_biology 86817c accuracy gen 76.19
ceval-middle_school_physics 8accf6 accuracy gen 57.89
ceval-middle_school_chemistry 167a15 accuracy gen 80
ceval-veterinary_medicine b4e08d accuracy gen 60.87
ceval-college_economics f3f4e6 accuracy gen 38.18
ceval-business_administration c1614e accuracy gen 33.33
ceval-marxism cf874c accuracy gen 73.68
ceval-mao_zedong_thought 51c7a4 accuracy gen 66.67
ceval-education_science 591fee accuracy gen 55.17
ceval-teacher_qualification 4e4ced accuracy gen 63.64
ceval-high_school_politics 5c0de2 accuracy gen 47.37
ceval-high_school_geography 865461 accuracy gen 47.37
ceval-middle_school_politics 5be3e7 accuracy gen 80.95
ceval-middle_school_geography 8a63be accuracy gen 83.33
ceval-modern_chinese_history fc01af accuracy gen 56.52
ceval-ideological_and_moral_cultivation a2aa4a accuracy gen 78.95
ceval-logic f5b022 accuracy gen 54.55
ceval-law a110a1 accuracy gen 29.17
ceval-chinese_language_and_literature 0f8b68 accuracy gen 39.13
ceval-art_studies 2a1300 accuracy gen 54.55
ceval-professional_tour_guide 4e673e accuracy gen 62.07
ceval-legal_professional ce8787 accuracy gen 52.17
ceval-high_school_chinese 315705 accuracy gen 42.11
ceval-high_school_history 7eb30a accuracy gen 65
ceval-middle_school_history 48ab4a accuracy gen 86.36
ceval-civil_servant 87d061 accuracy gen 46.81
ceval-sports_science 70f27b accuracy gen 47.37
ceval-plant_protection 8941f9 accuracy gen 54.55
ceval-basic_medicine c409d6 accuracy gen 73.68
ceval-clinical_medicine 49e82d accuracy gen 45.45
ceval-urban_and_rural_planner 95b885 accuracy gen 41.3
ceval-accountant 002837 accuracy gen 32.65
ceval-fire_engineer bc23f5 accuracy gen 32.26
ceval-environmental_impact_assessment_engineer c64e2d accuracy gen 45.16
ceval-tax_accountant 3a5e3c accuracy gen 36.73
ceval-physician 6e277d accuracy gen 38.78
ceval-stem - naive_average gen 42
ceval-social-science - naive_average gen 58.97
ceval-humanities - naive_average gen 56.42
ceval-other - naive_average gen 44.98
ceval-hard - naive_average gen 35.4
ceval - naive_average gen 48.94```