1 敏感性以及特异性定义
https://en.wikipedia.org/wiki/Sensitivity_and_specificity
2 zhang博客讲的挺好的,但是有错误,还是可以参考
http://blog.csdn.net/abcjennifer/article/details/7359370
3 precision以及recall定义
https://www.douban.com/note/284051363/
3 代码以及画roc曲线的图
参考:
https://stackoverflow.com/questions/25009284/how-to-plot-roc-curve-in-python
import pandas as pd
pred = pd.read_csv('prediction.csv')
gt = pd.read_csv('stage2_solution.csv')
pred_data = pred.values
gt_data = gt.values
gt_d = gt_data[:,0:2]
diction = {}
for i in range(len(gt_d)):
# print(gt_d[i,0],gt_d[i,1])
diction[gt_d[i,0]]=gt_d[i,1]
pred_label=[]
gt_label = []
tp=0
tn=0
fp=0
fn=0
for i in range(len(pred_data)):
# print (pred_data[i,0],pred_data[i,1],diction[pred_data[i,0]])
g = diction[pred_data[i,0]]
pred_label.append(pred_data[i,1])
gt_label.append(diction[pred_data[i,0]])
if(pred_data[i,1]>0.5):
p = 1
else:
p = 0
# print('g:',g,'p:',p)
if(g==1):
if(p==1):
tp = tp+1
else:
tn = tn +1
else:
if(p==1):
fp = fp+1
else:
fn = fn+1
if(p==0 and g==1):
print (pred_data[i,0],pred_data[i,1],diction[pred_data[i,0]])
print tp,tn,fp,fn
('07625e01a28581f0d854cd53f9fd46ee', 0.117159187794, 1)
('9fb26348e55b7b1b5247e59104beec5f', 0.23055803775800002, 1)
('d63bcca00bf9df3d3aca0973aba7f22a', 0.115243196487, 1)
('678b4a8e39c885e4220f87f4fdacb18a', 0.233972370625, 1)
('1093f95635baf659ad5b3a893b14a0d9', 0.321477591991, 1)
('c8690d3efb5cbfcb9f2871fe19a3b32f', 0.037960231304199996, 1)
('9e041750951de329e7c7a20acdb9c88f', 0.215780794621, 1)
('c5f3b3390e3f873360b8f91439e0f280', 0.4399498701100001, 1)
('5b162effadb4b71d97cc48f0d3472d1f', 0.346106171608, 1)
('66dfced7056ab28e85423be7839d1d7d', 0.0265184044838, 1)
('2805ecbf568656eff77e968f27f55711', 0.207600712776, 1)
('4434c5c933b67ce29d62b2556ebd4b0d', 0.354858398438, 1)
('15e96a2ba69d4f8af4d44e75a1cfc91f', 0.336322128773, 1)
('5bcb6eda63001cfce086f1fe5de51f41', 0.0494830608368, 1)
('951cb7ff32666ac91dd95968af53e3dc', 0.042300403118099995, 1)
('a77b5a7654310f555e04a046711b81b6', 0.116569340229, 1)
('e62c65f23653e8439a5176fe778cf8b1', 0.0626381635666, 1)
('87de7ae1cbcd60a3cc98684f03ff975b', 0.271269857883, 1)
('15df31b452c0c92bef1d60fd75a99df7', 0.287601053715, 1)
('65d6bc5aaa876d81dead5d02d5638f98', 0.29096144437800003, 1)
('35a388aafa5fae8987968dfc4341ba0b', 0.22209799289700002, 1)
('82689b6006173d5c9b37bb4b5731d77d', 0.290232181549, 1)
('a59a4d0518786ac8ab13081238dd0757', 0.0694873332977, 1)
('f68e914a9970ec8ecc17b479f9c30e69', 0.300102353096, 1)
('a966293e9872a02b8011e19219cd52a2', 0.12901556491900001, 1)
('e50e25554d4aad665f9ffbc7a6605a18', 0.355409502983, 1)
('ad9877d7153ba81280cf6fda3ec258ec', 0.42044192552599996, 1)
('c69b8fbe155444085faa2d8a77fa2077', 0.24609863758099998, 1)
('ee7d43a9c214c33d1ca97b6e8b62dfed', 0.344136297703, 1)
('5a06ebc438b934a360a5e469a6874505', 0.310246169567, 1)
('d2cdfbcadca86bcb7365bc86d9c29132', 0.474076628685, 1)
('ff910f590b8d19dbc135d69a4bb6dc3e', 0.410838961601, 1)
('9f2e0344d97a73499ac3c2f364177660', 0.127251148224, 1)
('a05dae5c271ef3048b80d22408bf0d4c', 0.44878578186000007, 1)
('4eb7ea2cb6e3ebf6a9ac680665c58970', 0.426235556602, 1)
('172f016737a31607818a98360f25a0d6', 0.0955578684807, 1)
('e65babe52dbff77a3efa66066cc5d9bd', 0.180333673954, 1)
('ad677d927dbfdca141d5f5e990301867', 0.383404612541, 1)
('7513342e12f6ff612403693c7fbf3cdc', 0.22137969732299997, 1)
('4d7c36fb4b7009d4a90fc5b6ba4609c7', 0.13613379001599998, 1)
('d7fc3c406dae05d72a3d932944cd4a39', 0.320285797119, 1)
('8bb78b08c671f1c52d03d29f56997640', 0.0917291641235, 1)
('825ea4ecda33bb0a3aa86188d8deec8b', 0.0786511301994, 1)
('2503439e08a1e9232b80a625eeb26614', 0.27218401431999995, 1)
('356f21ce2359091e3adbe4dd24247420', 0.388289391994, 1)
('37f046c4e2bbc062bb103b302b22beef', 0.213571429253, 1)
('63248fdafd2df302b57a479289d76105', 0.499042093754, 1)
('f81025619a4993f4e406f3ccd07e9c0b', 0.0994927883148, 1)
('07ba80a4520fd85c686715bc0f7d7170', 0.361474454403, 1)
('5d1dd71ccc0e5c54726daa38e0ec353c', 0.0834940075874, 1)
('c65c04ff2cab8677c509abc3d14615df', 0.26668071746800004, 1)
('5514032473557df7a4a0d484d0ed27e5', 0.181709766388, 1)
('b37b42625992ced93aa5b876292419e0', 0.0973561406136, 1)
('467fbaad4184bc2a030c27351549298f', 0.0871037840843, 1)
99 54 38 315
print tp+fn,tn+fp
414 92
pred_label
[0.306648731232,
0.8760701417919999,
0.0346515774727,
0.1649710536,
0.0914691090584,
0.0446593165398,
0.07621145248410001,
0.117159187794,
0.23253923654599998,
0.23055803775800002,
0.115243196487,
0.0224041938782,
0.0458522439003,
0.299911916256,
0.704319000244,
0.43548732996000006,
0.030129075050400002,
0.233972370625,
0.0498383045197,
0.321477591991,
0.812364935875,
0.0718807578087,
0.919714152813,
0.49185758829099996,
0.026584863662700003,
0.0679003000259,
0.0449317097664,
0.0711770057678,
0.11159145832100001,
0.037960231304199996,
0.5159543752669999,
0.215780794621,
0.044772207737,
0.321922183037,
0.8048728704449999,
0.4399498701100001,
0.243216395378,
0.346106171608,
0.0257115364075,
0.614302396774,
0.923866629601,
0.121345341206,
0.0645252466202,
0.26350492239,
0.0444534420967,
0.6119161844250001,
0.0265184044838,
0.129248678684,
0.09459382295610001,
0.8202509880069999,
0.0154209733009,
0.0690587162971,
0.108752191067,
0.8403134942049999,
0.810078263283,
0.787731289864,
0.049283266067500005,
0.207600712776,
0.0320543646812,
0.04333925247190001,
0.849267244339,
0.0329706668854,
0.0446615815163,
0.0807330608368,
0.861198723316,
0.048518240451800004,
0.05440914630890001,
0.0340330004692,
0.354858398438,
0.409967780113,
0.023639500141099997,
0.939375579357,
0.059544444084199996,
0.728375673294,
0.644544184208,
0.970956742764,
0.11147278547299999,
0.031562447547900004,
0.209877669811,
0.28941488266,
0.0336276292801,
0.101174592972,
0.133744955063,
0.525971651077,
0.45949858427,
0.22605919838000002,
0.850231289864,
0.060046672821000006,
0.0673955082893,
0.040676891803699995,
0.051343739032699995,
0.336322128773,
0.789554357529,
0.0494830608368,
0.887484014034,
0.333477795124,
0.115098059177,
0.0343714952469,
0.0166711807251,
0.042300403118099995,
0.116569340229,
0.02067476511,
0.0536320209503,
0.162982702255,
0.019909024238599998,
0.019078195095099997,
0.848688483238,
0.0237035155296,
0.07821965217589999,
0.0748621821404,
0.12407445907600001,
0.0260081887245,
0.549250781536,
0.64028608799,
0.5554161071779999,
0.0626381635666,
0.337692081928,
0.166900634766,
0.7358903288840001,
0.271269857883,
0.7449672222139999,
0.754090726376,
0.036576986312899996,
0.0154871940613,
0.162472724915,
0.8283893466,
0.561336040497,
0.022459089756,
0.12467682361600001,
0.15985864400900002,
0.12295264005699999,
0.0408948659897,
0.08322954177860001,
0.0155268907547,
0.253732264042,
0.287601053715,
0.5059261918069999,
0.7878701686859999,
0.0340597033501,
0.052699804306,
0.179771304131,
0.11872249841700001,
0.29096144437800003,
0.0258894562721,
0.796294391155,
0.0793922543526,
0.20181220769900002,
0.625052571297,
0.9538311362270001,
0.7827150821690001,
0.966531693935,
0.683534145355,
0.025528848171200002,
0.398672938347,
0.052346765994999996,
0.988933742046,
0.0205284953117,
0.057099699974100006,
0.533129096031,
0.108986318111,
0.8740727305410001,
0.054328620433800004,
0.815580070019,
0.0532991290092,
0.153633058071,
0.0325220823288,
0.025053441524499998,
0.898797214031,
0.055256009101900004,
0.9507915973659999,
0.8566108345990001,
0.0243853926659,
0.22209799289700002,
0.022739171981799998,
0.910795509815,
0.169241070747,
0.0378332734108,
0.0367866158485,
0.23314499855,
0.8260697722429999,
0.0478984713554,
0.225474655628,
0.8025456666949999,
0.290232181549,
0.947477519512,
0.0706942677498,
0.0694873332977,
0.493386030197,
0.864034891129,
0.0584462285042,
0.300102353096,
0.06039792299269999,
0.301897644997,
0.7810286283490001,
0.9301441907879999,
0.99861240387,
0.12901556491900001,
0.135593295097,
0.134292602539,
0.033970952034,
0.0749914646149,
0.355409502983,
0.134282767773,
0.0326668024063,
0.6847266554829999,
0.42044192552599996,
0.032674074173,
0.960749745369,
0.24609863758099998,
0.344136297703,
0.896595299244,
0.310246169567,
0.035803914070099994,
0.37698054313699997,
0.06826096773149999,
0.053512275219000004,
0.7839746475220001,
0.010945975780499998,
0.0522285699844,
0.118953585625,
0.8401124477390001,
0.0912300944328,
0.9939808845520001,
0.05664479732509999,
0.575493991375,
0.043535888195000004,
0.869194030762,
0.119209587574,
0.061302781105,
0.193221330643,
0.11779540777200001,
0.786761283875,
0.0511956214905,
0.06733179092410001,
0.040928423404699994,
0.959544181824,
0.906318724155,
0.0444074273109,
0.0489168763161,
0.0294325351715,
0.816952228546,
0.027754008769999998,
0.956563353539,
0.672473907471,
0.267829358578,
0.9915587902069999,
0.0157701969147,
0.0807236433029,
0.277778983116,
0.0154266357422,
0.0809623003006,
0.040648162365000004,
0.652987837791,
0.137083649635,
0.0237013697624,
0.474076628685,
0.017674446106,
0.9826710820200001,
0.526813030243,
0.227248251438,
0.052958905696900004,
0.0703300833702,
0.0569689869881,
0.015280723571799998,
0.0175392627716,
0.05126118659969999,
0.083359181881,
0.66984641552,
0.043055236339600005,
0.8001756668089999,
0.104937016964,
0.0661010146141,
0.8102839589120001,
0.0547814369202,
0.107002198696,
0.709119081497,
0.555048823357,
0.022159814834599997,
0.71013122797,
0.410838961601,
0.0753583908081,
0.0222990512848,
0.639405250549,
0.054847359657300004,
0.0682095885277,
0.9155819416050001,
0.09025275707239999,
0.04651945829390001,
0.027292370796199998,
0.0821488499641,
0.07674360275270001,
0.07838195562360001,
0.0537794828415,
0.151747643948,
0.0403423905373,
0.035184204578400005,
0.047605812549599995,
0.942981719971,
0.7264840006829999,
0.0390491485596,
0.558655023575,
0.7652485370640001,
0.884135842323,
0.713637709618,
0.6731960177420001,
0.0918847322464,
0.635707080364,
0.813631594181,
0.0240622162819,
0.028998851776099996,
0.628164947033,
0.0184214711189,
0.6033244133,
0.09100937843319999,
0.903185069561,
0.7863504886630001,
0.109753608704,
0.238945126534,
0.0491129159927,
0.609125494957,
0.055827498436,
0.127251148224,
0.44878578186000007,
0.0890922546387,
0.430430710316,
0.0947884321213,
0.0657268762589,
0.426235556602,
0.0401832461357,
0.0955578684807,
0.39128816127800004,
0.18202227354000003,
0.0398209691048,
0.37694716453599997,
0.0157968401909,
0.07383424043660002,
0.042762398719800004,
0.04445165395740001,
0.04965978860859999,
0.109847545624,
0.6804876327509999,
0.506127297878,
0.0267729759216,
0.143815755844,
0.111534059048,
0.7794041633609999,
0.015767574310299997,
0.038128733634900006,
0.0511896014214,
0.180333673954,
0.702603936195,
0.103834271431,
0.159316837788,
0.383404612541,
0.0831456184387,
0.150595903397,
0.22137969732299997,
0.583897650242,
0.06986886262889999,
0.13613379001599998,
0.320285797119,
0.0450734496117,
0.621346950531,
0.0808102488518,
0.029326438903799996,
0.0917291641235,
0.0786511301994,
0.0845966339111,
0.500489234924,
0.139561355114,
0.944442391396,
0.841670095921,
0.706939518452,
0.139667630196,
0.10517472028699999,
0.8459794521330001,
0.014253795146899999,
0.128443598747,
0.0613714456558,
0.27218401431999995,
0.0393926501274,
0.0251014828682,
0.187129318714,
0.0212181806564,
0.051073253154800005,
0.722355008125,
0.0679240822792,
0.388289391994,
0.015026271343200002,
0.213571429253,
0.882094442844,
0.0288588404655,
0.0459799766541,
0.872308850288,
0.509616732597,
0.037440061569199996,
0.570501208305,
0.039659738540599995,
0.017843186855299998,
0.648983478546,
0.08827137947080001,
0.9297993183140001,
0.0254611372948,
0.021844089031199998,
0.0901657938957,
0.0737712979317,
0.612956285477,
0.0234156847,
0.067400932312,
0.159428536892,
0.0353515744209,
0.029090940952300002,
0.988542079926,
0.0594680905342,
0.9943150877949999,
0.876945018768,
0.05685520172119999,
0.041065394878400004,
0.645810365677,
0.375511944294,
0.43031364679300005,
0.375743567944,
0.145624101162,
0.0369937419891,
0.07281547784810001,
0.0326633453369,
0.194029033184,
0.499042093754,
0.0227205753326,
0.0271198749542,
0.024527013301800002,
0.0806727409363,
0.135131299496,
0.039363801479300006,
0.042473375797300005,
0.0994927883148,
0.033588647842399996,
0.13234364986400002,
0.130347132683,
0.762364864349,
0.0736729502678,
0.0309421420097,
0.10914081335100001,
0.165281057358,
0.035065472126,
0.019040286541,
0.00971162319183,
0.586828112602,
0.361474454403,
0.019100189209,
0.0812140703201,
0.961014151573,
0.539634466171,
0.0861241817474,
0.07396948337550001,
0.0709484815598,
0.18269032239900002,
0.056558668613400004,
0.0685266852379,
0.7308714389800001,
0.0642604231834,
0.920344531536,
0.0678136944771,
0.0234156847,
0.06957840919489999,
0.592350780964,
0.8675767183299999,
0.0428700447083,
0.051432013511699994,
0.0565760731697,
0.0401865839958,
0.703727483749,
0.0442606806755,
0.8268591165539999,
0.0834940075874,
0.840219736099,
0.455112397671,
0.8079274296760001,
0.021398961544,
0.0617161989212,
0.0770745873451,
0.9037966728210001,
0.07724809646610001,
0.7189496755600001,
0.056118667125699996,
0.029981791973099997,
0.541769862175,
0.027953922748599997,
0.018179237842599998,
0.774275898933,
0.0774254202843,
0.571707069874,
0.083989918232,
0.26668071746800004,
0.61296248436,
0.039847552776300005,
0.181709766388,
0.848829567432,
0.0736467838287,
0.0453742742538,
0.07220143079760001,
0.0161296129227,
0.0973561406136,
0.0580281019211,
0.0871037840843,
0.09880226850510002]
gt_label
[0,
1,
0,
0,
0,
0,
0,
1,
0,
1,
1,
0,
0,
0,
1,
0,
0,
1,
0,
1,
0,
0,
1,
0,
0,
0,
0,
0,
0,
1,
0,
1,
0,
0,
1,
1,
0,
1,
0,
1,
0,
0,
0,
0,
0,
1,
1,
0,
0,
1,
0,
0,
0,
1,
0,
1,
0,
1,
0,
0,
1,
0,
0,
0,
1,
0,
0,
0,
1,
0,
0,
1,
0,
1,
1,
1,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
1,
0,
0,
0,
0,
1,
1,
1,
1,
0,
0,
0,
0,
1,
1,
0,
0,
0,
0,
0,
1,
0,
0,
0,
0,
0,
1,
0,
1,
1,
0,
0,
1,
1,
1,
1,
0,
0,
0,
1,
1,
0,
0,
0,
0,
0,
0,
0,
0,
1,
0,
1,
0,
0,
0,
0,
1,
0,
1,
0,
0,
0,
1,
1,
1,
0,
0,
0,
0,
1,
0,
0,
0,
0,
1,
0,
1,
0,
0,
0,
0,
1,
0,
1,
0,
0,
1,
0,
0,
0,
0,
0,
0,
1,
0,
0,
1,
1,
1,
0,
1,
0,
1,
0,
1,
0,
0,
1,
1,
1,
1,
0,
0,
0,
0,
1,
0,
0,
0,
1,
0,
1,
1,
1,
1,
1,
0,
0,
0,
0,
1,
0,
0,
0,
1,
0,
1,
0,
1,
0,
1,
0,
0,
0,
0,
1,
0,
0,
0,
1,
1,
0,
0,
0,
1,
0,
0,
0,
0,
1,
0,
0,
0,
0,
0,
0,
1,
0,
0,
1,
0,
1,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
1,
0,
0,
1,
0,
0,
1,
0,
0,
0,
1,
0,
0,
1,
0,
0,
1,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
1,
0,
0,
0,
0,
1,
0,
1,
0,
0,
0,
0,
0,
1,
0,
1,
0,
1,
1,
0,
0,
0,
1,
0,
1,
1,
0,
0,
0,
0,
1,
0,
1,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
1,
0,
0,
0,
1,
0,
0,
0,
1,
0,
0,
0,
1,
0,
0,
1,
1,
0,
1,
1,
0,
1,
0,
0,
1,
1,
0,
1,
0,
0,
0,
1,
0,
0,
1,
0,
0,
0,
1,
0,
0,
0,
0,
0,
1,
0,
1,
0,
1,
1,
0,
0,
1,
1,
0,
0,
0,
0,
0,
0,
1,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
1,
0,
1,
1,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
1,
0,
0,
0,
0,
0,
0,
0,
1,
0,
0,
0,
1,
0,
0,
0,
0,
0,
0,
0,
0,
1,
0,
0,
1,
1,
0,
0,
0,
0,
0,
0,
1,
0,
0,
0,
0,
0,
0,
1,
0,
0,
0,
0,
0,
0,
1,
1,
1,
0,
1,
0,
0,
0,
1,
0,
1,
0,
0,
1,
0,
0,
1,
0,
1,
0,
1,
1,
0,
1,
0,
0,
0,
0,
0,
1,
0,
1,
0]
import numpy as np
y = np.array(gt_label)
scores = np.array(pred_label)
y[0:10]
array([0, 1, 0, 0, 0, 0, 0, 1, 0, 1])
scores[0:10]
array([ 0.30664873, 0.87607014, 0.03465158, 0.16497105, 0.09146911,
0.04465932, 0.07621145, 0.11715919, 0.23253924, 0.23055804])
from sklearn import metrics
fpr, tpr, thresholds = metrics.roc_curve(y, scores, pos_label=1)
roc_auc = metrics.auc(fpr, tpr)
# method I: plt
import matplotlib.pyplot as plt
plt.title('Receiver Operating Characteristic')
plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)
plt.legend(loc = 'lower right')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()
thresholds
array([ 0.9986124 , 0.95954418, 0.95656335, 0.94747752, 0.94444239,
0.92979932, 0.92034453, 0.91558194, 0.91079551, 0.86119872,
0.85661083, 0.84926724, 0.84882957, 0.84597945, 0.8416701 ,
0.81558007, 0.81236494, 0.81028396, 0.81007826, 0.7742759 ,
0.76524854, 0.72837567, 0.726484 , 0.71894968, 0.71013123,
0.704319 , 0.68048763, 0.67319602, 0.66984642, 0.65298784,
0.64581037, 0.64454418, 0.64028609, 0.63940525, 0.63570708,
0.62816495, 0.62505257, 0.61296248, 0.61295629, 0.60332441,
0.58682811, 0.57170707, 0.57050121, 0.56133604, 0.55865502,
0.55541611, 0.55504882, 0.53963447, 0.51595438, 0.5061273 ,
0.50592619, 0.49904209, 0.49185759, 0.47407663, 0.4551124 ,
0.43994987, 0.43031365, 0.41083896, 0.39128816, 0.38340461,
0.37551194, 0.3441363 , 0.33769208, 0.33632213, 0.32192218,
0.31024617, 0.30189764, 0.30010235, 0.29991192, 0.29023218,
0.28941488, 0.28760105, 0.27777898, 0.27126986, 0.26782936,
0.26668072, 0.25373226, 0.24609864, 0.23894513, 0.23397237,
0.23253924, 0.23055804, 0.22547466, 0.21357143, 0.20987767,
0.20760071, 0.18202227, 0.18033367, 0.13708365, 0.13613379,
0.12924868, 0.12901556, 0.1284436 , 0.12725115, 0.11779541,
0.1152432 , 0.10117459, 0.09949279, 0.09880227, 0.09555787,
0.09188473, 0.09172916, 0.08827138, 0.08710378, 0.08398992,
0.08349401, 0.07939225, 0.07865113, 0.06957841, 0.06948733,
0.06426042, 0.06263816, 0.04965979, 0.04948306, 0.04247338,
0.0423004 , 0.03812873, 0.03796023, 0.02658486, 0.0265184 ,
0.0236395 , 0.02341568, 0.00971162])