Delong检验AUC之间的比较

ROC的显著性检验：Delong test

转自matthewsyc_-CSDN博客的Python实现显著性检验delong

import numpy as np
from matplotlib import pyplot as plt
import scipy.stats as st
from sklearn import metrics

class DelongTest():
    def __init__(self,preds1,preds2,label,threshold=0.05):
        '''
        preds1:the output of model1
        preds2:the output of model2
        label :the actual label
        '''
        self._preds1=preds1
        self._preds2=preds2
        self._label=label
        self.threshold=threshold
        self._show_result()

    def _auc(self,X, Y)->float:
        return 1/(len(X)*len(Y)) * sum([self._kernel(x, y) for x in X for y in Y])

    def _kernel(self,X, Y)->float:
        '''
        Mann-Whitney statistic
        '''
        return .5 if Y==X else int(Y < X)

    def _structural_components(self,X, Y)->list:
        V10 = [1/len(Y) * sum([self._kernel(x, y) for y in Y]) for x in X]
        V01 = [1/len(X) * sum([self._kernel(x, y) for x in X]) for y in Y]
        return V10, V01

    def _get_S_entry(self,V_A, V_B, auc_A, auc_B)->float:
        return 1/(len(V_A)-1) * sum([(a-auc_A)*(b-auc_B) for a,b in zip(V_A, V_B)])
    
    def _z_score(self,var_A, var_B, covar_AB, auc_A, auc_B):
        return (auc_A - auc_B)/((var_A + var_B - 2*covar_AB )**(.5)+ 1e-8)

    def _group_preds_by_label(self,preds, actual)->list:
        X = [p for (p, a) in zip(preds, actual) if a]
        Y = [p for (p, a) in zip(preds, actual) if not a]
        return X, Y

    def _compute_z_p(self):
        X_A, Y_A = self._group_preds_by_label(self._preds1, self._label)
        X_B, Y_B = self._group_preds_by_label(self._preds2, self._label)

        V_A10, V_A01 = self._structural_components(X_A, Y_A)
        V_B10, V_B01 = self._structural_components(X_B, Y_B)

        auc_A = self._auc(X_A, Y_A)
        auc_B = self._auc(X_B, Y_B)

        # Compute entries of covariance matrix S (covar_AB = covar_BA)
        var_A = (self._get_S_entry(V_A10, V_A10, auc_A, auc_A) * 1/len(V_A10)+ self._get_S_entry(V_A01, V_A01, auc_A, auc_A) * 1/len(V_A01))
        var_B = (self._get_S_entry(V_B10, V_B10, auc_B, auc_B) * 1/len(V_B10)+ self._get_S_entry(V_B01, V_B01, auc_B, auc_B) * 1/len(V_B01))
        covar_AB = (self._get_S_entry(V_A10, V_B10, auc_A, auc_B) * 1/len(V_A10)+ self._get_S_entry(V_A01, V_B01, auc_A, auc_B) * 1/len(V_A01))

        # Two tailed test
        z = self._z_score(var_A, var_B, covar_AB, auc_A, auc_B)
        p = st.norm.sf(abs(z))*2

        return z,p

    def _show_result(self):
        z,p=self._compute_z_p()
        print(f"z score = {z:.5f};\np value = {p:.5f};")
        if p < self.threshold :print("There is a significant difference")
        else:        print("There is NO significant difference")


model_T1 = [0.9286671876907349, 0.12230029702186584, 0.8363653421401978, 0.7654389142990112, 0.7763422727584839, 0.6425679922103882, 0.6900917887687683, 0.0020835199393332005, 1.8482187442714348e-06, 0.05895061045885086, 0.5234459638595581, 0.02154107764363289, 0.0005529098561964929, 1.0255861525365617e-05, 1.6497815522598103e-05, 4.60243472844013e-06, 7.771797754685394e-06, 0.0026524162385612726, 0.008208156563341618, 0.0019362145103514194, 0.0007979585789144039, 0.002598818624392152, 0.0006110476097092032, 0.02920028753578663, 3.5067957924184157e-06, 2.712853529374115e-06, 0.002889233408495784, 0.0006173983565531671, 0.013871446251869202, 0.0013073674635961652, 0.0035311749670654535, 0.02041054517030716, 1.0161960744881071e-05, 0.014190172776579857, 0.014970371499657631, 0.3339996635913849, 0.5497852563858032, 0.0031937595922499895, 0.02309592254459858, 0.00341019150801003, 0.1611122041940689, 0.001523851417005062, 0.19950851798057556, 0.23235632479190826, 0.5881261825561523, 0.8668797612190247, 0.7946147322654724, 0.6447446942329407, 0.5585587024688721, 0.5180597305297852, 0.000724370707757771, 0.00020042371761519462, 0.0003843296435661614, 0.020368428900837898, 0.1158250942826271, 0.02185729332268238, 0.001407236559316516, 0.0006345538422465324, 0.0004269088676664978, 0.0002547753101680428, 0.0003697795618791133, 0.0002122873702319339, 0.6322190761566162, 0.6207689046859741, 0.736122727394104, 0.004311468917876482, 1.2255405863470514e-06, 0.005339937284588814, 0.0005346754915080965, 0.0003176368772983551, 0.6552878022193909, 0.0038374674040824175, 0.5052462220191956, 0.4525039494037628, 0.9005113244056702, 0.7788103222846985, 0.6716985106468201, 2.2297392206382938e-05, 0.00027775965281762183, 2.1535204723477364e-05, 0.3423946797847748, 0.006384789478033781, 5.828431676491164e-05, 0.003988982178270817, 0.3659616708755493, 0.8306800723075867, 2.961195177952902e-10, 0.6995818614959717, 0.4251277446746826, 0.16980202496051788, 2.8759855013049673e-06, 2.2440724933403544e-05, 0.45607003569602966, 0.9558064937591553, 0.9185711145401001, 0.024765780195593834, 0.004654869437217712, 0.0052179074846208096, 1.4105962691246532e-06, 3.189445487805642e-05, 3.553445640136488e-05, 0.0026363625656813383, 0.0002017530205193907, 0.0010894985171034932, 0.0008051302284002304, 0.00017342685896437615, 0.00022129248827695847, 0.0023354736622422934, 0.14349988102912903, 0.40188175439834595, 0.38590899109840393, 0.6591441035270691, 0.010944784618914127, 0.015424299985170364, 0.7817600965499878, 0.5953721404075623, 0.024464957416057587, 5.791781404695939e-06, 0.0010140695376321673, 1.2036892258038279e-05, 0.39592358469963074, 0.09853703528642654, 0.3674406409263611, 0.18882495164871216, 1.7917989680427127e-05, 0.00011450925376266241, 0.00043408083729445934, 0.0009347826708108187, 0.23674307763576508, 0.7380964756011963, 0.002266501309350133, 0.8758330345153809, 0.004354963544756174, 0.014250767417252064, 0.0002952649083454162, 0.0028194673359394073, 9.74805952864699e-05, 2.2088817786425352e-05, 0.006085223518311977, 0.7509129643440247, 0.9360244274139404, 0.4571159780025482, 0.03870671987533569, 0.06048654392361641, 0.8016455173492432, 0.6522192358970642, 0.3957277834415436, 0.2328018695116043, 0.44333183765411377, 0.07408204674720764, 0.6854544878005981, 0.004471771419048309, 0.002663085237145424, 0.03812319040298462, 0.18206274509429932, 0.506072461605072, 0.018420040607452393, 0.017227698117494583, 0.001227160799317062, 0.0002566325420048088, 0.6139092445373535, 0.1473468840122223, 0.0003598793118726462, 1.2675373000092804e-05, 4.167185761616565e-05, 0.0001726925402181223, 0.0001410742406733334, 0.7030008435249329, 0.7886523008346558, 0.7290905117988586, 0.00036186279612593353, 0.6218881607055664, 0.00011227317008888349, 0.17833451926708221, 0.16090083122253418, 0.5516718626022339, 0.2504293620586395, 0.47005969285964966, 0.007375859189778566, 0.004802790004760027, 0.00020438313367776573, 0.00037397813866846263, 0.22941498458385468, 0.47053080797195435, 0.653666079044342, 0.8458345532417297, 0.20129182934761047, 0.13508418202400208, 0.15991008281707764, 0.5368548035621643, 0.5966965556144714, 0.8113263845443726, 5.073296733826282e-07, 8.0895470091491e-06, 0.0005735920276492834, 0.023115145042538643, 0.08668030053377151, 0.5067580938339233, 1.4598244888475165e-06, 4.357683428679593e-05, 7.523944805143401e-05, 3.347306346768164e-06, 0.0028112195432186127, 0.16275006532669067, 0.01358186174184084, 0.7592234015464783, 0.4408063590526581, 0.003194011515006423, 0.03655749186873436, 5.423053153208457e-05, 0.7525954246520996, 0.4289085268974304, 0.008914526551961899, 0.006904531270265579, 0.47592300176620483, 0.008215970359742641, 1.4943753740226384e-05, 0.01762087270617485, 0.018269110471010208, 0.05632288381457329, 0.035676222294569016, 0.7647976279258728, 0.6912809014320374, 0.5513157248497009, 0.6677781939506531, 0.6517137289047241, 0.0057359603233635426, 0.007676746230572462, 0.06119336187839508, 0.1668495237827301, 0.0034770045895129442, 0.0025386479683220387, 0.00024918498820625246, 0.09083046019077301, 2.183920514653437e-05, 5.1931209554823e-07, 0.0009258990758098662, 1.2141274055466056e-05, 6.15610770182684e-05, 0.26974213123321533, 0.0014125185552984476, 0.5564538240432739, 0.0010192206827923656, 0.8496865034103394, 0.8566997051239014, 0.8942370414733887, 0.8981861472129822, 0.7795079946517944, 0.002306456910446286, 0.005913989618420601, 0.003595686051994562, 1.3953986126580276e-05, 3.086438127297697e-08, 7.963629286678042e-06, 0.0002710364933591336, 0.5425804853439331, 1.2679157407546882e-05, 1.963201430044137e-05, 2.3980843252502382e-05, 0.2404172122478485, 0.6362407803535461, 0.14535585045814514, 0.04779406264424324, 0.07770709693431854, 0.6030382513999939, 0.29453039169311523, 6.538783054566011e-05, 0.007034760899841785, 0.00018194783478975296, 0.0019962526857852936, 0.9063031673431396, 0.9573707580566406, 0.9621424078941345, 0.9424834847450256, 0.9296712875366211, 0.4719333350658417, 0.06917949765920639, 3.944115815102123e-05, 1.4037820619705599e-05, 0.15358926355838776, 0.008737289346754551, 0.0009830647613853216, 0.1828768402338028, 0.8401545286178589, 0.8633949160575867, 0.68779456615448, 7.574486176054052e-07, 1.38481173053151e-05, 0.009959177114069462, 0.002658088458701968, 0.06706203520298004, 0.11771391332149506, 0.6386122107505798, 0.0012216082541272044, 0.09727150201797485, 0.019265757873654366, 0.2419409602880478, 0.0002321281936019659, 2.1615656805806793e-05, 0.7208119034767151, 0.634853720664978, 0.6359359622001648, 0.19433660805225372, 0.014603053219616413, 0.8905642628669739, 0.8375570774078369, 0.9285385012626648, 0.7970498204231262, 0.1816151887178421, 0.11530590057373047, 0.4571355879306793, 0.1663857251405716, 0.8328610062599182, 0.6832456588745117, 0.7535916566848755, 0.9820402264595032, 0.7571716904640198, 0.15680605173110962, 0.0012799680698662996, 0.05533283203840256, 0.0002240943576907739, 0.5164972543716431, 0.6750808954238892, 0.9405986666679382, 0.9598041772842407, 0.9416008591651917, 0.9509901404380798, 0.9294530153274536, 0.8811376094818115, 1.0619881322782021e-05, 0.00016638459055684507, 0.7110522985458374, 0.6771866679191589, 0.7935450077056885, 0.47179946303367615, 0.003900413401424885, 0.0005847685970366001, 0.0028134502936154604, 0.000295858655590564, 0.05956973508000374, 0.02087830752134323, 0.14074532687664032, 0.47934263944625854, 0.12244205176830292, 1.414667508470302e-06, 0.014050351455807686, 0.022541530430316925, 0.7037399411201477, 0.7322210073471069, 0.6443852782249451, 0.7209482192993164, 3.82126345357392e-05, 3.180993417117861e-06, 0.004032370168715715, 0.002639957470819354, 0.09230037778615952, 0.11181439459323883, 0.011076772585511208, 0.32988476753234863, 0.06563466787338257, 0.0030806255526840687, 0.0001641338167246431, 0.0028539695776998997, 1.2160335245425813e-06, 2.8957825293218775e-07, 1.2796751889254665e-06, 0.0035487771965563297, 0.0001170950781670399, 0.001583161880262196, 1.1092753993580118e-05, 6.799487891839817e-05, 9.662444062996656e-05, 0.018053239211440086, 4.955257281835657e-06, 5.538342406907759e-07, 0.0018705923575907946, 0.014902561902999878, 0.5309826135635376, 0.20722231268882751, 0.7661179900169373, 0.964127242565155, 0.2560027241706848, 0.6160029172897339, 0.03812374919652939, 0.0002573214878793806, 0.26090648770332336, 0.756320595741272, 0.8255756497383118, 0.36558717489242554, 0.4078218936920166, 0.06870945543050766, 0.6766645908355713, 0.012080064043402672, 0.0006364831351675093, 0.5842103958129883, 0.5631831288337708, 0.6014434099197388, 0.0022539161145687103, 0.00036454887595027685, 2.7944428438786417e-05, 9.319582204625476e-06, 0.13292035460472107, 8.637007340439595e-06, 0.0019551541190594435, 0.0016147008864209056, 0.00011619490396697074, 0.7263587117195129, 0.7798292636871338, 0.8330521583557129, 0.004688818473368883, 0.01924714632332325, 0.6885802149772644, 0.7136117219924927, 1.318233103120292e-06, 0.0004597558290697634, 0.9344145059585571, 0.6939900517463684, 0.010470282286405563, 0.011886034160852432, 0.0030224535148590803, 0.5901885032653809, 0.08140185475349426, 0.3353444039821625, 0.8128919005393982, 0.879311203956604, 0.0008231668034568429, 0.26056015491485596, 0.05373772233724594, 0.001302599674090743, 0.7963628768920898, 0.7111307978630066, 0.7717320919036865, 0.7523367404937744, 0.5839848518371582, 0.6311432123184204, 0.00028566631954163313, 0.03402561694383621, 0.033389098942279816, 0.013400149531662464, 0.15802350640296936, 0.8897414803504944, 0.00014863809337839484, 0.0009304866543971002, 0.0799245685338974, 0.000697044306434691, 0.005263538099825382, 0.011733802035450935, 0.042783308774232864, 0.4235123097896576, 0.01737258769571781, 0.2735745310783386, 0.0008943561697378755, 0.0001965552510228008, 0.00013813770783599466, 2.0755064724653494e-06, 0.006828279700130224, 0.005371144041419029, 0.008843516930937767, 0.00022443027410190552, 2.3958989913808182e-05, 1.5728950529592112e-05, 0.00016341077571269125, 4.5532884541898966e-05, 0.24420082569122314, 0.014605071395635605, 0.0003333623753860593, 2.3438746211468242e-05, 0.00038885659887455404, 1.2642933143069968e-05, 3.3705625810398487e-06, 3.329839501020615e-06, 3.372342325747013e-05, 0.0011922131525352597, 0.0268186554312706, 8.28136908239685e-05, 0.06726405769586563, 0.5022422671318054, 0.06099864840507507, 0.012706654146313667, 0.00045231240801513195, 9.223391316481866e-06, 0.00037768748006783426, 0.0035813061986118555, 0.009778010658919811, 0.00022406273637898266, 0.00011620908480836079, 0.0011737713357433677, 0.0006806163000874221, 0.0006222869269549847, 0.1647057831287384, 0.7038504481315613, 0.5614456534385681, 0.3956218957901001, 8.016490227191753e-08, 5.635161414829781e-06, 0.016953380778431892, 0.011407014913856983, 0.12679357826709747, 0.8887414932250977, 0.8440614938735962, 7.178814121289179e-05, 0.001452427008189261, 0.00029703695327043533, 0.0009012226364575326, 0.027420086786150932, 0.4221166670322418, 0.7193593382835388, 0.15171384811401367, 0.10325798392295837, 0.010262860916554928, 0.039923880249261856, 0.11918654292821884, 0.002445633290335536, 0.001611630548723042, 0.009882799349725246, 4.402729609864764e-05, 6.566718184330966e-06, 0.0008725043735466897, 1.4151224604574963e-05, 4.068440830451436e-06, 1.0789297448354773e-05, 5.183698249311419e-06, 0.000138661649543792, 0.0005777851911261678, 1.2013313607894816e-05, 0.007317390758544207]


model_T2 =  [0.0007020804332569242, 0.2968996465206146, 0.0115611357614398, 0.013311983086168766, 0.2002752721309662, 0.17460332810878754, 0.1745566427707672, 0.005899535957723856, 0.476856529712677, 0.8484365940093994, 0.8539079427719116, 0.8302357196807861, 0.8368730545043945, 0.24093978106975555, 0.2741888165473938, 0.5919451117515564, 0.1489667445421219, 0.06903688609600067, 0.18941010534763336, 0.5722696781158447, 0.672296404838562, 0.6824051141738892, 0.6684012413024902, 0.6131160855293274, 0.013429501093924046, 0.00162671715952456, 0.000847682065796107, 0.0024565719068050385, 0.011971517466008663, 0.5489612817764282, 0.635345995426178, 0.6636718511581421, 0.6309570670127869, 0.5563052296638489, 0.209333136677742, 0.1672002673149109, 0.5015451312065125, 0.08207330107688904, 0.14773957431316376, 0.06032562628388405, 0.032043006271123886, 0.05560987815260887, 0.04639885574579239, 0.011995624750852585, 0.000639778038021177, 2.8195920094731264e-05, 0.04259299859404564, 0.6114986538887024, 0.6607894897460938, 0.005362281110137701, 0.36037543416023254, 0.39409348368644714, 0.666567862033844, 0.3850672245025635, 0.06499986350536346, 0.1611773520708084, 0.07750695198774338, 0.003928137011826038, 0.2495286464691162, 0.14534428715705872, 0.49271947145462036, 0.20547381043434143, 0.04559754580259323, 0.45805343985557556, 0.37294572591781616, 0.00017296479200012982, 0.00014043190458323807, 0.02865942008793354, 0.0001559094962431118, 0.11884113401174545, 0.30189481377601624, 5.8803354477277026e-05, 5.560594581766054e-05, 0.02099442481994629, 0.17365790903568268, 0.4670473635196686, 0.09137126058340073, 0.675428569316864, 0.00488136475905776, 0.04043930396437645, 0.24075429141521454, 0.5053150057792664, 0.06509161740541458, 0.0015505043556913733, 0.5900483131408691, 0.03131509944796562, 0.00618204427883029, 0.000992923742160201, 0.6214674711227417, 0.6650625467300415, 0.021787239238619804, 0.1305793821811676, 0.2895553708076477, 0.0013034249423071742, 0.004332731943577528, 0.3518234193325043, 0.039540644735097885, 0.026019616052508354, 0.0011406932026147842, 0.0010802766773849726, 0.6269184947013855, 0.3310069739818573, 0.6702204942703247, 0.6659959554672241, 0.6298815608024597, 0.6546012759208679, 0.6685041189193726, 0.005269553512334824, 0.016451016068458557, 0.0031846126075834036, 0.35565581917762756, 0.6696816682815552, 0.5129771828651428, 0.3221326172351837, 0.074521005153656, 0.012331731617450714, 0.4556482434272766, 0.016980750486254692, 0.04756464809179306, 0.6677854657173157, 0.6265979409217834, 0.4196363389492035, 0.45156189799308777, 0.751499593257904, 0.6298133134841919, 0.8313624262809753, 0.8279103636741638, 0.8607258200645447, 0.8948836326599121, 0.3115576207637787, 0.6356721520423889, 0.48344340920448303, 0.0008426422718912363, 0.3156064450740814, 0.15026158094406128, 0.5724818110466003, 0.5383831858634949, 0.0034918927121907473, 0.0006428186898119748, 0.6166383624076843, 0.3288925588130951, 0.022482892498373985, 0.0036157057620584965, 0.0005975252715870738, 9.98768737190403e-05, 0.042281441390514374, 5.863178739673458e-05, 0.00010129265865543857, 0.006663816049695015, 0.10797814279794693, 0.67229825258255, 0.0002786022378131747, 0.3501167297363281, 0.032899729907512665, 0.0006672089220955968, 0.0005640878225676715, 0.00021285690309014171, 0.035166557878255844, 0.01647760719060898, 0.05353037267923355, 0.002972563961520791, 0.004373507108539343, 0.5435131192207336, 0.6198729276657104, 0.002527418080717325, 0.06160985678434372, 0.018760012462735176, 0.003756130812689662, 0.48009902238845825, 0.028384290635585785, 0.006453181616961956, 0.08443202078342438, 0.002288583666086197, 0.0021818848326802254, 0.6502827405929565, 0.06868404895067215, 0.5146245360374451, 0.12983079254627228, 0.18777400255203247, 0.0043359482660889626, 0.000663791608531028, 0.0014715420547872782, 0.0023678175639361143, 0.013167532160878181, 0.5391507744789124, 0.38324078917503357, 0.001498343190178275, 0.027427803725004196, 1.2111725482100155e-05, 0.8381072282791138, 0.9724121689796448, 0.8074756860733032, 0.49338650703430176, 0.674272894859314, 0.4273991584777832, 0.003601273288950324, 0.022577153518795967, 0.6420342326164246, 0.673581600189209, 0.4927522540092468, 0.5743281841278076, 0.6314590573310852, 0.00038206257158890367, 0.002732706256210804, 9.326828148914501e-06, 0.6573835611343384, 0.0009966855868697166, 0.12486492842435837, 0.03793743997812271, 0.09217111766338348, 0.482607901096344, 0.3950778841972351, 0.7377353310585022, 0.48124346137046814, 0.018896065652370453, 0.004729888401925564, 0.3558165431022644, 1.7230157027370296e-05, 0.018559115007519722, 0.009672866202890873, 0.06754618138074875, 0.22382070124149323, 0.0021496324334293604, 0.0024687028490006924, 0.0011711926199495792, 0.04931230470538139, 0.0011124404845759273, 0.00019747194892261177, 0.05359483137726784, 0.2582226097583771, 0.04710490256547928, 0.0006825285963714123, 0.004477331414818764, 0.0005563073209486902, 0.01403745636343956, 0.09481959789991379, 0.006425788626074791, 0.9591269493103027, 0.9648312926292419, 0.9188762903213501, 0.815167248249054, 0.6245118379592896, 0.693330705165863, 0.016927868127822876, 0.0004937084740959108, 0.2308521419763565, 0.48625102639198303, 0.5714074969291687, 0.3505268394947052, 0.5945030450820923, 0.06285685300827026, 0.18560470640659332, 0.000613735057413578, 0.7068749666213989, 0.6503477692604065, 9.937662071024533e-06, 0.6122027039527893, 0.3908679485321045, 0.6248378157615662, 0.5916271805763245, 0.6392416954040527, 0.000275463709840551, 0.002400875324383378, 0.08096427470445633, 0.000854377809446305, 0.0023431212175637484, 0.0048413644544780254, 0.9165566563606262, 0.8798535466194153, 0.30874258279800415, 0.00019200329552404583, 3.2102777822728967e-06, 0.0005713824066333473, 0.0020089163444936275, 0.02406446449458599, 0.0019106242107227445, 0.1959811896085739, 0.5055606365203857, 0.34610775113105774, 0.2788758873939514, 0.21219098567962646, 0.26057741045951843, 0.20780827105045319, 0.43249040842056274, 0.20967219769954681, 0.3374312222003937, 0.0023960801772773266, 0.009893354028463364, 0.04170116037130356, 0.009524617344141006, 0.5180814266204834, 0.47823771834373474, 0.05145750567317009, 0.0005012672045268118, 0.14218541979789734, 0.3184467852115631, 0.6150031089782715, 0.7922473549842834, 0.015792347490787506, 0.21070726215839386, 0.028353482484817505, 0.006742686033248901, 0.00926970411092043, 0.0008134348317980766, 0.6376845836639404, 0.07729824632406235, 0.4973224997520447, 0.21197380125522614, 0.5515767335891724, 0.45659419894218445, 0.6651371121406555, 0.6476901173591614, 0.7115156054496765, 0.6453447937965393, 0.5254475474357605, 0.0023575881496071815, 0.03758431226015091, 0.5065690279006958, 0.5991615653038025, 0.6317957043647766, 0.8268820643424988, 0.1294158548116684, 0.5594586730003357, 0.4594947099685669, 0.1947304904460907, 0.1006975844502449, 0.6559655666351318, 0.412299782037735, 0.47961169481277466, 0.1454407274723053, 0.5624175071716309, 0.07217495888471603, 0.1838746815919876, 0.4496749937534332, 0.0005916818627156317, 0.17424514889717102, 0.060264818370342255, 0.17342260479927063, 0.12623046338558197, 0.8801029324531555, 0.07793943583965302, 0.554614245891571, 0.6715960502624512, 0.5075272917747498, 0.07669727504253387, 0.027024926617741585, 0.032293956726789474, 0.07651326805353165, 0.02478897012770176, 0.6110949516296387, 0.055710505694150925, 0.09311292320489883, 0.07716117054224014, 0.06721365451812744, 0.14703276753425598, 0.6003630757331848, 0.4567694365978241, 0.6638132333755493, 0.6373446583747864, 0.6098276376724243, 0.5165919065475464, 0.5216148495674133, 0.06296691298484802, 0.1292073130607605, 0.2755183279514313, 0.5292887091636658, 0.6873375177383423, 0.5843538641929626, 0.0004222942516207695, 0.35931816697120667, 0.5666448473930359, 0.03070819564163685, 0.6690975427627563, 1.8186627812610823e-06, 0.00013965790276415646, 0.26712602376937866, 0.485291063785553, 0.11865359544754028, 1.9965065803262405e-05, 0.029656725004315376, 0.0032811593264341354, 0.5314611196517944, 0.0007019366603344679, 0.0005068948375992477, 0.503909707069397, 0.6921981573104858, 0.16755995154380798, 0.02602839097380638, 0.00478530814871192, 3.0062532459851354e-05, 0.00021559761080425233, 0.00014363671652972698, 0.002631527604535222, 0.01663915626704693, 0.008537276647984982, 0.21088899672031403, 0.0010132432216778398, 0.5744059085845947, 0.0003527164226397872, 0.00026002846425399184, 0.17844243347644806, 0.3785109221935272, 0.5981534719467163, 0.39206624031066895, 0.41253840923309326, 0.004653099924325943, 0.15311455726623535, 0.19108544290065765, 0.00017891140305437148, 0.2115340232849121, 0.5550284385681152, 0.3112154006958008, 0.5203328728675842, 0.006524937227368355, 0.0003506591310724616, 0.026595495641231537, 0.023230843245983124, 0.006720579229295254, 0.00035543841659091413, 0.03806919977068901, 0.633804976940155, 0.547579824924469, 0.6209372282028198, 0.5605165362358093, 0.6973844766616821, 0.4154863655567169, 0.14888902008533478, 0.5520925521850586, 0.0458734855055809, 0.002011424396187067, 0.025307346135377884, 0.0002185381599701941, 0.0007853073766455054, 0.0021567794028669596, 0.005216516554355621, 0.017453555017709732, 0.047846172004938126, 0.027154590934515, 0.42575690150260925, 0.6814905405044556, 0.47828999161720276, 0.12265412509441376, 0.28555023670196533, 0.2451099008321762, 0.5831632018089294, 0.17721566557884216, 0.032415371388196945, 0.40869686007499695, 0.46929532289505005, 0.00018328879377804697, 0.10926248878240585, 0.5967662334442139, 0.00012623396469280124, 3.7815902032889426e-05, 0.004696671385318041, 0.273021936416626, 0.49918487668037415, 0.6690890192985535, 0.007159706670790911, 0.25744524598121643, 0.35030820965766907, 0.4902588427066803, 0.04657598212361336, 0.005511024966835976, 0.0005065646837465465, 0.029438726603984833, 0.18914936482906342, 0.0006277631036937237, 0.011449011042714119, 0.00021373711933847517, 0.0008039683452807367, 0.07929576933383942, 0.1353127360343933, 0.011661477386951447, 0.010458121076226234, 0.6804843544960022, 0.6068956851959229, 0.29750165343284607, 0.1107877865433693, 0.27566662430763245, 0.0790712833404541, 0.07273143529891968, 0.6513102650642395, 0.6550227403640747, 0.040296729654073715, 0.027308249846100807, 0.01414759736508131, 0.38702914118766785, 0.18038970232009888, 0.03265657648444176, 0.5815393924713135, 0.6797095537185669, 0.662988543510437, 0.005155664402991533, 0.07452616095542908, 0.025495272129774094, 0.010196640156209469, 0.05362448841333389, 0.6050799489021301, 0.10532937198877335, 0.4720572233200073, 0.01198667660355568, 0.03319794684648514, 0.011757935397326946, 0.06207773834466934, 0.006234779953956604, 0.07521171122789383, 0.019837461411952972, 0.05445077642798424, 0.46556025743484497, 0.3235531747341156, 0.31694841384887695, 0.13226483762264252, 0.013989241793751717, 0.5190746784210205, 0.011652907356619835, 0.017410993576049805, 0.7713167667388916, 0.6310662031173706, 0.061059530824422836, 0.052401404827833176, 1.3594242773251608e-05, 0.7342516779899597, 0.8011546730995178, 0.03248738870024681, 0.5227835774421692]


model_enhance =   [0.49104437232017517, 0.0046099708415567875, 0.7933430671691895, 0.7308081984519958, 0.5939068794250488, 0.0058775595389306545, 0.00021366006694734097, 0.0004950197180733085, 0.000314614298986271, 0.025339625775814056, 0.11430276930332184, 0.6385487914085388, 0.0074263582937419415, 0.05618023872375488, 0.08989442139863968, 0.022673483937978745, 4.222952702548355e-05, 0.0030953402165323496, 0.002787671983242035, 0.022184303030371666, 0.014683821238577366, 0.04024795815348625, 0.050097040832042694, 0.2325897514820099, 0.4904755651950836, 0.3971957266330719, 0.05542862042784691, 0.3470284044742584, 0.710574746131897, 0.6956165432929993, 0.6339473128318787, 0.6703693866729736, 0.7255199551582336, 0.7682428359985352, 0.6857794523239136, 0.5675339102745056, 0.39151838421821594, 0.2719265818595886, 0.49323132634162903, 0.3011181652545929, 0.5876427888870239, 0.5578424334526062, 0.39613038301467896, 0.5516050457954407, 0.027418900281190872, 0.596279501914978, 0.7273393273353577, 0.11327560245990753, 0.18026499450206757, 0.08118574321269989, 0.29902535676956177, 0.5131328701972961, 0.5425755977630615, 0.656814455986023, 0.6126159429550171, 0.6412121653556824, 0.04872024431824684, 0.019892368465662003, 0.00745631568133831, 0.6041919589042664, 0.6661133170127869, 0.6055419445037842, 0.3487075865268707, 0.5600353479385376, 0.15213629603385925, 0.22741740942001343, 0.08419957011938095, 0.5915631651878357, 0.32794860005378723, 0.4179318845272064, 0.033663664013147354, 0.0001480317150708288, 0.034390490502119064, 0.7072318196296692, 0.6746150851249695, 0.7399874329566956, 0.7146034836769104, 0.05327225476503372, 0.541407585144043, 0.19255755841732025, 0.015547025948762894, 0.6860074400901794, 0.7008016705513, 0.00032905986881814897, 0.5664354562759399, 0.010588248260319233, 0.5943024754524231, 0.2081579864025116, 0.0004227667232044041, 6.137823947938159e-05, 0.0011521257692947984, 0.11372119188308716, 0.0021193246357142925, 0.0022770045325160027, 0.006822500843554735, 0.10745489597320557, 0.02742820791900158, 0.008233730681240559, 0.6492651104927063, 0.15377312898635864, 0.7164788842201233, 0.6595593690872192, 0.6147886514663696, 0.5717105269432068, 0.49514293670654297, 0.30706292390823364, 0.0027964410837739706, 0.6316722631454468, 0.6925250887870789, 0.00044486080878414214, 0.7128796577453613, 0.5532905459403992, 0.15225501358509064, 0.12081891298294067, 0.719632089138031, 0.7186455130577087, 0.6794337034225464, 0.28193315863609314, 0.6080201268196106, 0.0034164413809776306, 0.6579349637031555, 0.4684072732925415, 0.7098644971847534, 0.7144832611083984, 0.004080129321664572, 0.0038650778587907553, 9.640456846682355e-05, 0.0018094254191964865, 0.06582339853048325, 0.018775498494505882, 0.6389769315719604, 0.6631955504417419, 0.7449101209640503, 0.7164248824119568, 0.366580992937088, 0.7314199209213257, 0.7542970180511475, 0.6994715332984924, 0.7285101413726807, 0.04878527298569679, 0.010140525177121162, 0.1698514074087143, 0.7347316145896912, 0.7603068351745605, 0.023245975375175476, 0.5778223872184753, 0.7366633415222168, 0.7659956216812134, 0.7707857489585876, 0.6920876502990723, 0.30496737360954285, 0.7308146953582764, 0.7167994379997253, 0.6055561304092407, 0.7667103409767151, 0.7614172101020813, 0.7408435940742493, 0.7190749049186707, 0.009814681485295296, 0.004594134166836739, 0.32430416345596313, 0.6304882764816284, 0.38415950536727905, 0.09051887691020966, 0.23469652235507965, 0.0789453387260437, 0.7379677891731262, 0.3390706479549408, 0.5511326193809509, 0.3630410432815552, 0.41310495138168335, 0.08354005217552185, 0.16223792731761932, 0.039516303688287735, 0.5220267176628113, 0.4839315712451935, 0.35424208641052246, 0.14426186680793762, 0.012869512662291527, 0.005235342308878899, 0.6134679317474365, 0.5328667163848877, 0.5667473077774048, 0.3035803735256195, 0.1405952274799347, 0.0011101390700787306, 0.07840228080749512, 0.04655313864350319, 0.0025343287270516157, 0.0002500613045413047, 0.4439745843410492, 0.024503501132130623, 0.7125786542892456, 0.38978278636932373, 0.685543417930603, 0.44731226563453674, 0.4790714979171753, 0.656743586063385, 0.5549431443214417, 0.6490336656570435, 0.10093902051448822, 0.5697217583656311, 0.592683732509613, 0.7729834318161011, 0.3941093981266022, 0.7189250588417053, 0.6066076159477234, 0.7484140396118164, 0.520448625087738, 0.06382910162210464, 0.7917677164077759, 0.6828535795211792, 0.5842903852462769, 0.5090375542640686, 0.38057926297187805, 0.6447773575782776, 0.6949726343154907, 0.5270105600357056, 0.7198072671890259, 0.7358319759368896, 0.7471132278442383, 0.7261866331100464, 0.6442791819572449, 0.7056608200073242, 0.63443922996521, 0.1449396312236786, 0.21159598231315613, 0.18463195860385895, 0.6555926203727722, 0.3367936909198761, 0.6319777965545654, 0.4133986234664917, 0.22720541059970856, 0.7347624897956848, 0.0058319177478551865, 0.2585030496120453, 0.00036536966217681766, 0.05151544511318207, 0.012509494088590145, 0.005556186661124229, 0.16222316026687622, 0.004560562316328287, 0.021661100909113884, 0.00037267396692186594, 0.0008715156000107527, 0.0041024088859558105, 0.008752691559493542, 0.010087354108691216, 0.012570103630423546, 0.004781346768140793, 0.004590003751218319, 0.7190380096435547, 0.6113893389701843, 0.5318098664283752, 0.13889645040035248, 0.15836162865161896, 0.1212681457400322, 0.046956028789281845, 0.048648733645677567, 0.6959618926048279, 0.0209672749042511, 0.018801718950271606, 0.14907845854759216, 0.6564871072769165, 0.7227582335472107, 0.7019285559654236, 0.7141021490097046, 0.41100049018859863, 0.46623966097831726, 0.539584755897522, 0.736219048500061, 0.7143000960350037, 0.7011867165565491, 0.6597432494163513, 0.7374700903892517, 0.41059693694114685, 0.07663081586360931, 0.5512334704399109, 0.4343661069869995, 0.7097408771514893, 0.0003089274396188557, 0.041036222130060196, 0.7740663290023804, 0.5029147863388062, 0.5728787183761597, 0.6915498971939087, 0.08576621860265732, 0.03223905712366104, 0.007226474117487669, 0.2671557366847992, 6.582603236893192e-05, 0.008424983359873295, 0.7247122526168823, 0.698523759841919, 0.4679115116596222, 0.6920439004898071, 0.7227068543434143, 0.629769504070282, 0.3637588918209076, 0.00443655950948596, 0.05238991975784302, 0.026079801842570305, 0.3996599018573761, 0.6262144446372986, 0.4529872238636017, 0.6829957962036133, 0.4356970191001892, 0.5894578099250793, 0.7301884293556213, 0.7332475185394287, 0.6403977274894714, 0.2836649417877197, 0.1437123566865921, 0.239141047000885, 0.5551893711090088, 0.42472267150878906, 0.652529239654541, 0.5433456301689148, 0.1607096940279007, 0.1424179971218109, 0.037110403180122375, 0.33647170662879944, 0.4192572236061096, 0.7749425172805786, 0.7798988223075867, 0.7955756783485413, 0.7253832817077637, 0.7504675388336182, 0.7486242651939392, 0.6095923781394958, 0.5494783520698547, 0.646192193031311, 0.7406136393547058, 0.5017513036727905, 0.6075132489204407, 0.46379414200782776, 0.5781270265579224, 0.6462863683700562, 0.6368050575256348, 0.5754066109657288, 0.03557078540325165, 0.3964519202709198, 0.04864601045846939, 0.37788060307502747, 0.005816136486828327, 0.5485402941703796, 0.5566387176513672, 0.7620170712471008, 0.6988906264305115, 0.17813773453235626, 0.5523940324783325, 0.384738951921463, 0.4494682848453522, 5.486462760018185e-05, 0.0022269082255661488, 0.0956621915102005, 0.07894661277532578, 0.11357351392507553, 0.10210483521223068, 0.3956911265850067, 0.6804952025413513, 0.6207043528556824, 0.5491510629653931, 0.07702402770519257, 0.004044895060360432, 0.4343716502189636, 0.003612831002101302, 0.0007046699174679816, 0.0013447649544104934, 0.05852140486240387, 0.056529391556978226, 0.07972094416618347, 0.6004849672317505, 8.930495823733509e-05, 0.0006994808209128678, 0.3554244041442871, 0.18464277684688568, 0.6496310830116272, 0.509319543838501, 0.6665493845939636, 0.7936999201774597, 0.6816418170928955, 0.21121715009212494, 0.5999415516853333, 0.6276393532752991, 0.2923305630683899, 0.2320573478937149, 0.25295835733413696, 0.051017388701438904, 0.04189672693610191, 0.2415195256471634, 0.5395299792289734, 0.31905895471572876, 0.6741589903831482, 0.5304057002067566, 0.02924525737762451, 0.7326505780220032, 0.7378993034362793, 0.6795021295547485, 0.6914877891540527, 0.7200818657875061, 0.5883511304855347, 0.667378842830658, 0.0034714313223958015, 0.0006583488429896533, 0.004176929127424955, 0.4690559506416321, 0.16914503276348114, 0.6386116147041321, 0.5215850472450256, 0.7149121761322021, 0.6411414742469788, 0.6844361424446106, 0.6126192808151245, 0.5826629400253296, 0.39340296387672424, 0.7308200597763062, 0.6914877891540527, 0.34994637966156006, 0.1439756453037262, 0.7248843312263489, 0.642027735710144, 0.5944335460662842, 0.6583343148231506, 0.6329526901245117, 0.35834741592407227, 0.01104868482798338, 0.05747123062610626, 0.3443070352077484, 0.2968401312828064, 0.25425541400909424, 0.39910805225372314, 0.2659827172756195, 0.19838809967041016, 0.7580121755599976, 0.09329623728990555, 0.7449678778648376, 0.7481277585029602, 0.5568138360977173, 0.26190924644470215, 0.6727721691131592, 0.5537561178207397, 0.6969426274299622, 0.6924300193786621, 0.6795870065689087, 0.16517023742198944, 0.008365511894226074, 0.5301941633224487, 0.6546461582183838, 0.7183555960655212, 0.2028215378522873, 0.6994555592536926, 0.5909652709960938, 0.2783302664756775, 0.13030047714710236, 0.4118131995201111, 0.6667314171791077, 0.3389904797077179, 0.6100792288780212, 0.6537708044052124, 0.6393662691116333, 0.6492815017700195, 0.417079895734787, 0.696862518787384, 0.7283753156661987, 0.5334644317626953, 0.008592445403337479, 0.12860926985740662, 0.05293595418334007, 0.42954394221305847, 0.43203914165496826, 0.6312083601951599, 0.7542460560798645, 0.681522786617279, 0.6436386108398438, 0.747838020324707, 0.7025184035301208, 0.61095130443573, 0.17397649586200714, 0.41933080554008484, 0.06038213521242142, 0.5487355589866638, 0.4345129132270813, 0.015652388334274292, 0.6328126192092896, 0.7548785209655762, 0.7300924062728882, 0.6698721647262573, 0.6187539100646973, 0.0032885645050555468, 0.5366550087928772, 0.5510576367378235, 0.48440223932266235, 0.680126965045929, 0.7002642154693604, 0.06292293220758438, 0.049728475511074066, 0.2610737383365631, 0.5849501490592957, 0.6976334452629089, 0.6679533123970032, 0.6333304047584534, 0.7118127346038818, 0.65781170129776, 0.7220643162727356, 0.7250059247016907, 0.673527181148529, 0.7799001932144165, 0.727616548538208, 0.7717617750167847, 0.7777869701385498, 0.7105962038040161, 0.6820797324180603, 0.3690793514251709, 0.6701405644416809, 0.6933114528656006, 0.7305803894996643, 0.7164450287818909, 0.6285778880119324, 0.6600467562675476, 0.0004331394156906754, 3.78053737222217e-05, 5.748212061007507e-06, 3.4391257486277027e-06, 0.005111427512019873, 0.7349907755851746]

# 创建对应的真实标签
true_labels = [1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1]
print ('========'+'以下为train 数据中各个模型与（T1+T2+enhance）的对比'+'========'+'\n')
print('1 model_T2  vs model_T1 :')
DelongTest(model_T2  , model_T1 , true_labels )
print('- - -'*30)
print('2 model_enhance vs model_T2  :')
DelongTest(model_enhance, model_T2  , true_labels )
print('- - -'*30)
print('3 model_T1 vs model_enhance  :')
DelongTest(model_T1 , model_enhance  , true_labels )

然后我使用MedCalc进行了比较，结果是一致的。 

然后选ROC curves 下的Comparision of ROC curves进入以下界面

然后OK就可以了 

关于medcalc的操作可以查看

lanjingling2019的

SPSS多条ROC曲线绘制，其中几条位于对角线下方，使用Medcalc 绘制解决，并进行Delong 检验​​​​​​​​​​​​​​