spark dataframe 行转列与列转行

最新推荐文章于 2023-12-21 12:29:06 发布

BigData_xgs

最新推荐文章于 2023-12-21 12:29:06 发布

阅读量1.1k

点赞数

文章标签： spark

本文链接：https://blog.csdn.net/weixin_53099973/article/details/120087563

版权

1 原始数据

logPrint("Curves", Curves)
Curves.show(10, false)
  //point的范围是1-96
  //可以看到共有30048条数据
  ======================「Curves:30048」 ======================
    +-----------------+----------------+------+-----+
    |CIS_TRAN_ID      |TRAN_ID         |P     |point|
    +-----------------+----------------+------+-----+
    |00000000000003489|3489            |0.3029|17   |
    |00000000008500445|8500445         |0.9975|86   |
    |00000000010005659|10005659        |0.006 |70   |
    |00000000030007744|30007744        |0.4036|90   |
    |00000000030011046|30011046        |3.174 |38   |
    |00000000050003546|50003546        |6.0E-4|21   |
    |00000000076005775|76005775        |0.0   |58   |
    |00000000085004674|85004674        |1.0E-4|75   |
    |00000000200023704|200023704       |0.0   |74   |
    |01000000103316188|1000000103316188|0.0   |37   |
    +-----------------+----------------+------+-----+

2 行转列之后


    val result1: DataFrame = Curves
      .groupBy("CIS_TRAN_ID", "TRAN_ID")
      .pivot("point")
      .sum("P")
      //空值用0.0填充
      .na.fill(0.0)
      .persist(StorageLevel.MEMORY_AND_DISK)

    logPrint("result1", result1)
    result1.show(10, false)
    //可以看到共有313条数据   30048=313*96

  ======================「result1:313」 ======================
+-----------------+------------------------------------------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+
|CIS_TRAN_ID      |TRAN_ID                                   |1     |2     |3     |4     |5     |6     |7     |8     |9     |10    |11    |12    |13    |14    |15    |16    |17    |18    |19    |20    |21    |22    |23    |24    |25    |26    |27    |28    |29    |30    |31    |32    |33    |34    |35    |36    |37    |38    |39    |40    |41    |42    |43    |44    |45    |46    |47    |48    |49    |50    |51    |52    |53    |54    |55    |56    |57    |58    |59    |60    |61    |62    |63    |64    |65    |66    |67    |68    |69    |70    |71    |72    |73    |74    |75    |76    |77    |78    |79    |80    |81    |82    |83    |84    |85    |86    |87    |88    |89    |90    |91    |92    |93    |94    |95    |96    |
+-----------------+------------------------------------------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+
|29M00000003398208|C762D418-8607-44DE-9C9E-1F76857FB676-25483|0.3073|0.3426|0.2789|0.2944|0.2351|0.2546|0.2432|0.2155|0.2241|0.2346|0.1888|0.1848|0.2148|0.1742|0.1645|0.2082|0.1522|0.1618|0.1994|0.1739|0.1545|0.1552|0.2159|0.1683|0.1667|0.1886|0.2608|0.1996|0.2276|0.204 |0.1818|0.204 |0.2135|0.2055|0.1933|0.2753|0.3206|0.2834|0.3118|0.3033|0.329 |0.4017|0.3695|0.4194|0.3665|0.2686|0.4175|0.4036|0.327 |0.3658|0.3118|0.3815|0.3552|0.3957|0.3144|0.287 |0.2645|0.2703|0.236 |0.3311|0.2335|0.2885|0.355 |0.2412|0.2426|0.2206|0.2259|0.2899|0.2794|0.2624|0.3446|0.3395|0.3633|0.4003|0.3459|0.4928|0.3721|0.498 |0.5839|0.6124|0.4458|0.4549|0.5606|0.5473|0.5326|0.4265|0.386 |0.4881|0.5248|0.5416|0.4818|0.3982|0.3758|0.3449|0.3368|0.3005|
|29M00000004874430|9d5a5c039d8a598cc95e23176e015e9d5856ad31dd|0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |
|29M00000221271479|52da75c8bc8a598ccb54f3d81b015552da33ec5fe8|0.0286|0.029 |0.0365|0.0262|0.028 |0.0302|0.0342|0.0317|0.0224|0.025 |0.0252|0.0303|0.0196|0.0246|0.0192|0.0235|0.0254|0.0282|0.0314|0.0302|0.0256|0.021 |0.0233|0.0314|0.0357|0.1333|0.0816|0.0407|0.0505|0.0542|0.087 |0.0969|0.0582|0.1384|0.073 |0.0497|0.1763|0.1211|0.0271|0.026 |0.0873|0.0523|0.0377|0.0374|0.0376|0.0441|0.0643|0.0488|0.0763|0.1291|0.0569|0.0909|0.0591|0.0446|0.0623|0.074 |0.0409|0.0307|0.04  |0.0428|0.0434|0.0448|0.0444|0.0372|0.0296|0.0317|0.0371|0.0317|0.0961|0.036 |0.0435|0.0403|0.0368|0.0747|0.1287|0.1266|0.1476|0.0481|0.0633|0.121 |0.0739|0.1684|0.0666|0.0717|0.0622|0.0534|0.0472|0.0516|0.0614|0.0448|0.0427|0.0502|0.0432|0.0395|0.0346|0.0394|
|02017000683449090|2017000683449090                          |0.1074|0.0981|0.0968|0.1162|0.0978|0.1022|0.0722|0.0568|0.0793|0.0847|0.0818|0.0862|0.073 |0.0679|0.0731|0.0764|0.083 |0.0631|0.074 |0.0701|0.062 |0.0639|0.0787|0.0737|0.0913|0.1311|0.1054|0.1021|0.0922|0.1063|0.1762|0.1906|0.2364|0.2778|0.3286|0.3077|0.2868|0.2545|0.3117|0.2872|0.2956|0.2975|0.2687|0.3456|0.3678|0.3376|0.3037|0.2592|0.2628|0.2132|0.2242|0.2726|0.2673|0.3959|0.3079|0.254 |0.3684|0.3243|0.3113|0.2624|0.2807|0.2985|0.2876|0.2582|0.3274|0.3354|0.2927|0.3177|0.3147|0.2938|0.2352|0.2488|0.2414|0.258 |0.1995|0.1808|0.149 |0.1689|0.1326|0.1732|0.1374|0.138 |0.1297|0.1289|0.1349|0.116 |0.1263|0.0994|0.1038|0.1042|0.1305|0.1128|0.0961|0.1097|0.1105|0.0888|
|29M00000007725181|26CBB0AD-705C-431B-9C24-C7AAB686D644-77331|0.0361|0.0395|0.0669|0.0441|0.0286|0.0384|0.032 |0.0384|0.0424|0.0346|0.0398|0.0395|0.0314|0.0447|0.0487|0.0661|0.0464|0.0323|0.0404|0.0312|0.043 |0.0661|0.0358|0.0467|0.0363|0.0482|0.0415|0.0479|0.0438|0.0603|0.0637|0.0329|0.1462|0.0424|0.0389|0.0678|0.026 |0.0312|0.032 |0.03  |0.0225|0.0323|0.0323|0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |
|29M00000006466858|FC8BC5F3-38E6-3B25-CE04-30AD80302B25-12345|0.0689|0.062 |0.0845|0.0623|0.0675|0.0565|0.1379|0.0663|0.0643|0.0672|0.0603|0.0764|0.0551|0.0557|0.0516|0.0499|0.0513|0.0508|0.0499|0.056 |0.0669|0.1004|0.0811|0.0562|0.0568|0.0672|0.0738|0.0669|0.0577|0.1837|0.3366|0.1832|0.169 |0.0764|0.0741|0.1131|0.197 |0.1174|0.0675|0.2498|0.1102|0.1142|0.1102|0.1033|0.0799|0.0516|0.0603|0.2227|0.2143|0.2489|0.1765|0.2449|0.1359|0.1912|0.1154|0.1826|0.1116|0.2962|0.0888|0.2126|0.06  |0.133 |0.1967|0.0678|0.0998|0.1653|0.1886|0.1618|0.1385|0.1434|0.086 |0.1814|0.152 |0.2466|0.2244|0.2409|0.1021|0.1292|0.1347|0.1814|0.2184|0.304 |0.3707|0.3597|0.1762|0.1376|0.1422|0.1312|0.1217|0.1134|0.1085|0.1134|0.1165|0.1736|0.1858|0.116 |
|02017000654950488|2017000654950488                          |0.0047|0.0048|0.0519|0.0067|0.0047|0.0047|0.0047|0.0046|0.0066|0.0092|0.0105|0.0045|0.0045|0.0546|0.0045|0.0064|0.0045|0.0045|0.0067|0.0045|0.0064|0.0061|0.006 |0.0532|0.006 |0.008 |0.006 |0.0057|0.0656|0.161 |0.0605|0.1067|0.004 |0.0058|0.0087|0.0081|0.0727|0.0126|0.0127|0.0146|0.0127|0.0129|0.0148|0.0129|0.0177|0.019 |0.0168|0.0131|0.0153|0.0136|0.0117|0.0176|0.0318|0.0201|0.0111|0.007 |0.1257|0.0251|0.011 |0.0104|0.0066|0.0051|0.0091|0.0051|0.0093|0.0087|0.0193|0.0073|0.0587|0.0051|0.0095|0.0088|0.0051|0.0052|0.0057|0.0143|0.0202|0.0249|0.0193|0.0223|0.02  |0.0221|0.0154|0.0149|0.0188|0.0133|0.0156|0.0135|0.0134|0.0135|0.0135|0.02  |0.0135|0.0136|0.016 |0.0136|
|02017000670743633|2017000670743633                          |1.2915|1.2935|1.292 |1.3625|1.2881|1.276 |1.2831|1.284 |1.2827|1.3031|1.2763|1.2894|1.2803|1.2807|1.2794|1.2798|1.2912|1.2788|1.2791|1.279 |1.2767|1.2784|1.2813|1.2743|1.2812|1.2796|1.3005|1.2901|1.3062|1.2855|1.3491|1.3785|1.3067|1.2831|1.3661|1.3104|1.2837|1.358 |1.3019|1.3049|1.3198|1.3314|1.3389|1.3311|1.3365|1.3305|1.3451|1.3087|1.3101|1.4937|1.3035|1.3913|1.353 |1.3751|1.2972|1.3046|1.3303|1.3868|1.3252|1.3047|1.326 |1.348 |1.3795|1.3087|1.3446|1.3234|1.3368|1.3711|1.3502|1.3303|1.3232|1.3345|1.3797|1.3058|1.2927|1.335 |1.2996|1.3098|1.3113|1.3414|1.3867|1.3219|1.3693|1.4876|1.3362|1.3667|1.416 |1.3782|1.3501|1.3926|1.3299|1.3225|1.2937|1.3303|1.2978|1.3075|
|02019000890401553|2019000890401553                          |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |0.0   |
|02010000237652468|2010000237652468                          |0.0385|0.0469|0.3381|0.3676|0.0385|0.0385|0.0474|0.0385|0.0385|0.3557|0.3729|0.2349|0.0385|0.0385|0.047 |0.0473|0.1752|0.361 |0.3679|0.0384|0.0385|0.0384|0.0468|0.1537|0.3582|0.3652|0.0384|0.0383|0.0383|0.0382|0.0465|0.3453|0.3586|0.3702|0.0387|0.0547|0.038 |0.038 |0.1716|0.3654|0.3647|0.0378|0.0379|0.0451|0.0464|0.0454|0.3662|0.3667|0.0469|0.0382|0.0471|0.0382|0.047 |0.2612|0.3737|0.2312|0.0477|0.0378|0.0378|0.0377|0.1737|0.3201|0.3722|0.0377|0.0377|0.0377|0.0377|0.1689|0.3598|0.3672|0.0378|0.0395|0.0534|0.0385|0.0386|0.3657|0.3686|0.0386|0.0474|0.0385|0.047 |0.0386|0.2722|0.3668|0.3758|0.0386|0.0387|0.0387|0.0472|0.1752|0.3624|0.3686|0.0381|0.0469|0.0384|0.0379|
+-----------------+------------------------------------------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+------+

3 列转行之后

//这是一个自定函数 作用是将string类型的数据转换成Double类型的数组
val myfunction: UserDefinedFunction = udf[ArrayBuffer[Double],String](xu)
    //列转行
    val result2: DataFrame = result1
      .select($"CIS_TRAN_ID"
        , $"TRAN_ID"
        //这个方法的作用是将里面字段的值组成一个新的结构体，作为一个新的字段的值
        , struct("1",
          "2",
          "3",
          "4",
          "5",
          "6",
          "7",
          "8",
          "9",
          "10",
          "11",
          "12",
          "13",
          "14",
          "15",
          "16",
          "17",
          "18",
          "19",
          "20",
          "21",
          "22",
          "23",
          "24",
          "25",
          "26",
          "27",
          "28",
          "29",
          "30",
          "31",
          "32",
          "33",
          "34",
          "35",
          "36",
          "37",
          "38",
          "39",
          "40",
          "41",
          "42",
          "43",
          "44",
          "45",
          "46",
          "47",
          "48",
          "49",
          "50",
          "51",
          "52",
          "53",
          "54",
          "55",
          "56",
          "57",
          "58",
          "59",
          "60",
          "61",
          "62",
          "63",
          "64",
          "65",
          "66",
          "67",
          "68",
          "69",
          "70",
          "71",
          "72",
          "73",
          "74",
          "75",
          "76",
          "77",
          "78",
          "79",
          "80",
          "81",
          "82",
          "83",
          "84",
          "85",
          "86",
          "87",
          "88",
          "89",
          "90",
          "91",
          "92",
          "93",
          "94",
          "95",
          "96"
        ).cast("string") as "point")
      .withColumn("point",myfunction($"point"))
      .persist(StorageLevel.MEMORY_AND_DISK_SER)

    result2.printSchema()

  //root
 //   |-- CIS_TRAN_ID: string (nullable = true)
 //   |-- TRAN_ID: string (nullable = true)
//    |-- point: array (nullable = true)
//    |    |-- element: double (containsNull = false)

    logPrint("result2", result2)
    result2.show(10, false)
======================「result2:313」 ======================
+-----------------+------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|CIS_TRAN_ID      |TRAN_ID                                   |point                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     |
+-----------------+------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
|29M00000003398208|C762D418-8607-44DE-9C9E-1F76857FB676-25483|[0.3073, 0.3426, 0.2789, 0.2944, 0.2351, 0.2546, 0.2432, 0.2155, 0.2241, 0.2346, 0.1888, 0.1848, 0.2148, 0.1742, 0.1645, 0.2082, 0.1522, 0.1618, 0.1994, 0.1739, 0.1545, 0.1552, 0.2159, 0.1683, 0.1667, 0.1886, 0.2608, 0.1996, 0.2276, 0.204, 0.1818, 0.204, 0.2135, 0.2055, 0.1933, 0.2753, 0.3206, 0.2834, 0.3118, 0.3033, 0.329, 0.4017, 0.3695, 0.4194, 0.3665, 0.2686, 0.4175, 0.4036, 0.327, 0.3658, 0.3118, 0.3815, 0.3552, 0.3957, 0.3144, 0.287, 0.2645, 0.2703, 0.236, 0.3311, 0.2335, 0.2885, 0.355, 0.2412, 0.2426, 0.2206, 0.2259, 0.2899, 0.2794, 0.2624, 0.3446, 0.3395, 0.3633, 0.4003, 0.3459, 0.4928, 0.3721, 0.498, 0.5839, 0.6124, 0.4458, 0.4549, 0.5606, 0.5473, 0.5326, 0.4265, 0.386, 0.4881, 0.5248, 0.5416, 0.4818, 0.3982, 0.3758, 0.3449, 0.3368, 0.3005]   |
|29M00000004874430|9d5a5c039d8a598cc95e23176e015e9d5856ad31dd|[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]                                                                                                                                                                                                                                                                                          |
|29M00000221271479|52da75c8bc8a598ccb54f3d81b015552da33ec5fe8|[0.0286, 0.029, 0.0365, 0.0262, 0.028, 0.0302, 0.0342, 0.0317, 0.0224, 0.025, 0.0252, 0.0303, 0.0196, 0.0246, 0.0192, 0.0235, 0.0254, 0.0282, 0.0314, 0.0302, 0.0256, 0.021, 0.0233, 0.0314, 0.0357, 0.1333, 0.0816, 0.0407, 0.0505, 0.0542, 0.087, 0.0969, 0.0582, 0.1384, 0.073, 0.0497, 0.1763, 0.1211, 0.0271, 0.026, 0.0873, 0.0523, 0.0377, 0.0374, 0.0376, 0.0441, 0.0643, 0.0488, 0.0763, 0.1291, 0.0569, 0.0909, 0.0591, 0.0446, 0.0623, 0.074, 0.0409, 0.0307, 0.04, 0.0428, 0.0434, 0.0448, 0.0444, 0.0372, 0.0296, 0.0317, 0.0371, 0.0317, 0.0961, 0.036, 0.0435, 0.0403, 0.0368, 0.0747, 0.1287, 0.1266, 0.1476, 0.0481, 0.0633, 0.121, 0.0739, 0.1684, 0.0666, 0.0717, 0.0622, 0.0534, 0.0472, 0.0516, 0.0614, 0.0448, 0.0427, 0.0502, 0.0432, 0.0395, 0.0346, 0.0394]      |
|02017000683449090|2017000683449090                          |[0.1074, 0.0981, 0.0968, 0.1162, 0.0978, 0.1022, 0.0722, 0.0568, 0.0793, 0.0847, 0.0818, 0.0862, 0.073, 0.0679, 0.0731, 0.0764, 0.083, 0.0631, 0.074, 0.0701, 0.062, 0.0639, 0.0787, 0.0737, 0.0913, 0.1311, 0.1054, 0.1021, 0.0922, 0.1063, 0.1762, 0.1906, 0.2364, 0.2778, 0.3286, 0.3077, 0.2868, 0.2545, 0.3117, 0.2872, 0.2956, 0.2975, 0.2687, 0.3456, 0.3678, 0.3376, 0.3037, 0.2592, 0.2628, 0.2132, 0.2242, 0.2726, 0.2673, 0.3959, 0.3079, 0.254, 0.3684, 0.3243, 0.3113, 0.2624, 0.2807, 0.2985, 0.2876, 0.2582, 0.3274, 0.3354, 0.2927, 0.3177, 0.3147, 0.2938, 0.2352, 0.2488, 0.2414, 0.258, 0.1995, 0.1808, 0.149, 0.1689, 0.1326, 0.1732, 0.1374, 0.138, 0.1297, 0.1289, 0.1349, 0.116, 0.1263, 0.0994, 0.1038, 0.1042, 0.1305, 0.1128, 0.0961, 0.1097, 0.1105, 0.0888]   |
|29M00000007725181|26CBB0AD-705C-431B-9C24-C7AAB686D644-77331|[0.0361, 0.0395, 0.0669, 0.0441, 0.0286, 0.0384, 0.032, 0.0384, 0.0424, 0.0346, 0.0398, 0.0395, 0.0314, 0.0447, 0.0487, 0.0661, 0.0464, 0.0323, 0.0404, 0.0312, 0.043, 0.0661, 0.0358, 0.0467, 0.0363, 0.0482, 0.0415, 0.0479, 0.0438, 0.0603, 0.0637, 0.0329, 0.1462, 0.0424, 0.0389, 0.0678, 0.026, 0.0312, 0.032, 0.03, 0.0225, 0.0323, 0.0323, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]                                                                                                                                                               |
|29M00000006466858|FC8BC5F3-38E6-3B25-CE04-30AD80302B25-12345|[0.0689, 0.062, 0.0845, 0.0623, 0.0675, 0.0565, 0.1379, 0.0663, 0.0643, 0.0672, 0.0603, 0.0764, 0.0551, 0.0557, 0.0516, 0.0499, 0.0513, 0.0508, 0.0499, 0.056, 0.0669, 0.1004, 0.0811, 0.0562, 0.0568, 0.0672, 0.0738, 0.0669, 0.0577, 0.1837, 0.3366, 0.1832, 0.169, 0.0764, 0.0741, 0.1131, 0.197, 0.1174, 0.0675, 0.2498, 0.1102, 0.1142, 0.1102, 0.1033, 0.0799, 0.0516, 0.0603, 0.2227, 0.2143, 0.2489, 0.1765, 0.2449, 0.1359, 0.1912, 0.1154, 0.1826, 0.1116, 0.2962, 0.0888, 0.2126, 0.06, 0.133, 0.1967, 0.0678, 0.0998, 0.1653, 0.1886, 0.1618, 0.1385, 0.1434, 0.086, 0.1814, 0.152, 0.2466, 0.2244, 0.2409, 0.1021, 0.1292, 0.1347, 0.1814, 0.2184, 0.304, 0.3707, 0.3597, 0.1762, 0.1376, 0.1422, 0.1312, 0.1217, 0.1134, 0.1085, 0.1134, 0.1165, 0.1736, 0.1858, 0.116]     |
|02017000654950488|2017000654950488                          |[0.0047, 0.0048, 0.0519, 0.0067, 0.0047, 0.0047, 0.0047, 0.0046, 0.0066, 0.0092, 0.0105, 0.0045, 0.0045, 0.0546, 0.0045, 0.0064, 0.0045, 0.0045, 0.0067, 0.0045, 0.0064, 0.0061, 0.006, 0.0532, 0.006, 0.008, 0.006, 0.0057, 0.0656, 0.161, 0.0605, 0.1067, 0.004, 0.0058, 0.0087, 0.0081, 0.0727, 0.0126, 0.0127, 0.0146, 0.0127, 0.0129, 0.0148, 0.0129, 0.0177, 0.019, 0.0168, 0.0131, 0.0153, 0.0136, 0.0117, 0.0176, 0.0318, 0.0201, 0.0111, 0.007, 0.1257, 0.0251, 0.011, 0.0104, 0.0066, 0.0051, 0.0091, 0.0051, 0.0093, 0.0087, 0.0193, 0.0073, 0.0587, 0.0051, 0.0095, 0.0088, 0.0051, 0.0052, 0.0057, 0.0143, 0.0202, 0.0249, 0.0193, 0.0223, 0.02, 0.0221, 0.0154, 0.0149, 0.0188, 0.0133, 0.0156, 0.0135, 0.0134, 0.0135, 0.0135, 0.02, 0.0135, 0.0136, 0.016, 0.0136]        |
|02017000670743633|2017000670743633                          |[1.2915, 1.2935, 1.292, 1.3625, 1.2881, 1.276, 1.2831, 1.284, 1.2827, 1.3031, 1.2763, 1.2894, 1.2803, 1.2807, 1.2794, 1.2798, 1.2912, 1.2788, 1.2791, 1.279, 1.2767, 1.2784, 1.2813, 1.2743, 1.2812, 1.2796, 1.3005, 1.2901, 1.3062, 1.2855, 1.3491, 1.3785, 1.3067, 1.2831, 1.3661, 1.3104, 1.2837, 1.358, 1.3019, 1.3049, 1.3198, 1.3314, 1.3389, 1.3311, 1.3365, 1.3305, 1.3451, 1.3087, 1.3101, 1.4937, 1.3035, 1.3913, 1.353, 1.3751, 1.2972, 1.3046, 1.3303, 1.3868, 1.3252, 1.3047, 1.326, 1.348, 1.3795, 1.3087, 1.3446, 1.3234, 1.3368, 1.3711, 1.3502, 1.3303, 1.3232, 1.3345, 1.3797, 1.3058, 1.2927, 1.335, 1.2996, 1.3098, 1.3113, 1.3414, 1.3867, 1.3219, 1.3693, 1.4876, 1.3362, 1.3667, 1.416, 1.3782, 1.3501, 1.3926, 1.3299, 1.3225, 1.2937, 1.3303, 1.2978, 1.3075]    |
|02019000890401553|2019000890401553                          |[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]                                                                                                                                                                                                                                                                                          |
|02010000237652468|2010000237652468                          |[0.0385, 0.0469, 0.3381, 0.3676, 0.0385, 0.0385, 0.0474, 0.0385, 0.0385, 0.3557, 0.3729, 0.2349, 0.0385, 0.0385, 0.047, 0.0473, 0.1752, 0.361, 0.3679, 0.0384, 0.0385, 0.0384, 0.0468, 0.1537, 0.3582, 0.3652, 0.0384, 0.0383, 0.0383, 0.0382, 0.0465, 0.3453, 0.3586, 0.3702, 0.0387, 0.0547, 0.038, 0.038, 0.1716, 0.3654, 0.3647, 0.0378, 0.0379, 0.0451, 0.0464, 0.0454, 0.3662, 0.3667, 0.0469, 0.0382, 0.0471, 0.0382, 0.047, 0.2612, 0.3737, 0.2312, 0.0477, 0.0378, 0.0378, 0.0377, 0.1737, 0.3201, 0.3722, 0.0377, 0.0377, 0.0377, 0.0377, 0.1689, 0.3598, 0.3672, 0.0378, 0.0395, 0.0534, 0.0385, 0.0386, 0.3657, 0.3686, 0.0386, 0.0474, 0.0385, 0.047, 0.0386, 0.2722, 0.3668, 0.3758, 0.0386, 0.0387, 0.0387, 0.0472, 0.1752, 0.3624, 0.3686, 0.0381, 0.0469, 0.0384, 0.0379]|
+-----------------+------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+

    val result3: DataFrame = result2
    //具体的列转行 explode函数
    //explode:Creates a new row for each element in the given array or map column
    //所以我大费周章的将 point这个字段转成数组 因为explode只能接受array or map
      .withColumn("P",explode($"point"))
        .drop("point")

    logPrint("result3", result3)
    result3.show(10, false)
======================「result3:30048」 ======================
+-----------------+------------------------------------------+------+
|CIS_TRAN_ID      |TRAN_ID                                   |P     |
+-----------------+------------------------------------------+------+
|29M00000003398208|C762D418-8607-44DE-9C9E-1F76857FB676-25483|0.3073|
|29M00000003398208|C762D418-8607-44DE-9C9E-1F76857FB676-25483|0.3426|
|29M00000003398208|C762D418-8607-44DE-9C9E-1F76857FB676-25483|0.2789|
|29M00000003398208|C762D418-8607-44DE-9C9E-1F76857FB676-25483|0.2944|
|29M00000003398208|C762D418-8607-44DE-9C9E-1F76857FB676-25483|0.2351|
|29M00000003398208|C762D418-8607-44DE-9C9E-1F76857FB676-25483|0.2546|
|29M00000003398208|C762D418-8607-44DE-9C9E-1F76857FB676-25483|0.2432|
|29M00000003398208|C762D418-8607-44DE-9C9E-1F76857FB676-25483|0.2155|
|29M00000003398208|C762D418-8607-44DE-9C9E-1F76857FB676-25483|0.2241|
|29M00000003398208|C762D418-8607-44DE-9C9E-1F76857FB676-25483|0.2346|
+-----------------+------------------------------------------+------+



  }

  def xu(cs: String): ArrayBuffer[Double] = {

    var arr01 = ArrayBuffer[Double]()
    val str: String = cs.replace("[","").replace("]","")

    val strings: Array[String] = str.split(",")
    for (i <- 0 to (strings.length-1)) {
      arr01 += strings(i).toDouble
    }
    arr01
  }

BigData_xgs

关注

0
点赞
踩
3

收藏

觉得还不错? 一键收藏
0
评论
spark dataframe 行转列与列转行

1 原始数据logPrint("Curves", Curves)Curves.show(10, false) ======================「Curves:30048」 ====================== +-----------------+----------------+------+-----+ |CIS_TRAN_ID |TRAN_ID |P |point| +-----------------+--
复制链接

扫一扫