pyspark 文章画像和用户画像（二）

文章特征

文章特征包括：
文章关键词权重、文章频道、文章向量
其中文章关键词和文章向量的求取可参考https://blog.csdn.net/qq_29153321/article/details/104680282

一、获取文章频道、k个关键词权重

# 文章关键词数据：article_id,channel_id,keywords，topics
article_profile = spark.sparkContext.textFile(r'data\backup\article.db\article_profile')
# article_profile = spark.sparkContext.textFile(r'article_profile')
article_profile = article_profile.map(lambda line: line.split('\x01'))
print("文章原始数据", article_profile.take(10))

# 每篇文章取关键词最高k个权重
def article_profile_to_feature(row):
    try:
        article_weights = list(row[2].split('\x02'))
        article_weights = dict((i.split('\x03')[0], i.split('\x03')[1]) for i in article_weights)
        article_weights = sorted(article_weights.values(), reverse=True)[:10]
    except Exception as e:
        article_weights = [0.0] * 10
    return row[0], row[1], article_weights

article_profile = article_profile.map(article_profile_to_feature).toDF(['article_id', 'channel_id', 'weights'])
# print("文章权重", article_profile.take(10))

二、获取文章向量

article_vector = spark.sparkContext.textFile(r'data\backup\article.db\article_vector')
article_vector = article_vector.map(lambda line: line.split('\x01')).toDF(['article_id', 'channel_id', 'articlevector'])
# print("文章向量", article_vector.take(10))

article_feature = article_profile.join(article_vector, on=['article_id'], how='inner')


# 转化为Vector
def feature_to_vector(row):
    from pyspark.ml.linalg import Vectors

    return row[0], int(row[1]), Vectors.dense(row[2]), Vectors.dense(row[4].split('\x02'))


article_feature = article_feature.rdd.map(feature_to_vector).toDF(['article_id', 'channel_id', 'weights', 'articlevector'])

三、将所有文章特征合并成一列
文章特征为1+10+100=111维

from pyspark.ml.feature import VectorAssembler

columns = ['article_id', 'channel_id', 'weights', 'articlevector']
article_feature = VectorAssembler().setInputCols(columns[1:4]).setOutputCol("article_features").transform(article_feature).drop('channel_id','weights','articlevector')
print("文章特征", article_feature.take(10))
# 文章特征 [Row(article_id='104454', article_features=DenseVector([15.0, 3.2094, 3.2039, 2.9953, 2.0482, 1.9617, 1.6548, 1.3788, 1.3693, 1.2726, 1.0749, 0.0779, -0.0466, -0.1271, -0.0513, -0.0978, 0.2169, 0.1831, -0.0239, -0.0674, 0.0952, 0.0276, 0.2655, -0.1464, -0.2151, 0.0582, 0.1312, -0.0142, -0.127, 0.1623, 0.0151, -0.0363, 0.0072, -0.057, -0.1306, -0.1808, -0.1364, 0.1409, 0.1924, 0.1071, 0.0211, -0.0334, 0.0638, -0.0415, -0.1193, -0.0863, -0.0558, -0.132, 0.0259, -0.0743, 0.1589, 0.0283, 0.1039, 0.0538, 0.2827, -0.172, 0.0103, -0.0252, -0.0788, 0.0087, 0.1353, -0.1259, -0.2982, 0.1853, 0.1347, 0.1359, 0.0385, -0.1453, 0.1341, -0.0524, -0.0562, -0.3002, 0.0933, 0.1789, -0.04, 0.0692, 0.0079, 0.3347, -0.1194, -0.0793, -0.1307, 0.0931, 0.1296, -0.2999, -0.1489, -0.155, 0.4116, -0.0542, 0.1598, 0.1106, -0.0151, -0.0186, 0.1038, 0.0456, 0.1378, 0.2447, -0.1183, -0.1822, 0.0063, 0.0177, 0.0143, 0.2606, -0.1036, -0.0728, 0.0099, 0.1133, 0.0096, 0.0549, 0.1118, -0.0809, -0.1995])), Row(article_id='11078', article_features=DenseVector([19.0, 4.3524, 2.6313, 1.8421, 1.7456, 1.708, 1.3605, 1.2479, 1.2043, 1.1133, 0.8934, 0.0921, 0.1117, -0.2751, 0.0562, 0.2314, -0.2011, -0.1391, -0.0455, 0.0326, -0.3225, 0.0686, 0.1775, -0.1634, 0.0508, -0.0243, -0.0589, -0.1448, 0.0018, -0.0765, 0.2269, 0.1435, 0.0894, 0.205, -0.1244, -0.4143, -0.2649, 0.2978, 0.1372, -0.2373, 0.0868, -0.1164, 0.1571, 0.0712, -0.4209, -0.1384, 0.059, -0.2506, 0.0653, -0.1703, -0.0296, 0.2106, -0.1498, -0.0433, -0.074, -0.1136, 0.0374, 0.1584, -0.4794, -0.0552, 0.0386, 0.1024, 0.0608, 0.1539, -0.1179, -0.2023, -0.145, -0.0226, 0.1391, -0.141, 0.091, -0.1656, 0.0819, 0.2499, -0.176, 0.0247, 0.0642, -0.2663, -0.1592, -0.0568, 0.0102, -0.243, -0.1601, 0.1592, -0.0506, 0.1318, -0.0145, -0.0471, -0.2726, 0.106, -0.2138, -0.0267, 0.1777, -0.1218, 0.1902, 0.0832, 0.108, 0.3634, 0.1479, -0.1266, 0.2028, 0.1931, -0.0179, -0.1039, -0.2265, -0.057, -0.478, -0.116, -0.3476, 0.2256, 0.0315])), Row(article_id='112261', article_features=DenseVector([19.0, 6.9986, 6.7458, 4.3174, 4.0892, 3.8024, 3.7027, 3.5808, 3.1863, 3.1441, 2.8555, 0.2672, -0.015, -0.5429, 0.2897, 0.5796, 0.219, 0.0784, -0.0887, 0.426, -0.4749, 0.0278, -0.1603, -0.5976, -0.092, -0.0429, -0.3129, -0.0699, -0.6033, -0.8889, 0.3618, -0.2609, 0.4682, 0.702, 0.4104, -0.5749, 0.2777, -0.0784, 0.386, 0.4163, 0.5496, -1.0026, 0.0563, 0.1473, 0.2207, 0.4276, -0.0062, 0.0977, 0.3181, -0.5622, 0.0398, -0.407, -0.1554, -0.367, -0.6229, 0.4178, -0.5088, 0.1394, 0.4287, -0.2425, -0.1597, -0.2231, 0.2254, 0.3132, 0.4591, -0.5523, 0.0653, -0.3522, -0.3025, 0.8189, -0.1365, 0.8583, -0.0341, 0.5411, -0.0189, -0.7076, 0.4366, 0.3908, 0.2022, -0.2131, -0.3759, 0.2754, -0.1044, 0.8178, 0.0268, 0.0513, 0.3592, -0.4084, 0.1466, -0.1035, 0.9015, -0.0975, -0.3203, 0.076, -0.0188, -0.3406, 0.0043, 0.7749, 0.1, -0.1054, 0.9776, -0.2702, -0.4802, -0.8697, -0.0798, -0.1149, -0.154, 0.1037, 0.4164, 0.1837, -0.3219])), Row(article_id='113417', article_features=DenseVector([19.0, 2.7855, 2.4906, 2.3413, 2.2181, 2.0936, 2.0288, 1.9961, 1.8992, 1.779, 1.5828, -0.0298, 0.1844, -0.1527, 0.2177, -0.0928, 0.1654, 0.0017, -0.1639, -0.0378, -0.058, 0.2614, 0.1186, -0.0639, -0.1151, -0.1751, -0.2461, 0.1065, 0.099, 0.0717, -0.0506, -0.1597, -0.1328, 0.1314, -0.1697, 0.3646, 0.1417, 0.2085, 0.2381, -0.0275, -0.1403, -0.1953, -0.3581, -0.0099, -0.2884, -0.0762, 0.1037, -0.001, -0.0134, -0.2092, 0.2685, -0.2462, 0.3999, -0.0012, 0.156, -0.2364, 0.3707, 0.1092, 0.1387, 0.0823, 0.0058, 0.0016, -0.0677, 0.252, 0.1782, 0.0018, -0.1861, 0.1731, -0.1112, 0.3227, -0.3457, -0.1461, -0.0333, -0.0747, -0.018, 0.2252, 0.0057, 0.0683, 0.2159, -0.0124, 0.1717, -0.1419, 0.1799, -0.2844, -0.1427, -0.1597, -0.337, 0.1603, 0.1057, -0.0095, -0.061, -0.1318, -0.1161, 0.2994, 0.2736, 0.1247, -0.2224, -0.1255, -0.3158, -0.0457, -0.2657, -0.3013, -0.1367, 0.145, -0.0659, -0.138, 0.0018, -0.1144, -0.092, -0.004, -0.2964])), Row(article_id='115722', article_features=DenseVector([19.0, 5.6859, 3.7446, 3.5655, 2.3104, 2.3089, 1.8003, 1.5114, 1.4648, 1.3759, 1.3268, 0.0901, -0.0201, -0.4115, 0.2405, 0.5276, -0.1327, -0.4406, -0.1985, -0.1297, 0.0191, 0.1782, -0.2119, -0.3124, -0.001, -0.1857, -0.0555, 0.1649, 0.0812, -0.1855, 0.3606, -0.2043, 0.2093, 0.0622, 0.2577, -0.0443, 0.001, -0.3585, 0.3761, -0.4314, -0.1804, -0.1869, 0.2003, -0.1028, -0.0559, -0.6038, 0.0253, -0.078, -0.1018, -0.025, 0.0137, 0.0066, -0.2711, 0.0667, -0.3556, -0.013, -0.1323, 0.4523, 0.0631, -0.1825, -0.1171, 0.0118, -0.266, 0.1047, -0.1262, -0.1043, -0.042, -0.2069, -0.142, 0.2523, 0.1945, 0.5276, -0.1675, 0.2908, -0.3185, -0.0355, -0.2327, 0.0476, 0.0345, 0.2535, -0.0535, -0.1166, -0.1315, -0.0552, -0.2849, -0.0781, 0.0929, -0.308, 0.1491, 0.055, 0.4864, -0.031, -0.1496, -0.1269, -0.0451, 0.0547, -0.2267, 0.293, 0.115, 0.1512, 0.5928, 0.2199, 0.0512, -0.1089, 0.2852, 0.6087, -0.1846, -0.2396, -0.1107, 0.3475, -0.1342])), Row(article_id='117630', article_features=DenseVector([18.0, 3.2094, 2.8729, 2.3898, 2.3177, 1.8147, 1.7998, 1.7648, 1.6284, 1.5127, 1.4225, -0.0538, -0.0768, 0.1833, 0.0466, -0.1089, 0.0345, -0.1708, -0.0353, 0.1568, -0.2158, -0.1089, -0.0345, 0.0828, 0.0192, 0.2037, -0.0531, -0.0118, 0.1785, -0.0062, -0.0975, -0.1063, 0.0481, -0.0017, -0.0836, -0.0247, -0.0351, -0.0074, 0.0669, -0.0032, -0.1687, -0.2153, 0.0899, 0.2705, 0.0298, 0.0909, -0.1071, 0.0713, -0.1074, -0.0697, 0.1991, 0.2544, -0.1318, 0.1003, 0.1545, -0.1852, -0.2174, 0.0665, 0.0743, -0.1158, 0.0718, -0.0107, 0.019, -0.0768, -0.1513, 0.0891, 0.0016, -0.3053, 0.0644, 0.0915, 0.0967, -0.0474, -0.0798, 0.0307, -0.25, 0.0339, 0.0657, 0.0784, -0.2457, 0.0702, 0.1407, -0.1035, -0.1085, -0.0559, -0.0471, 0.0617, -0.0366, -0.0106, -0.0236, 0.028, -0.0997, -0.0598, 0.0276, -0.2518, 0.057, -0.1273, 0.0526, -0.1048, -0.119, 0.0663, 0.1958, 0.4176, 0.0814, 0.1912, 0.0623, -0.0552, -0.0778, -0.0528, -0.1847, -0.024, 0.2985])), Row(article_id='119461', article_features=DenseVector([22.0, 4.7052, 4.0736, 3.7461, 3.7127, 2.6155, 2.4923, 2.3657, 2.3224, 2.1035, 2.0412, 0.2313, 0.0846, 0.1077, -0.0436, -0.0696, 0.1053, 0.0836, 0.086, 0.3162, -0.1226, 0.294, -0.0902, 0.1681, -0.095, -0.0726, -0.1178, 0.011, 0.0328, -0.0711, -0.0616, 0.0169, -0.038, 0.5369, 0.1958, -0.3401, 0.1829, -0.1733, 0.2435, 0.1673, -0.0613, -0.0666, -0.0395, 0.114, 0.0216, -0.0507, 0.1734, -0.0618, -0.0962, 0.0748, -0.0268, 0.0123, -0.0055, 0.1346, 0.0706, 0.1962, 0.0973, -0.0784, -0.0162, 0.1298, -0.195, 0.0533, -0.2352, 0.3798, -0.1018, -0.0351, -0.17, 0.0935, 0.2375, 0.0221, -0.0784, -0.2106, 0.1597, 0.1108, -0.1046, 0.0189, -0.0967, 0.1163, 0.1526, 0.2088, -0.3541, -0.009, -0.0031, -0.3506, -0.0292, -0.3302, -0.0121, 0.0246, 0.021, -0.0466, 0.011, 0.0307, -0.034, 0.1512, 0.1588, -0.0916, -0.1144, 0.0681, 0.2597, 0.0077, 0.1922, 0.1088, -0.0556, 0.3096, -0.0011, -0.0306, -0.0316, 0.1571, 0.0808, 0.0036, 0.2122])), Row(article_id='119625', article_features=DenseVector([14.0, 6.6868, 2.1803, 1.6385, 1.5585, 1.1332, 1.1292, 1.0648, 0.9441, 0.9112, 0.847, 0.2274, 0.0027, 0.1581, 0.2004, 0.1147, 0.1, 0.0026, 0.0451, 0.0752, -0.1187, -0.1038, 0.2343, -0.0648, -0.0299, -0.0367, 0.1381, -0.0247, 0.0181, 0.1119, 0.2383, -0.3084, -0.1319, 0.0567, -0.1086, 0.0871, -0.1053, 0.1258, -0.1489, 0.1292, -0.345, 0.0593, -0.0612, -0.1155, -0.2328, -0.1287, 0.1569, -0.069, 0.1631, 0.0166, 0.0122, -0.0538, -0.1723, -0.025, 0.1263, 0.161, -0.2254, 0.0802, 0.0076, -0.0287, -0.0893, 0.1625, -0.0175, 0.094, -0.0602, -0.0086, -0.0022, -0.041, -0.0488, -0.0633, 0.1452, -0.2495, -0.0223, 0.0455, 0.0092, -0.1723, 0.0727, -0.0169, -0.2192, -0.1675, 0.1552, -0.0563, -0.1085, -0.1911, 0.1988, -0.2169, -0.0669, 0.3274, 0.1026, 0.1535, 0.0335, 0.1547, 0.0512, -0.1815, -0.1877, -0.3684, 0.0793, 0.0444, 0.2351, -0.2766, -0.0722, -0.0111, 0.1854, 0.0716, 0.0789, -0.1626, 0.0555, -0.0326, -0.2006, 0.0213, 0.0792])), Row(article_id='120478', article_features=DenseVector([14.0, 4.883, 3.0421, 2.9081, 2.6926, 2.6628, 2.4354, 2.2788, 2.2385, 2.2055, 2.1157, 0.1775, 0.0412, 0.2481, -0.0082, 0.1446, -0.1056, -0.1029, 0.1595, 0.2366, -0.0192, 0.0454, -0.1919, -0.0384, 0.1176, -0.3365, -0.1762, 0.0891, 0.1654, -0.0951, -0.0456, 0.0075, -0.0786, 0.0896, -0.0216, 0.1773, 0.1729, 0.0378, 0.0203, 0.19, -0.0377, -0.0762, 0.0399, 0.1559, 0.1463, 0.1298, 0.0003, 0.0299, 0.0338, 0.1274, -0.0216, -0.2186, -0.0783, -0.0049, -0.0058, 0.0427, -0.1087, 0.1767, 0.1692, 0.0636, -0.1144, 0.0077, 0.0367, -0.1269, 0.1001, -0.0116, -0.0434, -0.1044, -0.0063, -0.1787, 0.115, 0.072, 0.0085, -0.0191, -0.1266, -0.0386, 0.0234, -0.0595, -0.1128, 0.0137, 0.0861, -0.0041, 0.0365, -0.1963, 0.1349, -0.0227, 0.1432, -0.0216, -0.0033, -0.0898, -0.0825, 0.137, 0.0864, -0.0345, -0.2399, -0.0038, 0.0524, -0.0676, 0.1706, -0.0474, -0.0872, -0.0323, -0.0652, 0.2064, 0.1674, 0.0156, -0.0713, -0.1472, -0.0351, -0.0888, 0.288])), Row(article_id='121867', article_features=DenseVector([9.0, 6.5255, 2.722, 1.0556, 0.9091, 0.8219, 0.5892, 0.5599, 0.5033, 0.5023, 0.4794, 0.0723, 0.1117, -0.167, -0.1872, 0.0355, -0.0896, -0.2632, -0.1381, -0.0657, 0.1413, 0.107, -0.2137, 0.1305, 0.1737, 0.048, 0.1009, 0.3092, 0.0021, 0.0745, 0.144, 0.0099, -0.1198, -0.0918, 0.0268, -0.1822, 0.3223, 0.1144, -0.0515, 0.0231, 0.036, -0.0902, -0.2924, -0.0907, 0.135, -0.0776, 0.0826, 0.1058, -0.0482, 0.0491, 0.4293, 0.1711, -0.2485, -0.1281, -0.1473, -0.0147, 0.1557, 0.1302, -0.0119, 0.0774, -0.2075, 0.1212, 0.0114, 0.1829, 0.0471, 0.0695, 0.2132, -0.1485, 0.0977, -0.1875, 0.1439, 0.1427, -0.0237, -0.1829, 0.1583, 0.039, 0.1121, 0.1051, -0.0652, 0.0332, 0.1478, -0.0499, -0.2293, -0.0226, -0.0338, -0.1036, 0.1063, -0.0885, -0.1743, -0.0995, -0.1402, -0.0352, 0.0531, 0.054, 0.1145, -0.154, 0.0186, -0.1418, -0.044, 0.0371, -0.1269, -0.0838, 0.0357, 0.0751, 0.0195, -0.0793, -0.0792, -0.0121, 0.012, 0.0552, 0.0009]))]

# article_feature.rdd.repartition(1).saveAsTextFile("article_feature/")
article_feature.coalesce(2).write.parquet(path='datas/article_features/',mode='overwrite')

用户特征

读取固定时间内的用户行为日志
首先根据用户行为得分方案，求出用户对每频道各主题词得分，用户特征为各频道最高k个得分

一、用户文章行为信息联合文章主题信息

# 用户文章行为数据：user_id, action_time, article_id, channel_id, shared, clicked, collected, exposure, read_time
user_article_basic = spark.sparkContext.textFile(r'data\backup\profile.db\user_article_basic')
# user_article_basic = spark.sparkContext.textFile(r'user_article_basic')

# 处理文本导致的字符串问题
def article_basic(row):
    row = row.split('\x01')
    for i in range(len(row)):
        if row[i] == 'false':
            row[i] = False
        elif row[i] == 'true':
            row[i] = True

    return row


user_article_basic = user_article_basic.map(article_basic).toDF(
['user_id', 'action_time', 'article_id', 'channel_id', 'shared', 'clicked', 'collected', 'exposure', 'read_time'])
print("用户文章行为信息", user_article_basic.take(10))


def user_topic(row):
    return row[0], row[3].split('\x02')


article_profile = spark.sparkContext.textFile(r'data\backup\article.db\article_profile')
article_profile = article_profile.map(lambda line: line.split('\x01'))
article_topic = article_profile.map(user_topic).toDF(['article_id', 'topics'])
print("文章主题信息", article_topic.take(10))
user_article_topic = user_article_basic.join(article_topic, how='left', on=['article_id'])
print("用户行为联合文章主题信息", user_article_topic.take(20))

import pyspark.sql.functions as F

# explode主题词拆开为一行
user_article_topic = user_article_topic.withColumn('topic', F.explode('topics')).drop('topics')
print(user_article_topic.take(10))

二、用户行为得分，各频道内各主题的分数

# 用户行为得分方案
def compute_user_label_weights(partitions):
    """ 计算用户标签权重
    """
    action_weight = {
        "read_min": 1,
        "read_middle": 2,
        "collect": 2,
        "share": 3,
        "click": 5
    }

    from datetime import datetime
    import numpy as np
    import json

    # 循环处理每个用户对应的每个主题词
    for row in partitions:
        # 计算时间衰减系数
        t = datetime.now() - datetime.strptime(row.action_time, '%Y-%m-%d %H:%M:%S')
        alpha = 1 / (np.log(t.days + 1) + 1)

        if row.read_time == '':
            read_t = 0
        else:
            read_t = int(row.read_time)

        # 计算阅读时间的行为分数
        read_score = action_weight['read_middle'] if read_t > 1000 else action_weight['read_min']

        # 计算各种行为的权重和并乘以时间衰减系数
        weights = alpha * (row.shared * action_weight['share'] + row.clicked * action_weight['click'] + row.collected *
                           action_weight['collect'] + read_score)

        yield row.user_id, {row.channel_id + ':' + row.topic: json.dumps(weights)}


user_profile = user_article_topic.rdd.mapPartitions(compute_user_label_weights).toDF(["user_id", "weights"])
user_profile = user_profile.groupBy("user_id").agg(F.collect_list('weights').alias('channel_weights'))
print("用户行为得分权重", user_profile.take(10))

三、用户各频道内选取最高k个得分最为用户特征

# 选取每个频道内最高k个标签特征
def frature_preprocess(row):
    user_weights = []
    for i in range(1, 26):
        try:
            initdict = {}
            for item in row.channel_weights:
                weights = dict(initdict, **item)
                initdict = weights

            channel_weights = sorted([float(weights[key]) for key in weights.keys() if key.split(':')[0] == str(i)], reverse=True)[:10]
            if len(channel_weights) < 10:
                channel_weights.extend([0.0] * (10 - len(channel_weights)))
            user_weights.append(channel_weights)
        except:
            user_weights.append([0.0] * 10)
    return row.user_id, user_weights


user_feature = user_profile.rdd.map(frature_preprocess).toDF(['user_id', 'user_features'])
print("用户特征", user_feature.take(10))
# 用户特征 [Row(user_id='1106396183141548032', user_features=[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]), Row(user_id='1111524501104885760', user_weights=[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]), Row(user_id='1111189494544990208', user_weights=[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]), Row(user_id='1108264901190615040', user_weights=[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]), Row(user_id='10', user_weights=[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]), Row(user_id='1112727762809913344', user_weights=[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.3156660516340701, 1.3156660516340701, 1.3156660516340701, 1.3156660516340701, 1.3156660516340701, 1.3156660516340701, 1.3156660516340701, 1.3156660516340701, 1.3156660516340701, 1.3156660516340701], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]), Row(user_id='4', user_weights=[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]]), Row(user_id='2', user_weights=[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.8702711669501366, 0.8702711669501366, 0.8702711669501366, 0.8702711669501366, 0.8702711669501366, 0.8702711669501366, 0.8702711669501366, 0.8702711669501366, 0.8702711669501366, 0.8702711669501366], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]])]

user_feature.coalesce(2).write.parquet(path='datas/user_features/',mode='overwrite')