#user_latest_hist_src = 5882
#user_latest_hist_tgt = 5824
#tgt(train,test) = 431110,5255600
max_item_id_tgt = 3952
max_user_id_tgt = 6040
#train_set_tgt = 431110
#src(train) = 587726
max_item_id_src = 3947
max_user_id_src = 6040
#users_train_transfer = 5824
percent_users_transfer = 1.0
#test_set_users_attack = 1098
---gender---
#user_gender_map = 5967
---age---
#user_age_map = 5967
---occupation---
#user_occupation_map = 5967
{'adversary_weight': 1.0,
 'clip_norm': 5,
 'clip_norm_attack': 10,
 'data_name_source': 'source',
 'data_name_target': 'target',
 'data_user_age_map': '../user_age_map.data',
 'data_user_gender_map': '../user_gender_map.data',
 'data_user_occupation_map': '../user_occupation_map.data',
 'fc_layer': 64,
 'hidden_units': 80,
 'lr': 0.0005,
 'lr_attack': 0.001,
 'nepoch': 50,
 'num_classes_age': 3,
 'num_classes_gender': 2,
 'num_classes_occupation': 21,
 'ratio_train_tgt': 1.0,
 'test_batch_size': 4098,
 'test_batch_size_attack': 1024,
 'topKs': [1, 5, 10, 20, 35],
 'train_attack_batch_size': 128,
 'train_batch_size': 128,
 'train_trans_batch_size': 128}
defaultdict(<class 'set'>,
            {'age': {0, 1, 2},
             'gender': {0, 1},
             'occupation': {0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11,
                            12,
                            13,
                            14,
                            15,
                            16,
                            17,
                            18,
                            19,
                            20}})
loading done, begin training...[7.64s]
2020-01-15 15:11:58.338216: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA
2020-01-15 15:11:58.573649: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1105] Found device 0 with properties: 
name: TITAN Xp major: 6 minor: 1 memoryClockRate(GHz): 1.582
pciBusID: 0000:88:00.0
totalMemory: 11.91GiB freeMemory: 7.57GiB
2020-01-15 15:11:58.573729: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1195] Creating TensorFlow device (/device:GPU:0) -> (device: 0, name: TITAN Xp, pci bus id: 0000:88:00.0, compute capability: 6.1)
WARNING:tensorflow:From /qydata/ghuac/ml-1m/Adversarial_neural_real/model.py:180: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.
Instructions for updating:

Future major versions of TensorFlow will allow gradients to flow
into the labels input on backprop by default.

See tf.nn.softmax_cross_entropy_with_logits_v2.

#train_set_attack_fake_source = 476036
#train_set_attack_real_source = 476036
Epoch=0, loss_attack_fake_src=0.015964, loss_attack_src=0.001562, [73.95s]
Epoch=0, loss_attack_real=0.000000, gender=0.011265, age=0.012085, occupation=0.021711, [125.26s]
Epoch=0, src_loss=0.001231, [165.43s]
#batches_src = 4591
{'age': defaultdict(<class 'int'>, {0: 626, 1: 220, 2: 252}),
 'gender': defaultdict(<class 'int'>, {1: 796, 0: 302}),
 'occupation': defaultdict(<class 'int'>,
                           {0: 136,
                            1: 89,
                            2: 53,
                            3: 31,
                            4: 141,
                            5: 16,
                            6: 47,
                            7: 122,
                            8: 3,
                            9: 17,
                            10: 38,
                            11: 22,
                            12: 74,
                            13: 26,
                            14: 46,
                            15: 25,
                            16: 41,
                            17: 96,
                            18: 11,
                            19: 18,
                            20: 46})}
/qydata/ghuac/home/lib/python3.6/site-packages/scikit_learn-0.19.1-py3.6-linux-x86_64.egg/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.
  'precision', 'predicted', average, warn_for)
---gender: accuracy=0.7213---
micro: 0.7213,0.7213,0.7213
macro: 0.5636,0.5057,0.4432
weighted: 0.6373,0.7213,0.6202
---age: accuracy=0.5692---
micro: 0.5692,0.5692,0.5692
macro: 0.3143,0.3447,0.2758
weighted: 0.4130,0.5692,0.4368
---occupation: accuracy=0.1311---
micro: 0.1311,0.1311,0.1311
macro: 0.0240,0.0536,0.0316
weighted: 0.0571,0.1311,0.0759
#train_set_transfer_tgt = 431110
#batches_transfer_tgt = 3368
Epoch=0, trans_loss_tgt=0.002257, [198.22s]
Epoch=0, tgt_loss=0.002174, [227.51s]
#batches_tgt = 3368
Epoch=0: (AUC,MRR) = 0.9521, 0.6527, [291.29s]
HR=0.5605, NDCG=0.5605 at top-1
HR=0.7585, NDCG=0.6632 at top-5
HR=0.8562, NDCG=0.6948 at top-10
HR=0.9404, NDCG=0.7162 at top-20
HR=0.9828, NDCG=0.7251 at top-35
Epoch=1, loss_attack_fake_src=0.014722, loss_attack_src=0.001209, [364.12s]
Epoch=1, loss_attack_real=0.000000, gender=0.011271, age=0.012120, occupation=0.021693, [416.86s]
Epoch=1, src_loss=0.001180, [456.91s]
---gender: accuracy=0.7240---
micro: 0.7240,0.7240,0.7240
macro: 0.3624,0.4994,0.4200
weighted: 0.5254,0.7240,0.6089
---age: accuracy=0.5701---
micro: 0.5701,0.5701,0.5701
macro: 0.3129,0.3491,0.2855
weighted: 0.4135,0.5701,0.4441
---occupation: accuracy=0.1193---
micro: 0.1193,0.1193,0.1193
macro: 0.0237,0.0490,0.0278
weighted: 0.0569,0.1193,0.0682
Epoch=1, trans_loss_tgt=0.001967, [490.50s]
Epoch=1, tgt_loss=0.001851, [520.32s]
Epoch=1: (AUC,MRR) = 0.9513, 0.6518, [589.82s]
HR=0.5613, NDCG=0.5613 at top-1
HR=0.7546, NDCG=0.6613 at top-5
HR=0.8526, NDCG=0.6930 at top-10
HR=0.9390, NDCG=0.7150 at top-20
HR=0.9828, NDCG=0.7242 at top-35
Epoch=2, loss_attack_fake_src=0.014583, loss_attack_src=0.001150, [660.06s]
Epoch=2, loss_attack_real=0.000000, gender=0.011259, age=0.012082, occupation=0.021695, [713.67s]
Epoch=2, src_loss=0.001121, [754.01s]
---gender: accuracy=0.7259---
micro: 0.7259,0.7259,0.7259
macro: 0.6963,0.5027,0.4270
weighted: 0.7097,0.7259,0.6132
---age: accuracy=0.5792---
micro: 0.5792,0.5792,0.5792
macro: 0.3550,0.3655,0.3130
weighted: 0.4441,0.5792,0.4636
---occupation: accuracy=0.1148---
micro: 0.1148,0.1148,0.1148
macro: 0.0244,0.0471,0.0254
weighted: 0.0579,0.1148,0.0630
Epoch=2, trans_loss_tgt=0.001764, [788.17s]
Epoch=2, tgt_loss=0.001659, [818.25s]
Epoch=2: (AUC,MRR) = 0.9511, 0.6534, [884.44s]
HR=0.5639, NDCG=0.5639 at top-1
HR=0.7559, NDCG=0.6628 at top-5
HR=0.8547, NDCG=0.6948 at top-10
HR=0.9371, NDCG=0.7158 at top-20
HR=0.9824, NDCG=0.7253 at top-35
Epoch=3, loss_attack_fake_src=0.014353, loss_attack_src=0.001088, [956.64s]
Epoch=3, loss_attack_real=0.000000, gender=0.011249, age=0.012078, occupation=0.021668, [1009.17s]
Epoch=3, src_loss=0.001055, [1049.84s]
---gender: accuracy=0.7122---
micro: 0.7122,0.7122,0.7122
macro: 0.4957,0.4994,0.4393
weighted: 0.5987,0.7122,0.6153
---age: accuracy=0.5683---
micro: 0.5683,0.5683,0.5683
macro: 0.3118,0.3528,0.2948
weighted: 0.4136,0.5683,0.4499
---occupation: accuracy=0.1384---
micro: 0.1384,0.1384,0.1384
macro: 0.0245,0.0552,0.0323
weighted: 0.0593,0.1384,0.0797
Epoch=3, trans_loss_tgt=0.001586, [1084.50s]
Epoch=3, tgt_loss=0.001492, [1113.78s]
Epoch=3: (AUC,MRR) = 0.9510, 0.6540, [1183.54s]
HR=0.5642, NDCG=0.5642 at top-1
HR=0.7569, NDCG=0.6637 at top-5
HR=0.8552, NDCG=0.6955 at top-10
HR=0.9374, NDCG=0.7164 at top-20
HR=0.9814, NDCG=0.7257 at top-35
Epoch=4, loss_attack_fake_src=0.014562, loss_attack_src=0.001020, [1254.70s]
Epoch=4, loss_attack_real=0.000000, gender=0.011238, age=0.012062, occupation=0.021652, [1306.83s]
Epoch=4, src_loss=0.000992, [1347.13s]
---gender: accuracy=0.7240---
micro: 0.7240,0.7240,0.7240
macro: 0.3624,0.4994,0.4200
weighted: 0.5254,0.7240,0.6089
---age: accuracy=0.5474---
micro: 0.5474,0.5474,0.5474
macro: 0.2801,0.3406,0.2853
weighted: 0.3887,0.5474,0.4373
---occupation: accuracy=0.1230---
micro: 0.1230,0.1230,0.1230
macro: 0.0215,0.0478,0.0247
weighted: 0.0532,0.1230,0.0625
Epoch=4, trans_loss_tgt=0.001419, [1381.79s]
Epoch=4, tgt_loss=0.001344, [1411.68s]
Epoch=4: (AUC,MRR) = 0.9504, 0.6531, [1482.33s]
HR=0.5627, NDCG=0.5627 at top-1
HR=0.7575, NDCG=0.6633 at top-5
HR=0.8533, NDCG=0.6944 at top-10
HR=0.9368, NDCG=0.7156 at top-20
HR=0.9810, NDCG=0.7249 at top-35
Epoch=5, loss_attack_fake_src=0.014526, loss_attack_src=0.000955, [1553.98s]
Epoch=5, loss_attack_real=0.000000, gender=0.011244, age=0.012082, occupation=0.021649, [1607.05s]
Epoch=5, src_loss=0.000938, [1647.37s]
---gender: accuracy=0.7250---
micro: 0.7250,0.7250,0.7250
macro: 0.3625,0.5000,0.4203
weighted: 0.5256,0.7250,0.6094
---age: accuracy=0.5656---
micro: 0.5656,0.5656,0.5656
macro: 0.3000,0.3496,0.2903
weighted: 0.4055,0.5656,0.4467
---occupation: accuracy=0.1357---
micro: 0.1357,0.1357,0.1357
macro: 0.0371,0.0531,0.0309
weighted: 0.0811,0.1357,0.0756
Epoch=5, trans_loss_tgt=0.001267, [1681.55s]
Epoch=5, tgt_loss=0.001215, [1710.86s]
Epoch=5: (AUC,MRR) = 0.9503, 0.6541, [1778.50s]
HR=0.5640, NDCG=0.5640 at top-1
HR=0.7564, NDCG=0.6636 at top-5
HR=0.8546, NDCG=0.6954 at top-10
HR=0.9365, NDCG=0.7163 at top-20
HR=0.9812, NDCG=0.7256 at top-35
Epoch=6, loss_attack_fake_src=0.014718, loss_attack_src=0.000904, [1849.51s]
Epoch=6, loss_attack_real=0.000000, gender=0.011245, age=0.012089, occupation=0.021647, [1901.81s]
Epoch=6, src_loss=0.000896, [1942.26s]
---gender: accuracy=0.7250---
micro: 0.7250,0.7250,0.7250
macro: 0.3625,0.5000,0.4203
weighted: 0.5256,0.7250,0.6094
---age: accuracy=0.5683---
micro: 0.5683,0.5683,0.5683
macro: 0.2824,0.3370,0.2567
weighted: 0.3900,0.5683,0.4241
---occupation: accuracy=0.1138---
micro: 0.1138,0.1138,0.1138
macro: 0.0209,0.0462,0.0239
weighted: 0.0497,0.1138,0.0576
Epoch=6, trans_loss_tgt=0.001139, [1976.53s]
Epoch=6, tgt_loss=0.001105, [2005.82s]
Epoch=6: (AUC,MRR) = 0.9497, 0.6533, [2074.74s]
HR=0.5634, NDCG=0.5634 at top-1
HR=0.7548, NDCG=0.6626 at top-5
HR=0.8522, NDCG=0.6941 at top-10
HR=0.9349, NDCG=0.7153 at top-20
HR=0.9805, NDCG=0.7248 at top-35
Epoch=7, loss_attack_fake_src=0.014718, loss_attack_src=0.000862, [2146.20s]
Epoch=7, loss_attack_real=0.000000, gender=0.011250, age=0.012104, occupation=0.021647, [2198.32s]
Epoch=7, src_loss=0.000860, [2237.73s]
---gender: accuracy=0.7250---
micro: 0.7250,0.7250,0.7250
macro: 0.3625,0.5000,0.4203
weighted: 0.5256,0.7250,0.6094
---age: accuracy=0.5710---
micro: 0.5710,0.5710,0.5710
macro: 0.3235,0.3441,0.2725
weighted: 0.4191,0.5710,0.4350
---occupation: accuracy=0.1321---
micro: 0.1321,0.1321,0.1321
macro: 0.0381,0.0520,0.0303
weighted: 0.0842,0.1321,0.0752
Epoch=7, trans_loss_tgt=0.001032, [2270.73s]
Epoch=7, tgt_loss=0.001020, [2299.60s]
Epoch=7: (AUC,MRR) = 0.9494, 0.6530, [2357.78s]
HR=0.5631, NDCG=0.5631 at top-1
HR=0.7546, NDCG=0.6624 at top-5
HR=0.8513, NDCG=0.6938 at top-10
HR=0.9326, NDCG=0.7145 at top-20
HR=0.9799, NDCG=0.7244 at top-35
Epoch=8, loss_attack_fake_src=0.014889, loss_attack_src=0.000825, [2396.68s]
Epoch=8, loss_attack_real=0.000000, gender=0.011244, age=0.012092, occupation=0.021640, [2415.28s]
Epoch=8, src_loss=0.000828, [2439.97s]
---gender: accuracy=0.7250---
micro: 0.7250,0.7250,0.7250
macro: 0.3625,0.5000,0.4203
weighted: 0.5256,0.7250,0.6094
---age: accuracy=0.5628---
micro: 0.5628,0.5628,0.5628
macro: 0.2589,0.3354,0.2598
weighted: 0.3746,0.5628,0.4256
---occupation: accuracy=0.1202---
micro: 0.1202,0.1202,0.1202
macro: 0.0216,0.0478,0.0270
weighted: 0.0513,0.1202,0.0664
Epoch=8, trans_loss_tgt=0.000946, [2462.87s]
Epoch=8, tgt_loss=0.000948, [2481.03s]
Epoch=8: (AUC,MRR) = 0.9490, 0.6542, [2531.43s]
HR=0.5651, NDCG=0.5651 at top-1
HR=0.7559, NDCG=0.6638 at top-5
HR=0.8495, NDCG=0.6941 at top-10
HR=0.9342, NDCG=0.7157 at top-20
HR=0.9798, NDCG=0.7253 at top-35
Epoch=9, loss_attack_fake_src=0.015027, loss_attack_src=0.000793, [2569.46s]
Epoch=9, loss_attack_real=0.000000, gender=0.011249, age=0.012111, occupation=0.021636, [2586.29s]
Epoch=9, src_loss=0.000801, [2610.09s]
---gender: accuracy=0.7250---
micro: 0.7250,0.7250,0.7250
macro: 0.3625,0.5000,0.4203
weighted: 0.5256,0.7250,0.6094
---age: accuracy=0.5674---
micro: 0.5674,0.5674,0.5674
macro: 0.2651,0.3349,0.2518
weighted: 0.3778,0.5674,0.4205
---occupation: accuracy=0.1175---
micro: 0.1175,0.1175,0.1175
macro: 0.0238,0.0486,0.0260
weighted: 0.0579,0.1175,0.0640
Epoch=9, trans_loss_tgt=0.000880, [2633.76s]
Epoch=9, tgt_loss=0.000892, [2649.64s]
Epoch=9: (AUC,MRR) = 0.9488, 0.6526, [2700.26s]
HR=0.5631, NDCG=0.5631 at top-1
HR=0.7541, NDCG=0.6620 at top-5
HR=0.8493, NDCG=0.6929 at top-10
HR=0.9336, NDCG=0.7143 at top-20
HR=0.9791, NDCG=0.7239 at top-35
Epoch=10, loss_attack_fake_src=0.014983, loss_attack_src=0.000772, [2740.46s]
Epoch=10, loss_attack_real=0.000000, gender=0.011249, age=0.012112, occupation=0.021635, [2758.95s]
Epoch=10, src_loss=0.000781, [2784.19s]
---gender: accuracy=0.7250---
micro: 0.7250,0.7250,0.7250
macro: 0.3625,0.5000,0.4203
weighted: 0.5256,0.7250,0.6094
---age: accuracy=0.5710---
micro: 0.5710,0.5710,0.5710
macro: 0.3239,0.3354,0.2476
weighted: 0.4178,0.5710,0.4181
---occupation: accuracy=0.1302---
micro: 0.1302,0.1302,0.1302
macro: 0.0222,0.0508,0.0271
weighted: 0.0545,0.1302,0.0685
Epoch=10, trans_loss_tgt=0.000828, [2807.69s]
Epoch=10, tgt_loss=0.000846, [2826.25s]
Epoch=10: (AUC,MRR) = 0.9484, 0.6516, [2877.10s]
HR=0.5620, NDCG=0.5620 at top-1
HR=0.7534, NDCG=0.6611 at top-5
HR=0.8508, NDCG=0.6925 at top-10
HR=0.9325, NDCG=0.7133 at top-20
HR=0.9788, NDCG=0.7230 at top-35
Epoch=11, loss_attack_fake_src=0.015094, loss_attack_src=0.000748, [2916.79s]
Epoch=11, loss_attack_real=0.000000, gender=0.011247, age=0.012113, occupation=0.021627, [2936.23s]
Epoch=11, src_loss=0.000760, [2962.64s]
---gender: accuracy=0.7250---
micro: 0.7250,0.7250,0.7250
macro: 0.3625,0.5000,0.4203
weighted: 0.5256,0.7250,0.6094
---age: accuracy=0.5719---
micro: 0.5719,0.5719,0.5719
macro: 0.3245,0.3376,0.2529
weighted: 0.4188,0.5719,0.4221
---occupation: accuracy=0.1302---
micro: 0.1302,0.1302,0.1302
macro: 0.0711,0.0525,0.0301
weighted: 0.1436,0.1302,0.0740
Epoch=11, trans_loss_tgt=0.000787, [2987.40s]
Epoch=11, tgt_loss=0.000811, [3007.00s]
Epoch=11: (AUC,MRR) = 0.9485, 0.6532, [3058.59s]
HR=0.5639, NDCG=0.5639 at top-1
HR=0.7539, NDCG=0.6625 at top-5
HR=0.8504, NDCG=0.6937 at top-10
HR=0.9328, NDCG=0.7146 at top-20
HR=0.9794, NDCG=0.7244 at top-35
Epoch=12, loss_attack_fake_src=0.014946, loss_attack_src=0.000730, [3097.11s]
Epoch=12, loss_attack_real=0.000000, gender=0.011248, age=0.012116, occupation=0.021629, [3115.67s]
Epoch=12, src_loss=0.000743, [3139.46s]
---gender: accuracy=0.7250---
micro: 0.7250,0.7250,0.7250
macro: 0.3625,0.5000,0.4203
weighted: 0.5256,0.7250,0.6094
---age: accuracy=0.5692---
micro: 0.5692,0.5692,0.5692
macro: 0.1899,0.3328,0.2418
weighted: 0.3248,0.5692,0.4136
---occupation: accuracy=0.1266---
micro: 0.1266,0.1266,0.1266
macro: 0.0211,0.0496,0.0271
weighted: 0.0513,0.1266,0.0683
Epoch=12, trans_loss_tgt=0.000753, [3163.24s]
Epoch=12, tgt_loss=0.000777, [3181.55s]
Epoch=12: (AUC,MRR) = 0.9475, 0.6483, [3241.06s]
HR=0.5588, NDCG=0.5588 at top-1
HR=0.7506, NDCG=0.6579 at top-5
HR=0.8480, NDCG=0.6893 at top-10
HR=0.9301, NDCG=0.7101 at top-20
HR=0.9783, NDCG=0.7203 at top-35
Epoch=13, loss_attack_fake_src=0.015057, loss_attack_src=0.000712, [3281.01s]
Epoch=13, loss_attack_real=0.000000, gender=0.011248, age=0.012116, occupation=0.021628, [3298.26s]
Epoch=13, src_loss=0.000727, [3319.94s]
---gender: accuracy=0.7250---
micro: 0.7250,0.7250,0.7250
macro: 0.3625,0.5000,0.4203
weighted: 0.5256,0.7250,0.6094
---age: accuracy=0.5710---
micro: 0.5710,0.5710,0.5710
macro: 0.5235,0.3347,0.2448
weighted: 0.5548,0.5710,0.4161
---occupation: accuracy=0.1239---
micro: 0.1239,0.1239,0.1239
macro: 0.0715,0.0517,0.0290
weighted: 0.1446,0.1239,0.0697
Epoch=13, trans_loss_tgt=0.000722, [3343.26s]
Epoch=13, tgt_loss=0.000751, [3360.17s]
Epoch=13: (AUC,MRR) = 0.9471, 0.6424, [3426.70s]
HR=0.5525, NDCG=0.5525 at top-1
HR=0.7425, NDCG=0.6507 at top-5
HR=0.8429, NDCG=0.6832 at top-10
HR=0.9318, NDCG=0.7058 at top-20
HR=0.9792, NDCG=0.7157 at top-35
early stopping...
{'AUC_best': 0.9521376332777505,
 'AUC_best_epoch': 0,
 'HRs_best': [0.5650931428143366,
              0.7585376925079049,
              0.8562399416422845,
              0.940350587527506,
              0.9828053290351646],
 'HRs_best_epoch': [8, 0, 0, 0, 0],
 'MRR_best': 0.6541703039336806,
 'MRR_best_epoch': 8,
 'NDCGs_best': [0.5650931428143366,
                0.6637895225046689,
                0.6954933253653754,
                0.7164276465565678,
                0.7256533708357495],
 'NDCGs_best_epoch': [8, 8, 3, 3, 3]}
{'age': {'accuracy': [0.5792349726775956, 2],
         'macro': {'f1': [0.312966323855616, 2],
                   'precision': [0.5235490732300213, 13],
                   'recall': [0.3655273255912233, 2]},
         'micro': {'f1': [0.5792349726775956, 2],
                   'precision': [0.5792349726775956, 2],
                   'recall': [0.5792349726775956, 2]},
         'weighted': {'f1': [0.46358279972275385, 2],
                      'precision': [0.554849872063734, 13],
                      'recall': [0.5792349726775956, 2]}},
 'gender': {'accuracy': [0.7258652094717668, 2],
            'macro': {'f1': [0.4432011772738421, 0],
                      'precision': [0.6963470319634704, 2],
                      'recall': [0.5057073446703717, 0]},
            'micro': {'f1': [0.7258652094717667, 2],
                      'precision': [0.7258652094717668, 2],
                      'recall': [0.7258652094717668, 2]},
            'weighted': {'f1': [0.6202456196439207, 0],
                         'precision': [0.7097004932172235, 2],
                         'recall': [0.7258652094717668, 2]}},
 'occupation': {'accuracy': [0.1384335154826958, 3],
                'macro': {'f1': [0.03225621907325845, 3],
                          'precision': [0.0714754832477835, 13],
                          'recall': [0.055164486717783325, 3]},
                'micro': {'f1': [0.1384335154826958, 3],
                          'precision': [0.1384335154826958, 3],
                          'recall': [0.1384335154826958, 3]},
                'weighted': {'f1': [0.07971629743433697, 3],
                             'precision': [0.14457981675756496, 13],
                             'recall': [0.1384335154826958, 3]}}}
---57.11min---
