--> -->
 
 
<type 'exceptions.KeyError'>
Python 2.7.3: /usr/bin/python
Sat Apr 19 02:33:06 2014

A problem occurred in a Python script. Here is the sequence of function calls leading up to the error, in the order they occurred.

 /srv/deployment/reporter/reporter/report.py in ()
     87 dbs = cache["_dbs"]
     88 
=>   89 total = fullprofile[db]["-"]["-total"]
     90 
     91 #cache.close()
total undefined, fullprofile = {u'1.23wmf21': {u'-': {u'-setup': {'count': 40334468, 'cpu': 42099.858822, 'cpusq': 168.393248, 'onecpu': 0.001043768789066463, 'onereal': 0.0010468397815486248, 'real': 42223.72567, 'realsq': 49.260114}, u'-total': {'count': 40334133, 'cpu': 9978763.844961, 'cpusq': 20514467.817275, 'onecpu': 0.24740246294524296, 'onereal': 0.28192059237234623, 'real': 11371022.668185, 'realsq': 26146479.848926}, u'API:abusefiltercheckmatch': {'count': 2, 'cpu': 0.036002, 'cpusq': 0.000656, 'onecpu': 0.018001, 'onereal': 0.0253695, 'real': 0.050739, 'realsq': 0.001287}, u'API:abusefilterchecksyntax': {'count': 1, 'cpu': 0.0, 'cpusq': 0.0, 'onecpu': 0.0, 'onereal': 0.000358, 'real': 0.000358, 'realsq': 0.0}, u'API:abusefilters': {'count': 60, 'cpu': 0.408029, 'cpusq': 0.00544, 'onecpu': 0.006800483333333333, 'onereal': 0.010130216666666667, 'real': 0.607813, 'realsq': 0.009991}, u'API:abusefilters-DB': {'count': 60, 'cpu': 0.028003, 'cpusq': 0.000112, 'onecpu': 0.0004667166666666667, 'onereal': 0.0016937999999999999, 'real': 0.101628, 'realsq': 0.000236}, u'API:abuselog': {'count': 7599, 'cpu': 203.824709, 'cpusq': 10.693117, 'onecpu': 0.02682256994341361, 'onereal': 0.04077475667851033, 'real': 309.847376, 'realsq': 38.94351}, u'API:abuselog-DB': {'count': 7593, 'cpu': 3.436224, 'cpusq': 0.013744, 'onecpu': 0.00045255156064796527, 'onereal': 0.0014677155274595022, 'real': 11.144364, 'realsq': 1.768138}, u'API:allcategories': {'count': 10, 'cpu': 0.076004, 'cpusq': 0.001072, 'onecpu': 0.0076004, 'onereal': 0.0159306, 'real': 0.159306, 'realsq': 0.005527}, u'API:allcategories-DB': {'count': 10, 'cpu': 0.016, 'cpusq': 6.4e-05, 'onecpu': 0.0016, 'onereal': 0.007742300000000001, 'real': 0.077423, 'realsq': 0.00318}, ...}}, u'1.23wmf22': {u'-': {u'-setup': {'count': 22755961, 'cpu': 23760.173678, 'cpusq': 95.037264, 'onecpu': 0.001044129653676239, 'onereal': 0.0010466636428582384, 'real': 23817.837037, 'realsq': 27.481845}, u'-total': {'count': 22755770, 'cpu': 5018836.999889, 'cpusq': 10166360.869729, 'onecpu': 0.22055228189988738, 'onereal': 0.2523362521657584, 'real': 5742105.716946, 'realsq': 13638518.586305}, u'API:abusefiltercheckmatch': {'count': 1, 'cpu': 0.004, 'cpusq': 1.6e-05, 'onecpu': 0.004, 'onereal': 0.001481, 'real': 0.001481, 'realsq': 2e-06}, u'API:abusefilterchecksyntax': {'count': 4, 'cpu': 0.028003, 'cpusq': 0.000336, 'onecpu': 0.00700075, 'onereal': 0.0085125, 'real': 0.03405, 'realsq': 0.000546}, u'API:abusefilters': {'count': 9, 'cpu': 0.116006, 'cpusq': 0.001744, 'onecpu': 0.012889555555555555, 'onereal': 0.017757777777777775, 'real': 0.15982, 'realsq': 0.003019}, u'API:abusefilters-DB': {'count': 9, 'cpu': 0.008, 'cpusq': 3.2e-05, 'onecpu': 0.0008888888888888889, 'onereal': 0.001993, 'real': 0.017937, 'realsq': 4.5e-05}, u'API:abuselog': {'count': 4248, 'cpu': 116.739295, 'cpusq': 68.079014, 'onecpu': 0.027481001647834274, 'onereal': 0.044599246233521656, 'real': 189.457598, 'realsq': 786.18679}, u'API:abuselog-DB': {'count': 4246, 'cpu': 2.208125, 'cpusq': 0.008864, 'onecpu': 0.0005200482807348092, 'onereal': 0.001840682995760716, 'real': 7.81554, 'realsq': 5.298925}, u'API:allcategories': {'count': 16, 'cpu': 0.140007, 'cpusq': 0.002608, 'onecpu': 0.0087504375, 'onereal': 0.0105455625, 'real': 0.168729, 'realsq': 0.002755}, u'API:allcategories-DB': {'count': 16, 'cpu': 0.02, 'cpusq': 8e-05, 'onecpu': 0.00125, 'onereal': 0.002293875, 'real': 0.036702, 'realsq': 8.7e-05}, ...}}, u'1.24wmf1': {u'-': {u'-setup': {'count': 21771, 'cpu': 25.169571, 'cpusq': 0.1008, 'onecpu': 0.0011561054154609343, 'onereal': 0.0011500170410178678, 'real': 25.037021, 'realsq': 0.034043}, u'-total': {'count': 21771, 'cpu': 5031.114307, 'cpusq': 5692.512878, 'onecpu': 0.2310924765513757, 'onereal': 0.26118490969638514, 'real': 5686.256669, 'realsq': 7496.151826}, u'API:allpages': {'count': 10, 'cpu': 0.136007, 'cpusq': 0.010209, 'onecpu': 0.013600699999999999, 'onereal': 0.022451699999999998, 'real': 0.224517, 'realsq': 0.036667}, u'API:allpages-DB': {'count': 10, 'cpu': 0.012, 'cpusq': 4.8e-05, 'onecpu': 0.0012000000000000001, 'onereal': 0.0011757999999999998, 'real': 0.011758, 'realsq': 1.7e-05}, u'API:blocks': {'count': 15, 'cpu': 0.056002, 'cpusq': 0.000384, 'onecpu': 0.0037334666666666667, 'onereal': 0.005657400000000001, 'real': 0.084861, 'realsq': 0.000659}, u'API:blocks-DB': {'count': 15, 'cpu': 0.016001, 'cpusq': 6.4e-05, 'onecpu': 0.0010667333333333334, 'onereal': 0.0014494, 'real': 0.021741, 'realsq': 3.5e-05}, u'API:categories': {'count': 6, 'cpu': 0.008, 'cpusq': 3.2e-05, 'onecpu': 0.0013333333333333333, 'onereal': 0.0012995, 'real': 0.007797, 'realsq': 1e-05}, u'API:categories-DB': {'count': 6, 'cpu': 0.0, 'cpusq': 0.0, 'onecpu': 0.0, 'onereal': 0.0006746666666666667, 'real': 0.004048, 'realsq': 3e-06}, u'API:categoryinfo': {'count': 1, 'cpu': 0.0, 'cpusq': 0.0, 'onecpu': 0.0, 'onereal': 0.000272, 'real': 0.000272, 'realsq': 0.0}, u'API:categorymembers': {'count': 1, 'cpu': 0.008001, 'cpusq': 6.4e-05, 'onecpu': 0.008001, 'onereal': 0.005782, 'real': 0.005782, 'realsq': 3.3e-05}, ...}}, u'all': {u'-': {u'-setup': {'count': 63223185, 'cpu': 66010.893867, 'cpusq': 264.034432, 'onecpu': 0.0010440931418909062, 'onereal': 0.0010469545990604554, 'real': 66191.804303, 'realsq': 76.928624}, u'-total': {'count': 63222652, 'cpu': 15022583.181657, 'cpusq': 30711526.104803, 'onecpu': 0.23761393593006822, 'onereal': 0.271277388223512, 'real': 17150875.911124, 'realsq': 40022280.995079}, u'API:abusefiltercheckmatch': {'count': 3, 'cpu': 0.040002, 'cpusq': 0.000672, 'onecpu': 0.013334, 'onereal': 0.017406666666666668, 'real': 0.05222, 'realsq': 0.001289}, u'API:abusefilterchecksyntax': {'count': 5, 'cpu': 0.028003, 'cpusq': 0.000336, 'onecpu': 0.0056006, 'onereal': 0.0068816, 'real': 0.034408, 'realsq': 0.000546}, u'API:abusefilters': {'count': 69, 'cpu': 0.524035, 'cpusq': 0.007184, 'onecpu': 0.007594710144927536, 'onereal': 0.011125115942028986, 'real': 0.767633, 'realsq': 0.01301}, u'API:abusefilters-DB': {'count': 69, 'cpu': 0.036003, 'cpusq': 0.000144, 'onecpu': 0.0005217826086956522, 'onereal': 0.0017328260869565219, 'real': 0.119565, 'realsq': 0.000281}, u'API:abuselog': {'count': 11847, 'cpu': 320.564004, 'cpusq': 78.772131, 'onecpu': 0.027058664978475564, 'onereal': 0.042146110745336376, 'real': 499.304974, 'realsq': 825.1303}, u'API:abuselog-DB': {'count': 11839, 'cpu': 5.644349, 'cpusq': 0.022608, 'onecpu': 0.00047675893234225866, 'onereal': 0.001601478503251964, 'real': 18.959904, 'realsq': 7.067063}, u'API:allcategories': {'count': 26, 'cpu': 0.216011, 'cpusq': 0.00368, 'onecpu': 0.008308115384615384, 'onereal': 0.01261673076923077, 'real': 0.328035, 'realsq': 0.008282}, u'API:allcategories-DB': {'count': 26, 'cpu': 0.036, 'cpusq': 0.000144, 'onecpu': 0.0013846153846153845, 'onereal': 0.004389423076923077, 'real': 0.114125, 'realsq': 0.003267}, ...}}, u'stats/1.23wmf21': {u'-': {u'diff_cache_hit': {'count': 177889820, 'cpu': 9138726.0, 'cpusq': 9138726.0, 'onecpu': 0.05137295658627346, 'onereal': 0.05137295658627346, 'real': 9138726.0, 'realsq': 9138726.0}, u'diff_cache_miss': {'count': 5780749, 'cpu': 4768363.0, 'cpusq': 4768363.0, 'onecpu': 0.8248694070612649, 'onereal': 0.8248694070612649, 'real': 4768363.0, 'realsq': 4768363.0}, u'diff_uncacheable': {'count': 127046, 'cpu': 127018.0, 'cpusq': 127018.0, 'onecpu': 0.999779607386301, 'onereal': 0.999779607386301, 'real': 127018.0, 'realsq': 127018.0}, u'image_cache_hit': {'count': 2341987307, 'cpu': 380124897.0, 'cpusq': 380124897.0, 'onecpu': 0.16230869222213082, 'onereal': 0.16230869222213082, 'real': 380124897.0, 'realsq': 380124897.0}, u'image_cache_miss': {'count': 67624791, 'cpu': 40347504.0, 'cpusq': 40347504.0, 'onecpu': 0.5966377626216989, 'onereal': 0.5966377626216989, 'real': 40347504.0, 'realsq': 40347504.0}, u'job-abandon': {'count': 1115, 'cpu': 704.0, 'cpusq': 704.0, 'onecpu': 0.6313901345291479, 'onereal': 0.6313901345291479, 'real': 704.0, 'realsq': 704.0}, u'job-abandon-AssembleUploadChunks': {'count': 7, 'cpu': 7.0, 'cpusq': 7.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 7.0, 'realsq': 7.0}, u'job-abandon-MassMessageJob': {'count': 2, 'cpu': 2.0, 'cpusq': 2.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 2.0, 'realsq': 2.0}, u'job-abandon-ParsoidCacheUpdateJobOnDependencyChange': {'count': 1, 'cpu': 1.0, 'cpusq': 1.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 1.0, 'realsq': 1.0}, u'job-abandon-UpdateRepoOnMove': {'count': 12, 'cpu': 10.0, 'cpusq': 10.0, 'onecpu': 0.8333333333333334, 'onereal': 0.8333333333333334, 'real': 10.0, 'realsq': 10.0}, ...}}, u'stats/1.23wmf22': {u'-': {u'diff_cache_hit': {'count': 33031332, 'cpu': 4388476.0, 'cpusq': 4388476.0, 'onecpu': 0.13285797860043913, 'onereal': 0.13285797860043913, 'real': 4388476.0, 'realsq': 4388476.0}, u'diff_cache_miss': {'count': 2278566, 'cpu': 1739096.0, 'cpusq': 1739096.0, 'onecpu': 0.7632414422053169, 'onereal': 0.7632414422053169, 'real': 1739096.0, 'realsq': 1739096.0}, u'diff_uncacheable': {'count': 66122, 'cpu': 65859.0, 'cpusq': 65859.0, 'onecpu': 0.9960225038565077, 'onereal': 0.9960225038565077, 'real': 65859.0, 'realsq': 65859.0}, u'image_cache_hit': {'count': 1634075757, 'cpu': 255186564.0, 'cpusq': 255186564.0, 'onecpu': 0.15616568748838, 'onereal': 0.15616568748838, 'real': 255186564.0, 'realsq': 255186564.0}, u'image_cache_miss': {'count': 36251359, 'cpu': 19269713.0, 'cpusq': 19269713.0, 'onecpu': 0.5315583617154876, 'onereal': 0.5315583617154876, 'real': 19269713.0, 'realsq': 19269713.0}, u'job-abandon': {'count': 3845, 'cpu': 617.0, 'cpusq': 617.0, 'onecpu': 0.16046814044213265, 'onereal': 0.16046814044213265, 'real': 617.0, 'realsq': 617.0}, u'job-abandon-MassMessageJob': {'count': 1, 'cpu': 1.0, 'cpusq': 1.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 1.0, 'realsq': 1.0}, u'job-abandon-MassMessageJob-mediawikiwiki': {'count': 1, 'cpu': 1.0, 'cpusq': 1.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 1.0, 'realsq': 1.0}, u'job-abandon-ParsoidCacheUpdateJobOnDependencyChange': {'count': 1, 'cpu': 1.0, 'cpusq': 1.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 1.0, 'realsq': 1.0}, u'job-abandon-ParsoidCacheUpdateJobOnDependencyChange-enwiki': {'count': 1, 'cpu': 1.0, 'cpusq': 1.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 1.0, 'realsq': 1.0}, ...}}, u'stats/1.24wmf1': {u'-': {u'diff_cache_hit': {'count': 95952, 'cpu': 8021.0, 'cpusq': 8021.0, 'onecpu': 0.08359388027347006, 'onereal': 0.08359388027347006, 'real': 8021.0, 'realsq': 8021.0}, u'diff_cache_miss': {'count': 1603, 'cpu': 1254.0, 'cpusq': 1254.0, 'onecpu': 0.7822832189644416, 'onereal': 0.7822832189644416, 'real': 1254.0, 'realsq': 1254.0}, u'diff_uncacheable': {'count': 419, 'cpu': 415.0, 'cpusq': 415.0, 'onecpu': 0.9904534606205251, 'onereal': 0.9904534606205251, 'real': 415.0, 'realsq': 415.0}, u'image_cache_hit': {'count': 343447, 'cpu': 89659.0, 'cpusq': 89659.0, 'onecpu': 0.2610562910725672, 'onereal': 0.2610562910725672, 'real': 89659.0, 'realsq': 89659.0}, u'image_cache_miss': {'count': 12037, 'cpu': 2475.0, 'cpusq': 2475.0, 'onecpu': 0.20561601728005316, 'onereal': 0.20561601728005316, 'real': 2475.0, 'realsq': 2475.0}, u'job-insert': {'count': 12453, 'cpu': 6008.0, 'cpusq': 6008.0, 'onecpu': 0.4824540271420541, 'onereal': 0.4824540271420541, 'real': 6008.0, 'realsq': 6008.0}, u'job-insert-MessageGroupStatesUpdaterJob': {'count': 408, 'cpu': 408.0, 'cpusq': 408.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 408.0, 'realsq': 408.0}, u'job-insert-MessageGroupStatesUpdaterJob-mediawikiwiki': {'count': 408, 'cpu': 408.0, 'cpusq': 408.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 408.0, 'realsq': 408.0}, u'job-insert-MessageUpdateJob': {'count': 126, 'cpu': 7.0, 'cpusq': 7.0, 'onecpu': 0.05555555555555555, 'onereal': 0.05555555555555555, 'real': 7.0, 'realsq': 7.0}, u'job-insert-MessageUpdateJob-mediawikiwiki': {'count': 126, 'cpu': 7.0, 'cpusq': 7.0, 'onecpu': 0.05555555555555555, 'onereal': 0.05555555555555555, 'real': 7.0, 'realsq': 7.0}, ...}}, u'stats/all': {u'-': {u'diff_cache_hit': {'count': 211017335, 'cpu': 13535232.0, 'cpusq': 13535232.0, 'onecpu': 0.06414274922010554, 'onereal': 0.06414274922010554, 'real': 13535232.0, 'realsq': 13535232.0}, u'diff_cache_miss': {'count': 8060921, 'cpu': 6508716.0, 'cpusq': 6508716.0, 'onecpu': 0.8074407378511711, 'onereal': 0.8074407378511711, 'real': 6508716.0, 'realsq': 6508716.0}, u'diff_uncacheable': {'count': 193588, 'cpu': 193293.0, 'cpusq': 193293.0, 'onecpu': 0.9984761452156126, 'onereal': 0.9984761452156126, 'real': 193293.0, 'realsq': 193293.0}, u'image_cache_hit': {'count': 3976411504, 'cpu': 635401837.0, 'cpusq': 635401837.0, 'onecpu': 0.15979277707069023, 'onereal': 0.15979277707069023, 'real': 635401837.0, 'realsq': 635401837.0}, u'image_cache_miss': {'count': 103888363, 'cpu': 59619784.0, 'cpusq': 59619784.0, 'onecpu': 0.5738831788118559, 'onereal': 0.5738831788118559, 'real': 59619784.0, 'realsq': 59619784.0}, u'job-abandon': {'count': 4960, 'cpu': 1321.0, 'cpusq': 1321.0, 'onecpu': 0.26633064516129035, 'onereal': 0.26633064516129035, 'real': 1321.0, 'realsq': 1321.0}, u'job-abandon-AssembleUploadChunks': {'count': 7, 'cpu': 7.0, 'cpusq': 7.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 7.0, 'realsq': 7.0}, u'job-abandon-MassMessageJob': {'count': 3, 'cpu': 3.0, 'cpusq': 3.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 3.0, 'realsq': 3.0}, u'job-abandon-MassMessageJob-mediawikiwiki': {'count': 1, 'cpu': 1.0, 'cpusq': 1.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 1.0, 'realsq': 1.0}, u'job-abandon-ParsoidCacheUpdateJobOnDependencyChange': {'count': 2, 'cpu': 2.0, 'cpusq': 2.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 2.0, 'realsq': 2.0}, ...}}, u'test2': {u'-': {u'-setup': {'count': 93214, 'cpu': 106.146564, 'cpusq': 0.424816, 'onecpu': 0.0011387405754500397, 'onereal': 0.0011418332868453236, 'real': 106.434848, 'realsq': 0.134999}, u'-total': {'count': 93207, 'cpu': 19058.24305, 'cpusq': 24933.133025, 'onecpu': 0.2044722290171339, 'onereal': 0.29691104138101215, 'real': 27674.187434, 'realsq': 226325.290695}, u'API:allimages': {'count': 9, 'cpu': 0.136006, 'cpusq': 0.002208, 'onecpu': 0.015111777777777776, 'onereal': 0.02025233333333333, 'real': 0.182271, 'realsq': 0.003865}, u'API:allimages-DB': {'count': 9, 'cpu': 0.016, 'cpusq': 6.4e-05, 'onecpu': 0.0017777777777777779, 'onereal': 0.0031845555555555555, 'real': 0.028661, 'realsq': 0.000152}, u'API:allpages': {'count': 68, 'cpu': 0.17201, 'cpusq': 0.000752, 'onecpu': 0.0025295588235294117, 'onereal': 0.0033246029411764704, 'real': 0.226073, 'realsq': 0.000929}, u'API:allpages-DB': {'count': 68, 'cpu': 0.032003, 'cpusq': 0.000128, 'onecpu': 0.0004706323529411764, 'onereal': 0.0011252352941176472, 'real': 0.076516, 'realsq': 0.000124}, u'API:allusers': {'count': 12, 'cpu': 0.220013, 'cpusq': 0.004272, 'onecpu': 0.018334416666666666, 'onereal': 0.024582999999999997, 'real': 0.294996, 'realsq': 0.007582}, u'API:allusers-DB': {'count': 12, 'cpu': 0.004, 'cpusq': 1.6e-05, 'onecpu': 0.0003333333333333333, 'onereal': 0.0010779166666666667, 'real': 0.012935, 'realsq': 1.6e-05}, u'API:categories': {'count': 8, 'cpu': 0.008001, 'cpusq': 3.2e-05, 'onecpu': 0.001000125, 'onereal': 0.0030435, 'real': 0.024348, 'realsq': 0.000183}, u'API:categories-DB': {'count': 3, 'cpu': 0.004, 'cpusq': 1.6e-05, 'onecpu': 0.0013333333333333333, 'onereal': 0.0072703333333333335, 'real': 0.021811, 'realsq': 0.000163}, ...}}, u'thumb-1.23wmf21': {u'-': {u'-setup': {'count': 5032, 'cpu': 5.392328, 'cpusq': 0.021632, 'onecpu': 0.0010716073131955485, 'onereal': 0.0010456883942766297, 'real': 5.261904, 'realsq': 0.004974}, u'-total': {'count': 5032, 'cpu': 249.427588, 'cpusq': 14.015862, 'onecpu': 0.04956828060413354, 'onereal': 0.22898129630365663, 'real': 1152.233883, 'realsq': 883.631761}, u'AbuseFilterHooks::onGetAutoPromoteGroups': {'count': 136, 'cpu': 0.032003, 'cpusq': 0.000128, 'onecpu': 0.0002353161764705882, 'onereal': 0.0007205955882352941, 'real': 0.098001, 'realsq': 9.7e-05}, u'BabelStatic::onParserFirstCallInit': {'count': 114, 'cpu': 0.016, 'cpusq': 6.4e-05, 'onecpu': 0.00014035087719298245, 'onereal': 0.00012917543859649122, 'real': 0.014726, 'realsq': 0.0}, u'CentralAuthHooks::onAuthPluginSetup': {'count': 5013, 'cpu': 0.200017, 'cpusq': 0.0008, 'onecpu': 3.989966088170756e-05, 'onereal': 4.8229004588071014e-05, 'real': 0.241772, 'realsq': 0.0}, u'CentralAuthHooks::onSetupAfterCache': {'count': 5013, 'cpu': 0.132011, 'cpusq': 0.000528, 'onecpu': 2.633373229603032e-05, 'onereal': 4.1269100339118295e-05, 'real': 0.206882, 'realsq': 0.0}, u'CentralAuthHooks::onUserGetRights': {'count': 968, 'cpu': 0.028001, 'cpusq': 0.000112, 'onecpu': 2.8926652892561985e-05, 'onereal': 4.21900826446281e-05, 'real': 0.04084, 'realsq': 0.0}, u'CentralAuthHooks::onUserLoadDefaults': {'count': 871, 'cpu': 0.020001, 'cpusq': 8e-05, 'onecpu': 2.296326061997704e-05, 'onereal': 3.6738231917336395e-05, 'real': 0.031999, 'realsq': 0.0}, u'CentralAuthHooks::onUserLoadFromSession': {'count': 969, 'cpu': 0.872052, 'cpusq': 0.005376, 'onecpu': 0.0008999504643962848, 'onereal': 0.001284754385964912, 'real': 1.244927, 'realsq': 0.011123}, u'CentralAuthUser::loadFromCache': {'count': 105, 'cpu': 0.044001, 'cpusq': 0.000176, 'onecpu': 0.00041905714285714286, 'onereal': 0.0007469619047619048, 'real': 0.078431, 'realsq': 6.4e-05}, ...}}, ...}, db = 'stats/all'

<type 'exceptions.KeyError'>: '-total'
      args = ('-total',)
      message = '-total'