--> -->
 
 
<type 'exceptions.KeyError'>
Python 2.7.3: /usr/bin/python
Thu Apr 17 12:48:59 2014

A problem occurred in a Python script. Here is the sequence of function calls leading up to the error, in the order they occurred.

 /srv/deployment/reporter/reporter/report.py in ()
     87 dbs = cache["_dbs"]
     88 
=>   89 total = fullprofile[db]["-"]["-total"]
     90 
     91 #cache.close()
total undefined, fullprofile = {u'1.23wmf21': {u'-': {u'-setup': {'count': 37847093, 'cpu': 39525.122558, 'cpusq': 158.09472, 'onecpu': 0.0010443370791516273, 'onereal': 0.001047106622852117, 'real': 39629.941736, 'realsq': 46.145085}, u'-total': {'count': 37846789, 'cpu': 9352382.185341, 'cpusq': 19196177.345632, 'onecpu': 0.2471116422939077, 'onereal': 0.28110313629935685, 'real': 10638851.08676, 'realsq': 24325070.443953}, u'API:abusefiltercheckmatch': {'count': 2, 'cpu': 0.036002, 'cpusq': 0.000656, 'onecpu': 0.018001, 'onereal': 0.0253695, 'real': 0.050739, 'realsq': 0.001287}, u'API:abusefilterchecksyntax': {'count': 1, 'cpu': 0.0, 'cpusq': 0.0, 'onecpu': 0.0, 'onereal': 0.000358, 'real': 0.000358, 'realsq': 0.0}, u'API:abusefilters': {'count': 52, 'cpu': 0.360027, 'cpusq': 0.004992, 'onecpu': 0.0069235961538461534, 'onereal': 0.010515288461538461, 'real': 0.546795, 'realsq': 0.009278}, u'API:abusefilters-DB': {'count': 52, 'cpu': 0.024003, 'cpusq': 9.6e-05, 'onecpu': 0.00046159615384615385, 'onereal': 0.0017536730769230767, 'real': 0.091191, 'realsq': 0.000221}, u'API:abuselog': {'count': 7188, 'cpu': 194.472104, 'cpusq': 10.430329, 'onecpu': 0.027055106288258208, 'onereal': 0.04108234015025042, 'real': 295.299861, 'realsq': 38.289345}, u'API:abuselog-DB': {'count': 7182, 'cpu': 3.240208, 'cpusq': 0.01296, 'onecpu': 0.00045115678084099134, 'onereal': 0.0014856563631300472, 'real': 10.669984, 'realsq': 1.767011}, u'API:allcategories': {'count': 10, 'cpu': 0.076004, 'cpusq': 0.001072, 'onecpu': 0.0076004, 'onereal': 0.0159306, 'real': 0.159306, 'realsq': 0.005527}, u'API:allcategories-DB': {'count': 10, 'cpu': 0.016, 'cpusq': 6.4e-05, 'onecpu': 0.0016, 'onereal': 0.007742300000000001, 'real': 0.077423, 'realsq': 0.00318}, ...}}, u'1.23wmf22': {u'-': {u'-setup': {'count': 5820188, 'cpu': 6317.681633, 'cpusq': 25.270464, 'onecpu': 0.0010854772445494888, 'onereal': 0.001089653552600019, 'real': 6341.988531, 'realsq': 7.59548}, u'-total': {'count': 5820142, 'cpu': 1010667.413267, 'cpusq': 1482255.0873, 'onecpu': 0.17364995789913718, 'onereal': 0.20284393078467844, 'real': 1180580.481005, 'realsq': 2575679.073283}, u'API:abusefilters': {'count': 1, 'cpu': 0.004, 'cpusq': 1.6e-05, 'onecpu': 0.004, 'onereal': 0.00864, 'real': 0.00864, 'realsq': 7.5e-05}, u'API:abusefilters-DB': {'count': 1, 'cpu': 0.0, 'cpusq': 0.0, 'onecpu': 0.0, 'onereal': 0.002678, 'real': 0.002678, 'realsq': 7e-06}, u'API:abuselog': {'count': 481, 'cpu': 22.957429, 'cpusq': 65.144728, 'onecpu': 0.047728542619542624, 'onereal': 0.104717841995842, 'real': 50.369282, 'realsq': 778.84296}, u'API:abuselog-DB': {'count': 481, 'cpu': 0.204011, 'cpusq': 0.000816, 'onecpu': 0.00042413929313929314, 'onereal': 0.005966654885654885, 'real': 2.869961, 'realsq': 4.964221}, u'API:allcategories': {'count': 9, 'cpu': 0.096005, 'cpusq': 0.002304, 'onecpu': 0.010667222222222222, 'onereal': 0.012114666666666668, 'real': 0.109032, 'realsq': 0.002243}, u'API:allcategories-DB': {'count': 9, 'cpu': 0.008, 'cpusq': 3.2e-05, 'onecpu': 0.0008888888888888889, 'onereal': 0.0022906666666666665, 'real': 0.020616, 'realsq': 4.9e-05}, u'API:allimages': {'count': 24949, 'cpu': 373.031198, 'cpusq': 31.642589, 'onecpu': 0.014951749488957474, 'onereal': 0.01940349653292717, 'real': 484.097835, 'realsq': 40.583766}, u'API:allimages-DB': {'count': 24948, 'cpu': 95.629937, 'cpusq': 0.957472, 'onecpu': 0.003833170474587141, 'onereal': 0.005308226471059804, 'real': 132.429634, 'realsq': 1.534156}, ...}}, u'all': {u'-': {u'-setup': {'count': 43744107, 'cpu': 45929.929598, 'cpusq': 183.713824, 'onecpu': 0.001049968389982221, 'onereal': 0.00105292021499033, 'real': 46059.054547, 'realsq': 53.846015}, u'-total': {'count': 43743750, 'cpu': 10376599.765457, 'cpusq': 20696359.33113, 'onecpu': 0.23721331082627806, 'onereal': 0.27068072145252753, 'real': 11840589.809039, 'realsq': 26944137.312686}, u'API:abusefiltercheckmatch': {'count': 2, 'cpu': 0.036002, 'cpusq': 0.000656, 'onecpu': 0.018001, 'onereal': 0.0253695, 'real': 0.050739, 'realsq': 0.001287}, u'API:abusefilterchecksyntax': {'count': 1, 'cpu': 0.0, 'cpusq': 0.0, 'onecpu': 0.0, 'onereal': 0.000358, 'real': 0.000358, 'realsq': 0.0}, u'API:abusefilters': {'count': 53, 'cpu': 0.364027, 'cpusq': 0.005008, 'onecpu': 0.0068684339622641505, 'onereal': 0.010479905660377358, 'real': 0.555435, 'realsq': 0.009353}, u'API:abusefilters-DB': {'count': 53, 'cpu': 0.024003, 'cpusq': 9.6e-05, 'onecpu': 0.0004528867924528302, 'onereal': 0.0017711132075471697, 'real': 0.093869, 'realsq': 0.000228}, u'API:abuselog': {'count': 7669, 'cpu': 217.429533, 'cpusq': 75.575057, 'onecpu': 0.028351745077585082, 'onereal': 0.045073561481288306, 'real': 345.669143, 'realsq': 817.132305}, u'API:abuselog-DB': {'count': 7663, 'cpu': 3.444219, 'cpusq': 0.013776, 'onecpu': 0.00044946091609030403, 'onereal': 0.0017669248336160772, 'real': 13.539945, 'realsq': 6.731232}, u'API:allcategories': {'count': 19, 'cpu': 0.172009, 'cpusq': 0.003376, 'onecpu': 0.009053105263157895, 'onereal': 0.014123052631578948, 'real': 0.268338, 'realsq': 0.00777}, u'API:allcategories-DB': {'count': 19, 'cpu': 0.024, 'cpusq': 9.6e-05, 'onecpu': 0.0012631578947368421, 'onereal': 0.005159947368421053, 'real': 0.098039, 'realsq': 0.003229}, ...}}, u'stats/1.23wmf21': {u'-': {u'diff_cache_hit': {'count': 173805750, 'cpu': 8582425.0, 'cpusq': 8582425.0, 'onecpu': 0.04937940775837393, 'onereal': 0.04937940775837393, 'real': 8582425.0, 'realsq': 8582425.0}, u'diff_cache_miss': {'count': 5408015, 'cpu': 4454399.0, 'cpusq': 4454399.0, 'onecpu': 0.8236661695649883, 'onereal': 0.8236661695649883, 'real': 4454399.0, 'realsq': 4454399.0}, u'diff_uncacheable': {'count': 118002, 'cpu': 117974.0, 'cpusq': 117974.0, 'onecpu': 0.9997627158861714, 'onereal': 0.9997627158861714, 'real': 117974.0, 'realsq': 117974.0}, u'image_cache_hit': {'count': 2211249761, 'cpu': 360457819.0, 'cpusq': 360457819.0, 'onecpu': 0.1630109024124842, 'onereal': 0.1630109024124842, 'real': 360457819.0, 'realsq': 360457819.0}, u'image_cache_miss': {'count': 63839983, 'cpu': 38135330.0, 'cpusq': 38135330.0, 'onecpu': 0.5973580851360816, 'onereal': 0.5973580851360816, 'real': 38135330.0, 'realsq': 38135330.0}, u'job-abandon': {'count': 765, 'cpu': 569.0, 'cpusq': 569.0, 'onecpu': 0.7437908496732026, 'onereal': 0.7437908496732026, 'real': 569.0, 'realsq': 569.0}, u'job-abandon-AssembleUploadChunks': {'count': 7, 'cpu': 7.0, 'cpusq': 7.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 7.0, 'realsq': 7.0}, u'job-abandon-MassMessageJob': {'count': 2, 'cpu': 2.0, 'cpusq': 2.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 2.0, 'realsq': 2.0}, u'job-abandon-ParsoidCacheUpdateJobOnDependencyChange': {'count': 1, 'cpu': 1.0, 'cpusq': 1.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 1.0, 'realsq': 1.0}, u'job-abandon-UpdateRepoOnMove': {'count': 12, 'cpu': 10.0, 'cpusq': 10.0, 'onecpu': 0.8333333333333334, 'onereal': 0.8333333333333334, 'real': 10.0, 'realsq': 10.0}, ...}}, u'stats/1.23wmf22': {u'-': {u'diff_cache_hit': {'count': 5176460, 'cpu': 382915.0, 'cpusq': 382915.0, 'onecpu': 0.07397236721620568, 'onereal': 0.07397236721620568, 'real': 382915.0, 'realsq': 382915.0}, u'diff_cache_miss': {'count': 385279, 'cpu': 288061.0, 'cpusq': 288061.0, 'onecpu': 0.7476685726447587, 'onereal': 0.7476685726447587, 'real': 288061.0, 'realsq': 288061.0}, u'diff_uncacheable': {'count': 10334, 'cpu': 10207.0, 'cpusq': 10207.0, 'onecpu': 0.9877104702922392, 'onereal': 0.9877104702922392, 'real': 10207.0, 'realsq': 10207.0}, u'image_cache_hit': {'count': 397107144, 'cpu': 73155178.0, 'cpusq': 73155178.0, 'onecpu': 0.18422025165077363, 'onereal': 0.18422025165077363, 'real': 73155178.0, 'realsq': 73155178.0}, u'image_cache_miss': {'count': 7731947, 'cpu': 3838603.0, 'cpusq': 3838603.0, 'onecpu': 0.49646007661459657, 'onereal': 0.49646007661459657, 'real': 3838603.0, 'realsq': 3838603.0}, u'job-abandon': {'count': 184, 'cpu': 104.0, 'cpusq': 104.0, 'onecpu': 0.5652173913043478, 'onereal': 0.5652173913043478, 'real': 104.0, 'realsq': 104.0}, u'job-abandon-MassMessageJob': {'count': 1, 'cpu': 1.0, 'cpusq': 1.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 1.0, 'realsq': 1.0}, u'job-abandon-MassMessageJob-mediawikiwiki': {'count': 1, 'cpu': 1.0, 'cpusq': 1.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 1.0, 'realsq': 1.0}, u'job-abandon-UpdateRepoOnMove': {'count': 6, 'cpu': 5.0, 'cpusq': 5.0, 'onecpu': 0.8333333333333334, 'onereal': 0.8333333333333334, 'real': 5.0, 'realsq': 5.0}, u'job-abandon-UpdateRepoOnMove-wikidatawiki': {'count': 6, 'cpu': 5.0, 'cpusq': 5.0, 'onecpu': 0.8333333333333334, 'onereal': 0.8333333333333334, 'real': 5.0, 'realsq': 5.0}, ...}}, u'stats/all': {u'-': {u'diff_cache_hit': {'count': 178982268, 'cpu': 8965351.0, 'cpusq': 8965351.0, 'onecpu': 0.05009072183619888, 'onereal': 0.05009072183619888, 'real': 8965351.0, 'realsq': 8965351.0}, u'diff_cache_miss': {'count': 5793296, 'cpu': 4742462.0, 'cpusq': 4742462.0, 'onecpu': 0.8186120647037541, 'onereal': 0.8186120647037541, 'real': 4742462.0, 'realsq': 4742462.0}, u'diff_uncacheable': {'count': 128336, 'cpu': 128181.0, 'cpusq': 128181.0, 'onecpu': 0.9987922328886673, 'onereal': 0.9987922328886673, 'real': 128181.0, 'realsq': 128181.0}, u'image_cache_hit': {'count': 2608358395, 'cpu': 433613263.0, 'cpusq': 433613263.0, 'onecpu': 0.1662399092974338, 'onereal': 0.1662399092974338, 'real': 433613263.0, 'realsq': 433613263.0}, u'image_cache_miss': {'count': 71571965, 'cpu': 41973963.0, 'cpusq': 41973963.0, 'onecpu': 0.5864581613764551, 'onereal': 0.5864581613764551, 'real': 41973963.0, 'realsq': 41973963.0}, u'job-abandon': {'count': 949, 'cpu': 673.0, 'cpusq': 673.0, 'onecpu': 0.7091675447839831, 'onereal': 0.7091675447839831, 'real': 673.0, 'realsq': 673.0}, u'job-abandon-AssembleUploadChunks': {'count': 7, 'cpu': 7.0, 'cpusq': 7.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 7.0, 'realsq': 7.0}, u'job-abandon-MassMessageJob': {'count': 3, 'cpu': 3.0, 'cpusq': 3.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 3.0, 'realsq': 3.0}, u'job-abandon-MassMessageJob-mediawikiwiki': {'count': 1, 'cpu': 1.0, 'cpusq': 1.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 1.0, 'realsq': 1.0}, u'job-abandon-ParsoidCacheUpdateJobOnDependencyChange': {'count': 1, 'cpu': 1.0, 'cpusq': 1.0, 'onecpu': 1.0, 'onereal': 1.0, 'real': 1.0, 'realsq': 1.0}, ...}}, u'test2': {u'-': {u'-setup': {'count': 64715, 'cpu': 73.776577, 'cpusq': 0.295152, 'onecpu': 0.0011400228231476474, 'onereal': 0.0011475351773159235, 'real': 74.262739, 'realsq': 0.09327}, u'-total': {'count': 64708, 'cpu': 12912.559048, 'cpusq': 17846.636381, 'onecpu': 0.1995511999752735, 'onereal': 0.2793011513568647, 'real': 18073.018902, 'realsq': 40872.798582}, u'API:allimages': {'count': 6, 'cpu': 0.096004, 'cpusq': 0.001664, 'onecpu': 0.016000666666666667, 'onereal': 0.020853, 'real': 0.125118, 'realsq': 0.002747}, u'API:allimages-DB': {'count': 6, 'cpu': 0.008, 'cpusq': 3.2e-05, 'onecpu': 0.0013333333333333333, 'onereal': 0.0030710000000000004, 'real': 0.018426, 'realsq': 9.1e-05}, u'API:allpages': {'count': 55, 'cpu': 0.140008, 'cpusq': 0.000624, 'onecpu': 0.0025456, 'onereal': 0.003396363636363636, 'real': 0.1868, 'realsq': 0.000789}, u'API:allpages-DB': {'count': 55, 'cpu': 0.028002, 'cpusq': 0.000112, 'onecpu': 0.0005091272727272727, 'onereal': 0.0011675636363636362, 'real': 0.064216, 'realsq': 0.000111}, u'API:categories': {'count': 5, 'cpu': 0.004, 'cpusq': 1.6e-05, 'onecpu': 0.0008, 'onereal': 0.0034256, 'real': 0.017128, 'realsq': 0.000138}, u'API:categories-DB': {'count': 2, 'cpu': 0.004, 'cpusq': 1.6e-05, 'onecpu': 0.002, 'onereal': 0.007824, 'real': 0.015648, 'realsq': 0.000125}, u'API:echomarkread': {'count': 25, 'cpu': 0.716041, 'cpusq': 0.021584, 'onecpu': 0.028641640000000003, 'onereal': 0.03469928, 'real': 0.867482, 'realsq': 0.031988}, u'API:edit': {'count': 275, 'cpu': 167.858479, 'cpusq': 1047.431246, 'onecpu': 0.610394469090909, 'onereal': 0.8276842436363636, 'real': 227.613167, 'realsq': 1263.425013}, ...}}, u'thumb-1.23wmf21': {u'-': {u'-setup': {'count': 5032, 'cpu': 5.392328, 'cpusq': 0.021632, 'onecpu': 0.0010716073131955485, 'onereal': 0.0010456883942766297, 'real': 5.261904, 'realsq': 0.004974}, u'-total': {'count': 5032, 'cpu': 249.427588, 'cpusq': 14.015862, 'onecpu': 0.04956828060413354, 'onereal': 0.22898129630365663, 'real': 1152.233883, 'realsq': 883.631761}, u'AbuseFilterHooks::onGetAutoPromoteGroups': {'count': 136, 'cpu': 0.032003, 'cpusq': 0.000128, 'onecpu': 0.0002353161764705882, 'onereal': 0.0007205955882352941, 'real': 0.098001, 'realsq': 9.7e-05}, u'BabelStatic::onParserFirstCallInit': {'count': 114, 'cpu': 0.016, 'cpusq': 6.4e-05, 'onecpu': 0.00014035087719298245, 'onereal': 0.00012917543859649122, 'real': 0.014726, 'realsq': 0.0}, u'CentralAuthHooks::onAuthPluginSetup': {'count': 5013, 'cpu': 0.200017, 'cpusq': 0.0008, 'onecpu': 3.989966088170756e-05, 'onereal': 4.8229004588071014e-05, 'real': 0.241772, 'realsq': 0.0}, u'CentralAuthHooks::onSetupAfterCache': {'count': 5013, 'cpu': 0.132011, 'cpusq': 0.000528, 'onecpu': 2.633373229603032e-05, 'onereal': 4.1269100339118295e-05, 'real': 0.206882, 'realsq': 0.0}, u'CentralAuthHooks::onUserGetRights': {'count': 968, 'cpu': 0.028001, 'cpusq': 0.000112, 'onecpu': 2.8926652892561985e-05, 'onereal': 4.21900826446281e-05, 'real': 0.04084, 'realsq': 0.0}, u'CentralAuthHooks::onUserLoadDefaults': {'count': 871, 'cpu': 0.020001, 'cpusq': 8e-05, 'onecpu': 2.296326061997704e-05, 'onereal': 3.6738231917336395e-05, 'real': 0.031999, 'realsq': 0.0}, u'CentralAuthHooks::onUserLoadFromSession': {'count': 969, 'cpu': 0.872052, 'cpusq': 0.005376, 'onecpu': 0.0008999504643962848, 'onereal': 0.001284754385964912, 'real': 1.244927, 'realsq': 0.011123}, u'CentralAuthUser::loadFromCache': {'count': 105, 'cpu': 0.044001, 'cpusq': 0.000176, 'onecpu': 0.00041905714285714286, 'onereal': 0.0007469619047619048, 'real': 0.078431, 'realsq': 6.4e-05}, ...}}, u'thumb-1.23wmf22': {u'-': {u'-setup': {'count': 6886, 'cpu': 7.712485, 'cpusq': 0.03088, 'onecpu': 0.0011200239616613419, 'onereal': 0.0010750775486494336, 'real': 7.402984, 'realsq': 0.006981}, u'-total': {'count': 6886, 'cpu': 345.781662, 'cpusq': 18.985876, 'onecpu': 0.05021517020040662, 'onereal': 0.2738289455416788, 'real': 1885.586119, 'realsq': 1579.126474}, u'AbuseFilterHooks::onGetAutoPromoteGroups': {'count': 164, 'cpu': 0.068006, 'cpusq': 0.000272, 'onecpu': 0.00041467073170731704, 'onereal': 0.0007729146341463415, 'real': 0.126758, 'realsq': 0.000125}, u'BabelStatic::onParserFirstCallInit': {'count': 107, 'cpu': 0.008, 'cpusq': 3.2e-05, 'onecpu': 7.476635514018692e-05, 'onereal': 0.0001265140186915888, 'real': 0.013537, 'realsq': 0.0}, u'CentralAuthHooks::onAuthPluginSetup': {'count': 6869, 'cpu': 0.228018, 'cpusq': 0.000912, 'onecpu': 3.319522492356966e-05, 'onereal': 4.838069588004076e-05, 'real': 0.332327, 'realsq': 0.0}, u'CentralAuthHooks::onSetupAfterCache': {'count': 6869, 'cpu': 0.232017, 'cpusq': 0.000928, 'onecpu': 3.377740573591498e-05, 'onereal': 4.153297423205707e-05, 'real': 0.28529, 'realsq': 0.0}, u'CentralAuthHooks::onUserGetRights': {'count': 1264, 'cpu': 0.016002, 'cpusq': 6.4e-05, 'onecpu': 1.2659810126582278e-05, 'onereal': 4.2401107594936707e-05, 'real': 0.053595, 'realsq': 0.0}, u'CentralAuthHooks::onUserLoadDefaults': {'count': 1135, 'cpu': 0.024, 'cpusq': 9.6e-05, 'onecpu': 2.1145374449339206e-05, 'onereal': 3.655242290748899e-05, 'real': 0.041487, 'realsq': 0.0}, u'CentralAuthHooks::onUserLoadFromSession': {'count': 1265, 'cpu': 1.084077, 'cpusq': 0.006352, 'onecpu': 0.0008569778656126482, 'onereal': 0.0012759905138339921, 'real': 1.614128, 'realsq': 0.013523}, u'CentralAuthUser::loadFromCache': {'count': 149, 'cpu': 0.052003, 'cpusq': 0.000208, 'onecpu': 0.00034901342281879193, 'onereal': 0.0007491275167785234, 'real': 0.11162, 'realsq': 8e-05}, ...}}}, db = 'stats/all'

<type 'exceptions.KeyError'>: '-total'
      args = ('-total',)
      message = '-total'