« back to all changes in this revision
Viewing changes to lib/lp/scripts/utilities/pageperformancereport.py
- browse files at revision 11872
- compare with another revision
- compare with revision 14585.2.2
- download diff
- download tarball
- view history from revision 11872
- Committer: Launchpad Patch Queue Manager
- Date: 2010-11-05 19:58:32 UTC
- mfrom: (11775.2.32 ppr-merge)
- Revision ID: launchpad@pqm.canonical.com-20101105195832-y4z0m8xl7ipiefg4
[r=allenap][ui=none][no-qa] Make page-performance-report.py work in
constant memory. Allow generating the weekly and monthly
reports using the daily ones. Bonus points: output some metrics
for import in tuolumne.
constant memory. Allow generating the weekly and monthly
reports using the daily ones. Bonus points: output some metrics
for import in tuolumne.
added
removed
lib/lp/scripts/utilities/pageperformancereport.py
6
6
__metaclass__ = type
7
7
__all__ = ['main']
8
8
9
import bz2
10
import cPickle
9
11
from cgi import escape as html_quote
12
import copy
10
13
from ConfigParser import RawConfigParser
14
import csv
11
15
from datetime import datetime
16
import gzip
17
import math
12
18
import os.path
13
19
import re
14
import subprocess
15
20
from textwrap import dedent
16
import sqlite3
17
import tempfile
21
import textwrap
18
22
import time
19
import warnings
20
23
21
import numpy
22
24
import simplejson as json
23
25
import sre_constants
24
26
import zc.zservertracelog.tracereport
27
29
from canonical.launchpad.scripts.logger import log
28
30
from lp.scripts.helpers import LPOptionParser
29
31
30
# We don't care about conversion to nan, they are expected.
31
warnings.filterwarnings(
32
'ignore', '.*converting a masked element to nan.', UserWarning)
33
32
34
33
class Request(zc.zservertracelog.tracereport.Request):
35
34
url = None
58
57
59
58
Requests belong to a Category if the URL matches a regular expression.
60
59
"""
60
61
61
def __init__(self, title, regexp):
62
62
self.title = title
63
63
self.regexp = regexp
70
70
def __cmp__(self, other):
71
71
return cmp(self.title.lower(), other.title.lower())
72
72
73
def __deepcopy__(self, memo):
74
# We provide __deepcopy__ because the module doesn't handle
75
# compiled regular expression by default.
76
return Category(self.title, self.regexp)
77
78
79
class OnlineStatsCalculator:
80
"""Object that can compute count, sum, mean, variance and median.
81
82
It computes these value incrementally and using minimal storage
83
using the Welford / Knuth algorithm described at
84
http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#On-line_algorithm
85
"""
86
87
def __init__(self):
88
self.count = 0
89
self.sum = 0
90
self.M2 = 0.0 # Sum of square difference
91
self.mean = 0.0
92
93
def update(self, x):
94
"""Incrementally update the stats when adding x to the set.
95
96
None values are ignored.
97
"""
98
if x is None:
99
return
100
self.count += 1
101
self.sum += x
102
delta = x - self.mean
103
self.mean = float(self.sum)/self.count
104
self.M2 += delta*(x - self.mean)
105
106
@property
107
def variance(self):
108
"""Return the population variance."""
109
if self.count == 0:
110
return 0
111
else:
112
return self.M2/self.count
113
114
@property
115
def std(self):
116
"""Return the standard deviation."""
117
if self.count == 0:
118
return 0
119
else:
120
return math.sqrt(self.variance)
121
122
def __add__(self, other):
123
"""Adds this and another OnlineStatsCalculator.
124
125
The result combines the stats of the two objects.
126
"""
127
results = OnlineStatsCalculator()
128
results.count = self.count + other.count
129
results.sum = self.sum + other.sum
130
if self.count > 0 and other.count > 0:
131
# This is 2.1b in Chan, Tony F.; Golub, Gene H.; LeVeque,
132
# Randall J. (1979), "Updating Formulae and a Pairwise Algorithm
133
# for Computing Sample Variances.",
134
# Technical Report STAN-CS-79-773,
135
# Department of Computer Science, Stanford University,
136
# ftp://reports.stanford.edu/pub/cstr/reports/cs/tr/79/773/CS-TR-79-773.pdf .
137
results.M2 = self.M2 + other.M2 + (
138
(float(self.count) / (other.count * results.count)) *
139
((float(other.count) / self.count) * self.sum - other.sum)**2)
140
else:
141
results.M2 = self.M2 + other.M2 # One of them is 0.
142
if results.count > 0:
143
results.mean = float(results.sum) / results.count
144
return results
145
146
147
class OnlineApproximateMedian:
148
"""Approximate the median of a set of elements.
149
150
This implements a space-efficient algorithm which only sees each value
151
once. (It will hold in memory log bucket_size of n elements.)
152
153
It was described and analysed in
154
D. Cantone and M.Hofri,
155
"Analysis of An Approximate Median Selection Algorithm"
156
ftp://ftp.cs.wpi.edu/pub/techreports/pdf/06-17.pdf
157
158
This algorithm is similar to Tukey's median of medians technique.
159
It will compute the median among bucket_size values. And the median among
160
those.
161
"""
162
163
def __init__(self, bucket_size=9):
164
"""Creates a new estimator.
165
166
It approximates the median by finding the median among each
167
successive bucket_size element. And then using these medians for other
168
rounds of selection.
169
170
The bucket size should be a low odd-integer.
171
"""
172
self.bucket_size = bucket_size
173
# Index of the median in a completed bucket.
174
self.median_idx = (bucket_size-1)//2
175
self.buckets = []
176
177
def update(self, x, order=0):
178
"""Update with x."""
179
if x is None:
180
return
181
182
i = order
183
while True:
184
# Create bucket on demand.
185
if i >= len(self.buckets):
186
for n in range((i+1)-len(self.buckets)):
187
self.buckets.append([])
188
bucket = self.buckets[i]
189
bucket.append(x)
190
if len(bucket) == self.bucket_size:
191
# Select the median in this bucket, and promote it.
192
x = sorted(bucket)[self.median_idx]
193
# Free the bucket for the next round.
194
del bucket[:]
195
i += 1
196
continue
197
else:
198
break
199
200
@property
201
def median(self):
202
"""Return the median."""
203
# Find the 'weighted' median by assigning a weight to each
204
# element proportional to how far they have been selected.
205
candidates = []
206
total_weight = 0
207
for i, bucket in enumerate(self.buckets):
208
weight = self.bucket_size ** i
209
for x in bucket:
210
total_weight += weight
211
candidates.append([x, weight])
212
if len(candidates) == 0:
213
return 0
214
215
# Each weight is the equivalent of having the candidates appear
216
# that number of times in the array.
217
# So buckets like [[1, 2], [2, 3], [4, 2]] would be expanded to
218
# [1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4,
219
# 4, 4, 4, 4, 4] and we find the median of that list (2).
220
# We don't expand the items to conserve memory.
221
median = (total_weight-1) / 2
222
weighted_idx = 0
223
for x, weight in sorted(candidates):
224
weighted_idx += weight
225
if weighted_idx > median:
226
return x
227
228
def __add__(self, other):
229
"""Merge two approximators together.
230
231
All candidates from the other are merged through the standard
232
algorithm, starting at the same level. So an item that went through
233
two rounds of selection, will be compared with other items having
234
gone through the same number of rounds.
235
"""
236
results = OnlineApproximateMedian(self.bucket_size)
237
results.buckets = copy.deepcopy(self.buckets)
238
for i, bucket in enumerate(other.buckets):
239
for x in bucket:
240
results.update(x, i)
241
return results
242
73
243
74
244
class Stats:
75
"""Bag to hold request statistics.
245
"""Bag to hold and compute request statistics.
76
246
77
247
All times are in seconds.
78
248
"""
82
252
mean = 0 # Mean time per hit.
83
253
median = 0 # Median time per hit.
84
254
std = 0 # Standard deviation per hit.
85
ninetyninth_percentile_time = 0
86
255
histogram = None # # Request times histogram.
87
256
88
257
total_sqltime = 0 # Total time spent waiting for SQL to process.
95
264
median_sqlstatements = 0
96
265
std_sqlstatements = 0
97
266
98
def __init__(self, times, timeout):
99
"""Compute the stats based on times.
100
101
Times is a list of (app_time, sql_statements, sql_times).
102
103
The histogram is a list of request counts per 1 second bucket.
104
ie. histogram[0] contains the number of requests taking between 0 and
105
1 second, histogram[1] contains the number of requests taking between
106
1 and 2 seconds etc. histogram is None if there are no requests in
107
this Category.
267
@property
268
def ninetyninth_percentile_time(self):
269
"""Time under which 99% of requests are rendered.
270
271
This is estimated as 3 std deviations from the mean. Given that
272
in a daily report, many URLs or PageIds won't have 100 requests, it's
273
more useful to use this estimator.
108
274
"""
109
if not times:
110
return
111
112
self.total_hits = len(times)
113
114
# Ignore missing values (-1) in computation.
115
times_array = numpy.ma.masked_values(
116
numpy.asarray(times, dtype=numpy.float32), -1.)
117
118
self.total_time, self.total_sqlstatements, self.total_sqltime = (
119
times_array.sum(axis=0))
120
121
self.mean, self.mean_sqlstatements, self.mean_sqltime = (
122
times_array.mean(axis=0))
123
124
self.median, self.median_sqlstatements, self.median_sqltime = (
125
numpy.median(times_array, axis=0))
126
127
self.std, self.std_sqlstatements, self.std_sqltime = (
128
numpy.std(times_array, axis=0))
129
130
# This is an approximation which may not be true: we don't know if we
131
# have a std distribution or not. We could just find the 99th
132
# percentile by counting. Shock. Horror; however this appears pretty
133
# good based on eyeballing things so far - once we're down in the 2-3
134
# second range for everything we may want to revisit.
135
self.ninetyninth_percentile_time = self.mean + self.std*3
136
137
histogram_width = int(timeout*1.5)
138
histogram_times = numpy.clip(times_array[:,0], 0, histogram_width)
139
histogram = numpy.histogram(
140
histogram_times, normed=True, range=(0, histogram_width),
141
bins=histogram_width)
142
self.histogram = zip(histogram[1], histogram[0])
143
144
145
class SQLiteRequestTimes:
146
"""SQLite-based request times computation."""
275
return self.mean + 3*self.std
276
277
@property
278
def relative_histogram(self):
279
"""Return an histogram where the frequency is relative."""
280
if self.histogram:
281
return [[x, float(f)/self.total_hits] for x, f in self.histogram]
282
else:
283
return None
284
285
def text(self):
286
"""Return a textual version of the stats."""
287
return textwrap.dedent("""
288
<Stats for %d requests:
289
Time: total=%.2f; mean=%.2f; median=%.2f; std=%.2f
290
SQL time: total=%.2f; mean=%.2f; median=%.2f; std=%.2f
291
SQL stmt: total=%.f; mean=%.2f; median=%.f; std=%.2f
292
>""" % (
293
self.total_hits, self.total_time, self.mean, self.median,
294
self.std, self.total_sqltime, self.mean_sqltime,
295
self.median_sqltime, self.std_sqltime,
296
self.total_sqlstatements, self.mean_sqlstatements,
297
self.median_sqlstatements, self.std_sqlstatements))
298
299
300
class OnlineStats(Stats):
301
"""Implementation of stats that can be computed online.
302
303
You call update() for each request and the stats are updated incrementally
304
with minimum storage space.
305
"""
306
307
def __init__(self, histogram_width):
308
self.time_stats = OnlineStatsCalculator()
309
self.time_median_approximate = OnlineApproximateMedian()
310
self.sql_time_stats = OnlineStatsCalculator()
311
self.sql_time_median_approximate = OnlineApproximateMedian()
312
self.sql_statements_stats = OnlineStatsCalculator()
313
self.sql_statements_median_approximate = OnlineApproximateMedian()
314
self._histogram = [
315
[x, 0] for x in range(histogram_width)]
316
317
@property
318
def total_hits(self):
319
return self.time_stats.count
320
321
@property
322
def total_time(self):
323
return self.time_stats.sum
324
325
@property
326
def mean(self):
327
return self.time_stats.mean
328
329
@property
330
def median(self):
331
return self.time_median_approximate.median
332
333
@property
334
def std(self):
335
return self.time_stats.std
336
337
@property
338
def total_sqltime(self):
339
return self.sql_time_stats.sum
340
341
@property
342
def mean_sqltime(self):
343
return self.sql_time_stats.mean
344
345
@property
346
def median_sqltime(self):
347
return self.sql_time_median_approximate.median
348
349
@property
350
def std_sqltime(self):
351
return self.sql_time_stats.std
352
353
@property
354
def total_sqlstatements(self):
355
return self.sql_statements_stats.sum
356
357
@property
358
def mean_sqlstatements(self):
359
return self.sql_statements_stats.mean
360
361
@property
362
def median_sqlstatements(self):
363
return self.sql_statements_median_approximate.median
364
365
@property
366
def std_sqlstatements(self):
367
return self.sql_statements_stats.std
368
369
@property
370
def histogram(self):
371
if self.time_stats.count:
372
return self._histogram
373
else:
374
return None
375
376
def update(self, request):
377
"""Update the stats based on request."""
378
self.time_stats.update(request.app_seconds)
379
self.time_median_approximate.update(request.app_seconds)
380
self.sql_time_stats.update(request.sql_seconds)
381
self.sql_time_median_approximate.update(request.sql_seconds)
382
self.sql_statements_stats.update(request.sql_statements)
383
self.sql_statements_median_approximate.update(request.sql_statements)
384
385
idx = int(min(len(self.histogram)-1, request.app_seconds))
386
self.histogram[idx][1] += 1
387
388
def __add__(self, other):
389
"""Merge another OnlineStats with this one."""
390
results = copy.deepcopy(self)
391
results.time_stats += other.time_stats
392
results.time_median_approximate += other.time_median_approximate
393
results.sql_time_stats += other.sql_time_stats
394
results.sql_time_median_approximate += (
395
other.sql_time_median_approximate)
396
results.sql_statements_stats += other.sql_statements_stats
397
results.sql_statements_median_approximate += (
398
other.sql_statements_median_approximate)
399
for i, (n, f) in enumerate(other._histogram):
400
results._histogram[i][1] += f
401
return results
402
403
404
class RequestTimes:
405
"""Collect statistics from requests.
406
407
Statistics are updated by calling the add_request() method.
408
409
Statistics for mean/stddev/total/median for request times, SQL times and
410
number of SQL statements are collected.
411
412
They are grouped by Category, URL or PageID.
413
"""
147
414
148
415
def __init__(self, categories, options):
149
if options.db_file is None:
150
fd, self.filename = tempfile.mkstemp(suffix='.db', prefix='ppr')
151
os.close(fd)
152
else:
153
self.filename = options.db_file
154
self.con = sqlite3.connect(self.filename, isolation_level='EXCLUSIVE')
155
log.debug('Using request database %s' % self.filename)
156
# Some speed optimization.
157
self.con.execute('PRAGMA synchronous = off')
158
self.con.execute('PRAGMA journal_mode = off')
159
160
self.categories = categories
161
self.store_all_request = options.pageids or options.top_urls
162
self.timeout = options.timeout
163
self.cur = self.con.cursor()
164
165
# Create the tables, ignore errors about them being already present.
166
try:
167
self.cur.execute('''
168
CREATE TABLE category_request (
169
category INTEGER,
170
time REAL,
171
sql_statements INTEGER,
172
sql_time REAL)
173
''');
174
except sqlite3.OperationalError, e:
175
if 'already exists' in str(e):
176
pass
177
else:
178
raise
179
180
if self.store_all_request:
181
try:
182
self.cur.execute('''
183
CREATE TABLE request (
184
pageid TEXT,
185
url TEXT,
186
time REAL,
187
sql_statements INTEGER,
188
sql_time REAL)
189
''');
190
except sqlite3.OperationalError, e:
191
if 'already exists' in str(e):
192
pass
193
else:
194
raise
416
self.by_pageids = options.pageids
417
self.top_urls = options.top_urls
418
# We only keep in memory 50 times the number of URLs we want to
419
# return. The number of URLs can go pretty high (because of the
420
# distinct query parameters).
421
#
422
# Keeping all in memory at once is prohibitive. On a small but
423
# representative sample, keeping 50 times the possible number of
424
# candidates and culling to 90% on overflow, generated an identical
425
# report than keeping all the candidates in-memory.
426
#
427
# Keeping 10 times or culling at 90% generated a near-identical report
428
# (it differed a little in the tail.)
429
#
430
# The size/cull parameters might need to change if the requests
431
# distribution become very different than what it currently is.
432
self.top_urls_cache_size = self.top_urls * 50
433
434
# Histogram has a bin per second up to 1.5 our timeout.
435
self.histogram_width = int(options.timeout*1.5)
436
self.category_times = [
437
(category, OnlineStats(self.histogram_width))
438
for category in categories]
439
self.url_times = {}
440
self.pageid_times = {}
195
441
196
442
def add_request(self, request):
197
"""Add a request to the cache."""
198
sql_statements = request.sql_statements
199
sql_seconds = request.sql_seconds
200
201
# Store missing value as -1, as it makes dealing with those
202
# easier with numpy.
203
if sql_statements is None:
204
sql_statements = -1
205
if sql_seconds is None:
206
sql_seconds = -1
207
for idx, category in enumerate(self.categories):
443
"""Add request to the set of requests we collect stats for."""
444
for category, stats in self.category_times:
208
445
if category.match(request):
209
self.con.execute(
210
"INSERT INTO category_request VALUES (?,?,?,?)",
211
(idx, request.app_seconds, sql_statements, sql_seconds))
446
stats.update(request)
212
447
213
if self.store_all_request:
448
if self.by_pageids:
214
449
pageid = request.pageid or 'Unknown'
215
self.con.execute(
216
"INSERT INTO request VALUES (?,?,?,?,?)",
217
(pageid, request.url, request.app_seconds, sql_statements,
218
sql_seconds))
450
stats = self.pageid_times.setdefault(
451
pageid, OnlineStats(self.histogram_width))
452
stats.update(request)
219
453
220
def commit(self):
221
"""Call commit on the underlying connection."""
222
self.con.commit()
454
if self.top_urls:
455
stats = self.url_times.setdefault(
456
request.url, OnlineStats(self.histogram_width))
457
stats.update(request)
458
# Whenever we have more URLs than we need to, discard 10%
459
# that is less likely to end up in the top.
460
if len(self.url_times) > self.top_urls_cache_size:
461
cutoff = int(self.top_urls_cache_size*0.90)
462
self.url_times = dict(
463
sorted(self.url_times.items(),
464
key=lambda (url, stats): stats.total_time,
465
reverse=True)[:cutoff])
223
466
224
467
def get_category_times(self):
225
468
"""Return the times for each category."""
226
category_query = 'SELECT * FROM category_request ORDER BY category'
227
228
empty_stats = Stats([], 0)
229
categories = dict(self.get_times(category_query))
230
return [
231
(category, categories.get(idx, empty_stats))
232
for idx, category in enumerate(self.categories)]
233
234
def get_top_urls_times(self, top_n):
469
return self.category_times
470
471
def get_top_urls_times(self):
235
472
"""Return the times for the Top URL by total time"""
236
top_url_query = '''
237
SELECT url, time, sql_statements, sql_time
238
FROM request WHERE url IN (
239
SELECT url FROM (SELECT url, sum(time) FROM request
240
GROUP BY url
241
ORDER BY sum(time) DESC
242
LIMIT %d))
243
ORDER BY url
244
''' % top_n
245
473
# Sort the result by total time
246
474
return sorted(
247
self.get_times(top_url_query), key=lambda x: x[1].total_time,
248
reverse=True)
475
self.url_times.items(),
476
key=lambda (url, stats): stats.total_time,
477
reverse=True)[:self.top_urls]
249
478
250
479
def get_pageid_times(self):
251
480
"""Return the times for the pageids."""
252
pageid_query = '''
253
SELECT pageid, time, sql_statements, sql_time
254
FROM request
255
ORDER BY pageid
256
'''
257
return self.get_times(pageid_query)
258
259
def get_times(self, query):
260
"""Return a list of key, stats based on the query.
261
262
The query should return rows of the form:
263
[key, app_time, sql_statements, sql_times]
264
265
And should be sorted on key.
266
"""
267
times = []
268
current_key = None
269
results = []
270
self.cur.execute(query)
271
while True:
272
rows = self.cur.fetchmany()
273
if len(rows) == 0:
274
break
275
for row in rows:
276
# We are encountering a new group...
277
if row[0] != current_key:
278
# Compute the stats of the previous group
279
if current_key != None:
280
results.append(
281
(current_key, Stats(times, self.timeout)))
282
# Initialize the new group.
283
current_key = row[0]
284
times = []
285
286
times.append(row[1:])
287
# Compute the stats of the last group
288
if current_key != None:
289
results.append((current_key, Stats(times, self.timeout)))
481
# Sort the result by pageid
482
return sorted(self.pageid_times.items())
483
484
def __add__(self, other):
485
"""Merge two RequestTimes together."""
486
results = copy.deepcopy(self)
487
for other_category, other_stats in other.category_times:
488
for i, (category, stats) in enumerate(self.category_times):
489
if category.title == other_category.title:
490
results.category_times[i] = (
491
category, stats + other_stats)
492
break
493
else:
494
results.category_times.append(
495
(other_category, copy.deepcopy(other_stats)))
496
497
url_times = results.url_times
498
for url, stats in other.url_times.items():
499
if url in url_times:
500
url_times[url] += stats
501
else:
502
url_times[url] = copy.deepcopy(stats)
503
# Only keep top_urls_cache_size entries.
504
if len(self.url_times) > self.top_urls_cache_size:
505
self.url_times = dict(
506
sorted(
507
url_times.items(),
508
key=lambda (url, stats): stats.total_time,
509
reverse=True)[:self.top_urls_cache_size])
510
511
pageid_times = results.pageid_times
512
for pageid, stats in other.pageid_times.items():
513
if pageid in pageid_times:
514
pageid_times[pageid] += stats
515
else:
516
pageid_times[pageid] = copy.deepcopy(stats)
290
517
291
518
return results
292
519
293
def close(self, remove=False):
294
"""Close the SQLite connection.
295
296
:param remove: If true, the DB file will be removed.
297
"""
298
self.con.close()
299
if remove:
300
log.debug('Deleting request database.')
301
os.unlink(self.filename)
302
else:
303
log.debug('Keeping request database %s.' % self.filename)
304
305
520
306
521
def main():
307
522
parser = LPOptionParser("%prog [args] tracelog [...]")
340
555
default=12, type="int",
341
556
help="The configured timeout value : determines high risk page ids.")
342
557
parser.add_option(
343
"--db-file", dest="db_file",
344
default=None, metavar="FILE",
345
help="Do not parse the records, generate reports from the DB file.")
558
"--merge", dest="merge",
559
default=False, action='store_true',
560
help="Files are interpreted as pickled stats and are aggregated for" +
561
"the report.")
346
562
347
563
options, args = parser.parse_args()
348
564
349
565
if not os.path.isdir(options.directory):
350
566
parser.error("Directory %s does not exist" % options.directory)
351
567
352
if len(args) == 0 and options.db_file is None:
568
if len(args) == 0:
353
569
parser.error("At least one zserver tracelog file must be provided")
354
570
355
571
if options.from_ts is not None and options.until_ts is not None:
357
573
parser.error(
358
574
"--from timestamp %s is before --until timestamp %s"
359
575
% (options.from_ts, options.until_ts))
576
if options.from_ts is not None or options.until_ts is not None:
577
if options.merge:
578
parser.error('--from and --until cannot be used with --merge')
360
579
361
580
for filename in args:
362
581
if not os.path.exists(filename):
383
602
if len(categories) == 0:
384
603
parser.error("No data in [categories] section of configuration.")
385
604
386
times = SQLiteRequestTimes(categories, options)
605
times = RequestTimes(categories, options)
387
606
388
if len(args) > 0:
607
if options.merge:
608
for filename in args:
609
log.info('Merging %s...' % filename)
610
f = bz2.BZ2File(filename, 'r')
611
times += cPickle.load(f)
612
f.close()
613
else:
389
614
parse(args, times, options)
390
times.commit()
391
615
392
log.debug('Generating category statistics...')
393
616
category_times = times.get_category_times()
394
617
395
618
pageid_times = []
396
619
url_times= []
397
620
if options.top_urls:
398
log.debug('Generating top %d urls statistics...' % options.top_urls)
399
url_times = times.get_top_urls_times(options.top_urls)
621
url_times = times.get_top_urls_times()
400
622
if options.pageids:
401
log.debug('Generating pageid statistics...')
402
623
pageid_times = times.get_pageid_times()
403
624
404
625
def _report_filename(filename):
436
657
open(report_filename, 'w'), None, pageid_times, None,
437
658
options.timeout - 2)
438
659
439
times.close(options.db_file is None)
660
# Save the times cache for later merging.
661
report_filename = _report_filename('stats.pck.bz2')
662
log.info("Saving times database in %s", report_filename)
663
stats_file = bz2.BZ2File(report_filename, 'w')
664
cPickle.dump(times, stats_file, protocol=cPickle.HIGHEST_PROTOCOL)
665
stats_file.close()
666
667
# Output metrics for selected categories.
668
report_filename = _report_filename('metrics.dat')
669
log.info('Saving category_metrics %s', report_filename)
670
metrics_file = open(report_filename, 'w')
671
writer = csv.writer(metrics_file, delimiter=':')
672
date = options.until_ts or options.from_ts or datetime.utcnow()
673
date = time.mktime(date.timetuple())
674
675
for option in script_config.options('metrics'):
676
name = script_config.get('metrics', option)
677
for category, stats in category_times:
678
if category.title == name:
679
writer.writerows([
680
("%s_99" % option, "%f@%d" % (
681
stats.ninetyninth_percentile_time, date)),
682
("%s_mean" % option, "%f@%d" % (stats.mean, date))])
683
break
684
else:
685
log.warning("Can't find category %s for metric %s" % (
686
option, name))
687
metrics_file.close()
688
440
689
return 0
441
690
442
691
447
696
"""
448
697
ext = os.path.splitext(filename)[1]
449
698
if ext == '.bz2':
450
p = subprocess.Popen(
451
['bunzip2', '-c', filename],
452
stdout=subprocess.PIPE, stdin=subprocess.PIPE)
453
p.stdin.close()
454
return p.stdout
699
return bz2.BZ2File(filename, 'r')
455
700
elif ext == '.gz':
456
p = subprocess.Popen(
457
['gunzip', '-c', filename],
458
stdout=subprocess.PIPE, stdin=subprocess.PIPE)
459
p.stdin.close()
460
return p.stdout
701
return gzip.GzipFile(filename, 'r')
461
702
else:
462
703
return open(filename, mode)
463
704
684
925
histograms = []
685
926
686
927
def handle_times(html_title, stats):
687
histograms.append(stats.histogram)
928
histograms.append(stats.relative_histogram)
688
929
print >> outf, dedent("""\
689
930
<tr>
690
931
<th class="category-title">%s</th>
810
1051
</body>
811
1052
</html>
812
1053
""")
813
Loggerhead is a web-based interface for Breezy
Version: 2.0.1