« back to all changes in this revision
Viewing changes to lib/lp/services/log/uniquefileallocator.py
- browse files at revision 14104.6.23
- compare with another revision
- compare with revision 14625.2.5
- download diff
- download tarball
- view history from revision 14104.6.23
- Committer: Robert Collins
- Date: 2011-10-17 05:41:34 UTC
- mto: This revision was merged to the branch mainline in revision 14213.
- Revision ID: robertc@robertcollins.net-20111017054134-m1bo7gi6s6aixc7h
Nuke setOopsToken unneeded in a concurrency safe world.
- files removed:
- lib/lp/services/log/uniquefileallocator.py
- files modified:
- lib/canonical/launchpad/webapp/errorlog.py
added
removed
lib/lp/services/log/uniquefileallocator.py
1
# Copyright 2010 Canonical Ltd. This software is licensed under the
2
# GNU Affero General Public License version 3 (see the file LICENSE).
3
4
"""Create uniquely named log files on disk."""
5
6
7
__all__ = ['UniqueFileAllocator']
8
9
__metaclass__ = type
10
11
12
import datetime
13
import errno
14
import os.path
15
import stat
16
import threading
17
18
import pytz
19
20
21
UTC = pytz.utc
22
23
# the section of the ID before the instance identifier is the
24
# days since the epoch, which is defined as the start of 2006.
25
epoch = datetime.datetime(2006, 01, 01, 00, 00, 00, tzinfo=UTC)
26
27
28
class UniqueFileAllocator:
29
"""Assign unique file names to logs being written from an app/script.
30
31
UniqueFileAllocator causes logs written from one process to be uniquely
32
named. It is not safe for use in multiple processes with the same output
33
root - each process must have a unique output root.
34
"""
35
36
def __init__(self, output_root, log_type, log_subtype):
37
"""Create a UniqueFileAllocator.
38
39
:param output_root: The root directory that logs should be placed in.
40
:param log_type: A string to use as a prefix in the ID assigned to new
41
logs. For instance, "OOPS".
42
:param log_subtype: A string to insert in the generate log filenames
43
between the day number and the serial. For instance "T" for
44
"Testing".
45
"""
46
self._lock = threading.Lock()
47
self._output_root = output_root
48
self._last_serial = 0
49
self._last_output_dir = None
50
self._log_type = log_type
51
self._log_subtype = log_subtype
52
self._log_token = ""
53
54
def _findHighestSerialFilename(self, directory=None, time=None):
55
"""Find details of the last log present in the given directory.
56
57
This function only considers logs with the currently
58
configured log_subtype.
59
60
One of directory, time must be supplied.
61
62
:param directory: Look in this directory.
63
:param time: Look in the directory that a log written at this time
64
would have been written to. If supplied, supercedes directory.
65
:return: a tuple (log_serial, log_filename), which will be (0,
66
None) if no logs are found. log_filename is a usable path, not
67
simply the basename.
68
"""
69
if directory is None:
70
directory = self.output_dir(time)
71
prefix = self.get_log_infix()
72
lastid = 0
73
lastfilename = None
74
for filename in os.listdir(directory):
75
logid = filename.rsplit('.', 1)[1]
76
if not logid.startswith(prefix):
77
continue
78
logid = logid[len(prefix):]
79
if logid.isdigit() and (lastid is None or int(logid) > lastid):
80
lastid = int(logid)
81
lastfilename = filename
82
if lastfilename is not None:
83
lastfilename = os.path.join(directory, lastfilename)
84
return lastid, lastfilename
85
86
def _findHighestSerial(self, directory):
87
"""Find the last serial actually applied to disk in directory.
88
89
The purpose of this function is to not repeat sequence numbers
90
if the logging application is restarted.
91
92
This method is not thread safe, and only intended to be called
93
from the constructor (but it is called from other places in
94
integration tests).
95
"""
96
return self._findHighestSerialFilename(directory)[0]
97
98
def getFilename(self, log_serial, time):
99
"""Get the filename for a given log serial and time."""
100
log_subtype = self.get_log_infix()
101
# TODO: Calling output_dir causes a global lock to be taken and a
102
# directory scan, which is bad for performance. It would be better
103
# to have a split out 'directory name for time' function which the
104
# 'want to use this directory now' function can call.
105
output_dir = self.output_dir(time)
106
second_in_day = time.hour * 3600 + time.minute * 60 + time.second
107
return os.path.join(
108
output_dir, '%05d.%s%s' % (
109
second_in_day, log_subtype, log_serial))
110
111
def get_log_infix(self):
112
"""Return the current log infix to use in ids and file names."""
113
return self._log_subtype + self._log_token
114
115
def newId(self, now=None):
116
"""Returns an (id, filename) pair for use by the caller.
117
118
The ID is composed of a short string to identify the Launchpad
119
instance followed by an ID that is unique for the day.
120
121
The filename is composed of the zero padded second in the day
122
followed by the ID. This ensures that reports are in date order when
123
sorted lexically.
124
"""
125
if now is not None:
126
now = now.astimezone(UTC)
127
else:
128
now = datetime.datetime.now(UTC)
129
# We look up the error directory before allocating a new ID,
130
# because if the day has changed, errordir() will reset the ID
131
# counter to zero.
132
self.output_dir(now)
133
self._lock.acquire()
134
try:
135
self._last_serial += 1
136
newid = self._last_serial
137
finally:
138
self._lock.release()
139
subtype = self.get_log_infix()
140
day_number = (now - epoch).days + 1
141
log_id = '%s-%d%s%d' % (self._log_type, day_number, subtype, newid)
142
filename = self.getFilename(newid, now)
143
return log_id, filename
144
145
def output_dir(self, now=None):
146
"""Find or make the directory to allocate log names in.
147
148
Log names are assigned within subdirectories containing the date the
149
assignment happened.
150
"""
151
if now is not None:
152
now = now.astimezone(UTC)
153
else:
154
now = datetime.datetime.now(UTC)
155
date = now.strftime('%Y-%m-%d')
156
result = os.path.join(self._output_root, date)
157
if result != self._last_output_dir:
158
self._lock.acquire()
159
try:
160
self._last_output_dir = result
161
# make sure the directory exists
162
try:
163
os.makedirs(result)
164
except OSError, e:
165
if e.errno != errno.EEXIST:
166
raise
167
# Make sure the directory permission is set to: rwxr-xr-x
168
permission = (
169
stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP |
170
stat.S_IROTH | stat.S_IXOTH)
171
os.chmod(result, permission)
172
# TODO: Note that only one process can do this safely: its not
173
# cross-process safe, and also not entirely threadsafe:
174
# another # thread that has a new log and hasn't written it
175
# could then use that serial number. We should either make it
176
# really safe, or remove the contention entirely and log
177
# uniquely per thread of execution.
178
self._last_serial = self._findHighestSerial(result)
179
finally:
180
self._lock.release()
181
return result
182
183
def listRecentReportFiles(self):
184
now = datetime.datetime.now(UTC)
185
yesterday = now - datetime.timedelta(days=1)
186
directories = [self.output_dir(now), self.output_dir(yesterday)]
187
for directory in directories:
188
report_names = os.listdir(directory)
189
for name in sorted(report_names, reverse=True):
190
yield directory, name
191
192
def setToken(self, token):
193
"""Append a string to the log subtype in filenames and log ids.
194
195
:param token: a string to append..
196
Scripts that run multiple processes can use this to create a
197
unique identifier for each process.
198
"""
199
self._log_token = token
Loggerhead is a web-based interface for Breezy
Version: 2.0.1