78
81
return date, message_dict
84
def _utc_datetime(dt):
85
return dt.astimezone(dateutil.tz.tzutc())
88
def _utc_timestamp(dt):
89
return calendar.timegm(_utc_datetime(dt).timetuple())
92
def _utc_timeuuid(dt, lowest_val=True):
93
return convert_time_to_uuid(_utc_timestamp(dt), lowest_val)
96
def _cmp_timeuuid(a, b):
98
return cmp(a.time, b.time)
102
def _bound_timeuuid(a, b, max=False):
103
if a == '' or _cmp_timeuuid(b, a) == (1 if max else -1):
108
def _make_bounds(memo, range_start, range_finish, backward):
111
memo = uuid.UUID(memo)
116
if range_start is not None:
117
start = _bound_timeuuid(
118
start, _utc_timeuuid(range_start), max=True)
119
if range_finish is not None:
120
finish = _bound_timeuuid(
121
finish, _utc_timeuuid(range_finish, lowest_val=False))
122
return memo, start, finish
125
LEGAL_HEADERS = set([
126
'date', 'from', 'subject', 'message-id',
130
def _format_message(message, headers=[], include_raw=False):
134
assert not set(headers).difference(LEGAL_HEADERS)
136
for header in headers:
137
hdict[header] = message.get(header)
138
data['headers'] = hdict
141
data['raw'] = message['raw']
81
146
class CassandraConnection(object):
83
def __init__(self, keyspace, host):
148
def __init__(self, keyspace, hosts):
84
149
self._keyspace = keyspace
86
151
self._connection = self._connect()
152
self._pool = self._connect()
87
153
self.messages = self._column_family('message')
88
154
self.archive_messages = self._column_family('archive_message')
90
156
def _connect(self):
91
return pycassa.connect(self._keyspace, self._host)
157
return pycassa.pool.ConnectionPool(self._keyspace, self._hosts)
93
159
def _column_family(self, name):
94
return pycassa.ColumnFamily(self._connection, name)
160
return pycassa.ColumnFamily(self._pool, name)
96
162
def add_message(self, archive_uuid, message):
97
163
message_uuid = uuid.uuid4()
98
164
message_date, message_dict = _parse_message(message)
99
message_dict['content'] = message
165
message_dict['raw'] = message
100
166
message_dict['date_created'] = (
101
167
datetime.datetime.utcnow().isoformat() + 'Z')
102
168
self.messages.insert(message_uuid, message_dict)
103
169
self.archive_messages.insert(
105
{message_date.astimezone(dateutil.tz.tzutc()): message_uuid})
171
{_utc_timestamp(message_date): message_uuid})
107
173
'Imported %s into %s'
108
174
% (message_dict.get('message-id', None), archive_uuid))
109
175
return message_uuid
111
def _format_message(self, message):
113
'date': message.get('date'),
114
'from': message.get('from'),
115
'subject': message.get('subject'),
116
'message-id': message.get('message-id'),
119
177
def _trim(self, sequence, end):
178
"""Return the sequence with one of the ends trimmed.
180
:param end: if true, remove the last element. otherwise remove
121
184
return sequence[:-1]
123
186
return sequence[1:]
125
def get_messages(self, archive_uuid, order, count, memo, backward=False):
188
def get_messages(self, archive_uuid, order, count, memo, backward=False,
189
start_date=None, finish_date=None, format='all',
190
headers=['from', 'date', 'subject', 'message-id']):
126
191
if order in ("date", "-date"):
127
192
reversed = order[0] == '-'
129
194
raise AssertionError("Unsupported order.")
131
memo = uuid.UUID(memo)
196
memo, start, finish = _make_bounds(
197
memo, start_date, finish_date, backward)
132
199
# Get up to n+1 messages from the memo: the last item of the
133
200
# previous batch (because that's where the memo starts) + this
141
202
pairs = self.archive_messages.get(
142
203
archive_uuid, column_count=count + 1, column_start=start,
143
204
column_finish=finish, column_reversed=reversed).items()
144
if memo and len(pairs) and pairs[0][0] <= memo:
206
if len(pairs) and memo and pairs[0][0] <= memo:
207
# The memo (from the previous batch) was included in the result.
145
209
pairs = self._trim(pairs, False ^ backward)
146
210
elif len(pairs) > count:
211
# There was no memo in the result, so the n+1th element is
212
# unnecessary. Kill it.
147
213
pairs = self._trim(pairs, True ^ backward)
149
215
if len(pairs) == 0: