1
# Copyright (c) 2012 Canonical Ltd
3
# This program is free software: you can redistribute it and/or modify
4
# it under the terms of the GNU Affero General Public License as published by
5
# the Free Software Foundation, either version 3 of the License, or
6
# (at your option) any later version.
8
# This program is distributed in the hope that it will be useful,
9
# but WITHOUT ANY WARRANTY; without even the implied warranty of
10
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11
# GNU Affero General Public License for more details.
13
# You should have received a copy of the GNU Affero General Public
14
# License along with this program. If not, see
15
# <http://www.gnu.org/licenses/>.
20
from email.utils import parsedate_tz
26
from pycassa.system_manager import (
32
from grackle.cassandra import workaround_1779
35
def create_schema(host, keyspace, clobber=False, create_keyspace=False):
36
mgr = SystemManager(host)
39
mgr.create_keyspace(keyspace, replication_factor=1)
42
for cf in mgr.get_keyspace_column_families(keyspace):
43
mgr.drop_column_family(keyspace, cf)
47
mgr.create_column_family, keyspace, 'message',
48
key_validation_class=LEXICAL_UUID_TYPE)
50
mgr.create_column_family, keyspace, 'archive_message',
51
comparator_type=TIME_UUID_TYPE,
52
default_validation_class=LEXICAL_UUID_TYPE)
58
def _parse_message(message):
59
"""Get a date and dict of an RFC822 message."""
60
parsed = email.parser.Parser().parsestr(message)
63
for key in ('from', 'to', 'subject', 'message-id'):
64
value = parsed.get(key, None)
66
message_dict[key] = value
68
date = parsed.get('date')
71
pdate = parsedate_tz(date)
72
date = datetime.datetime(
74
tzinfo=dateutil.tz.tzoffset('', pdate[9]))
77
message_dict['date'] = date.isoformat() if date is not None else None
79
return date, message_dict
82
def _utc_datetime(dt):
83
return dt.astimezone(dateutil.tz.tzutc())
86
def _utc_timestamp(dt):
87
return time.mktime(_utc_datetime(dt).timetuple()) - time.timezone
90
class CassandraConnection(object):
92
def __init__(self, keyspace, host):
93
self._keyspace = keyspace
95
self._connection = self._connect()
96
self.messages = self._column_family('message')
97
self.archive_messages = self._column_family('archive_message')
100
return pycassa.connect(self._keyspace, self._host)
102
def _column_family(self, name):
103
return pycassa.ColumnFamily(self._connection, name)
105
def add_message(self, archive_uuid, message):
106
message_uuid = uuid.uuid4()
107
message_date, message_dict = _parse_message(message)
108
message_dict['content'] = message
109
message_dict['date_created'] = (
110
datetime.datetime.utcnow().isoformat() + 'Z')
111
self.messages.insert(message_uuid, message_dict)
112
self.archive_messages.insert(
114
{_utc_timestamp(message_date): message_uuid})
116
'Imported %s into %s'
117
% (message_dict.get('message-id', None), archive_uuid))
120
def _format_message(self, message):
122
'date': message.get('date'),
123
'from': message.get('from'),
124
'subject': message.get('subject'),
125
'message-id': message.get('message-id'),
128
def _trim(self, sequence, end):
129
"""Return the sequence with one of the ends trimmed.
131
:param end: if true, remove the last element. otherwise remove
139
def get_messages(self, archive_uuid, order, count, memo, backward=False):
140
if order in ("date", "-date"):
141
reversed = order[0] == '-'
143
raise AssertionError("Unsupported order.")
145
memo = uuid.UUID(memo)
153
# Get up to n+1 messages from the memo: the last item of the
154
# previous batch (because that's where the memo starts) + this
156
pairs = self.archive_messages.get(
157
archive_uuid, column_count=count + 1, column_start=start,
158
column_finish=finish, column_reversed=reversed).items()
160
if len(pairs) and memo and pairs[0][0] <= memo:
161
# The memo (from the previous batch) was included in the result.
163
pairs = self._trim(pairs, False ^ backward)
164
elif len(pairs) > count:
165
# There was no memo in the result, so the n+1th element is
166
# unnecessary. Kill it.
167
pairs = self._trim(pairs, True ^ backward)
170
return (None, [], None)
172
assert 0 < len(pairs) <= count
174
# We've narrowed down the message references. Fetch the messages.
175
ids = [v for k, v in pairs]
176
messages = self.messages.multiget(
177
ids, columns=['date', 'from', 'subject', 'message-id'])
181
[self._format_message(messages[id]) for id in ids],