2
# Copyright (C) 2006 Robey Pointer <robey@lag.net>
4
# This program is free software; you can redistribute it and/or modify
5
# it under the terms of the GNU General Public License as published by
6
# the Free Software Foundation; either version 2 of the License, or
7
# (at your option) any later version.
9
# This program is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
# GNU General Public License for more details.
14
# You should have received a copy of the GNU General Public License
15
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20
indexing of the comment text of revisions, for fast searching.
22
two separate 'shelve' files are created:
24
- recorded: revid -> 1 (if the revid is indexed)
25
- index: 3-letter substring -> list(revids)
35
from loggerhead import util
36
from loggerhead.util import decorator
37
from loggerhead.lockfile import LockFile
39
# if any substring index reaches this many revids, replace the entry with
40
# an ALL marker -- it's not worth an explicit index.
45
with_lock = util.with_lock('_lock')
48
def normalize_string(s):
50
remove any punctuation and normalize all whitespace to a single space.
52
s = util.to_utf8(s).lower()
53
# remove apostrophes completely.
54
s = re.sub(r"'", '', s)
55
# convert other garbage into space
56
s = re.sub(r'[^\w\d]', ' ', s)
57
# compress multiple spaces into one.
58
s = re.sub(r'\s{2,}', ' ', s)
59
# and finally remove leading/trailing whitespace
64
class TextIndex (object):
65
def __init__(self, history, cache_path):
66
self.history = history
67
self.log = history.log
69
if not os.path.exists(cache_path):
72
self._recorded_filename = os.path.join(cache_path, 'textindex-recorded')
73
self._index_filename = os.path.join(cache_path, 'textindex')
75
# use a lockfile since the cache folder could be shared across different processes.
76
self._lock = LockFile(os.path.join(cache_path, 'index-lock'))
79
self.log.info('Using search index; %d entries.', len(self))
81
def _is_indexed(self, revid, recorded):
82
return recorded.get(util.to_utf8(revid), None) is not None
85
def is_indexed(self, revid):
86
recorded = shelve.open(self._recorded_filename, 'c', protocol=2)
88
return self._is_indexed(revid, recorded)
94
recorded = shelve.open(self._recorded_filename, 'c', protocol=2)
114
recorded = shelve.open(self._recorded_filename, 'c', protocol=2)
116
return (len(recorded) >= len(self.history.get_revision_history())) and (util.to_utf8(self.history.last_revid) in recorded)
120
def _index_change(self, change, recorded, index):
122
currently, only indexes the 'comment' field.
124
comment = normalize_string(change.comment)
127
for i in xrange(len(comment) - 2):
128
sub = comment[i:i + 3]
129
revid_set = index.get(sub, None)
130
if revid_set is None:
132
elif revid_set == ALL:
133
# this entry got too big
135
revid_set.add(change.revid)
136
if len(revid_set) > ALL_THRESHOLD:
138
index[sub] = revid_set
140
recorded[util.to_utf8(change.revid)] = True
143
def index_changes(self, revid_list):
144
recorded = shelve.open(self._recorded_filename, 'c', protocol=2)
145
index = shelve.open(self._index_filename, 'c', protocol=2)
147
revid_list = [r for r in revid_list if not self._is_indexed(r, recorded)]
148
change_list = self.history.get_changes(revid_list)
149
for change in change_list:
150
self._index_change(change, recorded, index)
156
def find(self, text, revid_list=None):
157
index = shelve.open(self._index_filename, 'c', protocol=2)
159
text = normalize_string(text)
164
if revid_list is not None:
165
total_set = set(revid_list)
168
for i in xrange(len(text) - 2):
170
revid_set = index.get(sub, None)
171
if revid_set is None:
172
# zero matches, stop here.
178
if total_set is None:
179
total_set = revid_set
181
total_set.intersection_update(revid_set)
182
if len(total_set) == 0:
187
# tricky: if seen_all is True, one of the substring indices was ALL
188
# (in other words, unindexed), so our results are actually a superset
189
# of the exact answer.
191
# if we cared, we could do a direct match on the result set and cull
192
# out any that aren't actually matches. for now, i'm gonna say that
193
# we DON'T care, and if one of the substrings hit ALL, there's a small
194
# chance that we'll give a few false positives.
197
def check_rebuild(self, max_time=3600):
199
check if there are any un-indexed revisions, and if so, index them.
200
but don't spend longer than C{max_time} on it.
202
if self.closed() or self.full():
206
self.log.info('Building search index...')
207
work = list(self.history.get_revision_history())
208
start_time = time.time()
209
last_update = time.time()
213
for i in xrange(0, len(work), jump):
215
self.index_changes(r)
221
if now - start_time > 3600:
222
# there's no point working for hours. eventually we might even
223
# hit the next re-index interval, which would suck mightily.
224
self.log.info('Search indexing has worked for an hour; giving up for now.')
226
if now - last_update > 60:
227
self.log.info('Search indexing continues: %d/%d' % (min(count, len(work)), len(work)))
228
last_update = time.time()
229
# give someone else a chance at the lock
231
self.log.info('Search index completed.')
added
removed
loggerhead/textindex.py
