2
# Copyright (C) 2006 Robey Pointer <robey@lag.net>
4
# This program is free software; you can redistribute it and/or modify
5
# it under the terms of the GNU General Public License as published by
6
# the Free Software Foundation; either version 2 of the License, or
7
# (at your option) any later version.
9
# This program is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
# GNU General Public License for more details.
14
# You should have received a copy of the GNU General Public License
15
# along with this program; if not, write to the Free Software
16
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20
indexing of the comment text of revisions, for fast searching.
22
two separate database files are created:
24
- recorded: revid -> 1 (if the revid is indexed)
25
- index: 3-letter substring -> list(revids)
34
from loggerhead import util
35
from loggerhead.util import decorator
36
from loggerhead.lockfile import LockFile
37
from loggerhead.changecache import FakeShelf
39
# if any substring index reaches this many revids, replace the entry with
40
# an ALL marker -- it's not worth an explicit index.
45
with_lock = util.with_lock('_lock')
48
def normalize_string(s):
50
remove any punctuation and normalize all whitespace to a single space.
52
s = util.to_utf8(s).lower()
53
# remove apostrophes completely.
54
s = re.sub(r"'", '', s)
55
# convert other garbage into space
56
s = re.sub(r'[^\w\d]', ' ', s)
57
# compress multiple spaces into one.
58
s = re.sub(r'\s{2,}', ' ', s)
59
# and finally remove leading/trailing whitespace
64
class TextIndex (object):
65
def __init__(self, history, cache_path):
66
self.history = history
67
self.log = history.log
69
if not os.path.exists(cache_path):
72
self._recorded_filename = os.path.join(cache_path, 'textindex-recorded.sql')
73
self._index_filename = os.path.join(cache_path, 'textindex.sql')
75
# use a lockfile since the cache folder could be shared across different processes.
76
self._lock = LockFile(os.path.join(cache_path, 'index-lock'))
79
self.log.info('Using search index; %d entries.', len(self))
82
return FakeShelf(self._index_filename)
85
return FakeShelf(self._recorded_filename)
87
def _is_indexed(self, revid, recorded):
88
return recorded.get(util.to_utf8(revid)) is not None
91
def is_indexed(self, revid):
92
recorded = self._recorded()
94
return self._is_indexed(revid, recorded)
100
recorded = self._recorded()
102
return recorded.count()
120
recorded = self._recorded()
121
last_revid = util.to_utf8(self.history.last_revid)
123
return (recorded.count() >= len(self.history.get_revision_history())
124
and recorded.get(last_revid) is not None)
128
def _index_change(self, change, recorded, index):
130
currently, only indexes the 'comment' field.
132
comment = normalize_string(change.comment)
135
for i in xrange(len(comment) - 2):
136
sub = comment[i:i + 3]
137
orig = revid_set = index.get(sub)
138
if revid_set is None:
140
elif revid_set == ALL:
141
# this entry got too big
143
revid_set.add(change.revid)
144
if len(revid_set) > ALL_THRESHOLD:
147
index.update([(sub, revid_set)], commit=False)
149
index.add([(sub, revid_set)], commit=False)
151
recorded.add([(util.to_utf8(change.revid), True)], commit=False)
154
def index_changes(self, revid_list):
155
recorded = self._recorded()
156
index = self._index()
158
revid_list = [r for r in revid_list if not self._is_indexed(r, recorded)]
159
change_list = self.history.get_changes(revid_list)
160
for change in change_list:
161
self._index_change(change, recorded, index)
163
index.close(commit=True)
164
recorded.close(commit=True)
167
def find(self, text, revid_list=None):
168
index = self._index()
170
text = normalize_string(text)
175
if revid_list is not None:
176
total_set = set(revid_list)
179
for i in xrange(len(text) - 2):
181
revid_set = index.get(sub)
182
if revid_set is None:
183
# zero matches, stop here.
189
if total_set is None:
190
total_set = revid_set
192
total_set.intersection_update(revid_set)
193
if len(total_set) == 0:
198
# tricky: if seen_all is True, one of the substring indices was ALL
199
# (in other words, unindexed), so our results are actually a superset
200
# of the exact answer.
202
# if we cared, we could do a direct match on the result set and cull
203
# out any that aren't actually matches. for now, i'm gonna say that
204
# we DON'T care, and if one of the substrings hit ALL, there's a small
205
# chance that we'll give a few false positives.
208
def check_rebuild(self, max_time=3600):
210
check if there are any un-indexed revisions, and if so, index them.
211
but don't spend longer than C{max_time} on it.
213
if self.closed() or self.full():
217
self.log.info('Building search index...')
218
work = list(self.history.get_revision_history())
219
start_time = time.time()
220
last_update = time.time()
224
for i in xrange(0, len(work), jump):
226
self.index_changes(r)
232
if now - start_time > 3600:
233
# there's no point working for hours. eventually we might even
234
# hit the next re-index interval, which would suck mightily.
235
self.log.info('Search indexing has worked for an hour; giving up for now.')
237
if now - last_update > 60:
238
self.log.info('Search indexing continues: %d/%d' % (min(count, len(work)), len(work)))
239
last_update = time.time()
240
# give someone else a chance at the lock
242
self.log.info('Search index completed.')
added
removed
loggerhead/textindex.py
