« back to all changes in this revision
Viewing changes to storage/myisam/ft_nlq_search.c
- browse files at revision 76.1.1
- compare with another revision
- compare with revision 1219
- download diff
- download tarball
- view history from revision 76.1.1
- Committer: Brian Aker
- Date: 2008-07-06 08:22:57 UTC
- mto: This revision was merged to the branch mainline in revision 78.
- Revision ID: brian@tangent.org-20080706082257-gni9cj1cdjlqomz0
Final removal of fulltext core from myisam.
- files removed:
- storage/myisam/ft_boolean_search.c
- files modified:
- sql/mysqld.cc
added
removed
storage/myisam/ft_nlq_search.c
1
/* Copyright (C) 2001-2005 MySQL AB
2
3
This program is free software; you can redistribute it and/or modify
4
it under the terms of the GNU General Public License as published by
5
the Free Software Foundation; version 2 of the License.
6
7
This program is distributed in the hope that it will be useful,
8
but WITHOUT ANY WARRANTY; without even the implied warranty of
9
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10
GNU General Public License for more details.
11
12
You should have received a copy of the GNU General Public License
13
along with this program; if not, write to the Free Software
14
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */
15
16
/* Written by Sergei A. Golubchik, who has a shared copyright to this code */
17
18
#define FT_CORE
19
#include "ftdefs.h"
20
21
/* search with natural language queries */
22
23
typedef struct ft_doc_rec
24
{
25
my_off_t dpos;
26
double weight;
27
} FT_DOC;
28
29
struct st_ft_info
30
{
31
struct _ft_vft *please;
32
MI_INFO *info;
33
int ndocs;
34
int curdoc;
35
FT_DOC doc[1];
36
};
37
38
typedef struct st_all_in_one
39
{
40
MI_INFO *info;
41
uint keynr;
42
CHARSET_INFO *charset;
43
uchar *keybuff;
44
TREE dtree;
45
} ALL_IN_ONE;
46
47
typedef struct st_ft_superdoc
48
{
49
FT_DOC doc;
50
FT_WORD *word_ptr;
51
double tmp_weight;
52
} FT_SUPERDOC;
53
54
static int FT_SUPERDOC_cmp(void* cmp_arg __attribute__((unused)),
55
FT_SUPERDOC *p1, FT_SUPERDOC *p2)
56
{
57
if (p1->doc.dpos < p2->doc.dpos)
58
return -1;
59
if (p1->doc.dpos == p2->doc.dpos)
60
return 0;
61
return 1;
62
}
63
64
static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio)
65
{
66
int subkeys, r;
67
uint keylen, doc_cnt;
68
FT_SUPERDOC sdoc, *sptr;
69
TREE_ELEMENT *selem;
70
double gweight=1;
71
MI_INFO *info=aio->info;
72
uchar *keybuff=aio->keybuff;
73
MI_KEYDEF *keyinfo=info->s->keyinfo+aio->keynr;
74
my_off_t key_root=info->s->state.key_root[aio->keynr];
75
uint extra=HA_FT_WLEN+info->s->base.rec_reflength;
76
#if HA_FT_WTYPE == HA_KEYTYPE_FLOAT
77
float tmp_weight;
78
#else
79
#error
80
#endif
81
82
DBUG_ENTER("walk_and_match");
83
84
word->weight=LWS_FOR_QUERY;
85
86
keylen=_ft_make_key(info,aio->keynr,keybuff,word,0);
87
keylen-=HA_FT_WLEN;
88
doc_cnt=0;
89
90
/* Skip rows inserted by current inserted */
91
for (r=_mi_search(info, keyinfo, keybuff, keylen, SEARCH_FIND, key_root) ;
92
!r &&
93
(subkeys=ft_sintXkorr(info->lastkey+info->lastkey_length-extra)) > 0 &&
94
info->lastpos >= info->state->data_file_length ;
95
r= _mi_search_next(info, keyinfo, info->lastkey,
96
info->lastkey_length, SEARCH_BIGGER, key_root))
97
;
98
99
info->update|= HA_STATE_AKTIV; /* for _mi_test_if_changed() */
100
101
/* The following should be safe, even if we compare doubles */
102
while (!r && gweight)
103
{
104
105
if (keylen &&
106
ha_compare_text(aio->charset,info->lastkey+1,
107
info->lastkey_length-extra-1, keybuff+1,keylen-1,0,0))
108
break;
109
110
if (subkeys<0)
111
{
112
if (doc_cnt)
113
DBUG_RETURN(1); /* index is corrupted */
114
/*
115
TODO here: unsafe optimization, should this word
116
be skipped (based on subkeys) ?
117
*/
118
keybuff+=keylen;
119
keyinfo=& info->s->ft2_keyinfo;
120
key_root=info->lastpos;
121
keylen=0;
122
r=_mi_search_first(info, keyinfo, key_root);
123
goto do_skip;
124
}
125
#if HA_FT_WTYPE == HA_KEYTYPE_FLOAT
126
tmp_weight=*(float*)&subkeys;
127
#else
128
#error
129
#endif
130
/* The following should be safe, even if we compare doubles */
131
if (tmp_weight==0)
132
DBUG_RETURN(doc_cnt); /* stopword, doc_cnt should be 0 */
133
134
sdoc.doc.dpos=info->lastpos;
135
136
/* saving document matched into dtree */
137
if (!(selem=tree_insert(&aio->dtree, &sdoc, 0, aio->dtree.custom_arg)))
138
DBUG_RETURN(1);
139
140
sptr=(FT_SUPERDOC *)ELEMENT_KEY((&aio->dtree), selem);
141
142
if (selem->count==1) /* document's first match */
143
sptr->doc.weight=0;
144
else
145
sptr->doc.weight+=sptr->tmp_weight*sptr->word_ptr->weight;
146
147
sptr->word_ptr=word;
148
sptr->tmp_weight=tmp_weight;
149
150
doc_cnt++;
151
152
gweight=word->weight*GWS_IN_USE;
153
if (gweight < 0 || doc_cnt > 2000000)
154
gweight=0;
155
156
if (_mi_test_if_changed(info) == 0)
157
r=_mi_search_next(info, keyinfo, info->lastkey, info->lastkey_length,
158
SEARCH_BIGGER, key_root);
159
else
160
r=_mi_search(info, keyinfo, info->lastkey, info->lastkey_length,
161
SEARCH_BIGGER, key_root);
162
do_skip:
163
while ((subkeys=ft_sintXkorr(info->lastkey+info->lastkey_length-extra)) > 0 &&
164
!r && info->lastpos >= info->state->data_file_length)
165
r= _mi_search_next(info, keyinfo, info->lastkey, info->lastkey_length,
166
SEARCH_BIGGER, key_root);
167
168
}
169
word->weight=gweight;
170
171
DBUG_RETURN(0);
172
}
173
174
175
static int walk_and_copy(FT_SUPERDOC *from,
176
uint32 count __attribute__((unused)), FT_DOC **to)
177
{
178
DBUG_ENTER("walk_and_copy");
179
from->doc.weight+=from->tmp_weight*from->word_ptr->weight;
180
(*to)->dpos=from->doc.dpos;
181
(*to)->weight=from->doc.weight;
182
(*to)++;
183
DBUG_RETURN(0);
184
}
185
186
static int walk_and_push(FT_SUPERDOC *from,
187
uint32 count __attribute__((unused)), QUEUE *best)
188
{
189
DBUG_ENTER("walk_and_copy");
190
from->doc.weight+=from->tmp_weight*from->word_ptr->weight;
191
set_if_smaller(best->elements, ft_query_expansion_limit-1);
192
queue_insert(best, (uchar *)& from->doc);
193
DBUG_RETURN(0);
194
}
195
196
197
static int FT_DOC_cmp(void *unused __attribute__((unused)),
198
FT_DOC *a, FT_DOC *b)
199
{
200
return sgn(b->weight - a->weight);
201
}
202
203
204
FT_INFO *ft_init_nlq_search(MI_INFO *info, uint keynr, uchar *query,
205
uint query_len, uint flags, uchar *record)
206
{
207
TREE wtree;
208
ALL_IN_ONE aio;
209
FT_DOC *dptr;
210
FT_INFO *dlist=NULL;
211
my_off_t saved_lastpos=info->lastpos;
212
struct st_mysql_ftparser *parser;
213
MYSQL_FTPARSER_PARAM *ftparser_param;
214
DBUG_ENTER("ft_init_nlq_search");
215
216
/* black magic ON */
217
if ((int) (keynr = _mi_check_index(info,keynr)) < 0)
218
DBUG_RETURN(NULL);
219
if (_mi_readinfo(info,F_RDLCK,1))
220
DBUG_RETURN(NULL);
221
/* black magic OFF */
222
223
aio.info=info;
224
aio.keynr=keynr;
225
aio.charset=info->s->keyinfo[keynr].seg->charset;
226
aio.keybuff=info->lastkey+info->s->base.max_key_length;
227
parser= info->s->keyinfo[keynr].parser;
228
if (! (ftparser_param= ftparser_call_initializer(info, keynr, 0)))
229
goto err;
230
231
bzero(&wtree,sizeof(wtree));
232
233
init_tree(&aio.dtree,0,0,sizeof(FT_SUPERDOC),(qsort_cmp2)&FT_SUPERDOC_cmp,0,
234
NULL, NULL);
235
236
ft_parse_init(&wtree, aio.charset);
237
ftparser_param->flags= 0;
238
if (ft_parse(&wtree, query, query_len, parser, ftparser_param,
239
&wtree.mem_root))
240
goto err;
241
242
if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,
243
left_root_right))
244
goto err;
245
246
if (flags & FT_EXPAND && ft_query_expansion_limit)
247
{
248
QUEUE best;
249
init_queue(&best,ft_query_expansion_limit,0,0, (queue_compare) &FT_DOC_cmp,
250
0);
251
tree_walk(&aio.dtree, (tree_walk_action) &walk_and_push,
252
&best, left_root_right);
253
while (best.elements)
254
{
255
my_off_t docid=((FT_DOC *)queue_remove(& best, 0))->dpos;
256
if (!(*info->read_record)(info,docid,record))
257
{
258
info->update|= HA_STATE_AKTIV;
259
ftparser_param->flags= MYSQL_FTFLAGS_NEED_COPY;
260
if (unlikely(_mi_ft_parse(&wtree, info, keynr, record, ftparser_param,
261
&wtree.mem_root)))
262
{
263
delete_queue(&best);
264
goto err;
265
}
266
}
267
}
268
delete_queue(&best);
269
reset_tree(&aio.dtree);
270
if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,
271
left_root_right))
272
goto err;
273
274
}
275
276
/*
277
If ndocs == 0, this will not allocate RAM for FT_INFO.doc[],
278
so if ndocs == 0, FT_INFO.doc[] must not be accessed.
279
*/
280
dlist=(FT_INFO *)my_malloc(sizeof(FT_INFO)+
281
sizeof(FT_DOC)*
282
(int)(aio.dtree.elements_in_tree-1),
283
MYF(0));
284
if (!dlist)
285
goto err;
286
287
dlist->please= (struct _ft_vft *) & _ft_vft_nlq;
288
dlist->ndocs=aio.dtree.elements_in_tree;
289
dlist->curdoc=-1;
290
dlist->info=aio.info;
291
dptr=dlist->doc;
292
293
tree_walk(&aio.dtree, (tree_walk_action) &walk_and_copy,
294
&dptr, left_root_right);
295
296
if (flags & FT_SORTED)
297
my_qsort2(dlist->doc, dlist->ndocs, sizeof(FT_DOC), (qsort2_cmp)&FT_DOC_cmp,
298
0);
299
300
err:
301
delete_tree(&aio.dtree);
302
delete_tree(&wtree);
303
info->lastpos=saved_lastpos;
304
DBUG_RETURN(dlist);
305
}
306
307
308
int ft_nlq_read_next(FT_INFO *handler, char *record)
309
{
310
MI_INFO *info= (MI_INFO *) handler->info;
311
312
if (++handler->curdoc >= handler->ndocs)
313
{
314
--handler->curdoc;
315
return HA_ERR_END_OF_FILE;
316
}
317
318
info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
319
320
info->lastpos=handler->doc[handler->curdoc].dpos;
321
if (!(*info->read_record)(info,info->lastpos,(uchar*) record))
322
{
323
info->update|= HA_STATE_AKTIV; /* Record is read */
324
return 0;
325
}
326
return my_errno;
327
}
328
329
330
float ft_nlq_find_relevance(FT_INFO *handler,
331
uchar *record __attribute__((unused)),
332
uint length __attribute__((unused)))
333
{
334
int a,b,c;
335
FT_DOC *docs=handler->doc;
336
my_off_t docid=handler->info->lastpos;
337
338
if (docid == HA_POS_ERROR)
339
return -5.0;
340
341
/* Assuming docs[] is sorted by dpos... */
342
343
for (a=0, b=handler->ndocs, c=(a+b)/2; b-a>1; c=(a+b)/2)
344
{
345
if (docs[c].dpos > docid)
346
b=c;
347
else
348
a=c;
349
}
350
/* bounds check to avoid accessing unallocated handler->doc */
351
if (a < handler->ndocs && docs[a].dpos == docid)
352
return (float) docs[a].weight;
353
else
354
return 0.0;
355
}
356
357
358
void ft_nlq_close_search(FT_INFO *handler)
359
{
360
my_free((uchar*)handler,MYF(0));
361
}
362
363
364
float ft_nlq_get_relevance(FT_INFO *handler)
365
{
366
return (float) handler->doc[handler->curdoc].weight;
367
}
368
369
370
void ft_nlq_reinit_search(FT_INFO *handler)
371
{
372
handler->curdoc=-1;
373
}
374
Loggerhead is a web-based interface for Breezy
Version: 2.0.1