« back to all changes in this revision
Viewing changes to storage/myisam/ft_parser.c
- browse files at revision 76.1.1
- compare with another revision
- compare with revision 1219
- download diff
- download tarball
- view history from revision 76.1.1
- Committer: Brian Aker
- Date: 2008-07-06 08:22:57 UTC
- mto: This revision was merged to the branch mainline in revision 78.
- Revision ID: brian@tangent.org-20080706082257-gni9cj1cdjlqomz0
Final removal of fulltext core from myisam.
- files removed:
- storage/myisam/ft_boolean_search.c
- files modified:
- sql/mysqld.cc
added
removed
storage/myisam/ft_parser.c
1
/* Copyright (C) 2000-2005 MySQL AB
2
3
This program is free software; you can redistribute it and/or modify
4
it under the terms of the GNU General Public License as published by
5
the Free Software Foundation; version 2 of the License.
6
7
This program is distributed in the hope that it will be useful,
8
but WITHOUT ANY WARRANTY; without even the implied warranty of
9
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10
GNU General Public License for more details.
11
12
You should have received a copy of the GNU General Public License
13
along with this program; if not, write to the Free Software
14
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */
15
16
/* Written by Sergei A. Golubchik, who has a shared copyright to this code */
17
18
#include "ftdefs.h"
19
20
typedef struct st_ft_docstat {
21
FT_WORD *list;
22
uint uniq;
23
double sum;
24
} FT_DOCSTAT;
25
26
typedef struct st_my_ft_parser_param
27
{
28
TREE *wtree;
29
MEM_ROOT *mem_root;
30
} MY_FT_PARSER_PARAM;
31
32
static int FT_WORD_cmp(CHARSET_INFO* cs, FT_WORD *w1, FT_WORD *w2)
33
{
34
return ha_compare_text(cs, (uchar*) w1->pos, w1->len,
35
(uchar*) w2->pos, w2->len, 0, 0);
36
}
37
38
static int walk_and_copy(FT_WORD *word,uint32 count,FT_DOCSTAT *docstat)
39
{
40
word->weight=LWS_IN_USE;
41
docstat->sum+=word->weight;
42
memcpy_fixed((docstat->list)++,word,sizeof(FT_WORD));
43
return 0;
44
}
45
46
/* transforms tree of words into the array, applying normalization */
47
48
FT_WORD * ft_linearize(TREE *wtree, MEM_ROOT *mem_root)
49
{
50
FT_WORD *wlist,*p;
51
FT_DOCSTAT docstat;
52
DBUG_ENTER("ft_linearize");
53
54
if ((wlist=(FT_WORD *) alloc_root(mem_root, sizeof(FT_WORD)*
55
(1+wtree->elements_in_tree))))
56
{
57
docstat.list=wlist;
58
docstat.uniq=wtree->elements_in_tree;
59
docstat.sum=0;
60
tree_walk(wtree,(tree_walk_action)&walk_and_copy,&docstat,left_root_right);
61
}
62
delete_tree(wtree);
63
if (!wlist)
64
DBUG_RETURN(NULL);
65
66
docstat.list->pos=NULL;
67
68
for (p=wlist;p->pos;p++)
69
{
70
p->weight=PRENORM_IN_USE;
71
}
72
73
for (p=wlist;p->pos;p++)
74
{
75
p->weight/=NORM_IN_USE;
76
}
77
78
DBUG_RETURN(wlist);
79
}
80
81
my_bool ft_boolean_check_syntax_string(const uchar *str)
82
{
83
uint i, j;
84
85
if (!str ||
86
(strlen((char*) str)+1 != sizeof(ft_boolean_syntax)) ||
87
(str[0] != ' ' && str[1] != ' '))
88
return 1;
89
for (i=0; i<sizeof(ft_boolean_syntax); i++)
90
{
91
/* limiting to 7-bit ascii only */
92
if ((unsigned char)(str[i]) > 127 || my_isalnum(default_charset_info, str[i]))
93
return 1;
94
for (j=0; j<i; j++)
95
if (str[i] == str[j] && (i != 11 || j != 10))
96
return 1;
97
}
98
return 0;
99
}
100
101
/*
102
RETURN VALUE
103
0 - eof
104
1 - word found
105
2 - left bracket
106
3 - right bracket
107
4 - stopword found
108
*/
109
uchar ft_get_word(CHARSET_INFO *cs, uchar **start, uchar *end,
110
FT_WORD *word, MYSQL_FTPARSER_BOOLEAN_INFO *param)
111
{
112
uchar *doc=*start;
113
int ctype;
114
uint mwc, length;
115
int mbl;
116
117
param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0);
118
param->weight_adjust= param->wasign= 0;
119
param->type= FT_TOKEN_EOF;
120
121
while (doc<end)
122
{
123
for (; doc < end; doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
124
{
125
mbl= cs->cset->ctype(cs, &ctype, (uchar*)doc, (uchar*)end);
126
if (true_word_char(ctype, *doc))
127
break;
128
if (*doc == FTB_RQUOT && param->quot)
129
{
130
param->quot= (char*) doc;
131
*start=doc+1;
132
param->type= FT_TOKEN_RIGHT_PAREN;
133
goto ret;
134
}
135
if (!param->quot)
136
{
137
if (*doc == FTB_LBR || *doc == FTB_RBR || *doc == FTB_LQUOT)
138
{
139
/* param->prev=' '; */
140
*start=doc+1;
141
if (*doc == FTB_LQUOT)
142
param->quot= (char*) *start;
143
param->type= (*doc == FTB_RBR ? FT_TOKEN_RIGHT_PAREN : FT_TOKEN_LEFT_PAREN);
144
goto ret;
145
}
146
if (param->prev == ' ')
147
{
148
if (*doc == FTB_YES ) { param->yesno=+1; continue; } else
149
if (*doc == FTB_EGAL) { param->yesno= 0; continue; } else
150
if (*doc == FTB_NO ) { param->yesno=-1; continue; } else
151
if (*doc == FTB_INC ) { param->weight_adjust++; continue; } else
152
if (*doc == FTB_DEC ) { param->weight_adjust--; continue; } else
153
if (*doc == FTB_NEG ) { param->wasign= !param->wasign; continue; }
154
}
155
}
156
param->prev=*doc;
157
param->yesno=(FTB_YES==' ') ? 1 : (param->quot != 0);
158
param->weight_adjust= param->wasign= 0;
159
}
160
161
mwc=length=0;
162
for (word->pos= doc; doc < end; length++,
163
doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
164
{
165
mbl= cs->cset->ctype(cs, &ctype, (uchar*)doc, (uchar*)end);
166
if (true_word_char(ctype, *doc))
167
mwc=0;
168
else if (!misc_word_char(*doc) || mwc)
169
break;
170
else
171
mwc++;
172
}
173
param->prev='A'; /* be sure *prev is true_word_char */
174
word->len= (uint)(doc-word->pos) - mwc;
175
if ((param->trunc=(doc<end && *doc == FTB_TRUNC)))
176
doc++;
177
178
if (((length >= ft_min_word_len && !is_stopword((char*) word->pos,
179
word->len))
180
|| param->trunc) && length < ft_max_word_len)
181
{
182
*start=doc;
183
param->type= FT_TOKEN_WORD;
184
goto ret;
185
}
186
else if (length) /* make sure length > 0 (if start contains spaces only) */
187
{
188
*start= doc;
189
param->type= FT_TOKEN_STOPWORD;
190
goto ret;
191
}
192
}
193
if (param->quot)
194
{
195
*start= doc;
196
param->quot= (char*) doc;
197
param->type= 3; /* FT_RBR */
198
goto ret;
199
}
200
ret:
201
return param->type;
202
}
203
204
uchar ft_simple_get_word(CHARSET_INFO *cs, uchar **start, const uchar *end,
205
FT_WORD *word, my_bool skip_stopwords)
206
{
207
uchar *doc= *start;
208
uint mwc, length;
209
int mbl;
210
int ctype;
211
DBUG_ENTER("ft_simple_get_word");
212
213
do
214
{
215
for (;; doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
216
{
217
if (doc >= end)
218
DBUG_RETURN(0);
219
mbl= cs->cset->ctype(cs, &ctype, (uchar*)doc, (uchar*)end);
220
if (true_word_char(ctype, *doc))
221
break;
222
}
223
224
mwc= length= 0;
225
for (word->pos= doc; doc < end; length++,
226
doc+= (mbl > 0 ? mbl : (mbl < 0 ? -mbl : 1)))
227
{
228
mbl= cs->cset->ctype(cs, &ctype, (uchar*)doc, (uchar*)end);
229
if (true_word_char(ctype, *doc))
230
mwc= 0;
231
else if (!misc_word_char(*doc) || mwc)
232
break;
233
else
234
mwc++;
235
}
236
237
word->len= (uint)(doc-word->pos) - mwc;
238
239
if (skip_stopwords == FALSE ||
240
(length >= ft_min_word_len && length < ft_max_word_len &&
241
!is_stopword((char*) word->pos, word->len)))
242
{
243
*start= doc;
244
DBUG_RETURN(1);
245
}
246
} while (doc < end);
247
DBUG_RETURN(0);
248
}
249
250
void ft_parse_init(TREE *wtree, CHARSET_INFO *cs)
251
{
252
DBUG_ENTER("ft_parse_init");
253
if (!is_tree_inited(wtree))
254
init_tree(wtree,0,0,sizeof(FT_WORD),(qsort_cmp2)&FT_WORD_cmp,0,NULL, cs);
255
DBUG_VOID_RETURN;
256
}
257
258
259
static int ft_add_word(MYSQL_FTPARSER_PARAM *param,
260
char *word, int word_len,
261
MYSQL_FTPARSER_BOOLEAN_INFO *boolean_info __attribute__((unused)))
262
{
263
TREE *wtree;
264
FT_WORD w;
265
MY_FT_PARSER_PARAM *ft_param=param->mysql_ftparam;
266
DBUG_ENTER("ft_add_word");
267
wtree= ft_param->wtree;
268
if (param->flags & MYSQL_FTFLAGS_NEED_COPY)
269
{
270
uchar *ptr;
271
DBUG_ASSERT(wtree->with_delete == 0);
272
ptr= (uchar *)alloc_root(ft_param->mem_root, word_len);
273
memcpy(ptr, word, word_len);
274
w.pos= ptr;
275
}
276
else
277
w.pos= (uchar*) word;
278
w.len= word_len;
279
if (!tree_insert(wtree, &w, 0, wtree->custom_arg))
280
{
281
delete_tree(wtree);
282
DBUG_RETURN(1);
283
}
284
DBUG_RETURN(0);
285
}
286
287
288
static int ft_parse_internal(MYSQL_FTPARSER_PARAM *param,
289
char *doc_arg, int doc_len)
290
{
291
uchar *doc= (uchar*) doc_arg;
292
uchar *end= doc + doc_len;
293
MY_FT_PARSER_PARAM *ft_param=param->mysql_ftparam;
294
TREE *wtree= ft_param->wtree;
295
FT_WORD w;
296
DBUG_ENTER("ft_parse_internal");
297
298
while (ft_simple_get_word(wtree->custom_arg, &doc, end, &w, TRUE))
299
if (param->mysql_add_word(param, (char*) w.pos, w.len, 0))
300
DBUG_RETURN(1);
301
DBUG_RETURN(0);
302
}
303
304
305
int ft_parse(TREE *wtree, uchar *doc, int doclen,
306
struct st_mysql_ftparser *parser,
307
MYSQL_FTPARSER_PARAM *param, MEM_ROOT *mem_root)
308
{
309
MY_FT_PARSER_PARAM my_param;
310
DBUG_ENTER("ft_parse");
311
DBUG_ASSERT(parser);
312
313
my_param.wtree= wtree;
314
my_param.mem_root= mem_root;
315
316
param->mysql_parse= ft_parse_internal;
317
param->mysql_add_word= ft_add_word;
318
param->mysql_ftparam= &my_param;
319
param->cs= wtree->custom_arg;
320
param->doc= (char*) doc;
321
param->length= doclen;
322
param->mode= MYSQL_FTPARSER_SIMPLE_MODE;
323
DBUG_RETURN(parser->parse(param));
324
}
325
326
#define MAX_PARAM_NR 2
327
MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info,
328
uint keynr, uint paramnr)
329
{
330
uint32 ftparser_nr;
331
struct st_mysql_ftparser *parser;
332
if (! info->ftparser_param)
333
{
334
/* info->ftparser_param can not be zero after the initialization,
335
because it always includes built-in fulltext parser. And built-in
336
parser can be called even if the table has no fulltext indexes and
337
no varchar/text fields. */
338
if (! info->s->ftparsers)
339
{
340
/* It's ok that modification to shared structure is done w/o mutex
341
locks, because all threads would set the same variables to the
342
same values. */
343
uint i, j, keys= info->s->state.header.keys, ftparsers= 1;
344
for (i= 0; i < keys; i++)
345
{
346
MI_KEYDEF *keyinfo= &info->s->keyinfo[i];
347
if (keyinfo->flag & HA_FULLTEXT)
348
{
349
for (j= 0;; j++)
350
{
351
if (j == i)
352
{
353
keyinfo->ftparser_nr= ftparsers++;
354
break;
355
}
356
if (info->s->keyinfo[j].flag & HA_FULLTEXT &&
357
keyinfo->parser == info->s->keyinfo[j].parser)
358
{
359
keyinfo->ftparser_nr= info->s->keyinfo[j].ftparser_nr;
360
break;
361
}
362
}
363
}
364
}
365
info->s->ftparsers= ftparsers;
366
}
367
/*
368
We have to allocate two MYSQL_FTPARSER_PARAM structures per plugin
369
because in a boolean search a parser is called recursively
370
ftb_find_relevance* calls ftb_check_phrase*
371
(MAX_PARAM_NR=2)
372
*/
373
info->ftparser_param= (MYSQL_FTPARSER_PARAM *)
374
my_malloc(MAX_PARAM_NR * sizeof(MYSQL_FTPARSER_PARAM) *
375
info->s->ftparsers, MYF(MY_WME|MY_ZEROFILL));
376
init_alloc_root(&info->ft_memroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
377
if (! info->ftparser_param)
378
return 0;
379
}
380
if (keynr == NO_SUCH_KEY)
381
{
382
ftparser_nr= 0;
383
parser= &ft_default_parser;
384
}
385
else
386
{
387
ftparser_nr= info->s->keyinfo[keynr].ftparser_nr;
388
parser= info->s->keyinfo[keynr].parser;
389
}
390
DBUG_ASSERT(paramnr < MAX_PARAM_NR);
391
ftparser_nr= ftparser_nr*MAX_PARAM_NR + paramnr;
392
if (! info->ftparser_param[ftparser_nr].mysql_add_word)
393
{
394
/* Note, that mysql_add_word is used here as a flag:
395
mysql_add_word == 0 - parser is not initialized
396
mysql_add_word != 0 - parser is initialized, or no
397
initialization needed. */
398
info->ftparser_param[ftparser_nr].mysql_add_word=
399
(int (*)(struct st_mysql_ftparser_param *, char *, int,
400
MYSQL_FTPARSER_BOOLEAN_INFO *)) 1;
401
if (parser->init && parser->init(&info->ftparser_param[ftparser_nr]))
402
return 0;
403
}
404
return &info->ftparser_param[ftparser_nr];
405
}
406
407
void ftparser_call_deinitializer(MI_INFO *info)
408
{
409
uint i, j, keys= info->s->state.header.keys;
410
free_root(&info->ft_memroot, MYF(0));
411
if (! info->ftparser_param)
412
return;
413
for (i= 0; i < keys; i++)
414
{
415
MI_KEYDEF *keyinfo= &info->s->keyinfo[i];
416
for (j=0; j < MAX_PARAM_NR; j++)
417
{
418
MYSQL_FTPARSER_PARAM *ftparser_param=
419
&info->ftparser_param[keyinfo->ftparser_nr*MAX_PARAM_NR + j];
420
if (keyinfo->flag & HA_FULLTEXT && ftparser_param->mysql_add_word)
421
{
422
if (keyinfo->parser->deinit)
423
keyinfo->parser->deinit(ftparser_param);
424
ftparser_param->mysql_add_word= 0;
425
}
426
else
427
break;
428
}
429
}
430
}
431
Loggerhead is a web-based interface for Breezy
Version: 2.0.1