~drizzle-trunk/drizzle/development : revision 76.1.1

40

only. And it is sufficient to calculate the checksum once only.

41

*/

42

43

#include "ftdefs.h"

43

#include "myisamdef.h"

44

#include <m_ctype.h>

45

#include <stdarg.h>

46

#include <my_getopt.h>

68

static int sort_one_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,

69

my_off_t pagepos, File new_file);

70

static int sort_key_read(MI_SORT_PARAM *sort_param,void *key);

71

static int sort_ft_key_read(MI_SORT_PARAM *sort_param,void *key);

72

71

static int sort_get_next_record(MI_SORT_PARAM *sort_param);

73

72

static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b);

74

static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a);

75

73

static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);

76

74

static my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo,

77

75

uchar *key);

829

827

(*key_checksum)+= mi_byte_checksum((uchar*) key,

830

828

key_length- info->s->rec_reflength);

831

829

record= _mi_dpos(info,0,key+key_length);

832

if (keyinfo->flag & HA_FULLTEXT) /* special handling for ft2 */

833

{

834

uint off;

835

int subkeys;

836

get_key_full_length_rdonly(off, key);

837

subkeys=ft_sintXkorr(key+off);

838

if (subkeys < 0)

839

{

840

ha_rows tmp_keys=0;

841

if (chk_index_down(param,info,&info->s->ft2_keyinfo,record,

842

temp_buff,&tmp_keys,key_checksum,1))

843

goto err;

844

if (tmp_keys + subkeys)

845

{

846

mi_check_print_error(param,

847

"Number of words in the 2nd level tree "

848

"does not match the number in the header. "

849

"Parent word in on the page %s, offset %u",

850

llstr(page,llbuff), (uint) (old_keypos-buff));

851

goto err;

852

}

853

(*keys)+=tmp_keys-1;

854

continue;

855

}

856

/* fall through */

857

}

858

830

if (record >= info->state->data_file_length)

859

831

{

860

832

#ifndef DBUG_OFF

1777

1749

{

1778

1750

if (mi_is_key_active(info->s->state.key_map, i))

1779

1751

{

1780

if (info->s->keyinfo[i].flag & HA_FULLTEXT )

1781

{

1782

if (_mi_ft_add(info, i, key, buff, filepos))

1783

goto err;

1784

}

1785

else

1786

1752

{

1787

1753

uint key_length=_mi_make_key(info,i,key,buff,filepos);

1788

1754

if (_mi_ck_write(info,i,key,key_length))

1800

1766

{

1801

1767

if (mi_is_key_active(info->s->state.key_map, i))

1802

1768

{

1803

if (info->s->keyinfo[i].flag & HA_FULLTEXT)

1804

{

1805

if (_mi_ft_del(info,i, key,buff,filepos))

1806

break;

1807

}

1808

else

1809

1769

{

1810

1770

uint key_length=_mi_make_key(info,i,key,buff,filepos);

1811

1771

if (_mi_ck_delete(info,i,key,key_length))

2048

2008

(key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)

2049

2009

break;

2050

2010

DBUG_ASSERT(keypos <= endpos);

2051

if (keyinfo->flag & HA_FULLTEXT)

2052

{

2053

uint off;

2054

int subkeys;

2055

get_key_full_length_rdonly(off, key);

2056

subkeys=ft_sintXkorr(key+off);

2057

if (subkeys < 0)

2058

{

2059

next_page= _mi_dpos(info,0,key+key_length);

2060

_mi_dpointer(info,keypos-nod_flag-info->s->rec_reflength,

2061

param->new_file_pos); /* Save new pos */

2062

if (sort_one_index(param,info,&info->s->ft2_keyinfo,

2063

next_page,new_file))

2064

goto err;

2065

}

2066

}

2067

2011

}

2068

2012

}

2069

2013

2294

2238

my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));

2295

2239

2296

2240

sort_param.wordlist=NULL;

2297

init_alloc_root(&sort_param.wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);

2298

2241

2299

2242

if (share->data_file_type == DYNAMIC_RECORD)

2300

2243

length=max(share->base.min_pack_length+1,share->base.min_block_length);

2359

2302

info->state->records=info->state->del=share->state.split=0;

2360

2303

info->state->empty=0;

2361

2304

2362

if (sort_param.keyinfo->flag & HA_FULLTEXT)

2363

{

2364

uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*

2365

sort_param.keyinfo->seg->charset->mbmaxlen;

2366

sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;

2367

/*

2368

fulltext indexes may have much more entries than the

2369

number of rows in the table. We estimate the number here.

2370

2371

Note, built-in parser is always nr. 0 - see ftparser_call_initializer()

2372

*/

2373

if (sort_param.keyinfo->ftparser_nr == 0)

2374

{

2375

/*

2376

for built-in parser the number of generated index entries

2377

cannot be larger than the size of the data file divided

2378

by the minimal word's length

2379

*/

2380

sort_info.max_records=

2381

(ha_rows) (sort_info.filelength/ft_min_word_len+1);

2382

}

2383

else

2384

{

2385

/*

2386

for external plugin parser we cannot tell anything at all :(

2387

so, we'll use all the sort memory and start from ~10 buffpeks.

2388

(see _create_index_by_sort)

2389

*/

2390

sort_info.max_records=

2391

10*param->sort_buffer_length/sort_param.key_length;

2392

}

2393

2394

sort_param.key_read=sort_ft_key_read;

2395

sort_param.key_write=sort_ft_key_write;

2396

}

2397

else

2398

2305

{

2399

2306

sort_param.key_read=sort_key_read;

2400

2307

sort_param.key_write=sort_key_write;

2808

2715

istep=1;

2809

2716

if ((!(param->testflag & T_SILENT)))

2810

2717

printf ("- Fixing index %d\n",key+1);

2811

if (sort_param[i].keyinfo->flag & HA_FULLTEXT)

2812

{

2813

sort_param[i].key_read=sort_ft_key_read;

2814

sort_param[i].key_write=sort_ft_key_write;

2815

}

2816

else

2817

2718

{

2818

2719

sort_param[i].key_read=sort_key_read;

2819

2720

sort_param[i].key_write=sort_key_write;

2850

2751

sort_param[i].key_length++;

2851

2752

}

2852

2753

total_key_length+=sort_param[i].key_length;

2853

2854

if (sort_param[i].keyinfo->flag & HA_FULLTEXT)

2855

{

2856

uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*

2857

sort_param[i].keyinfo->seg->charset->mbmaxlen;

2858

sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;

2859

init_alloc_root(&sort_param[i].wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);

2860

}

2861

2754

}

2862

2755

sort_info.total_keys=i;

2863

2756

sort_param[0].master= 1;

3118

3011

DBUG_RETURN(sort_write_record(sort_param));

3119

3012

} /* sort_key_read */

3120

3013

3121

static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)

3122

{

3123

int error;

3124

SORT_INFO *sort_info=sort_param->sort_info;

3125

MI_INFO *info=sort_info->info;

3126

FT_WORD *wptr=0;

3127

DBUG_ENTER("sort_ft_key_read");

3128

3129

if (!sort_param->wordlist)

3130

{

3131

for (;;)

3132

{

3133

free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));

3134

if ((error=sort_get_next_record(sort_param)))

3135

DBUG_RETURN(error);

3136

if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record,

3137

&sort_param->wordroot)))

3138

DBUG_RETURN(1);

3139

if (wptr->pos)

3140

break;

3141

error=sort_write_record(sort_param);

3142

}

3143

sort_param->wordptr=sort_param->wordlist=wptr;

3144

}

3145

else

3146

{

3147

error=0;

3148

wptr=(FT_WORD*)(sort_param->wordptr);

3149

}

3150

3151

sort_param->real_key_length=(info->s->rec_reflength+

3152

_ft_make_key(info, sort_param->key,

3153

key, wptr++, sort_param->filepos));

3154

#ifdef HAVE_purify

3155

if (sort_param->key_length > sort_param->real_key_length)

3156

bzero(key+sort_param->real_key_length,

3157

(sort_param->key_length-sort_param->real_key_length));

3158

#endif

3159

if (!wptr->pos)

3160

{

3161

free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));

3162

sort_param->wordlist=0;

3163

error=sort_write_record(sort_param);

3164

}

3165

else

3166

sort_param->wordptr=(void*)wptr;

3167

3168

DBUG_RETURN(error);

3169

} /* sort_ft_key_read */

3170

3171

3014

3172

3015

/*

3173

3016

Read next record from file using parameters in sort_info.

3813

3656

(uchar*) a, HA_OFFSET_ERROR));

3814

3657

} /* sort_key_write */

3815

3658

3816

int sort_ft_buf_flush(MI_SORT_PARAM *sort_param)

3817

{

3818

SORT_INFO *sort_info=sort_param->sort_info;

3819

SORT_KEY_BLOCKS *key_block=sort_info->key_block;

3820

MYISAM_SHARE *share=sort_info->info->s;

3821

uint val_off, val_len;

3822

int error;

3823

SORT_FT_BUF *ft_buf=sort_info->ft_buf;

3824

uchar *from, *to;

3825

3826

val_len=share->ft2_keyinfo.keylength;

3827

get_key_full_length_rdonly(val_off, ft_buf->lastkey);

3828

to=ft_buf->lastkey+val_off;

3829

3830

if (ft_buf->buf)

3831

{

3832

/* flushing first-level tree */

3833

error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,

3834

HA_OFFSET_ERROR);

3835

for (from=to+val_len;

3836

!error && from < ft_buf->buf;

3837

from+= val_len)

3838

{

3839

memcpy(to, from, val_len);

3840

error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,

3841

HA_OFFSET_ERROR);

3842

}

3843

return error;

3844

}

3845

/* flushing second-level tree keyblocks */

3846

error=flush_pending_blocks(sort_param);

3847

/* updating lastkey with second-level tree info */

3848

ft_intXstore(ft_buf->lastkey+val_off, -ft_buf->count);

3849

_mi_dpointer(sort_info->info, ft_buf->lastkey+val_off+HA_FT_WLEN,

3850

share->state.key_root[sort_param->key]);

3851

/* restoring first level tree data in sort_info/sort_param */

3852

sort_info->key_block=sort_info->key_block_end- sort_info->param->sort_key_blocks;

3853

sort_param->keyinfo=share->keyinfo+sort_param->key;

3854

share->state.key_root[sort_param->key]=HA_OFFSET_ERROR;

3855

/* writing lastkey in first-level tree */

3856

return error ? error :

3857

sort_insert_key(sort_param,sort_info->key_block,

3858

ft_buf->lastkey,HA_OFFSET_ERROR);

3859

}

3860

3861

static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a)

3862

{

3863

uint a_len, val_off, val_len, error;

3864

uchar *p;

3865

SORT_INFO *sort_info=sort_param->sort_info;

3866

SORT_FT_BUF *ft_buf=sort_info->ft_buf;

3867

SORT_KEY_BLOCKS *key_block=sort_info->key_block;

3868

3869

val_len=HA_FT_WLEN+sort_info->info->s->base.rec_reflength;

3870

get_key_full_length_rdonly(a_len, (uchar *)a);

3871

3872

if (!ft_buf)

3873

{

3874

/*

3875

use two-level tree only if key_reflength fits in rec_reflength place

3876

and row format is NOT static - for _mi_dpointer not to garble offsets

3877

*/

3878

if ((sort_info->info->s->base.key_reflength <=

3879

sort_info->info->s->base.rec_reflength) &&

3880

(sort_info->info->s->options &

3881

(HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))

3882

ft_buf=(SORT_FT_BUF *)my_malloc(sort_param->keyinfo->block_length +

3883

sizeof(SORT_FT_BUF), MYF(MY_WME));

3884

3885

if (!ft_buf)

3886

{

3887

sort_param->key_write=sort_key_write;

3888

return sort_key_write(sort_param, a);

3889

}

3890

sort_info->ft_buf=ft_buf;

3891

goto word_init_ft_buf; /* no need to duplicate the code */

3892

}

3893

get_key_full_length_rdonly(val_off, ft_buf->lastkey);

3894

3895

if (ha_compare_text(sort_param->seg->charset,

3896

((uchar *)a)+1,a_len-1,

3897

ft_buf->lastkey+1,val_off-1, 0, 0)==0)

3898

{

3899

if (!ft_buf->buf) /* store in second-level tree */

3900

{

3901

ft_buf->count++;

3902

return sort_insert_key(sort_param,key_block,

3903

((uchar *)a)+a_len, HA_OFFSET_ERROR);

3904

}

3905

3906

/* storing the key in the buffer. */

3907

memcpy (ft_buf->buf, (char *)a+a_len, val_len);

3908

ft_buf->buf+=val_len;

3909

if (ft_buf->buf < ft_buf->end)

3910

return 0;

3911

3912

/* converting to two-level tree */

3913

p=ft_buf->lastkey+val_off;

3914

3915

while (key_block->inited)

3916

key_block++;

3917

sort_info->key_block=key_block;

3918

sort_param->keyinfo=& sort_info->info->s->ft2_keyinfo;

3919

ft_buf->count=(ft_buf->buf - p)/val_len;

3920

3921

/* flushing buffer to second-level tree */

3922

for (error=0; !error && p < ft_buf->buf; p+= val_len)

3923

error=sort_insert_key(sort_param,key_block,p,HA_OFFSET_ERROR);

3924

ft_buf->buf=0;

3925

return error;

3926

}

3927

3928

/* flushing buffer */

3929

if ((error=sort_ft_buf_flush(sort_param)))

3930

return error;

3931

3932

word_init_ft_buf:

3933

a_len+=val_len;

3934

memcpy(ft_buf->lastkey, a, a_len);

3935

ft_buf->buf=ft_buf->lastkey+a_len;

3936

/*

3937

32 is just a safety margin here

3938

(at least max(val_len, sizeof(nod_flag)) should be there).

3939

May be better performance could be achieved if we'd put

3940

(sort_info->keyinfo->block_length-32)/XXX

3941

instead.

3942

TODO: benchmark the best value for XXX.

3943

*/

3944

ft_buf->end=ft_buf->lastkey+ (sort_param->keyinfo->block_length-32);

3945

return 0;

3946

} /* sort_ft_key_write */

3947

3948

3659

3949

3660

/* get pointer to record from a key */

3950

3661

4606

4317

static my_bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)

4607

4318

{

4608

4319

uint key_maxlength=key->maxlength;

4609

if (key->flag & HA_FULLTEXT)

4610

{

4611

uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*

4612

key->seg->charset->mbmaxlen;

4613

key_maxlength+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;

4614

}

4615

return (key->flag & HA_SPATIAL) ||

4616

(key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) &&

4320

return (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) &&

4617

4321

((ulonglong) rows * key_maxlength >

4618

4322

(ulonglong) myisam_max_temp_length));

4619

4323

}