1
/* Copyright (C) 2000-2006 MySQL AB
3
This program is free software; you can redistribute it and/or modify
4
it under the terms of the GNU General Public License as published by
5
the Free Software Foundation; version 2 of the License.
7
This program is distributed in the hope that it will be useful,
8
but WITHOUT ANY WARRANTY; without even the implied warranty of
9
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10
GNU General Public License for more details.
12
You should have received a copy of the GNU General Public License
13
along with this program; if not, write to the Free Software
14
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */
16
/* Describe, check and repair of MyISAM tables */
19
About checksum calculation.
21
There are two types of checksums. Table checksum and row checksum.
23
Row checksum is an additional byte at the end of dynamic length
24
records. It must be calculated if the table is configured for them.
25
Otherwise they must not be used. The variable
26
MYISAM_SHARE::calc_checksum determines if row checksums are used.
27
MI_INFO::checksum is used as temporary storage during row handling.
28
For parallel repair we must assure that only one thread can use this
29
variable. There is no problem on the write side as this is done by one
30
thread only. But when checking a record after read this could go
31
wrong. But since all threads read through a common read buffer, it is
32
sufficient if only one thread checks it.
34
Table checksum is an eight byte value in the header of the index file.
35
It can be calculated even if row checksums are not used. The variable
36
MI_CHECK::glob_crc is calculated over all records.
37
MI_SORT_PARAM::calc_checksum determines if this should be done. This
38
variable is not part of MI_CHECK because it must be set per thread for
39
parallel repair. The global glob_crc must be changed by one thread
40
only. And it is sufficient to calculate the checksum once only.
46
#include <my_getopt.h>
47
#ifdef HAVE_SYS_VADVISE_H
48
#include <sys/vadvise.h>
50
#ifdef HAVE_SYS_MMAN_H
56
#define my_raid_create(A,B,C,D,E,F,G) my_create(A,B,C,G)
57
#define my_raid_delete(A,B,C) my_delete(A,B)
60
/* Functions defined in this file */
62
static int check_k_link(MI_CHECK *param, MI_INFO *info,uint nr);
63
static int chk_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
64
my_off_t page, uchar *buff, ha_rows *keys,
65
ha_checksum *key_checksum, uint level);
66
static uint isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo);
67
static ha_checksum calc_checksum(ha_rows count);
68
static int writekeys(MI_SORT_PARAM *sort_param);
69
static int sort_one_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
70
my_off_t pagepos, File new_file);
71
static int sort_key_read(MI_SORT_PARAM *sort_param,void *key);
72
static int sort_ft_key_read(MI_SORT_PARAM *sort_param,void *key);
73
static int sort_get_next_record(MI_SORT_PARAM *sort_param);
74
static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b);
75
static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a);
76
static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);
77
static my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo,
79
static int sort_insert_key(MI_SORT_PARAM *sort_param,
80
register SORT_KEY_BLOCKS *key_block,
81
uchar *key, my_off_t prev_block);
82
static int sort_delete_record(MI_SORT_PARAM *sort_param);
83
/*static int flush_pending_blocks(MI_CHECK *param);*/
84
static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint blocks,
86
static ha_checksum mi_byte_checksum(const uchar *buf, uint length);
87
static void set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share);
89
void myisamchk_init(MI_CHECK *param)
91
bzero((uchar*) param,sizeof(*param));
92
param->opt_follow_links=1;
93
param->keys_in_use= ~(ulonglong) 0;
94
param->search_after_block=HA_OFFSET_ERROR;
95
param->auto_increment_value= 0;
96
param->use_buffers=USE_BUFFER_INIT;
97
param->read_buffer_length=READ_BUFFER_INIT;
98
param->write_buffer_length=READ_BUFFER_INIT;
99
param->sort_buffer_length=SORT_BUFFER_INIT;
100
param->sort_key_blocks=BUFFERS_WHEN_SORTING;
101
param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL;
102
param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
103
param->start_check_pos=0;
104
param->max_record_length= LONGLONG_MAX;
105
param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE;
106
param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
109
/* Check the status flags for the table */
111
int chk_status(MI_CHECK *param, register MI_INFO *info)
113
MYISAM_SHARE *share=info->s;
115
if (mi_is_crashed_on_repair(info))
116
mi_check_print_warning(param,
117
"Table is marked as crashed and last repair failed");
118
else if (mi_is_crashed(info))
119
mi_check_print_warning(param,
120
"Table is marked as crashed");
121
if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0))
123
/* Don't count this as a real warning, as check can correct this ! */
124
uint save=param->warning_printed;
125
mi_check_print_warning(param,
126
share->state.open_count==1 ?
127
"%d client is using or hasn't closed the table properly" :
128
"%d clients are using or haven't closed the table properly",
129
share->state.open_count);
130
/* If this will be fixed by the check, forget the warning */
131
if (param->testflag & T_UPDATE_STATE)
132
param->warning_printed=save;
137
/* Check delete links */
139
int chk_del(MI_CHECK *param, register MI_INFO *info, uint test_flag)
142
uint delete_link_length;
143
my_off_t empty, next_link, old_link= 0;
144
char buff[22],buff2[22];
145
DBUG_ENTER("chk_del");
147
param->record_checksum=0;
148
delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 :
149
info->s->rec_reflength+1);
151
if (!(test_flag & T_SILENT))
152
puts("- check record delete-chain");
154
next_link=info->s->state.dellink;
155
if (info->state->del == 0)
157
if (test_flag & T_VERBOSE)
159
puts("No recordlinks");
164
if (test_flag & T_VERBOSE)
165
printf("Recordlinks: ");
167
for (i= info->state->del ; i > 0L && next_link != HA_OFFSET_ERROR ; i--)
169
if (*killed_ptr(param))
171
if (test_flag & T_VERBOSE)
172
printf(" %9s",llstr(next_link,buff));
173
if (next_link >= info->state->data_file_length)
175
if (my_pread(info->dfile, (uchar*) buff,delete_link_length,
176
next_link,MYF(MY_NABP)))
178
if (test_flag & T_VERBOSE) puts("");
179
mi_check_print_error(param,"Can't read delete-link at filepos: %s",
180
llstr(next_link,buff));
185
if (test_flag & T_VERBOSE) puts("");
186
mi_check_print_error(param,"Record at pos: %s is not remove-marked",
187
llstr(next_link,buff));
190
if (info->s->options & HA_OPTION_PACK_RECORD)
192
my_off_t prev_link=mi_sizekorr(buff+12);
193
if (empty && prev_link != old_link)
195
if (test_flag & T_VERBOSE) puts("");
196
mi_check_print_error(param,"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2));
200
next_link=mi_sizekorr(buff+4);
201
empty+=mi_uint3korr(buff+1);
205
param->record_checksum+=(ha_checksum) next_link;
206
next_link=_mi_rec_pos(info->s,(uchar*) buff+1);
207
empty+=info->s->base.pack_reclength;
210
if (test_flag & T_VERBOSE)
212
if (empty != info->state->empty)
214
mi_check_print_warning(param,
215
"Found %s deleted space in delete link chain. Should be %s",
217
llstr(info->state->empty,buff));
219
if (next_link != HA_OFFSET_ERROR)
221
mi_check_print_error(param,
222
"Found more than the expected %s deleted rows in delete link chain",
223
llstr(info->state->del, buff));
228
mi_check_print_error(param,
229
"Found %s deleted rows in delete link chain. Should be %s",
230
llstr(info->state->del - i, buff2),
231
llstr(info->state->del, buff));
238
param->testflag|=T_RETRY_WITHOUT_QUICK;
239
if (test_flag & T_VERBOSE) puts("");
240
mi_check_print_error(param,"record delete-link-chain corrupted");
245
/* Check delete links in index file */
247
static int check_k_link(MI_CHECK *param, register MI_INFO *info, uint nr)
250
uint block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH;
252
char llbuff[21], llbuff2[21];
254
DBUG_ENTER("check_k_link");
255
DBUG_PRINT("enter", ("block_size: %u", block_size));
257
if (param->testflag & T_VERBOSE)
258
printf("block_size %4u:", block_size); /* purecov: tested */
260
next_link=info->s->state.key_del[nr];
261
records= (ha_rows) (info->state->key_file_length / block_size);
262
while (next_link != HA_OFFSET_ERROR && records > 0)
264
if (*killed_ptr(param))
266
if (param->testflag & T_VERBOSE)
267
printf("%16s",llstr(next_link,llbuff));
269
/* Key blocks must lay within the key file length entirely. */
270
if (next_link + block_size > info->state->key_file_length)
272
/* purecov: begin tested */
273
mi_check_print_error(param, "Invalid key block position: %s "
274
"key block size: %u file_length: %s",
275
llstr(next_link, llbuff), block_size,
276
llstr(info->state->key_file_length, llbuff2));
281
/* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
282
if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1))
284
/* purecov: begin tested */
285
mi_check_print_error(param, "Mis-aligned key block: %s "
286
"minimum key block length: %u",
287
llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
293
Read the key block with MI_MIN_KEY_BLOCK_LENGTH to find next link.
294
If the key cache block size is smaller than block_size, we can so
295
avoid unecessary eviction of cache block.
297
if (!(buff=key_cache_read(info->s->key_cache,
298
info->s->kfile, next_link, DFLT_INIT_HITS,
299
(uchar*) info->buff, MI_MIN_KEY_BLOCK_LENGTH,
300
MI_MIN_KEY_BLOCK_LENGTH, 1)))
302
/* purecov: begin tested */
303
mi_check_print_error(param, "key cache read error for block: %s",
304
llstr(next_link,llbuff));
308
next_link=mi_sizekorr(buff);
310
param->key_file_blocks+=block_size;
312
if (param->testflag & T_VERBOSE)
314
if (next_link != HA_OFFSET_ERROR)
315
printf("%16s\n",llstr(next_link,llbuff));
319
DBUG_RETURN (next_link != HA_OFFSET_ERROR);
323
/* Check sizes of files */
325
int chk_size(MI_CHECK *param, register MI_INFO *info)
328
register my_off_t skr,size;
329
char buff[22],buff2[22];
330
DBUG_ENTER("chk_size");
332
if (!(param->testflag & T_SILENT)) puts("- check file-size");
334
/* The following is needed if called externally (not from myisamchk) */
335
flush_key_blocks(info->s->key_cache,
336
info->s->kfile, FLUSH_FORCE_WRITE);
338
size= my_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(MY_THREADSAFE));
339
if ((skr=(my_off_t) info->state->key_file_length) != size)
341
/* Don't give error if file generated by myisampack */
342
if (skr > size && mi_is_any_key_active(info->s->state.key_map))
345
mi_check_print_error(param,
346
"Size of indexfile is: %-8s Should be: %s",
347
llstr(size,buff), llstr(skr,buff2));
350
mi_check_print_warning(param,
351
"Size of indexfile is: %-8s Should be: %s",
352
llstr(size,buff), llstr(skr,buff2));
354
if (!(param->testflag & T_VERY_SILENT) &&
355
! (info->s->options & HA_OPTION_COMPRESS_RECORD) &&
356
ulonglong2double(info->state->key_file_length) >
357
ulonglong2double(info->s->base.margin_key_file_length)*0.9)
358
mi_check_print_warning(param,"Keyfile is almost full, %10s of %10s used",
359
llstr(info->state->key_file_length,buff),
360
llstr(info->s->base.max_key_file_length-1,buff));
362
size=my_seek(info->dfile,0L,MY_SEEK_END,MYF(0));
363
skr=(my_off_t) info->state->data_file_length;
364
if (info->s->options & HA_OPTION_COMPRESS_RECORD)
365
skr+= MEMMAP_EXTRA_MARGIN;
367
if (info->data_file_type == STATIC_RECORD &&
368
skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length)
369
skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length;
373
info->state->data_file_length=size; /* Skip other errors */
374
if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN)
377
mi_check_print_error(param,"Size of datafile is: %-9s Should be: %s",
378
llstr(size,buff), llstr(skr,buff2));
379
param->testflag|=T_RETRY_WITHOUT_QUICK;
383
mi_check_print_warning(param,
384
"Size of datafile is: %-9s Should be: %s",
385
llstr(size,buff), llstr(skr,buff2));
388
if (!(param->testflag & T_VERY_SILENT) &&
389
!(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
390
ulonglong2double(info->state->data_file_length) >
391
(ulonglong2double(info->s->base.max_data_file_length)*0.9))
392
mi_check_print_warning(param, "Datafile is almost full, %10s of %10s used",
393
llstr(info->state->data_file_length,buff),
394
llstr(info->s->base.max_data_file_length-1,buff2));
401
int chk_key(MI_CHECK *param, register MI_INFO *info)
403
uint key,found_keys=0,full_text_keys=0,result=0;
405
ha_checksum old_record_checksum,init_checksum;
406
my_off_t all_keydata,all_totaldata,key_totlength,length;
407
ulong *rec_per_key_part;
408
MYISAM_SHARE *share=info->s;
410
char buff[22],buff2[22];
411
DBUG_ENTER("chk_key");
413
if (!(param->testflag & T_SILENT))
414
puts("- check key delete-chain");
416
param->key_file_blocks=info->s->base.keystart;
417
for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
418
if (check_k_link(param,info,key))
420
if (param->testflag & T_VERBOSE) puts("");
421
mi_check_print_error(param,"key delete-link-chain corrupted");
425
if (!(param->testflag & T_SILENT)) puts("- check index reference");
427
all_keydata=all_totaldata=key_totlength=0;
428
old_record_checksum=0;
429
init_checksum=param->record_checksum;
430
if (!(share->options &
431
(HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
432
old_record_checksum=calc_checksum(info->state->records+info->state->del-1)*
433
share->base.pack_reclength;
434
rec_per_key_part= param->rec_per_key_part;
435
for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
436
rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
438
param->key_crc[key]=0;
439
if (! mi_is_key_active(share->state.key_map, key))
441
/* Remember old statistics for key */
442
memcpy((char*) rec_per_key_part,
443
(char*) (share->state.rec_per_key_part +
444
(uint) (rec_per_key_part - param->rec_per_key_part)),
445
keyinfo->keysegs*sizeof(*rec_per_key_part));
450
param->record_checksum=init_checksum;
452
bzero((char*) ¶m->unique_count,sizeof(param->unique_count));
453
bzero((char*) ¶m->notnull_count,sizeof(param->notnull_count));
455
if ((!(param->testflag & T_SILENT)))
456
printf ("- check data record references index: %d\n",key+1);
457
if (keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL))
459
if (share->state.key_root[key] == HA_OFFSET_ERROR &&
460
(info->state->records == 0 || keyinfo->flag & HA_FULLTEXT))
462
if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],
463
DFLT_INIT_HITS,info->buff,0))
465
mi_check_print_error(param,"Can't read indexpage from filepos: %s",
466
llstr(share->state.key_root[key],buff));
467
if (!(param->testflag & T_INFO))
472
param->key_file_blocks+=keyinfo->block_length;
474
param->keydata=param->totaldata=0;
477
if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
478
&keys, param->key_crc+key,1))
480
if(!(keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL)))
482
if (keys != info->state->records)
484
mi_check_print_error(param,"Found %s keys of %s",llstr(keys,buff),
485
llstr(info->state->records,buff2));
486
if (!(param->testflag & T_INFO))
491
if (found_keys - full_text_keys == 1 &&
493
(HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
494
(param->testflag & T_DONT_CHECK_CHECKSUM)))
495
old_record_checksum=param->record_checksum;
496
else if (old_record_checksum != param->record_checksum)
499
mi_check_print_error(param,"Key %u doesn't point at same records that key 1",
502
mi_check_print_error(param,"Key 1 doesn't point at all records");
503
if (!(param->testflag & T_INFO))
509
if ((uint) share->base.auto_key -1 == key)
511
/* Check that auto_increment key is bigger than max key value */
512
ulonglong auto_increment;
514
_mi_read_key_record(info, 0L, info->rec_buff);
515
auto_increment= retrieve_auto_increment(info, info->rec_buff);
516
if (auto_increment > info->s->state.auto_increment)
518
mi_check_print_warning(param, "Auto-increment value: %s is smaller "
519
"than max used value: %s",
520
llstr(info->s->state.auto_increment,buff2),
521
llstr(auto_increment, buff));
523
if (param->testflag & T_AUTO_INC)
525
set_if_bigger(info->s->state.auto_increment,
527
set_if_bigger(info->s->state.auto_increment,
528
param->auto_increment_value);
531
/* Check that there isn't a row with auto_increment = 0 in the table */
532
mi_extra(info,HA_EXTRA_KEYREAD,0);
533
bzero(info->lastkey,keyinfo->seg->length);
534
if (!mi_rkey(info, info->rec_buff, key, (const uchar*) info->lastkey,
535
(key_part_map)1, HA_READ_KEY_EXACT))
537
/* Don't count this as a real warning, as myisamchk can't correct it */
538
uint save=param->warning_printed;
539
mi_check_print_warning(param, "Found row where the auto_increment "
540
"column has the value 0");
541
param->warning_printed=save;
543
mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
546
length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2;
547
if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
548
printf("Key: %2d: Keyblocks used: %3d%% Packed: %4d%% Max levels: %2d\n",
550
(int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)),
551
(int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/
552
my_off_t2double(length)),
554
all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length;
557
if (param->testflag & T_STATISTICS)
558
update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
559
param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
560
param->notnull_count: NULL,
561
(ulonglong)info->state->records);
563
if (param->testflag & T_INFO)
565
if (all_totaldata != 0L && found_keys > 0)
566
printf("Total: Keyblocks used: %3d%% Packed: %4d%%\n\n",
567
(int) (my_off_t2double(all_keydata)*100.0/
568
my_off_t2double(all_totaldata)),
569
(int) ((my_off_t2double(key_totlength) -
570
my_off_t2double(all_keydata))*100.0/
571
my_off_t2double(key_totlength)));
572
else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map))
575
if (param->key_file_blocks != info->state->key_file_length &&
576
param->keys_in_use != ~(ulonglong) 0)
577
mi_check_print_warning(param, "Some data are unreferenced in keyfile");
578
if (found_keys != full_text_keys)
579
param->record_checksum=old_record_checksum-init_checksum; /* Remove delete links */
581
param->record_checksum=0;
586
static int chk_index_down(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
587
my_off_t page, uchar *buff, ha_rows *keys,
588
ha_checksum *key_checksum, uint level)
590
char llbuff[22],llbuff2[22];
591
DBUG_ENTER("chk_index_down");
593
/* Key blocks must lay within the key file length entirely. */
594
if (page + keyinfo->block_length > info->state->key_file_length)
596
/* purecov: begin tested */
597
/* Give it a chance to fit in the real file size. */
598
my_off_t max_length= my_seek(info->s->kfile, 0L, MY_SEEK_END,
600
mi_check_print_error(param, "Invalid key block position: %s "
601
"key block size: %u file_length: %s",
602
llstr(page, llbuff), keyinfo->block_length,
603
llstr(info->state->key_file_length, llbuff2));
604
if (page + keyinfo->block_length > max_length)
606
/* Fix the remebered key file length. */
607
info->state->key_file_length= (max_length &
608
~ (my_off_t) (keyinfo->block_length - 1));
612
/* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
613
if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1))
615
/* purecov: begin tested */
616
mi_check_print_error(param, "Mis-aligned key block: %s "
617
"minimum key block length: %u",
618
llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
623
if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0))
625
mi_check_print_error(param,"Can't read key from filepos: %s",
629
param->key_file_blocks+=keyinfo->block_length;
630
if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level))
635
/* purecov: begin tested */
643
"Ignore NULLs" statistics collection method: process first index tuple.
646
mi_collect_stats_nonulls_first()
647
keyseg IN Array of key part descriptions
648
notnull INOUT Array, notnull[i] = (number of {keypart1...keypart_i}
649
tuples that don't contain NULLs)
650
key IN Key values tuple
653
Process the first index tuple - find out which prefix tuples don't
654
contain NULLs, and update the array of notnull counters accordingly.
658
void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg, ulonglong *notnull,
662
first_null= ha_find_null(keyseg, key) - keyseg;
664
All prefix tuples that don't include keypart_{first_null} are not-null
665
tuples (and all others aren't), increment counters for them.
667
for (kp= 0; kp < first_null; kp++)
673
"Ignore NULLs" statistics collection method: process next index tuple.
676
mi_collect_stats_nonulls_next()
677
keyseg IN Array of key part descriptions
678
notnull INOUT Array, notnull[i] = (number of {keypart1...keypart_i}
679
tuples that don't contain NULLs)
680
prev_key IN Previous key values tuple
681
last_key IN Next key values tuple
684
Process the next index tuple:
685
1. Find out which prefix tuples of last_key don't contain NULLs, and
686
update the array of notnull counters accordingly.
687
2. Find the first keypart number where the prev_key and last_key tuples
688
are different(A), or last_key has NULL value(B), and return it, so the
689
caller can count number of unique tuples for each key prefix. We don't
690
need (B) to be counted, and that is compensated back in
694
1 + number of first keypart where values differ or last_key tuple has NULL
698
int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, ulonglong *notnull,
699
uchar *prev_key, uchar *last_key)
702
uint first_null_seg, kp;
706
Find the first keypart where values are different or either of them is
707
NULL. We get results in diffs array:
708
diffs[0]= 1 + number of first different keypart
709
diffs[1]=offset: (last_key + diffs[1]) points to first value in
710
last_key that is NULL or different from corresponding
713
ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY,
714
SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
715
seg= keyseg + diffs[0] - 1;
717
/* Find first NULL in last_key */
718
first_null_seg= ha_find_null(seg, last_key + diffs[1]) - keyseg;
719
for (kp= 0; kp < first_null_seg; kp++)
723
Return 1+ number of first key part where values differ. Don't care if
724
these were NULLs and not .... We compensate for that in
731
/* Check if index is ok */
733
static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
734
my_off_t page, uchar *buff, ha_rows *keys,
735
ha_checksum *key_checksum, uint level)
738
uint used_length,comp_flag,nod_flag,key_length=0;
739
uchar key[HA_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*old_keypos,*endpos;
740
my_off_t next_page,record;
743
DBUG_ENTER("chk_index");
744
DBUG_DUMP("buff",(uchar*) buff,mi_getint(buff));
746
/* TODO: implement appropriate check for RTree keys */
747
if (keyinfo->flag & HA_SPATIAL)
750
if (!(temp_buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
752
mi_check_print_error(param,"Not enough memory for keyblock");
756
if (keyinfo->flag & HA_NOSAME)
757
comp_flag=SEARCH_FIND | SEARCH_UPDATE; /* Not real duplicates */
759
comp_flag=SEARCH_SAME; /* Keys in positionorder */
760
nod_flag=mi_test_if_nod(buff);
761
used_length=mi_getint(buff);
762
keypos=buff+2+nod_flag;
763
endpos=buff+used_length;
765
param->keydata+=used_length; param->totaldata+=keyinfo->block_length; /* INFO */
767
if (level > param->max_level)
768
param->max_level=level;
770
if (used_length > keyinfo->block_length)
772
mi_check_print_error(param,"Wrong pageinfo at page: %s",
778
if (*killed_ptr(param))
780
memcpy((char*) info->lastkey,(char*) key,key_length);
781
info->lastkey_length=key_length;
784
next_page=_mi_kpos(nod_flag,keypos);
785
if (chk_index_down(param,info,keyinfo,next_page,
786
temp_buff,keys,key_checksum,level+1))
790
if (keypos >= endpos ||
791
(key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
793
DBUG_ASSERT(key_length <= sizeof(key));
796
mi_check_print_error(param,"Wrong key block length at page: %s",llstr(page,llbuff));
800
(flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
801
comp_flag, diff_pos)) >=0)
803
DBUG_DUMP("old",(uchar*) info->lastkey, info->lastkey_length);
804
DBUG_DUMP("new",(uchar*) key, key_length);
805
DBUG_DUMP("new_in_page",(uchar*) old_keypos,(uint) (keypos-old_keypos));
807
if (comp_flag & SEARCH_FIND && flag == 0)
808
mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff));
810
mi_check_print_error(param,"Key in wrong position at page %s",llstr(page,llbuff));
813
if (param->testflag & T_STATISTICS)
815
if (*keys != 1L) /* not first_key */
817
if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
818
ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,
819
SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL,
821
else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
823
diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg,
824
param->notnull_count,
827
param->unique_count[diff_pos[0]-1]++;
831
if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
832
mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
836
(*key_checksum)+= mi_byte_checksum((uchar*) key,
837
key_length- info->s->rec_reflength);
838
record= _mi_dpos(info,0,key+key_length);
839
if (keyinfo->flag & HA_FULLTEXT) /* special handling for ft2 */
843
get_key_full_length_rdonly(off, key);
844
subkeys=ft_sintXkorr(key+off);
848
if (chk_index_down(param,info,&info->s->ft2_keyinfo,record,
849
temp_buff,&tmp_keys,key_checksum,1))
851
if (tmp_keys + subkeys)
853
mi_check_print_error(param,
854
"Number of words in the 2nd level tree "
855
"does not match the number in the header. "
856
"Parent word in on the page %s, offset %u",
857
llstr(page,llbuff), (uint) (old_keypos-buff));
865
if (record >= info->state->data_file_length)
868
char llbuff2[22], llbuff3[22];
870
mi_check_print_error(param,"Found key at page %s that points to record outside datafile",llstr(page,llbuff));
871
DBUG_PRINT("test",("page: %s record: %s filelength: %s",
872
llstr(page,llbuff),llstr(record,llbuff2),
873
llstr(info->state->data_file_length,llbuff3)));
874
DBUG_DUMP("key",(uchar*) key,key_length);
875
DBUG_DUMP("new_in_page",(uchar*) old_keypos,(uint) (keypos-old_keypos));
878
param->record_checksum+=(ha_checksum) record;
880
if (keypos != endpos)
882
mi_check_print_error(param,"Keyblock size at page %s is not correct. Block length: %d key length: %d",
883
llstr(page,llbuff), used_length, (keypos - buff));
886
my_afree((uchar*) temp_buff);
889
my_afree((uchar*) temp_buff);
894
/* Calculate a checksum of 1+2+3+4...N = N*(N+1)/2 without overflow */
896
static ha_checksum calc_checksum(ha_rows count)
899
DBUG_ENTER("calc_checksum");
913
DBUG_PRINT("exit",("sum: %lx",(ulong) sum));
914
DBUG_RETURN((ha_checksum) sum);
915
} /* calc_checksum */
918
/* Calc length of key in normal isam */
920
static uint isam_key_length(MI_INFO *info, register MI_KEYDEF *keyinfo)
924
DBUG_ENTER("isam_key_length");
926
length= info->s->rec_reflength;
927
for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++)
928
length+= keyseg->length;
930
DBUG_PRINT("exit",("length: %d",length));
935
/* Check that record-link is ok */
937
int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend)
939
int error,got_error,flag;
940
uint key, left_length= 0, b_type,field;
941
ha_rows records, del_blocks;
942
my_off_t used, empty, pos, splits, start_recpos= 0,
943
del_length, link_used, start_block;
944
uchar *record= NULL, *to= NULL;
945
char llbuff[22],llbuff2[22],llbuff3[22];
946
ha_checksum intern_record_checksum;
947
ha_checksum key_checksum[HA_MAX_POSSIBLE_KEY];
948
my_bool static_row_size;
950
MI_BLOCK_INFO block_info;
951
DBUG_ENTER("chk_data_link");
953
if (!(param->testflag & T_SILENT))
956
puts("- check records and index references");
958
puts("- check record links");
961
if (!mi_alloc_rec_buff(info, -1, &record))
963
mi_check_print_error(param,"Not enough memory for record");
966
records=del_blocks=0;
967
used=link_used=splits=del_length=0;
968
intern_record_checksum=param->glob_crc=0;
970
empty=info->s->pack.header_length;
972
/* Check how to calculate checksum of rows */
974
if (info->s->data_file_type == COMPRESSED_RECORD)
976
for (field=0 ; field < info->s->base.fields ; field++)
978
if (info->s->rec[field].base_type == FIELD_BLOB ||
979
info->s->rec[field].base_type == FIELD_VARCHAR)
987
pos=my_b_tell(¶m->read_cache);
988
bzero((char*) key_checksum, info->s->base.keys * sizeof(key_checksum[0]));
989
while (pos < info->state->data_file_length)
991
if (*killed_ptr(param))
993
switch (info->s->data_file_type) {
995
if (my_b_read(¶m->read_cache,(uchar*) record,
996
info->s->base.pack_reclength))
999
pos+=info->s->base.pack_reclength;
1001
if (*record == '\0')
1004
del_length+=info->s->base.pack_reclength;
1005
continue; /* Record removed */
1007
param->glob_crc+= mi_static_checksum(info,record);
1008
used+=info->s->base.pack_reclength;
1010
case DYNAMIC_RECORD:
1011
flag=block_info.second_read=0;
1012
block_info.next_filepos=pos;
1015
if (_mi_read_cache(¶m->read_cache,(uchar*) block_info.header,
1016
(start_block=block_info.next_filepos),
1017
sizeof(block_info.header),
1018
(flag ? 0 : READING_NEXT) | READING_HEADER))
1020
if (start_block & (MI_DYN_ALIGN_SIZE-1))
1022
mi_check_print_error(param,"Wrong aligned block at %s",
1023
llstr(start_block,llbuff));
1026
b_type=_mi_get_block_info(&block_info,-1,start_block);
1027
if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
1030
if (b_type & BLOCK_SYNC_ERROR)
1034
mi_check_print_error(param,"Unexpected byte: %d at link: %s",
1035
(int) block_info.header[0],
1036
llstr(start_block,llbuff));
1039
pos=block_info.filepos+block_info.block_len;
1042
if (b_type & BLOCK_DELETED)
1044
if (block_info.block_len < info->s->base.min_block_length)
1046
mi_check_print_error(param,
1047
"Deleted block with impossible length %lu at %s",
1048
block_info.block_len,llstr(pos,llbuff));
1051
if ((block_info.next_filepos != HA_OFFSET_ERROR &&
1052
block_info.next_filepos >= info->state->data_file_length) ||
1053
(block_info.prev_filepos != HA_OFFSET_ERROR &&
1054
block_info.prev_filepos >= info->state->data_file_length))
1056
mi_check_print_error(param,"Delete link points outside datafile at %s",
1061
del_length+=block_info.block_len;
1062
pos=block_info.filepos+block_info.block_len;
1066
mi_check_print_error(param,"Wrong bytesec: %d-%d-%d at linkstart: %s",
1067
block_info.header[0],block_info.header[1],
1068
block_info.header[2],
1069
llstr(start_block,llbuff));
1072
if (info->state->data_file_length < block_info.filepos+
1073
block_info.block_len)
1075
mi_check_print_error(param,
1076
"Recordlink that points outside datafile at %s",
1082
if (!flag++) /* First block */
1085
pos=block_info.filepos+block_info.block_len;
1086
if (block_info.rec_len > (uint) info->s->base.max_pack_length)
1088
mi_check_print_error(param,"Found too long record (%lu) at %s",
1089
(ulong) block_info.rec_len,
1090
llstr(start_recpos,llbuff));
1094
if (info->s->base.blobs)
1096
if (!(to= mi_alloc_rec_buff(info, block_info.rec_len,
1099
mi_check_print_error(param,
1100
"Not enough memory (%lu) for blob at %s",
1101
(ulong) block_info.rec_len,
1102
llstr(start_recpos,llbuff));
1109
left_length=block_info.rec_len;
1111
if (left_length < block_info.data_len)
1113
mi_check_print_error(param,"Found too long record (%lu) at %s",
1114
(ulong) block_info.data_len,
1115
llstr(start_recpos,llbuff));
1119
if (_mi_read_cache(¶m->read_cache,(uchar*) to,block_info.filepos,
1120
(uint) block_info.data_len,
1121
flag == 1 ? READING_NEXT : 0))
1123
to+=block_info.data_len;
1124
link_used+= block_info.filepos-start_block;
1125
used+= block_info.filepos - start_block + block_info.data_len;
1126
empty+=block_info.block_len-block_info.data_len;
1127
left_length-=block_info.data_len;
1130
if (b_type & BLOCK_LAST)
1132
mi_check_print_error(param,
1133
"Wrong record length %s of %s at %s",
1134
llstr(block_info.rec_len-left_length,llbuff),
1135
llstr(block_info.rec_len, llbuff2),
1136
llstr(start_recpos,llbuff3));
1140
if (info->state->data_file_length < block_info.next_filepos)
1142
mi_check_print_error(param,
1143
"Found next-recordlink that points outside datafile at %s",
1144
llstr(block_info.filepos,llbuff));
1149
} while (left_length);
1152
if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) ==
1155
mi_check_print_error(param,"Found wrong record at %s",
1156
llstr(start_recpos,llbuff));
1161
info->checksum=mi_checksum(info,record);
1162
if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE))
1164
if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len,
1165
test(info->s->calc_checksum)))
1167
mi_check_print_error(param,"Found wrong packed record at %s",
1168
llstr(start_recpos,llbuff));
1173
param->glob_crc+= info->checksum;
1177
pos=block_info.filepos+block_info.block_len;
1179
case COMPRESSED_RECORD:
1180
if (_mi_read_cache(¶m->read_cache,(uchar*) block_info.header, pos,
1181
info->s->pack.ref_length, READING_NEXT))
1185
VOID(_mi_pack_get_block_info(info, &info->bit_buff, &block_info,
1186
&info->rec_buff, -1, start_recpos));
1187
pos=block_info.filepos+block_info.rec_len;
1188
if (block_info.rec_len < (uint) info->s->min_pack_length ||
1189
block_info.rec_len > (uint) info->s->max_pack_length)
1191
mi_check_print_error(param,
1192
"Found block with wrong recordlength: %d at %s",
1193
block_info.rec_len, llstr(start_recpos,llbuff));
1197
if (_mi_read_cache(¶m->read_cache,(uchar*) info->rec_buff,
1198
block_info.filepos, block_info.rec_len, READING_NEXT))
1200
if (_mi_pack_rec_unpack(info, &info->bit_buff, record,
1201
info->rec_buff, block_info.rec_len))
1203
mi_check_print_error(param,"Found wrong record at %s",
1204
llstr(start_recpos,llbuff));
1207
if (static_row_size)
1208
param->glob_crc+= mi_static_checksum(info,record);
1210
param->glob_crc+= mi_checksum(info,record);
1211
link_used+= (block_info.filepos - start_recpos);
1212
used+= (pos-start_recpos);
1214
assert(0); /* Impossible */
1218
intern_record_checksum+=(ha_checksum) start_recpos;
1220
if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0)
1222
printf("%s\r", llstr(records,llbuff)); VOID(fflush(stdout));
1225
/* Check if keys match the record */
1227
for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys;
1230
if (mi_is_key_active(info->s->state.key_map, key))
1232
if(!(keyinfo->flag & HA_FULLTEXT))
1234
uint key_length=_mi_make_key(info,key,info->lastkey,record,
1238
/* We don't need to lock the key tree here as we don't allow
1239
concurrent threads when running myisamchk
1242
#ifdef HAVE_RTREE_KEYS
1243
(keyinfo->flag & HA_SPATIAL) ?
1244
rtree_find_first(info, key, info->lastkey, key_length,
1245
MBR_EQUAL | MBR_DATA) :
1247
_mi_search(info,keyinfo,info->lastkey,key_length,
1248
SEARCH_SAME, info->s->state.key_root[key]);
1251
mi_check_print_error(param,"Record at: %10s "
1252
"Can't find key for index: %2d",
1253
llstr(start_recpos,llbuff),key+1);
1254
if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1259
key_checksum[key]+=mi_byte_checksum((uchar*) info->lastkey,
1268
if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1271
next:; /* Next record */
1273
if (param->testflag & T_WRITE_LOOP)
1275
VOID(fputs(" \r",stdout)); VOID(fflush(stdout));
1277
if (records != info->state->records)
1279
mi_check_print_error(param,"Record-count is not ok; is %-10s Should be: %s",
1280
llstr(records,llbuff), llstr(info->state->records,llbuff2));
1283
else if (param->record_checksum &&
1284
param->record_checksum != intern_record_checksum)
1286
mi_check_print_error(param,
1287
"Keypointers and record positions doesn't match");
1290
else if (param->glob_crc != info->state->checksum &&
1292
(HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)))
1294
mi_check_print_warning(param,
1295
"Record checksum is not the same as checksum stored in the index file\n");
1300
for (key=0 ; key < info->s->base.keys; key++)
1302
if (key_checksum[key] != param->key_crc[key] &&
1303
!(info->s->keyinfo[key].flag & (HA_FULLTEXT | HA_SPATIAL)))
1305
mi_check_print_error(param,"Checksum for key: %2d doesn't match checksum for records",
1312
if (del_length != info->state->empty)
1314
mi_check_print_warning(param,
1315
"Found %s deleted space. Should be %s",
1316
llstr(del_length,llbuff2),
1317
llstr(info->state->empty,llbuff));
1319
if (used+empty+del_length != info->state->data_file_length)
1321
mi_check_print_warning(param,
1322
"Found %s record-data and %s unused data and %s deleted-data",
1323
llstr(used,llbuff),llstr(empty,llbuff2),
1324
llstr(del_length,llbuff3));
1325
mi_check_print_warning(param,
1326
"Total %s, Should be: %s",
1327
llstr((used+empty+del_length),llbuff),
1328
llstr(info->state->data_file_length,llbuff2));
1330
if (del_blocks != info->state->del)
1332
mi_check_print_warning(param,
1333
"Found %10s deleted blocks Should be: %s",
1334
llstr(del_blocks,llbuff),
1335
llstr(info->state->del,llbuff2));
1337
if (splits != info->s->state.split)
1339
mi_check_print_warning(param,
1340
"Found %10s parts Should be: %s parts",
1341
llstr(splits,llbuff),
1342
llstr(info->s->state.split,llbuff2));
1344
if (param->testflag & T_INFO)
1346
if (param->warning_printed || param->error_printed)
1348
if (used != 0 && ! param->error_printed)
1350
printf("Records:%18s M.recordlength:%9lu Packed:%14.0f%%\n",
1351
llstr(records,llbuff), (long)((used-link_used)/records),
1352
(info->s->base.blobs ? 0.0 :
1353
(ulonglong2double((ulonglong) info->s->base.reclength*records)-
1354
my_off_t2double(used))/
1355
ulonglong2double((ulonglong) info->s->base.reclength*records)*100.0));
1356
printf("Recordspace used:%9.0f%% Empty space:%12d%% Blocks/Record: %6.2f\n",
1357
(ulonglong2double(used-link_used)/ulonglong2double(used-link_used+empty)*100.0),
1358
(!records ? 100 : (int) (ulonglong2double(del_length+empty)/
1359
my_off_t2double(used)*100.0)),
1360
ulonglong2double(splits - del_blocks) / records);
1362
printf("Record blocks:%12s Delete blocks:%10s\n",
1363
llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2));
1364
printf("Record data: %12s Deleted data: %10s\n",
1365
llstr(used-link_used,llbuff),llstr(del_length,llbuff2));
1366
printf("Lost space: %12s Linkdata: %10s\n",
1367
llstr(empty,llbuff),llstr(link_used,llbuff2));
1369
my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
1370
DBUG_RETURN (error);
1372
mi_check_print_error(param,"got error: %d when reading datafile at record: %s",my_errno, llstr(records,llbuff));
1374
my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
1375
param->testflag|=T_RETRY_WITHOUT_QUICK;
1377
} /* chk_data_link */
1381
@brief Drop all indexes
1383
@param[in] param check parameters
1384
@param[in] info MI_INFO handle
1385
@param[in] force if to force drop all indexes
1392
Once allocated, index blocks remain part of the key file forever.
1393
When indexes are disabled, no block is freed. When enabling indexes,
1394
no block is freed either. The new indexes are create from new
1397
Before recreating formerly disabled indexes, the unused blocks
1398
must be freed. There are two options to do this:
1399
- Follow the tree of disabled indexes, add all blocks to the
1400
deleted blocks chain. Would require a lot of random I/O.
1401
- Drop all blocks by clearing all index root pointers and all
1402
delete chain pointers and resetting key_file_length to the end
1403
of the index file header. This requires to recreate all indexes,
1404
even those that may still be intact.
1405
The second method is probably faster in most cases.
1407
When disabling indexes, MySQL disables either all indexes or all
1408
non-unique indexes. When MySQL [re-]enables disabled indexes
1409
(T_CREATE_MISSING_KEYS), then we either have "lost" blocks in the
1410
index file, or there are no non-unique indexes. In the latter case,
1411
mi_repair*() would not be called as there would be no disabled
1414
If there would be more unique indexes than disabled (non-unique)
1415
indexes, we could do the first method. But this is not implemented
1416
yet. By now we drop and recreate all indexes when repair is called.
1418
However, there is an exception. Sometimes MySQL disables non-unique
1419
indexes when the table is empty (e.g. when copying a table in
1420
mysql_alter_table()). When enabling the non-unique indexes, they
1421
are still empty. So there is no index block that can be lost. This
1422
optimization is implemented in this function.
1424
Note that in normal repair (T_CREATE_MISSING_KEYS not set) we
1425
recreate all enabled indexes unconditonally. We do not change the
1426
key_map. Otherwise we invert the key map temporarily (outside of
1427
this function) and recreate the then "seemingly" enabled indexes.
1428
When we cannot use the optimization, and drop all indexes, we
1429
pretend that all indexes were disabled. By the inversion, we will
1430
then recrate all indexes.
1433
static int mi_drop_all_indexes(MI_CHECK *param, MI_INFO *info, my_bool force)
1435
MYISAM_SHARE *share= info->s;
1436
MI_STATE_INFO *state= &share->state;
1439
DBUG_ENTER("mi_drop_all_indexes");
1442
If any of the disabled indexes has a key block assigned, we must
1443
drop and recreate all indexes to avoid losing index blocks.
1445
If we want to recreate disabled indexes only _and_ all of these
1446
indexes are empty, we don't need to recreate the existing indexes.
1448
if (!force && (param->testflag & T_CREATE_MISSING_KEYS))
1450
DBUG_PRINT("repair", ("creating missing indexes"));
1451
for (i= 0; i < share->base.keys; i++)
1453
DBUG_PRINT("repair", ("index #: %u key_root: 0x%lx active: %d",
1454
i, (long) state->key_root[i],
1455
mi_is_key_active(state->key_map, i)));
1456
if ((state->key_root[i] != HA_OFFSET_ERROR) &&
1457
!mi_is_key_active(state->key_map, i))
1460
This index has at least one key block and it is disabled.
1461
We would lose its block(s) if would just recreate it.
1462
So we need to drop and recreate all indexes.
1464
DBUG_PRINT("repair", ("nonempty and disabled: recreate all"));
1468
if (i >= share->base.keys)
1471
All of the disabled indexes are empty. We can just recreate them.
1472
Flush dirty blocks of this index file from key cache and remove
1473
all blocks of this index file from key cache.
1475
DBUG_PRINT("repair", ("all disabled are empty: create missing"));
1476
error= flush_key_blocks(share->key_cache, share->kfile,
1481
We do now drop all indexes and declare them disabled. With the
1482
T_CREATE_MISSING_KEYS flag, mi_repair*() will recreate all
1483
disabled indexes and enable them.
1485
mi_clear_all_keys_active(state->key_map);
1486
DBUG_PRINT("repair", ("declared all indexes disabled"));
1489
/* Remove all key blocks of this index file from key cache. */
1490
if ((error= flush_key_blocks(share->key_cache, share->kfile,
1491
FLUSH_IGNORE_CHANGED)))
1492
goto end; /* purecov: inspected */
1494
/* Clear index root block pointers. */
1495
for (i= 0; i < share->base.keys; i++)
1496
state->key_root[i]= HA_OFFSET_ERROR;
1498
/* Clear the delete chains. */
1499
for (i= 0; i < state->header.max_block_size_index; i++)
1500
state->key_del[i]= HA_OFFSET_ERROR;
1502
/* Reset index file length to end of index file header. */
1503
info->state->key_file_length= share->base.keystart;
1505
DBUG_PRINT("repair", ("dropped all indexes"));
1506
/* error= 0; set by last (error= flush_key_bocks()). */
1513
/* Recover old table by reading each record and writing all keys */
1514
/* Save new datafile-name in temp_filename */
1516
int mi_repair(MI_CHECK *param, register MI_INFO *info,
1517
char * name, int rep_quick)
1519
int error,got_error;
1520
ha_rows start_records,new_header_length;
1523
MYISAM_SHARE *share=info->s;
1524
char llbuff[22],llbuff2[22];
1525
SORT_INFO sort_info;
1526
MI_SORT_PARAM sort_param;
1527
DBUG_ENTER("mi_repair");
1529
bzero((char *)&sort_info, sizeof(sort_info));
1530
bzero((char *)&sort_param, sizeof(sort_param));
1531
start_records=info->state->records;
1532
new_header_length=(param->testflag & T_UNPACK) ? 0L :
1533
share->pack.header_length;
1536
sort_param.sort_info=&sort_info;
1538
if (!(param->testflag & T_SILENT))
1540
printf("- recovering (with keycache) MyISAM-table '%s'\n",name);
1541
printf("Data records: %s\n", llstr(info->state->records,llbuff));
1543
param->testflag|=T_REP; /* for easy checking */
1545
if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
1546
param->testflag|=T_CALC_CHECKSUM;
1548
if (!param->using_global_keycache)
1549
VOID(init_key_cache(dflt_key_cache, param->key_cache_block_size,
1550
param->use_buffers, 0, 0));
1552
if (init_io_cache(¶m->read_cache,info->dfile,
1553
(uint) param->read_buffer_length,
1554
READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
1556
bzero(&info->rec_cache,sizeof(info->rec_cache));
1560
if (init_io_cache(&info->rec_cache,-1,(uint) param->write_buffer_length,
1561
WRITE_CACHE, new_header_length, 1,
1562
MYF(MY_WME | MY_WAIT_IF_FULL)))
1564
info->opt_flag|=WRITE_CACHE_USED;
1565
if (!mi_alloc_rec_buff(info, -1, &sort_param.record) ||
1566
!mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
1568
mi_check_print_error(param, "Not enough memory for extra record");
1574
/* Get real path for data file */
1575
if ((new_file=my_raid_create(fn_format(param->temp_filename,
1576
share->data_file_name, "",
1578
0,param->tmpfile_createflag,
1579
share->base.raid_type,
1580
share->base.raid_chunks,
1581
share->base.raid_chunksize,
1584
mi_check_print_error(param,"Can't create new tempfile: '%s'",
1585
param->temp_filename);
1588
if (new_header_length &&
1589
filecopy(param,new_file,info->dfile,0L,new_header_length,
1592
info->s->state.dellink= HA_OFFSET_ERROR;
1593
info->rec_cache.file=new_file;
1594
if (param->testflag & T_UNPACK)
1596
share->options&= ~HA_OPTION_COMPRESS_RECORD;
1597
mi_int2store(share->state.header.options,share->options);
1600
sort_info.info=info;
1601
sort_info.param = param;
1602
sort_param.read_cache=param->read_cache;
1603
sort_param.pos=sort_param.max_pos=share->pack.header_length;
1604
sort_param.filepos=new_header_length;
1605
param->read_cache.end_of_file=sort_info.filelength=
1606
my_seek(info->dfile,0L,MY_SEEK_END,MYF(0));
1608
sort_param.fix_datafile= (my_bool) (! rep_quick);
1609
sort_param.master=1;
1610
sort_info.max_records= ~(ha_rows) 0;
1612
set_data_file_type(&sort_info, share);
1613
del=info->state->del;
1614
info->state->records=info->state->del=share->state.split=0;
1615
info->state->empty=0;
1617
if (param->testflag & T_CALC_CHECKSUM)
1618
sort_param.calc_checksum= 1;
1620
info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1622
/* This function always recreates all enabled indexes. */
1623
if (param->testflag & T_CREATE_MISSING_KEYS)
1624
mi_set_all_keys_active(share->state.key_map, share->base.keys);
1625
mi_drop_all_indexes(param, info, TRUE);
1627
lock_memory(param); /* Everything is alloced */
1629
/* Re-create all keys, which are set in key_map. */
1630
while (!(error=sort_get_next_record(&sort_param)))
1632
if (writekeys(&sort_param))
1634
if (my_errno != HA_ERR_FOUND_DUPP_KEY)
1636
DBUG_DUMP("record",(uchar*) sort_param.record,share->base.pack_reclength);
1637
mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s",
1639
llstr(sort_param.start_recpos,llbuff),
1640
llstr(info->dupp_key_pos,llbuff2));
1641
if (param->testflag & T_VERBOSE)
1643
VOID(_mi_make_key(info,(uint) info->errkey,info->lastkey,
1644
sort_param.record,0L));
1645
_mi_print_key(stdout,share->keyinfo[info->errkey].seg,info->lastkey,
1649
if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
1651
param->testflag|=T_RETRY_WITHOUT_QUICK;
1652
param->error_printed=1;
1657
if (sort_write_record(&sort_param))
1660
if (error > 0 || write_data_suffix(&sort_info, (my_bool)!rep_quick) ||
1661
flush_io_cache(&info->rec_cache) || param->read_cache.error < 0)
1664
if (param->testflag & T_WRITE_LOOP)
1666
VOID(fputs(" \r",stdout)); VOID(fflush(stdout));
1668
if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
1670
mi_check_print_warning(param,
1671
"Can't change size of indexfile, error: %d",
1676
if (rep_quick && del+sort_info.dupp != info->state->del)
1678
mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
1679
mi_check_print_error(param,"Run recovery again without -q");
1681
param->retry_repair=1;
1682
param->testflag|=T_RETRY_WITHOUT_QUICK;
1685
if (param->testflag & T_SAFE_REPAIR)
1687
/* Don't repair if we loosed more than one row */
1688
if (info->state->records+1 < start_records)
1690
info->state->records=start_records;
1698
my_close(info->dfile,MYF(0));
1699
info->dfile=new_file;
1700
info->state->data_file_length=sort_param.filepos;
1701
share->state.version=(ulong) time((time_t*) 0); /* Force reopen */
1705
info->state->data_file_length=sort_param.max_pos;
1707
if (param->testflag & T_CALC_CHECKSUM)
1708
info->state->checksum=param->glob_crc;
1710
if (!(param->testflag & T_SILENT))
1712
if (start_records != info->state->records)
1713
printf("Data records: %s\n", llstr(info->state->records,llbuff));
1715
mi_check_print_warning(param,
1716
"%s records have been removed",
1717
llstr(sort_info.dupp,llbuff));
1721
/* If invoked by external program that uses thr_lock */
1722
if (&share->state.state != info->state)
1723
memcpy( &share->state.state, info->state, sizeof(*info->state));
1728
/* Replace the actual file with the temporary file */
1731
my_close(new_file,MYF(0));
1732
info->dfile=new_file= -1;
1733
if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
1734
DATA_TMP_EXT, share->base.raid_chunks,
1735
(param->testflag & T_BACKUP_DATA ?
1736
MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
1737
mi_open_datafile(info,share,-1))
1743
if (! param->error_printed)
1744
mi_check_print_error(param,"%d for record at pos %s",my_errno,
1745
llstr(sort_param.start_recpos,llbuff));
1748
VOID(my_close(new_file,MYF(0)));
1749
VOID(my_raid_delete(param->temp_filename,info->s->base.raid_chunks,
1751
info->rec_cache.file=-1; /* don't flush data to new_file, it's closed */
1753
mi_mark_crashed_on_repair(info);
1755
my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff),
1756
MYF(MY_ALLOW_ZERO_PTR));
1757
my_free(mi_get_rec_buff_ptr(info, sort_param.record),
1758
MYF(MY_ALLOW_ZERO_PTR));
1759
my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
1760
VOID(end_io_cache(¶m->read_cache));
1761
info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
1762
VOID(end_io_cache(&info->rec_cache));
1763
got_error|=flush_blocks(param, share->key_cache, share->kfile);
1764
if (!got_error && param->testflag & T_UNPACK)
1766
share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
1767
share->pack.header_length=0;
1768
share->data_file_type=sort_info.new_data_file_type;
1770
share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES |
1771
STATE_NOT_ANALYZED);
1772
DBUG_RETURN(got_error);
1776
/* Uppate keyfile when doing repair */
1778
static int writekeys(MI_SORT_PARAM *sort_param)
1782
MI_INFO *info= sort_param->sort_info->info;
1783
uchar *buff= sort_param->record;
1784
my_off_t filepos= sort_param->filepos;
1785
DBUG_ENTER("writekeys");
1787
key=info->lastkey+info->s->base.max_key_length;
1788
for (i=0 ; i < info->s->base.keys ; i++)
1790
if (mi_is_key_active(info->s->state.key_map, i))
1792
if (info->s->keyinfo[i].flag & HA_FULLTEXT )
1794
if (_mi_ft_add(info, i, key, buff, filepos))
1798
else if (info->s->keyinfo[i].flag & HA_SPATIAL)
1800
uint key_length=_mi_make_key(info,i,key,buff,filepos);
1801
if (rtree_insert(info, i, key, key_length))
1804
#endif /*HAVE_SPATIAL*/
1807
uint key_length=_mi_make_key(info,i,key,buff,filepos);
1808
if (_mi_ck_write(info,i,key,key_length))
1816
if (my_errno == HA_ERR_FOUND_DUPP_KEY)
1818
info->errkey=(int) i; /* This key was found */
1821
if (mi_is_key_active(info->s->state.key_map, i))
1823
if (info->s->keyinfo[i].flag & HA_FULLTEXT)
1825
if (_mi_ft_del(info,i, key,buff,filepos))
1830
uint key_length=_mi_make_key(info,i,key,buff,filepos);
1831
if (_mi_ck_delete(info,i,key,key_length))
1837
/* Remove checksum that was added to glob_crc in sort_get_next_record */
1838
if (sort_param->calc_checksum)
1839
sort_param->sort_info->param->glob_crc-= info->checksum;
1840
DBUG_PRINT("error",("errno: %d",my_errno));
1845
/* Change all key-pointers that points to a records */
1847
int movepoint(register MI_INFO *info, uchar *record, my_off_t oldpos,
1848
my_off_t newpos, uint prot_key)
1853
DBUG_ENTER("movepoint");
1855
key=info->lastkey+info->s->base.max_key_length;
1856
for (i=0 ; i < info->s->base.keys; i++)
1858
if (i != prot_key && mi_is_key_active(info->s->state.key_map, i))
1860
key_length=_mi_make_key(info,i,key,record,oldpos);
1861
if (info->s->keyinfo[i].flag & HA_NOSAME)
1862
{ /* Change pointer direct */
1865
keyinfo=info->s->keyinfo+i;
1866
if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY,
1867
(uint) (SEARCH_SAME | SEARCH_SAVE_BUFF),
1868
info->s->state.key_root[i]))
1870
nod_flag=mi_test_if_nod(info->buff);
1871
_mi_dpointer(info,info->int_keypos-nod_flag-
1872
info->s->rec_reflength,newpos);
1873
if (_mi_write_keypage(info,keyinfo,info->last_keypage,
1874
DFLT_INIT_HITS,info->buff))
1878
{ /* Change old key to new */
1879
if (_mi_ck_delete(info,i,key,key_length))
1881
key_length=_mi_make_key(info,i,key,record,newpos);
1882
if (_mi_ck_write(info,i,key,key_length))
1891
/* Tell system that we want all memory for our cache */
1893
void lock_memory(MI_CHECK *param __attribute__((unused)))
1895
#ifdef SUN_OS /* Key-cacheing thrases on sun 4.1 */
1896
if (param->opt_lock_memory)
1898
int success = mlockall(MCL_CURRENT); /* or plock(DATLOCK); */
1899
if (geteuid() == 0 && success != 0)
1900
mi_check_print_warning(param,
1901
"Failed to lock memory. errno %d",my_errno);
1907
/* Flush all changed blocks to disk */
1909
int flush_blocks(MI_CHECK *param, KEY_CACHE *key_cache, File file)
1911
if (flush_key_blocks(key_cache, file, FLUSH_RELEASE))
1913
mi_check_print_error(param,"%d when trying to write bufferts",my_errno);
1916
if (!param->using_global_keycache)
1917
end_key_cache(key_cache,1);
1919
} /* flush_blocks */
1922
/* Sort index for more efficent reads */
1924
int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name)
1927
register MI_KEYDEF *keyinfo;
1929
my_off_t index_pos[HA_MAX_POSSIBLE_KEY];
1930
uint r_locks,w_locks;
1932
MYISAM_SHARE *share=info->s;
1933
MI_STATE_INFO old_state;
1934
DBUG_ENTER("mi_sort_index");
1936
/* cannot sort index files with R-tree indexes */
1937
for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1939
if (keyinfo->key_alg == HA_KEY_ALG_RTREE)
1942
if (!(param->testflag & T_SILENT))
1943
printf("- Sorting index for MyISAM-table '%s'\n",name);
1945
/* Get real path for index file */
1946
fn_format(param->temp_filename,name,"", MI_NAME_IEXT,2+4+32);
1947
if ((new_file=my_create(fn_format(param->temp_filename,param->temp_filename,
1948
"", INDEX_TMP_EXT,2+4),
1949
0,param->tmpfile_createflag,MYF(0))) <= 0)
1951
mi_check_print_error(param,"Can't create new tempfile: '%s'",
1952
param->temp_filename);
1955
if (filecopy(param, new_file,share->kfile,0L,
1956
(ulong) share->base.keystart, "headerblock"))
1959
param->new_file_pos=share->base.keystart;
1960
for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1963
if (! mi_is_key_active(info->s->state.key_map, key))
1966
if (share->state.key_root[key] != HA_OFFSET_ERROR)
1968
index_pos[key]=param->new_file_pos; /* Write first block here */
1969
if (sort_one_index(param,info,keyinfo,share->state.key_root[key],
1974
index_pos[key]= HA_OFFSET_ERROR; /* No blocks */
1977
/* Flush key cache for this file if we are calling this outside myisamchk */
1978
flush_key_blocks(share->key_cache,share->kfile, FLUSH_IGNORE_CHANGED);
1980
share->state.version=(ulong) time((time_t*) 0);
1981
old_state= share->state; /* save state if not stored */
1982
r_locks= share->r_locks;
1983
w_locks= share->w_locks;
1984
old_lock= info->lock_type;
1986
/* Put same locks as old file */
1987
share->r_locks= share->w_locks= share->tot_locks= 0;
1988
(void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
1989
VOID(my_close(share->kfile,MYF(MY_WME)));
1991
VOID(my_close(new_file,MYF(MY_WME)));
1992
if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT,0,
1994
mi_open_keyfile(share))
1996
info->lock_type= F_UNLCK; /* Force mi_readinfo to lock */
1997
_mi_readinfo(info,F_WRLCK,0); /* Will lock the table */
1998
info->lock_type= old_lock;
1999
share->r_locks= r_locks;
2000
share->w_locks= w_locks;
2001
share->tot_locks= r_locks+w_locks;
2002
share->state= old_state; /* Restore old state */
2004
info->state->key_file_length=param->new_file_pos;
2005
info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2006
for (key=0 ; key < info->s->base.keys ; key++)
2007
info->s->state.key_root[key]=index_pos[key];
2008
for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
2009
info->s->state.key_del[key]= HA_OFFSET_ERROR;
2011
info->s->state.changed&= ~STATE_NOT_SORTED_PAGES;
2015
VOID(my_close(new_file,MYF(MY_WME)));
2017
VOID(my_delete(param->temp_filename,MYF(MY_WME)));
2019
} /* mi_sort_index */
2022
/* Sort records recursive using one index */
2024
static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
2025
my_off_t pagepos, File new_file)
2027
uint length,nod_flag,used_length, key_length;
2028
uchar *buff,*keypos,*endpos;
2029
uchar key[HA_MAX_POSSIBLE_KEY_BUFF];
2030
my_off_t new_page_pos,next_page;
2032
DBUG_ENTER("sort_one_index");
2034
/* cannot walk over R-tree indices */
2035
DBUG_ASSERT(keyinfo->key_alg != HA_KEY_ALG_RTREE);
2036
new_page_pos=param->new_file_pos;
2037
param->new_file_pos+=keyinfo->block_length;
2039
if (!(buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
2041
mi_check_print_error(param,"Not enough memory for key block");
2044
if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0))
2046
mi_check_print_error(param,"Can't read key block from filepos: %s",
2047
llstr(pagepos,llbuff));
2050
if ((nod_flag=mi_test_if_nod(buff)) || keyinfo->flag & HA_FULLTEXT)
2052
used_length=mi_getint(buff);
2053
keypos=buff+2+nod_flag;
2054
endpos=buff+used_length;
2059
next_page=_mi_kpos(nod_flag,keypos);
2060
_mi_kpointer(info,keypos-nod_flag,param->new_file_pos); /* Save new pos */
2061
if (sort_one_index(param,info,keyinfo,next_page,new_file))
2064
("From page: %ld, keyoffset: %lu used_length: %d",
2065
(ulong) pagepos, (ulong) (keypos - buff),
2066
(int) used_length));
2067
DBUG_DUMP("buff",(uchar*) buff,used_length);
2071
if (keypos >= endpos ||
2072
(key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
2074
DBUG_ASSERT(keypos <= endpos);
2075
if (keyinfo->flag & HA_FULLTEXT)
2079
get_key_full_length_rdonly(off, key);
2080
subkeys=ft_sintXkorr(key+off);
2083
next_page= _mi_dpos(info,0,key+key_length);
2084
_mi_dpointer(info,keypos-nod_flag-info->s->rec_reflength,
2085
param->new_file_pos); /* Save new pos */
2086
if (sort_one_index(param,info,&info->s->ft2_keyinfo,
2087
next_page,new_file))
2094
/* Fill block with zero and write it to the new index file */
2095
length=mi_getint(buff);
2096
bzero((uchar*) buff+length,keyinfo->block_length-length);
2097
if (my_pwrite(new_file,(uchar*) buff,(uint) keyinfo->block_length,
2098
new_page_pos,MYF(MY_NABP | MY_WAIT_IF_FULL)))
2100
mi_check_print_error(param,"Can't write indexblock, error: %d",my_errno);
2103
my_afree((uchar*) buff);
2106
my_afree((uchar*) buff);
2108
} /* sort_one_index */
2112
Let temporary file replace old file.
2113
This assumes that the new file was created in the same
2114
directory as given by realpath(filename).
2115
This will ensure that any symlinks that are used will still work.
2116
Copy stats from old file to new file, deletes orignal and
2117
changes new file name to old file name
2120
int change_to_newfile(const char * filename, const char * old_ext,
2121
const char * new_ext,
2122
uint raid_chunks __attribute__((unused)),
2125
char old_filename[FN_REFLEN],new_filename[FN_REFLEN];
2128
return my_raid_redel(fn_format(old_filename,filename,"",old_ext,2+4),
2129
fn_format(new_filename,filename,"",new_ext,2+4),
2131
MYF(MY_WME | MY_LINK_WARNING | MyFlags));
2133
/* Get real path to filename */
2134
(void) fn_format(old_filename,filename,"",old_ext,2+4+32);
2135
return my_redel(old_filename,
2136
fn_format(new_filename,old_filename,"",new_ext,2+4),
2137
MYF(MY_WME | MY_LINK_WARNING | MyFlags));
2138
} /* change_to_newfile */
2141
/* Locks a whole file */
2142
/* Gives an error-message if file can't be locked */
2144
int lock_file(MI_CHECK *param, File file, my_off_t start, int lock_type,
2145
const char *filetype, const char *filename)
2147
if (my_lock(file,lock_type,start,F_TO_EOF,
2148
param->testflag & T_WAIT_FOREVER ? MYF(MY_SEEK_NOT_DONE) :
2149
MYF(MY_SEEK_NOT_DONE | MY_DONT_WAIT)))
2151
mi_check_print_error(param," %d when locking %s '%s'",my_errno,filetype,filename);
2152
param->error_printed=2; /* Don't give that data is crashed */
2159
/* Copy a block between two files */
2161
int filecopy(MI_CHECK *param, File to,File from,my_off_t start,
2162
my_off_t length, const char *type)
2164
char tmp_buff[IO_SIZE],*buff;
2166
DBUG_ENTER("filecopy");
2168
buff_length=(ulong) min(param->write_buffer_length,length);
2169
if (!(buff=my_malloc(buff_length,MYF(0))))
2171
buff=tmp_buff; buff_length=IO_SIZE;
2174
VOID(my_seek(from,start,MY_SEEK_SET,MYF(0)));
2175
while (length > buff_length)
2177
if (my_read(from,(uchar*) buff,buff_length,MYF(MY_NABP)) ||
2178
my_write(to,(uchar*) buff,buff_length,param->myf_rw))
2180
length-= buff_length;
2182
if (my_read(from,(uchar*) buff,(uint) length,MYF(MY_NABP)) ||
2183
my_write(to,(uchar*) buff,(uint) length,param->myf_rw))
2185
if (buff != tmp_buff)
2186
my_free(buff,MYF(0));
2189
if (buff != tmp_buff)
2190
my_free(buff,MYF(0));
2191
mi_check_print_error(param,"Can't copy %s to tempfile, error %d",
2198
Repair table or given index using sorting
2202
param Repair parameters
2203
info MyISAM handler to repair
2204
name Name of table (for warnings)
2205
rep_quick set to <> 0 if we should not change data file
2212
int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
2213
const char * name, int rep_quick)
2218
ha_rows start_records;
2219
my_off_t new_header_length,del;
2221
MI_SORT_PARAM sort_param;
2222
MYISAM_SHARE *share=info->s;
2224
ulong *rec_per_key_part;
2226
SORT_INFO sort_info;
2227
ulonglong key_map= 0;
2228
DBUG_ENTER("mi_repair_by_sort");
2230
start_records=info->state->records;
2233
new_header_length=(param->testflag & T_UNPACK) ? 0 :
2234
share->pack.header_length;
2235
if (!(param->testflag & T_SILENT))
2237
printf("- recovering (with sort) MyISAM-table '%s'\n",name);
2238
printf("Data records: %s\n", llstr(start_records,llbuff));
2240
param->testflag|=T_REP; /* for easy checking */
2242
if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
2243
param->testflag|=T_CALC_CHECKSUM;
2245
bzero((char*)&sort_info,sizeof(sort_info));
2246
bzero((char *)&sort_param, sizeof(sort_param));
2247
if (!(sort_info.key_block=
2248
alloc_key_blocks(param,
2249
(uint) param->sort_key_blocks,
2250
share->base.max_key_block_length))
2251
|| init_io_cache(¶m->read_cache,info->dfile,
2252
(uint) param->read_buffer_length,
2253
READ_CACHE,share->pack.header_length,1,MYF(MY_WME)) ||
2255
init_io_cache(&info->rec_cache,info->dfile,
2256
(uint) param->write_buffer_length,
2257
WRITE_CACHE,new_header_length,1,
2258
MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
2260
sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
2261
info->opt_flag|=WRITE_CACHE_USED;
2262
info->rec_cache.file=info->dfile; /* for sort_delete_record */
2264
if (!mi_alloc_rec_buff(info, -1, &sort_param.record) ||
2265
!mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
2267
mi_check_print_error(param, "Not enough memory for extra record");
2272
/* Get real path for data file */
2273
if ((new_file=my_raid_create(fn_format(param->temp_filename,
2274
share->data_file_name, "",
2276
0,param->tmpfile_createflag,
2277
share->base.raid_type,
2278
share->base.raid_chunks,
2279
share->base.raid_chunksize,
2282
mi_check_print_error(param,"Can't create new tempfile: '%s'",
2283
param->temp_filename);
2286
if (new_header_length &&
2287
filecopy(param, new_file,info->dfile,0L,new_header_length,
2290
if (param->testflag & T_UNPACK)
2292
share->options&= ~HA_OPTION_COMPRESS_RECORD;
2293
mi_int2store(share->state.header.options,share->options);
2295
share->state.dellink= HA_OFFSET_ERROR;
2296
info->rec_cache.file=new_file;
2299
info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2301
/* Optionally drop indexes and optionally modify the key_map. */
2302
mi_drop_all_indexes(param, info, FALSE);
2303
key_map= share->state.key_map;
2304
if (param->testflag & T_CREATE_MISSING_KEYS)
2306
/* Invert the copied key_map to recreate all disabled indexes. */
2310
sort_info.info=info;
2311
sort_info.param = param;
2313
set_data_file_type(&sort_info, share);
2314
sort_param.filepos=new_header_length;
2317
param->read_cache.end_of_file=sort_info.filelength=
2318
my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));
2320
sort_param.wordlist=NULL;
2321
init_alloc_root(&sort_param.wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
2323
if (share->data_file_type == DYNAMIC_RECORD)
2324
length=max(share->base.min_pack_length+1,share->base.min_block_length);
2325
else if (share->data_file_type == COMPRESSED_RECORD)
2326
length=share->base.min_block_length;
2328
length=share->base.pack_reclength;
2329
sort_info.max_records=
2330
((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
2331
(ha_rows) (sort_info.filelength/length+1));
2332
sort_param.key_cmp=sort_key_cmp;
2333
sort_param.lock_in_memory=lock_memory;
2334
sort_param.tmpdir=param->tmpdir;
2335
sort_param.sort_info=&sort_info;
2336
sort_param.fix_datafile= (my_bool) (! rep_quick);
2337
sort_param.master =1;
2339
del=info->state->del;
2341
if (param->testflag & T_CALC_CHECKSUM)
2342
sort_param.calc_checksum= 1;
2344
rec_per_key_part= param->rec_per_key_part;
2345
for (sort_param.key=0 ; sort_param.key < share->base.keys ;
2346
rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++)
2348
sort_param.read_cache=param->read_cache;
2349
sort_param.keyinfo=share->keyinfo+sort_param.key;
2350
sort_param.seg=sort_param.keyinfo->seg;
2352
Skip this index if it is marked disabled in the copied
2353
(and possibly inverted) key_map.
2355
if (! mi_is_key_active(key_map, sort_param.key))
2357
/* Remember old statistics for key */
2358
memcpy((char*) rec_per_key_part,
2359
(char*) (share->state.rec_per_key_part +
2360
(uint) (rec_per_key_part - param->rec_per_key_part)),
2361
sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part));
2362
DBUG_PRINT("repair", ("skipping seemingly disabled index #: %u",
2367
if ((!(param->testflag & T_SILENT)))
2368
printf ("- Fixing index %d\n",sort_param.key+1);
2369
sort_param.max_pos=sort_param.pos=share->pack.header_length;
2370
keyseg=sort_param.seg;
2371
bzero((char*) sort_param.unique,sizeof(sort_param.unique));
2372
sort_param.key_length=share->rec_reflength;
2373
for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++)
2375
sort_param.key_length+=keyseg[i].length;
2376
if (keyseg[i].flag & HA_SPACE_PACK)
2377
sort_param.key_length+=get_pack_length(keyseg[i].length);
2378
if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
2379
sort_param.key_length+=2 + test(keyseg[i].length >= 127);
2380
if (keyseg[i].flag & HA_NULL_PART)
2381
sort_param.key_length++;
2383
info->state->records=info->state->del=share->state.split=0;
2384
info->state->empty=0;
2386
if (sort_param.keyinfo->flag & HA_FULLTEXT)
2388
uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
2389
sort_param.keyinfo->seg->charset->mbmaxlen;
2390
sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
2392
fulltext indexes may have much more entries than the
2393
number of rows in the table. We estimate the number here.
2395
Note, built-in parser is always nr. 0 - see ftparser_call_initializer()
2397
if (sort_param.keyinfo->ftparser_nr == 0)
2400
for built-in parser the number of generated index entries
2401
cannot be larger than the size of the data file divided
2402
by the minimal word's length
2404
sort_info.max_records=
2405
(ha_rows) (sort_info.filelength/ft_min_word_len+1);
2410
for external plugin parser we cannot tell anything at all :(
2411
so, we'll use all the sort memory and start from ~10 buffpeks.
2412
(see _create_index_by_sort)
2414
sort_info.max_records=
2415
10*param->sort_buffer_length/sort_param.key_length;
2418
sort_param.key_read=sort_ft_key_read;
2419
sort_param.key_write=sort_ft_key_write;
2423
sort_param.key_read=sort_key_read;
2424
sort_param.key_write=sort_key_write;
2427
if (_create_index_by_sort(&sort_param,
2428
(my_bool) (!(param->testflag & T_VERBOSE)),
2429
(uint) param->sort_buffer_length))
2431
param->retry_repair=1;
2434
/* No need to calculate checksum again. */
2435
sort_param.calc_checksum= 0;
2436
free_root(&sort_param.wordroot, MYF(0));
2438
/* Set for next loop */
2439
sort_info.max_records= (ha_rows) info->state->records;
2441
if (param->testflag & T_STATISTICS)
2442
update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
2443
param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
2444
sort_param.notnull: NULL,
2445
(ulonglong) info->state->records);
2446
/* Enable this index in the permanent (not the copied) key_map. */
2447
mi_set_key_active(share->state.key_map, sort_param.key);
2448
DBUG_PRINT("repair", ("set enabled index #: %u", sort_param.key));
2450
if (sort_param.fix_datafile)
2452
param->read_cache.end_of_file=sort_param.filepos;
2453
if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
2455
if (param->testflag & T_SAFE_REPAIR)
2457
/* Don't repair if we loosed more than one row */
2458
if (info->state->records+1 < start_records)
2460
info->state->records=start_records;
2464
share->state.state.data_file_length = info->state->data_file_length=
2466
/* Only whole records */
2467
share->state.version=(ulong) time((time_t*) 0);
2468
my_close(info->dfile,MYF(0));
2469
info->dfile=new_file;
2470
share->data_file_type=sort_info.new_data_file_type;
2471
share->pack.header_length=(ulong) new_header_length;
2472
sort_param.fix_datafile=0;
2475
info->state->data_file_length=sort_param.max_pos;
2477
param->read_cache.file=info->dfile; /* re-init read cache */
2478
reinit_io_cache(¶m->read_cache,READ_CACHE,share->pack.header_length,
2482
if (param->testflag & T_WRITE_LOOP)
2484
VOID(fputs(" \r",stdout)); VOID(fflush(stdout));
2487
if (rep_quick && del+sort_info.dupp != info->state->del)
2489
mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
2490
mi_check_print_error(param,"Run recovery again without -q");
2492
param->retry_repair=1;
2493
param->testflag|=T_RETRY_WITHOUT_QUICK;
2497
if (rep_quick & T_FORCE_UNIQUENESS)
2499
my_off_t skr=info->state->data_file_length+
2500
(share->options & HA_OPTION_COMPRESS_RECORD ?
2501
MEMMAP_EXTRA_MARGIN : 0);
2503
if (share->data_file_type == STATIC_RECORD &&
2504
skr < share->base.reloc*share->base.min_pack_length)
2505
skr=share->base.reloc*share->base.min_pack_length;
2507
if (skr != sort_info.filelength && !info->s->base.raid_type)
2508
if (my_chsize(info->dfile,skr,0,MYF(0)))
2509
mi_check_print_warning(param,
2510
"Can't change size of datafile, error: %d",
2513
if (param->testflag & T_CALC_CHECKSUM)
2514
info->state->checksum=param->glob_crc;
2516
if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
2517
mi_check_print_warning(param,
2518
"Can't change size of indexfile, error: %d",
2521
if (!(param->testflag & T_SILENT))
2523
if (start_records != info->state->records)
2524
printf("Data records: %s\n", llstr(info->state->records,llbuff));
2526
mi_check_print_warning(param,
2527
"%s records have been removed",
2528
llstr(sort_info.dupp,llbuff));
2532
if (&share->state.state != info->state)
2533
memcpy( &share->state.state, info->state, sizeof(*info->state));
2536
got_error|= flush_blocks(param, share->key_cache, share->kfile);
2537
VOID(end_io_cache(&info->rec_cache));
2540
/* Replace the actual file with the temporary file */
2543
my_close(new_file,MYF(0));
2544
info->dfile=new_file= -1;
2545
if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
2546
DATA_TMP_EXT, share->base.raid_chunks,
2547
(param->testflag & T_BACKUP_DATA ?
2548
MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
2549
mi_open_datafile(info,share,-1))
2555
if (! param->error_printed)
2556
mi_check_print_error(param,"%d when fixing table",my_errno);
2559
VOID(my_close(new_file,MYF(0)));
2560
VOID(my_raid_delete(param->temp_filename,share->base.raid_chunks,
2562
if (info->dfile == new_file)
2565
mi_mark_crashed_on_repair(info);
2567
else if (key_map == share->state.key_map)
2568
share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
2569
share->state.changed|=STATE_NOT_SORTED_PAGES;
2571
my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff),
2572
MYF(MY_ALLOW_ZERO_PTR));
2573
my_free(mi_get_rec_buff_ptr(info, sort_param.record),
2574
MYF(MY_ALLOW_ZERO_PTR));
2575
my_free((uchar*) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR));
2576
my_free((uchar*) sort_info.ft_buf, MYF(MY_ALLOW_ZERO_PTR));
2577
my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
2578
VOID(end_io_cache(¶m->read_cache));
2579
info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
2580
if (!got_error && (param->testflag & T_UNPACK))
2582
share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
2583
share->pack.header_length=0;
2585
DBUG_RETURN(got_error);
2589
Threaded repair of table using sorting
2592
mi_repair_parallel()
2593
param Repair parameters
2594
info MyISAM handler to repair
2595
name Name of table (for warnings)
2596
rep_quick set to <> 0 if we should not change data file
2599
Same as mi_repair_by_sort but do it multithreaded
2600
Each key is handled by a separate thread.
2601
TODO: make a number of threads a parameter
2603
In parallel repair we use one thread per index. There are two modes:
2607
Only the indexes are rebuilt. All threads share a read buffer.
2608
Every thread that needs fresh data in the buffer enters the shared
2609
cache lock. The last thread joining the lock reads the buffer from
2610
the data file and wakes all other threads.
2614
The data file is rebuilt and all indexes are rebuilt to point to
2615
the new record positions. One thread is the master thread. It
2616
reads from the old data file and writes to the new data file. It
2617
also creates one of the indexes. The other threads read from a
2618
buffer which is filled by the master. If they need fresh data,
2619
they enter the shared cache lock. If the masters write buffer is
2620
full, it flushes it to the new data file and enters the shared
2621
cache lock too. When all threads joined in the lock, the master
2622
copies its write buffer to the read buffer for the other threads
2630
int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info,
2631
const char * name, int rep_quick)
2634
return mi_repair_by_sort(param, info, name, rep_quick);
2637
uint i,key, total_key_length, istep;
2639
ha_rows start_records;
2640
my_off_t new_header_length,del;
2642
MI_SORT_PARAM *sort_param=0;
2643
MYISAM_SHARE *share=info->s;
2644
ulong *rec_per_key_part;
2647
IO_CACHE new_data_cache; /* For non-quick repair. */
2648
IO_CACHE_SHARE io_share;
2649
SORT_INFO sort_info;
2650
ulonglong key_map= 0;
2651
pthread_attr_t thr_attr;
2652
ulong max_pack_reclength;
2653
DBUG_ENTER("mi_repair_parallel");
2655
start_records=info->state->records;
2658
new_header_length=(param->testflag & T_UNPACK) ? 0 :
2659
share->pack.header_length;
2660
if (!(param->testflag & T_SILENT))
2662
printf("- parallel recovering (with sort) MyISAM-table '%s'\n",name);
2663
printf("Data records: %s\n", llstr(start_records,llbuff));
2665
param->testflag|=T_REP; /* for easy checking */
2667
if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
2668
param->testflag|=T_CALC_CHECKSUM;
2671
Quick repair (not touching data file, rebuilding indexes):
2673
Read cache is (MI_CHECK *param)->read_cache using info->dfile.
2676
Non-quick repair (rebuilding data file and indexes):
2680
Read cache is (MI_CHECK *param)->read_cache using info->dfile.
2681
Write cache is (MI_INFO *info)->rec_cache using new_file.
2685
Read cache is new_data_cache synced to master rec_cache.
2687
The final assignment of the filedescriptor for rec_cache is done
2688
after the cache creation.
2690
Don't check file size on new_data_cache, as the resulting file size
2693
As rec_cache and new_data_cache are synced, write_buffer_length is
2694
used for the read cache 'new_data_cache'. Both start at the same
2695
position 'new_header_length'.
2698
DBUG_PRINT("info", ("is quick repair: %d", rep_quick));
2699
bzero((char*)&sort_info,sizeof(sort_info));
2700
/* Initialize pthread structures before goto err. */
2701
pthread_mutex_init(&sort_info.mutex, MY_MUTEX_INIT_FAST);
2702
pthread_cond_init(&sort_info.cond, 0);
2704
if (!(sort_info.key_block=
2705
alloc_key_blocks(param, (uint) param->sort_key_blocks,
2706
share->base.max_key_block_length)) ||
2707
init_io_cache(¶m->read_cache, info->dfile,
2708
(uint) param->read_buffer_length,
2709
READ_CACHE, share->pack.header_length, 1, MYF(MY_WME)) ||
2711
(init_io_cache(&info->rec_cache, info->dfile,
2712
(uint) param->write_buffer_length,
2713
WRITE_CACHE, new_header_length, 1,
2714
MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw) ||
2715
init_io_cache(&new_data_cache, -1,
2716
(uint) param->write_buffer_length,
2717
READ_CACHE, new_header_length, 1,
2718
MYF(MY_WME | MY_DONT_CHECK_FILESIZE)))))
2720
sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
2721
info->opt_flag|=WRITE_CACHE_USED;
2722
info->rec_cache.file=info->dfile; /* for sort_delete_record */
2726
/* Get real path for data file */
2727
if ((new_file=my_raid_create(fn_format(param->temp_filename,
2728
share->data_file_name, "",
2731
0,param->tmpfile_createflag,
2732
share->base.raid_type,
2733
share->base.raid_chunks,
2734
share->base.raid_chunksize,
2737
mi_check_print_error(param,"Can't create new tempfile: '%s'",
2738
param->temp_filename);
2741
if (new_header_length &&
2742
filecopy(param, new_file,info->dfile,0L,new_header_length,
2745
if (param->testflag & T_UNPACK)
2747
share->options&= ~HA_OPTION_COMPRESS_RECORD;
2748
mi_int2store(share->state.header.options,share->options);
2750
share->state.dellink= HA_OFFSET_ERROR;
2751
info->rec_cache.file=new_file;
2754
info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2756
/* Optionally drop indexes and optionally modify the key_map. */
2757
mi_drop_all_indexes(param, info, FALSE);
2758
key_map= share->state.key_map;
2759
if (param->testflag & T_CREATE_MISSING_KEYS)
2761
/* Invert the copied key_map to recreate all disabled indexes. */
2765
sort_info.info=info;
2766
sort_info.param = param;
2768
set_data_file_type(&sort_info, share);
2771
param->read_cache.end_of_file=sort_info.filelength=
2772
my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));
2774
if (share->data_file_type == DYNAMIC_RECORD)
2775
rec_length=max(share->base.min_pack_length+1,share->base.min_block_length);
2776
else if (share->data_file_type == COMPRESSED_RECORD)
2777
rec_length=share->base.min_block_length;
2779
rec_length=share->base.pack_reclength;
2781
+1 below is required hack for parallel repair mode.
2782
The info->state->records value, that is compared later
2783
to sort_info.max_records and cannot exceed it, is
2784
increased in sort_key_write. In mi_repair_by_sort, sort_key_write
2785
is called after sort_key_read, where the comparison is performed,
2786
but in parallel mode master thread can call sort_key_write
2787
before some other repair thread calls sort_key_read.
2788
Furthermore I'm not even sure +1 would be enough.
2789
May be sort_info.max_records shold be always set to max value in
2792
sort_info.max_records=
2793
((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records + 1:
2794
(ha_rows) (sort_info.filelength/rec_length+1));
2796
del=info->state->del;
2798
/* for compressed tables */
2799
max_pack_reclength= share->base.pack_reclength;
2800
if (share->options & HA_OPTION_COMPRESS_RECORD)
2801
set_if_bigger(max_pack_reclength, share->max_pack_length);
2802
if (!(sort_param=(MI_SORT_PARAM *)
2803
my_malloc((uint) share->base.keys *
2804
(sizeof(MI_SORT_PARAM) + max_pack_reclength),
2807
mi_check_print_error(param,"Not enough memory for key!");
2811
rec_per_key_part= param->rec_per_key_part;
2812
info->state->records=info->state->del=share->state.split=0;
2813
info->state->empty=0;
2815
for (i=key=0, istep=1 ; key < share->base.keys ;
2816
rec_per_key_part+=sort_param[i].keyinfo->keysegs, i+=istep, key++)
2818
sort_param[i].key=key;
2819
sort_param[i].keyinfo=share->keyinfo+key;
2820
sort_param[i].seg=sort_param[i].keyinfo->seg;
2822
Skip this index if it is marked disabled in the copied
2823
(and possibly inverted) key_map.
2825
if (! mi_is_key_active(key_map, key))
2827
/* Remember old statistics for key */
2828
memcpy((char*) rec_per_key_part,
2829
(char*) (share->state.rec_per_key_part+
2830
(uint) (rec_per_key_part - param->rec_per_key_part)),
2831
sort_param[i].keyinfo->keysegs*sizeof(*rec_per_key_part));
2836
if ((!(param->testflag & T_SILENT)))
2837
printf ("- Fixing index %d\n",key+1);
2838
if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
2840
sort_param[i].key_read=sort_ft_key_read;
2841
sort_param[i].key_write=sort_ft_key_write;
2845
sort_param[i].key_read=sort_key_read;
2846
sort_param[i].key_write=sort_key_write;
2848
sort_param[i].key_cmp=sort_key_cmp;
2849
sort_param[i].lock_in_memory=lock_memory;
2850
sort_param[i].tmpdir=param->tmpdir;
2851
sort_param[i].sort_info=&sort_info;
2852
sort_param[i].master=0;
2853
sort_param[i].fix_datafile=0;
2854
sort_param[i].calc_checksum= 0;
2856
sort_param[i].filepos=new_header_length;
2857
sort_param[i].max_pos=sort_param[i].pos=share->pack.header_length;
2859
sort_param[i].record= (((uchar *)(sort_param+share->base.keys))+
2860
(max_pack_reclength * i));
2861
if (!mi_alloc_rec_buff(info, -1, &sort_param[i].rec_buff))
2863
mi_check_print_error(param,"Not enough memory!");
2867
sort_param[i].key_length=share->rec_reflength;
2868
for (keyseg=sort_param[i].seg; keyseg->type != HA_KEYTYPE_END;
2871
sort_param[i].key_length+=keyseg->length;
2872
if (keyseg->flag & HA_SPACE_PACK)
2873
sort_param[i].key_length+=get_pack_length(keyseg->length);
2874
if (keyseg->flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
2875
sort_param[i].key_length+=2 + test(keyseg->length >= 127);
2876
if (keyseg->flag & HA_NULL_PART)
2877
sort_param[i].key_length++;
2879
total_key_length+=sort_param[i].key_length;
2881
if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
2883
uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
2884
sort_param[i].keyinfo->seg->charset->mbmaxlen;
2885
sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
2886
init_alloc_root(&sort_param[i].wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
2889
sort_info.total_keys=i;
2890
sort_param[0].master= 1;
2891
sort_param[0].fix_datafile= (my_bool)(! rep_quick);
2892
sort_param[0].calc_checksum= test(param->testflag & T_CALC_CHECKSUM);
2894
sort_info.got_error=0;
2895
pthread_mutex_lock(&sort_info.mutex);
2898
Initialize the I/O cache share for use with the read caches and, in
2899
case of non-quick repair, the write cache. When all threads join on
2900
the cache lock, the writer copies the write cache contents to the
2906
init_io_cache_share(¶m->read_cache, &io_share, NULL, i);
2908
init_io_cache_share(&new_data_cache, &io_share, &info->rec_cache, i);
2911
io_share.total_threads= 0; /* share not used */
2913
(void) pthread_attr_init(&thr_attr);
2914
(void) pthread_attr_setdetachstate(&thr_attr,PTHREAD_CREATE_DETACHED);
2916
for (i=0 ; i < sort_info.total_keys ; i++)
2919
Copy the properly initialized IO_CACHE structure so that every
2920
thread has its own copy. In quick mode param->read_cache is shared
2921
for use by all threads. In non-quick mode all threads but the
2922
first copy the shared new_data_cache, which is synchronized to the
2923
write cache of the first thread. The first thread copies
2924
param->read_cache, which is not shared.
2926
sort_param[i].read_cache= ((rep_quick || !i) ? param->read_cache :
2928
DBUG_PRINT("io_cache_share", ("thread: %u read_cache: 0x%lx",
2929
i, (long) &sort_param[i].read_cache));
2932
two approaches: the same amount of memory for each thread
2933
or the memory for the same number of keys for each thread...
2934
In the second one all the threads will fill their sort_buffers
2935
(and call write_keys) at the same time, putting more stress on i/o.
2937
sort_param[i].sortbuff_size=
2938
#ifndef USING_SECOND_APPROACH
2939
param->sort_buffer_length/sort_info.total_keys;
2941
param->sort_buffer_length*sort_param[i].key_length/total_key_length;
2943
if (pthread_create(&sort_param[i].thr, &thr_attr,
2945
(void *) (sort_param+i)))
2947
mi_check_print_error(param,"Cannot start a repair thread");
2948
/* Cleanup: Detach from the share. Avoid others to be blocked. */
2949
if (io_share.total_threads)
2950
remove_io_thread(&sort_param[i].read_cache);
2951
DBUG_PRINT("error", ("Cannot start a repair thread"));
2952
sort_info.got_error=1;
2955
sort_info.threads_running++;
2957
(void) pthread_attr_destroy(&thr_attr);
2959
/* waiting for all threads to finish */
2960
while (sort_info.threads_running)
2961
pthread_cond_wait(&sort_info.cond, &sort_info.mutex);
2962
pthread_mutex_unlock(&sort_info.mutex);
2964
if ((got_error= thr_write_keys(sort_param)))
2966
param->retry_repair=1;
2969
got_error=1; /* Assume the following may go wrong */
2971
if (sort_param[0].fix_datafile)
2974
Append some nuls to the end of a memory mapped file. Destroy the
2975
write cache. The master thread did already detach from the share
2976
by remove_io_thread() in sort.c:thr_find_all_keys().
2978
if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
2980
if (param->testflag & T_SAFE_REPAIR)
2982
/* Don't repair if we loosed more than one row */
2983
if (info->state->records+1 < start_records)
2985
info->state->records=start_records;
2989
share->state.state.data_file_length= info->state->data_file_length=
2990
sort_param->filepos;
2991
/* Only whole records */
2992
share->state.version=(ulong) time((time_t*) 0);
2995
Exchange the data file descriptor of the table, so that we use the
2996
new file from now on.
2998
my_close(info->dfile,MYF(0));
2999
info->dfile=new_file;
3001
share->data_file_type=sort_info.new_data_file_type;
3002
share->pack.header_length=(ulong) new_header_length;
3005
info->state->data_file_length=sort_param->max_pos;
3007
if (rep_quick && del+sort_info.dupp != info->state->del)
3009
mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
3010
mi_check_print_error(param,"Run recovery again without -q");
3011
param->retry_repair=1;
3012
param->testflag|=T_RETRY_WITHOUT_QUICK;
3016
if (rep_quick & T_FORCE_UNIQUENESS)
3018
my_off_t skr=info->state->data_file_length+
3019
(share->options & HA_OPTION_COMPRESS_RECORD ?
3020
MEMMAP_EXTRA_MARGIN : 0);
3022
if (share->data_file_type == STATIC_RECORD &&
3023
skr < share->base.reloc*share->base.min_pack_length)
3024
skr=share->base.reloc*share->base.min_pack_length;
3026
if (skr != sort_info.filelength && !info->s->base.raid_type)
3027
if (my_chsize(info->dfile,skr,0,MYF(0)))
3028
mi_check_print_warning(param,
3029
"Can't change size of datafile, error: %d",
3032
if (param->testflag & T_CALC_CHECKSUM)
3033
info->state->checksum=param->glob_crc;
3035
if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
3036
mi_check_print_warning(param,
3037
"Can't change size of indexfile, error: %d", my_errno);
3039
if (!(param->testflag & T_SILENT))
3041
if (start_records != info->state->records)
3042
printf("Data records: %s\n", llstr(info->state->records,llbuff));
3044
mi_check_print_warning(param,
3045
"%s records have been removed",
3046
llstr(sort_info.dupp,llbuff));
3050
if (&share->state.state != info->state)
3051
memcpy(&share->state.state, info->state, sizeof(*info->state));
3054
got_error|= flush_blocks(param, share->key_cache, share->kfile);
3056
Destroy the write cache. The master thread did already detach from
3057
the share by remove_io_thread() or it was not yet started (if the
3058
error happend before creating the thread).
3060
VOID(end_io_cache(&info->rec_cache));
3062
Destroy the new data cache in case of non-quick repair. All slave
3063
threads did either detach from the share by remove_io_thread()
3064
already or they were not yet started (if the error happend before
3065
creating the threads).
3068
VOID(end_io_cache(&new_data_cache));
3071
/* Replace the actual file with the temporary file */
3074
my_close(new_file,MYF(0));
3075
info->dfile=new_file= -1;
3076
if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
3077
DATA_TMP_EXT, share->base.raid_chunks,
3078
(param->testflag & T_BACKUP_DATA ?
3079
MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
3080
mi_open_datafile(info,share,-1))
3086
if (! param->error_printed)
3087
mi_check_print_error(param,"%d when fixing table",my_errno);
3090
VOID(my_close(new_file,MYF(0)));
3091
VOID(my_raid_delete(param->temp_filename,share->base.raid_chunks,
3093
if (info->dfile == new_file)
3096
mi_mark_crashed_on_repair(info);
3098
else if (key_map == share->state.key_map)
3099
share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
3100
share->state.changed|=STATE_NOT_SORTED_PAGES;
3102
pthread_cond_destroy (&sort_info.cond);
3103
pthread_mutex_destroy(&sort_info.mutex);
3105
my_free((uchar*) sort_info.ft_buf, MYF(MY_ALLOW_ZERO_PTR));
3106
my_free((uchar*) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR));
3107
my_free((uchar*) sort_param,MYF(MY_ALLOW_ZERO_PTR));
3108
my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
3109
VOID(end_io_cache(¶m->read_cache));
3110
info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
3111
if (!got_error && (param->testflag & T_UNPACK))
3113
share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
3114
share->pack.header_length=0;
3116
DBUG_RETURN(got_error);
3120
/* Read next record and return next key */
3122
static int sort_key_read(MI_SORT_PARAM *sort_param, void *key)
3125
SORT_INFO *sort_info=sort_param->sort_info;
3126
MI_INFO *info=sort_info->info;
3127
DBUG_ENTER("sort_key_read");
3129
if ((error=sort_get_next_record(sort_param)))
3131
if (info->state->records == sort_info->max_records)
3133
mi_check_print_error(sort_info->param,
3134
"Key %d - Found too many records; Can't continue",
3138
sort_param->real_key_length=
3139
(info->s->rec_reflength+
3140
_mi_make_key(info, sort_param->key, (uchar*) key,
3141
sort_param->record, sort_param->filepos));
3143
bzero(key+sort_param->real_key_length,
3144
(sort_param->key_length-sort_param->real_key_length));
3146
DBUG_RETURN(sort_write_record(sort_param));
3147
} /* sort_key_read */
3149
static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
3152
SORT_INFO *sort_info=sort_param->sort_info;
3153
MI_INFO *info=sort_info->info;
3155
DBUG_ENTER("sort_ft_key_read");
3157
if (!sort_param->wordlist)
3161
free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
3162
if ((error=sort_get_next_record(sort_param)))
3164
if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record,
3165
&sort_param->wordroot)))
3169
error=sort_write_record(sort_param);
3171
sort_param->wordptr=sort_param->wordlist=wptr;
3176
wptr=(FT_WORD*)(sort_param->wordptr);
3179
sort_param->real_key_length=(info->s->rec_reflength+
3180
_ft_make_key(info, sort_param->key,
3181
key, wptr++, sort_param->filepos));
3183
if (sort_param->key_length > sort_param->real_key_length)
3184
bzero(key+sort_param->real_key_length,
3185
(sort_param->key_length-sort_param->real_key_length));
3189
free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
3190
sort_param->wordlist=0;
3191
error=sort_write_record(sort_param);
3194
sort_param->wordptr=(void*)wptr;
3197
} /* sort_ft_key_read */
3201
Read next record from file using parameters in sort_info.
3204
sort_get_next_record()
3205
sort_param Information about and for the sort process
3209
Dynamic Records With Non-Quick Parallel Repair
3211
For non-quick parallel repair we use a synchronized read/write
3212
cache. This means that one thread is the master who fixes the data
3213
file by reading each record from the old data file and writing it
3214
to the new data file. By doing this the records in the new data
3215
file are written contiguously. Whenever the write buffer is full,
3216
it is copied to the read buffer. The slaves read from the read
3217
buffer, which is not associated with a file. Thus read_cache.file
3218
is -1. When using _mi_read_cache(), the slaves must always set
3219
flag to READING_NEXT so that the function never tries to read from
3220
file. This is safe because the records are contiguous. There is no
3221
need to read outside the cache. This condition is evaluated in the
3222
variable 'parallel_flag' for quick reference. read_cache.file must
3223
be >= 0 in every other case.
3231
static int sort_get_next_record(MI_SORT_PARAM *sort_param)
3235
uint found_record,b_type,left_length;
3238
MI_BLOCK_INFO block_info;
3239
SORT_INFO *sort_info=sort_param->sort_info;
3240
MI_CHECK *param=sort_info->param;
3241
MI_INFO *info=sort_info->info;
3242
MYISAM_SHARE *share=info->s;
3243
char llbuff[22],llbuff2[22];
3244
DBUG_ENTER("sort_get_next_record");
3246
if (*killed_ptr(param))
3249
switch (share->data_file_type) {
3253
if (my_b_read(&sort_param->read_cache,sort_param->record,
3254
share->base.pack_reclength))
3256
if (sort_param->read_cache.error)
3257
param->out_flag |= O_DATA_LOST;
3258
param->retry_repair=1;
3259
param->testflag|=T_RETRY_WITHOUT_QUICK;
3262
sort_param->start_recpos=sort_param->pos;
3263
if (!sort_param->fix_datafile)
3265
sort_param->filepos=sort_param->pos;
3266
if (sort_param->master)
3267
share->state.split++;
3269
sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength);
3270
if (*sort_param->record)
3272
if (sort_param->calc_checksum)
3273
param->glob_crc+= (info->checksum=
3274
mi_static_checksum(info,sort_param->record));
3277
if (!sort_param->fix_datafile && sort_param->master)
3280
info->state->empty+=share->base.pack_reclength;
3283
case DYNAMIC_RECORD:
3284
pos= sort_param->pos;
3285
searching= (sort_param->fix_datafile && (param->testflag & T_EXTEND));
3286
parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0;
3289
found_record=block_info.second_read= 0;
3293
pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE);
3294
param->testflag|=T_RETRY_WITHOUT_QUICK;
3295
sort_param->start_recpos=pos;
3299
if (pos > sort_param->max_pos)
3300
sort_param->max_pos=pos;
3301
if (pos & (MI_DYN_ALIGN_SIZE-1))
3303
if ((param->testflag & T_VERBOSE) || searching == 0)
3304
mi_check_print_info(param,"Wrong aligned block at %s",
3309
if (found_record && pos == param->search_after_block)
3310
mi_check_print_info(param,"Block: %s used by record at %s",
3311
llstr(param->search_after_block,llbuff),
3312
llstr(sort_param->start_recpos,llbuff2));
3313
if (_mi_read_cache(&sort_param->read_cache,
3314
(uchar*) block_info.header,pos,
3315
MI_BLOCK_INFO_HEADER_LENGTH,
3316
(! found_record ? READING_NEXT : 0) |
3317
parallel_flag | READING_HEADER))
3321
mi_check_print_info(param,
3322
"Can't read whole record at %s (errno: %d)",
3323
llstr(sort_param->start_recpos,llbuff),errno);
3328
if (searching && ! sort_param->fix_datafile)
3330
param->error_printed=1;
3331
param->retry_repair=1;
3332
param->testflag|=T_RETRY_WITHOUT_QUICK;
3333
DBUG_RETURN(1); /* Something wrong with data */
3335
b_type=_mi_get_block_info(&block_info,-1,pos);
3336
if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
3337
((b_type & BLOCK_FIRST) &&
3338
(block_info.rec_len < (uint) share->base.min_pack_length ||
3339
block_info.rec_len > (uint) share->base.max_pack_length)))
3342
if (param->testflag & T_VERBOSE || searching == 0)
3343
mi_check_print_info(param,
3344
"Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
3345
block_info.header[0],block_info.header[1],
3346
block_info.header[2],llstr(pos,llbuff));
3349
block_info.second_read=0;
3351
/* Search after block in read header string */
3352
for (i=MI_DYN_ALIGN_SIZE ;
3353
i < MI_BLOCK_INFO_HEADER_LENGTH ;
3354
i+= MI_DYN_ALIGN_SIZE)
3355
if (block_info.header[i] >= 1 &&
3356
block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
3359
sort_param->start_recpos=pos;
3362
if (b_type & BLOCK_DELETED)
3365
if (block_info.block_len+ (uint) (block_info.filepos-pos) <
3366
share->base.min_block_length)
3369
mi_check_print_info(param,
3370
"Deleted block with impossible length %u at %s",
3371
block_info.block_len,llstr(pos,llbuff));
3376
if ((block_info.next_filepos != HA_OFFSET_ERROR &&
3377
block_info.next_filepos >=
3378
info->state->data_file_length) ||
3379
(block_info.prev_filepos != HA_OFFSET_ERROR &&
3380
block_info.prev_filepos >= info->state->data_file_length))
3383
mi_check_print_info(param,
3384
"Delete link points outside datafile at %s",
3394
pos+= MI_DYN_ALIGN_SIZE;
3395
sort_param->start_recpos=pos;
3396
block_info.second_read=0;
3402
if (block_info.block_len+ (uint) (block_info.filepos-pos) <
3403
share->base.min_block_length ||
3404
block_info.block_len > (uint) share->base.max_pack_length+
3408
mi_check_print_info(param,
3409
"Found block with impossible length %u at %s; Skipped",
3410
block_info.block_len+ (uint) (block_info.filepos-pos),
3415
pos+= MI_DYN_ALIGN_SIZE;
3416
sort_param->start_recpos=pos;
3417
block_info.second_read=0;
3421
if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))
3423
if (!sort_param->fix_datafile && sort_param->master &&
3424
(b_type & BLOCK_DELETED))
3426
info->state->empty+=block_info.block_len;
3428
share->state.split++;
3434
pos+=MI_DYN_ALIGN_SIZE;
3435
sort_param->start_recpos=pos;
3438
pos=block_info.filepos+block_info.block_len;
3439
block_info.second_read=0;
3443
if (!sort_param->fix_datafile && sort_param->master)
3444
share->state.split++;
3445
if (! found_record++)
3447
sort_param->find_length=left_length=block_info.rec_len;
3448
sort_param->start_recpos=pos;
3449
if (!sort_param->fix_datafile)
3450
sort_param->filepos=sort_param->start_recpos;
3451
if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
3452
sort_param->pos=block_info.filepos+1;
3454
sort_param->pos=block_info.filepos+block_info.block_len;
3455
if (share->base.blobs)
3457
if (!(to=mi_alloc_rec_buff(info,block_info.rec_len,
3458
&(sort_param->rec_buff))))
3460
if (param->max_record_length >= block_info.rec_len)
3462
mi_check_print_error(param,"Not enough memory for blob at %s (need %lu)",
3463
llstr(sort_param->start_recpos,llbuff),
3464
(ulong) block_info.rec_len);
3469
mi_check_print_info(param,"Not enough memory for blob at %s (need %lu); Row skipped",
3470
llstr(sort_param->start_recpos,llbuff),
3471
(ulong) block_info.rec_len);
3477
to= sort_param->rec_buff;
3479
if (left_length < block_info.data_len || ! block_info.data_len)
3481
mi_check_print_info(param,
3482
"Found block with too small length at %s; Skipped",
3483
llstr(sort_param->start_recpos,llbuff));
3486
if (block_info.filepos + block_info.data_len >
3487
sort_param->read_cache.end_of_file)
3489
mi_check_print_info(param,
3490
"Found block that points outside data file at %s",
3491
llstr(sort_param->start_recpos,llbuff));
3495
Copy information that is already read. Avoid accessing data
3496
below the cache start. This could happen if the header
3497
streched over the end of the previous buffer contents.
3500
uint header_len= (uint) (block_info.filepos - pos);
3501
uint prefetch_len= (MI_BLOCK_INFO_HEADER_LENGTH - header_len);
3503
if (prefetch_len > block_info.data_len)
3504
prefetch_len= block_info.data_len;
3507
memcpy(to, block_info.header + header_len, prefetch_len);
3508
block_info.filepos+= prefetch_len;
3509
block_info.data_len-= prefetch_len;
3510
left_length-= prefetch_len;
3514
if (block_info.data_len &&
3515
_mi_read_cache(&sort_param->read_cache,to,block_info.filepos,
3516
block_info.data_len,
3517
(found_record == 1 ? READING_NEXT : 0) |
3520
mi_check_print_info(param,
3521
"Read error for block at: %s (error: %d); Skipped",
3522
llstr(block_info.filepos,llbuff),my_errno);
3525
left_length-=block_info.data_len;
3526
to+=block_info.data_len;
3527
pos=block_info.next_filepos;
3528
if (pos == HA_OFFSET_ERROR && left_length)
3530
mi_check_print_info(param,"Wrong block with wrong total length starting at %s",
3531
llstr(sort_param->start_recpos,llbuff));
3534
if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file)
3536
mi_check_print_info(param,"Found link that points at %s (outside data file) at %s",
3538
llstr(sort_param->start_recpos,llbuff));
3541
} while (left_length);
3543
if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff,
3544
sort_param->find_length) != MY_FILE_ERROR)
3546
if (sort_param->read_cache.error < 0)
3548
if (sort_param->calc_checksum)
3549
info->checksum= mi_checksum(info, sort_param->record);
3550
if ((param->testflag & (T_EXTEND | T_REP)) || searching)
3552
if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
3553
sort_param->find_length,
3554
(param->testflag & T_QUICK) &&
3555
sort_param->calc_checksum &&
3556
test(info->s->calc_checksum)))
3558
mi_check_print_info(param,"Found wrong packed record at %s",
3559
llstr(sort_param->start_recpos,llbuff));
3563
if (sort_param->calc_checksum)
3564
param->glob_crc+= info->checksum;
3568
mi_check_print_info(param,"Key %d - Found wrong stored record at %s",
3570
llstr(sort_param->start_recpos,llbuff));
3572
pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE);
3575
case COMPRESSED_RECORD:
3576
for (searching=0 ;; searching=1, sort_param->pos++)
3578
if (_mi_read_cache(&sort_param->read_cache,(uchar*) block_info.header,
3580
share->pack.ref_length,READING_NEXT))
3582
if (searching && ! sort_param->fix_datafile)
3584
param->error_printed=1;
3585
param->retry_repair=1;
3586
param->testflag|=T_RETRY_WITHOUT_QUICK;
3587
DBUG_RETURN(1); /* Something wrong with data */
3589
sort_param->start_recpos=sort_param->pos;
3590
if (_mi_pack_get_block_info(info, &sort_param->bit_buff, &block_info,
3591
&sort_param->rec_buff, -1, sort_param->pos))
3593
if (!block_info.rec_len &&
3594
sort_param->pos + MEMMAP_EXTRA_MARGIN ==
3595
sort_param->read_cache.end_of_file)
3597
if (block_info.rec_len < (uint) share->min_pack_length ||
3598
block_info.rec_len > (uint) share->max_pack_length)
3601
mi_check_print_info(param,"Found block with wrong recordlength: %d at %s\n",
3603
llstr(sort_param->pos,llbuff));
3606
if (_mi_read_cache(&sort_param->read_cache,(uchar*) sort_param->rec_buff,
3607
block_info.filepos, block_info.rec_len,
3611
mi_check_print_info(param,"Couldn't read whole record from %s",
3612
llstr(sort_param->pos,llbuff));
3615
if (_mi_pack_rec_unpack(info, &sort_param->bit_buff, sort_param->record,
3616
sort_param->rec_buff, block_info.rec_len))
3619
mi_check_print_info(param,"Found wrong record at %s",
3620
llstr(sort_param->pos,llbuff));
3623
if (!sort_param->fix_datafile)
3625
sort_param->filepos=sort_param->pos;
3626
if (sort_param->master)
3627
share->state.split++;
3629
sort_param->max_pos=(sort_param->pos=block_info.filepos+
3630
block_info.rec_len);
3631
info->packed_length=block_info.rec_len;
3632
if (sort_param->calc_checksum)
3633
param->glob_crc+= (info->checksum=
3634
mi_checksum(info, sort_param->record));
3638
assert(0); /* Impossible */
3640
DBUG_RETURN(1); /* Impossible */
3645
Write record to new file.
3649
sort_param Sort parameters.
3652
This is only called by a master thread if parallel repair is used.
3659
int sort_write_record(MI_SORT_PARAM *sort_param)
3663
ulong block_length,reclength;
3665
uchar block_buff[8];
3666
SORT_INFO *sort_info=sort_param->sort_info;
3667
MI_CHECK *param=sort_info->param;
3668
MI_INFO *info=sort_info->info;
3669
MYISAM_SHARE *share=info->s;
3670
DBUG_ENTER("sort_write_record");
3672
if (sort_param->fix_datafile)
3674
switch (sort_info->new_data_file_type) {
3676
if (my_b_write(&info->rec_cache,sort_param->record,
3677
share->base.pack_reclength))
3679
mi_check_print_error(param,"%d when writing to datafile",my_errno);
3682
sort_param->filepos+=share->base.pack_reclength;
3683
info->s->state.split++;
3684
/* sort_info->param->glob_crc+=mi_static_checksum(info, sort_param->record); */
3686
case DYNAMIC_RECORD:
3688
from=sort_param->rec_buff;
3691
/* must be sure that local buffer is big enough */
3692
reclength=info->s->base.pack_reclength+
3693
_my_calc_total_blob_length(info,sort_param->record)+
3694
ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+
3695
MI_DYN_DELETE_BLOCK_HEADER;
3696
if (sort_info->buff_length < reclength)
3698
if (!(sort_info->buff=my_realloc(sort_info->buff, (uint) reclength,
3699
MYF(MY_FREE_ON_ERROR |
3700
MY_ALLOW_ZERO_PTR))))
3702
sort_info->buff_length=reclength;
3704
from= sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
3706
/* We can use info->checksum here as only one thread calls this. */
3707
info->checksum=mi_checksum(info,sort_param->record);
3708
reclength=_mi_rec_pack(info,from,sort_param->record);
3710
/* sort_info->param->glob_crc+=info->checksum; */
3714
block_length=reclength+ 3 + test(reclength >= (65520-3));
3715
if (block_length < share->base.min_block_length)
3716
block_length=share->base.min_block_length;
3717
info->update|=HA_STATE_WRITE_AT_END;
3718
block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE);
3719
if (block_length > MI_MAX_BLOCK_LENGTH)
3720
block_length=MI_MAX_BLOCK_LENGTH;
3721
if (_mi_write_part_record(info,0L,block_length,
3722
sort_param->filepos+block_length,
3723
&from,&reclength,&flag))
3725
mi_check_print_error(param,"%d when writing to datafile",my_errno);
3728
sort_param->filepos+=block_length;
3729
info->s->state.split++;
3730
} while (reclength);
3731
/* sort_info->param->glob_crc+=info->checksum; */
3733
case COMPRESSED_RECORD:
3734
reclength=info->packed_length;
3735
length= save_pack_length((uint) share->pack.version, block_buff,
3737
if (info->s->base.blobs)
3738
length+= save_pack_length((uint) share->pack.version,
3739
block_buff + length, info->blob_length);
3740
if (my_b_write(&info->rec_cache,block_buff,length) ||
3741
my_b_write(&info->rec_cache,(uchar*) sort_param->rec_buff,reclength))
3743
mi_check_print_error(param,"%d when writing to datafile",my_errno);
3746
/* sort_info->param->glob_crc+=info->checksum; */
3747
sort_param->filepos+=reclength+length;
3748
info->s->state.split++;
3751
assert(0); /* Impossible */
3754
if (sort_param->master)
3756
info->state->records++;
3757
if ((param->testflag & T_WRITE_LOOP) &&
3758
(info->state->records % WRITE_COUNT) == 0)
3761
printf("%s\r", llstr(info->state->records,llbuff));
3762
VOID(fflush(stdout));
3766
} /* sort_write_record */
3769
/* Compare two keys from _create_index_by_sort */
3771
static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,
3775
return (ha_key_cmp(sort_param->seg, *((uchar**) a), *((uchar**) b),
3776
USE_WHOLE_KEY, SEARCH_SAME, not_used));
3777
} /* sort_key_cmp */
3780
static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
3783
char llbuff[22],llbuff2[22];
3784
SORT_INFO *sort_info=sort_param->sort_info;
3785
MI_CHECK *param= sort_info->param;
3788
if (sort_info->key_block->inited)
3790
cmp=ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
3791
(uchar*) a, USE_WHOLE_KEY,SEARCH_FIND | SEARCH_UPDATE,
3793
if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
3794
ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
3795
(uchar*) a, USE_WHOLE_KEY,
3796
SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
3797
else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
3799
diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg,
3800
sort_param->notnull,
3801
sort_info->key_block->lastkey,
3804
sort_param->unique[diff_pos[0]-1]++;
3809
if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
3810
mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
3813
if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0)
3816
sort_info->info->lastpos=get_record_for_key(sort_info->info,
3817
sort_param->keyinfo,
3819
mi_check_print_warning(param,
3820
"Duplicate key for record at %10s against record at %10s",
3821
llstr(sort_info->info->lastpos,llbuff),
3822
llstr(get_record_for_key(sort_info->info,
3823
sort_param->keyinfo,
3824
sort_info->key_block->
3827
param->testflag|=T_RETRY_WITHOUT_QUICK;
3828
if (sort_info->param->testflag & T_VERBOSE)
3829
_mi_print_key(stdout,sort_param->seg,(uchar*) a, USE_WHOLE_KEY);
3830
return (sort_delete_record(sort_param));
3835
mi_check_print_error(param,
3836
"Internal error: Keys are not in order from sort");
3840
return (sort_insert_key(sort_param,sort_info->key_block,
3841
(uchar*) a, HA_OFFSET_ERROR));
3842
} /* sort_key_write */
3844
int sort_ft_buf_flush(MI_SORT_PARAM *sort_param)
3846
SORT_INFO *sort_info=sort_param->sort_info;
3847
SORT_KEY_BLOCKS *key_block=sort_info->key_block;
3848
MYISAM_SHARE *share=sort_info->info->s;
3849
uint val_off, val_len;
3851
SORT_FT_BUF *ft_buf=sort_info->ft_buf;
3854
val_len=share->ft2_keyinfo.keylength;
3855
get_key_full_length_rdonly(val_off, ft_buf->lastkey);
3856
to=ft_buf->lastkey+val_off;
3860
/* flushing first-level tree */
3861
error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,
3863
for (from=to+val_len;
3864
!error && from < ft_buf->buf;
3867
memcpy(to, from, val_len);
3868
error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,
3873
/* flushing second-level tree keyblocks */
3874
error=flush_pending_blocks(sort_param);
3875
/* updating lastkey with second-level tree info */
3876
ft_intXstore(ft_buf->lastkey+val_off, -ft_buf->count);
3877
_mi_dpointer(sort_info->info, ft_buf->lastkey+val_off+HA_FT_WLEN,
3878
share->state.key_root[sort_param->key]);
3879
/* restoring first level tree data in sort_info/sort_param */
3880
sort_info->key_block=sort_info->key_block_end- sort_info->param->sort_key_blocks;
3881
sort_param->keyinfo=share->keyinfo+sort_param->key;
3882
share->state.key_root[sort_param->key]=HA_OFFSET_ERROR;
3883
/* writing lastkey in first-level tree */
3884
return error ? error :
3885
sort_insert_key(sort_param,sort_info->key_block,
3886
ft_buf->lastkey,HA_OFFSET_ERROR);
3889
static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a)
3891
uint a_len, val_off, val_len, error;
3893
SORT_INFO *sort_info=sort_param->sort_info;
3894
SORT_FT_BUF *ft_buf=sort_info->ft_buf;
3895
SORT_KEY_BLOCKS *key_block=sort_info->key_block;
3897
val_len=HA_FT_WLEN+sort_info->info->s->base.rec_reflength;
3898
get_key_full_length_rdonly(a_len, (uchar *)a);
3903
use two-level tree only if key_reflength fits in rec_reflength place
3904
and row format is NOT static - for _mi_dpointer not to garble offsets
3906
if ((sort_info->info->s->base.key_reflength <=
3907
sort_info->info->s->base.rec_reflength) &&
3908
(sort_info->info->s->options &
3909
(HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
3910
ft_buf=(SORT_FT_BUF *)my_malloc(sort_param->keyinfo->block_length +
3911
sizeof(SORT_FT_BUF), MYF(MY_WME));
3915
sort_param->key_write=sort_key_write;
3916
return sort_key_write(sort_param, a);
3918
sort_info->ft_buf=ft_buf;
3919
goto word_init_ft_buf; /* no need to duplicate the code */
3921
get_key_full_length_rdonly(val_off, ft_buf->lastkey);
3923
if (ha_compare_text(sort_param->seg->charset,
3924
((uchar *)a)+1,a_len-1,
3925
ft_buf->lastkey+1,val_off-1, 0, 0)==0)
3927
if (!ft_buf->buf) /* store in second-level tree */
3930
return sort_insert_key(sort_param,key_block,
3931
((uchar *)a)+a_len, HA_OFFSET_ERROR);
3934
/* storing the key in the buffer. */
3935
memcpy (ft_buf->buf, (char *)a+a_len, val_len);
3936
ft_buf->buf+=val_len;
3937
if (ft_buf->buf < ft_buf->end)
3940
/* converting to two-level tree */
3941
p=ft_buf->lastkey+val_off;
3943
while (key_block->inited)
3945
sort_info->key_block=key_block;
3946
sort_param->keyinfo=& sort_info->info->s->ft2_keyinfo;
3947
ft_buf->count=(ft_buf->buf - p)/val_len;
3949
/* flushing buffer to second-level tree */
3950
for (error=0; !error && p < ft_buf->buf; p+= val_len)
3951
error=sort_insert_key(sort_param,key_block,p,HA_OFFSET_ERROR);
3956
/* flushing buffer */
3957
if ((error=sort_ft_buf_flush(sort_param)))
3962
memcpy(ft_buf->lastkey, a, a_len);
3963
ft_buf->buf=ft_buf->lastkey+a_len;
3965
32 is just a safety margin here
3966
(at least max(val_len, sizeof(nod_flag)) should be there).
3967
May be better performance could be achieved if we'd put
3968
(sort_info->keyinfo->block_length-32)/XXX
3970
TODO: benchmark the best value for XXX.
3972
ft_buf->end=ft_buf->lastkey+ (sort_param->keyinfo->block_length-32);
3974
} /* sort_ft_key_write */
3977
/* get pointer to record from a key */
3979
static my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
3982
return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key));
3983
} /* get_record_for_key */
3986
/* Insert a key in sort-key-blocks */
3988
static int sort_insert_key(MI_SORT_PARAM *sort_param,
3989
register SORT_KEY_BLOCKS *key_block, uchar *key,
3990
my_off_t prev_block)
3992
uint a_length,t_length,nod_flag;
3993
my_off_t filepos,key_file_length;
3994
uchar *anc_buff,*lastkey;
3995
MI_KEY_PARAM s_temp;
3997
MI_KEYDEF *keyinfo=sort_param->keyinfo;
3998
SORT_INFO *sort_info= sort_param->sort_info;
3999
MI_CHECK *param=sort_info->param;
4000
DBUG_ENTER("sort_insert_key");
4002
anc_buff=key_block->buff;
4003
info=sort_info->info;
4004
lastkey=key_block->lastkey;
4005
nod_flag= (key_block == sort_info->key_block ? 0 :
4006
info->s->base.key_reflength);
4008
if (!key_block->inited)
4010
key_block->inited=1;
4011
if (key_block == sort_info->key_block_end)
4013
mi_check_print_error(param,"To many key-block-levels; Try increasing sort_key_blocks");
4016
a_length=2+nod_flag;
4017
key_block->end_pos=anc_buff+2;
4018
lastkey=0; /* No previous key in block */
4021
a_length=mi_getint(anc_buff);
4023
/* Save pointer to previous block */
4025
_mi_kpointer(info,key_block->end_pos,prev_block);
4027
t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,
4028
(uchar*) 0,lastkey,lastkey,key,
4030
(*keyinfo->store_key)(keyinfo, key_block->end_pos+nod_flag,&s_temp);
4032
mi_putint(anc_buff,a_length,nod_flag);
4033
key_block->end_pos+=t_length;
4034
if (a_length <= keyinfo->block_length)
4036
VOID(_mi_move_key(keyinfo,key_block->lastkey,key));
4037
key_block->last_length=a_length-t_length;
4041
/* Fill block with end-zero and write filled block */
4042
mi_putint(anc_buff,key_block->last_length,nod_flag);
4043
bzero((uchar*) anc_buff+key_block->last_length,
4044
keyinfo->block_length- key_block->last_length);
4045
key_file_length=info->state->key_file_length;
4046
if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
4049
/* If we read the page from the key cache, we have to write it back to it */
4050
if (key_file_length == info->state->key_file_length)
4052
if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
4055
else if (my_pwrite(info->s->kfile,(uchar*) anc_buff,
4056
(uint) keyinfo->block_length,filepos, param->myf_rw))
4058
DBUG_DUMP("buff",(uchar*) anc_buff,mi_getint(anc_buff));
4060
/* Write separator-key to block in next level */
4061
if (sort_insert_key(sort_param,key_block+1,key_block->lastkey,filepos))
4064
/* clear old block and write new key in it */
4065
key_block->inited=0;
4066
DBUG_RETURN(sort_insert_key(sort_param, key_block,key,prev_block));
4067
} /* sort_insert_key */
4070
/* Delete record when we found a duplicated key */
4072
static int sort_delete_record(MI_SORT_PARAM *sort_param)
4077
SORT_INFO *sort_info=sort_param->sort_info;
4078
MI_CHECK *param=sort_info->param;
4079
MI_INFO *info=sort_info->info;
4080
DBUG_ENTER("sort_delete_record");
4082
if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
4084
mi_check_print_error(param,
4085
"Quick-recover aborted; Run recovery without switch -q or with switch -qq");
4088
if (info->s->options & HA_OPTION_COMPRESS_RECORD)
4090
mi_check_print_error(param,
4091
"Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);;
4095
old_file=info->dfile;
4096
info->dfile=info->rec_cache.file;
4097
if (sort_info->current_key)
4099
key=info->lastkey+info->s->base.max_key_length;
4100
if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) &&
4101
error != HA_ERR_RECORD_DELETED)
4103
mi_check_print_error(param,"Can't read record to be removed");
4104
info->dfile=old_file;
4108
for (i=0 ; i < sort_info->current_key ; i++)
4110
uint key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos);
4111
if (_mi_ck_delete(info,i,key,key_length))
4113
mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1);
4114
info->dfile=old_file;
4118
if (sort_param->calc_checksum)
4119
param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record);
4121
error=flush_io_cache(&info->rec_cache) || (*info->s->delete_record)(info);
4122
info->dfile=old_file; /* restore actual value */
4123
info->state->records--;
4125
} /* sort_delete_record */
4127
/* Fix all pending blocks and flush everything to disk */
4129
int flush_pending_blocks(MI_SORT_PARAM *sort_param)
4131
uint nod_flag,length;
4132
my_off_t filepos,key_file_length;
4133
SORT_KEY_BLOCKS *key_block;
4134
SORT_INFO *sort_info= sort_param->sort_info;
4135
myf myf_rw=sort_info->param->myf_rw;
4136
MI_INFO *info=sort_info->info;
4137
MI_KEYDEF *keyinfo=sort_param->keyinfo;
4138
DBUG_ENTER("flush_pending_blocks");
4140
filepos= HA_OFFSET_ERROR; /* if empty file */
4142
for (key_block=sort_info->key_block ; key_block->inited ; key_block++)
4144
key_block->inited=0;
4145
length=mi_getint(key_block->buff);
4147
_mi_kpointer(info,key_block->end_pos,filepos);
4148
key_file_length=info->state->key_file_length;
4149
bzero((uchar*) key_block->buff+length, keyinfo->block_length-length);
4150
if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
4153
/* If we read the page from the key cache, we have to write it back */
4154
if (key_file_length == info->state->key_file_length)
4156
if (_mi_write_keypage(info, keyinfo, filepos,
4157
DFLT_INIT_HITS, key_block->buff))
4160
else if (my_pwrite(info->s->kfile,(uchar*) key_block->buff,
4161
(uint) keyinfo->block_length,filepos, myf_rw))
4163
DBUG_DUMP("buff",(uchar*) key_block->buff,length);
4166
info->s->state.key_root[sort_param->key]=filepos; /* Last is root for tree */
4168
} /* flush_pending_blocks */
4170
/* alloc space and pointers for key_blocks */
4172
static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint blocks,
4176
SORT_KEY_BLOCKS *block;
4177
DBUG_ENTER("alloc_key_blocks");
4179
if (!(block=(SORT_KEY_BLOCKS*) my_malloc((sizeof(SORT_KEY_BLOCKS)+
4180
buffer_length+IO_SIZE)*blocks,
4183
mi_check_print_error(param,"Not enough memory for sort-key-blocks");
4186
for (i=0 ; i < blocks ; i++)
4189
block[i].buff=(uchar*) (block+blocks)+(buffer_length+IO_SIZE)*i;
4192
} /* alloc_key_blocks */
4195
/* Check if file is almost full */
4197
int test_if_almost_full(MI_INFO *info)
4199
if (info->s->options & HA_OPTION_COMPRESS_RECORD)
4201
return my_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(MY_THREADSAFE)) / 10 * 9 >
4202
(my_off_t) info->s->base.max_key_file_length ||
4203
my_seek(info->dfile, 0L, MY_SEEK_END, MYF(0)) / 10 * 9 >
4204
(my_off_t) info->s->base.max_data_file_length;
4207
/* Recreate table with bigger more alloced record-data */
4209
int recreate_table(MI_CHECK *param, MI_INFO **org_info, char *filename)
4214
MI_KEYDEF *keyinfo,*key,*key_end;
4215
HA_KEYSEG *keysegs,*keyseg;
4216
MI_COLUMNDEF *recdef,*rec,*end;
4217
MI_UNIQUEDEF *uniquedef,*u_ptr,*u_end;
4218
MI_STATUS_INFO status_info;
4219
uint unpack,key_parts;
4220
ha_rows max_records;
4221
ulonglong file_length,tmp_length;
4222
MI_CREATE_INFO create_info;
4223
DBUG_ENTER("recreate_table");
4225
error=1; /* Default error */
4227
status_info= (*org_info)->state[0];
4228
info.state= &status_info;
4229
share= *(*org_info)->s;
4230
unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
4231
(param->testflag & T_UNPACK);
4232
if (!(keyinfo=(MI_KEYDEF*) my_alloca(sizeof(MI_KEYDEF)*share.base.keys)))
4234
memcpy((uchar*) keyinfo,(uchar*) share.keyinfo,
4235
(size_t) (sizeof(MI_KEYDEF)*share.base.keys));
4237
key_parts= share.base.all_key_parts;
4238
if (!(keysegs=(HA_KEYSEG*) my_alloca(sizeof(HA_KEYSEG)*
4239
(key_parts+share.base.keys))))
4241
my_afree((uchar*) keyinfo);
4244
if (!(recdef=(MI_COLUMNDEF*)
4245
my_alloca(sizeof(MI_COLUMNDEF)*(share.base.fields+1))))
4247
my_afree((uchar*) keyinfo);
4248
my_afree((uchar*) keysegs);
4251
if (!(uniquedef=(MI_UNIQUEDEF*)
4252
my_alloca(sizeof(MI_UNIQUEDEF)*(share.state.header.uniques+1))))
4254
my_afree((uchar*) recdef);
4255
my_afree((uchar*) keyinfo);
4256
my_afree((uchar*) keysegs);
4260
/* Copy the column definitions */
4261
memcpy((uchar*) recdef,(uchar*) share.rec,
4262
(size_t) (sizeof(MI_COLUMNDEF)*(share.base.fields+1)));
4263
for (rec=recdef,end=recdef+share.base.fields; rec != end ; rec++)
4265
if (unpack && !(share.options & HA_OPTION_PACK_RECORD) &&
4266
rec->type != FIELD_BLOB &&
4267
rec->type != FIELD_VARCHAR &&
4268
rec->type != FIELD_CHECK)
4269
rec->type=(int) FIELD_NORMAL;
4272
/* Change the new key to point at the saved key segments */
4273
memcpy((uchar*) keysegs,(uchar*) share.keyparts,
4274
(size_t) (sizeof(HA_KEYSEG)*(key_parts+share.base.keys+
4275
share.state.header.uniques)));
4277
for (key=keyinfo,key_end=keyinfo+share.base.keys; key != key_end ; key++)
4280
for (; keyseg->type ; keyseg++)
4282
if (param->language)
4283
keyseg->language=param->language; /* change language */
4285
keyseg++; /* Skip end pointer */
4288
/* Copy the unique definitions and change them to point at the new key
4290
memcpy((uchar*) uniquedef,(uchar*) share.uniqueinfo,
4291
(size_t) (sizeof(MI_UNIQUEDEF)*(share.state.header.uniques)));
4292
for (u_ptr=uniquedef,u_end=uniquedef+share.state.header.uniques;
4293
u_ptr != u_end ; u_ptr++)
4296
keyseg+=u_ptr->keysegs+1;
4298
if (share.options & HA_OPTION_COMPRESS_RECORD)
4299
share.base.records=max_records=info.state->records;
4300
else if (share.base.min_pack_length)
4301
max_records=(ha_rows) (my_seek(info.dfile,0L,MY_SEEK_END,MYF(0)) /
4302
(ulong) share.base.min_pack_length);
4305
unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
4306
(param->testflag & T_UNPACK);
4307
share.options&= ~HA_OPTION_TEMP_COMPRESS_RECORD;
4309
file_length=(ulonglong) my_seek(info.dfile,0L,MY_SEEK_END,MYF(0));
4310
tmp_length= file_length+file_length/10;
4311
set_if_bigger(file_length,param->max_data_file_length);
4312
set_if_bigger(file_length,tmp_length);
4313
set_if_bigger(file_length,(ulonglong) share.base.max_data_file_length);
4315
VOID(mi_close(*org_info));
4316
bzero((char*) &create_info,sizeof(create_info));
4317
create_info.max_rows=max(max_records,share.base.records);
4318
create_info.reloc_rows=share.base.reloc;
4319
create_info.old_options=(share.options |
4320
(unpack ? HA_OPTION_TEMP_COMPRESS_RECORD : 0));
4322
create_info.data_file_length=file_length;
4323
create_info.auto_increment=share.state.auto_increment;
4324
create_info.language = (param->language ? param->language :
4325
share.state.header.language);
4326
create_info.key_file_length= status_info.key_file_length;
4328
Allow for creating an auto_increment key. This has an effect only if
4329
an auto_increment key exists in the original table.
4331
create_info.with_auto_increment= TRUE;
4332
/* We don't have to handle symlinks here because we are using
4333
HA_DONT_TOUCH_DATA */
4334
if (mi_create(filename,
4335
share.base.keys - share.state.header.uniques,
4336
keyinfo, share.base.fields, recdef,
4337
share.state.header.uniques, uniquedef,
4339
HA_DONT_TOUCH_DATA))
4341
mi_check_print_error(param,"Got error %d when trying to recreate indexfile",my_errno);
4344
*org_info=mi_open(filename,O_RDWR,
4345
(param->testflag & T_WAIT_FOREVER) ? HA_OPEN_WAIT_IF_LOCKED :
4346
(param->testflag & T_DESCRIPT) ? HA_OPEN_IGNORE_IF_LOCKED :
4347
HA_OPEN_ABORT_IF_LOCKED);
4350
mi_check_print_error(param,"Got error %d when trying to open re-created indexfile",
4354
/* We are modifing */
4355
(*org_info)->s->options&= ~HA_OPTION_READ_ONLY_DATA;
4356
VOID(_mi_readinfo(*org_info,F_WRLCK,0));
4357
(*org_info)->state->records=info.state->records;
4358
if (share.state.create_time)
4359
(*org_info)->s->state.create_time=share.state.create_time;
4360
(*org_info)->s->state.unique=(*org_info)->this_unique=
4362
(*org_info)->state->checksum=info.state->checksum;
4363
(*org_info)->state->del=info.state->del;
4364
(*org_info)->s->state.dellink=share.state.dellink;
4365
(*org_info)->state->empty=info.state->empty;
4366
(*org_info)->state->data_file_length=info.state->data_file_length;
4367
if (update_state_info(param,*org_info,UPDATE_TIME | UPDATE_STAT |
4372
my_afree((uchar*) uniquedef);
4373
my_afree((uchar*) keyinfo);
4374
my_afree((uchar*) recdef);
4375
my_afree((uchar*) keysegs);
4380
/* write suffix to data file if neaded */
4382
int write_data_suffix(SORT_INFO *sort_info, my_bool fix_datafile)
4384
MI_INFO *info=sort_info->info;
4386
if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile)
4388
uchar buff[MEMMAP_EXTRA_MARGIN];
4389
bzero(buff,sizeof(buff));
4390
if (my_b_write(&info->rec_cache,buff,sizeof(buff)))
4392
mi_check_print_error(sort_info->param,
4393
"%d when writing to datafile",my_errno);
4396
sort_info->param->read_cache.end_of_file+=sizeof(buff);
4401
/* Update state and myisamchk_time of indexfile */
4403
int update_state_info(MI_CHECK *param, MI_INFO *info,uint update)
4405
MYISAM_SHARE *share=info->s;
4407
if (update & UPDATE_OPEN_COUNT)
4409
share->state.open_count=0;
4410
share->global_changed=0;
4412
if (update & UPDATE_STAT)
4414
uint i, key_parts= mi_uint2korr(share->state.header.key_parts);
4415
share->state.rec_per_key_rows=info->state->records;
4416
share->state.changed&= ~STATE_NOT_ANALYZED;
4417
if (info->state->records)
4419
for (i=0; i<key_parts; i++)
4421
if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i]))
4422
share->state.changed|= STATE_NOT_ANALYZED;
4426
if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC))
4428
if (update & UPDATE_TIME)
4430
share->state.check_time= (long) time((time_t*) 0);
4431
if (!share->state.create_time)
4432
share->state.create_time=share->state.check_time;
4435
When tables are locked we haven't synched the share state and the
4436
real state for a while so we better do it here before synching
4437
the share state to disk. Only when table is write locked is it
4438
necessary to perform this synch.
4440
if (info->lock_type == F_WRLCK)
4441
share->state.state= *info->state;
4442
if (mi_state_info_write(share->kfile,&share->state,1+2))
4446
{ /* Force update of status */
4448
uint r_locks=share->r_locks,w_locks=share->w_locks;
4449
share->r_locks= share->w_locks= share->tot_locks= 0;
4450
error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
4451
share->r_locks=r_locks;
4452
share->w_locks=w_locks;
4453
share->tot_locks=r_locks+w_locks;
4458
mi_check_print_error(param,"%d when updating keyfile",my_errno);
4463
Update auto increment value for a table
4464
When setting the 'repair_only' flag we only want to change the
4465
old auto_increment value if its wrong (smaller than some given key).
4466
The reason is that we shouldn't change the auto_increment value
4467
for a table without good reason when only doing a repair; If the
4468
user have inserted and deleted rows, the auto_increment value
4469
may be bigger than the biggest current row and this is ok.
4471
If repair_only is not set, we will update the flag to the value in
4472
param->auto_increment is bigger than the biggest key.
4475
void update_auto_increment_key(MI_CHECK *param, MI_INFO *info,
4476
my_bool repair_only)
4479
DBUG_ENTER("update_auto_increment_key");
4481
if (!info->s->base.auto_key ||
4482
! mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1))
4484
if (!(param->testflag & T_VERY_SILENT))
4485
mi_check_print_info(param,
4486
"Table: %s doesn't have an auto increment key\n",
4487
param->isam_file_name);
4490
if (!(param->testflag & T_SILENT) &&
4491
!(param->testflag & T_REP))
4492
printf("Updating MyISAM file: %s\n", param->isam_file_name);
4494
We have to use an allocated buffer instead of info->rec_buff as
4495
_mi_put_key_in_record() may use info->rec_buff
4497
if (!mi_alloc_rec_buff(info, -1, &record))
4499
mi_check_print_error(param,"Not enough memory for extra record");
4503
mi_extra(info,HA_EXTRA_KEYREAD,0);
4504
if (mi_rlast(info, record, info->s->base.auto_key-1))
4506
if (my_errno != HA_ERR_END_OF_FILE)
4508
mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4509
my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
4510
mi_check_print_error(param,"%d when reading last record",my_errno);
4514
info->s->state.auto_increment=param->auto_increment_value;
4518
ulonglong auto_increment= retrieve_auto_increment(info, record);
4519
set_if_bigger(info->s->state.auto_increment,auto_increment);
4521
set_if_bigger(info->s->state.auto_increment, param->auto_increment_value);
4523
mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4524
my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
4525
update_state_info(param, info, UPDATE_AUTO_INC);
4531
Update statistics for each part of an index
4535
keyinfo IN Index information (only key->keysegs used)
4536
rec_per_key_part OUT Store statistics here
4537
unique IN Array of (#distinct tuples)
4538
notnull_tuples IN Array of (#tuples), or NULL
4539
records Number of records in the table
4542
This function is called produce index statistics values from unique and
4543
notnull_tuples arrays after these arrays were produced with sequential
4544
index scan (the scan is done in two places: chk_index() and
4547
This function handles all 3 index statistics collection methods.
4550
unique[0]= (#different values of {keypart1}) - 1
4551
unique[1]= (#different values of {keypart1,keypart2} tuple)-unique[0]-1
4554
For MI_STATS_METHOD_IGNORE_NULLS method, notnull_tuples is an array too:
4555
notnull_tuples[0]= (#of {keypart1} tuples such that keypart1 is not NULL)
4556
notnull_tuples[1]= (#of {keypart1,keypart2} tuples such that all
4557
keypart{i} are not NULL)
4559
For all other statistics collection methods notnull_tuples==NULL.
4562
rec_per_key_part[k] =
4563
= E(#records in the table such that keypart_1=c_1 AND ... AND
4564
keypart_k=c_k for arbitrary constants c_1 ... c_k)
4566
= {assuming that values have uniform distribution and index contains all
4567
tuples from the domain (or that {c_1, ..., c_k} tuple is choosen from
4570
= #tuples-in-the-index / #distinct-tuples-in-the-index.
4572
The #tuples-in-the-index and #distinct-tuples-in-the-index have different
4573
meaning depending on which statistics collection method is used:
4575
MI_STATS_METHOD_* how are nulls compared? which tuples are counted?
4576
NULLS_EQUAL NULL == NULL all tuples in table
4577
NULLS_NOT_EQUAL NULL != NULL all tuples in table
4578
IGNORE_NULLS n/a tuples that don't have NULLs
4581
void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
4582
ulonglong *unique, ulonglong *notnull,
4585
ulonglong count=0,tmp, unique_tuples;
4586
ulonglong tuples= records;
4588
for (parts=0 ; parts < keyinfo->keysegs ; parts++)
4590
count+=unique[parts];
4591
unique_tuples= count + 1;
4594
tuples= notnull[parts];
4596
#(unique_tuples not counting tuples with NULLs) =
4597
#(unique_tuples counting tuples with NULLs as different) -
4598
#(tuples with NULLs)
4600
unique_tuples -= (records - notnull[parts]);
4603
if (unique_tuples == 0)
4605
else if (count == 0)
4606
tmp= tuples; /* 1 unique tuple */
4608
tmp= (tuples + unique_tuples/2) / unique_tuples;
4611
for some weird keys (e.g. FULLTEXT) tmp can be <1 here.
4612
let's ensure it is not
4614
set_if_bigger(tmp,1);
4615
if (tmp >= (ulonglong) ~(ulong) 0)
4616
tmp=(ulonglong) ~(ulong) 0;
4618
*rec_per_key_part=(ulong) tmp;
4624
static ha_checksum mi_byte_checksum(const uchar *buf, uint length)
4627
const uchar *end=buf+length;
4628
for (crc=0; buf != end; buf++)
4629
crc=((crc << 1) + *((uchar*) buf)) +
4630
test(crc & (((ha_checksum) 1) << (8*sizeof(ha_checksum)-1)));
4634
static my_bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)
4636
uint key_maxlength=key->maxlength;
4637
if (key->flag & HA_FULLTEXT)
4639
uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
4640
key->seg->charset->mbmaxlen;
4641
key_maxlength+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
4643
return (key->flag & HA_SPATIAL) ||
4644
(key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) &&
4645
((ulonglong) rows * key_maxlength >
4646
(ulonglong) myisam_max_temp_length));
4650
Deactivate all not unique index that can be recreated fast
4651
These include packed keys on which sorting will use more temporary
4652
space than the max allowed file length or for which the unpacked keys
4653
will take much more space than packed keys.
4654
Note that 'rows' may be zero for the case when we don't know how many
4655
rows we will put into the file.
4658
void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows)
4660
MYISAM_SHARE *share=info->s;
4661
MI_KEYDEF *key=share->keyinfo;
4664
DBUG_ASSERT(info->state->records == 0 &&
4665
(!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
4666
for (i=0 ; i < share->base.keys ; i++,key++)
4668
if (!(key->flag & (HA_NOSAME | HA_SPATIAL | HA_AUTO_KEY)) &&
4669
! mi_too_big_key_for_sort(key,rows) && info->s->base.auto_key != i+1)
4671
mi_clear_key_active(share->state.key_map, i);
4672
info->update|= HA_STATE_CHANGED;
4679
Return TRUE if we can use repair by sorting
4680
One can set the force argument to force to use sorting
4681
even if the temporary file would be quite big!
4684
my_bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows,
4685
ulonglong key_map, my_bool force)
4687
MYISAM_SHARE *share=info->s;
4688
MI_KEYDEF *key=share->keyinfo;
4692
mi_repair_by_sort only works if we have at least one key. If we don't
4693
have any keys, we should use the normal repair.
4695
if (! mi_is_any_key_active(key_map))
4696
return FALSE; /* Can't use sort */
4697
for (i=0 ; i < share->base.keys ; i++,key++)
4699
if (!force && mi_too_big_key_for_sort(key,rows))
4707
set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share)
4709
if ((sort_info->new_data_file_type=share->data_file_type) ==
4710
COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK)
4714
if (share->options & HA_OPTION_PACK_RECORD)
4715
sort_info->new_data_file_type = DYNAMIC_RECORD;
4717
sort_info->new_data_file_type = STATIC_RECORD;
4719
/* Set delete_function for sort_delete_record() */
4720
memcpy((char*) &tmp, share, sizeof(*share));
4721
tmp.options= ~HA_OPTION_COMPRESS_RECORD;
4722
mi_setup_functions(&tmp);
4723
share->delete_record=tmp.delete_record;
added
removed
storage/myisam/mi_check.c
