~drizzle-trunk/drizzle/development

1 by brian
clean slate
1
/* Copyright (C) 2000-2006 MySQL AB
2
3
   This program is free software; you can redistribute it and/or modify
4
   it under the terms of the GNU General Public License as published by
5
   the Free Software Foundation; version 2 of the License.
6
7
   This program is distributed in the hope that it will be useful,
8
   but WITHOUT ANY WARRANTY; without even the implied warranty of
9
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
10
   GNU General Public License for more details.
11
12
   You should have received a copy of the GNU General Public License
13
   along with this program; if not, write to the Free Software
14
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
15
16
/* Describe, check and repair of MyISAM tables */
17
18
/*
19
  About checksum calculation.
20
21
  There are two types of checksums. Table checksum and row checksum.
22
23
  Row checksum is an additional byte at the end of dynamic length
24
  records. It must be calculated if the table is configured for them.
25
  Otherwise they must not be used. The variable
26
  MYISAM_SHARE::calc_checksum determines if row checksums are used.
27
  MI_INFO::checksum is used as temporary storage during row handling.
28
  For parallel repair we must assure that only one thread can use this
29
  variable. There is no problem on the write side as this is done by one
30
  thread only. But when checking a record after read this could go
31
  wrong. But since all threads read through a common read buffer, it is
32
  sufficient if only one thread checks it.
33
34
  Table checksum is an eight byte value in the header of the index file.
35
  It can be calculated even if row checksums are not used. The variable
36
  MI_CHECK::glob_crc is calculated over all records.
37
  MI_SORT_PARAM::calc_checksum determines if this should be done. This
38
  variable is not part of MI_CHECK because it must be set per thread for
39
  parallel repair. The global glob_crc must be changed by one thread
40
  only. And it is sufficient to calculate the checksum once only.
41
*/
42
43
#include "ftdefs.h"
44
#include <m_ctype.h>
45
#include <stdarg.h>
46
#include <my_getopt.h>
47
#ifdef HAVE_SYS_VADVISE_H
48
#include <sys/vadvise.h>
49
#endif
50
#ifdef HAVE_SYS_MMAN_H
51
#include <sys/mman.h>
52
#endif
53
#include "rt_index.h"
54
55
#ifndef USE_RAID
56
#define my_raid_create(A,B,C,D,E,F,G) my_create(A,B,C,G)
57
#define my_raid_delete(A,B,C) my_delete(A,B)
58
#endif
59
60
	/* Functions defined in this file */
61
62
static int check_k_link(MI_CHECK *param, MI_INFO *info,uint nr);
63
static int chk_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
64
		     my_off_t page, uchar *buff, ha_rows *keys,
65
		     ha_checksum *key_checksum, uint level);
66
static uint isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo);
67
static ha_checksum calc_checksum(ha_rows count);
68
static int writekeys(MI_SORT_PARAM *sort_param);
69
static int sort_one_index(MI_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
70
			  my_off_t pagepos, File new_file);
71
static int sort_key_read(MI_SORT_PARAM *sort_param,void *key);
72
static int sort_ft_key_read(MI_SORT_PARAM *sort_param,void *key);
73
static int sort_get_next_record(MI_SORT_PARAM *sort_param);
74
static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b);
75
static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a);
76
static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);
77
static my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo,
78
				uchar *key);
79
static int sort_insert_key(MI_SORT_PARAM  *sort_param,
80
                           register SORT_KEY_BLOCKS *key_block,
81
			   uchar *key, my_off_t prev_block);
82
static int sort_delete_record(MI_SORT_PARAM *sort_param);
83
/*static int flush_pending_blocks(MI_CHECK *param);*/
84
static SORT_KEY_BLOCKS	*alloc_key_blocks(MI_CHECK *param, uint blocks,
85
					  uint buffer_length);
86
static ha_checksum mi_byte_checksum(const uchar *buf, uint length);
87
static void set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share);
88
89
void myisamchk_init(MI_CHECK *param)
90
{
91
  bzero((uchar*) param,sizeof(*param));
92
  param->opt_follow_links=1;
93
  param->keys_in_use= ~(ulonglong) 0;
94
  param->search_after_block=HA_OFFSET_ERROR;
95
  param->auto_increment_value= 0;
96
  param->use_buffers=USE_BUFFER_INIT;
97
  param->read_buffer_length=READ_BUFFER_INIT;
98
  param->write_buffer_length=READ_BUFFER_INIT;
99
  param->sort_buffer_length=SORT_BUFFER_INIT;
100
  param->sort_key_blocks=BUFFERS_WHEN_SORTING;
101
  param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL;
102
  param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
103
  param->start_check_pos=0;
104
  param->max_record_length= LONGLONG_MAX;
105
  param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE;
106
  param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
107
}
108
109
	/* Check the status flags for the table */
110
111
int chk_status(MI_CHECK *param, register MI_INFO *info)
112
{
113
  MYISAM_SHARE *share=info->s;
114
115
  if (mi_is_crashed_on_repair(info))
116
    mi_check_print_warning(param,
117
			   "Table is marked as crashed and last repair failed");
118
  else if (mi_is_crashed(info))
119
    mi_check_print_warning(param,
120
			   "Table is marked as crashed");
121
  if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0))
122
  {
123
    /* Don't count this as a real warning, as check can correct this ! */
124
    uint save=param->warning_printed;
125
    mi_check_print_warning(param,
126
			   share->state.open_count==1 ? 
127
			   "%d client is using or hasn't closed the table properly" : 
128
			   "%d clients are using or haven't closed the table properly",
129
			   share->state.open_count);
130
    /* If this will be fixed by the check, forget the warning */
131
    if (param->testflag & T_UPDATE_STATE)
132
      param->warning_printed=save;
133
  }
134
  return 0;
135
}
136
137
	/* Check delete links */
138
139
int chk_del(MI_CHECK *param, register MI_INFO *info, uint test_flag)
140
{
141
  register ha_rows i;
142
  uint delete_link_length;
143
  my_off_t empty, next_link, old_link= 0;
144
  char buff[22],buff2[22];
145
  DBUG_ENTER("chk_del");
146
147
  param->record_checksum=0;
148
  delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 :
149
		      info->s->rec_reflength+1);
150
151
  if (!(test_flag & T_SILENT))
152
    puts("- check record delete-chain");
153
154
  next_link=info->s->state.dellink;
155
  if (info->state->del == 0)
156
  {
157
    if (test_flag & T_VERBOSE)
158
    {
159
      puts("No recordlinks");
160
    }
161
  }
162
  else
163
  {
164
    if (test_flag & T_VERBOSE)
165
      printf("Recordlinks:    ");
166
    empty=0;
167
    for (i= info->state->del ; i > 0L && next_link != HA_OFFSET_ERROR ; i--)
168
    {
169
      if (*killed_ptr(param))
170
        DBUG_RETURN(1);
171
      if (test_flag & T_VERBOSE)
172
	printf(" %9s",llstr(next_link,buff));
173
      if (next_link >= info->state->data_file_length)
174
	goto wrong;
175
      if (my_pread(info->dfile, (uchar*) buff,delete_link_length,
176
		   next_link,MYF(MY_NABP)))
177
      {
178
	if (test_flag & T_VERBOSE) puts("");
179
	mi_check_print_error(param,"Can't read delete-link at filepos: %s",
180
		    llstr(next_link,buff));
181
	DBUG_RETURN(1);
182
      }
183
      if (*buff != '\0')
184
      {
185
	if (test_flag & T_VERBOSE) puts("");
186
	mi_check_print_error(param,"Record at pos: %s is not remove-marked",
187
		    llstr(next_link,buff));
188
	goto wrong;
189
      }
190
      if (info->s->options & HA_OPTION_PACK_RECORD)
191
      {
192
	my_off_t prev_link=mi_sizekorr(buff+12);
193
	if (empty && prev_link != old_link)
194
	{
195
	  if (test_flag & T_VERBOSE) puts("");
196
	  mi_check_print_error(param,"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2));
197
	  goto wrong;
198
	}
199
	old_link=next_link;
200
	next_link=mi_sizekorr(buff+4);
201
	empty+=mi_uint3korr(buff+1);
202
      }
203
      else
204
      {
205
	param->record_checksum+=(ha_checksum) next_link;
206
	next_link=_mi_rec_pos(info->s,(uchar*) buff+1);
207
	empty+=info->s->base.pack_reclength;
208
      }
209
    }
210
    if (test_flag & T_VERBOSE)
211
      puts("\n");
212
    if (empty != info->state->empty)
213
    {
214
      mi_check_print_warning(param,
215
			     "Found %s deleted space in delete link chain. Should be %s",
216
			     llstr(empty,buff2),
217
			     llstr(info->state->empty,buff));
218
    }
219
    if (next_link != HA_OFFSET_ERROR)
220
    {
221
      mi_check_print_error(param,
222
			   "Found more than the expected %s deleted rows in delete link chain",
223
			   llstr(info->state->del, buff));
224
      goto wrong;
225
    }
226
    if (i != 0)
227
    {
228
      mi_check_print_error(param,
229
			   "Found %s deleted rows in delete link chain. Should be %s",
230
			   llstr(info->state->del - i, buff2),
231
			   llstr(info->state->del, buff));
232
      goto wrong;
233
    }
234
  }
235
  DBUG_RETURN(0);
236
237
wrong:
238
  param->testflag|=T_RETRY_WITHOUT_QUICK;
239
  if (test_flag & T_VERBOSE) puts("");
240
  mi_check_print_error(param,"record delete-link-chain corrupted");
241
  DBUG_RETURN(1);
242
} /* chk_del */
243
244
245
	/* Check delete links in index file */
246
247
static int check_k_link(MI_CHECK *param, register MI_INFO *info, uint nr)
248
{
249
  my_off_t next_link;
250
  uint block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH;
251
  ha_rows records;
252
  char llbuff[21], llbuff2[21];
253
  uchar *buff;
254
  DBUG_ENTER("check_k_link");
255
  DBUG_PRINT("enter", ("block_size: %u", block_size));
256
257
  if (param->testflag & T_VERBOSE)
258
    printf("block_size %4u:", block_size); /* purecov: tested */
259
260
  next_link=info->s->state.key_del[nr];
261
  records= (ha_rows) (info->state->key_file_length / block_size);
262
  while (next_link != HA_OFFSET_ERROR && records > 0)
263
  {
264
    if (*killed_ptr(param))
265
      DBUG_RETURN(1);
266
    if (param->testflag & T_VERBOSE)
267
      printf("%16s",llstr(next_link,llbuff));
268
269
    /* Key blocks must lay within the key file length entirely. */
270
    if (next_link + block_size > info->state->key_file_length)
271
    {
272
      /* purecov: begin tested */
273
      mi_check_print_error(param, "Invalid key block position: %s  "
274
                           "key block size: %u  file_length: %s",
275
                           llstr(next_link, llbuff), block_size,
276
                           llstr(info->state->key_file_length, llbuff2));
277
      DBUG_RETURN(1);
278
      /* purecov: end */
279
    }
280
281
    /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
282
    if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1))
283
    {
284
      /* purecov: begin tested */
285
      mi_check_print_error(param, "Mis-aligned key block: %s  "
286
                           "minimum key block length: %u",
287
                           llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
288
      DBUG_RETURN(1);
289
      /* purecov: end */
290
    }
291
292
    /*
293
      Read the key block with MI_MIN_KEY_BLOCK_LENGTH to find next link.
294
      If the key cache block size is smaller than block_size, we can so
295
      avoid unecessary eviction of cache block.
296
    */
297
    if (!(buff=key_cache_read(info->s->key_cache,
298
                              info->s->kfile, next_link, DFLT_INIT_HITS,
299
                              (uchar*) info->buff, MI_MIN_KEY_BLOCK_LENGTH,
300
                              MI_MIN_KEY_BLOCK_LENGTH, 1)))
301
    {
302
      /* purecov: begin tested */
303
      mi_check_print_error(param, "key cache read error for block: %s",
304
			   llstr(next_link,llbuff));
305
      DBUG_RETURN(1);
306
      /* purecov: end */
307
    }
308
    next_link=mi_sizekorr(buff);
309
    records--;
310
    param->key_file_blocks+=block_size;
311
  }
312
  if (param->testflag & T_VERBOSE)
313
  {
314
    if (next_link != HA_OFFSET_ERROR)
315
      printf("%16s\n",llstr(next_link,llbuff));
316
    else
317
      puts("");
318
  }
319
  DBUG_RETURN (next_link != HA_OFFSET_ERROR);
320
} /* check_k_link */
321
322
323
	/* Check sizes of files */
324
325
int chk_size(MI_CHECK *param, register MI_INFO *info)
326
{
327
  int error=0;
328
  register my_off_t skr,size;
329
  char buff[22],buff2[22];
330
  DBUG_ENTER("chk_size");
331
332
  if (!(param->testflag & T_SILENT)) puts("- check file-size");
333
334
  /* The following is needed if called externally (not from myisamchk) */
335
  flush_key_blocks(info->s->key_cache,
336
		   info->s->kfile, FLUSH_FORCE_WRITE);
337
338
  size= my_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(MY_THREADSAFE));
339
  if ((skr=(my_off_t) info->state->key_file_length) != size)
340
  {
341
    /* Don't give error if file generated by myisampack */
342
    if (skr > size && mi_is_any_key_active(info->s->state.key_map))
343
    {
344
      error=1;
345
      mi_check_print_error(param,
346
			   "Size of indexfile is: %-8s        Should be: %s",
347
			   llstr(size,buff), llstr(skr,buff2));
348
    }
349
    else
350
      mi_check_print_warning(param,
351
			     "Size of indexfile is: %-8s      Should be: %s",
352
			     llstr(size,buff), llstr(skr,buff2));
353
  }
354
  if (!(param->testflag & T_VERY_SILENT) &&
355
      ! (info->s->options & HA_OPTION_COMPRESS_RECORD) &&
356
      ulonglong2double(info->state->key_file_length) >
357
      ulonglong2double(info->s->base.margin_key_file_length)*0.9)
358
    mi_check_print_warning(param,"Keyfile is almost full, %10s of %10s used",
359
			   llstr(info->state->key_file_length,buff),
360
			   llstr(info->s->base.max_key_file_length-1,buff));
361
362
  size=my_seek(info->dfile,0L,MY_SEEK_END,MYF(0));
363
  skr=(my_off_t) info->state->data_file_length;
364
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
365
    skr+= MEMMAP_EXTRA_MARGIN;
366
#ifdef USE_RELOC
367
  if (info->data_file_type == STATIC_RECORD &&
368
      skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length)
369
    skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length;
370
#endif
371
  if (skr != size)
372
  {
373
    info->state->data_file_length=size;	/* Skip other errors */
374
    if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN)
375
    {
376
      error=1;
377
      mi_check_print_error(param,"Size of datafile is: %-9s         Should be: %s",
378
		    llstr(size,buff), llstr(skr,buff2));
379
      param->testflag|=T_RETRY_WITHOUT_QUICK;
380
    }
381
    else
382
    {
383
      mi_check_print_warning(param,
384
			     "Size of datafile is: %-9s       Should be: %s",
385
			     llstr(size,buff), llstr(skr,buff2));
386
    }
387
  }
388
  if (!(param->testflag & T_VERY_SILENT) &&
389
      !(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
390
      ulonglong2double(info->state->data_file_length) >
391
      (ulonglong2double(info->s->base.max_data_file_length)*0.9))
392
    mi_check_print_warning(param, "Datafile is almost full, %10s of %10s used",
393
			   llstr(info->state->data_file_length,buff),
394
			   llstr(info->s->base.max_data_file_length-1,buff2));
395
  DBUG_RETURN(error);
396
} /* chk_size */
397
398
399
	/* Check keys */
400
401
int chk_key(MI_CHECK *param, register MI_INFO *info)
402
{
403
  uint key,found_keys=0,full_text_keys=0,result=0;
404
  ha_rows keys;
405
  ha_checksum old_record_checksum,init_checksum;
406
  my_off_t all_keydata,all_totaldata,key_totlength,length;
407
  ulong   *rec_per_key_part;
408
  MYISAM_SHARE *share=info->s;
409
  MI_KEYDEF *keyinfo;
410
  char buff[22],buff2[22];
411
  DBUG_ENTER("chk_key");
412
413
  if (!(param->testflag & T_SILENT))
414
    puts("- check key delete-chain");
415
416
  param->key_file_blocks=info->s->base.keystart;
417
  for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
418
    if (check_k_link(param,info,key))
419
    {
420
      if (param->testflag & T_VERBOSE) puts("");
421
      mi_check_print_error(param,"key delete-link-chain corrupted");
422
      DBUG_RETURN(-1);
423
    }
424
425
  if (!(param->testflag & T_SILENT)) puts("- check index reference");
426
427
  all_keydata=all_totaldata=key_totlength=0;
428
  old_record_checksum=0;
429
  init_checksum=param->record_checksum;
430
  if (!(share->options &
431
	(HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
432
    old_record_checksum=calc_checksum(info->state->records+info->state->del-1)*
433
      share->base.pack_reclength;
434
  rec_per_key_part= param->rec_per_key_part;
435
  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
436
       rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
437
  {
438
    param->key_crc[key]=0;
439
    if (! mi_is_key_active(share->state.key_map, key))
440
    {
441
      /* Remember old statistics for key */
442
      memcpy((char*) rec_per_key_part,
443
	     (char*) (share->state.rec_per_key_part +
444
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
445
	     keyinfo->keysegs*sizeof(*rec_per_key_part));
446
      continue;
447
    }
448
    found_keys++;
449
450
    param->record_checksum=init_checksum;
451
    
452
    bzero((char*) &param->unique_count,sizeof(param->unique_count));
453
    bzero((char*) &param->notnull_count,sizeof(param->notnull_count));
454
455
    if ((!(param->testflag & T_SILENT)))
456
      printf ("- check data record references index: %d\n",key+1);
457
    if (keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL))
458
      full_text_keys++;
459
    if (share->state.key_root[key] == HA_OFFSET_ERROR &&
460
	(info->state->records == 0 || keyinfo->flag & HA_FULLTEXT))
461
      goto do_stat;
462
    if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],
463
                           DFLT_INIT_HITS,info->buff,0))
464
    {
465
      mi_check_print_error(param,"Can't read indexpage from filepos: %s",
466
		  llstr(share->state.key_root[key],buff));
467
      if (!(param->testflag & T_INFO))
468
	DBUG_RETURN(-1);
469
      result= -1;
470
      continue;
471
    }
472
    param->key_file_blocks+=keyinfo->block_length;
473
    keys=0;
474
    param->keydata=param->totaldata=0;
475
    param->key_blocks=0;
476
    param->max_level=0;
477
    if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
478
		  &keys, param->key_crc+key,1))
479
      DBUG_RETURN(-1);
480
    if(!(keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL)))
481
    {
482
      if (keys != info->state->records)
483
      {
484
	mi_check_print_error(param,"Found %s keys of %s",llstr(keys,buff),
485
		    llstr(info->state->records,buff2));
486
	if (!(param->testflag & T_INFO))
487
	DBUG_RETURN(-1);
488
	result= -1;
489
	continue;
490
      }
491
      if (found_keys - full_text_keys == 1 &&
492
	  ((share->options &
493
	    (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
494
	   (param->testflag & T_DONT_CHECK_CHECKSUM)))
495
	old_record_checksum=param->record_checksum;
496
      else if (old_record_checksum != param->record_checksum)
497
      {
498
	if (key)
499
	  mi_check_print_error(param,"Key %u doesn't point at same records that key 1",
500
		      key+1);
501
	else
502
	  mi_check_print_error(param,"Key 1 doesn't point at all records");
503
	if (!(param->testflag & T_INFO))
504
	  DBUG_RETURN(-1);
505
	result= -1;
506
	continue;
507
      }
508
    }
509
    if ((uint) share->base.auto_key -1 == key)
510
    {
511
      /* Check that auto_increment key is bigger than max key value */
512
      ulonglong auto_increment;
513
      info->lastinx=key;
514
      _mi_read_key_record(info, 0L, info->rec_buff);
515
      auto_increment= retrieve_auto_increment(info, info->rec_buff);
516
      if (auto_increment > info->s->state.auto_increment)
517
      {
518
        mi_check_print_warning(param, "Auto-increment value: %s is smaller "
519
                               "than max used value: %s",
520
                               llstr(info->s->state.auto_increment,buff2),
521
                               llstr(auto_increment, buff));
522
      }
523
      if (param->testflag & T_AUTO_INC)
524
      {
525
        set_if_bigger(info->s->state.auto_increment,
526
                      auto_increment);
527
        set_if_bigger(info->s->state.auto_increment,
528
                      param->auto_increment_value);
529
      }
530
531
      /* Check that there isn't a row with auto_increment = 0 in the table */
532
      mi_extra(info,HA_EXTRA_KEYREAD,0);
533
      bzero(info->lastkey,keyinfo->seg->length);
534
      if (!mi_rkey(info, info->rec_buff, key, (const uchar*) info->lastkey,
535
		   (key_part_map)1, HA_READ_KEY_EXACT))
536
      {
537
	/* Don't count this as a real warning, as myisamchk can't correct it */
538
	uint save=param->warning_printed;
539
        mi_check_print_warning(param, "Found row where the auto_increment "
540
                               "column has the value 0");
541
	param->warning_printed=save;
542
      }
543
      mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
544
    }
545
546
    length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2;
547
    if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
548
      printf("Key: %2d:  Keyblocks used: %3d%%  Packed: %4d%%  Max levels: %2d\n",
549
	     key+1,
550
	     (int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)),
551
	     (int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/
552
		    my_off_t2double(length)),
553
	     param->max_level);
554
    all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length;
555
556
do_stat:
557
    if (param->testflag & T_STATISTICS)
558
      update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
559
                       param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
560
                       param->notnull_count: NULL, 
561
                       (ulonglong)info->state->records);
562
  }
563
  if (param->testflag & T_INFO)
564
  {
565
    if (all_totaldata != 0L && found_keys > 0)
566
      printf("Total:    Keyblocks used: %3d%%  Packed: %4d%%\n\n",
567
	     (int) (my_off_t2double(all_keydata)*100.0/
568
		    my_off_t2double(all_totaldata)),
569
	     (int) ((my_off_t2double(key_totlength) -
570
		     my_off_t2double(all_keydata))*100.0/
571
		     my_off_t2double(key_totlength)));
572
    else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map))
573
      puts("");
574
  }
575
  if (param->key_file_blocks != info->state->key_file_length &&
576
      param->keys_in_use != ~(ulonglong) 0)
577
    mi_check_print_warning(param, "Some data are unreferenced in keyfile");
578
  if (found_keys != full_text_keys)
579
    param->record_checksum=old_record_checksum-init_checksum;	/* Remove delete links */
580
  else
581
    param->record_checksum=0;
582
  DBUG_RETURN(result);
583
} /* chk_key */
584
585
586
static int chk_index_down(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
587
                     my_off_t page, uchar *buff, ha_rows *keys,
588
                     ha_checksum *key_checksum, uint level)
589
{
590
  char llbuff[22],llbuff2[22];
591
  DBUG_ENTER("chk_index_down");
592
593
  /* Key blocks must lay within the key file length entirely. */
594
  if (page + keyinfo->block_length > info->state->key_file_length)
595
  {
596
    /* purecov: begin tested */
597
    /* Give it a chance to fit in the real file size. */
598
    my_off_t max_length= my_seek(info->s->kfile, 0L, MY_SEEK_END,
599
                                 MYF(MY_THREADSAFE));
600
    mi_check_print_error(param, "Invalid key block position: %s  "
601
                         "key block size: %u  file_length: %s",
602
                         llstr(page, llbuff), keyinfo->block_length,
603
                         llstr(info->state->key_file_length, llbuff2));
604
    if (page + keyinfo->block_length > max_length)
605
      goto err;
606
    /* Fix the remebered key file length. */
607
    info->state->key_file_length= (max_length &
608
                                   ~ (my_off_t) (keyinfo->block_length - 1));
609
    /* purecov: end */
610
  }
611
612
  /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
613
  if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1))
614
  {
615
    /* purecov: begin tested */
616
    mi_check_print_error(param, "Mis-aligned key block: %s  "
617
                         "minimum key block length: %u",
618
                         llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
619
    goto err;
620
    /* purecov: end */
621
  }
622
623
  if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0))
624
  {
625
    mi_check_print_error(param,"Can't read key from filepos: %s",
626
        llstr(page,llbuff));
627
    goto err;
628
  }
629
  param->key_file_blocks+=keyinfo->block_length;
630
  if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level))
631
    goto err;
632
633
  DBUG_RETURN(0);
634
635
  /* purecov: begin tested */
636
err:
637
  DBUG_RETURN(1);
638
  /* purecov: end */
639
}
640
641
642
/*
643
  "Ignore NULLs" statistics collection method: process first index tuple.
644
645
  SYNOPSIS
646
    mi_collect_stats_nonulls_first()
647
      keyseg   IN     Array of key part descriptions
648
      notnull  INOUT  Array, notnull[i] = (number of {keypart1...keypart_i}
649
                                           tuples that don't contain NULLs)
650
      key      IN     Key values tuple
651
652
  DESCRIPTION
653
    Process the first index tuple - find out which prefix tuples don't
654
    contain NULLs, and update the array of notnull counters accordingly.
655
*/
656
657
static
658
void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg, ulonglong *notnull,
659
                                    uchar *key)
660
{
661
  uint first_null, kp;
662
  first_null= ha_find_null(keyseg, key) - keyseg;
663
  /*
664
    All prefix tuples that don't include keypart_{first_null} are not-null
665
    tuples (and all others aren't), increment counters for them.
666
  */
667
  for (kp= 0; kp < first_null; kp++)
668
    notnull[kp]++;
669
}
670
671
672
/*
673
  "Ignore NULLs" statistics collection method: process next index tuple.
674
675
  SYNOPSIS
676
    mi_collect_stats_nonulls_next()
677
      keyseg   IN     Array of key part descriptions
678
      notnull  INOUT  Array, notnull[i] = (number of {keypart1...keypart_i}
679
                                           tuples that don't contain NULLs)
680
      prev_key IN     Previous key values tuple
681
      last_key IN     Next key values tuple
682
683
  DESCRIPTION
684
    Process the next index tuple:
685
    1. Find out which prefix tuples of last_key don't contain NULLs, and
686
       update the array of notnull counters accordingly.
687
    2. Find the first keypart number where the prev_key and last_key tuples
688
       are different(A), or last_key has NULL value(B), and return it, so the 
689
       caller can count number of unique tuples for each key prefix. We don't 
690
       need (B) to be counted, and that is compensated back in 
691
       update_key_parts().
692
693
  RETURN
694
    1 + number of first keypart where values differ or last_key tuple has NULL
695
*/
696
697
static
698
int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, ulonglong *notnull,
699
                                  uchar *prev_key, uchar *last_key)
700
{
701
  uint diffs[2];
702
  uint first_null_seg, kp;
703
  HA_KEYSEG *seg;
704
705
  /* 
706
     Find the first keypart where values are different or either of them is
707
     NULL. We get results in diffs array:
708
     diffs[0]= 1 + number of first different keypart
709
     diffs[1]=offset: (last_key + diffs[1]) points to first value in
710
                      last_key that is NULL or different from corresponding
711
                      value in prev_key.
712
  */
713
  ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY, 
714
             SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
715
  seg= keyseg + diffs[0] - 1;
716
717
  /* Find first NULL in last_key */
718
  first_null_seg= ha_find_null(seg, last_key + diffs[1]) - keyseg;
719
  for (kp= 0; kp < first_null_seg; kp++)
720
    notnull[kp]++;
721
722
  /* 
723
    Return 1+ number of first key part where values differ. Don't care if
724
    these were NULLs and not .... We compensate for that in
725
    update_key_parts.
726
  */
727
  return diffs[0];
728
}
729
730
731
	/* Check if index is ok */
732
733
static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
734
		     my_off_t page, uchar *buff, ha_rows *keys,
735
		     ha_checksum *key_checksum, uint level)
736
{
737
  int flag;
738
  uint used_length,comp_flag,nod_flag,key_length=0;
739
  uchar key[HA_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*old_keypos,*endpos;
740
  my_off_t next_page,record;
741
  char llbuff[22];
742
  uint diff_pos[2];
743
  DBUG_ENTER("chk_index");
744
  DBUG_DUMP("buff",(uchar*) buff,mi_getint(buff));
745
746
  /* TODO: implement appropriate check for RTree keys */
747
  if (keyinfo->flag & HA_SPATIAL)
748
    DBUG_RETURN(0);
749
750
  if (!(temp_buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
751
  {
752
    mi_check_print_error(param,"Not enough memory for keyblock");
753
    DBUG_RETURN(-1);
754
  }
755
756
  if (keyinfo->flag & HA_NOSAME)
757
    comp_flag=SEARCH_FIND | SEARCH_UPDATE;	/* Not real duplicates */
758
  else
759
    comp_flag=SEARCH_SAME;			/* Keys in positionorder */
760
  nod_flag=mi_test_if_nod(buff);
761
  used_length=mi_getint(buff);
762
  keypos=buff+2+nod_flag;
763
  endpos=buff+used_length;
764
765
  param->keydata+=used_length; param->totaldata+=keyinfo->block_length;	/* INFO */
766
  param->key_blocks++;
767
  if (level > param->max_level)
768
    param->max_level=level;
769
770
  if (used_length > keyinfo->block_length)
771
  {
772
    mi_check_print_error(param,"Wrong pageinfo at page: %s",
773
			 llstr(page,llbuff));
774
    goto err;
775
  }
776
  for ( ;; )
777
  {
778
    if (*killed_ptr(param))
779
      goto err;
780
    memcpy((char*) info->lastkey,(char*) key,key_length);
781
    info->lastkey_length=key_length;
782
    if (nod_flag)
783
    {
784
      next_page=_mi_kpos(nod_flag,keypos);
785
      if (chk_index_down(param,info,keyinfo,next_page,
786
                         temp_buff,keys,key_checksum,level+1))
787
	goto err;
788
    }
789
    old_keypos=keypos;
790
    if (keypos >= endpos ||
791
	(key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
792
      break;
793
    DBUG_ASSERT(key_length <= sizeof(key));
794
    if (keypos > endpos)
795
    {
796
      mi_check_print_error(param,"Wrong key block length at page: %s",llstr(page,llbuff));
797
      goto err;
798
    }
799
    if ((*keys)++ &&
800
	(flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
801
			 comp_flag, diff_pos)) >=0)
802
    {
803
      DBUG_DUMP("old",(uchar*) info->lastkey, info->lastkey_length);
804
      DBUG_DUMP("new",(uchar*) key, key_length);
805
      DBUG_DUMP("new_in_page",(uchar*) old_keypos,(uint) (keypos-old_keypos));
806
807
      if (comp_flag & SEARCH_FIND && flag == 0)
808
	mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff));
809
      else
810
	mi_check_print_error(param,"Key in wrong position at page %s",llstr(page,llbuff));
811
      goto err;
812
    }
813
    if (param->testflag & T_STATISTICS)
814
    {
815
      if (*keys != 1L)				/* not first_key */
816
      {
817
        if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
818
          ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,
819
                     SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL,
820
                     diff_pos);
821
        else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
822
        {
823
          diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg, 
824
                                                  param->notnull_count,
825
                                                  info->lastkey, key);
826
        }
827
	param->unique_count[diff_pos[0]-1]++;
828
      }
829
      else
830
      {  
831
        if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
832
          mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
833
                                         key);
834
      }
835
    }
836
    (*key_checksum)+= mi_byte_checksum((uchar*) key,
837
				       key_length- info->s->rec_reflength);
838
    record= _mi_dpos(info,0,key+key_length);
839
    if (keyinfo->flag & HA_FULLTEXT) /* special handling for ft2 */
840
    {
841
      uint off;
842
      int  subkeys;
843
      get_key_full_length_rdonly(off, key);
844
      subkeys=ft_sintXkorr(key+off);
845
      if (subkeys < 0)
846
      {
847
        ha_rows tmp_keys=0;
848
        if (chk_index_down(param,info,&info->s->ft2_keyinfo,record,
849
                           temp_buff,&tmp_keys,key_checksum,1))
850
          goto err;
851
        if (tmp_keys + subkeys)
852
        {
853
          mi_check_print_error(param,
854
                               "Number of words in the 2nd level tree "
855
                               "does not match the number in the header. "
856
                               "Parent word in on the page %s, offset %u",
857
                               llstr(page,llbuff), (uint) (old_keypos-buff));
858
          goto err;
859
        }
860
        (*keys)+=tmp_keys-1;
861
        continue;
862
      }
863
      /* fall through */
864
    }
865
    if (record >= info->state->data_file_length)
866
    {
867
#ifndef DBUG_OFF
868
      char llbuff2[22], llbuff3[22];
869
#endif
870
      mi_check_print_error(param,"Found key at page %s that points to record outside datafile",llstr(page,llbuff));
871
      DBUG_PRINT("test",("page: %s  record: %s  filelength: %s",
872
			 llstr(page,llbuff),llstr(record,llbuff2),
873
			 llstr(info->state->data_file_length,llbuff3)));
874
      DBUG_DUMP("key",(uchar*) key,key_length);
875
      DBUG_DUMP("new_in_page",(uchar*) old_keypos,(uint) (keypos-old_keypos));
876
      goto err;
877
    }
878
    param->record_checksum+=(ha_checksum) record;
879
  }
880
  if (keypos != endpos)
881
  {
882
    mi_check_print_error(param,"Keyblock size at page %s is not correct.  Block length: %d  key length: %d",
883
                llstr(page,llbuff), used_length, (keypos - buff));
884
    goto err;
885
  }
886
  my_afree((uchar*) temp_buff);
887
  DBUG_RETURN(0);
888
 err:
889
  my_afree((uchar*) temp_buff);
890
  DBUG_RETURN(1);
891
} /* chk_index */
892
893
894
	/* Calculate a checksum of 1+2+3+4...N = N*(N+1)/2 without overflow */
895
896
static ha_checksum calc_checksum(ha_rows count)
897
{
898
  ulonglong sum,a,b;
899
  DBUG_ENTER("calc_checksum");
900
901
  sum=0;
902
  a=count; b=count+1;
903
  if (a & 1)
904
    b>>=1;
905
  else
906
    a>>=1;
907
  while (b)
908
  {
909
    if (b & 1)
910
      sum+=a;
911
    a<<=1; b>>=1;
912
  }
913
  DBUG_PRINT("exit",("sum: %lx",(ulong) sum));
914
  DBUG_RETURN((ha_checksum) sum);
915
} /* calc_checksum */
916
917
918
	/* Calc length of key in normal isam */
919
920
static uint isam_key_length(MI_INFO *info, register MI_KEYDEF *keyinfo)
921
{
922
  uint length;
923
  HA_KEYSEG *keyseg;
924
  DBUG_ENTER("isam_key_length");
925
926
  length= info->s->rec_reflength;
927
  for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++)
928
    length+= keyseg->length;
929
930
  DBUG_PRINT("exit",("length: %d",length));
931
  DBUG_RETURN(length);
932
} /* key_length */
933
934
935
	/* Check that record-link is ok */
936
937
int chk_data_link(MI_CHECK *param, MI_INFO *info,int extend)
938
{
939
  int	error,got_error,flag;
940
  uint	key, left_length= 0, b_type,field;
941
  ha_rows records, del_blocks;
942
  my_off_t used, empty, pos, splits, start_recpos= 0,
943
           del_length, link_used, start_block;
944
  uchar	*record= NULL, *to= NULL;
945
  char llbuff[22],llbuff2[22],llbuff3[22];
946
  ha_checksum intern_record_checksum;
947
  ha_checksum key_checksum[HA_MAX_POSSIBLE_KEY];
948
  my_bool static_row_size;
949
  MI_KEYDEF *keyinfo;
950
  MI_BLOCK_INFO block_info;
951
  DBUG_ENTER("chk_data_link");
952
953
  if (!(param->testflag & T_SILENT))
954
  {
955
    if (extend)
956
      puts("- check records and index references");
957
    else
958
      puts("- check record links");
959
  }
960
961
  if (!mi_alloc_rec_buff(info, -1, &record))
962
  {
963
    mi_check_print_error(param,"Not enough memory for record");
964
    DBUG_RETURN(-1);
965
  }
966
  records=del_blocks=0;
967
  used=link_used=splits=del_length=0;
968
  intern_record_checksum=param->glob_crc=0;
969
  got_error=error=0;
970
  empty=info->s->pack.header_length;
971
972
  /* Check how to calculate checksum of rows */
973
  static_row_size=1;
974
  if (info->s->data_file_type == COMPRESSED_RECORD)
975
  {
976
    for (field=0 ; field < info->s->base.fields ; field++)
977
    {
978
      if (info->s->rec[field].base_type == FIELD_BLOB ||
979
	  info->s->rec[field].base_type == FIELD_VARCHAR)
980
      {
981
	static_row_size=0;
982
	break;
983
      }
984
    }
985
  }
986
987
  pos=my_b_tell(&param->read_cache);
988
  bzero((char*) key_checksum, info->s->base.keys * sizeof(key_checksum[0]));
989
  while (pos < info->state->data_file_length)
990
  {
991
    if (*killed_ptr(param))
992
      goto err2;
993
    switch (info->s->data_file_type) {
994
    case STATIC_RECORD:
995
      if (my_b_read(&param->read_cache,(uchar*) record,
996
		    info->s->base.pack_reclength))
997
	goto err;
998
      start_recpos=pos;
999
      pos+=info->s->base.pack_reclength;
1000
      splits++;
1001
      if (*record == '\0')
1002
      {
1003
	del_blocks++;
1004
	del_length+=info->s->base.pack_reclength;
1005
	continue;					/* Record removed */
1006
      }
1007
      param->glob_crc+= mi_static_checksum(info,record);
1008
      used+=info->s->base.pack_reclength;
1009
      break;
1010
    case DYNAMIC_RECORD:
1011
      flag=block_info.second_read=0;
1012
      block_info.next_filepos=pos;
1013
      do
1014
      {
1015
	if (_mi_read_cache(&param->read_cache,(uchar*) block_info.header,
1016
			   (start_block=block_info.next_filepos),
1017
			   sizeof(block_info.header),
1018
			   (flag ? 0 : READING_NEXT) | READING_HEADER))
1019
	  goto err;
1020
	if (start_block & (MI_DYN_ALIGN_SIZE-1))
1021
	{
1022
	  mi_check_print_error(param,"Wrong aligned block at %s",
1023
			       llstr(start_block,llbuff));
1024
	  goto err2;
1025
	}
1026
	b_type=_mi_get_block_info(&block_info,-1,start_block);
1027
	if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
1028
		      BLOCK_FATAL_ERROR))
1029
	{
1030
	  if (b_type & BLOCK_SYNC_ERROR)
1031
	  {
1032
	    if (flag)
1033
	    {
1034
	      mi_check_print_error(param,"Unexpected byte: %d at link: %s",
1035
			  (int) block_info.header[0],
1036
			  llstr(start_block,llbuff));
1037
	      goto err2;
1038
	    }
1039
	    pos=block_info.filepos+block_info.block_len;
1040
	    goto next;
1041
	  }
1042
	  if (b_type & BLOCK_DELETED)
1043
	  {
1044
	    if (block_info.block_len < info->s->base.min_block_length)
1045
	    {
1046
	      mi_check_print_error(param,
1047
				   "Deleted block with impossible length %lu at %s",
1048
				   block_info.block_len,llstr(pos,llbuff));
1049
	      goto err2;
1050
	    }
1051
	    if ((block_info.next_filepos != HA_OFFSET_ERROR &&
1052
		 block_info.next_filepos >= info->state->data_file_length) ||
1053
		(block_info.prev_filepos != HA_OFFSET_ERROR &&
1054
		 block_info.prev_filepos >= info->state->data_file_length))
1055
	    {
1056
	      mi_check_print_error(param,"Delete link points outside datafile at %s",
1057
			  llstr(pos,llbuff));
1058
	      goto err2;
1059
	    }
1060
	    del_blocks++;
1061
	    del_length+=block_info.block_len;
1062
	    pos=block_info.filepos+block_info.block_len;
1063
	    splits++;
1064
	    goto next;
1065
	  }
1066
	  mi_check_print_error(param,"Wrong bytesec: %d-%d-%d at linkstart: %s",
1067
			       block_info.header[0],block_info.header[1],
1068
			       block_info.header[2],
1069
			       llstr(start_block,llbuff));
1070
	  goto err2;
1071
	}
1072
	if (info->state->data_file_length < block_info.filepos+
1073
	    block_info.block_len)
1074
	{
1075
	  mi_check_print_error(param,
1076
			       "Recordlink that points outside datafile at %s",
1077
			       llstr(pos,llbuff));
1078
	  got_error=1;
1079
	  break;
1080
	}
1081
	splits++;
1082
	if (!flag++)				/* First block */
1083
	{
1084
	  start_recpos=pos;
1085
	  pos=block_info.filepos+block_info.block_len;
1086
	  if (block_info.rec_len > (uint) info->s->base.max_pack_length)
1087
	  {
1088
	    mi_check_print_error(param,"Found too long record (%lu) at %s",
1089
				 (ulong) block_info.rec_len,
1090
				 llstr(start_recpos,llbuff));
1091
	    got_error=1;
1092
	    break;
1093
	  }
1094
	  if (info->s->base.blobs)
1095
	  {
1096
	    if (!(to= mi_alloc_rec_buff(info, block_info.rec_len,
1097
					&info->rec_buff)))
1098
	    {
1099
	      mi_check_print_error(param,
1100
				   "Not enough memory (%lu) for blob at %s",
1101
				   (ulong) block_info.rec_len,
1102
				   llstr(start_recpos,llbuff));
1103
	      got_error=1;
1104
	      break;
1105
	    }
1106
	  }
1107
	  else
1108
	    to= info->rec_buff;
1109
	  left_length=block_info.rec_len;
1110
	}
1111
	if (left_length < block_info.data_len)
1112
	{
1113
	  mi_check_print_error(param,"Found too long record (%lu) at %s",
1114
			       (ulong) block_info.data_len,
1115
			       llstr(start_recpos,llbuff));
1116
	  got_error=1;
1117
	  break;
1118
	}
1119
	if (_mi_read_cache(&param->read_cache,(uchar*) to,block_info.filepos,
1120
			   (uint) block_info.data_len,
1121
			   flag == 1 ? READING_NEXT : 0))
1122
	  goto err;
1123
	to+=block_info.data_len;
1124
	link_used+= block_info.filepos-start_block;
1125
	used+= block_info.filepos - start_block + block_info.data_len;
1126
	empty+=block_info.block_len-block_info.data_len;
1127
	left_length-=block_info.data_len;
1128
	if (left_length)
1129
	{
1130
	  if (b_type & BLOCK_LAST)
1131
	  {
1132
	    mi_check_print_error(param,
1133
				 "Wrong record length %s of %s at %s",
1134
				 llstr(block_info.rec_len-left_length,llbuff),
1135
				 llstr(block_info.rec_len, llbuff2),
1136
				 llstr(start_recpos,llbuff3));
1137
	    got_error=1;
1138
	    break;
1139
	  }
1140
	  if (info->state->data_file_length < block_info.next_filepos)
1141
	  {
1142
	    mi_check_print_error(param,
1143
				 "Found next-recordlink that points outside datafile at %s",
1144
				 llstr(block_info.filepos,llbuff));
1145
	    got_error=1;
1146
	    break;
1147
	  }
1148
	}
1149
      } while (left_length);
1150
      if (! got_error)
1151
      {
1152
	if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) ==
1153
	    MY_FILE_ERROR)
1154
	{
1155
	  mi_check_print_error(param,"Found wrong record at %s",
1156
			       llstr(start_recpos,llbuff));
1157
	  got_error=1;
1158
	}
1159
	else
1160
	{
1161
	  info->checksum=mi_checksum(info,record);
1162
	  if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE))
1163
	  {
1164
	    if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len,
1165
                              test(info->s->calc_checksum)))
1166
	    {
1167
	      mi_check_print_error(param,"Found wrong packed record at %s",
1168
			  llstr(start_recpos,llbuff));
1169
	      got_error=1;
1170
	    }
1171
	  }
1172
	  if (!got_error)
1173
	    param->glob_crc+= info->checksum;
1174
	}
1175
      }
1176
      else if (!flag)
1177
	pos=block_info.filepos+block_info.block_len;
1178
      break;
1179
    case COMPRESSED_RECORD:
1180
      if (_mi_read_cache(&param->read_cache,(uchar*) block_info.header, pos,
1181
			 info->s->pack.ref_length, READING_NEXT))
1182
	goto err;
1183
      start_recpos=pos;
1184
      splits++;
1185
      VOID(_mi_pack_get_block_info(info, &info->bit_buff, &block_info,
1186
                                   &info->rec_buff, -1, start_recpos));
1187
      pos=block_info.filepos+block_info.rec_len;
1188
      if (block_info.rec_len < (uint) info->s->min_pack_length ||
1189
	  block_info.rec_len > (uint) info->s->max_pack_length)
1190
      {
1191
	mi_check_print_error(param,
1192
			     "Found block with wrong recordlength: %d at %s",
1193
			     block_info.rec_len, llstr(start_recpos,llbuff));
1194
	got_error=1;
1195
	break;
1196
      }
1197
      if (_mi_read_cache(&param->read_cache,(uchar*) info->rec_buff,
1198
			block_info.filepos, block_info.rec_len, READING_NEXT))
1199
	goto err;
1200
      if (_mi_pack_rec_unpack(info, &info->bit_buff, record,
1201
                              info->rec_buff, block_info.rec_len))
1202
      {
1203
	mi_check_print_error(param,"Found wrong record at %s",
1204
			     llstr(start_recpos,llbuff));
1205
	got_error=1;
1206
      }
1207
      if (static_row_size)
1208
	param->glob_crc+= mi_static_checksum(info,record);
1209
      else
1210
	param->glob_crc+= mi_checksum(info,record);
1211
      link_used+= (block_info.filepos - start_recpos);
1212
      used+= (pos-start_recpos);
1213
    case BLOCK_RECORD:
1214
      assert(0);                                /* Impossible */
1215
    } /* switch */
1216
    if (! got_error)
1217
    {
1218
      intern_record_checksum+=(ha_checksum) start_recpos;
1219
      records++;
1220
      if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0)
1221
      {
1222
	printf("%s\r", llstr(records,llbuff)); VOID(fflush(stdout));
1223
      }
1224
1225
      /* Check if keys match the record */
1226
1227
      for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys;
1228
	   key++,keyinfo++)
1229
      {
1230
        if (mi_is_key_active(info->s->state.key_map, key))
1231
	{
1232
          if(!(keyinfo->flag & HA_FULLTEXT))
1233
	  {
1234
	    uint key_length=_mi_make_key(info,key,info->lastkey,record,
1235
					 start_recpos);
1236
	    if (extend)
1237
	    {
1238
	      /* We don't need to lock the key tree here as we don't allow
1239
		 concurrent threads when running myisamchk
1240
	      */
1241
              int search_result=
1242
#ifdef HAVE_RTREE_KEYS
1243
                (keyinfo->flag & HA_SPATIAL) ?
1244
                rtree_find_first(info, key, info->lastkey, key_length,
1245
                                 MBR_EQUAL | MBR_DATA) : 
1246
#endif
1247
                _mi_search(info,keyinfo,info->lastkey,key_length,
1248
                           SEARCH_SAME, info->s->state.key_root[key]);
1249
              if (search_result)
1250
              {
1251
                mi_check_print_error(param,"Record at: %10s  "
1252
                                     "Can't find key for index: %2d",
1253
                                     llstr(start_recpos,llbuff),key+1);
1254
                if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1255
                  goto err2;
1256
              }
1257
	    }
1258
	    else
1259
	      key_checksum[key]+=mi_byte_checksum((uchar*) info->lastkey,
1260
						  key_length);
1261
	  }
1262
	}
1263
      }
1264
    }
1265
    else
1266
    {
1267
      got_error=0;
1268
      if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1269
	goto err2;
1270
    }
1271
  next:;				/* Next record */
1272
  }
1273
  if (param->testflag & T_WRITE_LOOP)
1274
  {
1275
    VOID(fputs("          \r",stdout)); VOID(fflush(stdout));
1276
  }
1277
  if (records != info->state->records)
1278
  {
1279
    mi_check_print_error(param,"Record-count is not ok; is %-10s   Should be: %s",
1280
		llstr(records,llbuff), llstr(info->state->records,llbuff2));
1281
    error=1;
1282
  }
1283
  else if (param->record_checksum &&
1284
	   param->record_checksum != intern_record_checksum)
1285
  {
1286
    mi_check_print_error(param,
1287
			 "Keypointers and record positions doesn't match");
1288
    error=1;
1289
  }
1290
  else if (param->glob_crc != info->state->checksum &&
1291
	   (info->s->options &
1292
	    (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)))
1293
  {
1294
    mi_check_print_warning(param,
1295
			   "Record checksum is not the same as checksum stored in the index file\n");
1296
    error=1;
1297
  }
1298
  else if (!extend)
1299
  {
1300
    for (key=0 ; key < info->s->base.keys;  key++)
1301
    {
1302
      if (key_checksum[key] != param->key_crc[key] &&
1303
          !(info->s->keyinfo[key].flag & (HA_FULLTEXT | HA_SPATIAL)))
1304
      {
1305
	mi_check_print_error(param,"Checksum for key: %2d doesn't match checksum for records",
1306
		    key+1);
1307
	error=1;
1308
      }
1309
    }
1310
  }
1311
1312
  if (del_length != info->state->empty)
1313
  {
1314
    mi_check_print_warning(param,
1315
			   "Found %s deleted space.   Should be %s",
1316
			   llstr(del_length,llbuff2),
1317
			   llstr(info->state->empty,llbuff));
1318
  }
1319
  if (used+empty+del_length != info->state->data_file_length)
1320
  {
1321
    mi_check_print_warning(param,
1322
			   "Found %s record-data and %s unused data and %s deleted-data",
1323
			   llstr(used,llbuff),llstr(empty,llbuff2),
1324
			   llstr(del_length,llbuff3));
1325
    mi_check_print_warning(param,
1326
			   "Total %s, Should be: %s",
1327
			   llstr((used+empty+del_length),llbuff),
1328
			   llstr(info->state->data_file_length,llbuff2));
1329
  }
1330
  if (del_blocks != info->state->del)
1331
  {
1332
    mi_check_print_warning(param,
1333
			   "Found %10s deleted blocks       Should be: %s",
1334
			   llstr(del_blocks,llbuff),
1335
			   llstr(info->state->del,llbuff2));
1336
  }
1337
  if (splits != info->s->state.split)
1338
  {
1339
    mi_check_print_warning(param,
1340
			   "Found %10s parts                Should be: %s parts",
1341
			   llstr(splits,llbuff),
1342
			   llstr(info->s->state.split,llbuff2));
1343
  }
1344
  if (param->testflag & T_INFO)
1345
  {
1346
    if (param->warning_printed || param->error_printed)
1347
      puts("");
1348
    if (used != 0 && ! param->error_printed)
1349
    {
1350
      printf("Records:%18s    M.recordlength:%9lu   Packed:%14.0f%%\n",
1351
	     llstr(records,llbuff), (long)((used-link_used)/records),
1352
	     (info->s->base.blobs ? 0.0 :
1353
	      (ulonglong2double((ulonglong) info->s->base.reclength*records)-
1354
	       my_off_t2double(used))/
1355
	      ulonglong2double((ulonglong) info->s->base.reclength*records)*100.0));
1356
      printf("Recordspace used:%9.0f%%   Empty space:%12d%%  Blocks/Record: %6.2f\n",
1357
	     (ulonglong2double(used-link_used)/ulonglong2double(used-link_used+empty)*100.0),
1358
	     (!records ? 100 : (int) (ulonglong2double(del_length+empty)/
1359
				      my_off_t2double(used)*100.0)),
1360
	     ulonglong2double(splits - del_blocks) / records);
1361
    }
1362
    printf("Record blocks:%12s    Delete blocks:%10s\n",
1363
	   llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2));
1364
    printf("Record data:  %12s    Deleted data: %10s\n",
1365
	   llstr(used-link_used,llbuff),llstr(del_length,llbuff2));
1366
    printf("Lost space:   %12s    Linkdata:     %10s\n",
1367
	   llstr(empty,llbuff),llstr(link_used,llbuff2));
1368
  }
1369
  my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
1370
  DBUG_RETURN (error);
1371
 err:
1372
  mi_check_print_error(param,"got error: %d when reading datafile at record: %s",my_errno, llstr(records,llbuff));
1373
 err2:
1374
  my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
1375
  param->testflag|=T_RETRY_WITHOUT_QUICK;
1376
  DBUG_RETURN(1);
1377
} /* chk_data_link */
1378
1379
1380
/**
1381
  @brief Drop all indexes
1382
1383
  @param[in]    param           check parameters
1384
  @param[in]    info            MI_INFO handle
1385
  @param[in]    force           if to force drop all indexes
1386
1387
  @return       status
1388
    @retval     0               OK
1389
    @retval     != 0            Error
1390
1391
  @note
1392
    Once allocated, index blocks remain part of the key file forever.
1393
    When indexes are disabled, no block is freed. When enabling indexes,
1394
    no block is freed either. The new indexes are create from new
1395
    blocks. (Bug #4692)
1396
1397
    Before recreating formerly disabled indexes, the unused blocks
1398
    must be freed. There are two options to do this:
1399
    - Follow the tree of disabled indexes, add all blocks to the
1400
      deleted blocks chain. Would require a lot of random I/O.
1401
    - Drop all blocks by clearing all index root pointers and all
1402
      delete chain pointers and resetting key_file_length to the end
1403
      of the index file header. This requires to recreate all indexes,
1404
      even those that may still be intact.
1405
    The second method is probably faster in most cases.
1406
1407
    When disabling indexes, MySQL disables either all indexes or all
1408
    non-unique indexes. When MySQL [re-]enables disabled indexes
1409
    (T_CREATE_MISSING_KEYS), then we either have "lost" blocks in the
1410
    index file, or there are no non-unique indexes. In the latter case,
1411
    mi_repair*() would not be called as there would be no disabled
1412
    indexes.
1413
1414
    If there would be more unique indexes than disabled (non-unique)
1415
    indexes, we could do the first method. But this is not implemented
1416
    yet. By now we drop and recreate all indexes when repair is called.
1417
1418
    However, there is an exception. Sometimes MySQL disables non-unique
1419
    indexes when the table is empty (e.g. when copying a table in
1420
    mysql_alter_table()). When enabling the non-unique indexes, they
1421
    are still empty. So there is no index block that can be lost. This
1422
    optimization is implemented in this function.
1423
1424
    Note that in normal repair (T_CREATE_MISSING_KEYS not set) we
1425
    recreate all enabled indexes unconditonally. We do not change the
1426
    key_map. Otherwise we invert the key map temporarily (outside of
1427
    this function) and recreate the then "seemingly" enabled indexes.
1428
    When we cannot use the optimization, and drop all indexes, we
1429
    pretend that all indexes were disabled. By the inversion, we will
1430
    then recrate all indexes.
1431
*/
1432
1433
static int mi_drop_all_indexes(MI_CHECK *param, MI_INFO *info, my_bool force)
1434
{
1435
  MYISAM_SHARE *share= info->s;
1436
  MI_STATE_INFO *state= &share->state;
1437
  uint i;
1438
  int error;
1439
  DBUG_ENTER("mi_drop_all_indexes");
1440
1441
  /*
1442
    If any of the disabled indexes has a key block assigned, we must
1443
    drop and recreate all indexes to avoid losing index blocks.
1444
1445
    If we want to recreate disabled indexes only _and_ all of these
1446
    indexes are empty, we don't need to recreate the existing indexes.
1447
  */
1448
  if (!force && (param->testflag & T_CREATE_MISSING_KEYS))
1449
  {
1450
    DBUG_PRINT("repair", ("creating missing indexes"));
1451
    for (i= 0; i < share->base.keys; i++)
1452
    {
1453
      DBUG_PRINT("repair", ("index #: %u  key_root: 0x%lx  active: %d",
1454
                            i, (long) state->key_root[i],
1455
                            mi_is_key_active(state->key_map, i)));
1456
      if ((state->key_root[i] != HA_OFFSET_ERROR) &&
1457
          !mi_is_key_active(state->key_map, i))
1458
      {
1459
        /*
1460
          This index has at least one key block and it is disabled.
1461
          We would lose its block(s) if would just recreate it.
1462
          So we need to drop and recreate all indexes.
1463
        */
1464
        DBUG_PRINT("repair", ("nonempty and disabled: recreate all"));
1465
        break;
1466
      }
1467
    }
1468
    if (i >= share->base.keys)
1469
    {
1470
      /*
1471
        All of the disabled indexes are empty. We can just recreate them.
1472
        Flush dirty blocks of this index file from key cache and remove
1473
        all blocks of this index file from key cache.
1474
      */
1475
      DBUG_PRINT("repair", ("all disabled are empty: create missing"));
1476
      error= flush_key_blocks(share->key_cache, share->kfile,
1477
                              FLUSH_FORCE_WRITE);
1478
      goto end;
1479
    }
1480
    /*
1481
      We do now drop all indexes and declare them disabled. With the
1482
      T_CREATE_MISSING_KEYS flag, mi_repair*() will recreate all
1483
      disabled indexes and enable them.
1484
    */
1485
    mi_clear_all_keys_active(state->key_map);
1486
    DBUG_PRINT("repair", ("declared all indexes disabled"));
1487
  }
1488
1489
  /* Remove all key blocks of this index file from key cache. */
1490
  if ((error= flush_key_blocks(share->key_cache, share->kfile,
1491
                               FLUSH_IGNORE_CHANGED)))
1492
    goto end; /* purecov: inspected */
1493
1494
  /* Clear index root block pointers. */
1495
  for (i= 0; i < share->base.keys; i++)
1496
    state->key_root[i]= HA_OFFSET_ERROR;
1497
1498
  /* Clear the delete chains. */
1499
  for (i= 0; i < state->header.max_block_size_index; i++)
1500
    state->key_del[i]= HA_OFFSET_ERROR;
1501
1502
  /* Reset index file length to end of index file header. */
1503
  info->state->key_file_length= share->base.keystart;
1504
1505
  DBUG_PRINT("repair", ("dropped all indexes"));
1506
  /* error= 0; set by last (error= flush_key_bocks()). */
1507
1508
 end:
1509
  DBUG_RETURN(error);
1510
}
1511
1512
1513
	/* Recover old table by reading each record and writing all keys */
1514
	/* Save new datafile-name in temp_filename */
1515
1516
int mi_repair(MI_CHECK *param, register MI_INFO *info,
1517
	      char * name, int rep_quick)
1518
{
1519
  int error,got_error;
1520
  ha_rows start_records,new_header_length;
1521
  my_off_t del;
1522
  File new_file;
1523
  MYISAM_SHARE *share=info->s;
1524
  char llbuff[22],llbuff2[22];
1525
  SORT_INFO sort_info;
1526
  MI_SORT_PARAM sort_param;
1527
  DBUG_ENTER("mi_repair");
1528
1529
  bzero((char *)&sort_info, sizeof(sort_info));
1530
  bzero((char *)&sort_param, sizeof(sort_param));
1531
  start_records=info->state->records;
1532
  new_header_length=(param->testflag & T_UNPACK) ? 0L :
1533
    share->pack.header_length;
1534
  got_error=1;
1535
  new_file= -1;
1536
  sort_param.sort_info=&sort_info;
1537
1538
  if (!(param->testflag & T_SILENT))
1539
  {
1540
    printf("- recovering (with keycache) MyISAM-table '%s'\n",name);
1541
    printf("Data records: %s\n", llstr(info->state->records,llbuff));
1542
  }
1543
  param->testflag|=T_REP; /* for easy checking */
1544
1545
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
1546
    param->testflag|=T_CALC_CHECKSUM;
1547
1548
  if (!param->using_global_keycache)
1549
    VOID(init_key_cache(dflt_key_cache, param->key_cache_block_size,
1550
                        param->use_buffers, 0, 0));
1551
1552
  if (init_io_cache(&param->read_cache,info->dfile,
1553
		    (uint) param->read_buffer_length,
1554
		    READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
1555
  {
1556
    bzero(&info->rec_cache,sizeof(info->rec_cache));
1557
    goto err;
1558
  }
1559
  if (!rep_quick)
1560
    if (init_io_cache(&info->rec_cache,-1,(uint) param->write_buffer_length,
1561
		      WRITE_CACHE, new_header_length, 1,
1562
		      MYF(MY_WME | MY_WAIT_IF_FULL)))
1563
      goto err;
1564
  info->opt_flag|=WRITE_CACHE_USED;
1565
  if (!mi_alloc_rec_buff(info, -1, &sort_param.record) ||
1566
      !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
1567
  {
1568
    mi_check_print_error(param, "Not enough memory for extra record");
1569
    goto err;
1570
  }
1571
1572
  if (!rep_quick)
1573
  {
1574
    /* Get real path for data file */
1575
    if ((new_file=my_raid_create(fn_format(param->temp_filename,
1576
					   share->data_file_name, "",
1577
					   DATA_TMP_EXT, 2+4),
1578
				 0,param->tmpfile_createflag,
1579
				 share->base.raid_type,
1580
				 share->base.raid_chunks,
1581
				 share->base.raid_chunksize,
1582
				 MYF(0))) < 0)
1583
    {
1584
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
1585
			   param->temp_filename);
1586
      goto err;
1587
    }
1588
    if (new_header_length &&
1589
        filecopy(param,new_file,info->dfile,0L,new_header_length,
1590
		 "datafile-header"))
1591
      goto err;
1592
    info->s->state.dellink= HA_OFFSET_ERROR;
1593
    info->rec_cache.file=new_file;
1594
    if (param->testflag & T_UNPACK)
1595
    {
1596
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
1597
      mi_int2store(share->state.header.options,share->options);
1598
    }
1599
  }
1600
  sort_info.info=info;
1601
  sort_info.param = param;
1602
  sort_param.read_cache=param->read_cache;
1603
  sort_param.pos=sort_param.max_pos=share->pack.header_length;
1604
  sort_param.filepos=new_header_length;
1605
  param->read_cache.end_of_file=sort_info.filelength=
1606
    my_seek(info->dfile,0L,MY_SEEK_END,MYF(0));
1607
  sort_info.dupp=0;
1608
  sort_param.fix_datafile= (my_bool) (! rep_quick);
1609
  sort_param.master=1;
1610
  sort_info.max_records= ~(ha_rows) 0;
1611
1612
  set_data_file_type(&sort_info, share);
1613
  del=info->state->del;
1614
  info->state->records=info->state->del=share->state.split=0;
1615
  info->state->empty=0;
1616
  param->glob_crc=0;
1617
  if (param->testflag & T_CALC_CHECKSUM)
1618
    sort_param.calc_checksum= 1;
1619
1620
  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1621
1622
  /* This function always recreates all enabled indexes. */
1623
  if (param->testflag & T_CREATE_MISSING_KEYS)
1624
    mi_set_all_keys_active(share->state.key_map, share->base.keys);
1625
  mi_drop_all_indexes(param, info, TRUE);
1626
1627
  lock_memory(param);			/* Everything is alloced */
1628
1629
  /* Re-create all keys, which are set in key_map. */
1630
  while (!(error=sort_get_next_record(&sort_param)))
1631
  {
1632
    if (writekeys(&sort_param))
1633
    {
1634
      if (my_errno != HA_ERR_FOUND_DUPP_KEY)
1635
	goto err;
1636
      DBUG_DUMP("record",(uchar*) sort_param.record,share->base.pack_reclength);
1637
      mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s",
1638
			  info->errkey+1,
1639
			  llstr(sort_param.start_recpos,llbuff),
1640
			  llstr(info->dupp_key_pos,llbuff2));
1641
      if (param->testflag & T_VERBOSE)
1642
      {
1643
	VOID(_mi_make_key(info,(uint) info->errkey,info->lastkey,
1644
			  sort_param.record,0L));
1645
	_mi_print_key(stdout,share->keyinfo[info->errkey].seg,info->lastkey,
1646
		      USE_WHOLE_KEY);
1647
      }
1648
      sort_info.dupp++;
1649
      if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
1650
      {
1651
        param->testflag|=T_RETRY_WITHOUT_QUICK;
1652
	param->error_printed=1;
1653
	goto err;
1654
      }
1655
      continue;
1656
    }
1657
    if (sort_write_record(&sort_param))
1658
      goto err;
1659
  }
1660
  if (error > 0 || write_data_suffix(&sort_info, (my_bool)!rep_quick) ||
1661
      flush_io_cache(&info->rec_cache) || param->read_cache.error < 0)
1662
    goto err;
1663
1664
  if (param->testflag & T_WRITE_LOOP)
1665
  {
1666
    VOID(fputs("          \r",stdout)); VOID(fflush(stdout));
1667
  }
30 by Brian Aker
Large file and ftruncate() support
1668
  if (ftruncate(share->kfile, info->state->key_file_length))
1 by brian
clean slate
1669
  {
1670
    mi_check_print_warning(param,
1671
			   "Can't change size of indexfile, error: %d",
1672
			   my_errno);
1673
    goto err;
1674
  }
1675
1676
  if (rep_quick && del+sort_info.dupp != info->state->del)
1677
  {
1678
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
1679
    mi_check_print_error(param,"Run recovery again without -q");
1680
    got_error=1;
1681
    param->retry_repair=1;
1682
    param->testflag|=T_RETRY_WITHOUT_QUICK;
1683
    goto err;
1684
  }
1685
  if (param->testflag & T_SAFE_REPAIR)
1686
  {
1687
    /* Don't repair if we loosed more than one row */
1688
    if (info->state->records+1 < start_records)
1689
    {
1690
      info->state->records=start_records;
1691
      got_error=1;
1692
      goto err;
1693
    }
1694
  }
1695
1696
  if (!rep_quick)
1697
  {
1698
    my_close(info->dfile,MYF(0));
1699
    info->dfile=new_file;
1700
    info->state->data_file_length=sort_param.filepos;
1701
    share->state.version=(ulong) time((time_t*) 0);	/* Force reopen */
1702
  }
1703
  else
1704
  {
1705
    info->state->data_file_length=sort_param.max_pos;
1706
  }
1707
  if (param->testflag & T_CALC_CHECKSUM)
1708
    info->state->checksum=param->glob_crc;
1709
1710
  if (!(param->testflag & T_SILENT))
1711
  {
1712
    if (start_records != info->state->records)
1713
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
1714
    if (sort_info.dupp)
1715
      mi_check_print_warning(param,
1716
			     "%s records have been removed",
1717
			     llstr(sort_info.dupp,llbuff));
1718
  }
1719
1720
  got_error=0;
1721
  /* If invoked by external program that uses thr_lock */
1722
  if (&share->state.state != info->state)
1723
    memcpy( &share->state.state, info->state, sizeof(*info->state));
1724
1725
err:
1726
  if (!got_error)
1727
  {
1728
    /* Replace the actual file with the temporary file */
1729
    if (new_file >= 0)
1730
    {
1731
      my_close(new_file,MYF(0));
1732
      info->dfile=new_file= -1;
1733
      if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
1734
			    DATA_TMP_EXT, share->base.raid_chunks,
1735
			    (param->testflag & T_BACKUP_DATA ?
1736
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
1737
	  mi_open_datafile(info,share,-1))
1738
	got_error=1;
1739
    }
1740
  }
1741
  if (got_error)
1742
  {
1743
    if (! param->error_printed)
1744
      mi_check_print_error(param,"%d for record at pos %s",my_errno,
1745
		  llstr(sort_param.start_recpos,llbuff));
1746
    if (new_file >= 0)
1747
    {
1748
      VOID(my_close(new_file,MYF(0)));
1749
      VOID(my_raid_delete(param->temp_filename,info->s->base.raid_chunks,
1750
			  MYF(MY_WME)));
1751
      info->rec_cache.file=-1; /* don't flush data to new_file, it's closed */
1752
    }
1753
    mi_mark_crashed_on_repair(info);
1754
  }
1755
  my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff),
1756
                            MYF(MY_ALLOW_ZERO_PTR));
1757
  my_free(mi_get_rec_buff_ptr(info, sort_param.record),
1758
          MYF(MY_ALLOW_ZERO_PTR));
1759
  my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
1760
  VOID(end_io_cache(&param->read_cache));
1761
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
1762
  VOID(end_io_cache(&info->rec_cache));
1763
  got_error|=flush_blocks(param, share->key_cache, share->kfile);
1764
  if (!got_error && param->testflag & T_UNPACK)
1765
  {
1766
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
1767
    share->pack.header_length=0;
1768
    share->data_file_type=sort_info.new_data_file_type;
1769
  }
1770
  share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES |
1771
			  STATE_NOT_ANALYZED);
1772
  DBUG_RETURN(got_error);
1773
}
1774
1775
1776
/* Uppate keyfile when doing repair */
1777
1778
static int writekeys(MI_SORT_PARAM *sort_param)
1779
{
1780
  register uint i;
1781
  uchar    *key;
1782
  MI_INFO  *info=   sort_param->sort_info->info;
1783
  uchar    *buff=   sort_param->record;
1784
  my_off_t filepos= sort_param->filepos;
1785
  DBUG_ENTER("writekeys");
1786
1787
  key=info->lastkey+info->s->base.max_key_length;
1788
  for (i=0 ; i < info->s->base.keys ; i++)
1789
  {
1790
    if (mi_is_key_active(info->s->state.key_map, i))
1791
    {
1792
      if (info->s->keyinfo[i].flag & HA_FULLTEXT )
1793
      {
1794
        if (_mi_ft_add(info, i, key, buff, filepos))
1795
	  goto err;
1796
      }
1797
#ifdef HAVE_SPATIAL
1798
      else if (info->s->keyinfo[i].flag & HA_SPATIAL)
1799
      {
1800
	uint key_length=_mi_make_key(info,i,key,buff,filepos);
1801
	if (rtree_insert(info, i, key, key_length))
1802
	  goto err;
1803
      }
1804
#endif /*HAVE_SPATIAL*/
1805
      else
1806
      {
1807
	uint key_length=_mi_make_key(info,i,key,buff,filepos);
1808
	if (_mi_ck_write(info,i,key,key_length))
1809
	  goto err;
1810
      }
1811
    }
1812
  }
1813
  DBUG_RETURN(0);
1814
1815
 err:
1816
  if (my_errno == HA_ERR_FOUND_DUPP_KEY)
1817
  {
1818
    info->errkey=(int) i;			/* This key was found */
1819
    while ( i-- > 0 )
1820
    {
1821
      if (mi_is_key_active(info->s->state.key_map, i))
1822
      {
1823
	if (info->s->keyinfo[i].flag & HA_FULLTEXT)
1824
        {
1825
          if (_mi_ft_del(info,i, key,buff,filepos))
1826
	    break;
1827
        }
1828
        else
1829
	{
1830
	  uint key_length=_mi_make_key(info,i,key,buff,filepos);
1831
	  if (_mi_ck_delete(info,i,key,key_length))
1832
	    break;
1833
	}
1834
      }
1835
    }
1836
  }
1837
  /* Remove checksum that was added to glob_crc in sort_get_next_record */
1838
  if (sort_param->calc_checksum)
1839
    sort_param->sort_info->param->glob_crc-= info->checksum;
1840
  DBUG_PRINT("error",("errno: %d",my_errno));
1841
  DBUG_RETURN(-1);
1842
} /* writekeys */
1843
1844
1845
	/* Change all key-pointers that points to a records */
1846
1847
int movepoint(register MI_INFO *info, uchar *record, my_off_t oldpos,
1848
	      my_off_t newpos, uint prot_key)
1849
{
1850
  register uint i;
1851
  uchar *key;
1852
  uint key_length;
1853
  DBUG_ENTER("movepoint");
1854
1855
  key=info->lastkey+info->s->base.max_key_length;
1856
  for (i=0 ; i < info->s->base.keys; i++)
1857
  {
1858
    if (i != prot_key && mi_is_key_active(info->s->state.key_map, i))
1859
    {
1860
      key_length=_mi_make_key(info,i,key,record,oldpos);
1861
      if (info->s->keyinfo[i].flag & HA_NOSAME)
1862
      {					/* Change pointer direct */
1863
	uint nod_flag;
1864
	MI_KEYDEF *keyinfo;
1865
	keyinfo=info->s->keyinfo+i;
1866
	if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY,
1867
		       (uint) (SEARCH_SAME | SEARCH_SAVE_BUFF),
1868
		       info->s->state.key_root[i]))
1869
	  DBUG_RETURN(-1);
1870
	nod_flag=mi_test_if_nod(info->buff);
1871
	_mi_dpointer(info,info->int_keypos-nod_flag-
1872
		     info->s->rec_reflength,newpos);
1873
	if (_mi_write_keypage(info,keyinfo,info->last_keypage,
1874
                              DFLT_INIT_HITS,info->buff))
1875
	  DBUG_RETURN(-1);
1876
      }
1877
      else
1878
      {					/* Change old key to new */
1879
	if (_mi_ck_delete(info,i,key,key_length))
1880
	  DBUG_RETURN(-1);
1881
	key_length=_mi_make_key(info,i,key,record,newpos);
1882
	if (_mi_ck_write(info,i,key,key_length))
1883
	  DBUG_RETURN(-1);
1884
      }
1885
    }
1886
  }
1887
  DBUG_RETURN(0);
1888
} /* movepoint */
1889
1890
1891
	/* Tell system that we want all memory for our cache */
1892
1893
void lock_memory(MI_CHECK *param __attribute__((unused)))
1894
{
1895
#ifdef SUN_OS				/* Key-cacheing thrases on sun 4.1 */
1896
  if (param->opt_lock_memory)
1897
  {
1898
    int success = mlockall(MCL_CURRENT);	/* or plock(DATLOCK); */
1899
    if (geteuid() == 0 && success != 0)
1900
      mi_check_print_warning(param,
1901
			     "Failed to lock memory. errno %d",my_errno);
1902
  }
1903
#endif
1904
} /* lock_memory */
1905
1906
1907
	/* Flush all changed blocks to disk */
1908
1909
int flush_blocks(MI_CHECK *param, KEY_CACHE *key_cache, File file)
1910
{
1911
  if (flush_key_blocks(key_cache, file, FLUSH_RELEASE))
1912
  {
1913
    mi_check_print_error(param,"%d when trying to write bufferts",my_errno);
1914
    return(1);
1915
  }
1916
  if (!param->using_global_keycache)
1917
    end_key_cache(key_cache,1);
1918
  return 0;
1919
} /* flush_blocks */
1920
1921
1922
	/* Sort index for more efficent reads */
1923
1924
int mi_sort_index(MI_CHECK *param, register MI_INFO *info, char * name)
1925
{
1926
  register uint key;
1927
  register MI_KEYDEF *keyinfo;
1928
  File new_file;
1929
  my_off_t index_pos[HA_MAX_POSSIBLE_KEY];
1930
  uint r_locks,w_locks;
1931
  int old_lock;
1932
  MYISAM_SHARE *share=info->s;
1933
  MI_STATE_INFO old_state;
1934
  DBUG_ENTER("mi_sort_index");
1935
1936
  /* cannot sort index files with R-tree indexes */
1937
  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1938
       key++,keyinfo++)
1939
    if (keyinfo->key_alg == HA_KEY_ALG_RTREE)
1940
      DBUG_RETURN(0);
1941
1942
  if (!(param->testflag & T_SILENT))
1943
    printf("- Sorting index for MyISAM-table '%s'\n",name);
1944
1945
  /* Get real path for index file */
1946
  fn_format(param->temp_filename,name,"", MI_NAME_IEXT,2+4+32);
1947
  if ((new_file=my_create(fn_format(param->temp_filename,param->temp_filename,
1948
				    "", INDEX_TMP_EXT,2+4),
1949
			  0,param->tmpfile_createflag,MYF(0))) <= 0)
1950
  {
1951
    mi_check_print_error(param,"Can't create new tempfile: '%s'",
1952
			 param->temp_filename);
1953
    DBUG_RETURN(-1);
1954
  }
1955
  if (filecopy(param, new_file,share->kfile,0L,
1956
	       (ulong) share->base.keystart, "headerblock"))
1957
    goto err;
1958
1959
  param->new_file_pos=share->base.keystart;
1960
  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
1961
       key++,keyinfo++)
1962
  {
1963
    if (! mi_is_key_active(info->s->state.key_map, key))
1964
      continue;
1965
1966
    if (share->state.key_root[key] != HA_OFFSET_ERROR)
1967
    {
1968
      index_pos[key]=param->new_file_pos;	/* Write first block here */
1969
      if (sort_one_index(param,info,keyinfo,share->state.key_root[key],
1970
			 new_file))
1971
	goto err;
1972
    }
1973
    else
1974
      index_pos[key]= HA_OFFSET_ERROR;		/* No blocks */
1975
  }
1976
1977
  /* Flush key cache for this file if we are calling this outside myisamchk */
1978
  flush_key_blocks(share->key_cache,share->kfile, FLUSH_IGNORE_CHANGED);
1979
1980
  share->state.version=(ulong) time((time_t*) 0);
1981
  old_state= share->state;			/* save state if not stored */
1982
  r_locks=   share->r_locks;
1983
  w_locks=   share->w_locks;
1984
  old_lock=  info->lock_type;
1985
1986
	/* Put same locks as old file */
1987
  share->r_locks= share->w_locks= share->tot_locks= 0;
1988
  (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
1989
  VOID(my_close(share->kfile,MYF(MY_WME)));
1990
  share->kfile = -1;
1991
  VOID(my_close(new_file,MYF(MY_WME)));
1992
  if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT,0,
1993
			MYF(0)) ||
1994
      mi_open_keyfile(share))
1995
    goto err2;
1996
  info->lock_type= F_UNLCK;			/* Force mi_readinfo to lock */
1997
  _mi_readinfo(info,F_WRLCK,0);			/* Will lock the table */
1998
  info->lock_type=  old_lock;
1999
  share->r_locks=   r_locks;
2000
  share->w_locks=   w_locks;
2001
  share->tot_locks= r_locks+w_locks;
2002
  share->state=     old_state;			/* Restore old state */
2003
2004
  info->state->key_file_length=param->new_file_pos;
2005
  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2006
  for (key=0 ; key < info->s->base.keys ; key++)
2007
    info->s->state.key_root[key]=index_pos[key];
2008
  for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
2009
    info->s->state.key_del[key]=  HA_OFFSET_ERROR;
2010
2011
  info->s->state.changed&= ~STATE_NOT_SORTED_PAGES;
2012
  DBUG_RETURN(0);
2013
2014
err:
2015
  VOID(my_close(new_file,MYF(MY_WME)));
2016
err2:
2017
  VOID(my_delete(param->temp_filename,MYF(MY_WME)));
2018
  DBUG_RETURN(-1);
2019
} /* mi_sort_index */
2020
2021
2022
	 /* Sort records recursive using one index */
2023
2024
static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
2025
			  my_off_t pagepos, File new_file)
2026
{
2027
  uint length,nod_flag,used_length, key_length;
2028
  uchar *buff,*keypos,*endpos;
2029
  uchar key[HA_MAX_POSSIBLE_KEY_BUFF];
2030
  my_off_t new_page_pos,next_page;
2031
  char llbuff[22];
2032
  DBUG_ENTER("sort_one_index");
2033
2034
  /* cannot walk over R-tree indices */
2035
  DBUG_ASSERT(keyinfo->key_alg != HA_KEY_ALG_RTREE);
2036
  new_page_pos=param->new_file_pos;
2037
  param->new_file_pos+=keyinfo->block_length;
2038
2039
  if (!(buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
2040
  {
2041
    mi_check_print_error(param,"Not enough memory for key block");
2042
    DBUG_RETURN(-1);
2043
  }
2044
  if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0))
2045
  {
2046
    mi_check_print_error(param,"Can't read key block from filepos: %s",
2047
		llstr(pagepos,llbuff));
2048
    goto err;
2049
  }
2050
  if ((nod_flag=mi_test_if_nod(buff)) || keyinfo->flag & HA_FULLTEXT)
2051
  {
2052
    used_length=mi_getint(buff);
2053
    keypos=buff+2+nod_flag;
2054
    endpos=buff+used_length;
2055
    for ( ;; )
2056
    {
2057
      if (nod_flag)
2058
      {
2059
	next_page=_mi_kpos(nod_flag,keypos);
2060
	_mi_kpointer(info,keypos-nod_flag,param->new_file_pos); /* Save new pos */
2061
	if (sort_one_index(param,info,keyinfo,next_page,new_file))
2062
	{
2063
	  DBUG_PRINT("error",
2064
		     ("From page: %ld, keyoffset: %lu  used_length: %d",
2065
		      (ulong) pagepos, (ulong) (keypos - buff),
2066
		      (int) used_length));
2067
	  DBUG_DUMP("buff",(uchar*) buff,used_length);
2068
	  goto err;
2069
	}
2070
      }
2071
      if (keypos >= endpos ||
2072
	  (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
2073
	break;
2074
      DBUG_ASSERT(keypos <= endpos);
2075
      if (keyinfo->flag & HA_FULLTEXT)
2076
      {
2077
        uint off;
2078
        int  subkeys;
2079
        get_key_full_length_rdonly(off, key);
2080
        subkeys=ft_sintXkorr(key+off);
2081
        if (subkeys < 0)
2082
        {
2083
          next_page= _mi_dpos(info,0,key+key_length);
2084
          _mi_dpointer(info,keypos-nod_flag-info->s->rec_reflength,
2085
                       param->new_file_pos); /* Save new pos */
2086
          if (sort_one_index(param,info,&info->s->ft2_keyinfo,
2087
                             next_page,new_file))
2088
            goto err;
2089
        }
2090
      }
2091
    }
2092
  }
2093
2094
  /* Fill block with zero and write it to the new index file */
2095
  length=mi_getint(buff);
2096
  bzero((uchar*) buff+length,keyinfo->block_length-length);
2097
  if (my_pwrite(new_file,(uchar*) buff,(uint) keyinfo->block_length,
2098
		new_page_pos,MYF(MY_NABP | MY_WAIT_IF_FULL)))
2099
  {
2100
    mi_check_print_error(param,"Can't write indexblock, error: %d",my_errno);
2101
    goto err;
2102
  }
2103
  my_afree((uchar*) buff);
2104
  DBUG_RETURN(0);
2105
err:
2106
  my_afree((uchar*) buff);
2107
  DBUG_RETURN(1);
2108
} /* sort_one_index */
2109
2110
2111
	/*
2112
	  Let temporary file replace old file.
2113
	  This assumes that the new file was created in the same
2114
	  directory as given by realpath(filename).
2115
	  This will ensure that any symlinks that are used will still work.
2116
	  Copy stats from old file to new file, deletes orignal and
2117
	  changes new file name to old file name
2118
	*/
2119
2120
int change_to_newfile(const char * filename, const char * old_ext,
2121
		      const char * new_ext,
2122
		      uint raid_chunks __attribute__((unused)),
2123
		      myf MyFlags)
2124
{
2125
  char old_filename[FN_REFLEN],new_filename[FN_REFLEN];
2126
#ifdef USE_RAID
2127
  if (raid_chunks)
2128
    return my_raid_redel(fn_format(old_filename,filename,"",old_ext,2+4),
2129
			 fn_format(new_filename,filename,"",new_ext,2+4),
2130
			 raid_chunks,
2131
			 MYF(MY_WME | MY_LINK_WARNING | MyFlags));
2132
#endif
2133
  /* Get real path to filename */
2134
  (void) fn_format(old_filename,filename,"",old_ext,2+4+32);
2135
  return my_redel(old_filename,
2136
		  fn_format(new_filename,old_filename,"",new_ext,2+4),
2137
		  MYF(MY_WME | MY_LINK_WARNING | MyFlags));
2138
} /* change_to_newfile */
2139
2140
2141
	/* Locks a whole file */
2142
	/* Gives an error-message if file can't be locked */
2143
2144
int lock_file(MI_CHECK *param, File file, my_off_t start, int lock_type,
2145
	      const char *filetype, const char *filename)
2146
{
2147
  if (my_lock(file,lock_type,start,F_TO_EOF,
2148
	      param->testflag & T_WAIT_FOREVER ? MYF(MY_SEEK_NOT_DONE) :
2149
	      MYF(MY_SEEK_NOT_DONE |  MY_DONT_WAIT)))
2150
  {
2151
    mi_check_print_error(param," %d when locking %s '%s'",my_errno,filetype,filename);
2152
    param->error_printed=2;		/* Don't give that data is crashed */
2153
    return 1;
2154
  }
2155
  return 0;
2156
} /* lock_file */
2157
2158
2159
	/* Copy a block between two files */
2160
2161
int filecopy(MI_CHECK *param, File to,File from,my_off_t start,
2162
	     my_off_t length, const char *type)
2163
{
2164
  char tmp_buff[IO_SIZE],*buff;
2165
  ulong buff_length;
2166
  DBUG_ENTER("filecopy");
2167
2168
  buff_length=(ulong) min(param->write_buffer_length,length);
2169
  if (!(buff=my_malloc(buff_length,MYF(0))))
2170
  {
2171
    buff=tmp_buff; buff_length=IO_SIZE;
2172
  }
2173
2174
  VOID(my_seek(from,start,MY_SEEK_SET,MYF(0)));
2175
  while (length > buff_length)
2176
  {
2177
    if (my_read(from,(uchar*) buff,buff_length,MYF(MY_NABP)) ||
2178
	my_write(to,(uchar*) buff,buff_length,param->myf_rw))
2179
      goto err;
2180
    length-= buff_length;
2181
  }
2182
  if (my_read(from,(uchar*) buff,(uint) length,MYF(MY_NABP)) ||
2183
      my_write(to,(uchar*) buff,(uint) length,param->myf_rw))
2184
    goto err;
2185
  if (buff != tmp_buff)
2186
    my_free(buff,MYF(0));
2187
  DBUG_RETURN(0);
2188
err:
2189
  if (buff != tmp_buff)
2190
    my_free(buff,MYF(0));
2191
  mi_check_print_error(param,"Can't copy %s to tempfile, error %d",
2192
		       type,my_errno);
2193
  DBUG_RETURN(1);
2194
}
2195
2196
2197
/*
2198
  Repair table or given index using sorting
2199
2200
  SYNOPSIS
2201
    mi_repair_by_sort()
2202
    param		Repair parameters
2203
    info		MyISAM handler to repair
2204
    name		Name of table (for warnings)
2205
    rep_quick		set to <> 0 if we should not change data file
2206
2207
  RESULT
2208
    0	ok
2209
    <>0	Error
2210
*/
2211
2212
int mi_repair_by_sort(MI_CHECK *param, register MI_INFO *info,
2213
		      const char * name, int rep_quick)
2214
{
2215
  int got_error;
2216
  uint i;
2217
  ulong length;
2218
  ha_rows start_records;
2219
  my_off_t new_header_length,del;
2220
  File new_file;
2221
  MI_SORT_PARAM sort_param;
2222
  MYISAM_SHARE *share=info->s;
2223
  HA_KEYSEG *keyseg;
2224
  ulong   *rec_per_key_part;
2225
  char llbuff[22];
2226
  SORT_INFO sort_info;
2227
  ulonglong key_map= 0;
2228
  DBUG_ENTER("mi_repair_by_sort");
2229
2230
  start_records=info->state->records;
2231
  got_error=1;
2232
  new_file= -1;
2233
  new_header_length=(param->testflag & T_UNPACK) ? 0 :
2234
    share->pack.header_length;
2235
  if (!(param->testflag & T_SILENT))
2236
  {
2237
    printf("- recovering (with sort) MyISAM-table '%s'\n",name);
2238
    printf("Data records: %s\n", llstr(start_records,llbuff));
2239
  }
2240
  param->testflag|=T_REP; /* for easy checking */
2241
2242
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
2243
    param->testflag|=T_CALC_CHECKSUM;
2244
2245
  bzero((char*)&sort_info,sizeof(sort_info));
2246
  bzero((char *)&sort_param, sizeof(sort_param));
2247
  if (!(sort_info.key_block=
2248
	alloc_key_blocks(param,
2249
			 (uint) param->sort_key_blocks,
2250
			 share->base.max_key_block_length))
2251
      || init_io_cache(&param->read_cache,info->dfile,
2252
		       (uint) param->read_buffer_length,
2253
		       READ_CACHE,share->pack.header_length,1,MYF(MY_WME)) ||
2254
      (! rep_quick &&
2255
       init_io_cache(&info->rec_cache,info->dfile,
2256
		     (uint) param->write_buffer_length,
2257
		     WRITE_CACHE,new_header_length,1,
2258
		     MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
2259
    goto err;
2260
  sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
2261
  info->opt_flag|=WRITE_CACHE_USED;
2262
  info->rec_cache.file=info->dfile;		/* for sort_delete_record */
2263
2264
  if (!mi_alloc_rec_buff(info, -1, &sort_param.record) ||
2265
      !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
2266
  {
2267
    mi_check_print_error(param, "Not enough memory for extra record");
2268
    goto err;
2269
  }
2270
  if (!rep_quick)
2271
  {
2272
    /* Get real path for data file */
2273
    if ((new_file=my_raid_create(fn_format(param->temp_filename,
2274
					   share->data_file_name, "",
2275
					   DATA_TMP_EXT, 2+4),
2276
				 0,param->tmpfile_createflag,
2277
				 share->base.raid_type,
2278
				 share->base.raid_chunks,
2279
				 share->base.raid_chunksize,
2280
				 MYF(0))) < 0)
2281
    {
2282
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
2283
			   param->temp_filename);
2284
      goto err;
2285
    }
2286
    if (new_header_length &&
2287
        filecopy(param, new_file,info->dfile,0L,new_header_length,
2288
		 "datafile-header"))
2289
      goto err;
2290
    if (param->testflag & T_UNPACK)
2291
    {
2292
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
2293
      mi_int2store(share->state.header.options,share->options);
2294
    }
2295
    share->state.dellink= HA_OFFSET_ERROR;
2296
    info->rec_cache.file=new_file;
2297
  }
2298
2299
  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2300
2301
  /* Optionally drop indexes and optionally modify the key_map. */
2302
  mi_drop_all_indexes(param, info, FALSE);
2303
  key_map= share->state.key_map;
2304
  if (param->testflag & T_CREATE_MISSING_KEYS)
2305
  {
2306
    /* Invert the copied key_map to recreate all disabled indexes. */
2307
    key_map= ~key_map;
2308
  }
2309
2310
  sort_info.info=info;
2311
  sort_info.param = param;
2312
2313
  set_data_file_type(&sort_info, share);
2314
  sort_param.filepos=new_header_length;
2315
  sort_info.dupp=0;
2316
  sort_info.buff=0;
2317
  param->read_cache.end_of_file=sort_info.filelength=
2318
    my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));
2319
2320
  sort_param.wordlist=NULL;
2321
  init_alloc_root(&sort_param.wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
2322
2323
  if (share->data_file_type == DYNAMIC_RECORD)
2324
    length=max(share->base.min_pack_length+1,share->base.min_block_length);
2325
  else if (share->data_file_type == COMPRESSED_RECORD)
2326
    length=share->base.min_block_length;
2327
  else
2328
    length=share->base.pack_reclength;
2329
  sort_info.max_records=
2330
    ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
2331
     (ha_rows) (sort_info.filelength/length+1));
2332
  sort_param.key_cmp=sort_key_cmp;
2333
  sort_param.lock_in_memory=lock_memory;
2334
  sort_param.tmpdir=param->tmpdir;
2335
  sort_param.sort_info=&sort_info;
2336
  sort_param.fix_datafile= (my_bool) (! rep_quick);
2337
  sort_param.master =1;
2338
  
2339
  del=info->state->del;
2340
  param->glob_crc=0;
2341
  if (param->testflag & T_CALC_CHECKSUM)
2342
    sort_param.calc_checksum= 1;
2343
2344
  rec_per_key_part= param->rec_per_key_part;
2345
  for (sort_param.key=0 ; sort_param.key < share->base.keys ;
2346
       rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++)
2347
  {
2348
    sort_param.read_cache=param->read_cache;
2349
    sort_param.keyinfo=share->keyinfo+sort_param.key;
2350
    sort_param.seg=sort_param.keyinfo->seg;
2351
    /*
2352
      Skip this index if it is marked disabled in the copied
2353
      (and possibly inverted) key_map.
2354
    */
2355
    if (! mi_is_key_active(key_map, sort_param.key))
2356
    {
2357
      /* Remember old statistics for key */
2358
      memcpy((char*) rec_per_key_part,
2359
	     (char*) (share->state.rec_per_key_part +
2360
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
2361
	     sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part));
2362
      DBUG_PRINT("repair", ("skipping seemingly disabled index #: %u",
2363
                            sort_param.key));
2364
      continue;
2365
    }
2366
2367
    if ((!(param->testflag & T_SILENT)))
2368
      printf ("- Fixing index %d\n",sort_param.key+1);
2369
    sort_param.max_pos=sort_param.pos=share->pack.header_length;
2370
    keyseg=sort_param.seg;
2371
    bzero((char*) sort_param.unique,sizeof(sort_param.unique));
2372
    sort_param.key_length=share->rec_reflength;
2373
    for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++)
2374
    {
2375
      sort_param.key_length+=keyseg[i].length;
2376
      if (keyseg[i].flag & HA_SPACE_PACK)
2377
	sort_param.key_length+=get_pack_length(keyseg[i].length);
2378
      if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
2379
	sort_param.key_length+=2 + test(keyseg[i].length >= 127);
2380
      if (keyseg[i].flag & HA_NULL_PART)
2381
	sort_param.key_length++;
2382
    }
2383
    info->state->records=info->state->del=share->state.split=0;
2384
    info->state->empty=0;
2385
2386
    if (sort_param.keyinfo->flag & HA_FULLTEXT)
2387
    {
2388
      uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
2389
                                    sort_param.keyinfo->seg->charset->mbmaxlen;
2390
      sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
2391
      /*
2392
        fulltext indexes may have much more entries than the
2393
        number of rows in the table. We estimate the number here.
2394
2395
        Note, built-in parser is always nr. 0 - see ftparser_call_initializer()
2396
      */
2397
      if (sort_param.keyinfo->ftparser_nr == 0)
2398
      {
2399
        /*
2400
          for built-in parser the number of generated index entries
2401
          cannot be larger than the size of the data file divided
2402
          by the minimal word's length
2403
        */
2404
        sort_info.max_records=
2405
          (ha_rows) (sort_info.filelength/ft_min_word_len+1);
2406
      }
2407
      else
2408
      {
2409
        /*
2410
          for external plugin parser we cannot tell anything at all :(
2411
          so, we'll use all the sort memory and start from ~10 buffpeks.
2412
          (see _create_index_by_sort)
2413
        */
2414
        sort_info.max_records=
2415
          10*param->sort_buffer_length/sort_param.key_length;
2416
      }
2417
2418
      sort_param.key_read=sort_ft_key_read;
2419
      sort_param.key_write=sort_ft_key_write;
2420
    }
2421
    else
2422
    {
2423
      sort_param.key_read=sort_key_read;
2424
      sort_param.key_write=sort_key_write;
2425
    }
2426
2427
    if (_create_index_by_sort(&sort_param,
2428
			      (my_bool) (!(param->testflag & T_VERBOSE)),
2429
			      (uint) param->sort_buffer_length))
2430
    {
2431
      param->retry_repair=1;
2432
      goto err;
2433
    }
2434
    /* No need to calculate checksum again. */
2435
    sort_param.calc_checksum= 0;
2436
    free_root(&sort_param.wordroot, MYF(0));
2437
2438
    /* Set for next loop */
2439
    sort_info.max_records= (ha_rows) info->state->records;
2440
2441
    if (param->testflag & T_STATISTICS)
2442
      update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
2443
                       param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
2444
                       sort_param.notnull: NULL,
2445
                       (ulonglong) info->state->records);
2446
    /* Enable this index in the permanent (not the copied) key_map. */
2447
    mi_set_key_active(share->state.key_map, sort_param.key);
2448
    DBUG_PRINT("repair", ("set enabled index #: %u", sort_param.key));
2449
2450
    if (sort_param.fix_datafile)
2451
    {
2452
      param->read_cache.end_of_file=sort_param.filepos;
2453
      if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
2454
	goto err;
2455
      if (param->testflag & T_SAFE_REPAIR)
2456
      {
2457
	/* Don't repair if we loosed more than one row */
2458
	if (info->state->records+1 < start_records)
2459
	{
2460
	  info->state->records=start_records;
2461
	  goto err;
2462
	}
2463
      }
2464
      share->state.state.data_file_length = info->state->data_file_length=
2465
	sort_param.filepos;
2466
      /* Only whole records */
2467
      share->state.version=(ulong) time((time_t*) 0);
2468
      my_close(info->dfile,MYF(0));
2469
      info->dfile=new_file;
2470
      share->data_file_type=sort_info.new_data_file_type;
2471
      share->pack.header_length=(ulong) new_header_length;
2472
      sort_param.fix_datafile=0;
2473
    }
2474
    else
2475
      info->state->data_file_length=sort_param.max_pos;
2476
2477
    param->read_cache.file=info->dfile;		/* re-init read cache */
2478
    reinit_io_cache(&param->read_cache,READ_CACHE,share->pack.header_length,
2479
                    1,1);
2480
  }
2481
2482
  if (param->testflag & T_WRITE_LOOP)
2483
  {
2484
    VOID(fputs("          \r",stdout)); VOID(fflush(stdout));
2485
  }
2486
2487
  if (rep_quick && del+sort_info.dupp != info->state->del)
2488
  {
2489
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
2490
    mi_check_print_error(param,"Run recovery again without -q");
2491
    got_error=1;
2492
    param->retry_repair=1;
2493
    param->testflag|=T_RETRY_WITHOUT_QUICK;
2494
    goto err;
2495
  }
2496
2497
  if (rep_quick & T_FORCE_UNIQUENESS)
2498
  {
2499
    my_off_t skr=info->state->data_file_length+
2500
      (share->options & HA_OPTION_COMPRESS_RECORD ?
2501
       MEMMAP_EXTRA_MARGIN : 0);
2502
#ifdef USE_RELOC
2503
    if (share->data_file_type == STATIC_RECORD &&
2504
	skr < share->base.reloc*share->base.min_pack_length)
2505
      skr=share->base.reloc*share->base.min_pack_length;
2506
#endif
2507
    if (skr != sort_info.filelength && !info->s->base.raid_type)
30 by Brian Aker
Large file and ftruncate() support
2508
      if (ftruncate(info->dfile, skr))
1 by brian
clean slate
2509
	mi_check_print_warning(param,
2510
			       "Can't change size of datafile,  error: %d",
2511
			       my_errno);
2512
  }
2513
  if (param->testflag & T_CALC_CHECKSUM)
2514
    info->state->checksum=param->glob_crc;
2515
30 by Brian Aker
Large file and ftruncate() support
2516
  if (ftruncate(share->kfile, info->state->key_file_length))
1 by brian
clean slate
2517
    mi_check_print_warning(param,
2518
			   "Can't change size of indexfile, error: %d",
2519
			   my_errno);
2520
2521
  if (!(param->testflag & T_SILENT))
2522
  {
2523
    if (start_records != info->state->records)
2524
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
2525
    if (sort_info.dupp)
2526
      mi_check_print_warning(param,
2527
			     "%s records have been removed",
2528
			     llstr(sort_info.dupp,llbuff));
2529
  }
2530
  got_error=0;
2531
2532
  if (&share->state.state != info->state)
2533
    memcpy( &share->state.state, info->state, sizeof(*info->state));
2534
2535
err:
2536
  got_error|= flush_blocks(param, share->key_cache, share->kfile);
2537
  VOID(end_io_cache(&info->rec_cache));
2538
  if (!got_error)
2539
  {
2540
    /* Replace the actual file with the temporary file */
2541
    if (new_file >= 0)
2542
    {
2543
      my_close(new_file,MYF(0));
2544
      info->dfile=new_file= -1;
2545
      if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
2546
			    DATA_TMP_EXT, share->base.raid_chunks,
2547
			    (param->testflag & T_BACKUP_DATA ?
2548
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
2549
	  mi_open_datafile(info,share,-1))
2550
	got_error=1;
2551
    }
2552
  }
2553
  if (got_error)
2554
  {
2555
    if (! param->error_printed)
2556
      mi_check_print_error(param,"%d when fixing table",my_errno);
2557
    if (new_file >= 0)
2558
    {
2559
      VOID(my_close(new_file,MYF(0)));
2560
      VOID(my_raid_delete(param->temp_filename,share->base.raid_chunks,
2561
			  MYF(MY_WME)));
2562
      if (info->dfile == new_file)
2563
	info->dfile= -1;
2564
    }
2565
    mi_mark_crashed_on_repair(info);
2566
  }
2567
  else if (key_map == share->state.key_map)
2568
    share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
2569
  share->state.changed|=STATE_NOT_SORTED_PAGES;
2570
2571
  my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff),
2572
                            MYF(MY_ALLOW_ZERO_PTR));
2573
  my_free(mi_get_rec_buff_ptr(info, sort_param.record),
2574
          MYF(MY_ALLOW_ZERO_PTR));
2575
  my_free((uchar*) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR));
2576
  my_free((uchar*) sort_info.ft_buf, MYF(MY_ALLOW_ZERO_PTR));
2577
  my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
2578
  VOID(end_io_cache(&param->read_cache));
2579
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
2580
  if (!got_error && (param->testflag & T_UNPACK))
2581
  {
2582
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
2583
    share->pack.header_length=0;
2584
  }
2585
  DBUG_RETURN(got_error);
2586
}
2587
2588
/*
2589
  Threaded repair of table using sorting
2590
2591
  SYNOPSIS
2592
    mi_repair_parallel()
2593
    param		Repair parameters
2594
    info		MyISAM handler to repair
2595
    name		Name of table (for warnings)
2596
    rep_quick		set to <> 0 if we should not change data file
2597
2598
  DESCRIPTION
2599
    Same as mi_repair_by_sort but do it multithreaded
2600
    Each key is handled by a separate thread.
2601
    TODO: make a number of threads a parameter
2602
2603
    In parallel repair we use one thread per index. There are two modes:
2604
2605
    Quick
2606
2607
      Only the indexes are rebuilt. All threads share a read buffer.
2608
      Every thread that needs fresh data in the buffer enters the shared
2609
      cache lock. The last thread joining the lock reads the buffer from
2610
      the data file and wakes all other threads.
2611
2612
    Non-quick
2613
2614
      The data file is rebuilt and all indexes are rebuilt to point to
2615
      the new record positions. One thread is the master thread. It
2616
      reads from the old data file and writes to the new data file. It
2617
      also creates one of the indexes. The other threads read from a
2618
      buffer which is filled by the master. If they need fresh data,
2619
      they enter the shared cache lock. If the masters write buffer is
2620
      full, it flushes it to the new data file and enters the shared
2621
      cache lock too. When all threads joined in the lock, the master
2622
      copies its write buffer to the read buffer for the other threads
2623
      and wakes them.
2624
2625
  RESULT
2626
    0	ok
2627
    <>0	Error
2628
*/
2629
2630
int mi_repair_parallel(MI_CHECK *param, register MI_INFO *info,
2631
			const char * name, int rep_quick)
2632
{
2633
#ifndef THREAD
2634
  return mi_repair_by_sort(param, info, name, rep_quick);
2635
#else
2636
  int got_error;
2637
  uint i,key, total_key_length, istep;
2638
  ulong rec_length;
2639
  ha_rows start_records;
2640
  my_off_t new_header_length,del;
2641
  File new_file;
2642
  MI_SORT_PARAM *sort_param=0;
2643
  MYISAM_SHARE *share=info->s;
2644
  ulong   *rec_per_key_part;
2645
  HA_KEYSEG *keyseg;
2646
  char llbuff[22];
2647
  IO_CACHE new_data_cache; /* For non-quick repair. */
2648
  IO_CACHE_SHARE io_share;
2649
  SORT_INFO sort_info;
2650
  ulonglong key_map= 0;
2651
  pthread_attr_t thr_attr;
2652
  ulong max_pack_reclength;
2653
  DBUG_ENTER("mi_repair_parallel");
2654
2655
  start_records=info->state->records;
2656
  got_error=1;
2657
  new_file= -1;
2658
  new_header_length=(param->testflag & T_UNPACK) ? 0 :
2659
    share->pack.header_length;
2660
  if (!(param->testflag & T_SILENT))
2661
  {
2662
    printf("- parallel recovering (with sort) MyISAM-table '%s'\n",name);
2663
    printf("Data records: %s\n", llstr(start_records,llbuff));
2664
  }
2665
  param->testflag|=T_REP; /* for easy checking */
2666
2667
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
2668
    param->testflag|=T_CALC_CHECKSUM;
2669
2670
  /*
2671
    Quick repair (not touching data file, rebuilding indexes):
2672
    {
2673
      Read  cache is (MI_CHECK *param)->read_cache using info->dfile.
2674
    }
2675
2676
    Non-quick repair (rebuilding data file and indexes):
2677
    {
2678
      Master thread:
2679
2680
        Read  cache is (MI_CHECK *param)->read_cache using info->dfile.
2681
        Write cache is (MI_INFO   *info)->rec_cache  using new_file.
2682
2683
      Slave threads:
2684
2685
        Read  cache is new_data_cache synced to master rec_cache.
2686
2687
      The final assignment of the filedescriptor for rec_cache is done
2688
      after the cache creation.
2689
2690
      Don't check file size on new_data_cache, as the resulting file size
2691
      is not known yet.
2692
2693
      As rec_cache and new_data_cache are synced, write_buffer_length is
2694
      used for the read cache 'new_data_cache'. Both start at the same
2695
      position 'new_header_length'.
2696
    }
2697
  */
2698
  DBUG_PRINT("info", ("is quick repair: %d", rep_quick));
2699
  bzero((char*)&sort_info,sizeof(sort_info));
2700
  /* Initialize pthread structures before goto err. */
2701
  pthread_mutex_init(&sort_info.mutex, MY_MUTEX_INIT_FAST);
2702
  pthread_cond_init(&sort_info.cond, 0);
2703
2704
  if (!(sort_info.key_block=
2705
	alloc_key_blocks(param, (uint) param->sort_key_blocks,
2706
			 share->base.max_key_block_length)) ||
2707
      init_io_cache(&param->read_cache, info->dfile,
2708
                    (uint) param->read_buffer_length,
2709
                    READ_CACHE, share->pack.header_length, 1, MYF(MY_WME)) ||
2710
      (!rep_quick &&
2711
       (init_io_cache(&info->rec_cache, info->dfile,
2712
                      (uint) param->write_buffer_length,
2713
                      WRITE_CACHE, new_header_length, 1,
2714
                      MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw) ||
2715
        init_io_cache(&new_data_cache, -1,
2716
                      (uint) param->write_buffer_length,
2717
                      READ_CACHE, new_header_length, 1,
2718
                      MYF(MY_WME | MY_DONT_CHECK_FILESIZE)))))
2719
    goto err;
2720
  sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
2721
  info->opt_flag|=WRITE_CACHE_USED;
2722
  info->rec_cache.file=info->dfile;         /* for sort_delete_record */
2723
2724
  if (!rep_quick)
2725
  {
2726
    /* Get real path for data file */
2727
    if ((new_file=my_raid_create(fn_format(param->temp_filename,
2728
					   share->data_file_name, "",
2729
					   DATA_TMP_EXT,
2730
					   2+4),
2731
				 0,param->tmpfile_createflag,
2732
				 share->base.raid_type,
2733
				 share->base.raid_chunks,
2734
				 share->base.raid_chunksize,
2735
				 MYF(0))) < 0)
2736
    {
2737
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
2738
			   param->temp_filename);
2739
      goto err;
2740
    }
2741
    if (new_header_length &&
2742
        filecopy(param, new_file,info->dfile,0L,new_header_length,
2743
		 "datafile-header"))
2744
      goto err;
2745
    if (param->testflag & T_UNPACK)
2746
    {
2747
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
2748
      mi_int2store(share->state.header.options,share->options);
2749
    }
2750
    share->state.dellink= HA_OFFSET_ERROR;
2751
    info->rec_cache.file=new_file;
2752
  }
2753
2754
  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2755
2756
  /* Optionally drop indexes and optionally modify the key_map. */
2757
  mi_drop_all_indexes(param, info, FALSE);
2758
  key_map= share->state.key_map;
2759
  if (param->testflag & T_CREATE_MISSING_KEYS)
2760
  {
2761
    /* Invert the copied key_map to recreate all disabled indexes. */
2762
    key_map= ~key_map;
2763
  }
2764
2765
  sort_info.info=info;
2766
  sort_info.param = param;
2767
2768
  set_data_file_type(&sort_info, share);
2769
  sort_info.dupp=0;
2770
  sort_info.buff=0;
2771
  param->read_cache.end_of_file=sort_info.filelength=
2772
    my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));
2773
2774
  if (share->data_file_type == DYNAMIC_RECORD)
2775
    rec_length=max(share->base.min_pack_length+1,share->base.min_block_length);
2776
  else if (share->data_file_type == COMPRESSED_RECORD)
2777
    rec_length=share->base.min_block_length;
2778
  else
2779
    rec_length=share->base.pack_reclength;
2780
  /*
2781
    +1 below is required hack for parallel repair mode.
2782
    The info->state->records value, that is compared later
2783
    to sort_info.max_records and cannot exceed it, is
2784
    increased in sort_key_write. In mi_repair_by_sort, sort_key_write
2785
    is called after sort_key_read, where the comparison is performed,
2786
    but in parallel mode master thread can call sort_key_write
2787
    before some other repair thread calls sort_key_read.
2788
    Furthermore I'm not even sure +1 would be enough.
2789
    May be sort_info.max_records shold be always set to max value in
2790
    parallel mode.
2791
  */
2792
  sort_info.max_records=
2793
    ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records + 1:
2794
     (ha_rows) (sort_info.filelength/rec_length+1));
2795
2796
  del=info->state->del;
2797
  param->glob_crc=0;
2798
  /* for compressed tables */
2799
  max_pack_reclength= share->base.pack_reclength;
2800
  if (share->options & HA_OPTION_COMPRESS_RECORD)
2801
    set_if_bigger(max_pack_reclength, share->max_pack_length);
2802
  if (!(sort_param=(MI_SORT_PARAM *)
2803
        my_malloc((uint) share->base.keys *
2804
		  (sizeof(MI_SORT_PARAM) + max_pack_reclength),
2805
		  MYF(MY_ZEROFILL))))
2806
  {
2807
    mi_check_print_error(param,"Not enough memory for key!");
2808
    goto err;
2809
  }
2810
  total_key_length=0;
2811
  rec_per_key_part= param->rec_per_key_part;
2812
  info->state->records=info->state->del=share->state.split=0;
2813
  info->state->empty=0;
2814
2815
  for (i=key=0, istep=1 ; key < share->base.keys ;
2816
       rec_per_key_part+=sort_param[i].keyinfo->keysegs, i+=istep, key++)
2817
  {
2818
    sort_param[i].key=key;
2819
    sort_param[i].keyinfo=share->keyinfo+key;
2820
    sort_param[i].seg=sort_param[i].keyinfo->seg;
2821
    /*
2822
      Skip this index if it is marked disabled in the copied
2823
      (and possibly inverted) key_map.
2824
    */
2825
    if (! mi_is_key_active(key_map, key))
2826
    {
2827
      /* Remember old statistics for key */
2828
      memcpy((char*) rec_per_key_part,
2829
	     (char*) (share->state.rec_per_key_part+
2830
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
2831
	     sort_param[i].keyinfo->keysegs*sizeof(*rec_per_key_part));
2832
      istep=0;
2833
      continue;
2834
    }
2835
    istep=1;
2836
    if ((!(param->testflag & T_SILENT)))
2837
      printf ("- Fixing index %d\n",key+1);
2838
    if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
2839
    {
2840
      sort_param[i].key_read=sort_ft_key_read;
2841
      sort_param[i].key_write=sort_ft_key_write;
2842
    }
2843
    else
2844
    {
2845
      sort_param[i].key_read=sort_key_read;
2846
      sort_param[i].key_write=sort_key_write;
2847
    }
2848
    sort_param[i].key_cmp=sort_key_cmp;
2849
    sort_param[i].lock_in_memory=lock_memory;
2850
    sort_param[i].tmpdir=param->tmpdir;
2851
    sort_param[i].sort_info=&sort_info;
2852
    sort_param[i].master=0;
2853
    sort_param[i].fix_datafile=0;
2854
    sort_param[i].calc_checksum= 0;
2855
2856
    sort_param[i].filepos=new_header_length;
2857
    sort_param[i].max_pos=sort_param[i].pos=share->pack.header_length;
2858
2859
    sort_param[i].record= (((uchar *)(sort_param+share->base.keys))+
2860
			   (max_pack_reclength * i));
2861
    if (!mi_alloc_rec_buff(info, -1, &sort_param[i].rec_buff))
2862
    {
2863
      mi_check_print_error(param,"Not enough memory!");
2864
      goto err;
2865
    }
2866
2867
    sort_param[i].key_length=share->rec_reflength;
2868
    for (keyseg=sort_param[i].seg; keyseg->type != HA_KEYTYPE_END;
2869
	 keyseg++)
2870
    {
2871
      sort_param[i].key_length+=keyseg->length;
2872
      if (keyseg->flag & HA_SPACE_PACK)
2873
        sort_param[i].key_length+=get_pack_length(keyseg->length);
2874
      if (keyseg->flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
2875
        sort_param[i].key_length+=2 + test(keyseg->length >= 127);
2876
      if (keyseg->flag & HA_NULL_PART)
2877
        sort_param[i].key_length++;
2878
    }
2879
    total_key_length+=sort_param[i].key_length;
2880
2881
    if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
2882
    {
2883
      uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
2884
                                    sort_param[i].keyinfo->seg->charset->mbmaxlen;
2885
      sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
2886
      init_alloc_root(&sort_param[i].wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
2887
    }
2888
  }
2889
  sort_info.total_keys=i;
2890
  sort_param[0].master= 1;
2891
  sort_param[0].fix_datafile= (my_bool)(! rep_quick);
2892
  sort_param[0].calc_checksum= test(param->testflag & T_CALC_CHECKSUM);
2893
2894
  sort_info.got_error=0;
2895
  pthread_mutex_lock(&sort_info.mutex);
2896
2897
  /*
2898
    Initialize the I/O cache share for use with the read caches and, in
2899
    case of non-quick repair, the write cache. When all threads join on
2900
    the cache lock, the writer copies the write cache contents to the
2901
    read caches.
2902
  */
2903
  if (i > 1)
2904
  {
2905
    if (rep_quick)
2906
      init_io_cache_share(&param->read_cache, &io_share, NULL, i);
2907
    else
2908
      init_io_cache_share(&new_data_cache, &io_share, &info->rec_cache, i);
2909
  }
2910
  else
2911
    io_share.total_threads= 0; /* share not used */
2912
2913
  (void) pthread_attr_init(&thr_attr);
2914
  (void) pthread_attr_setdetachstate(&thr_attr,PTHREAD_CREATE_DETACHED);
2915
2916
  for (i=0 ; i < sort_info.total_keys ; i++)
2917
  {
2918
    /*
2919
      Copy the properly initialized IO_CACHE structure so that every
2920
      thread has its own copy. In quick mode param->read_cache is shared
2921
      for use by all threads. In non-quick mode all threads but the
2922
      first copy the shared new_data_cache, which is synchronized to the
2923
      write cache of the first thread. The first thread copies
2924
      param->read_cache, which is not shared.
2925
    */
2926
    sort_param[i].read_cache= ((rep_quick || !i) ? param->read_cache :
2927
                               new_data_cache);
2928
    DBUG_PRINT("io_cache_share", ("thread: %u  read_cache: 0x%lx",
2929
                                  i, (long) &sort_param[i].read_cache));
2930
2931
    /*
2932
      two approaches: the same amount of memory for each thread
2933
      or the memory for the same number of keys for each thread...
2934
      In the second one all the threads will fill their sort_buffers
2935
      (and call write_keys) at the same time, putting more stress on i/o.
2936
    */
2937
    sort_param[i].sortbuff_size=
2938
#ifndef USING_SECOND_APPROACH
2939
      param->sort_buffer_length/sort_info.total_keys;
2940
#else
2941
      param->sort_buffer_length*sort_param[i].key_length/total_key_length;
2942
#endif
2943
    if (pthread_create(&sort_param[i].thr, &thr_attr,
2944
		       thr_find_all_keys,
2945
		       (void *) (sort_param+i)))
2946
    {
2947
      mi_check_print_error(param,"Cannot start a repair thread");
2948
      /* Cleanup: Detach from the share. Avoid others to be blocked. */
2949
      if (io_share.total_threads)
2950
        remove_io_thread(&sort_param[i].read_cache);
2951
      DBUG_PRINT("error", ("Cannot start a repair thread"));
2952
      sort_info.got_error=1;
2953
    }
2954
    else
2955
      sort_info.threads_running++;
2956
  }
2957
  (void) pthread_attr_destroy(&thr_attr);
2958
2959
  /* waiting for all threads to finish */
2960
  while (sort_info.threads_running)
2961
    pthread_cond_wait(&sort_info.cond, &sort_info.mutex);
2962
  pthread_mutex_unlock(&sort_info.mutex);
2963
2964
  if ((got_error= thr_write_keys(sort_param)))
2965
  {
2966
    param->retry_repair=1;
2967
    goto err;
2968
  }
2969
  got_error=1;				/* Assume the following may go wrong */
2970
2971
  if (sort_param[0].fix_datafile)
2972
  {
2973
    /*
2974
      Append some nuls to the end of a memory mapped file. Destroy the
2975
      write cache. The master thread did already detach from the share
2976
      by remove_io_thread() in sort.c:thr_find_all_keys().
2977
    */
2978
    if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
2979
      goto err;
2980
    if (param->testflag & T_SAFE_REPAIR)
2981
    {
2982
      /* Don't repair if we loosed more than one row */
2983
      if (info->state->records+1 < start_records)
2984
      {
2985
        info->state->records=start_records;
2986
        goto err;
2987
      }
2988
    }
2989
    share->state.state.data_file_length= info->state->data_file_length=
2990
      sort_param->filepos;
2991
    /* Only whole records */
2992
    share->state.version=(ulong) time((time_t*) 0);
2993
2994
    /*
2995
      Exchange the data file descriptor of the table, so that we use the
2996
      new file from now on.
2997
     */
2998
    my_close(info->dfile,MYF(0));
2999
    info->dfile=new_file;
3000
3001
    share->data_file_type=sort_info.new_data_file_type;
3002
    share->pack.header_length=(ulong) new_header_length;
3003
  }
3004
  else
3005
    info->state->data_file_length=sort_param->max_pos;
3006
3007
  if (rep_quick && del+sort_info.dupp != info->state->del)
3008
  {
3009
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
3010
    mi_check_print_error(param,"Run recovery again without -q");
3011
    param->retry_repair=1;
3012
    param->testflag|=T_RETRY_WITHOUT_QUICK;
3013
    goto err;
3014
  }
3015
3016
  if (rep_quick & T_FORCE_UNIQUENESS)
3017
  {
3018
    my_off_t skr=info->state->data_file_length+
3019
      (share->options & HA_OPTION_COMPRESS_RECORD ?
3020
       MEMMAP_EXTRA_MARGIN : 0);
3021
#ifdef USE_RELOC
3022
    if (share->data_file_type == STATIC_RECORD &&
3023
	skr < share->base.reloc*share->base.min_pack_length)
3024
      skr=share->base.reloc*share->base.min_pack_length;
3025
#endif
3026
    if (skr != sort_info.filelength && !info->s->base.raid_type)
30 by Brian Aker
Large file and ftruncate() support
3027
      if (ftruncate(info->dfile, skr))
1 by brian
clean slate
3028
	mi_check_print_warning(param,
3029
			       "Can't change size of datafile,  error: %d",
3030
			       my_errno);
3031
  }
3032
  if (param->testflag & T_CALC_CHECKSUM)
3033
    info->state->checksum=param->glob_crc;
3034
30 by Brian Aker
Large file and ftruncate() support
3035
  if (ftruncate(share->kfile, info->state->key_file_length))
1 by brian
clean slate
3036
    mi_check_print_warning(param,
3037
			   "Can't change size of indexfile, error: %d", my_errno);
3038
3039
  if (!(param->testflag & T_SILENT))
3040
  {
3041
    if (start_records != info->state->records)
3042
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
3043
    if (sort_info.dupp)
3044
      mi_check_print_warning(param,
3045
			     "%s records have been removed",
3046
			     llstr(sort_info.dupp,llbuff));
3047
  }
3048
  got_error=0;
3049
3050
  if (&share->state.state != info->state)
3051
    memcpy(&share->state.state, info->state, sizeof(*info->state));
3052
3053
err:
3054
  got_error|= flush_blocks(param, share->key_cache, share->kfile);
3055
  /*
3056
    Destroy the write cache. The master thread did already detach from
3057
    the share by remove_io_thread() or it was not yet started (if the
3058
    error happend before creating the thread).
3059
  */
3060
  VOID(end_io_cache(&info->rec_cache));
3061
  /*
3062
    Destroy the new data cache in case of non-quick repair. All slave
3063
    threads did either detach from the share by remove_io_thread()
3064
    already or they were not yet started (if the error happend before
3065
    creating the threads).
3066
  */
3067
  if (!rep_quick)
3068
    VOID(end_io_cache(&new_data_cache));
3069
  if (!got_error)
3070
  {
3071
    /* Replace the actual file with the temporary file */
3072
    if (new_file >= 0)
3073
    {
3074
      my_close(new_file,MYF(0));
3075
      info->dfile=new_file= -1;
3076
      if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
3077
			    DATA_TMP_EXT, share->base.raid_chunks,
3078
			    (param->testflag & T_BACKUP_DATA ?
3079
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
3080
	  mi_open_datafile(info,share,-1))
3081
	got_error=1;
3082
    }
3083
  }
3084
  if (got_error)
3085
  {
3086
    if (! param->error_printed)
3087
      mi_check_print_error(param,"%d when fixing table",my_errno);
3088
    if (new_file >= 0)
3089
    {
3090
      VOID(my_close(new_file,MYF(0)));
3091
      VOID(my_raid_delete(param->temp_filename,share->base.raid_chunks,
3092
			  MYF(MY_WME)));
3093
      if (info->dfile == new_file)
3094
	info->dfile= -1;
3095
    }
3096
    mi_mark_crashed_on_repair(info);
3097
  }
3098
  else if (key_map == share->state.key_map)
3099
    share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
3100
  share->state.changed|=STATE_NOT_SORTED_PAGES;
3101
3102
  pthread_cond_destroy (&sort_info.cond);
3103
  pthread_mutex_destroy(&sort_info.mutex);
3104
3105
  my_free((uchar*) sort_info.ft_buf, MYF(MY_ALLOW_ZERO_PTR));
3106
  my_free((uchar*) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR));
3107
  my_free((uchar*) sort_param,MYF(MY_ALLOW_ZERO_PTR));
3108
  my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
3109
  VOID(end_io_cache(&param->read_cache));
3110
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
3111
  if (!got_error && (param->testflag & T_UNPACK))
3112
  {
3113
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
3114
    share->pack.header_length=0;
3115
  }
3116
  DBUG_RETURN(got_error);
3117
#endif /* THREAD */
3118
}
3119
3120
	/* Read next record and return next key */
3121
3122
static int sort_key_read(MI_SORT_PARAM *sort_param, void *key)
3123
{
3124
  int error;
3125
  SORT_INFO *sort_info=sort_param->sort_info;
3126
  MI_INFO *info=sort_info->info;
3127
  DBUG_ENTER("sort_key_read");
3128
3129
  if ((error=sort_get_next_record(sort_param)))
3130
    DBUG_RETURN(error);
3131
  if (info->state->records == sort_info->max_records)
3132
  {
3133
    mi_check_print_error(sort_info->param,
3134
			 "Key %d - Found too many records; Can't continue",
3135
                         sort_param->key+1);
3136
    DBUG_RETURN(1);
3137
  }
3138
  sort_param->real_key_length=
3139
    (info->s->rec_reflength+
3140
     _mi_make_key(info, sort_param->key, (uchar*) key,
3141
		  sort_param->record, sort_param->filepos));
3142
#ifdef HAVE_purify
3143
  bzero(key+sort_param->real_key_length,
3144
	(sort_param->key_length-sort_param->real_key_length));
3145
#endif
3146
  DBUG_RETURN(sort_write_record(sort_param));
3147
} /* sort_key_read */
3148
3149
static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
3150
{
3151
  int error;
3152
  SORT_INFO *sort_info=sort_param->sort_info;
3153
  MI_INFO *info=sort_info->info;
3154
  FT_WORD *wptr=0;
3155
  DBUG_ENTER("sort_ft_key_read");
3156
3157
  if (!sort_param->wordlist)
3158
  {
3159
    for (;;)
3160
    {
3161
      free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
3162
      if ((error=sort_get_next_record(sort_param)))
3163
        DBUG_RETURN(error);
3164
      if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record,
3165
                                    &sort_param->wordroot)))
3166
        DBUG_RETURN(1);
3167
      if (wptr->pos)
3168
        break;
3169
      error=sort_write_record(sort_param);
3170
    }
3171
    sort_param->wordptr=sort_param->wordlist=wptr;
3172
  }
3173
  else
3174
  {
3175
    error=0;
3176
    wptr=(FT_WORD*)(sort_param->wordptr);
3177
  }
3178
3179
  sort_param->real_key_length=(info->s->rec_reflength+
3180
			       _ft_make_key(info, sort_param->key,
3181
					    key, wptr++, sort_param->filepos));
3182
#ifdef HAVE_purify
3183
  if (sort_param->key_length > sort_param->real_key_length)
3184
    bzero(key+sort_param->real_key_length,
3185
	  (sort_param->key_length-sort_param->real_key_length));
3186
#endif
3187
  if (!wptr->pos)
3188
  {
3189
    free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
3190
    sort_param->wordlist=0;
3191
    error=sort_write_record(sort_param);
3192
  }
3193
  else
3194
    sort_param->wordptr=(void*)wptr;
3195
3196
  DBUG_RETURN(error);
3197
} /* sort_ft_key_read */
3198
3199
3200
/*
3201
  Read next record from file using parameters in sort_info.
3202
3203
  SYNOPSIS
3204
    sort_get_next_record()
3205
      sort_param                Information about and for the sort process
3206
3207
  NOTE
3208
3209
    Dynamic Records With Non-Quick Parallel Repair
3210
3211
      For non-quick parallel repair we use a synchronized read/write
3212
      cache. This means that one thread is the master who fixes the data
3213
      file by reading each record from the old data file and writing it
3214
      to the new data file. By doing this the records in the new data
3215
      file are written contiguously. Whenever the write buffer is full,
3216
      it is copied to the read buffer. The slaves read from the read
3217
      buffer, which is not associated with a file. Thus read_cache.file
3218
      is -1. When using _mi_read_cache(), the slaves must always set
3219
      flag to READING_NEXT so that the function never tries to read from
3220
      file. This is safe because the records are contiguous. There is no
3221
      need to read outside the cache. This condition is evaluated in the
3222
      variable 'parallel_flag' for quick reference. read_cache.file must
3223
      be >= 0 in every other case.
3224
3225
  RETURN
3226
    -1          end of file
3227
    0           ok
3228
    > 0         error
3229
*/
3230
3231
static int sort_get_next_record(MI_SORT_PARAM *sort_param)
3232
{
3233
  int searching;
3234
  int parallel_flag;
3235
  uint found_record,b_type,left_length;
3236
  my_off_t pos;
3237
  uchar *to= NULL;
3238
  MI_BLOCK_INFO block_info;
3239
  SORT_INFO *sort_info=sort_param->sort_info;
3240
  MI_CHECK *param=sort_info->param;
3241
  MI_INFO *info=sort_info->info;
3242
  MYISAM_SHARE *share=info->s;
3243
  char llbuff[22],llbuff2[22];
3244
  DBUG_ENTER("sort_get_next_record");
3245
3246
  if (*killed_ptr(param))
3247
    DBUG_RETURN(1);
3248
3249
  switch (share->data_file_type) {
3250
  case STATIC_RECORD:
3251
    for (;;)
3252
    {
3253
      if (my_b_read(&sort_param->read_cache,sort_param->record,
3254
		    share->base.pack_reclength))
3255
      {
3256
	if (sort_param->read_cache.error)
3257
	  param->out_flag |= O_DATA_LOST;
3258
        param->retry_repair=1;
3259
        param->testflag|=T_RETRY_WITHOUT_QUICK;
3260
	DBUG_RETURN(-1);
3261
      }
3262
      sort_param->start_recpos=sort_param->pos;
3263
      if (!sort_param->fix_datafile)
3264
      {
3265
	sort_param->filepos=sort_param->pos;
3266
        if (sort_param->master)
3267
	  share->state.split++;
3268
      }
3269
      sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength);
3270
      if (*sort_param->record)
3271
      {
3272
	if (sort_param->calc_checksum)
3273
	  param->glob_crc+= (info->checksum=
3274
			     mi_static_checksum(info,sort_param->record));
3275
	DBUG_RETURN(0);
3276
      }
3277
      if (!sort_param->fix_datafile && sort_param->master)
3278
      {
3279
	info->state->del++;
3280
	info->state->empty+=share->base.pack_reclength;
3281
      }
3282
    }
3283
  case DYNAMIC_RECORD:
3284
    pos= sort_param->pos;
3285
    searching= (sort_param->fix_datafile && (param->testflag & T_EXTEND));
3286
    parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0;
3287
    for (;;)
3288
    {
3289
      found_record=block_info.second_read= 0;
3290
      left_length=1;
3291
      if (searching)
3292
      {
3293
	pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE);
3294
        param->testflag|=T_RETRY_WITHOUT_QUICK;
3295
	sort_param->start_recpos=pos;
3296
      }
3297
      do
3298
      {
3299
	if (pos > sort_param->max_pos)
3300
	  sort_param->max_pos=pos;
3301
	if (pos & (MI_DYN_ALIGN_SIZE-1))
3302
	{
3303
	  if ((param->testflag & T_VERBOSE) || searching == 0)
3304
	    mi_check_print_info(param,"Wrong aligned block at %s",
3305
				llstr(pos,llbuff));
3306
	  if (searching)
3307
	    goto try_next;
3308
	}
3309
	if (found_record && pos == param->search_after_block)
3310
	  mi_check_print_info(param,"Block: %s used by record at %s",
3311
		     llstr(param->search_after_block,llbuff),
3312
		     llstr(sort_param->start_recpos,llbuff2));
3313
	if (_mi_read_cache(&sort_param->read_cache,
3314
                           (uchar*) block_info.header,pos,
3315
			   MI_BLOCK_INFO_HEADER_LENGTH,
3316
			   (! found_record ? READING_NEXT : 0) |
3317
                           parallel_flag | READING_HEADER))
3318
	{
3319
	  if (found_record)
3320
	  {
3321
	    mi_check_print_info(param,
3322
				"Can't read whole record at %s (errno: %d)",
3323
				llstr(sort_param->start_recpos,llbuff),errno);
3324
	    goto try_next;
3325
	  }
3326
	  DBUG_RETURN(-1);
3327
	}
3328
	if (searching && ! sort_param->fix_datafile)
3329
	{
3330
	  param->error_printed=1;
3331
          param->retry_repair=1;
3332
          param->testflag|=T_RETRY_WITHOUT_QUICK;
3333
	  DBUG_RETURN(1);	/* Something wrong with data */
3334
	}
3335
	b_type=_mi_get_block_info(&block_info,-1,pos);
3336
	if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
3337
	   ((b_type & BLOCK_FIRST) &&
3338
	     (block_info.rec_len < (uint) share->base.min_pack_length ||
3339
	      block_info.rec_len > (uint) share->base.max_pack_length)))
3340
	{
3341
	  uint i;
3342
	  if (param->testflag & T_VERBOSE || searching == 0)
3343
	    mi_check_print_info(param,
3344
				"Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
3345
		       block_info.header[0],block_info.header[1],
3346
		       block_info.header[2],llstr(pos,llbuff));
3347
	  if (found_record)
3348
	    goto try_next;
3349
	  block_info.second_read=0;
3350
	  searching=1;
3351
	  /* Search after block in read header string */
3352
	  for (i=MI_DYN_ALIGN_SIZE ;
3353
	       i < MI_BLOCK_INFO_HEADER_LENGTH ;
3354
	       i+= MI_DYN_ALIGN_SIZE)
3355
	    if (block_info.header[i] >= 1 &&
3356
		block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
3357
	      break;
3358
	  pos+=(ulong) i;
3359
	  sort_param->start_recpos=pos;
3360
	  continue;
3361
	}
3362
	if (b_type & BLOCK_DELETED)
3363
	{
3364
	  my_bool error=0;
3365
	  if (block_info.block_len+ (uint) (block_info.filepos-pos) <
3366
	      share->base.min_block_length)
3367
	  {
3368
	    if (!searching)
3369
	      mi_check_print_info(param,
3370
				  "Deleted block with impossible length %u at %s",
3371
				  block_info.block_len,llstr(pos,llbuff));
3372
	    error=1;
3373
	  }
3374
	  else
3375
	  {
3376
	    if ((block_info.next_filepos != HA_OFFSET_ERROR &&
3377
		 block_info.next_filepos >=
3378
		 info->state->data_file_length) ||
3379
		(block_info.prev_filepos != HA_OFFSET_ERROR &&
3380
		 block_info.prev_filepos >= info->state->data_file_length))
3381
	    {
3382
	      if (!searching)
3383
		mi_check_print_info(param,
3384
				    "Delete link points outside datafile at %s",
3385
				    llstr(pos,llbuff));
3386
	      error=1;
3387
	    }
3388
	  }
3389
	  if (error)
3390
	  {
3391
	    if (found_record)
3392
	      goto try_next;
3393
	    searching=1;
3394
	    pos+= MI_DYN_ALIGN_SIZE;
3395
	    sort_param->start_recpos=pos;
3396
	    block_info.second_read=0;
3397
	    continue;
3398
	  }
3399
	}
3400
	else
3401
	{
3402
	  if (block_info.block_len+ (uint) (block_info.filepos-pos) <
3403
	      share->base.min_block_length ||
3404
	      block_info.block_len > (uint) share->base.max_pack_length+
3405
	      MI_SPLIT_LENGTH)
3406
	  {
3407
	    if (!searching)
3408
	      mi_check_print_info(param,
3409
				  "Found block with impossible length %u at %s; Skipped",
3410
				  block_info.block_len+ (uint) (block_info.filepos-pos),
3411
				  llstr(pos,llbuff));
3412
	    if (found_record)
3413
	      goto try_next;
3414
	    searching=1;
3415
	    pos+= MI_DYN_ALIGN_SIZE;
3416
	    sort_param->start_recpos=pos;
3417
	    block_info.second_read=0;
3418
	    continue;
3419
	  }
3420
	}
3421
	if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))
3422
	{
3423
          if (!sort_param->fix_datafile && sort_param->master &&
3424
              (b_type & BLOCK_DELETED))
3425
	  {
3426
	    info->state->empty+=block_info.block_len;
3427
	    info->state->del++;
3428
	    share->state.split++;
3429
	  }
3430
	  if (found_record)
3431
	    goto try_next;
3432
	  if (searching)
3433
	  {
3434
	    pos+=MI_DYN_ALIGN_SIZE;
3435
	    sort_param->start_recpos=pos;
3436
	  }
3437
	  else
3438
	    pos=block_info.filepos+block_info.block_len;
3439
	  block_info.second_read=0;
3440
	  continue;
3441
	}
3442
3443
	if (!sort_param->fix_datafile && sort_param->master)
3444
	  share->state.split++;
3445
	if (! found_record++)
3446
	{
3447
	  sort_param->find_length=left_length=block_info.rec_len;
3448
	  sort_param->start_recpos=pos;
3449
	  if (!sort_param->fix_datafile)
3450
	    sort_param->filepos=sort_param->start_recpos;
3451
	  if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
3452
	    sort_param->pos=block_info.filepos+1;
3453
	  else
3454
	    sort_param->pos=block_info.filepos+block_info.block_len;
3455
	  if (share->base.blobs)
3456
	  {
3457
	    if (!(to=mi_alloc_rec_buff(info,block_info.rec_len,
3458
				       &(sort_param->rec_buff))))
3459
	    {
3460
	      if (param->max_record_length >= block_info.rec_len)
3461
	      {
3462
		mi_check_print_error(param,"Not enough memory for blob at %s (need %lu)",
3463
				     llstr(sort_param->start_recpos,llbuff),
3464
				     (ulong) block_info.rec_len);
3465
		DBUG_RETURN(1);
3466
	      }
3467
	      else
3468
	      {
3469
		mi_check_print_info(param,"Not enough memory for blob at %s (need %lu); Row skipped",
3470
				    llstr(sort_param->start_recpos,llbuff),
3471
				    (ulong) block_info.rec_len);
3472
		goto try_next;
3473
	      }
3474
	    }
3475
	  }
3476
	  else
3477
	    to= sort_param->rec_buff;
3478
	}
3479
	if (left_length < block_info.data_len || ! block_info.data_len)
3480
	{
3481
	  mi_check_print_info(param,
3482
			      "Found block with too small length at %s; Skipped",
3483
			      llstr(sort_param->start_recpos,llbuff));
3484
	  goto try_next;
3485
	}
3486
	if (block_info.filepos + block_info.data_len >
3487
	    sort_param->read_cache.end_of_file)
3488
	{
3489
	  mi_check_print_info(param,
3490
			      "Found block that points outside data file at %s",
3491
			      llstr(sort_param->start_recpos,llbuff));
3492
	  goto try_next;
3493
	}
3494
        /*
3495
          Copy information that is already read. Avoid accessing data
3496
          below the cache start. This could happen if the header
3497
          streched over the end of the previous buffer contents.
3498
        */
3499
        {
3500
          uint header_len= (uint) (block_info.filepos - pos);
3501
          uint prefetch_len= (MI_BLOCK_INFO_HEADER_LENGTH - header_len);
3502
3503
          if (prefetch_len > block_info.data_len)
3504
            prefetch_len= block_info.data_len;
3505
          if (prefetch_len)
3506
          {
3507
            memcpy(to, block_info.header + header_len, prefetch_len);
3508
            block_info.filepos+= prefetch_len;
3509
            block_info.data_len-= prefetch_len;
3510
            left_length-= prefetch_len;
3511
            to+= prefetch_len;
3512
          }
3513
        }
3514
        if (block_info.data_len &&
3515
            _mi_read_cache(&sort_param->read_cache,to,block_info.filepos,
3516
                           block_info.data_len,
3517
                           (found_record == 1 ? READING_NEXT : 0) |
3518
                           parallel_flag))
3519
	{
3520
	  mi_check_print_info(param,
3521
			      "Read error for block at: %s (error: %d); Skipped",
3522
			      llstr(block_info.filepos,llbuff),my_errno);
3523
	  goto try_next;
3524
	}
3525
	left_length-=block_info.data_len;
3526
	to+=block_info.data_len;
3527
	pos=block_info.next_filepos;
3528
	if (pos == HA_OFFSET_ERROR && left_length)
3529
	{
3530
	  mi_check_print_info(param,"Wrong block with wrong total length starting at %s",
3531
			      llstr(sort_param->start_recpos,llbuff));
3532
	  goto try_next;
3533
	}
3534
	if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file)
3535
	{
3536
	  mi_check_print_info(param,"Found link that points at %s (outside data file) at %s",
3537
			      llstr(pos,llbuff2),
3538
			      llstr(sort_param->start_recpos,llbuff));
3539
	  goto try_next;
3540
	}
3541
      } while (left_length);
3542
3543
      if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff,
3544
			 sort_param->find_length) != MY_FILE_ERROR)
3545
      {
3546
	if (sort_param->read_cache.error < 0)
3547
	  DBUG_RETURN(1);
3548
	if (sort_param->calc_checksum)
3549
	  info->checksum= mi_checksum(info, sort_param->record);
3550
	if ((param->testflag & (T_EXTEND | T_REP)) || searching)
3551
	{
3552
	  if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
3553
                            sort_param->find_length,
3554
                            (param->testflag & T_QUICK) &&
3555
                            sort_param->calc_checksum &&
3556
                            test(info->s->calc_checksum)))
3557
	  {
3558
	    mi_check_print_info(param,"Found wrong packed record at %s",
3559
				llstr(sort_param->start_recpos,llbuff));
3560
	    goto try_next;
3561
	  }
3562
	}
3563
	if (sort_param->calc_checksum)
3564
	  param->glob_crc+= info->checksum;
3565
	DBUG_RETURN(0);
3566
      }
3567
      if (!searching)
3568
        mi_check_print_info(param,"Key %d - Found wrong stored record at %s",
3569
                            sort_param->key+1,
3570
                            llstr(sort_param->start_recpos,llbuff));
3571
    try_next:
3572
      pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE);
3573
      searching=1;
3574
    }
3575
  case COMPRESSED_RECORD:
3576
    for (searching=0 ;; searching=1, sort_param->pos++)
3577
    {
3578
      if (_mi_read_cache(&sort_param->read_cache,(uchar*) block_info.header,
3579
			 sort_param->pos,
3580
			 share->pack.ref_length,READING_NEXT))
3581
	DBUG_RETURN(-1);
3582
      if (searching && ! sort_param->fix_datafile)
3583
      {
3584
	param->error_printed=1;
3585
        param->retry_repair=1;
3586
        param->testflag|=T_RETRY_WITHOUT_QUICK;
3587
	DBUG_RETURN(1);		/* Something wrong with data */
3588
      }
3589
      sort_param->start_recpos=sort_param->pos;
3590
      if (_mi_pack_get_block_info(info, &sort_param->bit_buff, &block_info,
3591
                                  &sort_param->rec_buff, -1, sort_param->pos))
3592
	DBUG_RETURN(-1);
3593
      if (!block_info.rec_len &&
3594
	  sort_param->pos + MEMMAP_EXTRA_MARGIN ==
3595
	  sort_param->read_cache.end_of_file)
3596
	DBUG_RETURN(-1);
3597
      if (block_info.rec_len < (uint) share->min_pack_length ||
3598
	  block_info.rec_len > (uint) share->max_pack_length)
3599
      {
3600
	if (! searching)
3601
	  mi_check_print_info(param,"Found block with wrong recordlength: %d at %s\n",
3602
			      block_info.rec_len,
3603
			      llstr(sort_param->pos,llbuff));
3604
	continue;
3605
      }
3606
      if (_mi_read_cache(&sort_param->read_cache,(uchar*) sort_param->rec_buff,
3607
			 block_info.filepos, block_info.rec_len,
3608
			 READING_NEXT))
3609
      {
3610
	if (! searching)
3611
	  mi_check_print_info(param,"Couldn't read whole record from %s",
3612
			      llstr(sort_param->pos,llbuff));
3613
	continue;
3614
      }
3615
      if (_mi_pack_rec_unpack(info, &sort_param->bit_buff, sort_param->record,
3616
                              sort_param->rec_buff, block_info.rec_len))
3617
      {
3618
	if (! searching)
3619
	  mi_check_print_info(param,"Found wrong record at %s",
3620
			      llstr(sort_param->pos,llbuff));
3621
	continue;
3622
      }
3623
      if (!sort_param->fix_datafile)
3624
      {
3625
	sort_param->filepos=sort_param->pos;
3626
        if (sort_param->master)
3627
	  share->state.split++;
3628
      }
3629
      sort_param->max_pos=(sort_param->pos=block_info.filepos+
3630
			 block_info.rec_len);
3631
      info->packed_length=block_info.rec_len;
3632
      if (sort_param->calc_checksum)
3633
	param->glob_crc+= (info->checksum=
3634
                           mi_checksum(info, sort_param->record));
3635
      DBUG_RETURN(0);
3636
    }
3637
  case BLOCK_RECORD:
3638
    assert(0);                                  /* Impossible */
3639
  }
3640
  DBUG_RETURN(1);                               /* Impossible */
3641
}
3642
3643
3644
/*
3645
  Write record to new file.
3646
3647
  SYNOPSIS
3648
    sort_write_record()
3649
      sort_param                Sort parameters.
3650
3651
  NOTE
3652
    This is only called by a master thread if parallel repair is used.
3653
3654
  RETURN
3655
    0           OK
3656
    1           Error
3657
*/
3658
3659
int sort_write_record(MI_SORT_PARAM *sort_param)
3660
{
3661
  int flag;
3662
  uint length;
3663
  ulong block_length,reclength;
3664
  uchar *from;
3665
  uchar block_buff[8];
3666
  SORT_INFO *sort_info=sort_param->sort_info;
3667
  MI_CHECK *param=sort_info->param;
3668
  MI_INFO *info=sort_info->info;
3669
  MYISAM_SHARE *share=info->s;
3670
  DBUG_ENTER("sort_write_record");
3671
3672
  if (sort_param->fix_datafile)
3673
  {
3674
    switch (sort_info->new_data_file_type) {
3675
    case STATIC_RECORD:
3676
      if (my_b_write(&info->rec_cache,sort_param->record,
3677
		     share->base.pack_reclength))
3678
      {
3679
	mi_check_print_error(param,"%d when writing to datafile",my_errno);
3680
	DBUG_RETURN(1);
3681
      }
3682
      sort_param->filepos+=share->base.pack_reclength;
3683
      info->s->state.split++;
3684
      /* sort_info->param->glob_crc+=mi_static_checksum(info, sort_param->record); */
3685
      break;
3686
    case DYNAMIC_RECORD:
3687
      if (! info->blobs)
3688
	from=sort_param->rec_buff;
3689
      else
3690
      {
3691
	/* must be sure that local buffer is big enough */
3692
	reclength=info->s->base.pack_reclength+
3693
	  _my_calc_total_blob_length(info,sort_param->record)+
3694
	  ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+
3695
	  MI_DYN_DELETE_BLOCK_HEADER;
3696
	if (sort_info->buff_length < reclength)
3697
	{
3698
	  if (!(sort_info->buff=my_realloc(sort_info->buff, (uint) reclength,
3699
					   MYF(MY_FREE_ON_ERROR |
3700
					       MY_ALLOW_ZERO_PTR))))
3701
	    DBUG_RETURN(1);
3702
	  sort_info->buff_length=reclength;
3703
	}
3704
	from= sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
3705
      }
3706
      /* We can use info->checksum here as only one thread calls this. */
3707
      info->checksum=mi_checksum(info,sort_param->record);
3708
      reclength=_mi_rec_pack(info,from,sort_param->record);
3709
      flag=0;
3710
      /* sort_info->param->glob_crc+=info->checksum; */
3711
3712
      do
3713
      {
3714
	block_length=reclength+ 3 + test(reclength >= (65520-3));
3715
	if (block_length < share->base.min_block_length)
3716
	  block_length=share->base.min_block_length;
3717
	info->update|=HA_STATE_WRITE_AT_END;
3718
	block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE);
3719
	if (block_length > MI_MAX_BLOCK_LENGTH)
3720
	  block_length=MI_MAX_BLOCK_LENGTH;
3721
	if (_mi_write_part_record(info,0L,block_length,
3722
				  sort_param->filepos+block_length,
3723
				  &from,&reclength,&flag))
3724
	{
3725
	  mi_check_print_error(param,"%d when writing to datafile",my_errno);
3726
	  DBUG_RETURN(1);
3727
	}
3728
	sort_param->filepos+=block_length;
3729
	info->s->state.split++;
3730
      } while (reclength);
3731
      /* sort_info->param->glob_crc+=info->checksum; */
3732
      break;
3733
    case COMPRESSED_RECORD:
3734
      reclength=info->packed_length;
3735
      length= save_pack_length((uint) share->pack.version, block_buff,
3736
                               reclength);
3737
      if (info->s->base.blobs)
3738
	length+= save_pack_length((uint) share->pack.version,
3739
	                          block_buff + length, info->blob_length);
3740
      if (my_b_write(&info->rec_cache,block_buff,length) ||
3741
	  my_b_write(&info->rec_cache,(uchar*) sort_param->rec_buff,reclength))
3742
      {
3743
	mi_check_print_error(param,"%d when writing to datafile",my_errno);
3744
	DBUG_RETURN(1);
3745
      }
3746
      /* sort_info->param->glob_crc+=info->checksum; */
3747
      sort_param->filepos+=reclength+length;
3748
      info->s->state.split++;
3749
      break;
3750
    case BLOCK_RECORD:
3751
      assert(0);                                  /* Impossible */
3752
    }
3753
  }
3754
  if (sort_param->master)
3755
  {
3756
    info->state->records++;
3757
    if ((param->testflag & T_WRITE_LOOP) &&
3758
        (info->state->records % WRITE_COUNT) == 0)
3759
    {
3760
      char llbuff[22];
3761
      printf("%s\r", llstr(info->state->records,llbuff));
3762
      VOID(fflush(stdout));
3763
    }
3764
  }
3765
  DBUG_RETURN(0);
3766
} /* sort_write_record */
3767
3768
3769
	/* Compare two keys from _create_index_by_sort */
3770
3771
static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,
3772
			const void *b)
3773
{
3774
  uint not_used[2];
3775
  return (ha_key_cmp(sort_param->seg, *((uchar**) a), *((uchar**) b),
3776
		     USE_WHOLE_KEY, SEARCH_SAME, not_used));
3777
} /* sort_key_cmp */
3778
3779
3780
static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
3781
{
3782
  uint diff_pos[2];
3783
  char llbuff[22],llbuff2[22];
3784
  SORT_INFO *sort_info=sort_param->sort_info;
3785
  MI_CHECK *param= sort_info->param;
3786
  int cmp;
3787
3788
  if (sort_info->key_block->inited)
3789
  {
3790
    cmp=ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
3791
		   (uchar*) a, USE_WHOLE_KEY,SEARCH_FIND | SEARCH_UPDATE,
3792
		   diff_pos);
3793
    if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
3794
      ha_key_cmp(sort_param->seg,sort_info->key_block->lastkey,
3795
                 (uchar*) a, USE_WHOLE_KEY, 
3796
                 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
3797
    else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
3798
    {
3799
      diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg,
3800
                                                 sort_param->notnull,
3801
                                                 sort_info->key_block->lastkey,
3802
                                                 (uchar*)a);
3803
    }
3804
    sort_param->unique[diff_pos[0]-1]++;
3805
  }
3806
  else
3807
  {
3808
    cmp= -1;
3809
    if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
3810
      mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
3811
                                     (uchar*)a);
3812
  }
3813
  if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0)
3814
  {
3815
    sort_info->dupp++;
3816
    sort_info->info->lastpos=get_record_for_key(sort_info->info,
3817
						sort_param->keyinfo,
3818
						(uchar*) a);
3819
    mi_check_print_warning(param,
3820
			   "Duplicate key for record at %10s against record at %10s",
3821
			   llstr(sort_info->info->lastpos,llbuff),
3822
			   llstr(get_record_for_key(sort_info->info,
3823
						    sort_param->keyinfo,
3824
						    sort_info->key_block->
3825
						    lastkey),
3826
				 llbuff2));
3827
    param->testflag|=T_RETRY_WITHOUT_QUICK;
3828
    if (sort_info->param->testflag & T_VERBOSE)
3829
      _mi_print_key(stdout,sort_param->seg,(uchar*) a, USE_WHOLE_KEY);
3830
    return (sort_delete_record(sort_param));
3831
  }
3832
#ifndef DBUG_OFF
3833
  if (cmp > 0)
3834
  {
3835
    mi_check_print_error(param,
3836
			 "Internal error: Keys are not in order from sort");
3837
    return(1);
3838
  }
3839
#endif
3840
  return (sort_insert_key(sort_param,sort_info->key_block,
3841
			  (uchar*) a, HA_OFFSET_ERROR));
3842
} /* sort_key_write */
3843
3844
int sort_ft_buf_flush(MI_SORT_PARAM *sort_param)
3845
{
3846
  SORT_INFO *sort_info=sort_param->sort_info;
3847
  SORT_KEY_BLOCKS *key_block=sort_info->key_block;
3848
  MYISAM_SHARE *share=sort_info->info->s;
3849
  uint val_off, val_len;
3850
  int error;
3851
  SORT_FT_BUF *ft_buf=sort_info->ft_buf;
3852
  uchar *from, *to;
3853
3854
  val_len=share->ft2_keyinfo.keylength;
3855
  get_key_full_length_rdonly(val_off, ft_buf->lastkey);
3856
  to=ft_buf->lastkey+val_off;
3857
3858
  if (ft_buf->buf)
3859
  {
3860
    /* flushing first-level tree */
3861
    error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,
3862
			  HA_OFFSET_ERROR);
3863
    for (from=to+val_len;
3864
         !error && from < ft_buf->buf;
3865
         from+= val_len)
3866
    {
3867
      memcpy(to, from, val_len);
3868
      error=sort_insert_key(sort_param,key_block,ft_buf->lastkey,
3869
			    HA_OFFSET_ERROR);
3870
    }
3871
    return error;
3872
  }
3873
  /* flushing second-level tree keyblocks */
3874
  error=flush_pending_blocks(sort_param);
3875
  /* updating lastkey with second-level tree info */
3876
  ft_intXstore(ft_buf->lastkey+val_off, -ft_buf->count);
3877
  _mi_dpointer(sort_info->info, ft_buf->lastkey+val_off+HA_FT_WLEN,
3878
      share->state.key_root[sort_param->key]);
3879
  /* restoring first level tree data in sort_info/sort_param */
3880
  sort_info->key_block=sort_info->key_block_end- sort_info->param->sort_key_blocks;
3881
  sort_param->keyinfo=share->keyinfo+sort_param->key;
3882
  share->state.key_root[sort_param->key]=HA_OFFSET_ERROR;
3883
  /* writing lastkey in first-level tree */
3884
  return error ? error :
3885
                 sort_insert_key(sort_param,sort_info->key_block,
3886
                                 ft_buf->lastkey,HA_OFFSET_ERROR);
3887
}
3888
3889
static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a)
3890
{
3891
  uint a_len, val_off, val_len, error;
3892
  uchar *p;
3893
  SORT_INFO *sort_info=sort_param->sort_info;
3894
  SORT_FT_BUF *ft_buf=sort_info->ft_buf;
3895
  SORT_KEY_BLOCKS *key_block=sort_info->key_block;
3896
3897
  val_len=HA_FT_WLEN+sort_info->info->s->base.rec_reflength;
3898
  get_key_full_length_rdonly(a_len, (uchar *)a);
3899
3900
  if (!ft_buf)
3901
  {
3902
    /*
3903
      use two-level tree only if key_reflength fits in rec_reflength place
3904
      and row format is NOT static - for _mi_dpointer not to garble offsets
3905
     */
3906
    if ((sort_info->info->s->base.key_reflength <=
3907
         sort_info->info->s->base.rec_reflength) &&
3908
        (sort_info->info->s->options &
3909
          (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
3910
      ft_buf=(SORT_FT_BUF *)my_malloc(sort_param->keyinfo->block_length +
3911
                                      sizeof(SORT_FT_BUF), MYF(MY_WME));
3912
3913
    if (!ft_buf)
3914
    {
3915
      sort_param->key_write=sort_key_write;
3916
      return sort_key_write(sort_param, a);
3917
    }
3918
    sort_info->ft_buf=ft_buf;
3919
    goto word_init_ft_buf; /* no need to duplicate the code */
3920
  }
3921
  get_key_full_length_rdonly(val_off, ft_buf->lastkey);
3922
3923
  if (ha_compare_text(sort_param->seg->charset,
3924
                      ((uchar *)a)+1,a_len-1,
3925
                      ft_buf->lastkey+1,val_off-1, 0, 0)==0)
3926
  {
3927
    if (!ft_buf->buf) /* store in second-level tree */
3928
    {
3929
      ft_buf->count++;
3930
      return sort_insert_key(sort_param,key_block,
3931
                             ((uchar *)a)+a_len, HA_OFFSET_ERROR);
3932
    }
3933
3934
    /* storing the key in the buffer. */
3935
    memcpy (ft_buf->buf, (char *)a+a_len, val_len);
3936
    ft_buf->buf+=val_len;
3937
    if (ft_buf->buf < ft_buf->end)
3938
      return 0;
3939
3940
    /* converting to two-level tree */
3941
    p=ft_buf->lastkey+val_off;
3942
3943
    while (key_block->inited)
3944
      key_block++;
3945
    sort_info->key_block=key_block;
3946
    sort_param->keyinfo=& sort_info->info->s->ft2_keyinfo;
3947
    ft_buf->count=(ft_buf->buf - p)/val_len;
3948
3949
    /* flushing buffer to second-level tree */
3950
    for (error=0; !error && p < ft_buf->buf; p+= val_len)
3951
      error=sort_insert_key(sort_param,key_block,p,HA_OFFSET_ERROR);
3952
    ft_buf->buf=0;
3953
    return error;
3954
  }
3955
3956
  /* flushing buffer */
3957
  if ((error=sort_ft_buf_flush(sort_param)))
3958
    return error;
3959
3960
word_init_ft_buf:
3961
  a_len+=val_len;
3962
  memcpy(ft_buf->lastkey, a, a_len);
3963
  ft_buf->buf=ft_buf->lastkey+a_len;
3964
  /*
3965
    32 is just a safety margin here
3966
    (at least max(val_len, sizeof(nod_flag)) should be there).
3967
    May be better performance could be achieved if we'd put
3968
      (sort_info->keyinfo->block_length-32)/XXX
3969
      instead.
3970
        TODO: benchmark the best value for XXX.
3971
  */
3972
  ft_buf->end=ft_buf->lastkey+ (sort_param->keyinfo->block_length-32);
3973
  return 0;
3974
} /* sort_ft_key_write */
3975
3976
3977
	/* get pointer to record from a key */
3978
3979
static my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
3980
				   uchar *key)
3981
{
3982
  return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key));
3983
} /* get_record_for_key */
3984
3985
3986
	/* Insert a key in sort-key-blocks */
3987
3988
static int sort_insert_key(MI_SORT_PARAM *sort_param,
3989
			   register SORT_KEY_BLOCKS *key_block, uchar *key,
3990
			   my_off_t prev_block)
3991
{
3992
  uint a_length,t_length,nod_flag;
3993
  my_off_t filepos,key_file_length;
3994
  uchar *anc_buff,*lastkey;
3995
  MI_KEY_PARAM s_temp;
3996
  MI_INFO *info;
3997
  MI_KEYDEF *keyinfo=sort_param->keyinfo;
3998
  SORT_INFO *sort_info= sort_param->sort_info;
3999
  MI_CHECK *param=sort_info->param;
4000
  DBUG_ENTER("sort_insert_key");
4001
4002
  anc_buff=key_block->buff;
4003
  info=sort_info->info;
4004
  lastkey=key_block->lastkey;
4005
  nod_flag= (key_block == sort_info->key_block ? 0 :
4006
	     info->s->base.key_reflength);
4007
4008
  if (!key_block->inited)
4009
  {
4010
    key_block->inited=1;
4011
    if (key_block == sort_info->key_block_end)
4012
    {
4013
      mi_check_print_error(param,"To many key-block-levels; Try increasing sort_key_blocks");
4014
      DBUG_RETURN(1);
4015
    }
4016
    a_length=2+nod_flag;
4017
    key_block->end_pos=anc_buff+2;
4018
    lastkey=0;					/* No previous key in block */
4019
  }
4020
  else
4021
    a_length=mi_getint(anc_buff);
4022
4023
	/* Save pointer to previous block */
4024
  if (nod_flag)
4025
    _mi_kpointer(info,key_block->end_pos,prev_block);
4026
4027
  t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,
4028
				(uchar*) 0,lastkey,lastkey,key,
4029
				 &s_temp);
4030
  (*keyinfo->store_key)(keyinfo, key_block->end_pos+nod_flag,&s_temp);
4031
  a_length+=t_length;
4032
  mi_putint(anc_buff,a_length,nod_flag);
4033
  key_block->end_pos+=t_length;
4034
  if (a_length <= keyinfo->block_length)
4035
  {
4036
    VOID(_mi_move_key(keyinfo,key_block->lastkey,key));
4037
    key_block->last_length=a_length-t_length;
4038
    DBUG_RETURN(0);
4039
  }
4040
4041
	/* Fill block with end-zero and write filled block */
4042
  mi_putint(anc_buff,key_block->last_length,nod_flag);
4043
  bzero((uchar*) anc_buff+key_block->last_length,
4044
	keyinfo->block_length- key_block->last_length);
4045
  key_file_length=info->state->key_file_length;
4046
  if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
4047
    DBUG_RETURN(1);
4048
4049
  /* If we read the page from the key cache, we have to write it back to it */
4050
  if (key_file_length == info->state->key_file_length)
4051
  {
4052
    if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
4053
      DBUG_RETURN(1);
4054
  }
4055
  else if (my_pwrite(info->s->kfile,(uchar*) anc_buff,
4056
		     (uint) keyinfo->block_length,filepos, param->myf_rw))
4057
    DBUG_RETURN(1);
4058
  DBUG_DUMP("buff",(uchar*) anc_buff,mi_getint(anc_buff));
4059
4060
	/* Write separator-key to block in next level */
4061
  if (sort_insert_key(sort_param,key_block+1,key_block->lastkey,filepos))
4062
    DBUG_RETURN(1);
4063
4064
	/* clear old block and write new key in it */
4065
  key_block->inited=0;
4066
  DBUG_RETURN(sort_insert_key(sort_param, key_block,key,prev_block));
4067
} /* sort_insert_key */
4068
4069
4070
	/* Delete record when we found a duplicated key */
4071
4072
static int sort_delete_record(MI_SORT_PARAM *sort_param)
4073
{
4074
  uint i;
4075
  int old_file,error;
4076
  uchar *key;
4077
  SORT_INFO *sort_info=sort_param->sort_info;
4078
  MI_CHECK *param=sort_info->param;
4079
  MI_INFO *info=sort_info->info;
4080
  DBUG_ENTER("sort_delete_record");
4081
4082
  if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
4083
  {
4084
    mi_check_print_error(param,
4085
			 "Quick-recover aborted; Run recovery without switch -q or with switch -qq");
4086
    DBUG_RETURN(1);
4087
  }
4088
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
4089
  {
4090
    mi_check_print_error(param,
4091
			 "Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);;
4092
    DBUG_RETURN(1);
4093
  }
4094
4095
  old_file=info->dfile;
4096
  info->dfile=info->rec_cache.file;
4097
  if (sort_info->current_key)
4098
  {
4099
    key=info->lastkey+info->s->base.max_key_length;
4100
    if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) &&
4101
	error != HA_ERR_RECORD_DELETED)
4102
    {
4103
      mi_check_print_error(param,"Can't read record to be removed");
4104
      info->dfile=old_file;
4105
      DBUG_RETURN(1);
4106
    }
4107
4108
    for (i=0 ; i < sort_info->current_key ; i++)
4109
    {
4110
      uint key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos);
4111
      if (_mi_ck_delete(info,i,key,key_length))
4112
      {
4113
	mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1);
4114
	info->dfile=old_file;
4115
	DBUG_RETURN(1);
4116
      }
4117
    }
4118
    if (sort_param->calc_checksum)
4119
      param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record);
4120
  }
4121
  error=flush_io_cache(&info->rec_cache) || (*info->s->delete_record)(info);
4122
  info->dfile=old_file;				/* restore actual value */
4123
  info->state->records--;
4124
  DBUG_RETURN(error);
4125
} /* sort_delete_record */
4126
4127
	/* Fix all pending blocks and flush everything to disk */
4128
4129
int flush_pending_blocks(MI_SORT_PARAM *sort_param)
4130
{
4131
  uint nod_flag,length;
4132
  my_off_t filepos,key_file_length;
4133
  SORT_KEY_BLOCKS *key_block;
4134
  SORT_INFO *sort_info= sort_param->sort_info;
4135
  myf myf_rw=sort_info->param->myf_rw;
4136
  MI_INFO *info=sort_info->info;
4137
  MI_KEYDEF *keyinfo=sort_param->keyinfo;
4138
  DBUG_ENTER("flush_pending_blocks");
4139
4140
  filepos= HA_OFFSET_ERROR;			/* if empty file */
4141
  nod_flag=0;
4142
  for (key_block=sort_info->key_block ; key_block->inited ; key_block++)
4143
  {
4144
    key_block->inited=0;
4145
    length=mi_getint(key_block->buff);
4146
    if (nod_flag)
4147
      _mi_kpointer(info,key_block->end_pos,filepos);
4148
    key_file_length=info->state->key_file_length;
4149
    bzero((uchar*) key_block->buff+length, keyinfo->block_length-length);
4150
    if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
4151
      DBUG_RETURN(1);
4152
4153
    /* If we read the page from the key cache, we have to write it back */
4154
    if (key_file_length == info->state->key_file_length)
4155
    {
4156
      if (_mi_write_keypage(info, keyinfo, filepos,
4157
                            DFLT_INIT_HITS, key_block->buff))
4158
	DBUG_RETURN(1);
4159
    }
4160
    else if (my_pwrite(info->s->kfile,(uchar*) key_block->buff,
4161
		       (uint) keyinfo->block_length,filepos, myf_rw))
4162
      DBUG_RETURN(1);
4163
    DBUG_DUMP("buff",(uchar*) key_block->buff,length);
4164
    nod_flag=1;
4165
  }
4166
  info->s->state.key_root[sort_param->key]=filepos; /* Last is root for tree */
4167
  DBUG_RETURN(0);
4168
} /* flush_pending_blocks */
4169
4170
	/* alloc space and pointers for key_blocks */
4171
4172
static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint blocks,
4173
                                         uint buffer_length)
4174
{
4175
  register uint i;
4176
  SORT_KEY_BLOCKS *block;
4177
  DBUG_ENTER("alloc_key_blocks");
4178
4179
  if (!(block=(SORT_KEY_BLOCKS*) my_malloc((sizeof(SORT_KEY_BLOCKS)+
4180
					    buffer_length+IO_SIZE)*blocks,
4181
					   MYF(0))))
4182
  {
4183
    mi_check_print_error(param,"Not enough memory for sort-key-blocks");
4184
    return(0);
4185
  }
4186
  for (i=0 ; i < blocks ; i++)
4187
  {
4188
    block[i].inited=0;
4189
    block[i].buff=(uchar*) (block+blocks)+(buffer_length+IO_SIZE)*i;
4190
  }
4191
  DBUG_RETURN(block);
4192
} /* alloc_key_blocks */
4193
4194
4195
	/* Check if file is almost full */
4196
4197
int test_if_almost_full(MI_INFO *info)
4198
{
4199
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
4200
    return 0;
4201
  return my_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(MY_THREADSAFE)) / 10 * 9 >
4202
         (my_off_t) info->s->base.max_key_file_length ||
4203
         my_seek(info->dfile, 0L, MY_SEEK_END, MYF(0)) / 10 * 9 >
4204
         (my_off_t) info->s->base.max_data_file_length;
4205
}
4206
4207
	/* Recreate table with bigger more alloced record-data */
4208
4209
int recreate_table(MI_CHECK *param, MI_INFO **org_info, char *filename)
4210
{
4211
  int error;
4212
  MI_INFO info;
4213
  MYISAM_SHARE share;
4214
  MI_KEYDEF *keyinfo,*key,*key_end;
4215
  HA_KEYSEG *keysegs,*keyseg;
4216
  MI_COLUMNDEF *recdef,*rec,*end;
4217
  MI_UNIQUEDEF *uniquedef,*u_ptr,*u_end;
4218
  MI_STATUS_INFO status_info;
4219
  uint unpack,key_parts;
4220
  ha_rows max_records;
4221
  ulonglong file_length,tmp_length;
4222
  MI_CREATE_INFO create_info;
4223
  DBUG_ENTER("recreate_table");
4224
4225
  error=1;					/* Default error */
4226
  info= **org_info;
4227
  status_info= (*org_info)->state[0];
4228
  info.state= &status_info;
4229
  share= *(*org_info)->s;
4230
  unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
4231
    (param->testflag & T_UNPACK);
4232
  if (!(keyinfo=(MI_KEYDEF*) my_alloca(sizeof(MI_KEYDEF)*share.base.keys)))
4233
    DBUG_RETURN(0);
4234
  memcpy((uchar*) keyinfo,(uchar*) share.keyinfo,
4235
	 (size_t) (sizeof(MI_KEYDEF)*share.base.keys));
4236
4237
  key_parts= share.base.all_key_parts;
4238
  if (!(keysegs=(HA_KEYSEG*) my_alloca(sizeof(HA_KEYSEG)*
4239
				       (key_parts+share.base.keys))))
4240
  {
4241
    my_afree((uchar*) keyinfo);
4242
    DBUG_RETURN(1);
4243
  }
4244
  if (!(recdef=(MI_COLUMNDEF*)
4245
	my_alloca(sizeof(MI_COLUMNDEF)*(share.base.fields+1))))
4246
  {
4247
    my_afree((uchar*) keyinfo);
4248
    my_afree((uchar*) keysegs);
4249
    DBUG_RETURN(1);
4250
  }
4251
  if (!(uniquedef=(MI_UNIQUEDEF*)
4252
	my_alloca(sizeof(MI_UNIQUEDEF)*(share.state.header.uniques+1))))
4253
  {
4254
    my_afree((uchar*) recdef);
4255
    my_afree((uchar*) keyinfo);
4256
    my_afree((uchar*) keysegs);
4257
    DBUG_RETURN(1);
4258
  }
4259
4260
  /* Copy the column definitions */
4261
  memcpy((uchar*) recdef,(uchar*) share.rec,
4262
	 (size_t) (sizeof(MI_COLUMNDEF)*(share.base.fields+1)));
4263
  for (rec=recdef,end=recdef+share.base.fields; rec != end ; rec++)
4264
  {
4265
    if (unpack && !(share.options & HA_OPTION_PACK_RECORD) &&
4266
	rec->type != FIELD_BLOB &&
4267
	rec->type != FIELD_VARCHAR &&
4268
	rec->type != FIELD_CHECK)
4269
      rec->type=(int) FIELD_NORMAL;
4270
  }
4271
4272
  /* Change the new key to point at the saved key segments */
4273
  memcpy((uchar*) keysegs,(uchar*) share.keyparts,
4274
	 (size_t) (sizeof(HA_KEYSEG)*(key_parts+share.base.keys+
4275
				      share.state.header.uniques)));
4276
  keyseg=keysegs;
4277
  for (key=keyinfo,key_end=keyinfo+share.base.keys; key != key_end ; key++)
4278
  {
4279
    key->seg=keyseg;
4280
    for (; keyseg->type ; keyseg++)
4281
    {
4282
      if (param->language)
4283
	keyseg->language=param->language;	/* change language */
4284
    }
4285
    keyseg++;					/* Skip end pointer */
4286
  }
4287
4288
  /* Copy the unique definitions and change them to point at the new key
4289
     segments*/
4290
  memcpy((uchar*) uniquedef,(uchar*) share.uniqueinfo,
4291
	 (size_t) (sizeof(MI_UNIQUEDEF)*(share.state.header.uniques)));
4292
  for (u_ptr=uniquedef,u_end=uniquedef+share.state.header.uniques;
4293
       u_ptr != u_end ; u_ptr++)
4294
  {
4295
    u_ptr->seg=keyseg;
4296
    keyseg+=u_ptr->keysegs+1;
4297
  }
4298
  if (share.options & HA_OPTION_COMPRESS_RECORD)
4299
    share.base.records=max_records=info.state->records;
4300
  else if (share.base.min_pack_length)
4301
    max_records=(ha_rows) (my_seek(info.dfile,0L,MY_SEEK_END,MYF(0)) /
4302
			   (ulong) share.base.min_pack_length);
4303
  else
4304
    max_records=0;
4305
  unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
4306
    (param->testflag & T_UNPACK);
4307
  share.options&= ~HA_OPTION_TEMP_COMPRESS_RECORD;
4308
4309
  file_length=(ulonglong) my_seek(info.dfile,0L,MY_SEEK_END,MYF(0));
4310
  tmp_length= file_length+file_length/10;
4311
  set_if_bigger(file_length,param->max_data_file_length);
4312
  set_if_bigger(file_length,tmp_length);
4313
  set_if_bigger(file_length,(ulonglong) share.base.max_data_file_length);
4314
4315
  VOID(mi_close(*org_info));
4316
  bzero((char*) &create_info,sizeof(create_info));
4317
  create_info.max_rows=max(max_records,share.base.records);
4318
  create_info.reloc_rows=share.base.reloc;
4319
  create_info.old_options=(share.options |
4320
			   (unpack ? HA_OPTION_TEMP_COMPRESS_RECORD : 0));
4321
4322
  create_info.data_file_length=file_length;
4323
  create_info.auto_increment=share.state.auto_increment;
4324
  create_info.language = (param->language ? param->language :
4325
			  share.state.header.language);
4326
  create_info.key_file_length=  status_info.key_file_length;
4327
  /*
4328
    Allow for creating an auto_increment key. This has an effect only if
4329
    an auto_increment key exists in the original table.
4330
  */
4331
  create_info.with_auto_increment= TRUE;
4332
  /* We don't have to handle symlinks here because we are using
4333
     HA_DONT_TOUCH_DATA */
4334
  if (mi_create(filename,
4335
		share.base.keys - share.state.header.uniques,
4336
		keyinfo, share.base.fields, recdef,
4337
		share.state.header.uniques, uniquedef,
4338
		&create_info,
4339
		HA_DONT_TOUCH_DATA))
4340
  {
4341
    mi_check_print_error(param,"Got error %d when trying to recreate indexfile",my_errno);
4342
    goto end;
4343
  }
4344
  *org_info=mi_open(filename,O_RDWR,
4345
		    (param->testflag & T_WAIT_FOREVER) ? HA_OPEN_WAIT_IF_LOCKED :
4346
		    (param->testflag & T_DESCRIPT) ? HA_OPEN_IGNORE_IF_LOCKED :
4347
		    HA_OPEN_ABORT_IF_LOCKED);
4348
  if (!*org_info)
4349
  {
4350
    mi_check_print_error(param,"Got error %d when trying to open re-created indexfile",
4351
		my_errno);
4352
    goto end;
4353
  }
4354
  /* We are modifing */
4355
  (*org_info)->s->options&= ~HA_OPTION_READ_ONLY_DATA;
4356
  VOID(_mi_readinfo(*org_info,F_WRLCK,0));
4357
  (*org_info)->state->records=info.state->records;
4358
  if (share.state.create_time)
4359
    (*org_info)->s->state.create_time=share.state.create_time;
4360
  (*org_info)->s->state.unique=(*org_info)->this_unique=
4361
    share.state.unique;
4362
  (*org_info)->state->checksum=info.state->checksum;
4363
  (*org_info)->state->del=info.state->del;
4364
  (*org_info)->s->state.dellink=share.state.dellink;
4365
  (*org_info)->state->empty=info.state->empty;
4366
  (*org_info)->state->data_file_length=info.state->data_file_length;
4367
  if (update_state_info(param,*org_info,UPDATE_TIME | UPDATE_STAT |
4368
			UPDATE_OPEN_COUNT))
4369
    goto end;
4370
  error=0;
4371
end:
4372
  my_afree((uchar*) uniquedef);
4373
  my_afree((uchar*) keyinfo);
4374
  my_afree((uchar*) recdef);
4375
  my_afree((uchar*) keysegs);
4376
  DBUG_RETURN(error);
4377
}
4378
4379
4380
	/* write suffix to data file if neaded */
4381
4382
int write_data_suffix(SORT_INFO *sort_info, my_bool fix_datafile)
4383
{
4384
  MI_INFO *info=sort_info->info;
4385
4386
  if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile)
4387
  {
4388
    uchar buff[MEMMAP_EXTRA_MARGIN];
4389
    bzero(buff,sizeof(buff));
4390
    if (my_b_write(&info->rec_cache,buff,sizeof(buff)))
4391
    {
4392
      mi_check_print_error(sort_info->param,
4393
			   "%d when writing to datafile",my_errno);
4394
      return 1;
4395
    }
4396
    sort_info->param->read_cache.end_of_file+=sizeof(buff);
4397
  }
4398
  return 0;
4399
}
4400
4401
	/* Update state and myisamchk_time of indexfile */
4402
4403
int update_state_info(MI_CHECK *param, MI_INFO *info,uint update)
4404
{
4405
  MYISAM_SHARE *share=info->s;
4406
4407
  if (update & UPDATE_OPEN_COUNT)
4408
  {
4409
    share->state.open_count=0;
4410
    share->global_changed=0;
4411
  }
4412
  if (update & UPDATE_STAT)
4413
  {
4414
    uint i, key_parts= mi_uint2korr(share->state.header.key_parts);
4415
    share->state.rec_per_key_rows=info->state->records;
4416
    share->state.changed&= ~STATE_NOT_ANALYZED;
4417
    if (info->state->records)
4418
    {
4419
      for (i=0; i<key_parts; i++)
4420
      {
4421
        if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i]))
4422
          share->state.changed|= STATE_NOT_ANALYZED;
4423
      }
4424
    }
4425
  }
4426
  if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC))
4427
  {
4428
    if (update & UPDATE_TIME)
4429
    {
4430
      share->state.check_time= (long) time((time_t*) 0);
4431
      if (!share->state.create_time)
4432
	share->state.create_time=share->state.check_time;
4433
    }
4434
    /*
4435
      When tables are locked we haven't synched the share state and the
4436
      real state for a while so we better do it here before synching
4437
      the share state to disk. Only when table is write locked is it
4438
      necessary to perform this synch.
4439
    */
4440
    if (info->lock_type == F_WRLCK)
4441
      share->state.state= *info->state;
4442
    if (mi_state_info_write(share->kfile,&share->state,1+2))
4443
      goto err;
4444
    share->changed=0;
4445
  }
4446
  {						/* Force update of status */
4447
    int error;
4448
    uint r_locks=share->r_locks,w_locks=share->w_locks;
4449
    share->r_locks= share->w_locks= share->tot_locks= 0;
4450
    error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
4451
    share->r_locks=r_locks;
4452
    share->w_locks=w_locks;
4453
    share->tot_locks=r_locks+w_locks;
4454
    if (!error)
4455
      return 0;
4456
  }
4457
err:
4458
  mi_check_print_error(param,"%d when updating keyfile",my_errno);
4459
  return 1;
4460
}
4461
4462
	/*
4463
	  Update auto increment value for a table
4464
	  When setting the 'repair_only' flag we only want to change the
4465
	  old auto_increment value if its wrong (smaller than some given key).
4466
	  The reason is that we shouldn't change the auto_increment value
4467
	  for a table without good reason when only doing a repair; If the
4468
	  user have inserted and deleted rows, the auto_increment value
4469
	  may be bigger than the biggest current row and this is ok.
4470
4471
	  If repair_only is not set, we will update the flag to the value in
4472
	  param->auto_increment is bigger than the biggest key.
4473
	*/
4474
4475
void update_auto_increment_key(MI_CHECK *param, MI_INFO *info,
4476
			       my_bool repair_only)
4477
{
4478
  uchar *record= 0;
4479
  DBUG_ENTER("update_auto_increment_key");
4480
4481
  if (!info->s->base.auto_key ||
4482
      ! mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1))
4483
  {
4484
    if (!(param->testflag & T_VERY_SILENT))
4485
      mi_check_print_info(param,
4486
			  "Table: %s doesn't have an auto increment key\n",
4487
			  param->isam_file_name);
4488
    DBUG_VOID_RETURN;
4489
  }
4490
  if (!(param->testflag & T_SILENT) &&
4491
      !(param->testflag & T_REP))
4492
    printf("Updating MyISAM file: %s\n", param->isam_file_name);
4493
  /*
4494
    We have to use an allocated buffer instead of info->rec_buff as 
4495
    _mi_put_key_in_record() may use info->rec_buff
4496
  */
4497
  if (!mi_alloc_rec_buff(info, -1, &record))
4498
  {
4499
    mi_check_print_error(param,"Not enough memory for extra record");
4500
    DBUG_VOID_RETURN;
4501
  }
4502
4503
  mi_extra(info,HA_EXTRA_KEYREAD,0);
4504
  if (mi_rlast(info, record, info->s->base.auto_key-1))
4505
  {
4506
    if (my_errno != HA_ERR_END_OF_FILE)
4507
    {
4508
      mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4509
      my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
4510
      mi_check_print_error(param,"%d when reading last record",my_errno);
4511
      DBUG_VOID_RETURN;
4512
    }
4513
    if (!repair_only)
4514
      info->s->state.auto_increment=param->auto_increment_value;
4515
  }
4516
  else
4517
  {
4518
    ulonglong auto_increment= retrieve_auto_increment(info, record);
4519
    set_if_bigger(info->s->state.auto_increment,auto_increment);
4520
    if (!repair_only)
4521
      set_if_bigger(info->s->state.auto_increment, param->auto_increment_value);
4522
  }
4523
  mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4524
  my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
4525
  update_state_info(param, info, UPDATE_AUTO_INC);
4526
  DBUG_VOID_RETURN;
4527
}
4528
4529
4530
/*
4531
  Update statistics for each part of an index
4532
4533
  SYNOPSIS
4534
    update_key_parts()
4535
      keyinfo           IN  Index information (only key->keysegs used)
4536
      rec_per_key_part  OUT Store statistics here
4537
      unique            IN  Array of (#distinct tuples)
4538
      notnull_tuples    IN  Array of (#tuples), or NULL
4539
      records               Number of records in the table
4540
4541
  DESCRIPTION
4542
    This function is called produce index statistics values from unique and 
4543
    notnull_tuples arrays after these arrays were produced with sequential
4544
    index scan (the scan is done in two places: chk_index() and
4545
    sort_key_write()).
4546
4547
    This function handles all 3 index statistics collection methods.
4548
4549
    Unique is an array:
4550
      unique[0]= (#different values of {keypart1}) - 1
4551
      unique[1]= (#different values of {keypart1,keypart2} tuple)-unique[0]-1
4552
      ...
4553
4554
    For MI_STATS_METHOD_IGNORE_NULLS method, notnull_tuples is an array too:
4555
      notnull_tuples[0]= (#of {keypart1} tuples such that keypart1 is not NULL)
4556
      notnull_tuples[1]= (#of {keypart1,keypart2} tuples such that all 
4557
                          keypart{i} are not NULL)
4558
      ...
4559
    For all other statistics collection methods notnull_tuples==NULL.
4560
4561
    Output is an array:
4562
    rec_per_key_part[k] = 
4563
     = E(#records in the table such that keypart_1=c_1 AND ... AND 
4564
         keypart_k=c_k for arbitrary constants c_1 ... c_k) 
4565
     
4566
     = {assuming that values have uniform distribution and index contains all
4567
        tuples from the domain (or that {c_1, ..., c_k} tuple is choosen from
4568
        index tuples}
4569
     
4570
     = #tuples-in-the-index / #distinct-tuples-in-the-index.
4571
    
4572
    The #tuples-in-the-index and #distinct-tuples-in-the-index have different 
4573
    meaning depending on which statistics collection method is used:
4574
    
4575
    MI_STATS_METHOD_*  how are nulls compared?  which tuples are counted?
4576
     NULLS_EQUAL            NULL == NULL           all tuples in table
4577
     NULLS_NOT_EQUAL        NULL != NULL           all tuples in table
4578
     IGNORE_NULLS               n/a             tuples that don't have NULLs
4579
*/
4580
4581
void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
4582
                      ulonglong *unique, ulonglong *notnull,
4583
                      ulonglong records)
4584
{
4585
  ulonglong count=0,tmp, unique_tuples;
4586
  ulonglong tuples= records;
4587
  uint parts;
4588
  for (parts=0 ; parts < keyinfo->keysegs  ; parts++)
4589
  {
4590
    count+=unique[parts];
4591
    unique_tuples= count + 1;    
4592
    if (notnull)
4593
    {
4594
      tuples= notnull[parts];
4595
      /* 
4596
        #(unique_tuples not counting tuples with NULLs) = 
4597
          #(unique_tuples counting tuples with NULLs as different) - 
4598
          #(tuples with NULLs)
4599
      */
4600
      unique_tuples -= (records - notnull[parts]);
4601
    }
4602
    
4603
    if (unique_tuples == 0)
4604
      tmp= 1;
4605
    else if (count == 0)
4606
      tmp= tuples; /* 1 unique tuple */
4607
    else
4608
      tmp= (tuples + unique_tuples/2) / unique_tuples;
4609
4610
    /* 
4611
      for some weird keys (e.g. FULLTEXT) tmp can be <1 here. 
4612
      let's ensure it is not
4613
    */
4614
    set_if_bigger(tmp,1);
4615
    if (tmp >= (ulonglong) ~(ulong) 0)
4616
      tmp=(ulonglong) ~(ulong) 0;
4617
4618
    *rec_per_key_part=(ulong) tmp;
4619
    rec_per_key_part++;
4620
  }
4621
}
4622
4623
4624
static ha_checksum mi_byte_checksum(const uchar *buf, uint length)
4625
{
4626
  ha_checksum crc;
4627
  const uchar *end=buf+length;
4628
  for (crc=0; buf != end; buf++)
4629
    crc=((crc << 1) + *((uchar*) buf)) +
4630
      test(crc & (((ha_checksum) 1) << (8*sizeof(ha_checksum)-1)));
4631
  return crc;
4632
}
4633
4634
static my_bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)
4635
{
4636
  uint key_maxlength=key->maxlength;
4637
  if (key->flag & HA_FULLTEXT)
4638
  {
4639
    uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
4640
                                  key->seg->charset->mbmaxlen;
4641
    key_maxlength+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
4642
  }
4643
  return (key->flag & HA_SPATIAL) ||
4644
          (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) &&
4645
	  ((ulonglong) rows * key_maxlength >
4646
	   (ulonglong) myisam_max_temp_length));
4647
}
4648
4649
/*
4650
  Deactivate all not unique index that can be recreated fast
4651
  These include packed keys on which sorting will use more temporary
4652
  space than the max allowed file length or for which the unpacked keys
4653
  will take much more space than packed keys.
4654
  Note that 'rows' may be zero for the case when we don't know how many
4655
  rows we will put into the file.
4656
 */
4657
4658
void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows)
4659
{
4660
  MYISAM_SHARE *share=info->s;
4661
  MI_KEYDEF    *key=share->keyinfo;
4662
  uint          i;
4663
4664
  DBUG_ASSERT(info->state->records == 0 &&
4665
              (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
4666
  for (i=0 ; i < share->base.keys ; i++,key++)
4667
  {
4668
    if (!(key->flag & (HA_NOSAME | HA_SPATIAL | HA_AUTO_KEY)) &&
4669
        ! mi_too_big_key_for_sort(key,rows) && info->s->base.auto_key != i+1)
4670
    {
4671
      mi_clear_key_active(share->state.key_map, i);
4672
      info->update|= HA_STATE_CHANGED;
4673
    }
4674
  }
4675
}
4676
4677
4678
/*
4679
  Return TRUE if we can use repair by sorting
4680
  One can set the force argument to force to use sorting
4681
  even if the temporary file would be quite big!
4682
*/
4683
4684
my_bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows,
4685
			    ulonglong key_map, my_bool force)
4686
{
4687
  MYISAM_SHARE *share=info->s;
4688
  MI_KEYDEF *key=share->keyinfo;
4689
  uint i;
4690
4691
  /*
4692
    mi_repair_by_sort only works if we have at least one key. If we don't
4693
    have any keys, we should use the normal repair.
4694
  */
4695
  if (! mi_is_any_key_active(key_map))
4696
    return FALSE;				/* Can't use sort */
4697
  for (i=0 ; i < share->base.keys ; i++,key++)
4698
  {
4699
    if (!force && mi_too_big_key_for_sort(key,rows))
4700
      return FALSE;
4701
  }
4702
  return TRUE;
4703
}
4704
4705
4706
static void
4707
set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share)
4708
{
4709
  if ((sort_info->new_data_file_type=share->data_file_type) ==
4710
      COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK)
4711
  {
4712
    MYISAM_SHARE tmp;
4713
4714
    if (share->options & HA_OPTION_PACK_RECORD)
4715
      sort_info->new_data_file_type = DYNAMIC_RECORD;
4716
    else
4717
      sort_info->new_data_file_type = STATIC_RECORD;
4718
4719
    /* Set delete_function for sort_delete_record() */
4720
    memcpy((char*) &tmp, share, sizeof(*share));
4721
    tmp.options= ~HA_OPTION_COMPRESS_RECORD;
4722
    mi_setup_functions(&tmp);
4723
    share->delete_record=tmp.delete_record;
4724
  }
4725
}