~drizzle-trunk/drizzle/development

1 by brian
clean slate
1
#!/usr/bin/perl
2
# ======================================================================
3
#                     MySQL server stress test system 
4
# ======================================================================
5
#
6
##########################################################################
7
#
8
#                       SCENARIOS AND REQUIREMENTS
9
#
10
#   The system should perform stress testing of MySQL server with 
11
# following requirements and basic scenarios:
12
# 
13
# Basic requirements:
14
# 
15
# Design of stress script should allow one:
16
# 
17
#   - To stress test the mysqltest binary test engine.
18
#   - To stress test the regular test suite and any additional test suites
19
#     (such as mysql-test-extra-5.0).
20
#   - To specify files with lists of tests both for initialization of
21
#     stress db and for further testing itself.
22
#   - To define the number of threads to be concurrently used in testing.
23
#   - To define limitations for the test run. such as the number of tests or
24
#     loops for execution or duration of testing, delay between test
25
#     executions, and so forth.
26
#   - To get a readable log file that can be used for identification of
27
#     errors that occur during testing.
28
# 
29
# Basic scenarios:
30
# 
31
#     * It should be possible to run stress script in standalone mode
32
#       which will allow to create various scenarios of stress workloads:
33
# 
34
#       simple ones:
35
#
36
#         box #1:
37
#           - one instance of script with list of tests #1
38
# 
39
#       and more advanced ones:
40
# 
41
#         box #1:
42
#           - one instance of script with list of tests #1
43
#           - another instance of script with list of tests #2
44
#         box #2:
45
#           - one instance of script with list of tests #3
46
#           - another instance of script with list of tests #4
47
#             that will recreate whole database to back it to clean
48
#             state
49
# 
50
#       One kind of such complex scenarios maybe continued testing
51
#       when we want to run stress tests from many boxes with various
52
#       lists of tests that will last very long time. And in such case
53
#       we need some wrapper for MySQL server that will restart it in
54
#       case of crashes.
55
# 
56
#     * It should be possible to run stress script in ad-hoc mode from
57
#       shell or perl versions of mysql-test-run. This allows developers
58
#       to reproduce and debug errors that was found in continued stress 
59
#       testing
60
#
61
########################################################################
62
63
use Config;
64
65
if (!defined($Config{useithreads}))
66
{
67
  die <<EOF;
68
It is unable to run threaded version of stress test on this system 
69
due to disabled ithreads. Please check that installed perl binary 
70
was built with support of ithreads. 
71
EOF
72
}
73
74
use threads;
75
use threads::shared;
76
77
use IO::Socket;
78
use Sys::Hostname;
79
use File::Copy;
80
use File::Spec;
81
use File::Find;
82
use File::Basename;
83
use File::Path;
84
use Cwd;
85
86
use Data::Dumper;
87
use Getopt::Long;
88
89
my $stress_suite_version="1.0";
90
91
$|=1;
92
93
$opt_server_host="";
94
$opt_server_logs_dir="";
95
$opt_help="";
96
$opt_server_port="";
97
$opt_server_socket="";
98
$opt_server_user="";
99
$opt_server_password="";
100
$opt_server_database="";
101
$opt_cleanup="";
102
$opt_verbose="";
103
$opt_log_error_details="";
104
105
106
$opt_suite="main";
107
$opt_stress_suite_basedir="";
108
$opt_stress_basedir="";
109
$opt_stress_datadir="";
110
$opt_test_suffix="";
111
112
$opt_stress_mode="random";
113
114
$opt_loop_count=0;
115
$opt_test_count=0;
116
$opt_test_duration=0;
117
$opt_abort_on_error=0;
118
$opt_sleep_time = 0;
119
$opt_threads=1;
120
$pid_file="mysql_stress_test.pid";
121
$opt_mysqltest= ($^O =~ /mswin32/i) ? "mysqltest.exe" : "mysqltest";
122
$opt_check_tests_file="";
123
@mysqltest_args=("--silent", "-v", "--skip-safemalloc");
124
125
# Client ip address
126
$client_ip=inet_ntoa((gethostbyname(hostname()))[4]);
127
$client_ip=~ s/\.//g;
128
129
%tests_files=(client => {mtime => 0, data => []},
130
              initdb => {mtime => 0, data => []});
131
132
# Error codes and sub-strings with corresponding severity 
133
#
134
# S1 - Critical errors - cause immediately abort of testing. These errors
135
#                        could be caused by server crash or impossibility
136
#                        of test execution
137
#
138
# S2 - Serious errors - these errors are bugs for sure as it knowns that 
139
#                       they shouldn't appear during stress testing  
140
#
141
# S3 - Non-seriuos errros - these errors could be caused by fact that 
142
#                           we execute simultaneously statements that
143
#                           affect tests executed by other threads
144
                            
145
%error_strings = ( 'Failed in mysql_real_connect()' => S1,
146
                   'not found (Errcode: 2)' => S1 );
147
  
148
%error_codes = ( 1012 => S2, 1015 => S2, 1021 => S2,
149
                 1027 => S2, 1037 => S2, 1038 => S2,
150
                 1039 => S2, 1040 => S2, 1046 => S2, 
151
                 1180 => S2, 1181 => S2, 1203 => S2,
152
                 1205 => S2, 1206 => S2, 1207 => S2, 
153
                 1223 => S2, 2013 => S1);
154
155
share(%test_counters);
156
%test_counters=( loop_count => 0, test_count=>0);
157
158
share($exiting);
159
$exiting=0;
160
161
share($test_counters_lock);
162
$test_counters_lock=0;
163
share($log_file_lock);
164
$log_file_lock=0;
165
166
$SIG{INT}= \&sig_INT_handler;
167
$SIG{TERM}= \&sig_TERM_handler;
168
169
170
GetOptions("server-host=s", "server-logs-dir=s", "server-port=s",
171
           "server-socket=s", "server-user=s", "server-password=s",
172
           "server-database=s",
173
           "stress-suite-basedir=s", "suite=s", "stress-init-file:s", 
174
           "stress-tests-file:s", "stress-basedir=s", "stress-mode=s",
175
           "stress-datadir=s",
176
           "threads=s", "sleep-time=s", "loop-count=i", "test-count=i",
177
           "test-duration=i", "test-suffix=s", "check-tests-file", 
178
           "verbose", "log-error-details", "cleanup", "mysqltest=s", 
179
           "abort-on-error", "help") || usage();
180
181
usage() if ($opt_help);
182
183
#$opt_abort_on_error=1;
184
185
$test_dirname=get_timestamp();
186
$test_dirname.="-$opt_test_suffix" if ($opt_test_suffix ne '');
187
188
print <<EOF;
189
#############################################################
190
                  CONFIGURATION STAGE
191
#############################################################
192
EOF
193
194
if ($opt_stress_basedir eq '' || $opt_stress_suite_basedir eq '' ||
195
    $opt_server_logs_dir eq '')
196
{
197
  die <<EOF;
198
199
Options --stress-basedir, --stress-suite-basedir and --server-logs-dir are 
200
required. Please use these options to specify proper basedir for 
201
client, test suite and location of server logs.
202
203
stress-basedir: '$opt_stress_basedir'
204
stress-suite-basedir: '$opt_stress_suite_basedir'
205
server-logs-dir: '$opt_server_logs_dir'
206
207
EOF
208
}
209
210
#Workaround for case when we got relative but not absolute path 
211
$opt_stress_basedir=File::Spec->rel2abs($opt_stress_basedir);
212
$opt_stress_suite_basedir=File::Spec->rel2abs($opt_stress_suite_basedir);
213
$opt_server_logs_dir=File::Spec->rel2abs($opt_server_logs_dir);
214
215
if ($opt_stress_datadir ne '')
216
{
217
  $opt_stress_datadir=File::Spec->rel2abs($opt_stress_datadir);
218
}
219
220
if (! -d "$opt_stress_basedir")
221
{
222
  die <<EOF;
223
  
224
Directory '$opt_stress_basedir' does not exist.
225
Use --stress-basedir option to specify proper basedir for client
226
227
EOF
228
}
229
230
if (!-d $opt_stress_suite_basedir)
231
{
232
  die <<EOF;
233
234
Directory '$opt_stress_suite_basedir' does not exist.
235
Use --stress-suite-basedir option to specify proper basedir for test suite
236
237
EOF
238
}
239
240
$test_dataset_dir=$opt_stress_suite_basedir;
241
if ($opt_stress_datadir ne '')
242
{
243
  if (-d $opt_stress_datadir)
244
  {
245
    $test_dataset_dir=$opt_stress_datadir;
246
247
  }
248
  else
249
  {
250
    die <<EOF;
251
Directory '$opt_stress_datadir' not exists. Please specify proper one 
252
with --stress-datadir option.
253
EOF
254
  }  
255
}
256
257
if ($^O =~ /mswin32/i)
258
{
259
  $test_dataset_dir=~ s/\\/\\\\/g;
260
}
261
else
262
{
263
  $test_dataset_dir.="/";
264
}
265
266
267
268
if (!-d $opt_server_logs_dir)
269
{
270
  die <<EOF;
271
272
Directory server-logs-dir '$opt_server_logs_dir' does not exist.
273
Use --server-logs-dir option to specify proper directory for storing 
274
logs 
275
276
EOF
277
}
278
else
279
{
280
  #Create sub-directory for test session logs
281
  mkpath(File::Spec->catdir($opt_server_logs_dir, $test_dirname), 0, 0755);
282
  #Define filename of global session log file
283
  $stress_log_file=File::Spec->catfile($opt_server_logs_dir, $test_dirname,
284
                                       "mysql-stress-test.log");
285
}
286
287
if ($opt_suite ne '' && $opt_suite ne 'main' && $opt_suite ne 'default')
288
{
289
  $test_suite_dir=File::Spec->catdir($opt_stress_suite_basedir, "suite", $opt_suite);
290
}
291
else
292
{
293
  $test_suite_dir= $opt_stress_suite_basedir;
294
}
295
296
if (!-d $test_suite_dir)
297
{
298
  die <<EOF
299
300
Directory '$test_suite_dir' does not exist.
301
Use --suite options to specify proper dir for test suite
302
303
EOF
304
}
305
306
$test_suite_t_path=File::Spec->catdir($test_suite_dir,'t');
307
$test_suite_r_path=File::Spec->catdir($test_suite_dir,'r');
308
309
foreach my $suite_dir ($test_suite_t_path, $test_suite_r_path)
310
{
311
  if (!-d $suite_dir)
312
  {
313
    die <<EOF;
314
315
Directory '$suite_dir' does not exist.
316
Please ensure that you specified proper source location for 
317
test/result files with --stress-suite-basedir option and name 
318
of test suite with --suite option
319
320
EOF
321
  }
322
}
323
324
$test_t_path=File::Spec->catdir($opt_stress_basedir,'t');
325
$test_r_path=File::Spec->catdir($opt_stress_basedir,'r');
326
327
foreach $test_dir ($test_t_path, $test_r_path)
328
{
329
  if (-d $test_dir)
330
  {
331
    if ($opt_cleanup)
332
    {
333
      #Delete existing 't', 'r', 'r/*' subfolders in $stress_basedir
334
      rmtree("$test_dir", 0, 0);
335
      print "Cleanup $test_dir\n";
336
    }
337
    else
338
    {
339
      die <<EOF;
340
Directory '$test_dir' already exist. 
341
Please ensure that you specified proper location of working dir
342
for current test run with --stress-basedir option or in case of staled
343
directories use --cleanup option to remove ones
344
EOF
345
    }
346
  }
347
  #Create empty 't', 'r' subfolders that will be filled later
348
  mkpath("$test_dir", 0, 0777);
349
}
350
351
if (!defined($opt_stress_tests_file) && !defined($opt_stress_init_file))
352
{
353
  die <<EOF;
354
You should run stress script either with --stress-tests-file or with 
355
--stress-init-file otions. See help for details.
356
EOF
357
}
358
359
if (defined($opt_stress_tests_file))
360
{ 
361
  if ($opt_stress_tests_file eq '')
362
  {
363
    #Default location of file with set of tests for current test run
364
    $tests_files{client}->{filename}= File::Spec->catfile($opt_stress_suite_basedir,
365
                                      "testslist_client.txt");
366
  }
367
  else
368
  {
369
    $tests_files{client}->{filename}= $opt_stress_tests_file;
370
  }
371
372
  if (!-f $tests_files{client}->{filename})
373
  {
374
    die <<EOF;
375
376
File '$tests_files{client}->{filename}' with list of tests not exists. 
377
Please ensure that this file exists, readable or specify another one with 
378
--stress-tests-file option.
379
380
EOF
381
  }
382
}
383
384
if (defined($opt_stress_init_file))
385
{
386
  if ($opt_stress_init_file eq '')
387
  {
388
    #Default location of file with set of tests for current test run
389
    $tests_files{initdb}->{filename}= File::Spec->catfile($opt_stress_suite_basedir,
390
                                      "testslist_initdb.txt");
391
  }
392
  else
393
  {
394
    $tests_files{initdb}->{filename}= $opt_stress_init_file;
395
  }
396
397
  if (!-f $tests_files{initdb}->{filename})
398
  {
399
    die <<EOF;
400
401
File '$tests_files{initdb}->{filename}' with list of tests for initialization of database
402
for stress test not exists. 
403
Please ensure that this file exists, readable or specify another one with 
404
--stress-init-file option.
405
406
EOF
407
  }
408
}
409
410
if ($opt_stress_mode !~ /^(random|seq)$/)
411
{
412
  die <<EOF
413
Was specified wrong --stress-mode. Correct values 'random' and 'seq'.
414
EOF
415
}
416
417
if (open(TEST, "$opt_mysqltest -V |"))
418
{
419
  $mysqltest_version=join("",<TEST>);
420
  close(TEST);
421
  print "FOUND MYSQLTEST BINARY: ", $mysqltest_version,"\n";
422
}
423
else
424
{
425
  die <<EOF;
426
ERROR: mysqltest binary $opt_mysqltest not found $!.
427
You must either specify file location explicitly using --mysqltest
428
option, or make sure path to mysqltest binary is listed 
429
in your PATH environment variable.
430
EOF
431
}
432
433
#        
434
#Adding mysql server specific command line options for mysqltest binary
435
#
436
$opt_server_host= $opt_server_host ? $opt_server_host : "localhost";
437
$opt_server_port= $opt_server_port ? $opt_server_port : "3306";
438
$opt_server_user= $opt_server_user ? $opt_server_user : "root";
439
$opt_server_socket= $opt_server_socket ? $opt_server_socket : "/tmp/mysql.sock";
440
$opt_server_database= $opt_server_database ? $opt_server_database : "test";
441
442
unshift @mysqltest_args, "--host=$opt_server_host";
443
unshift @mysqltest_args, "--port=$opt_server_port";
444
unshift @mysqltest_args, "--user=$opt_server_user";
445
unshift @mysqltest_args, "--password=$opt_server_password";
446
unshift @mysqltest_args, "--socket=$opt_server_socket";
447
unshift @mysqltest_args, "--database=$opt_server_database";
448
449
#Export variables that could be used in tests
450
$ENV{MYSQL_TEST_DIR}=$test_dataset_dir;
451
$ENV{MASTER_MYPORT}=$opt_server_port;
452
$ENV{MASTER_MYSOCK}=$opt_server_socket;
453
454
print <<EOF;
455
TEST-SUITE-BASEDIR: $opt_stress_suite_basedir
456
SUITE:              $opt_suite
457
TEST-BASE-DIR:      $opt_stress_basedir
458
TEST-DATADIR:       $test_dataset_dir
459
SERVER-LOGS-DIR:    $opt_server_logs_dir
460
461
THREADS:            $opt_threads
462
TEST-MODE:          $opt_stress_mode
463
464
EOF
465
466
#-------------------------------------------------------------------------------
467
#At this stage we've already checked all needed pathes/files 
468
#and ready to start the test
469
#-------------------------------------------------------------------------------
470
471
if (defined($opt_stress_tests_file) || defined($opt_stress_init_file))
472
{
473
  print <<EOF;
474
#############################################################
475
                  PREPARATION STAGE
476
#############################################################
477
EOF
478
479
  #Copy Test files from network share to 't' folder
480
  print "\nCopying Test files from $test_suite_t_path to $test_t_path folder...";
481
  find({wanted=>\&copy_test_files, bydepth=>1}, "$test_suite_t_path");
482
  print "Done\n";
483
484
  #$test_r_path/r0 dir reserved for initdb
485
  $count_start= defined($opt_stress_init_file) ? 0 : 1;
486
487
  our $r_folder='';
488
  print "\nCreating 'r' folder and copying Protocol files to each 'r#' sub-folder...";
489
  for($count=$count_start; $count <= $opt_threads; $count++)
490
  {
491
    $r_folder = File::Spec->catdir($test_r_path, "r".$count);
492
    mkpath("$r_folder", 0, 0777); 
493
     
494
    find(\&copy_result_files,"$test_suite_r_path");
495
  }  
496
  print "Done\n\n";
497
}
498
499
if (defined($opt_stress_init_file))
500
{
501
  print <<EOF;
502
#############################################################
503
                  INITIALIZATION STAGE
504
#############################################################
505
EOF
506
507
  #Set limits for stress db initialization 
508
  %limits=(loop_count => 1, test_count => undef);
509
510
  #Read list of tests from $opt_stress_init_file
511
  read_tests_names($tests_files{initdb});
512
  test_loop($client_ip, 0, 'seq', $tests_files{initdb});  
513
  #print Dumper($tests_files{initdb}),"\n";
514
  print <<EOF;
515
516
Done initialization of stress database by tests from 
517
$tests_files{initdb}->{filename} file.
518
519
EOF
520
}
521
522
if (defined($opt_stress_tests_file))
523
{
524
  print <<EOF;
525
#############################################################
526
                  STRESS TEST RUNNING STAGE
527
#############################################################
528
EOF
529
530
  $exiting=0;
531
  #Read list of tests from $opt_stress_tests_file 
532
  read_tests_names($tests_files{client});
533
534
  #Reset current counter and set limits
535
  %test_counters=( loop_count => 0, test_count=>0);
536
  %limits=(loop_count => $opt_loop_count, test_count => $opt_test_count);
537
538
  if (($opt_loop_count && $opt_threads > $opt_loop_count) || 
539
      ($opt_test_count && $opt_threads > $opt_test_count))
540
  {
541
    warn <<EOF;
542
543
WARNING: Possible inaccuracies in number of executed loops or 
544
         tests because number of threads bigger than number of 
545
         loops or tests:
546
         
547
         Threads will be started: $opt_threads
548
         Loops will be executed:  $opt_loop_count
549
         Tests will be executed:  $opt_test_count    
550
551
EOF
552
  }
553
554
  #Create threads (number depending on the variable )
555
  for ($id=1; $id<=$opt_threads && !$exiting; $id++)
556
  {
557
    $thrd[$id] = threads->create("test_loop", $client_ip, $id,
558
                                 $opt_stress_mode, $tests_files{client});
559
560
    print "main: Thread ID $id TID ",$thrd[$id]->tid," started\n";
561
    select(undef, undef, undef, 0.5);
562
  }
563
564
  if ($opt_test_duration)
565
  {
566
    sleep($opt_test_duration);
567
    kill INT, $$;                             #Interrupt child threads
568
  }
569
570
  #Let other threads to process INT signal
571
  sleep(1);
572
573
  for ($id=1; $id<=$opt_threads;$id++)
574
  {
575
    if (defined($thrd[$id]))
576
    {
577
      $thrd[$id]->join();
578
    }
579
  }
580
  print "EXIT\n";
581
}
582
583
sub test_init
584
{
585
  my ($env)=@_;
586
  
587
  $env->{session_id}=$env->{ip}."_".$env->{thread_id};
588
  $env->{r_folder}='r'.$env->{thread_id}; 
589
  $env->{screen_logs}=File::Spec->catdir($opt_server_logs_dir, $test_dirname, 
590
                                         "screen_logs", $env->{session_id});
591
  $env->{reject_logs}=File::Spec->catdir($opt_server_logs_dir, $test_dirname,
592
                                         "reject_logs", $env->{session_id});
593
  
594
  mkpath($env->{screen_logs}, 0, 0755) unless (-d $env->{screen_logs});
595
  mkpath($env->{reject_logs}, 0, 0755) unless (-d $env->{reject_logs});
596
597
  $env->{session_log}= File::Spec->catfile($env->{screen_logs}, $env->{session_id}.".log");     
598
}
599
600
sub test_execute
601
{
602
  my $env= shift;
603
  my $test_name= shift;
604
605
  my $g_start= "";
606
  my $g_end= "";
607
  my $mysqltest_cmd= "";
608
  my @mysqltest_test_args=();
609
  my @stderr=();
610
611
  #Get time stamp
612
  $g_start = get_timestamp();
613
  $env->{errors}={};
614
  @{$env->{test_status}}=();
615
616
  my $test_file= $test_name.".test";
617
  my $result_file= $test_name.".result";
618
  my $reject_file = $test_name.'.reject';
619
  my $output_file = $env->{session_id}.'_'.$test_name.'_'.$g_start."_".$env->{test_count}.'.txt';
620
621
  my $test_filename = File::Spec->catfile($test_t_path, $test_file);
622
  my $result_filename = File::Spec->catdir($test_r_path, $env->{r_folder}, $result_file);
623
  my $reject_filename = File::Spec->catdir($test_r_path, $env->{r_folder}, $reject_file);
624
  my $output_filename = File::Spec->catfile($env->{screen_logs}, $output_file);     
625
626
627
  push @mysqltest_test_args, "--basedir=$opt_stress_suite_basedir/",
628
                             "--tmpdir=$opt_stress_basedir",
629
                             "-x $test_filename",
630
                             "-R $result_filename",
631
                             "2>$output_filename";
632
                        
633
  $cmd= "$opt_mysqltest --no-defaults ".join(" ", @mysqltest_args)." ".
634
                                        join(" ", @mysqltest_test_args);
635
636
  system($cmd);
637
638
  $exit_value  = $? >> 8;
639
  $signal_num  = $? & 127;
640
  $dumped_core = $? & 128;
641
642
  my $tid= threads->self->tid;
643
644
  if (-s $output_filename > 0)
645
  { 
646
    #Read stderr for further analysis
647
    open (STDERR_LOG, $output_filename) or 
648
                             warn "Can't open file $output_filename";
649
    @stderr=<STDERR_LOG>;
650
    close(STDERR_LOG);
651
 
652
    if ($opt_verbose)
653
    {
654
      $session_debug_file="$opt_stress_basedir/error$tid.txt";
655
      
656
      stress_log($session_debug_file, 
657
                "Something wrong happened during execution of this command line:");
658
      stress_log($session_debug_file, "MYSQLTEST CMD - $cmd");    
659
      stress_log($session_debug_file, "STDERR:".join("",@stderr));      
660
661
      stress_log($session_debug_file, "EXIT STATUS:\n1. EXIT: $exit_value \n".
662
                                      "2. SIGNAL: $signal_num\n".
663
                                      "3. CORE: $dumped_core\n");
664
    }
665
  }
666
667
  #If something wrong trying to analyse stderr 
668
  if ($exit_value || $signal_num)
669
  {
670
    if (@stderr)
671
    {
672
      foreach my $line (@stderr)
673
      {
674
        #FIXME: we should handle case when for one sub-string/code 
675
        #       we have several different error messages        
676
        #       Now for both codes/substrings we assume that
677
        #       first found message will represent error
678
679
        #Check line for error codes
680
        if (($err_msg, $err_code)= $line=~/failed: ((\d+):.+?$)/)
681
        {
682
          if (!exists($error_codes{$err_code}))
683
          {
684
            $severity="S3";
685
            $err_code=0;
686
          }
687
          else
688
          {
689
            $severity=$error_codes{$err_code};
690
          }
691
692
          if (!exists($env->{errors}->{$severity}->{$err_code}))
693
          {
694
            $env->{errors}->{$severity}->{$err_code}=[0, $err_msg];
695
          }
696
          $env->{errors}->{$severity}->{$err_code}->[0]++;
697
          $env->{errors}->{$severity}->{total}++;          
698
        }
699
700
        #Check line for error patterns
701
        foreach $err_string (keys %error_strings)
702
        {
703
          $pattern= quotemeta $err_string;
704
          if ($line =~ /$pattern/i)
705
          {
706
            my $severity= $error_strings{$err_string};
707
            if (!exists($env->{errors}->{$severity}->{$err_string}))
708
            {
709
              $env->{errors}->{$severity}->{$err_string}=[0, $line];
710
            }
711
            $env->{errors}->{$severity}->{$err_string}->[0]++;
712
            $env->{errors}->{$severity}->{total}++;          
713
          }
714
        }
715
      }
716
    }
717
    else
718
    {
719
      $env->{errors}->{S3}->{'Unknown error'}=
720
                              [1,"Unknown error. Nothing was output to STDERR"];
721
      $env->{errors}->{S3}->{total}=1;
722
    }
723
  }
724
725
  #
726
  #FIXME: Here we can perform further analysis of recognized 
727
  #       error codes 
728
  #
729
730
  foreach my $severity (sort {$a cmp $b} keys %{$env->{errors}})
731
  {
732
    my $total=$env->{errors}->{$severity}->{total};
733
    if ($total)
734
    {
735
      push @{$env->{test_status}}, "Severity $severity: $total";
736
      $env->{errors}->{total}=+$total;
737
    }
738
  }
739
740
  #FIXME: Should we take into account $exit_value here?
741
  #       Now we assume that all stringified errors(i.e. errors without 
742
  #       error codes) which are not exist in %error_string structure 
743
  #       are OK
744
  if (!$env->{errors}->{total})
745
  {
746
    push @{$env->{test_status}},"No Errors. Test Passed OK";
747
  }
748
749
  log_session_errors($env, $test_file);
750
751
  if (!$exiting && ($signal_num == 2 || $signal_num == 15 || 
752
      ($opt_abort_on_error && $env->{errors}->{S1} > 0)))
753
  {
754
    #mysqltest was interrupted with INT or TERM signals or test was 
755
    #ran with --abort-on-error option and we got errors with severity S1
756
    #so we assume that we should cancel testing and exit
757
    $exiting=1;
758
    print STDERR<<EOF;
759
WARNING:
760
   mysqltest was interrupted with INT or TERM signals or test was 
761
   ran with --abort-on-error option and we got errors with severity S1
762
   (test cann't connect to the server or server crashed) so we assume that 
763
   we should cancel testing and exit. Please check log file for this thread 
764
   in $stress_log_file or 
765
   inspect below output of the last test case executed with mysqltest to 
766
   find out cause of error.
767
   
768
   Output of mysqltest:
769
   @stderr
770
   
771
EOF
772
  }
773
774
  if (-e $reject_filename)
775
  {  
776
    move_to_logs($env->{reject_logs}, $reject_filename, $reject_file);
777
  }    
778
  
779
  if (-e $output_filename)
780
  {  
781
    move_to_logs($env->{screen_logs}, $output_filename, $output_file);
782
  }    
783
784
}
785
786
sub test_loop
787
{     
788
  my %client_env=();
789
  my $test_name="";
790
791
  # KEY for session identification: IP-THREAD_ID
792
  $client_env{ip} = shift;
793
  $client_env{thread_id} = shift;
794
795
  $client_env{mode} = shift;
796
  $client_env{tests_file}=shift; 
797
798
  $client_env{test_seq_idx}=0;
799
800
  #Initialize session variables
801
  test_init(\%client_env);
802
803
LOOP:
804
805
  while(!$exiting)
806
  {
807
    if ($opt_check_tests_file)
808
    {
809
      #Check if tests_file was modified and reread it in this case
810
      read_tests_names($client_env{tests_file}, 0);
811
    }
812
813
    {
814
      lock($test_counters_lock);
815
816
      if (($limits{loop_count} && $limits{loop_count} <= $test_counters{loop_count}*1) ||
817
          ($limits{test_count} && $limits{test_count} <= $test_counters{test_count}*1) )
818
      {
819
        $exiting=1;
820
        next LOOP;
821
      }
822
    }
823
824
    #Get random file name 
825
    if (($test_name = get_test(\%client_env)) ne '')
826
    {
827
      {
828
        lock($test_counters_lock);
829
830
        #Save current counters values 
831
        $client_env{loop_count}=$test_counters{loop_count};
832
        $client_env{test_count}=$test_counters{test_count};
833
      }
834
      #Run test and analyze results
835
      test_execute(\%client_env, $test_name);
836
837
      print "test_loop[".$limits{loop_count}.":".
838
             $limits{test_count}." ".
839
             $client_env{loop_count}.":".
840
             $client_env{test_count}."]:".
841
             " TID ".$client_env{thread_id}.
842
             " test: '$test_name' ".
843
             " Errors: ".join(" ",@{$client_env{test_status}}),"\n";
844
      print "\n";
845
    }
846
  
847
    sleep($opt_sleep_time) if($opt_sleep_time);
848
849
  }
850
}
851
852
sub move_to_logs ($$$)
853
{
854
  my $path_to_logs = shift;
855
  my $src_file = shift;
856
  my $random_filename = shift;
857
858
  my $dst_file = File::Spec->catfile($path_to_logs, $random_filename);
859
  
860
  move ($src_file, $dst_file) or warn<<EOF;
861
ERROR: move_to_logs: File $src_file cannot be moved to $dst_file: $!
862
EOF
863
}
864
865
sub copy_test_files ()
866
{
867
  if (/\.test$/)
868
  { 
869
    $src_file = $File::Find::name;
870
    #print "## $File::Find::topdir - $File::Find::dir - $src_file\n";
871
872
    if ($File::Find::topdir eq $File::Find::dir && $src_file !~ /SCCS/)
873
    {
874
      $test_filename = basename($src_file);
875
      $dst_file = File::Spec->catfile($test_t_path, $test_filename);
876
877
      copy($src_file, $dst_file) or die "ERROR: copy_test_files: File cannot be copied. $!";
878
    }
879
  }
880
}
881
882
sub copy_result_files ()
883
{
884
  if (/\.result$/)
885
  { 
886
    $src_file = $File::Find::name;
887
888
    if ($File::Find::topdir eq $File::Find::dir && $src_file !~ /SCCS/)
889
    {
890
      $result_filename = basename($src_file) ;
891
      $dst_file = File::Spec->catfile($r_folder, $result_filename);
892
893
      copy($src_file, $dst_file) or die "ERROR: copy_result_files: File cannot be copied. $!";
894
    }
895
  }
896
}
897
898
sub get_timestamp
899
{
900
  my ($sec,$min,$hour,$mday,$mon,$year,$wday,$ydat,$isdst) = localtime();
901
902
  return sprintf("%04d%02d%02d%02d%02d%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec);
903
}
904
905
sub read_tests_names
906
{
907
  my $tests_file = shift;
908
  my $force_load = shift;
909
910
  if ($force_load || ( (stat($tests_file->{filename}))[9] != $tests_file->{mtime}) )
911
  {
912
    open (TEST, $tests_file->{filename}) || die ("Could not open file <".
913
                                                  $tests_file->{filename}."> $!");
914
    @{$tests_file->{data}}= grep {!/^[#\r\n]|^$/} map { s/[\r\n]//g; $_ } <TEST>;
915
916
    close (TEST); 
917
    $tests_file->{mtime}=(stat(_))[9];
918
  }
919
}
920
921
sub get_random_test
922
{
923
  my $envt=shift;
924
  my $tests= $envt->{tests_file}->{data};
925
926
  my $random = int(rand(@{$tests}));
927
  my $test = $tests->[$random];
928
929
  return $test;
930
}
931
932
sub get_next_test
933
{
934
  my $envt=shift;
935
  my $test;
936
937
  if (@{$envt->{tests_file}->{data}})
938
  {
939
    $test=${$envt->{tests_file}->{data}}[$envt->{test_seq_idx}];
940
    $envt->{test_seq_idx}++;
941
  }
942
  
943
  #If we reach bound of array, reset seq index and increment loop counter
944
  if ($envt->{test_seq_idx} == scalar(@{$envt->{tests_file}->{data}}))
945
  {
946
    $envt->{test_seq_idx}=0;
947
    {
948
      lock($test_counters_lock);
949
      $test_counters{loop_count}++; 
950
    }
951
  }
952
953
  return $test;  
954
}
955
956
sub get_test
957
{
958
   my $envt=shift;
959
960
   {
961
     lock($test_counters_lock);
962
     $test_counters{test_count}++;
963
   }
964
   
965
   if ($envt->{mode} eq 'seq')
966
   {
967
     return get_next_test($envt);
968
   }
969
   elsif ($envt->{mode} eq 'random')
970
   {
971
     return get_random_test($envt);
972
   }
973
}
974
975
sub stress_log
976
{
977
  my ($log_file, $line)=@_;
978
 
979
  {
980
    open(SLOG,">>$log_file") or warn "Error during opening log file $log_file";
981
    print SLOG $line,"\n";
982
    close(SLOG);
983
  }
984
}
985
986
sub log_session_errors
987
{
988
  my ($env, $test_name) = @_;
989
  my $line='';
990
991
  {
992
    lock ($log_file_lock);
993
994
    #header in the begining of log file
995
    if (!-e $stress_log_file)
996
    {
997
      stress_log($stress_log_file, 
998
                   "TestID TID      Suite         TestFileName Found Errors");
999
      stress_log($stress_log_file, 
1000
                   "=======================================================");    
1001
    }
1002
1003
    $line=sprintf('%6d %3d %10s %20s %s', $env->{test_count}, threads->self->tid, 
1004
                                          $opt_suite, $test_name, 
1005
                                          join(",", @{$env->{test_status}}));
1006
                                      
1007
    stress_log($stress_log_file, $line);
1008
    #stress_log_with_lock($stress_log_file, "\n");
1009
1010
    if ($opt_log_error_details)
1011
    {
1012
      foreach $severity (sort {$a cmp $b} keys %{$env->{errors}})
1013
      {
1014
        stress_log($stress_log_file, "");
1015
        foreach $error (keys %{$env->{errors}->{$severity}})
1016
        {
1017
          if ($error ne 'total')
1018
          {
1019
            stress_log($stress_log_file, "$severity: Count:".
1020
                      $env->{errors}->{$severity}->{$error}->[0].
1021
                      " Error:". $env->{errors}->{$severity}->{$error}->[1]);
1022
          }
1023
        }
1024
      }
1025
    }
1026
  }
1027
}
1028
1029
sub sig_INT_handler
1030
{
1031
  $SIG{INT}= \&sig_INT_handler;
1032
  $exiting=1;
1033
  print STDERR "$$: Got INT signal-------------------------------------------\n";
1034
1035
}
1036
1037
sub sig_TERM_handler
1038
{
1039
  $SIG{TERM}= \&sig_TERM_handler;
1040
  $exiting=1;
1041
  print STDERR "$$: Got TERM signal\n";
1042
}
1043
1044
sub usage
1045
{
1046
  print <<EOF;
1047
1048
The MySQL Stress suite Ver $stress_suite_version
1049
1050
mysql-stress-test.pl --stress-basedir=<dir> --stress-suite-basedir=<dir> --server-logs-dir=<dir>
1051
1052
--server-host
1053
--server-port
1054
--server-socket
1055
--server-user
1056
--server-password
1057
--server-logs-dir
1058
  Directory where all clients session logs will be stored. Usually 
1059
  this is shared directory associated with server that used 
1060
  in testing
1061
1062
  Required option.
1063
1064
--stress-suite-basedir=<dir>
1065
  Directory that has r/ t/ subfolders with test/result files
1066
  which will be used for testing. Also by default we are looking 
1067
  in this directory for 'stress-tests.txt' file which contains 
1068
  list of tests.  It is possible to specify other location of this 
1069
  file with --stress-tests-file option.
1070
1071
  Required option.
1072
1073
--stress-basedir=<dir>
1074
  Working directory for this test run. This directory will be used 
1075
  as temporary location for results tracking during testing
1076
  
1077
  Required option.
1078
1079
--stress-datadir=<dir>
1080
  Location of data files used which will be used in testing.
1081
  By default we search for these files in <dir>/data where dir 
1082
  is value of --stress-suite-basedir option.
1083
1084
--stress-init-file[=/path/to/file with tests for initialization of stress db]
1085
  Using of this option allows to perform initialization of database
1086
  by execution of test files. List of tests will be taken either from 
1087
  specified file or if it omited from default file 'stress-init.txt'
1088
  located in <--stress-suite-basedir/--suite> dir
1089
    
1090
--stress-tests-file[=/path/to/file with tests] 
1091
  Using of this option allows to run stress test itself. Tests for testing 
1092
  will be taken either from specified file or if it omited from default 
1093
  file 'stress-tests.txt' located in <--stress-suite-basedir/--suite> dir
1094
1095
--stress-mode= [random|seq]
1096
  There are two possible modes which affect order of selecting tests
1097
  from the list:
1098
    - in random mode tests will be selected in random order
1099
    - in seq mode each thread will execute tests in the loop one by one as 
1100
      they specified in the list file. 
1101
      
1102
--sleep-time=<time in seconds>
1103
  Delay between test execution. Could be usefull in continued testsing 
1104
  when one of instance of stress script perform periodical cleanup or
1105
  recreating of some database objects
1106
1107
--threads=#number of threads
1108
  Define number of threads
1109
1110
--check-tests-file
1111
  Check file with list of tests. If file was modified it will force to
1112
  reread list of tests. Could be usefull in continued testing for
1113
  adding/removing tests without script interruption 
1114
1115
--mysqltest=/path/to/mysqltest binary
1116
1117
--verbose
1118
1119
--cleanup
1120
  Force to clean up working directory (specified with --stress-basedir)
1121
1122
--log-error-details
1123
  Enable errors details in the global error log file. (Default: off)
1124
1125
--test-count=<number of executed tests before we have to exit>
1126
--loop-count=<number of executed loops in sequential mode before we have to exit>
1127
--test-duration=<number of seconds that stress test should run>
1128
1129
Example of tool usage:
1130
1131
perl mysql-stress-test.pl \
1132
--stress-suite-basedir=/opt/qa/mysql-test-extra-5.0/mysql-test \
1133
--stress-basedir=/opt/qa/test \
1134
--server-logs-dir=/opt/qa/logs \
1135
--test-count=20  \
1136
--stress-tests-file=innodb-tests.txt \
1137
--stress-init-file=innodb-init.txt \
1138
--threads=5 \
1139
--suite=funcs_1  \
1140
--mysqltest=/opt/mysql/mysql-5.0/client/mysqltest \
1141
--server-user=root \
1142
--server-database=test \
1143
--cleanup \
1144
1145
EOF
1146
exit(0);
1147
}
1148
1149