~drizzle-trunk/drizzle/development

2324.2.1 by patrick crews
Initial work for sql-bench mode. Added sql-bench to the tree. Test script for running entire suite added
1
#!/usr/bin/perl
2
# Copyright (C) 2000-2001, 2003 MySQL AB
3
#
4
# This library is free software; you can redistribute it and/or
5
# modify it under the terms of the GNU Library General Public
6
# License as published by the Free Software Foundation; version 2
7
# of the License.
8
#
9
# This library is distributed in the hope that it will be useful,
10
# but WITHOUT ANY WARRANTY; without even the implied warranty of
11
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12
# Library General Public License for more details.
13
#
14
# You should have received a copy of the GNU Library General Public
15
# License along with this library; if not, write to the Free
16
# Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
17
# MA 02111-1307, USA
18
#
19
# a little program to generate a table of results
20
# just read all the RUN-*.log files and format them nicely
21
# Made by Luuk de Boer
22
# Patched by Monty
23
24
use Getopt::Long;
25
26
$opt_server="mysql";
27
$opt_dir="output";
28
$opt_machine=$opt_cmp="";
29
$opt_relative=$opt_same_server=$opt_help=$opt_Information=$opt_skip_count=$opt_no_bars=$opt_verbose=0;
30
31
GetOptions("Information","help","server=s","cmp=s","machine=s","relative","same-server","dir=s","skip-count","no-bars","html","verbose") || usage();
32
33
usage() if ($opt_help || $opt_Information);
34
35
$opt_cmp=lc(join(",",sort(split(',',$opt_cmp))));
36
37
if ($opt_same_server)
38
{
39
  $files="$opt_dir/RUN-$opt_server*$opt_machine";
40
}
41
else
42
{
43
  $files="$opt_dir/RUN-*$opt_machine";
44
}
45
$files.= "-cmp-$opt_cmp" if (length($opt_cmp));
46
47
#
48
# Go trough all RUN files and gather statistics.
49
#
50
51
if ($#ARGV == -1)
52
{
53
  @ARGV=glob($files);
54
  $automatic_files=1;
55
}
56
57
foreach (@ARGV)
58
{
59
  next if (!$opt_cmp && /-cmp-/ && $automatic_files || defined($found{$_}));
60
  $prog=$filename = $_;
61
  $found{$_}=1;			# Remove dupplicates
62
  /RUN-(.*)$/;
63
  $tot{$prog}{'version'}=$1;
64
  push(@key_order,$prog);
65
  $next = 0;
66
  open(TMP, "<$filename") || die "Can't open $filename: $!\n";
67
  while (<TMP>)
68
  {
69
    chomp;
70
    if ($next == 0) {
71
      if (/Server version:\s+(\S+.*)/i)
72
      {
73
	$tot{$prog}{'server'} = $1;
74
      }
75
      elsif (/Arguments:\s+(.+)/i)
76
      {
77
	$arguments= $1;
78
	# Remove some standard, not informative arguments
79
	$arguments =~ s/--force|--log|--use-old\S*|--server=\S+|--cmp=\S+|--user=\S+|--pass=\S+|--machine=\S+|--dir=\S+//g;
80
	if (($tmp=index($arguments,"--comment")) >= 0)
81
	{
82
	  if (($end=index($arguments,$tmp+2,"--")) >= 0)
83
	  {
84
	    substr($arguments,$tmp,($end-$tmp))="";
85
	  }
86
	  else
87
	  {
88
	    $arguments=substr($arguments,0,$tmp);
89
	  }
90
	}
91
	$arguments =~ s/\s+/ /g;
92
	$tot{$prog}{'arguments'}=$arguments;
93
      }
94
      elsif (/Comments:\s+(.+)/i) {
95
	$tot{$prog}{'comments'} = $1;
96
      } elsif (/^(\S+):.*(estimated\s|)total\stime:\s+([\d.]+)\s+(wallclock\s|)secs/i)
97
      {
98
	$tmp = $1; $tmp =~ s/://;
99
	$tot{$prog}{$tmp} = [ $3, (length($2) ? "+" : "")];
100
	$op1{$tmp} = $tmp;
101
      } elsif (/Totals per operation:/i) {
102
	$next = 1;
103
	next;
104
      }
105
    }
106
    elsif ($next == 1)
107
    {
108
      if (/^(\S+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)\s*([+|?])*/)
109
      {
110
	$tot1{$prog}{$1} = [$2,$6,$7];
111
	$op{$1} = $1;
112
      }
113
    }
114
  }
115
}
116
117
if (!%op)
118
{
119
  print "Didn't find any files matching: '$files'\n";
120
  print "Use the --cmp=server,server option to compare benchmarks\n";
121
  exit 1;
122
}
123
124
125
# everything is loaded ...
126
# now we have to create a fancy output :-)
127
128
# I prefer to redirect scripts instead to force it to file ; Monty
129
#
130
# open(RES, ">$resultfile") || die "Can't write to $resultfile: $!\n";
131
# select(RES)
132
#
133
134
if ($opt_html) {
135
  html_output();
136
} else {
137
  ascii_output();
138
}
139
exit 0;
140
141
#
142
# some output + format functions;
143
#
144
145
sub ascii_output {
146
  print <<EOF;
147
This is the result file of the different benchmark tests.
148
149
The number in () after each tests shows how many SQL commands the particular
150
test did.  As one test may have many different parameters this gives only
151
a rough picture of what was done.  Check the source for more information :)
152
153
Keep in mind that one can\'t compare benchmarks run with different --cmp
154
options. The --cmp options sets the all limits according to the worst
155
limit for all server in the benchmark.
156
157
Numbers marked with '+' are estimated according to previous runs because
158
the query took longer than a given time-limit to finish. The estimation
159
shouldn\'t be far from the real result thought.
160
161
Numbers marked with '?' contains results that gave wrong result. This can only
162
be used as an indication of how long it took for the server to produce a wrong
163
result :)
164
165
Numbers marked with '*' are tests that was run a different number times
166
than the test in the first column.  The reason for this is normally that the
167
marked test was run with different options that affects the number of tests
168
or that the test was from another version of the MySQL benchmarks.
169
170
Hope this will give you some idea how each db is performing at what thing ....
171
Hope you like it .... Luuk & Monty (1997)
172
173
EOF
174
175
  if ($opt_relative)
176
  {
177
    print "Column 1 is in seconds. All other columns are presented relative\n";
178
    print "to this. 1.00 is the same, bigger numbers indicates slower\n\n";
179
  }
180
181
  if ($opt_verbose)
182
  {
183
    print "The test values is printed in the format 'time:number of tests'\n";
184
  }
185
186
  if (length($opt_cmp))
187
  {
188
    print "The test was run with limits from: $opt_cmp\n\n";
189
  }
190
  print "The result logs which where found and the options:\n";
191
  $bar= $opt_no_bars ? " " : "|";
192
193
  # Move $opt_server first in array if not filename on command line
194
  if ($automatic_files)
195
  {
196
    @key_order=sort {$a cmp $b} keys %tot;
197
    for ($i=0; $i <= $#key_order; $i++)
198
    {
199
      if ($tot{$key_order[$i]}{'version'} =~ /^$opt_server-/)
200
      {
201
	unshift(@key_order,$key_order[$i]);
202
	splice(@key_order,$i+1,1);
203
	last;
204
      }
205
    }
206
  }
207
  # Print header
208
209
  $column_count=0;
210
  foreach $key (@key_order)
211
  {
212
    $tmp=$tmp=$tot{$key}{'version'};
213
    $tmp =~ s/-cmp-$opt_cmp// if (length($opt_cmp));
214
    $column_count++;
215
    printf "%2d %-40.40s: %s %s\n", $column_count, $tmp,
216
    $tot{$key}{'server'}, $tot{$key}{'arguments'};
217
    print "  $tot{$key}{'comments'}\n"
218
      if ($tot{$key}{'comments'} =~ /\w+/);
219
  }
220
221
  print "\n";
222
223
  $namewidth=($opt_skip_count && !$opt_verbose) ? 29 : 36;
224
  $colwidth= $opt_relative ? 10 : 7;
225
  $count_width=7;
226
  $colwidth+=$count_width if ($opt_verbose);
227
228
  print_sep("=");
229
  printf "%-$namewidth.${namewidth}s${bar}", "Operation";
230
  $count = 1;
231
  foreach $key (@key_order)
232
  {
233
    printf "%${colwidth}d${bar}", $count;
234
    $count++;
235
  }
236
  printf "\n%-$namewidth.${namewidth}s${bar}", "";
237
  foreach $key (@key_order)
238
  {
239
    $ver=$tot{$key}{'version'};
240
    $ver =~ s/-[a-zA-Z0-9_\.]+-cmp-$opt-cmp$//;
241
    printf "%${colwidth}.${colwidth}s${bar}", $ver;
242
  }
243
  print "\n";
244
  print_sep("-");
245
  print_string($opt_relative ? "Relative results per test (First column is in seconds):" : "Results per test in seconds:");
246
  print_sep("-");
247
248
  foreach $key (sort {$a cmp $b} keys %op1)
249
  {
250
    printf "%-$namewidth.${namewidth}s${bar}", $key;
251
    $first=undef();
252
    foreach $server (@key_order)
253
    {
254
      print_value($first,$tot{$server}{$key}->[0],undef(),$tot{$server}{$key}->[1]);
255
      $first=$tot{$server}{$key}->[0] if (!defined($first));
256
    }
257
    print "\n";
258
  }
259
260
  print_sep("-");
261
  print_string("The results per operation:");
262
  print_sep("-");
263
264
  foreach $key (sort {$a cmp $b} keys %op)
265
  {
266
    next if ($key =~ /TOTALS/i);
267
    $tmp=$key;
268
    $count=$tot1{$key_order[0]}{$key}->[1];
269
    $tmp.= " (" . $count .  ")" if (!$skip_count);
270
    printf "%-$namewidth.${namewidth}s${bar}", $tmp;
271
    $first=undef();
272
    foreach $server (@key_order)
273
    {
274
      $tmp= $count != $tot1{$server}{$key}->[1] ? "*" : "";
275
      print_value($first,$tot1{$server}{$key}->[0],$tot1{$server}{$key}->[1],
276
		  $tot1{$server}{$key}->[2] . $tmp);
277
      $first=$tot1{$server}{$key}->[0] if (!defined($first));
278
    }
279
    print "\n";
280
  }
281
282
  print_sep("-");
283
  $key="TOTALS";
284
  printf "%-$namewidth.${namewidth}s${bar}", $key;
285
  $first=undef();
286
  foreach $server (@key_order)
287
  {
288
    print_value($first,$tot1{$server}{$key}->[0],undef(),
289
		$tot1{$server}{$key}->[2]);
290
    $first=$tot1{$server}{$key}->[0] if (!defined($first));
291
  }
292
  print "\n";
293
  print_sep("=");
294
}
295
296
297
sub html_output
298
{
299
  my $template="template.html";
300
  my $title="MySQL | | Information | Benchmarks | Compare with $opt_cmp";
301
  my $image="info.gif";
302
  $bar="";
303
304
  open(TEMPLATE, $template) || die;
305
  while (<TEMPLATE>)
306
  {
307
    if (/<center>/)
308
    {
309
      print $_;
310
      print "<!---- This is AUTOMATICALLY Generated. Do not edit here! ---->\n";
311
    }
312
    elsif (/TITLE:SUBTITLE/)
313
    {
314
      s|TITLE:SUBTITLE|$title|;
315
      print $_;
316
    }
317
    elsif (/TITLE:COMPARE/)
318
    {
319
      s|TITLE:COMPARE|$opt_cmp|;
320
      print $_;
321
    }
322
    elsif (/ subchapter name /)
323
    {
324
      # Nothing here for now
325
      print $_;
326
    }
327
    elsif (/ text of chapter /)
328
    {
329
      print $_;
330
      print_html_body();
331
    }
332
    else
333
    {
334
      print $_;
335
    }
336
  }
337
  close(TEMPLATE);
338
}
339
340
341
sub print_html_body
342
{
343
  my ($title,$count,$key);
344
  print <<EOF;
345
<center>
346
<font size=+4><b>MySQL Benchmark Results</b></font><br>
347
<font size=+1><b>Compare with $opt_cmp</b></font><p><p>
348
</center>
349
This is the result file of the different benchmark tests.
350
<p>
351
352
The number in () after each tests shows how many SQL commands the particular
353
test did.  As one test may have many different parameters this gives only
354
a rough picture of what was done.  Check the source for more information.
355
<p>
356
Keep in mind that one can\'t compare benchmarks run with different --cmp
357
options. The --cmp options sets the all limits according to the worst
358
limit for all server in the benchmark.
359
<p>
360
Numbers marked with '+' are estimated according to previous runs because
361
the query took longer than a given time-limit to finish. The estimation
362
shouldn\'t be far from the real result thought.
363
<p>
364
Numbers marked with '?' contains results that gave wrong result. This can only
365
be used as an indication of how long it took for the server to produce a wrong
366
result :)
367
<p>
368
Hope this will give you some idea how each db is performing at what thing ....
369
<br>
370
Hope you like it .... Luuk & Monty (1997)
371
<p><p>
372
EOF
373
374
  if ($opt_relative)
375
  {
376
    print "Column 1 is in seconds. All other columns are presented relative<br>\n";
377
    print "to this. 1.00 is the same, bigger numbers indicates slower<p>\n\n";
378
  }
379
380
  if (length($opt_cmp))
381
  {
382
    print "The test was run with limits from: $opt_cmp\n\n";
383
  }
384
  print "The result logs which where found and the options:<br>\n";
385
386
  # Move $opt_server first in array
387
  if ($automatic_files)
388
  {
389
    @key_order=sort {$a cmp $b} keys %tot;
390
    for ($i=0; $i <= $#key_order; $i++)
391
    {
392
      if ($tot{$key_order[$i]}{'version'} =~ /^$opt_server-/)
393
      {
394
	unshift(@key_order,$key_order[$i]);
395
	splice(@key_order,$i+1,1);
396
	last;
397
      }
398
    }
399
  }
400
  # Print header
401
  print "<p><center><table border=1 width=100%>\n";
402
  $column_count=0;
403
  foreach $key (@key_order)
404
  {
405
    $tmp=$tot{$key}{'version'};
406
    $tmp =~ s/-cmp-$opt_cmp// if (length($opt_cmp));
407
    $column_count++;
408
#    printf "<tr><td>%2d<td>%-36.36s<td>%s %s</tr>\n", $column_count, $tmp,
409
    printf "<tr><td>%2d</td><td>%s</td><td>%s %s</td></tr>\n",
410
    $column_count, $tmp, $tot{$key}{'server'}, $tot{$key}{'arguments'};
411
    print "<tr><td colspan=3>$tot{$key}{'comments'}</td></tr>\n"
412
      if ($tot{$key}{'comments'} =~ /\w+/);
413
  }
414
415
  print "</table></center><p><center><table border=1 width=100%>\n";
416
417
  $namewidth=$opt_skip_count ? 22 :29;
418
  $colwidth= $opt_relative ? 10 : 7;
419
  $count_width=7;
420
421
  printf "<tr><td><b>%s</b></td>\n", "Operation";
422
  $count = 1;
423
  foreach $key (@key_order)
424
  {
425
    $ver=$tot{$key}{'version'};
426
    printf "<td align=center><b>%d", $count;
427
    printf "<br>%${colwidth}.${colwidth}s</b></td>\n", substr($ver,0,index($ver,"-"));
428
    $count++;
429
  }
430
  print "</tr>\n";
431
  $title = $opt_relative ? "Relative results per test (First column is in seconds):" : "Results per test in seconds:";
432
  printf "<tr><td colspan=%d><b>%s</b></td></tr>\n", $count, $title;
433
434
  foreach $key (sort {$a cmp $b} keys %op1)
435
  {
436
    if (!$opt_html)
437
    {
438
      printf "<tr><td>%-$namewidth.${namewidth}s</td>", $key;
439
    }
440
    else
441
    {
442
      print "<tr><td>$key</td>";
443
    }
444
    $first=undef();
445
    foreach $server (@key_order)
446
    {
447
      print_value($first,$tot{$server}{$key}->[0],undef(),
448
		  $tot{$server}{$key}->[1]);
449
      $first=$tot{$server}{$key}->[0] if (!defined($first));
450
    }
451
    print "</tr>\n";
452
  }
453
454
  $title = "The results per operation:";
455
  printf "<tr><td colspan=%d><b>%s</b></td></tr>\n", $count, $title;
456
457
  foreach $key (sort {$a cmp $b} keys %op)
458
  {
459
    next if ($key =~ /TOTALS/i);
460
    $tmp=$key;
461
    $tmp.= " (" . $tot1{$key_order[0]}{$key}->[1] . ")" if (!$skip_count);
462
    if (!$opt_html)
463
    {
464
      printf "<tr><td>%-$namewidth.${namewidth}s</td>", $tmp;
465
    }
466
    else
467
    {
468
      print "<tr><td>$tmp</td>";
469
    }
470
    $first=undef();
471
    foreach $server (@key_order)
472
    {
473
      print_value($first,$tot1{$server}{$key}->[0],
474
		  $tot1{$server}{$key}->[1],
475
		  $tot1{$server}{$key}->[2]);
476
      $first=$tot1{$server}{$key}->[0] if (!defined($first));
477
    }
478
    print "</tr>\n";
479
  }
480
481
  $key="TOTALS";
482
  printf "<tr><td><b>%-$namewidth.${namewidth}s</b></td>", $key;
483
  $first=undef();
484
  foreach $server (@key_order)
485
  {
486
    print_value($first,$tot1{$server}{$key}->[0],undef(),
487
		$tot1{$server}{$key}->[2]);
488
    $first=$tot1{$server}{$key}->[0] if (!defined($first));
489
  }
490
  print "</tr>\n</table>\n";
491
}
492
493
494
sub print_sep
495
{
496
  my ($sep)=@_;
497
  print $sep x ($namewidth + (($colwidth+1) * $column_count)+1),"\n";
498
}
499
500
501
sub print_value
502
{
503
  my ($first,$value,$count,$flags)=@_;
504
  my ($tmp,$width);
505
506
  if (defined($value))
507
  {
508
    if (!defined($first) || !$opt_relative)
509
    {
510
      $tmp=sprintf("%.2f",$value);
511
    }
512
    else
513
    {
514
      $first=1 if ($first == 0); # Assume that it took one second instead of 0
515
      $tmp= sprintf("%.2f",$value/$first);
516
    }
517
    if (defined($flags))
518
    {
519
      $tmp="+".$tmp if ($flags =~ /\+/);
520
      $tmp="?".$tmp if ($flags =~ /\?/);
521
      $tmp="*".$tmp if ($flags =~ /\*/);
522
    }
523
  }
524
  else
525
  {
526
    $tmp="";
527
  }
528
  $width= ($opt_verbose ? $colwidth - $count_width : $colwidth);
529
  if (!$opt_html)
530
  {
531
    $tmp= " " x ($width-length($tmp)) . $tmp if (length($tmp) < $width);
532
  }
533
  if ($opt_verbose)
534
  {
535
    if ($count)
536
    {
537
      $tmp.= ":" . " " x ($count_width-1-length($count)) . $count;
538
    }
539
    else
540
    {
541
      $tmp.= " " x ($count_width);
542
    }
543
  }
544
545
  if (!$opt_html) {
546
    print $tmp . "${bar}";
547
  } else {
548
    print "<td align=right>$tmp</td>";
549
  }
550
}
551
552
553
sub print_string
554
{
555
  my ($str)=@_;
556
  if (!$opt_html)
557
  {
558
    my ($width);
559
    $width=$namewidth + ($colwidth+1)*$column_count;
560
    $str=substr($str,1,$width) if (length($str) > $width);
561
    print($str," " x ($width - length($str)),"${bar}\n");
562
  }
563
  else
564
  {
565
    print $str,"\n";
566
  }
567
}
568
569
570
sub usage
571
{
572
    print <<EOF;
573
$0  Ver 1.2
574
575
This program parses all RUN files from old 'run-all-tests --log' scripts
576
and makes a nice comparable table.
577
578
$0 takes currently the following options:
579
580
--help or --Information		
581
  Shows this help
582
583
--cmp=server,server,server (Default $opt_cmp)
584
Compares all runs that are done with the same --cmp options to run-all-tests.
585
The most normal options are '--cmp=mysql,pg,solid' and '--cmp ""'
586
587
--dir=...  (Default $opt_dir)
588
From which directory one should get the runs.  All runs made by
589
run-all-tests --log is saved in the 'output' directory.
590
In the 'results' directory you may have some example runs from different
591
databases.
592
593
--html
594
  Print the table in html format.
595
596
--machine='full-machine-name' (Default $opt_machine)
597
Use only runs that match this machine.
598
599
--relative
600
Show all numbers in times of the first server where the time for the
601
first server is 1.0
602
603
--same-server
604
Compare all runs for --server=....  The --machine is not used in this case
605
This is nice to compare how the same server runs on different machines.
606
607
--server='server name'  (Default $opt_server)
608
Put this server in the first result column.
609
610
--skip-count
611
Do not write the number of tests after the test-name.
612
613
--verbose
614
Write the number of tests in each column. This is useful when some column
615
is marked with '*'.
616
EOF
617
618
  exit(0);
619
}