2430.1.5
by Olaf van der Spek
Merge trunk |
1 |
#! /usr/bin/env perl
|
2 |
||
3 |
# Copyright (C) 2010 Sun Microsystems, Inc. All rights reserved.
|
|
4 |
# Use is subject to license terms.
|
|
5 |
#
|
|
6 |
# This program is free software; you can redistribute it and/or modify
|
|
7 |
# it under the terms of the GNU General Public License as published by
|
|
8 |
# the Free Software Foundation; version 2 of the License.
|
|
9 |
#
|
|
10 |
# This program is distributed in the hope that it will be useful, but
|
|
11 |
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
13 |
# General Public License for more details.
|
|
14 |
#
|
|
15 |
# You should have received a copy of the GNU General Public License
|
|
16 |
# along with this program; if not, write to the Free Software
|
|
17 |
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
|
|
18 |
# USA
|
|
19 |
||
20 |
use lib 'lib'; |
|
21 |
use lib "$ENV{RQG_HOME}/lib"; |
|
22 |
use lib 'randgen/lib'; |
|
23 |
||
24 |
use strict; |
|
25 |
use Carp; |
|
26 |
use Cwd; |
|
27 |
use DBI; |
|
28 |
use File::Find; |
|
29 |
use Getopt::Long; |
|
30 |
use POSIX; |
|
31 |
use Sys::Hostname; |
|
32 |
||
33 |
use GenTest; |
|
34 |
use GenTest::Properties; |
|
35 |
use GenTest::Constants; |
|
36 |
||
37 |
# TODO:
|
|
38 |
# - clean up imports (use) not used.
|
|
39 |
# - fix description after adjusting calling scripts (PB2)
|
|
40 |
# - handle env vars RQG_HOME / RQG_CONF?
|
|
41 |
# - add support for skipping tests and printing message (e.g.: If plugin not found)
|
|
42 |
# - make basedir option a hash, skip basedirForRelease option?
|
|
43 |
# - Reporting: Check and fix? More flexibility needed?
|
|
44 |
# - propagate MTR_BUILD_THREAD / MTR_PORT_BASE, if needed?
|
|
45 |
# - make vardir optional (mandatory vardir conflicts with --mem / MTR_MEM=1)
|
|
46 |
# - add to help text (see sub help())
|
|
47 |
# - add logic for process cleanup (left-over mysqlds), at least if Pushbuild
|
|
48 |
# - more defaults?
|
|
49 |
# - exit safely
|
|
50 |
# - See also other TODO comments below.
|
|
51 |
||
52 |
||
53 |
################################################################################
|
|
54 |
#
|
|
55 |
# DRAFT - INCOMPLETE - DRAFT - INCOMPLETE - DRAFT
|
|
56 |
#
|
|
57 |
# This script is a wrapper around certain RQG functionality, primarily running
|
|
58 |
# well-defined RQG tests in a convenient way.
|
|
59 |
#
|
|
60 |
# The script accepts arguments which uniquely identify the test case to be run.
|
|
61 |
# From this identifier the script grabs information from a separate file which
|
|
62 |
# defines the test case, including various options to the RQG and MySQL.
|
|
63 |
# Then, the RQG's runall.pl script is executed with the options gathered from
|
|
64 |
# the test definition. Options may be overridden on the command line.
|
|
65 |
# The runall.pl script takes care of starting the database server as well as
|
|
66 |
# executing the test itself.
|
|
67 |
#
|
|
68 |
# MySQL/Sun/Oracle notes:
|
|
69 |
#
|
|
70 |
# This script implements the interface between Pushbuild (automated testing
|
|
71 |
# infrastructure) and the Random Query Generator.
|
|
72 |
# The script is invoked by Pushbuild (PB2) slave instances when they have a job
|
|
73 |
# matching certain conditions.
|
|
74 |
# See tests_systemqa.py in pb2-trd-config repository for details.
|
|
75 |
#
|
|
76 |
# Pushbuild always provides the following arguments/oiptions to this script:
|
|
77 |
# --basedir : Location of binaries to be tested
|
|
78 |
# --vardir : Location of temporary test files (logs, database, etc.)
|
|
79 |
# --branch : MySQL bazaar source code branch nick or other identifier
|
|
80 |
# corresponding to the given binaries.
|
|
81 |
# --config : File specifying which test specification to use.
|
|
82 |
# The test specification contains test name, options, etc.
|
|
83 |
#
|
|
84 |
# We hope to merge/integrate this with what is currently runall.pl and the
|
|
85 |
# new modules for server start that is being developed, eventually.
|
|
86 |
################################################################################
|
|
87 |
||
88 |
#
|
|
89 |
# See bottom of file for subroutines.
|
|
90 |
#
|
|
91 |
||
92 |
||
93 |
# Autoflush output buffers (needed when using POSIX::_exit())
|
|
94 |
$| = 1; |
|
95 |
||
96 |
# Show in output when this script starts for ease of debugging when wrapped
|
|
97 |
# inside other scripts.
|
|
98 |
say("==================== Starting $0 ====================\n"); |
|
99 |
||
100 |
# Find out if this is Pushbuild (MySQL's automatic test system).
|
|
101 |
# We assume this is Pushbuild if environment variable PB2WORKDIR is set.
|
|
102 |
my $pushbuild = 1 if defined $ENV{'PB2WORKDIR'}; |
|
103 |
||
104 |
||
105 |
||
106 |
################################################################################
|
|
107 |
# Option handling
|
|
108 |
################################################################################
|
|
109 |
||
110 |
# We require --config to specify a test definition file.
|
|
111 |
# Properties.pm treats this option in a special way: As a file containing other
|
|
112 |
# options.
|
|
113 |
#
|
|
114 |
# Options set in this config file may be overridden on the command line.
|
|
115 |
# The resulting (merged) set of options is then passed to the RQG itself.
|
|
116 |
# Note: When given on command line, mysqld options should not include 'inner'
|
|
117 |
# "--" prefix, e.g. "--mysqld=--lock-wait-timeout=2". Because prefixes
|
|
118 |
# will be added automatically, specify "--mysqld=lock-wait-timeout=2"
|
|
119 |
# or "--mysqld lock-wait-timeout=2" instead
|
|
120 |
# (TODO: Add logic to handle inner -- on command line?).
|
|
121 |
#
|
|
122 |
# TODO: Have runall.pl use GenTest::Properties as well.
|
|
123 |
#
|
|
124 |
# Consider: Accept --test on command line and find test definition file based
|
|
125 |
# on this (see parsing of such a file in lib/GenTest/App/Gendata.pm),
|
|
126 |
# instead of having config as a special value, with path and all.
|
|
127 |
||
128 |
# Read options.
|
|
129 |
# old: my ($basedir, $vardir, $tree, $test) = @ARGV;
|
|
130 |
# new: We use GenTest::Properties mechanisms
|
|
131 |
||
132 |
my @ARGV_saved = @ARGV; |
|
133 |
||
134 |
# Print full command line used for calling this script, so that it will appear
|
|
135 |
# in logs and can be repeated relatively easily.
|
|
136 |
# Avoiding say()'s prefixing to make it easy to copy & paste.
|
|
137 |
say("\nInbound command line was: "); |
|
138 |
print("$0 \\ \n ".join(" \\ \n ", @ARGV_saved)."\n"); |
|
139 |
||
140 |
||
141 |
my $options = {}; |
|
142 |
my $opt_result = GetOptions($options, |
|
143 |
'basedir=s', |
|
144 |
'category=s', |
|
145 |
'config=s', # The actual test definition, containing options. |
|
146 |
'grammar=s', |
|
147 |
'name=s', |
|
148 |
'vardir=s', # TODO: Make optional? |
|
149 |
'basedirForRelease:s%', # hash of paths to basedir for releases to test/compare |
|
150 |
'branch:s', # TODO: Remove, get from ENV{BRANCH_NAME} instead if needed? |
|
151 |
'candidate:s', |
|
152 |
'dry_run', |
|
153 |
'duration:s', |
|
154 |
'help', |
|
155 |
'mysqld:s%', |
|
156 |
'threads:s', |
|
157 |
'verbose!'
|
|
158 |
);
|
|
159 |
my $config = GenTest::Properties->new( |
|
160 |
options => $options, |
|
161 |
legal => [ |
|
162 |
# Adds to the union of $options, $required and $defaults. Include here
|
|
163 |
# options that may be specified in separate files, otherwise they
|
|
164 |
# will be refused ($options only includes options actually given on
|
|
165 |
# the command line).
|
|
166 |
'basedir', |
|
167 |
'basedirForRelease', |
|
168 |
'candidate', |
|
169 |
'category', |
|
170 |
'description', |
|
171 |
'dry_run', |
|
172 |
'grammar', |
|
173 |
'help', |
|
174 |
'mysqld', |
|
175 |
'name', |
|
176 |
'threads', |
|
177 |
'vardir', |
|
178 |
],
|
|
179 |
required => [ |
|
180 |
'basedir', # Need to know what to test (binaries) |
|
181 |
'category', # A test with unknown category is not well-defined. |
|
182 |
'config', # File specifying a test. Used for forcing a well-defined approach. |
|
183 |
'grammar', # No random queries without a grammar... |
|
184 |
'name' # So that we can refer to it later and report results. |
|
185 |
],
|
|
186 |
defaults => { |
|
187 |
threads => 1, # Keep it simple by default. |
|
188 |
duration => 300, # Suitable for regular automated 'regression testing' |
|
189 |
candidate => '', |
|
190 |
verbose => '', |
|
191 |
basedirForRelease => { # Locations of binaries for various releases |
|
192 |
'5.0' => osWindows() ? |
|
193 |
'G:\mysql-releases\mysql-5.0.87-win32' : |
|
194 |
'/export/home/mysql-releases/mysql-5.0'
|
|
195 |
}
|
|
196 |
},
|
|
197 |
help => \&help |
|
198 |
);
|
|
199 |
||
200 |
# Print help text if --help or no options were specified
|
|
201 |
$config->printHelp if not $opt_result or $config->help; |
|
202 |
||
203 |
||
204 |
||
205 |
# List of options that are to be sent to the RQG runner script if defined.
|
|
206 |
my @randgen_opts_noprefix = ( |
|
207 |
'basedir', |
|
208 |
'vardir', |
|
209 |
'grammar', |
|
210 |
'threads', |
|
211 |
'duration'
|
|
212 |
);
|
|
213 |
||
214 |
||
215 |
||
216 |
### Process options...
|
|
217 |
||
218 |
# Get list of options as string with appropriate prefixes.
|
|
219 |
my $mysqld_options = $config->genOpt('--mysqld=--', 'mysqld'); |
|
220 |
my $randgen_opts = $config->collectOpt('--', @randgen_opts_noprefix); |
|
221 |
#say(my $all_non_complex_opts = $config->collectOpt('--'));
|
|
222 |
||
223 |
### Set helper variables
|
|
224 |
# candidate: Match only exact string 'true' (case insensitive)
|
|
225 |
# Candidate tests may be treated differently e.g. wrt. reporting.
|
|
226 |
my $candidate = 1 if $config->candidate =~ m{^true$}io; |
|
227 |
my $spec_file = $config->config; # file with test definition |
|
228 |
||
229 |
||
230 |
################################################################################
|
|
231 |
# Prepare test, display info.
|
|
232 |
################################################################################
|
|
233 |
||
234 |
say("Loaded test definition from file " . $spec_file) if $spec_file; |
|
235 |
||
236 |
#
|
|
237 |
# Prepare ENV variables and other settings.
|
|
238 |
#
|
|
239 |
setupPushbuildEnv() if ($pushbuild); |
|
240 |
||
241 |
#
|
|
242 |
# Print easy-to-read info about the environment (useful for debugging etc.)
|
|
243 |
#
|
|
244 |
say("\n======== Information on the host system: ========"); |
|
245 |
say(" - Local time : ".localtime()); |
|
246 |
say(" - Hostname : ".hostname()); |
|
247 |
say(" - PID : $$"); |
|
248 |
say(" - Working dir : ".cwd()); |
|
249 |
say(" - PATH : ".$ENV{'PATH'}); |
|
250 |
||
251 |
say("\n======== Configuration: ========"); |
|
252 |
$config->printProps(); # TODO or not TODO? Useful or cluttering? Repeated in gentest? |
|
253 |
||
254 |
# TODO: Only if this is a bzr branch and bzr is available...
|
|
255 |
#say("===== Information on Random Query Generator version (bzr): =====\n");
|
|
256 |
#system("bzr info");
|
|
257 |
#system("bzr version-info");
|
|
258 |
#say("\n");
|
|
259 |
||
260 |
# Print MTR-style output saying which test suite/mode this is.
|
|
261 |
# Needed for Pushbuild reporting. The Pushbuild parser requires a certain format
|
|
262 |
# of the output.
|
|
263 |
# NOTE: So far we only support running one test at a time, so we only do this
|
|
264 |
# here.
|
|
265 |
print("\n"); |
|
266 |
print("#" x 78 . "\n"); |
|
267 |
print("# " . $config->name . "\n"); |
|
268 |
print("#" x 78 . "\n"); |
|
269 |
print("\n"); |
|
270 |
||
271 |
## Debug/development output, temporary only?:
|
|
272 |
#say("Test description:");
|
|
273 |
#print("\n--------------------\n".$config->description . "\n---------------------\n\n");
|
|
274 |
||
275 |
say("This is a CANDIDATE TEST.") if $candidate; |
|
276 |
||
277 |
# TODO: Remove? Require path to test file instead? (e.g. conf/runtime/rqg_info_schema.test)
|
|
278 |
# If not absolute path, it is relative to cwd at run time.
|
|
279 |
my $conf = $ENV{RQG_CONF}; # currently not used |
|
280 |
$conf = 'conf' if not defined $conf; |
|
281 |
||
282 |
||
283 |
################################################################################
|
|
284 |
# Run the test
|
|
285 |
################################################################################
|
|
286 |
||
287 |
# Generate command-line
|
|
288 |
# TODO: Call/run runall or server+gentest using object/module calls instead.
|
|
289 |
my $cmd = 'perl runall.pl ' . $randgen_opts . ' ' . $mysqld_options; |
|
290 |
$cmd =~ s{[\r\n\t]}{ }sgio; # remove line breaks and tabs form command |
|
291 |
||
292 |
||
293 |
if($config->dry_run) { |
|
294 |
# Display outbound command-line and exit
|
|
295 |
$spec_file ? say("Dry run, pretending to run test defined by ".$spec_file."...") |
|
296 |
: say("Dry run, pretending to run test defined on command-line..."); |
|
297 |
say($cmd); |
|
298 |
say("\n$0 Done."); |
|
299 |
safe_exit(); |
|
300 |
}
|
|
301 |
||
302 |
||
303 |
# run for real, process results, report, etc.
|
|
304 |
say("Command line:"); |
|
305 |
say($cmd); |
|
306 |
say(""); |
|
307 |
||
308 |
my $command_result = system($cmd); |
|
309 |
# shift result code to the right to obtain the code returned from the called script
|
|
310 |
my $command_result_shifted = ($command_result >> 8); |
|
311 |
||
312 |
||
313 |
################################################################################
|
|
314 |
# Process results and report.
|
|
315 |
################################################################################
|
|
316 |
||
317 |
# TODO
|
|
318 |
||
319 |
### Report test result in an MTR fashion.
|
|
320 |
### This is done so that Pushbuild will see it, parse it and add to xref database
|
|
321 |
### etc.
|
|
322 |
### Format: TESTSUITE.TESTCASE 'TESTMODE' [ RESULT ]
|
|
323 |
### Example: ndb.ndb_dd_alter 'InnoDB plugin' [ fail ]
|
|
324 |
### Not using TESTMODE for now.
|
|
325 |
||
326 |
my $full_test_name = $config->category.'.'.$config->name; |
|
327 |
# keep test statuses more or less vertically aligned
|
|
328 |
while (length $full_test_name < 40) |
|
329 |
{
|
|
330 |
$full_test_name = $full_test_name.' '; |
|
331 |
}
|
|
332 |
||
333 |
if ($command_result_shifted > 0) { |
|
334 |
# test failed
|
|
335 |
# Marking candidate test as "experimental" by reporting exp-fail status,
|
|
336 |
# as used by Pushbuild. TODO: Support other variations if needed.
|
|
337 |
if ($candidate) { |
|
338 |
print($full_test_name." [ exp-fail ]\n"); |
|
339 |
} else { |
|
340 |
print($full_test_name." [ fail ]\n"); |
|
341 |
}
|
|
342 |
say('Command failed with exit code '.$command_result_shifted); |
|
343 |
say('Look above this message in the test log for failure details.'); |
|
344 |
} else { |
|
345 |
print($full_test_name." [ pass ]\n"); |
|
346 |
}
|
|
347 |
||
348 |
################################################################################
|
|
349 |
# Clean up
|
|
350 |
################################################################################
|
|
351 |
||
352 |
say("\n$0 Done\n"); |
|
353 |
||
354 |
||
355 |
||
356 |
||
357 |
################################################################################
|
|
358 |
# Subroutines
|
|
359 |
################################################################################
|
|
360 |
||
361 |
###
|
|
362 |
### Displays help text. TODO: List more / all options?
|
|
363 |
###
|
|
364 |
sub help { |
|
365 |
||
366 |
print <<EOF |
|
367 |
||
368 |
###### Help for $0 - Testing via random query generation ######
|
|
369 |
||
370 |
Options:
|
|
371 |
||
372 |
--basedir : Top-level directory (basedir) of the database installation to be tested.
|
|
373 |
--config : Path to configuration file specifying a set of options, e.g. a test definition (.test).
|
|
374 |
Options specified in the config file can be overridden on the command-line.
|
|
375 |
--dry_run : Do not actually run the test, but process options and show resulting commands.
|
|
376 |
--help : Display this help message.
|
|
377 |
||
378 |
Example command-line:
|
|
379 |
$0 --config=conf/examples/example.test --basedir=/home/user/mysql-trunk --duration=60
|
|
380 |
||
381 |
This says "Run the test example.test found here... and set duration to 60 seconds".
|
|
382 |
Other required options are set in the example.test file.
|
|
383 |
||
384 |
Example .test file (for --config option):
|
|
385 |
||
386 |
{
|
|
387 |
name => 'rqg_example',
|
|
388 |
category => 'example',
|
|
389 |
description => 'Example test showing how to use the RQG',
|
|
390 |
grammar => 'conf/examples/example.yy',
|
|
391 |
threads => 5,
|
|
392 |
mysqld => {
|
|
393 |
'log-output' => 'file'
|
|
394 |
}
|
|
395 |
}
|
|
396 |
||
397 |
Command line options override options in --config file.
|
|
398 |
|
|
399 |
EOF
|
|
400 |
;
|
|
401 |
safe_exit(1); |
|
402 |
}
|
|
403 |
||
404 |
||
405 |
###
|
|
406 |
### Sets up environment variables and other stuff required when running in
|
|
407 |
### MySQL's Pushbuild environment.
|
|
408 |
###
|
|
409 |
sub setupPushbuildEnv { |
|
410 |
if (osWindows()) { |
|
411 |
# For tail and for cdb
|
|
412 |
# TODO: Remove randgen\bin, not used?
|
|
413 |
$ENV{PATH} = |
|
414 |
'G:\pb2\scripts\randgen\bin'. |
|
415 |
';G:\pb2\scripts\bin;C:\Program Files\Debugging Tools for Windows (x86)'. |
|
416 |
';'.$ENV{PATH}; |
|
417 |
||
418 |
# For cdb (stack traces)
|
|
419 |
$ENV{_NT_SYMBOL_PATH} = 'srv*c:\\cdb_symbols*http://msdl.microsoft.com/download/symbols;cache*c:\\cdb_symbols'; |
|
420 |
||
421 |
# For vlad (~2008-09)
|
|
422 |
#ENV{MYSQL_FULL_MINIDUMP} = 1;
|
|
423 |
||
424 |
} elsif (osSolaris()) { |
|
425 |
# For libmysqlclient
|
|
426 |
$ENV{LD_LIBRARY_PATH} = |
|
427 |
$ENV{LD_LIBRARY_PATH}. |
|
428 |
':/export/home/pb2/scripts/lib/'; |
|
429 |
||
430 |
# For DBI and DBD::mysql (on hosts with special Perl setup)
|
|
431 |
$ENV{PERL5LIB} = |
|
432 |
$ENV{PERL5LIB}. |
|
433 |
':/export/home/pb2/scripts/DBI-1.607/'. |
|
434 |
':/export/home/pb2/scripts/DBI-1.607/lib'. |
|
435 |
':/export/home/pb2/scripts/DBI-1.607/blib/arch/'. |
|
436 |
':/export/home/pb2/scripts/DBD-mysql-4.008/lib/'. |
|
437 |
':/export/home/pb2/scripts/DBD-mysql-4.008/blib/arch/'; |
|
438 |
||
439 |
# For c++filt
|
|
440 |
$ENV{PATH} = $ENV{PATH}.':/opt/studio12/SUNWspro/bin'; |
|
441 |
}
|
|
442 |
||
443 |
# We currently assume current working dir is a 'scripts' directory
|
|
444 |
# containing a copy of the random query generator.
|
|
445 |
# Make sure current working dir is the top-level randgen dir.
|
|
446 |
# TODO: Check if really needed.
|
|
447 |
chdir('randgen'); |
|
448 |
}
|