#! /usr/bin/env perl # Copyright (C) 2010 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 # USA use lib 'lib'; use lib "$ENV{RQG_HOME}/lib"; use lib 'randgen/lib'; use strict; use Carp; use Cwd; use DBI; use File::Find; use Getopt::Long; use POSIX; use Sys::Hostname; use GenTest; use GenTest::Properties; use GenTest::Constants; # TODO: # - clean up imports (use) not used. # - fix description after adjusting calling scripts (PB2) # - handle env vars RQG_HOME / RQG_CONF? # - add support for skipping tests and printing message (e.g.: If plugin not found) # - make basedir option a hash, skip basedirForRelease option? # - Reporting: Check and fix? More flexibility needed? # - propagate MTR_BUILD_THREAD / MTR_PORT_BASE, if needed? # - make vardir optional (mandatory vardir conflicts with --mem / MTR_MEM=1) # - add to help text (see sub help()) # - add logic for process cleanup (left-over mysqlds), at least if Pushbuild # - more defaults? # - exit safely # - See also other TODO comments below. ################################################################################ # # DRAFT - INCOMPLETE - DRAFT - INCOMPLETE - DRAFT # # This script is a wrapper around certain RQG functionality, primarily running # well-defined RQG tests in a convenient way. # # The script accepts arguments which uniquely identify the test case to be run. # From this identifier the script grabs information from a separate file which # defines the test case, including various options to the RQG and MySQL. # Then, the RQG's runall.pl script is executed with the options gathered from # the test definition. Options may be overridden on the command line. # The runall.pl script takes care of starting the database server as well as # executing the test itself. # # MySQL/Sun/Oracle notes: # # This script implements the interface between Pushbuild (automated testing # infrastructure) and the Random Query Generator. # The script is invoked by Pushbuild (PB2) slave instances when they have a job # matching certain conditions. # See tests_systemqa.py in pb2-trd-config repository for details. # # Pushbuild always provides the following arguments/oiptions to this script: # --basedir : Location of binaries to be tested # --vardir : Location of temporary test files (logs, database, etc.) # --branch : MySQL bazaar source code branch nick or other identifier # corresponding to the given binaries. # --config : File specifying which test specification to use. # The test specification contains test name, options, etc. # # We hope to merge/integrate this with what is currently runall.pl and the # new modules for server start that is being developed, eventually. ################################################################################ # # See bottom of file for subroutines. # # Autoflush output buffers (needed when using POSIX::_exit()) $| = 1; # Show in output when this script starts for ease of debugging when wrapped # inside other scripts. say("==================== Starting $0 ====================\n"); # Find out if this is Pushbuild (MySQL's automatic test system). # We assume this is Pushbuild if environment variable PB2WORKDIR is set. my $pushbuild = 1 if defined $ENV{'PB2WORKDIR'}; ################################################################################ # Option handling ################################################################################ # We require --config to specify a test definition file. # Properties.pm treats this option in a special way: As a file containing other # options. # # Options set in this config file may be overridden on the command line. # The resulting (merged) set of options is then passed to the RQG itself. # Note: When given on command line, mysqld options should not include 'inner' # "--" prefix, e.g. "--mysqld=--lock-wait-timeout=2". Because prefixes # will be added automatically, specify "--mysqld=lock-wait-timeout=2" # or "--mysqld lock-wait-timeout=2" instead # (TODO: Add logic to handle inner -- on command line?). # # TODO: Have runall.pl use GenTest::Properties as well. # # Consider: Accept --test on command line and find test definition file based # on this (see parsing of such a file in lib/GenTest/App/Gendata.pm), # instead of having config as a special value, with path and all. # Read options. # old: my ($basedir, $vardir, $tree, $test) = @ARGV; # new: We use GenTest::Properties mechanisms my @ARGV_saved = @ARGV; # Print full command line used for calling this script, so that it will appear # in logs and can be repeated relatively easily. # Avoiding say()'s prefixing to make it easy to copy & paste. say("\nInbound command line was: "); print("$0 \\ \n ".join(" \\ \n ", @ARGV_saved)."\n"); my $options = {}; my $opt_result = GetOptions($options, 'basedir=s', 'category=s', 'config=s', # The actual test definition, containing options. 'grammar=s', 'name=s', 'vardir=s', # TODO: Make optional? 'basedirForRelease:s%', # hash of paths to basedir for releases to test/compare 'branch:s', # TODO: Remove, get from ENV{BRANCH_NAME} instead if needed? 'candidate:s', 'dry_run', 'duration:s', 'help', 'mysqld:s%', 'threads:s', 'verbose!' ); my $config = GenTest::Properties->new( options => $options, legal => [ # Adds to the union of $options, $required and $defaults. Include here # options that may be specified in separate files, otherwise they # will be refused ($options only includes options actually given on # the command line). 'basedir', 'basedirForRelease', 'candidate', 'category', 'description', 'dry_run', 'grammar', 'help', 'mysqld', 'name', 'threads', 'vardir', ], required => [ 'basedir', # Need to know what to test (binaries) 'category', # A test with unknown category is not well-defined. 'config', # File specifying a test. Used for forcing a well-defined approach. 'grammar', # No random queries without a grammar... 'name' # So that we can refer to it later and report results. ], defaults => { threads => 1, # Keep it simple by default. duration => 300, # Suitable for regular automated 'regression testing' candidate => '', verbose => '', basedirForRelease => { # Locations of binaries for various releases '5.0' => osWindows() ? 'G:\mysql-releases\mysql-5.0.87-win32' : '/export/home/mysql-releases/mysql-5.0' } }, help => \&help ); # Print help text if --help or no options were specified $config->printHelp if not $opt_result or $config->help; # List of options that are to be sent to the RQG runner script if defined. my @randgen_opts_noprefix = ( 'basedir', 'vardir', 'grammar', 'threads', 'duration' ); ### Process options... # Get list of options as string with appropriate prefixes. my $mysqld_options = $config->genOpt('--mysqld=--', 'mysqld'); my $randgen_opts = $config->collectOpt('--', @randgen_opts_noprefix); #say(my $all_non_complex_opts = $config->collectOpt('--')); ### Set helper variables # candidate: Match only exact string 'true' (case insensitive) # Candidate tests may be treated differently e.g. wrt. reporting. my $candidate = 1 if $config->candidate =~ m{^true$}io; my $spec_file = $config->config; # file with test definition ################################################################################ # Prepare test, display info. ################################################################################ say("Loaded test definition from file " . $spec_file) if $spec_file; # # Prepare ENV variables and other settings. # setupPushbuildEnv() if ($pushbuild); # # Print easy-to-read info about the environment (useful for debugging etc.) # say("\n======== Information on the host system: ========"); say(" - Local time : ".localtime()); say(" - Hostname : ".hostname()); say(" - PID : $$"); say(" - Working dir : ".cwd()); say(" - PATH : ".$ENV{'PATH'}); say("\n======== Configuration: ========"); $config->printProps(); # TODO or not TODO? Useful or cluttering? Repeated in gentest? # TODO: Only if this is a bzr branch and bzr is available... #say("===== Information on Random Query Generator version (bzr): =====\n"); #system("bzr info"); #system("bzr version-info"); #say("\n"); # Print MTR-style output saying which test suite/mode this is. # Needed for Pushbuild reporting. The Pushbuild parser requires a certain format # of the output. # NOTE: So far we only support running one test at a time, so we only do this # here. print("\n"); print("#" x 78 . "\n"); print("# " . $config->name . "\n"); print("#" x 78 . "\n"); print("\n"); ## Debug/development output, temporary only?: #say("Test description:"); #print("\n--------------------\n".$config->description . "\n---------------------\n\n"); say("This is a CANDIDATE TEST.") if $candidate; # TODO: Remove? Require path to test file instead? (e.g. conf/runtime/rqg_info_schema.test) # If not absolute path, it is relative to cwd at run time. my $conf = $ENV{RQG_CONF}; # currently not used $conf = 'conf' if not defined $conf; ################################################################################ # Run the test ################################################################################ # Generate command-line # TODO: Call/run runall or server+gentest using object/module calls instead. my $cmd = 'perl runall.pl ' . $randgen_opts . ' ' . $mysqld_options; $cmd =~ s{[\r\n\t]}{ }sgio; # remove line breaks and tabs form command if($config->dry_run) { # Display outbound command-line and exit $spec_file ? say("Dry run, pretending to run test defined by ".$spec_file."...") : say("Dry run, pretending to run test defined on command-line..."); say($cmd); say("\n$0 Done."); safe_exit(); } # run for real, process results, report, etc. say("Command line:"); say($cmd); say(""); my $command_result = system($cmd); # shift result code to the right to obtain the code returned from the called script my $command_result_shifted = ($command_result >> 8); ################################################################################ # Process results and report. ################################################################################ # TODO ### Report test result in an MTR fashion. ### This is done so that Pushbuild will see it, parse it and add to xref database ### etc. ### Format: TESTSUITE.TESTCASE 'TESTMODE' [ RESULT ] ### Example: ndb.ndb_dd_alter 'InnoDB plugin' [ fail ] ### Not using TESTMODE for now. my $full_test_name = $config->category.'.'.$config->name; # keep test statuses more or less vertically aligned while (length $full_test_name < 40) { $full_test_name = $full_test_name.' '; } if ($command_result_shifted > 0) { # test failed # Marking candidate test as "experimental" by reporting exp-fail status, # as used by Pushbuild. TODO: Support other variations if needed. if ($candidate) { print($full_test_name." [ exp-fail ]\n"); } else { print($full_test_name." [ fail ]\n"); } say('Command failed with exit code '.$command_result_shifted); say('Look above this message in the test log for failure details.'); } else { print($full_test_name." [ pass ]\n"); } ################################################################################ # Clean up ################################################################################ say("\n$0 Done\n"); ################################################################################ # Subroutines ################################################################################ ### ### Displays help text. TODO: List more / all options? ### sub help { print < 'rqg_example', category => 'example', description => 'Example test showing how to use the RQG', grammar => 'conf/examples/example.yy', threads => 5, mysqld => { 'log-output' => 'file' } } Command line options override options in --config file. EOF ; safe_exit(1); } ### ### Sets up environment variables and other stuff required when running in ### MySQL's Pushbuild environment. ### sub setupPushbuildEnv { if (osWindows()) { # For tail and for cdb # TODO: Remove randgen\bin, not used? $ENV{PATH} = 'G:\pb2\scripts\randgen\bin'. ';G:\pb2\scripts\bin;C:\Program Files\Debugging Tools for Windows (x86)'. ';'.$ENV{PATH}; # For cdb (stack traces) $ENV{_NT_SYMBOL_PATH} = 'srv*c:\\cdb_symbols*http://msdl.microsoft.com/download/symbols;cache*c:\\cdb_symbols'; # For vlad (~2008-09) #ENV{MYSQL_FULL_MINIDUMP} = 1; } elsif (osSolaris()) { # For libmysqlclient $ENV{LD_LIBRARY_PATH} = $ENV{LD_LIBRARY_PATH}. ':/export/home/pb2/scripts/lib/'; # For DBI and DBD::mysql (on hosts with special Perl setup) $ENV{PERL5LIB} = $ENV{PERL5LIB}. ':/export/home/pb2/scripts/DBI-1.607/'. ':/export/home/pb2/scripts/DBI-1.607/lib'. ':/export/home/pb2/scripts/DBI-1.607/blib/arch/'. ':/export/home/pb2/scripts/DBD-mysql-4.008/lib/'. ':/export/home/pb2/scripts/DBD-mysql-4.008/blib/arch/'; # For c++filt $ENV{PATH} = $ENV{PATH}.':/opt/studio12/SUNWspro/bin'; } # We currently assume current working dir is a 'scripts' directory # containing a copy of the random query generator. # Make sure current working dir is the top-level randgen dir. # TODO: Check if really needed. chdir('randgen'); }