~drizzle-trunk/drizzle/development : contents of mysql-test/lib/mtr

~drizzle-trunk/drizzle/development : (revision 1)

# -*- cperl -*-
# Copyright (C) 2006 MySQL AB
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

# This is a library file used by the Perl version of mysql-test-run,
# and is part of the translation of the Bourne shell script with the
# same name.

use strict;

# Private IM-related operations.

sub mtr_im_kill_process ($$$$);
sub mtr_im_load_pids ($);
sub mtr_im_terminate ($);
sub mtr_im_check_alive ($);
sub mtr_im_check_main_alive ($);
sub mtr_im_check_angel_alive ($);
sub mtr_im_check_mysqlds_alive ($);
sub mtr_im_check_mysqld_alive ($);
sub mtr_im_cleanup ($);
sub mtr_im_rm_file ($);
sub mtr_im_errlog ($);
sub mtr_im_kill ($);
sub mtr_im_wait_for_connection ($$$);
sub mtr_im_wait_for_mysqld($$$);

# Public IM-related operations.

sub mtr_im_start ($$);
sub mtr_im_stop ($);

##############################################################################
#
#  Private operations.
#
##############################################################################

sub mtr_im_kill_process ($$$$) {
  my $pid_lst= shift;
  my $signal= shift;
  my $total_retries= shift;
  my $timeout= shift;

  my %pids;

  foreach my $pid ( @{$pid_lst} )
  {
    $pids{$pid}= 1;
  }

  for ( my $cur_attempt= 1; $cur_attempt <= $total_retries; ++$cur_attempt )
  {
    foreach my $pid ( keys %pids )
    {
      mtr_debug("Sending $signal to $pid...");

      kill($signal, $pid);

      unless ( kill (0, $pid) )
      {
        mtr_debug("Process $pid died.");
        delete $pids{$pid};
      }
    }

    return if scalar keys %pids == 0;

    mtr_debug("Sleeping $timeout second(s) waiting for processes to die...");

    sleep($timeout);
  }

  mtr_debug("Process(es) " .
            join(' ', keys %pids) .
            " is still alive after $total_retries " .
            "of sending signal $signal.");
}

###########################################################################

sub mtr_im_load_pids($) {
  my $im= shift;

  mtr_debug("Loading PID files...");

  # Obtain mysqld-process pids.

  my $instances = $im->{'instances'};

  for ( my $idx= 0; $idx < 2; ++$idx )
  {
    mtr_debug("IM-guarded mysqld[$idx] PID file: '" .
              $instances->[$idx]->{'path_pid'} . "'.");

    my $mysqld_pid;

    if ( -r $instances->[$idx]->{'path_pid'} )
    {
      $mysqld_pid= mtr_get_pid_from_file($instances->[$idx]->{'path_pid'});
      mtr_debug("IM-guarded mysqld[$idx] PID: $mysqld_pid.");
    }
    else
    {
      $mysqld_pid= undef;
      mtr_debug("IM-guarded mysqld[$idx]: no PID file.");
    }

    $instances->[$idx]->{'pid'}= $mysqld_pid;
  }

  # Re-read Instance Manager PIDs from the file, since during tests Instance
  # Manager could have been restarted, so its PIDs could have been changed.

  #   - IM-main

  mtr_debug("IM-main PID file: '$im->{path_pid}'.");

  if ( -f $im->{'path_pid'} )
  {
    $im->{'pid'} =
      mtr_get_pid_from_file($im->{'path_pid'});

    mtr_debug("IM-main PID: $im->{pid}.");
  }
  else
  {
    mtr_debug("IM-main: no PID file.");
    $im->{'pid'}= undef;
  }

  #   - IM-angel

  mtr_debug("IM-angel PID file: '$im->{path_angel_pid}'.");

  if ( -f $im->{'path_angel_pid'} )
  {
    $im->{'angel_pid'} =
      mtr_get_pid_from_file($im->{'path_angel_pid'});

    mtr_debug("IM-angel PID: $im->{'angel_pid'}.");
  }
  else
  {
    mtr_debug("IM-angel: no PID file.");
    $im->{'angel_pid'} = undef;
  }
}

###########################################################################

sub mtr_im_terminate($) {
  my $im= shift;

  # Load pids from pid-files. We should do it first of all, because IM deletes
  # them on shutdown.

  mtr_im_load_pids($im);

  mtr_debug("Shutting Instance Manager down...");

  # Ignoring SIGCHLD so that all children could rest in peace.

  start_reap_all();

  # Send SIGTERM to IM-main.

  if ( defined $im->{'pid'} )
  {
    mtr_debug("IM-main pid: $im->{pid}.");
    mtr_debug("Stopping IM-main...");

    mtr_im_kill_process([ $im->{'pid'} ], 'TERM', 10, 1);
  }
  else
  {
    mtr_debug("IM-main pid: n/a.");
  }

  # If IM-angel was alive, wait for it to die.

  if ( defined $im->{'angel_pid'} )
  {
    mtr_debug("IM-angel pid: $im->{'angel_pid'}.");
    mtr_debug("Waiting for IM-angel to die...");

    my $total_attempts= 10;

    for ( my $cur_attempt=1; $cur_attempt <= $total_attempts; ++$cur_attempt )
    {
      unless ( kill (0, $im->{'angel_pid'}) )
      {
        mtr_debug("IM-angel died.");
        last;
      }

      sleep(1);
    }
  }
  else
  {
    mtr_debug("IM-angel pid: n/a.");
  }

  stop_reap_all();

  # Re-load PIDs.

  mtr_im_load_pids($im);
}

###########################################################################

sub mtr_im_check_alive($) {
  my $im= shift;

  mtr_debug("Checking whether IM-components are alive...");

  return 1 if mtr_im_check_main_alive($im);

  return 1 if mtr_im_check_angel_alive($im);

  return 1 if mtr_im_check_mysqlds_alive($im);

  return 0;
}

###########################################################################

sub mtr_im_check_main_alive($) {
  my $im= shift;

  # Check that the process, that we know to be IM's, is dead.

  if ( defined $im->{'pid'} )
  {
    if ( kill (0, $im->{'pid'}) )
    {
      mtr_debug("IM-main (PID: $im->{pid}) is alive.");
      return 1;
    }
    else
    {
      mtr_debug("IM-main (PID: $im->{pid}) is dead.");
    }
  }
  else
  {
    mtr_debug("No PID file for IM-main.");
  }

  # Check that IM does not accept client connections.

  if ( mtr_ping_port($im->{'port'}) )
  {
    mtr_debug("IM-main (port: $im->{port}) " .
              "is accepting connections.");

    mtr_im_errlog("IM-main is accepting connections on port " .
                  "$im->{port}, but there is no " .
                  "process information.");
    return 1;
  }
  else
  {
    mtr_debug("IM-main (port: $im->{port}) " .
              "does not accept connections.");
    return 0;
  }
}

###########################################################################

sub mtr_im_check_angel_alive($) {
  my $im= shift;

  # Check that the process, that we know to be the Angel, is dead.

  if ( defined $im->{'angel_pid'} )
  {
    if ( kill (0, $im->{'angel_pid'}) )
    {
      mtr_debug("IM-angel (PID: $im->{angel_pid}) is alive.");
      return 1;
    }
    else
    {
      mtr_debug("IM-angel (PID: $im->{angel_pid}) is dead.");
      return 0;
    }
  }
  else
  {
    mtr_debug("No PID file for IM-angel.");
    return 0;
  }
}

###########################################################################

sub mtr_im_check_mysqlds_alive($) {
  my $im= shift;

  mtr_debug("Checking for IM-guarded mysqld instances...");

  my $instances = $im->{'instances'};

  for ( my $idx= 0; $idx < 2; ++$idx )
  {
    mtr_debug("Checking mysqld[$idx]...");

    return 1
      if mtr_im_check_mysqld_alive($instances->[$idx]);
  }
}

###########################################################################

sub mtr_im_check_mysqld_alive($) {
  my $mysqld_instance= shift;

  # Check that the process is dead.

  if ( defined $mysqld_instance->{'pid'} )
  {
    if ( kill (0, $mysqld_instance->{'pid'}) )
    {
      mtr_debug("Mysqld instance (PID: $mysqld_instance->{pid}) is alive.");
      return 1;
    }
    else
    {
      mtr_debug("Mysqld instance (PID: $mysqld_instance->{pid}) is dead.");
    }
  }
  else
  {
    mtr_debug("No PID file for mysqld instance.");
  }

  # Check that mysqld does not accept client connections.

  if ( mtr_ping_port($mysqld_instance->{'port'}) )
  {
    mtr_debug("Mysqld instance (port: $mysqld_instance->{port}) " .
              "is accepting connections.");

    mtr_im_errlog("Mysqld is accepting connections on port " .
                  "$mysqld_instance->{port}, but there is no " .
                  "process information.");
    return 1;
  }
  else
  {
    mtr_debug("Mysqld instance (port: $mysqld_instance->{port}) " .
              "does not accept connections.");
    return 0;
  }
}

###########################################################################

sub mtr_im_cleanup($) {
  my $im= shift;

  mtr_im_rm_file($im->{'path_pid'});
  mtr_im_rm_file($im->{'path_sock'});

  mtr_im_rm_file($im->{'path_angel_pid'});

  for ( my $idx= 0; $idx < 2; ++$idx )
  {
    mtr_im_rm_file($im->{'instances'}->[$idx]->{'path_pid'});
    mtr_im_rm_file($im->{'instances'}->[$idx]->{'path_sock'});
  }
}

###########################################################################

sub mtr_im_rm_file($)
{
  my $file_path= shift;

  if ( -f $file_path )
  {
    mtr_debug("Removing '$file_path'...");

    unless ( unlink($file_path) )
    {
      mtr_warning("Can not remove '$file_path'.")
    }
  }
  else
  {
    mtr_debug("File '$file_path' does not exist already.");
  }
}

###########################################################################

sub mtr_im_errlog($) {
  my $msg= shift;

  # Complain in error log so that a warning will be shown.
  # 
  # TODO: unless BUG#20761 is fixed, we will print the warning to stdout, so
  # that it can be seen on console and does not produce pushbuild error.

  # my $errlog= "$opt_vardir/log/mysql-test-run.pl.err";
  # 
  # open (ERRLOG, ">>$errlog") ||
  #   mtr_error("Can not open error log ($errlog)");
  # 
  # my $ts= localtime();
  # print ERRLOG
  #   "Warning: [$ts] $msg\n";
  # 
  # close ERRLOG;

  my $ts= localtime();
  print "Warning: [$ts] $msg\n";
}

###########################################################################

sub mtr_im_kill($) {
  my $im= shift;

  # Re-load PIDs. That can be useful because some processes could have been
  # restarted.

  mtr_im_load_pids($im);

  # Ignoring SIGCHLD so that all children could rest in peace.

  start_reap_all();

  # Kill IM-angel first of all.

  if ( defined $im->{'angel_pid'} )
  {
    mtr_debug("Killing IM-angel (PID: $im->{angel_pid})...");
    mtr_im_kill_process([ $im->{'angel_pid'} ], 'KILL', 10, 1)
  }
  else
  {
    mtr_debug("IM-angel is dead.");
  }

  # Re-load PIDs again.

  mtr_im_load_pids($im);

  # Kill IM-main.
  
  if ( defined $im->{'pid'} )
  {
    mtr_debug("Killing IM-main (PID: $im->pid})...");
    mtr_im_kill_process([ $im->{'pid'} ], 'KILL', 10, 1);
  }
  else
  {
    mtr_debug("IM-main is dead.");
  }

  # Re-load PIDs again.

  mtr_im_load_pids($im);

  # Kill guarded mysqld instances.

  my @mysqld_pids;

  mtr_debug("Collecting PIDs of mysqld instances to kill...");

  for ( my $idx= 0; $idx < 2; ++$idx )
  {
    my $pid= $im->{'instances'}->[$idx]->{'pid'};

    unless ( defined $pid )
    {
      next;
    }

    mtr_debug("  - IM-guarded mysqld[$idx] PID: $pid.");

    push (@mysqld_pids, $pid);
  }

  if ( scalar @mysqld_pids > 0 )
  {
    mtr_debug("Killing IM-guarded mysqld instances...");
    mtr_im_kill_process(\@mysqld_pids, 'KILL', 10, 1);
  }

  # That's all.

  stop_reap_all();
}

##############################################################################

sub mtr_im_wait_for_connection($$$) {
  my $im= shift;
  my $total_attempts= shift;
  my $connect_timeout= shift;

  mtr_debug("Waiting for IM on port $im->{port} " .
            "to start accepting connections...");

  for ( my $cur_attempt= 1; $cur_attempt <= $total_attempts; ++$cur_attempt )
  {
    mtr_debug("Trying to connect to IM ($cur_attempt of $total_attempts)...");

    if ( mtr_ping_port($im->{'port'}) )
    {
      mtr_debug("IM is accepting connections " .
                "on port $im->{port}.");
      return 1;
    }

    mtr_debug("Sleeping $connect_timeout...");
    sleep($connect_timeout);
  }

  mtr_debug("IM does not accept connections " .
            "on port $im->{port} after " .
            ($total_attempts * $connect_timeout) . " seconds.");

  return 0;
}

##############################################################################

sub mtr_im_wait_for_mysqld($$$) {
  my $mysqld= shift;
  my $total_attempts= shift;
  my $connect_timeout= shift;

  mtr_debug("Waiting for IM-guarded mysqld on port $mysqld->{port} " .
            "to start accepting connections...");

  for ( my $cur_attempt= 1; $cur_attempt <= $total_attempts; ++$cur_attempt )
  {
    mtr_debug("Trying to connect to mysqld " .
              "($cur_attempt of $total_attempts)...");

    if ( mtr_ping_port($mysqld->{'port'}) )
    {
      mtr_debug("Mysqld is accepting connections " .
                "on port $mysqld->{port}.");
      return 1;
    }

    mtr_debug("Sleeping $connect_timeout...");
    sleep($connect_timeout);
  }

  mtr_debug("Mysqld does not accept connections " .
            "on port $mysqld->{port} after " .
            ($total_attempts * $connect_timeout) . " seconds.");

  return 0;
}

##############################################################################
#
#  Public operations.
#
##############################################################################

sub mtr_im_start($$) {
  my $im = shift;
  my $opts = shift;

  mtr_debug("Starting Instance Manager...");

  my $args;
  mtr_init_args(\$args);
  mtr_add_arg($args, "--defaults-file=%s", $im->{'defaults_file'});

  foreach my $opt ( @{$opts} )
  {
    mtr_add_arg($args, $opt);
  }

  $im->{'spawner_pid'} =
    mtr_spawn(
      $::exe_im,                        # path to the executable
      $args,                            # cmd-line args
      '',                               # stdin
      $im->{'path_log'},                # stdout
      $im->{'path_err'},                # stderr
      '',                               # pid file path (not used)
      { append_log_file => 1 }          # append log files
      );

  unless ( $im->{'spawner_pid'} )
  {
    mtr_error('Could not start Instance Manager.')
  }

  # Instance Manager can be run in daemon mode. In this case, it creates
  # several processes and the parent process, created by mtr_spawn(), exits just
  # after start. So, we have to obtain Instance Manager PID from the PID file.

  mtr_debug("Waiting for IM to create PID file (" .
            "path: '$im->{path_pid}'; " .
            "timeout: $im->{start_timeout})...");

  unless ( sleep_until_file_created($im->{'path_pid'},
                                    $im->{'start_timeout'},
                                    -1) ) # real PID is still unknown
  {
    mtr_debug("IM has not created PID file in $im->{start_timeout} secs.");
    mtr_debug("Aborting test suite...");

    mtr_kill_leftovers();

    mtr_report("IM has not created PID file in $im->{start_timeout} secs.");
    return 0;
  }

  $im->{'pid'}= mtr_get_pid_from_file($im->{'path_pid'});

  mtr_debug("Instance Manager started. PID: $im->{pid}.");

  # Wait until we can connect to IM.

  my $IM_CONNECT_TIMEOUT= 30;

  unless ( mtr_im_wait_for_connection($im,
                                      $IM_CONNECT_TIMEOUT, 1) )
  {
    mtr_debug("Can not connect to Instance Manager " .
              "in $IM_CONNECT_TIMEOUT seconds after start.");
    mtr_debug("Aborting test suite...");

    mtr_kill_leftovers();

    mtr_report("Can not connect to Instance Manager " .
               "in $IM_CONNECT_TIMEOUT seconds after start.");
    return 0;
  }

  # Wait for IM to start guarded instances:
  #   - wait for PID files;

  mtr_debug("Waiting for guarded mysqlds instances to create PID files...");

  for ( my $idx= 0; $idx < 2; ++$idx )
  {
    my $mysqld= $im->{'instances'}->[$idx];

    if ( exists $mysqld->{'nonguarded'} )
    {
      next;
    }

    mtr_debug("Waiting for mysqld[$idx] to create PID file (" .
              "path: '$mysqld->{path_pid}'; " .
              "timeout: $mysqld->{start_timeout})...");

    unless ( sleep_until_file_created($mysqld->{'path_pid'},
                                      $mysqld->{'start_timeout'},
                                      -1) ) # real PID is still unknown
    {
      mtr_debug("mysqld[$idx] has not created PID file in " .
                 "$mysqld->{start_timeout} secs.");
      mtr_debug("Aborting test suite...");

      mtr_kill_leftovers();

      mtr_report("mysqld[$idx] has not created PID file in " .
                 "$mysqld->{start_timeout} secs.");
      return 0;
    }

    mtr_debug("PID file for mysqld[$idx] ($mysqld->{path_pid} created.");
  }

  # Wait until we can connect to guarded mysqld-instances
  # (in other words -- wait for IM to start guarded instances).

  mtr_debug("Waiting for guarded mysqlds to start accepting connections...");

  for ( my $idx= 0; $idx < 2; ++$idx )
  {
    my $mysqld= $im->{'instances'}->[$idx];

    if ( exists $mysqld->{'nonguarded'} )
    {
      next;
    }

    mtr_debug("Waiting for mysqld[$idx] to accept connection...");

    unless ( mtr_im_wait_for_mysqld($mysqld, 30, 1) )
    {
      mtr_debug("Can not connect to mysqld[$idx] " .
                "in $IM_CONNECT_TIMEOUT seconds after start.");
      mtr_debug("Aborting test suite...");

      mtr_kill_leftovers();

      mtr_report("Can not connect to mysqld[$idx] " .
                 "in $IM_CONNECT_TIMEOUT seconds after start.");
      return 0;
    }

    mtr_debug("mysqld[$idx] started.");
  }

  mtr_debug("Instance Manager and its components are up and running.");

  return 1;
}

##############################################################################

sub mtr_im_stop($) {
  my $im= shift;

  mtr_debug("Stopping Instance Manager...");

  # Try graceful shutdown.

  mtr_im_terminate($im);

  # Check that all processes died.

  unless ( mtr_im_check_alive($im) )
  {
    mtr_debug("Instance Manager has been stopped successfully.");
    mtr_im_cleanup($im);
    return 1;
  }

  # Instance Manager don't want to die. We should kill it.

  mtr_im_errlog("Instance Manager did not shutdown gracefully.");

  mtr_im_kill($im);

  # Check again that all IM-related processes have been killed.

  my $im_is_alive= mtr_im_check_alive($im);

  mtr_im_cleanup($im);

  if ( $im_is_alive )
  {
    mtr_debug("Can not kill Instance Manager or its children.");
    return 0;
  }

  mtr_debug("Instance Manager has been killed successfully.");
  return 1;
}

###########################################################################

1;

1 by brian clean slate	1	# -- cperl --
	2	# Copyright (C) 2006 MySQL AB
	3	#
	4	# This program is free software; you can redistribute it and/or modify
	5	# it under the terms of the GNU General Public License as published by
	6	# the Free Software Foundation; version 2 of the License.
	7	#
	8	# This program is distributed in the hope that it will be useful,
	9	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	10	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	11	# GNU General Public License for more details.
	12	#
	13	# You should have received a copy of the GNU General Public License
	14	# along with this program; if not, write to the Free Software
	15	# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	16
	17	# This is a library file used by the Perl version of mysql-test-run,
	18	# and is part of the translation of the Bourne shell script with the
	19	# same name.
	20
	21	use strict;
	22
	23	# Private IM-related operations.
	24
	25	sub mtr_im_kill_process ($$$$);
	26	sub mtr_im_load_pids ($);
	27	sub mtr_im_terminate ($);
	28	sub mtr_im_check_alive ($);
	29	sub mtr_im_check_main_alive ($);
	30	sub mtr_im_check_angel_alive ($);
	31	sub mtr_im_check_mysqlds_alive ($);
	32	sub mtr_im_check_mysqld_alive ($);
	33	sub mtr_im_cleanup ($);
	34	sub mtr_im_rm_file ($);
	35	sub mtr_im_errlog ($);
	36	sub mtr_im_kill ($);
	37	sub mtr_im_wait_for_connection ($$$);
	38	sub mtr_im_wait_for_mysqld($$$);
	39
	40	# Public IM-related operations.
	41
	42	sub mtr_im_start ($$);
	43	sub mtr_im_stop ($);
	44
	45	##############################################################################
	46	#
	47	# Private operations.
	48	#
	49	##############################################################################
	50
	51	sub mtr_im_kill_process ($$$$) {
	52	my $pid_lst= shift;
	53	my $signal= shift;
	54	my $total_retries= shift;
	55	my $timeout= shift;
	56
	57	my %pids;
	58
	59	foreach my $pid ( @{$pid_lst} )
	60	{
	61	$pids{$pid}= 1;
	62	}
	63
	64	for ( my $cur_attempt= 1; $cur_attempt <= $total_retries; ++$cur_attempt )
65	{
66	foreach my $pid ( keys %pids )
67	{
68	mtr_debug("Sending $signal to $pid...");
69
70	kill($signal, $pid);
71
72	unless ( kill (0, $pid) )
73	{
74	mtr_debug("Process $pid died.");
75	delete $pids{$pid};
76	}
77	}
78
79	return if scalar keys %pids == 0;
80
81	mtr_debug("Sleeping $timeout second(s) waiting for processes to die...");
82
83	sleep($timeout);
84	}
85
86	mtr_debug("Process(es) " .
87	join(' ', keys %pids) .
88	" is still alive after $total_retries " .
89	"of sending signal $signal.");
90	}
91
92	###########################################################################
93
94	sub mtr_im_load_pids($) {
95	my $im= shift;
96
97	mtr_debug("Loading PID files...");
98
99	# Obtain mysqld-process pids.
100
101	my $instances = $im->{'instances'};
102
103	for ( my $idx= 0; $idx < 2; ++$idx )
104	{
105	mtr_debug("IM-guarded mysqld[$idx] PID file: '" .
106	$instances->[$idx]->{'path_pid'} . "'.");
107
108	my $mysqld_pid;
109
110	if ( -r $instances->[$idx]->{'path_pid'} )
111	{
112	$mysqld_pid= mtr_get_pid_from_file($instances->[$idx]->{'path_pid'});
113	mtr_debug("IM-guarded mysqld[$idx] PID: $mysqld_pid.");
114	}
115	else
116	{
117	$mysqld_pid= undef;
118	mtr_debug("IM-guarded mysqld[$idx]: no PID file.");
119	}
120
121	$instances->[$idx]->{'pid'}= $mysqld_pid;
122	}
123
124	# Re-read Instance Manager PIDs from the file, since during tests Instance
125	# Manager could have been restarted, so its PIDs could have been changed.
126
127	# - IM-main
128
129	mtr_debug("IM-main PID file: '$im->{path_pid}'.");
130
131	if ( -f $im->{'path_pid'} )
132	{
133	$im->{'pid'} =
134	mtr_get_pid_from_file($im->{'path_pid'});
135
136	mtr_debug("IM-main PID: $im->{pid}.");
137	}
138	else
139	{
140	mtr_debug("IM-main: no PID file.");
141	$im->{'pid'}= undef;
142	}
143
144	# - IM-angel
145
146	mtr_debug("IM-angel PID file: '$im->{path_angel_pid}'.");
147
148	if ( -f $im->{'path_angel_pid'} )
149	{
150	$im->{'angel_pid'} =
151	mtr_get_pid_from_file($im->{'path_angel_pid'});
152
153	mtr_debug("IM-angel PID: $im->{'angel_pid'}.");
154	}
155	else
156	{
157	mtr_debug("IM-angel: no PID file.");
158	$im->{'angel_pid'} = undef;
159	}
160	}
161
162	###########################################################################
163
164	sub mtr_im_terminate($) {
165	my $im= shift;
166
167	# Load pids from pid-files. We should do it first of all, because IM deletes
168	# them on shutdown.
169
170	mtr_im_load_pids($im);
171
172	mtr_debug("Shutting Instance Manager down...");
173
174	# Ignoring SIGCHLD so that all children could rest in peace.
175
176	start_reap_all();
177
178	# Send SIGTERM to IM-main.
179
180	if ( defined $im->{'pid'} )
181	{
182	mtr_debug("IM-main pid: $im->{pid}.");
183	mtr_debug("Stopping IM-main...");
184
185	mtr_im_kill_process([ $im->{'pid'} ], 'TERM', 10, 1);
186	}
187	else
188	{
189	mtr_debug("IM-main pid: n/a.");
190	}
191
192	# If IM-angel was alive, wait for it to die.
193
194	if ( defined $im->{'angel_pid'} )
195	{
196	mtr_debug("IM-angel pid: $im->{'angel_pid'}.");
197	mtr_debug("Waiting for IM-angel to die...");
198
199	my $total_attempts= 10;
200
201	for ( my $cur_attempt=1; $cur_attempt <= $total_attempts; ++$cur_attempt )
202	{
203	unless ( kill (0, $im->{'angel_pid'}) )
204	{
205	mtr_debug("IM-angel died.");
206	last;
207	}
208
209	sleep(1);
210	}
211	}
212	else
213	{
214	mtr_debug("IM-angel pid: n/a.");
215	}
216
217	stop_reap_all();
218
219	# Re-load PIDs.
220
221	mtr_im_load_pids($im);
222	}
223
224	###########################################################################
225
226	sub mtr_im_check_alive($) {
227	my $im= shift;
228
229	mtr_debug("Checking whether IM-components are alive...");
230
231	return 1 if mtr_im_check_main_alive($im);
232
233	return 1 if mtr_im_check_angel_alive($im);
234
235	return 1 if mtr_im_check_mysqlds_alive($im);
236
237	return 0;
238	}
239
240	###########################################################################
241
242	sub mtr_im_check_main_alive($) {
243	my $im= shift;
244
245	# Check that the process, that we know to be IM's, is dead.
246
247	if ( defined $im->{'pid'} )
248	{
249	if ( kill (0, $im->{'pid'}) )
250	{
251	mtr_debug("IM-main (PID: $im->{pid}) is alive.");
252	return 1;
253	}
254	else
255	{
256	mtr_debug("IM-main (PID: $im->{pid}) is dead.");
257	}
258	}
259	else
260	{
261	mtr_debug("No PID file for IM-main.");
262	}
263
264	# Check that IM does not accept client connections.
265
266	if ( mtr_ping_port($im->{'port'}) )
267	{
268	mtr_debug("IM-main (port: $im->{port}) " .
269	"is accepting connections.");
270
271	mtr_im_errlog("IM-main is accepting connections on port " .
272	"$im->{port}, but there is no " .
273	"process information.");
274	return 1;
275	}
276	else
277	{
278	mtr_debug("IM-main (port: $im->{port}) " .
279	"does not accept connections.");
280	return 0;
281	}
282	}
283
284	###########################################################################
285
286	sub mtr_im_check_angel_alive($) {
287	my $im= shift;
288
289	# Check that the process, that we know to be the Angel, is dead.
290
291	if ( defined $im->{'angel_pid'} )
292	{
293	if ( kill (0, $im->{'angel_pid'}) )
294	{
295	mtr_debug("IM-angel (PID: $im->{angel_pid}) is alive.");
296	return 1;
297	}
298	else
299	{
300	mtr_debug("IM-angel (PID: $im->{angel_pid}) is dead.");
301	return 0;
302	}
303	}
304	else
305	{
306	mtr_debug("No PID file for IM-angel.");
307	return 0;
308	}
309	}
310
311	###########################################################################
312
313	sub mtr_im_check_mysqlds_alive($) {
314	my $im= shift;
315
316	mtr_debug("Checking for IM-guarded mysqld instances...");
317
318	my $instances = $im->{'instances'};
319
320	for ( my $idx= 0; $idx < 2; ++$idx )
321	{
322	mtr_debug("Checking mysqld[$idx]...");
323
324	return 1
325	if mtr_im_check_mysqld_alive($instances->[$idx]);
326	}
327	}
328
329	###########################################################################
330
331	sub mtr_im_check_mysqld_alive($) {
332	my $mysqld_instance= shift;
333
334	# Check that the process is dead.
335
336	if ( defined $mysqld_instance->{'pid'} )
337	{
338	if ( kill (0, $mysqld_instance->{'pid'}) )
339	{
340	mtr_debug("Mysqld instance (PID: $mysqld_instance->{pid}) is alive.");
341	return 1;
342	}
343	else
344	{
345	mtr_debug("Mysqld instance (PID: $mysqld_instance->{pid}) is dead.");
346	}
347	}
348	else
349	{
350	mtr_debug("No PID file for mysqld instance.");
351	}
352
353	# Check that mysqld does not accept client connections.
354
355	if ( mtr_ping_port($mysqld_instance->{'port'}) )
356	{
357	mtr_debug("Mysqld instance (port: $mysqld_instance->{port}) " .
358	"is accepting connections.");
359
360	mtr_im_errlog("Mysqld is accepting connections on port " .
361	"$mysqld_instance->{port}, but there is no " .
362	"process information.");
363	return 1;
364	}
365	else
366	{
367	mtr_debug("Mysqld instance (port: $mysqld_instance->{port}) " .
368	"does not accept connections.");
369	return 0;
370	}
371	}
372
373	###########################################################################
374
375	sub mtr_im_cleanup($) {
376	my $im= shift;
377
378	mtr_im_rm_file($im->{'path_pid'});
379	mtr_im_rm_file($im->{'path_sock'});
380
381	mtr_im_rm_file($im->{'path_angel_pid'});
382
383	for ( my $idx= 0; $idx < 2; ++$idx )
384	{
385	mtr_im_rm_file($im->{'instances'}->[$idx]->{'path_pid'});
386	mtr_im_rm_file($im->{'instances'}->[$idx]->{'path_sock'});
387	}
388	}
389
390	###########################################################################
391
392	sub mtr_im_rm_file($)
393	{
394	my $file_path= shift;
395
396	if ( -f $file_path )
397	{
398	mtr_debug("Removing '$file_path'...");
399
400	unless ( unlink($file_path) )
401	{
402	mtr_warning("Can not remove '$file_path'.")
403	}
404	}
405	else
406	{
407	mtr_debug("File '$file_path' does not exist already.");
408	}
409	}
410
411	###########################################################################
412
413	sub mtr_im_errlog($) {
414	my $msg= shift;
415
416	# Complain in error log so that a warning will be shown.
417	#
418	# TODO: unless BUG#20761 is fixed, we will print the warning to stdout, so
419	# that it can be seen on console and does not produce pushbuild error.
420
421	# my $errlog= "$opt_vardir/log/mysql-test-run.pl.err";
422	#
423	# open (ERRLOG, ">>$errlog") \|\|
424	# mtr_error("Can not open error log ($errlog)");
425	#
426	# my $ts= localtime();
427	# print ERRLOG
428	# "Warning: [$ts] $msg\n";
429	#
430	# close ERRLOG;
431
432	my $ts= localtime();
433	print "Warning: [$ts] $msg\n";
434	}
435
436	###########################################################################
437
438	sub mtr_im_kill($) {
439	my $im= shift;
440
441	# Re-load PIDs. That can be useful because some processes could have been
442	# restarted.
443
444	mtr_im_load_pids($im);
445
446	# Ignoring SIGCHLD so that all children could rest in peace.
447
448	start_reap_all();
449
450	# Kill IM-angel first of all.
451
452	if ( defined $im->{'angel_pid'} )
453	{
454	mtr_debug("Killing IM-angel (PID: $im->{angel_pid})...");
455	mtr_im_kill_process([ $im->{'angel_pid'} ], 'KILL', 10, 1)
456	}
457	else
458	{
459	mtr_debug("IM-angel is dead.");
460	}
461
462	# Re-load PIDs again.
463
464	mtr_im_load_pids($im);
465
466	# Kill IM-main.
467
468	if ( defined $im->{'pid'} )
469	{
470	mtr_debug("Killing IM-main (PID: $im->pid})...");
471	mtr_im_kill_process([ $im->{'pid'} ], 'KILL', 10, 1);
472	}
473	else
474	{
475	mtr_debug("IM-main is dead.");
476	}
477
478	# Re-load PIDs again.
479
480	mtr_im_load_pids($im);
481
482	# Kill guarded mysqld instances.
483
484	my @mysqld_pids;
485
486	mtr_debug("Collecting PIDs of mysqld instances to kill...");
487
488	for ( my $idx= 0; $idx < 2; ++$idx )
489	{
490	my $pid= $im->{'instances'}->[$idx]->{'pid'};
491
492	unless ( defined $pid )
493	{
494	next;
495	}
496
497	mtr_debug(" - IM-guarded mysqld[$idx] PID: $pid.");
498
499	push (@mysqld_pids, $pid);
500	}
501
502	if ( scalar @mysqld_pids > 0 )
503	{
504	mtr_debug("Killing IM-guarded mysqld instances...");
505	mtr_im_kill_process(\@mysqld_pids, 'KILL', 10, 1);
506	}
507
508	# That's all.
509
510	stop_reap_all();
511	}
512
513	##############################################################################
514
515	sub mtr_im_wait_for_connection($$$) {
516	my $im= shift;
517	my $total_attempts= shift;
518	my $connect_timeout= shift;
519
520	mtr_debug("Waiting for IM on port $im->{port} " .
521	"to start accepting connections...");
522
523	for ( my $cur_attempt= 1; $cur_attempt <= $total_attempts; ++$cur_attempt )
524	{
525	mtr_debug("Trying to connect to IM ($cur_attempt of $total_attempts)...");
526
527	if ( mtr_ping_port($im->{'port'}) )
528	{
529	mtr_debug("IM is accepting connections " .
530	"on port $im->{port}.");
531	return 1;
532	}
533
534	mtr_debug("Sleeping $connect_timeout...");
535	sleep($connect_timeout);
536	}
537
538	mtr_debug("IM does not accept connections " .
539	"on port $im->{port} after " .
540	($total_attempts * $connect_timeout) . " seconds.");
541
542	return 0;
543	}
544
545	##############################################################################
546
547	sub mtr_im_wait_for_mysqld($$$) {
548	my $mysqld= shift;
549	my $total_attempts= shift;
550	my $connect_timeout= shift;
551
552	mtr_debug("Waiting for IM-guarded mysqld on port $mysqld->{port} " .
553	"to start accepting connections...");
554
555	for ( my $cur_attempt= 1; $cur_attempt <= $total_attempts; ++$cur_attempt )
556	{
557	mtr_debug("Trying to connect to mysqld " .
558	"($cur_attempt of $total_attempts)...");
559
560	if ( mtr_ping_port($mysqld->{'port'}) )
561	{
562	mtr_debug("Mysqld is accepting connections " .
563	"on port $mysqld->{port}.");
564	return 1;
565	}
566
567	mtr_debug("Sleeping $connect_timeout...");
568	sleep($connect_timeout);
569	}
570
571	mtr_debug("Mysqld does not accept connections " .
572	"on port $mysqld->{port} after " .
573	($total_attempts * $connect_timeout) . " seconds.");
574
575	return 0;
576	}
577
578	##############################################################################
579	#
580	# Public operations.
581	#
582	##############################################################################
583
584	sub mtr_im_start($$) {
585	my $im = shift;
586	my $opts = shift;
587
588	mtr_debug("Starting Instance Manager...");
589
590	my $args;
591	mtr_init_args(\$args);
592	mtr_add_arg($args, "--defaults-file=%s", $im->{'defaults_file'});
593
594	foreach my $opt ( @{$opts} )
595	{
596	mtr_add_arg($args, $opt);
597	}
598
599	$im->{'spawner_pid'} =
600	mtr_spawn(
601	$::exe_im, # path to the executable
602	$args, # cmd-line args
603	'', # stdin
604	$im->{'path_log'}, # stdout
605	$im->{'path_err'}, # stderr
606	'', # pid file path (not used)
607	{ append_log_file => 1 } # append log files
608	);
609
610	unless ( $im->{'spawner_pid'} )
611	{
612	mtr_error('Could not start Instance Manager.')
613	}
614
615	# Instance Manager can be run in daemon mode. In this case, it creates
616	# several processes and the parent process, created by mtr_spawn(), exits just
617	# after start. So, we have to obtain Instance Manager PID from the PID file.
618
619	mtr_debug("Waiting for IM to create PID file (" .
620	"path: '$im->{path_pid}'; " .
621	"timeout: $im->{start_timeout})...");
622
623	unless ( sleep_until_file_created($im->{'path_pid'},
624	$im->{'start_timeout'},
625	-1) ) # real PID is still unknown
626	{
627	mtr_debug("IM has not created PID file in $im->{start_timeout} secs.");
628	mtr_debug("Aborting test suite...");
629
630	mtr_kill_leftovers();
631
632	mtr_report("IM has not created PID file in $im->{start_timeout} secs.");
633	return 0;
634	}
635
636	$im->{'pid'}= mtr_get_pid_from_file($im->{'path_pid'});
637
638	mtr_debug("Instance Manager started. PID: $im->{pid}.");
639
640	# Wait until we can connect to IM.
641
642	my $IM_CONNECT_TIMEOUT= 30;
643
644	unless ( mtr_im_wait_for_connection($im,
645	$IM_CONNECT_TIMEOUT, 1) )
646	{
647	mtr_debug("Can not connect to Instance Manager " .
648	"in $IM_CONNECT_TIMEOUT seconds after start.");
649	mtr_debug("Aborting test suite...");
650
651	mtr_kill_leftovers();
652
653	mtr_report("Can not connect to Instance Manager " .
654	"in $IM_CONNECT_TIMEOUT seconds after start.");
655	return 0;
656	}
657
658	# Wait for IM to start guarded instances:
659	# - wait for PID files;
660
661	mtr_debug("Waiting for guarded mysqlds instances to create PID files...");
662
663	for ( my $idx= 0; $idx < 2; ++$idx )
664	{
665	my $mysqld= $im->{'instances'}->[$idx];
666
667	if ( exists $mysqld->{'nonguarded'} )
668	{
669	next;
670	}
671
672	mtr_debug("Waiting for mysqld[$idx] to create PID file (" .
673	"path: '$mysqld->{path_pid}'; " .
674	"timeout: $mysqld->{start_timeout})...");
675
676	unless ( sleep_until_file_created($mysqld->{'path_pid'},
677	$mysqld->{'start_timeout'},
678	-1) ) # real PID is still unknown
679	{
680	mtr_debug("mysqld[$idx] has not created PID file in " .
681	"$mysqld->{start_timeout} secs.");
682	mtr_debug("Aborting test suite...");
683
684	mtr_kill_leftovers();
685
686	mtr_report("mysqld[$idx] has not created PID file in " .
687	"$mysqld->{start_timeout} secs.");
688	return 0;
689	}
690
691	mtr_debug("PID file for mysqld[$idx] ($mysqld->{path_pid} created.");
692	}
693
694	# Wait until we can connect to guarded mysqld-instances
695	# (in other words -- wait for IM to start guarded instances).
696
697	mtr_debug("Waiting for guarded mysqlds to start accepting connections...");
698
699	for ( my $idx= 0; $idx < 2; ++$idx )
700	{
701	my $mysqld= $im->{'instances'}->[$idx];
702
703	if ( exists $mysqld->{'nonguarded'} )
704	{
705	next;
706	}
707
708	mtr_debug("Waiting for mysqld[$idx] to accept connection...");
709
710	unless ( mtr_im_wait_for_mysqld($mysqld, 30, 1) )
711	{
712	mtr_debug("Can not connect to mysqld[$idx] " .
713	"in $IM_CONNECT_TIMEOUT seconds after start.");
714	mtr_debug("Aborting test suite...");
715
716	mtr_kill_leftovers();
717
718	mtr_report("Can not connect to mysqld[$idx] " .
719	"in $IM_CONNECT_TIMEOUT seconds after start.");
720	return 0;
721	}
722
723	mtr_debug("mysqld[$idx] started.");
724	}
725
726	mtr_debug("Instance Manager and its components are up and running.");
727
728	return 1;
729	}
730
731	##############################################################################
732
733	sub mtr_im_stop($) {
734	my $im= shift;
735
736	mtr_debug("Stopping Instance Manager...");
737
738	# Try graceful shutdown.
739
740	mtr_im_terminate($im);
741
742	# Check that all processes died.
743
744	unless ( mtr_im_check_alive($im) )
745	{
746	mtr_debug("Instance Manager has been stopped successfully.");
747	mtr_im_cleanup($im);
748	return 1;
749	}
750
751	# Instance Manager don't want to die. We should kill it.
752
753	mtr_im_errlog("Instance Manager did not shutdown gracefully.");
754
755	mtr_im_kill($im);
756
757	# Check again that all IM-related processes have been killed.
758
759	my $im_is_alive= mtr_im_check_alive($im);
760
761	mtr_im_cleanup($im);
762
763	if ( $im_is_alive )
764	{
765	mtr_debug("Can not kill Instance Manager or its children.");
766	return 0;
767	}
768
769	mtr_debug("Instance Manager has been killed successfully.");
770	return 1;
771	}
772
773	###########################################################################
774
775	1;