Subject: Ontape and Shell Script
From: border@col.hp.com (Bill Border)
Newsgroups: comp.databases.informix
Date: 6 Mar 1997 20:59:37 GMT

: If the data volume to be backed up is larger than the tape volume, you =
: will have problem as ontape will actually rewind the tape and write from =
: the beginning again!

  We have a perl script here which handle multiple tapes, errors, or
 whatever. I'll post a copy here on an un-supported basis. It works
 with Informix ODS 6 and 7 on HP-UX 9 and 10. 

  There are two components, a ksh and perl script.




 Piece 1 - ksh:





#!/usr/bin/ksh
###############################################################################
#
# db_ontape
#
###############################################################################

#
# Set traps to return fail code on interrupt
#

trap "exit 1" KILL INT HUP

function UsageMessage
{
  date
  print "Usage: '$P' -e <INSTANCE | nosap> -t <TAPE FILE NAME> -l <ONTAPE ARCHIVE LEVEL (0,1,2)> [-p <ENVIRONMENT SET SCRIPT>]"
}

error_proc ()
{
    MSG=$1

    echo " "
    echo "db_ontape | Fatal Error Encountered: $MSG"

    echo "db_ontape | Sending EMail Error Notification to sapgap/dbagap"
    mailx -s "db_ontape Error (`hostname`) - $MSG" <an email address> << EOF
      Host=`hostname`
      Instance=$ENV_TYPE
      Msg=$MSG
EOF

    #echo "db_ontape | Sending OpC Message"
    #/opt/OV/bin/OpC/opcmsg object=DB_Backup severity=critical msg_text="$MSG" application="SAP R/3" msg_grp=General

    if [ "$PROD_SYS" = "Y" ]; then
      echo "db_check | Sending Pager Notification"
      page_dbagap db_ontape `hostname` << EOF
        db_ontape Critical Error
        Msg=$MSG
        Host=`hostname`
        Instance=$ENV_TYPE
EOF
    fi

    echo "db_ontape | Exiting - \c"
    date
    exit 1
}

#
# Initializations
#

P=$(basename $0)
PGMPATH=$0
PGMDIR=${0%/$P}
export PATH=$PGMDIR:/usr/local/bin:$PATH

export HOST=`hostname`

export LOCATION=`nslookup $(hostname) | grep Name: | awk '{ \
split($2,array,".")
print array[2]
}'`

ID=`/usr/bin/whoami`

echo "------------------------------------------------------------------------"
print "$P | Starting - \c"
date
print "$P | PGMPATH=$PGMPATH PGMDIR=$PGMDIR LOCATION=$LOCATION ID=$ID"
print "PATH=$PATH"

#
# Get command line options
#

while getopts :e:t:l:p: option
do case $option in
   e) ENV_TYPE=$OPTARG;;
   t) TAPEFILE=$OPTARG;;
   l) ARCLEVEL=$OPTARG;;
   p) ENVSCRIPT=$OPTARG;;
   :) print "$(basename $0): -$OPTARG requires a value"
      exit 2;;
  \?) print "$(basename $0) unknown option $OPTARG"
      UsageMessage
      exit 2;;
   esac
done

export ENV_TYPE

print "$P | ENV_TYPE=$ENV_TYPE TAPEFILE=$TAPEFILE ARCLEVEL=$ARCLEVEL ENVSCRIPT=$ENVSCRIPT"
 
#
# Edits
#

if [ -z "$TAPEFILE" ]; then
  UsageMessage
  exit 1
fi

if [ -z "$ARCLEVEL" ]; then
  UsageMessage
  exit 1
fi

if [ -z "$ENV_TYPE" ]; then
  UsageMessage
  exit 1
fi

#
# Main
# 

if [[ "$ID" != "root" && "$ID" != "informix" ]];then
  error_proc "db_ontape must be run be either root or informix"
fi

if [ $ENV_TYPE = nosap ]; then
  if [ ! -z $ENVSCRIPT ];then
    print "$P | Executing . $ENVSCRIPT"
    . $ENVSCRIPT
    if [ $? != 0 ];then
      error_proc "Error running user specified ENV script"
    fi
    print "$P | Environment after $ENVSCRIPT"
    env
  else
    print "$P | Executing . /etc/profile"
    . /etc/profile

    print "$P | Executing . /etc/profile.d/*"
    PROFILE_FRAGMENT_DIR=/etc/profile.d

    if [ -d $PROFILE_FRAGMENT_DIR ]
    then
	for FRAGMENT in `/bin/ls $PROFILE_FRAGMENT_DIR`
	do
	    if  [   -x ${PROFILE_FRAGMENT_DIR}/${FRAGMENT}  \
		 -a -r ${PROFILE_FRAGMENT_DIR}/${FRAGMENT}  \
		 -a -f ${PROFILE_FRAGMENT_DIR}/${FRAGMENT}   ]
				# these files must be readable AND executable
	    then
		.  ${PROFILE_FRAGMENT_DIR}/${FRAGMENT}
		RETURN_VALUE=$?
		if [ $RETURN_VALUE -ne 0 ]
		then
	     echo "# ${PROFILE_FRAGMENT_DIR}/${FRAGMENT} exited abnormally"
	     echo "#       with value $RETURN_VALUE"
	     echo "###########################################################"
		fi
	    fi
	done
    fi

    print "$P | Environment after /etc/profile"
    env
    export ERROR_EMAIL="<an email address>"
  fi
else
  . dbeset $ENV_TYPE
  if [ $? != 0 ];then
    error_proc "dbeset Error"
  fi
  export ERROR_EMAIL="<an email address>"
fi

print "$P | INFORMIXDIR=$INFORMIXDIR"

if [ -z "$ENV_TYPE" ]; then
  error_proc "INFORMIXDIR is null - env var error"
fi

#
# Check for Informix active
#
onstat - | grep "On-Line"
if [ $? != 0 ];then
  error_proc "Informix not On-Line"
fi

#
# Check for TAPEDEV set to /dev/null 
#
onstat -c | grep ^TAPEDEV | grep /dev/null
if [ $? = 0 ];then
  error_proc "TAPEDEV set to /dev/null"
fi

#
# Check for other db_ontapes executing
#

PCNT=`ps -efuinformix | grep "db_ontape.pl" | grep -v grep | wc -l `
echo "db_ontape | Total number of db_ontape running: $PCNT"
if [ $PCNT -gt 0 ];then
  ps -efuinformix | grep "db_ontape.pl" | grep -v grep 
  error_proc "db_ontape | Another db_ontape.pl Already running"
  exit 1
fi

#
# Start perl pgm
#

export PATH=$PGMDIR:$PATH
print "PATH=$PATH"

#####################################################################
# Start - Code to stop SAP before archive
#

grep $INFORMIXSERVER /usr/local/sap/tools/db_bounce_sap.cfg
if [ $? = 0 ];then
  echo "db_ontape | Issuing r3_auto_shutdown to stop SAP"

  r3_auto_shutdown $ENV_TYPE r3

  if [ $? != 0 ];then
    error_proc "r3_auto_shutdown non-zero RC"
  fi
  echo "db_ontape | r3_auto_shutdown successful completion"
fi

#
# End - Code to stop SAP before archive
#####################################################################

if [ "$ID" = "informix" ];then
  db_ontape.pl $ARCLEVEL $TAPEFILE
else
  su informix -c "db_ontape.pl $ARCLEVEL $TAPEFILE"
fi

LASTRC=$?
print "$P | db_ontape.pl RC=$LASTRC"

if [ $LASTRC != 0 ];then
  if [ $LASTRC = 1 ];then
    print "$P | $P.awk failed - exiting"
    exit 1
  fi
  if [ $LASTRC = 202 ];then
    print "$P | $P.awk failed - Rerunning - Executing db_ontape_rerun_auto $ENV_TYPE $TAPEFILE $ARCLEVEL after sleeping 60"
    sleep 60
    db_ontape_rerun_auto $ENV_TYPE $TAPEFILE $ARCLEVEL
    print "$P | Exiting"
    exit 1
  fi
fi

#####################################################################
# Start - Code to start SAP after archive
#

grep $INFORMIXSERVER /usr/local/sap/tools/db_bounce_sap.cfg
if [ $? = 0 ];then
  echo "$P | Executing r3_auto_startup"
  r3_auto_startup $ENV_TYPE

  if [ $? != 1 ];then
    error_proc "r3_auto_startup was not able to start SAP"
  fi
  echo "$P | r3_auto_startup successful completion"
fi

#
# End - Code to start SAP after archive
#####################################################################

#
# Terminate
#

print "$P | Done - \c"
date
echo "------------------------------------------------------------------------"
exit
  




 Piece 2 - perl:



#!/usr/local/bin/perl5
# =========================================================
# db_ontape.pl
#
# Changes:
# 
#   bb 23oct96 -
#     Added code to extract the error string and errno from
#     the ontape output. This data is then sent in the email
#     failure message and logged to 
#     /csitc/dba/misc/db_backup_failure.txt .
#   bb 26oct96 -
#     Added code to create tapes-used report.
#     Changed pager code to use page_dbagap script.
#   bb 04Nov96 -
#     Added code to create success report in
#     /csitc/dba/misc/db_backup_success.txt .
#   bb 15Nov96 -
#     Added code to exit with rc=202 if errno from backup 
#     failure is 5. db_ontape will then run db_ontape_restart_auto
#     to automatically restart the backup.
#   bb 26Nov96 -
#     Added code to restart for other conditions (see analyze 
#     function).
#   bb 12Dec96 -
#     Added code to create sms ticket for failures.
#   bb 26Dec96 -
#     Added code to auto-restart rc's 2 and 25. Also added code to
#     auto-retry db_ontape_tapesreq once for tape checkout failures.
#   bb 03Jan97 -
#     Modified code to better handle 2+ tapes requests that fail  
#     (undef $tape_vol).
# =========================================================

$P = "db_ontape.pl";

# =========================================================
# Arguments
# =========================================================

$backup_level = @ARGV[0];
$tape_file    = @ARGV[1];

# 
# Check for an INSTANCEadm .forward file
#
$forwfile = "/home/" . $ENV{"INFORMIX_DBID"} . "adm/.forward";
$admemail = $ENV{"INFORMIX_DBID"} . "adm";
if (-e $forwfile)
{
  $error_email = "$ENV{\"ERROR_EMAIL\"} $admemail\@$ENV{\"HOST\"}.zzzzzzzzz";
}
else
{
  $error_email = $ENV{"ERROR_EMAIL"};
}

print "$P | Starting - backup_level=$backup_level tape_file=$tape_file error_email=$error_email\n";

# =========================================================
# Global vars
# =========================================================

$debug = 0;

$| = 1;

$backup_cmd   = "ontape";

$errstr = "?";
$errno  = 9999;

$mount_count = 0;
$sleep_time = 60;

$pid = $$;

$tmp_file   = "/tmp/db_ontape_w_$pid.out";
$work_file  = "/tmp/db_ontape_w_$pid.tmp";
$pipe_file  = "/tmp/db_ontape_w_$pid.pipe";

if ($tape_file eq "test") { $mount_hst = "xxxxxxxxxxxxxxx"; }
else                      { $mount_hst = "xxxxxxxxxxxxxxx"; }

# =========================================================
# Subs
# =========================================================

#
# sms_ticket
#

sub sms_ticket {

  if (! -e "/usr/local/sap/tools/db_ontape_autosms")
  {
    print "$P | SMS ticket trigger file /usr/local/sap/tools/db_ontape_autosms Not Found - returning\n"
;
    return;
  }

  $teamname = "DATABASE ADMINISTRATION (DBA) - CTSC";

  #open(MAIL, "| mailx -s '$teamname' border\@zzzzzzzzzzz ") ||
  open(MAIL, "| mailx -s '$teamname' smsemail\@zzzzzzzzzz ") ||
  die "Can't do mailx!\n";

print MAIL <<END;
FORM_NAME:db_ontape.pl
CUSTOMER_NAME:DBA Team
CUSTOMER_PHONE:590-3402
CUSTOMER_EMAIL:dbagap\@zzzzzzzzzzzzzzz
CATEGORY:DATABASE/PROBLEM (INFORMIX BACK-UPS)
TEAMNAME:Database Administration (DBA) - CTSC
PRIORITY:4
PROBLEM_SUMMARY:db_ontape - $ENV{"HOST"} - $ENV{"INFORMIXSERVER"} - Error=$errno ($msg)
DESCRIPTION:
-------------------------------------
Host:           $ENV{"HOST"}
INFORMIXSERVER: $ENV{"INFORMIXSERVER"}
Backup Level:   $backup_level
Errno:          $errno
Errstr:         $errstr
Msg:            $msg
-------------------------------------
To enable/disable the automatic creation of SMS tickets,
touch/remove file /usr/local/sap/tools/db_ontape_autosms
END

  close(MAIL);
  print "$P | Created SMS ticket\n"
}


#
# dismount_tape
#

sub dismount_tape
{
  print "\n\n";

  if ($mount_count > 0)
  {
    $rc = system("db_ontape_dismount -o $mount_hst:0 -d $tape_dev");
    if ($rc != 0)
    {
      $errstr = "Dismount";
      $errno  = 90;
      &error("Error during tape dismount - exiting");
    }
  }
}

#
# request_tape
#

sub request_tape
{

  print "$P | Requesting new tape from tape mgmt system ...\n";

  undef $tape_vol;

  if ($tape_file eq "test")
  {
    print "$P | (test mode) Enter tape volume ...\n";
    $tape_vol = <STDIN>;
    push(@tape_list,$tape_vol);
    print "$P | Using tape volume $tape_vol\n";
    return;
  }

  # 
  # Execute this loop a maximum of 2 times
  #

  for ($attempt_num = 1;$attempt_num <= 2; $attempt_num++)
  {
    print "$P | Attempt $attempt_num - Invoking db_ontape_tapesreq for tape file $tape_file\n";

    #
    # Remove old work file (just in case this tapesreq fails)
    #

    system("rm $work_file > /dev/null 2>&1");

    $rc = system("db_ontape_tapesreq $tape_file > $work_file 2>&1");
    print "$P | db_ontape_tapesreq rc=$rc\n";
    if ($rc != 0)
    {
      if ($attempt_num == 1)
      {
        print "$P | db_ontape_tapesreq rc=$rc - will sleep 30 min. before retrying once\n";
        sleep 1800;
        next;
      }
      system("cat $work_file");
      $errstr = "TapesreqError";
      $errno  = 91;
      &error("Error rc=$rc during db_ontape_tapesreq - exiting");
    }

    open(TAPESREQ, "$work_file") || die "can't open $work_file - $!";
    while (<TAPESREQ>)
    {
      if (/Use output tape/) 
      { 
	@words = split;
	$tape_vol = @words[4];
      } 
    }
    close(TAPESREQ);

    if (! defined($tape_vol))
    {
      if ($attempt_num == 1)
      {
        print "$P | didn't find tape volume - will sleep 30 min. before retrying once\n";
        sleep 1800;
        next;
      }
      system("cat $work_file");
      $errstr = "Tapesreq No Vol";
      $errno  = 93;
      &error("Tape volume not found after db_ontape_tapesreq");
    }
    #
    # If we do have a good tape vol, leave the "attempt" for loop
    # 
    else
    {
      last;
    }
  }

  push(@tape_list,$tape_vol);
  print "$P | Using tape volume $tape_vol\n";

}


#
# mount_tape
#

sub mount_tape
{
  print "\n\n";

  print "$P | Requesting mount of new tape ...\n";

  $rc = system("db_ontape_mount -o $mount_hst:0 -d $tape_dev -v $tape_vol");
  if ($rc != 0)
  {
    $errstr = "TapeMount";
    $errno  = 94;
    &error("Error during tape mount - exiting");
  }
    
  $mount_count++;

}

#
# analyze
#

sub analyze
{
  print "$P | Analyzing Error\n";

  $flag1 = 0;
  $flag2 = 0;

  open(ERROUT, "$tmp_file") || die "can't open ERROUT - $!";
  while (<ERROUT>)
  {
    if (/failed/)
    {
      @words = split;
     
      foreach $word (@words)
      {
        if ($flag2 == 1)
        {
          $errno = $word;
        }

        if ($word eq "errno")
        {
          $flag2 = 1;
        }

        if ($word eq "failed")
        {
          $flag1 = 1;
          $errstr = "Archive failed";
          next;
        }

        if ($flag1 == 1)
        {
          $errstr = $errstr . " " . $word;
        }

      }
    }
  }
  print "$P | errstr=$errstr\n";
  print "$P | errno =$errno\n";

  # 
  # Further analyze error
  # 

  if ($errstr =~ /backup terminated prem/) { $errno = "80" }
  if ($errstr =~ /access sysmaster datab/) { $errno = "81" }

  # 
  # Determine whether to auto-restart
  # 

  if (    $errno eq "2" 
       || $errno eq "5" 
       || $errno eq "9" 
       || $errno eq "25" 
       || $errno eq "80" 
       || $errno eq "81" 
       || $errno eq "82" 
      ) 
  {
    $restart_flag = 1;
  }
  else
  {
    $restart_flag = 0;
  }

  # 
  # If errno=5, send opc message to operator to nuke bad tape
  # 

  if ($errno eq "5")
  {

    print "$P | Sending OpC bad tape message\n";
    $rc = system("/opt/OV/bin/OpC/opcmsg severity=critical application=Informix object=DAT msg_text='Mark Tape $tape_vol as BAD and process accordingly' msg_grp=Backup");
    print "$P | Sent OpC message - rc=$rc\n";

    print "$P | Sending OpC clean drive message\n";
    $rc = system("/opt/OV/bin/OpC/opcmsg severity=critical application=Informix object=DAT msg_text='Please clean tapes drive(s) on system $ENV{\"HOST\"}' msg_grp=Backup");
    print "$P | Sent OpC message - rc=$rc\n";

    $rc = system("echo 'This message generated by db_ontape.pl' | mailx -s \"Possible Bad Tape $tape_vol - Instance: $ENV{\"HOST\"}/$ENV{\"INFORMIXSERVER\"} \" murphya\@cs.itc deanf\@cs.itc dbagap\@cs.itc");

  }

}


#
# error
#

sub error
{
  &analyze();

  $msg = @_[0];

  print "\n";
  print "$P | Fatal Error: $msg\n";
  print "$P | ontape output:\n";
    print "-cat start--------------------------------------------------------------\n";
  system("cat $tmp_file | grep -v percent");
    print "\n-cat end----------------------------------------------------------------\n";
  print "\n\n";

  $date = `date`;

  if ($restart_flag == 1)
  {
    $errhdr = "(Auto-Restartable)";
  }
  else
  {
    $errhdr = "";
  }

  $rc = system("echo 'Informix Instance: $ENV{\"INFORMIXSERVER\"}\nHost: $ENV{\"HOST\"}\nLevel: $backup_level\nError: $errstr\nErrno: $errno\nDate: $date\nSee file: $tmp_file for ontape output.' | mailx -s \"$errhdr db_ontape Error - Instance: $ENV{\"HOST\"}/$ENV{\"INFORMIXSERVER\"} - Error Message: $msg\" $error_email");
  print "$P | Mailed message to $error_email - rc=$rc\n";

  #
  # Create SMS ticket (if /usr/local/sap/tools/db_ontape_autosms exists)
  #
  &sms_ticket();

  #
  # Send OpC message
  #

  #$rc = system("/opt/OV/bin/OpC/opcmsg severity=critical application=Informix object=db_ontape msg_text='db_ontape - Error - $msg' msg_grp=Job");
  #print "$P | Sent OpC message - rc=$rc\n";

  #
  # Log a message indicating failure
  #

  if (-d "/csitc/dba/misc")
  {
    $fdate = `date +%y/%m/%d-%H:%M`;
    chop $fdate;
    $logstr = "$fdate db_ontape $ENV{\"HOST\"} $ENV{\"INFORMIXSERVER\"} $backup_level $errno [$errstr]";
    open(LOGOUT, ">>/csitc/dba/misc/db_backup_failure.txt") || die "can't open LOGOUT - $!";
    print LOGOUT "$logstr\n";
  }
  else
  {
    print "$P | No status messages logged - /csitc/dba/misc does not exist\n";
  }

  #
  # If PROD system, send page
  #

  if ($ENV{"PROD_SYS"} eq "Y")
  { 
    print "$P | Sending pager msg to page_dbagap script\n";

    $rc = system("echo 'db_ontape Error: $msg\nInformix Instance: $ENV{\"INFORMIXSERVER\"}\nHost: $ENV{\"HOST\"}' | page_dbagap");

    print "$P | After page_dbagap - rc=$rc\n";
  }

  &cleanup();

  # 
  # List tapes used
  # 

  print "$P | Tape volumes used by this backup:\n";
  foreach $this_tape (@tape_list)
  {
    print "$this_tape\n";
  }
  print "\n";

  # 
  # Process restart_flag
  # 

  if ($restart_flag == 1)
  {
    $exitrc = 202;
  }
  else
  {
    $exitrc = 1;
  }

  print "$P | Exiting rc=$exitrc\n";
  exit $exitrc;
}


#
# cleanup
#

sub cleanup
{
  open(PS, "ps -ef | grep $backup_cmd |") || die "can't open np - $!";
  while (<PS>)
  {
    if (/ontape -s/) 
    { 
      @words = split;
      $pid = @words[1];
      print "$P | Killing $backup_cmd PID=$pid\n";
      system("kill " . $pid);
      last;
    }
  }
}

sub get_tape_dev
{
  system("onstat -c | grep '^TAPEDEV'");

  open(ONSTAT, "onstat -c | grep '^TAPEDEV' |") || die "can't onstat -c - $!";
  while (<ONSTAT>)
  {
    print;
    if (/^TAPEDEV/) 
    { 
      @words = split;
      $tape_dev = @words[1];
      print "$P | Tape device is $tape_dev\n";

      if ($tape_dev =~ /:/)
      {
        @words = split(/:/,$tape_dev);
        $tape_hst = @words[0];
        $tape_dev = @words[1];
        print "$P | Remote tape device $tape_hst $tape_dev\n";
        $ENV{"REMOTE_TAPE_HOST"} = "$tape_hst";
        $ENV{"REMOTE_TAPE_CMD"} = "remsh $tape_hst";
      }
      return;
    }
  }
  $errstr = "NoTapedev";
  $errno  = 92;
  &error("Error: onstat -c did not find TAPEDEV");
}


# =========================================================
# Main 
# =========================================================

#
# Delete old files
#
print "$P | Deleting old pipes\n";
system("rm $pipe_file $tmp_file");

#
# Create ontape input pipe
#
print "$P | Creating new $pipe_file\n";
if (system("mkfifo $pipe_file") != 0)
{
  print "Error: Can't create $pipe_file - exiting\n";
  exit 1;
}

#
# Create ont tape ouptut file 
#
print "$P | Creating new $tmp_file\n";
if (system("touch $tmp_file") != 0)
{
  print "Error: Can't create $tmp_file - exiting\n";
  exit 1;
}

#
# Determine the name of the tape device from onstat -c
#

&get_tape_dev();

#
# Start backup process in background
#
print "$P | Starting background $backup_cmd\n";
if (system("$backup_cmd -s -L $backup_level < $pipe_file > $tmp_file &") != 0)
{
  system("echo $?");
  print "Error: Can't fire up $backup_cmd - exiting\n";
  exit 1;
}

#
# Open pipe
#
print "$P | Opening np_in named pipe\n";
open(NP_IN, ">$pipe_file") || die "can't open np - $!";


#
# Read from command output pipe
#

$stop = 0;
print "$P | Entering main processing loop - sleep time $sleep_time seconds\n";

READ:
while($stop==0)
{
  if ($debug)
  {
    print "-cat start-----------------------------------------------------------\n";
    system("cat $tmp_file | grep -v percent");
    print "\n";
    print "-cat end-------------------------------------------------------------\n";
  }

  #
  # read contents of out file to save the last record
  #

  #open(NP_OUT, "$tmp_file") || die "can't open np - $!";
  open(NP_OUT, "grep -v percent $tmp_file |") || die "can't $tmp_file np - $!";
  
  #
  # Just in case read fails
  #
  $rec = "crudcrudcrudcrudcrud";

  #while (read(NP_OUT,$_,100))
  while (<NP_OUT>)
  {
    if (
        /Interrupt/i 
     || /tape is still mounted/i 
     || /cannot open/i 
     || /failed/i 
     || /Terminated/i
     || /Abort/i 
     || /ERROR/i
     || /incompatible/i
    ) { &error("ontape failure"); }
    
    $len = length($_);
    if ($debug) { print "$P | rec length=$len\n"; }

    $rec = $_;
  }
  close(NP_OUT);

  # 
  # get 1st 20 bytes of last output line
  #

  $this_msghdr = substr($rec,0,20);

  # 
  # if last message is the same as previous last message, or 
  #   read failed cuz the record was too long, or length of
  #   last line > 100, reloop
  #

  if ($debug) { print "$P | this=$this_msghdr last=$last_msghdr\n"; }

  # 
  # Check to see if this message has been previously seen
  #   

  $found_it = 0;
  foreach $this_hist (@hist_msghdr)
  { 
    if ($this_hist eq $this_msghdr)
    { 
      $found_it = 1; 
      if ($debug) { print "$P | Found duplicate msg - ignoring\n"; }
    }
  }

  if ($debug)
  {
    $tlen = 0;
    foreach $this (@hist_msghdr) { $tlen += length($this) }
    print "$P | hist length=$tlen\n"; 
  }

  if (
   $this_msghdr eq $last_msghdr 
   || $rec eq "crudcrudcrudcrudcrud" 
   || $len > 100
   || $found_it == 1
  ) 
  {
    if ($debug) { print "$P | Sleeping $sleep_time\n"; }
    system("sleep $sleep_time");
    print ".";
    $sleep_count++;
    next READ;
  }

  push(@hist_msghdr,$this_msghdr);

  $sleep_tot = ($sleep_count * $sleep_time) / 60;
  $grand_sleep_tot += $sleep_tot;
  print "\n\n$P | Waking up from sleeping $sleep_tot minutes\n";
  print "\n\n$P | Encountered next message: ", $this_msghdr, "\n";

  $sleep_count = 0;

  # 
  # Mount the next tape
  #

  if (substr($rec,0,17) eq "Please mount tape")
  {
    &request_tape();
    &mount_tape();

    print "$P | Responding NEWLINE to $backup_cmd\n";
    syswrite(NP_IN,"\n",1);
  }

  # 
  # Normal completion
  #

  if (substr($rec,0,13) eq "Program over.")
  {
    # 
    # Check for errors 
    #
    open(NP_OUT, "$tmp_file") || die "can't $tmp_file np - $!";
    while (<NP_OUT>)
    {
      if (
          /Interrupt/i
       || /tape is still mounted/i
       || /cannot open/i 
       || /failed/i 
       || /Terminated/i
       || /Abort/i 
       || /ERROR/i
       || /incompatible/i
      ) { &error("ontape failure"); }
    }
    close(NP_OUT);

    print "$P | Backup complete - Total sleep time: $grand_sleep_tot\n";
    $stop = 1;
    next READ;
  }

  $last_msghdr = $this_msghdr;

}

print "$P | Closing named pipe\n";
close(NP);

# =========================================================
# Display ontape output to the log file
# =========================================================

print "$P | ontape output:\n";
print "-cat start--------------------------------------------------------------\n";
system("cat $tmp_file | grep -v percent");
print "\n-cat end----------------------------------------------------------------\n";

# =========================================================
# Dismount tape
# =========================================================

&dismount_tape();

# =========================================================
# Remove work files
# =========================================================

system("rm /tmp/db_ontape_w_$pid.*");

# =========================================================
# Issue onmode -F to free unused shared memory
# =========================================================

print "$P | Issuing onmode -F to free shared memory\n";
system("onstat -g seg");
system("onmode -F");
system("onstat -g seg");

# =========================================================
# List tapes used
# =========================================================

$tapelist = "";

print "$P | Tape volumes used by this backup:\n";
foreach $this_tape (@tape_list)
{
  print "$this_tape\n";
  $tapelist = $tapelist . $this_tape . " ";
}
print "\n";

#
# Log a message indicating success
#

$fdate = `date +%y/%m/%d-%H:%M`;
chop $fdate;
$logstr = "$fdate db_ontape $ENV{\"HOST\"} $ENV{\"INFORMIXSERVER\"} $backup_level 0 [ OK $tapelist]";
open(LOGOUT, ">>/csitc/dba/misc/db_backup_success.txt") || die "can't open LOGOUT - $!";
print LOGOUT "$logstr\n";

# =========================================================
# Terminate
# =========================================================

print "$P | Exiting\n";
exit 0;

 . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
          
                /***                  Hewlett-Packard - CTS/Colorado Springs
               /  ***\                Address: 1900 Garden of the gods Road 
             /        \      /                 Colorado Springs, CO 80907
      /*    /           \   /**\      Mailstop: C103G
     /***\ /              \/    \   
    /                      \      \   Email: border@cs.itc.hp.com  
   / Bill Border            \       \ Phone: (Telnet/719) 590-3402
  /  CTSC Database Admin./Engineering Fax:   (Telnet/719) 590-3596