/*
*         OpenPBS (Portable Batch System) v2.3 Software License
*
* Copyright (c) 1999-2000 Veridian Information Solutions, Inc.
* All rights reserved.
*
* ---------------------------------------------------------------------------
* For a license to use or redistribute the OpenPBS software under conditions
* other than those described below, or to purchase support for this software,
* please contact Veridian Systems, PBS Products Department ("Licensor") at:
*
*    www.OpenPBS.org  +1 650 967-4675                  sales@OpenPBS.org
*                        877 902-4PBS (US toll-free)
* ---------------------------------------------------------------------------
*
* This license covers use of the OpenPBS v2.3 software (the "Software") at
* your site or location, and, for certain users, redistribution of the
* Software to other sites and locations.  Use and redistribution of
* OpenPBS v2.3 in source and binary forms, with or without modification,
* are permitted provided that all of the following conditions are met.
* After December 31, 2001, only conditions 3-6 must be met:
*
* 1. Commercial and/or non-commercial use of the Software is permitted
*    provided a current software registration is on file at www.OpenPBS.org.
*    If use of this software contributes to a publication, product, or
*    service, proper attribution must be given; see www.OpenPBS.org/credit.html
*
* 2. Redistribution in any form is only permitted for non-commercial,
*    non-profit purposes.  There can be no charge for the Software or any
*    software incorporating the Software.  Further, there can be no
*    expectation of revenue generated as a consequence of redistributing
*    the Software.
*
* 3. Any Redistribution of source code must retain the above copyright notice
*    and the acknowledgment contained in paragraph 6, this list of conditions
*    and the disclaimer contained in paragraph 7.
*
* 4. Any Redistribution in binary form must reproduce the above copyright
*    notice and the acknowledgment contained in paragraph 6, this list of
*    conditions and the disclaimer contained in paragraph 7 in the
*    documentation and/or other materials provided with the distribution.
*
* 5. Redistributions in any form must be accompanied by information on how to
*    obtain complete source code for the OpenPBS software and any
*    modifications and/or additions to the OpenPBS software.  The source code
*    must either be included in the distribution or be available for no more
*    than the cost of distribution plus a nominal fee, and all modifications
*    and additions to the Software must be freely redistributable by any party
*    (including Licensor) without restriction.
*
* 6. All advertising materials mentioning features or use of the Software must
*    display the following acknowledgment:
*
*     "This product includes software developed by NASA Ames Research Center,
*     Lawrence Livermore National Laboratory, and Veridian Information
*     Solutions, Inc.
*     Visit www.OpenPBS.org for OpenPBS software support,
*     products, and information."
*
* 7. DISCLAIMER OF WARRANTY
*
* THIS SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND. ANY EXPRESS
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT
* ARE EXPRESSLY DISCLAIMED.
*
* IN NO EVENT SHALL VERIDIAN CORPORATION, ITS AFFILIATED COMPANIES, OR THE
* U.S. GOVERNMENT OR ANY OF ITS AGENCIES BE LIABLE FOR ANY DIRECT OR INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
* OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* This license will be governed by the laws of the Commonwealth of Virginia,
* without reference to its choice of law rules.
*/
#include <pbs_config.h>   /* the master config generated by configure */

#include <sstream>
#define PBS_MOM 1
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <errno.h>
#include <fcntl.h>
#include <pwd.h>
#include <signal.h>
#include <stdio.h>
#include <unistd.h>
#include <grp.h>
#include <dirent.h>
#include <ctype.h>
#include "libpbs.h"
#include "list_link.h"
#include "server_limits.h"
#include "attribute.h"
#include "pbs_job.h"
#include "log.h"
#include "../lib/Liblog/pbs_log.h"
#include "lib_ifl.h"
#include "../lib/Libutils/lib_utils.h"
#include "mom_mach.h"
#include "mom_func.h"
#include "resource.h"
#include "pbs_proto.h"
#include "net_connect.h"
#include "utils.h"
#include "mom_config.h"
#include "json/json.h"


const int   PELOG_DOESNT_EXIST = -1;


extern int  MOMPrologTimeoutCount;
extern int  MOMPrologFailureCount;

extern int  LOGLEVEL;
extern int  DEBUGMODE;

extern int  lockfds;
extern char *path_aux;

extern gid_t   pbsgroup;
extern uid_t   pbsuser;
extern char   *path_epilogp;

extern long TJobStartTimeout;
static pid_t child;
static pid_t childSessionID;
static int   run_exit;
extern char *path_jobs;
extern int   multi_mom;
extern unsigned int pbs_rm_port;

/* external prototypes */

extern int pe_input(char *);
extern void encode_used(job *, int, Json::Value *, tlist_head *);
#ifdef ENABLE_CSA
extern void add_wkm_end(uint64_t, int64_t, char *);

extern char             *path_epiloguser;
#endif /* ENABLE_CSA */


/* END extern prototypes */

const char *PPEType[] =
  {
  "NONE",
  "prolog",
  "epilog",
  "userprolog",
  "userepilog",
  "prolog_user_job",
  "epilog_user_job",
  NULL
  };



/*
 * resc_to_string - convert resources_[list or used] to a single string
 */

static char *resc_to_string(

  job       *pjob,      /* I (optional - if specified, report total job resources) */
  int        aindex,    /* I which pbs_attribute to convert */
  char      *buf,       /* O the buffer into which to convert */
  int        buflen)    /* I the length of the above buffer */

  {
  int            need;
  svrattrl      *patlist;
  tlist_head     svlist;
  pbs_attribute *pattr;

  int            isfirst = 1;

  CLEAR_HEAD(svlist);

  *buf = '\0';

  pattr = &pjob->ji_wattr[aindex];

  /* pack the list of resources into svlist */

  if (aindex == JOB_ATR_resource)
    {
    if (encode_resc(pattr, &svlist, (char *)"x", NULL, ATR_ENCODE_CLIENT, ATR_DFLAG_ACCESS) <= 0)
      {
      return(buf);
      }
    }
  else if (aindex == JOB_ATR_resc_used)
    {
    encode_used(pjob, ATR_DFLAG_RDACC, NULL, &svlist);
    }
  else
    {
    return(buf);
    }

  /* unpack svlist into a comma-delimited string */

  patlist = (svrattrl *)GET_NEXT(svlist);

  while (patlist != NULL)
    {
    need = strlen(patlist->al_resc) + strlen(patlist->al_value) + 3;

    if (need >= buflen)
      {
      patlist = (svrattrl *)GET_NEXT(patlist->al_link);

      continue;
      }

    if (LOGLEVEL >= 7)
      {
      fprintf(stderr, "Epilog:  %s=%s\n",
              patlist->al_resc,
              patlist->al_value);
      }

    if (isfirst == 1)
      {
      isfirst = 0;
      }
    else
      {
      strcat(buf, ",");
      buflen--;
      }

    strcat(buf, patlist->al_resc);

    strcat(buf, "=");
    strcat(buf, patlist->al_value);

    buflen -= need;

    patlist = (svrattrl *)GET_NEXT(patlist->al_link);
    }  /* END while (patlist != NULL) */

  free_attrlist(&svlist);

  return(buf);
  }  /* END resc_to_string() */





/*
 * pelog_err - record error for run_pelog()
 *
 * @see run_pelog() - parent
 *
 * @return (parameter 'n')
 */

static int pelog_err(

  job        *pjob,  /* I */
  const char *file,  /* I */
  int         n,     /* I - exit code */
  const char *text)  /* I */

  {
  sprintf(log_buffer,"prolog/epilog failed, file: %s, exit: %d, %s",
    file,
    n,
    text);

  sprintf(PBSNodeMsgBuf,"ERROR: %s",
    log_buffer);

  log_err(-1, __func__, log_buffer);

  return(n);
  }  /* END pelog_err() */





/*
 * pelogalm() - alarm handler for run_pelog()
 */

static void pelogalm(

  int sig)  /* I */

  {
  DIR            *pProcDir = NULL;
  struct dirent *pProc;
  proc_stat_t    *pProcStat;
  /* child is global */

  errno = 0;

  if(childSessionID > 0)
    {
    if((pProcDir = opendir("/proc")) == NULL)
      {
      childSessionID = -1;
      }
    }
  if(childSessionID > 0)
    {
    rewinddir(pProcDir);
    while((pProc = readdir(pProcDir)) != NULL)
      {
      if(!isdigit(pProc->d_name[0]))
        continue;
      int procID = atoi(pProc->d_name);
      if((pProcStat = get_proc_stat(procID)) == NULL)
        continue;
      if((childSessionID == pProcStat->session)&&(pProcStat->state != 'Z')&&(pProcStat->pid != 0))
        {
        kill(procID,SIGKILL);
        }
      }
    closedir(pProcDir);
    }


  if (childSessionID <= 0)
    {
    kill(child,SIGKILL);
    }

  run_exit = -4;

  return;
  }  /* END pelogalm() */




/* 
 * sets the user and group ids back to how they were
 *
 */

int undo_set_euid_egid(

  int         which,
  uid_t       real_uid,
  gid_t       real_gid,
  int         num_gids,
  gid_t      *real_gids,
  const char *id)
    
  {
  if ((which == PE_PROLOGUSER) || 
      (which == PE_EPILOGUSER) || 
      (which == PE_PROLOGUSERJOB) || 
      (which == PE_EPILOGUSERJOB))
    {
    if ((setuid_ext(real_uid, TRUE) != 0) ||
        (setegid(real_gid) != 0) ||
        (setgroups(num_gids,real_gids) != 0))
      {
      log_err(errno,id, (char *)"Couldn't revert back to the root user - IMMINENT FAILURE but will try to continue\n");
      }

    return(-1);
    }

  return(0);
  } /* END undo_set_euid_egid() */



/*
 * check_if_pelog_exists()
 *
 * checks if the pelog script exists. If it exists but can't be stat'ed this is an error. If it
 * doesn't exist there is no error but we don't need to execute anything, and if it exists and
 * can be stat'ed then we proceed.
 *
 * @param which - the type of pelog we're preparing to execute
 * @param pelog - the path to the pelog script
 * @param pelog_size - the size of the pelog buffer
 * @param sbuf - the stat buf we should save the result of stat into
 * @param specpelog - the path to a specific pelog
 * @param pjob - the job whose pelog we're preparing to execute
 * @param jobtypespecified - true if the job specified a pelog
 * @return - PBSE_NONE if the script exists and can be stat'ed, PELOG_DOESNT_EXIST if the script
 * doesn't exist, and the appropriate errno if the script exists but can't be stat'ed.
 */

int check_if_pelog_exists(
  
  int          which,
  char        *pelog,
  int          pelog_size,
  struct stat &sbuf,
  const char  *specpelog,
  job         &pjob,
  bool         jobtypespecified)

  {
  int               rc;

  rc = stat(pelog,&sbuf);

  if ((rc == -1) &&
      (jobtypespecified == true))
    {
    snprintf(pelog, pelog_size, "%s", specpelog);

    rc = stat(pelog, &sbuf);
    }

  if (rc == -1)
    {
    if (errno == ENOENT || errno == EBADF)
      {
      /* epilog/prolog script does not exist */

      if (LOGLEVEL >= 5)
        {
        char tmpBuf[1024];

        sprintf(log_buffer, "%s script '%s' for job %s does not exist (cwd: %s,pid: %d)",
          PPEType[which],
          (pelog[0] != '\0') ? pelog : "NULL",
          pjob.ji_qs.ji_jobid,
          getcwd(tmpBuf, sizeof(tmpBuf)),
          getpid());

        log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
        }

#ifdef ENABLE_CSA
      if ((which == PE_EPILOGUSER) &&
          (!strcmp(pelog, path_epiloguser)))
        {
        /*
          * Add a workload management end record
        */
        if (LOGLEVEL >= 8)
          {
          sprintf(log_buffer, "%s calling add_wkm_end from run_pelog() - no user epilog",
            pjob.ji_qs.ji_jobid);

          log_err(-1, __func__, log_buffer);
          }

        add_wkm_end(pjob.ji_wattr[JOB_ATR_pagg_id].at_val.at_ll,
            pjob.ji_qs.ji_un.ji_momt.ji_exitstat, pjob.ji_qs.ji_jobid);
        }

#endif /* ENABLE_CSA */

      return(PELOG_DOESNT_EXIST);
      }

    return(pelog_err(&pjob, pelog, errno, "cannot stat"));
    }

  return(PBSE_NONE);
  } /* check_if_pelog_exists() */



/*
 * check_pelog_permissions()
 *
 * Checks to verify that the pelog script's permissions are set correctly
 * @param sbuf - the stat buf for the pelog script
 * @param reduceprologchecks - a parameter indicating how strict the checks should be. If 
 * set to TRUE the script only has to be executable
 * @param pjob - the job whose pelog we're preparing to execute
 * @param pelog - the path to the pelog script
 * @param which - the kind of pelog we're preparing to execute
 * @return - PBSE_NONE on success, and error code otherwise
 */

int check_pelog_permissions(
    
  struct stat &sbuf,
  int          reduceprologchecks,
  job         *pjob,
  const char  *pelog,
  int          which)

  {
  if (reduceprologchecks == TRUE)
    {
    if ((!S_ISREG(sbuf.st_mode)) ||
        (!(sbuf.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))))
      {
      return(pelog_err(pjob, pelog, -1, "permission Error"));
      }
    }
  else
    {
    if ((which == PE_PROLOGUSERJOB) ||
        (which == PE_EPILOGUSERJOB))
      {
      if ((sbuf.st_uid != pjob->ji_qs.ji_un.ji_momt.ji_exuid) || 
          (!S_ISREG(sbuf.st_mode)) ||
          ((sbuf.st_mode & (S_IRUSR | S_IXUSR)) != (S_IRUSR | S_IXUSR)) ||
          (sbuf.st_mode & (S_IWGRP | S_IWOTH)))
        {
        return(pelog_err(pjob, pelog, -1, "permission Error"));
        }
      }
    else if ((sbuf.st_uid != 0) ||
             (!S_ISREG(sbuf.st_mode)) ||
             ((sbuf.st_mode & (S_IRUSR | S_IXUSR)) != (S_IRUSR | S_IXUSR)) ||
             (sbuf.st_mode & (S_IWGRP | S_IWOTH)))
      {
      return(pelog_err(pjob, pelog, -1, "permission Error"));
      }
    
    if ((which == PE_PROLOGUSER) ||
        (which == PE_EPILOGUSER))
      {
      /* script must also be read and execute by other */
      
      if ((sbuf.st_mode & (S_IROTH | S_IXOTH)) != (S_IROTH | S_IXOTH))
        {
        return(pelog_err(pjob, pelog, -1,  "permission Error"));
        }
      }
    } /* END !reduceprologchecks */

  return(PBSE_NONE);
  } /* END check_pelog_permissions() */



/*
 * get_complete_hostlist()
 *
 */

char *get_complete_hostlist(

  job *pjob)

  {
  std::string complete_hostlist;
  std::string previous_host;

  for (int i = 0; i < pjob->ji_numnodes; i++)
    {
    if (i != 0)
      complete_hostlist += ",";

    complete_hostlist += pjob->ji_hosts[i].hn_host;
    }

  return(strdup(complete_hostlist.c_str()));
  } // END get_complete_hostlist()



/*
 * setup_pelog_arguments()
 *
 * Sets arg with the correct arguments for the pelog script
 *
 * @param pelog (I) - the path to the pelog we're preparing to execute
 * @param pjob (I) - the job whose pelog we're preparing to execute
 * @param which (I) - the type of pelog we're preparing to execute
 * @param arg (O) - the arguments that will be passed to execv
 */

void setup_pelog_arguments(

  char   *pelog,
  job    *pjob,
  int     which,
  char  **arg)

  {
  int  LastArg;
  char resc_list[2048];
  char resc_used[2048];
  char namebuf[1024];
  
  if (multi_mom)
    {
    snprintf(namebuf, sizeof(namebuf), "%s%s%d%s",
      path_jobs, pjob->ji_qs.ji_fileprefix, pbs_rm_port, JOB_SCRIPT_SUFFIX);
    }
  else
    {
    snprintf(namebuf, sizeof(namebuf), "%s%s%s",
      path_jobs, pjob->ji_qs.ji_fileprefix, JOB_SCRIPT_SUFFIX);
    }


  arg[0] = pelog;

  arg[1] = pjob->ji_qs.ji_jobid;
  arg[2] = pjob->ji_wattr[JOB_ATR_euser].at_val.at_str;
  arg[3] = pjob->ji_wattr[JOB_ATR_egroup].at_val.at_str;
  arg[4] = pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str;

  /* NOTE:  inside child */

  if ((which == PE_EPILOG) || 
      (which == PE_EPILOGUSER) || 
      (which == PE_EPILOGUSERJOB))
    {
    /* for epilogue only */
    char *sid = (char *)calloc(1, 20);
    char *exit_stat = (char *)calloc(1, 12);

    sprintf(sid, "%ld",
            pjob->ji_wattr[JOB_ATR_session_id].at_val.at_long);
    sprintf(exit_stat,"%d",
            pjob->ji_qs.ji_un.ji_momt.ji_exitstat);

    arg[5] = sid;
    arg[6] = strdup(resc_to_string(pjob, JOB_ATR_resource, resc_list, sizeof(resc_list)));
    arg[7] = strdup(resc_to_string(pjob, JOB_ATR_resc_used, resc_used, sizeof(resc_used)));
    arg[8] = pjob->ji_wattr[JOB_ATR_in_queue].at_val.at_str;
    arg[9] = pjob->ji_wattr[JOB_ATR_account].at_val.at_str;
    arg[10] = exit_stat;
    arg[11] = strdup(namebuf);
    arg[12] = NULL;

    LastArg = 12;
    }
  else
    {
    /* prologue */
    int last_arg = 9;
    arg[5] = strdup(resc_to_string(pjob, JOB_ATR_resource, resc_list, sizeof(resc_list)));
    arg[6] = pjob->ji_wattr[JOB_ATR_in_queue].at_val.at_str;
    arg[7] = pjob->ji_wattr[JOB_ATR_account].at_val.at_str;
    arg[8] = strdup(namebuf);

    if (which == PE_PRESETUPPROLOG)
      {
      arg[9] = get_complete_hostlist(pjob);
      last_arg++;
      }

    arg[last_arg] = NULL;

    LastArg = last_arg;
    }

  for (int aindex = 0;aindex < LastArg; aindex++)
    {
    if (arg[aindex] == NULL)
      arg[aindex] = strdup("");
    }  /* END for (aindex) */
  } /* END setup_pelog_arguments() */



/*
 * setup_pelog_environment()
 *
 * Adds environment variables for the pelog
 *
 * @param pjob - the job whose pelog we're about to execute
 * @param which - the type of pelog we're executing
 */

void setup_pelog_environment(
    
  job *pjob,
  int  which)

  {
  char      buf[MAXPATHLEN + 1024];
  resource *r;
  /*
   * Pass Resource_List.nodes request in environment
   * to allow pro/epi-logue setup/teardown of system
   * settings.  --pw, 2 Jan 02
   * Fixed to use putenv for sysV compatibility.
   *  --troy, 11 jun 03
   *
   */

  r = find_resc_entry(
        &pjob->ji_wattr[JOB_ATR_resource],
        find_resc_def(svr_resc_def, (char *)"nodes", svr_resc_size));

  if (r != NULL)
    {
    const char *ppn_str = "ppn=";
    int         num_nodes = 1;
    int         num_ppn = 1;

    /* PBS_RESOURCE_NODES */
    put_env_var("PBS_RESOURCE_NODES", r->rs_value.at_val.at_str);

    /* PBS_NUM_NODES */
    num_nodes = strtol(r->rs_value.at_val.at_str, NULL, 10);

    /* 
     * InitUserEnv() also calculates num_nodes and num_ppn the same way
     */
    if (num_nodes != 0)
      {
      char *tmp;
      char *other_reqs;

      /* get the ppn */
      if ((tmp = strstr(r->rs_value.at_val.at_str,ppn_str)) != NULL)
        {
        tmp += strlen(ppn_str);

        num_ppn = strtol(tmp, NULL, 10);
        }

      other_reqs = r->rs_value.at_val.at_str;

      while ((other_reqs = strchr(other_reqs, '+')) != NULL)
        {
        other_reqs += 1;
        num_nodes += strtol(other_reqs, &other_reqs, 10);
        }
      }

    sprintf(buf, "%d", num_nodes);
    put_env_var("PBS_NUM_NODES", buf);

    /* PBS_NUM_PPN */
    sprintf(buf, "%d", num_ppn);
    put_env_var("PBS_NUM_PPN", buf);

    /* PBS_NP */
    sprintf(buf, "%d", pjob->ji_numvnod);
    put_env_var("PBS_NP", buf);
    }  /* END if (r != NULL) */

  r = find_resc_entry(
        &pjob->ji_wattr[JOB_ATR_resource],
        find_resc_def(svr_resc_def, (char *)"gres", svr_resc_size));

  if (r != NULL)
    {
    /* setenv("PBS_RESOURCE_NODES",r->rs_value.at_val.at_str,1); */
    put_env_var("PBS_RESOURCE_GRES", r->rs_value.at_val.at_str);
    }

  char *cpu_clock = arst_string("PBS_CPUCLOCK",&pjob->ji_wattr[JOB_ATR_variables]);
  if (cpu_clock != NULL)
    {
    cpu_clock = strchr(cpu_clock,'=');
    if (cpu_clock != NULL)
      {
      cpu_clock++;
      put_env_var("PBS_CPUCLOCK",cpu_clock);
      }
    }

  if (TTmpDirName(pjob, buf, sizeof(buf)))
    {
    put_env_var("TMPDIR", buf);
    }

  /* Set PBS_SCHED_HINT */
  char *envname = (char *)"PBS_SCHED_HINT";
  char *envval;

  if ((envval = get_job_envvar(pjob, envname)) != NULL)
    {
    put_env_var("PBS_SCHED_HINT", envval);
    }

  /* Set PBS_NODENUM */

  sprintf(buf, "%d", pjob->ji_nodeid);
  put_env_var("PBS_NODENUM", buf);

  /* Set PBS_MSHOST */
  put_env_var("PBS_MSHOST", pjob->ji_vnods[0].vn_host->hn_host);

  /* Set PBS_NODEFILE */
  if (pjob->ji_flags & MOM_HAS_NODEFILE)
    {
    sprintf(buf, "%s/%s",
      path_aux,
      pjob->ji_qs.ji_jobid);
    put_env_var("PBS_NODEFILE", buf);
    }

  /* Set PBS_O_WORKDIR */
  char *workdir_val = get_job_envvar(pjob,"PBS_O_WORKDIR");
  if (workdir_val != NULL)
    {
    put_env_var("PBS_O_WORKDIR", workdir_val);
    }

  /* SET BEOWULF_JOB_MAP */
  struct array_strings *vstrs;

  int VarIsSet = 0;
  int j;

  vstrs = pjob->ji_wattr[JOB_ATR_variables].at_val.at_arst;

  for (j = 0;j < vstrs->as_usedptr;++j)
    {
    if (!strncmp(
          vstrs->as_string[j],
          "BEOWULF_JOB_MAP=",
          strlen("BEOWULF_JOB_MAP=")))
      {
      VarIsSet = 1;

      break;
      }
    }

  if (VarIsSet == 1)
    {
    char *val = strchr(vstrs->as_string[j], '=');

    if (val != NULL)
      put_env_var("BEOWULF_JOB_MAP", val+1);
    }

  /* Set some Moab env variables if they exist */
  if ((which == PE_PROLOG) ||
      (which == PE_EPILOG))
    {
    char *tmp_val;
    int   moabenvcnt = 14;  /* # of entries in moabenvs */
    static char      *moabenvs[] = {
        (char *)"MOAB_NODELIST",
        (char *)"MOAB_JOBID",
        (char *)"MOAB_JOBNAME",
        (char *)"MOAB_USER",
        (char *)"MOAB_GROUP",
        (char *)"MOAB_CLASS",
        (char *)"MOAB_TASKMAP",
        (char *)"MOAB_QOS",
        (char *)"MOAB_PARTITION",
        (char *)"MOAB_PROCCOUNT",
        (char *)"MOAB_NODECOUNT",
        (char *)"MOAB_MACHINE",
        (char *)"MOAB_JOBARRAYINDEX",
        (char *)"MOAB_JOBARRAYRANGE"
        };
    
    for (int aindex=0; aindex < moabenvcnt; aindex++)
      {
      tmp_val = get_job_envvar(pjob,moabenvs[aindex]);
      if (tmp_val != NULL)
        {
        put_env_var(moabenvs[aindex], tmp_val);
        }
      }
    }
  } /* END setup_pelog_environment() */



/*
 * setup_prolog_outputs
 *
 * Sets the stdout and stderr file descriptors (1 and 2 respectively) to the 
 * appropriate streams according to the output type and the job's output and error files
 *
 * @param pjob - the job whose pelog we're preparing to execute
 * @param pe_io_type - specifies what kind of output this pelog should have
 * @param delete_job - TRUE if this comes as a result of a delete job request
 * @param specpelog - the specific pelog we're executing
 */

void setup_pelog_outputs(
    
  job  *pjob,
  int   pe_io_type,
  int   delete_job,
  char *specpelog)

  {
  int fds1 = 0;
  int fds2 = 0;

  if (pe_io_type == PE_IO_TYPE_NULL)
    {
    /* no output, force to /dev/null */

    fds1 = open("/dev/null", O_WRONLY, 0600);
    fds2 = open("/dev/null", O_WRONLY, 0600);
    }
  else if (pe_io_type == PE_IO_TYPE_STD)
    {
    /* open job standard out/error */

    /*
     * We need to know if files are joined or not.
     * If they are then open the correct file and duplicate it to the other
    */

    int isjoined = is_joined(pjob);

    switch (isjoined)
      {
      case -1:

        fds2 = open_std_file(pjob, StdErr, O_WRONLY | O_APPEND,
                             pjob->ji_qs.ji_un.ji_momt.ji_exgid);

        fds1 = (fds2 < 0)?-1:dup(fds2);

        break;

      case 1:

        fds1 = open_std_file(pjob, StdOut, O_WRONLY | O_APPEND,
                             pjob->ji_qs.ji_un.ji_momt.ji_exgid);

        fds2 = (fds1 < 0)?-1:dup(fds1);

        break;

      default:

        fds1 = open_std_file(pjob, StdOut, O_WRONLY | O_APPEND,
                             pjob->ji_qs.ji_un.ji_momt.ji_exgid);

        fds2 = open_std_file(pjob, StdErr, O_WRONLY | O_APPEND,
                             pjob->ji_qs.ji_un.ji_momt.ji_exgid);
        break;
      }
    }

  /*
   * dupeStdFiles is a flag for those that couldn't open their .OU/.ER files
  */
  int dupeStdFiles = 1;

  if (!delete_job)
    {
    if ((fds1 < 0) ||
        (fds2 < 0))
      {
      if (fds1 >= 0)
        {
        close(fds1);
        fds1 = -1;
        }

      if (fds2 >= 0)
        {
        close(fds2);
        fds2 = -1;
        }

      if ((pe_io_type == PE_IO_TYPE_STD) &&
          (strlen(specpelog) == strlen(path_epilogp)) &&
          (strcmp(path_epilogp, specpelog) == 0))
        dupeStdFiles = 0;
      }
    }

  if (pe_io_type != PE_IO_TYPE_ASIS)
    {
    /* If PE_IO_TYPE_ASIS, leave as is, already open to job */

    /* dup only for those fds1 >= 0 */

    if (fds1 != 1)
      {
      close(1);

      if (dupeStdFiles)
        {
        if ((fds1 >= 0) &&
            (dup(fds1) >= 0))
          close(fds1);
        }
      }

    if (fds2 != 2)
      {
      close(2);

      if (dupeStdFiles)
        {
        if ((fds2 >= 0) &&
            (dup(fds2) >= 0))
          close(fds2);
        }
      }
    }
  } /* END setup_pelog_outputs() */



/*
 * handle_pipes_as_child()
 *
 * Writes the session to the pipe to the parent after setting a new session, then
 * closes the pipes.
 *
 * @param parent_read - file_descriptor that the parent reads
 * @param parent_write - file descriptor that the parent writes
 * @param kid_read - file descriptor that the child reads
 * @param kid_write - file descriptor that the child writes
 */
void handle_pipes_as_child(

  int parent_read,
  int parent_write,
  int kid_read,
  int kid_write)

  {
  // detach from parent
  childSessionID = setsid();

  close(parent_read);
  close(parent_write);
  write(kid_write,(char *)&childSessionID,sizeof(childSessionID));
  close(kid_read);
  close(kid_write);
  } /* END handle_pipes_as_child() */



/*
 * close_handles_as_child()
 *
 * Closes the handles the child shouldn't have open: log, lockfile, network connections.
 *
 */
void close_handles_as_child()

  {
  log_close(0);

  if (lockfds >= 0)
    {
    close(lockfds);

    lockfds = -1;
    }

  net_close(-1);
  } /* END close_handles_as_child() */



/*
 * change_directory_as_needed()
 *
 * Changes the directory to the job's home directory for user pelogs
 *
 * @param which - the kind of pelog we're preparing to execute
 * @param pjob - the job whose pelog we're preparing to execute
 * @post-cond: we have changed to the appropriate directory if possible and necessary
 */
void change_directory_as_needed(

  int  which,
  job *pjob)

  {
  if ((which == PE_PROLOGUSER) || 
      (which == PE_EPILOGUSER) || 
      (which == PE_PROLOGUSERJOB) || 
      (which == PE_EPILOGUSERJOB))
    {
    if (chdir(pjob->ji_grpcache->gc_homedir) != 0)
      {
      /* warn only, no failure */

      sprintf(log_buffer,
        "PBS: chdir to %s failed: %s (running user %s in current directory)",
        pjob->ji_grpcache->gc_homedir,
        strerror(errno),
        which == PE_PROLOGUSER ? "prologue" : "epilogue");

      if (write_ac_socket(2, log_buffer, strlen(log_buffer)) == -1) {}

      fsync(2);
      }
    }
  } /* END change_directory_as_needed() */



/*
 * prepare_and_run_pelog_as_child()
 *
 * Closes file handles, sets up the output and error streams, arguments,
 * changes directory, etc. to get everything ready and then execs the 
 * prologue or epilogue script.
 *
 * @param pjob - the job whose pelog we're executing (I)
 * @param pe_io_type - specifies what kind of output the pelog should have (I)
 * @param delete_job - TRUE if this is an epilogue from a delete job request (I)
 * @param specpelog - the specific pelog script to execute (I)
 * @param pelog - the script we're executing (I)
 * @param which - which prologue we're executing (I)
 * @param parent_read - file descriptor for the pipe the parent reads
 * @param parent_write - file descriptor for the pipe the parent writes
 * @param kid_read - file descriptor for the pipe the kid reads
 * @param kid_write - file descriptor for the pipe the kid writes
 * @param fd_input - file descriptor for the pelog's stdin
 * 
 * NOTE: this function should never return as it calls execv
 */

void prepare_and_run_pelog_as_child(

  job  *pjob,
  int   pe_io_type,
  int   delete_job,
  char *specpelog,
  char *pelog,
  int   which,
  int   parent_read,
  int   parent_write,
  int   kid_read,
  int   kid_write,
  int   fd_input)

  {
  char *arg[13];

  handle_pipes_as_child(parent_read, parent_write, kid_read, kid_write);

  close_handles_as_child();

  // setup stdin for the pelog
  if (fd_input != 0)
    {
    close(0);

    if (dup(fd_input) == -1) {}

    close(fd_input);
    }

  setup_pelog_outputs(pjob, pe_io_type, delete_job, specpelog);

  change_directory_as_needed(which, pjob);

  /* for both prolog and epilog */
  if (DEBUGMODE == 1)
    {
    char resc_list[2048];
    fprintf(stderr, "PELOGINFO:  script:'%s'  jobid:'%s'  euser:'%s'  egroup:'%s'  jobname:'%s' SSID:'%ld'  RESC:'%s'\n",
            pelog,
            pjob->ji_qs.ji_jobid,
            pjob->ji_wattr[JOB_ATR_euser].at_val.at_str,
            pjob->ji_wattr[JOB_ATR_egroup].at_val.at_str,
            pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str,
            pjob->ji_wattr[JOB_ATR_session_id].at_val.at_long,
            resc_to_string(pjob, JOB_ATR_resource, resc_list, sizeof(resc_list)));
    }

  setup_pelog_arguments(pelog, pjob, which, arg);

  setup_pelog_environment(pjob,  which);
  
  /*
   * if we want to run as user then we need to reset real user permissions
   * since it seems that some OSs use real not effective user id when execv'ing
   */
  
  if ((which == PE_PROLOGUSER) || 
      (which == PE_EPILOGUSER) || 
      (which == PE_PROLOGUSERJOB) || 
      (which == PE_EPILOGUSERJOB))
    {
    setuid_ext(pbsuser, TRUE);
    setegid(pbsgroup);

    if (become_the_user(pjob) != PBSE_NONE)
      {
      exit(-1);
      }
    }

  execv(pelog,arg);

  /* should not be reached, but clean up if error */
  sprintf(log_buffer,"execv of %s failed: %s\n",
    pelog,
    strerror(errno));

  if (write_ac_socket(2, log_buffer, strlen(log_buffer)) == -1)
    {
    /* cannot write message to stderr */

    /* NO-OP */
    }

  fsync(2);

  exit(255);
  } /* END prepare_and_run_pelog_as_child() */



/*
 * get_child_exit_status()
 *
 * Gets the exit status of the child and sets run_exit appropriately
 * 
 * @post-cond: the child has either exited or it has been determined
 * that we can't kill the child and run_exit has been marked with an error.
 * On a normal (non-timeout) exit run_exit has been set with the exit code
 * of the epilogue or prologue
 * @return - the exit status of the pelog or an appropriate error code on
 * a timeout
 */

int get_child_exit_status(

  job  *pjob,
  char *pelog,
  int   which)

  {
  struct sigaction  act;
  struct sigaction  oldact;
  long              real_alarm_time = pe_alarm_time;
  int               waitst;
  int               KillSent = FALSE;
  
  act.sa_handler = pelogalm;
  sigemptyset(&act.sa_mask);

  act.sa_flags = 0;

  sigaction(SIGALRM, &act, &oldact);

  if (real_alarm_time > 10) //Kill the prolog at least ten seconds before the timeout.
    {
    real_alarm_time -= 10;
    }

  alarm(real_alarm_time);

  while (waitpid(child, &waitst, 0) < 0)
    {
    if (errno != EINTR)
      {
      /* exit loop. non-alarm based failure occurred */

      run_exit = -3;

      MOMPrologFailureCount++;

      break;
      }

    // this gets set if the alarm goes off and means a timeout occurred
    if (run_exit == -4)
      {
      if (KillSent == FALSE)
        {
        MOMPrologTimeoutCount++;

        KillSent = TRUE;

        /* NOTE:  prolog/epilog may be locked in KERNEL space and unkillable */

        alarm(5);
        }
      else
        {
        /* cannot kill prolog/epilog, give up */

        run_exit = -5;

        break;
        }
      }
    } /* END while (wait(&waitst) < 0) */

  /* epilog/prolog child completed */
#ifdef ENABLE_CSA
  if ((which == PE_EPILOGUSER) && (!strcmp(pelog, path_epiloguser)))
    {
    /*
     * Add a workload management end record
    */
    if (LOGLEVEL >= 8)
      {
      sprintf(log_buffer, "%s calling add_wkm_end from run_pelog() - after user epilog",
              pjob->ji_qs.ji_jobid);

      log_err(-1, __func__, log_buffer);
      }

    add_wkm_end(pjob->ji_wattr[JOB_ATR_pagg_id].at_val.at_ll,
        pjob->ji_qs.ji_un.ji_momt.ji_exitstat, pjob->ji_qs.ji_jobid);
    }

#endif /* ENABLE_CSA */

  alarm(0);

  /* restore the previous handler */

  sigaction(SIGALRM, &oldact, 0);

  if (run_exit == 0)
    {
    if (WIFEXITED(waitst))
      {
      run_exit = WEXITSTATUS(waitst);
      }
    }
  
  switch (run_exit)
    {
    case 0:

      /* SUCCESS */

      /* NO-OP */

      break;

    case -3:

      pelog_err(pjob, pelog, run_exit,  "child wait interrupted");

      break;

    case -4:

      pelog_err(pjob, pelog, run_exit,  "prolog/epilog timeout occurred, child cleaned up");

      break;

    case -5:

      pelog_err(pjob, pelog, run_exit, "prolog/epilog timeout occurred, cannot kill child");

      break;

    default:

      pelog_err(pjob, pelog, run_exit, "nonzero p/e exit status");

      break;
    }  /* END switch (run_exit) */

  return(run_exit);
  } /* END get_child_exit_status() */




/*
 * run_pelog() - Run the Prologue/Epilogue script
 *
 * Script is run under uid of root, prologue and the epilogue have:
 *  - argv[1] is the jobid
 *  - argv[2] is the user's name
 *  - argv[3] is the user's group name
 *  - argv[4] is the job name
 *  - the input file is an architecture-dependent file
 *  - the output and error are the job's output and error
 * The epilogue also has:
 *   - argv[5] is the session id
 *   - argv[6] is the list of resource limits specified
 *   - argv[7] is the list of resources used
 *   - argv[8] is the queue in which the job resides
 *   - argv[9] is the account under which the job run
 *   - argv[10] is the job's exit status
 * The prologue also has:
 *   - argv[5] is the list of resource limits specified
 *   - argv[6] is the queue in which the job resides
 *   - argv[7] is the account under which the job is run
 * 
 * @see TMomFinalizeChild() - parent
 * @see pelog_err() - child
 *
 * @return = 0 - SUCCESS - file does not exist or execution successful
 * @return < 0 - FAILURE - general internal failure 
 *   -1 file permission issue
 *   -2 no pro/epi input file
 *   -3 child wait interrupted
 *   -4 prolog/epilog timeout occurred, child cleaned up
 *   -5 prolog/epilog timeout occurred, cannot kill child
 * @return > 0 - FAILURE - system failure (rc = errno)
 */

int run_pelog(

  int   which,      /* I (one of PE_*) */
  char *specpelog,  /* I - script path */
  job  *pjob,       /* I - associated job */
  int   pe_io_type, /* I - io type */
  int   delete_job)  /* I - called before a job being deleted (purge -p) */

  {
  int               fd_input;

  struct stat       sbuf;
  char              pelog[MAXPATHLEN + 1024];

  uid_t             real_uid;
  gid_t            *real_gids = NULL;
  gid_t             real_gid;
  int               num_gids;

  bool              jobtypespecified = false;

  int               rc;

  char             *ptr;

  int               pipes[2];
  int               kid_read;
  int               kid_write;
  int               parent_read;
  int               parent_write;

  if ((pjob == NULL) ||
      (specpelog == NULL) ||
      (specpelog[0] == '\0'))
    {
    return(0);
    }

  ptr = pjob->ji_wattr[JOB_ATR_jobtype].at_val.at_str;

  if (ptr != NULL)
    {
    jobtypespecified = true;

    snprintf(pelog,sizeof(pelog),"%s.%s",
      specpelog,
      ptr);
    }
  else
    {
    snprintf(pelog, sizeof(pelog), "%s", specpelog);
    }
    
  real_uid = getuid();
  real_gid = getgid();
  if ((num_gids = getgroups(0, real_gids)) < 0)
    {
    log_err(errno, __func__, (char *)"getgroups failed\n");
    
    return(-1);
    }

  /* to support root squashing, become the user before performing file checks */
  if ((which == PE_PROLOGUSER) || 
      (which == PE_EPILOGUSER) || 
      (which == PE_PROLOGUSERJOB) || 
      (which == PE_EPILOGUSERJOB))
    {

    real_gids = (gid_t *)calloc(num_gids, sizeof(gid_t));
    
    if (real_gids == NULL)
      {
      log_err(ENOMEM, __func__, (char *)"Cannot allocate memory! FAILURE\n");
      
      return(-1);
      }
    
    if (getgroups(num_gids,real_gids) < 0)
      {
      log_err(errno, __func__, (char *)"getgroups failed\n");
      free(real_gids);
      
      return(-1);
      }
    
    /* pjob->ji_grpcache will not be set if using LDAP and LDAP not set */
    /* It is possible that ji_grpcache failed to allocate as well. 
       Make sure ji_grpcache is not NULL */
    if (pjob->ji_grpcache != NULL)
      {
      if (setgroups(
            pjob->ji_grpcache->gc_ngroup,
            (gid_t *)pjob->ji_grpcache->gc_groups) != 0)
        {
        snprintf(log_buffer,sizeof(log_buffer),
          "setgroups() for UID = %lu failed: %s\n",
          (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid,
          strerror(errno));
      
        log_err(errno, __func__, log_buffer);
      
        undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
        free(real_gids);
      
        return(-1);
        }
      }
    else
      {
      sprintf(log_buffer, "pjob->ji_grpcache is null. check_pwd likely failed.");
      log_err(-1, __func__, log_buffer);
      undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
      free(real_gids);
      return(-1);
      }
    
    if (setegid(pjob->ji_qs.ji_un.ji_momt.ji_exgid) != 0)
      {
      snprintf(log_buffer,sizeof(log_buffer),
        "setegid(%lu) for UID = %lu failed: %s\n",
        (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exgid,
        (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid,
        strerror(errno));
      
      log_err(errno, __func__, log_buffer);
      
      undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
      free(real_gids);
      
      return(-1);
      }
    
    if (setuid_ext(pjob->ji_qs.ji_un.ji_momt.ji_exuid, TRUE) != 0)
      {
      snprintf(log_buffer,sizeof(log_buffer),
        "seteuid(%lu) failed: %s\n",
        (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid,
        strerror(errno));
      
      log_err(errno, __func__, log_buffer);
      
      undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
      free(real_gids);

      return(-1);
      }
    }

  rc = check_if_pelog_exists(which, pelog, sizeof(pelog), sbuf, specpelog, *pjob, jobtypespecified);

  switch (rc)
    {
      
    case PBSE_NONE:

      // continue
      break;

    case PELOG_DOESNT_EXIST:

      // not an error but we are done
      rc = PBSE_NONE;
      
      // fall through

    default:

      // error, notify caller 
      undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
      free(real_gids);
      return(rc);
    }

  if (LOGLEVEL >= 5)
    {
    sprintf(log_buffer,"running %s script '%s' for job %s",
      PPEType[which],
      (pelog[0] != '\0') ? pelog : "NULL",
      pjob->ji_qs.ji_jobid);

    log_ext(-1, __func__, log_buffer, LOG_DEBUG);
    }

  /* script must be owned by root, be regular file, read and execute by user *
   * and not writeable by group or other */

  if ((rc = check_pelog_permissions(sbuf, reduceprologchecks, pjob, pelog, which)) != PBSE_NONE)
    {
    undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
    free(real_gids);
    return(rc);
    }

  fd_input = pe_input(pjob->ji_qs.ji_jobid);

  if (fd_input < 0)
    {
    undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
    free(real_gids);
    return(pelog_err(pjob, pelog, -2,  "no pro/epilogue input file"));
    }

  run_exit = 0;

  // Set up communications between parent and child so that
  // child can send back the session id.
  if (pipe(pipes) == -1)
    {
    free(real_gids);
    return(-1);
    }

  if (pipes[1] < 3)
    {
    kid_write = fcntl(pipes[1], F_DUPFD, 3);

    close(pipes[1]);
    }
  else
    {
    kid_write = pipes[1];
    }

  parent_read = pipes[0];

  if (pipe(pipes) == -1)
    {
    free(real_gids);
    return(-1);
    }

  if (pipes[0] < 3)
    {
    kid_read = fcntl(pipes[0], F_DUPFD, 3);

    close(pipes[0]);
    }
  else
    {
    kid_read = pipes[0];
    }

  parent_write = pipes[1];

  if ((kid_read < 0) ||
      (kid_write < 0))
    {
    free(real_gids);
    return(-1);
    }

  child = fork();

  if (child > 0)
    {
    /* parent - watch for prolog/epilog to complete */

    close(fd_input);

    close(kid_read);
    close(kid_write);
    read(parent_read,(char *)&childSessionID,sizeof(childSessionID));
    close(parent_read);
    close(parent_write);


    /* switch back to root if necessary */
    undo_set_euid_egid(which,real_uid,real_gid,num_gids,real_gids,__func__);
    free(real_gids);

    rc = get_child_exit_status(pjob, pelog, which);

    return(rc);
    }
  else if (child == 0)
    {
    /* child - run script */
    prepare_and_run_pelog_as_child(pjob, pe_io_type, delete_job, specpelog, pelog, which,
        parent_read, parent_write, kid_read, kid_write, fd_input);
    
    // NOTREACHED: the above function doesn't return
    exit(255);
    }  /* END else () */
  else
    {
    // ERROR
    log_err(errno, __func__, "Fork failed");
    rc = -1;
    }
    
  free(real_gids);

  return(rc);
  }  /* END run_pelog() */
