#include "license_pbs.h" /* See here for the software license */

/*
 *
 * qsub_functions - (PBS) submit batch job
 *
 * Authors:
 *      Terry Heidelberg
 *      Livermore Computing
 *
 *      Bruce Kelly
 *      National Energy Research Supercomputer Center
 *
 *      Lawrence Livermore National Laboratory
 *      University of California
 */

#include <pbs_config.h>   /* the master config generated by configure */
#include <pbs_ifl.h>      /* pbs_submit_hash */
#include <pbs_cmds.h>     /* add_verify_resources */
#include <pbs_error.h>    /* all static defines,  message & error codes */
#include "qsub_functions.h"
#include "common_cmds.h"
#include "lib_ifl.h"

#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/wait.h>
#include <netinet/in.h>
#include <errno.h>
#include <fcntl.h>
#include <netdb.h>
#include <signal.h>
#include <unistd.h>
#include <stdlib.h>
#include <assert.h>
#include <grp.h>
#include <csv.h>
#include <pwd.h>

#ifdef sun
#include <sys/stream.h>
#endif /* sun */

#if defined(HAVE_SYS_TTY_H)
#include <sys/tty.h>
#endif

#if defined(FD_SET_IN_SYS_SELECT_H)
#include <sys/select.h>
#endif

#include "libcmds.h" /* TShowAbout_exit */
#include "cmds.h"
#include "net_connect.h"
#include "log.h"
#include "port_forwarding.h"
#include "common_cmds.h" 
#include "utils.h"
#include "complete_req.hpp"
#include "pbs_helper.h"

#if defined(PBS_NO_POSIX_VIOLATION)
#define GETOPT_ARGS "a:A:c:C:e:EF:hj:k:K:l:m:M:nN:o:p:q:r:S:u:v:VW:z"
#else
#define GETOPT_ARGS "a:A:b:c:C:d:D:e:EfF:hi:Ij:J:k:K:l:L:m:M:nN:o:p:P:q:r:S:t:T:u:v:Vw:W:Xxz-:"
#endif /* PBS_NO_POSIX_VIOLATION */

#define MAXBUF 2048

#define MAX_RETRIES  3
/* START: These are needed for bailout purposes */
int inter_sock = -1;
int interactivechild = 0;
int x11child = 0;
int have_terminal = TRUE;
char *new_jobname = NULL;           /* return from submit request */
/* for reference purposes:
 * pbs_server is defined in pbsD_connect.c and the extern is in pbs_ifl.h */
static char server_out[PBS_MAXSERVERNAME + PBS_MAXPORTNUM + 2];
struct termios oldtio;
/* END: bailout globals */

char *host_name_suffix = NULL;

/* state booleans for protecting already-set options */
int    J_opt = FALSE;
int    P_opt = FALSE;

const char *checkpoint_strings = "n,c,s,u,none,shutdown,periodic,enabled,interval,depth,dir";
char       *alternate_dependency;
int         alternate_data_type;
complete_req  cr;

/* adapted from openssh */
/* The parameter was EMsg, but was never used.
 * xauth_path was a global.  */

char *x11_get_proto(

  char *xauth_path, /* I */
  int   debug)        /* I */

  {
  char         line[X11_CHAR_SIZE];
  char         proto[X11_CHAR_SIZE];
  char         data[X11_CHAR_SIZE];
  char         screen[X11_CHAR_SIZE];
  char        *authstring;
  FILE        *f;
  int          got_data = 0;
  char        *display = NULL;
  char        *tmp;
  char        *p;
  struct stat  st;

  proto[0]  = '\0';
  data[0]   = '\0';
  screen[0] = '\0';

  if ((tmp = getenv("DISPLAY")) == NULL)
    {
    fprintf(stderr, "qsub: DISPLAY not set\n");
    return(NULL);
    }

  if ((display = strdup(tmp)) == NULL)
    {
    return(NULL);
    }

  if (!xauth_path)
    {
    free(display);
    return NULL;
    }

  if (stat(xauth_path, &st))
    {
    perror("qsub: xauth: ");
    free(display);
    return(NULL);
    }

  /* Try to get Xauthority information for the display. */

  if (strncmp(display, "localhost:", 10) == 0)
    {
    /*
     * Handle FamilyLocal case where $DISPLAY does
     * not match an authorization entry.  For this we
     * just try "xauth list unix:displaynum.screennum".
     * XXX: "localhost" match to determine FamilyLocal
     *      is not perfect.
     */

    snprintf(line, X11_CHAR_SIZE, "%s list unix:%s 2>/dev/null",
             xauth_path,
             display + 10);
    }
  else
    {
    snprintf(line, X11_CHAR_SIZE, "%s list %.200s 2>/dev/null",
             xauth_path,
             display);
    }

  p = strchr(display, ':');

  if (p != NULL)
    p = strchr(p, '.');

  if (p != NULL)
    snprintf(screen, sizeof(screen), "%s", p + 1);
  else
    snprintf(screen, sizeof(screen), "0");

  if (debug)
    fprintf(stderr, "x11_get_proto: %s\n",
            line);

  f = popen(line, "r");

  if (f == NULL)
    {
    fprintf(stderr, "execution of '%s' failed, errno=%d (%s)\n",
            line,
            errno,
            pbs_strerror(errno));
    }
  else if (fgets(line, X11_CHAR_SIZE, f) == 0)
    {
    fprintf(stderr, "cannot read data from '%s', errno=%d (%s)\n",
            line,
            errno,
            pbs_strerror(errno));
    }
  else if (sscanf(line, "%*s %511s %511s",
                  proto,
                  data) != 2)
    {
    fprintf(stderr, "cannot parse output from '%s'\n",
            line);
    }
  else
    {
    /* SUCCESS */

    got_data = 1;
    }

  if (f != NULL)
    pclose(f);

  if (!got_data)
    {
    /* FAILURE */
    free(display);
    return(NULL);
    }

  authstring = (char *)calloc(1, strlen(proto) + strlen(data) + strlen(screen) + 4);

  if (authstring == NULL)
    {
    /* FAILURE */

    free(display);
    return(NULL);
    }

  sprintf(authstring, "%s:%s:%s",
    proto,
    data,
    screen);

  free(display);
  return(authstring);
  }  /* END x11_get_proto() */



int find_job_script_index(

  int    start_index,
  int   *interactive,
  int   *prefix_index,
  int    argc,
  char **argv)

  {
  char search_str[3];
  int  ignore_next = FALSE;
  int  i;
  int  script_index = -1;

  search_str[1] = ':';
  search_str[2] = '\0';

  for (i = start_index; i < argc; i++)
    {
    if (ignore_next == FALSE)
      {
      if (*(argv[i]) == '-')
        {
        /* found an option */
        /* grab the first character to see if that character is in GETOPT_ARGS
         * as <char>:. If so, ignore the next argument, as it pertains to this
         * option */
        search_str[0] = *(argv[i] + 1);

        if (search_str[0] == 'I')
          *interactive = TRUE;
        else if (strstr(GETOPT_ARGS, search_str) != NULL)
          {
          int len = strlen(argv[i]);

          if (len <= 2)
            ignore_next = TRUE;

          if (search_str[0] == 'C')
            {
            *prefix_index = i + 1;
            }
          }
        }
      else
        {
        /* found a loose string with no index in front of it. This is the job script */
        script_index = i;
        break;
        }
      }
    else
      ignore_next = FALSE;
    }

  return(script_index);
  } /* END find_job_script_index() */




char *smart_strtok(

  char  *line,          /* I */
  const char  *delims,        /* I */
  char **ptrPtr,        /* O */
  int    ign_backslash) /* I */

  {
  char *head = NULL;
  char *start = NULL;

  int dindex;
  int ignchar;
  int ignore;

  int sq_count = 0;
  int dq_count = 0;
  int sb_count = 0;

  char *tmpLine = NULL;
  int   tmpLineSize;
  int   tindex;

  char *ptr;

  if (ptrPtr == NULL)
    {
    /* FAILURE */

    return(head);
    }
  else if (line != NULL)
    {
    *ptrPtr = line;
    }
  else if (*ptrPtr == NULL)
    {
    /* FAILURE */

    return(head);
    }

  start = *ptrPtr;

  tmpLineSize = (line == NULL) ? strlen(*ptrPtr) + 1 : strlen(line) + 1;
  tmpLine = (char *)calloc(1, tmpLineSize * sizeof(char));

  tmpLine[0] = '\0';

  tindex = 0;

  ignchar = FALSE;

  ptr = *ptrPtr;

  while (*ptr != '\0')
    {
    if (*ptr == '\'')
      {
      sq_count++;

      if ((head != NULL) && !(sq_count % 2) && !(dq_count % 2))
        {
        ptr++;

        ignchar = TRUE;
        }
      else 
        {
        ignore = TRUE;

        if (ign_backslash == TRUE)
          {
          /* check if backslash precedes delimiter */

          if ((ptr > start) && (*(ptr-1) == '\\'))
            {
            /* check if backslash is backslashed */

            if ((ptr > start + 1) && (*(ptr-2) != '\\'))
              {
              /* delimiter is backslashed, ignore */

              ignore = FALSE;
              
              sq_count--;
              }
            }
          }

        if (ignore == TRUE)
          {
          ptr++;

          ignchar = TRUE;
          }
        }
      }
    else if (*ptr == '\"')
      {
      dq_count++;

      if ((head != NULL) && !(sq_count % 2) && !(dq_count % 2))
        {
        ptr++;

        ignchar = TRUE;
        }
      else 
        {
        ignore = TRUE;

        if (ign_backslash == TRUE)
          {
          /* check if backslash precedes delimiter */

          if ((ptr > start) && (*(ptr-1) == '\\'))
            {
            /* check if backslash is backslashed */

            if ((ptr > start + 1) && (*(ptr-2) != '\\'))
              {
              /* delimiter is backslashed, ignore */

              ignore = FALSE;
              
              dq_count--;
              }
            }
          }

        if (ignore == TRUE)
          {
          ptr++;

          ignchar = TRUE;
          }
        }
      }
    else if (*ptr == '[' )
      {
      sb_count = 1;
      }
    else if (*ptr == ']')
      {
      sb_count = 0;
      }
    else if (*ptr == '{')
      {
      sb_count = 1;
      }
    else if (*ptr == '}')
      {
      sb_count = 0;
      }
    else if (!(sq_count % 2) && !(dq_count % 2) && (sb_count == 0))
      {
      /* not in quotations, locate delimiter */

      for (dindex = 0; delims[dindex] != '\0'; dindex++)
        {
        if (*ptr != delims[dindex])
          continue;

        if ((ign_backslash == TRUE) && (head != NULL))
          {
          /* check if backslash precedes delimiter */
          if ((ptr > head) && (*(ptr-1) == '\\'))
            {
            /* check if backslash is backslashed */

            if ((ptr > head + 1) && (*(ptr-1) != '\\'))
              {
              /* delimiter is backslashed, ignore */

              continue;
              }
            }
          }

        /* delimiter found */

        *ptr = '\0';
        
        ptr++;
        
        if (head != NULL)
          {
          *ptrPtr = ptr;

          tmpLine[tindex] = '\0';
          
          if (tindex > 0)
            strcpy(head,tmpLine);
          
          free(tmpLine);

          return(head);
          }

        ignchar = TRUE;

        break;
        } /* END for (dindex) */
      }

    if ((ignchar != TRUE) && (*ptr != '\0'))
      {
      if (head == NULL)
        head = ptr;

      tmpLine[tindex++] = ptr[0];

      ptr++;
      }

    ignchar = FALSE;
    } /* END while (*ptr != '\0') */

  tmpLine[tindex] = '\0';

  if (tindex > 0)
    strcpy(head,tmpLine);

  free(tmpLine);

  *ptrPtr = ptr;

  return(head);
  } /* END smart_strtok */





int get_name_value(

  char  *start,
  char **name,
  char **value)

  {
  static char *tok_ptr;
  char *curr_ptr;
  char *equals;
  static char tmpLine[65536];

  /* we've reached the end */
  if ((start == NULL) && (*tok_ptr == '\0'))
    return(0);

  if (start != NULL)
    snprintf(tmpLine, sizeof(tmpLine), "%s", start);

  curr_ptr = smart_strtok(tmpLine,"",&tok_ptr,FALSE);

  if (curr_ptr == NULL)
    return(0);
     
  if ((*curr_ptr == '=') || 
      (*curr_ptr == '\0'))
    {
    /* no name, fail */
    return(-1);
    }

  /* skip leading spaces */
  while (isspace((int)*curr_ptr) && (*curr_ptr))
    curr_ptr++;

  *name = curr_ptr;

  equals = *name;

  /* skip over name */
  while ((*equals) && (!isspace((int)*equals)) && (*equals != '='))
    equals++;

  /* strip blanks */
  while ((*equals) && (isspace((int)*equals)))
    *equals++ = '\0';

  if (*equals != '=')
    return (-1); /* should have found a = as first non blank */

  *equals++ = '\0';

  /* skip leading white space */
  while (isspace((int)*equals) && *equals)
    equals++;

  if (*equals == '\0')
    return(-1);

  *value = equals;

  return (1);
  }

int isexecutable(
  char *s)  /* I */
  {
  char *c;

  c = s;

  if ((*c == ':') || ((*c == '#') && (*(c + 1) == '!')))
    {
    return(FALSE);
    }

  while (isspace(*c))
    c++;

  if (notNULL(c))
    {
    return(*c != '#');
    }

  return(FALSE);
  }




char *ispbsdir(

  char *s,
  char *prefix)

  {
  char *it;
  int l;

  it = s;

  while (isspace(*it)) it++;

  l = strlen(prefix);

  if ((l > 0) && (strncmp(it, prefix, l) == 0))
    {
    return(it + l);
    }

  return(NULL);
  }



/*
 * isWindowsFormat detects whether a file is written in a DOS/Windows format
 * or not. It returns 1 if it is, 0 otherwise.
*/
int isWindowsFormat(

  FILE   *fd)      /* I */ /* File handler containing the file to check */

  {
  size_t len;
  char buffer[MAXBUF];
  int dosformat = 0;

  if (fd == NULL)
    {
    return(1);
    }

  if (fd == stdin)
    {
    return(0);
    }

  fseek(fd, 0, SEEK_SET);

  /* Read a line of text and check for the return character */
  /* If found, we assume it's a Windows format */
  while(fgets(buffer, sizeof(buffer), fd) != NULL)
    {
    len = strlen(buffer);
    if (len < MAXBUF)
      if (buffer[len - 2] == '\r')
        {
        dosformat = 1;
        break;
        }
     }

  fseek(fd, 0, SEEK_SET);
  return dosformat;
  }


int istext(

  FILE   *fd,      /* I */
  int    *IsText)  /* O (optional) */

  {
  int i;
  unsigned char bf[MMAX_VERIFY_BYTES];
  int len;

  if (IsText != NULL)
    *IsText = FALSE;

  if (fd == NULL)
    {
    return(0);
    }

  if (fd == stdin)
    {
    return(1);
    }

  /* read first characters to ensure this is ASCII text */
  fseek(fd, 0, SEEK_SET);
  len = fread(bf,1,MMAX_VERIFY_BYTES,fd);
  fseek(fd, 0, SEEK_SET);
  if(len < 0)
    {
    return(0);
    }

  for (i = 0;i < len;i++)
    {
    if (!isprint(bf[i]) && !isspace(bf[i]))
      {
      return(0);
      }
    }  /* END for (i) */

  if (IsText != NULL)
    *IsText = TRUE;

  return(1);
  }  /* END FileIsText() */




/* PBS_Filter check has ben consolidated
 * The validity preference follows:
 * Config file
 * SUBMIT_FILTER_PATH
 * /usr/local/sbin/torque_submitfilter   -- legacy
 */
int validate_submit_filter(
  job_data_container *a_hash)

  {
  int rc = 0;
  job_data *filter_info = NULL;
  struct stat  sfilter;
  const char *DefaultFilterPath = "/usr/local/sbin/torque_submitfilter";

  hash_find(a_hash, ATTR_pbs_o_submit_filter, &filter_info);
  if ((filter_info != NULL) && (filter_info->var_type == CONFIG_DATA))
    {
    if (stat(filter_info->value.c_str(), &sfilter) != -1)
      {
      rc = 1;
      }
    else
      rc = -1;
    }
  else if (stat(SUBMIT_FILTER_PATH, &sfilter) != -1)
    {
    hash_add_or_exit(a_hash, ATTR_pbs_o_submit_filter, SUBMIT_FILTER_PATH, STATIC_DATA);
    rc = 1;
    }
  else if (stat(DefaultFilterPath, &sfilter) != -1)
    {
    hash_add_or_exit(a_hash, ATTR_pbs_o_submit_filter, DefaultFilterPath, STATIC_DATA);
    rc = 1;
    }
  else
    hash_del_item(a_hash, ATTR_pbs_o_submit_filter);
  return rc;
  } /* validate_submit_filter() */



void validate_pbs_o_workdir(

  job_data_container *job_attr)

  {
  job_data     *tmp_job_info = NULL;
  const char *the_val = NULL;
  char        null_val[] = "\0";
  char        tmp_dir[MAXPATHLEN] = {""};

  if (hash_find(job_attr, ATTR_init_work_dir, &tmp_job_info) == FALSE)
    {
    if (hash_find(job_attr, "PWD",  &tmp_job_info))
      the_val = tmp_job_info->value.c_str();
    else
      {
      char *the_dir = NULL;
      if ((the_dir = getcwd(tmp_dir, MAXPATHLEN)) != NULL)
        the_val = the_dir;
      else
        the_val = null_val;
      }
    }
  else
    the_val = tmp_job_info->value.c_str();

  hash_add_or_exit(job_attr, ATTR_pbs_o_workdir, the_val, ENV_DATA);
  hash_add_or_exit(job_attr, ATTR_init_work_dir, the_val, ENV_DATA);
  } /* END validate_pbs_o_workdir() */




/*
 * validate qsub_host, if not valid fail
 * validate pbs_host, if not valid assign qsub_host as pbs_host
 */
void validate_qsub_host_pbs_o_server(
  job_data_container *job_attr)

  {
  job_data     *tmp_job_info = NULL;
  const char  *qsub_host = NULL;
  char         tmp_host_name[PBS_MAXHOSTNAME];
  char         tmp_host_name_with_suffix[PBS_MAXHOSTNAME];

  /* check if QSUBHOST was entered in torque.cfg */
  if (hash_find(job_attr, ATTR_submit_host, &tmp_job_info))
    qsub_host = tmp_job_info->value.c_str();
  else if (gethostname(tmp_host_name, PBS_MAXHOSTNAME) == 0)
    qsub_host = tmp_host_name;

  if (host_name_suffix != NULL)
    {
    snprintf((char *)tmp_host_name_with_suffix, PBS_MAXHOSTNAME, "%s%s", qsub_host, host_name_suffix);
    qsub_host = tmp_host_name_with_suffix;
    }

  if (qsub_host)
    {
    if (get_fullhostname((char *)qsub_host, tmp_host_name, PBS_MAXHOSTNAME, NULL) == 0)
      {
      hash_add_or_exit(job_attr, ATTR_submit_host, tmp_host_name, LOGIC_DATA);
      hash_add_or_exit(job_attr, ATTR_pbs_o_host, tmp_host_name, LOGIC_DATA);
      qsub_host = tmp_host_name;
      }
    else
      qsub_host = NULL;
    }

  if (!qsub_host)
    {
    fprintf(stderr, "qsub: cannot get (full) local host name\n");
    exit(3);
    }
  if (hash_find(job_attr, ATTR_pbs_o_server, &tmp_job_info))
    {
    char tmp_val[PBS_MAXHOSTNAME];
    if (get_fullhostname((char *)tmp_job_info->value.c_str(), tmp_val, PBS_MAXHOSTNAME, NULL) == 0)
      hash_add_or_exit(job_attr, ATTR_pbs_o_server, tmp_val, LOGIC_DATA);
    else
      {
      fprintf(stderr,"qsub: cannot get full server host name\n");
      exit(3);
      }
    }
  else
    {
    char *tmp_host = pbs_default();
    if (tmp_host == '\0')
      hash_add_or_exit(job_attr, ATTR_pbs_o_server, qsub_host, LOGIC_DATA);
    else
      hash_add_or_exit(job_attr, ATTR_pbs_o_server, tmp_host, LOGIC_DATA);
    }
  } /* END validate_qsub_host_pbs_o_server() */


int are_mpp_present(

  job_data_container  *resources,
  job_data            **dummy)

  {
  int mpp_present = hash_find(resources, "mppwidth", dummy);

  return(mpp_present);
  } /* END are_mpp_present() */



bool is_resource_request_valid(

  job_info    *ji,
  std::string &err_msg)

  {
  job_data_container *resources = ji->res_attr;
  job_data           *dummy;
  bool               nodes = false;
  bool               size = false;
  bool               mpp = false;
  bool               ncpus = false;

  nodes = hash_find(resources, "nodes", &dummy);
  size  = hash_find(resources, "size", &dummy);
  ncpus  = hash_find(resources, "ncpus", &dummy);

  if (((nodes == true) &&
      ((size == true) ||
       (ncpus == true))) ||
      ((size == true) &&
       (ncpus == true)))
    {
    err_msg =  "qsub: Jobs may not mix -l nodes with -l size or -l ncpus\n";
    return(false);
    }
  else if ((nodes == true) ||
           (size == true) ||
           (ncpus == true))
    {
    mpp = are_mpp_present(resources, &dummy);

    if (mpp == true)
      {
      if (nodes == true)
        {
        err_msg = "qsub: Specifying -l nodes is incompatible with specifying -l mppwidth\n";
        return(false);
        }
      else if (size == TRUE)
        {
        err_msg = "qsub: Specifying -l size is incompatible with specifying -l mppwidth\n";
        return(false);
        }
      else
        {
        err_msg = "qsub: Specifying -l ncpus is incompatible with specifying -l mppwidth\n";
        return(false);
        }
      }
    }

  // If req_count is > 0 that means -L was requested
  if (cr.req_count() > 0)
    {
    if ((nodes == true) ||
        (size == true) ||
        (mpp == true))
      {
      err_msg = "qsub: resource requests cannot combine -L with -l nodes, size, or mppwidth\n";
      return(false);
      }

    if ((hash_find(resources, "mem", &dummy)) ||
        (hash_find(resources, "hostlist", &dummy)) ||
        (hash_find(resources, "ncpus", &dummy)) ||
        (hash_find(resources, "procs", &dummy)) ||
        (hash_find(resources, "pvmem", &dummy)) ||
        (hash_find(resources, "pmem", &dummy)) ||
        (hash_find(resources, "vmem", &dummy)) ||
        (hash_find(resources, "reqattr", &dummy)) ||
        (hash_find(resources, "software", &dummy)) ||
        (hash_find(resources, "geometry", &dummy)) ||
        (hash_find(resources, "opsys", &dummy)) ||
        (hash_find(resources, "tpn", &dummy)) ||
        (hash_find(resources, "trl", &dummy)))
      {
      err_msg = "qsub: resource requests cannot combine -L with -l memory, gres, geometry, opsys, reqattr, hostlist, or proc count requests\n";
      return(false);
      }
    }

  return(true);
  } // is_resource_request_valid()



void validate_basic_resourcing(

  job_info *ji)

  {
  std::string err_msg;

  if (is_resource_request_valid(ji, err_msg) == false)
    {
    fprintf(stderr, "%s", err_msg.c_str());
    exit(4);
    }

  } /* END validate_basic_resourcing() */



void validate_array_options(

  job_info *ji)

  {
  job_data *dummy;

  if ((hash_find(ji->job_attr, ATTR_t, &dummy) == FALSE) &&
      (hash_find(ji->job_attr, ATTR_idle_slot_limit, &dummy)))
    {
    fprintf(stderr, "qsub: the idle array slot limit (-i) can only be applied to array jobs (-t)\n");
    exit(4);
    }
  } // END validate_basic_resourcing()



/*
 * Set up (or enforce) errpath or outpath when join option specified
 * so that qstat will diplay it properly.
 */
void validate_join_options (
  job_data_container *job_attr,
  char               * UNUSED(script_tmp))

  {

  job_data         *tmp_job_info = NULL;

  const char      *j_attr_value = NULL;
  const char      *o_attr_value = NULL;
  const char      *e_attr_value = NULL;

  /* obtain j, e, and o option values if they exist for further processing */
  if (hash_find(job_attr, ATTR_j, &tmp_job_info))
    {
    j_attr_value = tmp_job_info->value.c_str();
    }

  /* check needed only if j option is specified */
  if (j_attr_value != NULL)
    {
    if (hash_find(job_attr, ATTR_o, &tmp_job_info))
      {
      o_attr_value = tmp_job_info->value.c_str();
      }

    if (hash_find(job_attr, ATTR_e, &tmp_job_info))
      {
      e_attr_value = tmp_job_info->value.c_str();
      }

    if (strcmp(j_attr_value, "oe") == 0)
      {
      /* copy request outpath to errpath so that qstat displays errpath correctly */
      if (o_attr_value != NULL)
        {
        hash_add_or_exit(job_attr, ATTR_e, o_attr_value, CMDLINE_DATA);
        }
      }
    else if (strcmp(j_attr_value, "eo") == 0)
      {
      /* copy request errpath to outpath so that qstat displays outpath correctly */
      if (e_attr_value != NULL)
        {
        hash_add_or_exit(job_attr, ATTR_o, e_attr_value, CMDLINE_DATA);
        }
      }
    }
  }


void post_check_attributes(job_info *ji, char *script_tmp)
  {
  validate_pbs_o_workdir(ji->job_attr);
  validate_qsub_host_pbs_o_server(ji->job_attr);
  validate_basic_resourcing(ji);

  /* Make sure -j and -e or -o options are compatible so qstat will properly
   * display the outpath and errpath for the job.
   *
   * Fix for TRQ-1839 (job does not have matching output and errpath when
   * -j oe (or eo) specified.)
   */
  validate_join_options(ji->job_attr, script_tmp);

  validate_array_options(ji);
  } /* END post_check_attributes() */



/*
 * add_new_request_if_present()
 *
 * adds the -L request if it exists
 */

void add_new_request_if_present(

  job_info *ji)

  {
  if (cr.req_count() > 0)
    {
    std::string req_str;
    cr.toString(req_str);
    hash_add_or_exit(ji->job_attr, ATTR_req_information, req_str.c_str(), CMDLINE_DATA);
    }
  } // END add_new_request_if_present() 




/* return 3, 4, 5, 6, -1 on FAILURE, 0 on success */

static int get_script(

  int        ArgC,     /* I */
  char     **ArgV,     /* I */
  FILE      *file,     /* I */
  char      *script,   /* O (minsize=X) */
  job_info  *ji)       /* M */

  {
  char  s[MAX_LINE_LEN + 1];
  char *sopt;
  int   exec = FALSE;
  char *cont;
  char  tmp_name[] = "/tmp/qsub.XXXXXX";
  FILE *TMP_FILE;
  char *in;
  int   tmpfd;

  int   index;

  /* START WRAPPER */
  std::string cfilter;

  char tmp_name2[] = "/tmp/qsub.XXXXXX";

  FILE        *filesaved;
  FILE        *filter_pipe;
  int          rc;
  job_data    *tmp_job_info = NULL;
  bool         directive_prefix_on = false;

  /* if the submit_filter exists, run it.                               */

  /* check that the file is text */

  if (istext(file, NULL) == 0)
    {
    fprintf(stderr,
            "qsub:  file must be an ascii script\n");

    return(4);
    }

  if (isWindowsFormat(file))
    {
    fprintf(stderr,
            "qsub:  script is written in DOS/Windows text format\n");

    return(4);
    }
  if (hash_find(ji->job_attr, ATTR_pbs_o_submit_filter, &tmp_job_info))
    {
    /* run the copy through the submit filter. */
    if ((tmpfd = mkstemp(tmp_name2)) < 0)
      {
      fprintf(stderr,
              "qsub: could not create filter o/p %s\n",
              tmp_name2);

      return(4);
      }

    close(tmpfd);

    cfilter = tmp_job_info->value;

    for (index = 1;index < ArgC;index++)
      {
      if (ArgV[index] != NULL)
        {
        cfilter += " ";

        /* This is ugly. But we have to escape the '#' character
           of a -C directive prefix otherwise scripts interpret this
           as a comment and the rest of the qsub line is deleted */
        if (directive_prefix_on == true)
          {
          char directive_prefix[PBS_MAXHOSTNAME];

          memset(directive_prefix, 0, PBS_MAXHOSTNAME);
          directive_prefix[0] = '\\';
          strcat(directive_prefix, ArgV[index]);
          directive_prefix_on = false;
          continue;
          }

        if (!strcmp(ArgV[index], "-C"))
          {
          directive_prefix_on = true;
          }

        cfilter += ArgV[index];
        }
      }    /* END for (index) */

    cfilter += " >";
    cfilter += tmp_name2;

    if ((filter_pipe = popen(cfilter.c_str(), "w")) != NULL)
      {
      while ((in = fgets(s, MAX_LINE_LEN, file)) != NULL)
        {
        if (fputs(in, filter_pipe) < 0)
          {
          fprintf(stderr, "qsub: error writing to filter stdin\n");

          pclose(filter_pipe);
          unlink(tmp_name2);

          return(3);
          }
        }

      rc = pclose(filter_pipe);
      }
    else
      {
      rc = -1;
      }

    if (WEXITSTATUS(rc) == (unsigned char)SUBMIT_FILTER_ADMIN_REJECT_CODE)
      {
      fprintf(stderr, "qsub: Your job has been administratively rejected by the queueing system.\n");
      fprintf(stderr, "qsub: There may be a more detailed explanation prior to this notice.\n");

      unlink(tmp_name2);

      return(3);
      }

    if (WEXITSTATUS(rc))
      {
      fprintf(stderr, "qsub: submit filter returned an error code, aborting job submission.\n");

      unlink(tmp_name2);

      return(3);
      }

    /* get rid of the i/p copy. */

    /* preserve the original pointer. */

    filesaved = file;

    /* open the filtered script. */

    if ((file = fopen(tmp_name2, "r")) == NULL)
      {
      fprintf(stderr, "qsub: could not open filter o/p %s\n",
              tmp_name2);

      unlink(tmp_name2);

      file = filesaved;

      return(3);
      }

    /* Get rid of the filtered o/p; data remains accessible until    */
    /* file is closed.                                               */

    unlink(tmp_name2);

    /* Complete redirection.                                         */

    fclose(filesaved);
    }  /* END if (stat(PBS_Filter,&sfilter) != -1) */

  /* END WRAPPER */

  if ((tmpfd = mkstemp(tmp_name)) < 0)
    {
    fprintf(stderr, "qsub: could not create copy of script %s - %s\n",
            tmp_name, strerror(errno));

    return(4);
    }

  if ((TMP_FILE = fdopen(tmpfd, "w+")) == NULL)
    {
    fprintf(stderr, "qsub: could not create copy of script %s\n",
            tmp_name);

    unlink(tmp_name);

    return(4);
    }

  hash_find(ji->client_attr, "pbs_dprefix", &tmp_job_info);
  while ((in = fgets(s, MAX_LINE_LEN, file)) != NULL)
    {
    int len;

    /* replace DOS EOL ('^M') characters */

    len = strlen(in);

    if ((len >= 2) && (in[len - 2] == '\r') && (in[len - 1] == '\n'))
      {
      in[len - 2] = '\n';
      in[len - 1] = '\0';
      }

    if (!exec && ((sopt = ispbsdir(s, (char *)tmp_job_info->value.c_str())) != NULL))
      {
      while ((*(cont = in + strlen(in) - 2) == '\\') && (*(cont + 1) == '\n'))
        {
        /* next line is continuation of this line */

        *cont = '\0';  /* clear newline from our copy */

        if (fputs(in, TMP_FILE) < 0)
          {
          fprintf(stderr, "qsub: error writing copy of script, %s\n",
                  tmp_name);

          fclose(TMP_FILE);

          unlink(tmp_name);

          return(3);
          }

        in = cont;

        if ((in = fgets(in, MAX_LINE_LEN - (in - s), file)) == NULL)
          {
          fprintf(stderr, "qsub: unexpected end-of-file or read error in script\n");

          fclose(TMP_FILE);

          unlink(tmp_name);

          return(6);
          }
        }    /* END while ((*(cont = in + strlen(in) - 2) == '\\') && (*(cont + 1) == '\n')) */

      do_dir(sopt, ji, SCRIPT_DATA);
/*        {
        return(-1);
        }
        */
      }      /* END if (!exec && ((sopt = ispbsdir(s,prefix)) != NULL)) */
    else if (!exec && isexecutable(s))
      {
      exec = TRUE;
      }

    if (fputs(in, TMP_FILE) < 0)
      {
      fprintf(stderr, "qsub: error writing copy of script, %s\n",
              tmp_name);

      fclose(TMP_FILE);

      unlink(tmp_name);

      return(3);
      }
    }   /* END while ((in = fgets(s,MAX_LINE_LEN,file)) != NULL) */

  fclose(TMP_FILE);

  if (ferror(file))
    {
    fprintf(stderr, "qsub: error reading script file\n");

    return(5);
    }

  strcpy(script, tmp_name);

  return(0);
  }  /* END get_script() */



/**
 * Parse given character string in shell format to arc/argv format adding "qsub" as a first argument
 * making it look like qsub was called from command line.
 *
 * This function handles quotes and escape symbols as shell does:
 *  - arguments are splitted by any number of space characters as defined by isspace()
 *  - unescaped quotes enclose a single argument or its part. Quotes are dropped.
 *  - escaped characters treated as is: escaped quote and spaces are treated as a character without
 *    any special meaning
 */
void make_argv(

  int        *argc,   /* O - result argc value */
  char       *argv[], /* O - result argv value */
  char const *line)   /* I - input cmdline shell string */

  {
  char const *l, *c;
  char *b, *buffer;
  int len;
  char quote;

  buffer = (char *)calloc(1, strlen(line) + 1);

  if (buffer == NULL)
    {
    fprintf(stderr, "qsub: out of memory\n");

    exit(2);
    }

  *argc = 0;

  argv[(*argc)++] = (char *)"qsub";

  l = line;
  b = buffer;

  while (isspace(*l))
    l++;

  c = l;

  while (*c != '\0')
    {
    if ((*c == '"') || (*c == '\''))
      {
      quote = *c;
      /* don't include the quotes */
      c++;

      while ((*c != quote) && *c)
        *b++ = *c++;

      if (*c == '\0')
        {
        fprintf(stderr, "qsub: unmatched %c\n",
                *c);

        exit(1);
        }

      /* don't include the quotes */
      c++;
      }
    else if (*c == '\\')
      {
      c++;

      *b++ = *c++;
      }
    else if (isspace(*c))
      {
      len = c - l;

      assert(len > 0);

      if (argv[*argc] != NULL)
        free(argv[*argc]);

      argv[*argc] = (char *)calloc(1, len + 1);

      if (argv[*argc] == NULL)
        {
        fprintf(stderr, "qsub: out of memory\n");

        exit(2);
        }

      *b = '\0';

      strcpy(argv[(*argc)++], buffer);

      while (isspace(*c))
        c++;

      l = c;

      b = buffer;
      }
    else
      {
      *b++ = *c++;
      }
    }

  if (c != l)
    {
    len = c - l;

    assert(len > 0);

    if (argv[*argc] != NULL)
      free(argv[*argc]);

    argv[*argc] = (char *) calloc(1, len + 1);

    if (argv[*argc] == NULL)
      {
      fprintf(stderr, "qsub: out of memory\n");
      exit(2);
      }

    *b = '\0';

    strcpy(argv[(*argc)++], buffer);
    }

  free(buffer);

  return;
  }  /* END make_argv() */




void do_dir(

  char *opts,
  job_info *ji,
  int data_type)

  {
  int argc = 0;

  /* initialize all vect pointers with NULLs.
   * make_argv() frees an item if it's non-NULL and then allocates memory for a new one.
   */
  static char *vect[MAX_ARGV_LEN + 1] = {};

  make_argv(&argc, vect, opts);

  process_opts(argc, vect, ji, data_type);
  }  /* END do_dir() */


/*
 * The following bunch of functions support the "Interactive Job"
 * capability of PBS.
 */
/*
 * interactive_port - get a socket to listen to for "interactive" job
 * When the "interactive" job is run, its standard in, out, and error
 * will be connected to this socket.
 */
char *interactive_port(
    
  int *sock)

  {
  torque_socklen_t namelen;
  static char portstring[8];

  struct sockaddr_in myaddr;
  unsigned short port;

  *sock = socket(AF_INET, SOCK_STREAM, 0);

  if (*sock < 0)
    print_qsub_usage_exit("qsub: unable to obtain socket");

  namelen = sizeof(myaddr);

  memset(&myaddr, 0, sizeof(myaddr));
  myaddr.sin_family = AF_INET;
  myaddr.sin_addr.s_addr = INADDR_ANY;

  if (bind(*sock, (struct sockaddr *)&myaddr, namelen) < 0)
    print_qsub_usage_exit("qsub: unable to bind to socket");

  /* get port number assigned */

  if (getsockname(*sock, (struct sockaddr *)&myaddr, &namelen) < 0)
    print_qsub_usage_exit("qsub: unable to get port number");

  port = ntohs(myaddr.sin_port);

  sprintf(portstring, "%u",
          (unsigned int)port);

  if (listen(*sock, 1) < 0)
    print_qsub_usage_exit("qsub: listen on interactive socket");

  return(portstring);
  }  /* END interactive_port() */

/*
 * settermraw - set terminal into "raw" mode
 */
void settermraw(

  struct termios *ptio)

  {

  struct termios tio;

  tio = *ptio;

  tio.c_lflag &= ~(ICANON | ISIG | ECHO | ECHOE | ECHOK);
  tio.c_iflag &= ~(IGNBRK | INLCR | ICRNL | IXON | IXOFF);
  tio.c_oflag = 0;
  tio.c_oflag |= (OPOST); /* TAB3 */
  tio.c_cc[VMIN] = 1;
  tio.c_cc[VTIME] = 0;

#if defined(TABDLY) && defined(TAB3)

  if ((tio.c_oflag & TABDLY) == TAB3)
    tio.c_oflag &= ~TABDLY;

#endif

  tio.c_cc[VKILL]  = -1;

  tio.c_cc[VERASE] = -1;

  if (tcsetattr(0, TCSANOW, &tio) < 0)
    perror("qsub: set terminal mode");

  return;
  }  /* END settermraw() */

/*
 * stopme - suspend process on ~^Z or ~^Y
 * on suspend, reset terminal to normal "cooked" mode;
 * when resumed, again set terminal to raw.
 */
void stopme(

  pid_t p)  /* pid of 0 (process group) or just myself (writer) */

  {
  tcsetattr(0, TCSANOW, &oldtio); /* reset terminal */

  kill(p, SIGTSTP);

  settermraw(&oldtio);          /* back to raw when we resume */

  return;
  }

/*
 * Interactive Reader process: reads from the remote socket,
 *      and writes that out to the stdout
 */
int reader(

  int s, /* I - reading socket */
  int d) /* I - writing socket */

  {
  char  buf[4096];
  int   c;
  char *p;
  int   wc;

  /* read from the socket, and write to d */

  /* NOTE:  s should be blocking */

  while (1)
    {
    c = read_ac_socket(s, buf, sizeof(buf));

    if (c > 0)
      {
      p = buf;

      while (c)
        {
        if ((wc = write_ac_socket(d, p, c)) < 0)
          {
          if (errno == EINTR)
            {
            continue;
            }

          perror("qsub: write error");

          return(-1);
          }

        c -= wc;

        p += wc;
        }
      }
    else if (c == 0)
      {
      return(0);  /* EOF - all done */
      }
    else
      {
      if (errno == EINTR)
        continue;

      if (errno == EAGAIN)
        {
        sleep(1);

        continue;
        }

      perror("qsub: read error");

      return(-1);
      }
    }    /* END while (1) */

  return(0);
  }  /* END reader() */

/*
 * Writer process: reads from stdin, and writes
 * data out to the rem socket
 */
void writer(

  int s,  /* writing socket */
  int d)  /* reader socket */

  {
  char c;
  int i;
  int newline = 1;
  char tilde = '~';
  int wi;

  /* read from stdin, and write to the socket */

  while (1)
    {
    i = read_ac_socket(d, &c, 1);

    if (i > 0)
      {
      /* read data */

      if (newline)
        {
        if (c == tilde)
          {
          /* maybe escape character */

          /* read next character to check */

          while ((i = read_ac_socket(d, &c, 1)) != 1)
            {
            if ((i == -1) && (errno == EINTR))
              continue;

            break;
            }

          if (i != 1)
            break;

          if (c == '.') /* termination character */
            break;

          if (c == oldtio.c_cc[VSUSP])
            {
            stopme(0); /* ^Z suspend all */

            continue;
#ifdef VDSUSP
            }
          else if (c == oldtio.c_cc[VDSUSP])
            {
            stopme(getpid());

            continue;
#endif /* VDSUSP */
            }
          else
            {
            /* not escape, write out tilde */

            while ((wi = write_ac_socket(s, &tilde, 1)) != 1)
              {
              if ((wi == -1) && (errno == EINTR))
                continue;

              break;
              }

            if (wi != 1)
              break;
            }
          }

        newline = 0;   /* no longer at start of line */
        }
      else
        {
        /* reset to newline if \n \r kill or interrupt */

        newline = (c == '\n') ||
                  (c == oldtio.c_cc[VKILL]) ||
                  (c == oldtio.c_cc[VINTR]) ||
                  (c == '\r');
        }

      while ((wi = write_ac_socket(s, &c, 1)) != 1)
        {
        /* write out character */

        if ((wi == -1) && (errno == EINTR))
          continue;

        break;
        }

      if (wi != 1)
        break;
      }
    else if (i == 0)
      {
      /* EOF */

      break;
      }
    else if (i < 0)
      {
      /* error */

      if (errno == EAGAIN)
        {
        sleep(1);

        continue;
        }

      if (errno == EAGAIN)
        {
        sleep(1);

        continue;
        }

      if (errno != EINTR)
        {
        perror("qsub: read error");

        return;
        }
      }
    }  /* END while (1) */

  return;
  }  /* END writer() */

/*
 * getwinsize - get the current window size
 */
int getwinsize(

  struct winsize *wsz)

  {
  if (ioctl(0, TIOCGWINSZ, wsz) < 0)
    {
    perror("qsub: unable to get window size");

    return(-1);
    }

  return(0);
  }

/*
 * send_winsize = send the current tty's window size
 */
void send_winsize(

  int             sock,
  struct winsize *wsz)

  {
  char  buf[PBS_TERM_BUF_SZ];

  sprintf(buf, "WINSIZE %hu,%hu,%hu,%hu",
          wsz->ws_row,
          wsz->ws_col,
          wsz->ws_xpixel,
          wsz->ws_ypixel);

  if (write_ac_socket(sock, buf, PBS_TERM_BUF_SZ) != PBS_TERM_BUF_SZ)
    print_qsub_usage_exit("qsub: sending winsize");

  return;
  }




/*
 * send_term - send the current TERM type and certain control characters
 */
void send_term(

  int sock)

  {
  char  buf[PBS_TERM_BUF_SZ];
  char *term;
  char  cc_array[PBS_TERM_CCA];

  strcpy(buf, "TERM=");

  term = getenv("TERM");

  if (term == NULL)
    strcat(buf, "unknown");
  else
    safe_strncat(buf, term, PBS_TERM_BUF_SZ - 5);

  if (write_ac_socket(sock, buf, PBS_TERM_BUF_SZ) != PBS_TERM_BUF_SZ)
    print_qsub_usage_exit("qsub: sending term type");

  cc_array[0] = oldtio.c_cc[VINTR];

  cc_array[1] = oldtio.c_cc[VQUIT];
  cc_array[2] = oldtio.c_cc[VERASE];
  cc_array[3] = oldtio.c_cc[VKILL];
  cc_array[4] = oldtio.c_cc[VEOF];
  cc_array[5] = oldtio.c_cc[VSUSP];

  if (write_ac_socket(sock, cc_array, PBS_TERM_CCA) != PBS_TERM_CCA)
    print_qsub_usage_exit("qsub: sending term options");

  return;
  }




/*
 * catchchild = signal handler for Death of Child
 */

void catchchild(

  int UNUSED(sig) )

  {
  int status;
  int pid;

  while (1)
    {
    pid = waitpid(-1, &status, WNOHANG | WUNTRACED);

    if (pid == 0)
      {
      return;
      }

    if ((pid > 0) && (WIFSTOPPED(status) == 0))
      break;

    if ((pid == -1) && (errno != EINTR))
      {
      perror("qsub: bad status in catchchild: ");

      return;
      }
    }

  if (interactivechild > 0)
    kill(interactivechild, SIGTERM);

  if (x11child > 0)
    kill(x11child, SIGTERM);

  /* reset terminal to cooked mode */

  if (have_terminal)
    tcsetattr(0, TCSANOW, &oldtio);

  exit(0);

  /*NOTREACHED*/

  return;
  }  /* END catchchild() */




void no_suspend(

  int UNUSED(sig))

  {
  fprintf(stderr, "Sorry, you cannot suspend qsub until the job is started\n");

  fflush(stderr);
  }





/* does not return */

void bailout(void)

  {
  int c;
  int local_errno = 0;

  shutdown(inter_sock, 2);

  close(inter_sock);

  printf("Job %s is being deleted\n",
         new_jobname);

  c = cnt2server(server_out);

  if (c <= 0)
    {
    if (server_out[0] != 0)
      fprintf(stderr, "qsub: cannot connect to server %s (errno=%d) %s\n",
            server_out,
            c * -1,
            pbs_strerror(c * -1));
    else
      fprintf(stderr, "qsub: cannot connect to server %s (errno=%d) %s\n",
            pbs_server,
            c * -1,
            pbs_strerror(c * -1));


    fprintf(stderr, "qsub: pbs_server daemon may not be running on host %s or hostname in file '$TORQUEHOME/server_name' may be incorrect)\n", pbs_server);

    exit(1);
    }

  pbs_deljob_err(c, new_jobname, NULL, &local_errno);

  pbs_disconnect(c);

  exit(0);
  }



void toolong(

  int UNUSED(sig))

  {
  printf("Timeout -- deleting job\n");

  bailout();

  /*NOTREACHED*/

  exit(0);
  }



void catchint(

  int UNUSED(sig))

  {
  int c;

  printf("Do you wish to terminate the job and exit (y|[n])? ");

  fflush(stdout);

  while (1)
    {
    alarm(60); /* give a minute to think about it */

    c = getchar();

    if ((c == 'n') || (c == 'N') || (c == '\n'))
      break;

    if ((c == 'y') || (c == 'Y'))
      {
      bailout();

      /*NOTREACHED*/

      exit(0);
      }

    if (printf("yes or no please\n") < 0)
      {
      /* terminal probably went away */
      bailout();

      /*NOTREACHED*/

      exit(0);
      }

    while ((c != '\n') && (c != EOF))
      c = getchar();
    }  /* END while (1) */

  alarm(0);  /* reset alarm */

  while ((c != '\n') && (c != EOF))
    c = getchar();

  return;
  }  /* END catchint() */



void x11handler(

  int      param_sock)

  {
  struct pfwdsock *socks;
  int              n;
  char            *display;

  calloc_or_fail((char **)&socks, sizeof(struct pfwdsock) * NUM_SOCKS, "x11handler");

  for (n = 0;n < NUM_SOCKS;n++)
    (socks + n)->active = 0;

  (socks + 0)->sock = param_sock;

  (socks + 0)->active = 1;

  (socks + 0)->listening = 1;

  /* Try to open a socket for the local X server. */
  display = getenv("DISPLAY");

  if (!display)
    {
    fprintf(stderr, "DISPLAY not set.");
    free(socks);
    return;
    }

  port_forwarder(socks, x11_connect_display, display, 0, NULL);

  exit(EXIT_FAILURE);
  }



/*
 * interactive - set up for interactive communication with job
 */

void interactive(
  job_data_container  *client_attr)

  {
  int  amt;
  char cur_server[PBS_MAXSERVERNAME + PBS_MAXPORTNUM + 2];

  char momjobid[LOG_BUF_SIZE+1];
  int  news;
  int  nsel;
  char *pc;
  fd_set selset;

  struct sigaction act;

  struct sockaddr_in from;
  torque_socklen_t fromlen;

  struct timeval timeout;

  struct winsize wsz;
  job_data *tmp_job_info;

  /* Catch SIGINT and SIGTERM, and */
  /* setup to catch Death of child */

  sigemptyset(&act.sa_mask);
  act.sa_handler = catchint;
  act.sa_flags   = 0;

  if ((sigaction(SIGINT, &act, (struct sigaction *)0) < 0) ||
      (sigaction(SIGTERM, &act, (struct sigaction *)0) < 0))
    print_qsub_usage_exit("qsub: unable to catch signals");

  act.sa_handler = toolong;

  if ((sigaction(SIGALRM, &act, NULL) < 0))
    print_qsub_usage_exit("qsub: cannot catch alarm");

  /* save the old terminal setting */

  if (have_terminal && tcgetattr(0, &oldtio) < 0)
    print_qsub_usage_exit("qsub: unable to get terminal settings");

  /* Get the current window size, to be sent to MOM later */

  if (!have_terminal || getwinsize(&wsz))
    {
    wsz.ws_row = 20; /* unable to get actual values */
    wsz.ws_col = 80; /* set defaults   */
    wsz.ws_xpixel = 0;
    wsz.ws_ypixel = 0;
    }


  printf("qsub: waiting for job %s to start\n",

         new_jobname);

  /* Accept connection on socket set up earlier */

  nsel = 0;

  while (nsel == 0)
    {
    FD_ZERO(&selset);
    FD_SET(inter_sock, &selset);

    timeout.tv_usec = 0;
    timeout.tv_sec  = 30;

    nsel = select(FD_SETSIZE, &selset, NULL, NULL, &timeout);

    if (nsel > 0)
      {
      break;
      }
    else if (nsel == -1)
      {
      if (errno == EINTR)
        {
        nsel = 0;
        }
      else
        print_qsub_usage_exit("qsub: select failed");
      }

    /* connect to server, status job to see if still there */

    if (!locate_job(new_jobname, server_out, cur_server))
      {
      fprintf(stderr, "qsub: job %s apparently deleted\n",
              new_jobname);

      exit(1);
      }
    }

  /* apparently someone is attempting to connect to us */

  fromlen = sizeof(from);

  if ((news = accept(inter_sock, (struct sockaddr *) & from, &fromlen)) < 0)
    print_qsub_usage_exit("qsub: accept error");

  /* When MOM connects, she will send the job id for us to verify */

  amt = LOG_BUF_SIZE + 1;

  pc = momjobid;

  while (amt > 0)
    {
    fromlen = read_ac_socket(news, pc, amt);

    if (fromlen <= 0)
      break;

    pc += fromlen;

    if (*(pc - 1) == '\0')
      break;

    amt -= fromlen;
    }

  if (strncmp(momjobid, "PBS:", 4) == 0)
    {
    fprintf(stderr, "qsub: %s\n", momjobid);

    shutdown(news, 2);

    exit(1);
    }

  if (strncmp(momjobid, new_jobname, PBS_MAXSVRJOBID) != 0)
    {
    fprintf(stderr, "qsub: invalid job name from execution server\n");

    shutdown(news, 2);

    exit(1);
    }

  /*
   * got the right job, send:
   *  terminal type as "TERM=xxxx"
   *  window size as   "WINSIZE=r,c,x,y"
   */

  send_term(news);

  send_winsize(news, &wsz);

  printf("qsub: job %s ready\n\n",
         new_jobname);

  /* set SIGINT, SIGTERM processing to ignore */

  act.sa_handler = SIG_IGN;

  if ((sigaction(SIGINT, &act, (struct sigaction *)0) < 0)  ||
      (sigaction(SIGTERM, &act, (struct sigaction *)0) < 0) ||
      (sigaction(SIGALRM, &act, (struct sigaction *)0) < 0) ||
      (sigaction(SIGTSTP, &act, (struct sigaction *)0) < 0))
    print_qsub_usage_exit("unable to reset signals");

  fflush(NULL);

  interactivechild = fork();

  if (interactivechild == 0)
    {
    /*
     * child process - start the reader function
     *     set terminal into raw mode
     */

    if (have_terminal)
      settermraw(&oldtio);

    reader(news, fileno(stdout));

    /* reset terminal */

    if (have_terminal)
      tcsetattr(0, TCSANOW, &oldtio);

    printf("\nqsub: job %s completed\n",
           new_jobname);

    exit(0);
    }
  else if (interactivechild > 0)
    {
    /* parent - start the writer function */

    act.sa_handler = catchchild;

    if (sigaction(SIGCHLD, &act, (struct sigaction *)0) < 0)
      {
      exit(1);
      }

    if (hash_find(client_attr, "DISPLAY", &tmp_job_info))
      {
      if ((x11child = fork()) == 0)
        {
        act.sa_handler = SIG_DFL;

        sigaction(SIGTERM, &act, (struct sigaction *)0);

        x11handler(inter_sock);
        }
      }

    writer(news, fileno(stdin));

    /* all done - make sure reader child is gone and reset terminal */

    if (interactivechild > 0)
      kill(interactivechild, SIGTERM);

    if (x11child > 0)
      kill(x11child, SIGTERM);

    shutdown(inter_sock, SHUT_RDWR);

    close(inter_sock);

    if (have_terminal)
      tcsetattr(0, TCSANOW, &oldtio);

    exit(0);
    }
  else
    print_qsub_usage_exit("qsub: unable to fork");

  return;
  }  /* END interactive() */




int validate_group_list(

  char *glist)

  {
  /* check each group to determine if it is a valid group that the user can be a part of.
   * group list is of the form group[@host][,group[@host]...] */
  char           *buf = NULL;
  char           *groups = strdup(glist);
  const char     *delims = ",";
  char           *tmp_group = strtok(groups, delims); 
  char           *at;
  char           *u_name;
  char          **pmem;
  struct group   *grent;
  struct passwd  *pwent;

  if ((pwent = getpwuid(getuid())) == NULL)
    {
    free(groups);
    return(FALSE);
    }

  u_name = pwent->pw_name;
  
  while (tmp_group != NULL)
    {
    if ((at = strchr(tmp_group,'@')) != NULL)
      *at = '\0';
    
    if ((grent = getgrnam_ext(&buf, tmp_group)) == NULL)
      {
      free(groups);
      return(FALSE);
      }
    
    pmem = grent->gr_mem;
    free_grname(grent, buf);
    
    if (pmem == NULL)
      {
      free(groups);
      return(FALSE);
      }
    
    while (*pmem != NULL)
      {
      if (!strcmp(*pmem,u_name))
        break;

      pmem++;
      }

    if (*pmem == NULL)
      {
      /* match not found */
      free(groups);
      return(FALSE);
      }

    tmp_group = strtok(NULL,delims);
    }
      
  free(groups);

  return(TRUE);
  }


bool came_from_moab(
    
  const char  *src,
  std::string &escaped_semicolon)

  {
  char *p;
  if ((p = strstr((char *)src, "x=SID:Moab;")))
    {  
    char  buf[1024];
    char *s;

    for (s=buf; *p; p++, s++)
      {
      if (*p == ';')
        {
        *s = '\\';
        s++;
        }
      *s = *p;
      }

    *s = '\0';

    escaped_semicolon = std::string(buf);
    return true;
    }
  else
    return false;
  }



/*
 * process_opt_L()
 *
 * Verifies and adds the argument passed to -L
 * @param cmd_arg - the command line argument passed to qsub
 * @return PBSE_NONE if good, -1 otherwise
 */

void process_opt_L(

  const char *cmd_arg)

  {
  char        err_buf[MAXLINE*2];

  if (strncmp(cmd_arg, "tasks=", 6))
    {
    snprintf(err_buf, sizeof(err_buf), "qsub: illegal -L value: '%s'", cmd_arg);
    print_qsub_usage_exit(err_buf);
    }

  // check for errors
  char        *req_begin = strdup(cmd_arg + 6); // skip the 'tasks=' portion
  req          r;
  std::string  err;

  if (r.set_from_submission_string(req_begin, err) != PBSE_NONE)
    {
    snprintf(err_buf, sizeof(err_buf), "qsub: malformed piece of -L value: '%s'", err.c_str());
    print_qsub_usage_exit(err_buf);
    }

  if ((cr.req_count() > 0) &&
      (r.cgroup_preference_set() == true))
    {
    snprintf(err_buf, sizeof(err_buf),
      "qsub: cgroup_per_task (cpt) and cgroup_per_host (cph) may only be specified for the first req");
    print_qsub_usage_exit(err_buf);
    }

  cr.add_req(r);
  } // END process_opt_L()



/*
 * process_opt_d()
 *
 * Verifies and adds the argument passed to -d
 * @param ji - where we store the job information
 * @param cmd_arg - the command line argument passed to qsub
 * @param data_type - the source of this argument 
 * @param tmp_job_info - the source for information parsed from torque.cfg
 * @return PBSE_NONE if good, -1 otherwise
 */

int process_opt_d(

  job_info   *ji,
  const char *cmd_arg,
  int         data_type,
  job_data   *tmp_job_info)

  {
  if (cmd_arg == NULL)
    return(-1);
  else
    {
    if (cmd_arg[0] == '/')
      hash_add_or_exit(ji->job_attr, ATTR_pbs_o_initdir, cmd_arg, data_type);
    else
      {
      /* make '-d' relative to current directory, not $HOME */
      char tmpPWD[1024];
      char *mypwd;

      mypwd = getcwd(tmpPWD, sizeof(tmpPWD));

      if (mypwd == NULL)
        {
        char  err_buf[MAXLINE*2];
        snprintf(err_buf, sizeof(err_buf), "qsub: unable to get cwd: %d (%s)",
            errno, strerror(errno));
        print_qsub_usage_exit(err_buf);
        }
      
      std::string idir(mypwd);
      idir += "/";
      idir += cmd_arg;
      hash_add_or_exit(ji->job_attr, ATTR_pbs_o_initdir, idir.c_str(), data_type);
      }  /* END if (cmd_arg[0] != '/') */

    if (hash_find(ji->client_attr, "validate_path", &tmp_job_info))
      {
      /* validate local existence of '-d' working directory */

      if (chdir(cmd_arg) == -1)
        {
        char  err_buf[MAXLINE*2];
        snprintf(err_buf, sizeof(err_buf),
          "qsub: cannot chdir to '%s' errno: %d (%s)", cmd_arg, errno, strerror(errno));
        return(-1);
        }
      }
    }    /* END if (cmd_arg != NULL) */

  return(PBSE_NONE);
  } // END process_opt_d() 



/*
 * process_opt_i()
 *
 * Verifies and adds the idle slot limit argument to the job
 * @param ji - where we store the job information
 * @param cmd_arg - the command line argument passed to qsub after -i
 * @param data_type - the source of this argument
 * @return PBSE_NONE if a valid number, -1 if invalid
 */

int process_opt_i(

  job_info   *ji,
  const char *cmd_arg,
  int         data_type)

  {
  if (cmd_arg != NULL)
    {
    char *end;

    int limit = strtol(cmd_arg, &end, 10);

    if ((limit > 0) &&
        (end != cmd_arg))
      {
      hash_add_or_exit(ji->job_attr, ATTR_idle_slot_limit, cmd_arg, data_type);
     
      return(PBSE_NONE);
      }
    }

  return(-1);
  } // END process_opt_i()



/*
 * process_opt_j()
 *
 * Verifies and adds the argument passed to -j
 * @param ji - where we store the job information
 * @param cmd_arg - the command line argument passed to qsub
 * @param data_type - the source of this argument 
 * @return PBSE_NONE if good, -1 otherwise
 */

int process_opt_j(
  
  job_info   *ji,
  const char *cmd_arg,
  int         data_type)

  {
  if (cmd_arg == NULL)
    return(-1);

  if ((strcmp(cmd_arg, "oe") != 0) &&
      (strcmp(cmd_arg, "eo") != 0) &&
      (strcmp(cmd_arg, "n") != 0))
    return(-1);

  hash_add_or_exit(ji->job_attr, ATTR_j, cmd_arg, data_type);

  return(PBSE_NONE);
  } // END process_opt_j() 



/*
 * process_opt_k()
 *
 * Verifies and adds the argument passed to -k
 * @param ji - where we store the job information
 * @param cmd_arg - the command line argument passed to qsub
 * @param data_type - the source of this argument 
 * @return PBSE_NONE if good, -1 otherwise
 */

int process_opt_k(

  job_info   *ji,
  const char *cmd_arg,
  int         data_type)

  {
  if (cmd_arg == NULL)
    return(-1);

  if ((strcmp(cmd_arg, "o") != 0) &&
      (strcmp(cmd_arg, "e") != 0) &&
      (strcmp(cmd_arg, "oe") != 0) &&
      (strcmp(cmd_arg, "eo") != 0) &&
      (strcmp(cmd_arg, "n") != 0))
    return(-1);
  
  hash_add_or_exit(ji->job_attr, ATTR_k, cmd_arg, data_type);
  
  return(PBSE_NONE);
  } // END process_opt_k()



int process_opt_K(

  job_info   *ji,
  const char *cmd_arg,
  int         data_type)

  {
  if (cmd_arg == NULL)
    return(-1);

  long delay = strtol(cmd_arg, NULL, 10);

  // Must be a positive number
  if (delay < 1)
    return(-1);
  
  hash_add_or_exit(ji->job_attr, ATTR_user_kill_delay, cmd_arg, data_type);

  return(PBSE_NONE);
  } // END process_opt_K()



/*
 * process_opt_m()
 *
 * Verifies and adds the argument passed to -m
 * @param ji - where we store the job information
 * @param cmd_arg - the command line argument passed to qsub
 * @param data_type - the source of this argument 
 * @return PBSE_NONE if good, -1 otherwise
 */

int process_opt_m(
  
  job_info   *ji,
  const char *cmd_arg,
  int         data_type)

  {
  if (cmd_arg == NULL)
    return(-1);

  while (isspace((int)*cmd_arg))
    cmd_arg++;
  
  if (strlen(cmd_arg) == 0)
    return(-1);
  
  if (strcmp(cmd_arg, "n") != 0)
    {
    const char *pc = cmd_arg;
    
    while (*pc)
      {
      if ((*pc != 'a') &&
          (*pc != 'b') &&
          (*pc != 'e') &&
          (*pc != 'f') &&
          (*pc != 'p'))
        return(-1);

      pc++;
      }
    } /* END if (strcmp(cmd_arg,"n") != 0) */
  if (strcmp(cmd_arg, "p") != 0)
    {
    const char *pc = cmd_arg;
    
    while (*pc)
      {
      if ((*pc != 'a') &&
          (*pc != 'b') &&
          (*pc != 'e') &&
          (*pc != 'f') &&
          (*pc != 'n'))
        return(-1);

      pc++;
      }
    } /* END if (strcmp(cmd_arg,"p") != 0) */
    
          
  hash_add_or_exit(ji->job_attr, ATTR_m, cmd_arg, data_type);

  return(PBSE_NONE);
  } // END process_opt_m()



/*
 * process_opt_p()
 *
 * Verifies and adds the argument passed to -p
 * @param ji - where we store the job information
 * @param cmd_arg - the command line argument passed to qsub
 * @param data_type - the source of this argument 
 * @return PBSE_NONE if good, -1 otherwise
 */

int process_opt_p(
  
  job_info   *ji,
  const char *cmd_arg,
  int         data_type)

  {
  if (cmd_arg == NULL)
    return(-1);

  while (isspace((int)*cmd_arg))
    cmd_arg++;
  
  const char *pc = cmd_arg;
  
  if ((*pc == '-') ||
      (*pc == '+'))
    pc++;
  
  if (strlen(pc) == 0)
    return(-1);
  
  while (*pc != '\0')
    {
    if (!isdigit(*pc))
      return(-1);
    
    pc++;
    }
  
  int priority = strtol(cmd_arg, NULL, 10);
  
  if ((priority < -1024) ||
      (priority > 1023))
    return(-1);
  
  hash_add_or_exit(ji->job_attr, ATTR_p, cmd_arg, data_type);

  return(PBSE_NONE);
  } // END process_opt_p()



/** 
 * Process command line options.
 *
 * @see main() - parent
 *
 * NOTE:  return 0 on success 
 * NOTE:  only run submitfilter if pass < 10 
 */ 

void process_opts(

  int        argc,      /* I */
  char     **argv,      /* I */
  job_info  *ji,        /* M */
  int        data_type)

  {
  int          i;
  int          c;
  int          rc = 0;
  int          errflg = 0;
  time_t       after;
  char         a_value[80];
  char        *keyword;
  char        *valuewd;

  FILE        *fP = NULL;

  char         tmp_name[] = "/tmp/qsub.XXXXXX";
  char         tmp_name2[] = "/tmp/qsub.XXXXXX";

  char         cline[4096];
  std::string  cline_out;


  char         tmpResources[4096] = "";
  char        *cP;
  char        *ptr;
  char         flag;  /* submitfilter flag character */
  char        *vptr;  /* submitfilter flag value */


/*   struct stat sfilter; */

  int          tmpfd;
  int          nitems;
  char         search_string[256];
  job_data    *tmp_job_info = NULL;
  int          alloc_len = 0;
  char        *err_msg = NULL;
  /* Moved from global to local */
  char         path_out[MAXPATHLEN + 1];
  
  /* Note:
   * All other #ifdef's for PBS_NO_POSIX_VIOLATION are being removed because
   * the get_opts functionality will only process options in the list.
   * Due to the list above being set, the options that are not included
   * will never be processed
   */

  /* The following macro, together the value of passet (pass + 1) is used */
  /* to enforce the following rules: 1. option on the command line take */
  /* precedence over those in script directives.   2. With in the command */
  /* line or within the script, the last occurance of an option takes */
  /* precedence over the earlier occurance.    */

  /* Given the addition of the hashmap functionality where the last entry added
   * is the ONLY entry remaining (as the previous will be deleted) the need
   * to count which iteration has been removed
   */

#ifdef linux
    optind = 0;  /* prime getopt's starting point */
#else
    optind = 1;  /* prime getopt's starting point */
#endif

  while ((c = getopt(argc, argv, GETOPT_ARGS)) != EOF)
    {
    switch (c)
      {

      case '-':
        /**
         * We have already tested for --version and --about, in process_early_opts().
         * Any other opt that has - as the first char is invalid.
         */
        print_qsub_usage_exit("a single - is not a valid option");

        break;

      case 'a':

          if ((after = cvtdate(optarg)) < 0)
            print_qsub_usage_exit("qsub: illegal -a value");

          sprintf(a_value, "%ld", (long)after);
          hash_add_or_exit(ji->job_attr, ATTR_a, a_value, data_type);

        break;

      case 'A':

        hash_add_or_exit(ji->job_attr, ATTR_A, optarg, data_type);

        break;

      case 'b':

        hash_add_or_exit(ji->client_attr, "cnt2server_retry", optarg, data_type);

        break;

      case 'c':

          while (isspace((int)*optarg))
            optarg++;

          if (strlen(optarg) == 0)
            print_qsub_usage_exit("qsub: illegal -c value");

          /* OLD FORMAT:  -c { n | s | c | c=X }
           * New format: -c [ { <old format items> | <new items> } ',' ]
           * new items: none | shutdown | checkpoint | name=xyz | dir=xyz | interval=X
           */
          nitems = csv_length(optarg);

          for (i = 0; i < nitems; i++)
            {
            if ((ptr = csv_nth(optarg, i)) != NULL)
              {
              snprintf(search_string, sizeof(search_string), "%s", ptr);
              ptr = strchr(search_string, '=');

              if (ptr)
                *ptr = 0;
              else
                ptr = &search_string[strlen(search_string)];

              while (ptr > search_string && *(ptr - 1) == ' ')
                *--ptr = 0;

              if (csv_find_string(checkpoint_strings, search_string) == NULL)
                print_qsub_usage_exit("qsub: illegal -c value");
              }
            }

          hash_add_or_exit(ji->job_attr, ATTR_c, optarg, data_type);

        break;

      case 'C':

        hash_add_or_exit(ji->client_attr, "pbs_dprefix", optarg, data_type);

        break;

      case 'd':

        if (process_opt_d(ji, optarg, data_type, tmp_job_info) != PBSE_NONE)
          print_qsub_usage_exit("qsub: illegal -d value");

        break;

      case 'D':

        if (optarg != NULL)
          hash_add_or_exit(ji->job_attr, ATTR_pbs_o_rootdir, optarg, data_type);
        else
          print_qsub_usage_exit("qsub: illegal -D value");

        break;

      case 'e':

          if (hash_find(ji->job_attr, ATTR_submit_host, &tmp_job_info))
            rc = prepare_path(optarg,path_out,(char *)tmp_job_info->value.c_str());
          else
            rc = prepare_path(optarg,path_out,NULL);

          if ((rc == 0) || (rc == 3))
            hash_add_or_exit(ji->job_attr, ATTR_e, path_out, data_type);
          else
            print_qsub_usage_exit("qsub: illegal -e value");

        break;

      case 'E':

        hash_add_or_exit(ji->job_attr, ATTR_node_exclusive, "TRUE", data_type);

        break;

      case 'F':

        hash_add_or_exit(ji->job_attr, ATTR_args, optarg, data_type);
        break;


      case 'f':
      
        hash_add_or_exit(ji->job_attr, ATTR_f, "TRUE", data_type);
        break;
      
      case 'h':

        hash_add_or_exit(ji->job_attr, ATTR_h, "u", data_type);
        break;

      case 'i':

        if (process_opt_i(ji, optarg, data_type) != PBSE_NONE)
          {
          char buf[MAXLINE];
          snprintf(buf, sizeof(buf), "qsub: illegal -i value '%s'", optarg);
          print_qsub_usage_exit(buf);
          }

        break;

      case 'I':

        hash_add_or_exit(ji->job_attr, ATTR_inter, interactive_port(&inter_sock), data_type);

        break;

      case 'j':

        /* FORMAT:  {oe|eo|n} */

        if (process_opt_j(ji, optarg, data_type) != PBSE_NONE)
          print_qsub_usage_exit("qsub: illegal -j value");

        break;

      case 'J':

        hash_add_or_exit(ji->job_attr, ATTR_J, optarg, data_type);
        J_opt = TRUE;

        break;

      case 'k':

        /* FORMAT:  {o|e} */
        if (process_opt_k(ji, optarg, data_type) != PBSE_NONE)
          print_qsub_usage_exit("qsub: illegal -k value");

        break;

      case 'K':
        
        if (process_opt_K(ji, optarg, data_type) != PBSE_NONE)
          print_qsub_usage_exit("qsub: illegal -K value");

        break;

      case 'l':

        if ((strstr(optarg, ",procs=") != NULL) &&
            (strstr(optarg, "nodes=") != NULL))
          print_qsub_usage_exit("qsub: illegal -l value");

        if (add_verify_resources(ji->res_attr, optarg, data_type) != 0)
          print_qsub_usage_exit("qsub: illegal -l value");

          //If cpuclock gets set we need to set the node exclusive flag
          {
          job_data *pData = NULL;
          if (hash_find(ji->res_attr,"cpuclock",&pData))
            {
            hash_add_or_exit(ji->job_attr, ATTR_node_exclusive, "TRUE", data_type);
            }
          }

        break;

      case 'L':

        process_opt_L(optarg);

        break;

      case 'm':

        if (process_opt_m(ji, optarg, data_type) != PBSE_NONE)
          print_qsub_usage_exit("qsub: illegal -m value");

        break;

      case 'M':

          if (parse_at_list(optarg, FALSE, FALSE))
            print_qsub_usage_exit("qsub: illegal -M value");
          hash_add_or_exit(ji->job_attr, ATTR_M, optarg, data_type);
        break;

      case 'n':

        hash_add_or_exit(ji->job_attr, ATTR_node_exclusive, "TRUE", data_type);

        break;

      case 'N':
          /* NOTE:  did enforce alpha start previously - relax this constraint
                    allowing numeric job names (CRI - 6/26/07) */
          if (check_job_name(optarg, 0) == 0)
            hash_add_or_exit(ji->job_attr, ATTR_N, optarg, data_type);
          else
            print_qsub_usage_exit("qsub: illegal -N value");
        break;

      case 'o':
          if (hash_find(ji->job_attr, ATTR_submit_host, &tmp_job_info))
            rc = prepare_path(optarg,path_out,(char *)tmp_job_info->value.c_str());
          else
            rc = prepare_path(optarg,path_out,NULL);

          if ((rc == 0) || (rc == 3))
            hash_add_or_exit(ji->job_attr, ATTR_o, path_out, data_type);
          else
            print_qsub_usage_exit("qsub: illegal -o value");
        break;

      case 'p':
        
        if (process_opt_p(ji, optarg, data_type) != PBSE_NONE)
          print_qsub_usage_exit("qsub: illegal -p value");
        
        break;

      case 'P':

        if (strlen(optarg) > 0)
          {
          char *user;
          char *group;
          char *colon;

          /* don't check privileges, this happens on the server
             side as managers are allowed to submit jobs on behalf
             of other users */

          user = optarg;
          colon = strchr(user,':');

          if (colon != NULL)
            {
            group = colon+1;
            *colon = '\0';
            hash_add_or_exit(ji->job_attr, ATTR_g, group, data_type);
            }

          hash_add_or_exit(ji->job_attr, ATTR_P, user, data_type);

          P_opt = TRUE;
          }
        else
          print_qsub_usage_exit("qsub: -P requires a user name");

        break;

      case 'q':

        hash_add_or_exit(ji->client_attr, "destination", optarg, data_type);

        break;

      case 'r':

          if (strlen(optarg) != 1)
            print_qsub_usage_exit("qsub: illegal -r value (y/n)");

          if ((*optarg != 'y') && (*optarg != 'n'))
            print_qsub_usage_exit("qsub: illegal -r value (y/n)");

          hash_add_or_exit(ji->job_attr, ATTR_r, optarg, data_type);

        break;

      case 'S':

          if (parse_at_list(optarg, TRUE, TRUE))
            print_qsub_usage_exit("qsub: illegal -S value");

        hash_add_or_exit(ji->job_attr, ATTR_S, optarg, data_type);

        break;

      case 't':

        hash_add_or_exit(ji->job_attr, ATTR_t, optarg, data_type);

        break;

      case 'T':


          /* validate before sending request to server? */
          hash_add_or_exit(ji->job_attr, ATTR_jobtype, optarg, data_type);

        break;

      case 'u':

          if (parse_at_list(optarg, TRUE, FALSE))
            print_qsub_usage_exit("qsub: illegal -u value");

          hash_add_or_exit(ji->job_attr, ATTR_u, optarg, data_type);

        break;

      case 'v':

        rc = parse_variable_list(ji->job_attr, ji->user_attr, CMDLINE_DATA, SET, optarg);

        if (rc != PBSE_NONE)
          exit(rc);

        break;

      case 'V':

        hash_add_or_exit(ji->client_attr, "user_attr", "1", LOGIC_DATA);

        break;

      case 'w':

        if (optarg == NULL)
          print_qsub_usage_exit("qsub: illegal -w value");
        else
          hash_add_or_exit(ji->job_attr, ATTR_init_work_dir, optarg, data_type);

        break;

      case 'W':

        while (isspace((int)*optarg))
          optarg++;

        if (strlen(optarg) == 0)
          {
          /* value is empty */

          fprintf(stderr, "qsub: illegal -W value\n");

          errflg++;

          break;
          }

        i = get_name_value(optarg, &keyword, &valuewd);

        if (i != 1)
          {
          char tmpLine[65536];

          /* assume resource manager extension */

          snprintf(tmpLine, sizeof(tmpLine), "x=%s",
                   optarg);

          i = get_name_value(tmpLine, &keyword, &valuewd);
          }

        while (i == 1)
          {
          if (!strcmp(keyword, ATTR_depend))
            {
            std::vector<std::string> dependency_options;

            if (((rc = parse_depend_list(valuewd, dependency_options)) != PBSE_NONE) ||
                (dependency_options.size() == 0))
              {
              /* cannot parse 'depend' value */
              char err_msg[MAXLINE];

              if (rc == 2)
                {
                snprintf(err_msg, sizeof(err_msg),
                  "qsub: -W value exceeded max length (%d)", PBS_DEPEND_LEN);
                print_qsub_usage_exit(err_msg);
                }
              else
                {
                snprintf(err_msg, sizeof(err_msg),
                  "qsub: illegal -W dependency value: '%s'", valuewd);

                print_qsub_usage_exit(err_msg);
                }

              break;
              }

            hash_add_or_exit(ji->job_attr, ATTR_depend, dependency_options[0].c_str(), data_type);

            if (dependency_options.size() > 1)
              {
              alternate_dependency = strdup(dependency_options[1].c_str());
              alternate_data_type = data_type;
              }
            }
          else if (!strcmp(keyword, ATTR_job_radix))
            {
            int radix_value;
            int len;
            
              
              len = strlen(valuewd);
              if (len > MAX_RADIX_NUM_LEN)
                print_qsub_usage_exit("qsub: illegal -W value for job_radix");
              for (i = 0; i < len; i++)
                {
                if (!isdigit(valuewd[i])) /* verify the string is all digits */
                  break;
                }
              
              if (i == len) /* we parsed the whole valuewd string and it is a number */
                {
                radix_value = atoi(valuewd);
                if (radix_value < 2)
                  print_qsub_usage_exit("qsub: illegal -W. job_radix must be >= 2");
                else
                  hash_add_or_exit(ji->job_attr, ATTR_job_radix, valuewd, ENV_DATA);
                }
              else
                print_qsub_usage_exit("qsub: illegal -W value for job_radix");
            }
          else if (!strcmp(keyword, ATTR_stagein))
            {
            if (parse_stage_list(valuewd))
              print_qsub_usage_exit("qsub: illegal -W value for stagein");
            
            if (hash_find(ji->job_attr, ATTR_stagein, &tmp_job_info))
              {
              /* 
               * if this attribute already exists, we need to append this
               * value to it because multiples are allowed.
               */
              char *tmpBuf;

              if ((tmpBuf = (char *)malloc(strlen(valuewd) + tmp_job_info->value.length() + 2)) == (char *)0)
                {
                fprintf(stderr, "Out of memory.\n");
                exit(1);
                }
              strcpy(tmpBuf, tmp_job_info->value.c_str());
              strcat(tmpBuf, ",");
              strcat(tmpBuf, valuewd);
              hash_add_or_exit(ji->job_attr, ATTR_stagein, tmpBuf, data_type);
              free(tmpBuf);
              }
            else
              {
              hash_add_or_exit(ji->job_attr, ATTR_stagein, valuewd, data_type);
              }

            }
          else if (!strcmp(keyword, ATTR_stageout))
            {            
            if (parse_stage_list(valuewd))
              print_qsub_usage_exit("qsub: illegal -W value for stageout");
            
            if (hash_find(ji->job_attr, ATTR_stageout, &tmp_job_info))
              {
              /* 
               * if this attribute already exists, we need to append this
               * value to it because multiples are allowed.
               */
              char *tmpBuf;
              
              if ((tmpBuf = (char *)malloc(strlen(valuewd) + tmp_job_info->value.length() + 2)) == (char *)0)
                {
                fprintf(stderr, "Out of memory.\n");
                exit(1);
                }

              strcpy(tmpBuf, tmp_job_info->value.c_str());
              strcat(tmpBuf, ",");
              strcat(tmpBuf, valuewd);
              hash_add_or_exit(ji->job_attr, ATTR_stageout, tmpBuf, data_type);
              free(tmpBuf);
              }
            else
              {
              hash_add_or_exit(ji->job_attr, ATTR_stageout, valuewd, data_type);
              }
            }
          else if (!strcmp(keyword, ATTR_t))
            {
            hash_add_or_exit(ji->job_attr, ATTR_t, valuewd, data_type);
            }
          else if (!strcmp(keyword, ATTR_g))
            {
            if (parse_at_list(valuewd, TRUE, FALSE))
              print_qsub_usage_exit("qsub: illegal -W value grouplist");

            if (hash_find(ji->client_attr, "validate_group", &tmp_job_info))
              {
              if (validate_group_list(valuewd) == FALSE)
                {
                alloc_len = 80 + strlen(valuewd);
                calloc_or_fail(&err_msg, alloc_len, "-W attribute");
                snprintf(err_msg, alloc_len, "qsub: User isn't a member of one or more groups in %s", valuewd);
                print_qsub_usage_exit(err_msg);
                }
              }
            hash_add_or_exit(ji->job_attr, ATTR_g, valuewd, data_type);
            }
          else if (!strcmp(keyword, ATTR_inter))
            {
            /* specify interactive job */
            if (strcmp(valuewd, "true") != 0)
              print_qsub_usage_exit("qsub: illegal -W value");

            hash_add_or_exit(ji->job_attr, ATTR_inter, interactive_port(&inter_sock), data_type);
            }
          else if (!strcmp(keyword, ATTR_umask))
            {
            int len;
            len = strlen(valuewd);
            if (valuewd[0] == '0')
              --len;

            if (len > 3)
              {
              alloc_len = 80 + strlen(valuewd);
              calloc_or_fail(&err_msg, alloc_len, "-W attribute");
              snprintf(err_msg, alloc_len, "qsub: Invalid umask value, too many digits: %s", valuewd);
              print_qsub_usage_exit(err_msg);
              }
            
            if (valuewd[0] == '0')
              {
              /* value is octal, convert to decimal */
              long mask;
              char buf[4];

              mask = strtol(valuewd, NULL, 8);
              snprintf(buf, 4, "%ld", mask); 

              /* value is octal, convert to decimal */
              hash_add_or_exit(ji->job_attr, ATTR_umask, buf, data_type);
              }
            else
              hash_add_or_exit(ji->job_attr, ATTR_umask, valuewd, data_type);
            }
          else if (!strcmp(keyword, ATTR_f))
            {
            switch (valuewd[0])
              {
            
              /*accept 1, TRUE,true,YES,yes, 0, FALSE, false, NO, no */
              case 1:
              case 'T':
              case 't':
              case 'Y':
              case 'y':
                
                hash_add_or_exit(ji->job_attr, ATTR_f, "TRUE", data_type);
               
                break;
                
              case 0:
              case 'F':
              case 'f':
              case 'N':
              case 'n':
                
                hash_add_or_exit(ji->job_attr, ATTR_f, "FALSE", data_type);
                break;
              
              default:
                alloc_len = 80 + strlen(ATTR_f) + strlen(valuewd);
                calloc_or_fail(&err_msg, alloc_len, "-W attribute");
                snprintf(err_msg, alloc_len, "qsub: invalid %s value: %s", ATTR_f, valuewd);
                print_qsub_usage_exit(err_msg);

              }
            }
          else if ((!strcmp(keyword, "x")) &&
                   (!strncmp(valuewd, "mppnodes=", strlen("mppnodes="))))
            {
            // add this as a resource
            char *to_process = strdup(valuewd);
            if (add_verify_resources(ji->res_attr, to_process, data_type) != 0)
              {
              free(to_process);
              print_qsub_usage_exit("qsub: illegal -l value (mppnodes is processed as a -l value)");
              }

            free(to_process);
            }
          else
            {
            job_data *pVal;
            char tmpLine[65536];

            //Append if there is already a value here.
            if (hash_find(ji->job_attr,keyword,&pVal))
              {
              snprintf(tmpLine, sizeof(tmpLine), "%s;%s", pVal->value.c_str(), valuewd);
              valuewd = tmpLine;
              }
            hash_add_or_exit(ji->job_attr, keyword, valuewd, data_type);
            }

          i = get_name_value(NULL, &keyword, &valuewd);
          }  /* END while (i == 1) */

        if (i == -1)
          print_qsub_usage_exit("qsub: illegal -W value");

        break;

/* #if !defined(PBS_NO_POSIX_VIOLATION) */

      case 'X':

        if (hash_find(ji->user_attr, "DISPLAY", &tmp_job_info))
          hash_add_or_exit(ji->client_attr, "DISPLAY", tmp_job_info->value.c_str(), LOGIC_DATA);
        else
          print_qsub_usage_exit("qsub: DISPLAY not set");

        break;

      case 'x':
      
        if (!(hash_find(ji->job_attr, ATTR_inter, &tmp_job_info)))
          {
          print_qsub_usage_exit("qsub: '-x' invalid on non-interactive job");
          }

        if (hash_find(ji->client_attr, "cmdline_script", &tmp_job_info))
          {
          hash_add_or_exit(ji->job_attr, ATTR_intcmd, tmp_job_info->value.c_str(), CMDLINE_DATA);
          }
        else
          {
          print_qsub_usage_exit("qsub: '-x' used without a script specified");
          }

        break;
        
/* #endif */

      case 'z':

        hash_add_or_exit(ji->client_attr, "no_jobid_out", "1", data_type);

        break;

      case '?':

      default :
        if (optarg != NULL)
          {
          alloc_len = 80 + strlen(optarg);
          calloc_or_fail(&err_msg, alloc_len, "unknown attribute flag");
          snprintf(err_msg, alloc_len, "qsub: invalid attribute flag (%c) value: %s", c, optarg);
          print_qsub_usage_exit(err_msg);
          }
        else
          {
          print_qsub_usage_exit("exiting");
          }

        break;
      }
    }  /* END while ((c = getopt(argc,argv,GETOPT_ARGS)) != EOF) */

  if ((J_opt == TRUE) && 
      (P_opt != TRUE))
    {
    print_qsub_usage_exit("The -J option can only be used in conjunction with -P");
    }


  /* ORNL WRAPPER */

  if (!(hash_find(ji->client_attr, "no_submit_filter", &tmp_job_info)) &&
      (hash_find(ji->job_attr, ATTR_inter, &tmp_job_info)))
    {
    int original_optind = optind;

    /* Evaluate resources for interactive submission here. */

    /* Modified to reduce excess exit code */
  
    if ((tmpfd = mkstemp(tmp_name)) < 1)
      rc = 1;
    else if ((fP = fdopen(tmpfd, "w+")) == NULL)
      rc = 2;
    else if (fprintf(fP, "%s\n\n", tmpResources) < 0)
      rc = 3;
    if (rc != 0)
      {
      alloc_len = 80 + strlen(tmp_name);
      calloc_or_fail(&err_msg, alloc_len, "tmp file error");
      snprintf(err_msg, alloc_len, "qsub: %s tmp job file %s",
          rc<=2?"could not create":"unable to write to", tmp_name);
      if (rc >= 2)
        unlink(tmp_name);
      print_qsub_usage_exit(err_msg);
      }
    fclose(fP);
    fP = NULL;

    if (hash_find(ji->job_attr, ATTR_pbs_o_submit_filter, &tmp_job_info))
      {
      int index;

      if ((tmpfd = mkstemp(tmp_name2)) < 1)
        {
        alloc_len = 80  + strlen(tmp_name2);
        calloc_or_fail(&err_msg, alloc_len, "qsub: could not create tmp job file");
        snprintf(err_msg, alloc_len, "qsub: could not create tmp job file %s", tmp_name2);
        unlink(tmp_name);
        print_qsub_usage_exit(err_msg);
        }

      close(tmpfd);

      /* run the specified resources through the submitfilter. */
      cline_out = tmp_job_info->value;

      for (index = 1;index < argc;index++)
        {
        if (argv[index] != NULL)
          {
          cline_out += " ";
          cline_out += "\"";
          cline_out += argv[index];
          cline_out += "\"";
         }
        }    /* END for (index) */
       
      cline_out += " <";
      cline_out += tmp_name;
      cline_out += " >";
      cline_out += tmp_name2;

      rc = system(cline_out.c_str());

      alloc_len = 0;
      if (rc == -1)
        {
        alloc_len = 80  + strlen(tmp_name2);
        calloc_or_fail(&err_msg, alloc_len, "qsub: error writing filter o/p");
        snprintf(err_msg, alloc_len, "qsub: error writing filter o/p, %s", tmp_name2);
       }
      else if (WEXITSTATUS(rc) == (unsigned char)SUBMIT_FILTER_ADMIN_REJECT_CODE)
        {
        alloc_len = 160;
        calloc_or_fail(&err_msg, alloc_len, "qsub: administrative rejection");
        snprintf(err_msg, alloc_len, "qsub: Your job has been administratively rejected by the queueing system.\nqsub: There may be a more detailed explanation prior to this notice.");
        }
      else if (WEXITSTATUS(rc))
        {
        alloc_len = 80;
        calloc_or_fail(&err_msg, alloc_len, "qsub: filter error code");
        snprintf(err_msg, alloc_len, "qsub: submit filter returned an error code, aborting job submission.");
        }
      else
        {
        /* Success path */
        fP = fopen(tmp_name2, "r+");
        }

      unlink(tmp_name2);
      unlink(tmp_name);
      if (alloc_len != 0)
        print_qsub_usage_exit(err_msg);


      }  /* END if (stat(PBS_Filter,&sfilter) != -1) */
    else
      {
      fP = fopen(tmp_name, "r+");
      unlink(tmp_name);
      }

    /* evaluate the resources */
    /* If I'm not missing something, this can be optimized.
     * In the case of a filter the results need to be read from the 
     * resulting file.
     * However, if there is no filter, the results are read from a file that
     * written to the disk a bit earlier and never modified before read here.
     * If a string can be parsed instead it would speed up the whole process
     *  by not having another disk write/read access.
     */
    if (fP != NULL)
      {
      while (fgets(cline, sizeof(cline), fP) != NULL)
        {
        // Skip blank lines
        if (*cline == '\n')
          continue;

        if (strlen(cline) < 5)
          break;

        for (cP = cline;cP < cline + strlen(cline);cP++)
          {
          if (*cP == '\n')
            {
            *cP = '\0';
            }
          }

        /* NOTE:  allow for job attributes other than '-l' */

        /* FORMAT:  '#PBS -<FLAG> <VAL>' */

        if (strncasecmp(cline, "#pbs -", strlen("#pbs -")))
          {
          /* invalid line specified */

          continue;
          }

        /* NOTE:  a better design would be to process the submitfilter
         * outside of process_opts(),
         * add valid args to ArgC/ArgV, and call process_opts() once. (NYI)
         */

        /* NOTE:  can we utilize 'process_opts' to process submit filter lines? (NYI) */

        flag = cline[strlen("#pbs -")];

        vptr = cline + strlen("#pbs -x ");

        switch (flag)
          {

          case 'l':

            if (add_verify_resources(ji->res_attr, vptr, data_type))
              print_qsub_usage_exit("qsub: illegal -l value");

            break;

          default:

            {
            char FlagString[3];

            char *tmpArgV[4];

            int   aindex;

            FlagString[0] = '-';
            FlagString[1] = flag;
            FlagString[2] = '\0';

            /* Duplicate code */
            aindex = 1;  /* prime getopt's starting point */
            tmpArgV[0] = (char *)"";

            tmpArgV[aindex] = FlagString;
            tmpArgV[aindex + 1] = vptr;
            tmpArgV[aindex + 2] = NULL;

            tmpArgV[3] = NULL;

            /* To prevent recursion, set a flag in the client_attr */
            hash_add_or_exit(ji->client_attr, "no_submit_filter", "1", LOGIC_DATA);
            process_opts(aindex + 2, tmpArgV, ji, FILTER_DATA);
            hash_del_item(ji->client_attr, "no_submit_filter");
            }

          break;
          }  /* END switch (cptr[0]) */
        }    /* END while (fgets(cline,sizeof(cline),fP) != NULL) */
      
      fclose(fP);
      }

    /* restore optind */
    optind = original_optind;
    }    /* END if (Interact_opt == 1) */

  /* END ORNL WRAPPER */

  } /* END process_opts() */




/*
 * set_job_defaults - if not already set, set certain job attributes to
 * their default value. As this is run before all other functions,
 * the job_attr hashmap is empty on start and no checks for existing values are needed
 */

void set_job_defaults(
    
  job_info *ji)

  {
  job_data *tmp_job_info = NULL;
  hash_add_or_exit(ji->job_attr, ATTR_c, CHECKPOINT_UNSPECIFIED, STATIC_DATA);

  hash_add_or_exit(ji->job_attr, ATTR_h, NO_HOLD, STATIC_DATA);

  hash_add_or_exit(ji->job_attr, ATTR_j, NO_JOIN, STATIC_DATA);

  hash_add_or_exit(ji->job_attr, ATTR_k, NO_KEEP, STATIC_DATA);

  hash_add_or_exit(ji->job_attr, ATTR_m, MAIL_AT_ABORT, STATIC_DATA);

  hash_add_or_exit(ji->job_attr, ATTR_p, DEFAULT_PRIORITY, STATIC_DATA);

  /* rerunnable_by_default = true, if this changes later, that value will override this one */
  hash_add_or_exit(ji->job_attr, ATTR_r, "TRUE", STATIC_DATA);
  hash_add_or_exit(ji->job_attr, ATTR_f, "FALSE", STATIC_DATA);
  
  hash_add_or_exit(ji->client_attr, "pbs_dprefix", "#PBS", STATIC_DATA);
  hash_add_or_exit(ji->job_attr, ATTR_job_radix, "0", STATIC_DATA);
  if (hash_find(ji->user_attr, "PBS_CLIENTRETRY", &tmp_job_info))
    hash_add_or_exit(ji->client_attr, "cnt2server_retry", tmp_job_info->value.c_str(), ENV_DATA);
  return;
  }  /* END set_job_defaults() */






/* This is used to set options for the client to be used as part of making
 * the call to the pbs_server. This information is thrown out at the end
 * of the call.
 */
void set_client_attr_defaults(job_data_container *client_attr)
  {
  hash_add_or_exit(client_attr, "xauth_path", XAUTH_PATH, STATIC_DATA);
  hash_add_or_exit(client_attr, "validate_path", "1", STATIC_DATA);
  }

void update_job_env_names(job_info *ji)
  {
  job_data *tmp_job_info = NULL;

  if (hash_find(ji->user_attr, "HOME", &tmp_job_info))
    hash_add_or_exit(ji->job_attr, ATTR_pbs_o_home, tmp_job_info->value.c_str(), ENV_DATA);
  else
    hash_add_or_exit(ji->job_attr, ATTR_pbs_o_home, "/", ENV_DATA);

  if (hash_find(ji->user_attr, "LOGNAME", &tmp_job_info))
    hash_add_or_exit(ji->job_attr, ATTR_pbs_o_logname, tmp_job_info->value.c_str(), ENV_DATA);

  if (hash_find(ji->user_attr, "PATH", &tmp_job_info))
    hash_add_or_exit(ji->job_attr, ATTR_pbs_o_path, tmp_job_info->value.c_str(), ENV_DATA);

  if (hash_find(ji->user_attr, "MAIL", &tmp_job_info))
    hash_add_or_exit(ji->job_attr, ATTR_pbs_o_mail, tmp_job_info->value.c_str(), ENV_DATA);

  if (hash_find(ji->user_attr, "SHELL", &tmp_job_info))
    hash_add_or_exit(ji->job_attr, ATTR_pbs_o_shell, tmp_job_info->value.c_str(), ENV_DATA);

  if (hash_find(ji->user_attr, "TZ", &tmp_job_info))
    hash_add_or_exit(ji->job_attr, ATTR_pbs_o_tz, tmp_job_info->value.c_str(), ENV_DATA);

  if (hash_find(ji->user_attr, "LANG", &tmp_job_info))
    hash_add_or_exit(ji->job_attr, ATTR_pbs_o_lang, tmp_job_info->value.c_str(), ENV_DATA);

  }

/* Process all config files options */
void process_config_file(
    
  job_info *ji)

  {
  char torque_cfg_buf[MAX_LINE_LEN];      /* Buffer holds config file */
  char *param_val;

  if (load_config(torque_cfg_buf, sizeof(torque_cfg_buf)) == 0)
    {
    /* This config entry should most likely be removed in the future */
    if ((param_val = get_trq_param("QSUBSLEEP", torque_cfg_buf)) != NULL)
      {
      sleep(atoi(param_val));
      }

    if ((param_val = get_trq_param("SUBMITFILTER", torque_cfg_buf)) != NULL)
      {
      hash_add_or_exit(ji->job_attr, ATTR_pbs_o_submit_filter, param_val, CONFIG_DATA);
      }

    if ((param_val = get_trq_param("SERVERHOST", torque_cfg_buf)) != NULL)
      {
      hash_add_or_exit(ji->client_attr, "serverhost", param_val, CONFIG_DATA);
      }

    if ((param_val = get_trq_param("QSUBHOST", torque_cfg_buf)) != NULL)
      {
      hash_add_or_exit(ji->job_attr, ATTR_submit_host, param_val, CONFIG_DATA);
      }

    if ((param_val = get_trq_param("QSUBSENDUID", torque_cfg_buf)) != NULL)
      {
      hash_add_or_exit(ji->client_attr, ATTR_pbs_o_uid, param_val, ENV_DATA);
      }

    if (get_trq_param("QSUBSENDGROUPLIST", torque_cfg_buf) != NULL)
      {
      gid_t group_id = getgid();
      struct group *gpent = getgrgid(group_id);

      if (gpent != NULL)
        {
        hash_add_or_exit(ji->job_attr, ATTR_g, gpent->gr_name, ENV_DATA);
        hash_add_or_exit(ji->client_attr, "qsubsendgrouplist", gpent->gr_name, CONFIG_DATA);
        }
      }

    if ((param_val = get_trq_param("XAUTHPATH", torque_cfg_buf)) != NULL)
      {
      hash_add_or_exit(ji->client_attr, "xauth_path", param_val, CONFIG_DATA);
      }

    if ((param_val = get_trq_param("CLIENTRETRY", torque_cfg_buf)) != NULL)
      {
      /* The value of this will be verified later */
      hash_add_or_exit(ji->client_attr, "cnt2server_retry", param_val, CONFIG_DATA);
      }

    if ((param_val = get_trq_param("VALIDATEGROUP", torque_cfg_buf)) != NULL)
      {
      if (getgrgid(getgid()) == NULL)
        print_qsub_usage_exit("qsub: cannot validate submit group.");

      hash_add_or_exit(ji->client_attr, "validate_group", param_val, CONFIG_DATA);
      }

    if ((param_val = get_trq_param("DEFAULTCKPT", torque_cfg_buf)) != NULL)
      {
      hash_add_or_exit(ji->job_attr, ATTR_c, param_val, CONFIG_DATA);
      }

    if ((param_val = get_trq_param("VALIDATEPATH", torque_cfg_buf)) != NULL)
      {
      if (!strcasecmp(param_val, "false"))
        hash_del_item(ji->client_attr, "validate_path");
      }
    if ((param_val = get_trq_param("RERUNNABLEBYDEFAULT", torque_cfg_buf)) != NULL)
      {
      if (!strcasecmp(param_val, "false"))
        hash_add_or_exit(ji->job_attr, ATTR_r, "FALSE", STATIC_DATA);
      }
    if ((param_val = get_trq_param("FAULT_TOLERANT_BY_DEFAULT", torque_cfg_buf)) != NULL)
      {
      if (!strcasecmp(param_val, "true"))
        hash_add_or_exit(ji->job_attr, ATTR_r, "TRUE", STATIC_DATA);
      }
    if ((param_val = get_trq_param("HOST_NAME_SUFFIX", torque_cfg_buf)) != NULL)
      {
      if (param_val != NULL)
        {
        host_name_suffix = (char *)calloc(1, strlen(param_val));
        strcpy(host_name_suffix, param_val);
        }
      }
    }    /* END if (load_config(torque_cfg_buf,sizeof(torque_cfg_buf)) == 0) */
  }

/**
 * display the error
 * display the qsub usage information
 * exit
 */
void print_qsub_usage_exit(const char *error_msg)
  {
  static char usage[] =
    "usage: qsub [-a date_time] [-A account_string] [-b secs]\n\
    [-c [ none | { enabled | periodic | shutdown |\n\
    depth=<int> | dir=<path> | interval=<minutes>}... ]\n\
    [-C directive_prefix] [-d path] [-D path]\n\
    [-e path] [-h] [-I] [-j oe|eo|n] [-k {oe}] [-K <kill delay seconds>] \n\
    [-l resource_list] [-m n|{abe}] [-M user_list] [-N jobname] [-o path] \n\
    [-p priority] [-P proxy_user [-J <jobid]] [-q queue] [-r y|n] \n\
    [-S path] [-t number_to_submit] [-T type]  [-u user_list] [-w] path\n";

  /* need secondary usage since there appears to be a 512 byte size limit */

  static char usage2[] =
    "    [-W additional_attributes] [-v variable_list] [-V ] [-x] [-X] [-z] [script]\n";
    
  fprintf(stderr,"[%s]\n\n%s%s\n", error_msg, usage, usage2);

  exit(2);
  }

void add_submit_args_to_job(
  job_data_container *job_attr,
  int                argc,
  char               **argv)

  {
  int alloc_len = 1; /* Trailing \0 */
  int argi = 0;
  char *submit_args_str = NULL;
  for (argi = 1;argi < argc;argi++)
    {
    alloc_len += strlen(argv[argi]) + 1;
    }

  if (alloc_len > 0)
    {
    calloc_or_fail(&submit_args_str, alloc_len,
        "qsub:submit args out of memory");

    for (argi = 1;argi < argc;argi++)
      {
      strcat(submit_args_str, argv[argi]);

      if (argi != argc - 1)
        {
        strcat(submit_args_str, " ");
        }
      }
    hash_add_or_exit(job_attr, ATTR_submit_args, submit_args_str, CMDLINE_DATA);
    }
  }

void set_minwclimit(
  job_data_container *res_attr)

  {
  job_data *tmp_job_info;
  int dash_pos;
  if (hash_find(res_attr, "walltime", &tmp_job_info))
    {
    /* if walltime range specified, break into minwclimit and walltime resources */
    if((dash_pos = tmp_job_info->value.find('-') != std::string::npos))
      {
      std::string minwclimit =  tmp_job_info->value.substr(dash_pos);
      tmp_job_info->value = tmp_job_info->value.substr(dash_pos + 1);
      /* set minwclimit to min walltime range value */
      hash_add_or_exit(res_attr, "minwclimit", minwclimit.c_str(), LOGIC_DATA);
      /* Over write existing walltime value */
      hash_add_or_exit(res_attr, "walltime", tmp_job_info->value.c_str(), LOGIC_DATA);
      }
    }
  }

void add_variable_list(

  job_info           *ji,
  const char         *var_name,
  job_data_container *src_hash)

  {
  int                total_len = 0;
  int                count = 0;
  int                pos = 0;
  char              *var_list = NULL;
  job_data          *en;
  job_data_iterator *it = NULL;

  if (src_hash != NULL)
    {
    src_hash->lock();
    it = src_hash->get_iterator();
    src_hash->unlock();
    }

  job_data *v_value = NULL;

  /* if -v was used then it needs to be included as well. */
  if (hash_find(ji->job_attr, var_name, &v_value) != 0)
    {
    /* add the length of this + 1 for the comma */
    total_len = v_value->value.length() + 1;
    }

  total_len += hash_strlen(ji->user_attr);
  count = hash_count(ji->user_attr);
  total_len += count*2;
  var_list = (char *)calloc(1, total_len);

  if (v_value != NULL)
    {
    strcpy(var_list, v_value->value.c_str());
    if (src_hash != NULL)
      strcat(var_list, ",");
    }

  if (it != NULL)
    {
    src_hash->lock();
    while((en = it->get_next_item()) != NULL)
      {
      pos++;
      strcat(var_list, en->name.c_str());
      strcat(var_list, "=");
      if (en->value.length() != 0)
        {
        strcat(var_list, en->value.c_str());
        }
      if (pos != count)
        {
        strcat(var_list, ",");
        }
      }
    src_hash->unlock();

    delete it;
    }

  hash_add_or_exit(ji->job_attr, var_name, var_list, CMDLINE_DATA);
  } /* END add_variable_list() */



/**
 * Handle --about and --version, and any other options that would cause
 * the program to exit quickly instead of doing normal workflow.
 * This function should exit() if a short-circuit turns out to be what
 * the user requested.
 */
void process_early_opts(
    
  int    argc,
  char **argv)

  {
  int i;

  for (i = 0; i < argc; ++i)
    {
    char const *name = argv[i];

    if (name[0] == '-' && name[1] == '-')
      {
      name += 2;
      if (strcmp(name, "about") == 0)
        TShowAbout_exit();
      else if (strcmp(name, "version") == 0)
        {
        fprintf(stderr, "Version: %s\nCommit: %s\n", PACKAGE_VERSION, GIT_HASH);
        exit(0);
        }
      }
    }
  } /* END process_early_opts() */



/*
 * retry_submit_error()
 *
 * Returns true if the error from pbs_submit_hash() is one that may change
 * if retried, false if its a permanent error
 *
 * @param error - the error code returned from pbs_submit_hash()
 * @return true if the error is transient, false otherwise
 */

bool retry_submit_error(

  int error)

  {

  switch (error)

    {
    case PBSE_NONE:
    case PBSE_NOATTR:
    case PBSE_ATTRRO:
    case PBSE_IVALREQ:
    case PBSE_UNKREQ:
    case PBSE_PERM:
    case PBSE_BADHOST:
    case PBSE_JOBEXIST:
    case PBSE_UNKQUE:
    case PBSE_QUNOENB:
    case PBSE_QACESS:
    case PBSE_BADUSER:
    case PBSE_ATTRTYPE:
    case PBSE_UNKRESC:
    case PBSE_QUENODFLT:
    case PBSE_ROUTEREJ:
    case PBSE_BADSCRIPT:
    case PBSE_BADGRP:
    case PBSE_MAXQUED:
    case PBSE_EXLIMIT:
    case PBSE_BADACCT:
    case PBSE_SVRDOWN:
    case PBSE_JOBTYPE:
    case PBSE_MAXUSERQUED:
    case PBSE_NORERUNABLE:
    case PBSE_NONONRERUNABLE:
    case PBSE_BAD_ARRAY_REQ:
    case PBSE_NOFAULTTOLERANT:
    case PBSE_NOFAULTINTOLERANT:

      return(false);
    }

  return(true);
  } // END retry_submit_error() 



/** 
 * qsub main 
 *
 * @see process_opts() - child
 */

void main_func(

  int    argc,  /* I */
  char **argv,  /* I */
  char **envp)  /* I */

  {

  int               errflg;                         /* option error */
  char              script[MAXPATHLEN + 1] = ""; /* name of script file */
  char              script_tmp[MAXPATHLEN + 1] = "";    /* name of script file copy */
  int               script_index;
  char             *bnp;
  FILE             *script_fp;                    /* FILE pointer to the script */
  char             *destination = NULL;           /* Changed from global to local */
  char             *s_n_out;                      /* server part of destination */
  int               sock_num;                     /* return from pbs_connect */
  char             *errmsg = NULL;                /* return from pbs_geterrmsg */
  int               local_errno = 0;
  int               job_is_interactive = FALSE;
  int               prefix_index = -1;

  struct stat       statbuf;

  struct sigaction  act;

  int               script_idx = 0;
  int               idx;
  job_data         *tmp_job_info = NULL;
  /* Allocate Memmgr */
  int               debug = FALSE;
  job_info          ji;

  /**
   * Before we go to the trouble of allocating memory, initializing structures,
   * and setting up for ordinary workflow, check options to see if we'll be
   * short-circuiting. If yes, then we'll exit without ever returning to main_func.
   */
  process_early_opts(argc, argv);
  
  /* The order of precedence for processing options follows:
   * 1 - processing logic (includes submitfilter)
   * 2 - cmdline information
   * 3 - #PBS information & script
   * 4 - config file options
   * 5 - environment variables
   * 6 - predefined code defaults
   *
   * These are processed and added to the hash in reverse order.
   * The default hashmap functionality is to remove existing values to add
   *  new ones.
   */


  /* (5) adds all env variables to a tmp hash */
  set_env_opts(ji.user_attr, envp);
  /* (6) set option default job values */
  set_job_defaults(&ji);
  /* (6) Adds client default options */
  set_client_attr_defaults(ji.client_attr);
  /* The following call  also replaces the functionality of set_job_env
   * up to the v_opt and V_opt sections. Those are replaced below */
  /* The names currently used differ from the actual anvironment names,
   * this adds an expected set */
  update_job_env_names(&ji);
  add_submit_args_to_job(ji.job_attr, argc, argv);
  debug = hash_find(ji.job_attr, "pbsdebug", &tmp_job_info); /* Set debug state */

  /* (4) process config file options */
  process_config_file(&ji);

  /* check/set submit filter_path */
  if (validate_submit_filter(ji.job_attr) == -1)
    {
     hash_find(ji.job_attr, ATTR_pbs_o_submit_filter, &tmp_job_info);
     fprintf(stderr,
             "qsub: invalid submit filter: \"%s\"\n",
             tmp_job_info->value.c_str());

     exit(1);
    }

  /* NOTE:  load config before processing opts since config may modify how opts are handled */

#ifdef linux
  optind = 0;  /* prime getopt's starting point */
#else
  optind = 1;  /* prime getopt's starting point */
#endif

  script_index = find_job_script_index(optind + 1, &job_is_interactive, &prefix_index, argc, argv);

  if (script_index != -1)
    {
    snprintf(script, sizeof(script), "%s", argv[script_index]);
    /* store the script so it can be used later (e.g. '-x' option) */
    hash_add_or_exit(ji.client_attr, "cmdline_script", script, CMDLINE_DATA);
    }

  if (prefix_index != -1)
    hash_add_or_exit(ji.client_attr, "pbs_dprefix", argv[prefix_index], CMDLINE_DATA);

  script_idx = argc - optind;
  if (hash_find(ji.job_attr, ATTR_inter, &tmp_job_info))
    {
    for (idx = 1; idx < script_idx; idx++)
      {
      int len = strlen(script);
      snprintf(script + len, sizeof(script) - len, " %s", argv[optind + idx]);
      }
    }

  /* if script is empty, get standard input */
  if (!strcmp(script, "") || !strcmp(script, "-"))
    {
    if (hash_find(ji.job_attr, ATTR_N, &tmp_job_info) == FALSE)
      hash_add_or_exit(ji.job_attr, ATTR_N, "STDIN", CMDLINE_DATA);

    if (job_is_interactive == FALSE)
      {
      /* (3) */
      if ((errflg = get_script(
                      argc,
                      argv,
                      stdin,
                      script_tmp,    /* O */
                      &ji)) != 0)
        {
        unlink(script_tmp);

        exit(1);
        }

      }
    }    /* END if (!strcmp(script,"") || !strcmp(script,"-")) */
  else
    {
    /* non-empty script, read it for directives */

    if (stat(script, &statbuf) < 0)
      {
      fprintf(stderr, "qsub: script file '%s' cannot be loaded - %s\n",
              script,
              strerror(errno));

      exit(1);
      }

    if (!S_ISREG(statbuf.st_mode))
      {
      fprintf(stderr, "qsub: script not a file\n");

      exit(1);
      }

    if ((script_fp = fopen(script, "r")) != NULL)
      {
      if (hash_find(ji.job_attr, ATTR_N, &tmp_job_info) == FALSE)
        {
        if ((bnp = strrchr(script, (int)'/')))
          bnp++;
        else
          bnp = script;

        if (check_job_name(bnp, 0) == 0)
          hash_add_or_exit(ji.job_attr, ATTR_N, bnp, CMDLINE_DATA);
        else
          print_qsub_usage_exit("qsub: cannot form a valid job name from the script name");
        }

      /* (3) */
      if ((errflg = get_script(
                      argc,
                      argv,
                      script_fp,
                      script_tmp, /* O */
                      &ji)) != 0)
        {
        fclose(script_fp);
        unlink(script_tmp);

        exit(1);
        }
      }    /* END if ((script_fp = fopen(script,"r")) != NULL) */
    else
      {
      unlink(script_tmp);
      print_qsub_usage_exit("qsub: opening script file:");
      }
    }    /* END else (!strcmp(script,"") || !strcmp(script,"-")) */
 
  /* (2) cmdline options */
  process_opts(argc, argv, &ji, CMDLINE_DATA);

  if (((optind + 1) < argc) && (hash_find(ji.job_attr, ATTR_inter, &tmp_job_info) == FALSE))
    print_qsub_usage_exit("index issues");
  
  post_check_attributes(&ji, script_tmp);

  add_new_request_if_present(&ji);

  if (hash_find(ji.client_attr, "DISPLAY", &tmp_job_info))
    {
    char *x11authstr;
    hash_find(ji.client_attr, "xauth_path", &tmp_job_info);
    /* get the DISPLAY's auth proto, data, and screen number */
    if (debug)
      {
      fprintf(stderr, "xauth_path=%s\n",
              tmp_job_info->value.c_str());
      }

    if ((x11authstr = x11_get_proto((char *)tmp_job_info->value.c_str(), debug)) != NULL)
      {
      /* stuff this info into the job */
      hash_add_or_exit(ji.job_attr, ATTR_forwardx11, x11authstr, ENV_DATA);
      
      if (debug)
        fprintf(stderr, "x11auth string: %s\n",
                x11authstr);
      }
    else
      print_qsub_usage_exit("qsub: Failed to get xauth data (check $DISPLAY variable)");
    }


  /* interactive job can not be job array */
  if (hash_find(ji.job_attr, ATTR_inter, &tmp_job_info) &&
      hash_find(ji.job_attr, ATTR_t, &tmp_job_info))
    {
    fprintf(stderr, "qsub: interactive job can not be job array.\n");

    unlink(script_tmp);

    exit(2);
    }

  if (hash_find(ji.job_attr, ATTR_inter, &tmp_job_info) &&
      ((isatty(0) == 0) || (isatty(1) == 0)))
    {
    if (hash_find(ji.job_attr, ATTR_intcmd, &tmp_job_info))
      {
      have_terminal = FALSE;
      }
    else
      {
      fprintf(stderr, "qsub:\tstandard input and output must be a terminal for \n\tinteractive job submission\n");
      
      unlink(script_tmp);
      
      close(inter_sock);
      
      exit(1);
      }
    }

  /* Given the change in parameter management, this is moving to
   * the top of this function for ease of understanding */
  server_out[0] = '\0';

  if (hash_find(ji.client_attr, "destination", &tmp_job_info))
    {
    char *q_n_out;                      /* queue part of destination */
    if (parse_destination_id((char *)tmp_job_info->value.c_str(), &q_n_out, &s_n_out))
      {
      fprintf(stderr, "qsub: illegally formed destination: %s\n",
        tmp_job_info->value.c_str());
  
      unlink(script_tmp);
  
      exit(2);
      }
    destination = (char *)tmp_job_info->value.c_str();
    if (notNULL(s_n_out))
      {
      strcpy(server_out, s_n_out);
      }
    }
  else
    {
    /* Currently if the destination is null, it is replaced downstream
     * with the server_list */
    calloc_or_fail(&destination, 2, "destination");
    destination[0] = '\0';
    }

  /* if walltime range specified, break into minwclimit and walltime */
  set_minwclimit(ji.job_attr);

  /* Root user submission not allowed */
  local_errno = PBSE_NONE;
  if (hash_find(ji.job_attr, ATTR_P, &tmp_job_info) == TRUE)
    {
    if (strcmp("root", tmp_job_info->value.c_str()) == 0)
      {
      local_errno = PBSE_BADUSER;
      }
    }
  else if ((getuid() == 0) && (geteuid() == 0))
    {
    local_errno = PBSE_BADUSER;
    }
  if (local_errno != PBSE_NONE)
    {
    fprintf(stderr, "qsub can not be run as root\n");
    unlink(script_tmp);
    exit(1);
    }

  /* connect to the server */

  if (hash_find(ji.client_attr, "cnt2server_retry", &tmp_job_info))
    {
    int tmpNum = atoi(tmp_job_info->value.c_str());
    if (tmpNum > 0)
      {
      cnt2server_conf(tmpNum); /* set number of seconds to retry */
      }
    }

  sock_num = cnt2server(server_out);

  if (sock_num <= 0)
    {
    local_errno = -1 * sock_num;

    if (server_out[0] != 0)
      fprintf(stderr, "qsub: cannot connect to server %s (errno=%d) %s\n",
        server_out,
        local_errno,
        pbs_strerror(local_errno));
    else
      fprintf(stderr, "qsub: cannot connect to server %s (errno=%d) %s\n",
        pbs_server,
        local_errno,
        pbs_strerror(local_errno));


    if (debug)
      {
      fprintf(stderr, "qsub: pbs_server daemon may not be running on host %s or hostname in file '$TORQUEHOME/server_name' may be incorrect)\n",
        pbs_server);
      }

    unlink(script_tmp);
    exit(local_errno);
    }

  /* Get required environment variables to be sent to the server.
   * -V functionality */

  if (hash_find(ji.client_attr, "user_attr", &tmp_job_info))
    add_variable_list(&ji, ATTR_v, ji.user_attr);

  /* disallow ^Z which hangs up MOM starting an interactive job */

  sigemptyset(&act.sa_mask);

  act.sa_handler = no_suspend;

  act.sa_flags = 0;

  if (sigaction(SIGTSTP, &act, (struct sigaction *)0) < 0)
    {
    pbs_disconnect(sock_num);
    unlink(script_tmp);
    print_qsub_usage_exit("unable to catch signals");
    }

  // If we are doing an interactive job, don't send the job script even if one exists
  if (hash_find(ji.job_attr, ATTR_inter, &tmp_job_info))
    {
    unlink(script_tmp);
    memset(script_tmp, 0, sizeof(script_tmp));
    }

  /* Send submit request to the server. */

  int retries = 0;

  do
    {
    local_errno = pbs_submit_hash(
                  sock_num,
                  ji.job_attr,
                  ji.res_attr,
                  script_tmp,
                  destination,
                  NULL,
                  &new_jobname,
                  &errmsg);

    /* If we get a timeout the server is busy. Let the user 
       know what is taking so long */
    if (local_errno == PBSE_TIMEOUT)
      fprintf(stderr, "Connection to server timed out. Trying again");
    else if ((local_errno == PBSE_BADDEPEND) &&
             (alternate_dependency != NULL))
      {
      // Replace the old dependency string with the new one
      hash_add_or_exit(ji.job_attr, ATTR_depend, alternate_dependency, alternate_data_type);
      }
    else if ((local_errno != PBSE_STAGEIN) &&
	           (local_errno != PBSE_NOCOPYFILE) &&
	           (local_errno != PBSE_DISPROTO) &&
	           (local_errno != PBSE_SERVER_BUSY))
      {
      retries = MAX_RETRIES;
      }


    } while((++retries < MAX_RETRIES) && (local_errno != PBSE_NONE));

  if (alternate_dependency != NULL)
    free(alternate_dependency);

  if (local_errno != PBSE_NONE)
    {
    if (errmsg == NULL)
      {
      errmsg = pbs_strerror(local_errno);
      }
    
    if (errmsg != NULL)
      fprintf(stderr, "qsub: submit error (%s)\n", errmsg);
    
    else
      fprintf(stderr, "qsub: Error (%d - %s) submitting job\n",
              local_errno, pbs_strerror(local_errno));

    pbs_disconnect(sock_num);
    unlink(script_tmp);

    exit(local_errno);
    }
  else
    {
    if ((hash_find(ji.client_attr, "no_jobid_out", &tmp_job_info) == FALSE) &&
        (hash_find(ji.job_attr, ATTR_inter, &tmp_job_info) == FALSE))
      printf("%s\n", new_jobname);
    }

  /* disconnet from the server. */

  pbs_disconnect(sock_num);

  unlink(script_tmp);

  /* is this an interactive job ??? */

  if (hash_find(ji.job_attr, ATTR_inter, &tmp_job_info))
    interactive(ji.client_attr);

  }  /* END main_func() */
