#include "license_pbs.h" /* See here for the software license */
#include <pbs_config.h>   /* the master config generated by configure */
#include "lib_mom.h" /* header */

#include <string>
#include <vector>
#include <assert.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <dirent.h>
#include <errno.h>
#include <strings.h>
#include <mntent.h>
#include <asm/types.h>
#include <time.h>
#include <sys/quota.h>
#include <sys/time.h>
#include <sys/procfs.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <sys/vfs.h>
#include <sys/sysmacros.h>
#include <sys/resource.h>
#include <signal.h>
#include <syscall.h>
#include <ctype.h>
#include <string.h>
#include <csv.h>
#include <fcntl.h>
#include <map>

/* needed for oom_adj */
#include <linux/limits.h>

#ifdef Q_6_5_QUOTAON
/* remap dqblk for SUSE 9.0 */
#define dqblk if_dqblk
#endif /* Q_6_5_QUOTAON */

/*
#ifndef dqblk
#include <linux/quotaio_v1.h>
#define dqblk v1_disk_dqblk
#endif
*/

#include "pbs_error.h"
#include "portability.h"
#include "list_link.h"
#include "server_limits.h"
#include "attribute.h"
#include "resource.h"
#include "pbs_job.h"
#include "log.h"
#include "mom_mach.h"
#include "mom_func.h"
#include "resmon.h"
#include "utils.h"
#include "../rm_dep.h"
#include "pbs_nodes.h"
#include "node_frequency.hpp"
#ifdef PENABLE_LINUX26_CPUSETS
#include "pbs_cpuset.h"
#endif
#include "mom_config.h"
#include "timer.hpp"

#ifdef PENABLE_LINUX_CGROUPS
#include "machine.hpp"
#include "trq_cgroups.h"
#include "complete_req.hpp"
#include "req.hpp"
#endif

#ifdef USE_RESOURCE_PLUGIN
#include "trq_plugin_api.h"
#endif


/*
** System dependent code to gather information for the resource
** monitor for a Linux i386 machine.
**
** Resources known by this code:
**  cput      cpu time for a pid or session
**  mem       memory size for a pid or session in KB
**  resi      resident memory size for a pid or session in KB
**  sessions  list of sessions in the system
**  pids      list of pids in a session
**  nsessions number of sessions in the system
**  nusers    number of users in the system
**  totmem    total memory size in KB
**  availmem  available memory size in KB
**  ncpus     number of cpus
**  physmem   physical memory size in KB
**  size      size of a file or filesystem
**  idletime  seconds of idle time
**  walltime  wall clock time for a pid
**  loadave   current load average
**  quota     quota information (sizes in kb)
**  netload   number of bytes transferred for all interfaces
*/

#ifndef MAX_LINE
#define MAX_LINE 1024
#endif

#ifndef TRUE
#define FALSE 0
#define TRUE 1
#endif /* TRUE */

static char    procfs[] = "/proc";
static DIR    *pdir = NULL;
int     pagesize;

pid2procarrayindex_map_t pid2procarrayindex_map;

extern char *ret_string;

extern time_t   time_now;

extern  int     LOGLEVEL;

#define TBL_INC 200            /* initial proc table */
#define PMEMBUF_SIZE  2048

#ifdef PENABLE_LINUX_CGROUPS
extern Machine this_node;
extern char    mom_alias[];
#endif

proc_stat_t   *proc_array = NULL;
static int            nproc = 0;
static int            max_proc = 0;

extern pid2jobsid_map_t pid2jobsid_map; 

/*
** external functions and data
*/
extern job_pid_set_t    global_job_sid_set;
extern struct  config          *search(struct config *,char *);
extern struct  rm_attribute    *momgetattr(char *);
extern long     system_ncpus;
#ifdef NUMA_SUPPORT
extern int       num_node_boards;
extern nodeboard node_boards[];
extern int       numa_index;
#else
extern char  path_meminfo[MAX_LINE];
#endif /* NUMA_SUPPORT */

/*
** local functions and data
*/
static const char *resi     (struct rm_attribute *);
static const char *totmem   (struct rm_attribute *);
static const char *availmem (struct rm_attribute *);
static const char *physmem  (struct rm_attribute *);
static const char *ncpus    (struct rm_attribute *);
static const char *walltime (struct rm_attribute *);
static const char *quota    (struct rm_attribute *);
static const char *netload  (struct rm_attribute *);
static const char *cpuclock (struct rm_attribute *);
#ifdef NUMA_SUPPORT
const char *cpuact   (struct rm_attribute *);
#endif
#ifdef USELIBMEMACCT
#ifdef __cplusplus
extern "C"
{
#endif
long long   get_memacct_resi(pid_t pid);
extern long get_weighted_memory_size(pid_t);
#ifdef __cplusplus
}
#endif
#endif

#ifndef mbool_t
#define mbool_t char
#endif /* mbool_t */

mbool_t ProcIsChild(char *,pid_t,char *);

extern const char *loadave(struct rm_attribute *);
extern const char *nullproc(struct rm_attribute *);

time_t wait_time = 10;

#ifdef NUMA_SUPPORT
typedef struct proc_cpu
  {
  unsigned long long idle_total;
  unsigned long long busy_total;
  } proc_cpu_t;
static proc_cpu_t *cpu_array = NULL;
#endif

/*
** local resource array
*/

struct config dependent_config[] =
  {
    { "resi",     {resi}     },
    { "totmem",   {totmem}   },
    { "availmem", {availmem} },
    { "physmem",  {physmem}  },
    { "ncpus",    {ncpus}    },
#ifdef NUMA_SUPPORT
    { "loadave",  {cpuact}   },
#else
    { "loadave",  {loadave}  },
#endif
    { "walltime", {walltime} },
    { "quota",    {quota}    },
    { "netload",  {netload}  },
    { "size",     {size}     },
    { "cpuclock", {cpuclock} },
    { NULL,       {nullproc} }
  };

unsigned linux_time = 0;

/*
 * support routine for getting system time -- sets linux_time
 */

void proc_get_btime(void)

  {
  FILE *fp;
  char  label[256];

  if ((fp = fopen("/proc/stat", "r")) == NULL)
    {
    return;
    }

  while (!feof(fp))
    {
    if (fscanf(fp, "%s", label) != 1)
      {
      fclose(fp);
      return;
      }

    if (strcmp(label, "btime"))
      {
      if (fscanf(fp, "%*[^\n]%*c") != 0)
        {
        fclose(fp);
        return;
        }
      }
    else
      {
      if (fscanf(fp, "%u", &linux_time) != 1) {}

      fclose(fp);

      return;
      }
    }    /* END while (!feof(fp)) */

  fclose(fp);

  if (LOGLEVEL >= 7)
    {
    sprintf(log_buffer, "DRIFT debug: getting btime, setting linux_time to %ld", (long)linux_time);
    log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_MOM, "linux_time", log_buffer);
    }

  return;
  }  /* END proc_get_btime() */



/*
 * skip_space_delimited_values()
 *
 * skips number_to_skip space delimited values from str_to_advance and advances the
 * pointer appropriately
 *
 * @param number_to_skip - the number of space delimited values to skip
 * @param str_to_advance - the string we're advancing
 */

void skip_space_delimited_values(

  int    number_to_skip,
  char **str_to_advance)

  {
  if ((str_to_advance == NULL) ||
      (*str_to_advance == NULL))
    return;

  int   spaces_found = 0;
  char *curr_ptr = *str_to_advance;

  while (spaces_found < number_to_skip)
    {
    curr_ptr = strchr(curr_ptr, ' ');

    if (curr_ptr == NULL)
      break;

    spaces_found++;
    curr_ptr++;
    }

  *str_to_advance = curr_ptr;
  } /* END skip_space_delimited_values() */



/*
 * Convert jiffies to seconds.
 *
 * Hertz is sysconf(_SC_CLK_TCK) in get_proc_stat()
*/

#define JTOS(x)  (x) / Hertz;



/*
 * populate_stats_from_the_buffer
 *
 * reads a buffer in /proc/pid/stat format which is:
 * '%d (%s) %c %d %d %d %*d %*d %u %*u %*u %*u %*u %lu %lu %lu ' +
 * '%lu %*ld %*ld %*u %*ld %lu %llu %lld' these values correspond to:
 * <pid> (<command>) <state> <ppid> <pgrp> <session> <tty_nr>* <tpgid>*
 * <flags> <minflt>* <cminflt>* <majflt>* <cmajflt> <utime> <stime>
 * <cutime> <cstime> <priority>* <nice>* <0>* <itrealvalue>* <starttime>
 * <vsize> <rss> ...
 * and populates the relevant values into ps. Values marked with * are
 * ignored.
 *
 * @param buffer - the buffer containing this information. I
 * @param ps - the process statistics information struct. O
 * @param path - the buffer where we're saving the command name. O
 * @param path_size - the size of the buffer for the command. O
 * @return - PBSE_NONE on success, -1 on failure
 */

int populate_stats_from_the_buffer(

  char          *buffer,
  proc_stat_t   &ps,
  char          *path,
  int            path_size)

  {
  char          *lastbracket = strrchr(buffer, ')');
  char          *curr_ptr;
  char          *ptr;
  unsigned long  jstarttime;

  static int Hertz  = 0;
  static int Hertz_errored = 0;

  if (Hertz <= 0)
    {
    Hertz = sysconf(_SC_CLK_TCK);  /* returns 0 on error */

    if (Hertz <= 0)
      {
      /* FAILURE */

      if (!Hertz_errored)
        log_err(errno, "get_proc_stat", "sysconf(_SC_CLK_TCK) failed, unable to monitor processes");

      Hertz_errored = 1;

      return(-1);
      }
    }

  Hertz_errored = 0;


  if (lastbracket == NULL)
    return(-1);

  *lastbracket = '\0'; /* We basically split the string here, overwriting the ')'. */

  lastbracket++;

  ps.pid = strtol(buffer, &curr_ptr, 10);

  ptr = strchr(curr_ptr, '(');

  if (ptr == NULL)
    return(-1);

  snprintf(path, path_size, "%s", ptr + 1);

  curr_ptr = lastbracket;

  // last bracket is currently in the format
  //  '%c %d %d %d %*d %*d %u %*u %*u %*u %*u %lu %lu %lu' +
  //  ' %lu %*ld %*ld %*u %*ld %lu %llu %lld'

  // skip the ' ' and read the state
  curr_ptr++;
  ps.state = *curr_ptr;

  // move past this value and the next space
  curr_ptr += 2;

  // read the ppid and skip a space
  ps.ppid = strtol(curr_ptr, &curr_ptr, 10);
  curr_ptr++;

  // read the pgrp and skip a space
  ps.pgrp = strtol(curr_ptr, &curr_ptr, 10);
  curr_ptr++;

  // read the session and skip a space
  ps.session = strtol(curr_ptr, &curr_ptr, 10);
  curr_ptr++;

  // skip the next two values
  skip_space_delimited_values(2, &curr_ptr);

  // get the flags and skip a space
  ps.flags = strtol(curr_ptr, &curr_ptr, 10);
  curr_ptr++;

  // skip the next 4 values
  skip_space_delimited_values(4, &curr_ptr);

  // read the utime - cycles that the process has been in user mode
  ps.utime = strtoll(curr_ptr, &curr_ptr, 10);
  curr_ptr++;

  // read the stime - cycles that the process has been in system mode
  ps.stime = strtoll(curr_ptr, &curr_ptr, 10);
  curr_ptr++;

  // read the cutime - cycles that the process' children have been in user mode
  ps.cutime = strtoll(curr_ptr, &curr_ptr, 10);
  curr_ptr++;

  // read the cstime - cycles that the process' children have been in system mode
  ps.cstime = strtoll(curr_ptr, &curr_ptr, 10);
  curr_ptr++;

  // skip the next 4 values
  skip_space_delimited_values(4, &curr_ptr);

  // read the start time
  jstarttime = strtoll(curr_ptr, &curr_ptr, 10);
  curr_ptr++;

  // read the virtual memory size
  ps.vsize = strtoll(curr_ptr, &curr_ptr, 10);
  curr_ptr++;

  // read the stack size
  ps.rss = strtoll(curr_ptr, &curr_ptr, 10);

  ps.start_time = linux_time + JTOS(jstarttime);
  ps.name = path;

  ps.utime = JTOS(ps.utime);
  ps.stime = JTOS(ps.stime);
  ps.cutime = JTOS(ps.cutime);
  ps.cstime = JTOS(ps.cstime);

  return(PBSE_NONE);
  } /* END populate_stats_from_the_buffer() */



/* NOTE:  see 'man 5 proc' for /proc/pid/stat format and description */


/* FORMAT: <PID> <COMM> <STATE> <PPID> <PGRP> <SESSION> [<TTY_NR>] [<TPGID>] <FLAGS> [<MINFLT>] [<CMINFLT>] [<MAJFLT>] [<CMAJFLT>] <UTIME> <STIME> <CUTIME> <CSTIME> [<PRIORITY>] [<NICE>] [<0>] [<ITREALVALUE>] <STARTTIME> <VSIZE> <RSS> [<RLIM>] [<STARTCODE>] ... */

/*
 * Linux /proc status routine.
 *
 * Returns a pointer to a static proc_stat_t structure given
 * a process number, or NULL if there is an error.  Takes the
 * place of the ioctl call PIOCSTATUS in the irix imp of mom_mach.c
 *
 */

proc_stat_t *get_proc_stat(

  int pid)  /* I */

  {
  FUNCTION_TIMER
  static proc_stat_t  ps;
  static char         path[MAXLINE];
  static char         readbuf[MAXLINE << 2];
  FILE               *fd;

  struct stat         sb;

  sprintf(path, "/proc/%d/stat",
          pid);

  if ((fd = fopen(path, "r")) == NULL)
    {
    /* FAILURE */

    return(NULL);
    }

  /* use 'man 5 proc' for /proc/pid/stat format */

  if (!fgets(readbuf, sizeof(readbuf), fd))
    {
    fclose(fd);

    return(NULL);
    }

  if (populate_stats_from_the_buffer(readbuf, ps, path, sizeof(path)) != PBSE_NONE)
    {
    fclose(fd);
    return(NULL);
    }

  if (fstat(fileno(fd), &sb) == -1)
    {
    /* FAILURE */

    fclose(fd);

    return(NULL);
    }

  ps.uid = sb.st_uid;

  /* SUCCESS */

  fclose(fd);

  return(&ps);
  }  /* END get_proc_stat() */


#ifdef USELIBMEMACCT
/*
 * Retrieve weighted RSS value for process with pid from memacctd.
 * Returns the value in bytes on success, returns -1 on failure.
 */

long long get_memacct_resi(pid_t pid)
  {
  long long  w_rss;

  if ((w_rss = get_weighted_memory_size(pid)) == -1)
    {
    sprintf(log_buffer, "get_weighted_memory_size(%d) failed", pid);
    log_err(errno, __func__, log_buffer);
    }

  return(w_rss);
  } /* END get_memacct_resi() */
#endif



/*
 * get_proc_mem_from_path()
 * @returns a pointer to a struct containing the memory information
 * @pre-cond: path must point to a valid path of a meminfo system file
 */

proc_mem_t *get_proc_mem_from_path(

  const char *path)

  {
  proc_mem_t *mm;
  FILE       *fp;
  char        str[32];
  long long   bfsz  = -1;
  long long   casz  = -1;
  long long   fcasz = -1;

  if ((fp = fopen(path,"r")) == NULL)
    {
    return(NULL);
    }

  mm = (proc_mem_t *)calloc(1, sizeof(proc_mem_t));

  if (fscanf(fp,"%30s",str) != 1)
    {
    fclose(fp);
    free(mm);

    return(NULL);
    }

  if (!strncmp(str,"total:",sizeof(str)))
    {
    /* old format */
    if (fscanf(fp,"%*[^\n]%*c") != 0)     /* remove text header */
      {
      fclose(fp);
      free(mm);

      return(NULL);
      }

    /* umu vmem patch */
    if (fscanf(fp, "%*s %llu %llu %llu %*u %lld %lld",
               &mm->mem_total,
               &mm->mem_used,
               &mm->mem_free,
               &bfsz,
               &casz) != 5)
      {
      fclose(fp);
      free(mm);

      return(NULL);
      }

    mm->mem_free += casz + bfsz;

    if (fscanf(fp, "%*s %llu %llu %llu %*[^\n]%*c",
               &mm->swap_total,
               &mm->swap_used,
               &mm->swap_free) != 3)
      {
      fclose(fp);
      free(mm);

      return(NULL);
      }
    }
  else
    {
    do
      {
      /* new format (kernel > 2.4) the first 'str' has been read */
      if (!strncmp(str, "MemTotal:", sizeof(str)))
        {
        if (fscanf(fp, "%llu",
                   &mm->mem_total) != 1)
          {
          fclose(fp);
          free(mm);

          return(NULL);
          }

        mm->mem_total *= 1024; /* the unit is kB */
        }
      else if (!strncmp(str, "MemFree:", sizeof(str)))
        {
        if (fscanf(fp, "%llu",
                   &mm->mem_free) != 1)
          {
          fclose(fp);
          free(mm);

          return(NULL);
          }

        mm->mem_free *= 1024;
        }
      else if (!strncmp(str, "Buffers:", sizeof(str)))
        {
        if (fscanf(fp, "%lld",
                   &bfsz) != 1)
          {
          fclose(fp);
          free(mm);

          return(NULL);
          }

        bfsz *= 1024;
        }
      else if (!strncmp(str, "Cached:", sizeof(str)))
        {
        if (fscanf(fp, "%lld",
                   &casz) != 1)
          {
          fclose(fp);
          free(mm);

          return(NULL);
          }

        casz *= 1024;
        }
      else if (!strncmp(str, "FilePages:", sizeof(str)))
        {
        if (fscanf(fp, "%lld",
                   &fcasz) != 1)
          {
          fclose(fp);
          free(mm);

          return(NULL);
          }

        fcasz *= 1024;
        }
      else if (!strncmp(str, "SwapTotal:", sizeof(str)))
        {
        if (fscanf(fp, "%llu",
                   &mm->swap_total) != 1)
          {
          fclose(fp);
          free(mm);
          return(NULL);
          }

        mm->swap_total *= 1024;
        }
      else if (!strncmp(str, "SwapFree:", sizeof(str)))
        {
        if (fscanf(fp, "%llu",
                   &mm->swap_free) != 1)
          {
          fclose(fp);
          free(mm);
          return(NULL);
          }

        mm->swap_free *= 1024;
        }
      }
    while (fscanf(fp, "%30s", str) == 1);
    }    /* END else */

  fclose(fp);

  if (bfsz >= 0 || casz >= 0)
    {
    if (bfsz > 0)
      mm->mem_free += bfsz;
    if (casz > 0)
      mm->mem_free += casz;
    }
  else if (fcasz > 0)
    {
    mm->mem_free += fcasz;
    }

  return(mm);
  } /* END get_proc_mem_from_path() */



proc_mem_t *get_proc_mem(void)

  {
  static proc_mem_t   ret_mm;
#ifdef NUMA_SUPPORT
  int                 i;
#else
  proc_mem_t         *mem;
#endif

#ifdef NUMA_SUPPORT
  ret_mm.mem_total  = 0;
  ret_mm.mem_used   = 0;
  ret_mm.mem_free   = 0;
  ret_mm.swap_total = 0;
  ret_mm.swap_used  = 0;
  ret_mm.swap_free  = 0;

  for (i = 0; i < node_boards[numa_index].num_nodes; i++)
    {
    proc_mem_t *node_mem = get_proc_mem_from_path(node_boards[numa_index].path_meminfo[i]);

    if (node_mem == NULL)
      return(NULL);

    ret_mm.mem_total  += node_mem->mem_total;
    ret_mm.mem_used   += node_mem->mem_used;
    ret_mm.mem_free   += node_mem->mem_free;
    ret_mm.swap_total += node_mem->swap_total;
    ret_mm.swap_used  += node_mem->swap_used;
    ret_mm.swap_free  += node_mem->swap_free;

    free(node_mem);
    }
#else
  mem = get_proc_mem_from_path(path_meminfo);

  if(mem == NULL)
    return (NULL);

  ret_mm.mem_total  = mem->mem_total;
  ret_mm.mem_used   = mem->mem_used;
  ret_mm.mem_free   = mem->mem_free;
  ret_mm.swap_total = mem->swap_total;
  ret_mm.swap_used  = mem->swap_used;
  ret_mm.swap_free  = mem->swap_free;

  free(mem);
#endif

  return(&ret_mm);
  }  /* END get_proc_mem() */




#ifdef PNOT

proc_mem_t *get_proc_mem(void)

  {
  static proc_mem_t mm;
  FILE     *fp;
  unsigned long m_tot, m_use, m_free;
  unsigned long s_tot, s_use, s_free;

  if ((fp = fopen(path_meminfo, "r")) == NULL)
    {
    return(NULL);
    }

  fscanf(fp, "%*[^\n]%*c");      /* remove text header */;

  fscanf(fp, "%*s %lu %lu %lu %*[^\n]%*c",
         &m_tot,
         &m_use,
         &m_free);

  fscanf(fp, "%*s %lu %lu %lu %*[^\n]%*c",
         &s_tot,
         &s_use,
         &s_free);

  mm.total = m_tot + s_tot;
  mm.used = m_use + s_use;
  mm.free = m_free + s_free;

  fclose(fp);

  return(&mm);
  }  /* END get_proc_mem() */

#endif /* PNOT */

/*
 * sets oom_adj score for current process
 * requires root privileges or CAP_SYS_RESOURCE to succeed
 */

static int oom_adj(int score)
  
  {
  pid_t pid;
  int   rc;
  int   fd;

  char oom_adj_path[PATH_MAX] = "";
  char adj_value[128] = "";
   
  /* max valid values are -1000 to 1000 */
  if ((score > 1000) ||
      (score < -1000))
    return -1;
  
  pid = getpid();
  
  snprintf(oom_adj_path, sizeof(oom_adj_path), "/proc/%d/oom_score_adj", pid);
   
  /* try and open the oom_score_adj file for writing first */
  /* if it fails to open then try oom_adj */
  if ((fd = open(oom_adj_path, O_RDWR)) == -1 )
    {
    if (score < -17)
      score = -17;
    else if (score > 15)
      score = 15;
     
    snprintf(oom_adj_path, sizeof(oom_adj_path), "/proc/%d/oom_adj", pid);

    if ((fd = open(oom_adj_path, O_RDWR)) == -1)
      return -1;
     
    if (snprintf(adj_value,sizeof(adj_value),"%d",score) < 0)
      return -1;
     
    rc = write(fd,adj_value,strlen(adj_value));
     
    close(fd);
    return rc;   
    }

  snprintf(adj_value,sizeof(adj_value),"%d",score);

  rc = write(fd,adj_value,strlen(adj_value));

  close(fd);
  return rc;
  }


void dep_initialize(void)

  {
  pagesize = getpagesize();

  if ((pdir = opendir(procfs)) == NULL)
    {
    log_err(errno, __func__, "opendir");

    return;
    }


  /* NOTE:  /proc/<pid>/oom_adj tunable is linux specific */
  /* LKF: make pbs_mom processes immune to oom killer's killing frenzy if requested*/
  if (mom_oom_immunize != 0)
    {

    if (oom_adj(-1000) < 0)
      {
      log_record(
        PBSEVENT_SYSTEM,
        PBS_EVENTCLASS_SERVER,
        __func__,
        "failed to make pbs_mom oom-killer immune");
      }
    else
      {
      log_record(
        PBSEVENT_SYSTEM,
        PBS_EVENTCLASS_SERVER,
        __func__,
        "mom is now oom-killer safe");
      }
    }

  proc_get_btime();

  return;
  }  /* END dep_initialize() */





void dep_cleanup(void)

  {
  log_record(PBSEVENT_SYSTEM, 0, __func__, "dependent cleanup");

  if (pdir)
    {
    closedir(pdir);

    pdir = NULL;
    }

  return;
  }





/*
 * This routine is called on each cycle of the main loop.
 */

void
dep_main_loop_cycle(void)
  {
  /* No periodic functions. */
  }





/*
 * Internal size decoding routine.
 *
 * Accepts a resource pointer and a pointer to the unsigned long integer
 * to receive the decoded value.  It returns a PBS error code, and the
 * decoded value in the unsigned long integer.
 *
 *  sizeof(word) = sizeof(int)
 */




static int mm_getsize(

  resource *pres,  /* I */
  unsigned long *ret)   /* O */

  {
  unsigned long value;

  if (pres->rs_value.at_type != ATR_TYPE_SIZE)
    {
    return(PBSE_ATTRTYPE);
    }

  value = pres->rs_value.at_val.at_size.atsv_num;

  if (pres->rs_value.at_val.at_size.atsv_units == ATR_SV_WORDSZ)
    {
    if (value > ULONG_MAX / sizeof(int))
      {
      return(PBSE_BADATVAL);
      }

    value *= sizeof(int);
    }

  if (value > (ULONG_MAX >> pres->rs_value.at_val.at_size.atsv_shift))
    {
    return(PBSE_BADATVAL);
    }

  *ret = (value << pres->rs_value.at_val.at_size.atsv_shift);

  return(PBSE_NONE);
  }  /* END mm_getsize() */





/*
 * Internal time decoding routine.
 *
 * Accepts a resource pointer and a pointer to the unsigned long integer
 * to receive the decoded value.  It returns a PBS error code, and the
 * decoded value of time in seconds in the unsigned long integer.
 */

static int mm_gettime(

  resource      *pres,
  unsigned long *ret)

  {
  if (pres->rs_value.at_type != ATR_TYPE_LONG)
    {
    return(PBSE_ATTRTYPE);
    }

  if (pres->rs_value.at_val.at_long < 0)
    {
    return(PBSE_BADATVAL);
    }

  *ret = pres->rs_value.at_val.at_long;

  return(PBSE_NONE);
  }


bool injob(

  job   *pjob,
  pid_t  pid)

  {
  pid_t  job_sid_of_pid;

  /* look up the owning job session id */

  /* see if pid is in the map */
  pid2jobsid_map_t::const_iterator iter = pid2jobsid_map.find(pid);
  if (iter == pid2jobsid_map.end())
    {
    /* not in map */
    return(false);
    }

  /* set to the job sid that was found */
  job_sid_of_pid = iter->second;

  /* find the pid in ji_job_pid_set */
  job_pid_set_t::const_iterator job_pid_set_iter = pjob->ji_job_pid_set->find(job_sid_of_pid);
  if (job_pid_set_iter != pjob->ji_job_pid_set->end())
    {
    /* found it */
    return(true);
    }

  /* Next, check the job's tasks to see if they match the session id */
  for (unsigned int i = 0; i < pjob->ji_tasks->size(); i++)
    {
    task *ptask = pjob->ji_tasks->at(i);

    if (job_sid_of_pid == ptask->ti_qs.ti_sid)
      return(true);
    }

  return(false);
  }  /* END injob() */


/*
 * Internal session CPU time decoding routine.
 *
 * Accepts a job pointer.  Returns the sum of all cpu time
 * consumed for all tasks executed by the job, in seconds,
 * adjusted by cputfactor.
 */

#ifndef PENABLE_LINUX_CGROUPS

unsigned long cput_sum(

  job *pjob)  /* I */

  {
  ulong          cputime;
  int            nps = 0;
  proc_stat_t   *ps;

  cputime = 0;

  if (LOGLEVEL >= 6)
    {
    sprintf(log_buffer, "pid2jobsid_map loop start - jobid = %s",
            pjob->ji_qs.ji_jobid);

    log_record(PBSEVENT_DEBUG, 0, __func__, log_buffer);
    }

  /* iterate over pids of all job session ids */
  for (pid2jobsid_map_t::const_iterator iter = pid2jobsid_map.begin();
       iter != pid2jobsid_map.end();
       iter++)
    {
    int index;
    pid_t pid;

    pid = iter->first;

    if (!injob(pjob, pid))
      {
      continue;
      }
    
    /* pid is in the job */

    /* get the index of this pid in the proc_array */

    /* see if pid is in the map */
    pid2procarrayindex_map_t::const_iterator pa_iter = pid2procarrayindex_map.find(pid);
    if (pa_iter == pid2procarrayindex_map.end())
      {
      /* not in map so skip */
      continue;
      }

    /* assign index to the index of the pid in proc_array */
    index = pa_iter->second;

    ps = &proc_array[index];

    nps++;

    cputime += (ps->utime + ps->stime + ps->cutime + ps->cstime);

    if (LOGLEVEL >= 6)
      {
      sprintf(log_buffer, "%s: session=%d pid=%d cputime=%lu (cputfactor=%f)",
              __func__,
              ps->session,
              ps->pid,
              cputime,
              cputfactor);

      log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
      }
    }    /* END for */

  if (nps == 0)
    pjob->ji_flags |= MOM_NO_PROC;
  else
    pjob->ji_flags &= ~MOM_NO_PROC;

  return((unsigned long)((double)cputime * cputfactor));
  }  /* END cput_sum() */




#else
#define LOCAL_BUF_SIZE 256

unsigned long cput_sum(

    job *pjob)

  {
  ulong        cputime = 0; 
  std::string  full_cgroup_path;
  int          fd;
  int          rc;
  char         buf[LOCAL_BUF_SIZE];

  pbs_attribute *pattr;
  pattr = &pjob->ji_wattr[JOB_ATR_req_information];
  if ((pattr != NULL) && (pattr->at_flags & ATR_VFLAG_SET) == 1)
    {
    unsigned int    req_index;
    int    rc;
    const char  *job_id = pjob->ji_qs.ji_jobid;

    complete_req  *cr = (complete_req *)pattr->at_val.at_ptr;
    rc = cr->get_req_index_for_host(mom_alias, req_index);
    if (rc != PBSE_NONE)
      {
      sprintf(buf, "Could not find req for host %s, job_id %s", mom_alias, pjob->ji_qs.ji_jobid);
      log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, buf);
      return(cputime);
      }

    req &host_req = cr->get_req(req_index);

    for (unsigned int task_index = 0; task_index < host_req.get_req_allocation_count(); task_index++)
      {
      unsigned long cput_used;
      std::string   task_host;

      host_req.get_task_host_name(task_host, task_index);
      if (task_hosts_match(task_host.c_str(), mom_alias) == false)
        {
        /* names don't match. Got to next task */
        continue;
        }

      rc = trq_cg_get_task_cput_stats(job_id, req_index, task_index, cput_used);
      if (rc != PBSE_NONE)
        continue;

      cr->set_task_cput_used(req_index, task_index, cput_used/NANO_SECONDS);

      }
    pjob->ji_flags &= ~MOM_NO_PROC;
    }

    /* This is not a -L request */

    full_cgroup_path = cg_cpuacct_path + pjob->ji_qs.ji_jobid + "/cpuacct.usage";

    fd = open(full_cgroup_path.c_str(), O_RDONLY);
    if (fd <= 0)
      {
      if (pjob->ji_cgroups_created == true)
        {
        sprintf(buf, "failed to open %s: %s", full_cgroup_path.c_str(), strerror(errno));
        log_err(-1, __func__, buf);
        }
      return(0);
      }

    rc = read(fd, buf, LOCAL_BUF_SIZE);
    if (rc == -1)
      {
      sprintf(buf, "failed to read %s: %s", full_cgroup_path.c_str(), strerror(errno));
      log_err(-1, __func__, buf);
      close(fd);
      return(0);
      }
    else if (rc != 0) /* if rc is 0 something is not right but it is not a critical error. Don't do anything */
    {
    ulong nano_seconds;
    /* successful read. Should be a number in nano-seconds */

    nano_seconds = atol(buf);
    cputime += nano_seconds;
    }

    /* convert the nano seconds to seconds */
    cputime = cputime/NANO_SECONDS;

    pjob->ji_flags &= ~MOM_NO_PROC;

    close(fd);
    

  return(cputime);
  
  }

#endif /* #ifndef PENABLE_LINUX_CGROUPS */


/*
 * Return TRUE if any process in the job is over limit for cputime usage.
 */

int overcpu_proc(

  job  *pjob,
  unsigned long  limit)  /* I */

  {
  ulong          cputime;

  proc_stat_t   *ps;

  /* iterate over pids of all job session ids */
  for (pid2jobsid_map_t::const_iterator iter = pid2jobsid_map.begin();
       iter != pid2jobsid_map.end();
       iter++)
    {
    int index;
    pid_t pid;

    pid = iter->first;

    if (!injob(pjob, pid))
      {
      continue;
      }

    /* pid is in the job */

    /* get the index of this pid in the proc_array */

    /* see if pid is in the map */
    pid2procarrayindex_map_t::const_iterator pa_iter = pid2procarrayindex_map.find(pid);
    if (pa_iter == pid2procarrayindex_map.end())
      {
      /* not in map so skip */
      continue;
      }

    /* assign index to the index of the pid in proc_array */
    index = pa_iter->second;

    ps = &proc_array[index];

    /* change from ps->cutime to ps->utime, and ps->cstime to ps->stime */

    cputime = (ulong)((double)(ps->utime + ps->stime) * cputfactor);

    if (cputime > limit)
      {
      return(TRUE);
      }
    }

  return(FALSE);
  }  /* END overcpu_proc() */



/*
 * Internal session virtual memory usage function.
 *
 * Returns the total number of bytes of address
 * space consumed by all current processes within the job.
 */

unsigned long long mem_sum(

  job *pjob)

  {
  unsigned long long  segadd;
  proc_stat_t        *ps;

  segadd = 0;

  if (LOGLEVEL >= 6)
    {
    sprintf(log_buffer, "pid2jobsid_map loop start - jobid = %s",
            pjob->ji_qs.ji_jobid);
    log_record(PBSEVENT_DEBUG, 0, __func__, log_buffer);
    }

  /* iterate over pids of all job session ids */
  for (pid2jobsid_map_t::const_iterator iter = pid2jobsid_map.begin();
       iter != pid2jobsid_map.end();
       iter++)
    {
    int index;
    pid_t pid;

    pid = iter->first;

    if (!injob(pjob, pid))
      {
      continue;
      }

    /* pid is in the job */

    /* get the index of this pid in the proc_array */

    /* see if pid is in the map */
    pid2procarrayindex_map_t::const_iterator pa_iter = pid2procarrayindex_map.find(pid);
    if (pa_iter == pid2procarrayindex_map.end())
      {
      /* not in map so skip */
      continue;
      }

    /* assign index to the index of the pid in proc_array */
    index = pa_iter->second;

    ps = &proc_array[index];

    segadd += ps->vsize;

    if (LOGLEVEL >= 6)
      {
      sprintf(log_buffer, "%s: session=%d pid=%d vsize=%llu sum=%llu",
              __func__,
              ps->session,
              ps->pid,
              ps->vsize,
              segadd);
      log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
      }

    }  /* END for */

  return(segadd);
  }  /* END mem_sum() */



#ifndef PENABLE_LINUX_CGROUPS
/*
 * Internal session memory usage function.
 *
 * Returns the total number of bytes of resident memory
 * consumed by all current processes within the job.
 */

unsigned long long resi_sum(

  job *pjob)

  {
  unsigned long long  resisize;
  proc_stat_t        *ps;
#ifdef USELIBMEMACCT
  long long                w_rss;
#endif

  resisize = 0;

  if (LOGLEVEL >= 6)
    {
    sprintf(log_buffer, "proc_array loop start - jobid = %s",
            pjob->ji_qs.ji_jobid);
    log_record(PBSEVENT_DEBUG, 0, __func__, log_buffer);
    }

  /* iterate over pids of all job session ids */
  for (pid2jobsid_map_t::const_iterator iter = pid2jobsid_map.begin();
       iter != pid2jobsid_map.end();
       iter++)
    {
    int index;
    pid_t pid;

    /* get the pid (key) from the map */
    pid = iter->first;

    if (!injob(pjob, pid))
      {
      /* pid is not in job */
      continue;
      }

    /* pid is in the job */

    /* get the index of this pid in the proc_array */

    /* see if pid is in the map */
    pid2procarrayindex_map_t::const_iterator pa_iter = pid2procarrayindex_map.find(pid);
    if (pa_iter == pid2procarrayindex_map.end())
      {
      /* not in map so skip */
      continue;
      }

    /* assign index to the index of the pid in proc_array */
    index = pa_iter->second;

    ps = &proc_array[index];


#ifdef USELIBMEMACCT

    /* Ask memacctd for weighted rss of pid, use this instead of ps->rss */

    w_rss = get_memacct_resi(ps->pid);

    if (w_rss == -1)
      resisize += ps->rss * pagesize;
    else
      resisize += w_rss;

    if (LOGLEVEL >= 6)
      {
      sprintf(log_buffer, "%s: session=%d pid=%d rss=%llu w_rss=%ld sum=%llu",
              __func__,
              ps->session,
              ps->pid,
              ps->rss * pagesize,
              w_rss,
              resisize);
      log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
      }

#else

    resisize += ps->rss * pagesize;

    if (LOGLEVEL >= 6)
      {
      sprintf(log_buffer, "%s: session=%d pid=%d rss=%llu sum=%llu",
              __func__,
              ps->session,
              ps->pid,
              ps->rss * pagesize,
              resisize);
      log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
      }

#endif

    }  /* END for  */

  return(resisize);
  }  /* END resi_sum() */

#else

unsigned long long resi_sum(

  job *pjob)

  {
  unsigned long long resisize = 0;
  std::string  full_cgroup_path;
  char         buf[LOCAL_BUF_SIZE];
  int          fd;
  int          rc;

  pbs_attribute *pattr;
  pattr = &pjob->ji_wattr[JOB_ATR_req_information];
  if ((pattr != NULL) && (pattr->at_flags & ATR_VFLAG_SET) != 0)
    {
    int          rc;
    unsigned int req_index;

    complete_req  *cr = (complete_req *)pattr->at_val.at_ptr;
    rc = cr->get_req_index_for_host(mom_alias, req_index);
    if (rc != PBSE_NONE)
      {
      sprintf(buf, "Could not find req for host %s, job_id %s", mom_alias, pjob->ji_qs.ji_jobid);
      log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, buf);
      return(resisize);
      }

    req &host_req = cr->get_req(req_index);
    for (unsigned int task_index = 0; task_index < host_req.get_req_allocation_count(); task_index++)
      {
      unsigned long long mem_used;
      std::string   task_host;

      host_req.get_task_host_name(task_host, task_index);
      if (task_hosts_match(task_host.c_str(), mom_alias) == false)
        {
        /* names don't match. Got to next task */
        continue;
        }

      mem_used = 0;
      rc = trq_cg_get_task_memory_stats(pjob->ji_qs.ji_jobid, req_index, task_index, mem_used);
      if (rc != PBSE_NONE)
        continue;

      cr->set_task_memory_used(req_index, task_index, mem_used);
      resisize += mem_used;
      }
    }

  full_cgroup_path = cg_memory_path + pjob->ji_qs.ji_jobid + "/memory.max_usage_in_bytes";

  fd = open(full_cgroup_path.c_str(), O_RDONLY);
  if (fd <= 0)
    {
    if (pjob->ji_cgroups_created == true)
      {
      sprintf(buf, "failed to open %s: %s", full_cgroup_path.c_str(), strerror(errno));
      log_err(-1, __func__, buf);
      }

    return(0);
    }

  rc = read(fd, buf, LOCAL_BUF_SIZE);
  if (rc == -1)
    {
    sprintf(buf, "failed to read %s: %s", full_cgroup_path.c_str(), strerror(errno));
    log_err(-1, __func__, buf);
    close(fd);
    return(0);
    }
  else if (rc != 0) 
    {
    int hardwareStyle;
    unsigned long long mem_read;

    mem_read = strtoull(buf, NULL, 10);

    hardwareStyle = this_node.getHardwareStyle();

    if (hardwareStyle == AMD) /* AMD adds everything up in the parent cgroup hierarchy and Intel does not */
      resisize = mem_read;
    else
      resisize += mem_read;
    }

  close(fd);

  return(resisize);
  }
#endif



/*
 * Return TRUE if any process in the job is over limit for virtual memory usage.
 */

int overmem_proc(

  job       *pjob,  /* I */
  unsigned long long  limit) /* I */

  {
  proc_stat_t        *ps;

  if (LOGLEVEL >= 6)
    {
    sprintf(log_buffer, "pid2jobsid_map loop start - jobid = %s",
            pjob->ji_qs.ji_jobid);

    log_record(PBSEVENT_DEBUG, 0, __func__, log_buffer);
    }

  /* iterate over pids of all job session ids */
  for (pid2jobsid_map_t::const_iterator iter = pid2jobsid_map.begin();
       iter != pid2jobsid_map.end();
       iter++)
    {
    int index;
    pid_t pid;

    pid = iter->first;

    if (!injob(pjob, pid))
      {
      continue;
      }

    /* pid is in the job */

    /* get the index of this pid in the proc_array */

    /* see if pid is in the map */
    pid2procarrayindex_map_t::const_iterator pa_iter = pid2procarrayindex_map.find(pid);
    if (pa_iter == pid2procarrayindex_map.end())
      {
      /* not in map so skip */
      continue;
      }

    /* assign index to the index of the pid in proc_array */
    index = pa_iter->second;

    ps = &proc_array[index];

    if (ps->vsize > limit)
      {
      return(TRUE);
      }
    }    /* END for */

  return(FALSE);
  }  /* END overmem_proc() */




extern char *msg_momsetlim;

/*
 * Internal error routine
 */

int error(

  const char *string,
  int   value)

  {
  char  *message;

  assert(string != NULL);
  assert(*string != '\0');

  message = pbse_to_txt(value);

  assert(message != NULL);
  assert(*message != '\0');

  fprintf(stderr, msg_momsetlim, string, message);

  fflush(stderr);

  return(value);
  }  /* END error() */


/*
 * Establish system-enforced limits for the job.
 *
 * Run through the resource list, checking the values for all items
 * we recognize.
 *
 * If set_mode is SET_LIMIT_SET, then also set hard limits for the
 * system enforced limits (not-polled).
 * If anything goes wrong with the process, return a PBS error code
 * and print a message on standard error.  A zero-length resource list
 * is not an error.
 *
 * If set_mode is SET_LIMIT_SET the entry conditions are:
 *     1. MOM has already forked, and we are called from the child.
 *     2. The child is still running as root.
 *     3.  Standard error is open to the user's file.
 *
 * If set_mode is SET_LIMIT_ALTER, we are being called to modify
 * existing limits.  Cannot alter those set by setrlimit (kernel)
 * because we are the wrong process.
 */

int mom_set_limits(

  job *pjob,     /* I */
  int  set_mode) /* SET_LIMIT_SET or SET_LIMIT_ALTER */

  {
  const char  *pname = NULL;
  int  retval;
  unsigned long value; /* place in which to build resource value */

  struct rlimit reslim;
  unsigned long vmem_limit = 0;
  unsigned long mem_limit = 0;

  /* NOTE:  log_buffer is exported */

  if (LOGLEVEL >= 2)
    {
    sprintf(log_buffer, "%s(%s,%s) entered",
            __func__,
            (pjob != NULL) ? pjob->ji_qs.ji_jobid : "NULL",
            (set_mode == SET_LIMIT_SET) ? "set" : "alter");

    log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);

    log_buffer[0] = '\0';
    }

  assert(pjob != NULL);

  assert(pjob->ji_wattr[JOB_ATR_resource].at_type == ATR_TYPE_RESC);

  /*
   * cycle through all the resource specifications,
   * setting limits appropriately.
   */

  memset(&reslim, 0, sizeof(reslim));

  /* set oom_adj score for the starting job */
  /* if immunize mode is set to on, we have to set child score to 0 */
  if ( (set_mode == SET_LIMIT_SET) && ( job_oom_score_adjust != 0 || mom_oom_immunize != 0 ) )
    {
    retval = oom_adj(job_oom_score_adjust);

    if ( LOGLEVEL >= 2 )
      {
      sprintf(log_buffer, "setting oom_adj '%s'",
        (retval != -1) ? "succeeded" : "failed");
      log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
      }

    }

  if (pjob->ji_wattr[JOB_ATR_resource].at_val.at_ptr != NULL)
    {
    std::vector<resource> *resources = (std::vector<resource> *)pjob->ji_wattr[JOB_ATR_resource].at_val.at_ptr;

    for (size_t i = 0; i < resources->size(); i++)
      {
      resource &r = resources->at(i);

      if (r.rs_defin != NULL)
        pname = r.rs_defin->rs_name;
      else
        pname = NULL;

      if (LOGLEVEL >= 2)
        {
        sprintf(log_buffer, "setting limit for attribute '%s'",
                (pname != NULL) ? pname : "NULL");

        log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);

        log_buffer[0] = '\0';
        }

      assert(r.rs_defin != NULL);

      assert(pname != NULL);

      assert(pname[0] != '\0');

      if (!strcmp(pname, "cput"))
        {
        if (igncput == FALSE)
          {
          /* cpu time - check, if less than pcput use it */

          retval = mm_gettime(&r, &value);

          if (retval != PBSE_NONE)
            {
            sprintf(log_buffer, "cput mm_gettime failed in %s", __func__);

            return(error(pname, retval));
            }
          }
        }
      else if (!strcmp(pname, "pcput"))
        {
        if (igncput == FALSE)
          {
          if (set_mode == SET_LIMIT_SET)
            {
            /* process cpu time - set */

            retval = mm_gettime(&r, &value);

            if (retval != PBSE_NONE)
              {
              sprintf(log_buffer, "pcput mm_gettime failed in %s", __func__);

              return(error(pname, retval));
              }

            reslim.rlim_cur = reslim.rlim_max =
              (unsigned long)((double)value / cputfactor);

            if (LOGLEVEL >= 2)
              {
              sprintf(log_buffer, "setting cpu time limit to %ld for job %s",
                (long int)reslim.rlim_cur,
                pjob->ji_qs.ji_jobid);

              log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);

              log_buffer[0] = '\0';
              }

            /* NOTE: some versions of linux have a bug which causes the parent
                     process to receive a SIGKILL if the child's cpu limit is exceeded */

            if (setrlimit(RLIMIT_CPU, &reslim) < 0)
              {
              sprintf(log_buffer, "setrlimit for RLIMIT_CPU failed in %s, errno=%d (%s)",
                __func__,
                errno, strerror(errno));

              return(error("RLIMIT_CPU", PBSE_SYSTEM));
              }
            }    /* END if (set_mode == SET_LIMIT_SET) */
          }
        }
      else if (!strcmp(pname, "file"))
        {
        /* set */

        if (set_mode == SET_LIMIT_SET)
          {
          retval = mm_getsize(&r, &value);

          if (retval != PBSE_NONE)
            {
            sprintf(log_buffer, "mm_getsize() failed for file in %s",
                    __func__);

            return(error(pname, retval));
            }

          reslim.rlim_cur = reslim.rlim_max = value;

          if (setrlimit(RLIMIT_FSIZE, &reslim) < 0)
            {
            sprintf(log_buffer, "cannot set file limit to %ld for job %s (setrlimit failed - check default user limits)",
                    (long int)reslim.rlim_max,
                    pjob->ji_qs.ji_jobid);

            log_err(errno, __func__, log_buffer);

            log_buffer[0] = '\0';

            return(error(pname, PBSE_SYSTEM));
            }
          }
        }
      else if (!strcmp(pname, "vmem"))
        {
        if (ignvmem == FALSE)
          {
          /* check */

          retval = mm_getsize(&r, &value);

          if (retval != PBSE_NONE)
            {
            sprintf(log_buffer, "mm_getsize() failed for vmem in %s", __func__);

            return(error(pname, retval));
            }

          if ((vmem_limit == 0) || (value < vmem_limit))
            vmem_limit = value;
          }
        }
      else if (!strcmp(pname, "pvmem"))
        {
        if (ignvmem == FALSE)
          {
          /* set */

          if (set_mode == SET_LIMIT_SET)
            {
            retval = mm_getsize(&r, &value);

            if (retval != PBSE_NONE)
              {
              sprintf(log_buffer, "mm_getsize() failed for pvmem in %s",
                __func__);

              return(error(pname, retval));
              }

            if ((vmem_limit == 0) || (value < vmem_limit))
              vmem_limit = value;
            }
          }
        }
      else if ((!strcmp(pname,"mem") && (pjob->ji_numnodes != 1)) ||
                !strcmp(pname,"mppmem"))
        {
        /* ignore. If we ever get rid of support for the UNICOS OS then we can
           remove the ATR_DFLAG_MOM | ATR_DFLAG_ALTRUN flags from mppmem */
        }
      else if ((!strcmp(pname, "mem") && (pjob->ji_numnodes == 1)) ||
               !strcmp(pname, "pmem"))
        {
        if (ignmem == FALSE)
          {
          /* set */

          if (set_mode == SET_LIMIT_SET)
            {
            retval = mm_getsize(&r, &value);

            if (retval != PBSE_NONE)
              {
              sprintf(log_buffer, "mm_getsize() failed for mem/pmem in %s",
                __func__);

              return(error(pname, retval));
              }

            reslim.rlim_cur = reslim.rlim_max = value;

            if (setrlimit(RLIMIT_DATA, &reslim) < 0)
              {
              sprintf(log_buffer, "cannot set data limit to %ld for job %s (setrlimit failed w/errno=%d (%s) - check default user limits)",
                (long int)reslim.rlim_max,
                pjob->ji_qs.ji_jobid,
                errno,
                strerror(errno));

              return(error("RLIMIT_DATA", PBSE_SYSTEM));
              }

            if (setrlimit(RLIMIT_RSS, &reslim) < 0)
              {
              sprintf(log_buffer, "cannot set RSS limit to %ld for job %s (setrlimit failed w/errno=%d (%s) - check default user limits)",
                (long int)reslim.rlim_max,
                pjob->ji_qs.ji_jobid,
                errno,
                strerror(errno));

              return(error("RLIMIT_RSS", PBSE_SYSTEM));
              }

#ifdef __GATECH
            /* NOTE:  best patch may be to change to 'vmem_limit = value;' */

            if (setrlimit(RLIMIT_STACK, &reslim) < 0)
              {
              sprintf(log_buffer, "cannot set stack limit to %ld for job %s (setrlimit failed w/errno=%d (%s) - check default user limits)",
                (long int)reslim.rlim_max,
                pjob->ji_qs.ji_jobid,
                errno,
                strerror(errno));

              return(error("RLIMIT_STACK", PBSE_SYSTEM));
              }

            /* set address space */

            if (setrlimit(RLIMIT_AS, &reslim) < 0)
              {
              sprintf(log_buffer, "cannot set AS limit to %ld for job %s (setrlimit failed w/errno=%d (%s) - check default user limits)",
                (long int)reslim.rlim_max,
                pjob->ji_qs.ji_jobid,
                errno,
                strerror(errno));

              return(error("RLIMIT_AS", PBSE_SYSTEM));
              }

#endif /* __GATECH */

            mem_limit = value;

            if (getrlimit(RLIMIT_STACK, &reslim) >= 0)
              {
              /* NOTE:  mem_limit no longer used with UMU patch in place */

              mem_limit = value + reslim.rlim_cur;
              }
            }
          }
        }    /* END else if (!strcmp(pname,"mem") && ... */
      else if (!strcmp(pname, "walltime"))
        {
        /* check */

        retval = mm_gettime(&r, &value);

        if (retval != PBSE_NONE)
          {
          sprintf(log_buffer, "mm_gettime() failed for walltime in %s\n",
                  __func__);

          return(error(pname, retval));
          }
        }
      else if (!strcmp(pname, "nice"))
        {
        /* set nice */

        if (set_mode == SET_LIMIT_SET)
          {
          errno = 0;

          if ((nice((int)r.rs_value.at_val.at_long) == -1) && (errno != 0))
            {
            sprintf(log_buffer, "nice() failed w/errno=%d (%s) in %s\n",
                    errno,
                    strerror(errno),
                    __func__);

            return(error(pname, PBSE_BADATVAL));
            }
          }
        }
      else if (!strcmp(pname, "cpuclock"))   /* Set cpu frequency */
        {
        if(set_mode == SET_LIMIT_SET)
          {
          std::string beforeFreq;
          char requestedFreq[100];

          from_frequency(&(r.rs_value.at_val.at_frequency),requestedFreq);

          nd_frequency.get_frequency_string(beforeFreq);
          if (!nd_frequency.set_frequency((cpu_frequency_type)r.rs_value.at_val.at_frequency.frequency_type,
              r.rs_value.at_val.at_frequency.mhz,
              r.rs_value.at_val.at_frequency.mhz))
            {
            std::string msg = "Failed to change frequency.";
            log_ext(nd_frequency.get_last_error(),__func__,msg.c_str(),LOG_ERR);
            }
          else
            {
            std::string afterFreq;
            nd_frequency.get_frequency_string(afterFreq);
            std::string msg = "Changed frequency from " + beforeFreq + " to " + afterFreq + " requested frequency was " + requestedFreq;
            log_ext(PBSE_CHANGED_CPU_FREQUENCY,__func__, msg.c_str(),LOG_NOTICE);
            }
          }
        }
      else if (!strcmp(pname, "size"))
        {
        /* ignore */

        /* NO-OP */
        }
      else if (!strcmp(pname, "prologue"))
        {
        }
      else if (!strcmp(pname, "epilogue"))
        {
        }
      else if ((!strcmp(pname, "mppdepth"))  ||
               (!strcmp(pname, "mppnodect")) ||
               (!strcmp(pname, "mppwidth")) ||
               (!strcmp(pname, "mppnppn")) ||
               (!strcmp(pname, "mppnodes")) ||
               (!strcmp(pname, "mpplabels")) ||
               (!strcmp(pname, "mpparch")) ||
               (!strcmp(pname, "mpplabel")))
        {
        /* NO-OP */
        }
      else if ((r.rs_defin->rs_flags & ATR_DFLAG_RMOMIG) == 0)
        {
        /* don't recognize and not marked as ignore by mom */

        sprintf(log_buffer, "do not know how to process resource '%s' in %s\n",
                pname,
                __func__);

        return(error(pname, PBSE_UNKRESC));
        }
      }
    }

  if (set_mode == SET_LIMIT_SET)
    {
    /* if either of vmem or pvmem was given, set sys limit to lesser */

    if (vmem_limit != 0)
      {
      /* Don't make (p)vmem < pmem */

      if (mem_limit > vmem_limit)
        {
        vmem_limit = mem_limit;
        }

      reslim.rlim_cur = reslim.rlim_max = vmem_limit;

      if ((ignvmem == 0) && (setrlimit(RLIMIT_AS, &reslim) < 0))
        {
        sprintf(log_buffer, "setrlimit() failed setting AS for vmem_limit mod in %s\n",
                __func__);

        return(error("RLIMIT_AS", PBSE_SYSTEM));
        }

      /* UMU vmem patch sets RLIMIT_AS rather than RLIMIT_DATA and RLIMIT_STACK */

      /*
      reslim.rlim_cur = reslim.rlim_max = mem_limit;

      if (setrlimit(RLIMIT_DATA,&reslim) < 0)
        {
        sprintf(log_buffer,"setrlimit() failed setting data for vmem_limit mod in %s\n",
          id);

        return(error("RLIMIT_DATA",PBSE_SYSTEM));
        }

      if (setrlimit(RLIMIT_STACK,&reslim) < 0)
        {
        sprintf(log_buffer,"setrlimit() failed setting stack for vmem_limit mod in %s\n",
          id);

        return(error("RLIMIT_STACK",PBSE_SYSTEM));
        }
      */
      }
    }

  if (LOGLEVEL >= 5)
    {
    sprintf(log_buffer, "%s(%s,%s) completed", __func__, pjob->ji_qs.ji_jobid,
      (set_mode == SET_LIMIT_SET) ? "set" : "alter");

    log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);

    log_buffer[0] = '\0';
    }

  /* SUCCESS */

  return(PBSE_NONE);
  }  /* END mom_set_limits() */



/*
 * State whether MOM main loop has to poll this job to determine if some
 * limits are being exceeded.
 *
 * Sets flag TRUE if polling is necessary, FALSE otherwise.  Actual
 * polling is done using the mom_over_limit machine-dependent function.
 */

int mom_do_poll(

  job *pjob) /* I */

  {
  const char  *pname;

  assert(pjob != NULL);

  if (LOGLEVEL >= 4)
    {
    log_record(
      PBSEVENT_JOB,
      PBS_EVENTCLASS_JOB,
      pjob->ji_qs.ji_jobid,
      "evaluating limits for job");
    }

  assert(pjob != NULL);

  assert(pjob->ji_wattr[JOB_ATR_resource].at_type == ATR_TYPE_RESC);

  std::vector<resource> *resources = (std::vector<resource> *)pjob->ji_wattr[JOB_ATR_resource].at_val.at_ptr;

  if (resources != NULL)
    {
    for (size_t i = 0; i < resources->size(); i++)
      {
      resource &r = resources->at(i);

      assert(r.rs_defin != NULL);

      pname = r.rs_defin->rs_name;

      assert(pname != NULL);
      assert(*pname != '\0');

      if (strcmp(pname, "walltime") == 0 ||
          strcmp(pname, "cput") == 0 ||
          strcmp(pname, "pcput") == 0 ||
          strcmp(pname, "mem") == 0 ||
          strcmp(pname, "pvmem") == 0 ||
          strcmp(pname, "vmem") == 0)
        {
        return(TRUE);
        }
      }
    }

  return(FALSE);
  }  /* END mom_do_poll() */



/*
 * Setup for polling.
 *
 * Open kernel device and get namelist info.
 */

int mom_open_poll(void)

  {
  if (LOGLEVEL >= 6)
    {
    log_record(PBSEVENT_SYSTEM, 0, __func__, "started");
    }

  pagesize = getpagesize();

  proc_array = (proc_stat_t *)calloc(TBL_INC, sizeof(proc_stat_t));

  if (proc_array == NULL)
    {
    log_err(errno, __func__, "calloc");

    return(PBSE_SYSTEM);
    }

  max_proc = TBL_INC;

  return(PBSE_NONE);
  }  /* END mom_open_poll() */

/*
 * try to find owning job session id of a pid from pid's lineage
 */

int get_job_sid_from_pid(pid_t pid)
  {
  int  index;
  int  sid;
  job_pid_set_t::const_iterator it;

  if (pid < 2)
    {
    /* don't care about pids 0 and 1 */
    return(-1);
    }

  /* get the index of this pid in the proc_array */

  /* see if pid is in the map */
  pid2procarrayindex_map_t::const_iterator pa_iter = pid2procarrayindex_map.find(pid);
  if (pa_iter == pid2procarrayindex_map.end())
    {
    /* not in map so bail out */
    return(-1);
    }

  /* assign index to the index of the pid in proc_array */
  index = pa_iter->second;

  /* get the sid of the pid in the proc_array */
  sid = proc_array[index].session;

  /* find sid in global_job_sid_set */
  it = global_job_sid_set.find(sid);

  /* found? */
  if (it != global_job_sid_set.end())
    {
    /* yes, return the sid */
    return(sid);
    }

  /* sid not in global_job_sid_set so try to find an owning one from pid's lineage */
  return(get_job_sid_from_pid(proc_array[index].ppid));
  }


/*
 * Declare start of polling loop.
 *
 * This function caches information about all of processes
 * on the compute node (pbs_mom calls this function). Each process
 * in /proc/ is queried by looking at the 'stat' file. Statistics like
 * CPU usage time, memory consumption, etc. are gathered in the proc_array
 * list. This list is then used throughout the pbs_mom to get information
 * about tasks it is monitoring.
 *
 * This function is called from the main MOM loop once every "check_poll_interval"
 * seconds.
 *
 * @see get_proc_stat() - child
 * @see mom_set_use() - Aggregates data collected here
 *
 * NOTE:  populates global 'proc_array[]' variable.
 * NOTE:  reallocs proc_array[] as needed to accomodate processes.
 * NOTE:  populates global 'pid2jobsid_map' map (pid to owning job session id mapping for all pids).
 * NOTE:  populates global 'pid2procarrayindex_map' map (pid to index in proc_array map).
 *
 * @see mom_open_poll() - allocs proc_array table.
 * @see mom_close_poll() - frees procs_array.
 * @see setup_program_environment() - parent - called at pbs_mom start
 * @see main_loop() - parent - called once per iteration
 * @see mom_set_use() - populate job structure with usage data for local use or to send to mother superior
 */

int mom_get_sample(void)

  {
  proc_stat_t           *pi;
  proc_stat_t           *ps;
  pid_t                  pid;
#ifdef PENABLE_LINUX26_CPUSETS
  struct pidl           *pids = NULL;
  struct pidl           *pp;
#else
  struct dirent         *dent;
#endif

  if (proc_array == NULL)
    mom_open_poll();

  nproc = 0;

  /* clear the maps */
  pid2jobsid_map.clear();
  pid2procarrayindex_map.clear();

  pi = proc_array;

  if (LOGLEVEL >= 6)
    {
    log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_SERVER, __func__, "proc_array load started");
    }

#ifdef PENABLE_LINUX26_CPUSETS

  /* Instead of collect stats of all processes running on a large SMP system,
   * collect stats of processes running in and below the Torque cpuset, only
   * This relies on reliable process starters for MPI, which bind their tasks
   * to the cpuset of the job. */

#ifdef USELIBCPUSET
  pids = get_cpuset_pidlist(TTORQUECPUSET_BASE, pids);
#else
  pids = get_cpuset_pidlist(TTORQUECPUSET_PATH, pids);
#endif
  pp = pids;
  while (pp != NULL)
    {
    pid = pp->pid;
    pp  = pp->next;
#else
  if (pdir == NULL)
    {
    if ((pdir = opendir(procfs)) == NULL)
      return(PBSE_SYSTEM);
    }

  rewinddir(pdir);

  while ((dent = readdir(pdir)) != NULL)
    {
    if (!isdigit(dent->d_name[0]))
      continue;

    pid = atoi(dent->d_name);
#endif
    if ((ps = get_proc_stat(pid)) == NULL)
      {
      if (errno != ENOENT)
        {
        sprintf(log_buffer, "%d: get_proc_stat", pid);

        log_err(errno, __func__, log_buffer);
        }

      continue;
      }

    /* nproc++; -- we need to increment AFTER assigning this ps to
       the proc_array--otherwise we could skip it in for loops */

    if ((nproc + 1) >= max_proc)
      {
      proc_stat_t *hold;

      if (LOGLEVEL >= 9)
        {
        log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_SERVER, __func__, "alloc more proc_array");
        }

      max_proc *= 2;

      hold = (proc_stat_t *)calloc(1, max_proc * sizeof(proc_stat_t));

      if (hold == NULL)
        {
        log_err(errno, __func__, "unable to realloc space for proc_array sample");

        return(PBSE_SYSTEM);
        }

      memcpy(hold, proc_array, sizeof(proc_stat_t) * max_proc / 2);
      free(proc_array);

      proc_array = hold;
      }  /* END if ((nproc+1) == max_proc) */

    /* map pid to proc_array index */
    pid2procarrayindex_map[pid] = nproc;

    pi = &proc_array[nproc++];

    memcpy(pi, ps, sizeof(proc_stat_t));
    }  /* END while (...) != NULL) */

#ifdef PENABLE_LINUX26_CPUSETS
  free_pidlist(pids);
#endif

  if (LOGLEVEL >= 6)
    {
    sprintf(log_buffer, "proc_array loaded - nproc=%d",
            nproc);

    log_record(PBSEVENT_DEBUG, 0, __func__, log_buffer);
    }

  /* We have filled the proc_array table. Now find all of the processes that actually belong to a job and
     add them to the pid2jobsid_map associating the process id with the session id of the job to which it belongs */

  for ( int i = 0; i < nproc; i++)
    {
    int  job_sid;
    job_pid_set_t::const_iterator it;

    if ((proc_array[i].session < 2) || (proc_array[i].pid < 2) || (proc_array[i].ppid < 2))
      {
      /* process 0 and 1 are nothing we need to look at */
      continue;
      }

    /* If the session of this entry is in the global_job_sid_set then it belongs to the job.
       associate the pid with the session of the job */
    it = global_job_sid_set.find(proc_array[i].session);
    if (it != global_job_sid_set.end())
      {
      pid2jobsid_map[proc_array[i].pid] = proc_array[i].session;
      continue;
      }

    /* the entry was not in the global_job_sid_set so try to find owning job sid from entry's lineage */
    if ((job_sid = get_job_sid_from_pid(proc_array[i].ppid)) != -1)
      {
      pid2jobsid_map[proc_array[i].pid] = job_sid;
      continue;
      }

    /* If we get to here the proc_array entry does not belong to a current job */
    }

  return(PBSE_NONE);
  }  /* END mom_get_sample() */



/*
 * Measure job resource usage and compare with its limits.
 *
 * If it has exceeded any well-formed polled limit return the limit that
 * it exceeded.
 * Otherwise, return PBSE_NONE.  log_buffer is populated with failure.
 */

int mom_over_limit(

  job *pjob)  /* I */

  {
  const char       *pname;
  int                retval;
  unsigned long      value;
  unsigned long      num;
  unsigned long long numll;

  assert(pjob != NULL);
  assert(pjob->ji_wattr[JOB_ATR_resource].at_type == ATR_TYPE_RESC);

  std::vector<resource> *resources = (std::vector<resource> *)pjob->ji_wattr[JOB_ATR_resource].at_val.at_ptr;

  if (resources != NULL)
    {
    for (size_t i = 0; i < resources->size(); i++)
      {
      resource &r = resources->at(i);

      assert(r.rs_defin != NULL);

      pname = r.rs_defin->rs_name;

      assert(pname != NULL);
      assert(*pname != '\0');

      if ((igncput == FALSE) && (strcmp(pname, "cput") == 0))
        {
        retval = mm_gettime(&r, &value);

        if (retval != PBSE_NONE)
          continue;

        if ((num = cput_sum(pjob)) > value)
          {
          sprintf(log_buffer, "cput %lu exceeded limit %lu",
                  num,
                  value);

          return(JOB_EXEC_OVERLIMIT_CPUT);
          }
        }
      else if ((igncput == FALSE) && (strcmp(pname, "pcput") == 0))
        {
        retval = mm_gettime(&r, &value);

        if (retval != PBSE_NONE)
          continue;

        if (overcpu_proc(pjob, value))
          {
          sprintf(log_buffer, "pcput exceeded limit %lu",
                  value);

          return(JOB_EXEC_OVERLIMIT_CPUT);
          }
        }
      else if (strcmp(pname, "vmem") == 0)
        {
        retval = mm_getsize(&r, &value);

        if (retval != PBSE_NONE)
          continue;

        if ((ignvmem == 0) && ((numll = mem_sum(pjob)) > value))
          {
          sprintf(log_buffer, "vmem %llu exceeded limit %lu",
                  numll,
                  value);

          return(JOB_EXEC_OVERLIMIT_MEM);
          }
        }
      else if (strcmp(pname, "pvmem") == 0)
        {
        unsigned long long valuell;

        retval = mm_getsize(&r, &value);

        if (retval != PBSE_NONE)
          continue;

        valuell = (unsigned long long)value;

        if ((ignvmem == 0) && (overmem_proc(pjob, valuell)))
          {
          sprintf(log_buffer, "pvmem exceeded limit %llu",
                  valuell);

          return(JOB_EXEC_OVERLIMIT_MEM);
          }
        }
      else if (ignwalltime == 0 && strcmp(pname, "walltime") == 0)
        {

        /* no need to check walltime on sisters, MS will get it */
        if (am_i_mother_superior(*pjob) == false)
          continue;

        retval = mm_gettime(&r, &value);

        if (retval != PBSE_NONE)
          continue;

        num = (unsigned long)((double)(time_now - pjob->ji_qs.ji_stime) *
                              wallfactor);

        if (num > value)
          {
          sprintf(log_buffer, "walltime %ld exceeded limit %ld",
                  num,
                  value);

          return(JOB_EXEC_OVERLIMIT_WT);
          }
        }
      }  /* END for each resource */
    }

#ifdef PENABLE_LINUX26_CPUSETS
  /* Check memory_pressure */

  if (memory_pressure_threshold > 0)
    {
    /*
     * If last recorded memory_pressure is over threshold, increment counter.
     * If duration is enabled, throw over_limit if counter reaches duration.
     */

      if (pjob->ji_mempressure_curr < memory_pressure_threshold)
        {
        pjob->ji_mempressure_cnt = 0; /* reset */
        }
      else
        {
        pjob->ji_mempressure_cnt++;   /* count */

        sprintf(log_buffer, "job %s memory_pressure is over %d for %d (%d) cycles",
          pjob->ji_qs.ji_jobid,
          memory_pressure_threshold,
          pjob->ji_mempressure_cnt,
          memory_pressure_duration);
        log_ext(-1, __func__, log_buffer,LOG_ALERT);

        if (memory_pressure_duration && (pjob->ji_mempressure_cnt >= memory_pressure_duration))
          {
          sprintf(log_buffer, "swap rate due to memory oversubscription is too high");
          return(JOB_EXEC_OVERLIMIT_MEM);
          }

        }

    }
#endif

  return(PBSE_NONE);
  }  /* END mom_over_limit() */



/*
 * job_expected_resc_found: logs an error if an expected resource was not found
 */
int job_expected_resc_found(

  const resource *pres,
  const resource_def  *rd,
  const char *jobid)

  {
  if (!pres)
    {
    char log_buf[2048];
    snprintf(log_buf, sizeof(log_buf), "job %s missing expected resource %s for resource usage calculation",
      jobid, rd->rs_name);
    log_err(-1, __func__, log_buf);
    return -1;
    }
  return PBSE_NONE;
  }



#ifdef USE_RESOURCE_PLUGIN
/*
 * add_plugin_job_resource_usage()
 *
 * Calls the resource plugin to add arbitrary job resource usage information
 *
 * @param pjob - the job whose resource usage information we're adding
 */

void add_plugin_job_resource_usage(
    
  job *pjob)

  {
  static const int job_resource_alarm_seconds = 3;
  std::string jid(pjob->ji_qs.ji_jobid);
  std::set<pid_t> job_pids(*pjob->ji_job_pid_set);
  
  if (LOGLEVEL >= 3)
    log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_MOM, __func__, "Executing the job usage plugin");

  alarm(job_resource_alarm_seconds);
  report_job_resources(jid, job_pids, *pjob->ji_custom_usage_info);
  alarm(0);

  if (LOGLEVEL >= 3)
    log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_MOM, __func__,
      "Finished executing the job usage plugin");
  }
#endif


/*
 * Update the job attribute for resources used.
 *
 * The first time this function is called for a job,
 * it sets up resource entries for
 * each resource that can be reported for this machine.
 *
 * Subsequent calls update the resource usage information based on
 * stats gathered by the mom_get_sample() function. This function
 * is often called by "im_request()" as a result of POLL_JOB query
 * from the mother superior.
 *
 * @see im_request() - parent - respond to poll_job request from mother superior
 * @see examine_all_running_jobs() - parent - update local use on mother superior
 * @see TMomFinalizeJob1() - parent - update serial job immediately at job start
 *
 * @return An error code if something goes wrong.
 */

int mom_set_use(

  job *pjob)  /* I (modified) */

  {
  resource      *pres;
  pbs_attribute *at;
  resource_def  *rd;
  unsigned long *lp;
  unsigned long  lnum;
#ifdef PENABLE_LINUX26_CPUSETS
  int            inum;
#endif

  assert(pjob != NULL);
  at = &pjob->ji_wattr[JOB_ATR_resc_used];
  assert(at->at_type == ATR_TYPE_RESC);

#ifdef USESAVEDRESOURCES
  /* don't update jobs that are marked as recovery */
  if (pjob->ji_flags & MOM_JOB_RECOVERY)
    {
    return(PBSE_NONE);
    }
#endif    /* USESAVEDRESOURCES */

  at->at_flags |= ATR_VFLAG_MODIFY;

  if ((at->at_flags & ATR_VFLAG_SET) == 0)
    {
    /* This is the first time mom_set_use 
     * has been called for this job.
     * initialize usage structures */

    at->at_flags |= ATR_VFLAG_SET;

    rd = find_resc_def(svr_resc_def, "cput", svr_resc_size);

    assert(rd != NULL);

    pres = add_resource_entry(at, rd);
    pres->rs_value.at_flags |= ATR_VFLAG_SET;
    pres->rs_value.at_type = ATR_TYPE_LONG;

    rd = find_resc_def(svr_resc_def, "vmem", svr_resc_size);

    assert(rd != NULL);

    pres = add_resource_entry(at, rd);

    pres->rs_value.at_flags |= ATR_VFLAG_SET;
    pres->rs_value.at_type = ATR_TYPE_SIZE;
    pres->rs_value.at_val.at_size.atsv_shift = 10; /* KB */
    pres->rs_value.at_val.at_size.atsv_units = ATR_SV_BYTESZ;

    rd = find_resc_def(svr_resc_def, "walltime", svr_resc_size);

    assert(rd != NULL);

    pres = add_resource_entry(at, rd);
    pres->rs_value.at_flags |= ATR_VFLAG_SET;
    pres->rs_value.at_type = ATR_TYPE_LONG;

    rd = find_resc_def(svr_resc_def, "mem", svr_resc_size);

    assert(rd != NULL);

    pres = add_resource_entry(at, rd);

    pres->rs_value.at_flags |= ATR_VFLAG_SET;
    pres->rs_value.at_type = ATR_TYPE_SIZE;
    pres->rs_value.at_val.at_size.atsv_shift = 10; /* KB */
    pres->rs_value.at_val.at_size.atsv_units = ATR_SV_BYTESZ;

    rd = find_resc_def(svr_resc_def, "energy_used", svr_resc_size);

    assert(rd != NULL);

    pres = add_resource_entry(at, rd);
    pres->rs_value.at_flags |= ATR_VFLAG_SET;
    pres->rs_value.at_type = ATR_TYPE_LONG;

    }  /* END if ((at->at_flags & ATR_VFLAG_SET) == 0) */

  /* get cputime */

  rd = find_resc_def(svr_resc_def, "cput", svr_resc_size);

  assert(rd != NULL);

  pres = find_resc_entry(at, rd);

  if (job_expected_resc_found(pres, rd, pjob->ji_qs.ji_jobid))
    return -1;

  lp = (unsigned long *) &pres->rs_value.at_val.at_long;

  lnum = cput_sum(pjob);

  *lp = MAX(*lp, lnum);

#if 0
  /* get joules */

  rd = find_resc_def(svr_resc_def, "energy_used", svr_resc_size);

  assert(rd != NULL);

  pres = find_resc_entry(at, rd);

  pres->rs_value.at_val.at_long += 3;
#endif

  /* get swap */

  rd = find_resc_def(svr_resc_def, "vmem", svr_resc_size);

  assert(rd != NULL);

  pres = find_resc_entry(at, rd);

  if (job_expected_resc_found(pres, rd, pjob->ji_qs.ji_jobid))
    return -1;

  lp = &pres->rs_value.at_val.at_size.atsv_num;

  lnum = (mem_sum(pjob) + 1023) >> pres->rs_value.at_val.at_size.atsv_shift; /* as KB */

  *lp = MAX(*lp, lnum);

  /* get walltime */

  rd = find_resc_def(svr_resc_def, "walltime", svr_resc_size);

  assert(rd != NULL);

  pres = find_resc_entry(at, rd);

  if (job_expected_resc_found(pres, rd, pjob->ji_qs.ji_jobid))
    return -1;

  /* NOTE: starting jobs can come through here before stime is recorded */
  if (pjob->ji_qs.ji_stime == 0)
    pres->rs_value.at_val.at_long = 0;
  else
    pres->rs_value.at_val.at_long =
      (long)((double)(time_now - pjob->ji_qs.ji_stime) * wallfactor);

  /* get memory */

  rd = find_resc_def(svr_resc_def, "mem", svr_resc_size);

  assert(rd != NULL);

  pres = find_resc_entry(at, rd);

  if (job_expected_resc_found(pres, rd, pjob->ji_qs.ji_jobid))
    return -1;

  lp = &pres->rs_value.at_val.at_size.atsv_num;

  lnum = (resi_sum(pjob) + 1023) >> pres->rs_value.at_val.at_size.atsv_shift; /* as KB */

  *lp = MAX(*lp, lnum);

#ifdef PENABLE_LINUX26_CPUSETS
  /* get memory_pressure */

  if (memory_pressure_threshold > 0)
    {
    inum = get_cpuset_mempressure(pjob->ji_qs.ji_jobid);

    /* Store if success */
    if (inum != -1)
      pjob->ji_mempressure_curr = inum;

    /* Alert if there is pressure */
    if (inum > 0)
      {
      sprintf(log_buffer, "job %s causes memory_pressure %d", pjob->ji_qs.ji_jobid, inum);
      log_ext(-1, __func__, log_buffer, LOG_ALERT);
      }
    }
  else
    {
    pjob->ji_mempressure_curr = 0;
    }
#endif

#ifdef USE_RESOURCE_PLUGIN
  add_plugin_job_resource_usage(pjob);
#endif

  return(PBSE_NONE);
  }  /* END mom_set_use() */





/**
 * Kill a task session.
 * Call with the task pointer and a signal number.
 *
 * @return number of tasks signalled (0 = failure)
 *
 * @see kill_job() - parent
 *
 * NOTE:  should support killpg() or killpidtree() - (NYI)
 *        may be required for suspend/resume
 */

int kill_task(

  job  *pjob,   /* I */
  task *ptask,  /* I */
  int   sig,    /* I */
  int   pg)     /* I (1=signal process group, 0=signal master process only) */

  {
  int            ct = 0;  /* num of processes killed */
  int            ctThisIteration = 0;
  int            ctCleanIterations = 0;
  int            loopCt = 0;
  int            NumProcessesFound = 0; /* number of processes found with session ID */
#ifdef PENABLE_LINUX26_CPUSETS
  struct pidl   *pids = NULL;
  struct pidl   *pp;
#else
   struct dirent *dent;
#endif
  pid_t          pid;

  proc_stat_t   *ps;
  int            sesid;
  pid_t          mompid;

  sesid = ptask->ti_qs.ti_sid;
  mompid = getpid();

  if (LOGLEVEL >= 5)
    {
    sprintf(log_buffer, "%s: sending signal %d to task %d, session %d",
      __func__,
      sig,
      ptask->ti_qs.ti_task,
      sesid);

    log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, ptask->ti_qs.ti_parentjobid, log_buffer);
    }

  if (sesid <= 1)
    {
    if (LOGLEVEL >= 3)
      {
      sprintf(log_buffer, "cannot send signal %d to task (no session id)",
        sig);

      log_record(PBSEVENT_ERROR, PBS_EVENTCLASS_JOB, ptask->ti_qs.ti_parentjobid, log_buffer);
      }

    /* FAILURE */

    return(0);
    }

  do
    {
    ctThisIteration = 0;

    /* NOTE:  do not use cached proc-buffer since we need up-to-date info */
#ifdef PENABLE_LINUX26_CPUSETS

    /* Instead of collecting stats of all processes running on a large SMP system,
     * collect stats of processes running in and below the Torque cpuset, only
     * This relies on reliable process starters for MPI, which bind their tasks
     * to the cpuset of the job. */

#ifdef USELIBCPUSET
    pids = get_cpuset_pidlist(TTORQUECPUSET_BASE, pids);
#else
    pids = get_cpuset_pidlist(TTORQUECPUSET_PATH, pids);
#endif /* USELIBCPUSET */
    pp = pids;
    while (pp != NULL)
      {
      pid = pp->pid;
      pp  = pp->next;
#else
    if (pdir == NULL)
      {
      if ((pdir = opendir(procfs)) == NULL)
        return(PBSE_SYSTEM);
      }

    /* pdir is global */
    rewinddir(pdir);

    while ((dent = readdir(pdir)) != NULL)
      {
      if (!isdigit(dent->d_name[0]))
        continue;

      pid = atoi(dent->d_name);
#endif /* PENABLE_LINUX26_CPUSETS */
      if ((ps = get_proc_stat(pid)) == NULL)
        {
        if (errno != ENOENT)
          {
          sprintf(log_buffer, "%d: get_proc_stat", pid);

          log_err(errno, __func__, log_buffer);
          }

        continue;
        }

      if ((sesid == ps->session) ||
          (ProcIsChild(procfs,pid,ptask->ti_qs.ti_parentjobid) == TRUE))

        {
        NumProcessesFound++;

        if ((ps->state == 'Z') ||
            (ps->pid == 0))
          {
          /*
           * Killing a zombie is sure death! Its pid is zero,
           * which to kill(2) means 'every process in the process
           * group of the current process'.
           */

          sprintf(log_buffer, "%s: not killing process (pid=%d/state=%c) with sig %d",
            __func__, ps->pid, ps->state, sig);

          log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, ptask->ti_qs.ti_parentjobid, log_buffer);
          }  /* END if ((ps->state == 'Z') || (ps->pid == 0)) */
        else
          {
          int i = 0;

          if (ps->pid == mompid)
            {
            /*
             * there is a race condition with newly started jobs that
             * can be killed before they've established their own
             * session id.  This means the child tasks still have MOM's
             * session id.  We check this to make sure MOM doesn't kill
             * herself.
             */

            if (LOGLEVEL >= 3)
              {
              sprintf(log_buffer, "%s: not killing process %d. Avoid sending signal because child task still has MOM's session id", __func__, ps->pid);

              log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, ptask->ti_qs.ti_parentjobid, log_buffer);
              }

            if ((sig == SIGKILL) ||
                (sig == SIGTERM))
              {
              ++ctThisIteration; //Ultimately this is  task that will need to be killed.
              }

            continue;
            }  /* END if (ps->pid == mompid) */

          if (sig == SIGKILL)
            {
            if (pg == 0)
              {
              /* make sure we only send a SIGTERM one time per process */
              if ((ps->state != 'Z') &&
                  (pjob->ji_sigtermed_processes->find(pid) == pjob->ji_sigtermed_processes->end()))
                {
                sprintf(log_buffer, "%s: killing pid %d task %d gracefully with sig %d",
                  __func__, ps->pid, ptask->ti_qs.ti_task, SIGTERM);
                
                log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB,
                    ptask->ti_qs.ti_parentjobid, log_buffer);
            
                kill(ps->pid, SIGTERM);
                pjob->ji_sigtermed_processes->insert(ps->pid);
                }
              }
            else
              {
              if ((ps->state != 'Z') &&
                  (pjob->ji_sigtermed_processes->find(pid) == pjob->ji_sigtermed_processes->end()))
                {
                sprintf(log_buffer, "%s: killing pid %d task %d gracefully with sig %d",
                  __func__, ps->pid, ptask->ti_qs.ti_task, SIGTERM);
                
                log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB,
                    ptask->ti_qs.ti_parentjobid, log_buffer);
            
                killpg(ps->pid, SIGTERM);
                pjob->ji_sigtermed_processes->insert(ps->pid);
                }
              }

            for (i = 0;i < 20;i++)
              {
              /* check if process is gone */
              if ((ps = get_proc_stat(ps->pid)) == NULL)
                {
                break;
                }
              else
                {
                sprintf(log_buffer, "%s: process (pid=%d/state=%c) after sig %d",
                  __func__, ps->pid, ps->state, SIGTERM);

                log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, ptask->ti_qs.ti_parentjobid, log_buffer);

                if (ps->state == 'Z')
                  break;
                }

              /* try to kill again */
              if (kill(ps->pid, 0) == -1)
                break;

              }  /* END for (i = 0) */
            }    /* END if (sig == SIGKILL) */
          else
            {
            i = 20;
            }

          if (i >= 20)
            {
            /* NOTE: handle race-condition where process goes zombie as a result
             * of previous SIGTERM */

            /* update proc info from /proc/<PID>/stat */
            if ((ps = get_proc_stat(ps->pid)) != NULL)
              {
              if (ps->state == 'Z')
                {
                /*
                 * Killing a zombie is sure death! Its pid is zero,
                 * which to kill(2) means 'every process in the process
                 * group of the current process'.
                 */

                sprintf(log_buffer, "%s: not killing process (pid=%d/state=%c) with sig %d",
                  __func__, ps->pid, ps->state, sig);

                log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB,	ptask->ti_qs.ti_parentjobid, log_buffer);
                }  /* END if ((ps->state == 'Z') || (ps->pid == 0)) */
              else
                {
                /* kill process hard */

                /* why is this not killing with SIGKILL? */
                sprintf(log_buffer, "%s: killing pid %d task %d with sig %d",
                  __func__, ps->pid, ptask->ti_qs.ti_task, sig);

                if (pg == 0)
                  {
                  if (sig != SIGTERM)
                    {
                    log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, ptask->ti_qs.ti_parentjobid, log_buffer);
                    kill(ps->pid, sig);
                    }
                  /* make sure we only send a SIGTERM one time */
                  else if (pjob->ji_sigtermed_processes->find(ps->pid) == pjob->ji_sigtermed_processes->end())
                    {
                    log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, ptask->ti_qs.ti_parentjobid, log_buffer);
                    killpg(ps->pid, SIGTERM);
                    pjob->ji_sigtermed_processes->insert(ps->pid);
                    }
                  }
                else
                  {
                  if (sig != SIGTERM)
                    {
                    log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, ptask->ti_qs.ti_parentjobid, log_buffer);
                    killpg(ps->pid, sig);
                    }
                  else if (pjob->ji_sigtermed_processes->find(ps->pid) == pjob->ji_sigtermed_processes->end())
                    {
                    log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, ptask->ti_qs.ti_parentjobid, log_buffer);
                    killpg(ps->pid, SIGTERM);
                    pjob->ji_sigtermed_processes->insert(ps->pid);
                    }
                  }
                }
              }    /* END if ((ps = get_proc_stat(ps->pid)) != NULL) */
            }      /* END if (i >= 20) */

          ++ct;
          }  /* END else ((ps->state == 'Z') || (ps->pid == 0)) */
        }    /* END if (sesid == ps->session) */
      }      /* END while (...) != NULL) */

#ifdef PENABLE_LINUX26_CPUSETS
    free_pidlist(pids);
    pids = NULL;
#endif
    if (ctThisIteration == 0)
      {
      ctCleanIterations++;
      }
    else
      {
      ctCleanIterations=0;
      }
    } while ((ctCleanIterations <= 3) && (loopCt++ < 20));

  /* NOTE:  to fix bad state situations resulting from a hard crash, the logic
            below should be triggered any time no processes are found (NYI) */

  if ((IS_ADOPTED_TASK(ptask->ti_qs.ti_task)) &&
      (NumProcessesFound == 0))
    {
    /* no process was found, but for an adopted task this is OK (we don't find
     * out about the adopted task's termination via waitpid()--so we can safely
     * say that we have "killed" the task, even though the task was killed/died
     * some other way */

    ct++;

    /* do code to mark task as finished (borrowed from Linux scan_for_terminated())... */

    ptask->ti_qs.ti_exitstat = 0;  /* assume successful completion */
    ptask->ti_qs.ti_status   = TI_STATE_EXITED;

    task_save(ptask);

    sprintf(log_buffer,
      "%s: job %s adopted task %d was marked as terminated because task's PID was no longer found, sid=%d",
      __func__,
      ptask->ti_qs.ti_parentjobid,
      ptask->ti_qs.ti_task,
      ptask->ti_qs.ti_sid);

    log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, ptask->ti_qs.ti_parentjobid, log_buffer);
    }

  if ((NumProcessesFound == 0) &&
      (ct <= 0))
    {
    /* we can't find any processes belonging to given session, so we can safely say
     * that we "killed" the task and have TORQUE clean it up */

    ct++;

    /* do code to mark task as finished (borrowed from Linux scan_for_terminated())... */

    ptask->ti_qs.ti_exitstat = 0;  /* assume successful completion */
    ptask->ti_qs.ti_status   = TI_STATE_EXITED;

    task_save(ptask);

    if (LOGLEVEL >= 5)
      {
      sprintf(log_buffer,
        "%s: could not send signal %d to task %d (session %d)--no process was found with this session ID (marking task as killed)!",
        __func__,
        sig,
        ptask->ti_qs.ti_task,
        sesid);

      log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, ptask->ti_qs.ti_parentjobid, log_buffer);
      }
    }

  /* SUCCESS */

  return(ct);
  }  /* END kill_task() */





/*
 * Clean up everything related to polling.
 */

int mom_close_poll(void)

  {
  if (LOGLEVEL >= 6)
    {
    log_record(PBSEVENT_SYSTEM, 0, __func__, "entered");
    }

  if (pdir != NULL)
    {
    if (closedir(pdir) != 0)
      {
      log_err(errno, __func__, "closedir");

      return(PBSE_SYSTEM);
      }

    pdir = NULL;
    }

  if (proc_array != NULL)
    {
    free(proc_array);
    proc_array = NULL;
    nproc = 0;
    max_proc = TBL_INC;
    }

  return(PBSE_NONE);
  }  /* END mom_close_poll() */




/*
 * mom_does_checkpoint
 *
 * @returns CST values as described in resmon.h.
 */

int mom_does_checkpoint(void)
  {
  return(CST_BLCR); /* Use the BLCR checkpointing system. */
  }

/*
 * Checkpoint the job.
 *
 * If abort is true, kill it too.
 */

int mach_checkpoint(

  task *ptask,  /* I */
  char *file,  /* I */
  int  abort)  /* I */

  {
  return(-1);
  }  /* END mach_checkpoint() */





/*
 * Restart the job from the checkpoint file.
 *
 * Return -1 on error or sid if okay.
 */

long mach_restart(

  task *ptask,
  char *file)

  {
  return(-1);
  }





#define dsecs(val) ( (double)(val) )

char *cput_job(

  pid_t jobid)

  {
  int           found = 0;
  int           i;

  double        cputime, addtime;
  proc_stat_t  *ps;

  cputime = 0.0;

  if (LOGLEVEL >= 6)
    {
    sprintf(log_buffer, "proc_array loop start - jobid = %d",
            jobid);

    log_record(PBSEVENT_DEBUG, 0, __func__, log_buffer);
    }

  for (i = 0;i < nproc;i++)
    {
    ps = &proc_array[i];

    if (jobid != ps->session)
      continue;

    found = 1;

    /* add utime and stime (AKE) */

    addtime =
      dsecs(ps->utime) +
      dsecs(ps->stime) +
      dsecs(ps->cutime) +
      dsecs(ps->cstime);

    cputime += addtime;

    DBPRT(("%s: total %.2f pid %d %.2f\n",
           __func__,
           cputime,
           ps->pid,
           addtime))
    }  /* END for (i) */

  if (!found)
    {
    rm_errno = RM_ERR_EXIST;

    return(NULL);
    }

  sprintf(ret_string, "%.2f",

          cputime * cputfactor);

  return(ret_string);
  }  /* END cput_job() */




char *cput_proc(

  pid_t pid)

  {
  double cputime;
  proc_stat_t *ps;

  cputime = 0.0;


  if ((ps = get_proc_stat(pid)) == NULL)
    {
    if (errno != ENOENT)
      {
      sprintf(log_buffer, "%d: get_proc_stat",
              pid);

      log_err(errno, __func__, log_buffer);
      }

    rm_errno = RM_ERR_SYSTEM;

    return(NULL);
    }

  cputime = dsecs(ps->utime) + dsecs(ps->stime);

  sprintf(ret_string, "%.2f",
          cputime * cputfactor);

  return(ret_string);
  }  /* END cput_proc() */




const char *cput(

  struct rm_attribute *attrib)

  {
  int value;

  if (attrib == NULL)
    {
    log_err(-1, __func__, no_parm);

    rm_errno = RM_ERR_NOPARAM;

    return(NULL);
    }

  if ((value = atoi(attrib->a_value)) == 0)
    {
    sprintf(log_buffer, "bad param: %s",
            attrib->a_value);

    log_err(-1, __func__, log_buffer);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if (momgetattr(NULL))
    {
    log_err(-1, __func__, extra_parm);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if (strcmp(attrib->a_qualifier, "session") == 0)
    {
    return(cput_job((pid_t)value));
    }

  if (strcmp(attrib->a_qualifier, "proc") == 0)
    {
    return(cput_proc((pid_t)value));
    }

  rm_errno = RM_ERR_BADPARAM;

  return(NULL);
  }  /* END cput() */




char *mem_job(

  pid_t sid)  /* I */

  {
  unsigned long long  memsize;
  int                 i;

  proc_stat_t        *ps;

  /* max memsize ??? */

  memsize = 0;

  if (LOGLEVEL >= 6)
    {
    sprintf(log_buffer, "proc_array loop start - sid = %d",
            sid);

    log_record(PBSEVENT_DEBUG, 0, __func__, log_buffer);
    }

  for (i = 0;i < nproc;i++)
    {
    ps = &proc_array[i];

    if (sid != ps->session)
      continue;

    memsize += ps->vsize;
    }  /* END for (i) */

  if (memsize == 0)
    {
    rm_errno = RM_ERR_EXIST;

    return(NULL);
    }

  sprintf(ret_string, "%llukb",

          memsize >> 10); /* KB */

  return(ret_string);
  }  /* END mem_job() */




char *mem_proc(

  pid_t pid)

  {
  proc_stat_t  *ps;

  if ((ps = get_proc_stat(pid)) == NULL)
    {
    if (errno != ENOENT)
      {
      sprintf(log_buffer, "%d: get_proc_stat",
              pid);

      log_err(errno, __func__, log_buffer);
      }

    rm_errno = RM_ERR_SYSTEM;

    return(NULL);
    }

  sprintf(ret_string, "%llukb",

          (unsigned long long)ps->vsize >> 10); /* KB */

  return(ret_string);
  }  /* END mem_proc() */





const char *mem(

  struct rm_attribute *attrib)

  {
  int value;

  if (attrib == NULL)
    {
    log_err(-1, __func__, no_parm);

    rm_errno = RM_ERR_NOPARAM;

    return(NULL);
    }

  if ((value = atoi(attrib->a_value)) == 0)
    {
    sprintf(log_buffer, "bad param: %s",
            attrib->a_value);

    log_err(-1, __func__, log_buffer);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if (momgetattr(NULL))
    {
    log_err(-1, __func__, extra_parm);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if (strcmp(attrib->a_qualifier, "session") == 0)
    {
    return(mem_job((pid_t)value));
    }
  else if (strcmp(attrib->a_qualifier, "proc") == 0)
    {
    return(mem_proc((pid_t)value));
    }
  else
    {
    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  return(NULL);
  }  /* END mem() */




static char *resi_job(

  pid_t jobid)

  {
  int                 i;
  int                 found = 0;
  unsigned long long  resisize;
  proc_stat_t        *ps;
#ifdef USELIBMEMACCT
  long long           w_rss;
#endif

  resisize = 0;

  if (LOGLEVEL >= 6)
    {
    sprintf(log_buffer, "proc_array loop start - jobid = %d",
            jobid);
    log_record(PBSEVENT_DEBUG, 0, __func__, log_buffer);
    }

  for (i = 0;i < nproc;i++)
    {
    ps = &proc_array[i];

    if (jobid != ps->session)
      continue;

    found = 1;

#ifdef USELIBMEMACCT

    /* Ask memacctd for weighted rss of pid, use this instead of ps->rss */

    w_rss = get_memacct_resi(ps->pid);
    if (w_rss == -1)
      resisize += ps->rss * pagesize;
    else
      resisize += w_rss;

#else

    resisize += ps->rss;

#endif
    }  /* END for (i) */

  if (found)
    {
    /* in KB */

#ifdef USELIBMEMACCT
    sprintf(ret_string, "%llukb", resisize >> 10);
#else
    sprintf(ret_string, "%llukb",
            (resisize * (unsigned long long)pagesize) >> 10);
#endif

    return(ret_string);
    }

  rm_errno = RM_ERR_EXIST;

  return(NULL);
  }  /* END resi_job() */





static char *resi_proc(

  pid_t pid)

  {
  proc_stat_t *ps;

#ifdef USELIBMEMACCT
  long long     w_rss;
#endif

  if ((ps = get_proc_stat(pid)) == NULL)
    {
    if (errno != ENOENT)
      {
      sprintf(log_buffer, "%d: get_proc_stat(PIOCPSINFO)",
              pid);
      log_err(errno, __func__, log_buffer);
      }

    rm_errno = RM_ERR_SYSTEM;

    return(NULL);
    }

#ifdef USELIBMEMACCT

  /* Ask memacctd for weighted rss of pid, use this instead of ps->rss */

  if ((w_rss = get_memacct_resi(ps->pid)) == -1)
    sprintf(ret_string, "%llukb", (ps->rss * (unsigned long long)pagesize) >> 10);
  else
    sprintf(ret_string, "%ldkb", w_rss >> 10);

#else
  /* in KB */

  sprintf(ret_string, "%lukb",
          ((ulong)ps->rss * (ulong)pagesize) >> 10);
#endif

  return(ret_string);
  }  /* END resi_proc() */




static const char *resi(

  struct rm_attribute *attrib)

  {
  int   value;

  if (attrib == NULL)
    {
    log_err(-1, __func__, no_parm);
    rm_errno = RM_ERR_NOPARAM;

    return(NULL);
    }

  if ((value = atoi(attrib->a_value)) == 0)
    {
    sprintf(log_buffer,
            "bad param: %s",
            attrib->a_value);

    log_err(-1, __func__, log_buffer);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if (momgetattr(NULL))
    {
    log_err(-1, __func__, extra_parm);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if (strcmp(attrib->a_qualifier, "session") == 0)
    {
    return(resi_job((pid_t)value));
    }

  if (strcmp(attrib->a_qualifier, "proc") == 0)
    {
    return(resi_proc((pid_t)value));
    }

  rm_errno = RM_ERR_BADPARAM;

  return(NULL);
  }  /* END resi() */



const char *sessions(

  struct rm_attribute *attrib) /* I */

  {
  int                nsids = 0;
  pid_t              sid;
  char              *s;
#ifdef NUMA_SUPPORT
  char               mom_check_name[PBS_MAXSERVERNAME];
  job               *pjob;
#else
  proc_stat_t       *ps;
  struct pidl       *sids  = NULL, *sl = NULL, *sp;
  int                i;
#endif

  if (attrib != NULL)
    {
    log_err(-1, __func__, extra_parm);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  ret_string[0] = '\0';

#ifdef NUMA_SUPPORT

  /* Initialize the node name to check for for this NUMA node */

  strcpy(mom_check_name, mom_host);
  if ((s = strchr(mom_check_name, '.')) != NULL)
    *s = '\0';
  sprintf(mom_check_name + strlen(mom_check_name), "-%d/", numa_index);

  /* Initialize the return string */

  s = ret_string;

  /* Walk through job list, look for jobs running on this NUMA node */
  std::list<job *>::iterator iter;

  // get a list of jobs in start time order, first to last
  for (iter = alljobs_list.begin(); iter != alljobs_list.end(); iter++)
    {
    pjob = *iter;

    if (strstr(pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str, mom_check_name) == NULL)
      continue;

    /* Show all tasks registered for this job */

    for (unsigned int i = 0; i < pjob->ji_tasks->size(); i++)
      {
      task *ptask = pjob->ji_tasks->at(i);

      if (ptask->ti_qs.ti_status != TI_STATE_RUNNING)
        continue;

      sid = ptask->ti_qs.ti_sid;

      if (LOGLEVEL >= 9)
        {
        sprintf(log_buffer, "%s[%d]: job %s on %s? sid %d",
          __func__,
          nsids,
          pjob->ji_qs.ji_jobid,
          mom_check_name,
          sid);
        log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
        }

      checkret(&s, 100);
      sprintf(s, "%s%d", (ret_string[0] != '\0') ? " " : "", sid);
      s += strlen(s);
      nsids++;

      } /* END for each task */

    } /* END for(pjob) */

#else

  /* Walk through proc_array, store unique session IDs in the pids list */

  for (i = 0;i < nproc;i++)
    {
    ps = &proc_array[i];

    if (ps->uid == 0)
      continue;

    if ((sid = ps->session) == 0)
      continue;

    if (LOGLEVEL >= 9)
      {
      sprintf(log_buffer, "%s[%d]: pid %d sid %d", __func__, nsids, ps->pid, sid);
      log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
      }

    sp = sids;
    while (sp)
      {
      if (sp->pid == sid)  /* found */
        break;
      sp = sp->next;
      }

    if (sp)
      continue;

    /* not found */
    if ((sp = (struct pidl *)calloc(1, sizeof(struct pidl))) == NULL)
      {
      log_err(errno, __func__, "no memory");
      rm_errno = RM_ERR_SYSTEM;
      if (sids)
        free_pidlist(sids);
      return(NULL);
      }

    sp->pid  = sid;
    sp->next = NULL;
    nsids++;
    if (sl)
      sl->next = sp;
    else
      sids = sp;
    sl = sp;

    } /* END for(i) */

  /*
   * Assemble return string.
   * Return empty string if no sessions.
   */

  s  = ret_string;
  sp = sids;
  while (sp)
    {
    checkret(&s, 100);

    if (sp == sids)
      sprintf(s, "%d", sp->pid);
    else
      sprintf(s, " %d", sp->pid);

    s += strlen(s);

    sp = sp->next;

    } /* END while(sp) */

  /* Done */

  if (sids)
    free_pidlist(sids);

#endif

  if (LOGLEVEL >= 6)
    {
    sprintf(log_buffer, "nsessions=%d", nsids);
    log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
    }

  return(ret_string);

  }



const char *nsessions(

  struct rm_attribute *attrib)

  {
  const char *result;
  const char *ch;
  int    num;

  if ((result = sessions(attrib)) == NULL)
    return(result);

  if (result[0] == '\0')
    {
    num = 0;
    }
  else
    {
    num = 1;
    for (ch = result;*ch;ch++)
      if (*ch == ' ')  /* count blanks */
        num++;
    }  /* END for (ch) */

  sprintf(ret_string, "%d",
          num);

  return(ret_string);
  }  /* END nsessions() */





const char *pids(

  struct rm_attribute *attrib)  /* I */

  {
  pid_t         jobid;
  proc_stat_t  *ps;
  char         *fmt;
  int           i;
  int           num_pids = 0;

  if (attrib == NULL)
    {
    log_err(-1, __func__, no_parm);

    rm_errno = RM_ERR_NOPARAM;

    return(NULL);
    }

  if ((jobid = (pid_t)atoi(attrib->a_value)) == 0)
    {
    sprintf(log_buffer, "bad param: %s",
            attrib->a_value);

    log_err(-1, __func__, log_buffer);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if (momgetattr(NULL))
    {
    log_err(-1, __func__, extra_parm);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if (strcmp(attrib->a_qualifier, "session") != 0)
    {
    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  /* Search for members of session */

  fmt = ret_string;

  for (i = 0;i < nproc;i++)
    {
    ps = &proc_array[i];

    if (LOGLEVEL >= 6)
      {
      DBPRT(("%s[%d]: pid: %d  sid: %d\n",
             __func__,
             num_pids,
             ps->pid,
             ps->session))
      }

    if (jobid != ps->session)
      continue;

    sprintf(fmt, "%d ",
            ps->pid);

    fmt += strlen(fmt);

    num_pids++;
    }  /* END for (i) */

  if (num_pids == 0)
    {
    rm_errno = RM_ERR_EXIST;

    return(NULL);
    }

  return(ret_string);
  }  /* END pids() */





const char *nusers(

  struct rm_attribute *attrib)

  {
  int                j;
  int                nuids = 0;
  uid_t             *uids, *hold;
  static int         maxuid = 200;
  register uid_t     uid;
#ifdef NUMA_SUPPORT
  char               mom_check_name[PBS_MAXSERVERNAME], *s;
  job               *pjob;
#else
  int                i;
  proc_stat_t       *ps;
#endif

  if (attrib != NULL)
    {
    log_err(-1, __func__, extra_parm);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if ((uids = (uid_t *)calloc(maxuid, sizeof(uid_t))) == NULL)
    {
    log_err(errno, __func__, "no memory");

    rm_errno = RM_ERR_SYSTEM;

    return(NULL);
    }

#ifdef NUMA_SUPPORT

  /* Initialize the node name to check for for this NUMA node */

  strcpy(mom_check_name, mom_host);
  if ((s = strchr(mom_check_name, '.')) != NULL)
    *s = '\0';
  sprintf(mom_check_name + strlen(mom_check_name), "-%d/", numa_index);

  /* Walk through job list, look for jobs running on this NUMA node */
  std::list<job *>::iterator iter;

  // get a list of jobs in start time order, first to last
  for (iter = alljobs_list.begin(); iter != alljobs_list.end(); iter++)
    {
    pjob = *iter;

    if (strstr(pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str, mom_check_name) == NULL)
      continue;

    /* Store uid of job owner */

    uid = pjob->ji_qs.ji_un.ji_momt.ji_exuid;

    if (LOGLEVEL >= 9)
      {
      sprintf(log_buffer, "%s[%d]: job %s on %s? uid %d",
        __func__,
        nuids,
        pjob->ji_qs.ji_jobid,
        mom_check_name,
        uid);

      log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
      }

#else

  for (i = 0;i < nproc;i++)
    {
    ps = &proc_array[i];

    if ((uid = ps->uid) == 0)
      continue;

    if (LOGLEVEL >= 9)
      {
      sprintf(log_buffer, "%s[%d]: pid %d uid %d",
              __func__,
              nuids,
              ps->pid,
              uid);

      log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
      }

#endif

    for (j = 0;j < nuids;j++)
      {
      if (uids[j] == uid)
        break;
      }

    if (j == nuids)
      {
      /* not found */

      if (nuids == maxuid)
        {
        /* need more space */

        maxuid += 100;

        hold = (uid_t *)realloc(uids, maxuid);

        if (hold == NULL)
          {
          log_err(errno, __func__, "realloc");

          rm_errno = RM_ERR_SYSTEM;

          free(uids);

          return(NULL);
          }

        memset(hold+(maxuid-100), 0, 100*sizeof(uid_t));

        if (LOGLEVEL >= 7)
          {
          sprintf(log_buffer, "%s[%d]: need more space: %d", __func__, nuids, maxuid);
          log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
          }

        hold[nuids++] = uid; /* add uid to list */
        uids          = hold;
        }
      else
        {
        uids[nuids++] = uid; /* add uid to list */
        }
      }
    }    /* END for (i) */

  sprintf(ret_string, "%d",
          nuids);

  free(uids);

  if (LOGLEVEL >= 6)
    {
    sprintf(log_buffer, "nusers=%d", nuids);
    log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
    }

  return(ret_string);
  }  /* END nusers() */




const char *totmem(

  struct rm_attribute *attrib)

  {
  proc_mem_t *mm;

  if (attrib)
    {
    log_err(-1, __func__, extra_parm);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if ((mm = get_proc_mem()) == NULL)
    {
    log_err(errno, __func__, "get_proc_mem");

    rm_errno = RM_ERR_SYSTEM;

    return(NULL);
    }

  if (LOGLEVEL >= 6)
    {
    sprintf(log_buffer, "%s: total mem=%llu",
            __func__,
            mm->mem_total + mm->swap_total);

    log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
    }

  sprintf(ret_string, "%lukb",

          (ulong)((mm->mem_total >> 10) + (mm->swap_total >> 10))); /* KB */

  return(ret_string);
  }  /* END totmem() */





const char *availmem(

  struct rm_attribute *attrib)

  {
  proc_mem_t *mm;

  if (attrib != NULL)
    {
    log_err(-1, __func__, extra_parm);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if ((mm = get_proc_mem()) == NULL)
    {
    log_err(errno, __func__, "get_proc_mem");

    rm_errno = RM_ERR_SYSTEM;

    return(NULL);
    }  /* END availmem() */

  if (LOGLEVEL >= 6)
    {
    sprintf(log_buffer, "%s: free mem=%llu",
            __func__,
            mm->mem_free + mm->swap_free);

    log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);
    }

  sprintf(ret_string, "%lukb",

          (ulong)((mm->mem_free >> 10) + (mm->swap_free >> 10))); /* KB */

  return(ret_string);
  }  /* END availmem() */




const char *ncpus(

  struct rm_attribute *attrib)

  {
#ifdef NUMA_SUPPORT
  /* report the configured ncpus for this numa node */
  sprintf(ret_string,"%d",node_boards[numa_index].num_cpus);
#else
  char   label[128];
  FILE  *fp;
  int    procs;

  if (attrib != NULL)
    {
    log_err(-1, __func__, extra_parm);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if ((fp = fopen("/proc/cpuinfo", "r")) == NULL)
    {
    return(NULL);
    }

  procs = 0;

  while (!feof(fp))
    {
    if (fscanf(fp, "%s %*[^\n]%*c", label) == 0)
      {
      getc(fp);  /* must do something to get to eof */
      }
    else if (strcmp("processor", label) == 0)
      procs++;
    }

  sprintf(ret_string, "%d", procs);

  system_ncpus = procs;

  fclose(fp);

#endif /* NUMA_SUPPORT */
  if (LOGLEVEL >= 6)
    {
    sprintf(log_buffer, "ncpus=%s", ret_string);
    log_record(PBSEVENT_SYSTEM, PBS_EVENTCLASS_NODE, "ncpus", log_buffer);
    }

  return(ret_string);
  }  /* END ncpus() */


/* find_file checks for the existence of filename
 * in the ':' delimited path string
 * Return TRUE if file exists
 *        FALSE if file not found
 */

int find_file(

  const char *path,
  const char *filename)

  {
  char        *token;
  std::string  buf;
  char        *work_str;

  int          rc;
  struct stat  statBuf;

  if ((path == NULL) ||
      (filename == NULL))
    {
    return(FALSE);
    }

  work_str = strdup(path);
  char *malloc_str = work_str; //tokenizer can change work_str;

  while ((token = threadsafe_tokenizer(&work_str, ":")) != NULL)
    {
    buf = token;
    buf += "/";
    buf += filename;

    rc = stat(buf.c_str(), &statBuf);
    if (rc == 0)
      {
      free(malloc_str);
      return(TRUE);
      }
    }

  free(malloc_str);

  return(FALSE);
  } /* END find_file() */





static const char *physmem(

  struct rm_attribute *attrib)

  {
  char tmpBuf[PMEMBUF_SIZE];

  char *tmp_ptr;
  char *BPtr;
  int   BSpace;

  unsigned long long mem;
  unsigned long long mem_total;
  FILE *fp;
#ifdef NUMA_SUPPORT
  int i;
#endif

  if (attrib != NULL)
    {
    log_err(-1, __func__, extra_parm);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  mem_total = 0;

#ifdef NUMA_SUPPORT

  for (i = 0; i < node_boards[numa_index].num_nodes; i++)
#endif /* NUMA_SUPPORT */
    {
#ifdef NUMA_SUPPORT
    if (!(fp = fopen(node_boards[numa_index].path_meminfo[i],"r")))
#else
    if (!(fp = fopen(path_meminfo, "r")))
#endif
      {
      rm_errno = RM_ERR_SYSTEM;

      return(NULL);
      }

    BPtr = tmpBuf;

    BSpace = sizeof(tmpBuf);

    BPtr[0] = '\0';

    while (!feof(fp))
      {
      if (fgets(BPtr, BSpace, fp) == NULL)
        {
        break;
        }

      BSpace -= strlen(BPtr);

      BPtr   += strlen(BPtr);
      }

    fclose(fp);

    /* FORMAT:  '...\nMemTotal:   XXX kB\n' */

    if ((tmp_ptr = strstr(tmpBuf, "MemTotal:")) != NULL)
      {
      BPtr = tmp_ptr + strlen("MemTotal:");

      if (sscanf(BPtr, "%llu",
                 &mem) != 1)
        {
        rm_errno = RM_ERR_SYSTEM;

        return(NULL);
        }

      /* value specified in kb */
      }
    else
      {
      /* attempt to load first numeric value */

      if (sscanf(BPtr, "%*s %llu",
                 &mem) != 1)
        {
        rm_errno = RM_ERR_SYSTEM;

        return(NULL);
        }

      /* value specified in bytes */

      mem >>= 10;
      }
    mem_total += mem;
    }

  sprintf(ret_string, "%llukb", mem_total);

  return(ret_string);
  }  /* END physmem() */






char *size_fs(

  char *param)

  {
  struct statfs fsbuf;

  if (param[0] != '/')
    {
    sprintf(log_buffer, "%s: not full path filesystem name: %s",
            __func__,
            param);

    log_err(-1, __func__, log_buffer);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if (statfs(param, &fsbuf) == -1)
    {
    log_err(errno, __func__, "statfs");

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

#ifdef RPT_BAVAIL
#define RPT_STATFS_MEMBER f_bavail
#else
#define RPT_STATFS_MEMBER f_bfree
#endif

  sprintf(ret_string, "%lukb:%lukb",
          (ulong)(((double)fsbuf.f_bsize * (double)fsbuf.RPT_STATFS_MEMBER) / 1024.0),
          (ulong)(((double)fsbuf.f_bsize * (double)fsbuf.f_blocks) / 1024.0)); /* KB */

  return(ret_string);
  }  /* END size_fs() */




char *size_file(

  char *param)

  {
  struct stat sbuf;

  if (param[0] != '/')
    {
    sprintf(log_buffer, "%s: not full path filesystem name: %s",
            __func__, param);

    log_err(-1, __func__, log_buffer);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if (stat(param, &sbuf) == -1)
    {
    log_err(errno, __func__, "stat");

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  sprintf(ret_string, "%lukb",

          (unsigned long)sbuf.st_size >> 10); /* KB */

  return(ret_string);
  }  /* END size_file() */




const char *size(

  struct rm_attribute *attrib)

  {
  char *param;

  if (attrib == NULL)
    {
    log_err(-1, __func__, no_parm);

    rm_errno = RM_ERR_NOPARAM;

    return(NULL);
    }

  if (momgetattr(NULL))
    {
    log_err(-1, __func__, extra_parm);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  param = attrib->a_value;

  if (strcmp(attrib->a_qualifier, "file") == 0)
    {
    return(size_file(param));
    }

  if (strcmp(attrib->a_qualifier, "fs") == 0)
    {
    return(size_fs(param));
    }

  rm_errno = RM_ERR_BADPARAM;

  return(NULL);
  }  /* END size() */

const char *cpuclock(

  struct rm_attribute *attrib)

  {
  std::string str;
  if(nd_frequency.get_frequency_string(str))
    {
    strcpy(ret_string,str.c_str());
    }
  else
    {
    strcpy(ret_string,"Fixed");
    }
  return ret_string;
  }  /* END cpuclock() */



/*
 * For a recovering (-p) mom, look through existing tasks in existing
 * jobs for things that have exited that are not owned by us through a
 * parent-child relationship.  Otherwise we cannot report back to tm
 * clients when tasks have exited.
 */

void scan_non_child_tasks(void)

  {
  job *pJob;
  static int first_time = TRUE;
  int log_drift_event = 0;

  DIR *pdir;  /* use local pdir to prevent race conditions associated w/global pdir (VPAC) */

  pdir = opendir(procfs);
  std::list<job *>::iterator iter;

  // get a list of jobs in start time order, first to last
  for (iter = alljobs_list.begin(); iter != alljobs_list.end(); iter++)
    {
    pJob = *iter;

    long job_start_time = 0;
    long job_session_id = 0;
    long session_start_time = 0;
    proc_stat_t *ps = NULL;
    if (pJob->ji_wattr[JOB_ATR_system_start_time].at_flags&ATR_VFLAG_SET)
      {
      job_start_time = pJob->ji_wattr[JOB_ATR_system_start_time].at_val.at_long;
      }

    if (pJob->ji_wattr[JOB_ATR_session_id].at_flags&ATR_VFLAG_SET)
      {
      job_session_id = pJob->ji_wattr[JOB_ATR_session_id].at_val.at_long;
      }

    if ((ps = get_proc_stat(job_session_id)) != NULL)
      {
      session_start_time = (long)ps->start_time;
      }

    for (unsigned int i = 0; i < pJob->ji_tasks->size(); i++)
      {
      task *pTask = pJob->ji_tasks->at(i);

#ifdef PENABLE_LINUX26_CPUSETS
      struct pidl   *pids = NULL;
      struct pidl   *pp;
#else
      struct dirent *dent;
#endif
      pid_t          pid;
      int            found;

      /*
       * Check for tasks that were exiting when mom went down, set back to
       * running so we can reprocess them and send the obit
       */
      if ((first_time) &&
          (pTask->ti_qs.ti_sid != 0) &&
          ((pTask->ti_qs.ti_status == TI_STATE_EXITED) ||
           (pTask->ti_qs.ti_status == TI_STATE_DEAD)))
        {

        if (LOGLEVEL >= 7)
          {
          sprintf(log_buffer, "marking task %d as TI_STATE_RUNNING was %d",
              pTask->ti_qs.ti_task,
              pTask->ti_qs.ti_status);

          log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, pJob->ji_qs.ji_jobid, log_buffer);
          }

        pTask->ti_qs.ti_status = TI_STATE_RUNNING;
        }

      /* only check on tasks that we think should still be around */

      if (pTask->ti_qs.ti_status != TI_STATE_RUNNING)
        continue;

      /* look for processes with this session id */

      found = 0;

      /* NOTE:  on linux systems, the session master should have pid == sessionid */

      if (kill(pTask->ti_qs.ti_sid, 0) != -1)
        {
        if((job_start_time != 0)&&
            (session_start_time != 0))
          {
          if(labs(job_start_time - session_start_time) < 3600)
            {
            found = 1;
            if(job_start_time != session_start_time)
              {
              log_drift_event = 1;
              }
            }
          else
            {
            log_drift_event = 1;
            }
          }
        else
          {
          found = 1;
          }
        }
      if(!found)
        {
        /* session master cannot be found, look for other pid in session */
#ifdef PENABLE_LINUX26_CPUSETS
        pids = get_cpuset_pidlist(pJob->ji_qs.ji_jobid, pids);
        pp   = pids;
        while (pp != NULL)
          {
          pid = pp->pid;
          pp  = pp->next;
#else
        if (pdir == NULL)
          {
          if ((pdir = opendir(procfs)) == NULL)
            return;
          }

        rewinddir(pdir);

        while ((dent = readdir(pdir)) != NULL)
          {
          if (!isdigit(dent->d_name[0]))
            continue;

          pid = atoi(dent->d_name);
#endif /* PENABLE_LINUX26_CPUSETS */
          if ((ps = get_proc_stat(pid)) == NULL)
            continue;

          if (ps->session == pTask->ti_qs.ti_sid)
            {
            if(pJob->ji_wattr[JOB_ATR_system_start_time].at_flags&ATR_VFLAG_SET)
              {
              proc_stat_t *ts = get_proc_stat(ps->session);
              long         job_start_time = pJob->ji_wattr[JOB_ATR_system_start_time].at_val.at_long;
              if (ts == NULL)
                continue;

              if (abs((long)ts->start_time - job_start_time) < 3600)
                {
                found = 1;
                break;
                }
              }
            else
              {
              found = 1;
              break;
              }
            }
          }    /* END while ((dent) != NULL) */
#ifdef PENABLE_LINUX26_CPUSETS
        free_pidlist(pids);
#endif
        }

      if (!found)
        {
        char buf[MAXLINE];

        extern int exiting_tasks;

        sprintf(buf, "found exited session %d for task %d in job %s",
            pTask->ti_qs.ti_sid,
            pTask->ti_qs.ti_task,
            pJob->ji_qs.ji_jobid);

        log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, buf);

        pTask->ti_qs.ti_exitstat = 0;  /* actually unknown */
        pTask->ti_qs.ti_status = TI_STATE_EXITED;

        task_save(pTask);

#ifdef USESAVEDRESOURCES
        if (first_time)
          {
          pJob->ji_flags |= MOM_JOB_RECOVERY;

          if (LOGLEVEL >= 7)
            {
            sprintf(buf, "marking job as MOM_JOB_RECOVERY for task %d",
                pTask->ti_qs.ti_task);

            log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, pJob->ji_qs.ji_jobid, buf);
            }
          }
#endif    /* USESAVEDRESOURCES */

        exiting_tasks = 1;
        }
      }
      
      if ((log_drift_event) && 
          (LOGLEVEL >= 7))
        {
        sprintf(log_buffer, "DRIFT debug: comparing linux_time %u; job_start_time %ld and session_start_time[%ld] %ld: difference %d",
          linux_time,
          job_start_time,
          job_session_id,
          session_start_time,
          (int)abs(job_start_time - session_start_time));
        log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, pJob->ji_qs.ji_jobid, log_buffer);
        }
    } /* END for each job */

  if (pdir != NULL)
    closedir(pdir);

  first_time = FALSE;

  return;
  }  /* END scan_non_child_tasks() */





time_t maxtm;

void setmax(

  const char *dev)

  {

  struct stat sb;

  if (stat(dev, &sb) == -1)
    {
    return;
    }

  if (maxtm < sb.st_atime)
    maxtm = sb.st_atime;

  return;
  }  /* END setmax() */




const char *idletime(

  struct rm_attribute *attrib)

  {
  DIR         *dp;

  struct dirent *de;
  char          ttyname[50];
  time_t         curtm;

  if (attrib)
    {
    log_err(-1, __func__, extra_parm);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if ((dp = opendir("/dev")) == NULL)
    {
    log_err(errno, __func__, "opendir /dev");

    rm_errno = RM_ERR_SYSTEM;

    return(NULL);
    }

  maxtm = 0;

  curtm = time(NULL);

  setmax("/dev/mouse");

  while ((de = readdir(dp)) != NULL)
    {
    if (maxtm >= curtm)
      break;

    if (strncmp(de->d_name, "tty", 3))
      continue;

    sprintf(ttyname, "/dev/%s",
            de->d_name);

    setmax(ttyname);
    }

  closedir(dp);

  sprintf(ret_string, "%ld",
          (long)MAX(0, curtm - maxtm));

  return(ret_string);
  }  /* END idletime() */




static const char *walltime(

  struct rm_attribute *attrib)

  {
  int  value, job, found = 0;
  time_t now, start;
  proc_stat_t  *ps;

  int           i;

  if (attrib == NULL)
    {
    log_err(-1, __func__, no_parm);

    rm_errno = RM_ERR_NOPARAM;

    return(NULL);
    }

  if ((value = atoi(attrib->a_value)) == 0)
    {
    sprintf(log_buffer, "bad param: %s",
            attrib->a_value);

    log_err(-1, __func__, log_buffer);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if (momgetattr(NULL))
    {
    log_err(-1, __func__, extra_parm);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if (strcmp(attrib->a_qualifier, "proc") == 0)
    {
    job = 0;
    }
  else if (strcmp(attrib->a_qualifier, "session") == 0)
    {
    job = 1;
    }
  else
    {
    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if ((now = time(NULL)) <= 0)
    {
    log_err(errno, __func__, "time");

    rm_errno = RM_ERR_SYSTEM;

    return(NULL);
    }

  start = now;

  for (i = 0;i < nproc;i++)
    {
    ps = &proc_array[i];

    if (job != 0)
      {
      if (value != ps->session)
        continue;
      }
    else
      {
      if (value != ps->pid)
        continue;
      }

    found = 1;

    start = MIN((unsigned)start, ps->start_time);
    }  /* END for (i) */

  if (found)
    {
    sprintf(ret_string, "%ld",
            (long)((double)(now - start) * wallfactor));

    return(ret_string);
    }

  rm_errno = RM_ERR_EXIST;

  return(NULL);
  }  /* END walltime() */



/* Get the load average for this node */

int get_la(

  double *rv)  /* O */

  {
  FILE *fp;
  float load;

  if ((fp = fopen("/proc/loadavg", "r")) == NULL)
    {
    rm_errno = RM_ERR_SYSTEM;

    return(rm_errno);
    }

  if (fscanf(fp, "%f",
             &load) != 1)
    {
    log_err(errno, __func__, "fscanf of load in /proc/loadavg");

    fclose(fp);

    rm_errno = RM_ERR_SYSTEM;

    return(rm_errno);
    }

  *rv = (double)load;

  fclose(fp);

  return(0);
  }  /* END get_la() */




#ifdef NUMA_SUPPORT

/*
 * Calculate cpu activities for numa nodeboards.
 *
 * This is a very preliminary attempt to provide useful load data for NUMA nodeboards.
 * Instead of a load average, we report the cpu activities of all cpus of a NUMA board.
 * Calculated numbers range from 0.0 (no CPU activity) to the number of
 * CPUs of a NUMA board (all CPUs are busy to 100%).
 *
 * Note that this is NOT the load average. However, it almost looks the same.
 *
 * The activity of a cpu is calculated from the content of /proc/stat like done
 * by top and related tools.
 */

void collect_cpuact(void)
  {
  FILE  *fp;
  char   label[128];
  long   procs;
  int    cpu_id;
  int    i;
  unsigned long long usr, nice, sys, idle, wait;
  unsigned long long totidle, totbusy, prevtot;
  unsigned long long dtot, dbusy;

  /*
   * Allocate cpu_array, if not already done.
   * Need to figure out number of cpus in the system, first.
   */

  if (cpu_array == NULL)
    {
    if ((fp = fopen("/proc/cpuinfo", "r")) == NULL)
      /* Failure */
      return;

    procs = 0;
    while (! feof(fp))
      {
      if (fscanf(fp, "%s %*[^\n]%*c", label) == 0)
        getc(fp);
      else if (strcmp("processor", label) == 0)
        procs++;
      }

    fclose(fp);
    system_ncpus = procs;

    sprintf(log_buffer, "system contains %ld CPUs", system_ncpus);
    log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer);

    if (system_ncpus)
      {
      if ((cpu_array = (proc_cpu_t *)calloc(system_ncpus, sizeof(proc_cpu_t))) == NULL)
        {
        log_err(errno, __func__, "failed to allocate memory");
        return;
        }
      }
    }

  /* Zero out cpu_array */
  memset(cpu_array, 0, system_ncpus * sizeof(proc_cpu_t));

  /* Parse CPU counters from /proc/stat */
  if ((fp = fopen("/proc/stat", "r")) != NULL)
    {
    while (! feof(fp))
      {
      if (fscanf(fp, "%s", label) != 1)
        /* Format error */
        break;

      if (sscanf(label, "cpu%d", &cpu_id) != 1)
        /* Line does not report cpu activities */
        continue;

      if (cpu_id >= system_ncpus)
        /* Ups, more cpus than found in /proc/cpuinfo */
        break;

      if (fscanf(fp, " %llu %llu %llu %llu %llu", &usr, &nice, &sys, &idle, &wait) != 5)
        /* Format error */
        break;

      cpu_array[cpu_id].idle_total = idle;
      cpu_array[cpu_id].busy_total = usr + nice + sys + wait;

      }
    fclose(fp);
    } /* END if (fp) */

  /* Calculate cpu activity for each nodeboard */
  for (i = 0; i < num_node_boards; i++)
    {

    /* Sum up cpu counters of relevant CPUs */
    totidle = totbusy = 0;
    hwloc_bitmap_foreach_begin(cpu_id, node_boards[i].cpuset)
      {
      totidle += cpu_array[cpu_id].idle_total;
      totbusy += cpu_array[cpu_id].busy_total;
      }
    hwloc_bitmap_foreach_end();

    /* If there are counters from a previous call, evaluate */
    if ((prevtot = node_boards[i].pstat_idle + node_boards[i].pstat_busy) != 0)
      {
      dbusy = totbusy - node_boards[i].pstat_busy; /* diff busy  counter sum */
      dtot  = totbusy + totidle - prevtot;        /* diff total counter sum */
      node_boards[i].cpuact = (float)(node_boards[i].num_cpus * dbusy / (double)dtot);
      }
    else
      {
      node_boards[i].cpuact = 0;
      }

    /* Remember counter sums */
    node_boards[i].pstat_idle = totidle;
    node_boards[i].pstat_busy = totbusy;

    } /* END for(i) */

  return;
  } /* END collect_cpuact() */





const char *cpuact(

  struct rm_attribute *attrib)

  {
  if (attrib != NULL)
    {
    log_err(-1, __func__, extra_parm);
    rm_errno = RM_ERR_BADPARAM;
    return(NULL);
    }

  sprintf(ret_string, "%.2f", node_boards[numa_index].cpuact);

  if (LOGLEVEL >= 6)
    {
    sprintf(log_buffer, "cpuact=%s", ret_string);
    log_record(PBSEVENT_SYSTEM, PBS_EVENTCLASS_NODE, __func__, log_buffer);
    }

  return(ret_string);
  } /* END cpuact() */
#endif






u_long gracetime(

  u_long secs)

  {
  time_t now = time((time_t *)NULL);

  if (secs > (u_long)now)  /* time is in the future */
    return(secs - now);

  return(0);
  }





static const char *quota(

  struct rm_attribute *attrib)

  {
  int  type;
  dev_t  dirdev;
  uid_t  uid;

  struct stat sb;

  struct mntent *me;

  struct dqblk qi;
  FILE  *m;

  struct passwd *pw;

  static const char *type_array[] =
    {
        "harddata",
        "softdata",
        "currdata",
        "hardfile",
        "softfile",
        "currfile",
        "timedata",
        "timefile",
    NULL
    };

  enum type_name
    {
    harddata,
    softdata,
    currdata,
    hardfile,
    softfile,
    currfile,
    timedata,
    timefile,
    type_end
    };

  if (attrib == NULL)
    {
    log_err(-1, __func__, no_parm);

    rm_errno = RM_ERR_NOPARAM;

    return(NULL);
    }

  if (strcmp(attrib->a_qualifier, "type"))
    {
    sprintf(log_buffer, "unknown qualifier %s",
            attrib->a_qualifier);

    log_err(-1, __func__, log_buffer);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  for (type = 0;type < type_end;type++)
    {
    if (strcmp(attrib->a_value, type_array[type]) == 0)
      break;
    }

  if (type == type_end)
    {
    /* check to see if command is legal */

    sprintf(log_buffer, "bad param: %s=%s",
            attrib->a_qualifier,
            attrib->a_value);

    log_err(-1, __func__, log_buffer);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if ((attrib = momgetattr(NULL)) == NULL)
    {
    log_err(-1, __func__, no_parm);

    rm_errno = RM_ERR_NOPARAM;

    return(NULL);
    }

  if (strcmp(attrib->a_qualifier, "dir") != 0)
    {
    sprintf(log_buffer, "bad param: %s=%s",
            attrib->a_qualifier,
            attrib->a_value);

    log_err(-1, __func__, log_buffer);

    rm_errno = RM_ERR_BADPARAM;

    return(NULL);
    }

  if (attrib->a_value[0] != '/')   /* must be absolute path */
    {
    sprintf(log_buffer,
            "not an absolute path: %s", attrib->a_value);
    log_err(-1, __func__, log_buffer);
    rm_errno = RM_ERR_BADPARAM;
    return NULL;
    }

  if (stat(attrib->a_value, &sb) == -1)
    {
    sprintf(log_buffer, "stat: %s", attrib->a_value);
    log_err(errno, __func__, log_buffer);
    rm_errno = RM_ERR_EXIST;
    return NULL;
    }

  dirdev = (dev_t)sb.st_dev;

  DBPRT(("dir has devnum %d\n", (int)dirdev))

  if ((m = setmntent(MOUNTED, "r")) == NULL)
    {
    log_err(errno, __func__, "setmntent");
    rm_errno = RM_ERR_SYSTEM;
    return NULL;
    }

  while ((me = getmntent(m)) != NULL)
    {
    if (strcmp(me->mnt_type, MNTTYPE_IGNORE) == 0)
      continue;

    if (stat(me->mnt_dir, &sb) == -1)
      {
      sprintf(log_buffer, "stat: %s", me->mnt_dir);
      log_err(errno, __func__, log_buffer);
      continue;
      }

    if (LOGLEVEL >= 6)
      DBPRT(("%s\t%s\t%d\n", me->mnt_fsname, me->mnt_dir, (int)dirdev))

      if (!memcmp(&sb.st_dev, &dirdev, sizeof(dev_t)))
        break;
    }

  endmntent(m);

  if (me == NULL)
    {
    sprintf(log_buffer,
            "filesystem %s not found", attrib->a_value);
    log_err(-1, __func__, log_buffer);
    rm_errno = RM_ERR_EXIST;
    return NULL;
    }

#if defined(MNTOPT_NOQUOTA)
  if (hasmntopt(me, MNTOPT_NOQUOTA) != NULL)
    {
    sprintf(log_buffer,
            "no quotas on filesystem %s", me->mnt_dir);
    log_err(-1, __func__, log_buffer);
    rm_errno = RM_ERR_EXIST;
    return NULL;
    }

#endif /* MNTOPT_NOQUOTA */

  if ((attrib = momgetattr(NULL)) == NULL)
    {
    log_err(-1, __func__, no_parm);
    rm_errno = RM_ERR_NOPARAM;
    return NULL;
    }

  if (strcmp(attrib->a_qualifier, "user") != 0)
    {
    sprintf(log_buffer, "bad param: %s=%s",
            attrib->a_qualifier, attrib->a_value);
    log_err(-1, __func__, log_buffer);
    rm_errno = RM_ERR_BADPARAM;
    return NULL;
    }

  if ((uid = (uid_t)atoi(attrib->a_value)) == 0)
    {
    char *buf;
    if ((pw = getpwnam_ext(&buf, attrib->a_value)) == NULL)
      {
      sprintf(log_buffer,
              "user not found: %s", attrib->a_value);
      log_err(-1, __func__, log_buffer);
      rm_errno = RM_ERR_EXIST;
      return NULL;
      }

    uid = pw->pw_uid;
    free_pwnam(pw, buf);
    }

  if (syscall(
        SYS_quotactl,
        QCMD(Q_GETQUOTA, USRQUOTA),
        me->mnt_fsname,
        uid,
        (caddr_t)&qi) == -1)
    {
    log_err(errno, __func__, "quotactl");

    rm_errno = RM_ERR_SYSTEM;

    return(NULL);
    }

  /* sizes in KB */

  switch (type)
    {

    case harddata:

      sprintf(ret_string, "%lukb",
              (u_long)qi.dqb_bhardlimit >> 10);

      break;

    case softdata:

      sprintf(ret_string, "%lukb",
              (u_long)qi.dqb_bsoftlimit >> 10);

      break;

    case currdata:

#if defined(TENABLEQUOTA)
#if _LINUX_QUOTA_VERSION < 2
      sprintf(ret_string, "%lukb",
              (u_long)qi.dqb_curblocks >> 10);
#else /* _LINUX_QUOTA_VERSION < 2 */
      sprintf(ret_string, "%lukb",
              (u_long)qi.dqb_curspace >> 10);
#endif /* _LINUX_QUOTA_VERSION < 2 */
#endif /* TENABLEQUOTA */

      break;

    case hardfile:

      sprintf(ret_string, "%lu",
              (u_long)qi.dqb_ihardlimit);

      break;

    case softfile:

      sprintf(ret_string, "%lu",
              (u_long)qi.dqb_isoftlimit);

      break;

    case currfile:

      sprintf(ret_string, "%lu",
              (u_long)qi.dqb_curinodes);

      break;

    case timedata:

      sprintf(ret_string, "%lu",
              gracetime((u_long)qi.dqb_btime));

      break;

    case timefile:

      sprintf(ret_string, "%lu",
              gracetime((u_long)qi.dqb_itime));

      break;
    }  /* END switch() */

  return(ret_string);
  }  /* END quota() */



/* tested for linux 2.4 kernel (not tested on 2.6) */

#define MAX_INTERFACES 10 /*the maximum number of interfaces*/
#define HEADER_STR "%*[^\n]\n%*[^\n]\n"
#define INTERFACE_STR "%[^:]:%lu %*d %*d %*d %*d %*d %*d %*d %lu %*d %*d %*d %*d %*d %*d %*d\n"

static const char *netload(

  struct rm_attribute *attrib)

  {
#ifdef NUMA_SUPPORT
  /* there's no way to determine these numbers for a numa node */
  return(NULL);
#else
  FILE *fp;
  int   rc; /*read count*/

  char interfaceName[MAX_INTERFACES][32];
  unsigned long int bytesRX[MAX_INTERFACES + 1];
  unsigned long int bytesTX[MAX_INTERFACES + 1];

  int interface = 0;
  /* int ethNum = 0; */

  if ((fp = fopen("/proc/net/dev", "r")) == NULL)
    {
    rm_errno = RM_ERR_SYSTEM;

    return(NULL);
    }

  rc = fscanf(fp, HEADER_STR); /*strip off header lines*/

  if (rc < 0)
    {
    log_err(errno, __func__, "fscanf of header lines in /proc/net/dev");

    fclose(fp);

    rm_errno = RM_ERR_SYSTEM;

    return(NULL);
    }

  /* read in interface stats until we can't */
  /* sum all interface stats, excluding 'lo'*/

  memset(bytesRX, 0, sizeof(bytesRX));

  memset(bytesTX, 0, sizeof(bytesTX));

  for (interface = 0;interface < MAX_INTERFACES;interface++)
    {
    rc = fscanf(fp, INTERFACE_STR,
                interfaceName[interface],
                &bytesRX[interface],
                &bytesTX[interface]);

    if (rc != 3)
      {

      interface++; /*adjust counter for future decrement*/

      break;
      }

    if (strcmp(interfaceName[interface], "lo") != 0) /* don't count 'lo' interfaces' stats */
      {
      /* For singling out ethernet interfaces */

      /*
      if (strncmp(interfaceName[interface],"eth",3) == 0)
        {
        rc = sscanf(interfaceName[interface],"eth%d",
          &ethNum);
        }
      */

      bytesRX[MAX_INTERFACES] += bytesRX[interface];
      bytesTX[MAX_INTERFACES] += bytesTX[interface];
      }
    }   /* END for (interface) */

  /* remove lo from interface count */

  --interface;

  fclose(fp);

  sprintf(ret_string, "%lu",
          bytesRX[MAX_INTERFACES] + bytesTX[MAX_INTERFACES]);

  return(ret_string);
#endif /* NUMA_SUPPORT */
  }  /* END netload() */





mbool_t ProcIsChild(

  char  *Dir,    /* I */
  pid_t  PID,    /* I */
  char  *JobID)  /* I */

  {
  return(FALSE);
  }  /* END ProcIsChild() */

/* END mom_mach.c */
