/*
*         OpenPBS (Portable Batch System) v2.3 Software License
*
* Copyright (c) 1999-2000 Veridian Information Solutions, Inc.
* All rights reserved.
*
* ---------------------------------------------------------------------------
* For a license to use or redistribute the OpenPBS software under conditions
* other than those described below, or to purchase support for this software,
* please contact Veridian Systems, PBS Products Department ("Licensor") at:
*
*    www.OpenPBS.org  +1 650 967-4675                  sales@OpenPBS.org
*                        877 902-4PBS (US toll-free)
* ---------------------------------------------------------------------------
*
* This license covers use of the OpenPBS v2.3 software (the "Software") at
* your site or location, and, for certain users, redistribution of the
* Software to other sites and locations.  Use and redistribution of
* OpenPBS v2.3 in source and binary forms, with or without modification,
* are permitted provided that all of the following conditions are met.
* After December 31, 2001, only conditions 3-6 must be met:
*
* 1. Commercial and/or non-commercial use of the Software is permitted
*    provided a current software registration is on file at www.OpenPBS.org.
*    If use of this software contributes to a publication, product, or
*    service, proper attribution must be given; see www.OpenPBS.org/credit.html
*
* 2. Redistribution in any form is only permitted for non-commercial,
*    non-profit purposes.  There can be no charge for the Software or any
*    software incorporating the Software.  Further, there can be no
*    expectation of revenue generated as a consequence of redistributing
*    the Software.
*
* 3. Any Redistribution of source code must retain the above copyright notice
*    and the acknowledgment contained in paragraph 6, this list of conditions
*    and the disclaimer contained in paragraph 7.
*
* 4. Any Redistribution in binary form must reproduce the above copyright
*    notice and the acknowledgment contained in paragraph 6, this list of
*    conditions and the disclaimer contained in paragraph 7 in the
*    documentation and/or other materials provided with the distribution.
*
* 5. Redistributions in any form must be accompanied by information on how to
*    obtain complete source code for the OpenPBS software and any
*    modifications and/or additions to the OpenPBS software.  The source code
*    must either be included in the distribution or be available for no more
*    than the cost of distribution plus a nominal fee, and all modifications
*    and additions to the Software must be freely redistributable by any party
*    (including Licensor) without restriction.
*
* 6. All advertising materials mentioning features or use of the Software must
*    display the following acknowledgment:
*
*     "This product includes software developed by NASA Ames Research Center,
*     Lawrence Livermore National Laboratory, and Veridian Information
*     Solutions, Inc.
*     Visit www.OpenPBS.org for OpenPBS software support,
*     products, and information."
*
* 7. DISCLAIMER OF WARRANTY
*
* THIS SOFTWARE IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND. ANY EXPRESS
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT
* ARE EXPRESSLY DISCLAIMED.
*
* IN NO EVENT SHALL VERIDIAN CORPORATION, ITS AFFILIATED COMPANIES, OR THE
* U.S. GOVERNMENT OR ANY OF ITS AGENCIES BE LIABLE FOR ANY DIRECT OR INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
* OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* This license will be governed by the laws of the Commonwealth of Virginia,
* without reference to its choice of law rules.
*/
#include <pbs_config.h>   /* the master config generated by configure */

#include <assert.h>
#include <stdlib.h>
#include <ctype.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <pthread.h>
#include <string>
#include "libpbs.h"
#include "list_link.h"
#include "attribute.h"
#include "server_limits.h"
#include "credential.h"
#include "batch_request.h"
#include "pbs_job.h"
#include "queue.h"
#include "server.h"
#include "work_task.h"
#include "pbs_error.h"
#include "log.h"
#include "../lib/Liblog/pbs_log.h"
#include "../lib/Liblog/log_event.h"
#include "svrfunc.h"
#include "array.h"
#include "svr_func.h" /* get_svr_attr_* */
#include "ji_mutex.h"
#include "mutex_mgr.hpp"
#include "utils.h"
#include "job_func.h"


#define SYNC_SCHED_HINT_NULL 0
#define SYNC_SCHED_HINT_FIRST 1
#define SYNC_SCHED_HINT_OTHER 2


/*
 * This file holds all the functions dealing with job dependency
 */

/* External functions */

extern int que_to_local_svr(struct batch_request *preq);
extern long calc_job_cost(job *);
char *get_correct_jobname(const char *jobid);


/* Local Private Functions */

void set_depend_hold(job *, pbs_attribute *, job_array *);
int register_sync(struct depend *,  char *child, char *host, long);
int register_dep(pbs_attribute *, struct batch_request *, int, int *);
int unregister_dep(pbs_attribute *, struct batch_request *);
int unregister_sync(pbs_attribute *, struct batch_request *);

struct depend *find_depend(int type, pbs_attribute *pattr);

struct depend *make_depend(int type, pbs_attribute *pattr);

depend_job *find_dependjob(struct depend *, const char *name);

depend_job *make_dependjob(struct depend *, const char *jobid);
void   del_depend_job(struct depend *pdep, depend_job *pdj);
int    build_depend(pbs_attribute *, const char *);
void   clear_depend(struct depend *, int type, int exists);
int    release_cheapest(job *, struct depend *);
int    send_depend_req(job *, depend_job *pparent, int, int, int, void (*postfunc)(batch_request *),bool bAsyncOk);
depend_job *alloc_dependjob(const char *jobid);

/* External Global Data Items */

extern struct server server;
extern char  server_name[];
extern char *msg_err_malloc;
extern char *msg_illregister;
extern char *msg_registerdel;
extern char *msg_registerrel;
extern char *msg_regrej;
extern int   LOGLEVEL;
extern char *PJobState[];


extern int   svr_chk_owner(struct batch_request *, job *);


 /*
 * check_dependency_job()
 * checks to make the sure the dependency request is legitimate
 *
 * @parent req_register
 * @param jobid - the id of the job
 * @param job_ptr - the job 
 * @param preq - the request for the dependency
 */
 
int check_dependency_job(
 
  char           *jobid,   /* I */
  batch_request  *preq,    /* M */
  job           **job_ptr) /* O */

  {
  job   *pjob;
  char   log_buf[LOCAL_LOG_BUF_SIZE + 1];
  int    type;
  int    rc = PBSE_NONE;

  if ((jobid == NULL) ||
      (preq == NULL) ||
      (job_ptr == NULL))
    return(PBSE_BAD_PARAMETER);

  *job_ptr = NULL;

  /*  make sure request is from a server */
  if (!preq->rq_fromsvr)
    {
    req_reject(PBSE_IVALREQ, 0, preq, NULL, NULL);

    return(PBSE_IVALREQ);
    }
   
  if (LOGLEVEL >= 10)
    {
    snprintf(log_buf, sizeof(log_buf), "rq_parent: %s", jobid);
    log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_QUEUE, __func__, log_buf);
    }

  /* find the "parent" job specified in the request */
  if ((pjob = svr_find_job(jobid, TRUE)) == NULL)
    {
    /*
     * job not found... if server is initializing, it may not
     * yet recovered, that is not an error.
     */
    long state = SV_STATE_DOWN;

    get_svr_attr_l(SRV_ATR_State, &state);

    if (state != SV_STATE_INIT)
      {
      log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, jobid, pbse_to_txt(PBSE_UNKJOBID));

      rc = PBSE_UNKJOBID;
      req_reject(rc, 0, preq, NULL, NULL);
      }
    else
      {
      reply_ack(preq);
      rc = PBSE_JOBNOTFOUND;
      }

    return(rc);
    }

  mutex_mgr job_mutex(pjob->ji_mutex, true);

  type = preq->rq_ind.rq_register.rq_dependtype;

  if (((pjob->ji_qs.ji_state == JOB_STATE_COMPLETE) ||
        (pjob->ji_qs.ji_state == JOB_STATE_EXITING)) &&
      ((type == JOB_DEPEND_TYPE_AFTERSTART) ||
       (type == JOB_DEPEND_TYPE_AFTERANY) ||
       ((type == JOB_DEPEND_TYPE_AFTEROK) &&
        (pjob->ji_qs.ji_un.ji_exect.ji_exitstat == 0)) ||
       ((type == JOB_DEPEND_TYPE_AFTERNOTOK) &&
        (pjob->ji_qs.ji_un.ji_exect.ji_exitstat != 0))))
    {
    log_event(PBSEVENT_DEBUG, PBS_EVENTCLASS_JOB, jobid, pbse_to_txt(PBSE_BADSTATE));
    
    rc = PBSE_BADSTATE;
    req_reject(rc, 0, preq, NULL, NULL);
    
    return(rc);
    }

  job_mutex.set_unlock_on_exit(false);
  *job_ptr = pjob;

  return(PBSE_NONE);
  } /* END check_dependency_job() */




/*
 * register_syncwith()
 * registers the syncwith dependency type
 * @parent register_dependency()
 */

int register_syncwith(

  batch_request *preq,
  job           *pjob)

  {
  pbs_attribute *pattr = &pjob->ji_wattr[JOB_ATR_depend];
  int            rc = PBSE_NONE;
  struct depend *pdep = find_depend(JOB_DEPEND_TYPE_SYNCCT, pattr);
  
  if (pdep != NULL)
    {
    rc = register_sync(
        pdep,
        preq->rq_ind.rq_register.rq_child,
        preq->rq_ind.rq_register.rq_svr,
        preq->rq_ind.rq_register.rq_cost);
    
    if (pdep->dp_numreg > pdep->dp_numexp)
      {
      /* all registered - release first */
      
      release_cheapest(pjob, pdep);
      }
    }
  else
    {
    rc = PBSE_NOSYNCMSTR;
    }

  return(rc);
  } /* END register_syncwith() */




int register_before_dep(
    
  batch_request *preq,
  job           *pjob,
  int            type)

  {
  int            rc = PBSE_NONE;
  depend        *pdep = NULL;
  pbs_attribute *pattr = &pjob->ji_wattr[JOB_ATR_depend];
  int            revtype;
  int            made = FALSE;

  /*
   * Check job owner for permission, use the real
   * job owner, not the sending server's name.
   */
  strcpy(preq->rq_user, preq->rq_ind.rq_register.rq_owner);
  
  if (svr_chk_owner(preq, pjob))
    {
    rc = PBSE_PERM;  /* not same user */
    }
  else
    {
    /* ok owner, see if job has "on" */
    pdep = find_depend(JOB_DEPEND_TYPE_ON, pattr);
    
    if (pdep == NULL)
      {
      /* on "on", see if child already registered */
      revtype = type ^(JOB_DEPEND_TYPE_BEFORESTART - JOB_DEPEND_TYPE_AFTERSTART);
      
      pdep = find_depend(revtype, pattr);
      
      if (pdep == NULL)
        {
        /* no "on" and no prior - return error */
        printf("in the on code\n");
        
        rc = PBSE_BADDEPEND;
        }
      }
    else if ((rc = register_dep(pattr, preq, type, &made)) == PBSE_NONE)
      {
      if (made)
        {
        /* first time registered */
        if (--pdep->dp_numexp <= 0)
          delete pdep;
        }
      }
    }

  return(rc);
  } /* END register_before_dep() */



bool job_ids_match(

  const char *parent,
  const char *child)

  {
  bool match;

  if ((is_svr_attr_set(SRV_ATR_display_job_server_suffix)) ||
      (is_svr_attr_set(SRV_ATR_job_suffix_alias)))
    {
    char *correct_parent = get_correct_jobname(parent);
    char *correct_child = get_correct_jobname(child);

    match = strcmp(correct_parent, correct_child) == 0;

    free(correct_parent);
    free(correct_child);
    }
  else
    match = strcmp(parent, child) == 0;

  return(match);
  } /* END job_ids_match() */



/*
 * register_dependency()
 * handles the registering of a dependency on a job
 * @parent req_register()
 */
 
int register_dependency(
 
  batch_request *preq,
  job           *pjob,
  int            type)

  {
  int rc = PBSE_NONE;
  int made = FALSE;

  if (job_ids_match(preq->rq_ind.rq_register.rq_parent, preq->rq_ind.rq_register.rq_child))
    return(PBSE_IVALREQ);

  switch (type)
    {
    
    case JOB_DEPEND_TYPE_SYNCWITH:
     
      rc = register_syncwith(preq, pjob);
      
      break;
      
    case JOB_DEPEND_TYPE_AFTERSTART:
      
    case JOB_DEPEND_TYPE_AFTERANY:
      
    case JOB_DEPEND_TYPE_AFTEROK:
      
    case JOB_DEPEND_TYPE_AFTERNOTOK:
      
      rc = register_dep(&pjob->ji_wattr[JOB_ATR_depend], preq, type, &made);
      
      break;

    case JOB_DEPEND_TYPE_BEFORESTART:
      
    case JOB_DEPEND_TYPE_BEFOREANY:
      
    case JOB_DEPEND_TYPE_BEFOREOK:
      
    case JOB_DEPEND_TYPE_BEFORENOTOK:

      rc = register_before_dep(preq, pjob, type);

      break;
 
    default:

       rc = PBSE_IVALREQ;
      
      break;
    }
 
  return(rc);
  } /* END register_dependency() */





int release_before_dependency(

  batch_request *preq,
  job           *pjob,
  int            type)
 
  {
  int                rc = PBSE_NONE;
  pbs_attribute     *pattr = &pjob->ji_wattr[JOB_ATR_depend];
  struct depend     *pdep = NULL;
  depend_job        *pdj = NULL;
  char               log_buf[LOCAL_LOG_BUF_SIZE];
  /*char               job_id[PBS_MAXSVRJOBID+1];*/
 
  /* predecessor sent release-reduce "on", */
  /* see if this job can now run    */
  
  type ^= (JOB_DEPEND_TYPE_BEFORESTART - JOB_DEPEND_TYPE_AFTERSTART);
  
  if ((pdep = find_depend(type, pattr)))
    {
    if ((pdj = find_dependjob(pdep, preq->rq_ind.rq_register.rq_child)))
      {
      del_depend_job(pdep, pdj);
      
      sprintf(log_buf, msg_registerrel, preq->rq_ind.rq_register.rq_child);
      log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,pjob->ji_qs.ji_jobid,log_buf);
      
      if (pdep->dp_jobs.size() == 0)
        {
        /* no more dependencies of this type */
        delete pdep;
        
        set_depend_hold(pjob, pattr, NULL);
        }
      
      return(rc);
      }
    } /* END if ((pdep = find_depend(type,pattr))) */
  
  rc = PBSE_IVALREQ;
 
  return(rc);
  } /* END release_before_dependency() */




int release_syncwith_dependency(
 
  batch_request *preq,
  job           *pjob)
 
  {
  pbs_attribute     *pattr = &pjob->ji_wattr[JOB_ATR_depend];
  struct depend     *pdep = NULL;
  int                rc = PBSE_NONE;
  char               tmpcoststr[64];
  /*char               job_id[PBS_MAXSVRJOBID+1];*/
 
  pdep = find_depend(JOB_DEPEND_TYPE_SYNCCT, pattr);
  
  if (pdep == NULL)
    pdep = find_depend(JOB_DEPEND_TYPE_SYNCWITH, pattr);
  
  if (pdep != NULL)
    {
    pdep->dp_released = 1;
    
    set_depend_hold(pjob, pattr, NULL);
    
    sprintf(tmpcoststr, "%ld", preq->rq_ind.rq_register.rq_cost);

    if (pjob->ji_wattr[JOB_ATR_sched_hint].at_val.at_str != NULL)
      free(pjob->ji_wattr[JOB_ATR_sched_hint].at_val.at_str);

    pjob->ji_wattr[JOB_ATR_sched_hint].at_val.at_str = strdup(tmpcoststr);
    pjob->ji_wattr[JOB_ATR_sched_hint].at_flags |= ATR_VFLAG_SET;
    }
  else
    {
    rc = PBSE_NOSYNCMSTR;
    }

  return(rc);
  } /* END release_syncwith_dependency() */




int release_dependency(

  batch_request *preq,
  job           *pjob,
  int            type)

  {
  int rc = PBSE_NONE;

  switch (type)
    {
    
    case JOB_DEPEND_TYPE_BEFORESTART:
      
    case JOB_DEPEND_TYPE_BEFOREANY:
      
    case JOB_DEPEND_TYPE_BEFOREOK:
      
    case JOB_DEPEND_TYPE_BEFORENOTOK:
      
      rc = release_before_dependency(preq, pjob, type);
      
      break;
      
    case JOB_DEPEND_TYPE_SYNCWITH:
      
      rc = release_syncwith_dependency(preq, pjob);
      
      break;
    }
 
  return(rc);
  } /* END release_dependency() */



int ready_dependency(

  batch_request *preq,
  job           *pjob)

  {
  int                rc = PBSE_NONE;
  pbs_attribute     *pattr = &pjob->ji_wattr[JOB_ATR_depend];
  struct depend     *pdep = NULL;
  depend_job        *pdj = NULL;

  if ((pdep = find_depend(JOB_DEPEND_TYPE_SYNCCT, pattr)))
    {
    /* mark sender as running */
    
    unsigned int dp_jobs_size = pdep->dp_jobs.size();
    for (unsigned int i = 0; i < dp_jobs_size; i++)
      {
      pdj = pdep->dp_jobs[i];
      
      if (pdj->dc_child == preq->rq_ind.rq_register.rq_child)
        {
        pdj->dc_state = JOB_DEPEND_OP_READY;
        
        break;
        }
      }
    
    release_cheapest(pjob, pdep); /* release next one */
    }
  else
    {
    rc = PBSE_NOSYNCMSTR;
    }

  return(rc);
  } /* END ready_dependency() */



/*
 * delete_dependency_job()
 *
 * Deletes the dependent job. We are here because the dependency can never be satisfied.
 * @param preq - the batch request specifying the request
 * @param pjob_ptr - a pointer to a pointer to the job
 * @return PBSE_NONE on success or PBSE_IVALREQ if a job is detected to be dependent on
 * itself.
 */

int delete_dependency_job(
 
  batch_request *preq,
  job           **pjob_ptr)
 
  {
  job *pjob = *pjob_ptr;
  int  rc = PBSE_NONE;
  char log_buf[LOCAL_LOG_BUF_SIZE];
  
  // Do not take action for a job depending on itself.
  if (!strcmp(preq->rq_ind.rq_register.rq_parent,
        preq->rq_ind.rq_register.rq_child))
    {
    rc = PBSE_IVALREQ; /* prevent an infinite loop */
    }
  // Only abort if the job isn't already exiting, and never abort
  // running jobs for any job dependency reasons.
  else if ((pjob->ji_qs.ji_state < JOB_STATE_EXITING) &&
           (pjob->ji_qs.ji_state != JOB_STATE_RUNNING))
    {
    sprintf(log_buf, msg_registerdel, preq->rq_ind.rq_register.rq_child);
    log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf);
    
    /* pjob freed and set to NULL */
    job_abt(pjob_ptr, log_buf, true);
    }

  return(rc);
  } /* END delete_dependency_job() */



/*
 * unregister_dependency()
 *
 * Remove a dependency from a job
 * @param preq - the request in question
 * @param pjob - the job who needs to have a dependency removed.
 * @param type - the dependency type
 * @return PBSE_NONE on success or a failure code
 */

int unregister_dependency(
 
  batch_request *preq,
  job           *pjob,
  int            type)

  {
  pbs_attribute *pattr = &pjob->ji_wattr[JOB_ATR_depend];
  int            rc = PBSE_NONE;
 
  if (type == JOB_DEPEND_TYPE_SYNCWITH)
    {
    if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING)
      {
      rc = PBSE_IVALREQ;
      }
    else
      {
      unregister_sync(pattr, preq);
      }
    }
  else
    {
    unregister_dep(pattr, preq);
    }
  
  set_depend_hold(pjob, pattr, NULL);

  return(rc);
  } /* END unregister_dependency() */




/*
 * req_register - process the Register Dependency Request
 *
 * We have an interesting problem here in that the request may well
 * orginate from ourself.  In that case we doen't really reply.
 *
 * Note: It is possible, though it doesn't make sense, for a job
 * to have a self-referencing depend.  We reject these for the register
 * and delete operations, but allow it for others in an attempt
 * to be graceful.
 */

int req_register(
 
  batch_request *preq)  /* I */
 
  {
  job  *pjob = NULL;
  char *ps;
  int   rc = PBSE_NONE;
  int   type = preq->rq_ind.rq_register.rq_dependtype;
  char  log_buf[LOCAL_LOG_BUF_SIZE + 1];

  if ((rc = check_dependency_job(preq->rq_ind.rq_register.rq_parent, preq, &pjob)) != PBSE_NONE)
    return(rc);

  if (LOGLEVEL >= 8)
    {
    sprintf(log_buf,"Dependency requested parent job %s state (%s), child job %s",
      preq->rq_ind.rq_register.rq_parent,
      PJobState[pjob->ji_qs.ji_state],
      preq->rq_ind.rq_register.rq_child);

    log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,pjob->ji_qs.ji_jobid,log_buf);
    }
 
  pjob->ji_modified = 1;
 
  /* more of the server:port fix kludge */
  if ((ps = strchr(preq->rq_ind.rq_register.rq_child, (int)'@')))
    {
    snprintf(preq->rq_ind.rq_register.rq_svr, sizeof(preq->rq_ind.rq_register.rq_svr), "%s", ps + 1);

    *ps = '\0';
    }
  else
    {
    /* We ignore what qsub sent us for the host. We need to use the 
     * server_name so we know we can connect to the active server.
     * This is an high availability consideration */
    if (server_name[0] != '\0')
      strcpy(preq->rq_ind.rq_register.rq_svr, server_name);
    else
      strcpy(preq->rq_ind.rq_register.rq_svr, preq->rq_host);
    }
 
  /* Handle the dependency */
  try
    {
    switch (preq->rq_ind.rq_register.rq_op)
      {
      case JOB_DEPEND_OP_REGISTER:

        rc = register_dependency(preq, pjob, type);

        break;
        
      case JOB_DEPEND_OP_RELEASE:

        rc = release_dependency(preq, pjob, type);

        break;

      case JOB_DEPEND_OP_READY:

        rc = ready_dependency(preq, pjob);
   
        break;
   
      case JOB_DEPEND_OP_DELETE:
   
        rc = delete_dependency_job(preq, &pjob);
   
        break;
   
      case JOB_DEPEND_OP_UNREG:

        rc =  unregister_dependency(preq, pjob, type);

        break;

      default:

        sprintf(log_buf, msg_illregister, preq->rq_ind.rq_register.rq_parent);

        log_event(
          PBSEVENT_DEBUG | PBSEVENT_SYSTEM | PBSEVENT_ERROR,
          PBS_EVENTCLASS_REQUEST,
          preq->rq_host,
          log_buf);

        rc = PBSE_IVALREQ;

        break;
      }  /* END switch (preq->rq_ind.rq_register.rq_op) */
    }
  catch (int pbs_errcode)
    {
    if (pbs_errcode == PBSE_JOBNOTFOUND)
      pjob = NULL;
    }

  if (rc)
    {
    if (pjob != NULL)
      pjob->ji_modified = 0;

    req_reject(rc, 0, preq, NULL, NULL);
    }
  else
    {
    if ((pjob != NULL) &&
        (pjob->ji_modified != 0))
      job_save(pjob, SAVEJOB_FULL, 0);

    reply_ack(preq);
    }

  if (pjob != NULL)
    unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL);

  return(rc);
  }  /* END req_register() */




/*
 * req_registerarray() 
 * registers a dependency on an array
 */

int req_registerarray(

  struct batch_request *preq)  /* I */

  {
  job_array  *pa;
  char        array_name[PBS_MAXSVRJOBID + 1];
  char        range[MAXPATHLEN];
  int         num_jobs = -1;
  char       *dot_server;
  char       *bracket_ptr;
  int         rc = PBSE_NONE;
  int         type;

  /*  make sure request is from a server */
  if (!preq->rq_fromsvr)
    {
    req_reject(PBSE_IVALREQ, 0, preq, NULL, NULL);

    return(PBSE_IVALREQ);
    }

  strcpy(array_name,preq->rq_ind.rq_register.rq_parent);

  /* if brackets are present, remove the brackets and 
   * store the number in range */
  range[0] = '\0';
  if ((bracket_ptr = strchr(array_name,'[')) != NULL)
    {
    /* go to the 2nd pair of brackets, if present */
    if ((bracket_ptr = strchr(bracket_ptr+1,'[')) != NULL)
      {
      *bracket_ptr = '\0';
      bracket_ptr++;

      snprintf(range, sizeof(range), "%s", bracket_ptr);
      if ((bracket_ptr = strchr(range,']')) != NULL)
        {
        *bracket_ptr = '\0';

        /* only do this if range isn't empty */
        if (strcmp(range,""))
          {
          num_jobs = atoi(range);
          }

        if ((dot_server = strchr(bracket_ptr+1,'.')) != NULL)
          {
          safe_strncat(array_name, dot_server, sizeof(array_name) - strlen(array_name) - 1);
          }
        else 
          {
          /* error, no server. shouldn't get here ever due
           * to checks in depend_on_que */

          req_reject(PBSE_IVALREQ,0,preq,NULL,
            "No server specified");
          return(PBSE_IVALREQ);
          }
        }
      else
        {
        /* error, if bracket opens must close */

        req_reject(PBSE_IVALREQ,0,preq,NULL,
          "Array range format invalid, must have closed bracket ']'");
        return(PBSE_IVALREQ);
        }
      } /* end second brackets if */
    }

  /* get the array */
  if ((pa = get_array(array_name)) == NULL)
    {
    /*
     * array not found... if server is initializing, it may not
     * yet be recovered, that is not an error.
     */
    long state = SV_STATE_DOWN;
    get_svr_attr_l(SRV_ATR_State, &state);

    rc = PBSE_UNKARRAYID;
    if (state != SV_STATE_INIT)
      {
      log_event(
        PBSEVENT_DEBUG,
        PBS_EVENTCLASS_JOB,
        preq->rq_ind.rq_register.rq_parent,
        pbse_to_txt(PBSE_UNKJOBID));

      req_reject(rc, 0, preq, NULL, "unable to find array");
      }
    else
      {
      reply_ack(preq);
      }

    return(rc);
    }

  type = preq->rq_ind.rq_register.rq_dependtype;

  /* register the dependency on the array */

  switch (preq->rq_ind.rq_register.rq_op)
    {

    case JOB_DEPEND_OP_REGISTER:

      if ((rc = register_array_depend(pa,preq,type,num_jobs)))
        {
        req_reject(rc,0,preq,NULL,NULL);
        }

      set_array_depend_holds(pa);

      break;

    case JOB_DEPEND_OP_RELEASE:

    case JOB_DEPEND_OP_DELETE:
    case JOB_DEPEND_OP_UNREG:

      /* NYI */

      break;
    } /* END switch (preq->rq_ind.rq_register.rq_op */

  unlock_ai_mutex(pa, __func__, "2", LOGLEVEL);

  return(rc);
  } /* END req_registerarray() */



int register_array_depend(

  job_array            *pa,    /* I/O */
  struct batch_request *preq,  /* I */
  int                   type,  /* I */
  int                   num_jobs) /* I */

  {
  struct array_depend     *pdep = NULL;

  array_depend_job        *pdj = NULL;

  /* check for existing dependencies of that type */
  std::list<array_depend *>::iterator it = pa->ai_qs.deps.begin();

  while (it != pa->ai_qs.deps.end())
    {
    if (type == (*it)->dp_type)
      {
      pdep = *it;
      break; 
      }

    it++;
    }

  /* make dependency if none exists */
  if (pdep == NULL)
    {
    pdep = new array_depend();

    if (pdep != NULL)
      {
      pdep->dp_type = type;

      pa->ai_qs.deps.push_back(pdep);
      }
    else
      return(PBSE_SYSTEM);
    }

  /* now we have the dependency, add the job to it */
  /* verify the job isn't already there */
  unsigned int dp_jobs_size = pdep->dp_jobs.size();
  for (unsigned int i = 0; i < dp_jobs_size; i++)
    {
    if (pdep->dp_jobs[i]->dc_child == preq->rq_ind.rq_register.rq_child)
      {
      pdj = pdep->dp_jobs[i];
      break;
      }

    }

  /* assume success if the job is there */
  if (pdj != NULL)
    return(PBSE_NONE);

  /* try to create the job */
  pdj = new array_depend_job();

  if (pdj != NULL)
    {
    pdj->dc_child = preq->rq_ind.rq_register.rq_child;

    if (num_jobs == -1)
      {
      /* not specified, make it the entire array */
      pdj->dc_num = pa->ai_qs.num_jobs;
      }
    else
      {
      /* specified, use parameter */
      pdj->dc_num = num_jobs;
      }

    pdep->dp_jobs.push_back(pdj);
    }
  else
    {
    return(PBSE_SYSTEM);
    }

  /* SUCCESS */

  return(PBSE_NONE);
  } /* END register_array_depend */



bool remove_array_dependency_job_from_job(

  struct array_depend *pdep,
  job                 *pjob,
  char                *job_array_id)

  {
  struct depend_job *job_pdj;
  struct depend     *job_pdep = NULL;
  bool               removed = false;
  char               log_buf[LOCAL_LOG_BUF_SIZE];

  if ((job_pdep = find_depend(pdep->dp_type, &pjob->ji_wattr[JOB_ATR_depend])) != NULL)
    {
    if ((job_pdj = find_dependjob(job_pdep, job_array_id)) != NULL)
      {
      removed = true;
      del_depend_job(job_pdep, job_pdj);
      
      snprintf(log_buf, sizeof(log_buf), msg_registerrel, job_array_id);
      log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf);
      
      if (job_pdep->dp_jobs.size() == 0)
        {
        /* no more dependencies of this type */
        delete job_pdep;
        }
      }
    }

  return(removed);
  } /* remove_array_dependency_job_from_job() */



/*
 * set_array_depend_holds()
 *
 * looks through the array which is passed for all jobs depending on this
 * array.
 * Checks to see if the dependencies are satisfied, and updates the holds
 * and dependencies on the job as needed.
 * @pre-cond: pa must point to a valid array
 * @post-cond: all jobs depending on pa have their holds up to date. If a job
 * has an after dependency on pa and it is satisfied, the dependency is removed
 * from the job.
 * @return true if at least one dependency was satisfied, false if none were.
 */

bool set_array_depend_holds(

  job_array *pa)

  {
  int                      compareNumber;
  bool                     dependency_satisfied = false;

  job                     *pjob;
  array_depend_job        *pdj;

  /* loop through dependencies to update holds */
  for (std::list<array_depend *>::iterator it = pa->ai_qs.deps.begin();
       it != pa->ai_qs.deps.end(); it++)
    {
    array_depend *pdep = *it;

    compareNumber = -1;

    switch (pdep->dp_type)
      {
      case JOB_DEPEND_TYPE_AFTERSTARTARRAY:

        compareNumber = pa->ai_qs.num_started;

        break;

      case JOB_DEPEND_TYPE_AFTEROKARRAY:

        compareNumber = pa->ai_qs.num_successful;

        break;

      case JOB_DEPEND_TYPE_AFTERNOTOKARRAY:

        compareNumber = pa->ai_qs.num_failed;

        break;

      case JOB_DEPEND_TYPE_AFTERANYARRAY:

        compareNumber = pa->ai_qs.jobs_done;

        break;

      case JOB_DEPEND_TYPE_BEFORESTARTARRAY:

        compareNumber = pa->ai_qs.num_started;

        break;

      case JOB_DEPEND_TYPE_BEFOREOKARRAY:

        compareNumber = pa->ai_qs.num_successful;

        break;

      case JOB_DEPEND_TYPE_BEFORENOTOKARRAY:

        compareNumber = pa->ai_qs.num_failed;

        break;

      case JOB_DEPEND_TYPE_BEFOREANYARRAY:

        compareNumber = pa->ai_qs.jobs_done;

        break;
      }

    /* update each job with that dependency type */

    unsigned int dp_jobs_size = pdep->dp_jobs.size();
    for (unsigned int i = 0; i < dp_jobs_size; i++)
      {
      pdj = pdep->dp_jobs[i];
      pjob = svr_find_job(pdj->dc_child.c_str(), TRUE);

      if (pjob != NULL)
        {
        mutex_mgr job_mutex(pjob->ji_mutex, true);

        if (((compareNumber < pdj->dc_num) &&
             (pdep->dp_type < JOB_DEPEND_TYPE_BEFORESTARTARRAY)) ||
            ((compareNumber >= pdj->dc_num) && 
             (pdep->dp_type > JOB_DEPEND_TYPE_AFTERANYARRAY)))
          {
          /* hold */
          pjob->ji_wattr[JOB_ATR_hold].at_val.at_long |= HOLD_s;
          pjob->ji_wattr[JOB_ATR_hold].at_flags |= ATR_VFLAG_SET;

          if (LOGLEVEL >= 8)
            {
            log_event(
              PBSEVENT_JOB,
              PBS_EVENTCLASS_JOB,
              pjob->ji_qs.ji_jobid,
              "Setting HOLD_s due to dependencies\n");
            }

          svr_setjobstate(pjob, JOB_STATE_HELD, JOB_SUBSTATE_DEPNHOLD, FALSE);
          }
        else 
          {
          dependency_satisfied = true;

          if (pdep->dp_type < JOB_DEPEND_TYPE_BEFORESTARTARRAY)
            {
            remove_array_dependency_job_from_job(pdep, pjob, pa->ai_qs.parent_id);
            }
          
          /* release the array's hold - set_depend_hold
           * will clear holds if there are no other dependencies
           * logged in set_depend_hold */
          try 
            {
            set_depend_hold(pjob, &pjob->ji_wattr[JOB_ATR_depend], pa);
            }
          catch (int err)
            {
            job_mutex.set_unlock_on_exit(false);
            }
          }
        }

      }
    }

  return(dependency_satisfied);
  } /* END set_array_depend_holds */


/*
 * post_doq (que not dog) - post request/reply processing for depend_on_que
 * i.e. the sending of register operations.
 */

void post_doq(

  batch_request *preq)

  {
  char                  log_buf[LOCAL_LOG_BUF_SIZE];
  char                 *msg;
  job                  *pjob;
  pbs_attribute        *pattr;

  struct depend        *pdp;
  depend_job           *pdjb;
  char                 *jobid;

  if (preq == NULL)
    return;

  jobid = preq->rq_ind.rq_register.rq_child;

  if (preq->rq_reply.brp_code)
    {
    /* request was rejected */
    snprintf(log_buf, sizeof(log_buf), "%s%s", msg_regrej, preq->rq_ind.rq_register.rq_parent);
    log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, jobid, log_buf);

    pjob = svr_find_job(jobid, TRUE);

    if ((msg = pbse_to_txt(preq->rq_reply.brp_code)) != NULL)
      {
      safe_strncat(log_buf, "\n", sizeof(log_buf) - strlen(log_buf) - 1);
      safe_strncat(log_buf, msg, sizeof(log_buf) - strlen(log_buf) - 1);
      }

    if (pjob != NULL)
      {
      mutex_mgr job_mutex(pjob->ji_mutex, true);

      safe_strncat(log_buf,
        "\nJob held for unknown job dep, use 'qrls' to release",
        sizeof(log_buf) - strlen(log_buf) - 1);

      if (preq->rq_reply.brp_code != PBSE_BADSTATE)
        {
        svr_mailowner(pjob, MAIL_ABORT, MAIL_FORCE, log_buf);
        }

      pattr = &pjob->ji_wattr[JOB_ATR_depend];

      if (((pdp = find_depend(preq->rq_ind.rq_register.rq_dependtype, pattr)) != 0) &&
          ((pdjb = find_dependjob(pdp, preq->rq_ind.rq_register.rq_parent)) != 0))
        {
        del_depend_job(pdp, pdjb);

        if (preq->rq_reply.brp_code != PBSE_BADSTATE)
          {
          pjob->ji_wattr[JOB_ATR_hold].at_val.at_long |= HOLD_u;
          pjob->ji_wattr[JOB_ATR_hold].at_flags |= ATR_VFLAG_SET;
          pjob->ji_modified = 1;
          }

        try
          {
          set_depend_hold(pjob, pattr, NULL);
          }
        catch (int err)
          {
          job_mutex.set_unlock_on_exit(false);
          }
        }
      }
    }

  free_br(preq);

  return;
  }  /* END post_doq() */





/*
 * alter_unreg - if required, unregister dependencies on alter of pbs_attribute
 * This is called from depend_on_que() when is is acting as the at_action
 * routine for the dependency pbs_attribute.
 */

int alter_unreg(

  job           *pjob,
  pbs_attribute *old,  /* current job dependency attribure */
  pbs_attribute *new_attr)  /* job dependency pbs_attribute after alter */

  {
  struct depend *poldd;
  struct depend *pnewd;
  depend_job    *oldjd;

  int            type;

  for (poldd = (struct depend *)GET_NEXT(old->at_val.at_list);
       poldd;
       poldd = (struct depend *)GET_NEXT(poldd->dp_link))
    {
    type = poldd->dp_type;

    if ((type != JOB_DEPEND_TYPE_ON) && (type != JOB_DEPEND_TYPE_SYNCCT))
      {
      pnewd = find_depend(type, new_attr);

      for (unsigned int i = 0; i < poldd->dp_jobs.size(); i++)
        {
        oldjd = poldd->dp_jobs[i];

        if ((pnewd == 0) || 
            (find_dependjob(pnewd, oldjd->dc_child.c_str()) == 0))
          {
          int rc = send_depend_req(pjob, oldjd, type, JOB_DEPEND_OP_UNREG, SYNC_SCHED_HINT_NULL, free_br,false);

          if (rc == PBSE_JOBNOTFOUND)
            return(rc);
          }

        }
      }
    }

  return(PBSE_NONE);
  }  /* END alter_unreg() */



/*
 * depend_on_que - Perform a series of actions if job has a dependency
 * that needs action when the job is queued into an execution queue.
 *
 * Called from svr_enquejob() when a job enters an
 * execution queue.  Also  the at_action routine for the pbs_attribute.
 */

int depend_on_que(

  pbs_attribute *pattr,
  void          *pj,
  int            mode)

  {
  long               cost;

  struct depend     *pdep;
  struct depend     *next;

  int                rc = PBSE_NONE;
  int                type;
  job               *pjob = (job *)pj;
  pbs_queue         *pque;
  char               job_id[PBS_MAXSVRJOBID+1];

  strcpy(job_id, pjob->ji_qs.ji_jobid);
  pque = get_jobs_queue(&pjob);

  if (pque == NULL)
    {
    if (pjob == NULL)
      {
      log_err(PBSE_JOBNOTFOUND, __func__, "Job lost while acquiring queue 8");
      return(PBSE_JOBNOTFOUND);
      }
    else
      return(PBSE_NONE);
    }

  mutex_mgr pque_mutex = mutex_mgr(pque->qu_mutex, true);
  if (((mode != ATR_ACTION_ALTER) && 
       (mode != ATR_ACTION_NOOP)) ||
       (pque->qu_qs.qu_type != QTYPE_Execution))
    {
    return(PBSE_NONE);
    }
  pque_mutex.unlock();

  if (mode == ATR_ACTION_ALTER)
    {
    /* if there are dependencies being removed, unregister them */

    if (alter_unreg(pjob, &(pjob)->ji_wattr[JOB_ATR_depend], pattr) == PBSE_JOBNOTFOUND)
      return(PBSE_JOBNOTFOUND);
    }

  /* First set a System hold if required */
  set_depend_hold(pjob, pattr, NULL);

  /* Check if there are dependencies that require registering */

  pdep = (struct depend *)GET_NEXT(pattr->at_val.at_list);

  while (pdep != NULL)
    {
    // pdep can be freed during the loop so we need to get the next value now
    next = (struct depend *)GET_NEXT(pdep->dp_link);
    type = pdep->dp_type;

    if (type == JOB_DEPEND_TYPE_SYNCCT)
      {
      /* register myself - this calculates and records the cost */

      cost = calc_job_cost(pjob);

      register_sync(pdep, (pjob)->ji_qs.ji_jobid, server_name, cost);

      if (pdep->dp_numreg > pdep->dp_numexp)
        release_cheapest(pjob, pdep);
      }
    else if (type != JOB_DEPEND_TYPE_ON)
      {

      // initialize rc to PBSE_BADDEPEND to make sure that send_depend_req is called at least
      // once if we are queuing a job
      if (mode != ATR_ACTION_ALTER)
        rc = PBSE_BADDEPEND;

      // send_depend_req could end up deleting the dependency
      // so make a copy first so we don't try to walk an array that's
      // being deleted. We need to make a deep copy or we're referencing freed memory
      std::vector<depend_job *> pparent;

      for (unsigned int i = 0; i < pdep->dp_jobs.size(); i++)
        {
        depend_job *dj = alloc_dependjob(pdep->dp_jobs[i]->dc_child.c_str());
        pparent.push_back(dj);
        }

      for (unsigned int i = 0; i < pparent.size(); i++)
        {
        if ((rc = send_depend_req(pjob, pparent[i], type, JOB_DEPEND_OP_REGISTER,
                                  SYNC_SCHED_HINT_NULL, post_doq,false)) != PBSE_NONE)
          break;
        }

      // free the allocated array
      for (unsigned int i = 0; i < pparent.size(); i++)
        delete pparent[i];

      if (rc != PBSE_NONE)
        return(rc);
      }

    pdep = next;
    }

  return(rc);
  }  /* END depend_on_que() */




/*
 * post_doe - Post (reply) processing of requests processing for depend_on_exec
 */

void post_doe(

  batch_request *preq)

  {
  pbs_attribute        *pattr;

  struct depend        *pdep;

  depend_job           *pdj;
  job                  *pjob;
  char                 *jobid;
 
  if (preq == NULL)
    return;

  jobid = preq->rq_ind.rq_register.rq_child;

  pjob = svr_find_job(jobid, TRUE);

  if (pjob != NULL)
    {
    mutex_mgr job_mutex(pjob->ji_mutex, true);
    
    pattr = &pjob->ji_wattr[JOB_ATR_depend];
    pdep  = find_depend(JOB_DEPEND_TYPE_BEFORESTART, pattr);

    if (pdep != NULL)
      {
      if ((pdj = find_dependjob(pdep, preq->rq_ind.rq_register.rq_parent)) != NULL)
        del_depend_job(pdep, pdj);

      if (pdep->dp_jobs.size() == 0)
        {
        /* no more dependencies of this type */

        delete pdep;
        }
      }
    }

  free_br(preq);

  return;
  }  /* END post_doe() */



/*
 * depend_on_exec - Perform actions if job has
 * "beforestart" dependency - send "register-release" to child job; or
 * "syncct/syncwith" dependency - send "register-ready" to child job.
 *
 * This function is called from svr_startjob().
 */

int depend_on_exec(

  job *pjob)

  {
  struct depend     *pdep;
  depend_job        *pdj;
  char               jobid[PBS_MAXSVRJOBID+1];

  strcpy(jobid, pjob->ji_qs.ji_jobid);
  /* If any jobs come after my start, release them */
  pdep = find_depend(
           JOB_DEPEND_TYPE_BEFORESTART,
           &pjob->ji_wattr[JOB_ATR_depend]);

  if (pdep != NULL)
    {

    for (unsigned int i = 0; i < pdep->dp_jobs.size(); i++)
      {
      pdj = pdep->dp_jobs[i];
    
      if (send_depend_req(pjob,
            pdj,
            pdep->dp_type,
            JOB_DEPEND_OP_RELEASE,
            SYNC_SCHED_HINT_NULL,
            post_doe,false) == PBSE_JOBNOTFOUND)
        {
        return(PBSE_JOBNOTFOUND);
        }
      }
    }

  /* If I am a member of a sync set,             */
  /* send ready to master to release next in set */

  pdep = find_depend(
           JOB_DEPEND_TYPE_SYNCWITH,
           &pjob->ji_wattr[JOB_ATR_depend]);

  if (pdep != NULL)
    {
    if (pdep->dp_jobs.size() > 0)
      {
      pdj = pdep->dp_jobs[0];

      if (send_depend_req(pjob,
            pdj,
            pdep->dp_type,
            JOB_DEPEND_OP_READY,
            SYNC_SCHED_HINT_NULL,
            free_br,false) == PBSE_JOBNOTFOUND)
        {
        return(PBSE_JOBNOTFOUND);
        }
      }
    }

  /* If I am the master of a sync set, skip the sending (to myself) */
  /* a ready, just cut direct to releasing the next cheapest        */

  pdep = find_depend(JOB_DEPEND_TYPE_SYNCCT, &pjob->ji_wattr[JOB_ATR_depend]);

  if (pdep != NULL)
    {

    if (pdep->dp_jobs.size() > 0)
      {
      /* first will be myself, mark as running */
      pdep->dp_jobs[0]->dc_state = JOB_DEPEND_OP_READY;
      }

    release_cheapest(pjob, pdep);
    }

  return(PBSE_NONE);
  }  /* END depend_on_exec() */






/* depend_on_term - Perform actions if job has "afterany, afterok, afternotok"
 * dependencies, send "register-release" or register-delete" as
 * appropriate.
 *
 * This function is invoked from on_job_exit() in req_jobobit.c.
 * When there are no depends to deal with, free the pbs_attribute and
 * recall on_job_exit().
 */

int depend_on_term(

  job *pjob)

  {
  int                exitstat;
  int                op;
  pbs_attribute     *pattr;

  struct depend     *pdep;

  depend_job        *pparent;
  int                rc;
  int                shouldkill = 0;
  int                type;

  if (pjob == NULL)
    return(PBSE_BAD_PARAMETER);
 
  exitstat = pjob->ji_qs.ji_un.ji_exect.ji_exitstat;
  pattr = &pjob->ji_wattr[JOB_ATR_depend];

  pdep = (struct depend *)GET_NEXT(pattr->at_val.at_list);

  while ((pdep != NULL) &&
         (pjob != NULL))
    {
    op = -1;

    type = pdep->dp_type;

    /* for the first three, before... types, release or delete */
    /* next job depending on exit status                       */

    switch (type)
      {
      case JOB_DEPEND_TYPE_BEFORESTART:

        op = JOB_DEPEND_OP_DELETE;

        break;

      case JOB_DEPEND_TYPE_BEFOREOK:

        if (pjob->ji_qs.ji_substate == JOB_SUBSTATE_ABORT)
          {
          op = JOB_DEPEND_OP_DELETE;
          }
        else
          {
          if (exitstat == 0)
            op = JOB_DEPEND_OP_RELEASE;
          else
            op = JOB_DEPEND_OP_DELETE;
          }

        break;

      case JOB_DEPEND_TYPE_BEFORENOTOK:

        if (exitstat != 0)
          op = JOB_DEPEND_OP_RELEASE;
        else
          op = JOB_DEPEND_OP_DELETE;

        break;

      case JOB_DEPEND_TYPE_BEFOREANY:

        op = JOB_DEPEND_OP_RELEASE;

        break;

      case JOB_DEPEND_TYPE_SYNCCT:

        /* Master of sync set has ended, if any members were */
        /* never started, kill all the jobs in the set  */

        for (unsigned int i = 0; i < pdep->dp_jobs.size(); i++)
          {
          pparent = pdep->dp_jobs[i];

          if (pparent->dc_state != JOB_DEPEND_OP_READY)
            shouldkill = 1;
          }

        if (shouldkill)
          {

          for (unsigned int i = 1; i < pdep->dp_jobs.size(); i++)
            {
            pparent = pdep->dp_jobs[i];

            rc = send_depend_req(pjob, pparent, type, JOB_DEPEND_OP_DELETE, SYNC_SCHED_HINT_NULL, free_br,true);
            
            if (rc == PBSE_JOBNOTFOUND)
              {
              return(rc);
              }
            }

          }

        break;

      } /* END switch(type) */

    if (op != -1)
      {
      for (unsigned int i = 0; i < pdep->dp_jobs.size(); i++)
        {
        pparent = pdep->dp_jobs[i];

        /* "release" the job to execute */
        if ((rc = send_depend_req(pjob, pparent, type, op, SYNC_SCHED_HINT_NULL, free_br,true)) != PBSE_NONE)
          {
          return(rc);
          }
        }
      }

    pdep = (struct depend *)GET_NEXT(pdep->dp_link);
    } /* END loop over each dependency */

  return(PBSE_NONE);
  }  /* END depend_on_term() */





/*
 * release_cheapest - release the cheapest of the unreleased jobs in the
 * sync set.
 */

int release_cheapest(

  job           *pjob,
  struct depend *pdep)

  {
  long               lowestcost = 0;
  depend_job        *cheapest = NULL;
  int                hint = SYNC_SCHED_HINT_OTHER;
  int                nreleased = 0;
  depend_job        *pdj;
  int                rc = PBSE_NONE;


  for (unsigned int i = 0; i < pdep->dp_jobs.size(); i++)
    {
    pdj = pdep->dp_jobs[i];

    if (pdj->dc_state == 0)
      {
      if ((cheapest == NULL) || (pdj->dc_cost < lowestcost))
        {
        cheapest = pdj;

        lowestcost  = pdj->dc_cost;
        }
      }
    else
      {
      ++nreleased; /* incr number already released */
      }
    }

  if (cheapest)
    {
    if (nreleased == 0)
      hint = SYNC_SCHED_HINT_FIRST;

    if ((rc = send_depend_req(pjob, cheapest, JOB_DEPEND_TYPE_SYNCWITH,
          JOB_DEPEND_OP_RELEASE, hint, free_br,false)) == PBSE_NONE)
      {
      cheapest->dc_state = JOB_DEPEND_OP_RELEASE;
      }
    else if (rc == PBSE_JOBNOTFOUND)
      return(rc);

    }

  return(PBSE_NONE);
  } /* END release_cheapest() */






/**
 * set_depend_hold - set a hold on the job required by the type of dependency
 *
 * NOTE:  determine where dependency hold is cleared and comment
 *
 * @param pjob - the job whose dependency hold we're checking out
 * @param pattr - the job's dependency attribute
 * @param pa - optional, if non NULL then array's lock is already held and shouldn't be re-locked
 */

void set_depend_hold(

  job           *pjob,
  pbs_attribute *pattr, 
  job_array     *pa)

  {
  int                loop = 1;
  int                newstate;
  int                newsubst;

  struct depend     *pdp = NULL;
  depend_job        *djob = NULL;

  struct job        *djp = NULL;
  int                substate = -1;

  std::vector<std::string>  array_names;

  if (pattr->at_flags & ATR_VFLAG_SET)
    pdp = (struct depend *)GET_NEXT(pattr->at_val.at_list);

  while ((pdp != NULL) && (loop != 0))
    {
    switch (pdp->dp_type)
      {

      case JOB_DEPEND_TYPE_SYNCCT:

        if (pdp->dp_released == 0)
          substate = JOB_SUBSTATE_SYNCHOLD;

        break;

      case JOB_DEPEND_TYPE_SYNCWITH:

        if ((pdp->dp_jobs.size() != 0) &&
            (pdp->dp_released == 0))
          substate = JOB_SUBSTATE_SYNCHOLD;

        break;

      case JOB_DEPEND_TYPE_AFTERSTART:

      case JOB_DEPEND_TYPE_AFTEROK:

      case JOB_DEPEND_TYPE_AFTERNOTOK:

      case JOB_DEPEND_TYPE_AFTERANY:

        /* If the job we are depending on has already completed */
        /* Then don't set this job on dependent Hold, just leave it as Queued */
        if (pdp->dp_jobs.size() > 0)
          {
          djob = pdp->dp_jobs[0];

          bool jobids_match = job_ids_match(pjob->ji_qs.ji_jobid, djob->dc_child.c_str());
          
          /* if dc_child is the same job id as pjob don't
             lock the job. It is already locked */
          if (!jobids_match)
            djp = svr_find_job(djob->dc_child.c_str(), TRUE);

          if (!djp ||
              ((pdp->dp_type == JOB_DEPEND_TYPE_AFTERSTART) &&
               (djp->ji_qs.ji_state < JOB_STATE_RUNNING)))
            {
            substate = JOB_SUBSTATE_DEPNHOLD;
            }
          else if ((pdp->dp_type != JOB_DEPEND_TYPE_AFTERSTART) &&
                   (djp->ji_qs.ji_state != JOB_STATE_COMPLETE))
            {
            substate = JOB_SUBSTATE_DEPNHOLD;
            }
          else if (((pdp->dp_type == JOB_DEPEND_TYPE_AFTEROK) &&
                    (djp->ji_qs.ji_un.ji_exect.ji_exitstat != 0)) ||
                   ((pdp->dp_type == JOB_DEPEND_TYPE_AFTERNOTOK) &&
                    (djp->ji_qs.ji_un.ji_exect.ji_exitstat == 0)))
            {
            substate = JOB_SUBSTATE_DEPNHOLD;
            }

          if (djp != NULL)
            unlock_ji_mutex(djp, __func__, "1", LOGLEVEL);
          }

        break;

      case JOB_DEPEND_TYPE_ON:

        if (pdp->dp_numexp)
          substate = JOB_SUBSTATE_DEPNHOLD;

        break;

      default:

        if (pdp->dp_type >= JOB_DEPEND_TYPE_AFTERSTARTARRAY)
          {
          // make sure the dependency still present
          if (pdp->dp_jobs.size() > 0)
            {
            array_names.push_back(pdp->dp_jobs[0]->dc_child);
            }
          else
            {
            if (LOGLEVEL >= 2)
              {
              log_event(
                PBSEVENT_JOB,
                PBS_EVENTCLASS_JOB,
                pjob->ji_qs.ji_jobid,
                "Array dependency already satisfied. Skipping array dependency handling for job.");
              }
            }
          }
      }  /* END switch (pdp->dp_type) */

    pdp = (struct depend *)GET_NEXT(pdp->dp_link);
    }  /* END while ((pdp != NULL) && (loop != 0)) */

  if (substate == -1)
    {
    /* No (more) dependencies, clear system hold and set state */

    if ((pjob->ji_qs.ji_substate == JOB_SUBSTATE_SYNCHOLD) ||
        (pjob->ji_qs.ji_substate == JOB_SUBSTATE_DEPNHOLD))
      {
      pjob->ji_wattr[JOB_ATR_hold].at_val.at_long &= ~HOLD_s;

      /* newstate is job's 'natural state - ie, what it would be if dependency did not exist */

      if (LOGLEVEL >= 8)
        {
        log_event(
          PBSEVENT_JOB,
          PBS_EVENTCLASS_JOB,
          pjob->ji_qs.ji_jobid,
          "Clearing HOLD_s due to dependencies\n");
        }

      svr_evaljobstate(*pjob, newstate, newsubst, 0);
      svr_setjobstate(pjob, newstate, newsubst, FALSE);
      }

    // If we have array dependencies, we need to update them
    if (array_names.size() != 0)
      {
      std::string jobid(pjob->ji_qs.ji_jobid);
      unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL);

      for (size_t i = 0; i < array_names.size(); i++)
        {
        job_array *to_update;

        if ((pa != NULL) &&
            ((array_names[i] == pa->ai_qs.parent_id) ||
             ((!strncmp(array_names[i].c_str(), pa->ai_qs.parent_id, array_names[i].size())) &&
              (pa->ai_qs.parent_id[array_names[i].size()] == '.'))))
          {
          continue;
          }
          
        to_update = get_array(array_names[i].c_str());

        if (to_update != NULL)
          {
          set_array_depend_holds(to_update);

          unlock_ai_mutex(to_update, __func__, "2", LOGLEVEL);
          }
        }

      pjob = svr_find_job(jobid.c_str(), TRUE);

      if (pjob == NULL)
        {
        throw (int)PBSE_JOBNOTFOUND;
        }
      }
    }
  else
    {
    /* there are dependencies, set system hold accordingly */

    pjob->ji_wattr[JOB_ATR_hold].at_val.at_long |= HOLD_s;
    pjob->ji_wattr[JOB_ATR_hold].at_flags |= ATR_VFLAG_SET;

    if (LOGLEVEL >= 8)
      {
      log_event(
        PBSEVENT_JOB,
        PBS_EVENTCLASS_JOB,
        pjob->ji_qs.ji_jobid,
        "Setting HOLD_s due to dependencies\n");
      }

    svr_setjobstate(pjob, JOB_STATE_HELD, substate, FALSE);
    }

  return;
  }  /* END set_depend_hold() */



/*
 * depend_clrrdy - clear state ready flags in job dependency pbs_attribute
 */

void depend_clrrdy(

  job *pjob)

  {
  struct depend     *pdp;

  pdp = (struct depend *)GET_NEXT(pjob->ji_wattr[JOB_ATR_depend].at_val.at_list);

  while ((pdp != NULL) &&
         (pdp->dp_type == JOB_DEPEND_TYPE_SYNCCT))
    {

    for (unsigned int i = 0; i < pdp->dp_jobs.size(); i++)
      pdp->dp_jobs[i]->dc_state = 0;

    pdp = (struct depend *)GET_NEXT(pdp->dp_link);
    }

  return;
  }  /* END depend_clrrdy() */





/*
 * find_depend - find a dependency struct of a certain type for a job
 */

struct depend *find_depend(

  int            type,
  pbs_attribute *pattr)

  {
  struct depend *pdep = NULL;

  if (pattr->at_flags & ATR_VFLAG_SET)
    {
    pdep = (struct depend *)GET_NEXT(pattr->at_val.at_list);

    while (pdep != NULL)
      {
      if (pdep->dp_type == type)
        break;

      pdep = (struct depend *)GET_NEXT(pdep->dp_link);
      }
    }

  return(pdep);
  }  /* END find_depend() */





/*
 * make_depend - allocate and attach a depend struct to the attribute
 */

struct depend *make_depend(

  int            type,
  pbs_attribute *pattr)

  {
  struct depend *pdep = NULL;

  pdep = new depend(type);

  if (pdep != NULL)
    {
    append_link(&pattr->at_val.at_list, &pdep->dp_link, pdep);
    pattr->at_flags |= ATR_VFLAG_SET;
    }

  return(pdep);
  }  /* END make_depend() */





/*
 * register_sync - a "child job" is registering sync with its "parent"
 */

int register_sync(

  struct depend *pdep,
  char          *child,
  char          *host,
  long           cost)

  {
  depend_job *pdj;

  if ((pdj = find_dependjob(pdep, child)))
    {
    /* existing regist., just update the location of the child */

    return(PBSE_NONE);
    }

  /* a new registration, create depend_job entry */

  pdj = make_dependjob(pdep, child);

  if (pdj == NULL)
    {
    return(PBSE_SYSTEM);
    }

  pdj->dc_cost = cost;

  /* increment number registered */
  if (++pdep->dp_numreg > pdep->dp_numexp + 1)
    {
    return(PBSE_IVALREQ); /* too many registered */
    }

  return(PBSE_NONE);
  }  /* END register_sync() */





/*
 * register_dep - Some job wants to run before/after the local job, so set up
 * a dependency on the local job.
 *
 * @see req_register() - parent
 *
 */

int register_dep(

  pbs_attribute        *pattr,
  struct batch_request *preq,
  int                   type,
  int                  *made) /* RETURN */

  {
  struct depend     *pdep;
  depend_job        *pdj;

  /* change into the mirror image type */

  type ^= (JOB_DEPEND_TYPE_BEFORESTART - JOB_DEPEND_TYPE_AFTERSTART);

  if ((pdep = find_depend(type, pattr)) == NULL)
    {
    if ((pdep = make_depend(type, pattr)) == NULL)
      {
      return(PBSE_SYSTEM);
      }
    }

  if ((pdj = find_dependjob(pdep, preq->rq_ind.rq_register.rq_child)))
    {
    *made = 0;

    return(PBSE_NONE);
    }

  if (make_dependjob(pdep, preq->rq_ind.rq_register.rq_child) == NULL)
    {
    return(PBSE_SYSTEM);
    }

  /* SUCCESS */
  *made = 1;

  return(PBSE_NONE);
  }  /* END register_dep() */





/**
 * unregister_dep - remove a registered dependency
 * Results from a qalter call to remove existing dependencies
 *
 * @see register_dep()
 */

int unregister_dep(

  pbs_attribute        *pattr,
  struct batch_request *preq)

  {
  int                type;

  struct depend     *pdp;
  depend_job        *pdjb;

  /* get mirror image of dependency type */

  type = preq->rq_ind.rq_register.rq_dependtype ^
         (JOB_DEPEND_TYPE_BEFORESTART - JOB_DEPEND_TYPE_AFTERSTART);

  if (((pdp = find_depend(type, pattr)) == NULL) ||
      ((pdjb = find_dependjob(pdp, preq->rq_ind.rq_register.rq_child)) == NULL))
    {
    return(PBSE_IVALREQ);
    }

  del_depend_job(pdp, pdjb);

  return(PBSE_NONE);
  }  /* END unregister_dep() */




/*
 * unregister_sync - remove a registered sycn-dependency
 * Results from a qalter call to remove existing dependencies
 */

int unregister_sync(

  pbs_attribute        *pattr,
  struct batch_request *preq)

  {

  struct depend   *pdp;

  depend_job *pdjb;

  if (((pdp = find_depend(JOB_DEPEND_TYPE_SYNCCT, pattr)) == 0) ||
      ((pdjb = find_dependjob(pdp, preq->rq_ind.rq_register.rq_child)) == 0))
    {
    return(PBSE_IVALREQ);
    }

  del_depend_job(pdp, pdjb);

  if (--pdp->dp_numreg <= pdp->dp_numexp)
    {
    if (pdp->dp_released == 1)
      {
      if (pdp->dp_jobs.size() > 0)
        {
        pdp->dp_jobs[0]->dc_state = 0;
        pdp->dp_released = 0;
        }
      }
    }

  return(PBSE_NONE);
  }  /* END unregister_sync() */




/*
 * find_dependjob - find a child dependent job with a certain job id
 */

depend_job *find_dependjob(

  struct depend *pdep,
  const char    *name)

  {
  depend_job *pdj;
  bool        display_server_suffix = true;

  get_svr_attr_b(SRV_ATR_display_job_server_suffix, &display_server_suffix);

  unsigned int dp_jobs_size = pdep->dp_jobs.size();
  for (unsigned int i = 0; i < dp_jobs_size; i++)
    {
    pdj = pdep->dp_jobs[i];
    
    if ((pdj->dc_child == name) ||
        ((display_server_suffix == FALSE) &&
         (!strncmp(name, pdj->dc_child.c_str(), strlen(name))) &&
         (strchr(name, '.') == NULL)))
      return(pdj);
    }

  return(NULL);
  }  /* END find_dependjob() */


depend_job *alloc_dependjob(

  const char *jobid)

  {
  depend_job *pdj = new depend_job();

  pdj->dc_child = jobid;
  
  return(pdj);
  } // END alloc_dependjob() 



/*
 * make_dependjob - add a depend_job structue
 */

depend_job *make_dependjob(

  struct depend *pdep,
  const char    *jobid)

  {
  depend_job *pdj = alloc_dependjob(jobid);

  if (pdj != NULL)
    pdep->dp_jobs.push_back(pdj);

  return(pdj);
  }  /* END make_dependjob() */





/*
 * send_depend_req - build and send a Register Dependent request
 */

int send_depend_req(

  job         *pjob,
  depend_job  *pparent,
  int          type,
  int          op,
  int          schedhint,
  void       (*postfunc)(batch_request *),
  bool         bAsyncOk)

  {
  int                   rc = 0;
  int                   i;
  char                  job_id[PBS_MAXSVRJOBID + 1];
  char                  br_id[MAXLINE];

  struct batch_request *preq;
  char                  log_buf[LOCAL_LOG_BUF_SIZE];

  preq = alloc_br(PBS_BATCH_RegistDep);

  if (preq == NULL)
    {
    log_err(errno, __func__, msg_err_malloc);
    return(PBSE_SYSTEM);
    }

  for (i = 0;i < PBS_MAXUSER;++i)
    {
    if (pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str == NULL)
      {
      free_br(preq);
      return(PBSE_BADATVAL);
      }

    preq->rq_ind.rq_register.rq_owner[i] =
      pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str[i];

    if (preq->rq_ind.rq_register.rq_owner[i] == '@')
      break;
    }

  preq->rq_ind.rq_register.rq_owner[i] = '\0';

  snprintf(preq->rq_ind.rq_register.rq_parent,
    sizeof(preq->rq_ind.rq_register.rq_parent), "%s", pparent->dc_child.c_str());
  snprintf(preq->rq_ind.rq_register.rq_child, sizeof(preq->rq_ind.rq_register.rq_child),
    "%s", pjob->ji_qs.ji_jobid);

  /* kludge for server:port follows */

  if (strchr(server_name, (int)':'))
    {
    strcat(preq->rq_ind.rq_register.rq_child, "@");
    strcat(preq->rq_ind.rq_register.rq_child, server_name);
    }

  preq->rq_ind.rq_register.rq_dependtype = type;

  preq->rq_ind.rq_register.rq_op = op;
  strcpy(preq->rq_host, server_name);  /* for issue_to_svr() */

  /* if registering sync, include job cost for scheduling */

  if (type == JOB_DEPEND_TYPE_SYNCWITH)
    {
    if (op == JOB_DEPEND_OP_REGISTER)
      /* for sync-register, cost = job resource cost */

      preq->rq_ind.rq_register.rq_cost = calc_job_cost(pjob);
    else
      /* for sync-release, cost = scheduling hint */

      preq->rq_ind.rq_register.rq_cost = schedhint;
    }
  else
    {
    /* otherwise, cost = null */
    preq->rq_ind.rq_register.rq_cost = 0;
    }

  /* save jobid and unlock mutex */
  strcpy(job_id, pjob->ji_qs.ji_jobid);
  unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL);

  get_batch_request_id(preq);
  snprintf(br_id, sizeof(br_id), "%s", preq->rq_id);

  if (bAsyncOk)
    {
    snprintf(preq->rq_host, sizeof(preq->rq_host), "%s", server_name);
    rc = que_to_local_svr(preq);
    preq = NULL;
    }
  else
    {
    rc = issue_to_svr(server_name, &preq, NULL);
    }

  if (rc != PBSE_NONE)
    {
    /* requests are conditionally freed in issue_to_svr() */
    if (preq != NULL)
      {
      free_br(preq);
      }

    sprintf(log_buf, "Unable to perform dependency with job %s\n", pparent->dc_child.c_str());
    log_err(rc, __func__, log_buf);

    if ((preq = get_remove_batch_request(br_id)) != NULL)
      free_br(preq);

    if ((pjob = svr_find_job(job_id, TRUE)) == NULL)
      {
      return(PBSE_JOBNOTFOUND);
      }

    return(rc);
    }
  /* local requests have already been processed and freed. Do not attempt to
   * free or reference again. */
  else if (preq != NULL)
    postfunc(preq);

  if ((pjob = svr_find_job(job_id, TRUE)) == NULL)
    {
    return(PBSE_JOBNOTFOUND);
    }

  return(PBSE_NONE);
  }  /* END send_depend_req() */





/*
 * This section contains general function for dependency attributes
 *
 * Each pbs_attribute has functions for:
 * Decoding the value string to the machine representation.
 * Encoding the internal representation of the pbs_attribute to external
 * Setting the value by =, + or - operators.
 * Comparing a (decoded) value with the pbs_attribute value.
 * Freeing the space calloc-ed to the pbs_attribute value.
 *
 * The prototypes are declared in "attribute.h"
 *
 * ----------------------------------------------------------------------------
 * pbs_Attribute functions for attributes of type "dependency".
 *
 * The "encoded" or external form of the value is a string with sub-strings
 * separated by commas and terminated by a null.
 *
 * The "decoded" or internal form is a list of depend (and depend_child)
 * structures, which are defined in pbs_job.h.
 * ----------------------------------------------------------------------------
 */



struct dependnames dependnames[] =

  {
    {JOB_DEPEND_TYPE_AFTERSTART, "after" },
  {JOB_DEPEND_TYPE_AFTEROK,    "afterok" },
  {JOB_DEPEND_TYPE_AFTERNOTOK, "afternotok" },
  {JOB_DEPEND_TYPE_AFTERANY,   "afterany" },
  {JOB_DEPEND_TYPE_BEFORESTART, "before" },
  {JOB_DEPEND_TYPE_BEFOREOK,   "beforeok" },
  {JOB_DEPEND_TYPE_BEFORENOTOK, "beforenotok" },
  {JOB_DEPEND_TYPE_BEFOREANY,  "beforeany" },
  {JOB_DEPEND_TYPE_ON,         "on" },
  {JOB_DEPEND_TYPE_SYNCWITH,   "syncwith" },
  {JOB_DEPEND_TYPE_SYNCCT,     "synccount" },
  {JOB_DEPEND_TYPE_AFTERSTARTARRAY, "afterstartarray" },
  {JOB_DEPEND_TYPE_AFTEROKARRAY, "afterokarray" },
  {JOB_DEPEND_TYPE_AFTERNOTOKARRAY, "afternotokarray" },
  {JOB_DEPEND_TYPE_AFTERANYARRAY, "afteranyarray" },
  {JOB_DEPEND_TYPE_BEFORESTARTARRAY, "beforestartarray" },
  {JOB_DEPEND_TYPE_BEFOREOKARRAY, "beforeokarray" },
  {JOB_DEPEND_TYPE_BEFORENOTOKARRAY, "beforenotokarray" },
  {JOB_DEPEND_TYPE_BEFOREANYARRAY, "beforeanyarray" },
  { -1, (char *)0 }
  };





/*
 * decode_depend - decode a string into an attr of type dependency
 * String is of form: depend_type:job_id[:job_id:...][,depend_type:job_id]
 *
 * Returns: 0 if ok,
 *  >0 error number if error,
 *  *patr members set
 */

int decode_depend(

  pbs_attribute *patr,
  const char    *name,  /* attribute name */
  const char    *rescn, /* resource name, unused here */
  const char    *val,   /* attribute value */
  int            perm)  /* only used for resources */

  {
  int  rc;
  char *valwd;
  char *ptr = NULL;
  char *work_val;

  if ((val == NULL) || (*val == '\0'))
    {
    free_depend(patr);

    patr->at_flags |= ATR_VFLAG_MODIFY;

    return(PBSE_NONE);
    }
    
  free_depend(patr);

  work_val = strdup(val);

  /*
   * for each sub-string (terminated by comma or new-line),
   * add a depend or depend_child structure.
   */

  valwd = parse_comma_string(work_val,&ptr);

  while ((valwd != NULL) &&
         (*valwd != '\0'))
    {
    if ((rc = build_depend(patr, valwd)) != 0)
      {
      free_depend(patr);

      free(work_val);

      return(rc);
      }

    valwd = parse_comma_string(NULL,&ptr);
    }

  patr->at_flags |= ATR_VFLAG_SET | ATR_VFLAG_MODIFY;
      
  free(work_val);

  return(PBSE_NONE);
  }  /* END decode_depend() */





/*
 * cpy_jobsvr() - a version of strcat() that watches for an embedded colon
 * and escapes it with a leading blackslash.  This is needed because
 * the colon is overloaded, both as job_id separater within a set of
 *	depend jobs, and as the server:port separater. Ugh!
 *
 *	This code is horribly inefficient, because we have to walk the length of d, when called in a tight loop. We are
 *	trying to replace this with cat_jobsvr.
 */
#if 0
static void cpy_jobsvr(

  char *d,
  char *s)

  {
  while (*d)
    d++;

  while (*s)
    {
    if (*s == ':')
      *d++ = '\\';

    *d++ = *s++;
    }

  *d = '\0';

  return;
  }  /* END cpy_jobsvr() */
#endif /* 0 */

/*
 *  cat_jobsvr() - a version of strcat() that watches for an embedded colon
 *	and escapes it with a leading blackslash.  This is needed because
 *	the colon is overloaded, both as job_id separater within a set of
 *	depend jobs, and as the server:port separater. Ugh!
 */

void cat_jobsvr(

  char       **Dest,
  const char  *Src)

  {
  char *d;

  if (Dest == NULL)
    return;

  d = *Dest;

  while (*Src) 
    {
    if (*Src == ':')
      *d++ = '\\';

    *d++ = *Src++;
    }

  *d = '\0';

  *Dest = d;

  return;
  }


/*
 * fast_strcat() - an improved version of strcat() that is more efficient in
 * a tight loop
 */

void fast_strcat(

  char **Dest,
  const char  *Src)

  {
  char *d;

  if (Dest == NULL)
    return;

  d = *Dest;

  while (*Src)
    *d++ = *Src++;

  *d = '\0';

  *Dest = d;

  return;
  }




/*
 * dup_depend - duplicate a dependency pd and store it in pattr
 * @see set_depend() - parent
 */

int dup_depend(

  pbs_attribute *pattr,
  struct depend *pd)

  {
  depend     *pnwd;
  depend_job *poldj;
  depend_job *pnwdj;
  int         type;

  type = pd->dp_type;

  if ((pnwd = make_depend(type, pattr)) == 0)
    {
    return(-1);
    }

  pnwd->dp_numexp = pd->dp_numexp;

  pnwd->dp_numreg   = pd->dp_numreg;
  pnwd->dp_released = pd->dp_released;

  unsigned int dp_jobs_size = pd->dp_jobs.size();
  for (unsigned int i = 0; i < dp_jobs_size; i++)
    {
    poldj = pd->dp_jobs[i];

    if ((pnwdj = make_dependjob(pnwd, poldj->dc_child.c_str())) == 0)
      {
      return(-1);
      }

    pnwdj->dc_state = poldj->dc_state;

    pnwdj->dc_cost  = poldj->dc_cost;
    }

  return(PBSE_NONE);
  }  /* END dup_depend() */





/*
 * encode_depend - encode dependency attr into attrlist entry
 *
 * Returns: >0 if ok, entry created and linked into list
 *   =0 no value to encode, entry not created
 *   -1 if error
 */
/*ARGSUSED*/

int encode_depend(

  pbs_attribute *attr,   /* ptr to attribute to encode */
  tlist_head    *phead,  /* ptr to head of attrlist list */
  const char   *atname, /* attribute name */
  const char   *rsname, /* resource name or null */
  int            mode,   /* encode mode, unused here */
  int            perm)   /* only used for resources */

  {
  int                 ct = 0;
  char                cvtbuf[22];
  int                 numdep = 0;

  depend             *nxdp;
  struct svrattrl    *pal;
  depend             *pdp;
  depend_job         *pdjb = NULL;
  struct dependnames *pn;

  char               *BPtr = 0;

  if (!attr)
    return (-1);

  if (!(attr->at_flags & ATR_VFLAG_SET))
    return (0); /* no values */

  pdp = (struct depend *)GET_NEXT(attr->at_val.at_list);

  if (pdp == (struct depend *)0)
    return (0);

  /* scan dependencies types to compute needed base size of svrattrl */

  for (nxdp = pdp; nxdp; nxdp = (struct depend *)GET_NEXT(nxdp->dp_link))
    {
    if ((nxdp->dp_type == JOB_DEPEND_TYPE_SYNCCT) ||
        (nxdp->dp_type == JOB_DEPEND_TYPE_ON))
      {
      /* Doesn't this mean it could be too short? *MUTSU* */
      ct += 30;   /* a guess at a reasonable amt of space */
      }
    else
      {
      ct += 20; /* for longest type */
      ct += nxdp->dp_jobs.size() * (PBS_MAXSVRJOBID + PBS_MAXSERVERNAME + 3);
      }
    }

  if ((pal = attrlist_create(atname, rsname, ct+1)) == NULL)
    {
    return (-1);
    }

  *pal->al_value = '\0';

  BPtr = pal->al_value;

  for (nxdp = pdp; nxdp; nxdp = (struct depend *)GET_NEXT(nxdp->dp_link))
    {
    if ((nxdp->dp_type != JOB_DEPEND_TYPE_SYNCCT) &&
        (nxdp->dp_type != JOB_DEPEND_TYPE_ON)       &&
        (nxdp->dp_jobs.size() == 0))
      {
      continue; /* no value, skip this one */
      }
    
    if (nxdp != pdp)
      fast_strcat(&BPtr,",");  /* comma between */

    pn = &dependnames[nxdp->dp_type];
    fast_strcat(&BPtr,pn->name);

    if ((pn->type == JOB_DEPEND_TYPE_SYNCCT) ||
        (pn->type == JOB_DEPEND_TYPE_ON))
      {
      sprintf(cvtbuf, ":%d", nxdp->dp_numexp);
      fast_strcat(&BPtr,cvtbuf);
      }
    else
      {
      unsigned int dp_jobs_size = nxdp->dp_jobs.size();
      for (unsigned int i = 0; i < dp_jobs_size; i++)
        {
        pdjb = nxdp->dp_jobs[i];

        fast_strcat(&BPtr,":");
        cat_jobsvr(&BPtr,pdjb->dc_child.c_str());
        } 
      }	
    
    ++numdep;
    }

  if (numdep)
    {
    /* there are dependencies recorded, added to the list */
    pal->al_flags = attr->at_flags;
    append_link(phead, &pal->al_link, pal);
    return (1);
    }
  else
    {
    /* there are no dependencies, just the base structure, */
    /* so remove this svrattrl from ths list  */
    (void)free(pal);
    return (0);
    }
  }  /* END encode_depend() */




/*
 * set_depend - set value of pbs_attribute of dependency type to another
 *
 * A=B --> set of dependencies in A replaced by set in B
 * A+B --> dependencies in B added to list in A
 * A-B --> not defined
 *
 * Returns: 0 if ok
 *  >0 if error
 */

int set_depend(

  pbs_attribute *attr,
  pbs_attribute *new_attr,
  enum batch_op  op)

  {
  struct depend *pdnew;
  struct depend *pdold;
  int            rc;

  assert(attr && new_attr);

  switch (op)
    {
    case SET:

      /*
       * if the type of dependency entry already exists, we are
       * going to replace it, so get rid of the old and dup the new
       */

      pdnew = (struct depend *)GET_NEXT(new_attr->at_val.at_list);

      while (pdnew != NULL)
        {
        pdold = find_depend(pdnew->dp_type, attr);

        if (pdold != NULL)
          delete pdold;

        if ((rc = dup_depend(attr, pdnew)) != 0)
          {
          return(rc);
          }

        pdnew = (struct depend *)GET_NEXT(pdnew->dp_link);
        }

      break;


    case INCR: /* not defined */

    case DECR: /* not defined */

    default:

      return(PBSE_IVALREQ);

      /*NOTREACHED*/

      break;
    }

  attr->at_flags |= ATR_VFLAG_SET | ATR_VFLAG_MODIFY;

  return(PBSE_NONE);
  }  /* END set_depend() */





/*
 * comp_depend - compare two attributes of type dependency
 * This operation is undefined.
 *
 * Returns: 0
 *  +1
 *  -1
 */

int comp_depend(
    
  pbs_attribute *attr, 
  pbs_attribute *with)

  {
  return (-1);
  }



void free_depend(

  pbs_attribute *attr)

  {
  depend     *pdp;

  while ((pdp = (struct depend *)GET_NEXT(attr->at_val.at_list)))
    {
    delete pdp;
    }

  attr->at_flags &= ~ATR_VFLAG_SET;

  return;
  }  /* END free_depend() */





/*
 * build_depend -  build a dependency structure
 *  parse the string and turn it into a list of depend structures
 *
 * Return 0 if ok, otherwise non-zero error number
 */

int build_depend(

  pbs_attribute *pattr,
  const char    *value)

  {
  struct depend      *have[JOB_DEPEND_NUMBER_TYPES];
  int                 i;
  int                 numwds;

  struct depend      *pd;

  depend_job         *pdjb;

  struct dependnames *pname;
  char               *pwhere;
  char               *valwd;
  char               *nxwrd;
  int                 type;
  char               *work_val;

  if ((value == NULL) ||
      (*value == '\0'))
    return(PBSE_BADATVAL);
 
  work_val = strdup(value);

  /*
   * Map first subword into dependency type.  If there is just the type
   * with no following job id or count, then leave an empty depend
   * struct;  set_depend will "remove" any of that kind.
   */

  if ((nxwrd = strchr((char *)work_val, (int)':')) != NULL)
    *nxwrd++ = '\0';

  for (pname = dependnames; pname->type != -1; pname++)
    {
    if (!strcmp(work_val, pname->name))
      break;
    }

  if (pname->type == -1)
    {
    free(work_val);
    return(PBSE_BADATVAL);
    }

  type = pname->type;

  /* what types do we have already? */
  for (i = 0;i < JOB_DEPEND_NUMBER_TYPES;i++)
    have[i] = NULL;

  for (pd = (struct depend *)GET_NEXT(pattr->at_val.at_list);
       pd;
       pd = (struct depend *)GET_NEXT(pd->dp_link))
    {
    have[pd->dp_type] = pd;
    }

  /* certain combinations are not allowed */

  switch (type)
    {

    case JOB_DEPEND_TYPE_SYNCWITH:

      if (have[JOB_DEPEND_TYPE_SYNCWITH]   ||
          have[JOB_DEPEND_TYPE_SYNCCT]     ||
          have[JOB_DEPEND_TYPE_AFTERSTART] ||
          have[JOB_DEPEND_TYPE_AFTEROK]    ||
          have[JOB_DEPEND_TYPE_AFTERNOTOK] ||
          have[JOB_DEPEND_TYPE_AFTERANY]   ||
          have[JOB_DEPEND_TYPE_ON])
        {
        free(work_val);
        return(PBSE_BADATVAL);
        }

      break;

    case JOB_DEPEND_TYPE_SYNCCT:

      if (have[JOB_DEPEND_TYPE_SYNCWITH] ||
          have[JOB_DEPEND_TYPE_SYNCCT])
        {
        free(work_val);
        return(PBSE_BADATVAL);
        }

      break;

    case JOB_DEPEND_TYPE_AFTERSTART:

    case JOB_DEPEND_TYPE_AFTEROK:

    case JOB_DEPEND_TYPE_AFTERNOTOK:

    case JOB_DEPEND_TYPE_AFTERANY:

    case JOB_DEPEND_TYPE_ON:

      if (have[JOB_DEPEND_TYPE_SYNCWITH])
        {
        free(work_val);
        return(PBSE_BADATVAL);
        }

      break;

    }

  if ((pd = have[type]) == NULL)
    {
    pd = make_depend(type, pattr);

    if (pd == NULL)
      {
      free(work_val);
      return(PBSE_SYSTEM);
      }
    }

  /* now process the value string */

  numwds = 0;

  while (nxwrd && (*nxwrd != '\0'))
    {
    numwds++;  /* number of arguments */
    valwd = nxwrd;

    /* find end of next word delimited by a : but not a '\:' */
    while (((*nxwrd != ':') || (*(nxwrd - 1) == '\\')) && *nxwrd)
      nxwrd++;

    if (*nxwrd)
      *nxwrd++ = '\0';

    /* now process word (argument) depending on "depend type" */
    if ((type == JOB_DEPEND_TYPE_ON) ||
        (type == JOB_DEPEND_TYPE_SYNCCT))
      {

      /* a single word argument, a count */
      if (numwds == 1)
        {
        pd->dp_numexp = strtol(valwd, &pwhere, 10);

        if ((pd->dp_numexp < 1) ||
            (pwhere && (*pwhere != '\0')))
          {
          free(work_val);
          return(PBSE_BADATVAL);
          }
        }
      else
        {
        free(work_val);
        return(PBSE_BADATVAL);
        }

      }
    else   /* all other dependency types */
      {
      /* a set of job_id[\:port][@server[\:port]] */
      pdjb = new depend_job();

      if (pdjb)
        {
        pdjb->dc_state = 0;
        pdjb->dc_cost  = 0;
        pdjb->dc_child = valwd;

        std::size_t pos = pdjb->dc_child.find("@");

        if (pos != std::string::npos)
          {
          pdjb->dc_child.erase(pos);
          }

        pd->dp_jobs.push_back(pdjb);
        }
      else
        {
        free(work_val);
        return(PBSE_SYSTEM);
        }

      }
    }

  free(work_val);
  /* SUCCESS */

  return(PBSE_NONE);
  }  /* END build_depend() */





/*
 * clear_depend - clear a single dependency set
 * If the "exist" flag is set, any depend_job sub-structures are freed.
 */

void clear_depend(

  struct depend *pd,
  int            type,
  int            exist)

  {
  if (exist)
    {
    unsigned int dp_jobs_size = pd->dp_jobs.size();
    for (unsigned int i = 0; i < dp_jobs_size; i++)
      {
      delete pd->dp_jobs[i];
      }

    pd->dp_jobs.clear();
    }
  else
    {
    pd->dp_jobs.clear();
    CLEAR_LINK(pd->dp_link);
    }

  pd->dp_type = type;

  pd->dp_numexp = 0;
  pd->dp_numreg = 0;
  pd->dp_released = 0;

  return;
  }  /* END clear_depend() */



/*
 * del_depend_job - delete a single depend_job structure
 */

void del_depend_job(

  depend     *pdep,
  depend_job *pdj)

  {
  // erase pdj from my list
  unsigned int dp_jobs_size = pdep->dp_jobs.size();
  for (unsigned int i = 0; i < dp_jobs_size; i++)
    {
    if (pdj == pdep->dp_jobs[i])
      {
      pdep->dp_jobs.erase(pdep->dp_jobs.begin() + i);
      break;
      }
    }

  delete pdj;
  }  /* END del_depend_job() */

/*
 * If pJob has an AFTERANY dependency on targetJob, remove it.
 * pJob is passed in with the ji_mutex locked.
 */
void removeAfterAnyDependency(
    
  const char *pJId,
  const char *targetJobID)

  {
  char *pTargetJobID = (char *)targetJobID;

  if (!strcmp((char *)pJId,pTargetJobID))
    return;

  job *pLockedJob = svr_find_job((char *)pJId,FALSE);
  
  if (pLockedJob == NULL)
    return;
  
  mutex_mgr job_mutex(pLockedJob->ji_mutex,true);
  pbs_attribute *pattr = &pLockedJob->ji_wattr[JOB_ATR_depend];
  struct depend *pDep = find_depend(JOB_DEPEND_TYPE_AFTERANY,pattr);
  
  if (pDep != NULL)
    {
    depend_job *pDepJob = find_dependjob(pDep,pTargetJobID);
    if (pDepJob != NULL)
      {
      del_depend_job(pDep, pDepJob);

      try
        {
        set_depend_hold(pLockedJob, pattr, NULL);
        }

      catch (int e)
        {
        job_mutex.set_unlock_on_exit(false);
        }
      }
    }
  }

/*
 * removeBeforeAnyDependencies()
 *
 *
 */

void removeBeforeAnyDependencies(
    
  const char *pJId)

  {
  job *pLockedJob = svr_find_job(pJId,FALSE);

  if (pLockedJob != NULL)
    {
    mutex_mgr job_mutex(pLockedJob->ji_mutex,true);
    pbs_attribute *pattr = &pLockedJob->ji_wattr[JOB_ATR_depend];

    struct depend *pDep = find_depend(JOB_DEPEND_TYPE_BEFOREANY,pattr);
    if (pDep != NULL)
      {

      unsigned int dp_jobs_size = pDep->dp_jobs.size();
      for (unsigned int i = 0; i < dp_jobs_size; i++)
        {
        depend_job *pDepJob = pDep->dp_jobs[i];
        std::string depID(pDepJob->dc_child);
        job_mutex.unlock();
        removeAfterAnyDependency(depID.c_str(),pJId);
        pLockedJob = svr_find_job(pJId,FALSE);
        if (pLockedJob == NULL)
          return;

        job_mutex.mark_as_locked();
        }
      }
    }

  } // END removeBeforeAnyDependencies()



/* END req_register.c */

