# @package      hubzero-submit-server
# @file         SubmitServer.py
# @copyright    Copyright (c) 2012-2020 The Regents of the University of California.
# @license      http://opensource.org/licenses/MIT MIT
#
# Copyright (c) 2012-2020 The Regents of the University of California.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# HUBzero is a registered trademark of The Regents of the University of California.
#
__version__ = '3.11.3'

import os
import sys
import signal
import select
import socket
import glob
import re
import uuid
import hashlib
import pwd
import subprocess
import ldap
import traceback
import math
import random
import shutil
import time
import string
import copy
import tempfile
import csv
import json
import pprint
import base64
import logging
from xml.dom import minidom, Node
import yaml
from errno import EPIPE

from hubzero.submit.LogMessage            import getLogIDMessage as getLogMessage, logSetJobId
from hubzero.submit.CommandParser         import CommandParser
from hubzero.submit.BoundConnection       import BoundConnection
from hubzero.submit.UnboundConnection     import UnboundConnection
from hubzero.submit.MySQLDatabase         import MySQLDatabase
from hubzero.submit.InfosInfo             import InfosInfo
from hubzero.submit.SubmissionScriptsInfo import SubmissionScriptsInfo

if sys.version_info > (3,):
   long = int
ZERO = long(0)

class SubmitServer:
   def __init__(self,
                configurationDirectory,
                submitConfigurationFile,
                infosDirectory,
                infosConfigurationFile,
                distributorDirectory,
                distributorScript,
                harvesterDirectory,
                harvesterScript,
                jobStatusDirectory,
                jobStatusScript,
                jobKillDirectory,
                jobKillScript,
                venueProbeDirectory,
                venueProbeScript):
      self.logger = logging.getLogger(__name__)
      self.logger.log(logging.INFO,getLogMessage(80*'#'))
      self.logger.log(logging.INFO,getLogMessage("Submit server version: %s" % (__version__)))

      self.configData             = {}
      self.configFilePath         = os.path.join(configurationDirectory,submitConfigurationFile)
      self.infosConfigurationPath = os.path.join(infosDirectory,infosConfigurationFile)
      self.publicKeyDirectory     = os.path.join(configurationDirectory,'.ssh')

      self.infosInfo             = None
      self.submissionScriptsInfo = None

      self.mySQLDatabase    = None
      self.clientListener   = None
      self.attachListener   = None
      self.serverConnection = None
      self.commandParser    = None

      self.clientData                              = {}
      self.clientData['clientId']                  = None
      self.clientData['version']                   = ""
      self.clientData['doubleDashTerminator']      = False
      self.clientData['userName']                  = ""
      self.clientData['sudoUserName']              = ""
      self.clientData['wsUserName']                = ""
      self.clientData['password']                  = ""
      self.clientData['sessionId']                 = 0
      self.clientData['sessionToken']              = ""
      self.clientData['cacheHosts']                = ""
      self.clientData['privateFingerPrint']        = ""
      self.clientData['clientToken']               = ""
      self.clientData['args']                      = []
      self.clientData['envs']                      = {}
      self.clientData['umask']                     = 0o027
      self.clientData['workDirectory']             = ""
      self.clientData['workDirectoryProperties']   = {}
      self.clientData['signaturePath']             = {}
      self.clientData['isClientTTY']               = True
      self.clientData['pegasusVersion']            = None
      self.clientData['reportMetrics']             = False
      self.clientData['isParametric']              = False
      self.clientData['localExecution']            = False
      self.clientData['isLocalPegasus']            = False
      self.clientData['inputFileMapping']          = {}
      self.clientData['suspended']                 = False
      self.clientData['submitCommandFileMapping']  = {}
      self.clientData['submitCommandFilesMapped']  = False
      self.clientData['submitCommandTransferPath'] = ""
      self.clientData['isBuildingInputTar']        = False

      self.serverData                         = {}
      self.serverData['serverId']             = uuid.uuid4()
      self.serverData['authenticationHash']   = ""
      self.serverData['hostFQDN']             = socket.getfqdn()
      self.serverData['version']              = __version__
      self.serverData['operationMode']        = ZERO
      self.serverData['distributorDirectory'] = distributorDirectory
      self.serverData['distributorScript']    = distributorScript
      self.serverData['distributorExists']    = False
      self.serverData['harvesterDirectory']   = harvesterDirectory
      self.serverData['harvesterScript']      = harvesterScript
      self.serverData['harvesterExists']      = False
      self.serverData['jobStatusDirectory']   = jobStatusDirectory
      self.serverData['jobStatusScript']      = jobStatusScript
      self.serverData['jobStatusExists']      = False
      self.serverData['jobKillDirectory']     = jobKillDirectory
      self.serverData['jobKillScript']        = jobKillScript
      self.serverData['jobKillExists']        = False
      self.serverData['venueProbeDirectory']  = venueProbeDirectory
      self.serverData['venueProbeScript']     = venueProbeScript
      self.serverData['venueProbeExists']     = False
      self.serverData['pegasusExists']        = False
      self.serverData['pegasusVersion']       = None
      self.serverData['pegasusHome']          = None
      self.serverData['pegasusPythonPath']    = None
      self.serverData['type']                 = 'Listener'
      self.serverData['squidInJob']           = False
      self.serverData['exitCode']             = 2
      self.serverData['distributorExitCode']  = 2
      self.serverData['uid']                  = None
      self.serverData['gid']                  = None
      self.serverData['gids']                 = None
      self.serverData['progressReport']       = 'text'
      self.serverData['homeDirectory']        = None
      self.serverData['email']                = None
      self.serverData['submitterClass']       = 1
      self.serverData['maximumAuthAttempts']  = 3
      self.serverData['authAttempts']         = 0
      self.serverData['authz']                = False
      self.serverData['handshakeComplete']    = False
      self.serverData['createdSessions']      = []
      self.serverData['sessionDirectory']     = ""
      self.serverData['session']              = 0
      self.serverData['jobToken']             = ""
      self.serverData['jobId']                = 0
      self.serverData['localJobId']           = "%08d" % (self.serverData['jobId'])
      self.serverData['superJobId']           = 0
      self.serverData['runName']              = ""
      self.serverData['args']                 = []
      self.serverData['envs']                 = {}
      self.serverData['event']                = "[simulation]"
      self.serverData['ncpus']                = 1
      self.serverData['venue']                = "any"
      self.serverData['metricsRecorded']      = False
      self.serverData['metrics']              = ""
      self.serverData['waitForLowLoad']       = False
      self.serverData['detach']               = False
      self.serverData['detached']             = False
      self.serverData['socketPath']           = ""
      self.serverData['socketPathPrefix']     = 'file://'
      self.serverData['attachId']             = ZERO
      self.serverData['attachJobId']          = "%08d" % (self.serverData['attachId'])
      self.serverData['attached']             = False
# Assume that a db entry exists.  That's where the jobid came from.
# If we've improperly assumed it exists, then finalizeJob() will
# indicate failure when a job is aborted prematurely.
      self.serverData['readyToExit']          = False
      self.serverData['dbJobEntryExists']     = False
      self.serverData['doFinalizeOperations'] = True
      self.serverData['finalized']            = False
      self.serverData['mysqlAuthenticated']   = False
      self.serverData['ldapAuthenticated']    = False
      self.serverData['sessionLimit']         = 0
      self.serverData['timeoutInterval']      = 60
      self.serverData['doHeartbeat']          = True
      self.serverData['jobScanInterval']      = 30
      self.serverData['disconnectTime']       = 0
      self.serverData['isImportingInputTar']  = False
      self.serverData['isImportingSubmitTar'] = False
      self.serverData['isExportingOutputTar'] = False
      self.serverData['isServerReadyForIO']   = False
      self.serverData['selectTimeout']        = 1*60.
      self.serverData['selectTimeoutSpread']  = 0.15
      self.serverData['disconnectMax']        = 3*self.serverData['selectTimeout']
      self.serverData['childPid']             = None
      self.serverData['workDirectory']        = ""
      self.serverData['workDirectoryShared']  = True
      self.serverData['clientTransferPath']   = ""
      self.serverData['inputObjects']         = {}
      self.serverData['outputObjects']        = {}
      self.serverData['importObjects']        = {}
      self.serverData['exportObjects']        = {}
      self.serverData['closePendingObjects']  = []
      self.serverData['childHasExited']       = False
      self.serverData['whenChildExited']      = -time.time()
      self.serverData['childKillPending']     = False
      self.serverData['jobScanPath']          = ""
      self.serverData['runStatusPath']        = ""
      self.serverData['ignoreMountPoints']    = [ os.sep + mp for mp in ['boot','dev','etc','proc','run','sys']]
      self.serverData['mountPoints']          = {}
      self.serverData['inputFileMapping']     = {}
      dfCommand = ['df','--portability','--all']
      child = subprocess.Popen(dfCommand,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               close_fds=True)
      dfStdOutput,dfStdError = child.communicate()
      dfExitStatus = child.returncode
      if dfExitStatus == 0:
         mounts = ''.join(dfStdOutput.decode('utf-8')).strip().split('\n')[1:]
         for mount in mounts:
            device     = mount.split()[0]
            mountPoint = mount.split()[-1]

            if mountPoint in self.serverData['ignoreMountPoints']:
               pass
            else:
               skipMountPoint = False
               for ignoreMountPoint in self.serverData['ignoreMountPoints']:
                  if mountPoint.startswith(ignoreMountPoint + os.sep):
                     skipMountPoint = True
                     break

               if not skipMountPoint:
                  if ':' in device:
                     self.serverData['mountPoints'][mountPoint] = {'device':device}
                  else:
                     self.serverData['mountPoints'][mountPoint] = {'device':':'.join([self.serverData['hostFQDN'],device])}
      else:
         self.logger.log(logging.ERROR,getLogMessage("df command failed: %s\n" % (dfStdError.decode('utf-8'))))
      self.logger.log(logging.DEBUG,getLogMessage("mountPoints:\n%s" % (pprint.pformat(self.serverData['mountPoints']))))

      signal.signal(signal.SIGINT,self.handleSignal)
      signal.signal(signal.SIGHUP,self.handleSignal)
      signal.signal(signal.SIGQUIT,self.handleSignal)
      signal.signal(signal.SIGABRT,self.handleSignal)
      signal.signal(signal.SIGTERM,self.handleSignal)
      signal.signal(signal.SIGCHLD,self.handleChild)


   def exit(self,
            exitCode=0):
      if   self.serverData['type'] == 'JobExecuter':
         if self.serverData['readyToExit']:
            self.killChild()
            self.unsetupWorkDirectory()
            if self.serverData['socketPath']:
               if self.serverData['socketPath'].startswith(self.serverData['socketPathPrefix']):
                  socketPath = self.serverData['socketPath'][len(self.serverData['socketPathPrefix']):]
               else:
                  socketPath = self.serverData['socketPath']
               if os.path.exists(socketPath):
                  try:
                     os.remove(socketPath)
                  except:
                     self.logger.log(logging.DEBUG,getLogMessage("socket removal failed: %s" % (socketPath)))
               self.serverData['socketPath'] = ""
            if self.serverData['clientTransferPath']:
               if os.path.isdir(self.serverData['clientTransferPath']):
                  shutil.rmtree(self.serverData['clientTransferPath'],True)
            if self.clientData['submitCommandTransferPath']:
               if os.path.isdir(self.clientData['submitCommandTransferPath']):
                  shutil.rmtree(self.clientData['submitCommandTransferPath'],True)
            exitCode = self.serverData['exitCode']
            self.logger.log(logging.INFO,getLogMessage("Server(%s) exiting(%d)." % (self.serverData['type'],exitCode)))
            sys.exit(exitCode)
         else:
            if not self.serverData['finalized']:
               if self.serverData['handshakeComplete']:
                  self.finalizeJob()
                  self.logger.log(logging.DEBUG,getLogMessage("exit finalizeJob complete"))
                  self.finalizeCreatedSessions()
                  self.logger.log(logging.DEBUG,getLogMessage("exit finalizeCreatedSessions complete"))
               self.serverData['finalized'] = True
               if self.serverData['exitCode'] == 2:
                  self.serverData['exitCode'] = exitCode
            if not self.serverData['detached']:
               if self.serverData['handshakeComplete']:
                  serverMessage = {'messageType':'serverExit',
                                   'exitCode':self.serverData['exitCode']}
                  self.clientListener.postJsonMessage(serverMessage)
                  self.logger.log(logging.DEBUG,getLogMessage("exit serverExit posted"))
               self.serverData['readyToExit'] = True
            else:
               if self.serverData['childHasExited']:
                  if time.time() > self.serverData['whenChildExited']+self.configData['maximumTimeDetachedAfterCompletion']:
                     self.logger.log(logging.DEBUG,getLogMessage("maximum detached time since completion"))
                     self.serverData['readyToExit'] = True
            if self.serverData['handshakeComplete']:
               self.logger.log(logging.DEBUG,getLogMessage("exit finalized = %s" % (self.serverData['finalized'])))
               self.logger.log(logging.DEBUG,getLogMessage("exit detached = %s" % (self.serverData['detached'])))
               self.logger.log(logging.DEBUG,getLogMessage("exit readyToExit = %s" % (self.serverData['readyToExit'])))
      elif self.serverData['type'] == 'Cache':
         if self.serverData['readyToExit']:
            self.unsetupWorkDirectory()
            exitCode = self.serverData['exitCode']
            self.logger.log(logging.INFO,getLogMessage("Server(%s) exiting(%d)." % (self.serverData['type'],exitCode)))
            sys.exit(exitCode)
         else:
            if not self.serverData['finalized']:
               self.serverData['finalized'] = True
               if self.serverData['exitCode'] == 2:
                  self.serverData['exitCode'] = exitCode
            serverMessage = {'messageType':'serverExit',
                             'exitCode':self.serverData['exitCode']}
            self.clientListener.postJsonMessage(serverMessage)
            self.serverData['readyToExit'] = True
      elif self.serverData['type'] == 'Proxy':
         if self.serverData['readyToExit']:
            self.logger.log(logging.INFO,getLogMessage("Server(%s) exiting(%d)." % (self.serverData['type'],exitCode)))
            sys.exit(exitCode)
         else:
            if not self.serverData['finalized']:
               self.finalizeCreatedSessions()
               self.serverData['finalized'] = True
            serverMessage = {'messageType':'serverExit',
                             'exitCode':exitCode}
            self.clientListener.postJsonMessage(serverMessage)
            self.serverData['readyToExit'] = True
      elif self.serverData['type'] == 'Listener':
         self.logger.log(logging.INFO,getLogMessage("Server(%s) exiting(%d)." % (self.serverData['type'],exitCode)))
         sys.exit(exitCode)


   def killChild(self):
      if self.serverData['childPid']:
         self.serverData['childKillPending'] = True
         try:
            os.kill(-self.serverData['childPid'],signal.SIGINT)
            for retry in range(0,12):
               time.sleep(10)
               os.kill(self.serverData['childPid'],0)
               if not self.serverData['childKillPending']:
                  break
            if self.serverData['childKillPending']:
               os.kill(-self.serverData['childPid'],signal.SIGTERM)
               for retry in range(0,6):
                  time.sleep(10)
                  os.kill(self.serverData['childPid'],0)
                  if not self.serverData['childKillPending']:
                     break
            if self.serverData['childKillPending']:
               os.kill(-self.serverData['childPid'],signal.SIGKILL)
         except:
            pass


   def stopForMissingSocket(self):
      self.logger.log(logging.INFO,getLogMessage("Server stopping for missing socket, terminating child process."))
      self.serverData['readyToExit'] = True
      self.serverData['exitCode']    = 1
      self.killChild()
      self.logger.log(logging.INFO,getLogMessage("Server stopping for missing socket, terminated child process."))


   def stop(self,
            exitCode=0):
      self.logger.log(logging.INFO,getLogMessage("Server stopping, terminating child process."))
      self.serverData['readyToExit'] = True
      self.serverData['exitCode']    = exitCode
      self.killChild()
      serverMessage = {'messageType':'wait'}
      self.clientListener.postJsonMessage(serverMessage)
      self.logger.log(logging.INFO,getLogMessage("Server stopping, terminated child process."))


   def handleSignal(self,
                    signalNumber,
                    frame):
      self.logger.log(logging.ERROR,getLogMessage("Server(%d) was terminated by a signal %d." % (os.getpid(),signalNumber)))
      exitCode = (1 << 7) + signalNumber
      self.serverData['exitCode'] = exitCode
      self.exit(exitCode)


   def handleChild(self,
                   signalNumber,
                   frame):
      self.serverData['childKillPending'] = False


   def scheduleHeartbeat(self):
      signal.signal(signal.SIGALRM,self.heartbeat)
      signal.alarm(self.configData['heartbeatInterval'])


   def heartbeat(self,
                 signalNumber,
                 frame):
      if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
         sqlCommand = """UPDATE job SET heartbeat=now()
                          WHERE jobid='%d'
                      """ % (self.serverData['jobId'])
         try:
            self.mySQLDatabase.update(sqlCommand)
         except:
            self.logger.log(logging.ERROR,getLogMessage("ERROR: heartbeat"))
         else:
            self.mySQLDatabase.commit()

         self.mySQLDatabase.disconnect()

      if self.clientListener.isConnected() or self.serverData['detached']:
         self.serverData['disconnectTime'] = 0
         self.scheduleHeartbeat()
      else:
         self.serverData['disconnectTime'] = time.time() - self.clientListener.getConnectionReadTime()
#        self.logger.log(logging.INFO,getLogMessage("Server has been disconnected for %f seconds." % \
#                                                               (self.serverData['disconnectTime'])))
         if self.serverData['disconnectTime'] >= self.serverData['disconnectMax']:
            message = "disconnect time (%f) >= disconnect max (%f)" % (self.serverData['disconnectTime'],
                                                                       self.serverData['disconnectMax'])
            self.logger.log(logging.ERROR,getLogMessage(message))
            if self.serverData['childPid']:
               self.killChild()
            else:
               self.exit(1)
         else:
            self.scheduleHeartbeat()


   def clientHeartbeat(self):
      if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
         sqlCommand = """UPDATE job SET heartbeat=now()
                          WHERE jobid='%d'
                      """ % (self.serverData['jobId'])
         try:
            self.mySQLDatabase.update(sqlCommand)
         except:
            self.logger.log(logging.ERROR,getLogMessage("ERROR: clientHeartbeat"))
         else:
            self.mySQLDatabase.commit()

         self.mySQLDatabase.disconnect()
         self.logger.log(logging.INFO,getLogMessage("ClientHeartbeat applied"))


   def scheduleTimeout(self,
                       interval=0):
      if interval == 0:
         interval = self.serverData['timeoutInterval']

      if self.serverData['authAttempts'] >= self.serverData['maximumAuthAttempts']:
         self.handleTimeout(0,0)
      signal.signal(signal.SIGALRM,self.handleTimeout)
      interval = int(math.ceil(interval))
      signal.alarm(interval)
      self.logger.log(logging.INFO,getLogMessage("Server will time out in %d seconds." % (interval)))


   def handleTimeout(self,
                     signalNumber,
                     frame):
      if   not self.serverData['authz']:
         remoteIP = self.clientListener.getRemoteIP()
         if self.clientData['sudoUserName']:
            message = "User '%s(%s)' at '%s' failed to authenticate (%d) and timed out." % (self.clientData['userName'],
                                                                                            self.clientData['sudoUserName'],
                                                                                            remoteIP,
                                                                                            self.serverData['authAttempts'])
         else:
            message = "User '%s' at '%s' failed to authenticate (%d) and timed out." % (self.clientData['userName'],
                                                                                        remoteIP,
                                                                                        self.serverData['authAttempts'])
         self.logger.log(logging.CRITICAL,getLogMessage(message))
         self.exit(1)
      elif self.serverData['waitForLowLoad']:
         self.checkLoadAndLaunch()
      else:
         self.logger.log(logging.ERROR,getLogMessage("Unknown reason for timeout."))
         self.exit(1)


   def scheduleJobScan(self):
      signal.signal(signal.SIGALRM,self.handleJobScan)
      signal.alarm(self.serverData['jobScanInterval'])


   def handleJobScan(self,
                     signalNumber,
                     frame):
      scheduleNextJobScan = False
      if   self.clientData['isParametric'] and self.serverData['progressReport'] == 'curses':
         if self.serverData['jobScanPath'] == "":
            jobScanPath = os.path.join(self.serverData['workDirectory'],
                                       self.serverData['runName'],
                                       'remoteParameterCombinations.csv')
            self.serverData['jobScanPath'] = jobScanPath
         if os.path.exists(self.serverData['jobScanPath']):
            try:
               fpJobScan = open(self.serverData['jobScanPath'],'r')
               try:
                  jobScanText = fpJobScan.readlines()
               except (IOError,OSError):
                  self.logger.log(logging.ERROR,getLogMessage("%s could not be read" % (self.serverData['jobScanPath'])))
               else:
                  serverMessage = {'messageType':'jobScanUpdate',
                                   'text':''.join(jobScanText)}
                  self.clientListener.postJsonMessage(serverMessage)
               finally:
                  fpJobScan.close()
            except (IOError,OSError):
               self.logger.log(logging.ERROR,getLogMessage("%s could not be opened" % (self.serverData['jobScanPath'])))
         scheduleNextJobScan = True
      elif self.clientData['isParametric'] and self.clientData['reportMetrics']:
         if self.serverData['jobScanPath'] == "":
            jobScanPath = os.path.join(self.serverData['workDirectory'],
                                       self.serverData['runName'],
                                       'remoteParameterCombinations.csv')
            self.serverData['jobScanPath'] = jobScanPath
         if os.path.exists(self.serverData['jobScanPath']):
            try:
               fpJobScan = open(self.serverData['jobScanPath'],'r')
               try:
                  csvReader = csv.reader(fpJobScan)
                  jobsComplete,jobsTotal = [0,0]
                  parameterNames = []
                  while len(parameterNames) <= 1:
                     parameterNames = next(csvReader)
                     if parameterNames[0].split(':')[0] == '# completed':
                        submitCompleted = parameterNames[0].split(':')[1].strip()
                        jobsInfo = submitCompleted.split()[0]
                        jobsComplete,jobsTotal = [int(nJob) for nJob in jobsInfo.split('/')]
                  jobStatusStates = {}
                  for parameterCombination in csvReader:
                     jobStatusState = parameterCombination[1]
                     if not jobStatusState in jobStatusStates:
                        jobStatusStates[jobStatusState] = 0
                     jobStatusStates[jobStatusState] += 1
               except StopIteration:
                  self.logger.log(logging.ERROR,getLogMessage("%s header row is missing" % (self.serverData['jobScanPath'])))
               except csv.Error:
                  self.logger.log(logging.ERROR,getLogMessage("csv reader failed on %s" % (self.serverData['jobScanPath'])))
               else:
                  msg =  "=SUBMIT-METRICS=>"
                  msg += " job=%s" % (self.serverData['localJobId'])
                  msg += " completed=%d total=%d" % (jobsComplete,jobsTotal)
                  for state,nState in jobStatusStates.items():
                     msg += " %s=%d" % (state,nState)
                  serverMessage = {'messageType':'writeStderr',
                                   'text':msg + '\n'}
                  self.clientListener.postJsonMessage(serverMessage)
               finally:
                  fpJobScan.close()
            except (IOError,OSError):
               self.logger.log(logging.ERROR,getLogMessage("%s could not be opened" % (self.serverData['jobScanPath'])))
         scheduleNextJobScan = True

      if self.clientData['isParametric']:
         if self.serverData['runStatusPath'] == "":
            runStatusPath = os.path.join(self.serverData['workDirectory'],
                                         self.serverData['runName'],
                                         'pegasusstatus.txt')
            self.serverData['runStatusPath'] = runStatusPath
         if os.path.exists(self.serverData['runStatusPath']):
            try:
               fpRunStatus = open(self.serverData['runStatusPath'],'r')
               try:
                  runStatusText = fpRunStatus.readlines()
               except (IOError,OSError):
                  self.logger.log(logging.ERROR,getLogMessage("%s could not be read" % (self.serverData['runStatusPath'])))
               else:
                  serverMessage = {'messageType':'runStatusUpdate',
                                   'text':''.join(runStatusText),
                                   'saveFile':not self.serverData['workDirectoryShared']}
                  self.clientListener.postJsonMessage(serverMessage)
               finally:
                  fpRunStatus.close()
            except (IOError,OSError):
               self.logger.log(logging.ERROR,getLogMessage("%s could not be opened" % (self.serverData['runStatusPath'])))
         scheduleNextJobScan = True

      if scheduleNextJobScan:
         if self.clientListener.isConnected():
            self.serverData['disconnectTime'] = 0
            self.scheduleJobScan()
         else:
            self.serverData['disconnectTime'] = time.time() - self.clientListener.getConnectionReadTime()
#           self.logger.log(logging.INFO,getLogMessage("Server has been disconnected for %f seconds." % \
#                                                                  (self.serverData['disconnectTime'])))
            if self.serverData['disconnectTime'] >= self.serverData['disconnectMax']:
               message = "disconnect time (%d) >= disconnect max (%d)" % (self.serverData['disconnectTime'],
                                                                          self.serverData['disconnectMax'])
               self.logger.log(logging.ERROR,getLogMessage(message))
               if self.serverData['childPid']:
                  self.killChild()
               else:
                  self.exit(1)
            else:
               self.scheduleJobScan()


   def cancelTimeout(self):
      alarmFunction = signal.getsignal(signal.SIGALRM)
      if not alarmFunction in [None,signal.SIG_IGN,signal.SIG_DFL]:
         try:
            if alarmFunction.__qualname__ == self.handleTimeout.__qualname__:
               signal.alarm(0)
         except:
            if cmp(alarmFunction,self.handleTimeout) == 0:
               signal.alarm(0)


   def configure(self):
      sectionPattern  = re.compile(r'(\s*\[)([^\s]*)(]\s*)')
      keyValuePattern = re.compile(r'( *)(\w*)( *= *)(.*[^\s$])( *)')
      commentPattern  = re.compile(r'\s*#.*')
      inServerSection = False

      configured = False
      try:
         fpConfig = open(self.configFilePath,'r')
         try:
            eof = False
            while not eof:
               record = fpConfig.readline()
               if record != "":
                  record = commentPattern.sub("",record)
                  if   sectionPattern.match(record):
                     sectionName = sectionPattern.match(record).group(2)
                     inServerSection = (sectionName == 'server')
                     if inServerSection:
                        self.configData = {'listenURIs':[],
                                           'sudoUsers':[],
                                           'trustedUsers':[],
                                           'submitSSLcert':"/etc/submit/submit_server.crt",
                                           'submitSSLkey':"/etc/submit/submit_server.key",
                                           'submitSSLCA':"/etc/submit/submit_server_ca.crt",
                                           'mysqlHost':"",
                                           'mysqlUser':"",
                                           'mysqlPassword':"",
                                           'mysqlCA':"",
                                           'mysqlCiphers':[],
                                           'mysqlMiddlewareDB':"",
                                           'mysqlUserDB':"",
                                           'ldapHosts':[],
                                           'ldapBaseDN':"",
                                           'ldapUserDN':"",
                                           'ldapTLSCipherSuite':"",
                                           'loadLimit':510,
                                           'loadHalflife':3600,
                                           'loadHorizon':86400,
                                           'heartbeatInterval':3600,
                                           'maximumTimeDetachedAfterCompletion':86400,
                                           'mountPointTranslations':{},
                                           'useSetup':'/etc/environ.sh',
                                           'emailFrom':"",
                                           'userClassMap':{'PRO Member':2},
                                           'cacheLoadUsers':[],
                                           'enableDistributor':True,
                                           'enableHarvester':True,
                                           'enableJobStatus':True,
                                           'enableJobKill':True,
                                           'enableVenueProbe':True
                                          }
                  elif inServerSection:
                     if keyValuePattern.match(record):
                        key,value = keyValuePattern.match(record).group(2,4)
                        if key in self.configData:
                           if   isinstance(self.configData[key],list):
                              self.configData[key] = [e.strip() for e in value.split(',')]
                           elif isinstance(self.configData[key],bool):
                              self.configData[key] = bool(value.lower() == 'true')
                           elif isinstance(self.configData[key],float):
                              self.configData[key] = float(value)
                           elif isinstance(self.configData[key],int):
                              self.configData[key] = int(value)
                           elif isinstance(self.configData[key],dict):
                              try:
                                 sampleKey   = self.configData[key].keys()[0]
                                 sampleValue = self.configData[key][sampleKey]
                              except:
                                 try:
                                    self.configData[key] = json.loads(value)
                                 except:
                                    self.configData[key] = {}
                                    sampleKey   = "key"
                                    sampleValue = "value"
                              else:
                                 self.configData[key] = {}
               
                              if not self.configData[key]:
                                 for e in value.split(','):
                                    dictKey,dictValue = e.split(':')
                                    if isinstance(sampleKey,int):
                                       dictKey = int(dictKey)
                                    if   isinstance(sampleValue,int):
                                       dictValue = int(dictValue)
                                    elif isinstance(sampleValue,float):
                                       dictValue = float(dictValue)
                                    elif isinstance(sampleValue,bool):
                                       dictValue = bool(dictValue.lower() == 'true')
                                    self.configData[key][dictKey] = dictValue
                           else:
                              self.configData[key] = value
                        else:
                           self.logger.log(logging.WARNING,getLogMessage("Undefined key = value pair %s = %s" % (key,value)))
               else:
                  eof = True
                  configured = True
         except (IOError,OSError):
            self.logger.log(logging.ERROR,getLogMessage("%s could not be read" % (self.configFilePath)))
         finally:
            fpConfig.close()
      except (IOError,OSError):
         self.logger.log(logging.ERROR,getLogMessage("%s could not be opened" % (self.configFilePath)))

      if configured:
         if self.configData['enableDistributor']:
            if self.serverData['distributorDirectory'] and self.serverData['distributorScript']:
               if os.path.exists(os.path.join(self.serverData['distributorDirectory'],self.serverData['distributorScript'])):
                  self.serverData['distributorExists'] = True
         if not self.serverData['distributorExists']:
            self.serverData['distributorDirectory']    = None
            self.serverData['distributorScript']       = None

         if self.configData['enableHarvester']:
            if self.serverData['harvesterDirectory'] and self.serverData['harvesterScript']:
               if os.path.exists(os.path.join(self.serverData['harvesterDirectory'],self.serverData['harvesterScript'])):
                  self.serverData['harvesterExists']   = True
         if not self.serverData['harvesterExists']:
            self.serverData['harvesterDirectory']      = None
            self.serverData['harvesterScript']         = None

         if self.configData['enableJobStatus']:
            if self.serverData['jobStatusDirectory'] and self.serverData['jobStatusScript']:
               if os.path.exists(os.path.join(self.serverData['jobStatusDirectory'],self.serverData['jobStatusScript'])):
                  self.serverData['jobStatusExists']   = True
         if not self.serverData['jobStatusExists']:
            self.serverData['jobStatusDirectory']      = None
            self.serverData['jobStatusScript']         = None

         if self.configData['enableJobKill']:
            if self.serverData['jobKillDirectory'] and self.serverData['jobKillScript']:
               if os.path.exists(os.path.join(self.serverData['jobKillDirectory'],self.serverData['jobKillScript'])):
                  self.serverData['jobKillExists']     = True
         if not self.serverData['jobKillExists']:
            self.serverData['jobKillDirectory']        = None
            self.serverData['jobKillScript']           = None

         if self.configData['enableVenueProbe']:
            if self.serverData['venueProbeDirectory'] and self.serverData['venueProbeScript']:
               if os.path.exists(os.path.join(self.serverData['venueProbeDirectory'],self.serverData['venueProbeScript'])):
                  self.serverData['venueProbeExists']  = True
         if not self.serverData['venueProbeExists']:
            self.serverData['venueProbeDirectory']     = None
            self.serverData['venueProbeScript']        = None

         self.logger.log(logging.INFO,getLogMessage("distributorExists: %s" % (self.serverData['distributorExists'])))
         self.logger.log(logging.INFO,getLogMessage("  harvesterExists: %s" % (self.serverData['harvesterExists'])))
         self.logger.log(logging.INFO,getLogMessage("  jobStatusExists: %s" % (self.serverData['jobStatusExists'])))
         self.logger.log(logging.INFO,getLogMessage("    jobKillExists: %s" % (self.serverData['jobKillExists'])))
         self.logger.log(logging.INFO,getLogMessage(" venueProbeExists: %s" % (self.serverData['venueProbeExists'])))

      if configured:
         if len(self.configData['listenURIs']) == 0:
            self.logger.log(logging.ERROR,getLogMessage("listenURIs missing from %s" % (self.configFilePath)))
            configured = False

         if self.configData['mysqlHost'] == "" or \
            self.configData['mysqlUser'] == "" or \
            self.configData['mysqlPassword'] == "" or \
            self.configData['mysqlMiddlewareDB'] == "":
            self.logger.log(logging.ERROR,getLogMessage("MySQL information missing from %s" % (self.configFilePath)))
            configured = False
         else:
            if self.configData['mysqlUserDB'] == "":
               self.configData['mysqlUserDB'] = self.configData['mysqlMiddlewareDB']

         if len(self.configData['ldapHosts']) == 0 or \
            self.configData['ldapUserDN'] == "":
            self.logger.log(logging.ERROR,getLogMessage("LDAP information missing from %s" % (self.configFilePath)))
            configured = False

      return(configured)


   def setupMySQL(self):
      setup = False
      self.mySQLDatabase = MySQLDatabase(mysqlHost=self.configData['mysqlHost'],
                                         mysqlUser=self.configData['mysqlUser'],
                                         mysqlPassword=self.configData['mysqlPassword'],
                                         mysqlCA=self.configData['mysqlCA'],
                                         mysqlCiphers=self.configData['mysqlCiphers'])
      if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
         self.mySQLDatabase.disconnect()
         if self.configData['mysqlUserDB'] != self.configData['mysqlMiddlewareDB']:
            if self.mySQLDatabase.connect(self.configData['mysqlUserDB']):
               self.mySQLDatabase.disconnect()
               setup = True
         else:
            setup = True

      return(setup)


   def setupClientListeners(self):
      self.clientListener = BoundConnection(self.configData['listenURIs'],
                                            submitSSLcert=self.configData['submitSSLcert'],
                                            submitSSLkey=self.configData['submitSSLkey'],
                                            submitSSLCA=self.configData['submitSSLCA'],
                                            bufferSize=1024)

      return(self.clientListener.isListening())


   def setupAttachListener(self):
      if not self.serverData['socketPath']:
         self.serverData['socketPath'] = self.serverData['socketPathPrefix'] + \
                                         os.path.join(os.sep,'tmp','submit_%s' % (self.serverData['localJobId']))
         self.attachListener = BoundConnection([self.serverData['socketPath']])

      return(self.attachListener.isListening())


   def setupServerConnection(self):
      self.serverData['socketPath'] = self.serverData['socketPathPrefix'] + \
                                      os.path.join(os.sep,'tmp','submit_%s' % (self.serverData['attachJobId']))
      self.serverConnection = UnboundConnection(UnboundConnection.TLSREQUIREMENTNONE,
                                                listenURIs=[self.serverData['socketPath']],
                                                maximumConnectionPasses=1)
      isConnected = self.serverConnection.isConnected()

      return(isConnected)


   def showHelpUsage(self,
                     operationMode):
      if operationMode & (self.commandParser.OPERATIONMODEHELPUSAGE | \
                          self.commandParser.OPERATIONMODEHELPEXAMPLES):
         if operationMode & (self.commandParser.OPERATIONMODEHELPTOOLS | \
                             self.commandParser.OPERATIONMODEHELPVENUES | \
                             self.commandParser.OPERATIONMODEHELPMANAGERS | \
                             self.commandParser.OPERATIONMODEVERSIONDISTRIBUTOR):
            showHelp = False
         else:
            showHelp = True
      else:
         showHelp = False

      return(showHelp)


   def parseCommandArguments(self):
      argumentsOK = False
      continueExecution = False
      commandEnabled = True
      self.commandParser = CommandParser(self.clientData['doubleDashTerminator'])

      if len(self.clientData['submitCommandFileMapping']) > 0:
         for arg in self.clientData['args']:
            serverArg = arg
            for clientInputPath in self.clientData['submitCommandFileMapping']:
               argValue = self.clientData['submitCommandFileMapping'][clientInputPath]['argValue']
               if argValue == arg:
                  mapInputFile = self.clientData['submitCommandFileMapping'][clientInputPath]['mappingRequired']
                  if mapInputFile:
                     serverInputPath = os.path.join(self.clientData['submitCommandTransferPath'],clientInputPath[1:])
                     if os.path.exists(serverInputPath):
                        serverArg = serverInputPath
            self.serverData['args'].append(serverArg)
      else:
         for arg in self.clientData['args']:
            self.serverData['args'].append(arg)
      self.logger.log(logging.INFO,getLogMessage("Args are:" + str(self.serverData['args'])))

      self.commandParser.parseArguments(self.serverData['args'][1:])

      if self.commandParser.validateArguments():
         self.serverData['operationMode'] = self.commandParser.getOperationMode()
#        self.commandParser.logOperationMode(self.serverData['operationMode'])
         if not self.serverData['distributorExists']:
            if self.serverData['operationMode'] & self.commandParser.OPERATIONMODEVERSIONDISTRIBUTOR:
               self.serverData['operationMode'] = self.serverData['operationMode'] & \
                                                  ~self.commandParser.OPERATIONMODEVERSIONDISTRIBUTOR
               self.serverData['operationMode'] = self.serverData['operationMode'] & \
                                                  ~self.commandParser.OPERATIONMODERUNDISTRIBUTOR
            if self.serverData['operationMode'] & self.commandParser.OPERATIONMODEVERSIONSERVER:
               self.serverData['operationMode'] = self.serverData['operationMode'] | \
                                                  self.commandParser.OPERATIONMODERUNSERVER
#        self.commandParser.logOperationMode(self.serverData['operationMode'])

         if   self.serverData['operationMode'] & (self.commandParser.OPERATIONMODERUNDISTRIBUTOR | \
                                                  self.commandParser.OPERATIONMODERUNDISTRIBUTORID):
            if not self.serverData['distributorExists']:
               commandEnabled = False
         elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNHARVESTID:
            if not self.serverData['harvesterExists']:
               commandEnabled = False
         elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNSTATUS:
            if not self.serverData['jobStatusExists']:
               commandEnabled = False
         elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNKILL:
            if not self.serverData['jobKillExists']:
               commandEnabled = False
         elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNVENUESTATUS:
            if not self.serverData['venueProbeExists']:
               commandEnabled = False
         elif self.serverData['operationMode'] == 0:
            commandEnabled = False

         if commandEnabled:
            if self.serverData['operationMode'] & self.commandParser.OPERATIONMODEVERSIONSERVER:
               message = "Submit server version: %s\n" % (self.serverData['version'])
               serverMessage = {'messageType':'writeStdout',
                                'text':message}
               self.clientListener.postJsonMessage(serverMessage)

            if   self.showHelpUsage(self.serverData['operationMode']):
               self.commandParser.showUsage()
               argumentsOK = True
            elif self.serverData['operationMode'] & (self.commandParser.OPERATIONMODERUNSTATUS | \
                                                     self.commandParser.OPERATIONMODERUNKILL | \
                                                     self.commandParser.OPERATIONMODERUNVENUESTATUS | \
                                                     self.commandParser.OPERATIONMODERUNDISTRIBUTOR):
               argumentsOK = True
               continueExecution = True
            elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNSERVER:
               argumentsOK = True
            elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNPROXY:
               self.serverData['type']        = 'Proxy'
               self.serverData['attachId']    = self.commandParser.getOption('attachId')
               self.serverData['attachJobId'] = "%08d" % (self.serverData['attachId'])
               argumentsOK = True
               continueExecution = True
            elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNCACHEHIT:
               self.serverData['type'] = 'Cache'
               argumentsOK = True
               continueExecution = True
            elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNCACHEMISS:
               self.serverData['type'] = 'Cache'
               argumentsOK = True
               continueExecution = True
            elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNCACHEPUBLISH:
               self.serverData['type'] = 'Cache'
               argumentsOK = True
               continueExecution = True
            elif self.serverData['operationMode'] & (self.commandParser.OPERATIONMODERUNDISTRIBUTORID | \
                                                     self.commandParser.OPERATIONMODERUNCLIENTLOCAL):
               exitCode = self.commandParser.setSweepParameters()
               if exitCode == 0:
                  exitCode = self.commandParser.setCSVDataParameters()
                  if exitCode == 0:
                     if self.commandParser.getParameterCombinationCount() > 0:
                        self.clientData['isParametric'] = True

                     self.clientData['localExecution'] = self.serverData['operationMode'] & \
                                                         self.commandParser.OPERATIONMODERUNCLIENTLOCAL
                     if self.clientData['localExecution']:
                        enteredExecutable = self.commandParser.getEnteredExecutable()
                        self.clientData['isLocalPegasus'] = os.path.basename(enteredExecutable).startswith('pegasus-')
                     self.clientData['reportMetrics']  = self.commandParser.getOption('metrics')
                     self.serverData['doHeartbeat']    = self.commandParser.getOption('doHeartbeat')
                     self.serverData['detach']         = self.commandParser.getOption('detach')
                     if self.commandParser.getOption('runName'):
                        self.serverData['runName']     = self.commandParser.getOption('runName')
                     argumentsOK = True
                     continueExecution = True
            elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNHARVESTID:
               self.clientData['localExecution'] = False
               self.clientData['reportMetrics']  = self.commandParser.getOption('metrics')
               if self.commandParser.getOption('runName'):
                  self.serverData['runName']     = self.commandParser.getOption('runName')
               argumentsOK = True
               continueExecution = True

      if   not self.commandParser.getOption('progress'):
         self.serverData['progressReport'] = 'silent'
      elif self.commandParser.getOption('progressCurses'):
         self.serverData['progressReport'] = 'curses'
      elif self.commandParser.getOption('progressSubmit'):
         self.serverData['progressReport'] = 'submit'
      elif self.commandParser.getOption('progressText'):
         self.serverData['progressReport'] = 'text'
      else:
         if self.clientData['isParametric'] and self.clientData['isClientTTY']:
            self.serverData['progressReport'] = 'curses'
         else:
            self.serverData['progressReport'] = 'text'

      if argumentsOK and continueExecution:
         if self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNHARVESTID:
            serverMessage = {'messageType':'harvestArgumentsParsed'}
         else:
            serverMessage = {'messageType':'argumentsParsed'}
         self.clientListener.postJsonMessage(serverMessage)

      return(argumentsOK,continueExecution,commandEnabled)


   def ldapAuthenticate(self,
                        username,
                        password):
      if username and password:
         # Check the following note for more info on ldaps...
         # http://sourceforge.net/mailarchive/forum.php?forum_id=4346&max_rows=25&style=flat&viewmonth=200508

         for ldapHost in self.configData['ldapHosts']:
            try:
               if   ldapHost.startswith('ldaps://'):
#                 ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT,ldap.OPT_X_TLS_ALLOW)
# From the ldap.conf manpage
                  if self.configData['ldapTLSCipherSuite']:
                     ldap.set_option(ldap.OPT_X_TLS_CIPHER_SUITE,self.configData['ldapTLSCipherSuite'])
                  ldapObject = ldap.initialize(ldapHost)
               elif ldapHost.startswith('ldap://'):
                  ldapObject = ldap.initialize(ldapHost)
               else:
                  self.logger.log(logging.ERROR,getLogMessage("Invalid ldapHost declaration.  Must be ldaps:// or ldap://"))
                  continue
            except ldap.LDAPError as e:
               self.logger.log(logging.ERROR,getLogMessage("%s: %s" % (ldapHost,e)))
               continue

            try:
               ldapObject.simple_bind_s(self.configData['ldapUserDN'] % (username),password)
            except ldap.LDAPError as e:
               self.logger.log(logging.ERROR,getLogMessage("%s: %s" % (ldapHost,e)))
            else:
               self.serverData['ldapAuthenticated'] = True
            finally:
               ldapObject.unbind_s()

            if self.serverData['ldapAuthenticated']:
               break

      return(self.serverData['ldapAuthenticated'])


   def isValidUsername(self):
      validUsername = False
      try:
         username = self.clientData['userName']
         (login,pw,uid,gid,name,homedir,shell) = pwd.getpwnam(username)
         self.serverData['uid']           = uid
         self.serverData['gid']           = gid
         self.serverData['homeDirectory'] = homedir
         validUsername = True
      except:
         self.logger.log(logging.ERROR,getLogMessage("Unable to get info for user '%s'" % (username)))

      return(validUsername)


   def getUserGroups(self):
      userInGroups = False
      self.serverData['gids'] = []
      if   self.clientData['sudoUserName']:
         username = self.clientData['sudoUserName']
      elif self.clientData['wsUserName']:
         username = self.clientData['wsUserName']
      else:
         username = self.clientData['userName']
      try:
         idCommand = ['/usr/bin/id',username,'--groups']
         child = subprocess.Popen(idCommand,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE,
                                  close_fds=True)
         idStdOutput,idStdError = child.communicate()
         idExitStatus = child.returncode
         if idExitStatus == 0:
            idStdOutput = idStdOutput.decode('utf-8').strip()
            for gid in idStdOutput.split():
               self.serverData['gids'].append(int(gid))
            userInGroups = True
         else:
            self.logger.log(logging.ERROR,getLogMessage("Unable to get gidlist for user '%s'" % (username)))
            if idStdOutput:
               self.logger.log(logging.ERROR,getLogMessage(idStdOutput.decode('utf-8')))
            if idStdError:
               self.logger.log(logging.ERROR,getLogMessage(idStdError.decode('utf-8')))
      except:
         # Not fatal...
         self.logger.log(logging.ERROR,getLogMessage("Unable to get gidlist for user '%s'" % (username)))
         self.logger.log(logging.ERROR,getLogMessage(traceback.format_exc()))

      return(userInGroups)


   @staticmethod
   def __generateToken(requestedTokenLength,
                       characterSet=""):
      """Generate a cleartext token using urandom, with even probability of each character.
      this is done by discarding random bytes that don't fit into an even multiple of the number
      of allowed characters. Based on maxwell middleware funcction.
      """
      if not characterSet:
         characterSet = string.ascii_letters + string.digits
      nCharacterSet = len(characterSet)
      # get numbers in a batch, it's faster than one by one
      randomCharacters = os.urandom(requestedTokenLength*2)
      tokenLength      = 0 # how many random characters we've generated so far
      iRandomCharacter = 0 # index to next unused random number in array
      token = ""
      maximumCharacterOrdinal = (256 // nCharacterSet) * nCharacterSet
      if sys.version_info > (3,):
         while tokenLength < requestedTokenLength:
            # reject results that would skew the freq. distribution of characters
            if randomCharacters[iRandomCharacter] < maximumCharacterOrdinal:
               token += characterSet[randomCharacters[iRandomCharacter] % nCharacterSet]
               tokenLength += 1
            iRandomCharacter += 1
            if iRandomCharacter >= requestedTokenLength*2:
               # we've run out of random numbers, get some more
               randomCharacters = os.urandom(requestedTokenLength*2)
               iRandomCharacter = 0
      else:
         while tokenLength < requestedTokenLength:
            # reject results that would skew the freq. distribution of characters
            if ord(randomCharacters[iRandomCharacter]) < maximumCharacterOrdinal:
               token += characterSet[ord(randomCharacters[iRandomCharacter]) % nCharacterSet]
               tokenLength += 1
            iRandomCharacter += 1
            if iRandomCharacter >= requestedTokenLength*2:
               # we've run out of random numbers, get some more
               randomCharacters = os.urandom(requestedTokenLength*2)
               iRandomCharacter = 0

      return(token)


   def getUserEmailAddress(self,
                           username):
      emailAddress = ""
      if self.mySQLDatabase.connect(self.configData['mysqlUserDB']):
         sqlCommand = """SELECT email FROM jos_users WHERE username="%s"
                      """ % (username)
         try:
            result = self.mySQLDatabase.select(sqlCommand)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               emailAddress = row[0]
         self.mySQLDatabase.disconnect()

      return(emailAddress)


   def getUserSubmitterClass(self,
                             username):
      submitterClass = 1
      if self.mySQLDatabase.connect(self.configData['mysqlUserDB']):
         sqlCommand = """SELECT jos_users.id,jos_usergroups.title
                           FROM jos_users INNER JOIN jos_user_usergroup_map on jos_user_usergroup_map.user_id=jos_users.id
                                          INNER JOIN jos_usergroups on jos_user_usergroup_map.group_id=jos_usergroups.id
                          WHERE jos_users.username="%s"
                      """ % (username)
         try:
            result = self.mySQLDatabase.select(sqlCommand)
         except:
            pass
         else:
            if len(result) > 0:
               for row in result:
                  if row[1] in self.configData['userClassMap']:
                     submitterClass = max(submitterClass,self.configData['userClassMap'][row[1]])
         self.mySQLDatabase.disconnect()

      return(submitterClass)


   def getUserSessionLimit(self,
                           username):
      sessionLimit = 0
      if self.mySQLDatabase.connect(self.configData['mysqlUserDB']):
         sqlCommand = """SELECT CASE
                                     WHEN jos_users_tool_preferences.jobs IS NOT NULL AND
                                          jos_users_tool_preferences.jobs != ''           THEN jos_users_tool_preferences.jobs
                                     ELSE (SELECT jos_tool_session_classes.jobs FROM jos_tool_session_classes
                                                                               WHERE jos_tool_session_classes.alias='default')
                               END AS jobs
                              FROM jos_users LEFT JOIN jos_users_tool_preferences ON
                                                       jos_users_tool_preferences.user_id=jos_users.id
                             WHERE jos_users.username="%s"
                      """ % (username)
         try:
            result = self.mySQLDatabase.select(sqlCommand)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               sessionLimit = int(row[0])
         self.mySQLDatabase.disconnect()

      return(sessionLimit)


   def getUserSessionCount(self,
                           username):
      sessionCount = -1
      if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
         sqlCommand = """SELECT COUNT(*) FROM session WHERE username='%s' AND dispnum!=0
                      """ % (username)
         try:
            result = self.mySQLDatabase.select(sqlCommand)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               sessionCount = int(row[0])
         self.mySQLDatabase.disconnect()

      return(sessionCount)


   def signon(self,
              clientAuthenticationHash):
      authorized = False
      if self.isValidUsername() and self.getUserGroups():
         authUsername = ""
         try:
            sudoUsername = self.clientData['sudoUserName']
         except:
            sudoUsername = ""
         try:
            wsUsername = self.clientData['wsUserName']
         except:
            wsUsername = ""

         try:
            username = self.clientData['userName']
            authUsername = username
         except:
            username = ""
         try:
            password = self.clientData['password']
         except:
            password = ""

         try:
            sessionToken = self.clientData['sessionToken']
         except:
            sessionToken = ""
         jobToken = self.serverData['jobToken']

         if   username and jobToken:
            if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
               sqlCommand = """SELECT sessnum FROM job
                               WHERE username='%s' AND jobtoken='%s'
                            """ % (username,jobToken)
               try:
                  result = self.mySQLDatabase.select(sqlCommand)
               except:
                  pass
               else:
                  if len(result) != 0:
                     row = result[0]
                     session = int(row[0])
                     if session > 0:
                        self.serverData['session'] = session
                        authorized = True
                        self.serverData['mysqlAuthenticated'] = True
                     else:
                        message = "User %s job token %s not matched." % (username,jobToken)
                        self.logger.log(logging.ERROR,getLogMessage(message))
                  else:
                     message = "User %s job token %s not matched." % (username,jobToken)
                     self.logger.log(logging.ERROR,getLogMessage(message))
               self.mySQLDatabase.disconnect()
         elif username and sessionToken:
            if username in self.configData['trustedUsers']:
               self.logger.log(logging.INFO,getLogMessage("session = %d" % (self.clientData['sessionId'])))
               if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
                  sqlCommand = """SELECT username FROM session
                                  WHERE sessnum=%d AND sesstoken='%s'
                               """ % (self.clientData['sessionId'],sessionToken)
                  try:
                     result = self.mySQLDatabase.select(sqlCommand)
                  except:
                     pass
                  else:
                     if len(result) != 0:
                        row = result[0]
                        authUsername = row[0]
                        self.clientData['sudoUserName'] = authUsername
                        self.serverData['session'] = self.clientData['sessionId']
                        authorized = True
                        self.serverData['mysqlAuthenticated'] = True
                  self.mySQLDatabase.disconnect()
            else:
               if username in self.configData['sudoUsers'] and sudoUsername:
                  authUsername = sudoUsername
               self.logger.log(logging.INFO,getLogMessage("session = %d" % (self.clientData['sessionId'])))
               if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
                  sqlCommand = """SELECT sessnum FROM session
                                  WHERE username='%s' AND sesstoken='%s'
                               """ % (authUsername,sessionToken)
                  try:
                     result = self.mySQLDatabase.select(sqlCommand)
                  except:
                     pass
                  else:
                     if len(result) != 0:
                        row = result[0]
                        self.serverData['session'] = int(row[0])
                        authorized = True
                        self.serverData['mysqlAuthenticated'] = True
                  self.mySQLDatabase.disconnect()
         elif username and clientAuthenticationHash:
            authorized = clientAuthenticationHash == self.serverData['authenticationHash']
            if authorized:
#              self.logger.log(logging.ERROR,getLogMessage("PPK authentication succeeded for user '%s'" % (username)))
               remoteIP = self.clientListener.getRemoteIP()
               appname = "submit"

               authUsername = username
               if username in self.configData['trustedUsers']:
                  authUsername = self.clientData['wsUserName']

               self.serverData['sessionLimit'] = self.getUserSessionLimit(authUsername)
               if self.serverData['sessionLimit'] > 0:
                  sessionCount = self.getUserSessionCount(authUsername)
                  if sessionCount >= 0:
                     if sessionCount >= self.serverData['sessionLimit']:
                        message = "User %s cannot exceed session limit of %d" % (authUsername,self.serverData['sessionLimit'])
                        self.logger.log(logging.ERROR,getLogMessage(message))
                        serverMessage = {'messageType':'writeStderr',
                                         'text':"Session limit of %d reached.\n" % (self.serverData['sessionLimit'])}
                        self.clientListener.postJsonMessage(serverMessage)
                        authorized = False
                     else:
                        sessionToken = self.__generateToken(32)
                        timeout = 60*60*24
                        if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
                           sqlCommand = """INSERT INTO
                                   session(username,remoteip,exechost,dispnum,start,accesstime,appname,sessname,  sesstoken,timeout)
                                    VALUES(    '%s',    '%s',    '%s',      0,now(),     now(),   '%s',    '%s',       '%s',     %f)
                                """ % (authUsername,remoteIP,"submit",                         appname,appname,sessionToken,timeout)
                           try:
                              self.mySQLDatabase.insert(sqlCommand)
                           except:
                              message = "ERROR: Unable to create session record for '%s'" % (authUsername)
                              self.logger.log(logging.ERROR,getLogMessage(message))
                              serverMessage = {'messageType':'message',
                                               'text':"Internal database problem."}
                              self.clientListener.postJsonMessage(serverMessage)
                              authorized = False
                           else:
                              sqlCommand = """SELECT LAST_INSERT_ID()"""
                              try:
                                 result = self.mySQLDatabase.select(sqlCommand)
                              except:
                                 pass
                              else:
                                 row = result[0]
                                 session = int(row[0])
                                 if session > 0:
                                    self.serverData['session'] = session
                                    self.logger.log(logging.INFO,getLogMessage("Created session %d" % (self.serverData['session'])))

                                    self.serverData['createdSessions'].append(self.serverData['session'])
                           self.mySQLDatabase.commit()
                           self.mySQLDatabase.disconnect()
                        else:
                           message = "Unable to connect to database to create session for user %s" % (authUsername)
                           self.logger.log(logging.ERROR,getLogMessage(message))
                           authorized = False
                  else:
                     message = "Unable to determine current number of sessions for user %s" % (authUsername)
                     self.logger.log(logging.ERROR,getLogMessage(message))
                     authorized = False
               else:
                  message = "User %s cannot exceed session limit of %d" % (authUsername,self.serverData['sessionLimit'])
                  self.logger.log(logging.ERROR,getLogMessage(message))
                  authorized = False

               if authorized:
                  self.clientData['sudoUserName'] = authUsername
                  self.serverData['mysqlAuthenticated'] = True
            else:
               self.logger.log(logging.ERROR,getLogMessage("PPK authentication failed for user '%s'" % (username)))
# prevent doomed authentication attempts
               self.serverData['maximumAuthAttempts'] = 0
         elif username and password:
            authorized = self.ldapAuthenticate(username,password)
            if authorized:
               remoteIP = self.clientListener.getRemoteIP()
               appname = "submit"
               self.serverData['sessionLimit'] = self.getUserSessionLimit(username)

               if self.serverData['sessionLimit'] > 0:
                  sessionCount = self.getUserSessionCount(username)
                  if sessionCount >= 0:
                     if sessionCount >= self.serverData['sessionLimit']:
                        message = "User %s cannot exceed session limit of %d" % (username,self.serverData['sessionLimit'])
                        self.logger.log(logging.ERROR,getLogMessage(message))
                        serverMessage = {'messageType':'writeStderr',
                                         'text':"Session limit of %d reached.\n" % (self.serverData['sessionLimit'])}
                        self.clientListener.postJsonMessage(serverMessage)
                        authorized = False
                     else:
                        sessionToken = self.__generateToken(32)
                        timeout = 60*60*24
                        if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
                           sqlCommand = """INSERT INTO
                                   session(username,remoteip,exechost,dispnum,start,accesstime,appname,sessname,  sesstoken,timeout)
                                    VALUES(    '%s',    '%s',    '%s',      0,now(),     now(),   '%s',    '%s',       '%s',     %f)
                                    """ % (username,remoteIP,"submit",                         appname,appname,sessionToken,timeout)
                           try:
                              self.mySQLDatabase.insert(sqlCommand)
                           except:
                              message = "ERROR: Unable to create session record for '%s'" % (username)
                              self.logger.log(logging.ERROR,getLogMessage(message))
                              serverMessage = {'messageType':'message',
                                               'text':"Internal database problem."}
                              self.clientListener.postJsonMessage(serverMessage)
                              authorized = False
                           else:
                              sqlCommand = """SELECT LAST_INSERT_ID()"""
                              try:
                                 result = self.mySQLDatabase.select(sqlCommand)
                              except:
                                 pass
                              else:
                                 row = result[0]
                                 session = int(row[0])
                                 if session > 0:
                                    self.serverData['session'] = session
                                    self.logger.log(logging.INFO,getLogMessage("Created session %d" % (self.serverData['session'])))

                                    self.serverData['createdSessions'].append(self.serverData['session'])
                           self.mySQLDatabase.commit()
                           self.mySQLDatabase.disconnect()
                        else:
                           authorized = False
                  else:
                     authorized = False
               else:
                  authorized = False
            else:
               self.logger.log(logging.ERROR,getLogMessage("LDAP authentication failed for user '%s'" % (username)))

         if authorized:
            if self.serverData['operationMode'] & CommandParser.OPERATIONMODERUNMEASUREMENT:
               if not username in self.configData['trustedUsers']:
# prevent doomed authentication attempts
                  self.serverData['maximumAuthAttempts'] = 0
                  authorized = False

         if authorized:
            self.serverData['email']          = self.getUserEmailAddress(authUsername)
            self.serverData['submitterClass'] = self.getUserSubmitterClass(authUsername)

      return(authorized)


   def getInputObjects(self):
      clientListeningSockets,clientReader = self.clientListener.getInputObjects()
      if self.attachListener:
         attachListeningSocket,attachReader = self.attachListener.getInputObjects()
      else:
         attachListeningSocket = []
         attachReader          = []
      if self.serverConnection:
         serverReader = self.serverConnection.getInputObject()
      else:
         serverReader = []

      return(clientListeningSockets,
             clientReader,
             attachListeningSocket,
             attachReader,
             serverReader,
             list(self.serverData['inputObjects'].keys()),
             list(self.serverData['exportObjects'].keys()))


   def getOutputObjects(self):
      clientWriter = self.clientListener.getOutputObjects()
      if self.attachListener:
         attachWriter = self.attachListener.getOutputObjects()
      else:
         attachWriter = []
      if self.serverConnection:
         serverWriter = self.serverConnection.getOutputObject()
      else:
         serverWriter = []

      importObjects = []
      for importObject in self.serverData['importObjects']:
         if self.serverData['importObjects'][importObject]['buffer'] != b"":
            importObjects.append(importObject)

      outputObjects = []
      for outputObject in self.serverData['outputObjects']:
         if self.serverData['outputObjects'][outputObject]['buffer'] != b"":
            outputObjects.append(outputObject)

      return(clientWriter,
             attachWriter,
             serverWriter,
             importObjects,
             outputObjects)


   def readFile(self,
                fileObject):
      if   fileObject in self.serverData['inputObjects']:
         inputFile = self.serverData['inputObjects'][fileObject]
         if   inputFile == '#STDOUT#' or inputFile == '#STDERR#':
            try:
               text = os.read(fileObject.fileno(),1024).decode('utf-8')
            except:
               self.logger.log(logging.ERROR,getLogMessage("Exception reading STDOUT or STDERR"))
               self.exit(1)
            else:
               if not text:
                  self.closeFile(fileObject)
               else:
                  if inputFile == '#STDOUT#':
                     serverMessage = {'messageType':'writeStdout',
                                      'text':text}
                     self.clientListener.postJsonMessage(serverMessage)
                  else:
                     serverMessage = {'messageType':'writeStderr',
                                      'text':text}
                     self.clientListener.postJsonMessage(serverMessage)
         elif inputFile == '#METRICS#':
            try:
               text = os.read(fileObject.fileno(),1024).decode('utf-8')
            except:
               self.logger.log(logging.ERROR,getLogMessage("Exception reading METRICS"))
               self.exit(1)
            else:
               if not text:
                  self.closeFile(fileObject)
               else:
                  self.serverData['metrics'] += text
         else:
            try:
               text = os.read(fileObject.fileno(),1024)
            except:
               self.logger.log(logging.ERROR,getLogMessage("Exception reading inputObject"))
               self.exit(1)
            else:
               if not text:
                  serverMessage = {'messageType':'close',
                                   'file':inputFile}
                  self.clientListener.postJsonMessage(serverMessage)
                  self.closeFile(fileObject)
               else:
                  serverMessage = {'messageType':'write',
                                   'file':inputFile,
                                   'text':base64.b64encode(text).decode('utf-8')}
                  self.clientListener.postJsonMessage(serverMessage)
      elif fileObject in self.serverData['exportObjects']:
         inputFile = self.serverData['exportObjects'][fileObject]
         try:
            text = os.read(fileObject.fileno(),1024)
         except:
            self.logger.log(logging.ERROR,getLogMessage("Exception reading exportObject"))
            self.exit(1)
         else:
            if not text:
               serverMessage = {'messageType':'close',
                                'file':inputFile}
               self.clientListener.postJsonMessage(serverMessage)
               self.closeFile(fileObject)
            else:
               serverMessage = {'messageType':'write',
                                'file':inputFile,
                                'text':base64.b64encode(text).decode('utf-8')}
               self.clientListener.postJsonMessage(serverMessage)


   def writeFile(self,
                 fileObject):
      if   fileObject in self.serverData['importObjects']:
         writeBuffer = self.serverData['importObjects'][fileObject]['buffer']
         try:
            textLength = os.write(fileObject.fileno(),writeBuffer)
         except IOError as e:
            if e.args[0] in [EPIPE]:
               self.logger.log(logging.ERROR,getLogMessage("BrokenPipe exception writing importObject %s: %s" % \
                                       (self.serverData['importObjects'][fileObject]['path'],writeBuffer[0:25])))
               self.serverData['importObjects'][fileObject]['buffer'] = b""
               self.serverData['closePendingObjects'].append(fileObject)
            else:
               self.logger.log(logging.ERROR,getLogMessage("Exception writing importObject %s" % \
                                          (self.serverData['importObjects'][fileObject]['path'])))
               self.logger.log(logging.ERROR,getLogMessage(traceback.format_exc()))
               self.exit(1)
         except:
            self.logger.log(logging.ERROR,getLogMessage("Exception writing importObject %s" % \
                                       (self.serverData['importObjects'][fileObject]['path'])))
            self.logger.log(logging.ERROR,getLogMessage(traceback.format_exc()))
            self.exit(1)
         else:
            self.serverData['importObjects'][fileObject]['buffer'] = writeBuffer[textLength:]
#           self.logger.log(logging.DEBUG,getLogMessage("importObject: %d bytes written" % (textLength)))
         self.clientData['isBuildingInputTar'] = False
      elif fileObject in self.serverData['outputObjects']:
         writeBuffer = self.serverData['outputObjects'][fileObject]['buffer']
         try:
            textLength = os.write(fileObject.fileno(),writeBuffer)
         except IOError as e:
            if e.args[0] in [EPIPE]:
               self.logger.log(logging.ERROR,getLogMessage("BrokenPipe exception writing outputObject %s: %s" % \
                                       (self.serverData['outputObjects'][fileObject]['path'],writeBuffer[0:25])))
               self.serverData['outputObjects'][fileObject]['buffer'] = b""
               self.serverData['closePendingObjects'].append(fileObject)
            else:
               self.logger.log(logging.ERROR,getLogMessage("Exception writing outputObject %s" % \
                                          (self.serverData['outputObjects'][fileObject]['path'])))
               self.logger.log(logging.ERROR,getLogMessage(traceback.format_exc()))
               self.exit(1)
         except:
            self.logger.log(logging.ERROR,getLogMessage("Exception writing outputObject %s" % \
                                       (self.serverData['outputObjects'][fileObject]['path'])))
            self.logger.log(logging.ERROR,getLogMessage(traceback.format_exc()))
            self.exit(1)
         else:
            self.serverData['outputObjects'][fileObject]['buffer'] = writeBuffer[textLength:]
#           self.logger.log(logging.DEBUG,getLogMessage("outputObject: %d bytes written" % (textLength)))


   def closeFile(self,
                 fileObject):
      if fileObject in self.serverData['inputObjects']:
         del self.serverData['inputObjects'][fileObject]

      if fileObject in self.serverData['exportObjects']:
         exportFile = self.serverData['exportObjects'][fileObject]
         exportPath = os.path.join(os.sep,"tmp",exportFile)
         if os.path.exists(exportPath):
            os.remove(exportPath)
         del self.serverData['exportObjects'][fileObject]

      transferTarPath = ""
      if fileObject in self.serverData['importObjects']:
         transferTarPath = self.serverData['importObjects'][fileObject]['path']
         del self.serverData['importObjects'][fileObject]
         if len(self.serverData['importObjects']) == 0:
            if self.clientData['submitCommandFilesMapped']:
               serverMessage = {'messageType':'exportFilesComplete'}
               self.clientData['isBuildingInputTar']  = False
               self.serverData['isImportingInputTar'] = False
            else:
               serverMessage = {'messageType':'exportCommandFilesComplete'}
               self.clientData['isBuildingInputTar']       = False
               self.serverData['isImportingSubmitTar']     = False
               self.clientData['submitCommandFilesMapped'] = True
            self.clientListener.postJsonMessage(serverMessage)

      if fileObject in self.serverData['outputObjects']:
         del self.serverData['outputObjects'][fileObject]

      try:
         fileObject.close()
         if transferTarPath:
#           self.logger.log(logging.DEBUG,getLogMessage("Extracting transfer tarfile %s" % (transferTarPath)))
            if   transferTarPath.endswith('.tar.gz'):
               transferDirectory = os.path.basename(transferTarPath)[:-7]
            elif transferTarPath.endswith('.tar'):
               transferDirectory = os.path.basename(transferTarPath)[:-4]
            else:
               transferDirectory = "%s_transfer" % (self.serverData['jobId'])
            if os.path.isdir(self.serverData['workDirectory']):
               transferPath = os.path.join(self.serverData['workDirectory'],transferDirectory)
            else:
               transferPath = os.path.join(os.sep,"tmp",transferDirectory)
            if not os.path.isdir(transferPath):
               os.makedirs(transferPath)

            tarCommand = ['tar','xv','-C',transferPath,'-f',transferTarPath]
            self.logger.log(logging.INFO,getLogMessage("command: %s" % (tarCommand)))
            child = subprocess.Popen(tarCommand,
                                     stdout=subprocess.PIPE,
                                     stderr=subprocess.PIPE,
                                     close_fds=True)
            tarStdOutput,tarStdError = child.communicate()
            tarExitStatus = child.returncode
            if tarExitStatus == 0:
               os.remove(transferTarPath)
               self.serverData['clientTransferPath'] = transferPath
            else:
               self.logger.log(logging.ERROR,getLogMessage("Failed to extract transfer tarfile %s" % (transferTarPath)))
               if tarStdOutput:
                  self.logger.log(logging.ERROR,getLogMessage(tarStdOutput.decode('utf-8')))
               if tarStdError:
                  self.logger.log(logging.ERROR,getLogMessage(tarStdError.decode('utf-8')))
      except:
         self.logger.log(logging.ERROR,getLogMessage(traceback.format_exc()))


   def checkClosePendingObjects(self):
      markedForDeletion = []
      for fileObject in self.serverData['closePendingObjects']:
         if   fileObject in self.serverData['importObjects']:
            if self.serverData['importObjects'][fileObject]['buffer'] == b"":
               self.closeFile(fileObject)
               markedForDeletion.append(fileObject)
         elif fileObject in self.serverData['outputObjects']:
            if self.serverData['outputObjects'][fileObject]['buffer'] == b"":
               self.closeFile(fileObject)
               markedForDeletion.append(fileObject)
      for fileObject in markedForDeletion:
         self.serverData['closePendingObjects'].remove(fileObject)
      del markedForDeletion


   def checkLoad(self):
      loadOK = False
      if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
         #
         # Check that the load is low enough
         #
         sqlCommand = """SELECT SUM( POW(.5, TIMESTAMPDIFF(second,start,now())/%d) )
                           FROM job WHERE username='%s' AND TIMESTAMPDIFF(second,start,now()) < %d
                      """ % (self.configData['loadHalflife'],self.clientData['userName'],self.configData['loadHorizon'])
         try:
            result = self.mySQLDatabase.select(sqlCommand)
         except:
            pass
         else:
            if len(result) != 1:
               self.logger.log(logging.ERROR,getLogMessage("Error retrieving load for user %s" % (self.clientData['userName'])))
               self.scheduleTimeout()
            else:
               row = result[0]
               try:
                  load = float(row[0])
               except TypeError:
                  # If there are no entries for this user, result will be NULL
                  load = 0.

               if load < self.configData['loadLimit']:
                  message = "Cumulative job load is %.2f.  (Max: %.2f)" % (load,self.configData['loadLimit'])
                  self.logger.log(logging.INFO,getLogMessage(message))
                  loadOK = True
               else:
                  message = "User %s cannot exceed load limit %d." % (self.clientData['userName'],self.configData['loadLimit'])
                  self.logger.log(logging.ERROR,getLogMessage(message))
                  if not self.serverData['waitForLowLoad']:
                     msg = "You cannot exceed your active job limit of %f. (cur: %f)" % (self.configData['loadLimit'],load)
                     serverMessage = {'messageType':'message',
                                      'text':msg}
                     self.clientListener.postJsonMessage(serverMessage)
                     serverMessage = {'messageType':'jobId',
                                      'jobId':0}
                     self.clientListener.postJsonMessage(serverMessage)
                  else:
                     loadRatio = load/self.configData['loadLimit']
                     waitPeriod = math.ceil(self.configData['loadHalflife'] * math.log(loadRatio) / math.log(2))
                     msg = "Cumulative job load is %.2f.  (Max: %.2f)  Sleeping %d seconds." % (load,self.configData['loadLimit'],
                                                                                                waitPeriod)
                     self.logger.log(logging.INFO,getLogMessage(msg))
                     serverMessage = {'messageType':'message',
                                      'text':msg}
                     self.clientListener.postJsonMessage(serverMessage)
                     self.scheduleTimeout(waitPeriod)

         self.mySQLDatabase.disconnect()

      return(loadOK)


   def checkLoadAndLaunch(self):
      if self.checkLoad():
         self.insertJob()
         if not self.serverData['runName']:
            self.serverData['runName'] = self.serverData['localJobId']
         self.logger.log(logging.INFO,getLogMessage("Args are:" + str(self.clientData['args'])))
         if self.clientData['localExecution']:
            self.launchJob()
         else:
            serverMessage = {'messageType':'serverReadyForInputMapping'}
            self.clientListener.postJsonMessage(serverMessage)


   def doCacheSquid(self,
                    event,
                    cacheSquid):
      if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
         sqlCommand = """INSERT INTO squidlog(sessnum,  utc,event,squid)
                              VALUES(              %d,now(), '%s', '%s')
                      """ % (self.serverData['session'],event,cacheSquid)
         try:
            self.mySQLDatabase.insert(sqlCommand)
         except:
            self.logger.log(logging.ERROR,getLogMessage("ERROR: Unable to record cache action"))
         else:
            self.mySQLDatabase.commit()

         self.mySQLDatabase.disconnect()

      self.exit()


   def createDetachedSession(self):
      sessionOK = False
      if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
         sessionToken = self.__generateToken(32)
         timeout = 60*60*24
         sqlCommand = """INSERT INTO session(username,remoteip,exechost,dispnum,start,accesstime,appname,sessname,sesstoken,timeout)
                              SELECT         username,remoteip,    '%s',      0,now(),     now(),appname,sessname,     '%s',     %f
                                FROM session WHERE sessnum=%d
                      """ % ("submit",sessionToken,timeout,self.serverData['session'])
         try:
            self.mySQLDatabase.insert(sqlCommand)
         except:
            message = "ERROR: Unable to create detached session record for '%s'" % (self.clientData['userName'])
            serverMessage = {'messageType':'writeStderr',
                             'text':message + '\n'}
            self.clientListener.postJsonMessage(serverMessage)
            self.logger.log(logging.ERROR,getLogMessage(message))
         else:
            sqlCommand = """SELECT LAST_INSERT_ID()"""
            try:
               result = self.mySQLDatabase.select(sqlCommand)
            except:
               pass
            else:
               row = result[0]
               session = int(row[0])
               if session > 0:
                  self.serverData['session'] = session
                  self.logger.log(logging.INFO,getLogMessage("Created detached session %d" % (self.serverData['session'])))

                  self.serverData['createdSessions'].append(self.serverData['session'])
                  sessionOK = True

         self.mySQLDatabase.commit()
         self.mySQLDatabase.disconnect()

      return(sessionOK)


   def updateJobSession(self):
      jobSessionUpdated = True
      if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
         sqlCommand = """UPDATE job SET sessnum=%d
                          WHERE jobid='%d'
                      """ % (self.serverData['session'],self.serverData['jobId'])
         try:
            self.mySQLDatabase.update(sqlCommand)
         except:
            self.logger.log(logging.ERROR,getLogMessage("ERROR: updateJobSession"))
            jobSessionUpdated = False
         else:
            self.mySQLDatabase.commit()

         self.mySQLDatabase.disconnect()

      return(jobSessionUpdated)


   def insertJob(self):
      if self.serverData['operationMode'] & (self.commandParser.OPERATIONMODERUNDISTRIBUTORID | \
                                             self.commandParser.OPERATIONMODERUNCLIENTLOCAL | \
                                             self.commandParser.OPERATIONMODERUNCACHEHIT):
         if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
            #
            # Check that the superjob is a job that we own.
            #
            if self.serverData['superJobId'] != 0:
               sqlCommand = """SELECT jobid FROM job
                                WHERE jobid=%d AND sessnum=%d
                            """ % (self.serverData['superJobId'],self.serverData['session'])
               try:
                  result = self.mySQLDatabase.select(sqlCommand)
               except:
                  pass
               else:
                  if len(result) != 1:
                     message = "ERROR: Trying to claim superjob of %d." % (self.serverData['superJobId'])
                     self.logger.log(logging.ERROR,getLogMessage(message))
                     self.serverData['superJobId'] = 0
            self.serverData['jobToken'] = self.__generateToken(32)
            #
            # Insert the job into the job table.
            #
            # 0 - job completed
            # 1 - heartbeat expected
            # 2 - no heartbeat expected or cache hit
            # 3 - non monitored job
            # 4 - non monitored job complete
            #
            if   self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNCACHEHIT:
               active = 2
            elif self.commandParser.getOption('disableJobMonitoring'):
               active = 3
            else:
               if self.serverData['doHeartbeat']:
                  active = 1
               else:
                  active = 2

            if   self.serverData['squidInJob'] and 'RAPPTURE_CACHE_SQUID' in self.clientData['envs']:
               sqlCommand = """INSERT INTO job(sessnum,superjob,username,event,ncpus,venue,start,heartbeat,active,jobtoken,squid)
                                    VALUES(         %d,      %d,    '%s', '%s',   %d, '%s',now(),    now(),    %d,    '%s', '%s')
                            """ % (self.serverData['session'],self.serverData['superJobId'],self.clientData['userName'],
                                   self.serverData['event'],self.serverData['ncpus'],self.serverData['venue'],
                                   active,self.serverData['jobToken'],self.clientData['envs']['RAPPTURE_CACHE_SQUID'])
            elif self.serverData['squidInJob'] and 'SIM2L_CACHE_SQUID' in self.clientData['envs']:
               sqlCommand = """INSERT INTO job(sessnum,superjob,username,event,ncpus,venue,start,heartbeat,active,jobtoken,squid)
                                    VALUES(         %d,      %d,    '%s', '%s',   %d, '%s',now(),    now(),    %d,    '%s', '%s')
                            """ % (self.serverData['session'],self.serverData['superJobId'],self.clientData['userName'],
                                   self.serverData['event'],self.serverData['ncpus'],self.serverData['venue'],
                                   active,self.serverData['jobToken'],self.clientData['envs']['SIM2L_CACHE_SQUID'])
            else:
               sqlCommand = """INSERT INTO job(sessnum,superjob,username,event,ncpus,venue,start,heartbeat,active,jobtoken)
                                    VALUES(         %d,      %d,    '%s', '%s',   %d, '%s',now(),    now(),    %d,    '%s')
                            """ % (self.serverData['session'],self.serverData['superJobId'],self.clientData['userName'],
                                   self.serverData['event'],self.serverData['ncpus'],self.serverData['venue'],
                                   active,self.serverData['jobToken'])
            try:
               self.mySQLDatabase.insert(sqlCommand)
            except:
               self.logger.log(logging.ERROR,getLogMessage("ERROR: Unable to insert job."))
            else:
               sqlCommand = """SELECT LAST_INSERT_ID()"""
               try:
                  result = self.mySQLDatabase.select(sqlCommand)
               except:
                  pass
               else:
                  row = result[0]
                  self.serverData['jobId'] = int(row[0])
                  self.logger.log(logging.INFO,getLogMessage("Assigning jobId = %d:" % (self.serverData['jobId'])))
                  logSetJobId(self.serverData['jobId'])
                  self.serverData['localJobId'] = "%08d" % (self.serverData['jobId'])
                  self.serverData['dbJobEntryExists'] = True

                  sqlCommand = """UPDATE session SET accesstime=now()
                                   WHERE sessnum=%d
                               """ % (self.serverData['session'])
                  try:
                     self.mySQLDatabase.update(sqlCommand)
                  except:
                     message = "ERROR: Unable to update session accesstime for jobid %d." % (self.serverData['jobId'])
                     self.logger.log(logging.ERROR,getLogMessage(message))

            self.mySQLDatabase.commit()
            self.mySQLDatabase.disconnect()


   def updateJob(self):
      if self.serverData['operationMode'] & (self.commandParser.OPERATIONMODERUNDISTRIBUTORID | \
                                             self.commandParser.OPERATIONMODERUNCLIENTLOCAL):
         if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
            sqlCommand = """UPDATE job SET event='%s',ncpus=%d
                             WHERE jobid=%d
                         """ % (self.serverData['event'],self.serverData['ncpus'],self.serverData['jobId'])
            try:
               self.mySQLDatabase.update(sqlCommand)
            except:
               message = "ERROR: Unable to update job fields for jobid %d." % (self.serverData['jobId'])
               self.logger.log(logging.ERROR,getLogMessage(message))
            else:
               self.mySQLDatabase.commit()

            self.mySQLDatabase.disconnect()


   def invokeLocal(self):
      self.infosInfo           = InfosInfo(self.infosConfigurationPath)
      pegasusTemplateDirectory = self.infosInfo.getInfoPath('pegasusTemplates')

      pegasusRCPath    = None
      pegasusSitesPath = None
      if self.serverData['pegasusVersion']:
         pegasusName = 'pegasus_local.rc'
         pegasusPath = os.path.join(pegasusTemplateDirectory,"pegasus-%s" % (self.serverData['pegasusVersion']),pegasusName)
         if os.path.exists(pegasusPath):
            pegasusRCPath = pegasusPath
         else:
            self.logger.log(logging.ERROR,getLogMessage("%s does not exist" % (pegasusPath)))

         pegasusName = 'sites_local.xml'
         pegasusPath = os.path.join(pegasusTemplateDirectory,"pegasus-%s" % (self.serverData['pegasusVersion']),pegasusName)
         if os.path.exists(pegasusPath):
            pegasusSitesPath = pegasusPath
         else:
            pegasusName = 'sites_local.yml'
            pegasusPath = os.path.join(pegasusTemplateDirectory,"pegasus-%s" % (self.serverData['pegasusVersion']),pegasusName)
            if os.path.exists(pegasusPath):
               pegasusSitesPath = pegasusPath
            else:
               pegasusName = 'sites_local.[xy]ml'
               pegasusPath = os.path.join(pegasusTemplateDirectory,"pegasus-%s" % (self.serverData['pegasusVersion']),pegasusName)
               self.logger.log(logging.ERROR,getLogMessage("%s does not exist" % (pegasusPath)))

      if pegasusRCPath and pegasusSitesPath:
         try:
            fpPegasus = open(pegasusRCPath,'r')
            try:
               pegasusRCText = fpPegasus.readlines()
            except (IOError,OSError):
               self.logger.log(logging.ERROR,getLogMessage("%s could not be read" % (pegasusRCPath)))
            else:
               serverMessage = {'messageType':'pegasusRC',
                                'text':''.join(pegasusRCText)}
               self.clientListener.postJsonMessage(serverMessage)
            finally:
               fpPegasus.close()
         except (IOError,OSError):
            self.logger.log(logging.ERROR,getLogMessage("%s could not be opened" % (pegasusRCPath)))

         try:
            fpPegasus = open(pegasusSitesPath,'r')
            try:
               pegasusSitesText = fpPegasus.readlines()
            except (IOError,OSError):
               self.logger.log(logging.ERROR,getLogMessage("%s could not be read" % (pegasusSitesPath)))
            else:
               pegasusLocalSite = {}
               if   pegasusSitesPath.endswith('.xml'):
                  siteDoc = minidom.parse(pegasusSitesPath)
                  rootNode = siteDoc.documentElement
                  children = rootNode.getElementsByTagName("site")
                  for child in children:
                     handle = child.getAttribute('handle')
                     if handle == 'local':
                        try:
                           arch      = child.getAttribute('arch')
                        except:
                           pass
                        else:
                           pegasusLocalSite['arch'] = arch

                        try:
                           osFlavor  = child.getAttribute('os')
                        except:
                           pass
                        else:
                           pegasusLocalSite['osFlavor'] = osFlavor

                        try:
                           osVersion = child.getAttribute('osversion')
                        except:
                           pass
                        else:
                           pegasusLocalSite['osVersion'] = osVersion

                        try:
                           osRelease = child.getAttribute('osrelease')
                        except:
                           pass
                        else:
                           pegasusLocalSite['osRelease'] = osRelease
               elif pegasusSitesPath.endswith('.yml'):
                  with open(pegasusSitesPath,'r') as fpTemplate:
                     templateDictionary = yaml.load(fpTemplate,Loader=yaml.CSafeLoader)
                  for templateSite in templateDictionary['sites']:
                     handle = templateSite['name']
                     if handle == 'local':
                        try:
                           arch      = templateSite['arch']
                        except:
                           pass
                        else:
                           pegasusLocalSite['arch'] = arch

                        try:
                           osFlavor  = templateSite['os.type']
                        except:
                           pass
                        else:
                           pegasusLocalSite['osFlavor'] = osFlavor

                        try:
                           osVersion = templateSite['os.version']
                        except:
                           pass
                        else:
                           pegasusLocalSite['osVersion'] = osVersion

                        try:
                           osRelease = templateSite['os.release']
                        except:
                           pass
                        else:
                           pegasusLocalSite['osRelease'] = osRelease

               pegasusLocalSite['gridstartDistribute'] = False
               serverMessage = {'messageType':'pegasusSites',
                                'text':''.join(pegasusSitesText),
                                'localSite':pegasusLocalSite}
               self.clientListener.postJsonMessage(serverMessage)
            finally:
               fpPegasus.close()
         except (IOError,OSError):
            self.logger.log(logging.ERROR,getLogMessage("%s could not be opened" % (pegasusSitesPath)))
      else:
         if self.clientData['isParametric'] or self.clientData['isLocalPegasus']:
            msg = "Pegasus client version (%s), server version (%s) local configuration files are missing" % \
                                         (self.serverData['pegasusVersion'],self.clientData['pegasusVersion'])
            self.logger.log(logging.ERROR,getLogMessage(msg))
            serverMessage = {'messageType':'writeStderr',
                             'text':msg + '\n'}
            self.clientListener.postJsonMessage(serverMessage)

      self.submissionScriptsInfo = SubmissionScriptsInfo('Client',
                                                         submissionScriptRootPath=self.infosInfo.getInfoPath('submissionscripts'))
      submissionScripts = self.submissionScriptsInfo.getSubmissionScripts()
      serverMessage = {'messageType':'submissionScripts',
                       'submissionScripts':submissionScripts}
      self.clientListener.postJsonMessage(serverMessage)

      serverMessage = {'messageType':'serverReadyForIO'}
      self.clientListener.postJsonMessage(serverMessage)
      self.scheduleHeartbeat()


   def __getFileProperties(self,
                           filePath):
      fileProperties = {}
      fileProperties['filePath']    = os.path.abspath(filePath)
      fileProperties['mountPoint']  = ""
      fileProperties['mountDevice'] = ""
      if os.path.exists(fileProperties['filePath']):
         if len(self.serverData['mountPoints']) > 0:
            mountPoint = fileProperties['filePath']
            while mountPoint:
               if os.path.ismount(mountPoint):
                  fileProperties['mountPoint']  = mountPoint
                  fileProperties['mountDevice'] = self.serverData['mountPoints'][mountPoint]['device']
                  break
               else:
                  if mountPoint != os.sep:
                     mountPoint = os.path.dirname(mountPoint)
                  else:
                     break

         if not os.path.isdir(fileProperties['filePath']):
            md5Hash = hashlib.md5()
            with open(fileProperties['filePath'],'rb') as f:
               # Read and update hash in chunks of 4K
               for block in iter(lambda: f.read(4096),b""):
                  md5Hash.update(block)
               fileProperties['checksum'] = md5Hash.hexdigest()
         else:
            fileProperties['checksum'] = ""

         fileProperties['fileSize'] = os.lstat(fileProperties['filePath']).st_size
         fileProperties['inode']    = os.lstat(fileProperties['filePath']).st_ino
      else:
         fileProperties['checksum'] = ""
         fileProperties['fileSize'] = 0
         fileProperties['inode']    = 0

      return fileProperties


   def __isClientFileShared(self,
                            clientFileProperties):
      clientFileShared = True
      serverFile = clientFileProperties['filePath']
      matchInodeDevice = True
      for clientMountPoint in self.configData['mountPointTranslations']:
         if clientFileProperties['filePath'].startswith(clientMountPoint): 
            serverMountPoint = self.configData['mountPointTranslations'][clientMountPoint]['serverMountPoint']
            matchInodeDevice = self.configData['mountPointTranslations'][clientMountPoint]['matchInode']
            serverFile = os.path.join(serverMountPoint,clientFileProperties['filePath'][len(clientMountPoint):])
            break
#     self.logger.log(logging.DEBUG,getLogMessage("isClientFileShared: clientFile = %s" % (clientFileProperties['filePath'])))
#     self.logger.log(logging.DEBUG,getLogMessage("isClientFileShared: serverFile = %s" % (serverFile)))

      if os.path.exists(serverFile):
         fileProperties = self.__getFileProperties(serverFile)
#        self.logger.log(logging.DEBUG,getLogMessage("isClientFileShared: clientInode = %d serverInode = %d" % \
#                                                      (clientFileProperties['inode'],fileProperties['inode'])))
         if matchInodeDevice:
            if fileProperties['inode'] != clientFileProperties['inode']:
               if clientFileProperties['mountDevice'] and len(self.serverData['mountPoints']) > 0:
                  mountDevice = None
                  mountPoint = serverFile
                  while mountPoint:
                     if os.path.ismount(mountPoint):
                        mountDevice = self.serverData['mountPoints'][mountPoint]['device']
                        break
                     else:
                        if mountPoint != os.sep:
                           mountPoint = os.path.dirname(mountPoint)
                        else:
                           break
#                 self.logger.log(logging.DEBUG,getLogMessage("isClientFileShared: clientMountDevice = %s" % \
#                                                                      (clientFileProperties['mountDevice'])))
#                 self.logger.log(logging.DEBUG,getLogMessage("isClientFileShared: serverMountDevice = %s" % \
#                                                                                              (mountDevice)))
                  if mountDevice != clientFileProperties['mountDevice']:
                     clientFileShared = False
               else:
                  clientFileShared = False
         else:
            serverFileProperties = self.__getFileProperties(serverFile)
            if serverFileProperties['checksum'] != clientFileProperties['checksum'] or \
               serverFileProperties['fileSize'] != clientFileProperties['fileSize']:
               clientFileShared = False
      else:
         clientFileShared = False

      if clientFileShared:
         clientFileProperties['serverFilePath'] = serverFile

      return clientFileShared 


   def setupWorkDirectory(self):
      workDirectorySetup = True
      if self.__isClientFileShared(self.clientData['signaturePath']):
         self.serverData['workDirectory'] = os.path.dirname(self.clientData['signaturePath']['serverFilePath'])
      else:
         sessionDirectory = os.path.join(self.serverData['homeDirectory'],'data','sessions',str(self.serverData['session']))
         if not os.path.isdir(sessionDirectory):
            if os.path.islink(sessionDirectory):
# session directory is a broken link.  Happens when using local disk on execution hosts.
               sessionsDirectory = os.path.join(self.serverData['homeDirectory'],'data','sessions')
               if not os.path.isdir(sessionsDirectory):
                  try:
                     os.makedirs(sessionsDirectory)
                  except OSError as e:
                     message = "setupWorkDirectory: %s - %s" % (e.args[1],sessionsDirectory)
                     self.logger.log(logging.ERROR,getLogMessage(message))

                     serverMessage = {'messageType':'writeStderr',
                                      'text':"%s\n" % (message)}
                     self.clientListener.postJsonMessage(serverMessage)
                     workDirectorySetup = False

               if workDirectorySetup:
                  try:
                     sessionDirectory = tempfile.mkdtemp(prefix="%d-submit_" % (self.serverData['session']),
                                                         dir=sessionsDirectory)
                  except OSError as e:
                     message = "setupWorkDirectory: %s - %s" % (e.args[1],sessionDirectory)
                     self.logger.log(logging.ERROR,getLogMessage(message))

                     serverMessage = {'messageType':'writeStderr',
                                      'text':"%s\n" % (message)}
                     self.clientListener.postJsonMessage(serverMessage)
                     workDirectorySetup = False
                  else:
                     self.serverData['sessionDirectory'] = sessionDirectory
            else:
               try:
                  os.makedirs(sessionDirectory)
               except OSError as e:
                  message = "setupWorkDirectory: %s - %s" % (e.args[1],sessionDirectory)
                  self.logger.log(logging.ERROR,getLogMessage(message))

                  serverMessage = {'messageType':'writeStderr',
                                   'text':"%s\n" % (message)}
                  self.clientListener.postJsonMessage(serverMessage)
                  workDirectorySetup = False
               else:
                  self.serverData['sessionDirectory'] = sessionDirectory
         if workDirectorySetup:
            try:
               self.serverData['workDirectory'] = tempfile.mkdtemp(prefix='run_',dir=sessionDirectory)
            except OSError as e:
               message = "setupWorkDirectory: %s - %s" % (e.args[1],sessionDirectory)
               self.logger.log(logging.ERROR,getLogMessage(message))

               serverMessage = {'messageType':'writeStderr',
                                'text':"%s\n" % (message)}
               self.clientListener.postJsonMessage(serverMessage)
               workDirectorySetup = False
            else:
               self.serverData['workDirectoryShared'] = False

      if workDirectorySetup:
         os.chdir(self.serverData['workDirectory'])

      return(workDirectorySetup)


   def unsetupWorkDirectory(self):
      if not self.serverData['workDirectoryShared']:
         os.chdir(os.path.join(os.sep,'tmp'))
         if os.path.isdir(self.serverData['workDirectory']):
            shutil.rmtree(self.serverData['workDirectory'],True)
         sessionDirectory = self.serverData['sessionDirectory']
         if sessionDirectory:
            if os.path.isdir(sessionDirectory):
               try:
# remove empty directory
                  os.rmdir(sessionDirectory)
               except:
# directory is not empty so rename it
                  expiredSessionDirectory = sessionDirectory + "-expired"
                  try:
                     os.rename(sessionDirectory,expiredSessionDirectory)
                  except:
                     pass


   def filterEnvironment(self):
      self.serverData['envs'] = {}
      if   self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNDISTRIBUTORID:
         for env in self.clientData['envs']:
            if not env[0].isalpha():
               continue
            if env == "SUBMIT_JOB":
               self.serverData['envs'][env] = self.clientData['envs'][env]
               continue
            if env == "SUBMIT_APPLICATION_REVISION":
               self.serverData['envs'][env] = self.clientData['envs'][env]
               continue
            if env.endswith("_CHOICE"):
               self.serverData['envs'][env] = self.clientData['envs'][env]
               continue
            if env == "WS_JOBID":
               self.serverData['envs'][env] = self.clientData['envs'][env]
               continue

         if self.clientData['isClientTTY']:
            self.serverData['envs']['SUBMIT_ISCLIENTTTY'] = '1'

         if self.clientData['workDirectory']:
            self.serverData['envs']['CLIENT_WORK_DIRECTORY'] = self.clientData['workDirectory']
         if self.serverData['clientTransferPath']:
            self.serverData['envs']['CLIENT_TRANSFER_PATH'] = self.serverData['clientTransferPath']
         if self.serverData['pegasusVersion']:
            self.serverData['envs']['PEGASUS_VERSION'] = self.serverData['pegasusVersion']
         if self.serverData['pegasusHome']:
            self.serverData['envs']['PEGASUS_HOME'] = self.serverData['pegasusHome']
         if self.serverData['pegasusPythonPath']:
            self.serverData['envs']['PYTHONPATH'] = self.serverData['pegasusPythonPath']
         if self.configData['useSetup']:
            self.serverData['envs']['USE_SETUP_SCRIPT'] = self.configData['useSetup']
         if self.serverData['email']:
            self.serverData['envs']['USER_EMAIL'] = self.serverData['email']
         if self.configData['emailFrom']:
            self.serverData['envs']['HUB_EMAIL_FROM'] = self.configData['emailFrom']
         if self.clientData['cacheHosts']:
            self.serverData['envs']['CACHE_HOSTS'] = self.clientData['cacheHosts']
         if self.clientData['sudoUserName']:
            self.serverData['envs']['CLIENT_SUDO_USER'] = self.clientData['sudoUserName']
      elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNVENUESTATUS:
         if self.serverData['pegasusVersion']:
            self.serverData['envs']['PEGASUS_VERSION'] = self.serverData['pegasusVersion']

      self.serverData['envs']['DOUBLE_DASH_TERMINATOR'] = str(self.clientData['doubleDashTerminator'])
      self.serverData['envs']['HOME']                   = self.serverData['homeDirectory']
      self.serverData['envs']['USER']                   = self.clientData['userName']
      self.serverData['envs']['LOGNAME']                = self.clientData['userName']
      self.serverData['envs']['SUBMITTER_CLASS']        = str(self.serverData['submitterClass'])
      self.serverData['envs']['SESSION']                = str(self.serverData['session'])
      self.serverData['envs']['PATH']                   = "/usr/bin:/bin"


   def mapArguments(self):
      if   self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNSTATUS:
         jobStatusPath = os.path.join(self.serverData['jobStatusDirectory'],
                                      self.serverData['jobStatusScript'])
         self.serverData['args'][0] = jobStatusPath
      elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNKILL:
         jobKillPath = os.path.join(self.serverData['jobKillDirectory'],
                                    self.serverData['jobKillScript'])
         self.serverData['args'][0] = jobKillPath
      elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNVENUESTATUS:
         venueProbePath = os.path.join(self.serverData['venueProbeDirectory'],
                                       self.serverData['venueProbeScript'])
         self.serverData['args'][0] = venueProbePath
      elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNHARVESTID:
         harvesterPath = os.path.join(self.serverData['harvesterDirectory'],
                                      self.serverData['harvesterScript'])
         self.serverData['args'][0] = harvesterPath
      else:
         distributorPath = os.path.join(self.serverData['distributorDirectory'],
                                        self.serverData['distributorScript'])
         self.serverData['args'][0] = distributorPath
         if self.serverData['jobId'] > 0:
            self.serverData['args'].insert(1, "--jobid")
            self.serverData['args'].insert(2, str(self.serverData['jobId']))


   def invokeRunStatus(self):
      stdinPipe  = os.pipe()
      stdoutPipe = os.pipe()
      stderrPipe = os.pipe()
      self.serverData['childPid'] = os.fork()
      if self.serverData['childPid'] == 0:
         self.clientListener.closeConnection()
         logging.shutdown()

         os.setsid()
         os.dup2(stdinPipe[0],0)
         os.dup2(stdoutPipe[1],1)
         os.dup2(stderrPipe[1],2)
         os.close(stdinPipe[1])
         os.close(stdoutPipe[0])
         os.close(stderrPipe[0])

         try:
            os.execve(self.serverData['args'][0],self.serverData['args'],self.serverData['envs'])
         except OSError as e:
            self.logger.log(logging.ERROR,getLogMessage("invokeRunStatus: exec %s failed" % (self.serverData['args'][0])))
            os.write(2,"Can not invoke runstatus.\n".encode('utf-8'))
            os.write(2,(e.args[1]+'\n').encode('utf-8'))
            sys.exit(e.args[0])
      else:
         fpStdinPipe  = os.fdopen(stdinPipe[1],'wb',0)
         fpStdoutPipe = os.fdopen(stdoutPipe[0],'rb',0)
         fpStderrPipe = os.fdopen(stderrPipe[0],'rb',0)
         os.close(stdinPipe[0])
         os.close(stdoutPipe[1])
         os.close(stderrPipe[1])
         self.serverData['outputObjects'][fpStdinPipe]           = {}
         self.serverData['outputObjects'][fpStdinPipe]['path']   = '#STDIN#'
         self.serverData['outputObjects'][fpStdinPipe]['buffer'] = b""
         self.serverData['inputObjects'][fpStdoutPipe] = '#STDOUT#'
         self.serverData['inputObjects'][fpStderrPipe] = '#STDERR#'

         # At this point, the I/O is ready to receive things from the client.
         # Inform the client.
         serverMessage = {'messageType':'serverReadyForIO'}
         self.clientListener.postJsonMessage(serverMessage)


   def invokeRunKill(self):
      stdinPipe  = os.pipe()
      stdoutPipe = os.pipe()
      stderrPipe = os.pipe()
      self.serverData['childPid'] = os.fork()
      if self.serverData['childPid'] == 0:
         self.clientListener.closeConnection()
         logging.shutdown()

         os.setsid()
         os.dup2(stdinPipe[0],0)
         os.dup2(stdoutPipe[1],1)
         os.dup2(stderrPipe[1],2)
         os.close(stdinPipe[1])
         os.close(stdoutPipe[0])
         os.close(stderrPipe[0])

         try:
            os.execve(self.serverData['args'][0],self.serverData['args'],self.serverData['envs'])
         except OSError as e:
            os.write(2,"Can not invoke runkill.\n".encode('utf-8'))
            os.write(2,(e.args[1]+'\n').encode('utf-8'))
            sys.exit(e.args[0])
      else:
         fpStdinPipe  = os.fdopen(stdinPipe[1],'wb',0)
         fpStdoutPipe = os.fdopen(stdoutPipe[0],'rb',0)
         fpStderrPipe = os.fdopen(stderrPipe[0],'rb',0)
         os.close(stdinPipe[0])
         os.close(stdoutPipe[1])
         os.close(stderrPipe[1])
         self.serverData['outputObjects'][fpStdinPipe]           = {}
         self.serverData['outputObjects'][fpStdinPipe]['path']   = '#STDIN#'
         self.serverData['outputObjects'][fpStdinPipe]['buffer'] = b""
         self.serverData['inputObjects'][fpStdoutPipe] = '#STDOUT#'
         self.serverData['inputObjects'][fpStderrPipe] = '#STDERR#'

         # At this point, the I/O is ready to receive things from the client.
         # Inform the client.
         serverMessage = {'messageType':'serverReadyForIO'}
         self.clientListener.postJsonMessage(serverMessage)


   def invokeVenueStatus(self):
      stdinPipe  = os.pipe()
      stdoutPipe = os.pipe()
      stderrPipe = os.pipe()
      self.serverData['childPid'] = os.fork()
      if self.serverData['childPid'] == 0:
         self.clientListener.closeConnection()
         logging.shutdown()

         os.setsid()
         os.dup2(stdinPipe[0],0)
         os.dup2(stdoutPipe[1],1)
         os.dup2(stderrPipe[1],2)
         os.close(stdinPipe[1])
         os.close(stdoutPipe[0])
         os.close(stderrPipe[0])

         try:
            os.execve(self.serverData['args'][0],self.serverData['args'],self.serverData['envs'])
         except OSError as e:
            os.write(2,"Can not invoke venue status.\n".encode('utf-8'))
            os.write(2,(e.args[1]+'\n').encode('utf-8'))
            sys.exit(e.args[0])
      else:
         fpStdinPipe  = os.fdopen(stdinPipe[1],'wb',0)
         fpStdoutPipe = os.fdopen(stdoutPipe[0],'rb',0)
         fpStderrPipe = os.fdopen(stderrPipe[0],'rb',0)
         os.close(stdinPipe[0])
         os.close(stdoutPipe[1])
         os.close(stderrPipe[1])
         self.serverData['outputObjects'][fpStdinPipe]           = {}
         self.serverData['outputObjects'][fpStdinPipe]['path']   = '#STDIN#'
         self.serverData['outputObjects'][fpStdinPipe]['buffer'] = b""
         self.serverData['inputObjects'][fpStdoutPipe] = '#STDOUT#'
         self.serverData['inputObjects'][fpStderrPipe] = '#STDERR#'

         # At this point, the I/O is ready to receive things from the client.
         # Inform the client.
         serverMessage = {'messageType':'serverReadyForIO'}
         self.clientListener.postJsonMessage(serverMessage)


   def invokeHarvester(self):
      stdinPipe  = os.pipe()
      stdoutPipe = os.pipe()
      stderrPipe = os.pipe()
      metricPipe = os.pipe()
      self.serverData['childPid'] = os.fork()
      if self.serverData['childPid'] == 0:
         self.clientListener.closeConnection()
         logging.shutdown()

         os.setsid()
         os.dup2(stdinPipe[0],0)
         os.dup2(stdoutPipe[1],1)
         os.dup2(stderrPipe[1],2)
         os.dup2(metricPipe[1],3)
         os.close(stdinPipe[1])
         os.close(stdoutPipe[0])
         os.close(stderrPipe[0])
         os.close(metricPipe[0])
         os.close(stdinPipe[0])
         os.close(stdoutPipe[1])
         os.close(stderrPipe[1])
         os.close(metricPipe[1])

         try:
            os.execve(self.serverData['args'][0],self.serverData['args'],self.serverData['envs'])
         except OSError as e:
            os.write(2,"Can not invoke harvester.\n".encode('utf-8'))
            os.write(2,(e.args[1]+'\n').encode('utf-8'))
            sys.exit(e.args[0])
      else:
         fpStdinPipe  = os.fdopen(stdinPipe[1],'wb',0)
         fpStdoutPipe = os.fdopen(stdoutPipe[0],'rb',0)
         fpStderrPipe = os.fdopen(stderrPipe[0],'rb',0)
         fpMetricPipe = os.fdopen(metricPipe[0],'rb',0)
         os.close(stdinPipe[0])
         os.close(stdoutPipe[1])
         os.close(stderrPipe[1])
         os.close(metricPipe[1])
         self.serverData['outputObjects'][fpStdinPipe]           = {}
         self.serverData['outputObjects'][fpStdinPipe]['path']   = '#STDIN#'
         self.serverData['outputObjects'][fpStdinPipe]['buffer'] = b""
         self.serverData['inputObjects'][fpStdoutPipe] = '#STDOUT#'
         self.serverData['inputObjects'][fpStderrPipe] = '#STDERR#'
         self.serverData['inputObjects'][fpMetricPipe] = '#METRICS#'

         # At this point, the I/O is ready to receive things from the client.
         # Inform the client.
         serverMessage = {'messageType':'serverReadyForIO'}
         self.clientListener.postJsonMessage(serverMessage)

         self.scheduleHeartbeat()


   def invokeDistributor(self):
      if not self.serverData['workDirectoryShared']:
         try:
            fpMarker = open('.__fileTimeMarkerSubmit','w')
         except (IOError,OSError):
            pass
         finally:
            fpMarker.close()
         time.sleep(1)

      stdinPipe  = os.pipe()
      stdoutPipe = os.pipe()
      stderrPipe = os.pipe()
      metricPipe = os.pipe()
      self.serverData['childPid'] = os.fork()
      if self.serverData['childPid'] == 0:
         self.clientListener.closeConnection()
         logging.shutdown()

         os.setsid()
         os.dup2(stdinPipe[0],0)
         os.dup2(stdoutPipe[1],1)
         os.dup2(stderrPipe[1],2)
         os.dup2(metricPipe[1],3)
         os.close(stdinPipe[1])
         os.close(stdoutPipe[0])
         os.close(stderrPipe[0])
         os.close(metricPipe[0])
         os.close(stdinPipe[0])
         os.close(stdoutPipe[1])
         os.close(stderrPipe[1])
         os.close(metricPipe[1])

         try:
            os.execve(self.serverData['args'][0],self.serverData['args'],self.serverData['envs'])
         except OSError as e:
            os.write(2,"Can not invoke distributor.\n".encode('utf-8'))
            os.write(2,(e.args[1]+'\n').encode('utf-8'))
            sys.exit(e.args[0])
      else:
         fpStdinPipe  = os.fdopen(stdinPipe[1],'wb',0)
         fpStdoutPipe = os.fdopen(stdoutPipe[0],'rb',0)
         fpStderrPipe = os.fdopen(stderrPipe[0],'rb',0)
         fpMetricPipe = os.fdopen(metricPipe[0],'rb',0)
         os.close(stdinPipe[0])
         os.close(stdoutPipe[1])
         os.close(stderrPipe[1])
         os.close(metricPipe[1])
         self.serverData['outputObjects'][fpStdinPipe]           = {}
         self.serverData['outputObjects'][fpStdinPipe]['path']   = '#STDIN#'
         self.serverData['outputObjects'][fpStdinPipe]['buffer'] = b""
         self.serverData['inputObjects'][fpStdoutPipe] = '#STDOUT#'
         self.serverData['inputObjects'][fpStderrPipe] = '#STDERR#'
         self.serverData['inputObjects'][fpMetricPipe] = '#METRICS#'

         # At this point, the I/O is ready to receive things from the client.
         # Inform the client.
         serverMessage = {'messageType':'serverReadyForIO'}
         self.clientListener.postJsonMessage(serverMessage)

         if self.clientData['isParametric'] and not self.serverData['detach']:
            self.scheduleJobScan()
         else:
            self.scheduleHeartbeat()


   def startCommand(self):
      def __addConfigurationVariables(configurationVariables):
         for configurationVariable in configurationVariables:
            try:
               configurationValue = os.getenv(configurationVariable)
            except:
               pass
            else:
               if configurationValue is not None:
                  self.serverData['envs'][configurationVariable] = configurationValue

      self.scheduleHeartbeat()

      self.setUserGroupIds()
      self.filterEnvironment()
      self.mapArguments()
      os.umask(self.clientData['umask'])

      if   self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNSTATUS:
         configurationVariables = ["JOBSTATUSLOGLEVEL",
                                   "DAEMONSDIRECTORY",
                                   "DAEMONSCONFIGURATIONFILE"]
         __addConfigurationVariables(configurationVariables)
         self.invokeRunStatus()
      elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNKILL:
         configurationVariables = ["JOBKILLLOGLEVEL",
                                   "DAEMONSDIRECTORY",
                                   "DAEMONSCONFIGURATIONFILE"]
         __addConfigurationVariables(configurationVariables)
         self.invokeRunKill()
      elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNVENUESTATUS:
         configurationVariables = ["VENUESTATUSLOGLEVEL",
                                   "CONFIGURATIONDIRECTORY",
                                   "DISTRIBUTORCONFIGURATIONFILE",
                                   "DAEMONSDIRECTORY",
                                   "DAEMONSCONFIGURATIONFILE",
                                   "INFOSDIRECTORY",
                                   "INFOSCONFIGURATIONFILE"]
         __addConfigurationVariables(configurationVariables)
         self.invokeVenueStatus()
      elif self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNHARVESTID:
         configurationVariables = ["DISTRIBUTORLOGLEVEL",
                                   "DISTRIBUTORSYSLOGLOCAL",
                                   "CONFIGURATIONDIRECTORY",
                                   "DISTRIBUTORCONFIGURATIONFILE",
                                   "DAEMONSDIRECTORY",
                                   "DAEMONSCONFIGURATIONFILE",
                                   "INFOSDIRECTORY",
                                   "INFOSCONFIGURATIONFILE"]
         __addConfigurationVariables(configurationVariables)
         self.invokeHarvester()
      else:
         configurationVariables = ["DISTRIBUTORLOGLEVEL",
                                   "DISTRIBUTORSYSLOGLOCAL",
                                   "CONFIGURATIONDIRECTORY",
                                   "DISTRIBUTORCONFIGURATIONFILE",
                                   "DAEMONSDIRECTORY",
                                   "DAEMONSCONFIGURATIONFILE",
                                   "INFOSDIRECTORY",
                                   "INFOSCONFIGURATIONFILE"]
         __addConfigurationVariables(configurationVariables)
         self.invokeDistributor()
         self.serverData['detach'] = False


   def launchJob(self):
      self.updateJob()
      if self.serverData['operationMode'] & (self.commandParser.OPERATIONMODERUNDISTRIBUTORID | \
                                             self.commandParser.OPERATIONMODERUNCLIENTLOCAL):
         serverMessage = {'messageType':'jobId',
                          'jobId':self.serverData['jobId']}
         self.clientListener.postJsonMessage(serverMessage)
         serverMessage = {'messageType':'jobToken',
                          'token':self.serverData['jobToken']}
         self.clientListener.postJsonMessage(serverMessage)
         serverMessage = {'messageType':'submitterClass',
                          'class':self.serverData['submitterClass']}
         self.clientListener.postJsonMessage(serverMessage)
         serverMessage = {'messageType':'event',
                          'text':self.serverData['event']}
         self.clientListener.postJsonMessage(serverMessage)

      if self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNCLIENTLOCAL:
         serverMessage = {'messageType':'createdSessions',
                          'createdSessions':self.serverData['createdSessions']}
         self.clientListener.postJsonMessage(serverMessage)
         serverMessage = {'messageType':'operationMode',
                          'operationMode':self.serverData['operationMode']}
         self.clientListener.postJsonMessage(serverMessage)
         if self.serverData['doHeartbeat']:
            serverMessage = {'messageType':'heartbeatInterval',
                             'interval':self.configData['heartbeatInterval']}
            self.clientListener.postJsonMessage(serverMessage)
         self.invokeLocal()
      else:
         self.startCommand()


   def updateJobMetrics(self,
                        metricsRecord):
      if not self.serverData['metricsRecorded']:
         if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
            status                      = 0
            redundantJobKillStatus      = 1 << 7 | signal.SIGUSR1
            waittime                    = 0.
            cputime                     = 0.
            realtime                    = 0.
            ncpus                       = 0
            program                     = self.serverData['event']
            generated                   = ""
            allowedGeneratedPattern     = re.compile("^[0-9a-zA-z._-]*$")
            executionMode               = 'block'
            allowedExecutionModePattern = re.compile("^[a-z]*$")

            metricsRecord = metricsRecord.strip()
            metrics = metricsRecord.split()
            for metric in metrics:
               try:
                  key,value = metric.split('=')
                  if   key == 'status':
                     status = int(value)
                     if status != redundantJobKillStatus:
                        if   self.serverData['exitCode'] == 2:
                           self.serverData['exitCode'] = status
                        elif self.serverData['exitCode'] == 0:
                           if status != 0:
                              self.serverData['exitCode'] = status
                  elif key == 'cputime':
                     cputime = float(value)
                  elif key == 'realtime':
                     realtime = float(value)
                  elif key == 'waittime':
                     waittime = float(value)
                  elif key == 'ncpus':
                     ncpus = int(value)
                  elif key == 'venue':
                     self.serverData['venue'] = value
                  elif key == 'event':
                     program = value
                  elif key == 'generated':
                     if allowedGeneratedPattern.match(value):
                        generated = value
                  elif key == 'executionMode':
                     if allowedExecutionModePattern.match(value):
                        executionMode = value
                  else:
                     self.logger.log(logging.ERROR,getLogMessage("Unknown status item: '%s'" % (metric)))
               except:
                  self.logger.log(logging.ERROR,getLogMessage("Erroneous status item: '%s'" % (metric)))

            if executionMode == 'block':
               metricsMessage = " venue=%s status=%d cpu=%f real=%f wait=%f ncpus=%d" % (self.serverData['venue'],
                                                                                         status,cputime,realtime,waittime,ncpus)
               self.logger.log(logging.INFO,getLogMessage("Job Status:" + metricsMessage))
               if self.clientData['reportMetrics'] and self.serverData['venue'] != "":
                  self.logger.log(logging.INFO,getLogMessage("Sending requested metrics."))
                  msg = "=SUBMIT-METRICS=>"
                  msg += " job=%s" % (self.serverData['localJobId'])
                  msg += metricsMessage + '\n'
                  serverMessage = {'messageType':'writeStderr',
                                   'text':msg}
                  self.clientListener.postJsonMessage(serverMessage)

               if waittime > 0:
                  sqlCommand = """INSERT INTO joblog(sessnum,  job,superjob,event,start,walltime,venue)
                                       SELECT        sessnum,jobid,superjob, '%s',start,      %f, '%s'
                                         FROM job WHERE jobid=%d
                               """ % ("[waiting]",waittime,self.serverData['venue'],self.serverData['jobId'])
                  try:
                     self.mySQLDatabase.insert(sqlCommand)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("ERROR: Unable to create wait time record. (%f)" % (waittime)))

               if generated != "" and generated != "human":
                  try:
#                 combine instanceId with generated
                     uniqueGenerated = ':'.join((self.serverData['venue'].split(':')[0],generated))
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("ERROR: Unable to create generated record. (%s)" % \
                                                                                                (uniqueGenerated)))
                  else:
                     sqlCommand = """INSERT INTO joblog(sessnum,  job,superjob,event,start,venue)
                                          SELECT        sessnum,jobid,superjob, '%s',start, '%s'
                                            FROM job WHERE jobid=%d
                                  """ % ("[generated]",uniqueGenerated,self.serverData['jobId'])
                     try:
                        self.mySQLDatabase.insert(sqlCommand)
                     except:
                        self.logger.log(logging.ERROR,getLogMessage("ERROR: Unable to create generated record. (%s)" % \
                                                                                                   (uniqueGenerated)))

               if self.serverData['squidInJob']:
                  sqlCommand = """INSERT INTO joblog(sessnum,  job,superjob,event,start,walltime,cputime,ncpus,status,venue,squid)
                                       SELECT        sessnum,jobid,superjob, '%s',start,      %f,     %f,   %d,    %d, '%s',squid
                                         FROM job WHERE jobid=%d
                               """ % (program,realtime,cputime,ncpus,status,self.serverData['venue'],self.serverData['jobId'])
               else:
                  sqlCommand = """INSERT INTO joblog(sessnum,  job,superjob,event,start,walltime,cputime,ncpus,status,venue)
                                       SELECT        sessnum,jobid,superjob, '%s',start,      %f,     %f,   %d,    %d, '%s'
                                         FROM job WHERE jobid=%d
                               """ % (program,realtime,cputime,ncpus,status,self.serverData['venue'],self.serverData['jobId'])
               try:
                  self.mySQLDatabase.insert(sqlCommand)
               except:
                  self.logger.log(logging.ERROR,getLogMessage("ERROR: Unable to copy job %d to joblog" % \
                                                                            (self.serverData['jobId'])))

               self.serverData['metricsRecorded'] = True

               self.mySQLDatabase.commit()
            else:
               self.serverData['doFinalizeOperations'] = False

            self.mySQLDatabase.disconnect()


   def finalizeJob(self):
      self.logger.log(logging.DEBUG,getLogMessage("finalizeJob dbJobEntryExists = %s" % \
                                                (self.serverData['dbJobEntryExists'])))
      self.logger.log(logging.DEBUG,getLogMessage("finalizeJob doFinalizeOperations = %s" % \
                                                (self.serverData['doFinalizeOperations'])))
      if self.serverData['dbJobEntryExists'] and self.serverData['doFinalizeOperations']:
         if self.serverData['jobId'] > 0:
            if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
               sqlCommand = """SELECT sessnum,active FROM job
                                WHERE jobid=%d
                            """ % (self.serverData['jobId'])
               try:
                  result = self.mySQLDatabase.select(sqlCommand)
               except:
                  pass
               else:
                  updatedActive = 0
                  if len(result) == 0:
                     message = "ERROR: Unable to find session for job %d." % (self.serverData['jobId'])
                     self.logger.log(logging.ERROR,getLogMessage(message))
                  else:
                     row = result[0]
                     session = row[0]
                     active  = row[1]
                     if active == 3:
                        updatedActive = 4

                     sqlCommand = """UPDATE session SET accesstime=now()
                                      WHERE sessnum=%d
                                  """ % (session)
                     try:
                        self.mySQLDatabase.update(sqlCommand)
                     except:
                        self.logger.log(logging.ERROR,getLogMessage("ERROR: Unable to update session accesstime for job."))
                  self.logger.log(logging.DEBUG,getLogMessage("finalizeJob accesstime set"))

                  sqlCommand = """UPDATE job SET active=%d
                                   WHERE jobid=%d
                               """ % (updatedActive,self.serverData['jobId'])
                  try:
                     self.mySQLDatabase.update(sqlCommand)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("Unable to deactivate job."))
                  self.logger.log(logging.DEBUG,getLogMessage("finalizeJob job deactivated"))

                  #
                  # Clear inactive jobs beyond the load horizon.
                  #
                  sqlCommand = """DELETE FROM job
                                   WHERE active=0 AND TIMESTAMPDIFF(second,job.start,now()) > %d
                               """ % (self.configData['loadHorizon'])
                  try:
                     nDeleted = self.mySQLDatabase.delete(sqlCommand)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("Unable to clear old inactive jobs."))
                  self.logger.log(logging.DEBUG,getLogMessage("finalizeJob inactive jobs cleared"))

                  #
                  # Mark as noheartbeat jobs that have start time = heartbeat time.
                  #
                  sqlCommand = """UPDATE job SET active=2
                                   WHERE active=1 AND TIMESTAMPDIFF(second,job.start,now()) > %d AND
                                                      TIMESTAMPDIFF(second,job.start,job.heartbeat) < 2
                               """ % (self.configData['heartbeatInterval']*2)
                  try:
                     self.mySQLDatabase.update(sqlCommand)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("Unable to set noHeartbeat job entries."))
                  self.logger.log(logging.DEBUG,getLogMessage("finalizeJob mark noheartbeat"))

                  #
                  # Mark inactive any jobs that have not had a recent heartbeat.
                  #
                  sqlCommand = """UPDATE job SET active=0
                                   WHERE active=1 AND TIMESTAMPDIFF(second,job.heartbeat,now()) > %d
                               """ % (self.configData['heartbeatInterval']*3)
                  try:
                     self.mySQLDatabase.update(sqlCommand)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("Unable to deactivate moribund job entries."))
                  self.logger.log(logging.DEBUG,getLogMessage("finalizeJob mark inactive"))

                  #
                  # Mark inactive any jobs that have no heartbeat and have been moved to joblog.
                  #
                  sqlCommand = """UPDATE job INNER JOIN joblog ON jobid=job SET job.active=0
                                   WHERE job.active=2 AND TIMESTAMPDIFF(second,job.start,now()) > %d
                               """ % (self.configData['loadHorizon'])
                  try:
                     self.mySQLDatabase.update(sqlCommand)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("Unable to deactivate recorded non-heartbeat job entries."))
                  self.logger.log(logging.DEBUG,getLogMessage("finalizeJob mark noheartbeat inactive"))

                  #
                  # Copy any jobs that have no heartbeat and whose session has been terminated from job to joblog
                  #
                  sqlCommand = """SELECT job.jobid
                                    FROM job LEFT JOIN session ON job.sessnum=session.sessnum
                                            INNER JOIN sessionlog ON job.sessnum=sessionlog.sessnum
                                             LEFT JOIN joblog ON job.jobid=joblog.job
                                   WHERE (session.sessnum IS NULL AND
                                               joblog.job IS NULL AND job.active=2 AND TIMESTAMPDIFF(second,job.start,now()) > %d)
                               """ % (self.configData['loadHorizon'])
                  try:
                     resultSelect = self.mySQLDatabase.select(sqlCommand)
                  except:
                     pass
                  else:
                     self.logger.log(logging.DEBUG,getLogMessage("finalizeJob get jobid list"))
                     if len(resultSelect) > 0:
                        realtime = 0.
                        cputime  = 0.
                        ncpus    = 1
                        status   = 17
                        for row in resultSelect:
                           jobId = row[0]

                           if self.serverData['squidInJob']:
                              sqlCommand = """INSERT INTO joblog(sessnum,  job,superjob,event,start,walltime,cputime,ncpus,status,venue,squid)
                                                   SELECT        sessnum,jobid,superjob,event,start,      %f,     %f,   %d,    %d,venue,squid
                                                     FROM job WHERE jobid=%d
                                           """ % (realtime,cputime,ncpus,status,jobId)
                           else:
                              sqlCommand = """INSERT INTO joblog(sessnum,  job,superjob,event,start,walltime,cputime,ncpus,status,venue)
                                                   SELECT        sessnum,jobid,superjob,event,start,      %f,     %f,   %d,    %d,venue
                                                     FROM job WHERE jobid=%d
                                           """ % (realtime,cputime,ncpus,status,jobId)
                           try:
                              self.mySQLDatabase.insert(sqlCommand)
                           except:
                              self.logger.log(logging.ERROR,getLogMessage("ERROR: Unable to copy orphaned job %d to joblog" % (jobId)))
                     self.logger.log(logging.DEBUG,getLogMessage("finalizeJob copy job to joblog"))

                  #
                  # Mark inactive any unmonitored jobs that have completed and have been moved to joblog.
                  #
                  sqlCommand = """UPDATE job INNER JOIN joblog ON jobid=job SET job.active=0
                                   WHERE job.active=4 AND TIMESTAMPDIFF(second,job.start,now()) > %d
                               """ % (self.configData['loadHorizon'])
                  try:
                     self.mySQLDatabase.update(sqlCommand)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("Unable to deactivate recorded unmonitored job entries."))
                  self.logger.log(logging.DEBUG,getLogMessage("finalizeJob mark unmonitored inactive"))

                  #
                  # Copy any jobs that are not being monitored and whose session has been terminated from job to joblog
                  #
                  sqlCommand = """SELECT job.jobid
                                    FROM job LEFT JOIN session ON job.sessnum=session.sessnum
                                            INNER JOIN sessionlog ON job.sessnum=sessionlog.sessnum
                                             LEFT JOIN joblog ON job.jobid=joblog.job
                                   WHERE (session.sessnum IS NULL AND
                                               joblog.job IS NULL AND job.active=4 AND TIMESTAMPDIFF(second,job.start,now()) > %d)
                               """ % (self.configData['loadHorizon'])
                  try:
                     resultSelect = self.mySQLDatabase.select(sqlCommand)
                  except:
                     pass
                  else:
                     self.logger.log(logging.DEBUG,getLogMessage("finalizeJob get unmonitored jobid list"))
                     if len(resultSelect) > 0:
                        realtime = 0.
                        cputime  = 0.
                        ncpus    = 1
                        status   = 18
                        for row in resultSelect:
                           jobId = row[0]

                           if self.serverData['squidInJob']:
                              sqlCommand = """INSERT INTO joblog(sessnum,  job,superjob,event,start,walltime,cputime,ncpus,status,venue,squid)
                                                   SELECT        sessnum,jobid,superjob,event,start,      %f,     %f,   %d,    %d,venue,squid
                                                     FROM job WHERE jobid=%d
                                           """ % (realtime,cputime,ncpus,status,jobId)
                           else:
                              sqlCommand = """INSERT INTO joblog(sessnum,  job,superjob,event,start,walltime,cputime,ncpus,status,venue)
                                                   SELECT        sessnum,jobid,superjob,event,start,      %f,     %f,   %d,    %d,venue
                                                     FROM job WHERE jobid=%d
                                           """ % (realtime,cputime,ncpus,status,jobId)
                           try:
                              self.mySQLDatabase.insert(sqlCommand)
                           except:
                              self.logger.log(logging.ERROR,getLogMessage("ERROR: Unable to copy unmonitored orphaned job %d to joblog" % (jobId)))
                     self.logger.log(logging.DEBUG,getLogMessage("finalizeJob copy unmonitored jobs to joblog"))

               self.mySQLDatabase.commit()
               self.mySQLDatabase.disconnect()

         self.serverData['dbJobEntryExists'] = False


   def finalizeCreatedSession(self,
                              createdSession):
      if createdSession > 0:
         if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
            status = 0
            sqlCommand = """INSERT INTO
           sessionlog(sessnum,username,remoteip,exechost,dispnum,start,appname,status,                                     walltime)
       SELECT         sessnum,username,remoteip,exechost,dispnum,start,appname,    %d,TIMESTAMPDIFF(SECOND,start,now()) AS walltime
                           FROM session WHERE sessnum=%d
                         """ % (status,createdSession)
            try:
               self.mySQLDatabase.insert(sqlCommand)
            except:
               message = "ERROR: Unable to create sessionlog record for '%s'" % (self.clientData['userName'])
               serverMessage = {'messageType':'writeStderr',
                                'text':message + '\n'}
               self.clientListener.postJsonMessage(serverMessage)
               self.logger.log(logging.ERROR,getLogMessage(message))
            else:
               sqlCommand = """DELETE FROM session
                                WHERE sessnum=%d
                            """ % (createdSession)
               try:
                  nDeleted = self.mySQLDatabase.delete(sqlCommand)
               except:
                  self.logger.log(logging.ERROR,getLogMessage("Unable to delete created session %d" % (createdSession)))

            self.mySQLDatabase.commit()
            self.mySQLDatabase.disconnect()


   def finalizeCreatedSessions(self):
      if self.serverData['doFinalizeOperations']  and (len(self.serverData['createdSessions']) > 0):
         if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
            for createdSession in reversed(self.serverData['createdSessions']):
               status = 0
               sqlCommand = """INSERT INTO
           sessionlog(sessnum,username,remoteip,exechost,dispnum,start,appname,status,                                     walltime)
       SELECT         sessnum,username,remoteip,exechost,dispnum,start,appname,    %d,TIMESTAMPDIFF(SECOND,start,now()) AS walltime
                              FROM session WHERE sessnum=%d
                            """ % (status,createdSession)
               try:
                  self.mySQLDatabase.insert(sqlCommand)
               except:
                  message = "ERROR: Unable to create sessionlog record for '%s'" % (self.clientData['userName'])
                  serverMessage = {'messageType':'writeStderr',
                                   'text':message + '\n'}
                  self.clientListener.postJsonMessage(serverMessage)
                  self.logger.log(logging.ERROR,getLogMessage(message))
               else:
                  sqlCommand = """DELETE FROM session
                                   WHERE sessnum=%d
                               """ % (createdSession)
                  try:
                     nDeleted = self.mySQLDatabase.delete(sqlCommand)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("Unable to delete created session %d" % (createdSession)))

            self.mySQLDatabase.commit()
            self.mySQLDatabase.disconnect()


   def getSubmitSessionsStartedBefore(self,
                                      ageDays):
      submitSessionsStartedBefore = []
      if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
         sqlCommand = """SELECT DISTINCT(sessnum) FROM session
                          WHERE TIMESTAMPDIFF(day,start,now())>%d and exechost="submit"
                       ORDER BY sessnum
                      """ % (ageDays)
         try:
            result = self.mySQLDatabase.select(sqlCommand)
         except:
            message = "ERROR: Unable to get sessions older than %d days" % (ageDays)
            self.logger.log(logging.ERROR,getLogMessage(message))
         else:
            if len(result) > 0:
               for row in result:
                  submitSessionsStartedBefore.append(row[0])

         self.mySQLDatabase.disconnect()

      return(submitSessionsStartedBefore)


   def setUserGroupIds(self):
      if os.getuid() == 0:
         os.setregid(self.serverData['gid'],self.serverData['gid'])
         os.setgroups(self.serverData['gids'])
         os.setreuid(self.serverData['uid'],self.serverData['uid'])


   def attach(self):
# Proxy
      if self.setupServerConnection():
         self.logger.log(logging.INFO,getLogMessage("Attach to %d" % (self.serverData['attachId'])))
         serverMessage = {'messageType':'attached'}
         self.serverConnection.postJsonMessage(serverMessage)
      else:
         message = "Unable to attach to job %s,\nPerhaps it has already completed." % (self.serverData['attachJobId'])
         serverMessage = {'messageType':'writeStderr',
                          'text':message + '\n'}
         self.clientListener.postJsonMessage(serverMessage)
         self.logger.log(logging.ERROR,getLogMessage(message))
         self.exit(1)


   def processAttachRequests(self):
# JobExecuter
      if self.attachListener:
         message = self.attachListener.pullMessage(0)
         while message:
            args = message.split()
            if args[0] != 'null' and args[0] != 'json':
               self.logger.log(logging.DEBUG,getLogMessage("attach request = %s" % (args[0])))

            if args[0] == 'json':
               jsonMessageLength = int(args[1])
               jsonMessage = self.attachListener.pullMessage(jsonMessageLength)
               if len(jsonMessage) > 0:
                  try:
                     attachJsonObject = json.loads(jsonMessage)
                  except ValueError:
                     self.logger.log(logging.ERROR,getLogMessage("JSON object %s could not be decoded" % (jsonMessage)))
                  else:
                     self.logger.log(logging.DEBUG,getLogMessage("attach request = %s" % (attachJsonObject['messageType'])))
                     if attachJsonObject['messageType'] == 'attached':
                        attachMessage = {'messageType':'attached'}
                        self.attachListener.postJsonMessage(attachMessage)
                        attachChannel,fromAttachBuffer,toAttachBuffer = self.attachListener.setChannelAndBuffers(None,"","")
                        self.clientListener.closeConnection()
                        clientChannel,fromClientBuffer,toClientBuffer = self.clientListener.setChannelAndBuffers(attachChannel,
                                                                                                                 fromAttachBuffer,
                                                                                                                 toAttachBuffer)
                        self.clientListener.pushMessage(fromClientBuffer)
                        attachMessage = {'messageType':'attached'}
                        self.clientListener.postJsonMessage(attachMessage)
                        attachMessage = {'messageType':'runName',
                                         'runName':self.serverData['runName'],
                                         'jobId':self.serverData['jobId']}
                        self.clientListener.postJsonMessage(attachMessage)
                        attachMessage = {'messageType':'operationMode',
                                         'operationMode':self.serverData['operationMode']}
                        self.clientListener.postJsonMessage(attachMessage)

#                       self.logger.log(logging.DEBUG,getLogMessage("proxy->client(%s,%s)" % (type(toClientBuffer),toClientBuffer)))
                        self.clientListener.postMessage(toClientBuffer)
#                       attachMessage = {'messageType':'childHasExited',
#                                        'childHasExited':self.serverData['childHasExited']}
#                       self.clientListener.postJsonMessage(attachMessage)
                        self.serverData['detached'] = False
               else:
                  self.attachListener.pushMessage(message + '\n')
                  break
            else:
               self.logger.log(logging.ERROR,getLogMessage("Discarded attach message: %s" % (message)))

            message = self.attachListener.pullMessage(0)


   def processServerRequests(self):
# Proxy
      if self.serverConnection:
         message = self.serverConnection.pullMessage(0)
         while message:
            args = message.split()
            if args[0] != 'null' and args[0] != 'json':
               self.logger.log(logging.DEBUG,getLogMessage("server request = %s" % (args[0])))

            if args[0] == 'json':
               jsonMessageLength = int(args[1])
               jsonMessage = self.serverConnection.pullMessage(jsonMessageLength)
               if len(jsonMessage) > 0:
                  try:
                     serverJsonObject = json.loads(jsonMessage)
                  except ValueError:
                     self.logger.log(logging.ERROR,getLogMessage("JSON object %s could not be decoded" % (jsonMessage)))
                  else:
                     self.logger.log(logging.DEBUG,getLogMessage("server request = %s" % (serverJsonObject['messageType'])))
                     if   serverJsonObject['messageType'] == 'attached':
                        self.serverData['attached'] = True
                        break
                     elif serverJsonObject['messageType'] == 'attachFailed':
                        message = "Unable to attach to job %s,\nPerhaps it is in the final stages of completion." % \
                                                                                     (self.serverData['attachJobId'])
                        serverMessage = {'messageType':'writeStderr',
                                         'text':message + '\n'}
                        self.clientListener.postJsonMessage(serverMessage)
                        self.logger.log(logging.ERROR,getLogMessage(message))
                        self.exit(1)
               else:
                  self.serverConnection.pushMessage(message + '\n')
                  break
            else:
               self.logger.log(logging.ERROR,getLogMessage("Discarded server message: %s" % (message)))

            message = self.serverConnection.pullMessage(0)


   def processClientRequests(self):
      nMessagesProcessed = 0
      message = self.clientListener.pullMessage(0)
      while message:
         args = message.split()
         if args[0] != 'null' and args[0] != 'json':
            self.logger.log(logging.DEBUG,getLogMessage("client request = %s" % (args[0])))

         nMessagesProcessed += 1
         if args[0] == 'json':
            jsonMessageLength = int(args[1])
            jsonMessage = self.clientListener.pullMessage(jsonMessageLength)
            if len(jsonMessage) > 0:
               try:
                  clientJsonObject = json.loads(jsonMessage)
               except ValueError:
                  self.logger.log(logging.ERROR,getLogMessage("JSON object %s could not be decoded" % (jsonMessage)))
               else:
                  if clientJsonObject['messageType'] != 'null':
                     tStart = time.time()
                     self.logger.log(logging.DEBUG,getLogMessage("client request = %s" % (clientJsonObject['messageType'])))
                  if   clientJsonObject['messageType'] == 'userName':
                     self.clientData['userName'] = clientJsonObject['userName']
                     if 'sudoUserName' in clientJsonObject:
                        self.clientData['sudoUserName'] = clientJsonObject['sudoUserName']
                     if 'wsUserName' in clientJsonObject:
                        self.clientData['wsUserName'] = clientJsonObject['wsUserName']
                  elif clientJsonObject['messageType'] == 'password':
                     self.clientData['password'] = clientJsonObject['password']
                     self.serverData['authAttempts'] += 1
                  elif clientJsonObject['messageType'] == 'sessionId':
                     self.clientData['sessionId'] = int(clientJsonObject['sessionId'])
                  elif clientJsonObject['messageType'] == 'sessionToken':
                     self.clientData['sessionToken'] = clientJsonObject['sessionToken']
                     self.serverData['authAttempts'] += 1
                  elif clientJsonObject['messageType'] == 'cacheHosts':
                     self.clientData['cacheHosts'] = clientJsonObject['cacheHosts']
                  elif clientJsonObject['messageType'] == 'privateFingerPrint':
                     self.clientData['privateFingerPrint'] = clientJsonObject['privateFingerPrint']
#                    self.logger.log(logging.INFO,getLogMessage("PPK private fingerprint: %s" % \
#                                                     (self.clientData['privateFingerPrint'])))
                     self.serverData['authAttempts'] += 1
                  elif clientJsonObject['messageType'] == 'clientToken':
                     self.clientData['clientToken'] = clientJsonObject['clientToken']
                  elif clientJsonObject['messageType'] == 'clientReadyForSignon':
                     serverMessage = {'messageType':'serverReadyForSignon'}
                     if self.clientData['privateFingerPrint']:
                        for publicKeyFile in glob.iglob(os.path.join(self.publicKeyDirectory,'*.pub_pem')):
#                          self.logger.log(logging.INFO,getLogMessage("Checking %s for PPK match" % (publicKeyFile)))
                           command = ['openssl','rsa','-pubin','-in',publicKeyFile,'-noout','-modulus']
                           child = subprocess.Popen(command,
                                                    stdout=subprocess.PIPE,
                                                    stderr=subprocess.PIPE,
                                                    close_fds=True)
#                                                   universal_newlines=True)
                           stdOutput,stdError = child.communicate()
                           exitStatus = child.returncode
                           if exitStatus == 0:
                              modulus = stdOutput.decode('utf-8').split('=')[1].strip()

                              command = ['openssl','dgst','-sha256']
                              child = subprocess.Popen(command,
                                                       stdin=subprocess.PIPE,
                                                       stdout=subprocess.PIPE,
                                                       stderr=subprocess.PIPE,
                                                       close_fds=True)
#                                                      universal_newlines=True)
                              stdOutput,stdError = child.communicate(modulus.encode('utf-8'))
                              exitStatus = child.returncode
                              if exitStatus == 0:
                                 publicFingerPrint = stdOutput.decode('utf-8').split(' ')[1].strip()
#                                self.logger.log(logging.INFO,getLogMessage("PPK public fingerprint: %s" % (publicFingerPrint)))
                                 if publicFingerPrint == self.clientData['privateFingerPrint']:
                                    secretMessage = uuid.uuid4().hex
#                                   self.logger.log(logging.INFO,getLogMessage("secret message: %s" % (secretMessage)))
                                    command = ['openssl','rsautl','-encrypt','-inkey',publicKeyFile,'-pubin']
                                    child = subprocess.Popen(command,
                                                             stdin=subprocess.PIPE,
                                                             stdout=subprocess.PIPE,
                                                             stderr=subprocess.PIPE,
                                                             close_fds=True)
                                    stdOutput,stdError = child.communicate(secretMessage.encode('utf-8'))
                                    exitStatus = child.returncode
                                    if exitStatus == 0:
                                       serverMessage['encryptedMessage'] = base64.b64encode(stdOutput).decode('utf-8')
                                       authenticationHash = hashlib.sha256()
                                       authenticationHash.update(self.clientData['clientId'].hex.encode('utf-8'))
                                       authenticationHash.update(secretMessage.encode('utf-8'))
                                       authenticationHash.update(self.serverData['serverId'].hex.encode('utf-8'))
                                       self.serverData['authenticationHash'] = authenticationHash.hexdigest()
#                                      self.logger.log(logging.INFO,getLogMessage("authentication hash: %s" % \
#                                                                   (self.serverData['authenticationHash'])))
                                    else:
                                       self.logger.log(logging.ERROR,getLogMessage("%s: %s" % \
                                                   (publicKeyFile,stdError.decode('utf-8'))))
                                    break
                     self.clientListener.postJsonMessage(serverMessage)
                  elif clientJsonObject['messageType'] == 'signon':
                     try:
                        clientAuthenticationHash = clientJsonObject['authenticationHash']
#                       self.logger.log(logging.INFO,getLogMessage("client authentication hash: %s" % (clientAuthenticationHash)))
                     except:
                        clientAuthenticationHash = None
                     self.serverData['authz'] = self.signon(clientAuthenticationHash)
                     if self.serverData['authz']:
                        serverMessage = {'messageType':'authz',
                                         'success':self.serverData['authz'],
                                         'retry':False,
                                         'hasDistributor':self.serverData['distributorExists'],
                                         'hasHarvester':self.serverData['harvesterExists'],
                                         'hasJobStatus':self.serverData['jobStatusExists'],
                                         'hasJobKill':self.serverData['jobKillExists'],
                                         'hasVenueProbe':self.serverData['venueProbeExists']}
                        self.clientListener.postJsonMessage(serverMessage)
                        self.setUserGroupIds()
                        serverMessage = {'messageType':'message',
                                         'text':"Congratulations - you have successfully authenticated."}
                        self.clientListener.postJsonMessage(serverMessage)
                        if self.serverData['jobId'] != 0:
                           self.serverData['dbJobEntryExists'] = True
                     else:
                        if self.serverData['authAttempts'] < self.serverData['maximumAuthAttempts']:
                           serverMessage = {'messageType':'authz',
                                            'success':self.serverData['authz'],
                                            'retry':True}
                           self.clientListener.postJsonMessage(serverMessage)
                        else:
                           serverMessage = {'messageType':'authz',
                                            'success':self.serverData['authz'],
                                            'retry':False}
                           self.clientListener.postJsonMessage(serverMessage)
                        self.scheduleTimeout()
                  elif clientJsonObject['messageType'] == 'parseArguments':
                     if self.setupWorkDirectory():
                        argumentsOK,continueExecution,commandEnabled = self.parseCommandArguments()
                        if argumentsOK:
                           if continueExecution:
                              if self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNPROXY:
                                 self.attach()
                                 self.cancelTimeout()
                              else:
                                 self.scheduleHeartbeat()
                           else:
                              self.exit(0)
                        else:
                           if not commandEnabled:
                              self.logger.log(logging.ERROR,getLogMessage("Command not available"))
                              serverMessage = {'messageType':'writeStderr',
                                               'text':"Command not available\n"}
                           else:
                              self.logger.log(logging.ERROR,getLogMessage("Command line argument parsing failed"))
                              serverMessage = {'messageType':'writeStderr',
                                               'text':"Command line argument parsing failed\n"}
                           self.clientListener.postJsonMessage(serverMessage)
                           self.exit(1)
                     else:
                        self.exit(1)
                  elif clientJsonObject['messageType'] == 'parseHarvestArguments':
                     if self.setupWorkDirectory():
                        argumentsOK,continueExecution,commandEnabled = self.parseCommandArguments()
                        if argumentsOK:
                           if continueExecution:
                              self.scheduleHeartbeat()
                           else:
                              self.exit(0)
                        else:
                           if not commandEnabled:
                              self.logger.log(logging.ERROR,getLogMessage("Command not available"))
                              serverMessage = {'messageType':'writeStderr',
                                               'text':"Command not available\n"}
                           else:
                              self.logger.log(logging.ERROR,getLogMessage("Command line argument parsing failed"))
                              serverMessage = {'messageType':'writeStderr',
                                               'text':"Command line argument parsing failed\n"}
                           self.clientListener.postJsonMessage(serverMessage)
                           self.exit(1)
                     else:
                        self.exit(1)
                  elif clientJsonObject['messageType'] == 'clientId':
                     self.clientData['clientId'] = uuid.UUID(clientJsonObject['clientId'])
#                    self.logger.log(logging.INFO,getLogMessage("clientId: %s" % (self.clientData['clientId'])))
                     serverMessage = {'messageType':'serverId',
                                      'serverId':self.serverData['serverId'].hex}
                     self.clientListener.postJsonMessage(serverMessage)
                  elif clientJsonObject['messageType'] == 'clientVersion':
                     self.clientData['version'] = clientJsonObject['version']
                     serverMessage = {'messageType':'serverVersion',
                                      'version':self.serverData['version']}
                     self.clientListener.postJsonMessage(serverMessage)
                     serverMajor,serverMinor,serverPatch = self.serverData['version'].split('.')
                     clientMajor,clientMinor,clientPatch = self.clientData['version'].split('.')
                     if clientMajor != serverMajor or clientMinor != serverMinor or clientPatch != serverPatch:
                        self.logger.log(logging.WARNING,getLogMessage("Client version(%s) does not equal Server version(%s)" % \
                                                                       (self.clientData['version'],self.serverData['version'])))
                  elif clientJsonObject['messageType'] == 'doubleDashTerminator':
                     self.clientData['doubleDashTerminator'] = clientJsonObject['doubleDashTerminator']
                  elif clientJsonObject['messageType'] == 'arg':
                     self.clientData['args'].append(clientJsonObject['arg'])
                  elif clientJsonObject['messageType'] == 'args':
                     for arg in clientJsonObject['args']:
                        self.clientData['args'].append(arg)
                  elif clientJsonObject['messageType'] == 'var':
                     envName  = clientJsonObject['name']
                     envValue = clientJsonObject['value']
                     self.clientData['envs'][envName] = envValue
                     if envName == "SUBMIT_JOB":
                        try:
                           self.serverData['superJobId'] = int(self.clientData['envs'][envName])
                        except:
                           self.logger.log(logging.ERROR,getLogMessage("SUBMIT_JOB = %s" % (self.clientData['envs'][envName])))
                  elif clientJsonObject['messageType'] == 'vars':
                     for envName in clientJsonObject['vars']:
                        envValue = clientJsonObject['vars'][envName]
                        self.clientData['envs'][envName] = envValue
                        if envName == "SUBMIT_JOB":
                           try:
                              self.serverData['superJobId'] = int(self.clientData['envs'][envName])
                           except:
                              self.logger.log(logging.ERROR,getLogMessage("SUBMIT_JOB = %s" % (self.clientData['envs'][envName])))
                  elif clientJsonObject['messageType'] == 'umask':
                     self.clientData['umask'] = clientJsonObject['umask']
                  elif clientJsonObject['messageType'] == 'pwd':
                     self.clientData['workDirectory']           = clientJsonObject['path']
                     self.clientData['workDirectoryProperties'] = clientJsonObject['properties']
                     self.clientData['signaturePath']           = clientJsonObject['signature']
#                 elif clientJsonObject['messageType'] == 'pwdWithMount':
#                    self.clientData['workDirectory']            = clientJsonObject['path']
#                    self.clientData['workDirectoryProperties']  = clientJsonObject['properties']
#                    self.clientData['signaturePath']            = clientJsonObject['signature']
                  elif clientJsonObject['messageType'] == 'isClientTTY':
                     self.clientData['isClientTTY'] = clientJsonObject['isClientTTY']
#                    self.logger.log(logging.INFO,getLogMessage("isClientTTY(%s) = %s" % (args[1],self.clientData['isClientTTY'])))
                  elif clientJsonObject['messageType'] == 'pegasusVersion':
                     self.clientData['pegasusVersion'] = clientJsonObject['version']
                     if self.clientData['pegasusVersion'] != '0.0.0':
                        pegasusCommand = ". %s\n" % (self.configData['useSetup']) + \
                                         "use -e -r pegasus-%s\n" % (self.clientData['pegasusVersion']) + \
                                         "pegasus-version\n" + \
                                         "pegasus-config --bin\n" + \
                                         "pegasus-config --python"
                        child = subprocess.Popen(pegasusCommand,shell=True,
                                                 stdout=subprocess.PIPE,
                                                 stderr=subprocess.PIPE,
                                                 close_fds=True)
                        pegasusStdOutput,pegasusStdError = child.communicate()
                        pegasusExitStatus = child.returncode
                        if pegasusExitStatus == 0:
                           try:
                              pegasusVersion,pegasusBin,pegasusPythonPath = ''.join(pegasusStdOutput.decode('utf-8')).strip().split()
                           except:
                              self.logger.log(logging.ERROR,getLogMessage("Setting Pegasus version failed\ncommand and result"))
                              self.logger.log(logging.ERROR,getLogMessage(pegasusCommand))
                              self.logger.log(logging.ERROR,getLogMessage(pegasusStdOutput.decode('utf-8')))
                              self.logger.log(logging.ERROR,getLogMessage(pegasusStdError.decode('utf-8')))
                           else:
                              self.serverData['pegasusVersion']    = pegasusVersion
                              self.serverData['pegasusHome']       = os.path.dirname(pegasusBin)
                              self.serverData['pegasusPythonPath'] = pegasusPythonPath
                              self.serverData['pegasusExists']     = True
                        else:
                           self.logger.log(logging.ERROR,getLogMessage("Getting Pegasus version failed\ncommand and result"))
                           self.logger.log(logging.ERROR,getLogMessage("exitStatus = %d" % pegasusExitStatus))
                           self.logger.log(logging.ERROR,getLogMessage(pegasusCommand))
                           self.logger.log(logging.ERROR,getLogMessage(pegasusStdOutput.decode('utf-8')))
                           self.logger.log(logging.ERROR,getLogMessage(pegasusStdError.decode('utf-8')))
                  elif clientJsonObject['messageType'] == 'inputFile':
                     clientInputPath      = clientJsonObject['path']
                     clientFileProperties = clientJsonObject['properties']
                     mapInputFile = not self.__isClientFileShared(clientFileProperties)
                     self.clientData['inputFileMapping'][clientInputPath] = mapInputFile
                     if not mapInputFile:
                        self.serverData['inputFileMapping'][clientInputPath] = clientFileProperties['serverFilePath']
#                 elif clientJsonObject['messageType'] == 'inputFileWithMount':
#                    clientInputPath      = clientJsonObject['path']
#                    clientFileProperties = clientJsonObject['properties']
#                    mapInputFile = not self.__isClientFileShared(clientFileProperties)
#                    self.clientData['inputFileMapping'][clientInputPath] = mapInputFile
#                    if not mapInputFile:
#                       self.serverData['inputFileMapping'][clientInputPath] = clientFileProperties['serverFilePath']
                  elif clientJsonObject['messageType'] == 'inputFileInodesSent':
                     fileMappingRequired = False
                     for clientInputPath,mapInputFile in self.clientData['inputFileMapping'].items():
                        if mapInputFile:
                           serverMessage = {'messageType':'addExportFile',
                                            'file':clientInputPath}
                           self.clientListener.postJsonMessage(serverMessage)
                           fileMappingRequired = True
                     if fileMappingRequired:
                        transferTarFile = "%s_transfer.tar.gz" % (self.serverData['jobId'])
                        transferTarPath = os.path.join(os.sep,"tmp",transferTarFile)
                        try:
                           fp = open(transferTarPath,'wb')
                        except (IOError,OSError):
                           self.logger.log(logging.ERROR,getLogMessage("input file open failed: %s" % (transferTarPath)))
                        else:
                           self.serverData['importObjects'][fp]           = {}
                           self.serverData['importObjects'][fp]['path']   = transferTarPath
                           self.serverData['importObjects'][fp]['buffer'] = b""
                           serverMessage = {'messageType':'exportFiles',
                                            'file':transferTarFile}
                           self.clientListener.postJsonMessage(serverMessage)
                           self.clientData['isBuildingInputTar']  = True
                           self.serverData['isImportingInputTar'] = True
                     else:
                        serverMessage = {'messageType':'noExportFiles'}
                        self.clientListener.postJsonMessage(serverMessage)
                  elif clientJsonObject['messageType'] == 'serverReadyForIO':
                     self.serverData['isServerReadyForIO'] = True
                  elif clientJsonObject['messageType'] == 'submitCommandFile':
                     clientInputPath      = clientJsonObject['path']
                     clientFileProperties = clientJsonObject['properties']
                     clientArgValue       = clientJsonObject['argValue']
                     mapInputFile = not self.__isClientFileShared(clientFileProperties)
                     self.clientData['submitCommandFileMapping'][clientInputPath] = {'mappingRequired':mapInputFile,
                                                                                     'argValue':clientArgValue}
#                 elif clientJsonObject['messageType'] == 'submitCommandFileWithMount':
#                    clientInputPath      = clientJsonObject['path']
#                    clientFileProperties = clientJsonObject['properties']
#                    clientArgValue       = clientJsonObject['argValue']
#                    mapInputFile = not self.__isClientFileShared(clientFileProperties)
#                    self.clientData['submitCommandFileMapping'][clientInputPath] = {'mappingRequired':mapInputFile,
#                                                                                    'argValue':clientArgValue}
                  elif clientJsonObject['messageType'] == 'submitCommandFileInodesSent':
                     fileMappingRequired = False
                     for clientInputPath in self.clientData['submitCommandFileMapping']:
                        mapInputFile = self.clientData['submitCommandFileMapping'][clientInputPath]['mappingRequired']
                        if mapInputFile:
                           serverMessage = {'messageType':'addExportCommandFile',
                                            'file':clientInputPath}
                           self.clientListener.postJsonMessage(serverMessage)
                           fileMappingRequired = True
                     if fileMappingRequired:
                        try:
                           transferTarDirectory = os.path.join(os.sep,"tmp")
                           fd,transferTarPath = tempfile.mkstemp(prefix='command_transfer_',suffix=".tar",dir=transferTarDirectory)
                           os.close(fd)
                           fp = open(transferTarPath,'wb')
                        except (IOError,OSError):
                           self.logger.log(logging.ERROR,getLogMessage("command files list file open failed"))
                        else:
                           self.clientData['submitCommandTransferPath']   = transferTarPath[:-4]
                           self.serverData['importObjects'][fp]           = {}
                           self.serverData['importObjects'][fp]['path']   = transferTarPath
                           self.serverData['importObjects'][fp]['buffer'] = b""
                           serverMessage = {'messageType':'exportCommandFiles',
                                            'file':transferTarPath}
                           self.clientListener.postJsonMessage(serverMessage)
                           self.clientData['isBuildingInputTar']   = True
                           self.serverData['isImportingSubmitTar'] = True
                     else:
                        serverMessage = {'messageType':'noExportCommandFiles'}
                        self.clientListener.postJsonMessage(serverMessage)
                        self.clientData['submitCommandFilesMapped'] = True
                  elif clientJsonObject['messageType'] == 'write':
                     clientInputPath = clientJsonObject['path']
                     text = base64.b64decode(clientJsonObject['text'].encode('utf-8'))
                     found = False
                     for importObject in self.serverData['importObjects']:
                        if self.serverData['importObjects'][importObject]['path'] == clientInputPath:
                           self.serverData['importObjects'][importObject]['buffer'] += text
#                          self.logger.log(logging.DEBUG,getLogMessage("importObject buffer length = %d" % \
#                                          (len(self.serverData['importObjects'][importObject]['buffer']))))
                           found = True
                           break
                     if not found:
                        for outputObject in self.serverData['outputObjects']:
                           if self.serverData['outputObjects'][outputObject]['path'] == clientInputPath:
                              self.serverData['outputObjects'][outputObject]['buffer'] += text
#                             self.logger.log(logging.DEBUG,getLogMessage("outputObject buffer length = %d" % \
#                                             (len(self.serverData['outputObjects'][outputObject]['buffer']))))
                              found = True
                              break
                     if not found:
                        self.logger.log(logging.ERROR,getLogMessage("File object not found to match read:%s" % (clientInputPath)))
                  elif clientJsonObject['messageType'] == 'close':
                     clientInputPath = clientJsonObject['path']
                     found = False
                     for importObject in self.serverData['importObjects']:
                        if self.serverData['importObjects'][importObject]['path'] == clientInputPath:
                           self.serverData['closePendingObjects'].append(importObject)
                           found = True
                           break
                     if not found:
                        for outputObject in self.serverData['outputObjects']:
                           if self.serverData['outputObjects'][outputObject]['path'] == clientInputPath:
                              self.serverData['closePendingObjects'].append(outputObject)
                              found = True
                              break
                     if not found:
                        self.logger.log(logging.ERROR,getLogMessage("File object not found to match close:%s" % (clientInputPath)))
                  elif clientJsonObject['messageType'] == 'clientReadyForIO':
                     if self.serverData['operationMode'] & (self.commandParser.OPERATIONMODERUNDISTRIBUTORID | \
                                                            self.commandParser.OPERATIONMODERUNCLIENTLOCAL):
                        if self.serverData['workDirectoryShared']:
                           serverMessage = {'messageType':'noImportFile'}
                           self.clientListener.postJsonMessage(serverMessage)
                        else:
                           importMessagePosted = False
                           try:
                              transferTarFile = "%s_output.tar" % (self.serverData['jobId'])
                              transferTarPath = os.path.join(os.sep,"tmp",transferTarFile)
                              if os.path.exists(transferTarPath):
                                 os.remove(transferTarPath)
                              findCommand = ['find','.','-newer','.__fileTimeMarkerSubmit','-not','-name','.','-print0']
                              tarCommand = ['xargs','--null','--no-run-if-empty','tar','rvf',transferTarPath,'--no-recursion']
                              self.logger.log(logging.INFO,getLogMessage("command: %s | %s" % (' '.join(findCommand), \
                                                                                               ' '.join(tarCommand))))
                              findChild = subprocess.Popen(findCommand,
                                                           stdout=subprocess.PIPE,
                                                           stderr=subprocess.PIPE,
                                                           close_fds=True)
                              tarChild = subprocess.Popen(tarCommand,
                                                          stdin=findChild.stdout,
                                                          stdout=subprocess.PIPE,
                                                          stderr=subprocess.PIPE,
                                                          close_fds=True)
                              tarStdOutput,tarStdError = tarChild.communicate()
                              tarExitStatus = tarChild.returncode
                              if tarExitStatus != 0:
                                 self.logger.log(logging.ERROR,getLogMessage("Failed to write transfer tarfile %s" % \
                                                                                                   (transferTarPath)))
                                 if tarStdOutput:
                                    self.logger.log(logging.ERROR,getLogMessage(tarStdOutput.decode('utf-8')))
                                 if tarStdError:
                                    self.logger.log(logging.ERROR,getLogMessage(tarStdError.decode('utf-8')))
                                 try:
                                    os.remove(transferTarPath)
                                 except:
                                    pass
                              else:
                                 if os.path.exists(transferTarPath):
                                    serverMessage = {'messageType':'writeStdout',
                                                     'text':"Import file transfer initiated. %s\n" % (time.ctime())}
                                    self.clientListener.postJsonMessage(serverMessage)
                                    zipCommand = ['gzip',transferTarPath]
                                    self.logger.log(logging.INFO,getLogMessage("command: %s" % (' '.join(zipCommand))))
                                    zipChild = subprocess.Popen(zipCommand,
                                                                stdout=subprocess.PIPE,
                                                                stderr=subprocess.PIPE,
                                                                close_fds=True)
                                    zipStdOutput,zipStdError = zipChild.communicate()
                                    zipExitStatus = zipChild.returncode
                                    if zipExitStatus != 0:
                                       self.logger.log(logging.ERROR,getLogMessage("Failed to gzip transfer tarfile %s" % \
                                                                                                        (transferTarPath)))
                                       if zipStdOutput:
                                          self.logger.log(logging.ERROR,getLogMessage(zipStdOutput.decode('utf-8')))
                                       if zipStdError:
                                          self.logger.log(logging.ERROR,getLogMessage(zipStdError.decode('utf-8')))
                                       try:
                                          os.remove(transferTarPath)
                                       except:
                                          pass
                                       transferTarPath += '.gz'
                                       try:
                                          os.remove(transferTarPath)
                                       except:
                                          pass
                                    else:
#                                      time.sleep(self.serverData['disconnectMax']+60)
                                       transferTarFile += '.gz'
                                       serverMessage = {'messageType':'importFile',
                                                        'file':transferTarFile}
                                       self.clientListener.postJsonMessage(serverMessage)
                                       self.serverData['isExportingOutputTar'] = True
                                       importMessagePosted = True
                           except OSError as e:
                              self.logger.log(logging.ERROR,getLogMessage("Failed to create transfer tarfile %s" % \
                                                                                                 (transferTarPath)))
                              self.logger.log(logging.ERROR,getLogMessage(e.args[1]))
                              self.logger.log(logging.ERROR,getLogMessage(traceback.format_exc()))
                           if not importMessagePosted:
                              serverMessage = {'messageType':'noImportFile'}
                              self.clientListener.postJsonMessage(serverMessage)
                     else:
                        serverMessage = {'messageType':'noImportFile'}
                        self.clientListener.postJsonMessage(serverMessage)
                  elif clientJsonObject['messageType'] == 'importFileReady':
                     transferTarFile = clientJsonObject['file']
                     transferTarPath = os.path.join(os.sep,"tmp",transferTarFile)
                     try:
                        fp = open(transferTarPath,'rb')
                     except (IOError,OSError):
                        self.logger.log(logging.ERROR,getLogMessage("%s could not be opened" % (transferTarPath)))
                        serverMessage = {'messageType':'importFileFailed',
                                         'file':transferTarFile}
                        self.clientListener.postJsonMessage(serverMessage)
                     else:
                        self.serverData['exportObjects'][fp] = transferTarFile
                  elif clientJsonObject['messageType'] == 'importFilesComplete':
                     self.serverData['isExportingOutputTar'] = False
                     self.logger.log(logging.DEBUG,getLogMessage("metrics recorded = %s" % (self.serverData['metricsRecorded'])))
                     self.logger.log(logging.DEBUG,getLogMessage("finalized = %s" % (self.serverData['finalized'])))
                     self.logger.log(logging.DEBUG,getLogMessage("detached = %s" % (self.serverData['detached'])))
                     self.logger.log(logging.DEBUG,getLogMessage("readyToExit = %s" % (self.serverData['readyToExit'])))
                     if self.serverData['metricsRecorded']:
                        self.logger.log(logging.DEBUG,getLogMessage("exitCode = %s" % (self.serverData['exitCode'])))
                        self.exit(self.serverData['exitCode'])
                     else:
                        self.exit()
                  elif clientJsonObject['messageType'] == 'startLocal':
                     self.checkLoadAndLaunch()
                  elif clientJsonObject['messageType'] == 'startHarvest':
                     self.launchJob()
                  elif clientJsonObject['messageType'] == 'setupRemote':
                     self.checkLoadAndLaunch()
                  elif clientJsonObject['messageType'] == 'startRemote':
                     self.launchJob()
                  elif clientJsonObject['messageType'] == 'doCacheHit':
                     cacheSquid = clientJsonObject['cacheSquid']
                     self.doCacheSquid('cache/hit',cacheSquid)
                  elif clientJsonObject['messageType'] == 'doCacheMiss':
                     cacheSquid = clientJsonObject['cacheSquid']
                     self.doCacheSquid('cache/miss',cacheSquid)
                  elif clientJsonObject['messageType'] == 'doCachePublish':
                     cacheSquid = clientJsonObject['cacheSquid']
                     self.doCacheSquid('cache/publish',cacheSquid)
                  elif clientJsonObject['messageType'] == 'localWait':
                     self.stop()
                  elif clientJsonObject['messageType'] == 'detachSignal':
                     if not self.serverData['detach']:
                        if self.createDetachedSession():
                           if self.updateJobSession():
                              self.serverData['detach'] = True
                              serverMessage = {'messageType':'readyToDetach'}
                              self.clientListener.postJsonMessage(serverMessage)
                     else:
                        serverMessage = {'messageType':'readyToDetach'}
                        self.clientListener.postJsonMessage(serverMessage)
                  elif clientJsonObject['messageType'] == 'detach':
                     self.clientListener.closeConnection()
                     if self.setupAttachListener():
                        self.serverData['detached'] = True
                     else:
                        message = "Unable to detach job %d" % (self.serverData['jobId'])
                        self.logger.log(logging.ERROR,getLogMessage(message))
                  elif clientJsonObject['messageType'] == 'jobId':
                     self.serverData['jobId'] = clientJsonObject['jobId']
                     self.logger.log(logging.INFO,getLogMessage("Assigning client jobId = %d:" % (self.serverData['jobId'])))
                     logSetJobId(self.serverData['jobId'])
                     self.serverData['localJobId'] = "%08d" % (self.serverData['jobId'])
                     if 'localExecution' in clientJsonObject:
                        self.clientData['localExecution'] = clientJsonObject['localExecution']
#                       self.logger.log(logging.INFO,getLogMessage("localExecution = %s" % (self.clientData['localExecution'])))
                  elif clientJsonObject['messageType'] == 'jobToken':
                     self.serverData['jobToken'] = clientJsonObject['token']
                  elif clientJsonObject['messageType'] == 'event':
                     self.serverData['event'] = clientJsonObject['text']
                  elif clientJsonObject['messageType'] == 'operationMode':
                     self.serverData['operationMode'] = clientJsonObject['operationMode']
                  elif clientJsonObject['messageType'] == 'createdSessions':
                     self.serverData['createdSessions'] = clientJsonObject['createdSessions']
                  elif clientJsonObject['messageType'] == 'localMetrics':
                     instanceIndex = clientJsonObject['instanceIndex']
                     status        = clientJsonObject['status']
                     cpuTime       = clientJsonObject['cpuTime']
                     realTime      = clientJsonObject['realTime']
                     metricsMessage = "venue=%d:local status=%d cputime=%f realtime=%f" % \
                                                  (instanceIndex,status,cpuTime,realTime)
                     if 'event' in clientJsonObject:
                        metricsMessage += " event=%s" % (clientJsonObject['event'])
                     self.updateJobMetrics(metricsMessage)
                     if instanceIndex > 0:
                        self.serverData['metricsRecorded'] = False
                  elif clientJsonObject['messageType'] == 'clientHeartbeat':
                     self.clientHeartbeat()
                     self.stop()
                  elif clientJsonObject['messageType'] == 'localExit':
                     status   = clientJsonObject['status']
                     cpuTime  = clientJsonObject['cpuTime']
                     realTime = clientJsonObject['realTime']
                     metricsMessage = "venue=local status=%d cputime=%f realtime=%f" % (status,cpuTime,realTime)
                     if 'event' in clientJsonObject:
                        metricsMessage += " event=%s" % (clientJsonObject['event'])
                     if 'generated' in clientJsonObject:
                        metricsMessage += " generated=%s" % (clientJsonObject['generated'])
                     if 'squid' in clientJsonObject:
                        metricsMessage += " squid=%s" % (clientJsonObject['squid'])
                     self.updateJobMetrics(metricsMessage)
                     self.finalizeJob()
                     self.finalizeCreatedSessions()
                     self.serverData['finalized'] = True
                     serverMessage = {'messageType':'exit',
                                      'exitCode':status}
                     self.clientListener.postJsonMessage(serverMessage)
                     self.serverData['readyToExit'] = True
                  elif clientJsonObject['messageType'] == 'measurements':
#{
#   "stagingHandle": "00359536_01",
#   "jobStatistic": {
#      "jobSubmissionMechanism": "localTAPIS",
#      "remoteJobIdNumber": "",
#      "event": ":tapis_wheezy_simtool_serial:",
#      "generated": "",
#      "venue": "submit.dev.nanohub.org",
#      "realTime": 114.55,
#      "userTime": 11.55,
#      "sysTime": 3.15,
#      "exitCode": 0,
#      "transferCompleteTime": 1609961607.0,
#      "jobStartedTime": 1609961635.0,
#      "jobFinshedTime": 1609961750.0,
#      "waitingTime": 28.0,
#      "elapsedRunTime": 115.0,
#      "nCores": 1
#   },
#   "instanceId": 1,
#   "messageType": "activeJobJobStatistic",
#   "remoteJobId": "32728d39-85f3-4e5f-a282-9f611f2361d0-007",
#   "session": 23315,
#   "localJobId": 359536
#}
                     self.serverData['createdSessions'].append(clientJsonObject['session'])
                     self.serverData['jobId']      = clientJsonObject['localJobId']
                     self.logger.log(logging.INFO,getLogMessage("Assigning jobId = %d:" % (self.serverData['jobId'])))
                     logSetJobId(self.serverData['jobId'])
                     self.serverData['localJobId'] = "%08d" % (self.serverData['jobId'])
                     self.serverData['dbJobEntryExists'] = True

                     instanceId     = clientJsonObject['instanceId']
                     mechanism      = clientJsonObject['jobStatistic']['jobSubmissionMechanism']
                     remoteId       = clientJsonObject['remoteJobId']
                     remoteLocation = clientJsonObject['jobStatistic']['venue']
                     exitCode       = clientJsonObject['jobStatistic']['exitCode']
                     cpuTime        = clientJsonObject['jobStatistic']['userTime'] + clientJsonObject['jobStatistic']['sysTime']
                     elapsedRunTime = clientJsonObject['jobStatistic']['elapsedRunTime']
                     waitTime       = clientJsonObject['jobStatistic']['waitingTime']
                     nCpuUsed       = clientJsonObject['jobStatistic']['nCores']
                     metricsMessage = "venue=%d:%s:%s:%s status=%d cputime=%f realtime=%f waittime=%f ncpus=%d" % \
                            (instanceId,mechanism,remoteId,remoteLocation,exitCode,cpuTime,elapsedRunTime,waitTime,nCpuUsed)
                     if 'event' in clientJsonObject['jobStatistic']:
                        metricsMessage += " event=%s" % (clientJsonObject['jobStatistic']['event'])
                     self.logger.log(logging.INFO,getLogMessage("%s" % (metricsMessage)))
                     self.updateJobMetrics(metricsMessage)
                     self.finalizeJob()
                     self.finalizeCreatedSessions()
                     self.serverData['finalized'] = True
                     self.serverData['exitCode'] = exitCode
                     serverMessage = {'messageType':'serverExit',
                                      'exitCode':self.serverData['exitCode']}
                     self.clientListener.postJsonMessage(serverMessage)
                     self.serverData['readyToExit'] = True
                  elif clientJsonObject['messageType'] == 'clientSuspended':
                     self.clientData['suspended'] = True
                     serverMessage = {'messageType':'clientSuspended'}
                     self.clientListener.postJsonMessage(serverMessage)
                  elif clientJsonObject['messageType'] == 'clientContinued':
                     self.clientData['suspended'] = False
                     serverMessage = {'messageType':'clientContinued'}
                     self.clientListener.postJsonMessage(serverMessage)
                  elif clientJsonObject['messageType'] == 'clientSignal':
                     signalNumber = clientJsonObject['signal']
                     if self.serverData['childPid']:
                        self.killChild()
                     else:
                        self.handleSignal(signalNumber,None)
                     serverMessage = {'messageType':'null'}
                     self.clientListener.postJsonMessage(serverMessage)
                  elif clientJsonObject['messageType'] == 'clientExit':
                     exitCode = clientJsonObject['exitCode']
                     self.exit(exitCode)
                  elif clientJsonObject['messageType'] == 'exit':
                     self.exit()
                  elif clientJsonObject['messageType'] == 'null':
                     pass
                  else:
                     self.logger.log(logging.ERROR,getLogMessage("Discarded message type: %s" % \
                                                                  (clientJsonObject['messageType'])))
                  if clientJsonObject['messageType'] != 'null':
                     tFinish = time.time()
                     self.logger.log(logging.DEBUG,getLogMessage("client request = %s took %f secs" % \
                                                   (clientJsonObject['messageType'],tFinish-tStart)))
            else:
               self.clientListener.pushMessage(message + '\n')
               nMessagesProcessed -= 1
               break
         else:
            self.logger.log(logging.ERROR,getLogMessage("Discarded client message: %s" % (message)))

         message = self.clientListener.pullMessage(0)

#     self.logger.log(logging.DEBUG,getLogMessage("Processed %d client messages" % (nMessagesProcessed)))


   def run(self):
      if self.serverData['type'] == 'Listener':
         if self.mySQLDatabase.connect(self.configData['mysqlMiddlewareDB']):
            sqlCommand = """SELECT squid FROM job LIMIT 1"""
            try:
               result = self.mySQLDatabase.select(sqlCommand)
            except:
               pass
            else:
               if len(result) == 1:
                  self.serverData['squidInJob'] = True
            self.mySQLDatabase.disconnect()

      missingSocket = False
      stopForMissingSocketPending = False
      while True:
         if self.serverData['type'] == 'Proxy' and self.serverData['attached']:
            proxyMessage = self.clientListener.pullMessage(-1024)
            if proxyMessage:
#              self.logger.log(logging.DEBUG,getLogMessage("client->server(%s,%s)" % (type(proxyMessage),proxyMessage)))
               if not isinstance(proxyMessage,type("")):
                  if isinstance(proxyMessage,type(u"")):
                     proxyMessage = proxyMessage.encode('utf-8')
               self.serverConnection.postMessage(proxyMessage)
            proxyMessage = self.serverConnection.pullMessage(-1024)
            if proxyMessage:
#              self.logger.log(logging.DEBUG,getLogMessage("server->client(%s,%s)" % (type(proxyMessage),proxyMessage)))
               if not isinstance(proxyMessage,type("")):
                  if isinstance(proxyMessage,type(u"")):
                     proxyMessage = proxyMessage.encode('utf-8')
               self.clientListener.postMessage(proxyMessage)

         if self.serverData['type'] == 'JobExecuter' and self.serverData['detached']:
            if self.serverData['socketPath']:
               if self.serverData['socketPath'].startswith(self.serverData['socketPathPrefix']):
                  socketPath = self.serverData['socketPath'][len(self.serverData['socketPathPrefix']):]
               else:
                  socketPath = self.serverData['socketPath']
               if not os.path.exists(socketPath):
                  missingSocket = True
                  self.serverData['socketPath'] = ""

         if   self.serverData['childHasExited'] and not self.serverData['detached']:
            selectTimeout = 1.
         elif self.serverData['type'] == 'JobExecuter' and not self.serverData['isServerReadyForIO']:
            selectTimeout = 1.
         elif self.serverData['type'] == 'JobExecuter' and self.clientData['localExecution']:
            selectTimeout = 1.
         elif missingSocket:
            selectTimeout = 1.
         else:
            minSelectTimeout = self.serverData['selectTimeout']* \
                               (1.-self.serverData['selectTimeoutSpread']/2.)
            maxSelectTimeout = self.serverData['selectTimeout']* \
                               (1.+self.serverData['selectTimeoutSpread']/2.)
            selectTimeout = random.uniform(minSelectTimeout,maxSelectTimeout)

         if self.clientListener and self.clientListener.isConnected():
            if not self.clientListener.isMessagePending():
               timeSinceLastWrite = time.time() - self.clientListener.getConnectionWriteTime()
               if timeSinceLastWrite >= 0.25*selectTimeout:
                  if   self.serverData['isExportingOutputTar'] and not self.serverData['detached']:
                     serverMessage = {'messageType':'null'}
                     self.clientListener.postJsonMessage(serverMessage)
                  elif self.serverData['type'] == 'JobExecuter':
                     if self.serverData['isImportingInputTar'] or self.serverData['isImportingSubmitTar']:
                        serverMessage = {'messageType':'null'}
                        self.clientListener.postJsonMessage(serverMessage)

         tStart = time.time()
         selectTimedOut = False
         try:
            clientListenerSockets,clientReader, \
                                  attachListenerSocket,attachReader, \
                                  serverReader, \
                                  inputObjects,exportObjects  = self.getInputObjects()
            clientWriter, \
                                  attachWriter, \
                                  serverWriter, \
                                  importObjects,outputObjects = self.getOutputObjects()
            readers = clientListenerSockets + clientReader + \
                      attachListenerSocket + attachReader + \
                      serverReader + \
                      inputObjects + exportObjects
            writers = clientWriter + \
                      attachWriter + \
                      serverWriter + \
                      importObjects + outputObjects
            readyReaders,readyWriters,readyExceptions = select.select(readers,writers,[],selectTimeout)
            if not (readyReaders or readyWriters or readyExceptions):
               selectTimedOut = True
         except select.error:
            selectTimedOut = True
            self.logger.log(logging.DEBUG,getLogMessage("select timeout(%f): %d readers, %d writers" % \
                                                           (selectTimeout,len(readers),len(writers))))
            readyReaders = []
            readyWriters = []

         tFinish = time.time()
#        self.logger.log(logging.DEBUG,getLogMessage("select time(%f): %d readers, %d writers, %d readyReaders, %d readyWriters" % \
#                                                   (tFinish-tStart,len(readers),len(writers),len(readyReaders),len(readyWriters))))
#
# send keepalive to notify client server is still running
#
         if self.clientListener and self.clientListener.isConnected():
            if not self.serverData['readyToExit']:
               if not self.serverData['detached']:
                  timeSinceLastWrite = time.time() - self.clientListener.getConnectionWriteTime()
                  if selectTimedOut or (timeSinceLastWrite >= self.serverData['selectTimeout']):
                     serverMessage = {'messageType':'null'}
                     self.clientListener.postJsonMessage(serverMessage)

         for readyReader in readyReaders:
            if   readyReader in clientListenerSockets:
               if self.clientListener.acceptConnection(readyReader):
                  # Do a double-fork to dissociate from the listening server.
                  try:
                     fork1PID = os.fork()
                  except OSError as e:
                     self.logger.log(logging.ERROR,getLogMessage("Daemonizing fork1 failed: %s." % (e.args[1])))
                     self.exit(e.args[0])
                  else:
                     if fork1PID != 0:
                        self.clientListener.closeConnection()  # Close the client socket in the listening server
                        os.wait()                              # Wait for the intermediate child to exit
                     else:
                        try:
                           fork2PID = os.fork()
                        except OSError as e:
                           self.logger.log(logging.ERROR,getLogMessage("Daemonizing fork2 failed: %s." % (e.args[1])))
                           self.exit(e.args[0])
                        else:
                           if fork2PID != 0:
                              sys.exit(0) # This is the intermediate child.  Exit.
                           else:
                              # This is the real child.
                              os.setsid()
                              self.serverData['type']     = 'JobExecuter'
                              self.serverData['serverId'] = uuid.uuid4()
#                             self.logger.log(logging.INFO,getLogMessage("serverId: %s" % (self.serverData['serverId'])))
                              self.scheduleTimeout()
                              if self.clientListener.acceptHandshake(readyReader):
                                 self.serverData['handshakeComplete'] = True
                                 self.clientListener.closeListeningConnections()
                              else:
                                 self.logger.log(logging.ERROR,getLogMessage("Client connection acceptHandshake failed."))
                                 self.exit(1)
            elif readyReader in clientReader:
               self.clientListener.receiveMessage()
            elif readyReader in attachListenerSocket:
               if self.attachListener.acceptConnection(readyReader):
                  if not self.attachListener.acceptHandshake(readyReader):
                     self.logger.log(logging.ERROR,getLogMessage("Attach connection acceptHandshake failed."))
               else:
                  self.logger.log(logging.ERROR,getLogMessage("Attach connection failed."))
            elif readyReader in attachReader:
               self.attachListener.receiveMessage()
            elif readyReader in serverReader:
               self.serverConnection.receiveMessage()
            else:
               self.readFile(readyReader)

         if self.serverData['type'] == 'JobExecuter':
            self.processClientRequests()
            self.processAttachRequests()
            if missingSocket and not stopForMissingSocketPending:
               stopForMissingSocketPending = True
               self.stopForMissingSocket()
         if self.serverData['type'] == 'Proxy' and not self.serverData['attached']:
            self.processServerRequests()
         if self.serverData['type'] == 'Cache':
            self.processClientRequests()

         for readyWriter in readyWriters:
            if   readyWriter in clientWriter:
#              self.logger.log(logging.DEBUG,getLogMessage("clientListener.sendMessage"))
               self.clientListener.sendMessage()
            elif readyWriter in attachWriter:
               self.attachListener.sendMessage()
            elif readyWriter in serverWriter:
               self.serverConnection.sendMessage()
            elif readyWriter in importObjects:
               self.writeFile(readyWriter)
            elif readyWriter in outputObjects:
               self.writeFile(readyWriter)
         self.checkClosePendingObjects()

         if self.serverData['readyToExit']:
            if not self.serverData['detached']:
               if   not self.clientListener.isMessagePending():
                  message = "Server(%s) is ready to exit, no pending instructions." % (self.serverData['type'])
                  self.logger.log(logging.ERROR,getLogMessage(message))
                  self.exit()
               elif not self.clientListener.isConnected():
                  message = "Server(%s) is ready to exit, client is not connected." % (self.serverData['type'])
                  self.logger.log(logging.ERROR,getLogMessage(message))
                  self.exit()

         if self.serverData['childHasExited']:
            if not self.clientListener.isConnected():
               if not self.serverData['detached']:
                  self.logger.log(logging.ERROR,getLogMessage("Server child has exited, client is not connected."))
               self.exit()

         if self.serverData['type'] == 'Proxy':
            if not self.clientListener.isConnected() and not self.serverConnection.isConnected():
               self.logger.log(logging.ERROR,getLogMessage("Server proxy client is not connected, and server is not connected."))
               self.exit()
            if self.serverData['attached'] and not self.serverConnection.isConnected():
               self.logger.log(logging.ERROR,getLogMessage("Server proxy client is attached, and server is not connected."))
               self.serverData['attached'] = False
               self.exit()

         if self.serverData['type'] == 'JobExecuter':
            if not (self.clientListener.isConnected() or self.serverData['detached']):
               # equivalent to 'detachSignal'
               if not self.serverData['detach']:
                  if self.createDetachedSession():
                     if self.updateJobSession():
                        self.serverData['detach'] = True
                        # equivalent to 'detach'
                        self.clientListener.closeConnection()
                        if self.setupAttachListener():
                           self.serverData['detached'] = True
                        else:
                           message = "Unable to detach job %d" % (self.serverData['jobId'])
                           self.logger.log(logging.ERROR,getLogMessage(message))

         if self.serverData['authz'] and self.serverData['type'] == 'JobExecuter':
            lastIOTimes = []
            if self.clientListener and self.clientListener.isConnected():
               lastIOTimes.append(self.clientListener.getConnectionReadTime())
#              lastIOTimes.append(self.clientListener.getConnectionWriteTime())
            if self.attachListener and self.attachListener.isConnected():
               lastIOTimes.append(self.attachListener.getConnectionReadTime())
#              lastIOTimes.append(self.attachListener.getConnectionWriteTime())

            if lastIOTimes:
               timeSinceLastIO = time.time() - max(lastIOTimes)
               if self.clientListener and self.clientListener.isConnected():
                  if (not self.clientData['localExecution']) and \
                     (not self.clientData['suspended']) and \
                     (not self.clientData['isBuildingInputTar']) and \
                     (not self.serverData['isExportingOutputTar']):
                     if timeSinceLastIO > self.serverData['disconnectMax']:
                        self.logger.log(logging.ERROR,getLogMessage("client disconnected: detached = %s, lastIO = %s" % \
                                                            (self.serverData['detached'],time.ctime(max(lastIOTimes)))))
                        # equivalent to 'detachSignal'
                        if not self.serverData['detach']:
                           if self.createDetachedSession():
                              if self.updateJobSession():
                                 self.serverData['detach'] = True
                                 # equivalent to 'detach'
                                 self.clientListener.closeConnection()
                                 if self.setupAttachListener():
                                    self.serverData['detached'] = True
                                 else:
                                    message = "Unable to detach job %d" % (self.serverData['jobId'])
                                    self.logger.log(logging.ERROR,getLogMessage(message))

         if self.serverData['childPid']:
            pid = 0
            try:
               (pid,exitCode) = os.waitpid(self.serverData['childPid'],os.WNOHANG)
               if exitCode != 0:
                  if os.WIFSIGNALED(exitCode):
                     exitCode = (1 << 7) | os.WTERMSIG(exitCode)
                  else:
                     if os.WIFEXITED(exitCode):
                        exitCode = os.WEXITSTATUS(exitCode)
            except:
               try:
                  os.kill(self.serverData['childPid'],0)
               except:
                  self.logger.log(logging.INFO,getLogMessage("Child has exited."))
                  pid = self.serverData['childPid']
                  exitCode = 1
            if pid != 0:
               self.serverData['childHasExited']  = True
               self.serverData['whenChildExited'] = time.time()
               self.serverData['childPid'] = 0
               serverMessage = {'messageType':'childHasExited',
                                'childHasExited':self.serverData['childHasExited']}
               self.clientListener.postJsonMessage(serverMessage)

               while len(self.serverData['inputObjects']) > 0:
                  inputObjects = copy.copy(list(self.serverData['inputObjects'].keys()))
                  for inputObject in inputObjects:
                     self.readFile(inputObject)
                  del inputObjects

               if not self.commandParser.getOption('disableJobMonitoring') and \
                  (self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNDISTRIBUTORID or \
                   self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNCLIENTLOCAL or \
                   self.serverData['operationMode'] & self.commandParser.OPERATIONMODERUNHARVESTID):
                  exitCodePreset = self.serverData['exitCode'] != 2
                  metricsRecords = self.serverData['metrics'].split('\n')
                  metricsParsed = False
                  for metricsRecord in metricsRecords:
                     if metricsRecord != "":
                        if metricsRecord.startswith('runStatus='):
                           key,value = metricsRecord.split('=')
                           self.serverData['distributorExitCode'] = int(value)
                           if not exitCodePreset:
                              if self.serverData['distributorExitCode'] != self.serverData['exitCode']:
                                 self.logger.log(logging.WARNING,getLogMessage("exitCode mismatch: %d %d" % \
                                     (self.serverData['exitCode'],self.serverData['distributorExitCode'])))
                           self.serverData['exitCode'] = self.serverData['distributorExitCode']
                        else:
                           self.serverData['metricsRecorded'] = False
                           self.updateJobMetrics(metricsRecord)
                           metricsParsed = True
                  if not metricsParsed:
                     self.updateJobMetrics("status=%d" % (self.serverData['exitCode']))
               else:
                  self.serverData['exitCode'] = exitCode

               if os.path.exists(self.serverData['jobScanPath']):
                  try:
                     fpJobScan = open(self.serverData['jobScanPath'],'r')
                     try:
                        jobScanText = fpJobScan.readlines()
                     except (IOError,OSError):
                        pass
                     else:
                        serverMessage = {'messageType':'jobScanUpdate',
                                         'text':''.join(jobScanText)}
                        self.clientListener.postJsonMessage(serverMessage)
                     finally:
                        fpJobScan.close()
                  except (IOError,OSError):
                     self.logger.log(logging.ERROR,getLogMessage("%s could not be opened" % (self.serverData['jobScanPath'])))
                  os.remove(self.serverData['jobScanPath'])


