# @package      hubzero-submit-monitors
# @file         JobMonitorSQL.py
# @copyright    Copyright (c) 2012-2020 The Regents of the University of California.
# @license      http://opensource.org/licenses/MIT MIT
#
# Copyright (c) 2012-2020 The Regents of the University of California.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# HUBzero is a registered trademark of The Regents of the University of California.
#
import os
import re
import pwd
import copy
import math
import time
import signal
import traceback
import json
import base64
import logging
import requests

from hubzero.submit.LogMessage      import getLogIDMessage as getLogMessage
from hubzero.submit.BoundConnection import BoundConnection
from hubzero.submit.MySQLDatabase   import MySQLDatabase as MyDatabase
from hubzero.submit.JobInfo         import JobInfo
from hubzero.submit.InfosInfo       import InfosInfo
from hubzero.submit.SitesInfo       import SitesInfo
from hubzero.submit.AggregatorsInfo import AggregatorsInfo

try:
   iterRange = xrange
except NameError as e:
   iterRange = range

class JobMonitorSQL(BoundConnection):
   def __init__(self,
                configurationDirectory,
                monitorConfigurationFile,
                infosDirectory,
                infosConfigurationFile,
                listenURI):
      BoundConnection.__init__(self,[listenURI])

      self.logger = logging.getLogger(__name__)

      self.configurationDirectory = configurationDirectory
      self.infosDirectory         = infosDirectory
      self.infosConfigurationFile = infosConfigurationFile
      self.monitorConfigFilePath  = os.path.join(configurationDirectory,monitorConfigurationFile)

      self.infosInfo       = None
      self.sitesInfo       = None
      self.aggregatorsInfo = None
      self.settingInfo     = False

      self.terminating = False

      self.configData = {}
      self.myDatabase = None


   def configure(self):
      sectionPattern   = re.compile(r'(\s*\[)([^\s]*)(]\s*)')
      keyValuePattern  = re.compile(r'( *)(\w*)( *= *)(.*[^\s$])( *)')
      commentPattern   = re.compile(r'\s*#.*')
      inMonitorSection = False

      configured = False
      try:
         fpConfig = open(self.monitorConfigFilePath,'r')
         try:
            eof = False
            while not eof:
               record = fpConfig.readline()
               if record != "":
                  record = commentPattern.sub("",record)
                  if   sectionPattern.match(record):
                     sectionName = sectionPattern.match(record).group(2)
                     inMonitorSection = (sectionName == 'monitor')
                     if inMonitorSection:
                        self.configData = {'mysqlHost':"",
                                           'mysqlUser':"",
                                           'mysqlPassword':"",
                                           'mysqlDB':"",
                                           'timeBetweenJobRelease':5,
                                           'timeBetweenJobStageInput':2,
                                           'timeBetweenJobExecute':5,
                                           'timeBetweenJobMetrics':5,
                                           'timeBetweenJobCacheSimTools':5,
                                           'webServiceAPI':"",
                                           'webServiceRegister':"postWSjob",
                                           'webServiceStageInput':"stageInputsWSjob",
                                           'webServiceExecute':"executeStagedWSjob",
                                           'webServiceCacheResult':"cacheWSjob",
                                           'webServiceMetrics':"measureWSjob",
                                           'webServiceRegisterComplete':"postCompleteWSjob",
                                           'webServiceStageInputComplete':"stageInputCompleteWSjob",
                                           'webServiceCacheResultComplete':"cacheCompleteWSjob",
                                           'webServiceMetricsComplete':"measureCompleteWSjob"
                                          }
                  elif inMonitorSection:
                     if keyValuePattern.match(record):
                        key,value = keyValuePattern.match(record).group(2,4)
                        if key in self.configData:
                           if   isinstance(self.configData[key],list):
                              self.configData[key] = [e.strip() for e in value.split(',')]
                           elif isinstance(self.configData[key],bool):
                              self.configData[key] = bool(value.lower() == 'true')
                           elif isinstance(self.configData[key],float):
                              self.configData[key] = float(value)
                           elif isinstance(self.configData[key],int):
                              self.configData[key] = int(value)
                           elif isinstance(self.configData[key],dict):
                              try:
                                 sampleKey   = list(self.configData[key].keys())[0]
                                 sampleValue = self.configData[key][sampleKey]
                              except:
                                 try:
                                    self.configData[key] = json.loads(value)
                                 except:
                                    self.configData[key] = {}
                                    sampleKey   = "key"
                                    sampleValue = "value"
                              else:
                                 self.configData[key] = {}
               
                              if not self.configData[key]:
                                 for e in value.split(','):
                                    dictKey,dictValue = e.split(':')
                                    if isinstance(sampleKey,int):
                                       dictKey = int(dictKey)
                                    if   isinstance(sampleValue,int):
                                       dictValue = int(dictValue)
                                    elif isinstance(sampleValue,float):
                                       dictValue = float(dictValue)
                                    elif isinstance(sampleValue,bool):
                                       dictValue = bool(dictValue.lower() == 'true')
                                    self.configData[key][dictKey] = dictValue
                           else:
                              self.configData[key] = value
                        else:
                           self.logger.log(logging.WARNING,getLogMessage("Undefined key = value pair %s = %s" % (key,value)))
               else:
                  eof = True
                  if self.configData:
                     configured = True
         except (IOError,OSError):
            self.logger.log(logging.ERROR,getLogMessage("%s could not be read" % (self.monitorConfigFilePath)))
         finally:
            fpConfig.close()
      except (IOError,OSError):
         self.logger.log(logging.ERROR,getLogMessage("%s could not be opened" % (self.monitorConfigFilePath)))

      if configured:
         if self.configData['mysqlHost'] == "" or \
            self.configData['mysqlUser'] == "" or \
            self.configData['mysqlPassword'] == "" or \
            self.configData['mysqlDB'] == "":
            self.logger.log(logging.ERROR,getLogMessage("MySQL information missing from %s" % (self.monitorConfigFilePath)))
            configured = False
         else:
            self.myDatabase = MyDatabase(self.configData['mysqlHost'],self.configData['mysqlUser'],self.configData['mysqlPassword'])

      return(configured)


   def killJobSiteMonitors(self):
      self.signalJobSiteMonitors(signal.SIGTERM)
      time.sleep(5)
      if self.countRunningJobSiteMonitors() > 0:
         self.signalJobSiteMonitors(signal.SIGKILL)
      self.purgeActiveJobSites()


   def updateSchema(self,
                    targetSchema):
      schemaUpdated = False
      if self.myDatabase.connect(self.configData['mysqlDB']):
         userVersion = self.myDatabase.getDBuserVersion()
         if userVersion >= 0:
            while userVersion < targetSchema:
               if   userVersion == 0:
                  sqlScript = """
                                 CREATE TABLE activeJobs (
                                    aggregator     VARCHAR(32) DEFAULT '?',
                                    localJobId     INT         NOT NULL,
                                    instanceId     INT         NOT NULL,
                                    distributorPid INT         NOT NULL,
                                    hubUserId      INT         DEFAULT NULL,
                                    siteName       VARCHAR(32) DEFAULT '?',
                                    nCores         INT         DEFAULT 1,
                                    runName        VARCHAR(32) NOT NULL,
                                    siteDesignator VARCHAR(32) NOT NULL,
                                    remoteJobId    VARCHAR(32) NOT NULL,
                                    jobStatus      VARCHAR(32) DEFAULT '?',
                                    jobStage       VARCHAR(32) DEFAULT '?',
                                    jobQueue       VARCHAR(32) DEFAULT '?',
                                    destination    VARCHAR(32) DEFAULT '?',
                                    executionHost  VARCHAR(64) DEFAULT '?',
                                    timeRecorded   DOUBLE      DEFAULT 0.
                                 );
                                 CREATE UNIQUE INDEX activeJobs_globalJobId       ON activeJobs (siteDesignator,remoteJobId);
                                 CREATE        INDEX activeJobs_localJobId        ON activeJobs (localJobId,instanceId);
                                 CREATE        INDEX activeJobs_siteDesignator    ON activeJobs (siteDesignator);
                                 CREATE        INDEX activeJobs_siteQueue         ON activeJobs (siteDesignator,jobQueue);
                                 CREATE        INDEX activeJobs_siteStatus        ON activeJobs (siteDesignator,jobStatus);
                                 CREATE        INDEX activeJobs_instanceStatus    ON activeJobs (instanceId,jobStatus);
                                 CREATE        INDEX activeJobs_hubUserId         ON activeJobs (hubUserId);
                                 CREATE        INDEX activeJobs_userStatus        ON activeJobs (hubUserId,jobStatus);
                                 CREATE        INDEX activeJobs_userLocalIdStatus ON activeJobs (hubUserId,localJobId,jobStatus);
                                 CREATE        INDEX activeJobs_statusTime        ON activeJobs (jobStatus,timeRecorded);

                                 CREATE TABLE activeJobIdentities (
                                    localJobId   INT         NOT NULL,
                                    instanceId   INT         NOT NULL,
                                    hubUserId    INT         NOT NULL,
                                    identityName VARCHAR(32) NOT NULL
                                 );
                                 CREATE INDEX activeJobIdentities_localJobId   ON activeJobIdentities (localJobId,instanceId);
                                 CREATE INDEX activeJobIdentities_identityName ON activeJobIdentities (identityName);
                                 CREATE INDEX activeJobIdentities_identityUser ON activeJobIdentities (identityName,hubUserId);

                                 CREATE TABLE activeJobSites (
                                    jobSite     VARCHAR(32) UNIQUE PRIMARY KEY,
                                    pid         INT         NOT NULL,
                                    timeUpdated DOUBLE      DEFAULT 0.
                                 );

                                 CREATE TABLE userActivityScores (
                                    aggregator    VARCHAR(32) DEFAULT '?',
                                    siteName      VARCHAR(32) DEFAULT '?',
                                    hubUserId     INT         NOT NULL,
                                    activityScore DOUBLE      DEFAULT 0.5,
                                    timeUpdated   DOUBLE      DEFAULT -1.
                                 );
                                 CREATE INDEX userActivityScores_hubUserId  ON userActivityScores (hubUserId);
                                 CREATE INDEX userActivityScores_aggregator ON userActivityScores (aggregator,hubUserId);
                                 CREATE INDEX userActivityScores_siteUser   ON userActivityScores (siteName,hubUserId);

                                 CREATE TABLE fileTailings (
                                    siteDesignator VARCHAR(32)  NOT NULL,
                                    remoteJobId    VARCHAR(32)  NOT NULL,
                                    fileName       VARCHAR(256) NOT NULL,
                                    timeUpdated    DOUBLE       DEFAULT 0.,
                                    tailText       BLOB
                                 );
                                 CREATE UNIQUE INDEX fileTailings_globalTailId ON fileTailings (siteDesignator,remoteJobId,fileName);
                                 CREATE        INDEX fileTailings_globalRunId  ON fileTailings (siteDesignator,remoteJobId);

                                 CREATE TABLE registeredJobs (
                                    aggregator     VARCHAR(32) DEFAULT '?',
                                    state          INT         NOT NULL,
                                    localJobId     INT         NOT NULL,
                                    instanceId     INT         NOT NULL,
                                    distributorPid INT         NOT NULL,
                                    hubUserId      INT         DEFAULT NULL,
                                    siteName       VARCHAR(32) DEFAULT '?',
                                    timeRegistered DOUBLE      DEFAULT 0.
                                 );
                                 CREATE INDEX registeredJobs_aggregatorState ON registeredJobs (aggregator,state);
                                 CREATE INDEX registeredJobs_siteName        ON registeredJobs (siteName);
                                 CREATE INDEX registeredJobs_hubUserId       ON registeredJobs (hubUserId);
                                 CREATE INDEX registeredJobs_userSiteName    ON registeredJobs (hubUserId,siteName);
                                 CREATE INDEX registeredJobs_localJobId      ON registeredJobs (localJobId,instanceId);

                                 CREATE TABLE userSubmitterClasses (
                                    hubUserId      INT UNIQUE PRIMARY KEY,
                                    submitterClass INT NOT NULL
                                 );
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=1)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Creation of tables to hold job monitoring information for in flight jobs")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 1:
                  sqlScript = """
                                 CREATE TABLE registeredJobs_X (
                                    aggregator     VARCHAR(32) DEFAULT '?',
                                    state          INT         NOT NULL,
                                    runName        VARCHAR(32) NOT NULL,
                                    localJobId     INT         NOT NULL,
                                    instanceId     INT         NOT NULL,
                                    distributorPid INT         NOT NULL,
                                    hubUserId      INT         DEFAULT NULL,
                                    siteName       VARCHAR(32) DEFAULT '?',
                                    timeRegistered DOUBLE      DEFAULT 0.
                                 );

                                 INSERT INTO registeredJobs_X (aggregator,state,runName,localJobId,instanceId,
                                                               distributorPid,hubUserId,siteName,timeRegistered)
                                           SELECT aggregator,state,'?',localJobId,instanceId,
                                                  distributorPid,hubUserId,siteName,timeRegistered FROM registeredJobs;

                                 DROP TABLE registeredJobs;
                                 ALTER TABLE registeredJobs_X RENAME TO registeredJobs;

                                 CREATE INDEX registeredJobs_aggregatorState ON registeredJobs (aggregator,state);
                                 CREATE INDEX registeredJobs_siteName        ON registeredJobs (siteName);
                                 CREATE INDEX registeredJobs_hubUserId       ON registeredJobs (hubUserId);
                                 CREATE INDEX registeredJobs_userSiteName    ON registeredJobs (hubUserId,siteName);
                                 CREATE INDEX registeredJobs_localJobId      ON registeredJobs (localJobId,instanceId);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=2)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added runName to the registered job table")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 2:
                  sqlScript = """
                                 CREATE TABLE fileTailings_X (
                                    siteDesignator VARCHAR(32)  NOT NULL,
                                    remoteJobId    VARCHAR(32)  NOT NULL,
                                    fileName       VARCHAR(256) NOT NULL,
                                    nLines         INT          DEFAULT 10,
                                    timeUpdated    DOUBLE       DEFAULT 0.,
                                    tailText       BLOB
                                 );

                                 INSERT INTO fileTailings_X (siteDesignator,remoteJobId,fileName,nLines,timeUpdated,tailText)
                                           SELECT siteDesignator,remoteJobId,fileName,'?',timeUpdated,tailText 
                                                  FROM fileTailings;

                                 DROP TABLE fileTailings;
                                 ALTER TABLE fileTailings_X RENAME TO fileTailings;

                                 CREATE UNIQUE INDEX fileTailings_globalTailId ON fileTailings (siteDesignator,remoteJobId,fileName);
                                 CREATE        INDEX fileTailings_globalRunId  ON fileTailings (siteDesignator,remoteJobId);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=3)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added nLines to the file tailings table")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 3:
                  sqlScript = """
                                 CREATE TABLE pendingJobSites (
                                    jobSite     VARCHAR(32) UNIQUE PRIMARY KEY,
                                    timeUpdated DOUBLE      DEFAULT 0.
                                 );

                                 CREATE TABLE pendingJobs (
                                    siteDesignator   VARCHAR(32)   NOT NULL,
                                    remoteJobId      VARCHAR(32)   NOT NULL,
                                    jobWorkDirectory VARCHAR(2048) NOT NULL,
                                    localJobId       VARCHAR(32)   NOT NULL,
                                    instanceId       VARCHAR(32)   NOT NULL,
                                    runName          VARCHAR(32)   NOT NULL,
                                    timeRecorded     DOUBLE        DEFAULT 0.
                                 );
                                 CREATE UNIQUE INDEX pendingJobs_globalJobId    ON pendingJobs (siteDesignator,remoteJobId);
                                 CREATE        INDEX pendingJobs_siteDesignator ON pendingJobs (siteDesignator);

                                 CREATE TABLE restartJobSites (
                                    jobSite     VARCHAR(32) UNIQUE PRIMARY KEY,
                                    timeUpdated DOUBLE      DEFAULT 0.
                                 );
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=4)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added tables to share data between job and jobsite monitors")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 4:
                  sqlScript = """
                                 CREATE TABLE pendingJobs_X (
                                    state            INT           NOT NULL,
                                    siteDesignator   VARCHAR(32)   NOT NULL,
                                    remoteJobId      VARCHAR(32)   NOT NULL,
                                    jobWorkDirectory VARCHAR(2048) NOT NULL,
                                    localJobId       VARCHAR(32)   NOT NULL,
                                    instanceId       VARCHAR(32)   NOT NULL,
                                    runName          VARCHAR(32)   NOT NULL,
                                    timeRecorded     DOUBLE        DEFAULT 0.
                                 );

                                 INSERT INTO pendingJobs_X (state,siteDesignator,remoteJobId,jobWorkDirectory,
                                                            localJobId,instanceId,runName,timeRecorded)
                                           SELECT 1,siteDesignator,remoteJobId,jobWorkDirectory,
                                                  localJobId,instanceId,runName,timeRecorded
                                                  FROM pendingJobs;

                                 DROP TABLE pendingJobs;
                                 ALTER TABLE pendingJobs_X RENAME TO pendingJobs;

                                 CREATE UNIQUE INDEX pendingJobs_globalJobId         ON pendingJobs (siteDesignator,remoteJobId);
                                 CREATE        INDEX pendingJobs_siteDesignator      ON pendingJobs (siteDesignator);
                                 CREATE        INDEX pendingJobs_state               ON pendingJobs (state);
                                 CREATE        INDEX pendingJobs_stateSiteDesignator ON pendingJobs (state,siteDesignator);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=5)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added indexed state to pendingJobs table")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 5:
                  sqlScript = """
                                 ALTER TABLE activeJobs MODIFY remoteJobId VARCHAR(40);
                                 ALTER TABLE fileTailings MODIFY remoteJobId VARCHAR(40);
                                 ALTER TABLE pendingJobs MODIFY remoteJobId VARCHAR(40);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=6)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Increased maximum length of remoteJobId")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 6:
                  sqlScript = """
                                 CREATE TABLE registeredJobs_X (
                                    instanceToken  CHAR(32)    DEFAULT '?',
                                    aggregator     VARCHAR(32) DEFAULT '?',
                                    state          INT         NOT NULL,
                                    runName        VARCHAR(32) NOT NULL,
                                    localJobId     INT         NOT NULL,
                                    instanceId     INT         NOT NULL,
                                    distributorPid INT         NOT NULL,
                                    hubUserId      INT         DEFAULT NULL,
                                    siteName       VARCHAR(32) DEFAULT '?',
                                    timeRegistered DOUBLE      DEFAULT 0.
                                 );

                                 INSERT INTO registeredJobs_X (instanceToken,aggregator,state,runName,localJobId,instanceId,
                                                               distributorPid,hubUserId,siteName,timeRegistered)
                                           SELECT '?',aggregator,state,runName,localJobId,instanceId,
                                                  distributorPid,hubUserId,siteName,timeRegistered FROM registeredJobs;

                                 DROP TABLE registeredJobs;
                                 ALTER TABLE registeredJobs_X RENAME TO registeredJobs;

                                 CREATE INDEX registeredJobs_aggregatorState ON registeredJobs (aggregator,state);
                                 CREATE INDEX registeredJobs_siteName        ON registeredJobs (siteName);
                                 CREATE INDEX registeredJobs_hubUserId       ON registeredJobs (hubUserId);
                                 CREATE INDEX registeredJobs_userSiteName    ON registeredJobs (hubUserId,siteName);
                                 CREATE INDEX registeredJobs_localJobId      ON registeredJobs (localJobId,instanceId);

                                 CREATE TABLE activeJobs_X (
                                    instanceToken  CHAR(32)    DEFAULT '?',
                                    aggregator     VARCHAR(32) DEFAULT '?',
                                    localJobId     INT         NOT NULL,
                                    instanceId     INT         NOT NULL,
                                    distributorPid INT         NOT NULL,
                                    hubUserId      INT         DEFAULT NULL,
                                    siteName       VARCHAR(32) DEFAULT '?',
                                    nCores         INT         DEFAULT 1,
                                    runName        VARCHAR(32) NOT NULL,
                                    siteDesignator VARCHAR(32) NOT NULL,
                                    remoteJobId    VARCHAR(40) NOT NULL,
                                    jobStatus      VARCHAR(32) DEFAULT '?',
                                    jobStage       VARCHAR(32) DEFAULT '?',
                                    jobQueue       VARCHAR(32) DEFAULT '?',
                                    destination    VARCHAR(32) DEFAULT '?',
                                    executionHost  VARCHAR(64) DEFAULT '?',
                                    timeRecorded   DOUBLE      DEFAULT 0.
                                 );

                                 INSERT INTO activeJobs_X (instanceToken,aggregator,localJobId,instanceId,distributorPid,
                                                           hubUserId,siteName,nCores,runName,siteDesignator,remoteJobId,
                                                           jobStatus,jobStage,jobQueue,destination,executionHost,timeRecorded)
                                           SELECT '?',aggregator,localJobId,instanceId,distributorPid,
                                                  hubUserId,siteName,nCores,runName,siteDesignator,remoteJobId,
                                                  jobStatus,jobStage,jobQueue,destination,executionHost,timeRecorded FROM activeJobs;

                                 DROP TABLE activeJobs;
                                 ALTER TABLE activeJobs_X RENAME TO activeJobs;

                                 CREATE UNIQUE INDEX activeJobs_globalJobId       ON activeJobs (siteDesignator,remoteJobId);
                                 CREATE        INDEX activeJobs_localJobId        ON activeJobs (localJobId,instanceId);
                                 CREATE        INDEX activeJobs_siteDesignator    ON activeJobs (siteDesignator);
                                 CREATE        INDEX activeJobs_siteQueue         ON activeJobs (siteDesignator,jobQueue);
                                 CREATE        INDEX activeJobs_siteStatus        ON activeJobs (siteDesignator,jobStatus);
                                 CREATE        INDEX activeJobs_instanceStatus    ON activeJobs (instanceId,jobStatus);
                                 CREATE        INDEX activeJobs_hubUserId         ON activeJobs (hubUserId);
                                 CREATE        INDEX activeJobs_userStatus        ON activeJobs (hubUserId,jobStatus);
                                 CREATE        INDEX activeJobs_userLocalIdStatus ON activeJobs (hubUserId,localJobId,jobStatus);
                                 CREATE        INDEX activeJobs_statusTime        ON activeJobs (jobStatus,timeRecorded);
                                 CREATE        INDEX activeJobs_instanceToken     ON activeJobs (instanceToken);

                                 CREATE TABLE pendingJobs_X (
                                    instanceToken    CHAR(32)      DEFAULT '?',
                                    state            INT           NOT NULL,
                                    siteDesignator   VARCHAR(32)   NOT NULL,
                                    remoteJobId      VARCHAR(40)   NOT NULL,
                                    jobWorkDirectory VARCHAR(2048) NOT NULL,
                                    localJobId       VARCHAR(32)   NOT NULL,
                                    instanceId       VARCHAR(32)   NOT NULL,
                                    runName          VARCHAR(32)   NOT NULL,
                                    timeRecorded     DOUBLE        DEFAULT 0.
                                 );

                                 INSERT INTO pendingJobs_X (instanceToken,state,siteDesignator,remoteJobId,jobWorkDirectory,
                                                            localJobId,instanceId,runName,timeRecorded)
                                           SELECT '?',state,siteDesignator,remoteJobId,jobWorkDirectory,
                                                  localJobId,instanceId,runName,timeRecorded
                                                  FROM pendingJobs;

                                 DROP TABLE pendingJobs;
                                 ALTER TABLE pendingJobs_X RENAME TO pendingJobs;

                                 CREATE UNIQUE INDEX pendingJobs_globalJobId         ON pendingJobs (siteDesignator,remoteJobId);
                                 CREATE        INDEX pendingJobs_siteDesignator      ON pendingJobs (siteDesignator);
                                 CREATE        INDEX pendingJobs_state               ON pendingJobs (state);
                                 CREATE        INDEX pendingJobs_stateSiteDesignator ON pendingJobs (state,siteDesignator);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=7)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added instanceToken to job tables")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 7:
                  sqlScript = """
                                 CREATE TABLE activeJobs_X (
                                    instanceToken      CHAR(32)    DEFAULT '?',
                                    aggregator         VARCHAR(32) DEFAULT '?',
                                    state              INT         NOT NULL,
                                    localJobId         INT         NOT NULL,
                                    instanceId         INT         NOT NULL,
                                    distributorPid     INT         NOT NULL,
                                    hubUserId          INT         DEFAULT NULL,
                                    tapisSystemStageIn VARCHAR(64) DEFAULT '',
                                    stagingHandle      VARCHAR(32) DEFAULT '',
                                    siteName           VARCHAR(32) DEFAULT '?',
                                    nCores             INT         DEFAULT 1,
                                    runName            VARCHAR(32) NOT NULL,
                                    siteDesignator     VARCHAR(32) NOT NULL,
                                    remoteJobId        VARCHAR(40) NOT NULL,
                                    jobStatus          VARCHAR(32) DEFAULT '?',
                                    jobStage           VARCHAR(32) DEFAULT '?',
                                    jobQueue           VARCHAR(32) DEFAULT '?',
                                    destination        VARCHAR(32) DEFAULT '?',
                                    executionHost      VARCHAR(64) DEFAULT '?',
                                    timeRecorded       DOUBLE      DEFAULT 0.
                                 );

                                 INSERT INTO activeJobs_X (instanceToken,aggregator,state,localJobId,instanceId,distributorPid,
                                                           hubUserId,tapisSystemStageIn,stagingHandle,
                                                           siteName,nCores,runName,siteDesignator,remoteJobId,
                                                           jobStatus,jobStage,jobQueue,destination,executionHost,timeRecorded)
                                           SELECT instanceToken,aggregator,2,localJobId,instanceId,distributorPid,
                                                  hubUserId,'','',
                                                  siteName,nCores,runName,siteDesignator,remoteJobId,
                                                  jobStatus,jobStage,jobQueue,destination,executionHost,timeRecorded FROM activeJobs;

                                 DROP TABLE activeJobs;
                                 ALTER TABLE activeJobs_X RENAME TO activeJobs;

                                 CREATE UNIQUE INDEX activeJobs_globalJobId       ON activeJobs (siteDesignator,remoteJobId);
                                 CREATE        INDEX activeJobs_localJobId        ON activeJobs (localJobId,instanceId);
                                 CREATE        INDEX activeJobs_siteDesignator    ON activeJobs (siteDesignator);
                                 CREATE        INDEX activeJobs_siteQueue         ON activeJobs (siteDesignator,jobQueue);
                                 CREATE        INDEX activeJobs_siteStatus        ON activeJobs (siteDesignator,jobStatus);
                                 CREATE        INDEX activeJobs_instanceStatus    ON activeJobs (instanceId,jobStatus);
                                 CREATE        INDEX activeJobs_hubUserId         ON activeJobs (hubUserId);
                                 CREATE        INDEX activeJobs_userStatus        ON activeJobs (hubUserId,jobStatus);
                                 CREATE        INDEX activeJobs_userLocalIdStatus ON activeJobs (hubUserId,localJobId,jobStatus);
                                 CREATE        INDEX activeJobs_statusTime        ON activeJobs (jobStatus,timeRecorded);
                                 CREATE        INDEX activeJobs_instanceToken     ON activeJobs (instanceToken);
                                 CREATE        INDEX activeJobs_state             ON activeJobs (state);
                                 CREATE        INDEX activeJobs_userLocalJobId    ON activeJobs (localJobId,instanceId,hubUserId);
                                 CREATE        INDEX activeJobs_userToken         ON activeJobs (instanceToken,hubUserId);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=8)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added active job state to facilitate split mode execution")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 8:
                  sqlScript = """
                                 CREATE TABLE activeJobs_X (
                                    instanceToken      CHAR(32)     DEFAULT '?',
                                    aggregator         VARCHAR(32)  DEFAULT '?',
                                    state              INT          NOT NULL,
                                    localJobId         INT          NOT NULL,
                                    instanceId         INT          NOT NULL,
                                    distributorPid     INT          NOT NULL,
                                    hubUserId          INT          DEFAULT NULL,
                                    tapisSystemStageIn VARCHAR(64)  DEFAULT '',
                                    stagingHandle      VARCHAR(32)  DEFAULT '',
                                    siteName           VARCHAR(32)  DEFAULT '?',
                                    nCores             INT          DEFAULT 1,
                                    runName            VARCHAR(32)  NOT NULL,
                                    siteDesignator     VARCHAR(32)  NOT NULL,
                                    remoteJobId        VARCHAR(40)  NOT NULL,
                                    jobStatus          VARCHAR(32)  DEFAULT '?',
                                    jobStage           VARCHAR(32)  DEFAULT '?',
                                    jobQueue           VARCHAR(32)  DEFAULT '?',
                                    destination        VARCHAR(32)  DEFAULT '?',
                                    executionHost      VARCHAR(64)  DEFAULT '?',
                                    timeRecorded       DOUBLE       DEFAULT 0.,
                                    jobStatistic       VARCHAR(500) DEFAULT '{}'
                                 );

                                 INSERT INTO activeJobs_X (instanceToken,aggregator,state,localJobId,instanceId,distributorPid,
                                                           hubUserId,tapisSystemStageIn,stagingHandle,
                                                           siteName,nCores,runName,siteDesignator,remoteJobId,
                                                           jobStatus,jobStage,jobQueue,
                                                           destination,executionHost,timeRecorded,jobStatistic)
                                           SELECT instanceToken,aggregator,2,localJobId,instanceId,distributorPid,
                                                  hubUserId,tapisSystemStageIn,stagingHandle,
                                                  siteName,nCores,runName,siteDesignator,remoteJobId,
                                                  jobStatus,jobStage,jobQueue,
                                                  destination,executionHost,timeRecorded,'{}' FROM activeJobs;

                                 DROP TABLE activeJobs;
                                 ALTER TABLE activeJobs_X RENAME TO activeJobs;

                                 CREATE UNIQUE INDEX activeJobs_globalJobId       ON activeJobs (siteDesignator,remoteJobId);
                                 CREATE        INDEX activeJobs_localJobId        ON activeJobs (localJobId,instanceId);
                                 CREATE        INDEX activeJobs_siteDesignator    ON activeJobs (siteDesignator);
                                 CREATE        INDEX activeJobs_siteQueue         ON activeJobs (siteDesignator,jobQueue);
                                 CREATE        INDEX activeJobs_siteStatus        ON activeJobs (siteDesignator,jobStatus);
                                 CREATE        INDEX activeJobs_instanceStatus    ON activeJobs (instanceId,jobStatus);
                                 CREATE        INDEX activeJobs_hubUserId         ON activeJobs (hubUserId);
                                 CREATE        INDEX activeJobs_userStatus        ON activeJobs (hubUserId,jobStatus);
                                 CREATE        INDEX activeJobs_userLocalIdStatus ON activeJobs (hubUserId,localJobId,jobStatus);
                                 CREATE        INDEX activeJobs_statusTime        ON activeJobs (jobStatus,timeRecorded);
                                 CREATE        INDEX activeJobs_instanceToken     ON activeJobs (instanceToken);
                                 CREATE        INDEX activeJobs_state             ON activeJobs (state);
                                 CREATE        INDEX activeJobs_userLocalJobId    ON activeJobs (localJobId,instanceId,hubUserId);
                                 CREATE        INDEX activeJobs_userToken         ON activeJobs (instanceToken,hubUserId);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=9)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added JobStatistic to activeJobs")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 9:
                  sqlScript = """
                                 CREATE TABLE registeredJobs_X (
                                    instanceToken  CHAR(32)    DEFAULT '?',
                                    wsJobId        VARCHAR(40) DEFAULT '?',
                                    aggregator     VARCHAR(32) DEFAULT '?',
                                    state          INT         NOT NULL,
                                    runName        VARCHAR(32) NOT NULL,
                                    localJobId     INT         NOT NULL,
                                    instanceId     INT         NOT NULL,
                                    distributorPid INT         NOT NULL,
                                    hubUserId      INT         DEFAULT NULL,
                                    siteName       VARCHAR(32) DEFAULT '?',
                                    timeRegistered DOUBLE      DEFAULT 0.
                                 );

                                 INSERT INTO registeredJobs_X (instanceToken,wsJobId,
                                                               aggregator,state,runName,localJobId,instanceId,
                                                               distributorPid,hubUserId,siteName,timeRegistered)
                                           SELECT instanceToken,'?',
                                                  aggregator,state,runName,localJobId,instanceId,
                                                  distributorPid,hubUserId,siteName,timeRegistered FROM registeredJobs;

                                 DROP TABLE registeredJobs;
                                 ALTER TABLE registeredJobs_X RENAME TO registeredJobs;

                                 CREATE INDEX registeredJobs_aggregatorState ON registeredJobs (aggregator,state);
                                 CREATE INDEX registeredJobs_siteName        ON registeredJobs (siteName);
                                 CREATE INDEX registeredJobs_hubUserId       ON registeredJobs (hubUserId);
                                 CREATE INDEX registeredJobs_userSiteName    ON registeredJobs (hubUserId,siteName);
                                 CREATE INDEX registeredJobs_localJobId      ON registeredJobs (localJobId,instanceId);

                                 CREATE TABLE activeJobs_X (
                                    instanceToken      CHAR(32)     DEFAULT '?',
                                    wsJobId            VARCHAR(40)  DEFAULT '?',
                                    aggregator         VARCHAR(32)  DEFAULT '?',
                                    state              INT          NOT NULL,
                                    localJobId         INT          NOT NULL,
                                    instanceId         INT          NOT NULL,
                                    distributorPid     INT          NOT NULL,
                                    hubUserId          INT          DEFAULT NULL,
                                    tapisSystemStageIn VARCHAR(64)  DEFAULT '',
                                    stagingHandle      VARCHAR(32)  DEFAULT '',
                                    siteName           VARCHAR(32)  DEFAULT '?',
                                    nCores             INT          DEFAULT 1,
                                    runName            VARCHAR(32)  NOT NULL,
                                    siteDesignator     VARCHAR(32)  NOT NULL,
                                    remoteJobId        VARCHAR(40)  NOT NULL,
                                    jobStatus          VARCHAR(32)  DEFAULT '?',
                                    jobStage           VARCHAR(32)  DEFAULT '?',
                                    jobQueue           VARCHAR(32)  DEFAULT '?',
                                    destination        VARCHAR(32)  DEFAULT '?',
                                    executionHost      VARCHAR(64)  DEFAULT '?',
                                    timeRecorded       DOUBLE       DEFAULT 0.,
                                    jobStatistic       VARCHAR(500) DEFAULT '{}'
                                 );

                                 INSERT INTO activeJobs_X (instanceToken,wsJobId,
                                                           aggregator,state,localJobId,instanceId,distributorPid,
                                                           hubUserId,tapisSystemStageIn,stagingHandle,
                                                           siteName,nCores,runName,siteDesignator,remoteJobId,
                                                           jobStatus,jobStage,jobQueue,
                                                           destination,executionHost,timeRecorded,jobStatistic)
                                           SELECT instanceToken,'?',
                                                  aggregator,state,localJobId,instanceId,distributorPid,
                                                  hubUserId,tapisSystemStageIn,stagingHandle,
                                                  siteName,nCores,runName,siteDesignator,remoteJobId,
                                                  jobStatus,jobStage,jobQueue,
                                                  destination,executionHost,timeRecorded,jobStatistic FROM activeJobs;

                                 DROP TABLE activeJobs;
                                 ALTER TABLE activeJobs_X RENAME TO activeJobs;

                                 CREATE UNIQUE INDEX activeJobs_globalJobId       ON activeJobs (siteDesignator,remoteJobId);
                                 CREATE        INDEX activeJobs_localJobId        ON activeJobs (localJobId,instanceId);
                                 CREATE        INDEX activeJobs_siteDesignator    ON activeJobs (siteDesignator);
                                 CREATE        INDEX activeJobs_siteQueue         ON activeJobs (siteDesignator,jobQueue);
                                 CREATE        INDEX activeJobs_siteStatus        ON activeJobs (siteDesignator,jobStatus);
                                 CREATE        INDEX activeJobs_instanceStatus    ON activeJobs (instanceId,jobStatus);
                                 CREATE        INDEX activeJobs_hubUserId         ON activeJobs (hubUserId);
                                 CREATE        INDEX activeJobs_userStatus        ON activeJobs (hubUserId,jobStatus);
                                 CREATE        INDEX activeJobs_userLocalIdStatus ON activeJobs (hubUserId,localJobId,jobStatus);
                                 CREATE        INDEX activeJobs_statusTime        ON activeJobs (jobStatus,timeRecorded);
                                 CREATE        INDEX activeJobs_instanceToken     ON activeJobs (instanceToken);
                                 CREATE        INDEX activeJobs_state             ON activeJobs (state);
                                 CREATE        INDEX activeJobs_userLocalJobId    ON activeJobs (localJobId,instanceId,hubUserId);
                                 CREATE        INDEX activeJobs_userToken         ON activeJobs (instanceToken,hubUserId);

                                 CREATE TABLE pendingJobs_X (
                                    instanceToken    CHAR(32)      DEFAULT '?',
                                    wsJobId          VARCHAR(40)   DEFAULT '?',
                                    state            INT           NOT NULL,
                                    siteDesignator   VARCHAR(32)   NOT NULL,
                                    remoteJobId      VARCHAR(40)   NOT NULL,
                                    jobWorkDirectory VARCHAR(2048) NOT NULL,
                                    localJobId       VARCHAR(32)   NOT NULL,
                                    instanceId       VARCHAR(32)   NOT NULL,
                                    runName          VARCHAR(32)   NOT NULL,
                                    timeRecorded     DOUBLE        DEFAULT 0.
                                 );

                                 INSERT INTO pendingJobs_X (instanceToken,wsJobId,
                                                            state,siteDesignator,remoteJobId,jobWorkDirectory,
                                                            localJobId,instanceId,runName,timeRecorded)
                                           SELECT instanceToken,'?',
                                                  state,siteDesignator,remoteJobId,jobWorkDirectory,
                                                  localJobId,instanceId,runName,timeRecorded
                                                  FROM pendingJobs;

                                 DROP TABLE pendingJobs;
                                 ALTER TABLE pendingJobs_X RENAME TO pendingJobs;

                                 CREATE UNIQUE INDEX pendingJobs_globalJobId         ON pendingJobs (siteDesignator,remoteJobId);
                                 CREATE        INDEX pendingJobs_siteDesignator      ON pendingJobs (siteDesignator);
                                 CREATE        INDEX pendingJobs_state               ON pendingJobs (state);
                                 CREATE        INDEX pendingJobs_stateSiteDesignator ON pendingJobs (state,siteDesignator);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=10)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added wsJobId to job tables")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 10:
                  sqlScript = """
                                 CREATE TABLE registeredJobs_X (
                                    session        INT         NOT NULL,
                                    instanceToken  CHAR(32)    DEFAULT '?',
                                    wsJobId        VARCHAR(40) DEFAULT '?',
                                    aggregator     VARCHAR(32) DEFAULT '?',
                                    state          INT         NOT NULL,
                                    runName        VARCHAR(32) NOT NULL,
                                    localJobId     INT         NOT NULL,
                                    instanceId     INT         NOT NULL,
                                    distributorPid INT         NOT NULL,
                                    hubUserId      INT         DEFAULT NULL,
                                    siteName       VARCHAR(32) DEFAULT '?',
                                    timeRegistered DOUBLE      DEFAULT 0.
                                 );

                                 INSERT INTO registeredJobs_X (session,instanceToken,wsJobId,
                                                               aggregator,state,runName,localJobId,instanceId,
                                                               distributorPid,hubUserId,siteName,timeRegistered)
                                           SELECT 0,instanceToken,wsJobId,
                                                  aggregator,state,runName,localJobId,instanceId,
                                                  distributorPid,hubUserId,siteName,timeRegistered FROM registeredJobs;

                                 DROP TABLE registeredJobs;
                                 ALTER TABLE registeredJobs_X RENAME TO registeredJobs;

                                 CREATE INDEX registeredJobs_aggregatorState ON registeredJobs (aggregator,state);
                                 CREATE INDEX registeredJobs_siteName        ON registeredJobs (siteName);
                                 CREATE INDEX registeredJobs_hubUserId       ON registeredJobs (hubUserId);
                                 CREATE INDEX registeredJobs_userSiteName    ON registeredJobs (hubUserId,siteName);
                                 CREATE INDEX registeredJobs_localJobId      ON registeredJobs (localJobId,instanceId);
                                 CREATE INDEX registeredJobs_session         ON registeredJobs (session);

                                 CREATE TABLE activeJobs_X (
                                    session            INT          NOT NULL,
                                    instanceToken      CHAR(32)     DEFAULT '?',
                                    wsJobId            VARCHAR(40)  DEFAULT '?',
                                    aggregator         VARCHAR(32)  DEFAULT '?',
                                    state              INT          NOT NULL,
                                    localJobId         INT          NOT NULL,
                                    instanceId         INT          NOT NULL,
                                    distributorPid     INT          NOT NULL,
                                    hubUserId          INT          DEFAULT NULL,
                                    tapisSystemStageIn VARCHAR(64)  DEFAULT '',
                                    stagingHandle      VARCHAR(32)  DEFAULT '',
                                    siteName           VARCHAR(32)  DEFAULT '?',
                                    nCores             INT          DEFAULT 1,
                                    runName            VARCHAR(32)  NOT NULL,
                                    siteDesignator     VARCHAR(32)  NOT NULL,
                                    remoteJobId        VARCHAR(40)  NOT NULL,
                                    jobStatus          VARCHAR(32)  DEFAULT '?',
                                    jobStage           VARCHAR(32)  DEFAULT '?',
                                    jobQueue           VARCHAR(32)  DEFAULT '?',
                                    destination        VARCHAR(32)  DEFAULT '?',
                                    executionHost      VARCHAR(64)  DEFAULT '?',
                                    timeRecorded       DOUBLE       DEFAULT 0.,
                                    jobStatistic       VARCHAR(500) DEFAULT '{}'
                                 );

                                 INSERT INTO activeJobs_X (session,instanceToken,wsJobId,
                                                           aggregator,state,localJobId,instanceId,distributorPid,
                                                           hubUserId,tapisSystemStageIn,stagingHandle,
                                                           siteName,nCores,runName,siteDesignator,remoteJobId,
                                                           jobStatus,jobStage,jobQueue,
                                                           destination,executionHost,timeRecorded,jobStatistic)
                                           SELECT 0,instanceToken,wsJobId,
                                                  aggregator,state,localJobId,instanceId,distributorPid,
                                                  hubUserId,tapisSystemStageIn,stagingHandle,
                                                  siteName,nCores,runName,siteDesignator,remoteJobId,
                                                  jobStatus,jobStage,jobQueue,
                                                  destination,executionHost,timeRecorded,jobStatistic FROM activeJobs;

                                 DROP TABLE activeJobs;
                                 ALTER TABLE activeJobs_X RENAME TO activeJobs;

                                 CREATE UNIQUE INDEX activeJobs_globalJobId       ON activeJobs (siteDesignator,remoteJobId);
                                 CREATE        INDEX activeJobs_localJobId        ON activeJobs (localJobId,instanceId);
                                 CREATE        INDEX activeJobs_siteDesignator    ON activeJobs (siteDesignator);
                                 CREATE        INDEX activeJobs_siteQueue         ON activeJobs (siteDesignator,jobQueue);
                                 CREATE        INDEX activeJobs_siteStatus        ON activeJobs (siteDesignator,jobStatus);
                                 CREATE        INDEX activeJobs_instanceStatus    ON activeJobs (instanceId,jobStatus);
                                 CREATE        INDEX activeJobs_hubUserId         ON activeJobs (hubUserId);
                                 CREATE        INDEX activeJobs_userStatus        ON activeJobs (hubUserId,jobStatus);
                                 CREATE        INDEX activeJobs_userLocalIdStatus ON activeJobs (hubUserId,localJobId,jobStatus);
                                 CREATE        INDEX activeJobs_statusTime        ON activeJobs (jobStatus,timeRecorded);
                                 CREATE        INDEX activeJobs_instanceToken     ON activeJobs (instanceToken);
                                 CREATE        INDEX activeJobs_state             ON activeJobs (state);
                                 CREATE        INDEX activeJobs_userLocalJobId    ON activeJobs (localJobId,instanceId,hubUserId);
                                 CREATE        INDEX activeJobs_userToken         ON activeJobs (instanceToken,hubUserId);
                                 CREATE        INDEX activeJobs_session           ON activeJobs (session);

                                 CREATE TABLE pendingJobs_X (
                                    session          INT           NOT NULL,
                                    instanceToken    CHAR(32)      DEFAULT '?',
                                    wsJobId          VARCHAR(40)   DEFAULT '?',
                                    state            INT           NOT NULL,
                                    siteDesignator   VARCHAR(32)   NOT NULL,
                                    remoteJobId      VARCHAR(40)   NOT NULL,
                                    jobWorkDirectory VARCHAR(2048) NOT NULL,
                                    localJobId       VARCHAR(32)   NOT NULL,
                                    instanceId       VARCHAR(32)   NOT NULL,
                                    runName          VARCHAR(32)   NOT NULL,
                                    timeRecorded     DOUBLE        DEFAULT 0.
                                 );

                                 INSERT INTO pendingJobs_X (session,instanceToken,wsJobId,
                                                            state,siteDesignator,remoteJobId,jobWorkDirectory,
                                                            localJobId,instanceId,runName,timeRecorded)
                                           SELECT 0,instanceToken,wsJobId,
                                                  state,siteDesignator,remoteJobId,jobWorkDirectory,
                                                  localJobId,instanceId,runName,timeRecorded
                                                  FROM pendingJobs;

                                 DROP TABLE pendingJobs;
                                 ALTER TABLE pendingJobs_X RENAME TO pendingJobs;

                                 CREATE UNIQUE INDEX pendingJobs_globalJobId         ON pendingJobs (siteDesignator,remoteJobId);
                                 CREATE        INDEX pendingJobs_siteDesignator      ON pendingJobs (siteDesignator);
                                 CREATE        INDEX pendingJobs_state               ON pendingJobs (state);
                                 CREATE        INDEX pendingJobs_stateSiteDesignator ON pendingJobs (state,siteDesignator);
                                 CREATE        INDEX pendingJobs_session             ON pendingJobs (session);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=11)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added session to job tables")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 11:
                  sqlScript = """
                                 CREATE TABLE activeJobs_X (
                                    session            INT           NOT NULL,
                                    instanceToken      CHAR(32)      DEFAULT '?',
                                    wsJobId            VARCHAR(40)   DEFAULT '?',
                                    aggregator         VARCHAR(32)   DEFAULT '?',
                                    state              INT           NOT NULL,
                                    localJobId         INT           NOT NULL,
                                    instanceId         INT           NOT NULL,
                                    distributorPid     INT           NOT NULL,
                                    hubUserId          INT           DEFAULT NULL,
                                    preStageDirectory  VARCHAR(2048) DEFAULT '',
                                    tapisSystemStageIn VARCHAR(64)   DEFAULT '',
                                    stagingHandle      VARCHAR(32)   DEFAULT '',
                                    siteName           VARCHAR(32)   DEFAULT '?',
                                    nCores             INT           DEFAULT 1,
                                    runName            VARCHAR(32)   NOT NULL,
                                    siteDesignator     VARCHAR(32)   NOT NULL,
                                    remoteJobId        VARCHAR(40)   NOT NULL,
                                    jobStatus          VARCHAR(32)   DEFAULT '?',
                                    jobStage           VARCHAR(32)   DEFAULT '?',
                                    jobQueue           VARCHAR(32)   DEFAULT '?',
                                    destination        VARCHAR(32)   DEFAULT '?',
                                    executionHost      VARCHAR(64)   DEFAULT '?',
                                    timeRecorded       DOUBLE        DEFAULT 0.,
                                    jobStatistic       VARCHAR(500)  DEFAULT '{}'
                                 );

                                 INSERT INTO activeJobs_X (session,instanceToken,wsJobId,
                                                           aggregator,state,localJobId,instanceId,distributorPid,
                                                           hubUserId,preStageDirectory,tapisSystemStageIn,stagingHandle,
                                                           siteName,nCores,runName,siteDesignator,remoteJobId,
                                                           jobStatus,jobStage,jobQueue,
                                                           destination,executionHost,timeRecorded,jobStatistic)
                                           SELECT 0,instanceToken,wsJobId,
                                                  aggregator,state,localJobId,instanceId,distributorPid,
                                                  hubUserId,"",tapisSystemStageIn,stagingHandle,
                                                  siteName,nCores,runName,siteDesignator,remoteJobId,
                                                  jobStatus,jobStage,jobQueue,
                                                  destination,executionHost,timeRecorded,jobStatistic FROM activeJobs;

                                 DROP TABLE activeJobs;
                                 ALTER TABLE activeJobs_X RENAME TO activeJobs;

                                 CREATE UNIQUE INDEX activeJobs_globalJobId       ON activeJobs (siteDesignator,remoteJobId);
                                 CREATE        INDEX activeJobs_localJobId        ON activeJobs (localJobId,instanceId);
                                 CREATE        INDEX activeJobs_wsJobId           ON activeJobs (wsJobId);
                                 CREATE        INDEX activeJobs_siteDesignator    ON activeJobs (siteDesignator);
                                 CREATE        INDEX activeJobs_siteQueue         ON activeJobs (siteDesignator,jobQueue);
                                 CREATE        INDEX activeJobs_siteStatus        ON activeJobs (siteDesignator,jobStatus);
                                 CREATE        INDEX activeJobs_instanceStatus    ON activeJobs (instanceId,jobStatus);
                                 CREATE        INDEX activeJobs_hubUserId         ON activeJobs (hubUserId);
                                 CREATE        INDEX activeJobs_userStatus        ON activeJobs (hubUserId,jobStatus);
                                 CREATE        INDEX activeJobs_userLocalIdStatus ON activeJobs (hubUserId,localJobId,jobStatus);
                                 CREATE        INDEX activeJobs_statusTime        ON activeJobs (jobStatus,timeRecorded);
                                 CREATE        INDEX activeJobs_instanceToken     ON activeJobs (instanceToken);
                                 CREATE        INDEX activeJobs_state             ON activeJobs (state);
                                 CREATE        INDEX activeJobs_userLocalJobId    ON activeJobs (localJobId,instanceId,hubUserId);
                                 CREATE        INDEX activeJobs_userToken         ON activeJobs (instanceToken,hubUserId);
                                 CREATE        INDEX activeJobs_session           ON activeJobs (session);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=12)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added preStageDirectory")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 12:
                  sqlScript = """
                                 CREATE TABLE wsJobs (
                                    wsJobId       VARCHAR(40) NOT NULL,
                                    state         INT         NOT NULL,
                                    hubUserId     INT         DEFAULT NULL,
                                    siteName      VARCHAR(32) DEFAULT '?',
                                    nCores        INT         DEFAULT 1,
                                    wallTimeLimit DOUBLE      DEFAULT 0.,
                                    timeEnrolled  DOUBLE      DEFAULT 0.,
                                    timePosted    DOUBLE      DEFAULT 0.
                                 );
                                 CREATE UNIQUE INDEX wsJobs_wsJobId   ON wsJobs (wsJobId);
                                 CREATE        INDEX wsJobs_state     ON wsJobs (state);
                                 CREATE        INDEX wsJobs_hubUserId ON wsJobs (hubUserId);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=13)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added wsJobs table")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 13:
                  sqlScript = """
                                 CREATE TABLE wsJobs_X (
                                    id            INT         NOT NULL AUTO_INCREMENT PRIMARY KEY,
                                    wsJobId       VARCHAR(40) NOT NULL,
                                    state         INT         NOT NULL,
                                    hubUserId     INT         DEFAULT NULL,
                                    toolRevision  VARCHAR(32) NOT NULL,
                                    siteName      VARCHAR(32) DEFAULT '?',
                                    nCores        INT         DEFAULT 1,
                                    wallTimeLimit DOUBLE      DEFAULT 0.,
                                    timeEnrolled  DOUBLE      DEFAULT 0.,
                                    timePosted    DOUBLE      DEFAULT 0.
                                 );

                                 INSERT INTO wsJobs_X (wsJobId,state,hubUserId,toolRevision,siteName,
                                                       nCores,wallTimeLimit,timeEnrolled,timePosted)
                                           SELECT wsJobId,state,hubUserId,"?",siteName,
                                                  nCores,wallTimeLimit,timeEnrolled,timePosted
                                             FROM wsJobs ORDER BY timeEnrolled ASC;

                                 DROP TABLE wsJobs;
                                 ALTER TABLE wsJobs_X RENAME TO wsJobs;

                                 CREATE UNIQUE INDEX wsJobs_wsJobId      ON wsJobs (wsJobId);
                                 CREATE        INDEX wsJobs_state        ON wsJobs (state);
                                 CREATE        INDEX wsJobs_hubUserId    ON wsJobs (hubUserId);
                                 CREATE        INDEX wsJobs_toolRevision ON wsJobs (toolRevision);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=14)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added toolRevision to wsJobs table")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 14:
                  sqlScript = """
                                 CREATE TABLE wsJobs_X (
                                    id            INT           NOT NULL AUTO_INCREMENT PRIMARY KEY,
                                    wsJobId       VARCHAR(40)   NOT NULL,
                                    state         INT           NOT NULL,
                                    hubUserId     INT           DEFAULT NULL,
                                    toolRevision  VARCHAR(32)   NOT NULL,
                                    siteName      VARCHAR(32)   DEFAULT '?',
                                    nCores        INT           DEFAULT 1,
                                    wallTimeLimit DOUBLE        DEFAULT 0.,
                                    exitStatus    INT           DEFAULT 0,
                                    errorMessage  VARCHAR(2048) DEFAULT '',
                                    timeEnrolled  DOUBLE        DEFAULT 0.,
                                    timePosted    DOUBLE        DEFAULT 0.
                                 );

                                 INSERT INTO wsJobs_X (wsJobId,state,hubUserId,toolRevision,siteName,
                                                       nCores,wallTimeLimit,
                                                       exitStatus,errorMessage,timeEnrolled,timePosted)
                                           SELECT wsJobId,state,hubUserId,"?",siteName,
                                                  nCores,wallTimeLimit,0,"",timeEnrolled,timePosted
                                             FROM wsJobs;

                                 DROP TABLE wsJobs;
                                 ALTER TABLE wsJobs_X RENAME TO wsJobs;

                                 CREATE UNIQUE INDEX wsJobs_wsJobId      ON wsJobs (wsJobId);
                                 CREATE        INDEX wsJobs_state        ON wsJobs (state);
                                 CREATE        INDEX wsJobs_hubUserId    ON wsJobs (hubUserId);
                                 CREATE        INDEX wsJobs_toolRevision ON wsJobs (toolRevision);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=15)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added error message to wsJobs table")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 15:
                  sqlScript = """
                                 CREATE TABLE wsJobs_X (
                                    id                  INT           NOT NULL AUTO_INCREMENT PRIMARY KEY,
                                    wsJobId             VARCHAR(40)   NOT NULL,
                                    state               INT           NOT NULL,
                                    hubUserId           INT           DEFAULT NULL,
                                    toolRevision        VARCHAR(32)   NOT NULL,
                                    siteName            VARCHAR(32)   DEFAULT '?',
                                    nCores              INT           DEFAULT 1,
                                    wallTimeLimit       DOUBLE        DEFAULT 0.,
                                    exitStatus          INT           DEFAULT 0,
                                    errorMessage        VARCHAR(2048) DEFAULT '',
                                    timeEnrolled        DOUBLE        DEFAULT 0.,
                                    timePosted          DOUBLE        DEFAULT 0.,
                                    exitStatusPreStaged INT           DEFAULT 0,
                                    timePreStaged       DOUBLE        DEFAULT 0.,
                                    exitStatusStagedIn  INT           DEFAULT 0,
                                    timeStagedIn        DOUBLE        DEFAULT 0.,
                                    exitStatusExecuted  INT           DEFAULT 0,
                                    timeExecuted        DOUBLE        DEFAULT 0.,
                                    exitStatusMeasured  INT           DEFAULT 0,
                                    timeMeasured        DOUBLE        DEFAULT 0.,
                                    exitStatusCached    INT           DEFAULT 0,
                                    timeCached          DOUBLE        DEFAULT 0.
                                 );

                                 INSERT INTO wsJobs_X (wsJobId,state,hubUserId,toolRevision,siteName,
                                                       nCores,wallTimeLimit,exitStatus,errorMessage,
                                                       timeEnrolled,timePosted,
                                                       exitStatusPreStaged,timePreStaged,
                                                       exitStatusStagedIn,timeStagedIn,
                                                       exitStatusExecuted,timeExecuted,
                                                       exitStatusMeasured,timeMeasured,
                                                       exitStatusCached,timeCached)
                                           SELECT wsJobId,state,hubUserId,toolRevision,siteName,
                                                  nCores,wallTimeLimit,exitStatus,errorMessage,
                                                  timeEnrolled,timePosted,
                                                  0,0.,
                                                  0,0.,
                                                  0,0.,
                                                  0,0.,
                                                  0,0.
                                             FROM wsJobs;

                                 DROP TABLE wsJobs;
                                 ALTER TABLE wsJobs_X RENAME TO wsJobs;

                                 CREATE UNIQUE INDEX wsJobs_wsJobId      ON wsJobs (wsJobId);
                                 CREATE        INDEX wsJobs_state        ON wsJobs (state);
                                 CREATE        INDEX wsJobs_hubUserId    ON wsJobs (hubUserId);
                                 CREATE        INDEX wsJobs_toolRevision ON wsJobs (toolRevision);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=16)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added transaction exit status to wsJobs table")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 16:
                  sqlScript = """
                                 CREATE TABLE wsJobs_X (
                                    id                  INT           NOT NULL AUTO_INCREMENT PRIMARY KEY,
                                    wsJobId             VARCHAR(40)   NOT NULL,
                                    state               INT           NOT NULL,
                                    hubUserId           INT           DEFAULT NULL,
                                    toolRevision        VARCHAR(32)   NOT NULL,
                                    siteName            VARCHAR(32)   DEFAULT '?',
                                    nCores              INT           DEFAULT 1,
                                    wallTimeLimit       DOUBLE        DEFAULT 0.,
                                    errorMessage        VARCHAR(2048) DEFAULT '',
                                    timeEnrolled        DOUBLE        DEFAULT 0.,
                                    exitStatusPosted    INT           DEFAULT 0,
                                    timePosted          DOUBLE        DEFAULT 0.,
                                    exitStatusPreStaged INT           DEFAULT 0,
                                    timePreStaged       DOUBLE        DEFAULT 0.,
                                    exitStatusStagedIn  INT           DEFAULT 0,
                                    timeStagedIn        DOUBLE        DEFAULT 0.,
                                    exitStatusExecuted  INT           DEFAULT 0,
                                    timeExecuted        DOUBLE        DEFAULT 0.,
                                    exitStatusMeasured  INT           DEFAULT 0,
                                    timeMeasured        DOUBLE        DEFAULT 0.,
                                    exitStatusCached    INT           DEFAULT 0,
                                    timeCached          DOUBLE        DEFAULT 0.
                                 );

                                 INSERT INTO wsJobs_X (wsJobId,state,hubUserId,toolRevision,siteName,
                                                       nCores,wallTimeLimit,errorMessage,timeEnrolled,
                                                       exitStatusPosted,timePosted,
                                                       exitStatusPreStaged,timePreStaged,
                                                       exitStatusStagedIn,timeStagedIn,
                                                       exitStatusExecuted,timeExecuted,
                                                       exitStatusMeasured,timeMeasured,
                                                       exitStatusCached,timeCached)
                                           SELECT wsJobId,state,hubUserId,toolRevision,siteName,
                                                  nCores,wallTimeLimit,errorMessage,timeEnrolled,
                                                  exitStatus,timePosted,
                                                  exitStatusPreStaged,timePreStaged,
                                                  exitStatusStagedIn,timeStagedIn,
                                                  exitStatusExecuted,timeExecuted,
                                                  exitStatusMeasured,timeMeasured,
                                                  exitStatusCached,timeCached
                                             FROM wsJobs;

                                 DROP TABLE wsJobs;
                                 ALTER TABLE wsJobs_X RENAME TO wsJobs;

                                 CREATE UNIQUE INDEX wsJobs_wsJobId      ON wsJobs (wsJobId);
                                 CREATE        INDEX wsJobs_state        ON wsJobs (state);
                                 CREATE        INDEX wsJobs_hubUserId    ON wsJobs (hubUserId);
                                 CREATE        INDEX wsJobs_toolRevision ON wsJobs (toolRevision);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=17)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added posted transaction exit status to wsJobs table")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)
               elif userVersion == 17:
                  sqlScript = """
                                 CREATE TABLE jobActionProcesses (
                                    action         VARCHAR(32) NOT NULL,
                                    processPID     INT         NOT NULL,
                                    timeRegistered DOUBLE      DEFAULT 0.,
                                    timeSignaled   DOUBLE      DEFAULT 0.,
                                    timeActivity   DOUBLE      DEFAULT 0.
                                 );
                                 CREATE UNIQUE INDEX jobActionProcesses_action ON jobActionProcesses (action);
                              """
                  try:
                     self.myDatabase.script(sqlScript)
                  except:
                     self.logger.log(logging.ERROR,getLogMessage("SQL Error(schema=18)"))
                     break
                  else:
                     self.myDatabase.updateDBuserVersion("Added jobActionProcesses table")
                     self.myDatabase.commit()
                     userVersion = self.myDatabase.getDBuserVersion()
                     self.myDatabase.setDBuserVersion(userVersion)

            if userVersion == targetSchema:
               schemaUpdated = True
         self.myDatabase.disconnect()

      return(schemaUpdated)


   def setInfo(self):
      errorInSetInfo = False
      if not self.settingInfo:
         self.settingInfo = True

         configFilePath = os.path.join(self.infosDirectory,self.infosConfigurationFile)
         if self.infosInfo:
            del self.infosInfo
         self.infosInfo = InfosInfo(configFilePath)

         if self.sitesInfo:
            del self.sitesInfo
         self.sitesInfo = SitesInfo(self.infosInfo.getInfoPath('sites'),
                                    allowedVenueMechanisms=['*'],
                                    pegasusVersion='*')
         enabledSites      = self.sitesInfo.getEnabledSites()
         expandedSiteNames = self.sitesInfo.getExpandedSiteNames(enabledSites)

         if self.aggregatorsInfo:
            del self.aggregatorsInfo
         self.aggregatorsInfo = AggregatorsInfo(self.infosInfo.getInfoPath('aggregators'))

         for expandedSiteName in expandedSiteNames:
            aggregatorName = self.aggregatorsInfo.getSiteAggregator(expandedSiteName)
            if not aggregatorName:
               self.logger.log(logging.ERROR,getLogMessage("Site %s is not included in any aggregator" % (expandedSiteName)))
               errorInSetInfo = True

         del self.sitesInfo
         self.sitesInfo = None
         del expandedSiteNames
         del enabledSites

         self.settingInfo = False

      return(errorInSetInfo)


   def terminate(self):
      if not self.terminating:
         self.signalJobSiteMonitors(signal.SIGTERM)
         self.closeListeningConnections()
         self.terminating = True


   def dumpActiveJobs(self):
      if self.myDatabase:
         self.myDatabase.dump()


   def close(self):
      if self.myDatabase:
         self.myDatabase.close()
      self.closeListeningConnections()


   def getActiveJobSites(self):
      activeJobSites = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT jobSite FROM activeJobSites"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            for row in result:
               activeJobSites.append(row[0])
         self.myDatabase.disconnect()

      return(activeJobSites)


   def getActiveJobSitePID(self,
                           jobSite):
      pid = 0
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT pid FROM activeJobSites WHERE(jobSite=%s)"
         sqlParameters = (jobSite,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               pid = row[0]
         self.myDatabase.disconnect()

      return(pid)


   def getActiveJobSiteTimeUpdated(self,
                                   jobSite):
      timeUpdated = 0
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT timeUpdated FROM activeJobSites WHERE(jobSite=%s)"
         sqlParameters = (jobSite,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if result:
               row = result[0]
               timeUpdated = row[0]
         self.myDatabase.disconnect()

      return(timeUpdated)


   def getActiveJobSitesReport(self):
      activeJobSites = {}
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT jobSite,timeUpdated FROM activeJobSites"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            for row in result:
               activeJobSites[row[0]] = row[1]
         self.myDatabase.disconnect()

      return(activeJobSites)


   def deleteActiveJobSite(self,
                           jobSite):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "DELETE FROM activeJobSites WHERE(jobSite=%s)"
         sqlParameters = (jobSite,)
         try:
            nDeleted = self.myDatabase.delete(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def purgeActiveJobSites(self):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "DELETE FROM activeJobSites"
         try:
            nDeleted = self.myDatabase.delete(sqlCommand)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def isJobSiteActive(self,
                       jobSite):
      jobSiteIsActive = False
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT jobSite FROM activeJobSites WHERE(jobSite=%s)"
         sqlParameters = (jobSite,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               jobSiteIsActive = True
         self.myDatabase.disconnect()

      return(jobSiteIsActive)


   def getAggregatorsWithRegisteredJob(self):
      aggregatorsWithRegisteredJob = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT DISTINCT aggregator FROM registeredJobs"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            for row in result:
               aggregatorsWithRegisteredJob.append(row[0])
         self.myDatabase.disconnect()

      return(aggregatorsWithRegisteredJob)


   def getSitesWithRegisteredJob(self):
      sitesWithRegisteredJob = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT DISTINCT siteName FROM registeredJobs"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            for row in result:
               sitesWithRegisteredJob.append(row[0])
         self.myDatabase.disconnect()

      return(sitesWithRegisteredJob)


   def getUsersWithRegisteredJob(self):
      usersWithRegisteredJob = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT DISTINCT hubUserId FROM registeredJobs \
                                                WHERE state=%s"
         sqlParameters = (self.JOBREGISTRATIONSTATEHELD,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               usersWithRegisteredJob.append(row[0])
         self.myDatabase.disconnect()

      return(usersWithRegisteredJob)


   def getAggregatorUserRegisteredJob(self):
      aggregatorUserRegisteredJobs = {}
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = """SELECT DISTINCT aggregator,hubUserId \
                                    FROM registeredJobs \
                                   WHERE(state=%s AND aggregator!="")"""
         sqlParameters = (self.JOBREGISTRATIONSTATEHELD,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               if not row[0] in aggregatorUserRegisteredJobs:
                  aggregatorUserRegisteredJobs[row[0]] = []
               aggregatorUserRegisteredJobs[row[0]].append(row[1])
         self.myDatabase.disconnect()

      return(aggregatorUserRegisteredJobs)


   def getAggregatorRegisteredJobsForUser(self,
                                          aggregator,
                                          hubUserId):
      jobsInfo = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT localJobId,instanceId,siteName FROM registeredJobs \
                                                            WHERE(aggregator=%s AND hubUserId=%s AND state=%s) \
                                                         ORDER BY timeRegistered"
         sqlParameters = (aggregator,hubUserId,self.JOBREGISTRATIONSTATEHELD)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               jobInfo = JobInfo(localJobId=row[0],
                                 instanceId=row[1],
                                 siteName=row[2],
                                 aggregator=aggregator)
               jobsInfo.append(dict(jobInfo))
         self.myDatabase.disconnect()

      return(jobsInfo)


   def getHeldRegisteredJobsForUser(self,
                                    hubUserId):
      jobsInfo = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT runName,localJobId,instanceId,siteName FROM registeredJobs \
                                                                    WHERE(hubUserId=%s AND state=%s) \
                                                                 ORDER BY timeRegistered"
         sqlParameters = (hubUserId,self.JOBREGISTRATIONSTATEHELD)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               jobInfo = JobInfo(runName=row[0],
                                 localJobId=row[1],
                                 instanceId=row[2],
                                 siteName=row[3])
               jobsInfo.append(dict(jobInfo))
         self.myDatabase.disconnect()

      return(jobsInfo)


   def getReleasedRegisteredJobsForUser(self,
                                        hubUserId):
      jobsInfo = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT runName,localJobId,instanceId,siteName FROM registeredJobs \
                                                                    WHERE(hubUserId=%s AND state=%s) \
                                                                 ORDER BY timeRegistered"
         sqlParameters = (hubUserId,self.JOBREGISTRATIONSTATERELEASED)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               jobInfo = JobInfo(runName=row[0],
                                 localJobId=row[1],
                                 instanceId=row[2],
                                 siteName=row[3])
               jobsInfo.append(dict(jobInfo))
         self.myDatabase.disconnect()

      return(jobsInfo)


   def getRegisteredJobsForUser(self,
                                hubUserId):
      jobsInfo = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT runName,localJobId,instanceId,siteName FROM registeredJobs \
                                                                    WHERE(hubUserId=%s) \
                                                                 ORDER BY timeRegistered"
         sqlParameters = (hubUserId,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               jobInfo = JobInfo(runName=row[0],
                                 localJobId=row[1],
                                 instanceId=row[2],
                                 siteName=row[3])
               jobsInfo.append(dict(jobInfo))
         self.myDatabase.disconnect()

      return(jobsInfo)


   def getRegisteredJob(self,
                        localJobId,
                        instanceId):
      jobInfo = None
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT distributorPid,localJobId,instanceId,hubUserId, \
                              siteName,aggregator,timeRegistered FROM registeredJobs WHERE(localJobId=%s AND instanceId=%s)"
         sqlParameters = (localJobId,instanceId)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               jobInfo = JobInfo(distributorPid=row[0],
                                 localJobId=row[1],
                                 instanceId=row[2],
                                 hubUserId=row[3],
                                 siteName=row[4],
                                 aggregator=row[5],
                                 timeRecorded=row[6])
         self.myDatabase.disconnect()

      return(jobInfo)


   JOBREGISTRATIONSTATEHELD     = 1 << 0
   JOBREGISTRATIONSTATERELEASED = 1 << 1


   def addRegisteredJob(self,
                        jobInfo):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "INSERT INTO registeredJobs (session,instanceToken,wsJobId, \
                                                   aggregator,state,runName,localJobId,instanceId, \
                                                   distributorPid,hubUserId,siteName,timeRegistered) \
                            VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
         sqlParameters = (jobInfo['session'],
                          jobInfo['instanceToken'],
                          jobInfo['wsJobId'],
                          jobInfo['aggregator'],
                          self.JOBREGISTRATIONSTATEHELD,
                          jobInfo['runName'],
                          jobInfo['localJobId'],
                          jobInfo['instanceId'],
                          jobInfo['distributorPid'],
                          jobInfo['hubUserId'],
                          jobInfo['siteName'],
                          jobInfo['timeRecorded'])
         try:
            self.myDatabase.insert(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def deleteRegisteredJob(self,
                           localJobId,
                           instanceId):
      nDeletedRegisteredJobs = 0
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "DELETE FROM registeredJobs WHERE(localJobId=%s AND instanceId=%s)"
         sqlParameters = (localJobId,instanceId)
         try:
            nDeletedRegisteredJobs = self.myDatabase.delete(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()

      return(nDeletedRegisteredJobs)


   def getRegisteredJobCountAtAggregator(self,
                                         aggregator):
      nRegisteredJobCount = 0
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT COUNT(*) FROM registeredJobs WHERE(aggregator=%s)"
         sqlParameters = (aggregator,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               nRegisteredJobCount = row[0]
         self.myDatabase.disconnect()

      return(nRegisteredJobCount)


   def getAggregatorsReleasedJobCount(self):
      aggregatorsReleasedJobCount = {}
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = """SELECT aggregator,COUNT(*) AS nJobs \
                           FROM registeredJobs \
                          WHERE(state=%s AND aggregator!="") \
                       GROUP BY aggregator"""
         sqlParameters = (self.JOBREGISTRATIONSTATERELEASED,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               aggregatorsReleasedJobCount[row[0]] = row[1]
         self.myDatabase.disconnect()

      return(aggregatorsReleasedJobCount)


   def getRegisteredJobCountAtSite(self,
                                   siteName):
      nRegisteredJobCount = 0
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT COUNT(*) FROM registeredJobs WHERE(siteName=%s)"
         sqlParameters = (siteName,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               nRegisteredJobCount = row[0]
         self.myDatabase.disconnect()

      return(nRegisteredJobCount)


   def getSitesWithActiveJob(self):
      sitesWithActiveJob = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT DISTINCT siteDesignator FROM activeJobs"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            for row in result:
               sitesWithActiveJob.append(row[0])
         self.myDatabase.disconnect()

      return(sitesWithActiveJob)


   def getActiveJobSiteQueues(self,
                              jobSite):
      activeJobSiteQueues = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT DISTINCT jobQueue FROM activeJobs WHERE(siteDesignator=%s)"
         sqlParameters = (jobSite,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               activeJobSiteQueues.append(row[0])
         self.myDatabase.disconnect()

      return(activeJobSiteQueues)


   def getActiveJobsInQueue(self,
                            jobSite,
                            jobQueue):
      jobsInfo = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT aggregator,siteName,siteDesignator,remoteJobId,distributorPid,runName, \
                              nCores,localJobId,instanceId,jobStatus,jobStage,jobQueue,hubUserId, \
                              destination,executionHost,timeRecorded \
                         FROM activeJobs WHERE(siteDesignator=%s AND jobQueue=%s)"
         sqlParameters = (jobSite,jobQueue)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               jobInfo = JobInfo(aggregator=row[0],
                                 siteName=row[1],
                                 siteDesignator=row[2],
                                 remoteJobId=row[3],
                                 distributorPid=row[4],
                                 runName=row[5],
                                 nCores=row[6],
                                 localJobId=row[7],
                                 instanceId=row[8],
                                 jobStatus=row[9],
                                 jobStage=row[10],
                                 jobQueue=row[11],
                                 hubUserId=row[12],
                                 destination=row[13],
                                 executionHost=row[14],
                                 timeRecorded=row[15])
               jobsInfo.append(dict(jobInfo))
         self.myDatabase.disconnect()

      return(jobsInfo)


   def getActiveSiteDesignators(self):
      activeSiteDesignators = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT siteDesignator FROM activeJobs \
                                            WHERE(jobStatus!='Dr' AND jobStatus!='D')"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            for row in result:
               if not row[0] in activeSiteDesignators:
                  activeSiteDesignators.append(row[0])
         self.myDatabase.disconnect()

      return(activeSiteDesignators)


   def getActiveJobsForUser(self,
                            hubUserId):
      jobsInfo = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT aggregator,siteName,siteDesignator,remoteJobId,distributorPid, \
                              runName,nCores,localJobId,instanceId,jobStatus,jobStage, \
                              jobQueue,hubUserId,destination,executionHost,timeRecorded \
                         FROM activeJobs \
                               WHERE(hubUserId=%s AND jobStatus!='Dr' AND jobStatus!='D')"
         sqlParameters = (hubUserId,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               jobInfo = JobInfo(aggregator=row[0],
                                 siteName=row[1],
                                 siteDesignator=row[2],
                                 remoteJobId=row[3],
                                 distributorPid=row[4],
                                 runName=row[5],
                                 nCores=row[6],
                                 localJobId=row[7],
                                 instanceId=row[8],
                                 jobStatus=row[9],
                                 jobStage=row[10],
                                 jobQueue=row[11],
                                 hubUserId=row[12],
                                 destination=row[13],
                                 executionHost=row[14],
                                 timeRecorded=row[15])
               jobsInfo.append(dict(jobInfo))
         self.myDatabase.disconnect()

      return(jobsInfo)


   def getActiveJobPidForUser(self,
                              hubUserId,
                              localJobId):
      activeJobPid = None
      jobsInfo = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT siteDesignator,remoteJobId,distributorPid,localJobId,instanceId \
                         FROM activeJobs \
                               WHERE(hubUserId=%s AND localJobId=%s AND \
                                     jobStatus!='N' AND jobStatus!='Dr' AND jobStatus!='D') \
                               ORDER BY localJobId,instanceId"
         sqlParameters = (hubUserId,localJobId)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               jobInfo = JobInfo(siteDesignator=row[0],
                                 remoteJobId=row[1],
                                 distributorPid=row[2],
                                 localJobId=row[3],
                                 instanceId=row[4])
               jobsInfo.append(jobInfo)
         self.myDatabase.disconnect()

      if len(jobsInfo) > 0:
         for jobInfo in jobsInfo:
            if jobInfo['distributorPid'] > 0:
               activeJobPid = jobInfo['distributorPid']
               break

      del jobsInfo

      return(activeJobPid)


   def getSitesActiveJobCount(self):
      sitesActiveJobCount = {}
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT siteName,COUNT(*) AS nJobs \
                         FROM activeJobs \
                        WHERE(jobStatus!='Dr' AND jobStatus!='D' AND instanceId!=0) \
                     GROUP BY siteName"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            for row in result:
               sitesActiveJobCount[row[0]] = row[1]
         self.myDatabase.disconnect()

      return(sitesActiveJobCount)


   def getAggregatorsActiveJobCount(self):
      aggregatorsActiveJobCount = {}
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT aggregator,COUNT(*) AS nJobs \
                         FROM activeJobs \
                        WHERE(jobStatus!='Dr' AND jobStatus!='D' AND instanceId!=0) \
                     GROUP BY aggregator"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            for row in result:
               aggregatorsActiveJobCount[row[0]] = row[1]
         self.myDatabase.disconnect()

      return(aggregatorsActiveJobCount)


   def getUsersWithActiveJobWithIdentity(self,
                                         identityName):
      usersWithActiveJobWithIdentity = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT DISTINCT activeJobIdentities.hubUserId \
                                  FROM activeJobIdentities JOIN activeJobs \
                                    ON (activeJobIdentities.localJobId=activeJobs.localJobId AND \
                                        activeJobIdentities.instanceId=activeJobs.instanceId) \
                                 WHERE (activeJobs.jobStatus!='Dr' AND \
                                        activeJobs.jobStatus!='D' AND \
                                        activeJobs.instanceId!=0 AND \
                                        activeJobIdentities.identityName=%s)"
         sqlParameters = (identityName,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               usersWithActiveJobWithIdentity.append(row[0])
         self.myDatabase.disconnect()

      return(usersWithActiveJobWithIdentity)


   def getActiveJob(self,
                    siteDesignator,
                    remoteJobId):
      jobInfo = None
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT aggregator,state,siteName,distributorPid,runName,nCores,localJobId, \
                              instanceId,jobStatus,jobStage,jobQueue,hubUserId, \
                              preStageDirectory,tapisSystemStageIn,stagingHandle,destination, \
                              executionHost,timeRecorded \
                         FROM activeJobs WHERE(siteDesignator=%s AND remoteJobId=%s)"
         sqlParameters = (siteDesignator,remoteJobId)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               jobInfo = JobInfo(aggregator=row[0],
                                 state=row[1],
                                 siteName=row[2],
                                 siteDesignator=siteDesignator,
                                 remoteJobId=remoteJobId,
                                 distributorPid=row[3],
                                 runName=row[4],
                                 nCores=row[5],
                                 localJobId=row[6],
                                 instanceId=row[7],
                                 jobStatus=row[8],
                                 jobStage=row[9],
                                 jobQueue=row[10],
                                 hubUserId=row[11],
                                 preStageDirectory=row[12],
                                 tapisSystemStageIn=row[13],
                                 stagingHandle=row[14],
                                 destination=row[15],
                                 executionHost=row[16],
                                 timeRecorded=row[17])
         self.myDatabase.disconnect()

      return(jobInfo)


   ACTIVEJOBSTATEPRESTAGED    = 1 << 0
   ACTIVEJOBSTATESTAGING      = 1 << 1
   ACTIVEJOBSTATESTAGEDIN     = 1 << 2
   ACTIVEJOBSTATEEXECUTING    = 1 << 3
   ACTIVEJOBSTATEEXECUTED     = 1 << 4
   ACTIVEJOBSTATEMEASURING    = 1 << 5
   ACTIVEJOBSTATEMEASURED     = 1 << 6
   ACTIVEJOBSTATECACHING      = 1 << 7
   ACTIVEJOBSTATECACHED       = 1 << 8
   ACTIVEJOBSTATEENDOFTHELINE = 1 << 30
   ACTIVEJOBSTATEDONE         = ACTIVEJOBSTATECACHED

   activeStateMap = {'PRESTAGED':ACTIVEJOBSTATEPRESTAGED,
                     'STAGING':ACTIVEJOBSTATESTAGING,
                     'STAGEDIN':ACTIVEJOBSTATESTAGEDIN,
                     'EXECUTING':ACTIVEJOBSTATEEXECUTING,
                     'EXECUTED':ACTIVEJOBSTATEEXECUTED,
                     'MEASURING':ACTIVEJOBSTATEMEASURING,
                     'MEASURED':ACTIVEJOBSTATEMEASURED,
                     'CACHING':ACTIVEJOBSTATECACHING,
                     'CACHED':ACTIVEJOBSTATECACHED,
                     'ENDOFTHELINE':ACTIVEJOBSTATEENDOFTHELINE
                    }


   def addActiveJob(self,
                    jobInfo):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "INSERT INTO activeJobs (session,instanceToken,wsJobId, \
                                               aggregator,state,localJobId,instanceId,distributorPid,hubUserId, \
                                               preStageDirectory,tapisSystemStageIn,stagingHandle,siteName, \
                                               nCores,runName,siteDesignator,remoteJobId,jobStatus,jobStage,jobQueue,destination, \
                                               executionHost,timeRecorded,jobStatistic) \
                            VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
         sqlParameters = (jobInfo['session'],
                          jobInfo['instanceToken'],
                          jobInfo['wsJobId'],
                          jobInfo['aggregator'],
                          jobInfo['state'],
                          jobInfo['localJobId'],
                          jobInfo['instanceId'],
                          jobInfo['distributorPid'],
                          jobInfo['hubUserId'],
                          jobInfo['preStageDirectory'],
                          jobInfo['tapisSystemStageIn'],
                          jobInfo['stagingHandle'],
                          jobInfo['siteName'],
                          jobInfo['nCores'],
                          jobInfo['runName'],
                          jobInfo['siteDesignator'],
                          jobInfo['remoteJobId'],
                          jobInfo['jobStatus'],
                          jobInfo['jobStage'],
                          jobInfo['jobQueue'],
                          jobInfo['destination'],
                          jobInfo['executionHost'],
                          jobInfo['timeRecorded'],
                          jobInfo['jobStatistic'])
         try:
            self.myDatabase.insert(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for identityName in jobInfo['identityNames']:
               sqlCommand = "INSERT INTO activeJobIdentities (localJobId,instanceId,hubUserId,identityName) \
                                  VALUES(%s,%s,%s,%s)"
               sqlParameters = (jobInfo['localJobId'],
                                jobInfo['instanceId'],
                                jobInfo['hubUserId'],
                                identityName)
               try:
                  self.myDatabase.insert(sqlCommand,sqlParameters)
               except:
                  pass
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def updateActiveJob(self,
                       jobInfo):
      timeRecorded = time.time()
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "UPDATE activeJobs SET jobStatus=%s,jobStage=%s,jobQueue=%s,executionHost=%s,timeRecorded=%s \
                                       WHERE(siteDesignator=%s AND remoteJobId=%s)"
         sqlParameters = (jobInfo['jobStatus'],jobInfo['jobStage'], \
                          jobInfo['jobQueue'],jobInfo['executionHost'], \
                          timeRecorded, \
                          jobInfo['siteDesignator'],jobInfo['remoteJobId'])
         try:
            self.myDatabase.update(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def updateActiveJobStatus(self,
                             instanceToken,
                             jobStatus):
      if jobStatus == 'D':
         wsJobId = self.getActiveJobWSJobId(instanceToken)
         if wsJobId:
#           self.logger.log(logging.DEBUG,getLogMessage("wsJobId(%s): activeState = EXECUTED" % (wsJobId)))
            self.updateWSTransactionStatus(wsJobId,'EXECUTED')

      timeRecorded = time.time()
      if self.myDatabase.connect(self.configData['mysqlDB']):
         if jobStatus == 'D':
            sqlCommand = "UPDATE activeJobs SET state=%s,jobStatus=%s,timeRecorded=%s \
                                          WHERE(instanceToken=%s)"
            sqlParameters = (self.ACTIVEJOBSTATEEXECUTED,
                             jobStatus, \
                             timeRecorded, \
                             instanceToken)
         else:
            sqlCommand = "UPDATE activeJobs SET jobStatus=%s,timeRecorded=%s \
                                          WHERE(instanceToken=%s)"
            sqlParameters = (jobStatus, \
                             timeRecorded, \
                             instanceToken)
         try:
            self.myDatabase.update(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def getActiveJobStatus(self,
                          instanceToken):
      jobStatus = "?"
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT jobStatus FROM activeJobs \
                                       WHERE(instanceToken=%s)"
         sqlParameters = (instanceToken,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               jobStatus = row[0]
         self.myDatabase.disconnect()

      return(jobStatus)


   def getActiveJobInstanceToken(self,
                                 hubUserId,
                                 localJobId,
                                 instanceId):
      instanceToken = None
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT instanceToken FROM activeJobs \
                                           WHERE(hubUserId=%s AND localJobId=%s AND instanceId=%s)"
         sqlParameters = (hubUserId,localJobId,instanceId)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               instanceToken = row[0]
         self.myDatabase.disconnect()

      return(instanceToken)


   def getActiveJobWSJobId(self,
                           instanceToken):
      wsJobId = None
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT wsJobId FROM activeJobs \
                                     WHERE(instanceToken=%s)"
         sqlParameters = (instanceToken,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               wsJobId = row[0]
         self.myDatabase.disconnect()

      return(wsJobId)


   def getActiveJobRemoteJobId(self,
                               hubUserId,
                               instanceToken):
      remoteJobId = None
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT remoteJobId FROM activeJobs \
                                         WHERE(hubUserId=%s AND instanceToken=%s)"
         sqlParameters = (hubUserId,instanceToken)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               remoteJobId = row[0]
         self.myDatabase.disconnect()

      return(remoteJobId)


   def getActiveJobJobStatistic(self,
                                instanceToken):
      stagingHandle = ""
      remoteJobId   = ""
      jobStatistic  = "{}"
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT session,localJobId,instanceId,stagingHandle,remoteJobId,jobStatistic FROM activeJobs \
                                                                                                  WHERE(instanceToken=%s)"
         sqlParameters = (instanceToken,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               session       = row[0]
               localJobId    = row[1]
               instanceId    = row[2]
               stagingHandle = row[3]
               remoteJobId   = row[4]
               jobStatistic  = row[5]
         self.myDatabase.disconnect()

      return(session,localJobId,instanceId,stagingHandle,remoteJobId,jobStatistic)


   def updateActiveJobJobStatistic(self,
                                   instanceToken,
                                   jobStatistic):
      timeRecorded = time.time()
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "UPDATE activeJobs SET jobStatistic=%s,timeRecorded=%s \
                                       WHERE(instanceToken=%s)"
         sqlParameters = (jobStatistic,timeRecorded,instanceToken)
         try:
            self.myDatabase.update(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def purgeActiveJobs(self,
                       purgeJobStatus,
                       cutOffAge=0.):
      nPurgeActiveJobs = 0
      if self.myDatabase.connect(self.configData['mysqlDB']):
         cutOffTime = time.time()-cutOffAge
         sqlCommand = "DELETE FROM activeJobs WHERE(state=%s AND jobStatus=%s AND timeRecorded<=%s)"
         sqlParameters = (self.ACTIVEJOBSTATEEXECUTING,purgeJobStatus,cutOffTime)
         try:
            nPurgeActiveJobs = self.myDatabase.delete(sqlCommand,sqlParameters)
         except:
            pass
         else:
#           sqlCommand = "DELETE FROM activeJobIdentities \
#                               WHERE(localJobId || instanceId) \
#                                  IN(SELECT activeJobIdentities.localJobId || activeJobIdentities.instanceId \
#                                       FROM activeJobIdentities \
#                                  LEFT JOIN activeJobs ON (activeJobIdentities.localJobId=activeJobs.localJobId AND \
#                                                           activeJobIdentities.instanceId=activeJobs.instanceId) \
#                                      WHERE activeJobs.localJobId IS NULL)"
# https://stackoverflow.com/questions/5816840/delete-i-cant-specify-target-table
            sqlCommand = "DELETE FROM activeJobIdentities \
                                WHERE(localJobId,instanceId) IN \
                                    (SELECT * FROM (SELECT activeJobIdentities.localJobId,activeJobIdentities.instanceId \
                                        FROM activeJobIdentities \
                                   LEFT JOIN activeJobs ON (activeJobIdentities.localJobId=activeJobs.localJobId AND \
                                                            activeJobIdentities.instanceId=activeJobs.instanceId) \
                                       WHERE activeJobs.localJobId IS NULL) AS t)"
            try:
               nDeleted = self.myDatabase.delete(sqlCommand)
            except:
               pass
            else:
               self.myDatabase.commit()
         self.myDatabase.disconnect()

      return(nPurgeActiveJobs)


   def getActiveJobWSJobIds(self,
                            activeStateName):
      activeJobWSJobIds = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         state = self.activeStateMap[activeStateName]
         sqlCommand = "SELECT wsJobId FROM activeJobs WHERE(state=%s AND wsJobId!='')"
         sqlParameters = (state,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               activeJobWSJobIds.append(row[0])
         self.myDatabase.disconnect()

#     self.logger.log(logging.DEBUG,getLogMessage("getActiveJobWSJobIds(%s): %d" % (activeStateName,len(activeJobWSJobIds))))

      return(activeJobWSJobIds)


   def getPreStagedActiveJob(self,
                             wsJobId):
      preStagedActiveJob = None
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT instanceToken,preStageDirectory,tapisSystemStageIn,stagingHandle,siteName \
                         FROM activeJobs \
                        WHERE(wsJobId=%s)"
         sqlParameters = (wsJobId,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               jobInfo = JobInfo(instanceToken=row[0],
                                 preStageDirectory=row[1],
                                 tapisSystemStageIn=row[2],
                                 stagingHandle=row[3],
                                 siteName=row[4])
               preStagedActiveJob = dict(jobInfo)
         self.myDatabase.disconnect()

      return(preStagedActiveJob)


   def getStagedActiveJob(self,
                          wsJobId):
      stagedActiveJob = None
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT instanceToken,tapisSystemStageIn,stagingHandle,siteName \
                         FROM activeJobs \
                        WHERE(wsJobId=%s)"
         sqlParameters = (wsJobId,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               jobInfo = JobInfo(instanceToken=row[0],
                                 tapisSystemStageIn=row[1],
                                 stagingHandle=row[2],
                                 siteName=row[3])
               stagedActiveJob = dict(jobInfo)
         self.myDatabase.disconnect()

      return(stagedActiveJob)


   def updateActiveJobState(self,
                            wsJobId,
                            activeStateName,
                            requestParameters):
      if wsJobId:
         if 'exitStatus' in requestParameters:
#           self.logger.log(logging.DEBUG,getLogMessage("wsJobId(%s): activeState = %s, exitStatus = %d" % \
#                                              (wsJobId,activeStateName,requestParameters['exitStatus'])))
            self.updateWSTransactionStatus(wsJobId,activeStateName,exitStatus=requestParameters['exitStatus'])
         else:
#           self.logger.log(logging.DEBUG,getLogMessage("wsJobId(%s): activeState = %s" % \
#                                                             (wsJobId,activeStateName)))
            self.updateWSTransactionStatus(wsJobId,activeStateName)

      if self.myDatabase.connect(self.configData['mysqlDB']):
         state = self.activeStateMap[activeStateName]
         if 'remoteJobIdNumber' in requestParameters:
            sqlCommand = "UPDATE activeJobs SET remoteJobId=%s,jobStage=%s,state=%s \
                                          WHERE(wsJobId=%s)"
            sqlParameters = (requestParameters['remoteJobIdNumber'],"Simulation",state,wsJobId)
         else:
            sqlCommand = "UPDATE activeJobs SET state=%s \
                                          WHERE(wsJobId=%s)"
            sqlParameters = (state,wsJobId)
         try:
            self.myDatabase.update(sqlCommand,sqlParameters)
         except:
            pass
         self.myDatabase.commit()
         self.myDatabase.disconnect()


   def getCacheSimToolActiveJob(self,
                                wsJobId):
      cacheSimToolActiveJob = None
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT instanceToken,stagingHandle,siteName \
                         FROM activeJobs \
                        WHERE(wsJobId=%s)"
         sqlParameters = (wsJobId,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               jobInfo = JobInfo(instanceToken=row[0],
                                 stagingHandle=row[1],
                                 siteName=row[2])
               cacheSimToolActiveJob = dict(jobInfo)
         self.myDatabase.disconnect()

      return(cacheSimToolActiveJob)


   def getMetricsCollectionActiveJob(self,
                                     wsJobId):
      metricsCollectionActiveJob = None
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT instanceToken,stagingHandle,siteName \
                         FROM activeJobs \
                        WHERE(wsJobId=%s)"
         sqlParameters = (wsJobId,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               jobInfo = JobInfo(instanceToken=row[0],
                                 stagingHandle=row[1],
                                 siteName=row[2])
               metricsCollectionActiveJob = dict(jobInfo)
         self.myDatabase.disconnect()

      return(metricsCollectionActiveJob)


   def markActiveJobsAsComplete(self):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "UPDATE activeJobs SET state=%s \
                                       WHERE(state=%s)"
         sqlParameters = (self.ACTIVEJOBSTATEENDOFTHELINE,self.ACTIVEJOBSTATEDONE)
         try:
            self.myDatabase.update(sqlCommand,sqlParameters)
         except:
            pass

         sqlCommand = "UPDATE wsJobs SET state=%s \
                                   WHERE(state=%s)"
         sqlParameters = (self.WSJOBSTATEENDOFTHELINE,self.WSJOBSTATEDONE)

         try:
            self.myDatabase.update(sqlCommand,sqlParameters)
         except:
            pass

         self.myDatabase.commit()
         self.myDatabase.disconnect()


   @staticmethod
   def __textToBlob(text):
      try:
         blob = base64.b64encode(text).decode('utf-8')
      except UnicodeEncodeError:
         blob = base64.b64encode(text.encode('utf-8')).decode('utf-8')
      except TypeError:
         blob = base64.b64encode(text.encode('utf-8')).decode('utf-8')

      return(blob)


   @staticmethod
   def __blobToText(blob):
      try:
         text = base64.b64decode(blob.encode('utf-8'))
      except AttributeError:
         text = base64.b64decode(blob).decode('utf-8')

      return(text)


   def getFileTails(self,
                    siteDesignator,
                    remoteJobId):
      tailTexts = {}
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT fileName,tailText,timeUpdated FROM fileTailings \
                               WHERE(siteDesignator=%s AND remoteJobId=%s)"
         sqlParameters = (siteDesignator,remoteJobId)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               fileName    = row[0]
               tailBlob    = row[1]
               timeUpdated = row[2]
               tailText    = self.__blobToText(tailBlob)
               tailTexts[fileName] = {'tailText':tailText,
                                      'timeUpdated':timeUpdated}
         self.myDatabase.disconnect()

      return(tailTexts)


   def addFileTail(self,
                   siteDesignator,
                   remoteJobId,
                   fileName,
                   nLines,
                   tailText):
      timeUpdated = time.time()
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "INSERT INTO fileTailings (siteDesignator,remoteJobId,fileName,nLines,timeUpdated,tailText) \
                            VALUES(%s,%s,%s,%s,%s,%s)"
         tailBlob = self.__textToBlob(tailText)
#
# Maximum blob length in MySQL is 65535 characters
# If tailText is longer than the maximum allowed it must be truncated
# Reserve 4 bytes for padding and account for length increasing with
# base64 encoding
#
# maxTextLength = floor((65535-4)*6/8) = 49148
#
         tailBlob = tailBlob[-49148:]
         sqlParameters = (siteDesignator,
                          remoteJobId,
                          fileName,
                          nLines,
                          timeUpdated,
                          tailBlob)
         try:
            self.myDatabase.insert(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def clearFileTail(self,
                     siteDesignator,
                     remoteJobId,
                     fileName):
      tailText = ""
      tailBlob = self.__textToBlob(tailText)
      timeUpdated = time.time()
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "UPDATE fileTailings SET timeUpdated=%s,tailText=%s \
                                   WHERE(siteDesignator=%s AND remoteJobId=%s AND fileName=%s)"
         sqlParameters = (timeUpdated,tailBlob, \
                          siteDesignator,remoteJobId, \
                          fileName)
         try:
            self.myDatabase.update(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def clearFileTails(self,
                      siteDesignator,
                      remoteJobId):
      tailText = ""
      tailBlob = self.__textToBlob(tailText)
      timeUpdated = time.time()
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "UPDATE fileTailings SET timeUpdated=%s,tailText=%s \
                                   WHERE(siteDesignator=%s AND remoteJobId=%s)"
         sqlParameters = (timeUpdated,tailBlob, \
                          siteDesignator,remoteJobId)
         try:
            self.myDatabase.update(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def deleteFileTail(self,
                      siteDesignator,
                      remoteJobId,
                      fileName):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "DELETE FROM fileTailings \
                             WHERE(siteDesignator=%s AND remoteJobId=%s AND fileName=%s)"
         sqlParameters = (siteDesignator,remoteJobId,fileName)
         try:
            nDeleted = self.myDatabase.delete(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def deleteFileTails(self,
                       siteDesignator,
                       remoteJobId):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "DELETE FROM fileTailings \
                             WHERE(siteDesignator=%s AND remoteJobId=%s)"
         sqlParameters = (siteDesignator,remoteJobId)
         try:
            nDeleted = self.myDatabase.delete(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def purgeFileTails(self):
      nPurgeFileTails = 0
      if self.myDatabase.connect(self.configData['mysqlDB']):
#        sqlCommand = "DELETE FROM fileTailings \
#                            WHERE(siteDesignator || remoteJobId) \
#                               IN(SELECT fileTailings.siteDesignator || fileTailings.remoteJobId \
#                                    FROM fileTailings \
#                               LEFT JOIN activeJobs ON (fileTailings.siteDesignator=activeJobs.siteDesignator AND \
#                                                        fileTailings.remoteJobId=activeJobs.remoteJobId) \
#                                   WHERE activeJobs.siteDesignator IS NULL)"
         sqlCommand = "DELETE FROM fileTailings \
                             WHERE(siteDesignator,remoteJobId) IN \
                                 (SELECT * FROM (SELECT fileTailings.siteDesignator,fileTailings.remoteJobId \
                                     FROM fileTailings \
                                LEFT JOIN activeJobs ON (fileTailings.siteDesignator=activeJobs.siteDesignator AND \
                                                         fileTailings.remoteJobId=activeJobs.remoteJobId) \
                                    WHERE activeJobs.siteDesignator IS NULL) AS t)"
         try:
            nPurgeFileTails = self.myDatabase.delete(sqlCommand)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()

      return(nPurgeFileTails)


   def getActiveJobCount(self):
      nActiveJobCount = 0
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT COUNT(*) FROM activeJobs"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               nActiveJobCount = row[0]
         self.myDatabase.disconnect()

      return(nActiveJobCount)


   def getActiveJobInProgressCount(self):
      nActiveJobInProgressCount = 0
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT COUNT(*) FROM activeJobs \
                                      WHERE(jobStatus!='Dr' AND jobStatus!='D')"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               nActiveJobInProgressCount = row[0]
         self.myDatabase.disconnect()

      return(nActiveJobInProgressCount)


   def addUserActivity(self,
                       siteName,
                       hubUserId,
                       activityScore):
      timeUpdated = time.time()
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "INSERT INTO userActivityScores (siteName,hubUserId,activityScore,timeUpdated) \
                            VALUES(%s,%s,%s,%s)"
         sqlParameters = (siteName,hubUserId,activityScore,timeUpdated)
         try:
            self.myDatabase.insert(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def getUserActivity(self,
                       aggregator,
                       hubUserId):
      activityScore = 0.5
      timeUpdated   = -1.

      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT activityScore,timeUpdated FROM userActivityScores \
                                                       WHERE(aggregator=%s AND hubUserId=%s)"
         sqlParameters = (aggregator,hubUserId)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               activityScore = row[0]
               timeUpdated   = row[1]
         self.myDatabase.disconnect()

      return(activityScore,timeUpdated)


   def getUsersActivity(self,
                        hubUserIds):
      usersActivity = {}
      if self.myDatabase.connect(self.configData['mysqlDB']):
         if hubUserIds == '*':
            sqlCommand = "SELECT aggregator,hubUserId,activityScore FROM userActivityScores"
            sqlParameters = None
         elif len(hubUserIds) == 1:
            sqlCommand = "SELECT aggregator,hubUserId,activityScore FROM userActivityScores WHERE(hubUserId=%s)"
            sqlParameters = (hubUserIds[0],)
         else:
            sqlCommand = "SELECT aggregator,hubUserId,activityScore FROM userActivityScores"
            sqlParameters = None

         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               if hubUserIds == '*' or row[1] in hubUserIds:
                  if not row[0] in usersActivity:
                     usersActivity[row[0]] = {}
                  usersActivity[row[0]][row[1]] = row[2]
         self.myDatabase.disconnect()

      return(usersActivity)


   def getUsersPriority(self,
                        hubUserIds):
      usersPriority = {}

      usersActivity = self.getUsersActivity('*')
      for aggregator in usersActivity:
         usersPriority[aggregator] = {}
         sumPriorities = 0.
         if hubUserIds == '*':
            for hubUserId in usersActivity[aggregator]:
               submitterClass = self.getUserSubmitterClass(hubUserId)
               priorityBoosters = self.aggregatorsInfo.getAggregatorKeyValue(aggregator,'priorityBoosters')
               try:
                  if submitterClass in priorityBoosters:
                     priorityBooster = float(priorityBoosters[submitterClass])
                  else:
                     priorityBooster = 1.
               except:
#                 self.logger.log(logging.DEBUG,getLogMessage("getUsersPriority(%s): submitterClass = %s" % \
#                                                                             (aggregator,submitterClass)))
#                 self.logger.log(logging.DEBUG,getLogMessage("getUsersPriority(%s): priorityBoosters = %s" % \
#                                                                             (aggregator,priorityBoosters)))
                  priorityBooster = 0.5
               usersPriority[aggregator][hubUserId] = priorityBooster/usersActivity[aggregator][hubUserId]
               sumPriorities += usersPriority[aggregator][hubUserId]
         else:
            for hubUserId in hubUserIds:
               submitterClass = self.getUserSubmitterClass(hubUserId)
               priorityBoosters = self.aggregatorsInfo.getAggregatorKeyValue(aggregator,'priorityBoosters')
               try:
                  if submitterClass in priorityBoosters:
                     priorityBooster = float(priorityBoosters[submitterClass])
                  else:
                     priorityBooster = 1.
               except:
                  priorityBooster = 0.5
               if hubUserId in usersActivity[aggregator]:
                  usersPriority[aggregator][hubUserId] = priorityBooster/usersActivity[aggregator][hubUserId]
               else:
                  usersPriority[aggregator][hubUserId] = priorityBooster/0.5
               sumPriorities += usersPriority[aggregator][hubUserId]

         for hubUserId in usersPriority[aggregator]:
            usersPriority[aggregator][hubUserId] /= sumPriorities

      return(usersPriority)


   def getAggregatorsUsersPriorities(self,
                                     aggregatorsUsers):
      usersPriorities = {}

      usersActivity = self.getUsersActivity('*')
      for aggregator in aggregatorsUsers:
         usersPriorities[aggregator] = {}
         priorityBoosters = self.aggregatorsInfo.getAggregatorKeyValue(aggregator,'priorityBoosters')
         sumPriorities = 0.
         for hubUserId in aggregatorsUsers[aggregator]:
            submitterClass = self.getUserSubmitterClass(hubUserId)
            try:
               if submitterClass in priorityBoosters:
                  priorityBooster = float(priorityBoosters[submitterClass])
               else:
                  priorityBooster = 1.
            except:
               priorityBooster = 0.5

            if aggregator in usersActivity:
               if hubUserId in usersActivity[aggregator]:
                  usersPriorities[aggregator][hubUserId] = priorityBooster/usersActivity[aggregator][hubUserId]
               else:
                  usersPriorities[aggregator][hubUserId] = priorityBooster/0.5
            else:
               usersPriorities[aggregator][hubUserId] = priorityBooster/0.5
            sumPriorities += usersPriorities[aggregator][hubUserId]

         for hubUserId in usersPriorities[aggregator]:
            usersPriorities[aggregator][hubUserId] /= sumPriorities

      return(usersPriorities)


   def updateUserActivity(self,
                          aggregator,
                          hubUserId,
                          activityScore,
                          timeUpdated):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "UPDATE userActivityScores SET activityScore=%s,timeUpdated=%s \
                                               WHERE(aggregator=%s AND hubUserId=%s)"
         sqlParameters = (activityScore,timeUpdated,aggregator,hubUserId)
         try:
            self.myDatabase.update(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def updateUserActivityScores(self):
      now = time.time()
      currentActivityScores = {}
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT aggregator,hubUserId,SUM(nCores) AS activity FROM activeJobs \
                                                            WHERE(instanceId != 0 AND jobStatus != 'D' AND jobStatus != 'Dr') \
                                                            GROUP BY aggregator,hubUserId"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            for row in result:
               if not row[0] in currentActivityScores:
                  currentActivityScores[row[0]] = {}
               currentActivityScores[row[0]][row[1]] = float(row[2])
         self.myDatabase.disconnect()

      activityScores = {}
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT aggregator,hubUserId,activityScore,timeUpdated FROM userActivityScores"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            for row in result:
               aggregator    = row[0]
               hubUserId     = row[1]
               activityScore = row[2]
               timeUpdated   = row[3]
               if not aggregator in activityScores:
                  activityScores[aggregator] = {}
               activityScores[aggregator][hubUserId] = {'activityScore':activityScore,
                                                        'timeUpdated':timeUpdated}
         self.myDatabase.disconnect()

      newActivityScores     = {}
      updatedActivityScores = {}
      for aggregator in currentActivityScores:
         for hubUserId in currentActivityScores[aggregator]:
            currentActivityScore = currentActivityScores[aggregator][hubUserId]
            try:
               activityScore = activityScores[aggregator][hubUserId]['activityScore']
            except:
               if not aggregator in newActivityScores:
                  newActivityScores[aggregator] = {}
               newActivityScores[aggregator][hubUserId] = currentActivityScore
            else:
               timeUpdated   = activityScores[aggregator][hubUserId]['activityScore']
               beta = math.pow(0.5,(now-timeUpdated)/86400.)
#              self.logger.log(logging.DEBUG,getLogMessage("                type(beta): %s" % (type(beta))))
#              self.logger.log(logging.DEBUG,getLogMessage("       type(activityScore): %s" % (type(activityScore))))
#              self.logger.log(logging.DEBUG,getLogMessage("type(currentActivityScore): %s" % (type(currentActivityScore))))
               activity = beta*activityScore + (1.-beta)*currentActivityScore
               if not aggregator in updatedActivityScores:
                  updatedActivityScores[aggregator] = {}
               updatedActivityScores[aggregator][hubUserId] = max(activity,0.5)

      if len(newActivityScores)+len(updatedActivityScores) > 0:
         if self.myDatabase.connect(self.configData['mysqlDB']):
            for aggregator in newActivityScores:
               for hubUserId in newActivityScores[aggregator]:
                  sqlCommand = "INSERT INTO userActivityScores (aggregator,hubUserId,activityScore,timeUpdated) \
                                     VALUES(%s,%s,%s,%s)"
                  sqlParameters = (aggregator,hubUserId,newActivityScores[aggregator][hubUserId],now)
                  try:
                     self.myDatabase.insert(sqlCommand,sqlParameters)
                  except:
                     pass

            for aggregator in updatedActivityScores:
               for hubUserId in updatedActivityScores[aggregator]:
                  sqlCommand = "UPDATE userActivityScores SET activityScore=%s,timeUpdated=%s \
                                                        WHERE(aggregator=%s AND hubUserId=%s)"
                  sqlParameters = (updatedActivityScores[aggregator][hubUserId],now,aggregator,hubUserId)
                  try:
                     self.myDatabase.update(sqlCommand,sqlParameters)
                  except:
                     pass
            self.myDatabase.commit()
            self.myDatabase.disconnect()


   def purgeUserActivityScores(self):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         now = time.time()
         activityScore = 0.5
         mostRecentUpdate = now-60.*60.*24
         sqlCommand = "DELETE FROM userActivityScores WHERE(activityScore<=%s AND timeUpdated<=%s)"
         sqlParameters = (activityScore,mostRecentUpdate)
         try:
            nDeleted = self.myDatabase.delete(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def updateRegisteredJobStates(self,
                                 jobsInfo,
                                 jobRegistrationState):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         for jobInfo in jobsInfo:
            sqlCommand = "UPDATE registeredJobs SET state=%s \
                                              WHERE(localJobId=%s AND instanceId=%s)"
            sqlParameters = (jobRegistrationState,jobInfo['localJobId'],jobInfo['instanceId'])
            try:
               self.myDatabase.update(sqlCommand,sqlParameters)
            except:
               pass
         self.myDatabase.commit()
         self.myDatabase.disconnect()


   def getRegisteredJobsHeldCount(self):
      nRegisteredJobsHeld = 0
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT COUNT(*) FROM registeredJobs \
                                      WHERE state=%s"
         sqlParameters = (self.JOBREGISTRATIONSTATEHELD,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               nRegisteredJobsHeld = row[0]
         self.myDatabase.disconnect()

      return(nRegisteredJobsHeld)


   def releaseRegisteredJobs(self):
      nReleasedJobs   = 0
      releaseJobsInfo = []
# get the number of active jobs for each aggregator
      aggregatorsActiveJobCount = self.getAggregatorsActiveJobCount()
#     self.logger.log(logging.DEBUG,getLogMessage("releaseRegisteredJobs(aggregatorsActiveJobCount): %s" % (str(aggregatorsActiveJobCount))))
# get the number of released jobs pending activation for each aggregator
      aggregatorsReleasedJobCount = self.getAggregatorsReleasedJobCount()
#     self.logger.log(logging.DEBUG,getLogMessage("releaseRegisteredJobs(aggregatorsReleasedJobCount): %s" % (str(aggregatorsReleasedJobCount))))
# get users with held jobs at each aggregator
      aggregatorUserRegisteredJobs = self.getAggregatorUserRegisteredJob()
#     self.logger.log(logging.DEBUG,getLogMessage("releaseRegisteredJobs(aggregatorUserRegisteredJobs): %s" % (str(aggregatorUserRegisteredJobs))))
      usersPriority = self.getAggregatorsUsersPriorities(aggregatorUserRegisteredJobs)
#     self.logger.log(logging.DEBUG,getLogMessage("releaseRegisteredJobs(usersPriority): %s" % (str(usersPriority))))

      for aggregator in aggregatorUserRegisteredJobs:
         maximumAggregatorActiveJobs = self.aggregatorsInfo.getAggregatorKeyValue(aggregator,'maximumActiveJobs')
#        self.logger.log(logging.DEBUG,getLogMessage("releaseRegisteredJobs(maximumAggregatorActiveJobs): %s" % (str(maximumAggregatorActiveJobs))))
         nAggregatorJobs = self.getRegisteredJobCountAtAggregator(aggregator)
#        self.logger.log(logging.DEBUG,getLogMessage("releaseRegisteredJobs(nAggregatorJobs): %s" % (str(nAggregatorJobs))))
         for hubUserId in aggregatorUserRegisteredJobs[aggregator]:
            try:
               priority = usersPriority[aggregator][hubUserId]
            except KeyError:
               self.logger.log(logging.ERROR,getLogMessage("releaseRegisteredJobs: missing priority, aggregator %s, userId %d" % \
                                                                                                        (aggregator,hubUserId)))
            else:
               # get job info for registered held jobs at aggregator belonging to user
               jobsInfo = self.getAggregatorRegisteredJobsForUser(aggregator,hubUserId)
               nJobsToRelease = max(1,min(len(jobsInfo),int(round(float(nAggregatorJobs)*priority))))
#              self.logger.log(logging.DEBUG,getLogMessage("%s:nJobsToRelease(%d): %d of %d, %d*%f" % \
#                      (aggregator,hubUserId,nJobsToRelease,len(jobsInfo),nAggregatorJobs,priority)))
               for jobInfo in jobsInfo[:nJobsToRelease]:
                  nAggregatorInProcessJobs = 0
                  if jobInfo['aggregator'] in aggregatorsActiveJobCount:
                     nAggregatorInProcessJobs += aggregatorsActiveJobCount[jobInfo['aggregator']]
                  if jobInfo['aggregator'] in aggregatorsReleasedJobCount:
                     nAggregatorInProcessJobs += aggregatorsReleasedJobCount[jobInfo['aggregator']]
                  if nAggregatorInProcessJobs < maximumAggregatorActiveJobs:
                     releaseJobsInfo.append(jobInfo)
                     if not jobInfo['aggregator'] in aggregatorsReleasedJobCount:
                        aggregatorsReleasedJobCount[jobInfo['aggregator']] = 0
                     aggregatorsReleasedJobCount[jobInfo['aggregator']] += 1

      if releaseJobsInfo:
         nReleasedJobs = len(releaseJobsInfo)
         self.updateRegisteredJobStates(releaseJobsInfo,self.JOBREGISTRATIONSTATERELEASED)

      del releaseJobsInfo

      return(nReleasedJobs)


   def isRegisteredJobReleased(self,
                               localJobId,
                               instanceId):
      registeredJobReleased = False
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT state FROM registeredJobs WHERE(localJobId=%s AND instanceId=%s)"
         sqlParameters = (localJobId,instanceId)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               if row[0] == self.JOBREGISTRATIONSTATERELEASED:
                  registeredJobReleased = True
         self.myDatabase.disconnect()

      return(registeredJobReleased)


   def getWSJob(self,
                wsJobId):
      wsJobInfo = None
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT wsJobId,hubUserId,toolRevision,siteName,nCores,wallTimeLimit \
                         FROM wsJobs WHERE(wsJobId=%s)"
         sqlParameters = (wsJobId,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               wsJobInfo = {'wsJobId':row[0],
                            'hubUserId':row[1],
                            'toolRevision':row[2],
                            'siteName':row[3],
                            'nCores':row[4],
                            'wallTimeLimit':row[5]}
         self.myDatabase.disconnect()

      return(wsJobInfo)


   def getWSJobEnrolledIds(self):
      enrolledWSJobIds = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT wsJobId FROM wsJobs WHERE(state=%s)"
         sqlParameters = (self.WSJOBSTATEENROLLED,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               enrolledWSJobIds.append(row[0])
         self.myDatabase.disconnect()

      return(enrolledWSJobIds)


   def getWSJobPostedIds(self):
      postedWSJobIds = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT wsJobId FROM wsJobs WHERE(state=%s AND exitStatusPosted=0)"
         sqlParameters = (self.WSJOBSTATEPOSTED,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            for row in result:
               postedWSJobIds.append(row[0])
         self.myDatabase.disconnect()

      return(postedWSJobIds)


   WSJOBSTATEENROLLED     = 1 << 0
   WSJOBSTATEPOSTING      = 1 << 1
   WSJOBSTATEPOSTED       = 1 << 2
   WSJOBSTATEPRESTAGED    = 1 << 3
   WSJOBSTATESTAGING      = 1 << 4
   WSJOBSTATESTAGEDIN     = 1 << 5
   WSJOBSTATEEXECUTING    = 1 << 6
   WSJOBSTATEEXECUTED     = 1 << 7
   WSJOBSTATEMEASURING    = 1 << 8
   WSJOBSTATEMEASURED     = 1 << 9
   WSJOBSTATECACHING      = 1 << 10
   WSJOBSTATECACHED       = 1 << 11
   WSJOBSTATEDONE         = WSJOBSTATECACHED
   WSJOBSTATEENDOFTHELINE = 1 << 30

   wsStateMap = {'ENROLLED':WSJOBSTATEENROLLED,
                 'POSTING':WSJOBSTATEPOSTING,
                 'POSTED':WSJOBSTATEPOSTED,
                 'PRESTAGED':WSJOBSTATEPRESTAGED,
                 'STAGING':WSJOBSTATESTAGING,
                 'STAGEDIN':WSJOBSTATESTAGEDIN,
                 'EXECUTING':WSJOBSTATEEXECUTING,
                 'EXECUTED':WSJOBSTATEEXECUTED,
                 'MEASURING':WSJOBSTATEMEASURING,
                 'MEASURED':WSJOBSTATEMEASURED,
                 'CACHING':WSJOBSTATECACHING,
                 'CACHED':WSJOBSTATECACHED,
                 'ENDOFTHELINE':WSJOBSTATEENDOFTHELINE
                }


   def addWSJob(self,
                wsJobInfo):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         now = time.time()
         sqlCommand = "INSERT INTO wsJobs (wsJobId,state,hubUserId,toolRevision,siteName,nCores, \
                                           wallTimeLimit,timeEnrolled) \
                            VALUES(%s,%s,%s,%s,%s,%s,%s,%s)"
         sqlParameters = (wsJobInfo['wsJobId'],
                          self.WSJOBSTATEENROLLED,
                          wsJobInfo['hubUserId'],
                          wsJobInfo['toolRevision'],
                          wsJobInfo['siteName'],
                          wsJobInfo['nCores'],
                          wsJobInfo['wallTimeLimit'],
                          now)
         try:
            self.myDatabase.insert(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def deleteWSJob(self,
                   wsJobId):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "DELETE FROM wsJobs WHERE(wsJobId=%s)"
         sqlParameters = (wsJobId,)
         try:
            self.myDatabase.delete(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def updateWSTransactionStatus(self,
                                 wsJobId,
                                 wsStateName,
                                 exitStatus=None,
                                 errorMessage=None):
      signal = False
      if self.myDatabase.connect(self.configData['mysqlDB']):
         try:
            transactionState = self.wsStateMap[wsStateName]
         except:
            transactionState = 0
         now                 = time.time()

         if   transactionState == self.WSJOBSTATEPOSTING:
            sqlCommand = "UPDATE wsJobs SET state=%s \
                                      WHERE(wsJobId=%s)"
            sqlParameters = (transactionState,wsJobId)
         elif transactionState == self.WSJOBSTATEPOSTED:
            if   not exitStatus is None and not errorMessage is None:
               sqlCommand = "UPDATE wsJobs SET state=%s,errorMessage=%s,exitStatusPosted=%s,timePosted=%s \
                                         WHERE(wsJobId=%s)"
               sqlParameters = (transactionState,errorMessage,exitStatus,now,wsJobId)
            elif not exitStatus is None:
               sqlCommand = "UPDATE wsJobs SET state=%s,exitStatusPosted=%s,timePosted=%s \
                                         WHERE(wsJobId=%s)"
               sqlParameters = (transactionState,exitStatus,now,wsJobId)
            else:
               sqlCommand = "UPDATE wsJobs SET state=%s,timePosted=%s \
                                         WHERE(wsJobId=%s)"
               sqlParameters = (transactionState,now,wsJobId)
         elif transactionState == self.WSJOBSTATEPRESTAGED:
            if not exitStatus is None:
               sqlCommand = "UPDATE wsJobs SET state=%s,exitStatusPreStaged=%s,timePreStaged=%s \
                                         WHERE(wsJobId=%s)"
               sqlParameters = (transactionState,exitStatus,now,wsJobId)
            else:
               sqlCommand = "UPDATE wsJobs SET state=%s,timePreStaged=%s \
                                         WHERE(wsJobId=%s)"
               sqlParameters = (transactionState,now,wsJobId)
         elif transactionState == self.WSJOBSTATESTAGING:
            sqlCommand = "UPDATE wsJobs SET state=%s \
                                      WHERE(wsJobId=%s)"
            sqlParameters = (transactionState,wsJobId)
         elif transactionState == self.WSJOBSTATESTAGEDIN:
            if not exitStatus is None:
               sqlCommand = "UPDATE wsJobs SET state=%s,exitStatusStagedIn=%s,timeStagedIn=%s \
                                         WHERE(wsJobId=%s)"
               sqlParameters = (transactionState,exitStatus,now,wsJobId)
            else:
               sqlCommand = "UPDATE wsJobs SET state=%s,timeStagedIn=%s \
                                         WHERE(wsJobId=%s)"
               sqlParameters = (transactionState,now,wsJobId)
         elif transactionState == self.WSJOBSTATEEXECUTING:
            sqlCommand = "UPDATE wsJobs SET state=%s \
                                      WHERE(wsJobId=%s)"
            sqlParameters = (transactionState,wsJobId)
         elif transactionState == self.WSJOBSTATEEXECUTED:
            if not exitStatus is None:
               sqlCommand = "UPDATE wsJobs SET state=%s,exitStatusExecuted=%s,timeExecuted=%s \
                                         WHERE(wsJobId=%s)"
               sqlParameters = (transactionState,exitStatus,now,wsJobId)
            else:
               sqlCommand = "UPDATE wsJobs SET state=%s,timeExecuted=%s \
                                         WHERE(wsJobId=%s)"
               sqlParameters = (transactionState,now,wsJobId)
         elif transactionState == self.WSJOBSTATEMEASURING:
            sqlCommand = "UPDATE wsJobs SET state=%s \
                                      WHERE(wsJobId=%s)"
            sqlParameters = (transactionState,wsJobId)
         elif transactionState == self.WSJOBSTATEMEASURED:
            if not exitStatus is None:
               sqlCommand = "UPDATE wsJobs SET state=%s,exitStatusMeasured=%s,timeMeasured=%s \
                                         WHERE(wsJobId=%s)"
               sqlParameters = (transactionState,exitStatus,now,wsJobId)
            else:
               sqlCommand = "UPDATE wsJobs SET state=%s,timeMeasured=%s \
                                         WHERE(wsJobId=%s)"
               sqlParameters = (transactionState,now,wsJobId)
         elif transactionState == self.WSJOBSTATECACHING:
            sqlCommand = "UPDATE wsJobs SET state=%s \
                                      WHERE(wsJobId=%s)"
            sqlParameters = (transactionState,wsJobId)
         elif transactionState == self.WSJOBSTATECACHED:
            if not exitStatus is None:
               sqlCommand = "UPDATE wsJobs SET state=%s,exitStatusCached=%s,timeCached=%s \
                                         WHERE(wsJobId=%s)"
               sqlParameters = (transactionState,exitStatus,now,wsJobId)
            else:
               sqlCommand = "UPDATE wsJobs SET state=%s,timeCached=%s \
                                         WHERE(wsJobId=%s)"
               sqlParameters = (transactionState,now,wsJobId)
         else:
            sqlCommand    = None
            sqlParameters = None

         if sqlCommand and sqlParameters:
            try:
               self.myDatabase.update(sqlCommand,sqlParameters)
            except:
               pass
            self.myDatabase.commit()
            signal = True
         self.myDatabase.disconnect()

         if signal:
            if   transactionState == self.WSJOBSTATEPRESTAGED:
               self.signalWSJobActionProcess('WSStageInputSim2L')
            elif transactionState == self.WSJOBSTATESTAGEDIN:
               self.signalWSJobActionProcess('WSExecuteSim2L')
            elif transactionState == self.WSJOBSTATEEXECUTED:
               self.signalWSJobActionProcess('WSMeasureSim2L')
            elif transactionState == self.WSJOBSTATEMEASURED:
               self.signalWSJobActionProcess('WSCacheSim2L')


   def getWSJobStatus(self,
                      wsJobId):
      statusMap = {'InputsCached':'STAGINGJOB',
                   'PENDING':'STAGINGJOB',
                   'PROCESSING_INPUTS':'STAGINGJOB',
                   'STAGING_INPUTS':'STAGINGJOB',
                   'STAGING_INPUTS':'STAGINGJOB',
                   'STAGING_INPUTS':'STAGINGJOB',
                   'STAGED':'STAGINGJOB',
                   'STAGING_JOB':'STAGINGJOB',
                   'STAGING_JOB':'STAGINGJOB',
                   'STAGING_JOB':'STAGINGJOB',
                   'SUBMITTING':'STAGINGJOB',
                   'QUEUED':'QUEUED',
                   'RUNNING':'RUNNING',
                   'CLEANING_UP':'RUNNING',
                   'ARCHIVING':'ARCHIVING',
                   'FINISHED':'FINISHED',
                   'STOPPED':'FINISHED',
                   'D':'FINISHED'
                  }

      jobStatus    = 'INVALID'
      errorMessage = ""
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT state,errorMessage, \
                              exitStatusPosted,exitStatusPreStaged,exitStatusStagedIn, \
                              exitStatusExecuted,exitStatusMeasured,exitStatusCached \
                         FROM wsJobs WHERE(wsJobId=%s)"
         sqlParameters = (wsJobId,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               state               = row[0]
               wsErrorMessage      = row[1]
               exitStatusPosted    = row[2]
               exitStatusPreStaged = row[3]
               exitStatusStagedIn  = row[4]
               exitStatusExecuted  = row[5]
               exitStatusMeasured  = row[6]
               exitStatusCached    = row[7]

               if   state == self.WSJOBSTATEENDOFTHELINE:
                  if   self.WSJOBSTATEDONE == self.WSJOBSTATECACHED or \
                       self.WSJOBSTATEDONE == self.WSJOBSTATEMEASURED or \
                       self.WSJOBSTATEDONE == self.WSJOBSTATEEXECUTED:
                     exitCode = -1
                     sqlCommand = "SELECT jobStatistic \
                                     FROM activeJobs WHERE(wsJobId=%s)"
                     sqlParameters = (wsJobId,)
                     try:
                        result = self.myDatabase.select(sqlCommand,sqlParameters)
                     except:
                        pass
                     else:
                        if len(result) == 1:
                           row = result[0]
                           activeJobStatus = row[0]

                           try:
                              jobStatistic = json.loads(row[0])
                           except:
                              pass
                           else:
                              exitCode = jobStatistic['exitCode']

                     if   exitCode > 0:
                        errorMessage = "sim2L failed"
                     elif self.WSJOBSTATEDONE == self.WSJOBSTATECACHED:
                        if exitStatusCached == 0:
                           jobStatus = 'COMPLETE'
                        else:
                           if wsErrorMessage:
                              errorMessage = wsErrorMessage
                           else:
                              errorMessage = "Job caching failed"
                     elif self.WSJOBSTATEDONE == self.WSJOBSTATEMEASURED:
                        if exitStatusMeasured == 0:
                           jobStatus = 'COMPLETE'
                        else:
                           if wsErrorMessage:
                              errorMessage = wsErrorMessage
                           else:
                              errorMessage = "Job metrics failed"
                     elif self.WSJOBSTATEDONE == self.WSJOBSTATEEXECUTED:
                        if exitStatusExecuted == 0:
                           jobStatus = 'COMPLETE'
                        else:
                           if wsErrorMessage:
                              errorMessage = wsErrorMessage
                           else:
                              errorMessage = "Job execution failed"
                  elif self.WSJOBSTATEDONE == self.WSJOBSTATESTAGEDIN:
                     if exitStatusStagedIn == 0:
                        jobStatus = 'COMPLETE'
                     else:
                        if wsErrorMessage:
                           errorMessage = wsErrorMessage
                        else:
                           errorMessage = "Job staging input failed"
                  elif self.WSJOBSTATEDONE == self.WSJOBSTATEPRESTAGED:
                     if exitStatusPreStaged == 0:
                        jobStatus = 'COMPLETE'
                     else:
                        if wsErrorMessage:
                           errorMessage = wsErrorMessage
                        else:
                           errorMessage = "Job prestaging input failed"
                  elif self.WSJOBSTATEDONE == self.WSJOBSTATEPOSTED:
                     if exitStatusPosted == 0:
                        jobStatus = 'COMPLETE'
                     else:
                        if wsErrorMessage:
                           errorMessage = wsErrorMessage
                        else:
                           errorMessage = "Job registration failed"
               elif state == self.WSJOBSTATEENROLLED:
                  jobStatus = 'ENROLLED'
               elif state == self.WSJOBSTATEPOSTING:
                  jobStatus = 'ENROLLED'
               elif state == self.WSJOBSTATEPOSTED:
                  if exitStatusPosted == 0:
                     jobStatus = 'REGISTERED'
                  else:
                     if wsErrorMessage:
                        errorMessage = wsErrorMessage
                     else:
                        errorMessage = "Job registration failed"
               elif state == self.WSJOBSTATEPRESTAGED:
                  if exitStatusPreStaged == 0:
                     jobStatus = 'STAGINGINPUT'
                  else:
                     if wsErrorMessage:
                        errorMessage = wsErrorMessage
                     else:
                        errorMessage = "Job prestaging input failed"
               elif state == self.WSJOBSTATESTAGING:
                  jobStatus = 'STAGINGINPUT'
               elif state == self.WSJOBSTATESTAGEDIN:
                  if exitStatusStagedIn == 0:
                     jobStatus = 'STAGINGINPUT'
                  else:
                     errorMessage = "Job staging input failed"
               elif state == self.WSJOBSTATEEXECUTING or state == self.WSJOBSTATEEXECUTED:
                  if exitStatusExecuted == 0:
                     jobStatus = 'EXECUTING'

                     sqlCommand = "SELECT jobStatus \
                                     FROM activeJobs WHERE(wsJobId=%s)"
                     sqlParameters = (wsJobId,)
                     try:
                        result = self.myDatabase.select(sqlCommand,sqlParameters)
                     except:
                        pass
                     else:
                        if len(result) == 1:
                           row = result[0]
                           activeJobStatus = row[0]

                           try:
                              jobStatus = statusMap[activeJobStatus]
                           except:
                              self.logger.log(logging.WARNING,getLogMessage("Undefined job status: %s" % (activeJobStatus)))
                  else:
                     if wsErrorMessage:
                        errorMessage = wsErrorMessage
                     else:
                        errorMessage = "Job execution failed"
               elif state == self.WSJOBSTATEMEASURING:
                  jobStatus = 'REPORTINGMETRICS'
               elif state == self.WSJOBSTATEMEASURED:
                  if exitStatusMeasured == 0:
                     jobStatus = 'REPORTINGMETRICS'
                  else:
                     if wsErrorMessage:
                        errorMessage = wsErrorMessage
                     else:
                        errorMessage = "Job metrics failed"
               elif state == self.WSJOBSTATECACHING:
                  jobStatus = 'CACHINGOUTPUT'
               elif state == self.WSJOBSTATECACHED:
                  if exitStatusCached == 0:
                     jobStatus = 'CACHINGOUTPUT'
                  else:
                     if wsErrorMessage:
                        errorMessage = wsErrorMessage
                     else:
                        errorMessage = "Job caching failed"

               if jobStatus == 'INVALID':
                  self.logger.log(logging.ERROR,getLogMessage("INVALID: state = %d, exitStatusPosted = %d, exitStatusPreStaged = %d, exitStatusStagedIn = %d, exitStatusExecuted = %d, exitStatusMeasured = %d, exitStatusCached = %d" % (state,exitStatusPosted,exitStatusPreStaged,exitStatusStagedIn,exitStatusExecuted,exitStatusMeasured,exitStatusCached)))

         self.myDatabase.disconnect()

      return(jobStatus,errorMessage)


   def addUserSubmitterClass(self,
                             hubUserId,
                             submitterClass):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "INSERT INTO userSubmitterClasses (hubUserId,submitterClass) \
                            VALUES(%s,%s)"
         sqlParameters = (hubUserId,submitterClass)
         try:
            self.myDatabase.insert(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def updateUserSubmitterClass(self,
                                hubUserId,
                                submitterClass):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "UPDATE userSubmitterClasses SET submitterClass=%s WHERE(hubUserId=%s)"
         sqlParameters = (submitterClass,hubUserId)
         try:
            self.myDatabase.update(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def getUserSubmitterClass(self,
                             hubUserId):
      userSubmitterClass = -1
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT submitterClass FROM userSubmitterClasses WHERE(hubUserId=%s)"
         sqlParameters = (hubUserId,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               userSubmitterClass = row[0]
         self.myDatabase.disconnect()

      return(userSubmitterClass)


   def isJobSitePending(self,
                        jobSite):
      jobSitePending = False
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT COUNT(*) FROM pendingJobSites WHERE(jobSite=%s)"
         sqlParameters = (jobSite,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               nJobSitePending = row[0]
               if nJobSitePending > 0:
                  jobSitePending = True
         self.myDatabase.disconnect()

      return(jobSitePending)


   def addPendingJobSite(self,
                         jobSite):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         timeUpdated = time.time()
         sqlCommand = "INSERT INTO pendingJobSites (jobSite,timeUpdated) \
                            VALUES(%s,%s)"
         sqlParameters = (jobSite,
                          timeUpdated)
         try:
            self.myDatabase.insert(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def getPendingJobSitesReport(self):
      pendingJobSites = {}
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT jobSite,timeUpdated FROM pendingJobSites"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            for row in result:
               pendingJobSites[row[0]] = row[1]
         self.myDatabase.disconnect()

      return(pendingJobSites)


   def deletePendingJobSite(self,
                            jobSite):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "DELETE FROM pendingJobSites WHERE(jobSite=%s)"
         sqlParameters = (jobSite,)
         try:
            nDeleted = self.myDatabase.delete(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   JOBPENDINGSTATECREATED  = 1 << 0
   JOBPENDINGSTATESELECTED = 1 << 1


   def addPendingJobPosting(self,
                            session,
                            instanceToken,
                            wsJobId,
                            siteDesignator,
                            remoteJobId,
                            jobWorkDirectory,
                            localJobId,
                            instanceId,
                            runName):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         timeRecorded = time.time()
         sqlCommand = "INSERT INTO pendingJobs (session,instanceToken,wsJobId, \
                                                state,siteDesignator,remoteJobId,jobWorkDirectory,localJobId, \
                                                instanceId,runName,timeRecorded) \
                            VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
         sqlParameters = (session,
                          instanceToken,
                          wsJobId,
                          self.JOBPENDINGSTATECREATED,
                          siteDesignator,
                          remoteJobId,
                          jobWorkDirectory,
                          localJobId,
                          instanceId,
                          runName,
                          timeRecorded)
         try:
            self.myDatabase.insert(sqlCommand,sqlParameters)
         except:
            pass
         else:
#           self.logger.log(logging.DEBUG,getLogMessage("INSERT pendingJob %s:%s" % (siteDesignator,remoteJobId)))
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def getPendingJobPostingSites(self):
      pendingJobPostingSites = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT DISTINCT siteDesignator FROM pendingJobs"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            for row in result:
               pendingJobPostingSites.append(row[0])
         self.myDatabase.disconnect()

      return(pendingJobPostingSites)


   def getRestartJobSites(self):
      restartJobSites = []
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT DISTINCT jobSite FROM restartJobSites"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            for row in result:
               restartJobSites.append(row[0])
         self.myDatabase.disconnect()

      return(restartJobSites)


   def getRestartJobSitesReport(self):
      restartJobSites = {}
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT jobSite,timeUpdated FROM restartJobSites"
         try:
            result = self.myDatabase.select(sqlCommand)
         except:
            pass
         else:
            for row in result:
               restartJobSites[row[0]] = row[1]
         self.myDatabase.disconnect()

      return(restartJobSites)


   def deleteRestartJobSite(self,
                            jobSite):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "DELETE FROM restartJobSites WHERE(jobSite=%s)"
         sqlParameters = (jobSite,)
         try:
            nDeleted = self.myDatabase.delete(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def signalJobSiteMonitors(self,
                             signalNumber):
      activeJobSites = self.getActiveJobSites()
      sitesToDelete = []
      for siteDesignator in activeJobSites:
         sitePID = self.getActiveJobSitePID(siteDesignator)
         self.logger.log(logging.INFO,getLogMessage("Send signal %d to %d - %s" % (signalNumber,sitePID,siteDesignator)))
         try:
            os.kill(sitePID,signalNumber)
         except OSError:
            sitesToDelete.append(siteDesignator)
         except:
            sitesToDelete.append(siteDesignator)
            self.logger.log(logging.ERROR,getLogMessage(traceback.format_exc()))

      for siteDesignator in sitesToDelete:
         self.deleteActiveJobSite(siteDesignator)


   def countRunningJobSiteMonitors(self):
      nRunningJobSiteMonitors = 0
      signalNumber = 0
      activeJobSites = self.getActiveJobSites()
      for siteDesignator in activeJobSites:
         sitePID = self.getActiveJobSitePID(siteDesignator)
         self.logger.log(logging.INFO,getLogMessage("Send signal %d to %d - %s" % (signalNumber,sitePID,siteDesignator)))
         try:
            os.kill(sitePID,signalNumber)
         except:
            pass
         else:
            nRunningJobSiteMonitors += 1

      return(nRunningJobSiteMonitors)


   def addJobActionProcess(self,
                           action,
                           pid,
                           when):
      success = False
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "INSERT INTO jobActionProcesses (action,processPID,timeRegistered) \
                            VALUES(%s,%s,%s)"
         sqlParameters = (action,pid,when)
         try:
            self.myDatabase.insert(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
            success = True
         self.myDatabase.disconnect()

      return(success)


   def deleteJobActionProcess(self,
                              action):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "DELETE FROM jobActionProcesses WHERE(action=%s)"
         sqlParameters = (action,)
         try:
            nDeleted = self.myDatabase.delete(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def updateJobActionProcessSignaled(self,
                                      action,
                                      when):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "UPDATE jobActionProcesses SET timeSignaled=%s WHERE(action=%s)"
         sqlParameters = (when,action)
         try:
            self.myDatabase.update(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def updateJobActionProcessActivity(self,
                                      action,
                                      when):
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "UPDATE jobActionProcesses SET timeActivity=%s WHERE(action=%s)"
         sqlParameters = (when,action)
         try:
            self.myDatabase.update(sqlCommand,sqlParameters)
         except:
            pass
         else:
            self.myDatabase.commit()
         self.myDatabase.disconnect()


   def getJobActionProcess(self,
                           action):
      processPID   = -1
      timeSignaled = 0
      timeActivity = 0
      if self.myDatabase.connect(self.configData['mysqlDB']):
         sqlCommand = "SELECT processPID,timeSignaled,timeActivity FROM jobActionProcesses WHERE(action=%s)"
         sqlParameters = (action,)
         try:
            result = self.myDatabase.select(sqlCommand,sqlParameters)
         except:
            pass
         else:
            if len(result) == 1:
               row = result[0]
               processPID   = row[0]
               timeSignaled = row[1]
               timeActivity = row[2]
         self.myDatabase.disconnect()

      return(processPID,timeSignaled,timeActivity)


   def checkJobActionProcesses(self):
      signalNumber = 0
      for action in ('Processor','WSReleaseSim2L','WSStageInputSim2L','WSExecuteSim2L','WSMeasureSim2L','WSCacheSim2L'):
         processPID,timeSignaled,timeActivity = self.getJobActionProcess(action)
         if processPID > 0:
            try:
               os.kill(processPID,signalNumber)
            except:
               self.logger.log(logging.INFO,getLogMessage("Job action process %s (%d) is not running." % (action,processPID)))
               self.deleteJobActionProcess(action)
            else:
               self.logger.log(logging.INFO,getLogMessage("Job action process %s (%d) is running." % (action,processPID)))
         else:
            self.logger.log(logging.INFO,getLogMessage("Job action process %s is not registered." % (action)))


   def signalJobActionProcess(self,
                              action):
# Actions:
#    Processor

      processPID,timeSignaled,timeActivity = self.getJobActionProcess(action)
      if processPID > 0:
         try:
            os.kill(processPID,signal.SIGUSR1)
         except:
            self.logger.log(logging.ERROR,getLogMessage("Attempt to signal process %d (%s) failed." % (processPID,action)))
         else:
            signalTime = time.time()
            self.updateJobActionProcessSignaled(action,signalTime)


   def signalWSJobActionProcess(self,
                                action):
# Actions:
#    WSReleaseSim2L
#    WSStageInputSim2L
#    WSExecuteSim2L
#    WSMeasureSim2L
#    WSCacheSim2L

      processPID,timeSignaled,timeActivity = self.getJobActionProcess(action)
      if processPID > 0:
         try:
            os.kill(processPID,signal.SIGUSR2)
         except:
            self.logger.log(logging.ERROR,getLogMessage("Attempt to signal process %d (%s) failed." % (processPID,action)))
         else:
            signalTime = time.time()
            self.updateJobActionProcessSignaled(action,signalTime)


   def processRequests(self):
      message = self.pullMessage(0)
      while message:
         args = message.split()
         if args[0] == 'json':
            jsonMessageLength = int(args[1])
            jsonMessage = self.pullMessage(jsonMessageLength)
            if len(jsonMessage) > 0:
               try:
                  jsonObject = json.loads(jsonMessage)
               except ValueError:
                  self.logger.log(logging.ERROR,getLogMessage("JSON object %s could not be decoded" % (jsonMessage)))
               else:
                  if jsonObject['messageType'] != 'pipeFlusher':
                     self.logger.log(logging.DEBUG,getLogMessage("request = %s, message length = %d" % \
                                                        (jsonObject['messageType'],len(jsonMessage))))
                     requestStart = time.time()
                  if   jsonObject['messageType'] == 'registerJob':
                     session        = jsonObject['session']
                     instanceToken  = jsonObject['instanceToken']
                     wsJobId        = jsonObject['wsJobId']
                     localJobId     = int(jsonObject['localJobId'])
                     instanceId     = int(jsonObject['instanceId'])
                     distributorPid = jsonObject['distributorPid']
                     hubUserId      = jsonObject['hubUserId']
                     siteName       = jsonObject['siteName']
                     runName        = jsonObject['runName']
                     submitterClass = jsonObject['submitterClass']
                     aggregator     = self.aggregatorsInfo.getSiteAggregator(siteName)

                     oldSubmitterClass = self.getUserSubmitterClass(hubUserId)
                     if   oldSubmitterClass < 0:
                        self.addUserSubmitterClass(hubUserId,submitterClass)
                     elif oldSubmitterClass != submitterClass:
                        self.updateUserSubmitterClass(hubUserId,submitterClass)

                     jobInfo = JobInfo(session=session,
                                       instanceToken=instanceToken,
                                       wsJobId=wsJobId,
                                       aggregator=aggregator,
                                       siteName=siteName,
                                       runName=runName,
                                       distributorPid=distributorPid,
                                       localJobId=localJobId,
                                       instanceId=instanceId,
                                       hubUserId=hubUserId)
                     self.addRegisteredJob(jobInfo)
                     del jobInfo

                     returnMessage = {'messageType':'jobInfo',
                                      'aggregator':aggregator,
                                      'siteName':siteName,
                                      'runName':runName,
                                      'localJobId':localJobId,
                                      'instanceId':instanceId,
                                      'jobRegistrationState':self.JOBREGISTRATIONSTATEHELD}
                     self.postJsonMessage(returnMessage)
                     self.signalJobActionProcess('Processor')
                  elif jsonObject['messageType'] == 'deleteRegisteredJob':
                     localJobId = int(jsonObject['localJobId'])
                     instanceId = int(jsonObject['instanceId'])
                     nDeletedRegisteredJobs = self.deleteRegisteredJob(localJobId,instanceId)

                     returnMessage = {'messageType':'deletedRegisteredJobs',
                                      'localJobId':localJobId,
                                      'instanceId':instanceId,
                                      'nDeleted':nDeletedRegisteredJobs}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'releaseRegisteredJobs':
                     nReleasedJobs = self.releaseRegisteredJobs()

                     returnMessage = {'messageType':'releasedRegisteredJobs',
                                      'nReleasedJobs':nReleasedJobs}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'isRegisteredJobReleased':
                     localJobId = int(jsonObject['localJobId'])
                     instanceId = int(jsonObject['instanceId'])
                     registeredJobReleased = self.isRegisteredJobReleased(localJobId,instanceId)

                     returnMessage = {'messageType':'jobReleased',
                                      'localJobId':localJobId,
                                      'instanceId':instanceId,
                                      'jobReleased':registeredJobReleased}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'enrollWSjob':
                     self.addWSJob(jsonObject)

                     returnMessage = {'messageType':'enrolledWSjob',
                                      'wsJobId':jsonObject['wsJobId']}
                     self.postJsonMessage(returnMessage)
                     self.signalWSJobActionProcess('WSReleaseSim2L')
                  elif jsonObject['messageType'] == 'markWSjobAsPosting':
                     wsJobId = jsonObject['wsJobId']
                     self.updateWSTransactionStatus(wsJobId,'POSTING')

                     returnMessage = {'messageType':'markedWSjobAsPosting',
                                      'wsJobId':wsJobId}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'markWSjobAsPosted':
                     wsJobId           = jsonObject['wsJobId']
                     wsStateName       = jsonObject['wsStateName']
                     requestParameters = jsonObject['requestParameters']
                     exitStatus        = requestParameters['exitStatus']
                     errorMessage      = requestParameters['errorMessage']
                     self.updateWSTransactionStatus(wsJobId,
                                                    wsStateName,
                                                    exitStatus=exitStatus,
                                                    errorMessage=errorMessage)

                     returnMessage = {'messageType':'markedWSjobAsPosted',
                                      'wsJobId':wsJobId}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'getEnrolledWSJobIds':
                     enrolledWSJobIds = self.getWSJobEnrolledIds()

                     returnMessage = {'messageType':'fetchedEnrolledWSJobIds',
                                      'enrolledWSJobIds':enrolledWSJobIds}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'getWSJobStatus':
                     wsJobId = jsonObject['wsJobId']
                     jobStatus,errorMessage = self.getWSJobStatus(wsJobId)

                     returnMessage = {'messageType':'wsJobStatus',
                                      'jobStatus':jobStatus,
                                      'errorMessage':errorMessage}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'postJobStaging':
                     session            = jsonObject['session']
                     instanceToken      = jsonObject['instanceToken']
                     wsJobId            = jsonObject['wsJobId']
                     siteName           = jsonObject['siteName']
                     siteDesignator     = jsonObject['siteDesignator']
                     identityNames      = jsonObject['identityNames']
                     preStageDirectory  = jsonObject['preStageDirectory']
                     tapisStorageSystem = jsonObject['tapisStorageSystem']
                     hubUserId          = jsonObject['hubUserId']
                     jobWorkDirectory   = jsonObject['jobWorkDirectory']
                     stagingHandle      = "%s_%s" % (jsonObject['localJobId'],jsonObject['instanceId'])
                     localJobId         = int(jsonObject['localJobId'])
                     instanceId         = int(jsonObject['instanceId'])
                     destination        = jsonObject['destination']
                     runName            = jsonObject['runName']
                     nCores             = jsonObject['nCores']
                     distributorPid     = jsonObject['distributorPid']
                     jobStatistic       = jsonObject['jobStatistic']
                     aggregator         = self.aggregatorsInfo.getSiteAggregator(siteName)

# remoteJobId set because it is required to be unique in combination with siteDesignator
                     if preStageDirectory:
                        activeJobState = self.ACTIVEJOBSTATEPRESTAGED
#                       self.logger.log(logging.DEBUG,getLogMessage("wsJobId(%s): activeState = PRESTAGED" % (wsJobId)))
                        if wsJobId:
                           self.updateWSTransactionStatus(wsJobId,'PRESTAGED')
                     else:
                        activeJobState = self.ACTIVEJOBSTATESTAGEDIN
#                       self.logger.log(logging.DEBUG,getLogMessage("wsJobId(%s): activeState = STAGEDIN" % (wsJobId)))
                        if wsJobId:
                           self.updateWSTransactionStatus(wsJobId,'STAGEDIN')

                     jobInfo = JobInfo(state=activeJobState,
                                       session=session,
                                       instanceToken=instanceToken,
                                       wsJobId=wsJobId,
                                       aggregator=aggregator,
                                       siteName=siteName,
                                       identityNames=identityNames,
                                       siteDesignator=siteDesignator,
                                       preStageDirectory=preStageDirectory,
                                       tapisSystemStageIn=tapisStorageSystem,
                                       stagingHandle=stagingHandle,
                                       remoteJobId=wsJobId[::-1],
                                       distributorPid=distributorPid,
                                       jobStatus='InputsCached',
                                       runName=runName,
                                       nCores=nCores,
                                       localJobId=localJobId,
                                       instanceId=instanceId,
                                       hubUserId=hubUserId,
                                       destination=destination,
                                       jobStatistic=jobStatistic)
                     self.addActiveJob(jobInfo)
                     self.deleteRegisteredJob(localJobId,instanceId)
                     del jobInfo
                     self.dumpActiveJobs()

                     returnMessage = {'messageType':'jobInfo',
                                      'siteDesignator':siteDesignator,
                                      'instanceToken':instanceToken}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'postJob':
                     session          = jsonObject['session']
                     instanceToken    = jsonObject['instanceToken']
                     wsJobId          = jsonObject['wsJobId']
                     siteName         = jsonObject['siteName']
                     siteDesignator   = jsonObject['siteDesignator']
                     identityNames    = jsonObject['identityNames']
                     remoteJobId      = jsonObject['remoteJobId']
                     hubUserId        = jsonObject['hubUserId']
                     jobWorkDirectory = jsonObject['jobWorkDirectory']
                     localJobId       = int(jsonObject['localJobId'])
                     instanceId       = int(jsonObject['instanceId'])
                     destination      = jsonObject['destination']
                     runName          = jsonObject['runName']
                     nCores           = jsonObject['nCores']
                     distributorPid   = jsonObject['distributorPid']
                     tailFiles        = jsonObject['tailFiles']
                     aggregator       = self.aggregatorsInfo.getSiteAggregator(siteName)

                     jobInfo = self.getActiveJob(siteDesignator,remoteJobId)
                     if jobInfo:
                        jobStatus = jobInfo['jobStatus']
                        jobStage  = jobInfo['jobStage']
                     else:
                        jobStatus,jobStage,jobQueue = ('N','Job','?')
                        jobInfo = JobInfo(state=self.ACTIVEJOBSTATEEXECUTING,
                                          session=session,
                                          instanceToken=instanceToken,
                                          wsJobId=wsJobId,
                                          aggregator=aggregator,
                                          siteName=siteName,
                                          identityNames=identityNames,
                                          siteDesignator=siteDesignator,
                                          remoteJobId=remoteJobId,
                                          distributorPid=distributorPid,
                                          runName=runName,
                                          nCores=nCores,
                                          localJobId=localJobId,
                                          instanceId=instanceId,
                                          jobStatus=jobStatus,
                                          jobStage=jobStage,
                                          jobQueue=jobQueue,
                                          hubUserId=hubUserId,
                                          destination=destination)
                        self.addActiveJob(jobInfo)
                        self.deleteRegisteredJob(localJobId,instanceId)
                        del jobInfo
                        for tailFile in tailFiles:
                           nLines = tailFiles[tailFile]['nLines']
                           self.addFileTail(siteDesignator,
                                            remoteJobId,
                                            tailFile,
                                            nLines,
                                            "")
                     self.dumpActiveJobs()

                     returnMessage = {'messageType':'jobInfo',
                                      'siteDesignator':siteDesignator,
                                      'remoteJobId':remoteJobId,
                                      'status':jobStatus,
                                      'stage':jobStage}
                     self.postJsonMessage(returnMessage)
                     self.addPendingJobPosting(session,instanceToken,wsJobId,siteDesignator,remoteJobId,jobWorkDirectory,
                                               jsonObject['localJobId'],jsonObject['instanceId'],runName)
                  elif jsonObject['messageType'] == 'queryJob':
                     siteDesignator = jsonObject['siteDesignator']
                     remoteJobId    = jsonObject['remoteJobId']
                     markReportedAsDone = False
                     jobInfo = self.getActiveJob(siteDesignator,remoteJobId)
                     if jobInfo:
                        jobStatus = jobInfo['jobStatus']
                        jobStage  = jobInfo['jobStage']
                        jobSite   = jobInfo['executionHost']
                        if   jobStatus == 'D':
                           markReportedAsDone = True
                        elif jobStatus == 'Dr':
                           jobStatus = 'D'
                        tailFiles = self.getFileTails(siteDesignator,remoteJobId)
                        self.clearFileTails(siteDesignator,remoteJobId)
                     else:
                        jobStatus,jobStage,jobSite,jobQueue = ('?','?','?','?')
                        tailFiles = {}

                     returnMessage = {'messageType':'jobInfo',
                                      'siteDesignator':siteDesignator,
                                      'remoteJobId':remoteJobId,
                                      'status':jobStatus,
                                      'stage':jobStage,
                                      'site':jobSite,
                                      'tailFiles':tailFiles}
                     self.postJsonMessage(returnMessage)

                     if markReportedAsDone:
                        jobInfo['jobStatus'] = 'Dr'
                        self.updateActiveJob(jobInfo)
                  elif jsonObject['messageType'] == 'terminateJob':
                     siteDesignator = jsonObject['siteDesignator']
                     remoteJobId    = jsonObject['remoteJobId']
                     jobInfo = self.getActiveJob(siteDesignator,remoteJobId)
                     if jobInfo:
                        jobStatus = jobInfo['jobStatus']
                        jobStage  = jobInfo['jobStage']
                     else:
                        jobStatus,jobStage,jobQueue = ('?','?','?')

                     returnMessage = {'messageType':'jobInfo',
                                      'siteDesignator':siteDesignator,
                                      'remoteJobId':remoteJobId,
                                      'status':jobStatus,
                                      'stage':jobStage}
                     self.postJsonMessage(returnMessage)

                     if jobInfo:
                        jobInfo['jobStatus'] = 'D'
                        jobInfo['jobStage']  = 'Job'
                        self.updateActiveJob(jobInfo)
                  elif jsonObject['messageType'] == 'queryJobs':
                     jobsStatus = {}
                     jobsInfo = []
                     jobs = jsonObject['jobs']
                     for job in jobs:
                        siteDesignator = job['siteDesignator']
                        remoteJobId    = job['remoteJobId']
                        jobInfo = self.getActiveJob(siteDesignator,remoteJobId)
                        if not jobInfo:
                           jobStatus,jobStage,jobSite,jobQueue = ('?','?','?','?')
                           jobInfo = JobInfo(siteDesignator=siteDesignator,
                                             remoteJobId=remoteJobId,
                                             jobStatus=jobStatus,
                                             jobStage=jobStage,
                                             executionHost=jobSite,
                                             jobQueue=jobQueue)
                        jobsInfo.append(jobInfo)
                        if not siteDesignator in jobsStatus:
                           jobsStatus[siteDesignator] = {}

                     for jobInfo in jobsInfo:
                        siteDesignator = jobInfo['siteDesignator']
                        remoteJobId    = jobInfo['remoteJobId']
                        jobStatus      = jobInfo['jobStatus']
                        jobStage       = jobInfo['jobStage']
                        jobSite        = jobInfo['executionHost']
                        jobQueue       = jobInfo['jobQueue']

                        markReportedAsDone = False
                        if   jobStatus == 'D':
                           markReportedAsDone = True
                        elif jobStatus == 'Dr':
                           jobStatus = 'D'
                        if jobStatus != '?':
                           tailFiles = self.getFileTails(siteDesignator,remoteJobId)
                           self.clearFileTails(siteDesignator,remoteJobId)
                        else:
                           tailFiles = {}

                        jobsStatus[siteDesignator][remoteJobId] = {'status':jobStatus,
                                                                   'stage':jobStage,
                                                                   'site':jobSite,
                                                                   'tailFiles':tailFiles}
                        if markReportedAsDone:
                           jobInfo['jobStatus'] = 'Dr'
                           self.updateActiveJob(jobInfo)

                     returnMessage = {'messageType':'jobsStatus',
                                      'jobsStatus':jobsStatus}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'postWorkflow':
                     session                   = jsonObject['session']
                     instanceToken             = jsonObject['instanceToken']
                     wsJobId                   = jsonObject['wsJobId']
                     siteName                  = jsonObject['siteName']
                     siteDesignator            = jsonObject['siteDesignator']
                     identityNames             = jsonObject['identityNames']
                     remoteJobId               = jsonObject['remoteJobId']
                     hubUserId                 = jsonObject['hubUserId']
                     localJobId                = int(jsonObject['localJobId'])
                     nInstances                = jsonObject['nInstances']
                     instancesInitialJobStatus = jsonObject['instancesInitialJobStatus']
                     destination               = jsonObject['destination']
                     runName                   = jsonObject['runName']
                     nCores                    = jsonObject['nCores']
                     distributorPid            = jsonObject['distributorPid']
                     tailFiles                 = []
                     instanceId                = 0
                     aggregator                = self.aggregatorsInfo.getSiteAggregator(siteName)

                     jobInfo = self.getActiveJob(siteDesignator,remoteJobId)
                     if jobInfo:
                        jobStatus = jobInfo['jobStatus']
                        jobStage  = jobInfo['jobStage']
                     else:
                        jobStatus,jobStage,jobQueue = ('N','DAG','?')
                        jobInfo = JobInfo(state=self.ACTIVEJOBSTATEEXECUTING,
                                          session=session,
                                          instanceToken=instanceToken,
                                          wsJobId=wsJobId,
                                          aggregator=aggregator,
                                          siteName=siteName,
                                          identityNames=identityNames,
                                          siteDesignator=siteDesignator,
                                          remoteJobId=remoteJobId,
                                          distributorPid=distributorPid,
                                          runName=runName,
                                          nCores=1,
                                          localJobId=localJobId,
                                          instanceId=instanceId,
                                          jobStatus=jobStatus,
                                          jobStage=jobStage,
                                          jobQueue=jobQueue,
                                          hubUserId=hubUserId,
                                          destination=destination)
                        self.addActiveJob(jobInfo)
                        self.deleteRegisteredJob(localJobId,instanceId)
                        del jobInfo
                     self.dumpActiveJobs()

                     returnMessage = {'messageType':'jobInfo',
                                      'siteDesignator':siteDesignator,
                                      'remoteJobId':remoteJobId,
                                      'status':jobStatus,
                                      'stage':jobStage}
                     self.postJsonMessage(returnMessage)
                     jobWorkDirectory = ""
                     self.addPendingJobPosting(session,instanceToken,wsJobId,siteDesignator,remoteJobId,jobWorkDirectory,
                                               jsonObject['localJobId'],'00',runName)

                     if '.' in remoteJobId:
                        wfRemoteJobIdBase = remoteJobId.split('.')[0]
                     else:
                        wfRemoteJobIdBase = remoteJobId
                     nInstanceIdDigits = max(2,int(math.log10(nInstances)+1))
                     wfJobStatus,wfJobStage,wfJobQueue = ('WF','Simulation','?')
                     for instance in iterRange(1,nInstances+1):
                        if instancesInitialJobStatus:
                           wfJobStatus = instancesInitialJobStatus.pop(0)
                        wfRemoteJobId = wfRemoteJobIdBase + '.' + str(instance)
                        jobInfo = self.getActiveJob(siteDesignator,wfRemoteJobId)
                        if not jobInfo:
                           jobInfo = JobInfo(state=self.ACTIVEJOBSTATEEXECUTING,
                                             aggregator=aggregator,
                                             siteName=siteName,
                                             identityNames=identityNames,
                                             siteDesignator=siteDesignator,
                                             remoteJobId=wfRemoteJobId,
                                             distributorPid=distributorPid,
                                             runName=runName,
                                             nCores=nCores,
                                             localJobId=localJobId,
                                             instanceId=instance,
                                             jobStatus=wfJobStatus,
                                             jobStage=wfJobStage,
                                             jobQueue=wfJobQueue,
                                             hubUserId=hubUserId,
                                             destination=destination)
                           self.addActiveJob(jobInfo)
                           self.deleteRegisteredJob(localJobId,instance)
                           del jobInfo
                     self.dumpActiveJobs()
                  elif jsonObject['messageType'] == 'queryWorkflow':
                     siteDesignator = jsonObject['siteDesignator']
                     remoteJobId    = jsonObject['remoteJobId']
                     nInstances     = jsonObject['nInstances']
                     markReportedAsDone = False
                     jobInfo = self.getActiveJob(siteDesignator,remoteJobId)
                     if jobInfo:
                        jobStatus = jobInfo['jobStatus']
                        jobStage  = jobInfo['jobStage']
                        jobSite   = jobInfo['executionHost']
                        jobQueue  = jobInfo['jobQueue']
                        if   jobStatus == 'D':
                           markReportedAsDone = True
                        elif jobStatus == 'Dr':
                           jobStatus = 'D'
                     else:
                        jobStatus,jobStage,jobSite,jobQueue = ('?','?','?','?')
                     del jobInfo
                     wfInstances = {}
                     if '.' in remoteJobId:
                        wfRemoteJobIdBase = remoteJobId.split('.')[0]
                     else:
                        wfRemoteJobIdBase = remoteJobId
                     for instance in iterRange(1,nInstances+1):
                        wfMarkReportedAsDone = False
                        wfRemoteJobId = wfRemoteJobIdBase + '.' + str(instance)
                        jobInfo = self.getActiveJob(siteDesignator,wfRemoteJobId)
                        if jobInfo:
                           wfJobStatus = jobInfo['jobStatus']
                           wfJobStage  = jobInfo['jobStage']
                           wfJobSite   = jobInfo['executionHost']
                           wfJobQueue  = jobInfo['jobQueue']
                           if   wfJobStatus == 'D':
                              wfMarkReportedAsDone = True
                           elif wfJobStatus == 'Dr':
                              wfJobStatus = 'D'
                        else:
                           wfJobStatus,wfJobStage,wfJobSite,wfJobQueue = ('?','?','?','?')
                        wfInstances[instance] = {}
                        wfInstances[instance]['remoteJobId']        = wfRemoteJobId
                        wfInstances[instance]['jobStatus']          = wfJobStatus
                        wfInstances[instance]['jobStage']           = wfJobStage
                        wfInstances[instance]['jobSite']            = wfJobSite
                        wfInstances[instance]['jobQueue']           = wfJobQueue
                        wfInstances[instance]['markReportedAsDone'] = wfMarkReportedAsDone
                        del jobInfo

                     instanceStates = {}
                     for instance in iterRange(1,nInstances+1):
                        if wfInstances[instance]['jobStatus'] != '?':
                           instanceStates[instance] = {'status':wfInstances[instance]['jobStatus'],
                                                       'stage':wfInstances[instance]['jobStage'],
                                                       'site':wfInstances[instance]['jobSite']}

                     returnMessage = {'messageType':'workflowInfo',
                                      'siteDesignator':siteDesignator,
                                      'remoteJobId':remoteJobId,
                                      'status':jobStatus,
                                      'stage':jobStage,
                                      'site':jobSite,
                                      'instances':instanceStates}
                     self.postJsonMessage(returnMessage)

                     if markReportedAsDone:
                        jobStatus = 'Dr'
                        jobInfo = JobInfo(siteDesignator=siteDesignator,
                                          remoteJobId=remoteJobId,
                                          jobStatus=jobStatus,
                                          jobStage=jobStage,
                                          jobQueue=jobQueue)
                        self.updateActiveJob(jobInfo)
                        del jobInfo
                     for instance in iterRange(1,nInstances+1):
                        if wfInstances[instance]['markReportedAsDone']:
                           wfRemoteJobId = wfInstances[instance]['remoteJobId']
                           wfJobStatus   = 'Dr'
                           wfJobStage    = wfInstances[instance]['jobStage']
                           wfJobQueue    = wfInstances[instance]['jobQueue']
                           jobInfo = JobInfo(siteDesignator=siteDesignator,
                                             remoteJobId=wfRemoteJobId,
                                             jobStatus=wfJobStatus,
                                             jobStage=wfJobStage,
                                             jobQueue=wfJobQueue)
                           self.updateActiveJob(jobInfo)
                           del jobInfo
                     del instanceStates
                     del wfInstances
                  elif jsonObject['messageType'] == 'jobStatusUpdate':
                     jobStatusUpdate = jsonObject['jobStatusUpdate']
                     instanceToken   = jobStatusUpdate['instanceToken']
                     jobStatus       = jobStatusUpdate['jobStatus']
#                    self.logger.log(logging.DEBUG,getLogMessage("jobStatusUpdate: %s %s" % (instanceToken,jobStatus)))
                     self.updateActiveJobStatus(instanceToken,jobStatus)
                     returnMessage = {'messageType':'jobStatusUpdated',
                                      'instanceToken':instanceToken,
                                      'jobStatus':jobStatus}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'queryJobByToken':
                     instanceToken = jsonObject['instanceToken']
                     jobStatus = self.getActiveJobStatus(instanceToken)
                     returnMessage = {'messageType':'jobStatusByToken',
                                      'instanceToken':instanceToken,
                                      'jobStatus':jobStatus}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'queryUserInstanceToken':
                     hubUserId  = int(jsonObject['hubUserId'])
                     localJobId = int(jsonObject['localJobId'])
                     instanceId = int(jsonObject['instanceId'])
                     instanceToken = self.getActiveJobInstanceToken(hubUserId,
                                                                    localJobId,
                                                                    instanceId)
                     returnMessage = {'messageType':'userInstanceToken',
                                      'instanceToken':instanceToken}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'queryUserRemoteJobId':
                     hubUserId     = int(jsonObject['hubUserId'])
                     instanceToken = jsonObject['instanceToken']
                     remoteJobId = self.getActiveJobRemoteJobId(hubUserId,
                                                                instanceToken)
                     returnMessage = {'messageType':'userRemoteJobId',
                                      'remoteJobId':remoteJobId}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'queryActiveJobJobStatistic':
                     instanceToken = jsonObject['instanceToken']
                     session,localJobId,instanceId,stagingHandle,remoteJobId,jobStatistic = \
                                                                                  self.getActiveJobJobStatistic(instanceToken)
                     returnMessage = {'messageType':'activeJobJobStatistic',
                                      'session':session,
                                      'localJobId':localJobId,
                                      'instanceId':instanceId,
                                      'stagingHandle':stagingHandle,
                                      'remoteJobId':remoteJobId,
                                      'jobStatistic':jobStatistic}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'updateActiveJobJobStatistic':
                     instanceToken = jsonObject['instanceToken']
                     jobStatistic  = jsonObject['jobStatistic']
                     self.updateActiveJobJobStatistic(instanceToken,jobStatistic)
                     returnMessage = {'messageType':'updatedActiveJobJobStatistic',
                                      'instanceToken':instanceToken}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'getWSTaskURL':
                     wsTask = jsonObject['wsTask']
                     wsAPI  = self.configData['webServiceAPI']
                     wsTaskURL = ""
                     if wsAPI:
                        if   wsTask == 'register':
                           apiTask = self.configData['webServiceRegister']
                        elif wsTask == 'stageInput':
                           apiTask = self.configData['webServiceStageInput']
                        elif wsTask == 'execute':
                           apiTask = self.configData['webServiceExecute']
                        elif wsTask == 'cacheResult':
                           apiTask = self.configData['webServiceCacheResult']
                        elif wsTask == 'metrics':
                           apiTask = self.configData['webServiceMetrics']
                        elif wsTask == 'registerComplete':
                           apiTask = self.configData['webServiceRegisterComplete']
                        elif wsTask == 'stageInputComplete':
                           apiTask = self.configData['webServiceStageInputComplete']
                        elif wsTask == 'cacheResultComplete':
                           apiTask = self.configData['webServiceCacheResultComplete']
                        elif wsTask == 'metricsComplete':
                           apiTask = self.configData['webServiceMetricsComplete']
                        else:
                           apiTask = ""

                        if apiTask:
                           wsTaskURL = '/'.join([wsAPI,apiTask])

                     returnMessage = {'messageType':'wsTaskURL',
                                      'wsTaskURL':wsTaskURL}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'getActiveJobPreStaged':
                     wsJobId = jsonObject['wsJobId']
                     preStagedActiveJob = self.getPreStagedActiveJob(wsJobId)
                     returnMessage = {'messageType':'activeJobPreStaged',
                                      'instanceToken':preStagedActiveJob['instanceToken'],
                                      'preStageDirectory':preStagedActiveJob['preStageDirectory'],
                                      'tapisSystemStageIn':preStagedActiveJob['tapisSystemStageIn'],
                                      'stagingHandle':preStagedActiveJob['stagingHandle'],
                                      'siteName':preStagedActiveJob['siteName']}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'getActiveJobStaged':
                     wsJobId = jsonObject['wsJobId']
                     stagedActiveJob = self.getStagedActiveJob(wsJobId)

                     returnMessage = {'messageType':'activeJobStaged',
                                      'instanceToken':stagedActiveJob['instanceToken'],
                                      'tapisSystemStageIn':stagedActiveJob['tapisSystemStageIn'],
                                      'stagingHandle':stagedActiveJob['stagingHandle'],
                                      'siteName':stagedActiveJob['siteName']}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'updateActiveJobState':
                     wsJobId           = jsonObject['wsJobId']
                     activeStateName   = jsonObject['activeStateName']
                     requestParameters = jsonObject['requestParameters']
                     self.updateActiveJobState(wsJobId,activeStateName,requestParameters)

                     returnMessage = {'messageType':'updatedActiveJobState',
                                      'wsJobId':wsJobId}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'getActiveJobWSJobIds':
                     activeStateName = jsonObject['activeStateName']
                     activeJobWSJobIds = self.getActiveJobWSJobIds(activeStateName)
                     returnMessage = {'messageType':'fetchedActiveJobWSJobIds',
                                      'wsJobIds':activeJobWSJobIds}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'getActiveJobCacheSimTool':
                     wsJobId = jsonObject['wsJobId']
                     cacheSimToolActiveJob = self.getCacheSimToolActiveJob(wsJobId)
                     returnMessage = {'messageType':'activeJobCacheSimTool',
                                      'instanceToken':cacheSimToolActiveJob['instanceToken'],
                                      'stagingHandle':cacheSimToolActiveJob['stagingHandle'],
                                      'siteName':cacheSimToolActiveJob['siteName']}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'getActiveJobMetricsCollection':
                     wsJobId = jsonObject['wsJobId']
                     metricsCollectionActiveJob = self.getMetricsCollectionActiveJob(wsJobId)
                     returnMessage = {'messageType':'activeJobMetricsCollection',
                                      'instanceToken':metricsCollectionActiveJob['instanceToken'],
                                      'stagingHandle':metricsCollectionActiveJob['stagingHandle'],
                                      'siteName':metricsCollectionActiveJob['siteName']}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'markActiveJobsAsComplete':
                     self.markActiveJobsAsComplete()
                     returnMessage = {'messageType':'markedActiveJobsAsComplete'}
                     self.postJsonMessage(returnMessage)
                  elif jsonObject['messageType'] == 'reportJobSites':
                     jobSiteState = jsonObject['jobSiteState']
                     if   jobSiteState == 'active':
                        jobSites = self.getActiveJobSitesReport()
                     elif jobSiteState == 'pending':
                        jobSites = self.getPendingJobSitesReport()
                     elif jobSiteState == 'restart':
                        jobSites = self.getRestartJobSitesReport()
                     else:
                        jobSites = self.getActiveJobSitesReport()

                     report = {'messageType':'jobSitesReport',
                               'jobSites':jobSites}
                     self.postJsonMessage(report)
                  elif jsonObject['messageType'] == 'deleteJobSite':
                     jobSiteState   = jsonObject['jobSiteState']
                     siteDesignator = jsonObject['siteDesignator']
                     if   jobSiteState == 'active':
                        self.deleteActiveJobSite(siteDesignator)
                     elif jobSiteState == 'pending':
                        self.deletePendingJobSite(siteDesignator)
                     elif jobSiteState == 'restart':
                        self.deleteRestartJobSite(siteDesignator)
                     else:
                        self.deleteActiveJobSite(siteDesignator)

                     report = {'messageType':'deletedJobSite',
                               'siteDesignator':siteDesignator}
                     self.postJsonMessage(report)
                  elif jsonObject['messageType'] == 'reportSites':
                     siteDesignator = jsonObject['siteDesignator']
                     remoteJobId    = jsonObject['remoteJobId']

                     if siteDesignator == "*":
                        siteDesignators = self.getSitesWithActiveJob()
                     else:
                        siteDesignators = [siteDesignator]

                     report = {'messageType':'siteJobInfo',
                               'maxLastReportTime':0,
                               'siteDesignators':{}}
                     maxLastReportTime = 0
                     for siteDesignator in siteDesignators:
                        report['siteDesignators'][siteDesignator] = {}
                        lastReportTime = self.getActiveJobSiteTimeUpdated(siteDesignator)
                        maxLastReportTime = max(maxLastReportTime,lastReportTime)
                        report['siteDesignators'][siteDesignator]['lastReportTime'] = lastReportTime

                        report['siteDesignators'][siteDesignator]['queues'] = {}
                        activeQueues = self.getActiveJobSiteQueues(siteDesignator)
                        for activeQueue in activeQueues:
                           report['siteDesignators'][siteDesignator]['queues'][activeQueue] = {}

                           if remoteJobId == "*":
                              reportJobs = self.getActiveJobsInQueue(siteDesignator,activeQueue)
                              for reportJob in reportJobs:
                                 report['siteDesignators'][siteDesignator]['queues'][activeQueue][reportJob['remoteJobId']] = \
                                                                                            {'status':reportJob['jobStatus'], \
                                                                                             'stage':reportJob['jobStage'], \
                                                                                             'site':reportJob['executionHost']}
                           else:
                              reportJob = self.getActiveJob(siteDesignator,remoteJobId)
                              if reportJob:
                                 if reportJob['jobQueue'] == activeQueue:
                                    report['siteDesignators'][siteDesignator]['queues'][activeQueue][reportJob['remoteJobId']] = \
                                                                                           {'status':reportJob['jobStatus'], \
                                                                                            'stage':reportJob['jobStage'], \
                                                                                            'site':reportJob['executionHost']}

                     report['maxLastReportTime'] = maxLastReportTime
                     self.postJsonMessage(report)
                     del siteDesignators
                  elif jsonObject['messageType'] == 'listUserActivity':
                     hubUserIds = jsonObject['hubUserIds']
                     report = ""
                     usersActivity = self.getUsersActivity(hubUserIds)

                     report = {'messageType':'userActivityInfo',
                               'reportTime':time.time(),
                               'usersActivity':usersActivity}
                     self.postJsonMessage(report)
                     del usersActivity
                  elif jsonObject['messageType'] == 'listUserPriority':
                     hubUserIds = jsonObject['hubUserIds']
                     report = ""
                     usersPriority = self.getUsersPriority(hubUserIds)

                     report = {'messageType':'userPriorityInfo',
                               'reportTime':time.time(),
                               'usersPriority':usersPriority}
                     self.postJsonMessage(report)
                     del usersPriority
                  elif jsonObject['messageType'] == 'listUserJobs':
                     hubUserId = jsonObject['hubUserId']
                     reportActiveJobs     = self.getActiveJobsForUser(hubUserId)
                     reportRegisteredJobs = self.getHeldRegisteredJobsForUser(hubUserId)
                     report = {'messageType':'userJobInfo',
                               'reportTime':time.time(),
                               'userActiveJobs':reportActiveJobs,
                               'userRegisteredJobs':reportRegisteredJobs}
                     self.postJsonMessage(report)
                     del reportActiveJobs
                  elif jsonObject['messageType'] == 'getJobPID':
                     hubUserId  = jsonObject['hubUserId']
                     localJobId = jsonObject['localJobId']

                     activeJobPid = self.getActiveJobPidForUser(hubUserId,localJobId)
                     report = {'messageType':'jobPID',
                               'hubUserId':hubUserId,
                               'reportTime':time.time(),
                               'pid':activeJobPid}
                     self.postJsonMessage(report)
                  elif jsonObject['messageType'] == 'queryIdentityUsers':
                     identityName = jsonObject['identityName']
                     usersWithActiveJobWithIdentity = []
                     hubUserIdsWithActiveJobWithIdentity = self.getUsersWithActiveJobWithIdentity(identityName)
                     for hubUserId in hubUserIdsWithActiveJobWithIdentity:
                        try:
                           hubUserName = pwd.getpwuid(hubUserId).pw_name
                        except:
                           self.logger.log(logging.ERROR,getLogMessage("Unable to get info for user '%d'" % (hubUserId)))
                        else:
                           usersWithActiveJobWithIdentity.append(hubUserName)
                     report = {'messageType':'identityUsers',
                               'identityName':identityName,
                               'identityActiveJobUsers':usersWithActiveJobWithIdentity}
                     self.postJsonMessage(report)
                  elif jsonObject['messageType'] == 'registerJobActionProcess':
                     action = jsonObject['action']
                     pid    = jsonObject['pid']
                     when   = jsonObject['when']
                     self.deleteJobActionProcess(action)
                     success = self.addJobActionProcess(action,pid,when)
                     if success:
                        self.logger.log(logging.INFO,getLogMessage("Job action process %s (%d) registered." % (action,pid)))
                     else:
                        self.logger.log(logging.ERROR,getLogMessage("Job action process %s (%d) registration failed." % \
                                                                                                         (action,pid)))
                     if   action == 'Processor':
                        timeBetween = self.configData['timeBetweenJobRelease']
                     elif action == 'WSReleaseSim2L':
                        timeBetween = self.configData['timeBetweenJobRelease']
                     elif action == 'WSStageInputSim2L':
                        timeBetween = self.configData['timeBetweenJobStageInput']
                     elif action == 'WSExecuteSim2L':
                        timeBetween = self.configData['timeBetweenJobExecute']
                     elif action == 'WSMeasureSim2L':
                        timeBetween = self.configData['timeBetweenJobMetrics']
                     elif action == 'WSCacheSim2L':
                        timeBetween = self.configData['timeBetweenJobCacheSimTools']
                     else:
                        timeBetween = 999
                     report = {'messageType':'jobActionProcessRegistered',
                               'action':action,
                               'timeBetween':timeBetween,
                               'success':success}
                     self.postJsonMessage(report)
                  elif jsonObject['messageType'] == 'updateJobActionProcessSignalTime':
                     action = jsonObject['action']
                     when   = jsonObject['when']
                     success = self.updateJobActionProcessSignaled(action,when)
                     report = {'messageType':'jobActionProcessSignalTimeUpdated',
                               'action':action}
                     self.postJsonMessage(report)
                  elif jsonObject['messageType'] == 'updateJobActionProcessActivityTime':
                     action = jsonObject['action']
                     when   = jsonObject['when']
                     success = self.updateJobActionProcessActivity(action,when)
                     report = {'messageType':'jobActionProcessActivityTimeUpdated',
                               'action':action}
                     self.postJsonMessage(report)
                  elif jsonObject['messageType'] == 'pipeFlusher':
                     pass
                  else:
                     self.logger.log(logging.ERROR,getLogMessage("Discarded message type: %s" % (jsonObject['messageType'])))

                  if jsonObject['messageType'] != 'pipeFlusher':
                     requestTime = time.time()-requestStart
                     self.logger.log(logging.DEBUG,getLogMessage("request = %s, processing time = %f" % \
                                                              (jsonObject['messageType'],requestTime)))
            else:
               self.pushMessage(message + '\n')
               self.logger.log(logging.DEBUG,getLogMessage("incomplete request"))
               break
         else:
            self.logger.log(logging.ERROR,getLogMessage("Discarded message: %s" % (message)))

         message = self.pullMessage(0)


