# @package      hubzero-submit-monitors
# @file         JobMonitor.py
# @copyright    Copyright (c) 2012-2020 The Regents of the University of California.
# @license      http://opensource.org/licenses/MIT MIT
#
# Copyright (c) 2012-2020 The Regents of the University of California.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# HUBzero is a registered trademark of The Regents of the University of California.
#
import os
import re
import subprocess
import pwd
import copy
import math
import time
import signal
import traceback
import json
import base64
import logging

from hubzero.submit.LogMessage       import getLogIDMessage as getLogMessage
from hubzero.submit.BoundConnections import BoundConnections
from hubzero.submit.MySQLiteDatabase import MySQLiteDatabase
from hubzero.submit.JobInfo          import JobInfo
from hubzero.submit.InfosInfo        import InfosInfo
from hubzero.submit.SitesInfo        import SitesInfo
from hubzero.submit.AggregatorsInfo  import AggregatorsInfo

try:
   iterRange = xrange
except NameError as e:
   iterRange = range

class JobMonitor(BoundConnections):
   def __init__(self,
                infosDirectory,
                infosConfigurationFile,
                listenURI,
                activeJobDBPath="monitorJob.db",
                activeJobDumpPath="monitorJob.sql"):
      BoundConnections.__init__(self,listenURI,logConnection=False)

      self.logger = logging.getLogger(__name__)

      self.infosDirectory         = infosDirectory
      self.infosConfigurationFile = infosConfigurationFile

      self.infosInfo       = None
      self.sitesInfo       = None
      self.aggregatorsInfo = None
      self.settingInfo     = False

      self.pendingJobSites    = []
      self.pendingJobPostings = {}
      self.restartJobSites    = []
      self.terminating        = False
      self.channels           = {}

      self.activeJobDBPath    = activeJobDBPath
      self.activeJobDumpPath  = activeJobDumpPath
      self.myDatabase         = MySQLiteDatabase(self.activeJobDBPath,self.activeJobDumpPath)
      if not self.myDatabase.connect():
         self.logger.log(logging.ERROR,getLogMessage("Job monitor database connection failed"))


   def killJobSiteMonitors(self):
      self.signalJobSiteMonitors(signal.SIGTERM)
      time.sleep(5)
      if self.countRunningJobSiteMonitors() > 0:
         self.signalJobSiteMonitors(signal.SIGKILL)
      self.purgeActiveJobSites()


   def updateSchema(self,
                    targetSchema):
      schemaUpdated = False
      userVersion = self.myDatabase.getDBuserVersion()
      if userVersion >= 0:
         while userVersion < targetSchema:
            if   userVersion == 0:
               sqlScript = """
                              CREATE TABLE activeJobs (
                                 aggregator     VARCHAR(32) DEFAULT '?',
                                 localJobId     INT         NOT NULL,
                                 instanceId     INT         NOT NULL,
                                 distributorPid INT         NOT NULL,
                                 hubUserId      INT         DEFAULT NULL,
                                 siteName       VARCHAR(32) DEFAULT '?',
                                 nCores         INT         DEFAULT 1,
                                 runName        VARCHAR(32) NOT NULL,
                                 siteDesignator VARCHAR(32) NOT NULL,
                                 remoteJobId    VARCHAR(32) NOT NULL,
                                 jobStatus      VARCHAR(32) DEFAULT '?',
                                 jobStage       VARCHAR(32) DEFAULT '?',
                                 jobQueue       VARCHAR(32) DEFAULT '?',
                                 destination    VARCHAR(32) DEFAULT '?',
                                 executionHost  VARCHAR(64) DEFAULT '?',
                                 timeRecorded   DOUBLE      DEFAULT '0.'
                              );
                              CREATE UNIQUE INDEX activeJobs_globalJobId       ON activeJobs (siteDesignator,remoteJobId);
                              CREATE        INDEX activeJobs_localJobId        ON activeJobs (localJobId,instanceId);
                              CREATE        INDEX activeJobs_siteDesignator    ON activeJobs (siteDesignator);
                              CREATE        INDEX activeJobs_siteQueue         ON activeJobs (siteDesignator,jobQueue);
                              CREATE        INDEX activeJobs_siteStatus        ON activeJobs (siteDesignator,jobStatus);
                              CREATE        INDEX activeJobs_instanceStatus    ON activeJobs (instanceId,jobStatus);
                              CREATE        INDEX activeJobs_hubUserId         ON activeJobs (hubUserId);
                              CREATE        INDEX activeJobs_userStatus        ON activeJobs (hubUserId,jobStatus);
                              CREATE        INDEX activeJobs_userLocalIdStatus ON activeJobs (hubUserId,localJobId,jobStatus);
                              CREATE        INDEX activeJobs_statusTime        ON activeJobs (jobStatus,timeRecorded);

                              CREATE TABLE activeJobIdentities (
                                 localJobId   INT         NOT NULL,
                                 instanceId   INT         NOT NULL,
                                 hubUserId    INT         NOT NULL,
                                 identityName VARCHAR(32) NOT NULL
                              );
                              CREATE INDEX activeJobIdentities_localJobId   ON activeJobIdentities (localJobId,instanceId);
                              CREATE INDEX activeJobIdentities_identityName ON activeJobIdentities (identityName);
                              CREATE INDEX activeJobIdentities_identityUser ON activeJobIdentities (identityName,hubUserId);

                              CREATE TABLE activeJobSites (
                                 jobSite     VARCHAR(32) UNIQUE PRIMARY KEY,
                                 pid         INT         NOT NULL,
                                 timeUpdated DOUBLE      DEFAULT '0.'
                              );

                              CREATE TABLE userActivityScores (
                                 aggregator    VARCHAR(32) DEFAULT '?',
                                 siteName      VARCHAR(32) DEFAULT '?',
                                 hubUserId     INT         NOT NULL,
                                 activityScore DOUBLE      DEFAULT '0.5',
                                 timeUpdated   DOUBLE      DEFAULT '-1.'
                              );
                              CREATE INDEX userActivityScores_hubUserId  ON userActivityScores (hubUserId);
                              CREATE INDEX userActivityScores_aggregator ON userActivityScores (aggregator,hubUserId);
                              CREATE INDEX userActivityScores_siteUser   ON userActivityScores (siteName,hubUserId);

                              CREATE TABLE fileTailings (
                                 siteDesignator VARCHAR(32)  NOT NULL,
                                 remoteJobId    VARCHAR(32)  NOT NULL,
                                 fileName       VARCHAR(256) NOT NULL,
                                 timeUpdated    DOUBLE       DEFAULT '0.',
                                 tailText       BLOB
                              );
                              CREATE UNIQUE INDEX fileTailings_globalTailId ON fileTailings (siteDesignator,remoteJobId,fileName);
                              CREATE        INDEX fileTailings_globalRunId  ON fileTailings (siteDesignator,remoteJobId);

                              CREATE TABLE registeredJobs (
                                 aggregator     VARCHAR(32) DEFAULT '?',
                                 state          INT         NOT NULL,
                                 localJobId     INT         NOT NULL,
                                 instanceId     INT         NOT NULL,
                                 distributorPid INT         NOT NULL,
                                 hubUserId      INT         DEFAULT NULL,
                                 siteName       VARCHAR(32) DEFAULT '?',
                                 timeRegistered DOUBLE      DEFAULT '0.'
                              );
                              CREATE INDEX registeredJobs_aggregatorState ON registeredJobs (aggregator,state);
                              CREATE INDEX registeredJobs_siteName        ON registeredJobs (siteName);
                              CREATE INDEX registeredJobs_hubUserId       ON registeredJobs (hubUserId);
                              CREATE INDEX registeredJobs_userSiteName    ON registeredJobs (hubUserId,siteName);
                              CREATE INDEX registeredJobs_localJobId      ON registeredJobs (localJobId,instanceId);

                              CREATE TABLE userSubmitterClasses (
                                 hubUserId      INT UNIQUE PRIMARY KEY,
                                 submitterClass INT NOT NULL
                              );
                           """
               self.myDatabase.script(sqlScript)
               self.myDatabase.commit()
               self.myDatabase.setDBuserVersion(1)
            elif userVersion == 1:
               sqlScript = """
                              CREATE TABLE registeredJobs_X (
                                 aggregator     VARCHAR(32) DEFAULT '?',
                                 state          INT         NOT NULL,
                                 runName        VARCHAR(32) NOT NULL,
                                 localJobId     INT         NOT NULL,
                                 instanceId     INT         NOT NULL,
                                 distributorPid INT         NOT NULL,
                                 hubUserId      INT         DEFAULT NULL,
                                 siteName       VARCHAR(32) DEFAULT '?',
                                 timeRegistered DOUBLE      DEFAULT '0.'
                              );

                              INSERT INTO registeredJobs_X (aggregator,state,runName,localJobId,instanceId,
                                                            distributorPid,hubUserId,siteName,timeRegistered)
                                        SELECT aggregator,state,'?',localJobId,instanceId,
                                               distributorPid,hubUserId,siteName,timeRegistered FROM registeredJobs;

                              DROP TABLE registeredJobs;
                              ALTER TABLE registeredJobs_X RENAME TO registeredJobs;

                              CREATE INDEX registeredJobs_aggregatorState ON registeredJobs (aggregator,state);
                              CREATE INDEX registeredJobs_siteName        ON registeredJobs (siteName);
                              CREATE INDEX registeredJobs_hubUserId       ON registeredJobs (hubUserId);
                              CREATE INDEX registeredJobs_userSiteName    ON registeredJobs (hubUserId,siteName);
                              CREATE INDEX registeredJobs_localJobId      ON registeredJobs (localJobId,instanceId);
                           """
               self.myDatabase.script(sqlScript)
               self.myDatabase.commit()
               self.myDatabase.setDBuserVersion(2)
            elif userVersion == 2:
               sqlScript = """
                              CREATE TABLE fileTailings_X (
                                 siteDesignator VARCHAR(32)  NOT NULL,
                                 remoteJobId    VARCHAR(32)  NOT NULL,
                                 fileName       VARCHAR(256) NOT NULL,
                                 nLines         INT          DEFAULT 10,
                                 timeUpdated    DOUBLE       DEFAULT '0.',
                                 tailText       BLOB
                              );

                              INSERT INTO fileTailings_X (siteDesignator,remoteJobId,fileName,nLines,timeUpdated,tailText)
                                        SELECT siteDesignator,remoteJobId,fileName,'?',timeUpdated,tailText
                                               FROM fileTailings;

                              DROP TABLE fileTailings;
                              ALTER TABLE fileTailings_X RENAME TO fileTailings;

                              CREATE UNIQUE INDEX fileTailings_globalTailId ON fileTailings (siteDesignator,remoteJobId,fileName);
                              CREATE        INDEX fileTailings_globalRunId  ON fileTailings (siteDesignator,remoteJobId);
                           """
               self.myDatabase.script(sqlScript)
               self.myDatabase.commit()
               self.myDatabase.setDBuserVersion(3)
            elif userVersion == 3:
               sqlScript = """
                              ALTER TABLE activeJobs MODIFY remoteJobId VARCHAR(40);
                              ALTER TABLE fileTailings MODIFY remoteJobId VARCHAR(40);
                              ALTER TABLE pendingJobs MODIFY remoteJobId VARCHAR(40);
                           """
               self.myDatabase.script(sqlScript)
               self.myDatabase.commit()
               self.myDatabase.setDBuserVersion(4)

            userVersion = self.myDatabase.getDBuserVersion()

         if userVersion == targetSchema:
            schemaUpdated = True

      return(schemaUpdated)


   def setInfo(self):
      errorInSetInfo = False
      if not self.settingInfo:
         self.settingInfo = True

         configFilePath = os.path.join(self.infosDirectory,self.infosConfigurationFile)
         if self.infosInfo:
            del self.infosInfo
         self.infosInfo = InfosInfo(configFilePath)

         if self.sitesInfo:
            del self.sitesInfo
         self.sitesInfo = SitesInfo(self.infosInfo.getInfoPath('sites'),
                                    allowedVenueMechanisms=['*'],
                                    pegasusVersion='*')
         enabledSites      = self.sitesInfo.getEnabledSites()
         expandedSiteNames = self.sitesInfo.getExpandedSiteNames(enabledSites)

         if self.aggregatorsInfo:
            del self.aggregatorsInfo
         self.aggregatorsInfo = AggregatorsInfo(self.infosInfo.getInfoPath('aggregators'))

         for expandedSiteName in expandedSiteNames:
            aggregatorName = self.aggregatorsInfo.getSiteAggregator(expandedSiteName)
            if not aggregatorName:
               self.logger.log(logging.ERROR,getLogMessage("Site %s is not included in any aggregator" % (expandedSiteName)))
               errorInSetInfo = True

         del self.sitesInfo
         self.sitesInfo = None
         del expandedSiteNames
         del enabledSites

         self.settingInfo = False

      return(errorInSetInfo)


   def terminate(self):
      if not self.terminating:
         self.signalJobSiteMonitors(signal.SIGTERM)
         self.closeListeningConnection()
         self.terminating = True


   def dumpActiveJobs(self):
      if self.myDatabase:
         self.myDatabase.dump()


   def close(self):
      if self.myDatabase:
         self.myDatabase.close()
      self.closeListeningConnection()


   def addActiveJobSite(self,
                        jobSite,
                        channel,
                        pid):
      self.channels[jobSite] = channel
      now = time.time()
      sqlCommand = "INSERT INTO activeJobSites (jobSite,pid,timeUpdated) \
                           VALUES(?,?,?)"
      sqlParameters = (jobSite,pid,now)
      self.myDatabase.insert(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def updateActiveJobSite(self,
                           jobSite):
      now = time.time()
      sqlCommand = "UPDATE activeJobSites SET timeUpdated=? WHERE(jobSite=?)"
      sqlParameters = (now,jobSite)
      self.myDatabase.update(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def getActiveJobSites(self):
      activeJobSites = []
      sqlCommand = "SELECT jobSite FROM activeJobSites"
      result = self.myDatabase.select(sqlCommand)
      for row in result:
         activeJobSites.append(row['jobSite'])

      return(activeJobSites)


   def getActiveJobSiteChannel(self,
                               jobSite):
      try:
         channel = self.channels[jobSite]
      except ValueError:
         channel = None

      return(channel)


   def getActiveJobSitePID(self,
                           jobSite):
      pid = 0
      sqlCommand = "SELECT * FROM activeJobSites WHERE(jobSite=?)"
      sqlParameters = (jobSite,)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      if len(result) == 1:
         row = result[0]
         pid = row['pid']

      return(pid)


   def getActiveJobSiteTimeUpdated(self,
                                   jobSite):
      timeUpdated = 0
      sqlCommand = "SELECT * FROM activeJobSites WHERE(jobSite=?)"
      sqlParameters = (jobSite,)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      if result:
         row = result[0]
         timeUpdated = row['timeUpdated']

      return(timeUpdated)


   def getActiveJobSitesReport(self):
      activeJobSites = {}
      sqlCommand = "SELECT jobSite,timeUpdated FROM activeJobSites"
      result = self.myDatabase.select(sqlCommand)
      if result:
         for row in result:
            activeJobSites[row['jobSite']] = row['timeUpdated']

      return(activeJobSites)


   def deleteActiveJobSite(self,
                           jobSite):
      sqlCommand = "DELETE FROM activeJobSites WHERE(jobSite=?)"
      sqlParameters = (jobSite,)
      self.myDatabase.delete(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def purgeActiveJobSites(self):
      sqlCommand = "DELETE FROM activeJobSites"
      self.myDatabase.delete(sqlCommand)
      self.myDatabase.commit()


   def isJobSiteActive(self,
                       jobSite):
      jobSiteIsActive = False
      sqlCommand = "SELECT * FROM activeJobSites WHERE(jobSite=?)"
      sqlParameters = (jobSite,)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      if len(result) == 1:
         jobSiteIsActive = True

      return(jobSiteIsActive)


   def getAggregatorsWithRegisteredJob(self):
      aggregatorsWithRegisteredJob = []
      sqlCommand = "SELECT DISTINCT aggregator FROM registeredJobs"
      result = self.myDatabase.select(sqlCommand)
      for row in result:
         aggregatorsWithRegisteredJob.append(row['siteName'])

      return(aggregatorsWithRegisteredJob)


   def getSitesWithRegisteredJob(self):
      sitesWithRegisteredJob = []
      sqlCommand = "SELECT DISTINCT siteName FROM registeredJobs"
      result = self.myDatabase.select(sqlCommand)
      for row in result:
         sitesWithRegisteredJob.append(row['siteName'])

      return(sitesWithRegisteredJob)


   def getUsersWithRegisteredJob(self):
      usersWithRegisteredJob = []
      sqlCommand = "SELECT DISTINCT hubUserId FROM registeredJobs \
                                             WHERE state=?"
      sqlParameters = (self.JOBREGISTRATIONSTATEHELD,)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      for row in result:
         usersWithRegisteredJob.append(row['hubUserId'])

      return(usersWithRegisteredJob)


   def getAggregatorUserRegisteredJob(self):
      aggregatorUserRegisteredJobs = {}
      sqlCommand = "SELECT DISTINCT aggregator,hubUserId FROM registeredJobs \
                                                        WHERE state=?"
      sqlParameters = (self.JOBREGISTRATIONSTATEHELD,)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      for row in result:
         if not row['aggregator'] in aggregatorUserRegisteredJobs:
            aggregatorUserRegisteredJobs[row['aggregator']] = []
         aggregatorUserRegisteredJobs[row['aggregator']].append(row['hubUserId'])

      return(aggregatorUserRegisteredJobs)


   def getAggregatorRegisteredJobsForUser(self,
                                          aggregator,
                                          hubUserId):
      jobsInfo = []
      sqlCommand = "SELECT localJobId,instanceId,siteName FROM registeredJobs \
                                                         WHERE(aggregator=? AND hubUserId=? AND state=?) \
                                                      ORDER BY timeRegistered"
      sqlParameters = (aggregator,hubUserId,self.JOBREGISTRATIONSTATEHELD)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      for row in result:
         jobInfo = JobInfo(localJobId=row['localJobId'],
                           instanceId=row['instanceId'],
                           siteName=row['siteName'],
                           aggregator=aggregator)
         jobsInfo.append(dict(jobInfo))

      return(jobsInfo)


   def getHeldRegisteredJobsForUser(self,
                                    hubUserId):
      jobsInfo = []
      sqlCommand = "SELECT runName,localJobId,instanceId,siteName FROM registeredJobs \
                                                                 WHERE(hubUserId=? AND state=?) \
                                                              ORDER BY timeRegistered"
      sqlParameters = (hubUserId,self.JOBREGISTRATIONSTATEHELD)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      for row in result:
         jobInfo = JobInfo(runName=row['runName'],
                           localJobId=row['localJobId'],
                           instanceId=row['instanceId'],
                           siteName=row['siteName'])
         jobsInfo.append(dict(jobInfo))

      return(jobsInfo)


   def getReleasedRegisteredJobsForUser(self,
                                        hubUserId):
      jobsInfo = []
      sqlCommand = "SELECT runName,localJobId,instanceId,siteName FROM registeredJobs \
                                                                 WHERE(hubUserId=? AND state=?) \
                                                              ORDER BY timeRegistered"
      sqlParameters = (hubUserId,self.JOBREGISTRATIONSTATERELEASED)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      for row in result:
         jobInfo = JobInfo(runName=row['runName'],
                           localJobId=row['localJobId'],
                           instanceId=row['instanceId'],
                           siteName=row['siteName'])
         jobsInfo.append(dict(jobInfo))

      return(jobsInfo)


   def getRegisteredJobsForUser(self,
                                hubUserId):
      jobsInfo = []
      sqlCommand = "SELECT runName,localJobId,instanceId,siteName FROM registeredJobs \
                                                                 WHERE(hubUserId=?) \
                                                              ORDER BY timeRegistered"
      sqlParameters = (hubUserId,)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      for row in result:
         jobInfo = JobInfo(runName=row['runName'],
                           localJobId=row['localJobId'],
                           instanceId=row['instanceId'],
                           siteName=row['siteName'])
         jobsInfo.append(dict(jobInfo))

      return(jobsInfo)


   def getRegisteredJob(self,
                        localJobId,
                        instanceId):
      jobInfo = None
      sqlCommand = "SELECT * FROM registeredJobs WHERE(localJobId=? AND instanceId=?)"
      sqlParameters = (localJobId,instanceId)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      if len(result) == 1:
         row = result[0]
         jobInfo = JobInfo(distributorPid=row['distributorPid'],
                           localJobId=row['localJobId'],
                           instanceId=row['instanceId'],
                           hubUserId=row['hubUserId'],
                           siteName=row['siteName'],
                           aggregator=row['aggregator'],
                           timeRecorded=row['timeRegistered'])

      return(jobInfo)


   JOBREGISTRATIONSTATEHELD     = 1 << 0
   JOBREGISTRATIONSTATERELEASED = 1 << 1


   def addRegisteredJob(self,
                        jobInfo):
      sqlCommand = "INSERT INTO registeredJobs (aggregator,state,runName,localJobId,instanceId, \
                                                distributorPid,hubUserId,siteName,timeRegistered) \
                         VALUES(?,?,?,?,?,?,?,?,?)"
      sqlParameters = (jobInfo['aggregator'],
                       self.JOBREGISTRATIONSTATEHELD,
                       jobInfo['runName'],
                       jobInfo['localJobId'],
                       jobInfo['instanceId'],
                       jobInfo['distributorPid'],
                       jobInfo['hubUserId'],
                       jobInfo['siteName'],
                       jobInfo['timeRecorded'])
      self.myDatabase.insert(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def deleteRegisteredJob(self,
                           localJobId,
                           instanceId):
      sqlCommand = "DELETE FROM registeredJobs WHERE(localJobId=? AND instanceId=?)"
      sqlParameters = (localJobId,instanceId)
      nDeletedRegisteredJobs = self.myDatabase.delete(sqlCommand,sqlParameters)
      self.myDatabase.commit()

      return(nDeletedRegisteredJobs)


   def getRegisteredJobCountAtAggregator(self,
                                         aggregator):
      nRegisteredJobCount = 0
      sqlCommand = "SELECT COUNT(*) FROM registeredJobs WHERE(aggregator=?)"
      sqlParameters = (aggregator,)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      if len(result) == 1:
         row = result[0]
         nRegisteredJobCount = row[0]

      return(nRegisteredJobCount)


   def getRegisteredJobCountAtSite(self,
                                   siteName):
      nRegisteredJobCount = 0
      sqlCommand = "SELECT COUNT(*) FROM registeredJobs WHERE(siteName=?)"
      sqlParameters = (siteName,)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      if len(result) == 1:
         row = result[0]
         nRegisteredJobCount = row[0]

      return(nRegisteredJobCount)


   def getSitesWithActiveJob(self):
      sitesWithActiveJob = []
      sqlCommand = "SELECT DISTINCT siteDesignator FROM activeJobs"
      result = self.myDatabase.select(sqlCommand)
      for row in result:
         sitesWithActiveJob.append(row['siteDesignator'])

      return(sitesWithActiveJob)


   def getActiveJobSiteQueues(self,
                              jobSite):
      activeJobSiteQueues = []
      sqlCommand = "SELECT DISTINCT jobQueue FROM activeJobs WHERE(siteDesignator=?)"
      sqlParameters = (jobSite,)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      for row in result:
         activeJobSiteQueues.append(row['jobQueue'])

      return(activeJobSiteQueues)


   def getActiveJobsInQueue(self,
                            jobSite,
                            jobQueue):
      jobsInfo = []
      sqlCommand = "SELECT * FROM activeJobs WHERE(siteDesignator=? AND jobQueue=?)"
      sqlParameters = (jobSite,jobQueue)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      for row in result:
         jobInfo = JobInfo(aggregator=row['aggregator'],
                           siteName=row['siteName'],
                           siteDesignator=row['siteDesignator'],
                           remoteJobId=row['remoteJobId'],
                           distributorPid=row['distributorPid'],
                           runName=row['runName'],
                           nCores=row['nCores'],
                           localJobId=row['localJobId'],
                           instanceId=row['instanceId'],
                           jobStatus=row['jobStatus'],
                           jobStage=row['jobStage'],
                           jobQueue=row['jobQueue'],
                           hubUserId=row['hubUserId'],
                           destination=row['destination'],
                           executionHost=row['executionHost'],
                           timeRecorded=row['timeRecorded'])
         jobsInfo.append(dict(jobInfo))

      return(jobsInfo)


   def getActiveSiteDesignators(self):
      activeSiteDesignators = []
      sqlCommand = "SELECT siteDesignator FROM activeJobs \
                                         WHERE(jobStatus!='Dr' AND jobStatus!='D')"
      result = self.myDatabase.select(sqlCommand)
      for row in result:
         if not row['siteDesignator'] in activeSiteDesignators:
            activeSiteDesignators.append(row['siteDesignator'])

      return(activeSiteDesignators)


   def getActiveJobsForUser(self,
                            hubUserId):
      jobsInfo = []
      sqlCommand = "SELECT * FROM activeJobs \
                            WHERE(hubUserId=? AND jobStatus!='Dr' AND jobStatus!='D')"
      sqlParameters = (hubUserId,)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      for row in result:
         jobInfo = JobInfo(aggregator=row['aggregator'],
                           siteName=row['siteName'],
                           siteDesignator=row['siteDesignator'],
                           remoteJobId=row['remoteJobId'],
                           distributorPid=row['distributorPid'],
                           runName=row['runName'],
                           nCores=row['nCores'],
                           localJobId=row['localJobId'],
                           instanceId=row['instanceId'],
                           jobStatus=row['jobStatus'],
                           jobStage=row['jobStage'],
                           jobQueue=row['jobQueue'],
                           hubUserId=row['hubUserId'],
                           destination=row['destination'],
                           executionHost=row['executionHost'],
                           timeRecorded=row['timeRecorded'])
         jobsInfo.append(dict(jobInfo))

      return(jobsInfo)


   def getActiveJobPidForUser(self,
                              hubUserId,
                              localJobId):
      activeJobPid = None
      jobsInfo = []
      sqlCommand = "SELECT * FROM activeJobs \
                            WHERE(hubUserId=? AND localJobId=? AND \
                                  jobStatus!='N' AND jobStatus!='Dr' AND jobStatus!='D') \
                            ORDER BY localJobId,instanceId"
      sqlParameters = (hubUserId,localJobId)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      for row in result:
         jobInfo = JobInfo(siteDesignator=row['siteDesignator'],
                           remoteJobId=row['remoteJobId'],
                           distributorPid=row['distributorPid'],
                           localJobId=row['localJobId'],
                           instanceId=row['instanceId'])
         jobsInfo.append(jobInfo)

      if len(jobsInfo) > 0:
         for jobInfo in jobsInfo:
            if jobInfo['distributorPid'] > 0:
               activeJobPid = jobInfo['distributorPid']
               break

      del jobsInfo

      return(activeJobPid)


   def getSitesActiveJobCount(self):
      sitesActiveJobCount = {}
      sqlCommand = "SELECT siteName,COUNT(*) AS nJobs \
                      FROM activeJobs \
                     WHERE(jobStatus!='Dr' AND jobStatus!='D' AND instanceId!=0) \
                  GROUP BY siteName"
      result = self.myDatabase.select(sqlCommand)
      for row in result:
         sitesActiveJobCount[row['siteName']] = row['nJobs']

      return(sitesActiveJobCount)


   def getAggregatorsActiveJobCount(self):
      aggregatorsActiveJobCount = {}
      sqlCommand = "SELECT aggregator,COUNT(*) AS nJobs \
                      FROM activeJobs \
                     WHERE(jobStatus!='Dr' AND jobStatus!='D' AND instanceId!=0) \
                  GROUP BY aggregator"
      result = self.myDatabase.select(sqlCommand)
      for row in result:
         aggregatorsActiveJobCount[row['aggregator']] = row['nJobs']

      return(aggregatorsActiveJobCount)


   def getUsersWithActiveJobWithIdentity(self,
                                         identityName):
      usersWithActiveJobWithIdentity = []
      sqlCommand = "SELECT DISTINCT activeJobIdentities.hubUserId \
                               FROM activeJobIdentities JOIN activeJobs \
                                 ON (activeJobIdentities.localJobId=activeJobs.localJobId AND \
                                     activeJobIdentities.instanceId=activeJobs.instanceId) \
                              WHERE (activeJobs.jobStatus!='Dr' AND \
                                     activeJobs.jobStatus!='D' AND \
                                     activeJobs.instanceId!=0 AND \
                                     activeJobIdentities.identityName=?)"
      sqlParameters = (identityName,)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      for row in result:
         usersWithActiveJobWithIdentity.append(row['hubUserId'])

      return(usersWithActiveJobWithIdentity)


   def getActiveJob(self,
                    siteDesignator,
                    remoteJobId):
      jobInfo = None
      sqlCommand = "SELECT * FROM activeJobs WHERE(siteDesignator=? AND remoteJobId=?)"
      sqlParameters = (siteDesignator,remoteJobId)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      if len(result) == 1:
         row = result[0]
         jobInfo = JobInfo(aggregator=row['aggregator'],
                           siteName=row['siteName'],
                           siteDesignator=siteDesignator,
                           remoteJobId=remoteJobId,
                           distributorPid=row['distributorPid'],
                           runName=row['runName'],
                           nCores=row['nCores'],
                           localJobId=row['localJobId'],
                           instanceId=row['instanceId'],
                           jobStatus=row['jobStatus'],
                           jobStage=row['jobStage'],
                           jobQueue=row['jobQueue'],
                           hubUserId=row['hubUserId'],
                           destination=row['destination'],
                           executionHost=row['executionHost'],
                           timeRecorded=row['timeRecorded'])

      return(jobInfo)


   def addActiveJob(self,
                    jobInfo):
      sqlCommand = "INSERT INTO activeJobs (aggregator,localJobId,instanceId,distributorPid,hubUserId,siteName, \
                                            nCores,runName,siteDesignator,remoteJobId,jobStatus,jobStage,jobQueue,destination, \
                                            executionHost,timeRecorded) \
                           VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
      sqlParameters = (jobInfo['aggregator'],
                       jobInfo['localJobId'],
                       jobInfo['instanceId'],
                       jobInfo['distributorPid'],
                       jobInfo['hubUserId'],
                       jobInfo['siteName'],
                       jobInfo['nCores'],
                       jobInfo['runName'],
                       jobInfo['siteDesignator'],
                       jobInfo['remoteJobId'],
                       jobInfo['jobStatus'],
                       jobInfo['jobStage'],
                       jobInfo['jobQueue'],
                       jobInfo['destination'],
                       jobInfo['executionHost'],
                       jobInfo['timeRecorded'])
      self.myDatabase.insert(sqlCommand,sqlParameters)
      for identityName in jobInfo['identityNames']:
         sqlCommand = "INSERT INTO activeJobIdentities (localJobId,instanceId,hubUserId,identityName) \
                            VALUES(?,?,?,?)"
         sqlParameters = (jobInfo['localJobId'],
                          jobInfo['instanceId'],
                          jobInfo['hubUserId'],
                          identityName)
         self.myDatabase.insert(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def updateActiveJob(self,
                       jobInfo):
      timeRecorded = time.time()
      sqlCommand = "UPDATE activeJobs SET jobStatus=?,jobStage=?,jobQueue=?,executionHost=?,timeRecorded=? \
                                    WHERE(siteDesignator=? AND remoteJobId=?)"
      sqlParameters = (jobInfo['jobStatus'],jobInfo['jobStage'], \
                       jobInfo['jobQueue'],jobInfo['executionHost'], \
                       timeRecorded, \
                       jobInfo['siteDesignator'],jobInfo['remoteJobId'])
      self.myDatabase.update(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def purgeActiveJobs(self,
                       purgeJobStatus,
                       cutOffAge=0.):
      cutOffTime = time.time()-cutOffAge
      sqlCommand = "DELETE FROM activeJobs WHERE(jobStatus=? AND timeRecorded<=?)"
      sqlParameters = (purgeJobStatus,cutOffTime)
      nPurgeActiveJobs = self.myDatabase.delete(sqlCommand,sqlParameters)
      self.myDatabase.commit()

      sqlCommand = "DELETE FROM activeJobIdentities \
                          WHERE(localJobId || instanceId) \
                             IN(SELECT activeJobIdentities.localJobId || activeJobIdentities.instanceId \
                                  FROM activeJobIdentities \
                             LEFT JOIN activeJobs ON (activeJobIdentities.localJobId=activeJobs.localJobId AND \
                                                      activeJobIdentities.instanceId=activeJobs.instanceId) \
                                 WHERE activeJobs.localJobId IS NULL)"
      self.myDatabase.delete(sqlCommand)
      self.myDatabase.commit()

      return(nPurgeActiveJobs)


   @staticmethod
   def __textToBlob(text):
      try:
         blob = base64.b64encode(text).decode('utf-8')
      except UnicodeEncodeError:
         blob = base64.b64encode(text.encode('utf-8')).decode('utf-8')
      except TypeError:
         blob = base64.b64encode(text.encode('utf-8')).decode('utf-8')

      return(blob)


   @staticmethod
   def __blobToText(blob):
      try:
         text = base64.b64decode(blob.encode('utf-8'))
      except AttributeError:
         text = base64.b64decode(blob).decode('utf-8')

      return(text)


   def getFileTail(self,
                   siteDesignator,
                   remoteJobId,
                   fileName):
      nLines   = 0
      tailText = ""
      sqlCommand = "SELECT * FROM fileTailings \
                            WHERE(siteDesignator=? AND remoteJobId=? AND fileName=?)"
      sqlParameters = (siteDesignator,remoteJobId,fileName)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      if len(result) == 1:
         row = result[0]
         nLines   = row['nLines']
         tailBlob = row['tailText']
         tailText = self.__blobToText(tailBlob)

      return(nLines,tailText)


   def getFileTails(self,
                    siteDesignator,
                    remoteJobId):
      tailTexts = {}
      sqlCommand = "SELECT * FROM fileTailings \
                            WHERE(siteDesignator=? AND remoteJobId=?)"
      sqlParameters = (siteDesignator,remoteJobId)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      for row in result:
         fileName    = row['fileName']
         tailBlob    = row['tailText']
         timeUpdated = row['timeUpdated']
         tailText    = self.__blobToText(tailBlob)
         tailTexts[fileName] = {'tailText':tailText,
                                'timeUpdated':timeUpdated}

      return(tailTexts)


   def addFileTail(self,
                   siteDesignator,
                   remoteJobId,
                   fileName,
                   nLines,
                   tailText):
      timeUpdated = time.time()
      sqlCommand = "INSERT INTO fileTailings (siteDesignator,remoteJobId,fileName,nLines,timeUpdated,tailText) \
                         VALUES(?,?,?,?,?,?)"
      tailBlob = self.__textToBlob(tailText)
#
# Maximum blob length in MySQL is 65535 characters
# If tailText is longer than the maximum allowed it must be truncated
# Reserve 4 bytes for padding and account for length increasing with
# base64 encoding
#
# maxTextLength = floor((65535-4)*6/8) = 49148
#
      tailBlob = tailBlob[-49148:]
      sqlParameters = (siteDesignator,
                       remoteJobId,
                       fileName,
                       nLines,
                       timeUpdated,
                       tailBlob)
      self.myDatabase.insert(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def updateFileTail(self,
                      siteDesignator,
                      remoteJobId,
                      fileName,
                      tailText):
      nLines,tailTextDB = self.getFileTail(siteDesignator,
                                           remoteJobId,
                                           fileName)
      self.logger.log(logging.DEBUG,getLogMessage("updateFileTail: %d lines of %s" % (nLines,fileName)))
      updatedTailTextLines  = []
      if tailTextDB:
         updatedTailTextLines += tailTextDB.split('\n')
      if tailText:
         updatedTailTextLines += tailText.split('\n')
      updatedTailText = '\n'.join(updatedTailTextLines[-nLines:])
      updatedTailBlob = self.__textToBlob(updatedTailText)
#     self.logger.log(logging.DEBUG,getLogMessage("updateFileTail: updatedTailText = %s" % (updatedTailText)))

      timeUpdated = time.time()
      sqlCommand = "UPDATE fileTailings SET timeUpdated=?,tailText=? \
                                WHERE(siteDesignator=? AND remoteJobId=? AND fileName=?)"
      sqlParameters = (timeUpdated,updatedTailBlob, \
                       siteDesignator,remoteJobId, \
                       fileName)
      self.myDatabase.update(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def clearFileTail(self,
                     siteDesignator,
                     remoteJobId,
                     fileName):
      tailText = ""
      tailBlob = self.__textToBlob(tailText)
      timeUpdated = time.time()
      sqlCommand = "UPDATE fileTailings SET timeUpdated=?,tailText=? \
                                WHERE(siteDesignator=? AND remoteJobId=? AND fileName=?)"
      sqlParameters = (timeUpdated,tailBlob, \
                       siteDesignator,remoteJobId, \
                       fileName)
      self.myDatabase.update(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def clearFileTails(self,
                      siteDesignator,
                      remoteJobId):
      tailText = ""
      tailBlob = self.__textToBlob(tailText)
      timeUpdated = time.time()
      sqlCommand = "UPDATE fileTailings SET timeUpdated=?,tailText=? \
                                WHERE(siteDesignator=? AND remoteJobId=?)"
      sqlParameters = (timeUpdated,tailBlob, \
                       siteDesignator,remoteJobId)
      self.myDatabase.update(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def deleteFileTail(self,
                      siteDesignator,
                      remoteJobId,
                      fileName):
      sqlCommand = "DELETE FROM fileTailings \
                          WHERE(siteDesignator=? AND remoteJobId=? AND fileName=?)"
      sqlParameters = (siteDesignator,remoteJobId,fileName)
      self.myDatabase.delete(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def deleteFileTails(self,
                       siteDesignator,
                       remoteJobId):
      sqlCommand = "DELETE FROM fileTailings \
                          WHERE(siteDesignator=? AND remoteJobId=?)"
      sqlParameters = (siteDesignator,remoteJobId)
      self.myDatabase.delete(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def purgeFileTails(self):
      sqlCommand = "DELETE FROM fileTailings \
                          WHERE(siteDesignator || remoteJobId) \
                             IN(SELECT fileTailings.siteDesignator || fileTailings.remoteJobId \
                                  FROM fileTailings \
                             LEFT JOIN activeJobs ON (fileTailings.siteDesignator=activeJobs.siteDesignator AND \
                                                      fileTailings.remoteJobId=activeJobs.remoteJobId) \
                                 WHERE activeJobs.siteDesignator IS NULL)"
      nPurgeFileTails = self.myDatabase.delete(sqlCommand)
      self.myDatabase.commit()

      return(nPurgeFileTails)


   def updateWFJobs(self,
                    siteDesignator):
      jobsInfo = []
      sqlCommand = "SELECT * FROM activeJobs WHERE(siteDesignator=? AND jobStatus='WF')"
      sqlParameters = (siteDesignator,)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      for row in result:
         jobInfo = JobInfo(siteDesignator=row['siteDesignator'],
                           remoteJobId=row['remoteJobId'],
                           localJobId=row['localJobId'],
                           instanceId=row['instanceId'],
                           jobStatus=row['jobStatus'],
                           jobStage=row['jobStage'],
                           jobQueue=row['jobQueue'])
         jobsInfo.append(jobInfo)

      commit = False
      for jobInfo in jobsInfo:
         sqlCommand = "SELECT * FROM activeJobs WHERE(localJobId=? AND instanceId=0)"
         sqlParameters = (jobInfo['localJobId'],)
         result = self.myDatabase.select(sqlCommand,sqlParameters)
         if len(result) == 1:
            row = result[0]
            if row['jobStatus'] == 'D' or row['jobStatus'] == 'Dr':
               jobInfo['jobStatus'] = row['jobStatus']
         else:
            jobInfo['jobStatus'] = 'Dr'
         if jobInfo['jobStatus'] != 'WF':
            self.updateActiveJob(jobInfo)
            commit = True

      if commit:
         self.myDatabase.commit()

      del jobsInfo


   def getActiveJobCount(self):
      nActiveJobCount = 0
      sqlCommand = "SELECT COUNT(*) FROM activeJobs"
      result = self.myDatabase.select(sqlCommand)
      if len(result) == 1:
         row = result[0]
         nActiveJobCount = row[0]

      return(nActiveJobCount)


   def getActiveJobInProgressCount(self):
      nActiveJobInProgressCount = 0
      sqlCommand = "SELECT COUNT(*) FROM activeJobs \
                                   WHERE(jobStatus!='Dr' AND jobStatus!='D')"
      result = self.myDatabase.select(sqlCommand)
      if len(result) == 1:
         row = result[0]
         nActiveJobInProgressCount = row[0]

      return(nActiveJobInProgressCount)


   def addUserActivity(self,
                       siteName,
                       hubUserId,
                       activityScore):
      timeUpdated = time.time()
      sqlCommand = "INSERT INTO userActivityScores (siteName,hubUserId,activityScore,timeUpdated) \
                         VALUES(?,?,?,?)"
      sqlParameters = (siteName,hubUserId,activityScore,timeUpdated)
      self.myDatabase.insert(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def getUserActivity(self,
                       aggregator,
                       hubUserId):
      activityScore = 0.5
      timeUpdated   = -1.

      sqlCommand = "SELECT activityScore,timeUpdated FROM userActivityScores \
                                                    WHERE(aggregator=? AND hubUserId=?)"
      sqlParameters = (aggregator,hubUserId)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      if len(result) == 1:
         row = result[0]
         activityScore = row['activityScore']
         timeUpdated   = row['timeUpdated']

      return(activityScore,timeUpdated)


   def getUsersActivity(self,
                        hubUserIds):
      usersActivity = {}
      if hubUserIds == '*':
         sqlCommand = "SELECT aggregator,hubUserId,activityScore FROM userActivityScores"
         sqlParameters = None
      elif len(hubUserIds) == 1:
         sqlCommand = "SELECT aggregator,hubUserId,activityScore FROM userActivityScores WHERE(hubUserId=?)"
         sqlParameters = (hubUserIds[0],)
      else:
         sqlCommand = "SELECT aggregator,hubUserId,activityScore FROM userActivityScores"
         sqlParameters = None

      result = self.myDatabase.select(sqlCommand,sqlParameters)
      for row in result:
         if hubUserIds == '*' or row['hubUserId'] in hubUserIds:
            if not row['aggregator'] in usersActivity:
               usersActivity[row['aggregator']] = {}
            usersActivity[row['aggregator']][row['hubUserId']] = row['activityScore']

      return(usersActivity)


   def getUsersPriority(self,
                        hubUserIds):
      usersPriority = {}

      usersActivity = self.getUsersActivity('*')
      for aggregator in usersActivity:
         usersPriority[aggregator] = {}
         sumPriorities = 0.
         if hubUserIds == '*':
            for hubUserId in usersActivity[aggregator]:
               submitterClass = self.getUserSubmitterClass(hubUserId)
               priorityBoosters = self.aggregatorsInfo.getAggregatorKeyValue(aggregator,'priorityBoosters')
               if submitterClass in priorityBoosters:
                  priorityBooster = float(priorityBoosters[submitterClass])
               else:
                  priorityBooster = 1.
               usersPriority[aggregator][hubUserId] = priorityBooster/usersActivity[aggregator][hubUserId]
               sumPriorities += usersPriority[aggregator][hubUserId]
         else:
            for hubUserId in hubUserIds:
               submitterClass = self.getUserSubmitterClass(hubUserId)
               priorityBoosters = self.aggregatorsInfo.getAggregatorKeyValue(aggregator,'priorityBoosters')
               try:
                  if submitterClass in priorityBoosters:
                     priorityBooster = float(priorityBoosters[submitterClass])
                  else:
                     priorityBooster = 1.
               except:
                  priorityBooster = 0.5
               if hubUserId in usersActivity[aggregator]:
                  usersPriority[aggregator][hubUserId] = priorityBooster/usersActivity[aggregator][hubUserId]
               else:
                  usersPriority[aggregator][hubUserId] = priorityBooster/0.5
               sumPriorities += usersPriority[aggregator][hubUserId]

         for hubUserId in usersPriority[aggregator]:
            usersPriority[aggregator][hubUserId] /= sumPriorities

      return(usersPriority)


   def updateUserActivity(self,
                          aggregator,
                          hubUserId,
                          activityScore,
                          timeUpdated):
      sqlCommand = "UPDATE userActivityScores SET activityScore=?,timeUpdated=? \
                                            WHERE(aggregator=? AND hubUserId=?)"
      sqlParameters = (activityScore,timeUpdated,aggregator,hubUserId)
      self.myDatabase.update(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def updateUserActivityScores(self):
      now = time.time()
      currentActivityScores = {}

      sqlCommand = "SELECT aggregator,hubUserId,SUM(nCores) AS activity FROM activeJobs \
                                                             WHERE(instanceId != 0 AND jobStatus != 'D' AND jobStatus != 'Dr') \
                                                             GROUP BY aggregator,hubUserId"
      result = self.myDatabase.select(sqlCommand)
      for row in result:
         if not row['aggregator'] in currentActivityScores:
            currentActivityScores[row['aggregator']] = {}
         currentActivityScores[row['aggregator']][row['hubUserId']] = row['activity']

      updatedActivityScores = {}
      sqlCommand = "SELECT * FROM userActivityScores"
      result = self.myDatabase.select(sqlCommand)
      for row in result:
         aggregator    = row['aggregator']
         hubUserId     = row['hubUserId']
         activityScore = row['activityScore']
         timeUpdated   = row['timeUpdated']

         currentActivityScore = 0.
         if aggregator in currentActivityScores:
            if hubUserId in currentActivityScores[aggregator]:
               currentActivityScore = currentActivityScores[aggregator][hubUserId]

         beta = math.pow(0.5,(now-timeUpdated)/86400.)
         activity = beta*activityScore + (1.-beta)*currentActivityScore
         if not aggregator in updatedActivityScores:
            updatedActivityScores[aggregator] = {}
         updatedActivityScores[aggregator][hubUserId] = max(activity,0.5)

      if len(updatedActivityScores) > 0:
         for aggregator in updatedActivityScores:
            for hubUserId in updatedActivityScores[aggregator]:
               sqlCommand = "UPDATE userActivityScores SET activityScore=?,timeUpdated=? \
                                                     WHERE(aggregator=? AND hubUserId=?)"
               sqlParameters = (updatedActivityScores[aggregator][hubUserId],now,aggregator,hubUserId)
               self.myDatabase.update(sqlCommand,sqlParameters)
         self.myDatabase.commit()

      updatedActivityScores = {}
      for aggregator in currentActivityScores:
         for hubUserId in currentActivityScores[aggregator]:
            activityScore,timeUpdated = self.getUserActivity(aggregator,hubUserId)
            if timeUpdated < 0.:
               if not aggregator in updatedActivityScores:
                  updatedActivityScores[aggregator] = {}
               updatedActivityScores[aggregator][hubUserId] = currentActivityScores[aggregator][hubUserId]

      if len(updatedActivityScores) > 0:
         for aggregator in updatedActivityScores:
            for hubUserId in updatedActivityScores[aggregator]:
               sqlCommand = "INSERT INTO userActivityScores (aggregator,hubUserId,activityScore,timeUpdated) \
                                  VALUES(?,?,?,?)"
               sqlParameters = (aggregator,hubUserId,updatedActivityScores[aggregator][hubUserId],now)
               self.myDatabase.insert(sqlCommand,sqlParameters)
         self.myDatabase.commit()


   def updateRegisteredJobStates(self,
                                 jobsInfo,
                                 jobRegistrationState):
      for jobInfo in jobsInfo:
         sqlCommand = "UPDATE registeredJobs SET state=? \
                                           WHERE(localJobId=? AND instanceId=?)"
         sqlParameters = (jobRegistrationState,jobInfo['localJobId'],jobInfo['instanceId'])
         self.myDatabase.update(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def releaseRegisteredJobs(self):
      nReleasedJobs   = 0
      releaseJobsInfo = []
# get list of users with held registered jobs
      usersWithRegisteredJob = self.getUsersWithRegisteredJob()
# get user priority for each aggregator
      usersPriority = self.getUsersPriority(usersWithRegisteredJob)
# get the number of active jobs for each aggregator
      aggregatorsActiveJobCount = self.getAggregatorsActiveJobCount()
# get users with held jobs at each aggregator
      aggregatorUserRegisteredJobs = self.getAggregatorUserRegisteredJob()
      for aggregator in aggregatorUserRegisteredJobs:
         if not aggregator in usersPriority:
            usersPriority[aggregator] = {}
            nAggregatorUserRegisteredJobs = len(aggregatorUserRegisteredJobs)
            for hubUserId in aggregatorUserRegisteredJobs[aggregator]:
               submitterClass = self.getUserSubmitterClass(hubUserId)
               priorityBoosters = self.aggregatorsInfo.getAggregatorKeyValue(aggregator,'priorityBoosters')
               try:
                  if submitterClass in priorityBoosters:
                     priorityBooster = float(priorityBoosters[submitterClass])
                  else:
                     priorityBooster = 1.
               except:
                  priorityBooster = 1.
               usersPriority[aggregator][hubUserId] = priorityBooster/nAggregatorUserRegisteredJobs

      for aggregator in aggregatorUserRegisteredJobs:
         maximumAggregatorActiveJobs = self.aggregatorsInfo.getAggregatorKeyValue(aggregator,'maximumActiveJobs')
         nAggregatorJobs = self.getRegisteredJobCountAtAggregator(aggregator)
         for hubUserId in aggregatorUserRegisteredJobs[aggregator]:
            try:
               priority = usersPriority[aggregator][hubUserId]
            except KeyError:
               self.logger.log(logging.DEBUG,getLogMessage("releaseRegisteredJobs: priority not found for aggregator %s, userId %d" % (aggregator,hubUserId)))
            else:
               # get job info for registered held jobs at aggregator belonging to user
               jobsInfo = self.getAggregatorRegisteredJobsForUser(aggregator,hubUserId)
               nJobsToRelease = max(1,min(len(jobsInfo),int(round(float(nAggregatorJobs)*priority))))
               self.logger.log(logging.DEBUG,getLogMessage("nJobsToRelease: %d" % (nJobsToRelease)))
               for jobInfo in jobsInfo[:nJobsToRelease]:
                  if not jobInfo['aggregator'] in aggregatorsActiveJobCount:
                     aggregatorsActiveJobCount[jobInfo['aggregator']] = 0
                  if aggregatorsActiveJobCount[jobInfo['aggregator']] < maximumAggregatorActiveJobs:
                     releaseJobsInfo.append(jobInfo)
                     aggregatorsActiveJobCount[jobInfo['aggregator']] += 1

      if releaseJobsInfo:
         nReleasedJobs = len(releaseJobsInfo)
         self.updateRegisteredJobStates(releaseJobsInfo,self.JOBREGISTRATIONSTATERELEASED)

      del releaseJobsInfo

      return(nReleasedJobs)


   def isRegisteredJobReleased(self,
                               localJobId,
                               instanceId):
      registeredJobReleased = False
      sqlCommand = "SELECT state FROM registeredJobs WHERE(localJobId=? AND instanceId=?)"
      sqlParameters = (localJobId,instanceId)
      result = self.myDatabase.select(sqlCommand,sqlParameters)
      if len(result) == 1:
         row = result[0]
         if row['state'] == self.JOBREGISTRATIONSTATERELEASED:
            registeredJobReleased = True

      return(registeredJobReleased)


   def addUserSubmitterClass(self,
                             hubUserId,
                             submitterClass):
      sqlCommand = "INSERT INTO userSubmitterClasses (hubUserId,submitterClass) \
                         VALUES(?,?)"
      sqlParameters = (hubUserId,submitterClass)
      self.myDatabase.insert(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def updateUserSubmitterClass(self,
                                hubUserId,
                                submitterClass):
      sqlCommand = "UPDATE userSubmitterClasses SET submitterClass=? WHERE(hubUserId=?)"
      sqlParameters = (submitterClass,hubUserId)
      self.myDatabase.update(sqlCommand,sqlParameters)
      self.myDatabase.commit()


   def getUserSubmitterClass(self,
                             hubUserId):
      userSubmitterClass = -1
      sqlCommand = "SELECT submitterClass FROM userSubmitterClasses WHERE(hubUserId=?)"
      sqlParameters = (hubUserId,)
      result = self.myDatabase.select(sqlCommand,sqlParameters)

      if len(result) == 1:
         row = result[0]
         userSubmitterClass = row['submitterClass']

      return(userSubmitterClass)


   def isJobSitePending(self,
                        jobSite):
      return(jobSite in self.pendingJobSites)


   def addPendingJobSite(self,
                         jobSite):
      self.pendingJobSites.append(jobSite)


   def deletePendingJobSite(self,
                            jobSite):
      self.pendingJobSites.remove(jobSite)


   def getPendingJobSitesReport(self):
      pendingJobSites = {}
      if len(self.pendingJobSites) > 0:
         for jobSite in self.pendingJobSites:
            pendingJobSites[jobSite] = 0.

      return(pendingJobSites)


   def addPendingJobPosting(self,
                            siteDesignator,
                            remoteJobId,
                            jobWorkDirectory,
                            localJobId,
                            instanceId,
                            runName,
                            tailFiles):
      if not siteDesignator in self.pendingJobPostings:
         self.pendingJobPostings[siteDesignator] = {}
      self.pendingJobPostings[siteDesignator][remoteJobId] = {'jobWorkDirectory':jobWorkDirectory,
                                                              'localJobId':localJobId,
                                                              'instanceId':instanceId,
                                                              'runName':runName,
                                                              'tailFiles':tailFiles}


   def getPendingJobPostingSites(self):
      pendingJobPostingSites = list(self.pendingJobPostings.keys())

      return(pendingJobPostingSites)


   def postPendingJobPostings(self):
      markedForDeletion = []
      for siteDesignator in self.pendingJobPostings:
         if self.isJobSiteActive(siteDesignator):
            channel = self.getActiveJobSiteChannel(siteDesignator)
            for remoteJobId in self.pendingJobPostings[siteDesignator]:
               jsonObject = {'messageType':'newJobId',
                             'siteDesignator':siteDesignator,
                             'remoteJobId':remoteJobId,
                             'jobWorkDirectory':self.pendingJobPostings[siteDesignator][remoteJobId]['jobWorkDirectory'],
                             'localJobId':self.pendingJobPostings[siteDesignator][remoteJobId]['localJobId'],
                             'instanceId':self.pendingJobPostings[siteDesignator][remoteJobId]['instanceId'],
                             'runName':self.pendingJobPostings[siteDesignator][remoteJobId]['runName'],
                             'tailFiles':self.pendingJobPostings[siteDesignator][remoteJobId]['tailFiles']}
               self.postJsonMessage(channel,jsonObject)
            markedForDeletion.append(siteDesignator)

      for siteDesignator in markedForDeletion:
         del self.pendingJobPostings[siteDesignator]
      del markedForDeletion


   def addRestartJobSite(self,
                         jobSite):
      self.restartJobSites.append(jobSite)


   def getRestartJobSites(self):
      return(copy.copy(self.restartJobSites))


   def deleteRestartJobSite(self,
                            jobSite):
      self.restartJobSites.remove(jobSite)


   def getRestartJobSitesReport(self):
      restartJobSites = {}
      if len(self.restartJobSites) > 0:
         for jobSite in self.restartJobSites:
            restartJobSites[jobSite] = 0.

      return(restartJobSites)


   def signalJobSiteMonitors(self,
                             signalNumber):
      activeJobSites = self.getActiveJobSites()
      sitesToDelete = []
      for siteDesignator in activeJobSites:
         sitePID = self.getActiveJobSitePID(siteDesignator)
         self.logger.log(logging.INFO,getLogMessage("Send signal %d to %d - %s" % (signalNumber,sitePID,siteDesignator)))
         try:
            os.kill(sitePID,signalNumber)
         except OSError:
            sitesToDelete.append(siteDesignator)
         except:
            sitesToDelete.append(siteDesignator)
            self.logger.log(logging.ERROR,getLogMessage(traceback.format_exc()))

      for siteDesignator in sitesToDelete:
         self.deleteActiveJobSite(siteDesignator)


   def countRunningJobSiteMonitors(self):
      nRunningJobSiteMonitors = 0
      signalNumber = 0
      activeJobSites = self.getActiveJobSites()
      for siteDesignator in activeJobSites:
         sitePID = self.getActiveJobSitePID(siteDesignator)
         self.logger.log(logging.INFO,getLogMessage("Send signal %d to %d - %s" % (signalNumber,sitePID,siteDesignator)))
         try:
            os.kill(sitePID,signalNumber)
         except:
            pass
         else:
            nRunningJobSiteMonitors += 1

      return(nRunningJobSiteMonitors)


   def processRequests(self):
      for channel in self.activeChannels:
         message = self.pullMessage(channel,0)
         while message:
            args = message.split()
            if args[0] == 'json':
               jsonMessageLength = int(args[1])
               jsonMessage = self.pullMessage(channel,jsonMessageLength)
               if len(jsonMessage) > 0:
                  try:
                     jsonObject = json.loads(jsonMessage)
                  except ValueError:
                     self.logger.log(logging.ERROR,getLogMessage("JSON object %s could not be decoded" % (jsonMessage)))
                  else:
                     if jsonObject['messageType'] != 'pipeFlusher':
                        self.logger.log(logging.DEBUG,getLogMessage("request = %s, message length = %d" % \
                                                             (jsonObject['messageType'],len(jsonMessage))))
                        requestStart = time.time()
                     if   jsonObject['messageType'] == 'startJobSite':
                        self.logger.log(logging.DEBUG,getLogMessage(jsonMessage))
                        siteDesignator = jsonObject['siteDesignator']
                        if self.isJobSitePending(siteDesignator):
                           self.addActiveJobSite(siteDesignator,channel,jsonObject['pid'])
                           self.deletePendingJobSite(siteDesignator)
                        else:
                           self.logger.log(logging.ERROR,getLogMessage("Activated job site %s not pending" % (siteDesignator)))
                     elif jsonObject['messageType'] == 'siteUpdate':
                        siteDesignator = jsonObject['siteDesignator']
                        self.logger.log(logging.INFO,getLogMessage("%d/%d job status messages received from %s" % \
                                                                   (len(jsonObject['jobStates']), \
                                                                   jsonObject['nJobStates'],siteDesignator)))
                        for jobState in jsonObject['jobStates']:
                           try:
                              remoteJobId   = jobState.get('jobId')
                              jobStatus     = jobState.get('status')
                              jobStage      = jobState.get('stage')
                              jobQueue      = jobState.get('queue','?')
                              executionHost = jobState.get('executionHost','?')
                              tailFiles     = jobState.get('tailFiles',{})
#                             if tailFiles:
#                                self.logger.log(logging.DEBUG,getLogMessage("%s" % (str(tailFiles))))
                              jobInfo = JobInfo(siteDesignator=siteDesignator,
                                                remoteJobId=remoteJobId,
                                                jobStatus=jobStatus,
                                                jobStage=jobStage,
                                                jobQueue=jobQueue,
                                                executionHost=executionHost)
                              self.updateActiveJob(jobInfo)
                              for tailFile in tailFiles:
                                 tailText = tailFiles[tailFile]
                                 if tailText != '?':
                                    self.updateFileTail(siteDesignator,
                                                        remoteJobId,
                                                        tailFile,
                                                        tailText)

                              if jobStatus == 'D':
                                 distributorPid = 0
                                 self.logger.log(logging.DEBUG,getLogMessage("Getting distributorPid for %s:%s" % \
                                                                                     (siteDesignator,remoteJobId)))
                                 sqlCommand = "SELECT distributorPid \
                                                 FROM activeJobs WHERE(siteDesignator=? AND remoteJobId=?)"
                                 sqlParameters = (siteDesignator,remoteJobId)
                                 try:
                                    result = self.myDatabase.select(sqlCommand,sqlParameters)
                                 except:
                                    pass
                                 else:
                                    if len(result) == 1:
                                       row = result[0]
                                       distributorPid = row['distributorPid']

                                 self.logger.log(logging.DEBUG,getLogMessage("distributorPid = %d" % (distributorPid)))
                                 if distributorPid > 0:
                                    killCommand = ['sudo','/usr/bin/kill','-s','USR2',str(distributorPid)]
                                    child = subprocess.Popen(killCommand,
                                                             stdout=subprocess.PIPE,
                                                             stderr=subprocess.PIPE,
                                                             close_fds=True)
                                    killStdOutput,killStdError = child.communicate()
                                    killExitStatus = child.returncode
                                    if killExitStatus != 0:
                                       self.logger.log(logging.ERROR,getLogMessage("Unable to notify distributor %d" % \
                                                                                                      (distributorPid)))
                                       if killStdOutput:
                                          self.logger.log(logging.ERROR,getLogMessage(killStdOutput.decode('utf-8')))
                                       if killStdError:
                                          self.logger.log(logging.ERROR,getLogMessage(killStdError.decode('utf-8')))
                           except:
#                             pass
                              self.logger.log(logging.ERROR,getLogMessage(traceback.format_exc()))
                        self.updateWFJobs(siteDesignator)
                        self.dumpActiveJobs()
                        self.updateActiveJobSite(siteDesignator)
                     elif jsonObject['messageType'] == 'stopJobSite':
                        self.logger.log(logging.DEBUG,getLogMessage(jsonMessage))
                        siteDesignator = jsonObject['siteDesignator']
                        self.deleteActiveJobSite(siteDesignator)
                        exitStatus = jsonObject['exitStatus']
                        if exitStatus == 255:
                           self.addRestartJobSite(siteDesignator)
                     elif jsonObject['messageType'] == 'registerJob':
                        instanceToken  = jsonObject['instanceToken']
                        localJobId     = int(jsonObject['localJobId'])
                        instanceId     = int(jsonObject['instanceId'])
                        distributorPid = jsonObject['distributorPid']
                        hubUserId      = jsonObject['hubUserId']
                        siteName       = jsonObject['siteName']
                        runName        = jsonObject['runName']
                        submitterClass = jsonObject['submitterClass']
                        aggregator     = self.aggregatorsInfo.getSiteAggregator(siteName)

                        oldSubmitterClass = self.getUserSubmitterClass(hubUserId)
                        if   oldSubmitterClass < 0:
                           self.addUserSubmitterClass(hubUserId,submitterClass)
                        elif oldSubmitterClass != submitterClass:
                           self.updateUserSubmitterClass(hubUserId,submitterClass)

                        jobInfo = JobInfo(instanceToken=instanceToken,
                                          aggregator=aggregator,
                                          siteName=siteName,
                                          runName=runName,
                                          distributorPid=distributorPid,
                                          localJobId=localJobId,
                                          instanceId=instanceId,
                                          hubUserId=hubUserId)
                        self.addRegisteredJob(jobInfo)
                        del jobInfo

                        returnMessage = {'messageType':'jobInfo',
                                         'aggregator':aggregator,
                                         'siteName':siteName,
                                         'runName':runName,
                                         'localJobId':localJobId,
                                         'instanceId':instanceId,
                                         'jobRegistrationState':self.JOBREGISTRATIONSTATEHELD}
                        self.postJsonMessage(channel,returnMessage)
                     elif jsonObject['messageType'] == 'deleteRegisteredJob':
                        localJobId = int(jsonObject['localJobId'])
                        instanceId = int(jsonObject['instanceId'])
                        nDeletedRegisteredJobs = self.deleteRegisteredJob(localJobId,instanceId)

                        returnMessage = {'messageType':'deletedRegisteredJobs',
                                         'localJobId':localJobId,
                                         'instanceId':instanceId,
                                         'nDeleted':nDeletedRegisteredJobs}
                        self.postJsonMessage(channel,returnMessage)
                     elif jsonObject['messageType'] == 'isRegisteredJobReleased':
                        localJobId = int(jsonObject['localJobId'])
                        instanceId = int(jsonObject['instanceId'])
                        registeredJobReleased = self.isRegisteredJobReleased(localJobId,instanceId)

                        returnMessage = {'messageType':'jobReleased',
                                         'localJobId':localJobId,
                                         'instanceId':instanceId,
                                         'jobReleased':registeredJobReleased}
                        self.postJsonMessage(channel,returnMessage)
                     elif jsonObject['messageType'] == 'postJob':
                        instanceToken    = jsonObject['instanceToken']
                        siteName         = jsonObject['siteName']
                        siteDesignator   = jsonObject['siteDesignator']
                        identityNames    = jsonObject['identityNames']
                        remoteJobId      = jsonObject['remoteJobId']
                        hubUserId        = jsonObject['hubUserId']
                        jobWorkDirectory = jsonObject['jobWorkDirectory']
                        localJobId       = int(jsonObject['localJobId'])
                        instanceId       = int(jsonObject['instanceId'])
                        destination      = jsonObject['destination']
                        runName          = jsonObject['runName']
                        nCores           = jsonObject['nCores']
                        distributorPid   = jsonObject['distributorPid']
                        tailFiles        = jsonObject['tailFiles']
                        aggregator       = self.aggregatorsInfo.getSiteAggregator(siteName)

                        jobInfo = self.getActiveJob(siteDesignator,remoteJobId)
                        if jobInfo:
                           jobStatus = jobInfo['jobStatus']
                           jobStage  = jobInfo['jobStage']
                        else:
                           jobStatus,jobStage,jobQueue = ('N','Job','?')
                           jobInfo = JobInfo(instanceToken=instanceToken,
                                             aggregator=aggregator,
                                             siteName=siteName,
                                             identityNames=identityNames,
                                             siteDesignator=siteDesignator,
                                             remoteJobId=remoteJobId,
                                             distributorPid=distributorPid,
                                             runName=runName,
                                             nCores=nCores,
                                             localJobId=localJobId,
                                             instanceId=instanceId,
                                             jobStatus=jobStatus,
                                             jobStage=jobStage,
                                             jobQueue=jobQueue,
                                             hubUserId=hubUserId,
                                             destination=destination)
                           self.addActiveJob(jobInfo)
                           self.deleteRegisteredJob(localJobId,instanceId)
                           del jobInfo
                           for tailFile in tailFiles:
                              nLines = tailFiles[tailFile]['nLines']
                              self.addFileTail(siteDesignator,
                                               remoteJobId,
                                               tailFile,
                                               nLines,
                                               "")
                        self.dumpActiveJobs()

                        returnMessage = {'messageType':'jobInfo',
                                         'siteDesignator':siteDesignator,
                                         'remoteJobId':remoteJobId,
                                         'status':jobStatus,
                                         'stage':jobStage}
                        self.postJsonMessage(channel,returnMessage)
                        self.addPendingJobPosting(siteDesignator,remoteJobId,jobWorkDirectory,
                                                  jsonObject['localJobId'],jsonObject['instanceId'],runName,tailFiles)
                     elif jsonObject['messageType'] == 'queryJob':
                        siteDesignator = jsonObject['siteDesignator']
                        remoteJobId    = jsonObject['remoteJobId']
                        markReportedAsDone = False
                        jobInfo = self.getActiveJob(siteDesignator,remoteJobId)
                        if jobInfo:
                           jobStatus = jobInfo['jobStatus']
                           jobStage  = jobInfo['jobStage']
                           jobSite   = jobInfo['executionHost']
                           if   jobStatus == 'D':
                              markReportedAsDone = True
                           elif jobStatus == 'Dr':
                              jobStatus = 'D'
                           tailFiles = self.getFileTails(siteDesignator,remoteJobId)
                           self.clearFileTails(siteDesignator,remoteJobId)
                        else:
                           jobStatus,jobStage,jobSite,jobQueue = ('?','?','?','?')
                           tailFiles = {}

                        returnMessage = {'messageType':'jobInfo',
                                         'siteDesignator':siteDesignator,
                                         'remoteJobId':remoteJobId,
                                         'status':jobStatus,
                                         'stage':jobStage,
                                         'site':jobSite,
                                         'tailFiles':tailFiles}
                        self.postJsonMessage(channel,returnMessage)

                        if markReportedAsDone:
                           jobInfo['jobStatus'] = 'Dr'
                           self.updateActiveJob(jobInfo)
                     elif jsonObject['messageType'] == 'terminateJob':
                        siteDesignator = jsonObject['siteDesignator']
                        remoteJobId    = jsonObject['remoteJobId']
                        jobInfo = self.getActiveJob(siteDesignator,remoteJobId)
                        if jobInfo:
                           jobStatus = jobInfo['jobStatus']
                           jobStage  = jobInfo['jobStage']
                        else:
                           jobStatus,jobStage,jobQueue = ('?','?','?')

                        returnMessage = {'messageType':'jobInfo',
                                         'siteDesignator':siteDesignator,
                                         'remoteJobId':remoteJobId,
                                         'status':jobStatus,
                                         'stage':jobStage}
                        self.postJsonMessage(channel,returnMessage)

                        if jobInfo:
                           jobInfo['jobStatus'] = 'D'
                           jobInfo['jobStage']  = 'Job'
                           self.updateActiveJob(jobInfo)
                     elif jsonObject['messageType'] == 'queryJobs':
                        jobsStatus = {}
                        jobsInfo = []
                        jobs = jsonObject['jobs']
                        for job in jobs:
                           siteDesignator = job['siteDesignator']
                           remoteJobId    = job['remoteJobId']
                           jobInfo = self.getActiveJob(siteDesignator,remoteJobId)
                           if not jobInfo:
                              jobStatus,jobStage,jobSite,jobQueue = ('?','?','?','?')
                              jobInfo = JobInfo(siteDesignator=siteDesignator,
                                                remoteJobId=remoteJobId,
                                                jobStatus=jobStatus,
                                                jobStage=jobStage,
                                                executionHost=jobSite,
                                                jobQueue=jobQueue)
                           jobsInfo.append(jobInfo)
                           if not siteDesignator in jobsStatus:
                              jobsStatus[siteDesignator] = {}

                        for jobInfo in jobsInfo:
                           siteDesignator = jobInfo['siteDesignator']
                           remoteJobId    = jobInfo['remoteJobId']
                           jobStatus      = jobInfo['jobStatus']
                           jobStage       = jobInfo['jobStage']
                           jobSite        = jobInfo['executionHost']
                           jobQueue       = jobInfo['jobQueue']

                           markReportedAsDone = False
                           if   jobStatus == 'D':
                              markReportedAsDone = True
                           elif jobStatus == 'Dr':
                              jobStatus = 'D'
                           if jobStatus != '?':
                              tailFiles = self.getFileTails(siteDesignator,remoteJobId)
                              self.clearFileTails(siteDesignator,remoteJobId)
                           else:
                              tailFiles = {}

                           jobsStatus[siteDesignator][remoteJobId] = {'status':jobStatus,
                                                                      'stage':jobStage,
                                                                      'site':jobSite,
                                                                      'tailFiles':tailFiles}
                           if markReportedAsDone:
                              jobInfo['jobStatus'] = 'Dr'
                              self.updateActiveJob(jobInfo)

                        returnMessage = {'messageType':'jobsStatus',
                                         'jobsStatus':jobsStatus}
                        self.postJsonMessage(channel,returnMessage)
                     elif jsonObject['messageType'] == 'postWorkflow':
                        instanceToken             = jsonObject['instanceToken']
                        siteName                  = jsonObject['siteName']
                        siteDesignator            = jsonObject['siteDesignator']
                        identityNames             = jsonObject['identityNames']
                        remoteJobId               = jsonObject['remoteJobId']
                        hubUserId                 = jsonObject['hubUserId']
                        localJobId                = int(jsonObject['localJobId'])
                        nInstances                = jsonObject['nInstances']
                        instancesInitialJobStatus = jsonObject['instancesInitialJobStatus']
                        destination               = jsonObject['destination']
                        runName                   = jsonObject['runName']
                        nCores                    = jsonObject['nCores']
                        distributorPid            = jsonObject['distributorPid']
                        tailFiles                 = []
                        instanceId                = 0
                        aggregator                = self.aggregatorsInfo.getSiteAggregator(siteName)

                        jobInfo = self.getActiveJob(siteDesignator,remoteJobId)
                        if jobInfo:
                           jobStatus = jobInfo['jobStatus']
                           jobStage  = jobInfo['jobStage']
                        else:
                           jobStatus,jobStage,jobQueue = ('N','DAG','?')
                           jobInfo = JobInfo(instanceToken=instanceToken,
                                             aggregator=aggregator,
                                             siteName=siteName,
                                             identityNames=identityNames,
                                             siteDesignator=siteDesignator,
                                             remoteJobId=remoteJobId,
                                             distributorPid=distributorPid,
                                             runName=runName,
                                             nCores=1,
                                             localJobId=localJobId,
                                             instanceId=instanceId,
                                             jobStatus=jobStatus,
                                             jobStage=jobStage,
                                             jobQueue=jobQueue,
                                             hubUserId=hubUserId,
                                             destination=destination)
                           self.addActiveJob(jobInfo)
                           self.deleteRegisteredJob(localJobId,instanceId)
                           del jobInfo
                        self.dumpActiveJobs()

                        returnMessage = {'messageType':'jobInfo',
                                         'siteDesignator':siteDesignator,
                                         'remoteJobId':remoteJobId,
                                         'status':jobStatus,
                                         'stage':jobStage}
                        self.postJsonMessage(channel,returnMessage)
                        jobWorkDirectory = ""
                        self.addPendingJobPosting(siteDesignator,remoteJobId,jobWorkDirectory,
                                                  jsonObject['localJobId'],'00',runName,tailFiles)

                        if '.' in remoteJobId:
                           wfRemoteJobIdBase = remoteJobId.split('.')[0]
                        else:
                           wfRemoteJobIdBase = remoteJobId
                        nInstanceIdDigits = max(2,int(math.log10(nInstances)+1))
                        wfJobStatus,wfJobStage,wfJobQueue = ('WF','Simulation','?')
                        for instance in iterRange(1,nInstances+1):
                           if instancesInitialJobStatus:
                              wfJobStatus = instancesInitialJobStatus.pop(0)
                           wfRemoteJobId = wfRemoteJobIdBase + '.' + str(instance)
                           jobInfo = self.getActiveJob(siteDesignator,wfRemoteJobId)
                           if not jobInfo:
                              jobInfo = JobInfo(aggregator=aggregator,
                                                siteName=siteName,
                                                identityNames=identityNames,
                                                siteDesignator=siteDesignator,
                                                remoteJobId=wfRemoteJobId,
                                                distributorPid=distributorPid,
                                                runName=runName,
                                                nCores=nCores,
                                                localJobId=localJobId,
                                                instanceId=instance,
                                                jobStatus=wfJobStatus,
                                                jobStage=wfJobStage,
                                                jobQueue=wfJobQueue,
                                                hubUserId=hubUserId,
                                                destination=destination)
                              self.addActiveJob(jobInfo)
                              self.deleteRegisteredJob(localJobId,instance)
                              del jobInfo
                        self.dumpActiveJobs()
                     elif jsonObject['messageType'] == 'queryWorkflow':
                        siteDesignator = jsonObject['siteDesignator']
                        remoteJobId    = jsonObject['remoteJobId']
                        nInstances     = jsonObject['nInstances']
                        markReportedAsDone = False
                        jobInfo = self.getActiveJob(siteDesignator,remoteJobId)
                        if jobInfo:
                           jobStatus = jobInfo['jobStatus']
                           jobStage  = jobInfo['jobStage']
                           jobSite   = jobInfo['executionHost']
                           jobQueue  = jobInfo['jobQueue']
                           if   jobStatus == 'D':
                              markReportedAsDone = True
                           elif jobStatus == 'Dr':
                              jobStatus = 'D'
                        else:
                           jobStatus,jobStage,jobSite,jobQueue = ('?','?','?','?')
                        del jobInfo
                        wfInstances = {}
                        if '.' in remoteJobId:
                           wfRemoteJobIdBase = remoteJobId.split('.')[0]
                        else:
                           wfRemoteJobIdBase = remoteJobId
                        for instance in iterRange(1,nInstances+1):
                           wfMarkReportedAsDone = False
                           wfRemoteJobId = wfRemoteJobIdBase + '.' + str(instance)
                           jobInfo = self.getActiveJob(siteDesignator,wfRemoteJobId)
                           if jobInfo:
                              wfJobStatus = jobInfo['jobStatus']
                              wfJobStage  = jobInfo['jobStage']
                              wfJobSite   = jobInfo['executionHost']
                              wfJobQueue  = jobInfo['jobQueue']
                              if   wfJobStatus == 'D':
                                 wfMarkReportedAsDone = True
                              elif wfJobStatus == 'Dr':
                                 wfJobStatus = 'D'
                           else:
                              wfJobStatus,wfJobStage,wfJobSite,wfJobQueue = ('?','?','?','?')
                           wfInstances[instance] = {}
                           wfInstances[instance]['remoteJobId']        = wfRemoteJobId
                           wfInstances[instance]['jobStatus']          = wfJobStatus
                           wfInstances[instance]['jobStage']           = wfJobStage
                           wfInstances[instance]['jobSite']            = wfJobSite
                           wfInstances[instance]['jobQueue']           = wfJobQueue
                           wfInstances[instance]['markReportedAsDone'] = wfMarkReportedAsDone
                           del jobInfo

                        instanceStates = {}
                        for instance in iterRange(1,nInstances+1):
                           if wfInstances[instance]['jobStatus'] != '?':
                              instanceStates[instance] = {'status':wfInstances[instance]['jobStatus'],
                                                          'stage':wfInstances[instance]['jobStage'],
                                                          'site':wfInstances[instance]['jobSite']}

                        returnMessage = {'messageType':'workflowInfo',
                                         'siteDesignator':siteDesignator,
                                         'remoteJobId':remoteJobId,
                                         'status':jobStatus,
                                         'stage':jobStage,
                                         'site':jobSite,
                                         'instances':instanceStates}
                        self.postJsonMessage(channel,returnMessage)

                        if markReportedAsDone:
                           jobStatus = 'Dr'
                           jobInfo = JobInfo(siteDesignator=siteDesignator,
                                             remoteJobId=remoteJobId,
                                             jobStatus=jobStatus,
                                             jobStage=jobStage,
                                             jobQueue=jobQueue)
                           self.updateActiveJob(jobInfo)
                           del jobInfo
                        for instance in iterRange(1,nInstances+1):
                           if wfInstances[instance]['markReportedAsDone']:
                              wfRemoteJobId = wfInstances[instance]['remoteJobId']
                              wfJobStatus   = 'Dr'
                              wfJobStage    = wfInstances[instance]['jobStage']
                              wfJobQueue    = wfInstances[instance]['jobQueue']
                              jobInfo = JobInfo(siteDesignator=siteDesignator,
                                                remoteJobId=wfRemoteJobId,
                                                jobStatus=wfJobStatus,
                                                jobStage=wfJobStage,
                                                jobQueue=wfJobQueue)
                              self.updateActiveJob(jobInfo)
                              del jobInfo
                        del instanceStates
                        del wfInstances
                     elif jsonObject['messageType'] == 'jobStatusUpdate':
                        jobStatusUpdate = jsonObject['jobStatusUpdate']
                        instanceToken   = jobStatusUpdate['instanceToken']
                        self.logger.log(logging.DEBUG,getLogMessage("jobStatusUpdate: %s" % (jobStatusUpdate)))
                        returnMessage = {'messageType':'jobStatusUpdated',
                                         'instanceToken':instanceToken}
                        self.postJsonMessage(channel,returnMessage)
                     elif jsonObject['messageType'] == 'reportJobSites':
                        jobSiteState = jsonObject['jobSiteState']
                        if   jobSiteState == 'active':
                           jobSites = self.getActiveJobSitesReport()
                        elif jobSiteState == 'pending':
                           jobSites = self.getPendingJobSitesReport()
                        elif jobSiteState == 'restart':
                           jobSites = self.getRestartJobSitesReport()
                        else:
                           jobSites = self.getActiveJobSitesReport()

                        report = {'messageType':'jobSitesReport',
                                  'jobSites':jobSites}
                        self.postJsonMessage(channel,report)
                     elif jsonObject['messageType'] == 'deleteJobSite':
                        jobSiteState   = jsonObject['jobSiteState']
                        siteDesignator = jsonObject['siteDesignator']
                        if   jobSiteState == 'active':
                           self.deleteActiveJobSite(siteDesignator)
                        elif jobSiteState == 'pending':
                           self.deletePendingJobSite(siteDesignator)
                        elif jobSiteState == 'restart':
                           self.deleteRestartJobSite(siteDesignator)
                        else:
                           self.deleteActiveJobSite(siteDesignator)

                        report = {'messageType':'deletedJobSite',
                                  'siteDesignator':siteDesignator}
                        self.postJsonMessage(channel,report)
                     elif jsonObject['messageType'] == 'reportSites':
                        siteDesignator = jsonObject['siteDesignator']
                        remoteJobId    = jsonObject['remoteJobId']

                        if siteDesignator == "*":
                           siteDesignators = self.getSitesWithActiveJob()
                        else:
                           siteDesignators = [siteDesignator]

                        report = {'messageType':'siteJobInfo',
                                  'maxLastReportTime':0,
                                  'siteDesignators':{}}
                        maxLastReportTime = 0
                        for siteDesignator in siteDesignators:
                           report['siteDesignators'][siteDesignator] = {}
                           lastReportTime = self.getActiveJobSiteTimeUpdated(siteDesignator)
                           maxLastReportTime = max(maxLastReportTime,lastReportTime)
                           report['siteDesignators'][siteDesignator]['lastReportTime'] = lastReportTime

                           report['siteDesignators'][siteDesignator]['queues'] = {}
                           activeQueues = self.getActiveJobSiteQueues(siteDesignator)
                           for activeQueue in activeQueues:
                              report['siteDesignators'][siteDesignator]['queues'][activeQueue] = {}

                              if remoteJobId == "*":
                                 reportJobs = self.getActiveJobsInQueue(siteDesignator,activeQueue)
                                 for reportJob in reportJobs:
                                    report['siteDesignators'][siteDesignator]['queues'][activeQueue][reportJob['remoteJobId']] = \
                                                                                               {'status':reportJob['jobStatus'], \
                                                                                                'stage':reportJob['jobStage'], \
                                                                                                'site':reportJob['executionHost']}
                              else:
                                 reportJob = self.getActiveJob(siteDesignator,remoteJobId)
                                 if reportJob:
                                    if reportJob['jobQueue'] == activeQueue:
                                       report['siteDesignators'][siteDesignator]['queues'][activeQueue][reportJob['remoteJobId']] = \
                                                                                              {'status':reportJob['jobStatus'], \
                                                                                               'stage':reportJob['jobStage'], \
                                                                                               'site':reportJob['executionHost']}

                        report['maxLastReportTime'] = maxLastReportTime
                        self.postJsonMessage(channel,report)
                        del siteDesignators
                     elif jsonObject['messageType'] == 'listUserActivity':
                        hubUserIds = jsonObject['hubUserIds']
                        report = ""
                        usersActivity = self.getUsersActivity(hubUserIds)

                        report = {'messageType':'userActivityInfo',
                                  'reportTime':time.time(),
                                  'usersActivity':usersActivity}
                        self.postJsonMessage(channel,report)
                        del usersActivity
                     elif jsonObject['messageType'] == 'listUserPriority':
                        hubUserIds = jsonObject['hubUserIds']
                        report = ""
                        usersPriority = self.getUsersPriority(hubUserIds)

                        report = {'messageType':'userPriorityInfo',
                                  'reportTime':time.time(),
                                  'usersPriority':usersPriority}
                        self.postJsonMessage(channel,report)
                        del usersPriority
                     elif jsonObject['messageType'] == 'listUserJobs':
                        hubUserId = jsonObject['hubUserId']
                        reportActiveJobs     = self.getActiveJobsForUser(hubUserId)
                        reportRegisteredJobs = self.getHeldRegisteredJobsForUser(hubUserId)
                        report = {'messageType':'userJobInfo',
                                  'reportTime':time.time(),
                                  'userActiveJobs':reportActiveJobs,
                                  'userRegisteredJobs':reportRegisteredJobs}
                        self.postJsonMessage(channel,report)
                        del reportActiveJobs
                     elif jsonObject['messageType'] == 'getJobPID':
                        hubUserId  = jsonObject['hubUserId']
                        localJobId = jsonObject['localJobId']

                        activeJobPid = self.getActiveJobPidForUser(hubUserId,localJobId)
                        report = {'messageType':'jobPID',
                                  'hubUserId':hubUserId,
                                  'reportTime':time.time(),
                                  'pid':activeJobPid}
                        self.postJsonMessage(channel,report)
                     elif jsonObject['messageType'] == 'queryIdentityUsers':
                        identityName = jsonObject['identityName']
                        usersWithActiveJobWithIdentity = []
                        hubUserIdsWithActiveJobWithIdentity = self.getUsersWithActiveJobWithIdentity(identityName)
                        for hubUserId in hubUserIdsWithActiveJobWithIdentity:
                           try:
                              hubUserName = pwd.getpwuid(hubUserId).pw_name
                           except:
                              self.logger.log(logging.ERROR,getLogMessage("Unable to get info for user '%d'" % (hubUserId)))
                           else:
                              usersWithActiveJobWithIdentity.append(hubUserName)
                        report = {'messageType':'identityUsers',
                                  'identityName':identityName,
                                  'identityActiveJobUsers':usersWithActiveJobWithIdentity}
                        self.postJsonMessage(channel,report)
                     elif jsonObject['messageType'] == 'pipeFlusher':
                        pass
                     else:
                        self.logger.log(logging.ERROR,getLogMessage("Discarded message type: %s" % (jsonObject['messageType'])))
                     if jsonObject['messageType'] != 'pipeFlusher':
                        requestTime = time.time()-requestStart
                        self.logger.log(logging.DEBUG,getLogMessage("request = %s, processing time = %f" % \
                                                                   (jsonObject['messageType'],requestTime)))
               else:
                  self.pushMessage(channel,message + '\n')
                  self.logger.log(logging.DEBUG,getLogMessage("incomplete request"))
                  break
            else:
               self.logger.log(logging.ERROR,getLogMessage("Discarded message: %s" % (message)))

            message = self.pullMessage(channel,0)


