# @package      hubzero-submit-monitors
# @file         CloudMonitor.py
# @copyright    Copyright (c) 2012-2020 The Regents of the University of California.
# @license      http://opensource.org/licenses/MIT MIT
#
# Copyright (c) 2012-2020 The Regents of the University of California.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# HUBzero is a registered trademark of The Regents of the University of California.
#
import os
import signal
import re
import json
import logging

from hubzero.submit.LogMessage       import getLogIDMessage as getLogMessage
from hubzero.submit.BoundConnections import BoundConnections
from hubzero.submit.Cloud            import Cloud

try:
   iterRange = xrange
except NameError as e:
   iterRange = range

class CloudMonitor(BoundConnections):
   def __init__(self,
                listenURI,
                cloudsInfoPath,
                activeCloudDumpPath="monitorCloud.dump"):
      BoundConnections.__init__(self,listenURI,logConnection=False)

      self.logger = logging.getLogger(__name__)

      self.cloudsInfoPath = cloudsInfoPath
      self.cloudsInfo = CloudsInfo(self.cloudsInfoPath)

      self.activeCloudDumpPath = activeCloudDumpPath

      self.activeClouds = {}
      self.terminating  = False

      self.loadActiveClouds()


   def terminate(self):
      if not self.terminating:
         self.signalActiveClouds(signal.SIGTERM)
         self.closeListeningConnection()
         self.dumpActiveClouds()
         self.terminating = True


   def __createInstance(self,
                        cloudDesignator):
      instanceId = ""

      if cloudDesignator in self.activeClouds:
         cloud = self.activeClouds[cloudDesignator]
         instanceId = cloud.createInstance()

      return(instanceId)


   def __getInstanceHostname(self,
                             cloudDesignator,
                             instanceId):
      hostName = ""

      if cloudDesignator in self.activeClouds:
         cloud = self.activeClouds[cloudDesignator]
         hostName = cloud.getInstanceHostname(instanceId)

      return(hostName)


   def __reportAllInstances(self,
                            cloudDesignator):
      report = ""

      if cloudDesignator in self.activeClouds:
         cloud = self.activeClouds[cloudDesignator]
         report = cloud.reportAllInstances()

      return(report)


   def __incrementInstance(self,
                           cloudDesignator,
                           instanceId,
                           jobId):
      instanceIncremented = False
      if cloudDesignator in self.activeClouds:
         cloud = self.activeClouds[cloudDesignator]
         instanceIncremented = cloud.incrementInstance(instanceId,jobId)

      return(instanceIncremented)


   def __decrementInstance(self,
                           cloudDesignator,
                           instanceId,
                           jobId):
      instanceDecremented = False
      if cloudDesignator in self.activeClouds:
         cloud = self.activeClouds[cloudDesignator]
         instanceDecremented = cloud.decrementInstance(instanceId,jobId)

      return(instanceDecremented)


   def __terminateInstance(self,
                           cloudDesignator,
                           instanceId):
      instanceTerminated = False
      if cloudDesignator in self.activeClouds:
         cloud = self.activeClouds[cloudDesignator]
         instanceTerminated = cloud.terminateInstance(instanceId)

      return(instanceTerminated)


   def loadActiveClouds(self):
      if os.path.exists(self.activeCloudDumpPath):
         try:
            fpDump = open(self.activeCloudDumpPath,'r')
            try:
               line = fpDump.readline()
               nClouds = int(line)
               for cloudId in iterRange(nClouds):
                  line = fpDump.readline()
                  cloudDesignator = line.strip()
                  self.activeClouds[cloudDesignator] = Cloud(self.cloudsInfo,cloudDesignator)
                  self.activeClouds[cloudDesignator].loadActiveInstances(fpDump)
            except (IOError,OSError):
               pass
            finally:
               fpDump.close()
         except (IOError,OSError):
            pass


   def dumpActiveClouds(self):
      try:
         fpDump = open(self.activeCloudDumpPath,'w')
         try:
            nClouds = len(self.activeClouds)
            fpDump.write("%d\n" % (nClouds))
            for cloudDesignator in self.activeClouds:
               fpDump.write("%s\n" % (cloudDesignator))
               cloud = self.activeClouds[cloudDesignator]
               cloud.dumpActiveInstances(fpDump)
         except (IOError,OSError):
            pass
         finally:
            fpDump.close()
      except (IOError,OSError):
         pass


   def reloadCloudsInfo(self):
      del self.cloudsInfo

      self.cloudsInfo = CloudsInfo(self.cloudsInfoPath)
      self.logger.log(logging.INFO,getLogMessage("Cloud Info Reloaded!"))


   def signalActiveClouds(self,
                          signalNumber):
      for cloudDesignator in self.activeClouds:
         self.logger.log(logging.INFO,getLogMessage("terminating cloud %s" % (cloudDesignator)))
         self.activeClouds[cloudDesignator].terminate()

      self.dumpActiveClouds()


   def purgeIdleInstances(self):
      for cloudDesignator in self.activeClouds:
         cloud = self.activeClouds[cloudDesignator]
         cloud.purgeIdleInstances()


   def processRequests(self):
      for channel in self.activeChannels:
         message = self.pullMessage(channel,0)
         while message:
            args = message.split()
            if args[0] == 'json':
               jsonMessageLength = int(args[1])
               jsonMessage = self.pullMessage(channel,jsonMessageLength)
               if len(jsonMessage) > 0:
                  try:
                     jsonObject = json.loads(jsonMessage)
                  except ValueError:
                     self.logger.log(logging.ERROR,getLogMessage("JSON object %s could not be decoded" % (jsonMessage)))
                  else:
                     cloudDesignator = jsonObject['cloudDesignator']
                     if   jsonObject['messageType'] == 'createCloudInstance':
                        if not cloudDesignator in self.activeClouds:
                           self.activeClouds[cloudDesignator] = Cloud(self.cloudsInfo,cloudDesignator)
                        instanceId = self.__createInstance(cloudDesignator)
                        returnMessage = {'messageType':'cloudInfo',
                                         'cloudDesignator':cloudDesignator,
                                         'instanceId':instanceId}
                        self.postJsonMessage(channel,returnMessage)
                     elif jsonObject['messageType'] == 'incrementCloudInstanceUsage':
                        instanceId = jsonObject['instanceId']
                        jobId      = jsonObject['jobId']
                        instanceIncremented = self.__incrementInstance(cloudDesignator,instanceId,jobId)
                        if instanceIncremented:
                           message = "instance %s incremented on cloud %s, jobId = %s" % (instanceId,cloudDesignator,jobId)
                           self.logger.log(logging.INFO,getLogMessage(message))
                        else:
                           message = "incrementing instance %s on cloud %s failed, jobId = %s" % (instanceId,cloudDesignator,jobId)
                           self.logger.log(logging.INFO,getLogMessage(message))
                        returnMessage = {'messageType':'cloudIncrement',
                                         'cloudDesignator':cloudDesignator,
                                         'instanceId':instanceId,
                                         'jobId':jobId,
                                         'success':instanceIncremented}
                        self.postJsonMessage(channel,returnMessage)
                     elif jsonObject['messageType'] == 'decrementCloudInstanceUsage':
                        instanceId = jsonObject['instanceId']
                        jobId      = jsonObject['jobId']
                        instanceDecremented = self.__decrementInstance(cloudDesignator,instanceId,jobId)
                        if instanceDecremented:
                           message = "instance %s decremented on cloud %s, jobId = %s" % (instanceId,cloudDesignator,jobId)
                           self.logger.log(logging.INFO,getLogMessage(message))
                        else:
                           message = "decrementing instance %s on cloud %s failed, jobId = %s" % (instanceId,cloudDesignator,jobId)
                           self.logger.log(logging.ERROR,getLogMessage(message))
                        returnMessage = {'messageType':'cloudDecrement',
                                         'cloudDesignator':cloudDesignator,
                                         'instanceId':instanceId,
                                         'jobId':jobId,
                                         'success':instanceDecremented}
                        self.postJsonMessage(channel,returnMessage)
                     elif jsonObject['messageType'] == 'terminateCloudInstance':
                        instanceId = jsonObject['instanceId']
                        instanceTerminated = self.__terminateInstance(cloudDesignator,instanceId)
                        if instanceTerminated:
                           message = "instance %s terminated on cloud %s" % (instanceId,cloudDesignator)
                           self.logger.log(logging.INFO,getLogMessage(message))
                        else:
                           message = "termination of instance %s on cloud %s failed" % (instanceId,cloudDesignator)
                           self.logger.log(logging.ERROR,getLogMessage(message))
                        returnMessage = {'messageType':'cloudTerminate',
                                         'cloudDesignator':cloudDesignator,
                                         'instanceId':instanceId,
                                         'success':instanceTerminated}
                        self.postJsonMessage(channel,returnMessage)
                     elif jsonObject['messageType'] == 'reportActiveCloudInstances':
                        report = self.__reportAllInstances(cloudDesignator)
                        returnMessage = {'messageType':'cloudReport',
                                         'cloudDesignator':cloudDesignator,
                                         'report':report}
                        self.postJsonMessage(channel,returnMessage)
                     elif jsonObject['messageType'] == 'allocateInstance':
                        wallTime = jsonObject['wallTime']
                        jobId    = jsonObject['jobId']
                        if not cloudDesignator in self.activeClouds:
                           self.activeClouds[cloudDesignator] = Cloud(self.cloudsInfo,cloudDesignator)
                        minimumJobsInstanceId = self.activeClouds[cloudDesignator].getAvailableInstanceWithMinimumJobs(wallTime)
                        if not minimumJobsInstanceId:
                           nextInstanceId = self.activeClouds[cloudDesignator].getNextInstanceId()
                           self.logger.log(logging.INFO,getLogMessage("creating instance(%d) on cloud %s" % \
                                                                       (nextInstanceId,cloudDesignator)))
                           newInstanceId = self.__createInstance(cloudDesignator)
                           if newInstanceId:
                              minimumJobsInstanceId = newInstanceId
                              hostName = self.__getInstanceHostname(cloudDesignator,minimumJobsInstanceId)
                              message = "cloud %s instance(%d) %s created on %s" % \
                                         (cloudDesignator,nextInstanceId,minimumJobsInstanceId,hostName)
                              self.logger.log(logging.INFO,getLogMessage(message))
                           else:
                              minimumJobsInstanceId = ""
                              message = "cloud %s instance(%d) creation failed" % (cloudDesignator,nextInstanceId)
                              self.logger.log(logging.ERROR,getLogMessage(message))
#  lock out other requests
                        if minimumJobsInstanceId:
                           instanceIncremented = self.__incrementInstance(cloudDesignator,minimumJobsInstanceId,jobId)
                           if instanceIncremented:
                              message = "instance %s incremented on cloud %s, jobId = %s" % \
                                         (minimumJobsInstanceId,cloudDesignator,jobId)
                              self.logger.log(logging.INFO,getLogMessage(message))
                           else:
                              message = "incrementing instance %s on cloud %s failed, jobId = %s" % \
                                         (minimumJobsInstanceId,cloudDesignator,jobId)
                              self.logger.log(logging.ERROR,getLogMessage(message))
                        returnMessage = {'messageType':'cloudAllocate',
                                         'cloudDesignator':cloudDesignator,
                                         'jobId':jobId,
                                         'instanceId':minimumJobsInstanceId}
                        self.postJsonMessage(channel,returnMessage)
                     elif jsonObject['messageType'] == 'getInstanceHostname':
                        instanceId = jsonObject['instanceId']
                        hostName = self.__getInstanceHostname(cloudDesignator,instanceId)
                        returnMessage = {'messageType':'cloudHost',
                                         'cloudDesignator':cloudDesignator,
                                         'instanceId':instanceId,
                                         'hostname':hostName}
                        self.postJsonMessage(channel,returnMessage)
                     else:
                        self.logger.log(logging.ERROR,getLogMessage("Discarded message type: %s" % (jsonObject['messageType'])))
               else:
                  self.pushMessage(channel,message + '\n')
                  break
            else:
               self.logger.log(logging.ERROR,getLogMessage("Discarded message: %s" % (message)))

            message = self.pullMessage(channel,0)


