#!/usr/bin/env python
# analyze_queues
# focuses on statuses (less on client stats)

from __future__ import print_function
import os
import sys
import getopt
import re 
import datetime
import urllib
 
STARTUP_DIR=sys.path[0]
sys.path.append(os.path.join(STARTUP_DIR,"../../.."))

from glideinwms.factory.tools.lib import analyze
from glideinwms.lib import xmlParse


def list_print(frontend,zero_supp,entry_data,sorting,attr_list,sort_attribute,div):

   to_be_printed = []
   sum2 = 0

   for entry_name, entry in entry_data.iteritems():
      entry['RunDiff'] = entry['StatusRunning']-entry['ClientGlideTotal']
      entry['IdleDiff'] = entry['StatusIdle']-entry['ReqIdle']

      # avoid division by zero
      unmatched_percent = 0
      rundiff_percent = 0
      if entry['StatusRunning']!=0:
         rundiff_percent = float(entry['RunDiff'])/float(entry['StatusRunning'])*100  
      if entry['ClientGlideTotal']!=0: 
         unmatched_percent = float(entry['ClientGlideIdle'])/float(entry['ClientGlideTotal'])*100  
      entry['UM'] = unmatched_percent
      entry['RD'] = rundiff_percent

      if zero_supp==1:
         sum = 0
         for a in attr_list:
            sum += entry[a]
         if sum==0:
            continue 
         sum2 += sum

      to_be_printed.append((entry[attr_list[sort_attribute]],
             ("%-40s%7s %7s %7s %7s | %7s %7s %7s %7s | %8s %8s %4d%%"
           % (entry_name.lstrip("entry_"),
             analyze.km(float(entry['StatusRunning'])/div),
             analyze.km(float(entry['StatusHeld'])/div),
             analyze.km(float(entry['StatusIdle'])/div),
             analyze.km(float(entry['StatusIdleOther'])/div),

             analyze.km(float(entry['StatusPending'])/div),
             analyze.km(float(entry['StatusWait'])/div),
             analyze.km(float(entry['StatusStageIn'])/div),
             analyze.km(float(entry['StatusStageOut'])/div),

             analyze.km(float(entry['RunDiff'])/div),
             analyze.km(float(entry['IdleDiff'])/div),
             entry['RD']))))

   columns = ("%-40s%7s %7s %7s %7s | %7s %7s %7s %7s | %8s %8s %5s\n" % (
         frontend, "Run","Held","Idle","Unknwn","Pending","Wait","StgIn","StgOut","RunDiff","IdleDiff","%RD"))


   if sorting == 1:
      if zero_supp==1 and sum2==0:
         return 
      to_be_printed.sort()
      to_be_printed.reverse()
      to_be_printed.insert(0,(0,columns))
      for a in to_be_printed:
         print(a[1])
      print()
   else: 
      if zero_supp==1 and sum2==0:
         return 
      to_be_printed.insert(0,(0,columns))
      for a in to_be_printed:
         print(a[1])
      print()

##########################################################################

def main():

   usage="""
USAGE: 
    -x [#] : interval to do verbose stats (default 24)
    --source [ABSPATH or http addr] : factory base (default current directory)
    -s [attribute]: sort by attribute 
              (-s x to see list of choices) 
    -f [frontend] : filter by a single frontend
         (can omit "frontend_" before name)
    -z : zero suppression (don't show entries with 0s across all attributes)
    -p : show all periods (default off: just show 24 hours)
    -m : frontend mode - emphasize frontend data (don't show entry data)
                default unit: slots
    --ms : frontend mode, using seconds instead of slots
    --mh : frontend mode, using hours instead of slots
    -h : this usage message
"""

   # flags
   x = 24
   dir = os.getcwd()
   sorting = 0
   sort_attribute = 0
   filter_frontend = 0
   frontend_mode = 0
   show_all_periods = 0
   zero_supp = 0
 
   try:
       opts, args = getopt.getopt(sys.argv[1:], 'x:hwf:s:zmp', ['source=','ms','mh'])
   except getopt.GetoptError:
       print("\n Option not recognized or missing parameters.")
       print(" Use -h for usage.\n")
       sys.exit(0)
   for o, a in opts:
       if o == "-x":
           x = a
       elif o == "--source":
           dir = a
       elif o in ("-h", "-help"):
           print(usage)
           return
       elif o == "-s":
           if a=='%UM' or a=='UM': a='%unmatched'
           if a=='%RD' or a=='RD': a='%rundiff'
           sort_attribute = a.lower()
           sorting = 1
       elif o == "-z":
           zero_supp = 1 
       elif o == "-p":
           show_all_periods = 1 
       elif o == "-m":
           frontend_mode = 1 
       elif o == "--ms":
           frontend_mode = 2 
       elif o == "--mh":
           frontend_mode = 3 
       elif o == "-f":
           filter_frontend = a
           if 'frontend_' not in filter_frontend:
               filter_frontend = 'frontend_' + filter_frontend

   attributes = {'StatusRunning':0,'StatusHeld':0,'StatusIdle':0, 'StatusIdleOther':0,
                 'StatusPending':0,'StatusWait':0,'StatusStageIn':0, 'StatusStageOut':0,
                 'ReqIdle':0,'ClientGlideTotal':0,'ClientGlideIdle':0, 'ClientGlideRunning':0}
 
   #sort_attributes and attr_list are linked by index number. 
   sort_attributes = ['running','held','idle','unknown',
                      'pending','wait','stagein','stageout',
                      'rundiff','unmatched','idlediff',
                      '%unmatched','%rundiff']
   attr_list = ['StatusRunning','StatusHeld','StatusIdle','StatusIdleOther',
                          'StatusPending','StatusWait','StatusStageIn',
                          'StatusStageOut','RunDiff','ClientGlideIdle','IdleDiff','UM','RD']

   if sorting !=0:
      if sort_attribute not in sort_attributes:
         print("%s not in list of attributes. Choices are:\n" % sort_attribute)
         for a in sort_attributes:
            print(a)
         print()
         return   
      sort_attribute = sort_attributes.index(sort_attribute)

   data = {}  #sorted data
   rrd_data = {} 
   rrd = "rrd_Status_Attributes.xml" 

   if "http" in dir:
       file_dir = os.path.join(dir, rrd)
   else: 
       file_dir = os.path.join(dir, "monitor", rrd)
  
       if not os.path.exists(file_dir):
           #Try RPM location
           rpmdir="/var/lib/gwms-factory/work-dir"
           rpmfile_dir = os.path.join(rpmdir, "monitor",rrd)
           if not os.path.exists(rpmfile_dir):
               print("\nCannot open", file_dir)
               print("Please set --source to the factory work dir and try again.")
               sys.exit(1)
           if dir != os.getcwd():
               #Directory explicitly set, print warning
               print("\nWARNING: Cannot open", file_dir)
               print("Using RPM default %s instead" % rpmfile_dir)
           dir=rpmdir
           file_dir=rpmfile_dir
 
   try:  
       u = urllib.urlopen(file_dir)        
       rrd_data = xmlParse.xmlfile2dict(u)
   except:
       print("\nCannot open", file_dir,"\n\tor",rrd,"was not found there.\n")
       print("Please set --source to the factory work dir and try again.")
       sys.exit(1)
   u.close()

   # rrd_data[updated,total,entries[entry[total[periods], frontends[periods]]]]
   # rrd_data numbers verified by hand

##############################################################################
#   Rearranges rrd_data from data[entries][frontends][periods]  
#                   into data = [periods][frontends][entries][elements]
#      (periods are integers and in seconds)
###############################################################################


   frontend_list = []

   for entry in rrd_data['entries']:
      for frontend in rrd_data['entries'][entry]['frontends']:
         if frontend not in frontend_list:
            frontend_list.append(frontend)

   if filter_frontend != 0:
      if filter_frontend not in frontend_list:
         print("\nFrontend", filter_frontend, "not found at source.\n")
         print("Choices are:\n ")
         for frontend in frontend_list:
            print(frontend)
         print()
         sys.exit(1)
    
   for entry in rrd_data['entries']:
      for frontend in rrd_data['entries'][entry]['frontends']:
          
         #if filtering, only choose selected frontend 
         if filter_frontend != 0:
            if frontend != filter_frontend:
               continue
           
         for period, elements in rrd_data['entries'][entry]['frontends'][frontend]['periods'].iteritems():

            if int(period) not in data:
               data[int(period)] = {}
            if frontend not in data[int(period)]:
               data[int(period)][frontend] = {}
            if entry not in data[int(period)][frontend]:
               data[int(period)][frontend][entry] = {}
            
            for a in attributes.keys():
               if a not in data[int(period)][frontend][entry]:
                  data[int(period)][frontend][entry][a] = 0 
               try: data[int(period)][frontend][entry][a] += int(float(elements[a])*int(period))
               except: pass 

   # data[period[frontend[entry[element[value]]]]]
   #'data' numbers verified by hand
   #debug_print_dict(data)
  
#####################################################################
# Organize totals/stats for each period, frontend, and entry independantly
######################################################################


   if filter_frontend == 0:
      print("""
Status Attributes analysis for All Entries - %s
""" % datetime.datetime.now().strftime("%d-%m-%Y_%H:%M:%S"))
   else: print("""
Status Attributes analysis for %s - %s
""" % (filter_frontend, datetime.datetime.now().strftime("%d-%m-%Y_%H:%M:%S")))

          
   period_data = {} 
   frontend_data = {} 
   entry_data = {} 
   entry_data_all_frontends = {} 
 
   for period, frontends in data.iteritems():
      period = int(period)
      period_data[period] = {}
      frontend_data[period] = {}
      entry_data[period] = {}
      entry_data_all_frontends[period] = {}
      for a in attributes.keys(): period_data[period][a]=0

      for frontend, entries in frontends.iteritems(): 
         frontend_data[period][frontend]={}
         entry_data[period][frontend] = {}
         for a in attributes.keys(): frontend_data[period][frontend][a]=0

         for entry, elements in entries.iteritems():
            entry_data[period][frontend][entry] = {}
            if entry not in entry_data_all_frontends[period]:
               entry_data_all_frontends[period][entry] = {}
               for a in attributes.keys(): entry_data_all_frontends[period][entry][a]=0 

            for a in attributes.keys(): 
               entry_data[period][frontend][entry][a]=0

            for a in attributes.keys():
               entry_data[period][frontend][entry][a] += elements[a]  
               frontend_data[period][frontend][a] += elements[a] 
               period_data[period][a] += elements[a]
               entry_data_all_frontends[period][entry][a] += elements[a] 


######################################################################
#   Print
######################################################################

   # sort periods from least to greatest, with 24 hours at the top
   period_list = period_data.keys()
   period_list.sort()
   period_list.remove(86400)
   period_list.insert(0,86400)
      
   period = int(x)*3600

    # if filtering by period, make sure it's in the data
   if period not in period_list:
      print("Interval",x,"does not exist in data.\n Choices are:")
      for a in period_list:
         print(a/3600)
      print()
      return

   if show_all_periods==0:
      period_list=[period]

   for period in period_list:
        
     title = ("Past %.1f hours" % (float(period)/3600))
     p = period_data[period]
     status_total = p['StatusRunning'] + p['StatusHeld'] + p['StatusIdle'] + p['StatusIdleOther']+p['StatusPending']+p['StatusWait']+p['StatusStageIn']+p['StatusStageOut']

     print(
"""----------------------------------------
%s:

Status Running:  %s 
Status Held:     %s
Status Idle:     %s 
Status Unknown:  %s

Status Pending:  %s
Status Wait:     %s
Status Stage In: %s
Status Stage Out:%s

RunDiff (Running-ClientRegistered): %s
IdleDiff (Idle - RequestedIdle):    %s
"""
     % (title,
       analyze.printline(period_data[period]['StatusRunning'],status_total,period),
       analyze.printline(period_data[period]['StatusHeld'],status_total,period),
       analyze.printline(period_data[period]['StatusIdle'],status_total,period),
       analyze.printline(period_data[period]['StatusIdleOther'],status_total,period),

       analyze.printline(period_data[period]['StatusPending'],status_total,period),
       analyze.printline(period_data[period]['StatusWait'],status_total,period),
       analyze.printline(period_data[period]['StatusStageIn'],status_total,period),
       analyze.printline(period_data[period]['StatusStageOut'],status_total,period),

       analyze.printline(period_data[period]['StatusRunning']-period_data[period]['ClientGlideTotal'],period_data[period]['StatusRunning'],period),
       analyze.printline(period_data[period]['StatusIdle']-period_data[period]['ReqIdle'],1,period)))


################################################################################
#    Print list-style stats 
################################################################################

   period = int(x)*3600

   if filter_frontend == 0 and frontend_mode == 0:
      print("""
---------------------------------------
---------------------------------------
Per Entry (all frontends) stats for the past %s hours.\n""" % x) 
      list_print("",zero_supp, entry_data_all_frontends[period],sorting, attr_list, sort_attribute,1)

   if frontend_mode == 0:
      print("""
---------------------------------------
---------------------------------------
Per Entry (per frontend) stats for the past %s hours.\n""" % x) 

   if frontend_mode==0:
      for frontend, entries in data[period].iteritems():
         list_print(frontend, zero_supp, entry_data[period][frontend], sorting, attr_list, sort_attribute,1)

   else: # print frontend like entries, and omit entries
      units = ["Slots","Seconds","Hours"]
      divs = [period,1.0,3600.0]
      print("""
---------------------------------------
---------------------------------------
Frontend stats for the past %s hours, units = %s.\n""" % (x,units[frontend_mode-1])) 
      list_print("", zero_supp, frontend_data[period],sorting,attr_list,sort_attribute,divs[frontend_mode-1])


################################################################################
#    Print Key
################################################################################


   print("""-----------------------------------
LEGEND:

K = x   1,000
M = x 100,000

Run : Status Running
Held : Status Held
Idle : Status Idle
Unknwn : Status Unknown (StatusIdleOther)
Pending : Status Pending
Wait : Status Wait
StgIn : Status Stage In
StgOut : Status Stage Out
RunDiff : StatusRunning - ClientRegistered (ClientGlideTotal)
IdleDiff : StatusIdle - ReqIdle (Requested Idle)
%RD - Percent RunDiff over Running (RunDiff/StatusRunning)
-------------------------------------
        \n""")


if __name__ == "__main__":
    main()
