#!/usr/libexec/platform-python

import htcondor2 as htcondor
import classad2 as classad

import sys
import os
import re
import datetime
import optparse

from operator import itemgetter
from time import sleep
from copy import deepcopy

### command-line options ###
parser = optparse.OptionParser(usage=(
    'Usage: %prog [options]\n'
    '       %prog [options] <classad-filename> <classad-filename>\n\n'
    'Only -c, -s, and --attrs options considered when passing ClassAds via files.'))

# daemons
parser.set_defaults(daemon=False)
group = optparse.OptionGroup(parser, title='Daemon Options', description=(
    'If none of these options are used, the local DAEMON_LIST will be searched '
    'in the order: SCHEDD, STARTD, COLLECTOR, NEGOTIATOR, MASTER. '
    'If -n NAME is not specified, the first ClassAd matching the constraint '
    '"Machine == $(FULL_HOSTNAME)" (for Schedd/Startd/Master ads) '
    'or "Machine == $(COLLECTOR_HOST)" (for Collector/Negotiator ads) '
    'will be monitored.'))
group.add_option('--collector', action='store_const', const='Collector',
                     dest='daemon', help=(
                         'Monitor condor_collector ClassAds.'))
group.add_option('--negotiator', action='store_const', const='Negotiator',
                     dest='daemon', help=(
                         'Monitor condor_negotiator ClassAds.'))
group.add_option('--master', action='store_const', const='Master',
                     dest='daemon', help=(
                         'Monitor condor_master ClassAds.'))
group.add_option('--schedd', action='store_const', const='Schedd',
                     dest='daemon', help=(
                         'Monitor condor_schedd ClassAds.'))
group.add_option('--startd', action='store_const', const='Startd',
                     dest='daemon', help=(
                         'Monitor condor_startd ClassAds.'))
parser.add_option_group(group)

# live mode
parser.add_option('-l', '--live', action='store_const', const=True, dest='live',
                      default=False, help=(
    'Display results in a live, top-like mode.'))

# schedd history
parser.add_option('-H', '--history', action='store_true', dest='history',
                      default=False, help=(
    'Use the two most recent condor_schedd daemon ads from $(SPOOL)/schedd_daemon_history '
    'instead of querying the daemon twice.'))

# pool
parser.add_option('-p', '--pool', dest='pool', default=None, help=(
    'Query the specified Central Manager instead of $(COLLECTOR_HOST). '
    'Use of this option implies --collector unless otherwise specified.'))

# name
parser.add_option('-n', '--name', dest='name', default=None, help=(
    'Query using the constraint Name == NAME. If omitted '
    'and multiple ClassAds are returned, only the daemon named '
    'in the first ClassAd returned will be monitored.'))

# delay
parser.add_option('-d', '--delay', type='int', dest='delay',
                      default=htcondor.param['STATISTICS_WINDOW_QUANTUM'],
                      help=(
    'Specifies the delay between ClassAd updates, in integer seconds. '
    'If omitted, the value of the configuration variable '
    'STATISTICS_WINDOW_QUANTUM is used.'))

# display columns
parser.add_option('-c', '--columns', dest='column_set', default='default', help=(
    'Choose the set of columns displayed. Valid column sets are: '
    'default, runtime, count, all.'))

# sort column
parser.add_option('-s', '--sort', dest='column', default='', help=(
    'Sort table by column name. See the man page for a list of column names '
    'and definitions.'))
    
# additional ClassAd attributes
parser.add_option('--attrs', dest='attrs', default='', help=(
    'Comma-delimited list of additional ClassAd attributes to monitor. '
    'Values will be considered as "counts".'))

# set global variables based on command-line options
opts, args = parser.parse_args()
FILES = args
DAEMON = opts.daemon
LIVE = opts.live
HISTORY = opts.history
POOL = opts.pool
NAME = opts.name
DELAY = opts.delay
COL_SET = opts.column_set.lower()
SORT_COL = opts.column
ATTRIBUTES = [attr.strip() for attr in opts.attrs.split(',')]
    
# set default daemon from checking the DAEMON_LIST in specified order
if not DAEMON:
    if POOL is not None:
        DAEMON = 'Collector'
    else:
        for DAEMON in ['Schedd', 'Startd', 'Collector', 'Negotiator', 'Master']:
            if DAEMON.lower() in htcondor.param['DAEMON_LIST'].lower():
                break

# set machine name if name is not set
MACHINE = None
if (NAME is None) and (POOL is None):
    if DAEMON in ['Schedd', 'Startd', 'Master']:
        MACHINE = htcondor.param['FULL_HOSTNAME']
    else:
        MACHINE = htcondor.param['COLLECTOR_HOST']

# determine if we should do direct queries
DIRECT = DAEMON in ['Schedd', 'Startd']

# column names. These are the stable identifiers used internally and
# accepted by the -s/--sort option; the human-readable headers actually
# printed are defined in COL_LABELS below.
COL_NAMES = [
    'TotalRt',  # 0. Total Runtime
    'InstRt',   # 1. Runtime between ads
    'InstAvg',  # 2. Runtime/Count between ads
    'TotAvg',   # 3. Mean Runtime/Count since daemon start
    'TotMax',   # 4. Max Runtime/Count since daemon start
    'TotMin',   # 5. Min Runtime/Count since daemon start
    'RtPctAvg', # 6. InstAvg / TotAvg ("percent of mean runtime")
    'RtPctMax', # 7. InstAvg-TotMin / TotMax-TotMin ("percent of max runtime")
    'RtSigmas', # 8. InstAvg-TotAvg / TotStd ("std devs from mean runtime")
    'TotalCt',  # 9. Total Count
    'InstCt',   # 10. Count between ads
    'InstRate', # 11. Count/time between ads
    'AvgRate',  # 12. Count/time since daemon start
    'CtPctAvg', # 13. InstRate / AvgRate ("percent of mean count rate")
    'Item'      # 14. Attribute name
]

# Human-readable, three-row headers for each column: (scope, metric, units).
# The scope row names the time window (Recent = between the two ClassAds,
# Lifetime/Total = since daemon start); the metric row names the quantity;
# the units row gives its units (s = seconds, # = count, /s = count per
# second, % = percentage). Stacking the words keeps each column only as
# wide as its longest single word, so we get real words and units without
# the cryptic abbreviations.
COL_LABELS = {
    #            scope        metric     units
    'TotalRt':  ('Total',    'Runtime', '(s)'),
    'InstRt':   ('Recent',   'Runtime', '(s)'),
    'InstAvg':  ('Recent',   'Avg',     '(s)'),
    'TotAvg':   ('Lifetime', 'Avg',     '(s)'),
    'TotMax':   ('Lifetime', 'Max',     '(s)'),
    'TotMin':   ('Lifetime', 'Min',     '(s)'),
    'RtPctAvg': ('Pct of',   'Avg',     '(%)'),
    'RtPctMax': ('Pct of',   'Max',     '(%)'),
    'RtSigmas': ('Std',      'Devs',    ''),
    'TotalCt':  ('Total',    'Count',   '(#)'),
    'InstCt':   ('Recent',   'Count',   '(#)'),
    'InstRate': ('Recent',   'Rate',    '(/s)'),
    'AvgRate':  ('Lifetime', 'Rate',    '(/s)'),
    'CtPctAvg': ('Pct of',   'Rate',    '(%)'),
    'Item':     ('',         'Name',    ''),
}

# column sets
COL_SETS = {
    'all':     COL_NAMES,
    'default': ['InstRt', 'InstAvg', 'TotAvg', 'TotMax', 'RtPctAvg',
                    'InstRate', 'AvgRate', 'Item'],
    'runtime': ['InstRt', 'InstAvg', 'TotAvg', 'TotMax', 'RtPctAvg',
                    'RtPctMax', 'RtSigmas', 'Item'],
    'count':   ['TotalCt', 'InstCt', 'InstRate', 'AvgRate', 'CtPctAvg', 'Item'],
}

# use the default column set if not specified
if not (COL_SET in COL_SETS):
    COL_SET = 'default'

# check that SORT_COL is a valid column
if not (SORT_COL in COL_NAMES):
    for col_name in COL_NAMES:
        if SORT_COL.lower() == col_name.lower():
            SORT_COL = col_name
            break
    else: # if not a valid column, use the default
        if COL_SET != 'count':
            SORT_COL = 'InstRt'
        else:
            SORT_COL = 'InstRate'

# check if output being piped
PIPE = not sys.stdout.isatty()

# live mode does not apply when piped or using ClassAd files
if PIPE or FILES:
    LIVE = False

if HISTORY:
    LIVE = False

###

### ClassAd querying functions ###

def getConstraint(machine = None, name = None):
    """Figure out what the constraint string should be and return it"""

    constraints = []
    if machine is not None :
        constraints.append('(Machine =?= {0})'.format(classad.quote(machine)))
    if name is not None:
        constraints.append('(Name =?= {0})'.format(classad.quote(name)))

    constraint = ' && '.join(constraints)
    return constraint

def warnMultipleAds(n, daemon, pool):
    """Prints a warning to stderr if the Collector returns multiple ads"""
    
    if pool is None:
        pool = htcondor.param['COLLECTOR_HOST']
    sys.stderr.write((
        'Warning: Received {0} {1} ads from Collector {2}\n'
        'Consider specifying a daemon name with -n NAME.\n'
        ).format(n, daemon, pool))

def errNoAds(daemon, pool):
    """Prints an error to stderr and quits if the Collector returns no ClassAds"""
    
    if pool is None:
        pool = htcondor.param['COLLECTOR_HOST']
    sys.stderr.write((
        'Error: Did not receive any {0} ads from Collector {1}\n'
        ).format(daemon, pool))
    sys.exit(1)
    
def getDaemonName(pool, daemon, machine):
    """Returns Name attribute from the first ClassAd returned by the Collector"""

    # first try getting ads by constraining the Machine name
    collector = htcondor.Collector(pool)
    constraint = getConstraint(machine = machine)
    ads = collector.query(htcondor.AdTypes[daemon],
                                 constraint=constraint,
                                 projection=['Name'])

    # if that doesn't work, remove the constraint and try again
    if len(ads) == 0:
        ads = collector.query(htcondor.AdTypes[daemon],
                                  projection=['Name'])

    if len(ads) == 0:
        errNoAds(daemon, pool)
    if len(ads) > 1:
        warnMultipleAds(len(ads), daemon, pool)

    name = ads[0]['Name']
    return name

def queryClassAd(pool, daemon, name, direct = False, statistics = 'All:2'):
    """Returns the first ClassAd returned by the Collector"""

    collector = htcondor.Collector(pool)

    if direct:
        # direct queries take different arguments and return one ad
        kwargs = {
            'daemon_type': htcondor.DaemonTypes[daemon],
            'name': name,
            'statistics': statistics
        }
        ad = collector.directQuery(**kwargs)

    else:
        # indirect queries may return multiple ads
        kwargs = {
            'ad_type': htcondor.AdTypes[daemon],
            'constraint': getConstraint(name = name),
            'statistics': statistics
        }
        ads = collector.query(**kwargs)

        if len(ads) == 0:
            errNoAds(daemon, pool)
        if len(ads) > 1:
            warnMultipleAds(len(ads), daemon, pool)

        # return only the first ad
        ad = ads[0]

    return ad

def readScheddHistoryAds(history_path, count = 2):
    """Returns the last <count> ClassAds from schedd_daemon_history."""

    ads = []

    def flush_block(lines):
        if not lines:
            return
        text = '\n'.join(lines) + '\n\n'
        try:
            ad = classad.parseOne(text, parser=classad.Parser.Old)
        except Exception as exc:
            raise RuntimeError('Failed to parse schedd history ClassAd in {0}: {1}'.format(
                history_path, exc))
        ads.append(ad)
        lines.clear()

    with open(history_path) as history_file:
        current_lines = []
        for raw_line in history_file:
            line = raw_line.strip()
            if not line:
                continue
            if line.startswith('***'):
                flush_block(current_lines)
                continue
            current_lines.append(raw_line.rstrip('\n'))
        flush_block(current_lines)

    if len(ads) < count:
        raise RuntimeError(
            'History file {0} does not contain {1} ClassAds'.format(
                history_path, count))

    if count == 2:
        for idx in range(len(ads) - 1, 0, -1):
            older = ads[idx - 1]
            newer = ads[idx]
            if ('DaemonStartTime' in older and 'DaemonStartTime' in newer and
                    older['DaemonStartTime'] == newer['DaemonStartTime']):
                return [older, newer]
        return ads[-2:]

    return ads[-count:]

###

### ClassAd parsing ###

def getRuntimeAttrs(ad):
    """Returns a list of runtime attributes"""

    re_runtime = re.compile('^(.*)Runtime$')

    # some attributes should always be ignored
    re_ignore = re.compile('^DC(Socket|Pipe)')
    ignored_attrs = ['SCGetAutoCluster_cchit']

    attrs = []
    for key in ad.keys():
        match = re_runtime.match(key)
        if match:
            attr = match.groups()[0]
            if not (re_ignore.match(attr) or (attr in ignored_attrs)):
                attrs.append(attr)

    return attrs

### calculations ###

def computeUpdateTimes(old, new):
    """Returns the ClassAds' update times (Unix epoch)"""

    if (('DaemonStartTime' in new) and ('StatsLifetime' in new) and
        ('DaemonStartTime' in old) and ('StatsLifetime' in old)):
        t_0 = new['DaemonStartTime']
        if t_0 != old['DaemonStartTime']:
            sys.stderr.write('Error: Daemon ads have different start times.\n')
            sys.exit(1)
        t_old = t_0 + old['StatsLifetime']
        t_new = t_0 + new['StatsLifetime']
    else:
        t_old = old['MyCurrentTime']
        t_new = new['MyCurrentTime']

    # check for same timestamp
    if (t_new - t_old) == 0:
        if new['MyCurrentTime'] > old['MyCurrentTime']:  # try using MyCurrentTime
            t_new = new['MyCurrentTime']
            t_old = old['MyCurrentTime']
        else:  # quit and notify if the ads have the same timestamp
            sys.stderr.write('Error: ClassAds have the same timestamp.\n')
            sys.exit(1)
        
    return (t_old, t_new)

def computeInstDutyCycle(old, new):
    """Returns the daemon core duty cycle computed between two ClassAds"""

    if not (('DCSelectWaittime' in new) and ('DCPumpCycleSum' in new)):
        return None

    dPumpCycleCount = new['DCPumpCycleCount'] - old['DCPumpCycleCount']
    dPumpCycleSum = new['DCPumpCycleSum'] - old['DCPumpCycleSum']

    # check if a cycle has occured between ads
    if (dPumpCycleCount == 0) or (dPumpCycleSum < 1e-6):
        sys.stderr.write(('Duty cycle undefined, try setting a larger delay (-d).\n'))
        return None
    
    dSelectWaittime = new['DCSelectWaittime'] - old['DCSelectWaittime']
    
    duty_cycle = 100*(1 - dSelectWaittime/float(dPumpCycleSum))
    return duty_cycle    

def computeInstOpsPerSec(old, new):
    """Returns the daemon core commands per second computed between two ClassAds"""

    if not ('DCCommands' in new):
        return None

    t_old, t_new = computeUpdateTimes(old, new)
    dt = t_new - t_old

    dCommands = new['DCCommands'] - old['DCCommands']
        
    ops_per_sec = dCommands / float(dt)
    return ops_per_sec   
    
def computeRuntimeStats(old, new):
    """Returns a list of derived runtime and count statistics
    computed between two ClassAds for each runtime attribute"""
    
    t_old, t_new = computeUpdateTimes(old, new)
    dt = t_new - t_old

    # StatsLifetime is not reported by Startd, so we can't compute
    # lifetime stats (e.g. average counts/sec) from Startd ads.
    if 'StatsLifetime' in new:
        t = new['StatsLifetime']
    else:
        t = None

    # get the list of runtime attributes
    attrs = getRuntimeAttrs(new)

    # add any additional attributes specified by --attr
    for attr in ATTRIBUTES:
        attrs.append(attr)

    # build a list with tuples containing values of each statistic
    table = []
    for attr in attrs:

        # check that attribute counts exist in both ads
        if (attr in new) and (attr in old):
            C = new[attr]
            dC = new[attr] - old[attr]
        else:
            # don't bother keeping this attribute if there are no counts
            continue

        # check that attribute runtimes exist in both ads
        if ((attr + 'Runtime') in new) and ((attr + 'Runtime') in old):
            R = new[attr + 'Runtime']
            dR = new[attr + 'Runtime'] - old[attr + 'Runtime']
        else:
            R, dR = (None, None)

        # compute runtime/count between ads
        if (dC > 0) and (dR is not None):
            R_curr = dR / float(dC)
        else:
            R_curr = None

        # grab the attribute's lifetime runtime stats
        R_ = {}
        for stat in ['Avg', 'Max', 'Min', 'Std']:
            if (attr + 'Runtime' + stat) in new:
                R_[stat] = new[attr + 'Runtime' + stat]
            else:
                R_[stat] = None

        R_pct_avg, R_pct_max, R_sigmas = (None, None, None)
        if R_curr is not None:

            # compare current runtime/count to lifetime runtime/count
            if R_['Avg'] is not None:
                R_pct_avg = 100. * R_curr / float(R_['Avg'])

            # compare current runtime/count to lifetime max/min range
            if (R_['Max'] is not None) and (R_['Min'] is not None):
                if R_['Max'] == R_['Min']:
                    R_pct_max = 100.
                else:
                    R_pct_max = 100. * ( (R_curr - R_['Min']) /
                                             float(R_['Max'] - R_['Min']) )

            # compare difference between current and lifetime runtime/count
            # to the lifetime standard deviation in runtime/count
            if (R_['Avg'] is not None) and (R_['Std'] is not None):
                R_sigmas = (R_curr - R_['Avg']) / float(R_['Std'])

        # compute count/sec between ads
        C_curr = dC / float(dt)

        # compute lifetime counts/sec
        if t:
            C_avg = C / float(t)
        else:
            C_avg = None

        # compare current counts/sec to lifetime counts/sec
        C_pct_avg = None
        if (dC > 0) and (C_avg is not None):
            C_pct_avg = 100. * C_curr / float(C_avg)

        # cleanup item name
        if attr[0:2] == 'DC':
            attr = attr[2:]
            
        # store stats in a list in the same order as COL_NAMES
        row = [None]*len(COL_NAMES)
        row[COL_NAMES.index('Item')]      = attr
        row[COL_NAMES.index('TotalRt')]   = R
        row[COL_NAMES.index('InstRt')]    = dR
        row[COL_NAMES.index('InstAvg')]   = R_curr
        row[COL_NAMES.index('TotAvg')]    = R_['Avg']
        row[COL_NAMES.index('TotMax')]    = R_['Max']
        row[COL_NAMES.index('TotMin')]    = R_['Min']
        row[COL_NAMES.index('RtPctAvg')]  = R_pct_avg
        row[COL_NAMES.index('RtPctMax')]  = R_pct_max
        row[COL_NAMES.index('RtSigmas')]  = R_sigmas
        row[COL_NAMES.index('TotalCt')]   = C
        row[COL_NAMES.index('InstCt')]    = dC
        row[COL_NAMES.index('InstRate')]  = C_curr
        row[COL_NAMES.index('AvgRate')]   = C_avg
        row[COL_NAMES.index('CtPctAvg')]  = C_pct_avg

        # tupleize the stats list and add them to the larger list
        table.append(tuple(row))
        
    return table

### view ###

def getTerminalSize():
    """Returns the height (lines) and width (characters) of the terminal"""

    try: # works on Linux
        height, width = [int(x) for x in
                                  os.popen('stty size', 'r').read().split()]
        width -= 1 # leave a blank column on the right
        
    except Exception: # otherwise assume standard console size
        height = 25
        width = 80 - 1 # leave a blank column on the right

    return (height, width)

def getRuntimeColFmts(col_name, table, fmt_type):
    """Returns the text and number format that best fits a column"""

    # the only string-type data displayed is at the end of the row
    # so it doesn't require any special formatting
    column_fmt = '{0}'
    number_fmt = '{0}'

    if fmt_type == 'str':
        return (column_fmt, number_fmt)

    # for numerical data, compare the width of the (multi-row) header
    # to the length of the largest (in characters) number. The header
    # width is the longest single word stacked in the column.
    name_width = max(len(part) for part in COL_LABELS[col_name])

    # the "largest" number could be negative, so check both max and min
    i = COL_NAMES.index(col_name)
    max_val = max(table, key=lambda row: row[i] if row[i] is not None else 0)[i]
    min_val = min(table, key=lambda row: row[i] if row[i] is not None else 0)[i]

    try:
        max_width = len('{0:d}'.format(int(max_val)))
    except TypeError:
        max_width = 0

    try:
        min_width = len('{0:d}'.format(int(min_val)))
    except TypeError:
        min_width = 0

    n_width = max([max_width, min_width])

    # for floats, we want the largest number of decimal places
    # that we can get away with without needlessly expanding the column size
    if fmt_type in ['float', 'floatneg']:

        # add one to the length of columns that could have negative numbers
        if fmt_type == 'floatneg':
            n_width += 1 

        # compare against max int length + 1 because of decimal point
        if (n_width + 1) < name_width:
            decimals = name_width - (n_width + 1)
            n_width = (n_width + decimals + 1)
        else:
            decimals = 0

        col_width = max([n_width, name_width])

        column_fmt = '{0:>%ds}' % (col_width,)
        number_fmt = '{0:>%d.%df}' % (col_width, decimals)

    # for integers, we just need to make sure that we aren't needlessly
    # expanding the column size unless the numbers are really large
    if fmt_type == 'int':
        col_width = max([n_width, name_width])

        column_fmt = '{0:>%ds}' % (col_width,)
        number_fmt = '{0:>%dd}' % (col_width,)

    return (column_fmt, number_fmt)

def formatMemorySize(size):
    """Returns a string of human-readable memory size"""

    # ClassAds report memory in KB, so start with K
    suffixes = ['K', 'M', 'G', 'T', 'P']

    for i in range(len(suffixes)):
        if size/(10**(i*3)) < 1000:
            break

    size = size/float(10**(i*3))
    suffix = suffixes[i]

    return '{0:.1f} {1}'.format(size, suffix)

def printMonitorSelf(daemon, name, ad):
    """Prints the MonitorSelf stats and returns the number of lines printed"""

    out = []

    t = ad['MonitorSelfTime']
    out.append('{0} {1} status at {2}:'.format(daemon, name,
                    datetime.datetime.fromtimestamp(t)))
    out.append('\n')

    if 'MonitorSelfAge' in ad:
        uptime = datetime.timedelta(seconds = ad['MonitorSelfAge'])
        out.append('Uptime: {0}'.format(uptime))
        out.append('\n')
    if 'MonitorSelfSysCpuTime' in ad:
        sys_cpu_time = datetime.timedelta(seconds = ad['MonitorSelfSysCpuTime'])
        out.append('SysCpuTime: {0}'.format(sys_cpu_time))
        out.append(' '*4)
    if 'MonitorSelfUserCpuTime' in ad:
        user_cpu_time = datetime.timedelta(seconds=ad['MonitorSelfUserCpuTime'])
        out.append('UserCpuTime: {0}'.format(user_cpu_time))
        out.append('\n')
    if 'MonitorSelfImageSize' in ad:
        memory = formatMemorySize(ad['MonitorSelfImageSize'])
        out.append('Memory: {0}'.format(memory))
        out.append(' '*4)
    if 'MonitorSelfResidentSetSize' in ad:
        rss = formatMemorySize(ad['MonitorSelfResidentSetSize'])
        out.append('RSS: {0}'.format(rss))
        out.append('\n')
    out.append('\n')

    sys.stdout.write(''.join(out))

    lines = out.count('\n')
    
    return lines

def printRecentHealthStats(ad):
    """Prints the "recent" DC duty cycle and commands per second
    and returns the number of lines printed"""

    lines = 0

    # not all ClassAds report the RecentStatsTickTime,
    # but MonitorSelfTime is a close approximation
    if 'RecentStatsTickTime' in ad:
        t = datetime.datetime.fromtimestamp(ad['RecentStatsTickTime'])
    else:
        t = datetime.datetime.fromtimestamp(ad['MonitorSelfTime'])

    sys.stdout.write('Recent DC status at {0}:\n'.format(t))
    lines += 1
    
    if 'RecentDaemonCoreDutyCycle' in ad:
        duty_cycle = 100*ad['RecentDaemonCoreDutyCycle']
        if 'RecentStatsLifetime' in ad:
            duration = ad['RecentStatsLifetime']
            sys.stdout.write('Duty Cycle (last {0}s): {1:.2f}%    '.format(
                duration, duty_cycle))
        else:
            sys.stdout.write('Duty Cycle: {0:.2f}%    '.format(duty_cycle))

    for attr in ['DCCommandsPerSecond_' + l for l in ['4m', '20m', '4h']]:
        if attr in ad:
            ops_per_sec = ad[attr]
            duration = attr.split('_')[1]
            sys.stdout.write('Ops/second (last {0}): {1:.3f}'.format(
                duration, ops_per_sec))
            break

    sys.stdout.write('\n\n')
    lines += 2
        
    return lines    
    
def printInstHealthStats(old, new):
    """Prints the DC duty cycle and commands per second between two ads
    and returns the number of lines printed"""

    duty_cycle = computeInstDutyCycle(old, new)
    ops_per_sec = computeInstOpsPerSec(old, new)

    # in the case that we cannot get the values from between ads,
    # just print the recent DC stats
    if (duty_cycle is None) or (ops_per_sec is None):
        return printRecentHealthStats(new)

    lines = 0

    # print the time between ads
    t_old, t_new = computeUpdateTimes(old, new)
    t_old, t_new = [datetime.datetime.fromtimestamp(t) for t in [t_old, t_new]]
    sys.stdout.write('DC status from {0} to {1}:\n'.format(t_old, t_new))
    lines += 1

    # print the stats
    sys.stdout.write('Duty Cycle: {0:.2f}%    '.format(duty_cycle))
    sys.stdout.write('Ops/second: {0:.3f}\n'.format(ops_per_sec))
    lines += 1

    sys.stdout.write('\n')
    lines += 1

    return lines
    
    
def printRuntimeTable(old, new, col_set = COL_SET, sort_col = SORT_COL, lines = 0):
    """Prints the table of runtime statistics"""

    # check the terminal size
    height, width = getTerminalSize()

    # get the table of stats
    table = computeRuntimeStats(old, new)

    # get the text and number formats for each column
    col_fmts = {}
    for col in ['TotalCt', 'InstCt']:
        col_fmts[col] = getRuntimeColFmts(col, table, 'int')
    for col in ['RtSigmas']:
        col_fmts[col] = getRuntimeColFmts(col, table, 'floatneg')
    for col in ['Item']:
        col_fmts[col] = getRuntimeColFmts(col, table, 'str')
    for col in COL_NAMES:
        if not (col in col_fmts):
            col_fmts[col] = getRuntimeColFmts(col, table, 'float')

    # sort the table
    table.sort(key = itemgetter(COL_NAMES.index(sort_col)), reverse = True)

    # get the column names to be printed
    col_names = COL_SETS[col_set]

    # print the time between ads
    t_old, t_new = computeUpdateTimes(old, new)
    t_old, t_new = [datetime.datetime.fromtimestamp(t) for t in [t_old, t_new]]
    sys.stdout.write('Runtime stats from {0} to {1}:\n'.format(t_old, t_new))
    lines += 1
    
    # print the multi-row header: scope, then metric, then units
    for header_row in (0, 1, 2):
        cols = []
        for col_name in col_names:
            label = COL_LABELS[col_name][header_row]
            cols.append(col_fmts[col_name][0].format(label))
        col_string = ' '.join(cols).rstrip()

        # crop the header to the terminal window if needed
        if (not PIPE) and (len(col_string) > width):
            col_string = col_string[:width]
        sys.stdout.write(col_string + '\n')
        lines += 1

    # print a rule separating the header from the data, underlining each
    # column to its own width
    rule_cols = []
    for col_name in col_names:
        col_width = max(len(col_fmts[col_name][0].format(part))
                            for part in COL_LABELS[col_name])
        rule_cols.append('-' * col_width)
    rule_string = ' '.join(rule_cols).rstrip()
    if (not PIPE) and (len(rule_string) > width):
        rule_string = rule_string[:width]
    sys.stdout.write(rule_string + '\n')
    lines += 1

    # print the rows
    for row in table:

        # make a list of columns
        cols = []

        # go in the order of col_names
        for col_name in col_names:
            col_value = row[COL_NAMES.index(col_name)]

            # if the value can't be formatted, it's probably None ("n/a")
            try:
                cols.append(col_fmts[col_name][1].format(col_value))
            except (ValueError, TypeError):
                cols.append(col_fmts[col_name][0].format('n/a'))

        # merge the column list into a single string
        row_string = ' '.join(cols)

        # crop the row to the terminal window if needed
        if (not PIPE) and (len(row_string) > width):
            row_string = row_string[:width]

        # print the row
        sys.stdout.write(row_string + '\n')
        lines += 1

        # leave a blank row at the bottom of the table
        if (not PIPE) and (lines >= height - 1):
            break

def printCountdown(delay = DELAY):
    """Prints countdown to the next update to stderr"""
    now = datetime.datetime.now(datetime.timezone.utc)
    refresh_time = now + datetime.timedelta(seconds=delay)
    time_left = refresh_time - now
    while time_left > datetime.timedelta(seconds=0):
        # rewind the line (\r) and print countdown each second
        sys.stderr.write('\rUpdating ClassAd in {0:>5d} s'.format(
            time_left.seconds))
        sys.stderr.flush()
        sleep(1)
        time_left = refresh_time - datetime.datetime.now(datetime.timezone.utc)
    sys.stderr.write('\rUpdating ClassAd ' + 10*'.' + '\n\n')
    sys.stderr.flush()

def resetTerminal():
    """Clears the current terminal output"""
    sys.stdout.write('\n\n') # add a few blank lines
    sys.stdout.flush()
    if os.name == 'nt':
        os.system('cls')
    else:
        os.system('clear')
        
if __name__ == '__main__':

    # Two modes of input:
    # 1. ClassAds parsed from two files
    # 2. ClassAds queried from the Collector

    # Two modes of output:
    # 1. stdout to the terminal window
    # 2. stdout to a pipe

    # wrap everything to catch Ctrl-C
    try:

        if HISTORY and FILES:
            sys.stderr.write('Error: -history cannot be combined with ClassAd files.\n')
            sys.exit(1)

        # if we have files, parse them into old and new ClassAds
        if FILES:
            if len(FILES) != 2:
                sys.stderr.write(('Error: Bad arguments or more/less '
                                      'than 2 files passed:\n'))
                sys.stderr.write(' '.join(FILES))
                sys.stderr.write('\n')
                sys.exit(1)
            try:
                with open(FILES[0]) as f:
                    old = classad.parseNext(f)
                    if not old:
                        sys.stderr.write((
                            'Error: {0} does not contain any (valid) ClassAds\n'
                            ).format(FILES[0]))

            except Exception as e:
                sys.stderr.write(str(e))
                sys.stderr.write('\n')
                sys.exit(1)
            try:
                with open(FILES[1]) as f:
                    new = classad.parseNext(f)
                    if not new:
                        sys.stderr.write((
                            'Error: {0} does not contain any (valid) ClassAds\n'
                            ).format(FILES[1]))
            except Exception as e:
                sys.stderr.write(str(e))
                sys.stderr.write('\n')
                sys.exit(1)

            # make sure new is newer than old
            if new['MyCurrentTime'] < old['MyCurrentTime']:
                old, new = (new, old)

            # get the daemon type
            types = {
                'Collector': 'Collector',
                'DaemonMaster': 'Master',
                'Negotiator': 'Negotiator',
                'Scheduler': 'Schedd',
                'Machine': 'Startd'
            }

            if new['MyType'] in types:
                DAEMON = types[new['MyType']]
            else:
                DAEMON = new['MyType']

            # get the daemon name
            NAME = new['Name']

        # if we're querying the Collector, first get the "old" ClassAd
        elif HISTORY:
            if DAEMON != 'Schedd':
                sys.stderr.write('Error: -history is only supported for Schedd daemon ads.\n')
                sys.exit(1)
            if POOL is not None:
                sys.stderr.write('Error: -history cannot be used with --pool.\n')
                sys.exit(1)
            spool = htcondor.param.get('SPOOL')
            if not spool:
                sys.stderr.write('Error: SPOOL is not defined in the configuration; cannot locate schedd history file.\n')
                sys.exit(1)
            history_path = os.path.join(spool, 'schedd_daemon_history')
            try:
                old, new = readScheddHistoryAds(history_path, 2)
            except FileNotFoundError:
                sys.stderr.write('Error: Unable to open {0}\n'.format(history_path))
                sys.exit(1)
            except RuntimeError as exc:
                sys.stderr.write(str(exc) + '\n')
                sys.exit(1)
            except Exception as exc:
                sys.stderr.write(str(exc) + '\n')
                sys.exit(1)

            if new['MyCurrentTime'] < old['MyCurrentTime']:
                old, new = (new, old)

            if NAME is None and ('Name' in new):
                NAME = new['Name']

            if len(getRuntimeAttrs(new)) == 0:
                sys.stderr.write("Did not get any comparable runtime attributes from daemon ClassAd, exiting early.\n")
                sys.exit(0)

        else:
            if NAME is None:
                NAME = getDaemonName(POOL, DAEMON, MACHINE)

            old = queryClassAd(POOL, DAEMON, NAME, DIRECT)

            # go ahead and print the information we have
            printMonitorSelf(DAEMON, NAME, old)
            printRecentHealthStats(old)

            # if there aren't any runtime attributes in the classad,
            # print the information we have and exit now
            if len(getRuntimeAttrs(old)) == 0:
                sys.stderr.write("Did not get any comparable runtime attributes from daemon ClassAd, exiting early.\n")
                sys.exit(0)

            # countdown and get the "new" ClassAd from the Collector
            printCountdown()
            new = queryClassAd(POOL, DAEMON, NAME, DIRECT)

            # if in live mode, clear the terminal
            if LIVE:
                resetTerminal()
                pass

        # at this point, we have both old and new ClassAds, so print the info
        lines = printInstHealthStats(old, new)
        printRuntimeTable(old, new, lines = lines)

        # if in live mode, continue getting new ClassAds
        if LIVE:
            while True:
                old = deepcopy(new)
                printCountdown()
                new = queryClassAd(POOL, DAEMON, NAME, DIRECT)
                resetTerminal()
                lines = printMonitorSelf(DAEMON, NAME, new)
                lines += printInstHealthStats(old, new)
                printRuntimeTable(old, new, lines = lines)

    except KeyboardInterrupt:
        sys.stdout.write('\n')
        sys.stdout.flush()
