#!/usr/bin/python
#
####
#  Copyright (c) 2008-2014 Aerospike, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
#
# Description: Utility to monitor the status of the nodes and send commands to the cluster
####

from itertools import izip, chain
from pprint import pprint
import locale
import ConfigParser
import operator
import cmd
import sys, re, getopt
import inspect
from operator import itemgetter
from datetime import datetime
from time import sleep, time
import os, pickle
import shutil
import platform
import StringIO
import argparse
import getpass

try:
	import citrusleaf
except:
	print "citrusleaf tools package not installed ? Using self contained commands"	

################ self contained Info command logics  ###############

import socket
import struct
import types

class Data(object):
	pass

# ## Many of the functions just do a print without any valid returns
# ## This is to capture such prints so that they can be used as returns
def capture_stdout(func,printMsg=True):
	if printMsg is True:
		print "Data collection for " + func.func_name + " in progress.."
		sys.stdout.flush()
	old = sys.stdout
	capturer = StringIO.StringIO()
	sys.stdout = capturer
	func()
	sys.stdout = old
	output = capturer.getvalue()
	return output
#
# COMPATIBILITY COMPATIBILITY COMPATIBILITY
#
# So the 'struct' class went through lots of (good) improvements in 2.5, but we want to support
# old use as well as new. Write a few functions similar to the 2.5 ones, and either use builtin or
# pure based on what's available
#
# 

def my_unpack_from(fmtStr, buf, offset):
	sz = struct.calcsize(fmtStr)
	return struct.unpack(fmtStr, buf[offset:offset + sz])
	
def my_pack_into(fmtStr, buf, offset, *args):
	tmp_array = struct.pack(fmtStr, *args)
	buf[offset:offset + len(tmp_array)] = tmp_array

# 2.5+ has this nice partition call
def partition_25(s, sep):
	return(s.partition(sep))
	
# 2.4- doesn't
def partition_old(s, sep):
	idx = s.find(sep)
	if idx == -1:
		return(s, "", "")
	return(s[:idx], sep, s[idx + 1:])


g_proto_header = None
g_struct_header_in = None
g_struct_header_out = None
g_partition = None

# 2.5, this will succeed
try:
	g_proto_header = struct.Struct('! Q')
	g_cl_msg_header_in = struct.Struct('! B 2x B x B I 8x H H')
	g_struct_header_in = struct.Struct('! Q B 4x B I 8x H H')
	g_struct_header_out = struct.Struct('! Q B B B B B B I I I H H')
	g_partition = partition_25
	
# pre 2.5, if there's no Struct submember, so use my workaround pack/unpack
except:
	struct.unpack_from = my_unpack_from
	struct.pack_into = my_pack_into
	g_partition = partition_old

#
# Make an info request of the buffer
#


def citrusleaf_info_request(host, port, buf, debug=False):

	global g_netstopwatch_log
	
	stopwatch_big1 = None
	stopwatch_big2 = None
	topwatch_small1 = None
	topwatch_small2 = None
	if g_want_netstopwatch:
		stopwatch_big1 = time()
		g_netstopwatch_log_fd.write("\ninfo command = " + buf.rstrip())
		g_netstopwatch_log_fd.write("\ngrand begin time = " + str(stopwatch_big1))
	# request over TCP
	try:
		if g_want_netstopwatch:
			stopwatch_small1 = time()
			g_netstopwatch_log_fd.write("\n	begin cmd/header send/rec = " + str(stopwatch_small1))
		if debug:
			print 'debug info request'

		sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		sock.settimeout(0.5)
		sock.connect((host, int(port)))

		if user != None:
			rc = authenticate(sock, user, password)

			if rc != 0:
				print "Authentication failed for ", user, ": ", rc
				sock.close()
				return None

		sock.send(buf)

		if g_want_netstopwatch:
			stopwatch_small2 = time()
			g_netstopwatch_log_fd.write("\n	end   cmd/header send/rec = " + str(stopwatch_small2))
			g_netstopwatch_log_fd.write("		  lapse = " + str(stopwatch_small2 - stopwatch_small1))

		if debug:
			print "info get response"
		# get response
		rsp_hdr = sock.recv(8)

		q = struct.unpack_from("! Q", rsp_hdr, 0)
		sz = q[0] & 0xFFFFFFFFFFFF
		if debug:
			print "recv header length ", sz
		if sz > 0:
			pos = 0
			while (pos < sz): 
				if g_want_netstopwatch:
					stopwatch_small1 = time()
					g_netstopwatch_log_fd.write("\n		begin	 body chunk recv = " + str(stopwatch_small1))

				tmp_data = sock.recv(sz - pos)

				if g_want_netstopwatch:
					stopwatch_small2 = time()
					g_netstopwatch_log_fd.write("\n		end	   body chunk recv = " + str(stopwatch_small2))
					g_netstopwatch_log_fd.write("			  lapse = " + str(stopwatch_small2 - stopwatch_small1))
				if pos == 0:
					rsp_data = tmp_data
				else:
					rsp_data += tmp_data
				pos += len(tmp_data)
		sock.close()
	except Exception , ex:
		# print "info request got exception ",type(ex)," ",ex
		if g_want_netstopwatch:
			g_netstopwatch_log_fd.write("\n	exception encounted. " + type(ex) + "=" + str(ex))
			g_netstopwatch_log_fd.write("\n----------------------")
			g_netstopwatch_log_fd.flush()
		return -1	

	if g_want_netstopwatch:
		stopwatch_big2 = time()
		g_netstopwatch_log_fd.write("\ngrand  end time = " + str(stopwatch_big2))
		g_netstopwatch_log_fd.write("	lapse = " + str(stopwatch_big2 - stopwatch_big1))
		g_netstopwatch_log_fd.write("\n----------------------")
		g_netstopwatch_log_fd.flush()

	# parse out responses
	if sz == 0:
		return None

	if debug:
		print "receive as string: ", rsp_data

	return(rsp_data)




#
# Citrusleaf Info Command
#
# This is a special purpose request to do things to the citrusleaf cluster
# for a given node. 
#
# pass in the command - which is a command string, and a dict
# of name-value pairs.
#
# Wire format is cmd:name=value;name=value....
#
# Returns: values, a dict of with the name being the index, and the value being the value

CL_PROTO_SIZE	 = 8
CL_PROTO_TYPE_INFO = 1	 
CL_PROTO_TYPE_MSG = 3
CL_PROTO_VERSION = 2

def citrusleaf_info_command(host, port, cmd, parameters=None, debug=False):

	# Passed a set of names: created output buffer
	param_l = []
	param_l.append(cmd)
	param_l.append(":")
	if parameters != None:
		for name, value in parameters.iteritems():
			param_l.append(name)
			param_l.append("=")
			param_l.append(value)
			param_l.append(";")
		del param_l[len(param_l) - 1]
	param_l.append("\n")
	paramstr = "".join(param_l)
	# sometimes this string is unicode, if the parameters input were unicode not string
	# force to string just to be sure - this may be required elsewhere -
	# different versions of python are different about how they type stuff like this
	paramstr = str(paramstr)

	q = (CL_PROTO_VERSION << 56) | (CL_PROTO_TYPE_INFO << 48) | (len(paramstr))
	fmtStr = "! Q %ds" % len(paramstr)
	buf = struct.pack(fmtStr, q, paramstr)

	rsp_data = citrusleaf_info_request(host, port, buf, debug)

	if debug:
		print "citrusleaf info: response ", rsp_data
	return rsp_data


#
# Citrusleaf Info request
#
# This is a special purpose request to get informational name-value pairs
# from a given node. It's good for discovering the rest of the cluster,
# or trying to figure out which cluster members have which parts of the key space
#
# host, port are self explanatory
# 'names' is an iterable list of values to get, or None to get all. (well, actually, 'all' isn't 
# really *all*. It's a subset of all that someone decided was useful. YMMV)
#  being really nice, also supporting a single string as a name instead of requiring a list
#
# Returns: values, a dict of with the name being the index, and the value being the value


def citrusleaf_info(host, port, names=None, debug=False):

	# Passed a set of names: created output buffer

	if names == None:
		q = (CL_PROTO_VERSION << 56) | (CL_PROTO_TYPE_INFO << 48)
		if g_proto_header != None:
			buf = g_proto_header.pack(q)
		else:
			buf = struct.pack('! Q', q)

	elif type(names) == types.StringType:
		q = (CL_PROTO_VERSION << 56) | (CL_PROTO_TYPE_INFO << 48) | (len(names) + 1)
		fmtStr = "! Q %ds B" % len(names)
		buf = struct.pack(fmtStr, q, names, 10)

	else:  # better be iterable of strings
		# annoyingly, join won't post-pend a seperator. So make a new list
		# with all the seps in
		namestr = ""
		for name in names:
			namestr += name + '\n'
		q = (CL_PROTO_VERSION << 56) | (CL_PROTO_TYPE_INFO << 48) | (len(namestr) + 1)
		fmtStr = "! Q %ds B" % len(namestr)
		buf = struct.pack(fmtStr, q, namestr, 10)

	rsp_data = citrusleaf_info_request(host, port, buf, debug)

	if rsp_data == -1 or rsp_data == None:  # 'None' is returned when there is an invalid request
		return -1

	# if the original request was a single string, return a single string
	if type(names) == types.StringType:
		lines = rsp_data.split("\n")
		name, sep, value = g_partition(lines[0], "\t")

		if name != names:
			print " problem: requested name ", names, " got name ", name
			return(-1)
		return value

	else:
		rdict = dict()
		for line in rsp_data.split("\n"):
			if len(line) < 1:  # this accounts for the trailing '\n' - cheaper than chomp
				continue
			if debug:
				print " found line ", line
			name, sep, value = g_partition(line, "\t")
			if debug:
				print "	name: ", name, " value: ", value
			rdict[name] = value

		return rdict 

################ Info command logics end ###############

################ Persistent Dict ########################

# # {{{ http://code.activestate.com/recipes/576642/ (r10)


class PersistentDict(dict):
	''' Persistent dictionary with an API compatible with shelve and anydbm.

	The dict is kept in memory, so the dictionary operations run as fast as
	a regular dictionary.

	Write to disk is delayed until close or sync (similar to gdbm's fast mode).

	Input file format is automatically discovered.
	Output file format is selectable between pickle, json, and csv.
	All three serialization formats are backed by fast C implementations.

	'''
	
	def __init__(self, filename, flag='c', mode=None, format='pickle', *args, **kwds):
		self.flag = flag  # r=readonly, c=create, or n=new
		self.mode = mode  # None or an octal triple like 0644
		self.format = format  # Only 'pickle' 
		self.filename = filename
		if flag != 'n' and os.access(filename, os.R_OK):
			fileobj = open(filename, 'rb')
			try:
				self.load(fileobj)
			finally:
				fileobj.close()
		dict.__init__(self, *args, **kwds)

	def sync(self):
		'Write dict to disk'
		if self.flag == 'r':
			return
		filename = self.filename
		tempname = filename + '.tmp'
		fileobj = open(tempname, 'wb')
		try:
			try:
				self.dump(fileobj)
			except Exception:
				os.remove(tempname)
				raise 
		finally:
			fileobj.close()
	
		shutil.move(tempname, self.filename)  # atomic commit
		if self.mode is not None:
			os.chmod(self.filename, self.mode)

					

	def close(self):
		self.sync()

	def __enter__(self):
		return self

	def __exit__(self, *exc_info):
		self.close()

	def dump(self, fileobj):
		if self.format == 'pickle':
			pickle.dump(dict(self), fileobj, 2)
		else:
			raise NotImplementedError('Unknown format: ' + repr(self.format))

	def load(self, fileobj):
		# try formats from most restrictive to least restrictive
		loader = pickle.load
		fileobj.seek(0)
		try:
			return self.update(loader(fileobj))
		except Exception:
			pass

# # end of http://code.activestate.com/recipes/576642/ }}}
	
#
# Utility function to convert a String received from cinfo to a dictionary and return a dictionary
#
def string_to_dict(value):
	'''Simple function to convert string to dict'''
	
	stat_dict = {}
	stat_param = re.split(";", value)
	for value in stat_param:
		check = re.split("=", value)
		i1 = iter(check)
		MyDict = dict(izip(i1, i1))
		stat_dict.update(MyDict)
	return stat_dict

#
# Utility function to convert a String from sets and other new info command which dont follow the above format.
# It converts to a dictionary and return a dictionary
#
def string_colon_to_dict(value):
	'''Simple function to convert colon separated string to dict'''
	
	stat_dict = {}
	stat_param = re.split(":", value)
	for value in stat_param:
		check = re.split("=", value)
		i1 = iter(check)
		MyDict = dict(izip(i1, i1))
		stat_dict.update(MyDict)
	return stat_dict

#
# Crawl through all the known nodes in the cluster, and validate if each node has visibility to all other 
# nodes in the cluster. This also has the nice side benefite of auto-discovery of (all) the nodes.
# Logic: Passed in the list of IP from either the user or from hosts created by us in [donotedit] section
# "hosts" will have a list of active and inactive IP's
# "arg_machines" which is passed in is lost and is replaced by "cluster" when its returned.
# "stateDict" keeps track if the nodes can see the other nodes in the list
# "status" is basically NOT used for now , but it tracks list of all nodes, for future use.
# "host_alias" has a dict of DNS and IP address returned from services list


def cluster_crawler(arg_machines):
	'''go through the entire cluster through services list and created an updated list to use'''
	
	global hosts
	global host_alias, status
	stateDict = {}
	cluster = []
	rawIpServices = {}
	status = {}
	ipServices = {}
	uniqueNode = {}
	nodeList = []

	# first, translate all the cnames to IPs. we deal only in IPs, This creates a list of active IP
	# from the list which has been passed in.
	
	for raw_mcname in arg_machines:
		try:
			# skip over all the known aliases
			if raw_mcname in host_alias.keys():
				continue

			# cinfo return -1 mean the node is not responding
			allStatDict = cinfo(raw_mcname, ['service', 'node'])	
			mcname = allStatDict['service']	
			if allStatDict['node'] not in uniqueNode:
				uniqueNode[allStatDict['node']] = []
			
			if mcname == -1 or mcname == None:
				status[raw_mcname] = False
				continue
			else:
				try:			
					machine, port = split_ip_port(mcname)
					if machine == -1:   
						if mcname.find(";") != -1:
							multiple_interfaces_on_node = mcname.split(';')
							for node in multiple_interfaces_on_node:
								machine, port = split_ip_port(node)
								uniqueNode[allStatDict['node']].append(node)
								if machine != -1:			   
									mcname = machine + ":" + port
									break
					else:
						mcname = machine + ":" + port
						uniqueNode[allStatDict['node']].append(mcname)
								
				except Exception, i:
					print "Encountered Exception: ", i
					
				status[mcname] = True
				if mcname not in cluster and allStatDict['node'] not in nodeList :
					nodeList.append(allStatDict['node'])
					cluster.append(mcname)
					hosts.append(mcname)
					if mcname != raw_mcname:
						host_alias[raw_mcname] = mcname
		except:
			print "skipping", raw_mcname

	# Loop through the entire cluster to form an active cluster state
	for mcname in hosts:
		# Only call cinfo if we dont have services list for the node
		if mcname not in ipServices:
			try:
				allStatDict = cinfo(mcname, ['services-alumni', 'node'])
				if 'services-alumni' in allStatDict:
					allStatDict['services'] = allStatDict['services-alumni']
					del(allStatDict['services-alumni'])
				else:
					allStatDict = cinfo(mcname, ['services', 'node'])

				ipServices[mcname] = allStatDict['services']	
				if allStatDict['node'] not in uniqueNode:
					uniqueNode[allStatDict['node']] = []
			except:
				ipServices[mcname] = -1
					 
		if ipServices[mcname] == -1 or ipServices[mcname] == None:
			status[mcname] = False
		else:
			status[mcname] = True
			# Create a dict of IP -> services IPs
			rawIpServices[mcname] = parse_services(ipServices[mcname])
			try:
			# Add active nodes to cluster.
				if mcname not in cluster:
					uniqueNode[allStatDict['node']].append(mcname)
														
				if mcname not in cluster and allStatDict['node'] not in nodeList:
					nodeList.append(allStatDict['node'])
					cluster.append(mcname)
			except Exception, i:
				print "Error getting", i
			# examine the services for the current node, and add any newly discovered node to hosts.
			for ip in rawIpServices[mcname]:
				if ip not in hosts:
					hosts.append(ip)

	# Compare cluster with each services list and set state true if services list matches cluster	  
	for ip, services in rawIpServices.iteritems():
		update_cluster_visibility(ip, services)
	
	# This basically compares the list passed in when crawler is called and a list of active nodes
	# in crawler, The reason for doing this is to maintain history of all known nodes, and not using
	# union of arg_machines and cluster is because arg_machines can have dns names or ip without port
	# which messes the final list. 
	
	for m in arg_machines:
		if m not in host_alias.keys():
			if m not in hosts:
				hosts.append(m)	
	
	# Sort the by IP, but this is legacy , the sorting logic is done later on by Node or IP.
	# does not hurt to keep it.	
	cluster.sort()

	# Update fqdn for all nodes in cluster.
	for node in cluster:
		find_and_add_fqdn(node)

	return {'arg_machines':cluster}

# Utility function to replace hostname by IP, and return in same format (i.e. is :3000 exits append
# it to IP or just return IP), A list should be passed in. 

def modifyHosts(hosts):
	for i, v in enumerate(hosts):
		if v in host_alias or v + ":" + str(arg_port) in host_alias:
			if v.find(":") == -1:
				hosts.insert(i, str(filter(None, host_alias.get(v + ":" + str(arg_port)).split(":"))[0]))
				hosts.remove(v)
			else:
				hosts.insert(i, host_alias.get(v))
				hosts.remove(v)	
	return hosts
		
####
# Utility function to parse services list
####

def parse_services(servicesList):
	return re.findall('(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3})\.(?:[\d]{1,3}):(?:[\d]{1,6})', servicesList)

####
# Utility function to find and add fqdn to host_alias
####
def find_and_add_fqdn(ipPort):
	machine, port = filter(None, ipPort.split(':'))
	if machine not in host_alias.keys():
		if args.NoDns is True:
			host_alias[machine] = machine
		else:
			host_alias[machine] = socket.getfqdn(machine)

####
# Utility function to update cluster visibility state
####

def update_cluster_visibility(ipPort, services):

	if set([ipPort] + services) == set(get_active_nodes(True)):
		state[ipPort] = "true"
	else:
		state[ipPort] = "false" 

#
# Parse the config file. Parameters are in the "name=value" format 
# Using Config file instead of parameters makes deployment easier
# If the user didn't provide the config file, let defaults kick in
# Some background as this has become complex: The original host list in section [main] will be read
# and compared with [donotedit] section if it exists, If the ip exists, it means its the same cluster
# and [donotedit] section will be used as database to maintain cluster history. Original host list
# will be maintained only automagic in this list will be changing DNS to IP address.
# The reason being, becomes easier to compare hosts list from original to auto created list.
# parse_config is first called in the script and then cluster config finds the list of ip's
#
def parse_config(configfile, hosts_override):
	''' Parse config from the configfile and initialize all nodes/namespaces'''

	global arg_machines, arg_namespace, arg_xdr, arg_port_xdr, arg_namespace, arg_value, state, config 
	global hosts, host_alias, arg_color, arg_crawl, cluster_objects, arg_port, enable_fqdn

	cluster_objects = 0
	arg_namespace = ""
	state = {}
	arg_value = None
	hosts = []
	org_hosts = []
	arg_port = 3000
	# parm defaults
	arg_machines = ['127.0.0.1:' + str(arg_port)]
	host_alias = {}
	arg_color = True
	arg_xdr = False
	enable_fqdn = not args.NoDns
	
	arg_port_xdr = 3004
	arg_crawl = True
	
	defaultDict = {'hosts':arg_machines[0], 'xdr':arg_xdr, 'xdrPort':arg_port_xdr, \
						'color':arg_color, 'crawl':arg_crawl, 'port':arg_port, 'fqdn':enable_fqdn, \
						'##  Note  ##':'##### Please edit [main] section , this is a sample #####' 
				}
	
	# get config from the configfile and also set the [DEFAULT]section
	config = ConfigParser.SafeConfigParser(defaultDict) 
	
	readlist = config.read(configfile)
	if len(readlist) <= 0:
		print "Cannot read config file", configfile
	else:
		# Orginal list of hosts which user entered, can be the seed ip or comma separated list. 
		# DO NOT use this for the historical list of ip's in cluster.
		try:
			arg_port = config.getint('main', 'port')
		except:
			print "Adding port to config file"

		org_hosts = filter(None, config.get('main', 'hosts').split(','))
		
		if len(org_hosts) < 1:
			print "Missing seed host information in", configfile
			printusage(configfile)
			sys.exit(-1)
		else:
			# If org_hosts is found then check if the port exists, if it does not exists append :port
			arg_machines = []
			for x in org_hosts:
				if x.find(":") != -1:
					arg_machines.append(x)
				else:
					arg_machines.append(x + ":" + str(arg_port))
	
		# Now check if you have [donotedit] section from last run if you have it , use it only if the
		# IP from org_hosts exists in this list.
		
		if config.has_section('donotedit'):
			auto_hosts = filter(None, config.get('donotedit', 'hosts').split(','))
			for x in arg_machines:
				if x in auto_hosts:
					arg_machines = auto_hosts
					
		
		arg_xdr = config.get('main', 'xdr')
		if arg_xdr == "True" or arg_xdr == "true":
			arg_xdr = True
		else:
			arg_xdr = False
		arg_port_xdr = config.getint('main', 'xdrPort')

		arg_color = config.get('main', 'color')
		if arg_color == "True" or arg_color == "true":
			arg_color = True
		else:
			arg_color = False
		
		arg_crawl = config.get('main', 'crawl')
		if arg_crawl == "True" or arg_crawl == "true":
			arg_crawl = True
		else:
			arg_crawl = False
		
		try:
			enable_fqdn = config.get('main', 'fqdn')
			if enable_fqdn == "True" or enable_fqdn == "true":
				enable_fqdn = True
			else:
				enable_fqdn = False
			#override config by param. Above code becomes redundant
			enable_fqdn = not args.NoDns
		except:
			print ""
		
	if len(hosts_override) > 0:
		arg_machines = hosts_override
		# if override is used, Write back to the configuration so it uses this next time.
		org_hosts = arg_machines

	want_write_config = True
	if len(readlist) <= 0:
		# this is the first user invocation, need to create the initial config file regardless of situation
		write_config_rc = write_config(arg_machines)
		if write_config_rc:
			print "Created initial config file", configfile
		else:
			# if the write file here fail, no point to update it later
			want_write_config = False

	if arg_crawl:
		try:
			result = cluster_crawler(arg_machines)
			arg_machines = result['arg_machines']
		except:
			print "Unable to crawl cluster, Try crawl=False and enter all node IPs"

		# We need to continue even if no hosts identified in order to collect system data
		if 0 == len(arg_machines):
			print '***error* Failed to connect to any hosts.'
			print '		  Please edit the', configfile, 'file with the correct hosts parameter in the [main] section'
			print '		  or provide host connection host:port info via the -h parameter'
			return -1
		else:
			if sys.stdout.isatty():
				print len(arg_machines), 'hosts in cluster:', ','.join(arg_machines)
			# Call modifyHosts to replace DNS name by IP
			modifyHosts(org_hosts)

	else:
		for mcname in arg_machines:
			state[mcname] = 'n/a'
		want_write_config = False
		print 'Auto host discovery by crawler is disabled. To turn on, set crawl=True in the [main] section in', configfile
			
	# get namespaces from the any active node
	namespaces = None
	for m in arg_machines:
		namespaces = cinfo(m, "namespaces")
		if namespaces == -1 or namespaces == None :
			continue
		else:
			break
		
	if arg_machines == None:
		print '***error*** Cannot connect to any of the known machines:', ','.join(arg_machines)
		print '			Please edit the', configfile, 'file with the correct hosts parameter in the [main] section'
		print '			or provide host connection host:port info via the -h parameter'
		sys.exit(-1)

	if namespaces == None:
		print " No Namespaces received from servers, trying to get data from:" + configfile
		arg_namespace = filter(None, config.get('main', 'namespaces').split(','))
		if len(arg_namespace) < 1:
			print"***Error*** Unable to get namespace"
			sys.exit(-1)
	else:
		arg_namespace = namespaces.split(';')

	# write configuration back in the file. Must be done after crawl to pick up namespace info
	if want_write_config:
		write_config_rc = write_config(org_hosts)
		# if write_config_rc:
		# 	print "Updated configure file", configfile


# 	
# Persist the updated config info back to the file file
#
def write_config(org_hosts):
	'''Write the updated config back to the file'''

	global hosts, arg_namespace, arg_xdr, arg_port_xdr, arg_color, config, arg_port
	
	# write config into the configfile
	if not(config.has_section('main')):
		config.add_section('main')
		
	if not(config.has_section('donotedit')):
		config.add_section('donotedit')
	
	hoststr = ''
	need_comma = False
	for host in org_hosts:
		if need_comma:
			hoststr += ','
		hoststr += host
		need_comma = True
	
	if len(hoststr) > 0:
		config.set('main', 'hosts', hoststr)

	config.set('main', 'port', str(arg_port))
			
	config.set('main', 'xdr', str(arg_xdr))
	config.set('main', 'xdrPort', str(arg_port_xdr))

	config.set('main', 'color', str(arg_color))	
	config.set('main', 'crawl', str(arg_crawl))
	config.set('main', 'fqdn', str(enable_fqdn))
	
	hoststr = ''
	need_comma = False
	for host in hosts:
		if need_comma:
			hoststr += ','
		hoststr += host
		need_comma = True
	
	if len(hoststr) > 0:
		config.set('donotedit', 'hosts', hoststr)
	
	namestr = ''
	need_comma = False
	for namespace in arg_namespace:
		if need_comma:
			namestr += ','
		namestr += namespace
		need_comma = True
	if len(namestr) > 0:
		config.set('donotedit', 'namespaces', namestr + '	# Line is auto generated. Do not edit.')
	
	
	try:
		File = open(configfile, 'wb')
		try:
			config.write(File)
		finally:
			File.close()
	except IOError:
		print "*** warning: Cannot open", configfile, "for write ***"
		return False

	return True
	
#
# Display text in the selected color (for alerts, etc)
#
def color_text(text, color_name, bold=False):
	'''Display colored texts'''
	if sys.stdout.isatty():
		if color_name in COLORS:
			return '\033[' + str(int(bold)) + ';' + str(COLORS.index(color_name) + 90) + 'm' + str(text) + '\033[0m'
		elif color_name == 'BOLD':
			return '\033[1m' + str(text) + '\033[0m'
		sys.stderr.write('ERROR: "' + str(color_name) + '" is not a valid color.\n')
		sys.stderr.write('VALID COLORS: ' + str(COLORS) + '.\n')
	else:
		return str(text)
	
#
# Compare 2 items and print them out in appropriate colors based on the comparison result
# 	
def comparison(lhs, rhs=None, sign=None):
	'''Do simple comparison and return colored text'''
	if arg_color == True:
		if sign == '<':
			if lhs < rhs:
				return color_text(lhs, 'RED', True)
			else:
				return color_text(lhs, 'BLACK')
		elif sign == '>':
			if lhs > rhs:
				return color_text(lhs, 'RED', True)
			else:
				return color_text(lhs, 'BLACK')
		elif sign == '==':
			if lhs == rhs:
				return color_text(lhs, 'BLACK')
			else:
				return color_text(lhs, 'RED', True)
		elif sign == '!=':
			if lhs != rhs:
				return color_text(lhs, 'BLACK')
			else:
				return color_text(lhs, 'RED', True)	
		elif rhs == 'header':
			return color_text(lhs, 'BLACK', True)
		else:
			return lhs
	else:
		return lhs

#
# pprint_table accepts list of lists as input, 1st list should be the header and all columns 
# should be the same 
# Credit: table code is from http://ginstrom.com/scribbles/2007/09/04/pretty-printing-a-table-in-python/
#
# If language pack supporting locale is not installed, locale unset will fail. Hence put this in try
# Since locale is anyways not set/installed, and we are trying to unset it, this failure will not break pretty tables.
# AER-1660
try:
	locale.setlocale(locale.LC_NUMERIC, "")
except:
	pass
def format_num(num):
	"""Format a number according to given places.
	Adds commas, etc. Will truncate floats into ints!"""

	try:
		inum = int(num)
		return locale.format("%.*f", (0, inum), True)

	except (ValueError, TypeError):
		return str(num)
	
def get_max_width(table, index):
	"""Get the maximum width of the given column index"""
	return max([len(format_num(row[index])) for row in table]) 

def pprint_table(out, table):
	"""Prints out a table of data, padded for alignment
	@param out: Output stream (file-like object)
	@param table: The table to print. A list of lists.
	Each row must have the same number of columns. """
	col_paddings = []

	for i in range(len(table[0])):
		col_paddings.append(get_max_width(table, i))

	for row in table:
		# left col
		print >> out, row[0].ljust(col_paddings[0] + 1),
		# rest of the cols
		for i in range(1, len(row)):
			col = format_num(row[i]).rjust(col_paddings[i] + 2)
			print >> out, col,
		print >> out 


####
# utility function to Split ip port and return IP and port.
####
def split_ip_port(machine_port):
	
	try:
		# being nice if port is not there use default  port and ip passed in
		if machine_port.find(":") == -1:
			machine = machine_port
			port = arg_port
		else:
			# machine_port is in the format of 127.0.0.1:3000
			machine, port = machine_port.split(':')
		return machine, port
	except (ValueError):
		return -1, -1

####
# return a dict either containing all the info received or just the requested statistics
####
def cinfo(machine_port, arg_value):
	'''asinfo function equivalent'''

	# Handling cases where if the service list exports multiple IP's ,
	# Then split the ip's semi-colon separated and then just get data from the first IP.
	# Possibly might be some issues up the chain, when multiple IP's are exported.
	
	try:
		machine, port = split_ip_port(machine_port)
		if machine == -1:
			if machine_port.find(";") != -1:
				multiple_interfaces_on_node = machine_port.split(';')
				for node in multiple_interfaces_on_node:
					machine, port = split_ip_port(node)
					if machine != -1:
						break
				if machine == -1:
					return (-1)
			else:
				return (-1)
			
	except Exception, i:
		print "Encountered Exception: ", i


	try:
		
		try:
			r = citrusleaf.citrusleaf_info(machine, port, arg_value, user, password)
		except Exception, i:
			# print "Trying local asinfo logic"
			r = citrusleaf_info(machine, port, arg_value)
						
		# Should i return negative answers/rejections?
		  
		if r == -1:
			print 'request to ', machine, ':', port, ' returned error'
			status[str(machine) + ':' + str(port)] = False
		if r == None:
			print 'request to ', machine, ":", port, " returned no data"
			status[str(machine) + ':' + str(port)] = False
		if type(r) == types.StringType:
			status[str(machine) + ':' + str(port)] = True
			return(r)
		if type(r) == types.DictType:
			status[str(machine) + ':' + str(port)] = True
			return(r)
	
	except Exception, i:
		print 'request to ', machine_port, "for" , arg_value , " encountered exception: ", i
		return(r)	

#####
# Returns a dict of the table in format tableid={key:['value1','value2'].....}
# key is normally human readable header
# value is keys received from statistics/namespace or cinfo. this can be a list for different versions
# This is basically used when we dynamically print table, If table exists it will return that,
# if not , it will set default and return default dict.
#####

def get_table_header(tableid):

	# All Table headers should always have key "lhsh" which means , left hand side header 

	restore_default = False
	if tableid == 'Node' or tableid == 'Namespace' or tableid == 'XDR' or tableid == 'SETS' or tableid == 'restore': 
		if tableid == 'restore':
			restore_default = True
		if 'Node' not in persistent_dict or restore_default:
			
			persistent_dict['Node'] = {'ip:port':['mcname'], 'Node ID':['nodeid'], 'Principal ID':['paxos_principal'], 'Cluster Size':['cluster_size'],
						'Replicated Objects':['replicated_objects'], 'Free Disk pct':['free-pct-disk'],
						'Free Mem pct':['free-pct-memory'], 'Migrates':['migrates'],
						'Build':['nodebuild'], 'Sys Free Mem':['system_free_mem_pct'], 'Cluster Visibility':['state'],
						'lhsh':'ip:port'}
		if 'Namespace' not in persistent_dict or restore_default:
			persistent_dict['Namespace'] = {'ip/namespace':['mcnamens'], 'Master Objects':['unique_objects'], 'Used Mem %':['used-pct-memory'],
							'Used Disk %':['used-pct-disk'], 'Used Mem':['used-bytes-memory'],
							'Used Disk':['used-bytes-disk'], 'Avail Pct':['available_pct'], 'Evicted Objects':['evicted-objects'],
							'Repl Factor':['repl-factor'], 'hwm Disk':['high-water-disk-pct', 'high-water-disk'],
							'hwm Mem':['high-water-memory-pct', 'high-water-memory'], 'Stop Writes':['stop-writes'],
							'lhsh':'ip/namespace'}
		if 'XDR' not in persistent_dict or restore_default:
			persistent_dict['XDR'] = {'Nodes':['mcname'], 'Build':['nodebuild'], 'Req Outstanding':['stat_recs_outstanding'],
						'Lag secs':['timediff_lastship_cur_secs'], 'Bytes Shipped':['esmt-bytes-shipped'],
						'Req. Relog':['stat_recs_relogged'], 'Req. Shipped':['stat_recs_shipped'],
						'Free dlog':['free-dlog-pct'], 'Uptime':['xds-uptime', 'xdr-uptime'],
						'Throughput':['cur_throughput'], 'lhsh':'Nodes'}

		if 'SETS' not in persistent_dict or restore_default:
			persistent_dict['SETS'] = {'ip/ns:set':['mcnamesets'], 'Objects':['n_objects'],
									'Stop Writes Count':['set-stop-write-count'], 'Evict hwm':['set-evict-hwm-count'],
									'Delete Sets':['set-delete'], 'lhsh':'ip/ns:set'}
		if 'SINDEX' not in persistent_dict or restore_default:
			persistent_dict['SINDEX'] = {'ip/ns:set':['mcnamesets'], 'Secondary Index':['indexname'],
									'Number of Bins':['num_bins'], 'Bins':['bins'], 'Bin Type':['type'],
									'Sync State':['sync_state'], 'State':['state'], 'lhsh':'ip/ns:set'}
		
	if restore_default:
		return True
	elif persistent_dict[tableid]:
		return persistent_dict[tableid]
	else:
		return (-1)

#####
# TODO: Use this function to create rules dynamically, 
# Currently this returns set of rules to change color of particular cells based on rules
# Format for rules is 
# ruleid:{value_from_keylookup_from_header:['lhs','rhs','sign'],...}
# lhs can be 'self' or True or False (if you expect all values to be True, enter true. so, all false
# columns will be red and vise versa)
# DEPRECATED: rhs can be a function (func:) which returns some value or any other key from statistics 
# or namespace
# if rhs contains a list where the first element is 'func' the second element should be a function and the remaining
# elements can be arguments to that function
#####
	
def get_evaluatorDict(ruleid):
	if 'rules' not in persistent_dict:
		persistent_dict['rules'] = {'Node': {'cluster_size':['self', ['func', get_active_nodes], '=='],
                                                    'free-pct-disk':['self', '30', '<'], 'free-pct-memory':['self', '30', '<'],
                                                    'state':[True], 'paxos_principal':['self', ['func', get_paxos_principal], '==']},
                                           'Namespace' : {'used-pct-disk':['self', 'high-water-disk-pct', '>'],
                                                          'used-pct-memory':['self', 'high-water-memory-pct', '>'],
                                                          'stop-writes':[False]},
                                           'XDR': {'timediff_lastship_cur_secs':['self', '120', '>'],
                                                   'stat_recs_outstanding':['self', '100000', '>']}
						}
	
	try:
		
		return persistent_dict['rules'][ruleid]

	except:
		return (-1)

#####
# Utility function to check string is int or float.
#####	

def checkint(value):
	try:
		int(value)
		return True
	except:
		return False
	
def checkfloat(value):
	try:
		float(value)
		return True
	except:
		return False

#####
# Utlity function to check if its int or float or string and send back right values.
#####

def checkandreturn(value):
	if checkint(value):
		return int(value)
	elif checkfloat(value):
		return float(value)
	else:
		return value
#####
# Check if any cell has exceeded thresholds and color it appropriately.
# 
#####

def set_cell_color(cellKey, cellValue, ruleid, rowdict, header=False):
	try:	
		persistent_dict['rules'][ruleid]
		# print persistent_dict['rules'][ruleid]
	except:
		if get_evaluatorDict(ruleid) == -1:
			return cellKey, cellValue

	for key, value in persistent_dict['rules'][ruleid].iteritems():
		# key is the key to compare will cellValue, if match then compare.
		# value is a list [lhs,rhs,sign] or just [True/False]
		if header:
			for i in cellValue:
				if i == key:
					if value[0] == 'self':
						cellKey = comparison(cellKey, 'header')
					else:
						cellKey = comparison(cellKey, 'header')
	
		elif key == cellKey:
			if value[0] == 'self':
				lhs = checkandreturn(cellValue)
			elif value[0] == True:
				lhs = checkandreturn(cellValue)
				rhs = 'true'
				sign = '=='
			else:
				lhs = checkandreturn(cellValue)
				rhs = 'false'
				sign = '=='
	
			# if value can be just 1 if its Flag check or greater than it,
			if len(value) > 1:
				# Check if its a function or int or float or another key to compare againt.
				if type(value[1]) is list:
					val_list = value[1]
					if(val_list[0] == "func"):
						func = val_list[1]
						args = val_list[2:]
						j = func(*args)
						rhs = checkandreturn(j)
				elif value[1].startswith("func:"):  # deprecated, doesn't allow args
					i = value[1].split(":")
					# compatible with prior versions, should probably be removed from
					# persistent dict
					if i[1] == 'get_active_node()' or i[1] == 'get_active_nodes()': 
						j = get_active_nodes()
						rhs = checkandreturn(j)
				else:
					# # This should be another keylookup
					try:
						localcheck = value[1]
						localcheckstrip = localcheck[:-2]
						if checkint(localcheck) or checkfloat(localcheck):
							rhs = checkandreturn(localcheck)
						elif checkint(localcheckstrip) or checkfloat(localcheckstrip) :
							rhs = checkandreturn(localcheckstrip)
						else:
							rhs = checkandreturn(rowdict[value[1]])							
					except Exception, i:
						rhs = None
						
				sign = value[2]
			
			if lhs == 'n/a':
				rhs = 'n/a'
				sign = '=='
			cellValue = comparison(lhs, rhs, sign)					
	
	return cellKey, cellValue

#####
# Utility function to return either a list of active nodes or count of active nodes.
#####

def get_active_nodes(act_list=False, both=False):
	active_count = 0
	active_list = []
	for mcname in arg_machines:
		if mcname in status:
			active_count += 1
			active_list.append(mcname)
	if both:
		return active_list, active_count
	
	if act_list:
		return active_list
	else:
		return active_count

#####
# Utility function to determine which node should be principal
#####
def get_paxos_principal():
	try:
		t = time()
		last_principal, last_expire = get_paxos_principal.last_run
		if t < last_expire:
			return last_principal
		nodes = get_active_nodes(True)
		principal = max(map(lambda node: cinfo(node, "node"), nodes))
		expire = t + 1.0
		get_paxos_principal.last_run = (principal, expire)
		return principal
	except Exception, i:
		print "Caught Exception : ", i
get_paxos_principal.last_run = ("", 0.0)


####
# Add new node to list in runtime. After detecting cluster state change
####

def add_new_node(ipPort):
	
	services = cinfo(ipPort, 'services')
	if services == -1 or services == None:
		return False
	elif ipPort not in arg_machines:
		try:
			arg_machines.append(ipPort.strip())
			find_and_add_fqdn(ipPort)
			update_cluster_visibility(ipPort, parse_services(services))
			return True
		except Exception, i :
			print "Caught Exception : ", i
	return False
# ##
# Accepts "tableid", which is basically table to print...... 
# "valueDict" is basically dict in format {tableid:'Node', nodeid: {Dict of all elements from this node},nodeid2:{}}
# For namespace format is {tableid:'Namespace','namespaces':[list of namespace], nodeid:{ns1:{},ns2:{}}}
# note: valueDict: should always have mcname key in format ip:port
# "nodeidlist is list of all nodeid in the cluster
# ##

def create_table_list(valueDict):
	# get headers for the table to be created..
	headerDict = get_table_header(valueDict['tableid'])
	
	if headerDict == -1:
		return (-1)
	# get the value for the left hand side header/column
	lhsh = headerDict[headerDict['lhsh']]

	# Final list of lists to be returned to pprint_table format.
	headerForPrinting = []
	dataForPrinting = []
	
	# These are the keys from headerDict which need to be printed from valueDict.
	keysInvalueDict = [lhsh]	
	
	# Human readable header
	readableHeader = [[headerDict['lhsh']]]

	# Count the number of rows to be created for header
	headerRows = 1
	for headerKey, headerValue in headerDict.iteritems():	

		x = 0
		for key in headerKey.split(" "):
			x += 1
			if headerRows < x:
				headerRows = x

	# As the dict is not ordered! (for comptability with python 2.4!!)
	# Convert dict to a list of tuples, and then sort it. This should not change
	# when run under watch!	
	sortedHeaderList = headerDict.items()
	sortedHeaderList.sort(key=lambda tup: tup[0])

	temp_counter = 1
	
	# Call it kk because its a hack!! to make things line up.

	for kk in sortedHeaderList:
		headerKey = kk[0]
		headerValue = kk[1]
		if headerKey == headerDict['lhsh'] or headerKey == 'lhsh':
			continue
		else:
			try:
				if persistent_dict['sort'].has_key(valueDict['tableid']):
					if headerKey == persistent_dict['sort'][valueDict['tableid']][0]:
						sorting_by = temp_counter
						order_by = persistent_dict['sort'][valueDict['tableid']][1]
			except Exception, i:
				print "Cannot get sorting order", i

			# Get human readble header and keys for dict in valueDict.
			keysInvalueDict.append(headerValue)

			# Color messes up format if row is null for the column colored/bolded.
			# Hence finding null and replacing by '.' so we will bold it and maintain format
			# Don't enter . when color is disabled:
			if arg_color:
				while len(headerKey.split(" ")) < headerRows:
					headerKey = headerKey + " " + "."
					
			templist = []	
			for key in headerKey.split(" "):
				headerKey, headerValue = set_cell_color(key, headerValue, valueDict['tableid'], headerDict, True)
				templist.append(headerKey)
			readableHeader.append(templist)
		temp_counter += 1

	# Count number of headers to make, separated by space.

	for i in range(0, headerRows):
		temp = []
		for j in readableHeader:
			try:
				temp.append(operator.itemgetter(i)(j))
			except:
				temp.append(" ")

		if len(temp) > 0:
			# print (len(temp),temp)
			headerForPrinting.append(temp)
		else:
			continue

	# Create list of lists which can be fed into pprint_table for beautiful display.
	for num, mcname in enumerate(persistent_dict['ipNodeMap'].keys()):
		
		try:
			tempListofDict = []
			
			# For special tables like Namespace and sets , we need to flatten multi-level dicts
			# to row of dicts for eachnodes/namespaces/sets.
			table_type = valueDict['tableid']
			if persistent_dict['tables'][table_type] == 'namespace':
				for ns in valueDict[persistent_dict['ipNodeMap'][mcname]].get('namespaces'):
					tempListofDict.append(valueDict[persistent_dict['ipNodeMap'][mcname]].get(ns))
			elif persistent_dict['tables'][table_type] == 'sets':
				for ns in valueDict[persistent_dict['ipNodeMap'][mcname]].get('namespaces'):
					for setname in valueDict[persistent_dict['ipNodeMap'][mcname]][ns].get('SetNames'):
						tempListofDict.append(valueDict[persistent_dict['ipNodeMap'][mcname]][ns].get(setname))	
			elif persistent_dict['tables'][table_type] == 'setssummary':
				for ns in valueDict[persistent_dict['ipNodeMap'][mcname]].get('namespaces'):
					tempListofDict.append(valueDict[persistent_dict['ipNodeMap'][mcname]].get(ns))
			elif persistent_dict['tables'][table_type] == 'sindex':
				for index in valueDict[persistent_dict['ipNodeMap'][mcname]].get('indexes'):
					tempListofDict.append(valueDict[persistent_dict['ipNodeMap'][mcname]].get(index))
			else:
				try:
					# print type(valueDict[persistent_dict['ipNodeMap'][mcname]])
					tempListofDict.append(valueDict[persistent_dict['ipNodeMap'][mcname]])
				except:
					continue

			for rowdict in tempListofDict:
				noderow = []
				for keyList in keysInvalueDict:
					valueStr = ""
					for key in keyList:
						try:
							try:
								valueStr = rowdict[key]
								if isinstance(valueStr, list):
									valueStr = ','.join(valueStr)
									
								key, valueStr = set_cell_color(key, valueStr, valueDict['tableid'], rowdict)
								break
							except KeyError, inst:
								key, valueStr = set_cell_color(key, "n/a", valueDict['tableid'], rowdict) 
								continue
						except KeyError , inst:
							print "Error getting :", inst
					noderow.append(valueStr)
				dataForPrinting.append(noderow)

		except:
			# Become super resilient !!! if info timesout still keep running.... we are fault tolerant
			continue
		
	# Sort the table based on default rule or rule created by do_sorttable.
	try:
		if persistent_dict['sort'].has_key(valueDict['tableid']):
			dataForPrinting.sort(key=lambda x: x[sorting_by], reverse=order_by)
	except Exception, i:
		print "Sorting by IP, in Ascending order: "
		dataForPrinting.sort(key=lambda x: x[0], reverse=False)
		
	tableForPrinting = headerForPrinting + dataForPrinting

	return tableForPrinting, len(dataForPrinting)


#####
# Convert bytes to human readable format
#####

def bytes2human(n, format="%(value).2f%(symbol)s"):
	
	# print n
	"""
    >>> bytes2human(10000)
    '9K'
    >>> bytes2human(100001221)
    '95M'
    """
	symbols = (' B', ' K', ' M', ' G', ' T', ' P', ' E', ' Z', ' Y')
	prefix = {}
	for i, s in enumerate(symbols[1:]):
		prefix[s] = 1 << (i + 1) * 10
		# print prefix[s]
	for symbol in reversed(symbols[1:]):
		if n >= prefix[symbol]:
			value = float(n) / float(prefix[symbol])
			return format % locals()
	
	return format % dict(symbol=symbols[0], value=n)

####
# Return string of keys allowed for editing or creating tables, helper for humans
####

def allowed_keys(tabletype):
	return_str = ''
	if tabletype == 'node':
		for key in persistent_dict['STATISTICS']:
			return_str = key + ' , ' + return_str

	elif tabletype == 'namespace':
		for key in persistent_dict['NAMESPACE']:
			return_str = key + ' , ' + return_str
	elif tabletype == 'xdr':
		# Needed because XDR is disabled by default and it will not find the key
		try:
			for key in persistent_dict['XDRKEYS']:
				return_str = key + ' , ' + return_str
		except:
			return_str = 'No XDR keys in dict, run XDR once successfully'
	elif tabletype == 'sets':
		try:
			for key in persistent_dict['SETSKEYS']:
				return_str = key + ' , ' + return_str
		except:
			return_str = 'No SETS keys in dict, run SETS once sucessfully'
	elif tabletype == 'setssummary':
		try:
			for key in persistent_dict['SETSSUMMARY']:
				return_str = key + ' , ' + return_str
		except:
			return_str = 'No SETSSUMMARY keys in dict, run XDR once sucessfully'

	return return_str


#####
# Format data for each node so the function calling this can aggregate all values and
# display in rows/columns. This will basically get a dict from cinfo and will add new keys
# and merge all values in 1 dict.
#####

def parse_node(ipPort, nodeDetailsDict):
	
	global get_stat_file, get_stat_file2
	global xdr_stat_file, xdr_stat_file2

	mcnameDict = {}
	statistics = {}
	# arg_value=args[2]
	try:
		stat_file_fd = open(get_stat_file, "a")
	except:
		# let's try the alternative location
		get_stat_file = get_stat_file2
		stat_file_fd = open(get_stat_file, "a")

	try:
		stat_file_fd.write("\n------------------------------------------------------\n")
		stat_file_fd.write(ipPort)
		stat_file_fd.write("\n------------------------------------------------------\n")
		stat_file_fd.write(str(nodeDetailsDict))
		stat_file_fd.close()
	except:
		print "failed to write stat file: ", get_stat_file

	mcnameDict['nodebuild'] = nodeDetailsDict['build'] 
	
	if enable_fqdn:
		mcname = str(filter(None, ipPort.split(":"))[0])
		mcnameDict['mcname'] = host_alias[mcname]+':'+ipPort.split(":")[1]
	else:
		mcnameDict['mcname'] = ipPort

	for name, value in nodeDetailsDict.iteritems():
		if name == "node":
			mcnameDict['nodeid'] = value
		if name == "statistics":
			# Call String_to_dict function to convert a semi-colon separated string returned 
			# from asinfo to a Dictionary , So i can get individual property values 
			statistics = string_to_dict(value)

			persistent_dict['STATISTICS'] = statistics.keys()

			# mcnameDict['replicated_objects'] = "%.3f M" % (float(statistics['objects']) / 1000000.0)
			mcnameDict['replicated_objects'] = locale.format("%d", float(statistics['objects']), grouping=True)
			mcnameDict['migrates'] = "(" + statistics['migrate_progress_send'] + "," + statistics['migrate_progress_recv'] + ")"
			
			for key, value in statistics.iteritems():
				mcnameDict[key] = value

		if name == "services":
			try:
				update_cluster_visibility(ipPort, parse_services(value))
				mcnameDict['state'] = state[ipPort]
			except Exception, i:
				print i
			for node in filter(None, value.split(';')):
				if node not in state.keys():
					if not add_new_node(node):
						print "Could not add new node to list, Please restart asmonitor"
					else:
						print "Node added to Monitoring list"
	return mcnameDict

#####
# This fuction does cinfo calls to all the nodes and sends it to parse_node for processing and
# aggregates all per-node dicts for the entire cluster into 1 valueDict for Node.
#####

def get_node(tableid, machines):
		
	valueDict = {}
	
	for mcname in machines:
		try:
			node_info = cinfo(mcname, None)
			if node_info == -1 or node_info == None:  
				continue
			else:
				valueDict['tableid'] = tableid
				valueDict[node_info['node']] = parse_node(mcname, node_info)
				
				if 'ipNodeMap' not in persistent_dict:
					persistent_dict['ipNodeMap'] = {}
				
				persistent_dict['ipNodeMap'][mcname] = node_info['node']
		except:
			# Things fail so many times, just continue.
			continue
	return valueDict

#####
# Format per node namespace stat and return.
#####

def parse_ns(ipPort, ns, nameSpaceDict):
	'''get all namespace information and return in format get_namespace understands '''
	
	global get_stat_file, get_stat_file2

	nsDict = {}
	
	try:
		stat_file = open(get_stat_file, "a")
		try:
			stat_file.write("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n")
			stat_file.write(ipPort + "/" + ns)
			stat_file.write("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n")
			stat_file.write(str(nameSpaceDict))
		finally:
			stat_file.close()
	except IOError:
		# let's try the alternative location
		get_stat_file = get_stat_file2

	mcname = str(filter(None, ipPort.split(":"))[0])
	
	# unique_objects = "%.3f M" % (float(nameSpaceDict['master-objects']) / 1000000.0)
	unique_objects = "%s" % locale.format("%d", float(nameSpaceDict['master-objects']), grouping=True)
	
	try:
		for key, value in nameSpaceDict.iteritems():
			nsDict[key] = value
			
		if enable_fqdn:
			nsDict['mcnamens'] = host_alias[mcname] + "/" + ns
		else:
			nsDict['mcnamens'] = mcname + "/" + ns
			
		nsDict['mcname'] = ipPort
		nsDict['unique_objects'] = unique_objects
		if nameSpaceDict['type'] == 'device':
			nsDict['used-pct-disk'] = 100.0 - float(nameSpaceDict['free-pct-disk'])
			nsDict['used-bytes-disk'] = bytes2human(int(nameSpaceDict['used-bytes-disk']))
			
		nsDict['used-pct-memory'] = 100.0 - float(nameSpaceDict['free-pct-memory'])
		nsDict['used-bytes-memory'] = bytes2human(int(nameSpaceDict['used-bytes-memory']))
	
	except KeyError , inst:
		print "Error getting:", inst
	return nsDict

#####
# Aggregate all dicts for each namespace and each node into 1 large dict for displaying it.
#####

def get_namespace(tableid, machines):
	# obtain the server clocks here
	machine_epoch = {}
	base_time = time()

	valueDict = {}
		
	for ipPort in machines:	   
		# get all namespace information for this node
		try:
			nsDict = {}	
			cluster_objects = 0
			for ns in arg_namespace:
				name_info = cinfo(ipPort, "namespace/" + ns)
				if name_info == -1 or name_info == None:  
					continue
				else:
					
					# Call String_to_dict function to convert a semi-colon separated string 
					# returned from asinfo to a Dictionary , So i can get individual property values 
					
					nameSpaceDict = string_to_dict(name_info)
					
					persistent_dict['NAMESPACE'] = nameSpaceDict.keys()
					
					if nameSpaceDict['type'] == 'unknown':
						continue
					
					nsDict[ns] = parse_ns(ipPort, ns, nameSpaceDict)
					# need to adjust the machine time since we may have lots of machines,
					# and network could be slow for cinfo
					time_now = time()
					time_adjustment = int(time_now - base_time)
					machine_epoch[ipPort] = int(nameSpaceDict['current-time']) - time_adjustment
	
			valueDict[persistent_dict['ipNodeMap'][ipPort]] = nsDict
			valueDict[persistent_dict['ipNodeMap'][ipPort]]['numNS'] = len(arg_namespace)
			valueDict['tableid'] = tableid
			valueDict[persistent_dict['ipNodeMap'][ipPort]]['namespaces'] = arg_namespace
		except:
			# Continue on Fails, We are resillient !
			continue
	
	
	try:
		for ns in arg_namespace:
			cluster_objects = 0
			for ipPort in machines:
				try:
					repl_factor = int(valueDict[persistent_dict['ipNodeMap'][ipPort]][ns]['repl-factor'])
					cluster_objects = cluster_objects + \
					int(valueDict[persistent_dict['ipNodeMap'][ipPort]][ns]['master-objects'])
				except:
					continue
	
			print "Total (unique) objects in cluster for " + ns + \
					" : " + locale.format("%d", float(cluster_objects), grouping=True)
					# " : %.3f M" % (float(cluster_objects) / 1000000.0)

                print "Note: Total (unique) objects is an under estimate if migrations are in progress."
		print "\n"
	except Exception, e:
		print "cannot get sum of Objects from Namespace"
		print e

# cluster_objects=float(nsDict[ns]['objects'])/int(nsDict[ns]['repl-factor'])

	return valueDict

#####
# For each sets row, so that each set row has, information needed 
# eg of returned value: 
# {'SetNames': ['KK', 'KK2'], 'NoOfSets': 2, 'mcname': '192.168.105.70:3000', 
# 'KK': {'evict-hwm-count': '0', 'stop-write-count': '0', 'mcnamesets': '192.168.105.70/usermap:KK',
# 'n_objects': '39138274'}, 'KK2': {'evict-hwm-count': '0', 'stop-write-count': '0', 
# 'mcnamesets': '192.168.105.70/usermap:KK2', 'n_objects': '801'}}
#####

def parse_sets(ipPort, ns, insetsDict):

	setsDict = {}
	try:
		mcname = str(filter(None, ipPort.split(":"))[0])
		
		if enable_fqdn:
			setsDict['mcnamens'] = host_alias[mcname] + "/" + ns
		else:
			setsDict['mcnamens'] = mcname + "/" + ns

		setsDict['SetNames'] = insetsDict.keys()
		setsDict['NoOfSets'] = len(insetsDict.keys())
		persistent_dict['SETSSUMMARY'] = ['NoOfSets']
		persistent_dict['SETSSUMMARY'].append('mcnamens')
		persistent_dict['SETSSUMMARY'].append('SetNames')
		
		for key, value in insetsDict.iteritems():
			setsDict[key] = value
			setsDict[key]['mcnamesets'] = mcname + "/" + ns + ":" + key
		setsDict['mcname'] = ipPort
	
	except KeyError , inst:
		print "Error getting:", inst
	except Exception , i:
		print "Got Exception in parsing set data: ", i 

	return setsDict

#####
# Aggregate data for all nodes in cluster, for each namespace and sets in those namespaces
# Returns in format: {'1F419DEC05CA0568': {'usermap':{data from parse_sets}}, nextnode:{}} 
#####

def get_sets(tableid, machines):

	valueDict = {}
	# List of namespaces with sets.
	l_namespaces = []
		
	for ipPort in machines:	   
		# get all sets information for this node
		try:
			setsNsDict = {}	
			# Go through each namespace, request for sets for each and format it for printing.
			for ns in arg_namespace:
				setsDict = {}
				sets_info = cinfo(ipPort, "sets/" + ns)
				if sets_info == -1 or sets_info == None or len(sets_info) == 0:  
					continue
				else:
					setList = filter(None, sets_info.split(';'))
					
					for perSet in setList:
						perSetDict = string_colon_to_dict(perSet)
						setName = perSetDict['set_name']
						if perSetDict['ns_name'] not in l_namespaces:
							l_namespaces.append(perSetDict['ns_name'])
						setsDict[setName] = perSetDict
				
					persistent_dict['SETSKEYS'] = perSetDict.keys()
					setsNsDict[ns] = parse_sets(ipPort, ns, setsDict)

			valueDict[persistent_dict['ipNodeMap'][ipPort]] = setsNsDict
			valueDict[persistent_dict['ipNodeMap'][ipPort]]['namespaces'] = l_namespaces
			valueDict['tableid'] = tableid

		except:
			# Continue on Fails, We are resillient !
			continue

	return valueDict
	
#####
# Get SIndexes
#####

def get_sindex(tableid, machines):

	valueDict = {}
	l_indexes = []

	for ipPort in machines:
		# get all sindex information for this node
		try:
			sindexDict = {}
			sindex_info = cinfo(ipPort, "sindex")

			if sindex_info == -1 or sindex_info == None or len(sindex_info) == 0:
				continue
			else:
				sindexList = filter(None, sindex_info.split(';'))

			for perSindex in sindexList:
				perSindexDict = string_colon_to_dict(perSindex)
				index = perSindexDict['indexname']
				sindexDict[index] = perSindexDict
				sindexDict[index]['mcnamesets'] = ipPort.split(":")[0] + "/" + \
											perSindexDict['ns'] + ":" + perSindexDict['set']
				if index not in l_indexes:
					l_indexes.append(index)

			valueDict[persistent_dict['ipNodeMap'][ipPort]] = sindexDict
			valueDict[persistent_dict['ipNodeMap'][ipPort]]['indexes'] = l_indexes
			valueDict['tableid'] = tableid

		except Exception, e:
			# Continue on Fails, We are resillient !
			print e
			continue

	return valueDict

#####
# Parse xdr info from all nodes. This is not used in community edition.
#####

def parse_xdr(ipPort, xdrDetailsDict):
	'''Get xdr info from all nodes and return in format get_xdr understands'''

	global xdr_stat_file, xdr_stat_file2

	xdrDict = {}
	try:
		xdr_stat_file_fd = open(xdr_stat_file, "a")
	except:
		# let's try the alternative location
		xdr_stat_file = xdr_stat_file2
		xdr_stat_file_fd = open(xdr_stat_file, "a")

	try:
		xdr_stat_file_fd.write("\n------------------------------------------------------\n")
		xdr_stat_file_fd.write(ipPort)
		xdr_stat_file_fd.write("\n------------------------------------------------------\n")
		xdr_stat_file_fd.write(str(xdrDetailsDict))
		xdr_stat_file_fd.close()
	except:
		print "failed to write xdr stat file:", xdr_stat_file

	# Get xdr node_info

	xdrDict['nodebuild'] = xdrDetailsDict['build'] 
	machine, port = ipPort.split(':')
	
	if enable_fqdn:
		xdrDict['mcname'] = str(host_alias[machine]) + ':' + str(arg_port_xdr)
	else:
		xdrDict['mcname'] = str(machine) + ':' + str(arg_port_xdr) 
	
	for name, value in xdrDetailsDict.iteritems():
		if name == "statistics":
			# Call String_to_dict function to convert a semi-colon separated string returned from 
			# asinfo to a Dictionary , So i can get individual property values 
			statistics = string_to_dict(value)
			persistent_dict['XDRKEYS'] = statistics.keys()
			# Add xdr info from each node
			
			for key, value in statistics.iteritems():
				xdrDict[key] = value
			xdrDict['esmt-bytes-shipped'] = bytes2human(int(statistics['esmt-bytes-shipped']))
	return xdrDict

#####
# Aggregate data for all nodes and return for printing.
#####
def get_xdr(tableid, machines):
	valueDict = {}
	ipxdrmap = {}
	valueDict['tableid'] = tableid
	for ipPort in machines:
		try:
			# print persistent_dict['ipNodeMap']
			# mcname is in the format of 127.0.0.1:3000
			machine, port = ipPort.split(':')
			xdrip = str(machine) + ':' + str(arg_port_xdr)
			# Just allowing 1 XDR per machine
			if ipxdrmap.has_key(xdrip):
				continue
			else:
				xdrDetailsDict = cinfo(xdrip, None)
			
			if xdrDetailsDict == -1 or xdrDetailsDict == None:  
				continue
			else:
				ipxdrmap[xdrip] = ipPort	  
				valueDict[persistent_dict['ipNodeMap'][ipPort]] = parse_xdr(ipPort, xdrDetailsDict)
		except Exception, i:
			print i
			continue
	return valueDict

#####
# Hist-dump common code, This returns a dict in following format , Which is later parsed to display table
# histDump={'ttd-user': {'10.10.120.71:3000': [(19968, '99455'), (33792, '9383'), (20480, '93158'),
# (20992, '86236'), (43520, '854')], '10.10.120.72:3000': [(19968, '95592'), (33792, '8991'), 
# (20480, '88901'), (43520, '866'), (20992, '82837')],...}, 'ttd-map': 
# {'10.10.120.71:3000': [(512, '267877973'), (50688, '0'), (50176, '0'), (49664, '0'), (49152, '0')],
# '10.10.120.72:3000': [(512, '262718785'), (50688, '0'), (50176, '0'), (49664, '0'), (49152, '0')], ..}}
# Accepts histName= Name of histogram
#         topX= Number of columns to display.
#         machines= list of nodes.
#         ns= Namespace
#####

def get_hist_dump(histName, topX, machines, ns):
	
	histDump = {}
	asinfoList = []
	
	# Get all namespaces and create info list.
	for namespace in ns:
		valueString = "hist-dump:ns=" + namespace + ";hist=" + histName
		asinfoList.append(valueString)
	
		
	for ipPort in machines:
		try:
			histDict = cinfo(ipPort, asinfoList)
			if histDict is None or histDict == -1:
				continue
		except:
			pass
		
		try:
			for key, value in histDict.iteritems():
				
				if (-1 != value.find('error-unknown-hist-name')):
					print histName + " is not available on Server"
					continue
				# tempNS is a dict in format {header:value} e.g. {512:200000,...}
				tempNS = {}
				# # Value is returned as "ttd-map:objsz=100,1,0,44628603,.......;"
				if (-1 != value) and (-1 == value.find('error')):
					string, ignore = value.split(';')
					
					if ignore:
						print "I did find something here , Why ??"
					
					namespace, string = string.split(':')
					
					hist, string = string.split('=')
					
					buckets, size, string = string.split(',', 2)
					
					if hist == 'objsz':
						size = int(size) * 128
					else:
						size = int(size)
					
					bucket = 0
					
					for val in string.split(','):
						if hist == 'objsz':
							headerString = str(bytes2human(bucket)) + " to;" + str(bytes2human(bucket + size)) + "   "
						elif hist == 'ttl' or hist == 'evict':
							headerString = str(bucket) + "sec to;" + str(bucket + size) + "sec   "
						tempNS[headerString] = int(val)
						bucket = bucket + size
					
					if namespace not in histDump:
						histDump[namespace] = {}
					
					# # Sort the dict in decending order by value, i.e. larger number first 
					histDump[namespace][ipPort] = sorted(tempNS.items(), key=lambda(k, v):(v, k), reverse=True)[:topX]					
		except:
			continue
	
	return histDump

			
def printusage(configfile):
	parser.print_help()
	print ""
	print " The '[main]' section in configuration file (default is " + configfile + ", auto generated after the first", myprocname , "invocation):"
	print "	  hosts = Seed-HostIP:port or List of HostIP:port's (comma separated, each in format of 127.0.0.1:3000)"
	print "	  namespaces = List of namespaces (auto generated. user should not edit)"
	print "	  crawl = True Find all nodes in cluster- Works only with internal IPs"
	print "	  xdr = False. Disabled by default. To enable'xdr=True'"
	print "	  xdrport=xdr monitor port"
	print " Note the configuration file is auto updated after each run"
	print ""


#######======================================================================================#######

class RunCommand(cmd.Cmd):

	# pretty text...
	# Confirm you are in real console
	if sys.stdout.isatty():
		bold = "\033[1m"
		reset = "\033[0;0m"
	else:
		bold = "\'"
		reset = "\'"
		
	_VERSION_ = "3.5.11"
	prompt = "Monitor> "
	name = "Aerospike Interactive Shell"
	known_histnames = ['reads', 'writes_master', 'proxy', 'writes_reply', 'udf', 'query']	

	intro = bold + name + ", version " + _VERSION_ + reset
	
	def __init__(self):
		cmd.Cmd.__init__(self)
		import readline
		try:
			readline.read_history_file(hist_file)
		except Exception, i:
			print "Creating History file, " , i
			readline.write_history_file(hist_file)
			
		parse_config(configfile, hosts_from_parm)
		persistent_dict['default'] = ['Node', 'Namespace', 'XDR']
		if 'tables' not in persistent_dict:
			persistent_dict['tables'] = {'Node':'node', 'Namespace':'namespace', 'XDR':'xdr'}
		if 'SETS' not in persistent_dict['tables']:
			persistent_dict['tables']['SETS'] = 'sets'
		if 'SINDEX' not in persistent_dict['tables']:
			persistent_dict['tables']['SINDEX'] = 'sindex'
		if 'sort' not in persistent_dict:
			persistent_dict['sort'] = {'Node':('Node id', True), 'Namespace':('Avail Pct', False), 'XDR':('Lag secs', True), \
										'SINDEX':('Secondary Index', False)}
	

	help = { 
		'info': 	{'usage' : bold + 'info' + reset + '[ <TABLENAME>]' + \
					 '\n\t [-h comma separated hostip:port list]' + \
					 '\n\t [-p default port. 3000 assumed if missing]' + \
					 '\n\t [ <tab><tab> to see all table names]'
					 '\n\t [ ' + bold + 'info' + reset + ' TABLENAME (Autocomplete by tab) ]',
				 'desc': 'Gets cluster information in tabular format, Show custom tables or defaults \n' 
				 		'\t Default tables are Node,Namespace and XDR'},
		'asinfo': 	{'usage' : bold + 'asinfo' + reset + \
					 '\n\t -v <Value to get or set>' + \
					 '\n\t [-h <comma separated hostip:port list>]' + \
					 '\n\t [-p default port. 3000 assumed if missing]',
				 'desc': 'Sets configuration for the values for the entire cluster, unless the -h parameter is given.'},
		'latency': 	{'usage' : bold + 'latency' + reset + \
					 '\n\t [-v <histogram name filter>]' + \
					 '\n\t [-h <comma separated hostip:port list>]' + \
					 '\n\t [-p default port. 3000 assumed if missing]' + \
					 '\n\t [-k histogram key (reads, writes_master, writes, writes_reply, proxy)]' + \
					 '\n\t [-d duraction_sec]' + \
					 '\n\t [-v value spec for filtering with ? and * wildcard expressions]' + \
					 '\n\t [-b number of seconds (before now) to look back to]' + \
					 '\n\t [-d duration, the number of seconds from start to search]' + \
					 '\n\t [-s slice_sec, i.e. interval in sec to analyze]' + \
					 '\n\t [-t flag (want throughput). If present, display the throughtputs data instead]' + \
					 '\n\t [-m flag (by machine). If present, display the output group by machine names also]' + \
					 '\n\t [-c flag (config). If present, run the histogram configuration step]',
				 'desc': 'Get the latency histogram data'},
		'createtable':{'usage':bold + 'createtable' + reset ,
					'desc':'Create custom tables in Node,Namespace,Sets or XDR sections.'},
		'edittable': {'usage': bold + 'edittable' + reset,
					'desc': 'Edit default or custom tables (can only delete or replace columns)'},
		'sorttable': {'usage': bold + 'sorttable' + reset,
					'desc': 'Sort a table by a column, Ascending or descending (color will be disabled)'},
		'add': 		{'usage' : bold + 'add' + reset + \
					 '\n\t -h <comma separated hostip:port list>' + \
					 '\n\t [-p default port. 3000 assumed if missing]',
				 'desc'  : 'Add a single or list of hostip:port\'s to the current list of monitored nodes'},
		'remove': 	{'usage' : bold + 'remove' + reset + \
					 '\n\t -h <comma separated hostip:port list>' + \
					 '\n\t [-p default port. 3000 assumed if missing]',
				 'desc'  : 'Remove a single or list of hostip:port\'s from the current list of monitored nodes'},
		'stat': 	{'usage' : bold + 'stat' + reset + ' [-v <Value to filter>]' + \
					 '\n\t [-h <comma separated hostip:port list>]' + \
					 '\n\t [-p default port. 3000 assumed if missing]' + \
					 '\n\t [-v value spec for filtering with ? and * wildcard expressions]',
				 'desc': 'Get the statistics data of the entire cluster, unless -h parameter is given'},
		'watch':	{'usage' : bold + 'watch' + reset + ' -n <seconds> <command>',
				 'desc'  : '\t' + bold + 'watch' + reset + ': run ' + bold + '<command>' + reset + ' repeatedly, displaying its output.' \
				 '\n\t\t' + bold + '<command>' + reset + ' Has to be one of the legal commands in this shell.'},
		'histdump':  {'usage' : bold + 'histdump' + reset + ' -v <histogram name>' + \
					 '\n\t [-h <comma separated hostip:port list>]' + \
					 '\n\t [-p default port. 3000 assumed if missing]' + \
					 '\n\t [-t <Display Top X, where X is a number> Default is to display top 5 columns]' + \
					 '\n\t [-n <namespace name> Default is to display all namespaces]' + \
					 '\n\t Note: For -v, allowed options are: \n\t\t' + bold + 'objsz' + reset + ' For Object size histogram' + \
					 '\n\t\t' + bold + 'ttl' + reset + ' For Time to Live histogram' + \
					 '\n\t\t' + bold + 'evict' + reset + ' For evict histogram',
					'desc'  : bold + 'histdump' + reset + 'Get hist-dump statistics from server and' + \
					'display top <number> of columns'
					},
		'printconfig': 	{'usage' : bold + 'printconfig' + reset + \
					 '\n\t [-h <comma separated hostip:port list>]' + \
					 '\n\t [-p default port. 3000 assumed if missing]' + \
					 '\n\t [-v value spec for filtering with ? and * wildcard expressions]',
				 'desc': 'Print out server config info (EXPERIMENTAL)'},
		'compareconfig': 	{'usage' : bold + 'compareconfig' + reset + \
					 '\n\t [-h <comma separated hostip:port list>]' + \
					 '\n\t [-p default port. 3000 assumed if missing]',
				 'desc': 'Compare server config for the entire cluster, Prints the values which are'\
				 ' different, Does not compare Namespace config'},
		'shell' : {'usage': bold + 'shell' + reset + ' command_to_os',
				'desc': 'Execute shell commands from asmonitor and display output (EXPERIMENTAL)'},
		'pmap' : {'usage': bold + 'pmap' + reset + ' namespace',
				'desc': 'Gets the cluster wide partition map (and alarm any disagreement)'},
		'exit' : 	{'usage': bold + 'exit' + reset ,
				 'desc' : 'Exit the monitor console'},
		}

	
	def do_help(self, line):
		''' print list of commands and their usage '''
		print '' + self.bold + '\n' + self.name + self.reset
		print '' + self.bold + 'Version ' + self._VERSION_ + self.reset + '\n' 
		print '' + self.bold + 'Commands:' + self.reset + '\n'
		keys = sorted(self.help)
		for cmd in keys:
			print '\t' + self.help[cmd]['usage']
			print '\t' + self.help[cmd]['desc'] + '\n'

	def do_EOF(self, line):
		import readline
		readline.write_history_file(hist_file)
		print "\nConfig files location : " + str(configfile)
		print "Folder used by " + myprocname + " for files : " + monitorPath + "/"
		return True
	def do_exit(self, line):
		import readline
		readline.write_history_file(hist_file)
		print "\nConfig files location : " + str(configfile)
		print "Folder used by " + myprocname + " for files : " + monitorPath + "/"
		return True

	# Run shell commands, Dont know if i will use it. or not, but might be useful,
	last_output = ''

	def do_shell(self, line):
		'''Run a shell command'''
		print "running shell command:", line
		output = os.popen(line).read()
		print output
		self.last_output = output

	def do_echo(self, line):
		"Print the input, replacing '$out' with the output of the last shell command"
		# Obviously not robust
		print line.replace('$out', self.last_output)

	# Overriding base class which repeats last command
	def emptyline(self):
		return False

	def _getargs(self, line, optstr):
		''' utility function - get arguments and options from the command line'''
		myargv = line.split()
		opts, args = getopt.gnu_getopt(myargv, optstr)
		return opts, args
		
	# Utility function to parse the same set of arguments that are used by several commands
	def _getconfigargs(self, line):
		''' utility function - parse standard arguments used when manipulating hosts'''
		opts, args = self._getargs(line, 'h:p:v:')
		hostport = None
		port = None
		value = None
			
		for o, a in opts :
			if o == '-h':
				hostport = a
			if o == '-p':
				port = a
			elif o == '-v':
				value = a
		
		diffPort = False
		
		# port has to be numeric. If not numeric, an exception will be raised
		if None != port:
			port = str(int(port))

		hplist = []
		if hostport != None:
			hplist = hostport.split(',')
		else:
			hplist = arg_machines
			diffPort = True

		hostportlist = []
		for hp in hplist:
			# look for the : to assume the port. If not there, add it
			try: 
				h, p = hp.split(':')
				
				if diffPort and port:
					hostportlist.append(h + ':' + port)
				else:
					hostportlist.append(hp)
			except Exception, ex:
				if None == port:
					hostportlist.append(hp + ':' + str(arg_port))
				else:
					hostportlist.append(hp + ':' + port)

		return hostportlist, value

	# Utility function to parse the arguments that are used by do_latency
	def _getconfigargs_latency(self, line):
		''' utility function - parse standard arguments used when manipulating hosts'''
		opts, args = self._getargs(line, 'h:p:v:k:b:d:s:tmc')
		hostport = None
		port = None
		value = None
		lat_hist = None
		lat_back = None
		lat_duration = None
		lat_slice = None
		
		want_conf = False
		want_thruput = False
		want_bymachine = False
			
		for o, a in opts :
			if o == '-h':
				hostport = a
			if o == '-p':
				port = a
			elif o == '-v':
				value = a
			elif o == '-k':
				lat_hist = a
			elif o == '-b':
				lat_back = a
			elif o == '-d':
				lat_duration = a
			elif o == '-s':
				lat_slice = a
			elif o == '-t':
				want_thruput = True
			elif o == '-m':
				want_bymachine = True
			elif o == '-c':
				want_conf = True

		# port has to be numeric. If not numeric, an exception will be raised
		if None != port:
			port = str(int(port))

		hplist = []
		if hostport != None:
			hplist = hostport.split(',')

		hostportlist = []
		for hp in hplist:
			# look for the : to assume the port. If not there, add it
			try: 
				h, p = hp.split(':')
				hostportlist.append(hp)
			except Exception, ex:
				if None == port:
					hostportlist.append(hp + ':' + str(arg_port))
				else:
					hostportlist.append(hp + ':' + port)

		return hostportlist, value, lat_hist, lat_back, lat_duration, lat_slice, want_thruput, want_bymachine, want_conf


	def do_asinfo(self, line, raw=False):
		''' Dynamically set configuration on the entire cluster or a particular node- 
				Regular telnet/asinfo commands can be used in -v'''
		try:
			hostportlist, value = self._getconfigargs(line)
			print line
		except (KeyboardInterrupt, SystemExit):
			raise
		except:
			print self.help['asinfo']['usage']
			return False
		
		if value == None:
			print self.help['asinfo']['usage']
			return False
		value = value.strip('\"\'')

		machines = []
		if 0 == len(hostportlist):
			machines = arg_machines
		else:
			machines = hostportlist

		for mcname in machines:
			try:
				if(raw == False):
					print str(mcname) + " returned : " + cinfo(mcname, value)
				else:
					print cinfo(mcname, value)
			except:
				print "Error getting data from node", mcname
		
	def do_add(self, line):
		try:
			hostportlist, value = self._getconfigargs(line)
			print line
		except (KeyboardInterrupt, SystemExit):
			raise
		except:
			print self.help['add']['usage']
			return False

		machines = []
		if 0 == len(hostportlist):
			print self.help['add']['usage']
			return False
		else:
			machines = hostportlist

		for mcname in machines:
			try:
				hostport = str(mcname)
				state[hostport] = 'manual'
				arg_machines.append(hostport)
				print 'Added node', hostport, 'to node list'
			except:
				print 'Cannot add node', mcname, 'to node list'
		
	def do_remove(self, line):
		try:
			hostportlist, value = self._getconfigargs(line)
			print line
		except (KeyboardInterrupt, SystemExit):
			raise
		except:
			print self.help['remove']['usage']
			return False

		machines = []
		if 0 == len(hostportlist):
			print self.help['remove']['usage']
			return False
		else:
			machines = hostportlist
			

		for mcname in machines:
			try:
				hostport = str(mcname)
				# state[host]='manual'
				arg_machines.remove(hostport)
				print 'Removed node', hostport, 'from node list'
			except:
				print 'Cannot remove node', mcname, 'from node list'
				return False

	def do_stat(self, line):
		try:
			hostportlist, value = self._getconfigargs(line)
		except (KeyboardInterrupt, SystemExit):
			raise
		except:
			print self.help['stat']['usage']
			return False

		return self._cinfo_table("statistics", hostportlist, value)

	def do_printconfig(self, line):
		try:
			hostportlist, value = self._getconfigargs(line)
		except (KeyboardInterrupt, SystemExit):
			raise
		except:
			print self.help['printconfig']['usage']
			return False

		return self._cinfo_table("get-config:", hostportlist, value)

	def _cinfo_table(self, qtext, hostportlist, value):
		''' Get server statistical information '''

		if value != None:
			# translate the ? and * to proper regex patterns
			value = value.replace('?', '.')
			value = value.replace('*', '.*?')
			pattern = re.compile(value)

		machines = []
		if 0 == len(hostportlist):
			machines = arg_machines
		else:
			machines = hostportlist
				
		out = sys.stdout
		for mcname in machines:
			try:
				s = cinfo(mcname, qtext)
				stat = string_to_dict(s)
				sorted_stat = sorted(stat.iteritems(), key=itemgetter(0), reverse=False)
				t = []

				if value == None:
					found_match = True
				else:
					found_match = False

				for i, v in enumerate(sorted_stat):
					if value == None:
						t.append(sorted_stat[i])
					else:
						if None != pattern.match(sorted_stat[i][0]):
							t.append(sorted_stat[i])
							found_match = True

				print '	====' + self.bold + str(mcname) + self.reset + '===='
				if found_match:
					pprint_table(out, t)
				else:
					print "No match"

			except:
				print "Error getting data from", mcname
				
	#####
	# Compare config for each node and show the config values which are different.
	# TODO: Add check if the node is in the same cluster
	#####	
	
	def do_compareconfig(self, line):
		global arg_xdr, arg_port_xdr

		try:
			hostportlist, value = self._getconfigargs(line)
		except (KeyboardInterrupt, SystemExit):
			raise
		except:
			print self.help['printconfig']['usage']
			return False	
		
		machines = []
		if 0 == len(hostportlist):
			machines = arg_machines
		else:
			machines = hostportlist	
		
		clusterList = []
		mode_mesh = False
		for mcname in machines:
			try:
				node_config = cinfo(mcname, 'get-config:')
				if node_config == -1 or node_config == None:
					continue
				else:
					node_dict = string_to_dict(node_config)
					if node_dict['heartbeat-mode'] == 'mesh':
						mode_mesh = True	
			except Exception, i:
				print " Me no likey comparing : ", i 

			if arg_xdr:
				try:
					machine, port = mcname.split(':')
					xdrip = str(machine) + ':' + str(arg_port_xdr)
					node_config = cinfo(xdrip, 'get-config:')
					if node_config == -1 or node_config == None:
						continue
					else:
						node_dict = dict(chain(node_dict.iteritems(), string_to_dict(node_config).iteritems()))
				except Exception, i:
					print " Me no likey comparing : ", i 

			clusterList.append(node_dict)

		# Convert a list of dicts into iterable sequence of sets.
		configSets = (set(j.items()) for j in clusterList)
		
		# Using reduce to apply set.intersection to find common values in each sets.
		commonConfig = reduce(set.intersection, configSets)


		# Now once we have the common configurations, We will subtract that which each config
		# to find different configurations, ideally should be None
		listToPrint = [['k']]
		restOfList = []
		headerList = ['IP']
		for i, k in enumerate(machines):
			diffInConfig = dict(set(clusterList[i].items()) - commonConfig)
			restOfList = [k]
			for keyConfig, keyValue in sorted(diffInConfig.items()):
				if mode_mesh and keyConfig == 'heartbeat-address':
					continue
				
				if keyConfig == 'internal-address' or keyConfig == 'external-address' or \
				keyConfig == 'service-address' or keyConfig == 'heartbeat-interface-address' or \
				keyConfig == 'access-address' :
					continue

				if keyConfig not in headerList:
					headerList.append(keyConfig)
					# Prior rows need n/a appended
					for entry in listToPrint:
						entry.append('n/a')
				
			for column in headerList:
				if column == 'IP':
					continue
				if column in diffInConfig:
					restOfList.append(diffInConfig[column])
				else:
					restOfList.append('n/a')

			listToPrint.append(restOfList)	
		
		listToPrint[0] = headerList	
		out = sys.stdout
		if len(restOfList) > 1:
			pprint_table(out, listToPrint)
		else:
			print "All configurations are same across the cluster"
		
	# 
	# Get cluster latency history data
	#
	def do_latency(self, line):
		''' Get server latency history data '''
		global arg_machines

		try:
			hostportlist, value, lat_hist, lat_back, lat_duration, lat_slice, want_thruput, want_bymachine, want_conf \
				 = self._getconfigargs_latency(line)
		except (KeyboardInterrupt, SystemExit):
			raise
		except:
			print self.help['latency']['usage']
			return False

		machines = []
		if 0 < len(hostportlist):
			machines = hostportlist
		else:
			machines = arg_machines

		if want_conf:
			self._configure_latency_histogram(arg_machines)
			# Exit if we configuring do not run display later.
			return False

		self._display_latency(machines, value, lat_hist, lat_back, lat_duration, lat_slice, want_thruput, want_bymachine)


	def _configure_latency_histogram(self, machines):
		''' kick off latency histograms on the server side according to the user's will '''

		# verify all the machines on list have latency history turned on.
		# if not, ask if user would like to turn it on
		errornottracking_pattern = re.compile('error-not-tracking')
		error_pattern = re.compile('error')
		latency_back_parm = 3600
		latency_slice_parm = 10
		latency_thresholds_parm = '1,2,4,8,64'
		latency_duration_parm = 0
		target_machines = []
		for mcname in sorted(machines):
			try:
				print '	====' + self.bold + str(mcname) + self.reset + '===='
				try:
					s = cinfo(mcname, "latency:")
				except:
					print 'Error getting data from', mcname
					continue
	
				# found error, not sure what error yet
				if None != error_pattern.match(s):
					# error is not error-not-tracking?
					if None == errornottracking_pattern.match(s):
						# try again in case of timing error, which presents itself as a simple "error"
						print "Sleeping for 1 sec and trying again :" + str(mcname)
						sleep(1)
						s = cinfo(mcname, "latency:")
	
				want_configure = False
				# confirmed error-not-tracking, ask user for permission to turn it on
				if None != errornottracking_pattern.match(s):
					print 'Latency histogram is not active on', mcname
					yn = raw_input('Configure latency histograms on this server (Y/n)? ')
					if '' == yn or 'y' == yn or 'Y' == yn:
						want_configure = True
					else:
						print  '*** skipping ', mcname
				# the host already is tracking latency history
				else:
					print 'Latency histogram is already active on ', mcname
					target_machines.append(mcname)
					yn = raw_input('Configure latency histogram on this server (N/y)?')
					if 'y' == yn or 'Y' == yn:
						want_configure = True
	
				if want_configure:
					latencystart_cmd = 'hist-track-start:';
	
					### hist ###
					yn = raw_input('Start only a particular histogram (response N if you want all histograms) (y/N)? ')
					if 'y' == yn:
						response = raw_input('Enter only one or enter to skip: ' + ','.join(self.known_histnames) + ': ')
						if not response == '':
							latencystart_cmd += 'hist=' + response + ';'
							print 'hist response=', response
	
					### back ###
					response = 'x';
					while (not response.isdigit()):
						response = raw_input('Choose histogram window size in sec (1 day=86400)(enter=' + str(latency_back_parm) + ')? ')
						if response == '':
							response = str(latency_back_parm)
					latency_back_parm = int(response)
					latencystart_cmd += 'back=' + str(latency_back_parm) + ';'
								
					### slice ###
					yn = raw_input('Specify tracking time slice in sec (default=10)(y/N)? ')
					if 'y' == yn:
						response = 'x'
						while (not response.isdigit()):
							response = raw_input('Enter history time slice in sec (enter=' + str(latency_slice_parm) + ')? ')
							if response == '':
								response = str(latency_slice_parm)
						latency_slice_parm = int(response)
						latencystart_cmd += 'slice=' + str(latency_slice_parm) + ';'
							
					### thresholds ###
					yn = raw_input('Specify thresholds in sec (default=1,8,64) (y/N)? ')
					if 'y' == yn:
						response = raw_input('Enter comma separated thresholds (must be power of 2) (enter=' + latency_thresholds_parm + ')? ')
						if response == '':
							response = latency_thresholds_parm
						latency_thresholds_parm = response
						latencystart_cmd += 'thresholds=' + latency_thresholds_parm + ';'
	
					s = cinfo(mcname, latencystart_cmd)
					if 'ok' == s:
						print 'Command accepted by server. Command sent:', latencystart_cmd
						target_machines.append(mcname)
					else:
						print 'Failed to configure latency histogram on', mcname
						print 'Command used = ', latencystart_cmd
						print 'Response got from server = ', s
			except Exception, i:
				# Catch exceptions on stuff returned from server,
				# print i
				continue


	def _display_latency(self, machlist, filter_str, lat_hist, lat_back, lat_duration, lat_slice, want_thruput, want_bymachine):
		''' print out the latency data nicely '''

		errornottracking_pattern = re.compile('error-not-tracking')
		error_pattern = re.compile('error')

		cmd = 'latency'
		if want_thruput:
			cmd = 'throughput'

		raw_histograms = {}
		raw_thruputs = {}

		skipped_machlist = []
		data_collected = False
		non_tracking_host_count = 0

		# organize the data by histogram names and by machines names
		for mcname in machlist:
			parm = '';
			if (None != lat_hist):
				parm += 'hist=' + lat_hist + ';'
			if (None != lat_back):
				parm += 'back=' + lat_back + ';'
			if (None != lat_duration):
				parm += 'durartion=' + lat_duration + ';'
			if (None != lat_slice):
				parm += 'slice=' + lat_slice + ';'

			final_cmd = cmd + ':' + parm

			# retrieve the raw histogram/throughput data from server. 
			s = ''
			try: 
				s = cinfo(mcname, final_cmd)
				if None != error_pattern.match(s):
					# try again if needed
					print "Sleeping for 1 sec and trying again to :" + str(mcname)
					sleep(1)
					s = cinfo(mcname, final_cmd)
					if None == s or 0 == len(s) or None != error_pattern.match(s):
						if None != s and 0 < len(s):
							# handle known error situations
							if s == 'error-bad-hist-name':
								print "Bad histogram name '" + lat_hist + "'. Must be one of: ", ','.join(self.known_histnames)
								return
							if None != errornottracking_pattern.match(s):
								non_tracking_host_count += 1
						skipped_machlist.append(mcname)
						# print "***error getting latency info from server***"
						# print mcname, 'returns =', s
						continue
			except:
				skipped_machlist.append(mcname)
				# print "***error getting latency info from server***"
				# print mcname, 'returns =', s
				continue
				
			data_collected = True
			raw_histograms[mcname] = s

		if 0 < len(skipped_machlist):
			print "Histograms are not available on: ", ','.join(skipped_machlist)

		if not data_collected:
			print 'No histogram data to display.'
			if 0 < non_tracking_host_count:
				print str(non_tracking_host_count) + " servers do not have histogram tracking on. Consider 'latency -c' to configure histograms on servers"
			return

		if filter_str != None:
			filter_str = filter_str.replace('?', '.');
			filter_str = filter_str.replace('*', '.');
			filter = re.compile(filter_str);

		# group the raw data by histogram names as well as by machine names
		by_histogram = {}
		for h in self.known_histnames:
			# entries in each histogram group are indexed by machine name
			by_histogram[h] = {}

		by_machine = {}
		for mcname in raw_histograms.keys():
			# entries in each machine group are indexed by histogram name
			by_machine[mcname] = {}

		# could have combined with the last loop, but this is logically clearer
		for mcname in raw_histograms.keys():
			s = raw_histograms[mcname]

			# the data string comes back as op1:lab1a,lab1b,...;data1a1,data1b1...;data1an,data1bn;data;op2:lab2a1,lab2b1,..;data2a1,data2b1..;data2a1,data2b1
			# i.e. for each histogram, the 1st chunk is the label, the 2nd, and possibly subsequent chunks, is the the data. This 
			# 	  pattern repeats when an op name is found (identified by a historgram name folloowed by the the colon : sign)
			# e.g.
			# 	reads:22:26:42-GMT,ops/sec,>1ms,>8ms,>64ms;22:26:52,0.0,0.00,0.00,0.00;22:27:02,0.0,0.00,0.00,0.00;22:27:12,0.0,0.00,0.00,0.00;22:27:22,0.0,0.00,0.00,0.00;
			# 22:27:32,0.0,0.00,0.00,0.00;22:27:42,0.0,0.00,0.00,0.00;22:27:52,0.0,0.00,0.00,0.00;22:28:02,0.0,0.00,0.00,0.00;22:28:12,0.0,0.00,0.00,0.00;
			# 22:28:22,0.0,0.00,0.00,0.00;22:28:32,0.0,0.00,0.00,0.00;22:28:42,0.0,0.00,0.00,0.00;22:28:52,0.0,0.00,0.00,0.00;22:29:02,0.0,0.00,0.00,0.00;
			# 22:29:12,0.0,0.00,0.00,0.00;22:29:22,0.0,0.00,0.00,0.00;22:29:32,0.0,0.00,0.00,0.00;22:29:42,0.0,0.00,0.00,0.00;22:29:52,0.0,0.00,0.00,0.00;
			# 22:30:02,0.0,0.00,0.00,0.00;22:30:12,0.0,0.00,0.00,0.00;22:30:22,0.0,0.00,0.00,0.00;22:30:32,0.0,0.00,0.00,0.00;22:30:42,0.0,0.00,0.00,0.00;
			# 22:30:52,0.0,0.00,0.00,0.00;22:31:02,0.0,0.00,0.00,0.00;22:31:12,0.0,0.00,0.00,0.00;22:31:22,0.0,0.00,0.00,0.00;22:31:32,0.0,0.00,0.00,0.00;
			# writes_master:22:26:42-GMT,ops/sec,>1ms,>8ms,>64ms;22:26:52,0.0,0.00,0.00,0.00;22:27:02,0.0,0.00,0.00,0.00;22:27:12,0.0,0.00,0.00,0.00;
			# 22:27:22,0.0,0.00,0.00,0.00;22:27:32,0.0,0.00,0.00,0.00;22:27:42,0.0,0.00,0.00,0.00;22:27:52,0.0,0.00,0.00,0.00;22:28:02,0.0,0.00,0.00,0.00;
			# 22:28:12,0.0,0.00,0.00,0.00;22:28:22,0.0,0.00,0.00,0.00;22:28:32,0.0,0.00,0.00,0.00;22:28:42,0.0,0.00,0.00,0.00;22:28:52,0.0,0.00,0.00,0.00;
			# 22:29:02,0.0,0.00,0.00,0.00;22:29:12,0.0,0.00,0.00,0.00;22:29:22,0.0,0.00,0.00,0.00;22:29:32,0.0,0.00,0.00,0.00;22:29:42,0.0,0.00,0.00,0.00;
			# 22:29:52,0.0,0.00,0.00,0.00;22:30:02,0.0,0.00,0.00,0.00;22:30:12,0.0,0.00,0.00,0.00;22:30:22,0.0,0.00,0.00,0.00;22:30:32,0.0,0.00,0.00,0.00;
			# 22:30:42,0.0,0.00,0.00,0.00;22:30:52,0.0,0.00,0.00,0.00;22:31:02,0.0,0.00,0.00,0.00;22:31:12,0.0,0.00,0.00,0.00;22:31:22,0.0,0.00,0.00,0.00;22:31:32,0.0,0.00,0.00,0.00;

			chunks = s.split(';')
			chunk_count = len(chunks)
			i = 0
			while (i + 1) < chunk_count:

				label_chunk = chunks[i]
				labels = label_chunk.split(',')
				
				# extract the op name from the header label
				opname = ''
				starttime = ''
				firstcolonpos = labels[0].find(':')
				if 0 < firstcolonpos:
					opname = labels[0][:firstcolonpos]
					starttime = labels[0][firstcolonpos + 1:]
					labels[0] = 'timespan'

				records = []
				more_record = True
				i += 1
				while more_record:
					#
					# optimization: only save the record if the op is wanted
					#
					include_this_record = True
					if filter_str != None:
						if None == filter.match(opname):
							include_this_record = False

					if include_this_record:
						data_chunk = chunks[i];
						values = data_chunk.split(',')
							
						# frist column is the end time. Add start time to for a span
						values[0] = starttime + '->' + values[0]
							
						data_hash = {}
						for j in range(len(labels)):
							data_hash[labels[j]] = values[j]

						records.append(data_hash)

					# peek at next chunk for the next op
					if (i + 1) >= chunk_count:
						more_record = False
					elif '' == chunks[i + 1]:
						more_record = False
						i += 1
					else:
						peek_chunk = chunks[i + 1]
						peek_data = peek_chunk.split(':')
						if peek_data[0] in self.known_histnames:
							more_record = False

					# read the last record for this op, push out the data_pack
					if not more_record:
						# check user filter to determine to keep the data for the op or not
						if filter_str != None:
							if None == filter.match(opname):
								continue
						data_pack = []
						data_pack.append(labels)
						data_pack.append(records)

						by_machine[mcname][opname] = data_pack
						by_histogram[opname][mcname] = data_pack
					i += 1

		# print out, group by histograms
		if want_bymachine:
			print '				 ======' + self.bold + "by histogram" + self.reset + '======'
		for histname in by_histogram.keys():
			self._print_histogram_table(histname, by_histogram[histname])
			print ' '

		if want_bymachine:
			print '				 ======' + self.bold + "by machine" + self.reset + '======'
			for mcname in by_machine.keys():
				self._print_histogram_table(mcname, by_machine[mcname])
				print ' '

	def _derive_label_weight(self, l):
		''' get the latency label weight. Extract the number in >???ms or 0'''
		m = re.search(r'>(\d+)ms', l)
		if None == m:
			return 0;
		else:
			return int(m.group(1))

	def _merge_histogram_labels(self, indata):
		''' combine the labels in indata in the right order '''
		all_labels = []
		for k in indata.keys():
			if 0 == len(all_labels):
				all_labels = indata[k][0]
			if set(all_labels) != set(indata[k][0]):
				extra_labels = set(indata[k][0]) - set(all_labels)

				# found extra labels, pleace them in the right place in all_labels
				if (len(extra_labels) > 0):
					for el in extra_labels:
						i = 0
						el_label_weight = self._derive_label_weight(el);
						while i < len(all_labels):
							if el_label_weight < self._derive_label_weight(all_labels[i]):
								all_labels.insert(i, el)
								break
							i += 1
						# the extra label did find a right place in the middle of all_labels. Add it to the end
						if i >= len(all_labels):
							all_labels.append(el)
						# all_labels.append(el);
		return all_labels

	def _print_histogram_table(self, name, indata):
		''' print out the histogram data in table format '''

		# no participating entries. nothing to print
		if 0 == len(indata):
			return
		# first survey all the labels of all the data rows, we'll basically do a outer join later on
		all_labels = []
		# for k in indata.keys():
		# 	if set(all_labels) != set(indata[k].[0]):
		# 		all_labels = set(all_labels).union(set(indata[k].[0]))
		# all_labels = sorted(all_labels)
		all_labels = self._merge_histogram_labels(indata);

		print '	====' + self.bold + str(name) + self.reset + '===='
		t = []

		for k in sorted(indata.keys()):
			# all_labels = indata[k][0]
			data_records = indata[k][1]
			first_record = True
			for r in data_records:
				data_row = []
				
				# only the first record need to prefix with key name (machine or op)
				if first_record:
					data_row.append(k)
					first_record = False
				else:
					data_row.append(' ')
					
				for l in all_labels:
					if l in r.keys():
						data_row.append(r[l])
					else:
						data_row.append(' ')
				t.append(data_row)

		# the first row is the indata key name, so wedge that label 
		all_labels.insert(0, ' ')
		t.insert(0, all_labels)

		out = sys.stdout
		pprint_table(out, t)

	def do_enablexdr(self, line):
		global arg_xdr
		if arg_xdr:
			arg_xdr = False
		else:
			arg_xdr = True

	# 
	# Get cluster partition map consistency check
	#
	def do_pmap(self, line):
		''' Do partition map check'''
		try:
			hostportlist, value = self._getconfigargs(line)
		except (KeyboardInterrupt, SystemExit):
			raise
		except:
			print self.help['pmap']['usage']
			return False
		if line == None or line == "":	
			print self.help['pmap']['usage']
			return False
		
		# TODO: Check if namespace is valid
		ns = line.strip()

		machines = []
		if 0 == len(hostportlist):
			machines = arg_machines
		else:
			machines = hostportlist	
		
		pmap_table = []
		pmap_master = {}
		pmap_replica = {}
		
		# If IP needs to be printed
		# pmap_header= ['Partion ID','Master Node','Master IP','Replica Node','Replica IP']
		pmap_header = ['Partion ID', 'Master Node', 'Replica Node']
		pmap_table.append(pmap_header)
		
		for mcname in machines:
			r = cinfo(mcname, "partition-info");
			if r == -1 or r == None:
				continue
			else:
				pinfo = r.split(";");
				for p in pinfo:
					parts = p.split(':')
					pid = int(parts[1])
					if parts[0] == ns and parts[2] == 'S':
						if parts[3] == '0':
							pmap = pmap_master
						elif parts[3] == '1':
							pmap = pmap_replica
						else:
							print 'can only support 2 replicas now'
							return False
						if pid in pmap:
							pmap[pid] = [mcname] + 	pmap[pid]		
						else:
							pmap[pid] = [mcname] 
							
		#AER-1948 ipNodeMap is not available unless node info is run
		if 'ipNodeMap' not in persistent_dict:
			get_node('Node', machines)
		nodemap = persistent_dict['ipNodeMap']
		out = sys.stdout
		for pid in range(0, 4096):
			per_row = [str(pid)]
			try:
				for c in pmap_master[pid]:
					per_row.append(nodemap[c])
					# per_row.append(c)
				for c in pmap_replica[pid]:
					per_row.append(nodemap[c])
					# per_row.append(c)
				if (len(pmap_master[pid]) != 1 or len(pmap_replica[pid]) != 1):
					print "**problem***" 
				pmap_table.append(per_row)
			except:
				print str(pid) + " Did not get partition"
		pprint_table(out, pmap_table)
		
	# 
	# Get cluster statistics info
	#
	def do_info(self, commands):
		''' Get basic cluster statistics from nodes and display in pretty format'''

		print_default = True
		try:
			if persistent_dict['tables'].has_key(commands):
				print_table_id = commands
				print_default = False
				hostportlist = ''
			else:
				hostportlist, value = self._getconfigargs(commands)
		except (KeyboardInterrupt, SystemExit):
			raise
		except:
			print self.help['info']['usage']
			return False

		if 0 == len(hostportlist):
			machines = arg_machines
		else:
			machines = hostportlist

		if print_default:
			global get_stat_file, get_stat_file2
			global get_xdr_stat, get_xdr_stat2
			
			cluster_objects = 0
	
			# Traverse through all the nodes in cluster and request cinfo and print out the details about the cluster
			try:
				stat_file = open(get_stat_file, "wb")  # dump raw statistics in file
				try:
					stat_file.write(str(datetime.now()))
				finally:
					stat_file.close()
			except IOError:
				# let's try the alternative location
				get_stat_file = get_stat_file2
	
			print "===NODES==="
			print datetime.now()
			try:
				print_table_id = 'Node'
				node, rows = create_table_list(get_node(print_table_id, machines))
				out = sys.stdout
				pprint_table(out, node) 
				# print "Total number of objects in cluster : " + str(float(cluster_objects) / 1000000) + " M" 
				print "Number of nodes displayed: " + str(rows)
			except Exception, i:
				print "Getting error in Node", i 
	
			# Get all the namespace statistics from all nodes
			print "\n\n ===NAMESPACE==="
			try:
				print_table_id = 'Namespace'
				out = sys.stdout  
				name_sp, rows = create_table_list(get_namespace(print_table_id, machines))
				pprint_table(out, name_sp)
				print "Number of rows displayed: " + str(rows)
			except Exception, i:
				print "Getting error in Namespace", i

			# Get all the xdr statistics from all nodes
			if (arg_xdr == True):	
				print "\n\n ===XDR==="
				try:
					xdr_stat_file_fd = open(xdr_stat_file, "wb")  # Create xdr raw data file
					try:
						xdr_stat_file_fd.write(str(datetime.now()))
					finally:
						xdr_stat_file_fd.close()
				except IOError:
					# let's try the alternative location
					get_xdr_stat = get_xdr_stat2
		
				try:
					print_table_id = 'XDR'
					xdr_node, rows = create_table_list(get_xdr(print_table_id, machines))
					out = sys.stdout  
					pprint_table(out, xdr_node)
					print "Node of XDR responding: " + str(rows)
				except Exception, i:
					print"No XDR Data", i
		else:
			try:
				table_def = persistent_dict['tables']

				# Below if statement is added to fix the bug - AER-2427
				# info Node has to be run always before other info commands can work.

				if 'ipNodeMap' not in persistent_dict:
					get_node('Node', machines)

				if print_table_id in table_def.keys():
					out = sys.stdout 
					table_type = table_def[print_table_id]
				
					if table_type == 'node':
						customTableDict = get_node(print_table_id, machines)
						customTable, rows = create_table_list(customTableDict)
					elif table_type == 'namespace':
						customTableDict = get_namespace(print_table_id, machines)
						customTable, rows = create_table_list(customTableDict)
					elif table_type == 'xdr':
						customTableDict = get_xdr(print_table_id, machines)
						customTable, rows = create_table_list(customTableDict)
					# TODO: Create Summary of sets, i.e no of sets in namespace and setnames.
					elif table_type == 'sets' or table_type == 'setssummary':
						customTableDict = get_sets(print_table_id, machines)	
						customTable, rows = create_table_list(customTableDict)
					elif table_type == 'sindex':
						customTableDict = get_sindex(print_table_id, machines)
						customTable, rows = create_table_list(customTableDict)
					
					print  "\n === " + print_table_id.upper() + " ==="	
					pprint_table(out, customTable)
					print "No. of rows: " + str(rows) 
					
				else:
					raise ValueError('Table :' + print_table_id + ' does not have a type')
					
			except Exception, i:
				print "Getting error in custom table", i
# 		if len(machine_epoch) > 0:
# 			print "\n\n===CLOCK SYNC==="
# 			max_epoch = 0
# 			min_epoch = 0
# 			for m in machine_epoch.keys():
# 				if machine_epoch[m] > max_epoch:
# 					max_epoch = machine_epoch[m]
# 				if 0 == min_epoch:
# 					min_epoch = machine_epoch[m]
# 				if machine_epoch[m] < min_epoch:
# 					min_epoch = machine_epoch[m]
#
# 			max_min_epoch_diff = max_epoch - min_epoch
# 			if (3 > max_min_epoch_diff):
# 				print "All clocks on nodes are synced within", max_min_epoch_diff, "seconds"
# 			else:
# 				machine_deviation = []
# 				machine_deviation.append([comparison("ip:port"), comparison("clock deviation\n")])
# 				for m in sorted(machine_epoch.keys()):
# 					machine_deviation.append([m, str(machine_epoch[m] - min_epoch)+" sec"])
#
# 				out = sys.stdout
# 				try:
# 					pprint_table(out, machine_deviation)
# 				except Exception, i:
# 					print i

	# Nice featute for autocompleting info commands, e.g. info N<tab><tab> will give all tables
	# starting with N
	def complete_info(self, text, line, begidx, endidx):
		self.infolist = persistent_dict['tables'].keys()
		if not text:
			
			completions = self.infolist[:]
		else:
			completions = [ f
							for f in self.infolist
							if f.startswith(text)
							]
		return completions

	def do_netstopwatch(self, line):
		global g_want_netstopwatch
		global g_netstopwatch_log_fd

		if line == "on" or line == "yes" or line == "y" or line == "start" or line == "1":
			netstopwatch_log = "/tmp/" + myprocname + "_" + str(os.getpid()) + ".log"
			if not g_want_netstopwatch:
				try:
					g_netstopwatch_log_fd = open(netstopwatch_log, 'ab')
					print "Info I/O Stopwatch is on. Log file is:", netstopwatch_log
					g_want_netstopwatch = True
				except Exception, ex:
					print "Cannot open logfile", netstopwatch_log, "to write. Info Stopwatch is not on! Error get=", ex
			else:
				print "Info I/O Stopwatch was already on. Log file is:", netstopwatch_log
		elif line == "":
			if g_want_netstopwatch:
				print "Net stopwatch is currently on. Enter 'netstopwatch off' to turn it off"
			else:
				print "Net stopwatch is currently off. Enter 'netstopwatch on' to turn it on"
		else:
			if g_want_netstopwatch:
				g_want_netstopwatch = False
				g_netstopwatch_log_fd.close()
				print "Info I/O stopwatch is off."
	#####			
	# Sort table by column selected.
	#####
				
	def do_sorttable(self, line):
		
		table_name = ''
		
		try:
			table_def = persistent_dict['tables']
			### Sort table ###
			tn = raw_input('Do you want to Sort a table by column (response n if you do not want to edit) (Y/n)? ')
			if 'y' == tn or 'Y' == tn or '' == tn :
				response = ''
				while response == '':
					response = raw_input('Enter Table name : ')
					if not response == '':
						table_name = response
						print 'table name=', response
						if response not in table_def.keys():
							print 'Only following tables can be edited :'
							print "\t".join(persistent_dict['tables'].keys())
							response = ''
						else:
							try:
								if persistent_dict['sort'].has_key(table_name):
									print "Table sorted by Column: ", persistent_dict['sort'][table_name][0]
									if persistent_dict['sort'][table_name][1]:
										print "And it is sorted in Descending order"
									else:
										print "And it is sorted in Ascending order"
							except:
								print " Table does not seem to be sorted."
							
				tableHeader = persistent_dict[table_name].keys()

				
				for column in tableHeader:
					if column == 'lhsh':
						continue
					tn = raw_input('Sort ' + column + ' (no response to next) (y/N)? ')
					if 'y' == tn:
						tn2 = raw_input('Which order to sort ' + column + ' Ascending or Descending ?(default is Descending) (a/D)? ')
						if 'a' == tn2:
							persistent_dict['sort'][table_name] = (column, False)
						else:
							persistent_dict['sort'][table_name] = (column, True)
						break

				iskeycolor = persistent_dict[table_name][column][0]
				if persistent_dict['rules'].has_key(table_name):
					if persistent_dict['rules'][table_name].has_key(iskeycolor):
						del persistent_dict['rules'][table_name][iskeycolor]
						print "Color Alerting disabled for this column" 
					
				print "Table sorted by Column: ", persistent_dict['sort'][table_name][0]
				if persistent_dict['sort'][table_name][1]:
					print "And it is sorted in Descending order"
				else:
					print "And it is sorted in Ascending order"
						
					
		except (EOFError):
			print "\n Exit from Sorting Column"	
		except Exception, i:
			print "Exception Exit from sort function", i
	#####			
	# Allow user to edit a current table
	# TODO add the option to add a new row			
	#####
	def do_edittable(self, line):
		
		table_name = ''
		
		try:
			table_def = persistent_dict['tables']
			### edit table ###
			tn = raw_input('Do you want to Edit or delete a table (response n if you do not want to edit/delete) (Y/n)? ')
			if 'y' == tn or 'Y' == tn or '' == tn :
				response = ''
				while response == '':
					response = raw_input('Enter Table name : ')
					if not response == '':
						table_name = response
						print 'table name=', response
						if response not in table_def.keys() and persistent_dict.has_key(response):
							print 'Only following tables can be edited  :'
							print "\t".join(persistent_dict['tables'].keys())
							print 'If table is in list, then consider creating it again! '
							response = ''
				
				### delete table ###
				dn = raw_input('Do you want to Delete this table (response y if you want to Delete) (y/N)? ')
				if 'y' == dn or 'Y' == dn:
					try:
						if persistent_dict['tables'].has_key(table_name):
							del persistent_dict['tables'][table_name]
						if persistent_dict.has_key(table_name):
							del persistent_dict[table_name]
						print 'Table deleted Successfully.'
						return False
					except Exception, i:
						print 'Encountered exception deleting table ', i	
						return False	
				
				table_type = table_def[table_name]
				editTable = persistent_dict[table_name]

				print '---------------\n'
				print 'List of keys allowed for reference: ' + allowed_keys(table_type)
				print '\n---------------\n'
		
				tableHeader = editTable.keys()
				
				
				for column in tableHeader:
					# Do not allow users to delete/replace mcname or lhsh field
					if column == 'lhsh' or persistent_dict[table_name][column][0] == 'mcname' or \
					persistent_dict[table_name][column][0] == 'mcnamens' or \
					persistent_dict[table_name][column][0] == 'mcnamesets':
						continue
					tn = raw_input('Do you want to delete or replace ' + column + 
								' (no response to next) (d/r)? ')
					if 'd' == tn :
						del editTable[column]
					elif 'r' == tn:
						response = ''
						while response == '':
							response = raw_input('Enter New Column Name: ')
							if not response == '':
								column_name = response
								
								if response in editTable.keys():
									print 'Column already exists'
									response = ''
								else:
									print 'Column name=', response
						
						response_value = ''			
						while response_value == '':
							response_value = raw_input('Enter New key for ' + column_name + ' Column in the table: ')
							if table_type == 'node':
								if response_value not in persistent_dict['STATISTICS']:
									print 'Select key from list: ' + allowed_keys(table_type)
									response_value = ''
							elif table_type == 'namespace':
								if response_value not in persistent_dict['NAMESPACE']:
									print 'Select key from list: ' + allowed_keys(table_type)
									response_value = ''
							elif table_type == 'xdr':
								if response_value not in persistent_dict['XDRKEYS']:
									print 'Select key from list: ' + allowed_keys(table_type)
									response_value = ''	
							elif table_type == 'sets':
								if response_value not in persistent_dict['SETSKEYS']:
									print 'Select key from list: ' + allowed_keys(table_type)
									response_value = ''	
							elif table_type == 'setssummary':
								if response_value not in persistent_dict['SETSSUMMARY']:
									print 'Select key from list: ' + allowed_keys(table_type)
									response_value = ''		
						column_value = response_value	
						del editTable[column]		
						editTable[column_name] = [column_value]
				
				persistent_dict[table_name] = editTable

		except (EOFError , KeyError):
			print '\n Exiting from Editing table, No changes saved.'
		except Exception, i:
			print 'Exception/Exit in Edit table : ', i	
	#####
	# Restore defaults for the table, only for columns deleted or replaced.	
	#####	
	def do_restoredefault(self, line):
		if get_table_header('restore'):
			print "Restore Node,Namespace,SETS and XDR to defaults"
		else:
			return False
	
	#####
	# Create a new custom table , very useful as customization can allow users to have tables
	# with different values.
	# TODO: Allow user to have multiple values for each key, incase new build had those values
	# changed.
	#####
	
	def do_createtable(self, line):
	
		table_name = ''
		
		try:
			table_def = persistent_dict['tables']
			### create table ###
			tn = raw_input('Create new table (response n if you do not want to create) (Y/n)? ')
			if 'y' == tn or 'Y' == tn or '' == tn :
				response = ''
				while response == '':
					response = raw_input('Enter Table name : ')
					if not response == '':
						table_name = response
						print 'table name=', response
						if response in table_def.keys():
							print 'Table already exists'
							response = ''
							
	
				response = ''
				while response == '':
					response = raw_input('Type of table (node,namespace,xdr,sets) : ')
					if response == 'node' or response == 'namespace' or response == 'xdr' or  \
					response == 'sets' or response == 'setssummary':
						table_type = response
						print 'Table type=', response
					else:
						response = ''
				
				table_def[table_name] = table_type
				persistent_dict['tables'] = table_def
				print '---------------\n'
				print 'List of keys allowed for reference: ' + allowed_keys(table_type)
				print '\n---------------\n'
				### Get number of columns ###
				response = 'x';
				while (not response.isdigit()):
					if table_type == 'sets' or table_type == 'setssummary':
						response = raw_input('Enter the Number of Columns in the table : ')
					else:
						response = raw_input('Enter the Number of Columns in the table(excluding 1st column as it will be IP:port) : ')
					if response == '':
						continue
				no_table_columns = int(response) + 1

				TABLE_HEADER = ['Node IP']
				TABLE_KEY = ['mcname']
				
				if table_type == 'namespace':
					TABLE_HEADER = ['IP/ns']
					TABLE_KEY = ['mcnamens']

				if table_type == 'sets' or table_type == 'setssummary':
					lhsh_key = ''
					while lhsh_key == '':
						lhsh_key = raw_input('Enter Name of Left hand side header (lhsh) Column in the table (either ip:port or ip/ns): ')
						TABLE_HEADER = [lhsh_key]

					response_value = ''				
					while response_value == '':
						response_value = raw_input('Enter key for ' + lhsh_key + ' Column in the table: ')
						if response_value == 'mcnamesets' or response_value == 'mcnamens':
							TABLE_KEY = [response_value]
							if response_value == 'mcnamens':
								persistent_dict['tables'][table_name] = 'setssummary'
								table_type = 'setssummary'
								no_table_columns = 3
						else:
							print 'Only values allowed are: mcnamesets or mcnamens , use mcnamens if you want to see Sets Summary (i.e. To see setnames and Number of sets) use mcnamesets if you want to see Set details like objects in sets etc.'
							response_value = ''						
				
				cur_table = {TABLE_HEADER[0]:[TABLE_KEY[0]], 'lhsh':TABLE_HEADER[0]}
				for i in range(0, no_table_columns) :
					if i == 0:
						print '1st column will be ' + TABLE_HEADER[0]
						continue
					response_key = ''
					while response_key == '' and response_key not in TABLE_HEADER :
						response_key = raw_input('Enter Name of ' + str(i) + ' Column in the table: ')
					
					TABLE_HEADER.append(response_key)	
					
					response_value = ''
					while response_value == '' and response_value not in TABLE_KEY:
						response_value = raw_input('Enter key for ' + response_key + ' Column in the table: ')
						if table_type == 'node':
							if response_value not in persistent_dict['STATISTICS']:
								print 'Select key from list: ' + allowed_keys(table_type)
								response_value = ''
						elif table_type == 'namespace':
							if response_value not in persistent_dict['NAMESPACE']:
								print 'Select key from list: ' + allowed_keys(table_type)
								response_value = ''
						elif table_type == 'xdr':
							if response_value not in persistent_dict['XDRKEYS']:
								print 'Select key from list: ' + allowed_keys(table_type)
								response_value = ''
						elif table_type == 'sets':
							if response_value not in persistent_dict['SETSKEYS']:
								print 'Select key from list: ' + allowed_keys(table_type)
								response_value = ''	
						elif table_type == 'setssummary':
							if response_value not in persistent_dict['SETSSUMMARY']:
								print 'Select key from list: ' + allowed_keys(table_type)
								response_value = ''				
					
					cur_table[response_key] = [response_value]
				persistent_dict[table_name] = cur_table
				print 'Table ' + table_name + ' created!'
		except Exception, i:
			try:
				if persistent_dict['tables'].has_key(table_name):
					del persistent_dict['tables'][table_name]
				if persistent_dict.has_key(table_name):
					del persistent_dict[table_name]
				print '\n Exiting Creating table, No Changes saved.'
			except Exception, i:
				print '\n Exiting Creating table, There might be some changes saved.', i

# 	def do_looptest(self, line):
# 		''' Run a loop test with the server '''
#
# 		try:
# 			hostportlist, value = self._getconfigargs(line)
# 			if value != None:
# 				value = int(value)
# 		except (KeyboardInterrupt, SystemExit):
# 			raise
# 		except:
# 			print self.help['info']['usage']
# 			return False
#
# 		if 0 == len(hostportlist):
# 			machines = arg_machines
# 		else:
# 			machines = hostportlist
#
# 		if value < 1:
# 			value = 1000
#
# 		print "Submit", value, "info requests to each of the node:"
# 		for mcname in machines:
# 			time1 = time()
# 			for i in range(value):
# 				s = cinfo(mcname, 'service')
# 			time2 = time()
# 			print mcname, ": start time =", time1, "   end time =", time2,"   lapse=", time2-time1

	#####
	# Hacky way to make watch work, making format strict, i.e. requiring an order
	# for arguments : i.e. watch -n <seconds> [Free form of command passed to onecmd()]
	#####
	def do_watch(self, line):
		
		''' Run watch for any command available in console '''
		
		try:
			arg_for_me = ' '.join(line.split()[0:2])
			arg_to_watch = ' '.join(line.split()[2:])
			function_to_run = ''.join(line.split()[2])
		except:
			print "Invalid Format, Required Format:"
			print self.help['watch']['usage']
			print "\te.g. watch -n 10 info Node"
		
		opts, args = self._getargs(arg_for_me, 'n:')

		sleep_time = 0
		
		for o, a in opts:
			if o == '-n':
				if checkfloat(a):
					sleep_time = float(a)
				else:
					sleep_time = 0	
					
		if sleep_time != 0 and 'do_' + function_to_run in dir(self):			
			try:
				try:
					while True:
							print 'Every ' + str(sleep_time) + 's: ' + function_to_run + '\t\t\t' + str(datetime.now())
							self.onecmd(arg_to_watch)
							sleep(sleep_time)
				except (KeyboardInterrupt, SystemExit):
					raise
			except:
				print ""
	
	#####
	# Diplat top X number of columns for the histogram snapshot, (default is 5 columns)
	#####
	
	def do_histdump(self, line):
		
		try:
			opts, args = self._getargs(line, 'h:p:n:v:t:')
		except:
			print self.help['histdump']['usage']
			return False
					
		hostport = None
		port = None
		hist = None
		namespace = arg_namespace
		topX = 5
		
		try:
			for o, a in opts :
				if o == '-h':
					hostport = a
				if o == '-p':
					port = str(int(a))
				if o == '-n':
					namespace = a.split(',')
				if o == '-v':
					hist = a
				if o == '-t':
					topX = int(a)
		except:
			print self.help['histdump']['usage']
			return False

		# Hist name is required.	
		if hist is None:
			print self.help['histdump']['usage']
			return False
		elif hist not in ['ttl', 'evict', 'objsz']:
			print "Only hists allowed are : ttl,evict or objsz"
			return False
		
		diffPort = False

		hplist = []
		if hostport != None:
			hplist = hostport.split(',')
		else:
			hplist = arg_machines
			diffPort = True

		hostportlist = []
		for hp in hplist:
			# look for the : to assume the port. If not there, add it
			try: 
				h, p = hp.split(':')
				
				if diffPort and port:
					hostportlist.append(h + ':' + port)
				else:
					hostportlist.append(hp)
			except Exception, ex:
				if None == port:
					hostportlist.append(hp + ':' + str(arg_port))
				else:
					hostportlist.append(hp + ':' + port)
		
		out = sys.stdout
		
		# Call get_hist_dump to get histogram dump Dictionary.
		histDump = get_hist_dump(hist, topX, hostportlist, namespace)

		for namespace, value in histDump.iteritems():
			tableForPrinting = []
			lhsh = ''
			for ipPort, rows in value.iteritems():
				rowList = []
				lhsh = ipPort + '/' + namespace
				
				headerList = ['Node/Namespace; ']
				comparatorList = ['Node/Namespace; ']
				rowList = [lhsh]
				
				for tup in rows:
					if tup[0] not in comparatorList:
						comparatorList.append(tup[0])

					if len(headerList) <= topX + 1:
						headerList.append(tup[0])
						
					if set(comparatorList) == set (headerList):
						rowList.append(tup[1])
					else:
						rowList.append((tup[0], tup[1]))
				
				if len(tableForPrinting) == 0:
					# # Creating Header, Adding ";" in header to split long headers it in multiple rows to display
					headerList1 = []
					headerList2 = []
					for line in headerList:
						line1, line2 = line.split(";")
						headerList1.append(line1)
						headerList2.append(line2)
						
					tableForPrinting.append(headerList1)
					tableForPrinting.append(headerList2)
				
				tableForPrinting.append(rowList)
				
			pprint_table(out, tableForPrinting)
			print '\n'

	# Compare the total installed System RAM with total configured memory for all namespaces
	# NOTE :- This should be run on one of the node from the cluster.
	#####
	
	def do_checkmem(self, line):
		
		shell_cmd = "free | grep Mem: | awk '{print $2}'"
		bold = "\033[1m"
		reset = "\033[0;0m"

		hostportlist, value = self._getconfigargs(line)
		for mcname in hostportlist:
			s = cinfo(mcname, 'statistics')
			stat = string_to_dict(s)
			total_ns_memory = int(stat['total-bytes-memory'])
		
		output = os.popen(shell_cmd).read()
		sys_memory = int(output)*1024
		
		if total_ns_memory < sys_memory :
			COLOR = "\033[39m"			# Reset
			COMPARE = ' is less than '
		else:
			COLOR = "\033[31m"			# Red
			COMPARE = ' is greater than '
			
		msg = "Total Configured Namespace Memory : " + bold + COLOR + bytes2human(total_ns_memory) + reset
		msg += COLOR + COMPARE + reset
		msg += "Installed Memory : " + bold + COLOR + bytes2human(sys_memory) + reset

		print msg

	# ##Collect all info and put in a file to be shipped to support@aerospike.com
	def do_collectinfo(self, line):
		
		aslogdir = '/tmp/as_log_' + str(time())
		aslogfile = aslogdir + '/ascollectinfo.log'
		os.mkdir(aslogdir)
		collectedinfo = '';
		self.do_shell(line)
		info_params = ['Node', 'Namespace', 'XDR', 'SETS']
		monitor_params = ['printconfig', 'compareconfig', 'latency', 'stat']
		hist_params = ['objsz', 'ttl', 'evict']
		shell_cmds = ['date',
					 'hostname',
					 'ifconfig',
 					 'uname -a',
 					 'lsb_release -a',
 					 'ls /etc|grep release|xargs -I f cat /etc/f',
 					 'rpm -qa|grep -E "citrus|aero"',
 					 'dpkg -l|grep -E "citrus|aero"',
 					 'tail -n 1000 /var/log/aerospike/*.log',
 					 'tail -n 1000 /var/log/citrusleaf.log',
 					 'tail -n 1000 /var/log/*xdr.log',
 					 'netstat -pant|grep 3000',
 					 'top -n3 -b',
 					 'free -m',
 					 'df -h',
 					 'ls /sys/block/{sd*,xvd*}/queue/rotational |xargs -I f sh -c "echo f; cat f;"',
 					 'ls /sys/block/{sd*,xvd*}/device/model |xargs -I f sh -c "echo f; cat f;"',
 					 'lsof',
 					 'dmesg',
 					 'iostat -x 1 10',
 					 'vmstat -s',
 					 'vmstat -m',
 					 'iptables -L',
					 'cat /etc/aerospike/aerospike.conf',
					 'cat /etc/citrusleaf/citrusleaf.conf',
					  ]
		delimiter_begin = '===ASCOLLECTINFO==='
		def collect_asdcheck():
			self.do_info("Node")
			
		def collect_ns():
			for host in hosts_from_parm:
				for ns in arg_namespace:
					ns_info = cinfo(host, "namespace/" + ns)
					print ns
					ns_stats = ns_info.split(';')
					for stat in ns_stats:
						print stat
		
		def print_param(param):
			print delimiter_begin
			print param
		#If we are passing force, then do not do asinfo calls
		sys_online = True
		try:
			output = capture_stdout(collect_asdcheck)
			if "error" in output:
				raise Exception("error in connecting to localhost")
		except Exception,e:
			sys.stderr.write("\x1b[2J\x1b[H")
			if(line != 'force'):
				print e
				print "ERROR: Collectinfo works with a running aerospike server"
				print "run as --\n sudo asmonitor -e 'collectinfo force' \n-- to collect rest of system parameters while aerospike is not running"
				sys.exit(-1)
			sys_online = False
		
		if sys_online is True:		
			def collect_params():
				for param in info_params:
					print_param(param)
					self.do_info(param)
				for param in monitor_params:
					print_param(param)
					func = getattr(self, 'do_' + param)
					func(line)
				for param in hist_params:
					print_param(param)
					self.do_histdump('-v ' + param)
				print_param('NAMESPACE STATS')
				collect_ns()
				print_param('XDR STATS')
				self.do_stat('-p 3004')
					
			collectedinfo = capture_stdout(collect_params)
			
			def collect_loginfo():
				self.do_asinfo('-v logs ', True)
			logfiles = capture_stdout(collect_loginfo)
			logfile = logfiles.split(':')
			
			def collect_readlogs():
				print_param('')
				custom_log = str(logfile[1])
				self.do_shell('tail -n 10000 ' + custom_log)
				shutil.copy2(custom_log.split('\n',1)[0], aslogdir)
			collectedinfo += capture_stdout(collect_readlogs)

		def collect_sys():
			if sys_online is False:
				print_param("Running with Force on Offline Aerospike Server")
			lsof_cmd='sudo lsof|grep `sudo ps aux|grep -v grep|grep -E \'asd|cld\'|awk \'{print $2}\'` 2>/dev/null'
			print_param('')
			self.do_shell(lsof_cmd)
			print platform.platform()
			smd_home = '/opt/aerospike/smd'
			if os.path.isdir(smd_home):
				smd_files = [ f for f in os.listdir(smd_home) if os.path.isfile(os.path.join(smd_home, f)) ]
				for f in smd_files:
					smd_f = os.path.join(smd_home, f)    
					print smd_f
					smd_fp = open(smd_f, 'r')
					print smd_fp.read()
					smd_fp.close()
		collectedinfo += capture_stdout(collect_sys)

		def collect_shell():
			for param in shell_cmds:
				print_param(param)
				self.do_shell(param + " 2>/dev/null")
		collectedinfo += capture_stdout(collect_shell)
		f = open(str(aslogfile), 'w')
		f.write(str(collectedinfo))
		f.close()
		self.do_shell("tar -czvf " + aslogdir + ".tgz " + aslogdir)
		sys.stderr.write("\x1b[2J\x1b[H")
		print "\n\n\nFILE " + aslogdir + " and " + aslogdir + ".tgz saved. Please send it to support@aerospike.com"
		print "END OF ASCOLLECTINFO"
			
#######===========================End of class RunCommand====================================#######	
#####
# Initialization et al
####
# the image name may come in as ./asmonitor.py, we only want the asmonitor part
imagename = os.path.basename(inspect.stack()[-1][1])
imageparts = imagename.split('.')
myprocname = imageparts[0]

parser = argparse.ArgumentParser(add_help=False, conflict_handler='resolve')
parser.add_argument("-h", "--Host", default="127.0.0.1", help="Connection info of the host(s). Must be in the 127.0.0.1 or 127.0.0.1:3000 format. Comma separated for multi hosts")
parser.add_argument("-p", "--Port", type=int, default=3000, help="server port (default: %(default)s)")
parser.add_argument("-U", "--User", help="user name")
parser.add_argument("-P", "--Password", nargs="?", const="prompt", help="password")
parser.add_argument("-c", "--Config", default="filename", help="Location of configuration file (default: %(default)s)")
parser.add_argument("-r", "--OneCommand", help="Deprecated. use -e instead.")
parser.add_argument("-e", "--OneCommand", help="eg1: '-e info' execute the info command in the shell. eg2: '-e help' show the shell help text")
parser.add_argument("-d", "--Dir", help="Monitor Directory (default: users home directory)")
parser.add_argument("-n", "--NoDns", action="store_true", help="do not convert ip to FQDN. Defaults to False.")
parser.add_argument("-u", "--Usage", action="store_true", help="show program usage")
args = parser.parse_args()

if args.Usage:
	printusage(args.Config)
	sys.exit(0)

user = None
password = None

if args.User != None:
	user = args.User
	if args.Password == "prompt":
		args.Password = getpass.getpass("Enter Password:")
	password = citrusleaf.hashpassword(args.Password)

if sys.stdout.isatty():
	print "\nEnter help for commands\n"

hosts_from_parm = []
raw_hosts = []
COLORS = [
	'GREY', 'RED', 'GREEN', 'YELLOW',
	'BLUE', 'MAGENTA', 'CYAN', 'WHITE', 'BLACK'
]

# netstopwatch to measure network performance
g_want_netstopwatch = False
g_netstopwatch_log_fd = None

if args.Host != None:
	raw_hosts = filter(None, args.Host.split(','))
	if len(raw_hosts) < 1:
		print "Host info is malformed. It must be comma separated, and in the format of 127.0.0.1 or 127.0.0.1:3000"
		printusage(args.Config)
		sys.exit(-1)
elif args.Port != None:
	hosts_from_parm.append('127.0.0.1:' + str(args.Port))

# attach the port number to hosts if needed
for hp in raw_hosts:
	# look for the : to assume the port. If not there, add it
	try:
		h, p = hp.split(':')
		hosts_from_parm.append(hp)
	except Exception, ex:
		hosts_from_parm.append(hp + ':' + str(args.Port))

#####
# All files used by monitoring
# asmonitor.conf, Config file.
# get_stat_file.txt, raw dump of last run
# get_xdr_stat.txt, raw dump of xdr run.
# asmonitor.db, the database file (actually a dict) which we will use for a lot of stuff
# clhistory, history or last commands entered. 
#####

if args.Dir != None:
	monitorPath = args.Dir
else:
	monitorPath = os.environ['HOME'] + '/.' + myprocname

configfile = monitorPath + '/' + myprocname + '.conf'
get_stat_file = monitorPath + '/get_stat_file.txt'
get_stat_file2 = '/tmp/get_stat_file.txt'
xdr_stat_file = monitorPath + '/get_xdr_stat.txt'
xdr_stat_file2 = '/tmp/get_xdr_stat.txt'

db_file = monitorPath + '/' + myprocname + '.db'
hist_file = monitorPath + '/clhistory'

#####
# Move All files to 1 location , Under directory ~/.asmonitor/
#####

if not os.path.isdir(monitorPath):
	os.makedirs(monitorPath)
	
	old_config = os.environ['HOME'] + '/.' + myprocname + '.conf'
	old_stat = os.environ['HOME'] + '/.get_stat_file.txt'
	old_xdr = os.environ['HOME'] + '/.get_xdr_stat.txt' 
	old_db = '/tmp/' + myprocname + '.db'
	old_hist = os.environ['HOME'] + '/.clhistory'
	
	try:
		if os.path.exists(old_config):
			shutil.move(old_config, configfile)
	except Exception, i:
		print 'Encountered issue moving file , OK to continue ', i
		
	try:
		if os.path.exists(old_stat):
			shutil.move(old_stat, get_stat_file)
	except Exception, i:
		print 'Encountered issue moving file , OK to continue ', i
		
	try:			
		if os.path.exists(old_xdr):
			shutil.move(old_xdr, xdr_stat_file)
	except Exception, i:
		print 'Encountered issue moving file , OK to continue ', i
		
	try:
		if os.path.exists(old_db):
			shutil.move(old_db, db_file)
	except Exception, i:
		print 'Encountered issue moving file , OK to continue ', i
	
	try:	
		if os.path.exists(old_hist):
			shutil.move(old_hist, hist_file)
	except Exception, i:
		print 'Encountered issue moving file , OK to continue ', i

def main():
	global persistent_dict
	persistent_dict = PersistentDict(db_file, 'c', format='pickle')
	try:
		try:
			if args.OneCommand:
				RunCommand().onecmd(args.OneCommand)
			else:
				RunCommand().cmdloop()
		
		
		except (KeyboardInterrupt, SystemExit):
			import readline
			readline.write_history_file(hist_file)
			print "\nConfig files location : " + str(configfile)
			print "Folder used by " + myprocname + " for files : " + monitorPath + "/"
	finally:
		persistent_dict.close()



if __name__ == '__main__':
	main()
