#!/usr/bin/python
# -*- coding: UTF-8 -*-
# vim: set fileencoding=utf-8 :
#
# SZARP: SCADA software
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA

"""
Configuration parser for checking its correctness
and for displaying suggestions to user.
"""

import argparse
import subprocess
from subprocess import Popen, PIPE
import pysvn
import sys
sys.path.append("/opt/szarp/lib/python/")
from xmlns import add_xmlns
import os
from clint.textui import puts, indent, colored
from clint.textui.core import STDOUT, STDERR
from lxml import etree
import hashlib
import re
import glob
import unicodedata
import ConfigParser
import io

error_count = 0
quiet = False
def printerr(msg, prefix="ERROR: "):
	global error_count
	error_count = error_count + 1
	msg = unicode_to_ascii(msg)
	puts(colored.red(prefix + "(" + str(error_count) + ") " + str(msg)), stream=STDERR)

note_count = 0
def printnote(msg, prefix="NOTE: "):
	global note_count
	note_count = note_count + 1
	msg = unicode_to_ascii(msg)
	puts(colored.yellow(prefix + "(" + str(note_count) + ") " + str(msg)))

note_list = []
def add_note(msg):
	note_list.append(msg)

def print_notes():
	for note in note_list:
		printnote(note)

def printok(msg):
	puts(colored.green(str(msg)))

def unicode_to_ascii(string):
	"""
	Converts unicode string to printable ascii form, removes polish characters
	"""
	try:
		return unicodedata.normalize('NFKD', string).replace(u'ł', 'l').replace(u'Ł', 'L').encode('ascii', 'ignore')
	except TypeError:
		return string

# truncates long lines for debug prints
def trunc(s, width=80):
	long_line_suffix = '...'
	if len(s)>width:
		if width>len(long_line_suffix):
			return s[0:width-len(long_line_suffix)] + long_line_suffix
		else:
			return s[0:width]
	return s

def get_szarp_uncommited():
	client = pysvn.Client()
	is_svn_repo = True
	try:
		client.info(szarp_path)
	except:
		is_svn_repo = False
	if is_svn_repo:
		changes = client.status(szarp_path, get_all=False)
		interesting = set([pysvn.wc_status_kind.added, pysvn.wc_status_kind.deleted,\
			pysvn.wc_status_kind.modified, pysvn.wc_status_kind.conflicted])
		uncommited = [f.path for f in changes if f.text_status in interesting]
		uncommited_present = len(uncommited) > 0
	else:
		uncommited_present = None
	return is_svn_repo, uncommited_present

def get_szarp_path():
	if args.input_file is None:
		szarp_default = os.path.realpath("/etc/szarp/default")
		params_default = os.path.join(szarp_default, "config", "params.xml")
		return szarp_default, params_default
	else:
		if os.path.exists(args.input_file) and "/opt/szarp/" in os.path.abspath(args.input_file):
			params = os.path.abspath(args.input_file)
			config = os.path.dirname(params)
			szarp = os.path.dirname(config)
		elif os.path.exists(args.input_file):
			params = os.path.abspath(args.input_file)
			szarp = os.path.dirname(params)
		else:
			printerr(args.input_file + " does not exist")
			sys.exit(1)
		return szarp, params

def check_svn():
	szarp_svn, szarp_uncommited = get_szarp_uncommited()
	if szarp_svn:
		if szarp_uncommited:
			add_note("uncommited changes found in " + szarp_path + ": /opt/szarp/bin/conf-put.sh")
	else:
		if not quiet:
			add_note(szarp_path + " is not added to svn, maybe it should?")

def get_params_scheme():
	"""
	Getting params.xml to push it scheme to other functions
	"""
	options = etree.XMLParser(remove_blank_text=True, remove_comments=True)
	params_scheme = etree.parse(params_path, options)
	return params_scheme

def get_isl_list():
	"""
	Get list of isl file containing in szarp-server base from /config/isl/
	"""
	isl_regex = re.compile(r".{1,}\.isl")
	ls = subprocess.Popen(("ls", os.path.join(szarp_path, "config", "isl")), stdout=subprocess.PIPE)
	isl_ls = ls.communicate()[0]
	isl_list = []
	for isl in isl_ls.split('\n'):
		if isl.endswith(".isl"):
			isl_list.append(isl)
	return isl_list

def parse_config_file():
	config = params.getroot()
	elements = list(config)
	return config, elements

def check_for_daemons():
	"""
	Testing for:
		custom daemons
		extra:param base in dbmdmn daemon
		scripts or programs used in pythondmn/execdmn/testdmn
	"""

	md5file_path = "/var/lib/dpkg/info/szarp-daemons.md5sums"
	if not os.path.isfile(md5file_path):
		add_note("File does not exist: %s, will not check daemon sums" % md5file_path)
		return
	md5_map = {}
	with open(md5file_path, "r") as md5file:
		for line in md5file:
			if "opt/szarp/bin" in line:
				md5, daemon = line.split()
				# md5_map["/opt/szarp/bin/iecdmn"] = md5
				md5_map["/" + daemon] = md5

	ignore_daemon = ['/bin/true', '/dev/null', '/bin/false']
	daemons_list = []
	for device in add_xmlns('ipk', params_root.iter)("device"):
		daemon = device.get("daemon")
		if daemon in ignore_daemon:
			continue
		if not daemon in daemons_list:
			daemons_list.append(daemon)
			md5sum_check = 0
			try:
				daemon_md5sum = hashlib.md5(open(daemon, 'rb').read()).hexdigest()
				# search for given daemon md5sum in md5_map
				md5_map.keys()[md5_map.values().index(daemon_md5sum)]
			except ValueError:
				add_note("custom daemon found in " + daemon)
			except IOError:
				printerr(daemon + " does not exist!")

		# search for extra:param base at params under dbdmn
		if os.path.basename(daemon).startswith("dbdmn"):
			for unit in device:
				for param in unit:
					check_base(param)

		# check exec paths and scripts for pythdondmn, execdmn, testdmn
		check_daemon_script = ("pythondmn", "execdmn", "testdmn")
		if os.path.basename(daemon).startswith(check_daemon_script):
			script_path = device.get("path")
			if script_path.startswith("/opt/szarp"):
				if os.path.exists(script_path) is False:
					printerr("script %s called under device does not exist (line %d)"\
							% (script_path, device.sourceline))
					return
				if "testdmn" not in daemon:
					check_script(script_path)

		check_daemons_with_base_connection(device, daemon)

def check_daemons_with_base_connection(device, daemon):
	# check if daemons with base connection got path / extra:tcp-ip / extra:atc-ip
	daemons_to_check = ("k601dmn", "kamsdmn", "sbusdmn")
	base_conn_attrs = (add_xmlns("extra")("tcp-ip"), add_xmlns("extra")("atc-ip"), "path")
	curr_daemon_attrs = []
	for single_daemon in daemons_to_check:
		if os.path.basename(daemon).startswith(single_daemon):

			for attr in base_conn_attrs:
				if (device.get(attr)):
					curr_daemon_attrs.append(attr)

			intersec = len(set(base_conn_attrs).intersection(curr_daemon_attrs))

			# throw this code away when sbusdmn has atc-ip handling
			if os.path.basename(daemon).startswith("sbusdmn"):
				extra_atc_ip = add_xmlns('extra', device.get)("atc-ip")
				if extra_atc_ip != None:
					printerr("sbusdmn can't have extra:atc-ip attribute !")

			if intersec == 0:
				printerr("%s must have one of attributes: %s ! (line %d)" % (single_daemon, base_conn_attrs, device.sourceline))
			elif intersec > 1:
				printerr("%s must have ONLY ONE from: %s ! (line %d)" % (single_daemon, base_conn_attrs, device.sourceline))


def check_base(param):
	param_path = add_xmlns('extra', param.get)("param")
	if param_path is not None:
		base = re.search(r"(?P<base>[^:]*):", param_path).group("base")
		base_path = "/opt/szarp/" + base
		if not os.path.exists(base_path):
			printerr("extra:param base %s doesn't exist (line %d)"\
					% (base, param.sourceline))

def check_script(script_path):
	with open(script_path, "r") as script:
		shebang = script.readline()
		if not shebang.startswith("#!"):
			printerr("script %s does not contain shebang in first line"\
				% (script_path))
			return
		shebang_list = shebang.lstrip('#!').split()
		for exec_name in shebang_list:
			if exec_name.startswith('-'):
				continue
			if stat_shebang(exec_name) is False:
				printerr("script %s shebang is nonexecutable"\
								% (script_path))
			if "python" in exec_name and args.additional_test:
				check_python_script(script_path)

def stat_shebang(exec_name):
	return any(
			os.access(os.path.join(path, exec_name), os.X_OK)
			for path in os.environ["PATH"].split(os.pathsep)
	)

def check_python_script(script_path):
	dir_path = os.path.dirname(script_path)
	sys.path.append(dir_path)
	# we need only script name, without extension for __import__
	script_name = os.path.splitext(os.path.basename(script_path))[0]
	try:
		test_module = __import__(script_name)
	except Exception as ex:
		# if pysz{base,4} in message just ignore, it's included in pythondmn
		# other exception generates error
		if not "pysz" in ex.message:
			printerr("testing script %s ended up with error: %s"\
					% (script_path, ex.message))

def check_if_parameter_exist():
	"""
	Main function for going through isl files and check whether parameter exists
	"""
	options = etree.XMLParser(remove_blank_text=True, remove_comments=True)
	params_list = []
	for isl_file in isl_list:
		isl_path = os.path.join(szarp_path, "config", "isl", isl_file)
		xml_file = etree.parse(isl_path, options)
		isl = xml_file.getroot()
		isl_element = list(isl)
		find_parameter_from_isl(isl_element, params_list, isl_file)

invalid_chars = set(['\\', '\"', '<', '>', '`', '~', '*'])
def validate_parameter_name(param):
	name = param.get("name")
	components = name.split(":")
	valid = True
	if not len(components) in range(0,4):
		printerr("Was looking for 1 or 2 separators in param name, found " +
				str(len(components) - 1) + " (line " + str(param.sourceline) + ")")
		valid = False
	for component in components:
		if not component.strip():
			printerr("Parameter name contains empty or whitestring name (line " +
					str(param.sourceline) + ")")
			valid = False
		elif any(char in invalid_chars for char in component):
			printerr("Parameter name contains invalid character (line "
					+ str(param.sourceline) + ")")
			valid = False

	return valid

param_device = []
send_device = []
def get_parameters_from_device():
	for device in add_xmlns('ipk', params_root.iter)("device"):
		for unit in add_xmlns('ipk', device.iter)("unit"):
			for param in add_xmlns('ipk', unit.iter)("param"):
				if validate_parameter_name(param):
					param_device.append(param)
			for send in add_xmlns('ipk', unit.iter)("send"):
				send_device.append(send)

param_defined = []
def get_parameters_from_defined():
	for defined in add_xmlns('ipk', params_root.iter)("defined"):
		for param in add_xmlns('ipk', defined.iter)("param"):
			if validate_parameter_name(param):
				param_defined.append(param)

param_drawdefinable = []
def get_parameters_from_drawdefinable():
	for drawdefinable in add_xmlns('ipk', params_root.iter)("drawdefinable"):
		for param in add_xmlns('ipk', drawdefinable.iter)("param"):
			if validate_parameter_name(param):
				param_drawdefinable.append(param)

def get_param_list():
	get_parameters_from_device()
	get_parameters_from_defined()
	get_parameters_from_drawdefinable()

def check_no_base_ind():
	for param in param_device + param_defined:
		if param.get("base_ind") is None:
			name = unicode_to_ascii(param.get("name"))
			add_note("parameter %s no base_ind, won't be written to szbase (line %d)" % (name, param.sourceline))

def check_params_xmlschema():
	"""
	Testing params with XMLSchema
	"""
	try:
		xsdpath = os.path.join("/opt", "szarp", "resources", "dtd", "ipk-params.xsd")
		xsd = etree.XMLSchema(file=xsdpath)
		try:
			xsd.assertValid(params)
		except BaseException as ex:
			printerr(xsd.error_log)
	except Exception as e:
		printerr(e)

def check_under_defined_elements():
	"""
	Testing define elements for:
		RPN as attribute type of define element
		lua_formula as forbidden attribute in defined parameter
	"""
	for param in param_defined:
		define = add_xmlns('ipk', param.find)("define")
		if define is None:
			continue
		for attr, value in define.items():
			if attr == "type" and value != "RPN":
				printerr('found "' + value + '" as defined formula type, expected "RPN" (line ' + str(define.sourceline) + ')')
			elif attr == "lua_formula":
				printerr('found "' + attr + '" in defined parameter, use "formula" only (line "' + str(define.sourceline) + ')')

def check_under_device_elements():
	"""
	Testing draw elements for: value min/max ratio, valuescace/minscale/maxscale
		ratio
	Testing send elements for: param or value in send element, attribue min/max,
		separated by comma or dot
	"""
	for param in param_device:

		draw = add_xmlns('ipk',param.find)("draw")
		if draw is None:
			continue
		min_value = 0
		max_value = 0
		scale_value = 0
		min_scale_value = 0
		max_scale_value = 0
		for attr, value in draw.items():
			if attr == "min":
				min_value = value
			elif attr == "max":
				max_value = value
			elif attr == "scale":
				scale_value = value
			elif attr == "minscale":
				min_scale_value = value
			elif attr == "maxscale":
				max_scale_value = value
		if float(min_value) > float(max_value):
			add_note("in element draw found wrong min/max ratio (line " + str(draw.sourceline) + ")")
		if float(min_scale_value) > float(max_scale_value) > float(scale_value):
			add_note("in element draw found wrong scale/minscale/maxscale ratio (line " + str(draw.sourceline) + ")")

	send_req_keys = ['param', 'value']
	for send in send_device:
		if not any(key in send_req_keys for key in send.keys()):
			printerr("found send without param or value (line " + str(send.sourceline) + ")")
		send_param = send.get("param")
		if send_param:
			found = False
			for param in param_device + param_defined:
				if send_param in param.get("name"):
					found = True
					break
			if not found:
				printerr("could not find param " + send_param + " to send (line " +
					str(send.sourceline) + ")")
		for attr, value in send.items():
			if attr == "min" or attr == "max":
				if "," in value:
					printerr("min/max in 'send' contains comma, dot symbol requested (line " + str(send.sourceline) + ")")

def find_parameter_from_isl(branch, params_list, isl_file):
	"""
	Recursion function for finding parameter from isl
	"""
	for child_branch in branch:
		for attr, value in child_branch.items():
			if "localhost" in value:
				try:
					internal_parameter = re.search(r"(?<=[0-9]{4}\/).+(?=@)", value)
					parameter_name = str(internal_parameter.group(0))
					if not parameter_name in params_list:
						params_list.append(parameter_name)
					else:
						continue
					if not find_parameter_from_params(parameter_name):
						add_note("parameter \'" + parameter_name + "\' doesn't exist, called in isl:uri in " + isl_file + " (around line " + str(child_branch.sourceline) + ")")
				except:
					add_note("found bad call on isl:uri \'" + value + "\' in " + isl_file + " (around line " + str(child_branch.sourceline) + ")")
		find_parameter_from_isl(child_branch, params_list, isl_file)

def find_parameter_from_params(parameter_name):
	for param in param_list:
		name_to_check = prepare_param_name(param.get("name"))
		if parameter_name in name_to_check:
			return True
	return False

def prepare_param_name(param_name):
	def conv(x):
		if x in [ u"0", u"1", u"2", u"3", u"4", u"5", u"6", u"7", u"8", u"9",
			u"a", u"b", u"c", u"d", u"e", u"f", u"g", u"h", u"i", u"j",
			u"k", u"l", u"m", u"n", u"o", u"p", u"q", u"r", u"s", u"t",
			u"u", u"v", u"w", u"x", u"y", u"z",
			u"A", u"B", u"C", u"D", u"E", u"F", u"G", u"H", u"I", u"J",
			u"K", u"L", u"M", u"N", u"O", u"P", u"Q", u"R", u"S", u"T",
			u"U", u"V", u"W", u"X", u"Y", u"Z"]:
			return x

		if x == u":":
			return "/"

		pmap = { u"ą" : u"a", u"Ą" : u"A", u"ć" : u"c", u"Ć" : u"C",
			u"ę" : u"e", u"Ę" : u"E", u"ł" : u"l", u"Ł" : u"L",
			u"ń" : u"n", u"Ń" : u"N", u"ó" : u"o", u"Ó" : u"O",
			u"ś" : u"s", u"Ś" : u"S", u"ż" : u"z", u"Ż" : u"Z",
			u"ź" : u"z", u"Ź" : u"Z" }

		if x in pmap:
			return pmap[x]

		return u"_"

	return "".join([ conv(x) for x in param_name ])

def do_tests():
	get_param_list()
	check_params_xmlschema()
	check_for_daemons()
	check_under_defined_elements()
	check_under_device_elements()
	check_szarp_cfg()

def test_isl_file():
	check_if_parameter_exist()

def run_ipk2szarp(input_file):
	conf_parser = "/opt/szarp/bin/ipk2szarp"
	command = [conf_parser]
	if input_file:
		command.append(input_file)
	process = Popen(command, stdout=PIPE, stderr=PIPE)
	stdout, stderr = process.communicate()
	for stream in [stdout, stderr]:
		if not stream:
			continue
		for error in stream.strip().splitlines():
			printerr("ipk2szarp: " + error)

def get_szarp_cfg_path():
	# libpar searches for files in this order:
	#	./szarp.cfg
	#	$HOME/szarp.cfg
	#	/etc/szarp/szarp.cfg
	#	/etc/szarp.cfg
	for f in [
		args.szarp_cfg_file,
		"szarp.cfg",
		os.path.expanduser('~') + '/szarp.cfg',
		os.path.realpath("/etc/szarp/szarp.cfg"),
		os.path.realpath("/etc/szarp.cfg")
		]:
		if f != None and os.path.exists(f): return f

	return None

def put_breaked_lines_together_the_way_libpar_does(ss):
	ignoring_hash = False
	ss = ss.split('\n')
	val = ss[0]
	for s in ss[1:]:
		s = s.strip()
		if val[-1] == '\\':
			if s[0] != '#':
				ignoring_hash = True
				val = val[:-1] + s
			else:
				val = val[:-1] + s[1:].strip()
		else:
			val += '\n' + s
	val += '\n'
	if ignoring_hash and not quiet: add_note("Beware using line wrapping ('\\' char) in commented lines in szarp.cfg. It leads to  s t r a n g e  behaviour!")
	return val

def get_item(ss):
	match = re.search('(MENU|EXEC)[(]', ss)
	if match is None:
		return None, ss

	start = match.start()
	brackets = 0	# points to function name first char, so first opening bracket counts
	bracket_idxs = [ x.start() for x in re.finditer('(\(|\))', ss[start:]) ]
	for idx in bracket_idxs:
		if ss[start + idx] == '(':
			brackets += 1
		elif ss[start + idx] == ')':
			brackets -= 1
		if brackets == 0:
			end = start + idx + 1
			return ss[start:end], ss[0:start] + ss[end:]
	if brackets != 0:
		 printerr("Can't find matching ')' for '%s'" % trunc(ss[start:]))
	return None, ss

def get_item_params(ss):
	return re.findall('\"([^"]*)\"', ss)

def exec_item(i):
	p = get_item_params(i)
	if len(p) != 2:
		printerr("ERROR: wrong number of EXEC parameters (%s) for '%s'" % (len(p), trunc(i)))
		return False
	name = p[0]
	comm = p[1].split(' ')[0] # doesn't handle whitespaces in path
	if verbose: print nesting_lvl * '\t' + "Checking for %s ..." % comm,
	if which(comm) == None:
		if verbose: print 'no'
		printerr("Couldn't find '%s' added from %s" % (comm, trunc(i)))
		return False
	else:
		if verbose: print 'yes'
		return True

def menu_item(menu):
	status_ok = True
	global nesting_lvl
	if verbose: print nesting_lvl*'\t' + "Entering '%s'" % trunc(menu)
	nesting_lvl += 1
	if menu.startswidth('MENU'):
		match = re.findall('^MENU\(\"[^"]*\"(.*)\)', menu)
		if len(match) > 0:
			assert len(match) == 1
			menu = match[0]

	item, menu = get_item(menu)
	while item != None:
		if item.startswidth("EXEC"):
			if not exec_item(item): status_ok = False
			pass
		elif item.startswidth("MENU"):
			if not menu_item(item): status_ok = False
		else:
			assert False, 'NONAME:' + item
		item, menu = get_item(menu)
	nesting_lvl -= 1
	return status_ok

def check_szarp_cfg():
	szarp_cfg_path = get_szarp_cfg_path()
	if szarp_cfg_path == None:
		printerr("Cant find szarp.cfg file")
	with open(szarp_cfg_path) as szarp_cfg_file:
		if verbose: print "Reading config from '%s'" % os.path.realpath(szarp_cfg_path)
		data = szarp_cfg_file.read()
		data = put_breaked_lines_together_the_way_libpar_does(data)
	data = ':DUMMY_FIRST_HEADER\n' + data
	data = re.sub('^[:](.*)$', '[\\1]', data, flags=re.MULTILINE)
	parser = ConfigParser.RawConfigParser(allow_no_value=True)
	try:
		parser.readfp(io.BytesIO(data))
		# Python3: parser.read_string(data)
	except ConfigParser.ParsingError as e:
		printnote("File parsing errors - for more info use -v option")
		if verbose: print e

	try:
		menu = parser.get('scc', 'menu')
	except ConfigParser.NoOptionError as e:
		printnote("%s in '%s'" % (str(e), szarp_cfg_path))
		return None

	# depending on parsing errors parser returns list or string
	if type(menu) == list:
		menu = "\n".join(menu)

	if verbose: print 'Read menu value:\n%s' % menu

	status_ok = menu_item(menu)
	if not status_ok: add_note("Errors in '%s' found" % szarp_cfg_path)


# checks if program exists
def which(program):
	def is_exe(fpath):
		return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

	fpath, fname = os.path.split(program)
	if fpath:
		if is_exe(program):
			return program
	else:
		for path in os.environ["PATH"].split(os.pathsep):
			exe_file = os.path.join(path, program)
			if is_exe(exe_file):
				return exe_file
	return None

parser = argparse.ArgumentParser(description='Parses params.xml, szarp.cfg and checks for errors.')
parser.add_argument('-i', '--input-file',
	help='XML config file (default is params.xml)')
parser.add_argument('-a', '--additional-test', action='store_true',
	help='do additional test, including testing isls stored in ' + os.path.realpath("/etc/szarp/default"))
parser.add_argument('-q', '--quiet', action='store_true',
	help="hide not important messages, to be used e.g. in scripts")
parser.add_argument('-v', '--verbose', action='store_true',
	help="print debug info")
parser.add_argument('-j', '--szarp-cfg-file',
	help='szarp config file path (default are ./szarp.cfg, ~/szarp.cfg, /etc/default/szarp.cfg, /etc/szarp.cfg in given order)')

args = parser.parse_args()

run_ipk2szarp(args.input_file)

quiet = args.quiet
verbose = args.verbose
if verbose and not quiet:
	printnote("verbose mode on")

nesting_lvl = 0
if not quiet: add_note("if configuration has changed run: systemctl restart szarp-server.target")
szarp_path, params_path = get_szarp_path()
params = get_params_scheme()
params_root, elements = parse_config_file()
check_svn()
do_tests()
if args.additional_test:
	check_no_base_ind()
	if os.path.exists(os.path.join(szarp_path, "config", "isl")):
		isl_list = get_isl_list()
		test_isl_file()
print_notes()
if error_count == 0:
	if not quiet:
		printok("OK")
else:
	printerr("ERROR - params.xml or szarp.cfg not wellformed, check previous logs!", '')

	sys.exit(1)

