# from emails import *
from django.core.mail import send_mail
from rest_framework.response import Response
from rest_framework import status
from django.template.loader import render_to_string
from rest_framework.views import APIView

class MailView(APIView):
    def post(self, request,type):
        if type=="reject":
            print("reject")
            return self.RejectEmail("mahmoud","mahmoud.elbasiony904@gmail.com",["mahmoud.elbasiony904@gmail.com"])
        elif type =="accept":
            print("accept")
            return self.AcceptEmail("mahmoud","mahmoud.elbasiony904@gmail.com",["mahmoud.elbasiony904@gmail.com"])
        elif type == "interview":
            return self.InterviewEmail("mahmoud","mahmoud.elbasiony904@gmail.com",["mahmoud.elbasiony904@gmail.com"],"https://www.google.com","2023-10-30 10:30 AM")
    @classmethod
    def RejectEmail(cls,name,from_email,recipient_list):
        subject = 'rejection email'
        message = 'We regret to inform you that your application has been rejected because Your application did not meet the required criteria.'
        end_message = "Thank you for your interest."
        html_message = render_to_string('mail.html', {'subject': subject, 'message': message, "end_message": end_message, "name": name})
        send_mail(subject, message, from_email, recipient_list, html_message=html_message)

        return Response(
            {
                "success": True,
                "message": "Mail sent"
            },
            status=status.HTTP_201_CREATED
        )
    @classmethod
    def AcceptEmail(cls,name,from_email,recipient_list):
            subject = 'acceptance email'
            message = 'Congratulations! We are pleased to inform you that your application has been accepted.'
            end_message = "Thank you for your interest and we look forward to having you onboard."
            html_message = render_to_string('mail.html', {'subject': subject, 'message': message, "end_message": end_message, "name": name})
            send_mail(subject, message, from_email, recipient_list, html_message=html_message)
            return Response(
                {
                    "success": True,
                    "message": "Mail sent"
                },
                status=status.HTTP_201_CREATED
            )
    @classmethod
    def InterviewEmail(cls,name,from_email,recipient_list,date,room,meeting_url):
        subject = 'Interview'
        message = 'We would like to invite you to a technical interview'
        end_message = "Thank you for your interest."
        html_message = render_to_string('mail.html', {'subject': subject, 'message': message, "end_message": end_message, "name": name,"meeting_message":meeting_url,"date":date,"room":room})
        send_mail(subject, message, from_email, recipient_list, html_message=html_message)

        return Response(
            {
                "success": True,
                "message": "Mail sent"
            },
            status=status.HTTP_201_CREATED
        )


import matplotlib.pyplot as plt

from die import Die

# Creating two D6 dice.
die_1 = Die()
die_2 = Die()

# Make some roll and store result in a list.
results = []
for roll_num in range(1000):
    result = die_1.roll() * die_2.roll()
    results.append(result)

# Analyze the results.
frequencies = []
max_result = die_1.num_sides + die_2.num_sides
for value in range (2, max_result+1):
    frequency = results.count(value)
    frequencies.append(frequency)

# Visualize the results.
plt.title("Result of rolling two D6 dice 1000 times.")
plt.xticks(["2", '3', '4', '5', '6', '7', '8', '9', '10', '11', '12'])
plt.xlabel("Result")
plt.ylabel("Frequency of Result")
plt.plot(results, frequencies)

plt.show()


import streamlit as st
import requests
import pandas as pd
from io import StringIO

# Function to get TLS version
def get_tls_version(url, headers):
    try:
        with requests.get(url, headers=headers, stream=True) as response:
            st.write(f"Connected to {url}")

            if hasattr(response.raw, 'connection') and hasattr(response.raw.connection, 'sock') and hasattr(response.raw.connection.sock, 'version'):
                tls_version = response.raw.connection.sock.version()
                st.write(f"TLS version for {url}: {tls_version}")
                return tls_version
            else:
                st.write(f"TLS version info not available for {url}")
                return "TLS version info not available"
    except requests.RequestException as e:
        st.write(f"Error connecting to {url}: {e}")
        return f"Error: {e}"

# Function to check TLS versions for a list of URLs and return a DataFrame
def check_tls_versions(urls):
    results = []
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
    }

    for url in urls:
        version = get_tls_version(url, headers)
        results.append({"URL": url, "TLS Version": version})

    return pd.DataFrame(results)

# Streamlit interface
st.title("TLS Version Checker")

# Text area for URL input
url_input = st.text_area("Enter URLs (one per line)")

# Button to check TLS versions
if st.button("Check TLS Versions"):
    urls = url_input.split("\n")  # Split input into a list of URLs
    if urls:
        df_results = check_tls_versions(urls)
        st.write(df_results)

        # CSV Download
        csv = df_results.to_csv(index=False)
        st.download_button(
            label="Download data as CSV",
            data=csv,
            file_name='tls_versions.csv',
            mime='text/csv',
        )
    else:
        st.error("Please enter at least one URL.")


import binascii
import struct
from .util import ofpp, parseInt, get_token, parse_func
from .oxm import str2oxm, oxm2str
from .nx import nxast, register_nxast, NX_VENDOR_ID

align8 = lambda x:(x+7)//8*8

OFPT_FLOW_MOD = 14
OFPT_GROUP_MOD = 15
OFPT_MULTIPART_REQUEST = 18
OFPT_MULTIPART_REPLY = 19

OFPMP_FLOW = 1
OFPMP_GROUP = 6
OFPMP_GROUP_DESC = 7

OFPTT_MAX = 0xfe
OFPTT_ALL = 0xff

OFPCML_MAX = 0xffe5
OFPCML_NO_BUFFER = 0xffff

OFP_NO_BUFFER = 0xffffffff

for num,name in ofpp.items():
	globals()["OFPP_{:s}".format(name.upper())] = num

ofpg = {
	0xffffff00: "max",
	0xfffffffc: "all",
	0xffffffff: "any",
}
for num,name in ofpg.items():
	globals()["OFPG_{:s}".format(name.upper())] = num

ofpff = {
	"send_flow_rem": 1<<0,
	"check_overlap": 1<<1,
	"reset_counts": 1<<2,
	"no_pkt_counts": 1<<3,
	"no_byt_counts": 1<<4 }

for name,num in ofpff.items():
	globals()["OFPFF_{:s}".format(name.upper())] = num

OFPFC_ADD = 0
OFPFC_MODIFY = 1
OFPFC_MODIFY_STRICT = 2
OFPFC_DELETE = 3
OFPFC_DELETE_STRICT = 4

OFPMT_OXM = 1

action_names = {
	0: "output",
	11: "copy_ttl_out",
	12: "copy_ttl_in",
	15: "set_mpls_ttl",
	16: "dec_mpls_ttl",
	17: "push_vlan",
	18: "pop_vlan",
	19: "push_mpls",
	20: "pop_mpls",
	21: "set_queue",
	22: "group",
	23: "set_nw_ttl",
	24: "dec_nw_ttl",
	25: "set_field",
	26: "push_pbb",
	27: "pop_pbb",
	0xffff: "experimenter",
}
for n,name in action_names.items():
	globals()["OFPAT_{:s}".format(name.upper())] = n

ofpgc = ("add", "modify", "delete")
for num, name in enumerate(ofpgc):
	globals()["OFPGC_{:s}".format(name.upper())] = num

ofpgt = ("all", "select", "indirect", "ff")
for num, name in enumerate(ofpgt):
	globals()["OFPGT_{:s}".format(name.upper())] = num

def action_generic_str2act(ofpat):
	def str2act(payload, readarg):
		return struct.pack("!HH4x", ofpat, 8), 0
	return str2act

def action_generic_act2str(name):
	def act2str(payload):
		assert payload == bytearray(4)
		return name
	return act2str

def action_uint_str2act(ofpat, pack):
	def str2act(unparsed, readarg):
		h,b,unparsed = get_token(unparsed)
		num,l = parseInt(b)
		assert l==len(b)
		value = struct.pack(pack, num)
		return struct.pack("!HH", ofpat, 4+len(value))+value, len(h)+len(b)

	return str2act

def action_uint_act2str(fmt, pack):
	def act2str(payload):
		return fmt.format(*struct.unpack_from(pack, payload))
	return act2str

def action_output_str2act(unparsed, readarg):
	h,b,unparsed = get_token(unparsed)
	assert b.find("/")<0
	ps = b.split(":", 2)
	port = None
	for num,name in ofpp.items():
		if name == ps[0]:
			port = num
	if port is None:
		port,l = parseInt(ps[0])
		assert len(ps[0]) == l

	maxLen = OFPCML_NO_BUFFER
	if len(ps) > 1:
		maxLen,l = parseInt(ps[1])
		assert len(ps[1]) == l

	return struct.pack("!HHIH6x", OFPAT_OUTPUT, 16, port, maxLen), len(h) + len(b)

def action_output_act2str(payload):
	(port,maxLen) = struct.unpack("!IH6x", payload)
	name = ofpp.get(port, "{:d}".format(port))
	if port == OFPP_CONTROLLER and maxLen != OFPCML_NO_BUFFER:
		return "output={:s}:{:#x}".format(name, maxLen)
	else:
		return "output={:s}".format(name)

def action_push_str2act(ofpat):
	def str2act(unparsed, readarg):
		h,b,unparsed = get_token(unparsed)
		num, l = parseInt(b)
		return struct.pack("!HHH2x", ofpat, 8, num), len(h)+len(b)
	return str2act

def action_push_act2str(name):
	def act2str(payload):
		num = struct.unpack_from("!H", payload)[0]
		return "{:s}={:#06x}".format(name, num)
	return act2str

_str2act = {}
_act2str = {}

_str2act["output"] = action_output_str2act
_act2str[OFPAT_OUTPUT] = action_output_act2str

_str2act["copy_ttl_out"] = action_generic_str2act(OFPAT_COPY_TTL_OUT)
_act2str[OFPAT_COPY_TTL_OUT] = action_generic_act2str("copy_ttl_out")

_str2act["copy_ttl_in"] = action_generic_str2act(OFPAT_COPY_TTL_IN)
_act2str[OFPAT_COPY_TTL_IN] = action_generic_act2str("copy_ttl_in")

_str2act["set_mpls_ttl"] = action_uint_str2act(OFPAT_SET_MPLS_TTL, "!B3x")
_act2str[OFPAT_SET_MPLS_TTL] = action_uint_act2str("set_mpls_ttl={:d}", "!B3x")

_str2act["dec_mpls_ttl"] = action_generic_str2act(OFPAT_DEC_MPLS_TTL)
_act2str[OFPAT_DEC_MPLS_TTL] = action_generic_act2str("dec_mpls_ttl")

_str2act["push_vlan"] = action_push_str2act(OFPAT_PUSH_VLAN)
_act2str[OFPAT_PUSH_VLAN] = action_push_act2str("push_vlan")

_str2act["pop_vlan"] = action_generic_str2act(OFPAT_POP_VLAN)
_act2str[OFPAT_POP_VLAN] = action_generic_act2str("pop_vlan")

_str2act["push_mpls"] = action_push_str2act(OFPAT_PUSH_MPLS)
_act2str[OFPAT_PUSH_MPLS] = action_push_act2str("push_mpls")

_str2act["pop_mpls"] = action_push_str2act(OFPAT_POP_MPLS)
_act2str[OFPAT_POP_MPLS] = action_push_act2str("pop_mpls")

_str2act["push_mpls"] = action_push_str2act(OFPAT_PUSH_MPLS)
_act2str[OFPAT_PUSH_MPLS] = action_push_act2str("push_mpls")

_str2act["set_queue"] = action_uint_str2act(OFPAT_SET_QUEUE, "!I")
_act2str[OFPAT_SET_QUEUE] = action_uint_act2str("set_queue={:d}", "!I")

_str2act["group"] = action_uint_str2act(OFPAT_GROUP, "!I")
_act2str[OFPAT_GROUP] = action_uint_act2str("group={:d}", "!I")

_str2act["set_nw_ttl"] = action_uint_str2act(OFPAT_SET_NW_TTL, "!B")
_act2str[OFPAT_SET_NW_TTL] = action_uint_act2str("set_nw_ttl={:d}", "!B")

_str2act["dec_nw_ttl"] = action_generic_str2act(OFPAT_DEC_NW_TTL)
_act2str[OFPAT_DEC_NW_TTL] = action_generic_act2str("dec_nw_ttl")

_str2act["push_pbb"] = action_push_str2act(OFPAT_PUSH_PBB)
_act2str[OFPAT_PUSH_PBB] = action_push_act2str("push_pbb")

_str2act["pop_pbb"] = action_generic_str2act(OFPAT_POP_PBB)
_act2str[OFPAT_POP_PBB] = action_generic_act2str("pop_pbb")

register_nxast(_str2act, _act2str)

def str2act(s):
	h,name,arg = get_token(s)
	fname,farg = parse_func(name)

	if farg and fname in _str2act:
		b,p = _str2act[fname](farg, False)
		return bytes(b), len(h)+len(name)
	elif name in _str2act:
		b,p = _str2act[name](arg, True)
		return bytes(b), len(h)+len(name)+p
	elif name.startswith("set_"):
		if farg:
			op,payload,s = get_token(farg)
			oxm,p = str2oxm(fname[4:]+"="+payload, loop=False)
			consumed_length = len(h)+len(name)
		else:
			op,payload,s = get_token(arg)
			oxm,p = str2oxm(name[4:]+op+payload, loop=False)
			consumed_length = len(h)+4+p

		l = align8(len(oxm)+4)
		ret = bytearray(l)
		ret[:4] = struct.pack("!HH", OFPAT_SET_FIELD, l)
		ret[4:4+len(oxm)] = oxm
		return bytes(ret), consumed_length
	else:
		return b"", len(h)

def act2str(msg, loop=True):
	tokens = []
	while len(msg) > 4:
		(atype,l) = struct.unpack_from("!HH", msg)
		offset = 4
		if atype == OFPAT_EXPERIMENTER:
			vendor = struct.unpack_from("!I", msg, 4)[0]
			if vendor == NX_VENDOR_ID:
				atype = nxast(struct.unpack_from("!H", msg, 8)[0])
				offset = 10

		act = _act2str.get(atype)
		if atype == OFPAT_SET_FIELD:
			tokens.append("set_"+oxm2str(msg[4:], loop=False))
		elif act:
			tokens.append(act(msg[offset:l]))
		else:
			tokens.append("?")

		if loop:
			msg = msg[l:]
		else:
			break

	return ",".join(tokens)

instruction_names = {
	1: "goto_table",
	2: "write_metadata",
	3: "write_actions",
	4: "apply_actions",
	5: "clear_actions",
	6: "meter"
}
for n,name in instruction_names.items():
	globals()["OFPIT_{:s}".format(name.upper())] = n

def inst2str(msg, loop=True):
	tokens = []
	while len(msg) > 4:
		(itype,l) = struct.unpack_from("!HH", msg)
		if itype == OFPIT_GOTO_TABLE:
			assert l==8
			tokens.append("@goto={:d}".format(*struct.unpack_from("!B", msg, 4)))
		elif itype == OFPIT_WRITE_METADATA:
			assert l==24
			(v,m) = struct.unpack_from("!QQ", msg, 8)
			if m == 0:
				tokens.append("@metadata={:#x}".format(v))
			else:
				tokens.append("@metadata={:#x}/{:#x}".format(v,m))
		elif itype == OFPIT_WRITE_ACTIONS:
			assert l%8==0
			tokens.append("@write")
			arg = act2str(msg[8:l])
			if len(arg):
				tokens.append(arg)
		elif itype == OFPIT_APPLY_ACTIONS:
			assert l%8==0
			tokens.append("@apply")
			arg = act2str(msg[8:l])
			if len(arg):
				tokens.append(arg)
		elif itype == OFPIT_CLEAR_ACTIONS:
			assert l == 8
			tokens.append("@clear")
		elif itype == OFPIT_METER:
			assert l == 8, repr(msg)
			tokens.append("@meter={:d}".format(*struct.unpack_from("!I", msg, 4)))
		else:
			tokens.append("?")

		if loop:
			msg = msg[l:]
		else:
			break

	return ",".join(tokens)

PHASE_MATCH = 0
PHASE_ACTION = 1
PHASE_NOARG = 2

def str2dict(s):
	'''convert a string into a flow rule information dictionary'''
	ret = dict(
		match= b"",
		inst= b"",
	)

	actions = b""
	def inst_action(atype):
		def func():
			ret["inst"] += struct.pack("!HH4x", atype, 8+len(actions))+actions
		return func

	func = None
	phase = PHASE_MATCH
	while len(s) > 0:
		h,name,s = get_token(s)
		assert h.find("=")<0
		if name.startswith("@"):
			if func is not None:
				func()

			func = None
			phase = PHASE_NOARG
			if name in ("@goto", "@goto_table"):
				op,payload,s = get_token(s)
				assert op.find("=")>=0, "goto requires arg"
				num,l = parseInt(payload)
				assert l == len(payload)
				ret["inst"] += struct.pack("!HHB3x", OFPIT_GOTO_TABLE, 8, num)
			elif name in ("@metadata", "@write_metadata"):
				op,payload,s = get_token(s)
				assert op.find("=")>=0, "metadata requires arg"
				vm = payload.split("/", 1)
				num,l = parseInt(vm[0])
				assert l == len(vm[0])
				if len(vm) > 1:
					mask,l = parseInt(vm[1])
					assert l == len(vm[1])
					ret["inst"] += struct.pack("!HH4xQQ", OFPIT_WRITE_METADATA, 24, num, mask)
				else:
					ret["inst"] += struct.pack("!HH4xQQ", OFPIT_WRITE_METADATA, 24, num, 0)
			elif name in ("@apply", "@apply_actions"):
				func = inst_action(OFPIT_APPLY_ACTIONS)
				actions = b""
				phase = PHASE_ACTION
			elif name in ("@write", "@write_actions"):
				func = inst_action(OFPIT_WRITE_ACTIONS)
				actions = b""
				phase = PHASE_ACTION
			elif name in ("@clear", "@clear_actions"):
				ret["inst"] += struct.pack("!HH4x", OFPIT_CLEAR_ACTIONS, 8)
			elif name == "@meter":
				op,payload,s = get_token(s)
				assert op.find("=")>=0, "meter requires arg"
				assert payload.find("/") < 0, "meter does not take mask"
				num,l = parseInt(payload)
				assert l == len(payload)
				ret["inst"] += struct.pack("!HHI", OFPIT_METER, 8, num)
			else:
				raise ValueError("unknown {:s}".format(name))
		elif phase == PHASE_MATCH:
			def proc(field):
				op,payload,unparsed = get_token(s)
				assert op.find("=")>=0 and payload.find("/")<0
				num,l = parseInt(payload)
				assert len(payload) == l
				ret[field] = num
				return unparsed
			if name in ("table", "priority", "idle_timeout", "hard_timeout", "buffer"):
				s = proc(name)
			elif name in ofpff:
				ret["flags"] = ret.get("flags", 0) | ofpff[name]
			elif name == "cookie":
				op,payload,s = get_token(s)
				assert op.find("=")>=0, "cookie take value"
				vm = payload.split("/", 1)
				num,l = parseInt(vm[0])
				assert len(vm[0]) == l
				ret[name] = num
				if len(vm) > 1:
					num,l = parseInt(vm[1])
					ret["cookie_mask"] = num
			elif name == "out_port":
				op,payload,s = get_token(s)
				assert op.find("=") >= 0 and payload.find("/") < 0
				port = None
				for num,pname in ofpp.items():
					if pname == payload:
						port = num
				if port is None:
					port,l = parseInt(payload)
					assert l == len(payload)
				ret[name] = port
			elif name == "out_group":
				op,payload,s = get_token(s)
				assert op.find("=")>=0 and payload.find("/")<0
				port = None
				for num,gname in ofpg.items():
					if gname == payload:
						port = num
				if port is None:
					port,l = parseInt(payload)
					assert l == len(payload)
				ret[name] = port
			else:
				oxm, l = str2oxm(name+s, loop=False)
				if l == 0:
					raise ValueError("unknown match {:s}".format(s))
				ret["match"] += oxm
				s = (name+s)[l:]
		elif phase == PHASE_ACTION:
			act, l = str2act(name+s)
			if l == 0:
				raise ValueError("unknown action {:s}".format(s))
			actions += act
			s = (name+s)[l:]
		else:
			raise ValueError("invalid syntax")
	if func:
		func()

	return ret

def _fixed(default, **kwargs):
	'''
	@param default parameters the is equal in this will be suppressed
	'''
	def emit(name):
		return name in kwargs and kwargs[name] != default.get(name)

	ret = []
	if emit("cookie_mask"):
		ret.append("cookie={:#x}/{:#x}".format(kwargs["cookie"], kwargs["cookie_mask"]))
	elif emit("cookie"):
		ret.append("cookie={:#x}".format(kwargs["cookie"]))

	if emit("table"):
		ret.append("table={:d}".format(kwargs["table"]))

	if emit("priority"):
		ret.append("priority={:d}".format(kwargs["priority"]))

	if emit("buffer"):
		ret.append("buffer={:#x}".format(kwargs["buffer"]))

	if emit("out_port"):
		out_port = kwargs["out_port"]
		if out_port in ofpp:
			ret.append("out_port={:s}".format(ofpp[out_port]))
		else:
			ret.append("out_port={:d}".format(out_port))

	if emit("out_group"):
		out_group = kwargs["out_group"]
		if out_group in ofpg:
			ret.append("out_group={:s}".format(ofpg[out_group]))
		else:
			ret.append("out_group={:d}".format(out_group))

	if emit("idle_timeout"):
		ret.append("idle_timeout={:d}".format(kwargs["idle_timeout"]))

	if emit("hard_timeout"):
		ret.append("hard_timeout={:d}".format(kwargs["hard_timeout"]))

	if emit("flags"):
		for name,num in ofpff.items():
			if kwargs["flags"] & num:
				ret.append(name)

	return ret

def bucket2str(msg, group_type=OFPGT_ALL):
	(length,
	weight,
	watch_port,
	watch_group) = struct.unpack_from("!HHII4x", msg)

	ret = []
	if group_type == OFPGT_SELECT and weight != 1:
		ret.append("weight={:d}".format(weight))
	if watch_port != OFPP_ANY:
		ret.append("watch_port={:d}".format(watch_port))
	if watch_group != OFPG_ANY:
		ret.append("watch_group={:d}".format(watch_group))

	a = act2str(msg[16:length])
	if a:
		ret.append(a)

	return ",".join(ret)

def str2bucket(s, group_type=OFPGT_ALL):
	info = dict(
		weight=0,
		watch_port=OFPP_ANY,
		watch_group=OFPG_ANY)
	if group_type == OFPGT_SELECT:
		info["weight"] = 1

	actions = b""
	unparsed = s
	while len(unparsed):
		h,name,r = get_token(unparsed)
		assert h.find("=") < 0
		if name.startswith("@"):
			break
		elif name in ("weight", "watch_port", "watch_group"):
			op,payload,unparsed = get_token(r)
			assert op.find("=")>=0
			num,l = parseInt(payload)
			assert len(payload) == l
			info[name] = num
		else:
			b,l = str2act(unparsed)
			if l > 0:
				actions += b
				unparsed = unparsed[l:]
			else:
				break

	consumed_length = len(s) - len(unparsed)
	if consumed_length == 0:
		return b"", 0
	return struct.pack("!HHII4x",
		16 + len(actions),
		info["weight"],
		info["watch_port"],
		info["watch_group"]) + actions, consumed_length

# OFPT_FLOW_MOD

ofpfc_del_default = dict(
	cookie = 0,
	cookie_mask = 0,
	table = OFPTT_ALL,
	idle_timeout = 0,
	hard_timeout = 0,
	priority = 0x8000,
	buffer = 0,
	out_port = OFPP_ANY,
	out_group = OFPG_ANY,
	flags = 0,
)

ofpfc_default = dict(
	cookie = 0,
	cookie_mask = 0,
	table = 0,
	idle_timeout = 0,
	hard_timeout = 0,
	priority = 0x8000,
	buffer = OFP_NO_BUFFER,
	out_port = 0,
	out_group = 0,
	flags = 0,
)

def str2mod(s, command=OFPFC_ADD, xid=0):
	default = ofpfc_default
	if command in (OFPFC_DELETE, OFPFC_DELETE_STRICT):
		default = ofpfc_del_default

	info = dict(default)
	info.update(str2dict(s))

	OFPMT_OXM = 1
	oxm = info.get("match", b"")
	length = 4 + len(oxm)
	match = struct.pack("!HH", OFPMT_OXM, length) + oxm
	match += b"\0" * (align8(length)-length)

	inst = info.get("inst", b"")

	return struct.pack("!BBHIQQBBHHHIIIH2x", 4, OFPT_FLOW_MOD, 48+align8(length)+len(inst), xid,
		info["cookie"],
		info["cookie_mask"],
		info["table"],
		command,
		info["idle_timeout"],
		info["hard_timeout"],
		info["priority"],
		info["buffer"],
		info["out_port"],
		info["out_group"],
		info["flags"])+match+inst

def mod2str(msg):
	(hdr_version, hdr_type, hdr_length, hdr_xid,
	cookie,
	cookie_mask,
	table,
	cmd,
	idle_timeout,
	hard_timeout,
	priority,
	buffer_id,
	out_port,
	out_group,
	flags,
	match_type,
	match_length) = struct.unpack_from("!BBHIQQBBHHHIIIH2xHH", msg)

	default = ofpfc_default
	if cmd in (OFPFC_DELETE, OFPFC_DELETE_STRICT):
		default = ofpfc_del_default

	ret = _fixed(default,
		cookie=cookie,
		cookie_mask=cookie_mask,
		table=table,
		priority=priority,
		buffer=buffer_id,
		out_port=out_port,
		out_group=out_group,
		idle_timeout=idle_timeout,
		hard_timeout=hard_timeout,
		flags=flags)

	if match_type == OFPMT_OXM:
		rstr = oxm2str(msg[52:52+match_length-4])
		if len(rstr):
			ret.append(rstr)
	else:
		raise ValueError("match_type {:d} not supported".format(match_type))

	istr = inst2str(msg[48+align8(match_length):hdr_length])
	if len(istr):
		ret.append(istr)

	return ",".join(ret)

def mod2extra(msg):
	(hdr_version, hdr_type, hdr_length, hdr_xid,
	cookie,
	cookie_mask,
	table,
	cmd) = struct.unpack_from("!BBHIQQBB", msg)
	if cmd != OFPFC_ADD:
		return dict(command=cmd, xid=hdr_xid)
	return dict(xid=hdr_xid)

# OFPT_GROUP_MOD

def str2group(s, command=OFPGC_ADD, xid=0):
	group_id=OFPG_ANY
	group_type=OFPGT_ALL

	unparsed = s
	while len(unparsed) > 0:
		h,name,r = get_token(unparsed)
		if name == "group":
			op,payload,r = get_token(r)
			if payload.lower() == "all":
				group_id = OFPG_ALL
			else:
				num,l = parseInt(payload)
				assert len(payload) == l
				group_id = num

			unparsed = r
		elif name in ofpgt:
			group_type = ofpgt.index(name)
			unparsed = r
		else:
			break

	buckets = b""
	info = {}
	while len(unparsed) > 0:
		b, l = str2bucket(unparsed, group_type=group_type)
		if l:
			buckets += b
			unparsed = unparsed[l:]
		else:
			h,name,r = get_token(unparsed)
			if not name or name == "@bucket":
				unparsed = r
			else:
				break

	assert not unparsed, unparsed

	return struct.pack("!BBHIHBxI",
		4, OFPT_GROUP_MOD, 16+len(buckets), xid,
		command, group_type, group_id) + buckets

def group2str(msg):
	(hv, ht, hl, xid,
	command, group_type, group_id) = struct.unpack_from("!BBHIHBxI", msg)

	if group_id == OFPG_ALL:
		ret = ["group=all"]
	elif group_id == OFPG_ANY:
		ret = []
	else:
		ret = ["group={:d}".format(group_id)]

	ret.append(ofpgt[group_type])

	bin = msg[16:]
	while len(bin) > 16:
		(l,) = struct.unpack_from("!H", bin)
		ret += ["@bucket",
			bucket2str(bin[:l], group_type=group_type)]
		bin = bin[l:]

	return ",".join(ret)

def group2extra(msg):
	(hdr_version, hdr_type, hdr_length, hdr_xid,
	command,
	group_type,
	group_id) = struct.unpack_from("!BBHIHBxI", msg)

	assert hdr_type == OFPT_GROUP_MOD

	ret = dict(xid=hdr_xid, group_id=group_id)
	if command != OFPGC_ADD:
		ret["command"] = command
	return ret

# OFPMP_FLOW / OFPT_MULTIPART_*

def text2mpflow(txt, type=OFPT_MULTIPART_REPLY, xid=0):
	def parse(s):
		ret = {}
		while len(s) > 0:
			h,name,r = get_token(s)
			if name not in ("packet_count", "byte_count", "duration_sec", "duration_nsec"):
				break

			op,payload,s = get_token(r)
			assert op.find("=")>=0 and payload.find("/")<0
			num,l = parseInt(payload)
			assert l == len(payload)
			ret[name] = num

		ret.update(str2dict(s))
		return ret

	rules = [parse(s) for s in txt.split("\n")]

	if type!=OFPT_MULTIPART_REPLY and not rules:
		rules = [{}]

	msgs = b""
	capture = b""
	for info in rules:
		oxm = info["match"]
		length = 4 + len(oxm)
		match = struct.pack("!HH", OFPMT_OXM, length) + oxm
		match += b"\0" * (align8(length)-length)

		inst = info["inst"]

		body = b""
		if type!=OFPT_MULTIPART_REPLY:
			body = struct.pack("!B3xII4xQQ",
				info.get("table", OFPTT_ALL),
				info.get("out_port", OFPP_ANY),
				info.get("out_group", OFPG_ANY),
				info.get("cookie", 0),
				info.get("cookie_mask", 0)) + match
		else:
			body = struct.pack("!HBxIIHHHH4xQQQ",
				48 + len(match) + len(inst),
				info.get("table", 0),
				info.get("duration_sec", 0),
				info.get("duration_nsec", 0),
				info.get("priority", 0),
				info.get("idle_timeout", 0),
				info.get("hard_timeout", 0),
				info.get("flags", 0),
				info.get("cookie", 0),
				info.get("packet_count", 0),
				info.get("byte_count", 0)) + match + inst

		if len(capture) + len(body) > 0xffff - 16:
			flag = OFPMPF_REPLY_MORE
			if type==OFPT_MULTIPART_REQUEST:
				flag = OFPMPF_REQ_MORE
			msgs += struct.pack("!BBHIHH4x",
				4, type, 16+len(capture), xid,
				OFPMP_FLOW, flag)+capture
			capture = body
		else:
			capture += body

	msgs += struct.pack("!BBHIHH4x",
		4, type, 16+len(capture), xid,
		OFPMP_FLOW, 0) + capture

	return msgs

def mpflow2text_one(msg):
	(hdr_version, hdr_type, hdr_length, hdr_xid,
	mp_type,
	mp_flags) = struct.unpack_from("!BBHIHH4x", msg)

	assert mp_type == OFPMP_FLOW, "OFPMP_FLOW required"

	rules = []
	body = msg[16:hdr_length]
	if hdr_type != OFPT_MULTIPART_REPLY:
		(table_id,
		out_port,
		out_group,
		cookie,
		cookie_mask,
		match_type,
		match_length) = struct.unpack_from("!B3xII4xQQHH", body)
		defaults = dict(
			table = OFPTT_ALL,
			out_port = OFPP_ANY,
			out_group = OFPG_ANY,
			cookie=0,
			cookie_mask=0)
		ret = _fixed(defaults,
			table=table_id,
			out_port=out_port,
			out_group=out_group,
			cookie=cookie,
			cookie_mask=cookie_mask)
		if match_type == OFPMT_OXM:
			rstr = oxm2str(body[36:36+match_length-4])
			if len(rstr):
				ret.append(rstr)
		else:
			raise ValueError("match_type {:d} not supported".format(match_type))
		rules.append(flow=",".join(ret))
	else:
		while len(body) >= 56:
			(length,
			table_id,
			duration_sec,
			duration_nsec,
			priority,
			idle_timeout,
			hard_timeout,
			flags,
			cookie,
			packet_count,
			byte_count,
			match_type,
			match_length) = struct.unpack_from("!HBxIIHHHH4xQQQHH", body)

			defaults = dict(
				cookie = 0,
				table = 0,
				priority = 0x8000,
				idle_timeout = 0,
				hard_timeout = 0,
				flags = 0)
			ret = _fixed(defaults,
				cookie=cookie,
				table=table_id,
				priority=priority,
				idle_timeout=idle_timeout,
				hard_timeout=hard_timeout,
				flags=flags)

			if match_type == OFPMT_OXM:
				rstr = oxm2str(body[52:52+match_length-4])
				if len(rstr):
					ret.append(rstr)
			else:
				raise ValueError("match_type {:d} not supported".format(match_type))

			istr = inst2str(body[48+align8(match_length):length])
			if len(istr):
				ret.append(istr)

			c = []
			L = locals()
			for name in ("packet_count", "byte_count", "duration_sec", "duration_nsec"):
				if L[name]:
					c.append("{:s}={:d}".format(name, L[name]))

			rules.append(" ".join([",".join(x) for x in [c, ret] if x]))
			body = body[length:]
	return rules

def mpflow2text(msgs):
	rows = []
	while msgs:
		(hv, ht, hl, xid) = struct.unpack_from("!BBHI", msgs)
		rows += mpflow2text_one(msgs[:hl])
		msgs = msgs[hl:]
	return "\n".join(rows)

# OFPMP_GROUP_DESC / OFPT_MULTIPART_*

def text2mpgroupdesc(txt, type=OFPT_MULTIPART_REPLY, xid=0):
	if type != OFPT_MULTIPART_REPLY:
		return struct.pack("!BBHIHH4x",
			4, type, 16, xid, OFPMP_GROUP_DESC, 0)

	msgs = b""
	capture = b""
	for s in txt.split("\n"):
		group_id = 0
		group_type = OFPGT_ALL

		ret = {}
		while len(s) > 0:
			h,name,r = get_token(s)
			if name == "group":
				op,payload,s = get_token(r)
				assert op.find("=")>=0 and payload.find("/")<0
				num,l = parseInt(payload)
				assert l == len(payload)
				group_id = num
			elif name.lower() in ofpgt:
				group_type = ofpgt.index(name.lower())
				s = r
			else:
				break

		bs = b""
		while len(s) > 0:
			h,name,r = get_token(s)
			if name == "@bucket":
				s = r
				continue

			b,l = str2bucket(s, group_type=group_type)
			if l:
				s = s[l:]
				bs += b
			else:
				break

		body = struct.pack("!HBxI",
			8+len(bs), group_type, group_id) + bs

		if len(body)+len(capture) > 0xffff - 16:
			msgs += struct.pack("!BBHIHH4x",
				4, type, 16+len(capture), xid,
				OFPMP_GROUP_DESC,
				OFPMPF_REPLY_MORE)+capture
			capture = body
		else:
			capture += body
	msgs += struct.pack("!BBHIHH4x",
		4, type, 16+len(capture), xid,
		OFPMP_GROUP_DESC,
		0) + capture
	return msgs

def mpgroupdesc2text_one(msg):
	(hdr_version, hdr_type, hdr_length, hdr_xid,
	mp_type,
	mp_flags) = struct.unpack_from("!BBHIHH4x", msg)

	assert mp_type == OFPMP_GROUP_DESC, "OFPMP_GROUP_DESC required"

	if hdr_type != OFPT_MULTIPART_REPLY:
		return []

	groups = []
	r = msg[16:hdr_length]
	while len(r) >= 8:
		(length, group_type, group_id) = struct.unpack_from("!HBxI", r)
		assert length>=8

		ret = ["group={:d}".format(group_id),
			ofpgt[group_type]]
		bin = r[8:length]
		while len(bin) >= 16:
			(l,) = struct.unpack_from("!H", bin)
			ret += ["@bucket",
				bucket2str(bin[:l], group_type=group_type)]
			bin = bin[l:]

		groups.append(",".join(ret))
		r = r[length:]

	return groups

def mpgroupdesc2text(msgs):
	rows = []
	while msgs:
		(hv, ht, hl, xid) = struct.unpack_from("!BBHI", msgs)
		rows += mpgroupdesc2text_one(msgs[:hl])
		msgs = msgs[hl:]
	return "\n".join(rows)

# OFPMP_GROUP / OFPT_MULTIPART_*

def text2mpgroup(txt, type=OFPT_MULTIPART_REPLY, xid=0):
	msgs = b""
	capture = b""
	for s in txt.split("\n"):
		group_id = 0

		while len(s) > 0:
			h,name,r = get_token(s)
			if name == "group":
				op,payload,s = get_token(r)
				assert op.find("=")>=0 and payload.find("/")<0
				num,l = parseInt(payload)
				assert l == len(payload)

				group_id = num
			else:
				break

		if type != OFPT_MULTIPART_REPLY:
			capture = struct.pack("!I4x", group_id)
			break

		group = {}
		buckets = []
		while len(s) > 0:
			h,name,r = get_token(s)
			if name == "@bucket":
				buckets.append({})
				s = r
				continue

			op,payload,s = get_token(r)
			assert op.find("=")>=0 and payload.find("/")<0
			num,l = parseInt(payload)
			assert l == len(payload)

			if buckets:
				if name in ("packet_count", "byte_count"):
					buckets[-1][name] = num
				else:
					break
			elif name in ("ref_count", "packet_count", "byte_count", "duration_sec", "duration_nsec"):
				group[name] = num
			else:
				break

		bs = b"".join([struct.pack("!QQ",
			b.get("packet_count", 0), b.get("byte_count", 0)) for b in buckets])

		body = struct.pack("!H2xII4xQQII",
			40+len(bs), group_id,
			group.get("ref_count", 0),
			group.get("packet_count", 0),
			group.get("byte_count", 0),
			group.get("duration_sec", 0),
			group.get("duration_nsec", 0)) + bs

		if len(body)+len(capture) > 0xffff - 16:
			msgs += struct.pack("!BBHIHH4x",
				4, type, 16+len(capture), xid,
				OFPMP_GROUP,
				OFPMPF_REPLY_MORE) + capture
			capture = body
		else:
			capture += body
	msgs += struct.pack("!BBHIHH4x",
		4, type, 16+len(capture), xid,
		OFPMP_GROUP,
		0) + capture
	return msgs

def mpgroup2text_one(msg):
	(hdr_version, hdr_type, hdr_length, hdr_xid,
	mp_type,
	mp_flags) = struct.unpack_from("!BBHIHH4x", msg)

	assert mp_type == OFPMP_GROUP, "OFPMP_GROUP required"

	if hdr_type != OFPT_MULTIPART_REPLY:
		(group_id,) = struct.unpack_from("!I4x", msg, 16)
		if group_id == OFPG_ALL:
			return [] # default
		elif group_id == OFPG_ANY:
			return ["group=any"]
		else:
			return ["group={:d}".format(group_id)]

	groups = []
	r = msg[16:hdr_length]

	while len(r) >= 40:
		(length, group_id, ref_count,
		packet_count, byte_count,
		duration_sec, duration_nsec) = struct.unpack_from("!H2xII4xQQII", r)
		assert length>=40

		group = []
		L = locals()
		for k in ("ref_count", "packet_count", "byte_count",
				"duration_sec", "duration_nsec"):
			if L[k]:
				group.append("{:s}={:d}".format(k, L[k]))

		bs = r[40:length]
		while len(bs) >= 16:
			group.append("@bucket")
			(pc,bc) = struct.unpack_from("!QQ", bs)
			if pc != 0:
				group.append("packet_count={:d}".format(pc))
			if bc != 0:
				group.append("byte_count={:d}".format(bc))
			bs = bs[16:]

		stat = ",".join(group)
		if group_id == OFPG_ALL:
			groups.append("group=all " + stat)
		elif group_id == OFPG_ANY:
			groups.append(stat)
		else:
			groups.append("group={:d} ".format(group_id) + stat)

		r = r[length:]

	return groups

def mpgroup2text(msgs):
	rows = []
	while msgs:
		(hv, ht, hl, xid) = struct.unpack_from("!BBHI", msgs)
		rows += mpgroup2text_one(msgs[:hl])
		msgs = msgs[hl:]
	return "\n".join(rows)


"""
General Game Playing script. Trains a Tangled Program Graph model on multiple games.
Created by: Ryan Amaral
Created on: Feb. 25, 2019 (based on much earlier version)
"""

from __future__ import division
from tpg.tpg_trainer import TpgTrainer
from tpg.tpg_agent import TpgAgent
from optparse import OptionParser
from tpg_helpers import *
import random
import datetime
import multiprocessing as mp
import pickle
import time

"""
Command line arguments.
"""

def separgs(option, opt, value, parser):
    setattr(parser.values, option.dest, value.split(','))

parser = OptionParser()
# number of games to play
parser.add_option('-n', '--ngames', type='int', dest='numGames', default=16)
# number of previous titles to measure fitness on
parser.add_option('-f', '--fitGamesNum', type='int', dest='fitGamesNum', default=-1)
# specify any games
parser.add_option('--games', type='string', action='callback', callback=separgs, dest='games')
# choose games randomly
parser.add_option('-r', '--randg', action='store_true', dest='randomGames', default=False)
# starting tpg population size
parser.add_option('-p', '--pop', type='int', dest='popSize', default=360)
# population size to work up to
parser.add_option('-m', '--popMax', type='int', dest='popSizeMax', default=360)
# concurrent workers for parallization
parser.add_option('-w', '--workers', type='int', dest='workers', default=3)
# method of training
parser.add_option('-t', '--type', type='int', dest='trainType', default=0)
# number of frames to train on
parser.add_option('--trainFrames', type='int', dest='trainFrames', default=18000)
# number of frames to test on
parser.add_option('--testFrames', type='int', dest='testFrames', default=18000)
# number of episodes to train on
parser.add_option('--trainEps', type='int', dest='trainEps', default=3)
# number of episodes to train on
parser.add_option('--testEps', type='int', dest='testEps', default=10)
# every how many generations to do champ eval
parser.add_option('-c', '--champEvalGen', type='int', dest='champEvalGen', default=50)
# file for trainer with pretrained populations
parser.add_option('--popsFile', type='string', dest='popsFile', default=None)

# trainType is 0 for 'all at once', 1 for 'merge', 2 for 'gradual merge'.

(options, args) = parser.parse_args()

"""
General setup for GGP.
"""

# all of the titles we will be general game playing on
# we chose games that we know TPG does at-least OK in alone
allEnvNames = ['Alien-v0','Asteroids-v0','Atlantis-v0','BankHeist-v0',
               'BattleZone-v0','Bowling-v0','Boxing-v0','Centipede-v0',
               'ChopperCommand-v0','DoubleDunk-v0','FishingDerby-v0',
               'Freeway-v0','Frostbite-v0','Gravitar-v0','Hero-v0',
               'IceHockey-v0','Jamesbond-v0','Kangaroo-v0','Krull-v0',
               'KungFuMaster-v0','MsPacman-v0','PrivateEye-v0',
               'RoadRunner-v0','Skiing-v0','Tennis-v0','TimePilot-v0',
               'UpNDown-v0','Venture-v0','WizardOfWor-v0','Zaxxon-v0']

if options.games is not None:
    allEnvNames = sorted(options.games)
else:
    # shuffle if random and take certain number of games
    if options.randomGames:
        random.shuffle(allEnvNames)
    allEnvNames = sorted(allEnvNames[:options.numGames])
    #allEnvNames = ['Assault-v0', 'Boxing-v0'] # hardcode in games to use

print('All Games: ' + str(allEnvNames))

pool = mp.Pool(processes=options.workers, initializer=limit_cpu, maxtasksperchild=1)
man = mp.Manager() # manager for shared memory lists

"""
Create and initialize log files.
"""

timeStamp = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")

# game score summary of each game in each generation
logFileGameScoresName = 'ggp-log-gamescores-' + timeStamp + '.txt'
with open(logFileGameScoresName, 'a') as f:
    f.write('tpgGen,hoursElapsed,envName,tpgMin,tpgMax,tpgAvg,eliteSize,eliteUid\n')

# every few generations evaluate top overall few teams on all titles
logFileChampionsName = 'ggp-log-champions-' + timeStamp + '.txt'
with open(logFileChampionsName, 'a') as f:
    f.write('tpgGen,hoursElapsed,teamId')
    for envName in allEnvNames:
        f.write(',score' + envName)
    for envName in allEnvNames:
        f.write(',vis' + envName)
    f.write(',visTotal\n')

# every few generations, save the outcomes of the root teams
logFileOutcomesName = 'ggp-log-outcomes-' + timeStamp + '.txt'
with open(logFileOutcomesName, 'a') as f:
    f.write('tpgGen,hoursElapsed,teamId')
    for envName in allEnvNames:
        f.write(',score' + envName)
    f.write('\n')

# population fitness summary each generation
logFileFitnessName = 'ggp-log-fitness-' + timeStamp + '.txt'
with open(logFileFitnessName, 'a') as f:
    f.write('tpgGen,hoursElapsed,envs,fitMin,fitMax,fitAvg,champSize,champUid,totalTeams,totalRootTeams\n')

# and create file to save TPG
trainerFileName = 'tpg-trainer-' + timeStamp + '.pkl'

"""
Method for training TPG on all games at once. Each individual in the single population
will see all of the games before evolution will occur.
"""
def ggpTrainAllAtOnce():

    # create TPG
    trainer = TpgTrainer(actions=range(18), teamPopSize=options.popSize,
                    rTeamPopSize=options.popSize, maxProgramSize=128)

    envNamesSrt = sorted(list(allEnvNames)) # for reporting envs played

    fitnessEnvs = []

    while True: # train indefinately
        print('TPG Gen: ' + str(trainer.populations[None].curGen))

        if options.fitGamesNum < options.numGames and options.fitGamesNum > 0:
            fitnessEnvs.append(random.choice(list(set(allEnvNames)-set(fitnessEnvs))))
            if len(fitnessEnvs) > options.fitGamesNum:
                fitnessEnvs.pop(0)
        else:
            fitnessEnvs = list(allEnvNames)

        for envName in fitnessEnvs:
            print('Playing Game: ' + envName)

            scoreList = man.list()

            # run all agents on env
            pool.map(runAgent,
                [(agent, envName, scoreList, options.trainEps, options.trainFrames, None)
                    for agent in trainer.getAllAgents(skipTasks=[envName], noRef=True)])

            trainer.applyScores(scoreList)

            # report curEnv results to log
            scoreStats = trainer.getTaskScores(envName)
            bestTeam = trainer.getBestTeams(tasks=[envName])[0]
            with open(logFileGameScoresName, 'a') as f:
                f.write(str(trainer.populations[None].curGen) + ','
                    + str((time.time()-tstart)/3600) + ','
                    + envName + ','
                    + str(scoreStats['min']) + ','
                    + str(scoreStats['max']) + ','
                    + str(scoreStats['average']) +  ','
                    + str(len(bestTeam.getRootTeamGraph()[0])) + ','
                    + str(bestTeam.uid) + '\n')

        # do evolution after all envs played
        trainer.multiEvolve(tasks=[fitnessEnvs]+[[en] for en in fitnessEnvs],
                            weights=[0.5]+[0.5/len(fitnessEnvs) for _ in fitnessEnvs],
                            fitMethod='min', elitistTasks=allEnvNames)

        # report generational fitness results
        bestTeam = trainer.getBestTeams(tasks=fitnessEnvs)[0]
        with open(logFileFitnessName, 'a') as f:
            f.write(str(trainer.populations[None].curGen) + ','
                + str((time.time()-tstart)/3600) + ','
                + '/'.join(sorted(list(fitnessEnvs))) + ','
                + str(trainer.populations[None].scoreStats['min']) + ','
                + str(trainer.populations[None].scoreStats['max']) + ','
                + str(trainer.populations[None].scoreStats['average']) +  ','
                + str(len(bestTeam.getRootTeamGraph()[0])) + ','
                + str(bestTeam.uid) + ','
                + str(len(trainer.populations[None].teams)) + ','
                + str(len(trainer.populations[None].rootTeams)) + '\n')

        # save model after every gen
        with open(trainerFileName,'wb') as f:
            pickle.dump(trainer,f)

        # every 10 generations save all agent outcomes
        if trainer.populations[None].curGen % 10 == 0:
            for rt in trainer.populations[None].rootTeams:
                with open(logFileOutcomesName, 'a') as f:
                    f.write(str(trainer.populations[None].curGen) + ','
                        + str((time.time()-tstart)/3600) + ','
                        + str(rt.uid))
                    for envName in allEnvNames:
                        f.write(',' + str(rt.outcomes.get(envName, '-999999')))
                    f.write('\n')

        # every 50 generations evaluate top agents on all games
        if trainer.populations[None].curGen % options.champEvalGen == 0:
            champEval(envNamesSrt, trainer, logFileChampionsName, pool, man,
                        tstart, frames=options.testFrames, eps=options.testEps)

"""
Method for training TPG on each game separately, in separate populations. All of
the populations will be merged together at once.
"""
def ggpTrainMerge():
    # create TPG from existing file of populations
    trainer = pickle.load(open(options.popsFile, 'rb'))

    envNamesSrt = sorted(list(allEnvNames)) # for reporting envs played

    fitnessEnvs = []

    # merge populations together
    prevName = envNamesSrt[0]
    for i in range(len(envNamesSrt)-1):
        if i+1 < len(envNamesSrt)-1:
            curName = 'tmp'+str(i)
        else:
            curName = None

        trainer.merge2Populations(prevName, envNamesSrt[i+1], curName,
                teamPopSize=options.popSize, rTeamPopSize=options.popSize)
        prevName = curName

    # continue on as regular all at once
    while True: # train indefinately
        print('TPG Gen: ' + str(trainer.populations[None].curGen))

        if options.fitGamesNum < options.numGames and options.fitGamesNum > 0:
            fitnessEnvs.append(random.choice(list(set(allEnvNames)-set(fitnessEnvs))))
            if len(fitnessEnvs) > options.fitGamesNum:
                fitnessEnvs.pop(0)
            else:
                fitnessEnvs = list(allEnvNames)

        for envName in fitnessEnvs:
            print('Playing Game: ' + envName)

            scoreList = man.list()

            # run all agents on env
            pool.map(runAgent,
                [(agent, envName, scoreList, options.trainEps, options.trainFrames, None)
                    for agent in trainer.getAllAgents(skipTasks=[envName], noRef=True)])

            trainer.applyScores(scoreList)

            # report curEnv results to log
            scoreStats = trainer.getTaskScores(envName)
            bestTeam = trainer.getBestTeams(tasks=[envName])[0]
            with open(logFileGameScoresName, 'a') as f:
                f.write(str(trainer.populations[None].curGen) + ','
                    + str((time.time()-tstart)/3600) + ','
                    + envName + ','
                    + str(scoreStats['min']) + ','
                    + str(scoreStats['max']) + ','
                    + str(scoreStats['average']) +  ','
                    + str(len(bestTeam.getRootTeamGraph()[0])) + ','
                    + str(bestTeam.uid) + '\n')

        # do evolution after all envs played
        trainer.multiEvolve(tasks=[fitnessEnvs]+[[en] for en in fitnessEnvs],
                            weights=[0.5]+[0.5/len(fitnessEnvs) for _ in fitnessEnvs],
                            fitMethod='min', elitistTasks=allEnvNames)

        # report generational fitness results
        bestTeam = trainer.getBestTeams(tasks=fitnessEnvs)[0]
        with open(logFileFitnessName, 'a') as f:
            f.write(str(trainer.populations[None].curGen) + ','
                + str((time.time()-tstart)/3600) + ','
                + '/'.join(sorted(list(fitnessEnvs))) + ','
                + str(trainer.populations[None].scoreStats['min']) + ','
                + str(trainer.populations[None].scoreStats['max']) + ','
                + str(trainer.populations[None].scoreStats['average']) +  ','
                + str(len(bestTeam.getRootTeamGraph()[0])) + ','
                + str(bestTeam.uid) + ','
                + str(len(trainer.populations[None].teams)) + ','
                + str(len(trainer.populations[None].rootTeams)) + '\n')

        # save model after every gen
        with open(trainerFileName,'wb') as f:
            pickle.dump(trainer,f)

        # every 50 generations evaluate top agents on all games
        if trainer.populations[None].curGen % options.champEvalGen == 0:
            champEval(envNamesSrt, trainer, logFileChampionsName, pool, man,
                        tstart, frames=options.testFrames, eps=options.testEps)

"""
Method for training TPG on each game separately, in separate populations. The populations
will be merged gradually, meaning that first separate populations will be paired,
then those pairs will be paired, and so on untill in one population.
"""
def ggpTrainGradualMerge(envNames, popSize, popSizeMax, lfGSName, lfCName, lfFName,
        pool, man):
    pass

"""
Run the training unending.
"""
tstart = time.time() # start timing

if options.trainType == 0:
    ggpTrainAllAtOnce()
elif options.trainType == 1:
    ggpTrainMerge()
else:
    ggpTrainGradualMerge()


import numpy as np
import copy, math
from scipy.spatial import distance, Delaunay

def tri_area(vertices):
  u = vertices[0]
  v = vertices[1]
  w = vertices[2]
  return np.linalg.norm(np.cross( v-u, w-u)) / 2.

def tetra_volume(vertices):
  u = vertices[0]
  v = vertices[1]
  w = vertices[2]
  x = vertices[3]
  return np.cross(v-u, w-u).dot(x-u) / 6.

class MasterDict(dict):
  """
  A dict with reverse reference to a master mesh.
  """
  def __init__(self, master, *args,**kwargs):
    self.master = master
    dict.__init__(self,*args,**kwargs)

  def __setitem__(self, key, val):
    val.master = self.master
    dict.__setitem__(self, key, val)

#-------------------------------------------------------------------------------
# Fields

class Field(object):
  """
  A field meta class.
  """
  def __init__(self, label, loc, data = {}, mesh = None):
    self.label = label
    self.data = data
    self.master = master
    self.loc  = loc

  def toarrays(self):
    k = np.array(self.data.keys())
    v = np.array(self.data.values())
    return k, v

class ScalarField(Field):
  """
  A scalar field class.
  """
  pass

class VectorField(Field):
  """
  A vector field class.
  """
  pass

class TensorField(Field):
  """
  A tensor field class.
  """
  pass

#-------------------------------------------------------------------------------
# Nodes and related stuff

class Node(object):
  def __init__(self, coords, sets = set(), master = None):
    self.coords = np.asarray(coords, dtype= np.float64)
    self.sets = set(sets)
    self.master = master

  def __repr__(self):
    return "#NODE" + self.__str__()

  def __str__(self):
    return "{0}, sets={1}".format(self.coords, self.sets)

#-------------------------------------------------------------------------------
# Elements and related stuff

class Element(object):

  _extrudable = False

  def __init__(self, conn, sets = set(), surfaces = None, master = None):
    self.conn = np.asarray(conn[:self._nvert], dtype = np.uint32)
    self.master = master
    self.sets = set(sets)
    ns = self.ns()
    self.surfaces = [set() for i in xrange(ns)]
    if surfaces != None:
      for k in surfaces.keys():
        self.surfaces[k].add( surfaces[k] )

  def __repr__(self):
    return "#ELEMENT " + self.__str__()

  def __str__(self):
    name = self.type()
    conn = ",".join([str(l) for l in self.conn])
    return "{0}({1}), sets={2}, surf={3}".format(name, conn,
                                              self.sets,
                                              self.surfaces)

  def edges(self):
    """
    Returns the connectivity of the edges.
    """
    return self.conn[self._edges]

  def faces(self):
    """
    Returns the faces of a volumic element, None otherwise.
    """
    if self._space == 3:
      return np.array([ self.conn[f] for f in self._faces_conn])

  def simplex_conn(self):
    """
    Returns the connectivity of the simplexes forming the element.
    """
    return np.array([ self.conn[f] for f in self._simplex_conn])

  def type(self):
    return self.__class__.__name__

  def ns(self):
    if self._space == 1: n = 1
    if self._space == 2: n = len(self._edges)
    if self._space == 3: n = len(self._faces_conn)
    return n

  def extrude(self, offset, layer):
    if self._extrudable:
      newtype = self._extrude_as
      oldconn = self.conn
      newconn = np.concatenate([
                oldconn + offset * layer,
                oldconn + offset * (layer + 1)])[self._extrude_order]
      return globals()[newtype](conn = newconn, sets = self.sets)

  def collapsed(self):
    return len(self.conn) - len(set(self.conn))

  def collapsed_faces(self):
    if self._space == 3:
      faces = [self.conn[c] for c in self._faces_conn]
      return np.array([ len(f) - len(set(f)) for f in faces ])

  def collapsed_edges(self):
    if self._space  >= 2:
      edges = self.conn[self._edges]
      return np.array([ len(e) - len(set(e)) for e in edges ])

  def simplex_decomposition(self):
    conn = self.conn
    if self._space == 3:
        simplices = self.simplex_conn()
        simplices2 = []
        for i in range(len(simplices)):
          simplex = simplices[i]
          if (len(set(simplex)) == 4):
            if tetra_volume([self.master.nodes[l].coords for l in simplex]) > 0.:
              simplices2.append(simplices[i])

        return [Tetra4(simplex) for simplex in simplices2]
        """
        points = [self.master.nodes[l].coords for l in conn]
        tetras = Delaunay(points).simplices
        tetras2 = []
        for i in xrange(len(tetras)):
          if tetra_volume([points[j] for j in tetras[i]]) < 0.:
            t = np.array([tetras[i][j] for j in [1, 0, 2, 3]])
          else:
            t = np.array([tetras[i][j] for j in [0, 1, 2, 3]])
        return [Tetra4(conn[t]) for t in tetras]
        """

    if self._space == 2:
      if self.type() == "Tri3":
        if len(conn) == 3:
          return [self]
      if self.type() == "Quad4":
        if len(conn) == 4:
           return [Tri3(self.conn[c]) for c in [[0, 1, 2], [1, 2, 3]]]
        if len(conn) == 3:
          count = np.array([(self.conn == i).sum() for i in conn])
          rind = conn[np.where(count == 2)[0][0]]
          rconn = self.conn.copy()
          for i in range(4):
            if (rconn[1] == rind) and (rconn[-1] == rind):
              return [Tri3(rconn[[0,1,2]])]
            rconn = np.roll(rconn, 1)

  def clean_connectivity(self):
    if self.collapsed() == 0:
      return [self]
    else:
      return self.simplex_decomposition()

  def node_set_to_surface(self, nodesetlabel, surfacelabel):
    nodelabels = set([k for k in self.nodes.keys() if label in self.nodes[k].sets])
    for element in self.elements.values:
      for i in xrange(self.ns()):
        if self._space == 3: surfconn = self.conn[self._faces_conn[i]]
        # TO be completed
        if nodelabels.issuperset(surfconn):
          self.surfaces[i].add(surfacelabel)

  def volume(self, add = True):
    vertices = np.array([self.master.nodes[l].coords for l in self.conn ])
    simplices = vertices[self._simplex_conn]
    if self._space == 3:
      v = np.array([tetra_volume(simplex) for simplex in simplices])
      if add: v = v.sum()
      return v
    if self._space == 2:
      v = np.array([tri_area(simplex) for simplex in simplices])
      if add: v = v.sum()
      return v

  def centroid(self):
    vertices = np.array([self.master.nodes[l].coords for l in self.conn ])
    simplices = vertices[self._simplex_conn]
    centroids = simplices.mean(axis = 1)
    volumes = self.volume(add = False)[:,None]
    return (centroids * volumes).sum(axis = 0) / volumes.sum()

class Line2(Element):
  """
  A 1D 2 nodes line.
  """
  _nvert = 2
  _space = 1
  _extrudable = True
  _extrude_as = "Quad4"
  _extrude_order = np.array([0, 1, 3, 2])
  _simplex_conn = np.array([[0, 1]])

class Tri3(Element):
  """
  A 2D 3 noded triangular element
  """
  _nvert  = 3
  _space  = 2
  _edges = np.array([[0, 1],
                     [1, 2],
                     [2, 0]])
  _extrudable = True
  _extrude_as = "Prism6"
  _extrude_order = np.array([0, 1, 2, 3, 4, 5])
  _simplex_conn  = np.array([[0, 1, 2]])

class Quad4(Element):
  """
  A 2D 4 noded quadrangular element
  """
  _nvert  = 4
  _space  = 2
  _edges = np.array([[0, 1],
                     [1, 2],
                     [2, 3],
                     [3, 0]])
  _extrudable = True
  _extrude_as = "Hexa8"
  _extrude_order = np.array([0, 1, 2, 3, 4, 5, 6, 7])
  _simplex_conn  = np.array([[0, 1, 3],
                             [1, 2, 3]])

class Tetra4(Element):
  """
  A 3D 4 noded tetrahedral element
  """
  _nvert  = 4
  _space  = 3
  _faces_conn  = np.array([[0, 1, 2],
                           [0, 3, 1],
                           [1, 3, 2],
                           [2, 3, 0]])
  _faces_type  = ["Tri3", "Tri3", "Tri3", "Tri3"]
  _edges = np.array([[0, 1],
                     [1, 2],
                     [2, 0],
                     [0, 3],
                     [1, 3],
                     [2, 3]])
  _simplex_conn  = np.array([[0, 1, 3, 4]])

  def clean_connectivity(self):
    if self.collapsed():
      return None
    else:
      return [self]

class Pyra5(Element):
  """
  A 3D 5 noded pyramidal element
  """
  _nvert  = 5
  _space  = 3
  _faces_conn  = np.array([[0, 1, 2, 3],
                           [0, 1, 4],
                           [1, 2, 4],
                           [2, 3, 4],
                           [3, 0, 4]])
  _faces_type = ["Quad4", "Tri3", "Tri3", "Tri3", "Tri3"]
  _edges = np.array([[0, 1],
                     [1, 2],
                     [2, 3],
                     [3, 0],
                     [0, 4],
                     [1, 4],
                     [2, 4],
                     [3, 4]])
  _simplex_conn  = np.array([[0, 1, 3, 4],
                             [1, 2, 3, 4]])

class Prism6(Element):
  """
  A 3D 6 noded prismatic element
  """
  _nvert  = 6
  _space  = 3
  _faces_conn = np.array([[0, 1, 2],
                          [3, 5, 4],
                          [0, 3, 4, 1],
                          [1, 4, 5, 2],
                          [2, 5, 3, 0]])
  _faces_type = ["Tri3", "Tri3", "Quad4", "Quad4", "Quad4"]
  _edges = np.array([[0, 1],
                     [1, 2],
                     [2, 0],
                     [3, 4],
                     [4, 5],
                     [5, 3],
                     [0, 3],
                     [1, 4],
                     [2, 5]])
  _simplex_conn  = np.array([[0, 1, 2, 3],
                             [1, 2, 3, 4],
                             [2, 3, 4, 5]])

class Hexa8(Element):
  """
  A 3D 8 noded hexahedric element
  """
  _nvert  = 8
  _space  = 3
  _faces_conn  = np.array([[0, 1, 2, 3],
                           [4, 7, 6, 5],
                           [0, 4, 5, 1],
                           [1, 5, 6, 2],
                           [2, 6, 7, 3],
                           [3, 7, 4, 0]])
  _faces_type = ["Quad4",
                 "Quad4",
                 "Quad4",
                 "Quad4",
                 "Quad4",
                 "Quad4"]
  _edges = np.array([[0, 1],
                     [1, 2],
                     [2, 3],
                     [3, 0],
                     [4, 5],
                     [5, 6],
                     [6, 7],
                     [7, 4],
                     [0, 4],
                     [1, 5],
                     [2, 6],
                     [3, 7]])
  _simplex_conn = np.array([[0, 1, 3, 4],
                            [1, 2, 3, 4],
                            [3, 2, 7, 4],
                            [2, 6, 7, 4],
                            [1, 5, 2, 4],
                            [2, 5, 6, 4]])
#-------------------------------------------------------------------------------
# Mesh & related stuff
class Mesh(object):
  def __init__(self, nodes = {}, elements = {}):
    self.nodes    = MasterDict(self)
    for k, v in nodes.iteritems(): self.nodes[k] = v
    self.elements = MasterDict(self)
    for k, v in elements.iteritems(): self.elements[k] = v

  def __repr__(self):
    nn = len(self.nodes.values())
    ne = len(self.elements.values())
    return "#MESH: {0} nodes / {1} elements".format(nn, ne)

  def __str__(self):
    nodes, elements = self.nodes, self.elements
    nk = sorted(nodes.keys())
    ns = "\n".join( ["{0} {1}".format(k, str(nodes[k])) for k in nk])
    ek = sorted(elements.keys())
    es = "\n".join( ["{0} {1}".format(k, str(elements[k])) for k in ek])
    """
    nsets, esets = self.nsets, self.esets
    nsk = sorted(nsets.keys())
    nss = "\n".join( ["{0} {1}".format(k, str(nsets[k])) for k in nsk])
    esk = sorted(esets.keys())
    print esk
    ess = "\n".join( ["{0} {1}".format(k, str(esets[k])) for k in esk])
    """
    return "MESH:\n*NODES:\n{0}\n*ELEMENTS:\n{1}".format(ns, es)
  """
  def _add_set(self, kind, key, labels):
    if kind == "nset": target = self.nodes
    if kind == "eset": target = self.elements
    for label in labels:
      target[label].sets.add(key)

  def add_nset(self, *args, **kwargs):
    self._add_set(kind = "nset", *args, **kwargs)

  def add_eset(self, *args, **kwargs):
    self._add_set(kind = "eset", *args, **kwargs)
  """
  def export(path):
    return

  def load(path):
    return

  def extrude(self, translation, layers):
    translation = np.array(translation, dtype = np.float64)[:3]
    newmesh = Mesh()
    # Nodes:
    node_offset = max(self.nodes.keys())
    for l, n in self.nodes.iteritems():
      for j in xrange(layers+1):
        newnode = Node(coords = n.coords + translation * float(j) / layers,
                       sets = n.sets)
        newmesh.nodes[l + j * node_offset] = newnode
    # Elements:
    element_offset = max(self.elements.keys())
    for l, e in self.elements.iteritems():
      for layer in xrange(layers):
        newelement = e.extrude(offset = node_offset, layer = layer)
        if newelement != None:
          newmesh.elements[l + layer * element_offset] = newelement
    return newmesh

  def copy(self):
    return copy.deepcopy(self)

  def nodes_to_array(self):
    labels = np.array([k for k in self.nodes.keys()])
    n = len(labels)
    p = np.empty((n,3))
    for i in range(n):
      p[i] = self.nodes[labels[i]].coords
    return labels , p

  def transform(self, transformation):
    labels, p  = self.nodes_to_array()
    n = len(labels)
    x, y, z = p.transpose()
    newcoords = np.asarray(transformation(x, y, z),
                           dtype = np.float64).transpose()
    newmesh = self.copy()
    for i in range(n):
      newmesh.nodes[labels[i]].coords = newcoords[i]
    return newmesh

  def overlapping_nodes(self, crit_dist = 1.e-6):
    def calc_row_idx(k, n):
      return int(math.ceil((1/2.) * (- (-8*k + 4 *n**2 -4*n - 7)**0.5 + 2*n -1) - 1))

    def elem_in_i_rows(i, n):
      return i * (n - 1 - i) + (i*(i + 1))/2

    def calc_col_idx(k, i, n):
      return int(n - elem_in_i_rows(i + 1, n) + k)

    def condensed_to_square(k, n):
      i = calc_row_idx(k, n)
      j = calc_col_idx(k, i, n)
      return np.array([i, j])
    labels, points = self.nodes_to_array()
    dist = distance.pdist(points)
    n = len(labels)
    loc = np.where(dist<=crit_dist)[0]
    pairs = [labels[condensed_to_square(l, n)] for l in loc]
    mapping = dict(zip(labels, labels))
    for pair in pairs:
      pair.sort()
      mapping[pair[1]] = min(mapping[pair[1]],  pair[0])
    return mapping

  def merge_nodes(self, mapping):
    newmesh = self.copy()
    for elem in newmesh.elements.values():
        conn = elem.conn
        for i in range(len(conn)):
          conn[i] = mapping[conn[i]]

    for label in newmesh.nodes.keys():
      if mapping[label] != label:
        del newmesh.nodes[label]
    return newmesh

  def simplex_decomposition(self):
    """
    Returns a list of new simplex elements sharing the same vertices with the
    orginal one with a viable connectivity.
    """
    newmesh = self.copy()
    newmesh.elements.clear()
    label = 1
    for el in self.elements.values():
      simplices = el.simplex_decomposition()
      if simplices != None:
        for simplex in simplices:
          newmesh.elements[label] = simplex
          label += 1
    return newmesh

  def clean_connectivity(self):
    newmesh = self.copy()
    newmesh.elements.clear()
    label = 1
    for el in self.elements.values():
      newels = el.clean_connectivity()
      if newels != None:
        for newel in newels:
          newmesh.elements[label] = newel
          label += 1
    return newmesh
#-------------------------------------------------------------------------------
# Parsers & writers

# Abaqus INP

def parseInp(path):
  # Some useful tools
  def lineInfo(line):
    out =  {"type": "data"}
    if line[0] == "*":
      if line[1] == "*":
        out["type"] = "comment"
        out["text"] = line[2:]
      else:
        out["type"] = "command"
        words = line[1:].split(",")
        out["value"] = words[0].strip()
        out["options"] = {}
        for word in words[1:]:
          key, value =  [s.strip() for s in word.split("=")]
          out["options"][key] = value
    return out
  def elementMapper(inpeltype):
    if inpeltype == "t3d2": return "Line2"
    if inpeltype[:3] in ["cps", "cpe", "cax"]:
      if inpeltype[3] == "3": return "Tri3"
      if inpeltype[3] == "4": return "Quad4"
    if inpeltype[:3] in ["c3d"]:
      if inpeltype[3] == "4": return "Tetra4"
      if inpeltype[3] == "5": return "Pyra5"
      if inpeltype[3] == "6": return "Prism6"
      if inpeltype[3] == "8": return "Hexa8"

  # Output mesh
  m = Mesh()
  # File preprocessing
  lines = np.array([l.strip().lower() for l in open(path).readlines()])
  # Data processing
  env = None
  setlabel = None
  for line in lines:
    d = lineInfo(line)
    if d["type"] == "command":
      env = d["value"]
      # Nodes
      if env == "node":
        opt = d["options"]
        currentset = None
        if "nset" in opt.keys(): currentset = opt["nset"]

      # Elements
      if env == "element":
        opt = d["options"]
        eltype = elementMapper(opt["type"])
        currentset = None
        if "elset" in opt.keys(): currentset = opt["elset"]

      # Nsets
      if env == "nset":
        opt = d["options"]
        currentset = opt["nset"]

      # Elsets
      if env == "elset":
        opt = d["options"]
        currentset = opt["elset"]

    if d["type"] == "data":
      words = line.strip().split(",")
      if env == "node":
        label  = int(words[0])
        coords = np.array( [float(w) for w in words[1:]], dtype = np.float64 )
        if currentset == None:
          m.nodes[label] = Node(coords = coords)
        else:
          m.nodes[label] = Node(coords = coords, sets = set([currentset]))

      if env == "element":
        label  = int(words[0])
        conn = np.array( [int(w) for w in words[1:]], dtype = np.int32)
        if currentset == None:
          m.elements[label] = globals()[eltype](conn = conn)
        else:
          m.elements[label] = globals()[eltype](conn = conn, sets = set([currentset]))

      if env == "nset":
        [m.nodes[int(w)].sets.add(currentset) for w in words if len(w) != 0]

      if env == "elset":
        [m.elements[int(w)].sets.add(currentset) for w in words if len(w) != 0]

  return m

def writeInp(mesh, mapping, path = None):

  def exportset(s, d):
    out = ""
    labels = [str(k) for k,v in d.iteritems() if s in v.sets]
    for i in xrange(len(labels)):
      out += labels[i]
      if (i+1)%10 != 0:
        out += ", "
      else:
        out += ",\n"
    if out[-1] != "\n": out += "\n"
    return out

  # Nodes
  out = "*NODE\n"
  for label, node in mesh.nodes.iteritems():
    out += "{0}, {1}\n".format(label, ", ".join([ str(c) for c in node.coords]))
  # Elements
  etypes = set([e.type() for e in mesh.elements.values()])
  for etype in etypes:
    out +="*ELEMENT, TYPE={0}\n".format( mapping[etype])
    for label, elem in mesh.elements.iteritems():
      if elem.type() == etype:
        out += "{0}, {1}\n".format(label, ", ".join([ str(c) for c in elem.conn]))
  # Sets
  nsets = set().union(*[n.sets for n in mesh.nodes.values()])
  for s in nsets:
    out += "*NSET, NSET={0}\n".format(s) + exportset(s , mesh.nodes)
  esets = set().union(*[e.sets for e in mesh.elements.values()])
  for s in esets:
    out += "*ELSET, ELSET={0}\n".format(s) + exportset(s , mesh.elements)
  if path == None:
    return out
  else:
    open(path, "w").write(out)

def writeMsh(mesh, path = None):
  elementMap = {"Tri3": 2, "Quad4":3, "Tetra4":4, "Hexa8":5, "Prism6":6, "Pyra5": 7}
  pattern = """$MeshFormat
2.2 0 8
$EndMeshFormat
$Nodes
{0}
$EndNodes
$Elements
{1}
$EndElements"""
  nodeout = "{0}\n".format(len(mesh.nodes.keys()))
  nodelist = []
  for k in mesh.nodes.keys():
    node = mesh.nodes[k]
    coords = node.coords
    nodelist.append("{0} {1} {2} {3}".format(k, coords[0], coords[1], coords[2]) )
  nodeout += "\n".join(nodelist)
  elemout = ""
  elemout = "{0}\n".format(len(mesh.elements.keys()))
  elemlist = []
  for k in mesh.elements.keys():
    element = mesh.elements[k]
    coords = node.coords
    elemlist.append("{0} {1} 1 1 {2}".format(
        k,
        elementMap[element.__class__.__name__],
        " ".join( [str(l) for l in element.conn ] ) ))
  elemout += "\n".join(elemlist)
  if path == None:
    return pattern.format(nodeout, elemout)
  else:
    open(path, "w").write(pattern.format(nodeout, elemout))


import psycopg2
from psycopg2.extensions import AsIs

import argparse
import csv
from configparser import ConfigParser
import datetime
import logging
import os
import requests
import sys
import shutil

logger = None

def configure_logger(args):
    global logger
    if args.verbose:
        level = logging.DEBUG
    else:
        level = logging.INFO
    formatter = "[%(levelname)s] %(asctime)s | %(filename)s:%(lineno)d | %(message)s"
    logger = logging.getLogger(__name__)
    handler = logging.StreamHandler()
    fhandler = logging.FileHandler("run.log")
    handler.setLevel(level)
    fhandler.setLevel(logging.DEBUG)
    handler.setFormatter(logging.Formatter(formatter))
    fhandler.setFormatter(logging.Formatter(formatter))
    logger.setLevel(logging.DEBUG)
    logger.addHandler(handler)
    logger.addHandler(fhandler)

def set_db_config(dbname):
    config = ConfigParser()
    config.read('init.ini')
    config.set("Database","dbname",dbname)
    with open('init.ini',"w") as configfile:
        config.write(configfile)

def parse_args():
    config = ConfigParser()
    config.read("init.ini")
    print(config["Database"]["dbname"])
    todays_date = str(datetime.date.today())

    parser = argparse.ArgumentParser(
        prog="hr.py",
        description="This a program for managing Hr operations",
        epilog="use these subcommands to do specific operations",
    )

    parser.add_argument(
        "-b", "--db", help="Database name (Default : %(default)s)", action="store_true" , default=config["Database"]["dbname"]
    )
    parser.add_argument(
        "-v", "--verbose", help="print out detailed logs", action="store_true"
    )

    subparser = parser.add_subparsers(dest="subcommand", help="subcommand help")

    # database command
    parser_createdb = subparser.add_parser("createdb", help="create a database")
    parser_createdb.add_argument("-b", "--db", help="database name", type=str)

    # loading csv into database
    parser_load = subparser.add_parser(
        "loadcsv", help="insert data from file into employee table in database ",description="Imports list of employees into the database from specified file '-l'"
    )
    parser_load.add_argument("file",help="File name ",type=str,)

    parser_load.add_argument(
        "-d",
        "--address",
        help="Add a new address",
        type=str,
        default="100 Flat Grape Dr.;Fresno;CA;95555;United States of America",
    )

    # commad for creating vcards
    parser_vcard = subparser.add_parser("vcard", help="generate vcards")

    parser_qrcode = subparser.add_parser("qrcode", help="generate qrcode")
    parser_qrcode.add_argument("-s","--size",
        help="Qr dimension ",
        default=300,)

    parser_leave_emp = subparser.add_parser("leave-emp", help="add leaves for employees",description="Adds leaves taken by the employee")

    parser_leave_emp.add_argument("empid", help="Employee id ", type=str)
    parser_leave_emp.add_argument("-d", "--date", help="Date 'YYYY-MM-DD' (Default : %(default)s) ", type=str, default=todays_date)
    parser_leave_emp.add_argument( "-r", "--reason",help=" Reason for leave ",type=str,default="Not mentioned")
    parser_export = subparser.add_parser("export", help="Get the employee detail as csv file",description="Get a detailed summary of all employees in a csv file")

    args = parser.parse_args()
    return args

def parse_data(args):
    filename = args.file
    datalist = []
    try:
        with open(filename, "r") as data:
            detail = csv.reader(data)
            for i in detail:
                datalist.append(i)

        return datalist
    except FileNotFoundError as e:
        logger.error("Error %s", (e,))
        sys.exit(-1)

def fetch_from_db(args, user):
    try:
        connection = psycopg2.connect(f"dbname={args.db} user={user}")
        curs = connection.cursor()
        curs.execute(
            f"""SELECT first_name,last_name,designation,email,phone,company_address
                         FROM
                         employees """
        )
        data = [list(row) for row in curs.fetchall()]
        connection.commit()
        curs.close()
        connection.close()
        logger.info(f"Fetched all data from employees table in {args.db} database")
        return data
    except psycopg2.OperationalError as e:
        logger.error(e)
        sys.exit(-1)

def generate_vcard_content(lname, fname, designation, email, phone, address):
    return f"""BEGIN:VCARD
VERSION:2.1
N:{lname};{fname}
FN:{fname} {lname}
ORG:Authors, Inc.
TITLE:{designation}
TEL;WORK;VOICE:{phone}
ADR;WORK:;;{address}
EMAIL;PREF;INTERNET:{email}
REV:20150922T195243Z
END:VCARD
"""

def generate_qrcode(dimension, i):
    qr_code = requests.get(
        f"https://chart.googleapis.com/chart?cht=qr&chs={dimension}x{dimension}&chl={i}"
    )
    return qr_code

def create_vcards(data):
    if os.path.exists("vcard"):
        shutil.rmtree("vcard")
    os.mkdir("vcards")
    for i in data:
        lname, fname, designation, email, phone, address = i
        vcard = generate_vcard_content(lname, fname, designation, email, phone, address)
        with open(f"vcards/{lname}.vcf", "w") as d:
            d.write(vcard)
        logger.debug("Created vcard for %s ", lname)
    logger.info("Created all vcards")

def create_qrcode_images(data, args):
    if type(args.size) != int or int(args.size) > 500:
        logger.error("size must be an integer between 100 - 500")

    else:
        if os.path.exists("qrcode"):
            shutil.rmtree("vcard")
        os.mkdir("qrcode")
        for i in data:
            with open(f"qrcode/{i[0]}.qr.png", "wb") as Q:
                qr_code = generate_qrcode(args.size, i)
                Q.write(qr_code.content)
            logger.debug("Created QRcode for %s", i[0])
        logger.info("Created QRs for all")

def create_database(args, user):
    try:
        connection = psycopg2.connect(database="postgres", user=user)
        curs = connection.cursor()
        curs.execute("commit")
        curs.execute("create database %s ;", (AsIs(args.db),))
        curs.close()
        connection.close()
        logger.info("Database created")
    except psycopg2.errors.DuplicateDatabase as e:
        logger.error("Error %s", e)

def create_tables(args, user):
    try:
        dbname = args.db
        connnection = psycopg2.connect(f"dbname={dbname} user={user}")
        curs = connnection.cursor()
        query = open("queries.sql", "r")
        curs.execute(query.read())
        connnection.commit()
        curs.close()
        connnection.close()
        logger.info(f"Table employees created in {dbname} database")

    except psycopg2.errors.DuplicateTable as e:
        logger.error("Table already exists: %s", e)
        sys.exit(-1)

def load_csv_into_db(args, data, user):
    try:
        connection = psycopg2.connect(f"dbname={args.db} user={user}")
        curs = connection.cursor()
        fname, lname, designation, email, phone = data

        curs.execute(
            f"""INSERT
                         INTO
                         employees(first_name, last_name, designation, email, phone, company_address)
                          VALUES(%s,%s,%s,%s,%s,%s)""",
            (fname, lname, designation, email, phone, args.address),
        )

        connection.commit()
        curs.close()
        connection.close()
        logger.debug(" Inserted datas of %s into table employees ", fname)

    except psycopg2.OperationalError as e:
        logger.error("Error %s", e)
    except psycopg2.errors.ForeignKeyViolation as e:
        logger.warning("Foreign key violation: %s")

    except psycopg2.Error as e:
        logger.error(f"Error {e}")

def load_leave_employee(args, user):
    try:
        with psycopg2.connect(f"dbname={args.db} user={user}") as connection:
            with connection.cursor() as curs:
                try:
                    curs.execute(
                        "SELECT designation FROM employees WHERE s_no = %s ",
                        (args.empid,),
                    )
                    designation = curs.fetchone()[0]
                except (TypeError, IndexError) as e:
                    logger.error(f"Error while fetching designation: %s", (e,))
                    raise  # for exiting the fun'

                try:
                    curs.execute(
                        "SELECT total_leaves FROM designation WHERE id = %s ",
                        (designation,),
                    )
                    total_leave = curs.fetchone()[0]
                except (TypeError, IndexError) as e:
                    logger.error(f"Error while fetching total_leave: %s", (e,))
                    raise

                try:
                    curs.execute(
                        "SELECT count(leave_date) FROM employee_leave WHERE employee_id = %s ",
                        (args.empid,),
                    )
                    leave_count = curs.fetchone()[0]
                except (TypeError, IndexError) as e:
                    logger.error(f"Error while fetching leave count: %s", (e,))
                    raise

                try:
                    if leave_count < total_leave:
                        curs.execute(
                            "INSERT INTO employee_leave(leave_date, employee_id,reason) VALUES (%s,%s,%s)",
                            (args.date, args.empid, args.reason),
                        )
                        logger.info("Leave added")
                    else:
                        logger.warning("Maximum leave attained")
                except psycopg2.Error as e:
                    logger.error("Error while inserting leave data: %s", e)

    except psycopg2.Error as e:
        logger.error("Error: %s", e)

def join_tables(args, user):
    try:
        connection = psycopg2.connect(f"dbname={args.db} user={user}")
        curs = connection.cursor()
        curs.execute(
            """SELECT
  e.s_no,
  e.first_name,
  e.last_name,
  d.title,
  d.total_leaves,
  e.email,
  e.phone,
  e.company_address,
  COUNT(DISTINCT l.leave_date) AS leave_taken,
  d.total_leaves - COUNT(DISTINCT l.leave_date) AS leave_remaining
FROM employees e
JOIN designation d ON e.designation = d.id
LEFT JOIN employee_leave l ON e.s_no = l.employee_id
GROUP BY e.s_no, e.first_name, e.last_name, d.title, d.total_leaves, e.email, e.phone, e.company_address;
"""
        )
        data = curs.fetchall()
        connection.commit()
        curs.close()
        connection.close()
        logger.info(f"Joined tables and data fetched")
        return data
    except psycopg2.Error as e:
        logger.error("Error %s", e)

def export_employee_details(data):
    with open("employees_summary.csv", "w", newline="") as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(
            [
                "Employee ID",
                "First Name",
                "Last Name",
                "Designation",
                "Total Leaves",
                "Email",
                "Phone",
                "Company Address",
                "Leaves Taken",
                "Leaves Remaining",
            ]
        )

        for row in data:
            writer.writerow(row)
            logger.debug("Writed row on csv file")
    logger.info("Exported File as employee_summary.csv ")

def main():
    args = parse_args()
    user = os.getenv("USERNAME")
    configure_logger(args)

    if args.subcommand == "createdb":
        set_db_config(args.db)
        create_database(args, user)
        create_tables(args, user)

    if args.subcommand == "loadcsv":
        data = parse_data(args)
        for i in range(len(data)):
            load_csv_into_db(args, data[i], user)

    if args.subcommand == "vcard":
        data = fetch_from_db(args, user)
        create_vcards(data)

    if args.subcommand == "qrcode":
        data = fetch_from_db(args, user)
        create_qrcode_images(data, args)

    if args.subcommand == "leavemp":
        load_leave_employee(args, user)

    if args.subcommand == "export":
        data = join_tables(args, user)
        export_employee_details(data)

if __name__ == "__main__":
    main()


from pyblish import api
import pyblish.plugin
import json
import io
import logging

# does register_config imply we can have multiple filters?
# let's name it register because the filter only applies after discover has run.

# options
# - multiple pipeline filters. apply settings to plugins in order (could error on clash)
# - multiple pipeline filters, but only 1 active pipeline filter, able to switch pipeline
# - single pipeline only

# we need to support discover since the pyblish GUI uses it
# skip_attr = []#'repair', 'id', 'log', 'process', 'version', 'requires']

# plugin config: the config/settings of a single plugin

# pipeline config: the config/settings of all plugins in your pipeline, a dict of dicts of plugin configs
# since we use names you cant have 2 plugins  with the same name

# when it just says config this likely is pipeline config

class PipelineConfig(dict):
    # TODO doesnt yet support methods like deepcopy, returns dict instead of Config.

    def filter_default_attributes(config):  # filter_default_attrs
        """ filter out default pyblish plugin attributes. example: families, id, ... """
        default_attrs = [x for x in iter_default_plugin_attrs()]
        for plugin_name, plugin_config in config.items():
            for attr_name in list(plugin_config.keys()):
                if attr_name in default_attrs:
                    plugin_config.pop(attr_name)

    def filter_attributes(config, attributes):  # filter_attrs
        """ filter out custom attributes. """
        for plugin_name, plugin_config in config.items():
            for attr_name in list(plugin_config.keys()):
                if attr_name in attributes:
                    plugin_config.pop(attr_name)

    def filter_private_attributes(config):
        """ filter out custom attributes. """
        for plugin_name, plugin_config in config.items():
            for attr_name in list(plugin_config.keys()):
                if attr_name.startswith('_'):
                    plugin_config.pop(attr_name)

    def filter_empty_plugins(config):  # filter_empty_plugins
        for plugin_name in list(config.keys()):
            plugin_config = config[plugin_name]
            if not plugin_config:
                config.pop(plugin_name)

    def dump(config, path):  # save_config_as_json
        """save a config to a json. can be used on both pipeline and plugin configs"""
        with io.open(path, 'w', encoding='utf-8') as outfile:
            try:
                # python 2
                my_json_str = json.dumps(config, ensure_ascii=False, indent=4)
                if isinstance(my_json_str, str):
                    my_json_str = my_json_str.decode("utf-8")
                outfile.write(my_json_str)

            except:
                # python 3
                json.dump(config, outfile, indent=4)

    def register(config):

        def config_filter_callback(plugins):

            # raise error when an expected plugin is not found
            plugin_names = [plugin.__name__ for plugin in plugins]
            for plugin_name in config:
                if plugin_name not in plugin_names:
                    raise Exception("plugin not found in pipeline: " + plugin_name)

            # get settings per plugin
            for plugin in plugins[:]:

                logging.debug("applying config to plugin: " + plugin.__name__)
                plugin_config = config.get(plugin.__name__, None)  # plugins with same name might clash

                # skip Pyblish plugins not in the config, this might be unwanted in same cases
                if plugin_config is None:
                    logging.debug("disabling plugin not in config: " + plugin.__name__)
                    plugin.active = False
                    continue

                for attr_name, value in plugin_config.items():
                    # TODO  differentiate between input classes (action, plugin...) and raw input int,str...
                    # custom load if value starts with #action| or #plugin|. ex: #action|modulename.action_name

                    # apply settings to plugins
                    setattr(plugin, attr_name, value)

        api.register_discovery_filter(config_filter_callback)

def iter_default_plugin_attrs(private=False):
    """ filter default pyblish attributes """
    variables = vars(pyblish.plugin.Plugin)
    for x in variables:
        if not private and x.startswith('_'):
            continue
        yield(x)

def iter_non_default_plugin_attrs(plugin):
    """ get non default pyblish attributes """
    default_attrs = list(iter_default_plugin_attrs())
    for x in vars(plugin):
        if x.startswith('_'):
            continue
        if x in default_attrs:
            continue
        yield(x)

# TODO remove this convenience function
def register_pipeline_config_filter(config_path=None, config_dict=None):
    """
    register a plugin settings filter from a settings file,
    which overwrites the default plugin settings,
    and is applied when running pyblish discover

    :param config_path: path to config dict to load plugin settings from (json file)
    :param config_dict: pass a dict directly
    :return: None
    """

    # we expect either the config path or the config data
    assert config_path or config_dict, "register_config_filter requires either a config path or a config dict, not both"

    if config_path:
        f = open(config_path)
        config_dict = json.load(f)
    elif not config_dict:
        raise Exception

    # default_settings (ex all enabled/disabled ....)
    # any plugins not in pipeline, should they be included or exluded

    PipelineConfig(config_dict).register()

# def apply_config(config_path=None, config_dict=None):
#     register_config_filter(config_path=config_path)
#     plugins = api.discover()
#     return plugins

def load_config_from_json(config_path):
    """load a config from a json. can be used on both pipeline and plugin configs"""
    f = open(config_path)
    config_dict = json.load(f)
    return PipelineConfig(config_dict)

def diff_pipeline_configs(config_new, config_original):
    """get differences between config and plugin settings"""
    config_data = {}
    for k, v in config_new.items():
        config_data[k] = {}
        for k2, v2 in v.items():
            original_value = config_original[k][k2]
            if original_value != v2:

                if not original_value and not v2:  # example empty string and None
                    continue

                config_data[k][k2] = v2
    return PipelineConfig(config_data)

def get_pipeline_config_from_plugins(plugins):
    """
    from a list of plugins, get all plugin configs.
    save these in a single dict, the pipeline config
    """

    # store all plugin configs in the pipeline config
    pipeline_config = {}
    for plugin in plugins:
        pipeline_config[plugin.__name__] = get_plugin_config(plugin)

    return PipelineConfig(pipeline_config)

def type_is_supported(value):
    if hasattr(value, '__call__'):  # skip functions
        return
    if isinstance(value, type):  # skip classes
        return

    if isinstance(value, (list, tuple)) and len(value) > 0:
        value2 = value[0]
        return type_is_supported(value2)

    return True

def get_plugin_config(plugin):
    """
    get all attributes from the plugin class, and save them in a dict
    :param plugin: Pyblish plugin to get settings from
    :return: a dict containing plugin settings
    """
    # for every plugin, store all settings in the plugin config
    plugin_config = {}
    for attr in dir(plugin):
        if not attr.startswith('_') and attr not in ('id', 'log'):  # skip private, skip id & log, you should never change these
            value = getattr(plugin, attr)

            if not type_is_supported(value):
                continue

            plugin_config[attr] = value
    plugin_config['__doc__'] = plugin.__doc__
    return plugin_config


import requests
from bs4 import BeautifulSoup
import codecs

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'
}

data = [
    'https://item.rakuten.co.jp/thegoldshopping/c/0000000443/?s=4&i=1#risFil',
]

class Crawler():
    def __init__(self, data):
        self.data = data

    def get_urls(self, category_url):
        self.category_url = category_url
        r = requests.get(category_url,  headers = headers)
        soup = BeautifulSoup(r.text, "html.parser")

        a_tags = soup.select('#risFil a')
        a_tags_data = []

        for index, a_tag in enumerate(a_tags):
            if( index == 0 ):
                continue
            # 拿到9個商品後break
            elif( len(a_tags_data) == 6):
                break
            if a_tag['href'] in a_tags_data:
                continue
            a_tags_data.append(a_tag['href'])

        return a_tags_data

    def get_detail(self, urls):
        self.urls = urls

        detail_data = []
        detail = {}

        for index,url in enumerate(urls):
            detail = {}

            r = requests.get(url,  headers = headers)
            soup = BeautifulSoup(r.text, "html.parser")
            product = soup.find_all('td',bgcolor="#ffffff")

            # 網址
            detail["url"] = url
            # 品牌
            brand = product[1].text.split('/')[1].replace(' ','')
            detail['brand'] = brand
            # 商品名
            product_name = product[2].text
            detail['product_name'] = product_name
            # 價錢
            price = soup.find(class_="price2").text.replace('\n','')
            detail['price'] = price
            # 店名
            try:
                shop_name = product[len(product) - 1].text.split('GOLD')[1].split('T')[0].replace('（北海道札幌市）','').replace('お気軽にお問い合わせ下さい。','').replace(' 札幌','').replace(' ','').replace('\n','')
                detail['shop_name'] = shop_name
            except:
                print(f'第{index + 1}筆 店名資料有誤，請檢察')
                detail['shop_name'] =''
            detail_data.append(detail)
        print('爬蟲成功')
        return detail_data

    def get_imgs(self,url):
        self.url = url
        imgs_data = []

        r = requests.get(url,  headers = headers)
        soup = BeautifulSoup(r.text, "html.parser")
        imgs = soup.find_all(class_='noImage')

        for index,img in enumerate(imgs):
            if(index == 9):
                break
            img = img['src'].split('?')[0]
            imgs_data.append(img)
        return imgs_data

    def render_html(self, detail_data, imgs):

        html = ''

        for index, item in enumerate(detail_data):
            html += f"""

{item['brand']}

{item['product_name']}

{item['price']}(税込)

                    """
        # print(html)

        with open("data_sp.html", "w", encoding='UTF-8') as file:
            file.write(html)

    def run(self, data):
        urls =  self.get_urls(data)
        imgs = self.get_imgs(data)
        detail_data = self.get_detail(urls)
        self.render_html(detail_data, imgs)

c = Crawler(data)
c.run(data[0])


# -*- coding: utf-8 -*-
import re
import logging
from typing import Any, Dict, List, Text, Union, Optional
# from itertools import islice

from rasa_sdk import Action, Tracker
# from rasa_sdk.executor import CollectingDispatcher
from rasa_sdk.events import (
    SlotSet,
    UserUtteranceReverted,
    ConversationPaused,
    EventType,
    FollowupAction,
    ActionExecuted,
    UserUttered
)

# from actions import config
# from actions.api.algolia import AlgoliaAPI
# from actions.utils import text_to_float

logger = logging.getLogger(__name__)

class ActionInitializeAKissStory(Action):

    def name(self):
        return 'action_initialize_a_kiss_story'

    def run(self, dispatcher, tracker, domain):

        return [SlotSet("lesson_topic", 'a_kiss_story'),
                SlotSet('nlu_confused', None),
                SlotSet('nlu_confident', 'positive'),
                SlotSet('will_return', None),
                SlotSet("lesson_history", []),
                SlotSet("a_kiss_progress", 0)]

class ActionStoreLessonHistory__a_kiss(Action):

    def name(self):
        return 'action_store_lesson_history__a_kiss'

    def _extract_question_number(self, action_name):
        p = re.compile(r"_(\d{1,2})_")
        result = p.findall(action_name)

        return int(result[0]) if len(result) > 0 else None

    def run(self, dispatcher, tracker, domain):
        current_state = tracker.current_state()
        logger.debug(f"Current state: {current_state}")

        latest_action_name = tracker.latest_action_name

        question_num = self._extract_question_number(latest_action_name)
        if not question_num:
            return []

        question_num = int(self._extract_question_number(latest_action_name))

        logger.debug(f"latest utterance action: {latest_action_name}")
        logger.debug(f"latest question number: {question_num}")

        data = tracker.get_slot("lesson_history")
        if not data:
            data = []
        data.append(question_num)

        return [SlotSet('nlu_confused', None),
                SlotSet('nlu_confident', 'positive'),
                SlotSet('will_return', None),
                SlotSet("lesson_history", data),
                SlotSet("a_kiss_progress", question_num)]

class ActionResetNLUConfusedSlot(Action):

    def name(self):
        return 'action_reset_nlu_confused_slot'

    def run(self, dispatcher, tracker, domain):

        return [SlotSet('nlu_confused', None),
                SlotSet('nlu_confident', 'positive')]

class ActionNotUnderstandFallback(Action):
    def name(self):
        return 'action_not_understand_fallback'

    def run(self, dispatcher, tracker, domain):
        current_state = tracker.current_state()
        logger.debug(f"Current state: {current_state}")

        return [SlotSet("nlu_confused", "positive"),
                SlotSet('nlu_confident', None),
                FollowupAction('utter_can_not_understand')]

class ActionNotSureWhatToDoFallback(Action):
    def name(self):
        return 'action_not_sure_what_to_do_fallback'

    def _latest_user_uttered(self, events) -> UserUttered:
        utters = list(filter(lambda e: e["event"] == "user", events))
        length = len(utters)

        if length <= 0:
            return None

        latest_user_utter_data = self._latest_user_utter(events)

        user_uttered = UserUttered(
                text=latest_user_utter_data['text'],
                parse_data=latest_user_utter_data['parse_data'],
                input_channel=latest_user_utter_data['input_channel']
            )

        return user_uttered

    def run(self, dispatcher, tracker, domain):
        # events = tracker.current_state()['events']

        return [SlotSet('will_return', "positive"),
                FollowupAction('utter_return_to_previous_question')]

class ActionInitializeChangedStory(Action):

    def name(self):
        return 'action_initialize_changed_story'

    def run(self, dispatcher, tracker, domain):

        return [SlotSet("lesson_topic", 'changed_story'),
                SlotSet('nlu_confused', None),
                SlotSet('nlu_confident', 'positive'),
                SlotSet('will_return', None),
                SlotSet("lesson_history", []),
                SlotSet("changed_progress", 0)]

class ActionStoreLessonHistory__changed(Action):

    def name(self):
        return 'action_store_lesson_history__changed'

    def _extract_question_number(self, action_name):
        p = re.compile("_(\d{2})_")
        result = p.findall(action_name)

        return int(result[0]) if len(result) > 0 else None

    def run(self, dispatcher, tracker, domain):
        latest_action_name = tracker.latest_action_name

        question_num = self._extract_question_number(latest_action_name)
        if not question_num:
            return []

        question_num = int(self._extract_question_number(latest_action_name))

        logger.debug(f"latest utterance action: {latest_action_name}")
        logger.debug(f"latest question number: {question_num}")

        data = tracker.get_slot("lesson_history")
        if not data:
            data = []
        data.append(question_num)

        return [SlotSet('nlu_confused', None),
                SlotSet('nlu_confident', 'positive'),
                SlotSet('will_return', None),
                SlotSet("lesson_history", data),
                SlotSet("changed_progress", question_num)]


def get_file_from_azure_blob(sourcekey):
    sourcebucket = os.environ.get("S3_PRICING_DATA_BUCKET")

    # Download the file to /tmp/ folder
    file_name = os.path.basename(sourcekey)
    download_path = "/tmp/" + file_name  # nosec
    logger.info(
        f"sourcebucket {sourcebucket},  sourcekey {sourcekey}"
        + f" download_path {download_path}"
    )

    s3_client.download_file(sourcebucket, sourcekey, download_path)

    return download_path, file_name


import constants
import copy
import random

new_players = copy.deepcopy(constants.PLAYERS)
new_teams = copy.deepcopy(constants.TEAMS)
experience_yes = []
experience_no = []
panthers = []
bandits = []
warriors = []

def clean_data():
    for player in new_players:
        player['guardians'] = (','.join(player['guardians'].split(" and")))
        height = player["height"].split()
        player["height"] = int(height[0])

        if player["experience"] == "YES":
            player["experience"] = True
            experience_yes.append(player)

        else:
            player["experience"] = "NO"
            player["experience"] = False
            experience_no.append(player)

def balance_teams():
    random.shuffle(experience_yes)
    random.shuffle(experience_no)

    while experience_yes:
        warriors.append(experience_yes.pop())
        bandits.append(experience_yes.pop())
        panthers.append(experience_yes.pop())

    while experience_no:
        warriors.append(experience_no.pop())
        bandits.append(experience_no.pop())
        panthers.append(experience_no.pop())
    return panthers, bandits, warriors

def team_stats(team):
    player_count = len(team)
    experienced_count = 0
    inexperienced_count = 0
    total_height = 0
    names_players = []
    gaurdians_team = []
    for player in team:
        if player ["experience"] == True:
            experienced_count += 1
        elif player["experience"] == False:
            inexperienced_count += 1
    for player in team:
        total_height += player["height"]
    average_height = round(total_height / len(team))
    for player in team:
        names_players.append(player['name'])
        gaurdians_team.append(player['guardians'])

    print("\n-------------\n")
    print("Total Players: ", player_count)
    print("Total Experienced: ", experienced_count)
    print("Total Inexperienced: ", inexperienced_count)
    print("Average Height: ", average_height)
    print("Player Names: ")
    print(', '.join(names_players))
    print("Gaurdian Names: ")
    print(', '.join(gaurdians_team))
    print("\n")

def main_menu():

    while True:
        user_option = input("BASKETBALL TEAM STATS TOOL: \n\n"
                            "---MENU---\n\n"
                            "Here are your choices:\n"
                            "A) Display Team Stats\n"
                            "B) Quit\n\n Enter an option: \n").upper()
        if user_option == "A":
            print(
                "A) Panthers\n"
                "B) Warriors\n"
                "C) Bandits\n")

            team_option = input("Select a team from above\n").upper()
            if team_option == "A":
                print("\n\nDisplaying Panther Stats")
                team_stats(panthers)
            elif team_option == "B":
                print("\n\nDisplaying Warriors Stats")
                team_stats(warriors)
            elif team_option == "C":
                print("\n\nDisplaying Bandits Stats")
                team_stats(bandits)
            else:
                print("\n\nInvalid Selection, try again")
        elif user_option == "B":
            print("Okay bye...")
            break
        else:
            print("Invalid Selection")

if __name__ == "__main__":
    clean_data()
    balance_teams()
    main_menu()