from discord.ext import commands
import discord, sheets
import random, datetime, difflib, asyncio
from card import Card
from battle import Battle

async def longpm(ctx, content):
    if len(content) > 1990:
        found_too_long = True
        i = content.find('```')
        if i >= 0:
            msgs = content[:i].split('\n') + [content[i:]]
        else:
            msgs = content.split()
        while found_too_long:
            found_too_long = False
            for i, msg in enumerate(msgs):
                if len(msg) > 1990:
                    found_too_long = True
                    msgs[i] = msg[:1990]
                    msgs.insert(i+1,msg[1990:])
    else:
        msgs = [content]
    for msg in msgs:
        await ctx.author.send(msg)

class Capes(commands.Cog):

    def __init__(self):
        self.capelist = []
        self.trading = {}
        self.claims = {}
        self.bonus_claim = {}
        self.trades = {}
        self.triggers = {} # Completed reviews
        self.reviewing = {} # Reviews in progress
        self.battle_stats = {}
        self.battles = {}
        self.cached = False
        self.pause = False
        self.lock = asyncio.Lock()

    async def set_local_cache(self):
        await self.load_capes()
        await self.load_owners()
        await self.load_triggers()
        await self.load_battle_stats()
        self.battles = {}
        self.cached = True

    async def load_capes(self):
        capes = (await sheets.trading_capes())
        for cape in capes:
            new_cape = Card(*cape)
            self.capelist += [ new_cape ]

    async def load_owners(self):
        card_owns = (await sheets.owned_cards())
        for card in card_owns:
            if not card[0] in self.trading:
                self.trading[card[0]] = []
            self.trading[card[0]] += [self.capelist[int(card[1])]]

    async def load_triggers(self):
        self.triggers = (await sheets.waiting_triggers())

    async def load_battle_stats(self):
        self.battle_stats = (await sheets.get_battle_stats())

    async def random_claim(self, ctx, claimer_id):
        # Check if this person has already claimed today
        day = datetime.datetime.now().day
        if not day in self.claims:
            self.claims = {day: []}

        if claimer_id in self.claims[day] and claimer_id not in self.bonus_claim:
            if datetime.datetime.now().hour == 23:
                timeleft = "{} minutes".format(60-datetime.datetime.now().minute)
            else:
                timeleft = "{} hours".format(24-datetime.datetime.now().hour)
            await ctx.send("You have already made your daily claim! Next claim available in {}".format(timeleft))
            return

        amount = 0
        if claimer_id in self.bonus_claim:
            await ctx.send("Claiming {} bonus cards".format(self.bonus_claim[claimer_id]))
            amount += self.bonus_claim[claimer_id]
        if not claimer_id in self.claims[day]:
            amount += 1

        total = 0
        for i in self.capelist:
            if i.tier == "C":
                total += 27
            elif i.tier == "B":
                total += 9
            elif i.tier == "A":
                total += 3
            elif i.tier == "S":
                total += 1
        selectedValues = []
        for i in range(0, amount):
            selectedValues += [ random.randrange(0,total) ]
        collected = ""
        cards = []
        for selectedValue in selectedValues:
            counting = 0
            cape = None
            for i in self.capelist:
                if i.tier == "C":
                    val_to_add = 27
                elif i.tier == "B":
                    val_to_add = 9
                elif i.tier == "A":
                    val_to_add = 3
                elif i.tier == "S":
                    val_to_add = 1
                if selectedValue >= counting and selectedValue < counting + val_to_add:
                    cape = i
                    break
                else:
                    counting += val_to_add
            # If we didn't find a cape, make it the last one!
            if cape == None:
                cape = self.capelist[-1]
            if not claimer_id in self.trading:
                self.trading[claimer_id] = []
            self.trading[claimer_id] += [cape]
            cards += [self.capelist.index(cape)]
            collected += cape.name + ", "
        await sheets.gain_cards(claimer_id, cards)
        await ctx.send("You gained the card for {}".format(collected[:-2]))
        if not claimer_id in self.claims[day]:
            self.claims[day] += [ claimer_id ]
        if claimer_id in self.bonus_claim:
            del self.bonus_claim[claimer_id]

    async def view_card(self, ctx, cape):
        message =   "**{}** ({})\n".format(cape.name, cape.classification)
        if cape.tagline != "":
            message += "> ***{}***\n".format(cape.tagline)
        message +=  "> Tier: **{}**\n".format(cape.tier)
        if cape.civ != "Unknown":
            message +=  "> Civilian Identity: ||{}||\n".format(cape.civ)
        else:
            message +=  "> Civilian Identity: {}\n".format(cape.civ)
        message +=  "> Power: {}\n".format(cape.power)
        message +=  "> Campaign: **{}**, {}\n".format(cape.campaign, cape.pc)
        message +=  "> Affiliation: {}, {}".format(cape.affiliation, cape.alignment)
        await ctx.send(message)

    async def messagify_ownership(self, user, c_list, c_filter = ""):
        if c_filter != "":
            c_filter = " **{}** ".format(c_filter)
        owned_cards = 0
        owned = {}
        if user in self.trading:
            for i in self.trading[user]:
                if i in c_list:
                    owned_cards += 1
                    if not i.name in owned:
                        owned[i.name] = 0
                    owned[i.name] += 1
        initial = "You own {}{} cards\n".format(owned_cards,c_filter[:-1])
        m_index = 0
        messages = ["> "]
        for i in owned:
            messages[m_index] += i
            if owned[i] > 1:
                messages[m_index] += "x{}".format(owned[i])
            messages[m_index] += ", "
            if len(messages[m_index]) > 1800: # Don't let it get too long
                messages[m_index] = messages[m_index][:-1]
                m_index += 1
                messages += ["> "]
        messages[m_index] = messages[m_index][:-2]

        messages[0] = "{}{}Cards collected {}/{}\n{}".format(initial, c_filter,len(owned),len(c_list),messages[0])
        return messages

    async def view(self, ctx, owner, *args):
        if not owner in self.trading:
            await ctx.send("You own no cards")
            return
        if len(args) == 0:
            # Let's display all cards
            messages = (await self.messagify_ownership(owner, self.capelist))
            for i in messages:
                await ctx.send(i)
        else:
            name = ""
            for i in args:
                name += "{} ".format(i)
            name = name[:-1]
            capes_owned = []
            for i in self.trading[owner]:
                if i.name.lower() == name.lower():
                    return await self.view_card(ctx, i)
                if not i.name in capes_owned:
                    capes_owned += [ i.name ]

            closestCapes = difflib.get_close_matches(name, capes_owned)
            if len(closestCapes) > 1:
                await ctx.send("That cape could not be found, did you mean one of: " + closestCapes)
            elif len(closestCapes) == 1:
                await ctx.send("That cape could not be found, did you mean " + closestCapes[0])
            else:
                await ctx.send("You don't own that that card!")

    async def get_cape(self, name):
        for i in self.capelist:
            if i.name.lower() == name.lower():
                return i

    async def at_to_id(self, at):
        at = at.replace("<", "")
        at = at.replace("@", "")
        at = at.replace("!", "")
        at = at.replace(">", "")
        return at.strip()

    async def has_cards(self, user, cards):
        # Cards comes in as an array of strings
        if len(cards) == 0:
            # You always have some of nothing!
            return True
        if user not in self.trading:
            # Can't have the required cards if they have nothing
            return False
        card_dict = {}
        for i in self.trading[user]:
            if not i.name.lower() in card_dict:
                card_dict[i.name.lower()] = 0
            card_dict[i.name.lower()] += 1
        for i in cards:
            if i.lower() in card_dict and card_dict[i.lower()] > 0:
                card_dict[i.lower()] -= 1
            else:
                return False

        return True

    async def trade(self, ctx, user, subaction, *args):
        if len(args) > 0 and (await self.at_to_id(args[0])) == user:
            await ctx.send("Trading with yourself just don't make sense...")
            return
        if subaction ==  "help":
            message = "**Trading commands:**\n```"
            message += "%tc trade offer @username [Card you have, Another card you have] [Card they have, another card they have] - Make an offer to someone\n"
            message += "%tc trade view - View all open trades you have\n"
            message += "%tc trade view @username - View an offer someone has made/you have made to them\n"
            message += "%tc trade cancel @username - Cancel a trade you have with someone\n"
            message += "%tc trade reject @username - Reject a trade someone has made to you\n"
            message += "%tc trade accept @username - Accept an open trade you have with someone\n"
            message += "```"
            await ctx.send(message)
        elif subaction == "offer":
            if args[0] == "help":
                message = "Specify who you want to trade with, your offer in parenthesis, then what you want back\n"
                message += "`    %tc trade offer @someperson [My Card, My Other Card] [Their Card, Another of their cards, a third card of theirs]`"
                await ctx.send(message)
                return

            # First arg will be a person
            request_to_id = (await self.at_to_id(args[0]))
            # Make sure an existing offer with them doesn't already exist
            if (user in self.trades and request_to_id in self.trades[user]) or (request_to_id in self.trades and user in self.trades[request_to_id]):
                await ctx.send("Already have a trade proposed with them! Will have to cancel or reject trade to start a new one.")
                await self.trade(ctx, user, "view", *args[0:1])
                return
            offers = ""
            for i in args[1:]:
                offers += i + " "

            offer, desire, nil = offers.split("]")
            nil, offer = offer.split("[")
            offer = offer.split(",")
            nil, desire = desire.split("[")
            desire = desire.split(",")

            if len(offer) == 1 and offer[0] == "":
                offer = []
            if len(desire) == 1 and desire[0] == "":
                desire = []
            if len(offer) == 0 and len(desire) == 0:
                await ctx.send("You can't trade nothing for nothing, that's just silly")
                return

            for i in range(0, len(offer)):
                offer[i] = offer[i].strip()

            for i in range(0, len(desire)):
                desire[i] = desire[i].strip()

            # Check whether they even both have the required cards
            if not (await self.has_cards(user, offer)):
                await ctx.send("You do not have the cards required to trade!")
            elif not (await self.has_cards(request_to_id, desire)):
                await ctx.send("They don't have the cards you want!")
            else:
                if not user in self.trades:
                    self.trades[user] = {}
                self.trades[user][request_to_id] = [offer, desire]
                await ctx.send("Offer made")
        elif subaction == "view":
            if len(args) == 0:
                open_offers = ""
                for i in self.trades:
                    if user == i:
                        for j in self.trades[i]:
                            open_offers += "<@!{}>, ".format(j)
                    elif user in self.trades[i]:
                        open_offers += "<@!{}>, ".format(i)
                if open_offers == "":
                    await ctx.send("Don't have any open trades")
                else:
                    await ctx.send("Have open trades with {}".format(open_offers[:-2]))
                return
            request_to_id = (await self.at_to_id(args[0]))
            if user in self.trades and request_to_id in self.trades[user]:
                offer = self.trades[user][request_to_id][0]
                o = "["
                for i in offer:
                    o += i + ", "
                o = o[:-2] + "]"
                if len(offer) == 0:
                    o = "nothing"

                desire = self.trades[user][request_to_id][1]
                d = "["
                for i in desire:
                    d += i + ", "
                d = d[:-2] + "]"
                if len(desire) == 0:
                    d = "nothing"

                message = "You offered to trade {} for their {}\nYou can cancel this trade with `%tc trade cancel @username`".format(o, d)
                await ctx.send(message)
            elif request_to_id in self.trades and user in self.trades[request_to_id]:
                offer = self.trades[request_to_id][user][0]
                o = "["
                for i in offer:
                    o += i + ", "
                o = o[:-2] + "]"
                if len(offer) == 0:
                    o = "nothing"

                desire = self.trades[request_to_id][user][1]
                d = "["
                for i in desire:
                    d += i + ", "
                d = d[:-2] + "]"
                if len(desire) == 0:
                    d = "nothing"

                message = "They offered to trade {} for your {}\nYou can reject this trade with `%tc trade reject @username`".format(o, d)
                await ctx.send(message)
            else:
                await ctx.send("Found no trade offer between the two of you")
        elif subaction == "accept":
            request_to_id = (await self.at_to_id(args[0]))
            if request_to_id in self.trades and user in self.trades[request_to_id]:
                offer = self.trades[request_to_id][user][0]
                desire = self.trades[request_to_id][user][1]
                if not (await self.has_cards(request_to_id, offer)):
                    await ctx.send("They no longer have the required cards! Trade cancelled")
                    del self.trades[request_to_id][user]
                elif not (await self.has_cards(user, desire)):
                    await ctx.send("You no longer have the required cards! Trade cancelled")
                    del self.trades[request_to_id][user]
                else:
                    offer_indexes = []
                    for i in offer:
                        cape = await self.get_cape(i)
                        self.trading[request_to_id].remove(cape)
                        self.trading[user] += [cape]
                        offer_indexes += [ self.capelist.index(cape) ]

                    await sheets.move_card_owner(request_to_id, user, offer_indexes)

                    desire_indexes = []
                    for i in desire:
                        cape = await self.get_cape(i)
                        self.trading[user].remove(cape)
                        self.trading[request_to_id] += [cape]
                        desire_indexes += [ self.capelist.index(cape) ]

                    await sheets.move_card_owner(user, request_to_id, desire_indexes)

                    await ctx.send("Trade made!")
                    del self.trades[request_to_id][user]
                    return
            else:
                await ctx.send("They haven't sent you a trade")
        elif subaction == "reject" or subaction == "cancel":
            request_to_id = (await self.at_to_id(args[0]))
            if user in self.trades and request_to_id in self.trades[user]:
                del self.trades[user][request_to_id]
                await ctx.send("Trade cancelled")
            elif request_to_id in self.trades and user in self.trades[request_to_id]:
                del self.trades[request_to_id][user]
                await ctx.send("Trade rejected")
            else:
                await ctx.send("Found no trade offer between the two of you")

    async def filter(self, ctx, user, *args):
        s = ""
        for i in args:
            s += i + " "
        s = s[:-1]
        categories = [
            "mover", "shaker", "brute", "breaker", "master", "tinker",
            "blaster", "thinker", "striker", "changer", "trump", "stranger"
        ]
        relevant = []
        if s.lower() in ["s", "a", "b", "c"]:
            for i in self.capelist:
                if i.tier.lower() == s.lower():
                    relevant += [i]
            messages = (await self.messagify_ownership(user, relevant, "{} Tier".format(s.upper())))

        elif s.lower() in categories:
            for i in self.capelist:
                if s.lower() in i.classification.lower():
                    relevant += [i]
            messages = (await self.messagify_ownership(user, relevant, "{} Capes".format(s)))
        elif s.lower() in ["pc", "npc"]:
            for i in self.capelist:
                if i.pc.lower() == s.lower():
                    relevant += [i]
            messages = (await self.messagify_ownership(user, relevant, "{} Capes".format(s)))
        elif s.lower() in ["hero", "villain", "rogue"]:
            for i in self.capelist:
                if i.alignment.lower() == s.lower():
                    relevant += [i]
            messages = (await self.messagify_ownership(user, relevant, "{} Aligned".format(s)))
        else:
            # We're filtering by affiliation or campaign
            for i in self.capelist:
                if i.affiliation.lower() == s.lower():
                    relevant += [i]
            if len(relevant) > 0:
                messages = (await self.messagify_ownership(user, relevant, "{} Affiliates".format(s)))
            else:
                for i in self.capelist:
                    if i.campaign.lower() == s.lower():
                        relevant += [i]
                if len(relevant) > 0:
                    messages = (await self.messagify_ownership(user, relevant, "{} Campaign".format(s)))
                else:
                    for i in self.capelist:
                        if s.lower() in i.name.lower():
                            relevant += [i]
                    if len(relevant) > 0:
                        messages = (await self.messagify_ownership(user, relevant, "{} Cape".format(s)))
                    else:
                        messages = ["Could not find any cards that matched the **{}** filter".format(s)]
        for i in messages:
            await ctx.send(i)

    async def grant(self, who, amount):
        if not who in self.bonus_claim:
            self.bonus_claim[who] = 0
        self.bonus_claim[who] += amount

    async def craft(self, ctx, who, *args):
        s = ""
        for i in args:
            s += i + " "
        s = s[:-1]
        if s == "help":
            message = "Pass in three cards you own to craft one of a higher tier. "
            message += "Mixing card tiers spreads your probability of getting an upgraded tier "
            message += "(ie [Tier A, Tier A, Tier C] = 66% chance of Tier S, 33% chance of Tier B).\n"
            message += "`%tc craft [Card 1, Card 2, Card 3]`"
            await ctx.send(message)
            return
        nil,s = s.split("[")
        s,nil = s.split("]")
        c1,c2,c3 = s.split(",")
        c1 = c1.strip()
        c2 = c2.strip()
        c3 = c3.strip()
        my_cards = [c1,c2,c3]
        if not await self.has_cards(who, my_cards):
            await ctx.send("You don't have all those cards available to craft with!")
            return
        upgrade = {"C": "B", "B": "A", "A": "S"}
        roll = []
        cards_to_remove = []
        for i in range(0,len(my_cards)):
            for j in self.capelist:
                if my_cards[i] == j.name:
                    if j.tier == "S":
                        await ctx.send("Can't upgrade from S tier cards")
                        return
                    roll += [upgrade[j.tier]]
                    cards_to_remove += [j]
                    break

        target = roll[random.randrange(0,3)]
        if target == "A":
            await ctx.send("Crafting an **{} Tier** card".format(target))
        else:
            await ctx.send("Crafting a **{} Tier** card".format(target))
        relevant = []
        for i in self.capelist:
            if i.tier == target:
                relevant += [i]
        selection = relevant[random.randrange(0,len(relevant))]
        r_card_index = []
        for i in cards_to_remove:
            self.trading[who].remove(i)
            r_card_index += [self.capelist.index(i)]
        await sheets.remove_cards(who, r_card_index)
        self.trading[who] += [selection]
        await sheets.gain_cards(who, [self.capelist.index(selection)])
        await ctx.send("Crafted **{}** card".format(selection.name))

    async def autocraft(self, ctx, who):
        owned = {}
        for i in self.trading[who]:
            if i.tier == "S":
                # We can't upgrade S tier cards!
                continue
            if i.tier not in owned:
                owned[i.tier] = {}
            if i not in owned[i.tier]:
                owned[i.tier][i] = 0
            owned[i.tier][i] += 1

        crafted = False

        upgrade = {"C": "B", "B": "A", "A": "S"}
        tier_sorted = {}
        gained = {}
        r_card_index = []

        for i in self.capelist:
            if i.tier not in tier_sorted:
                tier_sorted[i.tier] = []
            tier_sorted[i.tier] += [i]

        for tier in owned:
            # We're looking for a collection of three things!
            group = []
            for card in owned[tier]:
                while owned[tier][card] > 1:
                    owned[tier][card] -= 1
                    group += [card]
                    if len(group) == 3:
                        crafted_card = random.choice(tier_sorted[upgrade[tier]])
                        if upgrade[tier] not in gained:
                            gained[upgrade[tier]] = []
                        gained[upgrade[tier]] += [crafted_card]
                        for i in group:
                            r_card_index += [self.capelist.index(i)]
                            self.trading[who].remove(i)
                        crafted = True
                        group = []

        if not crafted:
            await ctx.send("Couldn't find enough duplicate cards to craft together")
        else:
            string = "Auto-crafting complete! Here is what's gotten made: \n"
            for tier in gained:
                r_string = f"**{tier}**:\n> "
                for i in gained[tier]:
                    r_string += f"{i.name}, "
                r_string = r_string[:-2] + "\n"
                if len(string + r_string) > 1999:
                    await ctx.send(string)
                    string = r_string
                else:
                    string += r_string
            await ctx.send(string)
            await sheets.remove_cards(who, r_card_index)
            gain_list = []
            for i in gained:
                self.trading[who] += gained[i]
                for card in gained[i]:
                    gain_list += [self.capelist.index(card)]
            await sheets.gain_cards(who, gain_list)

    async def collection(self, ctx, who, coll="help"):
        collections = ["Tier", "Affiliation", "Alignment", "Campaign", "Classification"]
        if coll.lower() == "help" or coll.capitalize() not in collections:
            await ctx.send(f"Possible collections you can specify are [{', '.join(collections)}]")
            return
        elif coll.lower() == "tier":
            coll_all = { "S": [], "A": [], "B": [], "C": [] }
            for i in self.capelist:
                coll_all[i.tier] += [i]
            owned = { "S": [], "A": [], "B": [], "C": [] }
            for i in self.trading[who]:
                if i not in owned[i.tier]:
                    owned[i.tier] += [i]
        elif coll.lower() == "affiliation":
            coll_all = {}
            owned = {}
            for i in self.capelist:
                if i.affiliation not in coll_all:
                    coll_all[i.affiliation] = []
                    owned[i.affiliation] = []
                coll_all[i.affiliation] += [i]
            for i in self.trading[who]:
                if i not in owned[i.affiliation]:
                    owned[i.affiliation] += [i]
        elif coll.lower() == "alignment":
            coll_all = {}
            owned = {}
            for i in self.capelist:
                if i.alignment not in coll_all:
                    coll_all[i.alignment] = []
                    owned[i.alignment] = []
                coll_all[i.alignment] += [i]
            for i in self.trading[who]:
                if i not in owned[i.alignment]:
                    owned[i.alignment] += [i]
        elif coll.lower() == "campaign":
            coll_all = {}
            owned = {}
            for i in self.capelist:
                if i.campaign not in coll_all:
                    coll_all[i.campaign] = []
                    owned[i.campaign] = []
                coll_all[i.campaign] += [i]
            for i in self.trading[who]:
                if i not in owned[i.campaign]:
                    owned[i.campaign] += [i]
        elif coll.lower() == "classification":
            coll_all = {
                "Mover": [], "Shaker": [], "Brute": [], "Breaker": [],
                "Master": [], "Tinker": [], "Blaster": [], "Thinker": [],
                "Striker": [], "Changer": [], "Trump": [], "Stranger": []
            }
            owned = {
                "Mover": [], "Shaker": [], "Brute": [], "Breaker": [],
                "Master": [], "Tinker": [], "Blaster": [], "Thinker": [],
                "Striker": [], "Changer": [], "Trump": [], "Stranger": []
            }
            for classification in coll_all:
                for i in self.capelist:
                    if classification in i.classification:
                        coll_all[classification] += [i]
                for i in self.trading[who]:
                    if i not in owned[classification] and classification in i.classification:
                        owned[classification] += [i]

        coll_string = []
        for i in sorted(coll_all.keys()):
            if len(owned[i]) != 0:
                coll_string += [ f"**{i}** ({len(owned[i])}/{len(coll_all[i])})"]
        printouts = [f"**{coll.capitalize()}** Collections: \n> {coll_string[0]}"]
        counter = 0
        for i in coll_string[1:]:
            if len(f"{printouts[counter]}, {i}") > 2000:
                counter += 1
                printouts += [f"> {i}"]
            else:
                printouts[counter] += f", {i}"
        for printout in printouts:
            await ctx.send(printout)

    @commands.command()
    async def tc(self, ctx, action, *args):
        '''Do things for the Parahumans server trading card game.
            %tc claim - Claim your daily free card
            %tc view [Card] - View all the cards you own
            %tc filter PC/Campaign/Affiliation/Tier etc. - Filter your cards by some type of card element
            %tc collection help - View the various collections you can view
            %tc submit - Get the link to submit new cards for review
            %tc masterlist - View the existing cape masterlist
            %tc trade [offer/view/accept/reject/cancel/help] @username - Make a trade
            %tc craft [Card 1, Card 2, Card 3] - Craft three cards together into a new one
            %tc autocraft - Automatically craft away duplicates (as long as you have 3 or more in a specific rarity)
        '''
        async with self.lock:
            if action == "unpause" and str(ctx.author.id) == "227834498019098624":
                self.pause = False
                return
            if self.pause:
                # For when developing on a secondary bot
                return
            if not self.cached:
                await ctx.send("Caching things on first run")
                await self.set_local_cache()
            if action == "claim":
                #await ctx.send("Claiming paused while Wellwick experiments with things")
                await self.random_claim(ctx, str(ctx.author.id))
            elif action == "recache" and str(ctx.author.id) == "227834498019098624":
                self.capelist = []
                self.trading = {}
                self.triggers = {}
                self.reviewing = {}
                await self.set_local_cache()
            elif action == "view" or action == "list" or action == "check":
                await self.view(ctx, str(ctx.author.id), *args)
            elif action == "submit":
                await ctx.send("<https://docs.google.com/forms/d/e/1FAIpQLSdntX_uPBSttXxuYlHh_lLszN1YYk248xSBLbuFXiGAQ3PdIA/viewform>")
            elif action == "masterlist":
                await ctx.send("<https://docs.google.com/spreadsheets/d/1PcMDs_zm8xg22IdXl8hoHS8IPF5skX-_GK3djUZSlBw/edit#gid=0>")
            elif action == "trade":
                if len(args) == 0:
                    await ctx.send("Missing trade subcommand (ie offer, accept, reject, view, cancel)")
                else:
                    await self.trade(ctx, str(ctx.author.id), args[0], *args[1:])
            elif action == "offer" and len(args) > 0 and args[1] == "trade":
                await self.trade(ctx, str(ctx.author.id), "offer", *args[1:])
            elif action == "filter":
                await self.filter(ctx, str(ctx.author.id), *args)
            elif action == "pause" and str(ctx.author.id) == "227834498019098624":
                self.pause = True
            elif action == "grant" and str(ctx.author.id) == "227834498019098624":
                who = await self.at_to_id(args[0])
                amount = int(args[1])
                await self.grant(who, amount)
            elif action == "craft":
                await self.craft(ctx, str(ctx.author.id), *args)
            elif action == "autocraft":
                await self.autocraft(ctx, str(ctx.author.id))
            elif action == "collection":
                await self.collection(ctx, str(ctx.author.id), *args)

    async def start_battle(self, battle: Battle):
        for i in battle.teams:
            team = [cape.cape.name for cape in battle.teams[i].team]
            for cape in battle.teams[i].team:
                self.trading[i] += [ cape.cape ]
            await battle.channel.send(f"<@{i}> has brought their team: {', '.join(team)}")
        await battle.channel.send("It's time to fight! PM your strategy for the first round `%battle attack/defend/ambush/flee/stats`")

    async def do_battle_round(self, battle: Battle):
        starting_pools = {}
        gameover = False
        for i in battle.teams:
            starting_pools[i] = battle.teams[i].pool.copy()
        for i in battle.teams:
            for j in battle.teams:
                if j is not i:
                    enemy = battle.teams[j]
            battle.teams[i].make_move(enemy)
        for i in battle.teams:
            new_losses = [x.cape.name for x in battle.teams[i].pool if x not in starting_pools[i]]
            if len(new_losses) > 0:
                await battle.channel.send(f"<@{i}> has lost the capes {', '.join(new_losses)}")
            elif battle.teams[i].damage_taken:
                await battle.channel.send(f"<@{i}> has lost no units this round, but did take damage!")
            else:
                await battle.channel.send(f"<@{i}> has lost no units this round and took no damage!")
            battle.teams[i].new_round()
            if len(battle.teams[i].team) == 0:
                await battle.channel.send(f"<@{i}>'s team has been defeated!")
                gameover = True
            elif battle.teams[i].fled:
                await battle.channel.send(f"<@{i}>'s team has fled!")
                gameover = True

        if gameover:
            await battle.channel.send("That's it, gameover! Thanks for playing.")
            for i in starting_pools:
                del self.battles[i]
        else:
            await battle.channel.send("Time for a new round! PM your strategy `%battle attack/defend/ambush/flee/stats` (check stats for ambush success)")

    @commands.command()
    async def battle(self, ctx, *, args=""):
        '''Begin a battle against another player. Can specify size of game as well.'''
        if not self.cached:
            await ctx.send("Caching things on first run")
            await self.set_local_cache()

        player = str(ctx.author.id)
        if not player in self.battles:
            args = args.split()
            if len(args) == 0:
                args = [""]
            enemy = await self.at_to_id(args[0].strip())
            if len(args) > 1:
                cap = int(args[1])
            else:
                cap = 10
            if not enemy in self.trading:
                await ctx.send("Player not recognised!\nFor now, starting a testing battle")
                self.battles[player] = Battle({player: [], "test": self.capelist[:2]},self.battle_stats, cap=cap, channel=ctx.channel)
                self.battles[player].teams["test"].ready = True
                self.battles[player].teams["test"].move = "Defend"
                return
            else:
                await ctx.send("Starting a battle. Both go add your cards as fighting capes in PMs using `%battle add capename, capename, capename`!")
                self.battles[player] = Battle({player: [], enemy: []}, self.battle_stats, cap=cap, channel=ctx.channel)
                self.battles[enemy] = self.battles[player]
        elif not self.battles[player].started:
            if args.split()[0] == "add":
                splittup = args[4:].split(",")
                capes = []
                for i in splittup:
                    capes += [i.strip()]
                if await self.has_cards(player, capes):
                    for card in capes:
                        cape = await self.get_cape(card)
                        self.trading[player].remove(cape)
                        if not self.battles[player].teams[player].add_cape(cape):
                            await ctx.send(f"Adding {card} exceeds the quota for the team")
                else:
                    await ctx.send(f"You don't have all of those cards!")

                await ctx.send(self.battles[player].teams[player].info_string())
            elif args.split()[0].lower() == "quit" or args.split()[0].lower() == "cancel":
                # Quitting before the game starts!
                await ctx.send("Quitting the game")
                for i in self.battles[player].teams:
                    if i != player:
                        enemy = i
                await self.battles[player].channel.send(f"<@{enemy}>, <@{player}> has cancelled the battle!")
                for bc in self.battles[player].teams[player].team:
                    self.trading[player] += [bc.cape]
                del self.battles[player]
                if enemy in self.battles:
                    for bc in self.battles[enemy].teams[enemy].team:
                        self.trading[enemy] += [bc.cape]
                    del self.battles[enemy]
            elif args.split()[0].lower() == "start" or args.split()[0].lower() == "ready":
                p_team = self.battles[player].teams[player]
                if p_team.cap is not None and p_team.quota() < p_team.cap and not p_team.forcing:
                    p_team.forcing = True
                    await ctx.send("You haven't filled your team to the cap, are you sure you want to start? Resend the command if so!")
                    return
                p_team.ready = True
                if self.battles[player].ready():
                    self.battles[player].started = True
                    await ctx.send("Other player is ready. Game starting!")
                    await self.start_battle(self.battles[player])
                else:
                    await ctx.send("Waiting for other player!")
        elif self.battles[player].started:
            strategy = args.split()[0].lower()
            p_team = self.battles[player].teams[player]
            if strategy == "attack":
                p_team.move = "Attack"
            elif strategy == "defend":
                p_team.move = "Defend"
            elif strategy == "ambush":
                p_team.move = "Ambush"
            elif strategy == "flee":
                p_team.move = "Flee"
            elif strategy == "stats":
                await ctx.send(self.battles[player].teams[player].info_string())
                return
            else:
                await ctx.send("Strategy is not recognised")
                return

            if self.battles[player].do_turn():
                await ctx.send("Starting the new round!")
                await self.do_battle_round(self.battles[player])
            else:
                await ctx.send("Waiting for other player to decide strategy")

    @commands.command()
    async def submit(self, ctx, *, args):
        '''Submit a trigger to the sheet and gain a few cards. Please only submit if putting in actual effort or this feature will be removed/you will be blacklisted.
        If a mistake is made (ie spelling, trigger longer than a discord message can contain) PM Wellwick
        '''
        if not self.cached:
            await ctx.send("Caching things on first run")
            await self.set_local_cache()
        s = args.strip()
        if len(s) == 0:
            await ctx.send("Can't submit an empty trigger")
            return
        elif len(s) > 1850:
            await ctx.send(f"Your trigger is {1850 - len(s)} characters too long!")
            return
        if s in self.triggers:
            await ctx.send("This trigger already exists!")
            return
        self.triggers[s] = [str(ctx.author.id)]
        await sheets.submit_trigger(s, str(ctx.author.id), ctx.author.name)
        await self.grant(str(ctx.author.id),3)
        await ctx.send("Thanks for submitting a trigger. You've earned some bonus cards that you can `%tc claim`")

    @commands.command()
    async def r(self, ctx, subaction, *args):
        '''Do some reviewing of submitted triggers and earn some cards. Is done in PMs!
        %r get - Get a trigger to review
        %r approve - Approve the trigger you are reviewing
        %r critique [criticism] - Add a critique for the trigger
        %r skip - Skip the trigger you are currently reviewing'''
        if not self.cached:
            await ctx.send("Caching things on first run")
            await self.set_local_cache()
        user = str(ctx.author.id)
        if subaction == "get":
            message = ""
            if user in self.reviewing:
                message += "You are already reviewing a trigger\n"
                message += self.reviewing[user]
            else:
                # Randomly select from one they haven't already reviewed
                triggers = []
                for i in self.triggers:
                    if not user in self.triggers[i]:
                        triggers += [i]
                if len(triggers) == 0:
                    await ctx.author.send("You've already reviewed all the triggers available. Great job!")
                    return
                selected = triggers[random.randrange(0,len(triggers))]
                self.reviewing[user] = selected
                message += selected + "\n"
            message += "```%r approve comments - Approve the trigger. Comments optional\n"
            message += "%r critique comments - Critique the trigger\n"
            message += "%r skip - Skip reviewing this trigger```"
            await longpm(ctx, message)
        elif subaction == "approve":
            if not user in self.reviewing:
                await ctx.author.send("You aren't reviewing anything")
                return
            comments = ""
            for i in args:
                comments += i + " "
            comments[:-1]
            trigger = self.reviewing[user]
            self.triggers[trigger] += [user]
            try:
                await sheets.approve_trigger(trigger, user, ctx.author.name, comments)
            except:
                await ctx.send("Looks like this trigger may have changed remotely! Please ping Wellwick to ask for a recache. Skipping trigger")
                self.triggers[self.reviewing[user]] += [user]
                del self.reviewing[user]
                return
            del self.reviewing[user]
            await self.grant(user,1)
            await ctx.author.send("Thanks for reviewing a trigger. You've earned a bonus card that you can `%tc claim`")
        elif subaction == "critique":
            if not user in self.reviewing:
                await ctx.author.send("You aren't reviewing anything")
                return
            comments = ""
            if len(args) == 0:
                await ctx.author.send("You have provided no comments")
                return
            for i in args:
                comments += i + " "
            comments[:-1]
            trigger = self.reviewing[user]
            self.triggers[trigger] += [user]
            try:
                await sheets.critique_trigger(trigger, user, ctx.author.name, comments)
            except:
                await ctx.send("Looks like this trigger may have changed remotely so you can't review it right now, sorry! Please ping Wellwick to ask for a recache. Skipping trigger")
                self.triggers[self.reviewing[user]] += [user]
                del self.reviewing[user]
                return
            del self.reviewing[user]
            await self.grant(user,1)
            await ctx.author.send("Thanks for reviewing a trigger. You've earned a bonus card that you can `%tc claim`")
        elif subaction == "skip":
            if not user in self.reviewing:
                await ctx.author.send("You aren't reviewing anything")
                return
            self.triggers[self.reviewing[user]] += [user]
            del self.reviewing[user]

    @commands.command()
    async def cape(self, ctx, *args):
        '''Provide information on Weaverdice capes in the database.

        Can access the google sheets at https://docs.google.com/spreadsheets/d/1_syrsmptzWG0u3xdY3qzutYToY1t3I8s6yaryIpfckU/edit#gid=1668315016
        '''
        # Search for the cape name
        name = ""
        for arg in args:
            name += arg + " "

        name = name[:-1]
        info = (await sheets.cape(name))
        if info:
            output = "**" + info[0] + "**"
            if len(info) > 5 and not info[5] == "":
                output += " (" + info[5] + ")"
            output += "\n"
            if len(info) > 9 and not info[9] == "":
                output += "> Status: **" + info[9] + "**\n"
            if len(info) > 1 and not info[1] == "":
                output += "> Civilian identity: ||" + info[1] + "||\n"
            if (len(info) > 3 and not info[3] == "") or (len(info) > 4 and not info[4] == ""):
                output += "> Affiliation: " + info[3]
                if len(info) > 4 and (not info[3] == "" and not info[4] == ""):
                    output += ", "
                output += info[4] + "\n"
            if len(info) > 2 and not info[2] == "":
                output += "> Power: " + info[2] + "\n"
            if len(info) > 7 and not info[7] == "":
                output += "> Campaign: **" + info[7] + "**\n"
            if len(info) > 8 and not info[8] == "":
                output += "> Owner: **" + info[8] + "**"
                if len(info) > 6 and not info[6] == "":
                    output += ", " + info[6]
                output += "\n"
            elif len(info) > 6 and not info[6] == "":
                output += "> Unowned, **" + info[6] + "**\n"
            if len(info) > 10 and not info[10] == "":
                output += "> Additional notes: " + info[10]
            await ctx.send(output)
            #await ctx.send(str(info))
        else:
            await ctx.send("Couldn't find cape!")


import model as md
import datetime
import importlib
import os
importlib.reload(md)
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import roc_auc_score
from keras.models import Model
from keras.layers import Dense, Input, Embedding, Dropout, Activation, Reshape, Flatten
from keras.layers.merge import concatenate, dot, add, multiply
from keras.layers.normalization import BatchNormalization
from keras.layers.advanced_activations import LeakyReLU, PReLU, ELU
from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau, LearningRateScheduler
from keras.regularizers import l1, l2, l1_l2
from keras.initializers import RandomUniform
from keras.optimizers import RMSprop, Adam, SGD
from sklearn.metrics import roc_auc_score
import keras.backend as K
from nn_generator import DataGenerator
from keras.utils import multi_gpu_model
import tensorflow as tf
from nn_generator import DataGenerator
from sklearn.metrics import roc_auc_score
from keras.callbacks import Callback
import pandas as pd
from tqdm import tqdm
import numpy as np
import keras
from collections import defaultdict
import gc
import os
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import StandardScaler

from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot
# train_nn list of np.array [np.array]
os.environ["CUDA_VISIBLE_DEVICES"] = "0"

single_features = ['uid','LBS','carrier','consumptionAbility','age','gender','house','education']
# multi_features = ["appIdAction","appIdInstall","ct","interest1","interest2","interest3",
#                   "interest4","interest5","kw1","kw2","kw3","os","topic1","topic2","topic3","marriageStatus"]
multi_features = ["ct","interest1","interest2","interest3",
                  "interest4","interest5","kw1","kw2","kw3","os","topic1","topic2","topic3","marriageStatus"]

ad_embedding_features_single = ['aid','advertiserId','campaignId','adCategoryId','productId','productType'] #'creativeId'
ad_numeric=['creativeSize']
usr_embedding_features_single= ['LBS','carrier','gender','education','house','age','consumptionAbility']
usr_numeric = ['house','age','consumptionAbility']

def loaddata():

    train = pd.read_csv("data/preliminary_contest_data/train.csv",engine='c')
    test = pd.read_csv("data/preliminary_contest_data/test1.csv",engine='c')
    adattrs = pd.read_csv("data/preliminary_contest_data/adFeature.csv",engine='c')
    usersingle = pd.read_csv("data/preliminary_contest_data/unstack_single_train.csv",engine='c')
    train_nn = []
    for feat in tqdm(multi_features):
        tmp_trnn = pd.read_csv("data/preliminary_contest_data/%s_nn_train.csv" % feat,engine='c')
        tmp_trnn.fillna(0,inplace=True)

        tmp_trnn = tmp_trnn.astype('int32')
        train_nn.append(tmp_trnn.values)
        del tmp_trnn
    # encoding
    from sklearn.preprocessing import LabelEncoder
    le = LabelEncoder()
    for feat in ['advertiserId','campaignId','creativeId','adCategoryId','productId','productType']:
        adattrs[feat] = le.fit_transform(adattrs[feat])
    #merge all data together
    trainmerge = train.merge(adattrs,on='aid',how='left')
    trainmerge = trainmerge.merge(usersingle,on='uid',how='left')
    trainmerge['label'] = (trainmerge['label']+1)/2
    for feat in ['aid']:
        trainmerge[feat] = le.fit_transform(trainmerge[feat])
    trainmerge.fillna(0,inplace=True)
    trainmerge = trainmerge.astype('int32')
#     usercount = []
#     for i in tqdm(range(len(multi_features))):
#          usercount.append((train_nn[i].values!=0).sum(axis=1))
#     usercount = np.array(usercount).T
    del adattrs,usersingle
    return trainmerge,train_nn
def load_testdata():

    train = pd.read_csv("data/preliminary_contest_data/train.csv",engine='c')
    test = pd.read_csv("data/preliminary_contest_data/test1.csv",engine='c')
    adattrs = pd.read_csv("data/preliminary_contest_data/adFeature.csv",engine='c')
    usersingle = pd.read_csv("data/preliminary_contest_data/unstack_single_train.csv",engine='c')
    train_nn = []
    for feat in tqdm(multi_features):
        tmp_trnn = pd.read_csv("data/preliminary_contest_data/%s_nn_test.csv" % feat)
        tmp_trnn.fillna(0,inplace=True)
        tmp_trnn = tmp_trnn.values
        tmp_trnnmax = tmp_trnn.max()

        if tmp_trnnmax < 255:
            tmp_trnn.astype('uint8')
        elif tmp_trnnmax < 65535:
            tmp_trnn.astype('uint16')
        elif tmp_trnnmax < 2 ** 32 - 1:
            tmp_trnn.astype('uint32')
        else:
            tmp_trnn.astype('uint64')
        train_nn.append(tmp_trnn)
        del tmp_trnn
        gc.collect()
    # encoding
    from sklearn.preprocessing import LabelEncoder
    le = LabelEncoder()
    for feat in ['advertiserId','campaignId','creativeId','adCategoryId','productId','productType']:
        adattrs[feat] = le.fit_transform(adattrs[feat])
    #merge all data together
    trainmerge = test.merge(adattrs,on='aid',how='left')
    trainmerge = trainmerge.merge(usersingle,on='uid',how='left')
    #trainmerge['label'] = (trainmerge['label']+1)/2
    for feat in ['aid']:
        trainmerge[feat] = le.fit_transform(trainmerge[feat])
    trainmerge.fillna(0,inplace=True)
    trainmerge = trainmerge.astype('int32')
#     usercount = []
#     for i in tqdm(range(len(multi_features))):
#          usercount.append((train_nn[i].values!=0).sum(axis=1))
#     usercount = np.array(usercount).T
    del adattrs,usersingle
    return trainmerge,train_nn

def multifeat_onehot(train_nn,count_threshold = 20,):
    ohe = OneHotEncoder(sparse=False)
    train_nn_onehot = []
    onehot_index = []
    for i in tqdm(range(len(multi_features))):
        train_nnmax = train_nn[i].max()
        if  train_nnmax < count_threshold:
            tmpohe = ohe.fit_transform(train_nn[i].reshape((len(train_nn[i])*train_nn[i].shape[1],1)))[:,1:].reshape((len(train_nn[i]),train_nn[i].shape[1],train_nnmax)).sum(axis=1)
            train_nn_onehot.append(tmpohe.astype(int))
            onehot_index.append(i)
    train_nn_index = list(set(list(range(len(multi_features))))- set(onehot_index))
    return train_nn_onehot, onehot_index

def train_valid_split_multi(train_nn,train_nn_onehot=None,test_size=0.2,random_state=42):
    offttr_nn = []
    offtte_nn = []
    for feat in tqdm(range(len(multi_features))):
        a,b = train_test_split(train_nn[feat],test_size=test_size,random_state=random_state)
        offttr_nn.append(a)
        offtte_nn.append(b)

    offttr_nn_onehot = []
    offtte_nn_onehot = []
    if train_nn_onehot is not None:
        for feat in tqdm(range(len(train_nn_onehot))):
            a,b = train_test_split(train_nn_onehot[feat],test_size=test_size,random_state=random_state)
            offttr_nn_onehot.append(a)
            offtte_nn_onehot.append(b)
    return offttr_nn,offtte_nn,offttr_nn_onehot,offtte_nn_onehot

def train_valid_split_single(trainmerge,single_count_threshold = 200,test_size=0.2,random_state=42):
    suboffttr,subofftte = train_test_split(trainmerge,test_size=test_size,random_state=random_state)
    ohe = OneHotEncoder(sparse=False)
    train_embeddings = []
    valid_embeddings = []
    for feat in ad_embedding_features_single+usr_embedding_features_single:
        if int(trainmerge[feat].max()+1)< single_count_threshold:
            train_embeddings.append(ohe.fit_transform(suboffttr[[feat]].values))
            valid_embeddings.append(ohe.transform(subofftte[[feat]].values))
        else:
            train_embeddings.append(suboffttr[feat].values)
            valid_embeddings.append(subofftte[feat].values)
    ss_context = StandardScaler()
    usr_train_context = ss_context.fit_transform(suboffttr[usr_numeric].values)
    usr_valid_context = ss_context.transform(subofftte[usr_numeric].values)
    ad_train_context = ss_context.fit_transform(suboffttr[ad_numeric].values)
    ad_valid_context = ss_context.transform(subofftte[ad_numeric].values)
    #     usr_train_count = ss_context.fit_transform(suboffttr_count)
    #     usr_valid_count = ss_context.fit_transform(subofftte_count)
    return train_embeddings,valid_embeddings, usr_train_context,usr_valid_context,ad_train_context,ad_valid_context,suboffttr['label'],subofftte['label']
#offttr_count,offtte_count = train_test_split(usercount,test_size=0.2,random_state=443)
def takesample(trainmerge,train_nn ,rate=0.2,random_state=42):
    _,subtrainmerge = train_test_split(trainmerge,test_size=rate,random_state = random_state)
    subtrainmerge = subtrainmerge.reset_index(drop=True)
    subtrain_nn = []
    for feat in tqdm(train_nn):
        _,sub_nn = train_test_split(feat.values,test_size=rate,random_state = random_state)
        subtrain_nn.append(sub_nn)

    return subtrainmerge, subtrain_nn

def taketest(trainmerge,train_nn ,rate=0.2,random_state=42):
    subtrainmerge,testmerge = train_test_split(trainmerge,test_size=rate,random_state = random_state)
    subtrainmerge = subtrainmerge.reset_index(drop=True)
    testmerge = testmerge.reset_index(drop=True)
    subtrain_nn = []
    subtest_nn = []
    for feat in tqdm(train_nn):
        test_nn,sub_nn = train_test_split(feat,test_size=rate,random_state = random_state)
        subtrain_nn.append(sub_nn)
        subtest_nn.append(test_nn)
    return subtrainmerge, subtrain_nn,testmerge, subtest_nn
def downsample(trainmerge,train_nn):
    pos_trainmerge = trainmerge[trainmerge['label']==1]
    pos_count = len(pos_trainmerge)
    print(pos_count)
    neg_trainmerge = trainmerge[trainmerge['label'] == 0].sample(pos_count)
    subtrainmerge = pd.concat((pos_trainmerge,neg_trainmerge)).reset_index(drop=True)
    subtrain_nn = []
    for feat in tqdm(train_nn):
        subtrain_nn.append(pd.concat((feat.iloc[pos_trainmerge.index],feat.iloc[neg_trainmerge.index])).reset_index(drop=True).values)
    return subtrainmerge, subtrain_nn

def batch_train_test_split(tosplit,kfold=None,test_size=0.2,random_state=42):
    trainlist = []
    testlist = []
    for val in tqdm(tosplit):
        train,test = train_test_split(val,test_size=0.2,random_state=random_state,shuffle=True)
        trainlist.append(train)
        testlist.append(test)
    return trainlist,testlist

def batch_s_onehot(toonehot,toonehot_transform=None,count_threshold = None,toonehotindex=None):
    assert (count_threshold is not None or toonehotindex is not None) and not \
    (count_threshold is not None and toonehotindex is not None),"count_threshold and\
    toonehotindex can not all None, can not all not None"
    assert toonehot_transform is None or (toonehot_transform is not None and  len(toonehot_transform) == len(toonehot)),"len(toonehot_transform) should equal to len(toonehot)"
    ohe = OneHotEncoder(sparse=False)

    if count_threshold  is not None:
        for index in tqdm(range(len(toonehot))):
            if toonehot[index].max()<count_threshold:
                toonehot[index] = ohe.fit_transform(toonehot[index][:,np.newaxis])
                if toonehot_transform is not None:
                    toonehot_transform[index] = ohe.transform(toonehot_transform[index][:,np.newaxis] )
    elif toonehotindex is not None:
        for index in toonehotindex:
            toonehot[index] = ohe.fit_transform(toonehot[index][:,np.newaxis])
            if toonehot_transform is not None:
                toonehot_transform[index] = ohe.transform(toonehot_transform[index][:,np.newaxis])

    return toonehot,toonehot_transform

class auc_callback(Callback):
    def __init__(self,x,xn,y):
        self.x = x
        self.xn = xn
        self.y = y
    def on_train_begin(self, logs={}):
        if not ('val_auc_loss' in self.params['metrics']):
            self.params['metrics'].append('val_auc_loss')

    def on_epoch_end(self, epoch, logs={}):

        dataGenerator = DataGenerator()
        flow = dataGenerator.flow(self.x,self.xn, batch_size=16384, shuffle=False)
        y_pred = self.model.predict_generator(flow, flow.__len__(), workers=4)

        roc = roc_auc_score(self.y, y_pred)
        print("roc_auc %.6f\n" % roc)
        logs['val_auc_loss'] = roc

trainmerge,train_nn = loaddata()
gc.collect()

seed = np.random.randint(0,100000)
sser= StandardScaler()
usrcateg_s_train,usrcateg_s_valid = batch_train_test_split([trainmerge[feat].values for feat in
                                                      usr_embedding_features_single],
                                                     test_size=0.2,random_state=seed)

adcateg_s_train,adcateg_s_valid = batch_train_test_split([trainmerge[feat].values for feat in
                                                      ad_embedding_features_single],
                                                     test_size=0.2,random_state=seed)
adcateg_s_train,adcateg_s_valid = batch_s_onehot(adcateg_s_train,adcateg_s_valid,count_threshold=200)

usrnumer_s_train,usrnumer_s_valid = batch_train_test_split([trainmerge[feat].values for feat in usr_numeric ],
                                                     test_size=0.2,random_state=seed)
usrnumer_s_train = np.array(usrnumer_s_train).T
usrnumer_s_valid = np.array(usrnumer_s_valid).T
usrnumer_s_train = sser.fit_transform(usrnumer_s_train)
usrnumer_s_valid = sser.transform(usrnumer_s_valid)
adnumer_s_train,adnumer_s_valid = batch_train_test_split([trainmerge[feat].values for feat in ad_numeric ],
                                                     test_size=0.2,random_state=seed)
adnumer_s_train = np.array(adnumer_s_train).T
adnumer_s_valid = np.array(adnumer_s_valid).T
adnumer_s_train = sser.fit_transform(adnumer_s_train)
adnumer_s_valid = sser.transform(adnumer_s_valid)
usrcateg_m_train,usrcateg_m_valid = batch_train_test_split(train_nn,
                                                     test_size=0.2,random_state=seed)
m_oht_nn, m_oht_nnindex = multifeat_onehot(train_nn,count_threshold = 20,)
usrcateg_m_oh_train,usrcateg_m_oh_valid= batch_train_test_split(m_oht_nn,
                                                     test_size=0.2,random_state=seed)

y_train,y_valid = batch_train_test_split([trainmerge['label'].values],test_size=0.2,random_state=seed)
y_train = y_train[0]
y_valid = y_valid[0]
aid_train,aid_valid = batch_train_test_split([trainmerge['aid'].values],test_size=0.2,random_state=seed)
aid_train = aid_train[0]
aid_valid = aid_valid[0]
del train_nn
gc.collect()

#para = pd.read_csv('./nn_record.csv').sort_values(by='val_auc', ascending=False)
for i in range(100):
    K = np.random.randint(4, 128)  # 64
    lw = 7.5e-4 * (0.1 ** (np.random.rand() * 3 - 1.5))
    lw1 =  1e-3 * (0.1 ** (np.random.rand() * 3 - 1.5))
    lr = 7.5e-4 * (0.1 ** (np.random.rand() * 2 - 1.0))
    lr_decay = 0.65 + np.random.rand() * 0.3
    #activation = np.random.choice(['relu', 'tanh', 'prelu', 'leakyrelu', 'elu'])
    #batchnorm = np.random.choice([True, False])
    activation = 'relu'
    batchnorm = True

    sample_weight_rate = 0.0
#     lr = 0.0014
#     lr_decay = 0.57
#     lw1=0.001
#     lw = 0.001
#     K = 60
    epoch=40
    batch_size=8196

    #modelname =  "%f_%f_%f_%f_%d_%d_%d_%s_%d" % (lr,lr_decay,lw1,lw,K,epoch,batch_size,"withouttwoapp",seed,)
    print('K: %d, lw: %e, lw1: %e, lr: %e, lr_decay: %f, act: %s, batchnorm: %s'%(K, lw,lw1, lr, lr_decay, activation, batchnorm))
    model = md.get_model(K,adcateg_s_train,usrcateg_s_train,usrcateg_m_train,usrnumer_s_train,adnumer_s_train,
                         lw=lw,lr=lr,lw1=lw1,act=activation,batchnorm=batchnorm)

    dataGenerator = DataGenerator()
    train_flow = dataGenerator.flow(adcateg_s_train+usrcateg_s_train+usrcateg_m_train,
                                    [usrnumer_s_train,adnumer_s_train],
                                    y=y_train,
                                    batch_size=batch_size, shuffle=True)
    valid_flow = dataGenerator.flow(adcateg_s_valid+usrcateg_s_valid+usrcateg_m_valid,
                                    [usrnumer_s_valid,adnumer_s_valid],
                                    y=y_valid,
                                    batch_size=batch_size, shuffle=False)
    early_stopping =EarlyStopping(monitor='val_auc_loss', patience=2, min_delta=0.0000,verbose=2,mode='max')

    timenow = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
    modelseed =  np.random.randint(0,65536)
    model_path = 'model/model_1/bst_model_%s_%d.h5'%(timenow,modelseed)
    model_checkpoint = ModelCheckpoint(model_path, monitor='val_auc_loss', save_best_only=True, save_weights_only=True)
    lr_reducer = LearningRateScheduler(lambda x: lr*(lr_decay**x))
    hist = model.fit_generator(train_flow, train_flow.__len__(),
                               epochs=epoch, workers=4,#class_weight={1:20,0:1},
                               callbacks=[lr_reducer,
                                          #RocAucMetricCallback(predict_batch_size=batch_size),
                                          auc_callback(adcateg_s_valid+usrcateg_s_valid+usrcateg_m_valid,
                                                      [usrnumer_s_valid,adnumer_s_valid],
                                                      y_valid),early_stopping, model_checkpoint,
                                         ],
                               validation_data = valid_flow,
                               validation_steps=valid_flow.__len__())

    model.load_weights(model_path)

    bst_epoch = np.argmax(hist.history['val_auc_loss'])
    trn_loss = hist.history['loss'][bst_epoch]
    val_loss = hist.history['val_loss'][bst_epoch]
    val_auc_loss = hist.history['val_auc_loss'][bst_epoch]

#     valid_flow = dataGenerator.flow(adcateg_s_valid+usrcateg_s_valid+usrcateg_m_valid,
#                                     [usrnumer_s_valid,adnumer_s_valid],
#                                     y=y_valid,
#                                     batch_size=batch_size, shuffle=False)
#    model.save_weights("model/weights_%s.h5" % modelname)
#     valid_pred = model.predict_generator(valid_flow, valid_flow.__len__(), workers=1)
#     roc_auc_score(y_valid,valid_pred[:,0])

    res = '%s,%s,%d,%d,%s,%s,%d,%e,%e,%e,%f,%e,%.5f,%.5f,%.5f,\n'%(timenow, \
            'model_1',modelseed,seed, activation, batchnorm, K, lw, lw1, lr, lr_decay,  bst_epoch+1, trn_loss, \
            val_loss, val_auc_loss)
    f = open('./nn_record.csv', 'a')
    f.write(res)
    f.close()


"The drop metadata object and related functions"""
import logging
import os
import shutil
from typing import Any
from typing import Dict
from typing import Iterator
from typing import List
from typing import Optional
from typing import Tuple
from typing import Union

import aiofiles  # type: ignore
import bencode  # type: ignore

from syncr_backend.constants import DEFAULT_METADATA_LOOKUP_LOCATION
from syncr_backend.constants import DEFAULT_PUB_KEY_LOOKUP_LOCATION
from syncr_backend.external_interface.public_key_store import \
    get_public_key_store
from syncr_backend.init import node_init
from syncr_backend.init.node_init import get_full_init_directory
from syncr_backend.init.node_init import load_private_key_from_disk
from syncr_backend.util import crypto_util
from syncr_backend.util.async_util import async_cache
from syncr_backend.util.crypto_util import load_public_key
from syncr_backend.util.crypto_util import node_id_from_private_key
from syncr_backend.util.crypto_util import VerificationException
from syncr_backend.util.log_util import get_logger

LATEST = "LATEST"

logger = get_logger(__name__)

class DropVersion(object):
    """A drop version"""

    def __init__(self, version: int, nonce: int) -> None:
        self.version = version
        self.nonce = nonce

    def __iter__(self) -> Iterator[Tuple[str, Union['DropVersion', int]]]:
        """Used for calling dict() on this object, so it becomes
        {'version': version, 'nonce': nonce}
        """
        yield 'version', self.version
        yield 'nonce', self.nonce

    def __str__(self) -> str:
        return "%s_%s" % (self.version, self.nonce)

    def __eq__(self, other: object) -> bool:
        if isinstance(other, DropVersion):
            return self.version == other.version and self.nonce == other.nonce
        return False

    def __lt__(self, other: object) -> bool:
        if not isinstance(other, DropVersion):
            raise TypeError(other)
        return self.version < other.version

    def __le__(self, other: object) -> bool:
        if not isinstance(other, DropVersion):
            raise TypeError(other)
        return self.version <= other.version

class DropMetadata(object):
    """Representation of a drop's metadata file"""

    def __init__(
        self, drop_id: bytes, name: str, version: DropVersion,
        previous_versions: List[DropVersion], primary_owner: bytes,
        other_owners: Dict[bytes, int], signed_by: bytes,
        files: Dict[str, bytes],
        files_hash: Optional[bytes]=None, sig: Optional[bytes]=None,
        protocol_version: int=1,
    ) -> None:
        self.id = drop_id
        self.name = name
        self.version = version
        self.previous_versions = previous_versions
        self.owner = primary_owner
        self.other_owners = other_owners
        self.signed_by = signed_by
        self.files = files
        self.sig = sig
        self._protocol_version = protocol_version
        self._files_hash = files_hash
        self._log = None  # type: Optional[logging.Logger]

    @property
    def log(self) -> logging.Logger:
        """
        A logger for this object

        :return: a logger object for this class
        """
        if self._log is None:
            self._log = get_logger(
                '.'.join([
                    __name__, self.__class__.__name__,
                    crypto_util.b64encode(self.id).decode('utf-8'),
                ]),
            )
        return self._log

    @property
    async def files_hash(self) -> bytes:
        """Generate the hash of the files dictionary

        :return: The hash of the bencoded files dict
        """
        if self._files_hash is not None:
            return self._files_hash
        else:
            h = await self._gen_files_hash()
            self._files_hash = h
            return h

    async def _gen_files_hash(self) -> bytes:
        return await crypto_util.hash_dict(self.files)

    async def verify_files_hash(self) -> None:
        """Verify the file hash in this object

        Returns None if the hash is OK, throwns a VerificationException if the
        hash is not good or has not been set
        :raises VerificationException: If the files hash is not found or wrong
        """
        if self._files_hash is None:
            self.log.error("no files hash found when verifying")
            raise VerificationException()
        given = self._files_hash
        expected = await self._gen_files_hash()
        if given != expected:
            self.log.error("files verification failed!")
            raise VerificationException()

    @property
    async def unsigned_header(self) -> Dict[str, Any]:
        """Get the unsigned version of the header
        The signature is set to b"", and the files list is {}

        :return: A dict that is the drop metadata header, without a signature
        """
        h = {
            "protocol_version": self._protocol_version,
            "drop_id": self.id,
            "name": self.name,
            "version": self.version.version,
            "version_nonce": self.version.nonce,
            "previous_versions": [dict(v) for v in self.previous_versions],
            "primary_owner": self.owner,
            "other_owners": self.other_owners,
            "header_signature": b"",
            "signed_by": self.signed_by,
            "files_hash": await self.files_hash,
            "files": {},
        }
        return h

    @property
    async def header(self) -> Dict[str, Any]:
        """Get the full header, including signature
        If there is not signature already, will generate it, which requires
        to the private key of signed_by

        :return: The full drop metadata header in dict form
        """
        h = await self.unsigned_header
        if self.sig is None:
            self.log.debug("signing header")
            key = await node_init.load_private_key_from_disk()
            self.sig = await crypto_util.sign_dictionary(key, h)
        h["header_signature"] = self.sig
        return h

    async def verify_header(self) -> None:
        """Verify the signature in the header

        If the signature is OK, returns none, if the signature is None or is
        invalid throws a VerificationException
        :raises VerificationException: If the header signature doesn't match
        """
        if self.sig is None:
            self.log.error("header signature not found when verifying")
            raise VerificationException()
        key = await get_pub_key(self.signed_by)
        await crypto_util.verify_signed_dictionary(
            key, self.sig, (await self.unsigned_header),
        )

    def get_file_name_from_id(self, file_hash: bytes) -> str:
        """Get the file name of a file id

        :param file_hash: the file id
        :raises FileNotFoundError: If the file was not found in the metadata
        :return: the file name string
        """
        for (fname, fhash) in self.files.items():
            if fhash == file_hash:
                return fname

        self.log.error("tried to lookup a file that doesn't exist")
        raise FileNotFoundError()

    def unsubscribe(self) -> None:
        """Removes the refrence in the .5yncr folder therefore preventing
        future updates

        :return: None
        """
        save_path = _get_save_path()
        encoded_drop_id = crypto_util.b64encode(self.id).decode('utf-8')
        drop_loc_file = os.path.join(save_path, encoded_drop_id)
        logger.info("removing file: ", drop_loc_file)
        os.remove(drop_loc_file)

    async def delete(self) -> None:
        """Deletes the drop from the local system and unsubscribes

        :return: None
        """
        self.unsubscribe()
        drop_loc = await get_drop_location(self.id)
        self.log.debug("deleteing drop folder: %s", self.id)
        shutil.rmtree(drop_loc)

    @staticmethod
    def make_filename(
        id: bytes, version: Union[str, DropVersion],
    ) -> str:
        """Make the filename for a drop metadata"""
        return "%s_%s" % (
            crypto_util.b64encode(id).decode("utf-8"), str(version),
        )

    async def write_file(
        self, metadata_location: str, is_latest: bool=True,
    ) -> None:
        """Write the representation of this objec to disk

        :param metadata_location: where to write to disk
        :param is_latest: whether to also write the LATEST file
        :return: None
        """
        self.log.debug("writing file")
        file_name = DropMetadata.make_filename(self.id, self.version)
        if not os.path.exists(metadata_location):
            os.makedirs(metadata_location)
        async with aiofiles.open(
            os.path.join(metadata_location, file_name), 'wb',
        ) as f:
            await f.write(await self.encode())
        if is_latest:
            await DropMetadata.write_latest(
                self.id, self.version, metadata_location,
            )

    @staticmethod
    async def write_latest(
        id: bytes, version: DropVersion,
        metadata_location: str,
    ) -> None:
        """Write the latest version to disk

        :param id: the drop id
        :param version: the latest version
        :para metadata_location: where to write it
        """
        file_name = DropMetadata.make_filename(id, LATEST)
        async with aiofiles.open(
            os.path.join(metadata_location, file_name), 'w',
        ) as f:
            to_write = DropMetadata.make_filename(id, version)
            await f.write(to_write)

    @staticmethod
    async def read_latest(
        id: bytes, metadata_location: str,
    ) -> Optional[str]:
        """Read the latest drop version

        :param id: the drop id
        :param metadata_location: where to find it
        :return: The latest version, or none if not found
        """
        file_name = DropMetadata.make_filename(id, LATEST)
        logger.debug(
            "trying to read file %s from %s", file_name, metadata_location,
        )
        if not os.path.isfile(os.path.join(metadata_location, file_name)):
            logger.debug("File not found")
            return None
        async with aiofiles.open(
            os.path.join(metadata_location, file_name), 'r',
        ) as f:
            logger.debug("Reading file")
            return await f.readline()

    @staticmethod
    @async_cache()
    async def read_file(
        id: bytes, metadata_location: str, version: Optional[DropVersion]=None,
    ) -> Optional['DropMetadata']:
        """Read a drop metadata file from disk

        :param id: the drop id
        :param metadata_location: where to look for the file
        :param version: the drop version
        :return: A DropMetadata object, or maybe None
        """
        logger.debug("reading from file")
        if version is None:
            logger.debug(
                "Version is None, looking it up in %s", metadata_location,
            )
            file_name = await DropMetadata.read_latest(
                id, metadata_location,
            )
        else:
            logger.debug("Getting version %s", version)
            file_name = DropMetadata.make_filename(id, version)
        if file_name is None:
            logger.warning(
                "latest drop metadata not found for %s",
                crypto_util.b64encode(id),
            )
            return None

        if not os.path.isfile(os.path.join(metadata_location, file_name)):
            logger.warning(
                "drop metadata not found for %s",
                crypto_util.b64encode(id),
            )
            return None

        async with aiofiles.open(
            os.path.join(metadata_location, file_name), 'rb',
        ) as f:
            b = b''
            while True:
                data = await f.read()
                if not data:
                    break
                b += data
            return await DropMetadata.decode(b)

    async def encode(self) -> bytes:
        """Encode the full drop metadata file, including files, to bytes

        :return: The bencoded full metadata file
        """
        h = await self.header
        h["files"] = self.files
        return bencode.encode(h)

    @staticmethod
    async def decode(b: bytes) -> 'DropMetadata':
        """Decodes a bencoded drop metadata file to a DropMetadata object
        Also verifies the files hash and header signature, and throws an
        exception if they're not OK

        :param b: The bencoded file
        :return: A DropMetadata object from b
        """
        # Note: assumes signed header
        decoded = bencode.decode(b)
        dm = DropMetadata(
            drop_id=decoded["drop_id"],
            name=decoded["name"],
            version=DropVersion(decoded["version"], decoded["version_nonce"]),
            previous_versions=[
                DropVersion(
                    v["version"],
                    v["nonce"],
                ) for v in decoded["previous_versions"]
            ],
            primary_owner=decoded["primary_owner"],
            other_owners=decoded["other_owners"],
            signed_by=decoded["signed_by"],
            files_hash=decoded["files_hash"],
            files=decoded["files"],
            sig=decoded["header_signature"],
        )
        await dm.verify_files_hash()
        await dm.verify_header()
        return dm

async def save_drop_location(drop_id: bytes, location: str) -> None:
    """Save a drop's location in the central data dir

    :param drop_id: The unencoded drop id
    :param location: Where the drop is located on disk
    """
    save_path = _get_save_path()

    encoded_drop_id = crypto_util.b64encode(drop_id).decode('utf-8')

    if not os.path.exists(save_path):
        os.makedirs(save_path)

    async with aiofiles.open(
        os.path.join(save_path, encoded_drop_id), 'w',
    ) as f:
        await f.write(location)

async def get_drop_location(drop_id: bytes) -> str:
    """Get a drop's location from the central data dir

    :param drop_id: The drop id to look up
    :return: The drops save dir
    """
    save_path = _get_save_path()

    encoded_drop_id = crypto_util.b64encode(drop_id).decode('utf-8')

    async with aiofiles.open(
        os.path.join(save_path, encoded_drop_id), 'r',
    ) as f:
        return await f.read()

def list_drops() -> List[bytes]:
    """
    List the drops on this node

    :return: List of drop IDs
    """
    save_path = _get_save_path()
    names = os.listdir(save_path)

    return [crypto_util.b64decode(os.fsencode(e)) for e in names]

def _get_save_path() -> str:
    node_info_path = node_init.get_full_init_directory()
    save_path = os.path.join(node_info_path, DEFAULT_METADATA_LOOKUP_LOCATION)
    if not os.path.exists(save_path):
        os.mkdir(save_path)
    return save_path

async def get_pub_key(node_id: bytes) -> crypto_util.rsa.RSAPublicKey:
    """
    Gets the public key from disk if possible otherwise request it from
    PublicKeyStore

    :param node_id: bytes for the node you want public key of
    :raises VerificationException: If the pub key cannot be retrieved
    :return: PublicKey
    """
    init_directory = get_full_init_directory(None)
    pub_key_directory = os.path.join(
        init_directory,
        DEFAULT_PUB_KEY_LOOKUP_LOCATION,
    )

    if not os.path.isdir(pub_key_directory):
        os.makedirs(pub_key_directory)

    key_file_name = "{}.pub".format(crypto_util.b64encode(node_id))
    key_path = os.path.join(pub_key_directory, key_file_name)

    if os.path.isfile(key_path):
        async with aiofiles.open(key_path, 'rb') as pub_file:
            pub_key = await pub_file.read()
            return load_public_key(pub_key)
    else:
        key_bytes = await load_private_key_from_disk()
        this_node_id = await node_id_from_private_key(key_bytes)
        public_key_store = await get_public_key_store(this_node_id)
        key_request = await public_key_store.request_key(node_id)
        if key_request[0] and key_request[1] is not None:
            pub_key = key_request[1].encode('utf-8')
            await _save_key_to_disk(key_path, pub_key)
            return load_public_key(pub_key)
        else:
            raise VerificationException()

async def send_my_pub_key() -> None:
    """Send the pub key for this node the the Key Store"""
    this_node_id = await node_id_from_private_key(
        await load_private_key_from_disk(),
    )
    logger.info(
        "Sending pub key for %s to tracker",
        crypto_util.b64encode(this_node_id),
    )
    public_key_store = await get_public_key_store(this_node_id)
    pub_key = (await load_private_key_from_disk()).public_key()
    pub_key_bytes = crypto_util.dump_public_key(pub_key)
    await public_key_store.set_key(pub_key_bytes)

async def _save_key_to_disk(key_path: str, pub_key: bytes) -> None:
    """
    Saves the public key to the specified location

    :param key_path: absolute path to location of public key
    :param pub_key: bytes to be saved
    """
    async with aiofiles.open(key_path, 'wb') as pub_file:
        await pub_file.write(pub_key)

def gen_drop_id(first_owner: bytes) -> bytes:
    """
    Geterate a drop id

    :param first_owner: The initial primary owner
    :return: A drop id, which is the first owner's ID plus random bytes
    """
    return first_owner + crypto_util.random_bytes()


import time
from collections import defaultdict
import iso8601
import logging
import arrow

from impactstoryanalytics.widgets.widget import Widget, get_raw_dataclip_data
from impactstoryanalytics.widgets.widget_api_helpers import Converter

logger = logging.getLogger("impactstoryanalytics.widgets.signup_growth_target")

class Signup_growth_target(Widget):
    signups_with_products_url = "https://dataclips.heroku.com/feblvvoknanzuiumyiawutmqdwbo.json"
    raw_signups_data_url = "https://dataclips.heroku.com/qaclkwteaqnrltwxojgzytkgwczs.json"

    def __init__(self):
        Widget.__init__(self)
        self.start_date = arrow.get("2014-02-03")
        self.daily_growth = .0073  # times 7 = 5.11% weekly
        self.signups_total_endpoints = {
            "start": 4255,
            "finish": 10000  # 10k
        }

    def get_data(self):

        signups_with_products_data = get_raw_dataclip_data(self.signups_with_products_url)
        raw_signups_dataclip_data = get_raw_dataclip_data(self.raw_signups_data_url)

        actual_line = {}

        # make the raw signups line
        for row in raw_signups_dataclip_data["values"]:
            day_str = row[0][0:10]
            actual_line[day_str] = [int(row[1])]

        # make the signups-with-products line
        for row in signups_with_products_data["values"]:
            day_str = row[0][0:10]
            actual_line[day_str].append(int(row[1]))

        target_line = self.make_target_line()

        rows = []
        for date in sorted(target_line.keys()):
            actual_values = actual_line.get(date, ["", ""])
            target_values = target_line[date]

            rows.append([date] + actual_values + target_values)

        fields = [
            "date",
            "signups_raw",
            "signups_with_products",
            "target_signups_with_products",
            "target_signups_with_products_total"
        ]

        return {"fields": fields, "values": rows}

    def get_signup_line(self):
        pass

    def make_target_line(self):
        target_line = {}
        datapoint = {
            "date": self.start_date,
            "signups": "",
            "signups_total": self.signups_total_endpoints["start"]
        }

        while datapoint["signups_total"] < self.signups_total_endpoints["finish"]:

            # update the target line
            date_str = datapoint["date"].format('YYYY-MM-DD')
            target_line[date_str] = [
                datapoint["signups"],
                datapoint["signups_total"]
            ]

            # increment values for next iteration
            datapoint["date"] = datapoint["date"].replace(days=+1)
            datapoint["signups"] = self.daily_growth * datapoint["signups_total"]
            datapoint["signups_total"] *= (1 + self.daily_growth)

        return target_line


#!/usr/bin/python

'''
Update ansibleHosts to register/unregister hosts with ansible

@author : rene.ribaud@hp.com
'''
import sys
import pprint
import argparse
import os
import ConfigParser

# Variable needed

confFile = "/home/sdi/ansible/ansibleHosts"
prog = sys.argv[0]

if __name__ == '__main__':
    pp = pprint.PrettyPrinter(indent=4)

    parser = argparse.ArgumentParser(version = prog + ' version 1.0')

    parser.add_argument('action', action='store',
                        choices=['add','remove'],
                    help='Action to do either add or remove hostname to ansibleHosts')

    parser.add_argument('group', action='store',
                    help='Group to add or remove the hostname')

    parser.add_argument('hostname', action='store',
                    help='hostname to add or remove')

    arguments = parser.parse_args()

    # Load json

    config = ConfigParser.RawConfigParser(allow_no_value=True)
    config.read(confFile)

    if (arguments.action == "add"):
        config.add_section(arguments.group)
        config.set(arguments.group, arguments.hostname)

    if (arguments.action == "remove"):
        config.remove_option(arguments.group, arguments.hostname)

    with open(confFile, 'wb') as configfile:
        config.write(configfile)


from re import search, finditer, findall
from collections import defaultdict

with open('19.in', 'r') as f:
    raw_input = f.read()

TASK_REGEX = '(\\w+) => (\\w+)'

rules = defaultdict(set)
for line in raw_input.split("\n\n")[0].splitlines():
    begin, end = search(TASK_REGEX, line).group(1, 2)
    rules[begin].add(end)

medicine_molecule = raw_input.split("\n\n")[1]

def part1():
    generated = set()
    for r in rules:
        for m in finditer(r, medicine_molecule):
            for to in rules[r]:
                generated.add(medicine_molecule[:m.start()] + to + medicine_molecule[m.start() + len(r):])
    print(len(generated))

def normalize_chomsky(rules):
    rs = defaultdict(set)
    temp_num = 0
    for r in rules:
        for right_side in rules[r]:
            non_terminals = findall('[A-Z][^A-Z]*', right_side)
            if len(non_terminals) == 2:
                rs[r].add("".join(non_terminals))
            else:
                for i in range(len(non_terminals) - 1):
                    if i == 0:
                        left_side = r
                        new_right_side = non_terminals[0] + "Tmp" + str(temp_num)
                    elif i == len(non_terminals) - 2:
                        left_side = "Tmp" + str(temp_num - 1)
                        new_right_side = non_terminals[-2] + non_terminals[-1]
                    else:
                        left_side = "Tmp" + str(temp_num - 1)
                        new_right_side = non_terminals[i] + "Tmp" + str(temp_num)
                    rs[left_side].add(new_right_side)
                    temp_num += 1
    return rs

def search_for_rules(right_side):
    found = set()
    for l, r in rules.items():
        if right_side in r:
            found.add(l)
    return found

def part2():
    global rules
    rules = normalize_chomsky(rules)

    parts = findall('[A-Z][^A-Z]*', medicine_molecule)

    Xs = defaultdict(set)
    for i in range(len(parts)):
        Xs[(i+1, i+1)].add(parts[i])

    Ys = defaultdict(set)
    for i in range(len(parts)):
        Ys[(i+1, i+1)].add(0)

    for subword_length in range(2, len(parts) + 1):
        for start in range(1, len(parts) + 1 - subword_length + 1):
            currently_calculated = (start, start + subword_length - 1)
            for left_side in range(subword_length - 1):
                pyramid_left_element = (start, start + left_side)
                pyramid_right_element = (start + left_side + 1, start + subword_length - 1)
                for a in Xs[pyramid_left_element]:
                    for b in Xs[pyramid_right_element]:
                        for r in search_for_rules(a + b):
                            Xs[currently_calculated].add(r)
                            Ys[currently_calculated].add(min(Ys[pyramid_left_element]) + min(Ys[pyramid_right_element]) + [0, 1]['Tmp' not in r])

    print(min(Ys[(1, len(parts))]))

part1()
part2()


import os

import torch
from torch import nn

class ResidualGRU(nn.Module):
    def __init__(self, input_size, output_size, hidden_size, num_layers, dropout_p):
        super(ResidualGRU, self).__init__()
        self.input_size = input_size
        self.output_size = output_size
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.dropout_p = dropout_p

        self.input_layer = nn.Linear(self.input_size, self.hidden_size)
        self.grus = nn.ModuleList([nn.GRU(2 * hidden_size, hidden_size,
                                          batch_first=True) for _ in range(self.num_layers)])
        self.output_layer = nn.Linear(self.hidden_size, output_size)

    def init_hidden(self, batch_size, device):
        return torch.zeros(1, batch_size, self.hidden_size, device=device)

    def forward(self, x, hidden):
        x = nn.SELU()(self.input_layer(x))
        x_prev = x
        for i in range(len(self.grus)):
            gru_input = torch.cat((x_prev, x), dim=-1)
            x_prev = x
            x, hidden = self.grus[i](gru_input, hidden)
        x = self.output_layer(x)
        return x

    def save(self, archive_dir):
        print("Saving model to {}".format(archive_dir))
        if not os.path.exists(archive_dir):
            os.makedirs(archive_dir)
        torch.save({
            "state_dict": self.state_dict(),
            "input_size": self.input_size,
            "output_size": self.output_size,
            "hidden_size": self.hidden_size,
            "num_layers": self.num_layers,
            "dropout_p": self.dropout_p
        }, os.path.join(archive_dir, "model.pt"))

    @staticmethod
    def load(archive_dir, device):
        print("Loading model from {}".format(archive_dir))
        checkpoint = torch.load(os.path.join(archive_dir, "model.pt"), map_location=device)
        model = ResidualGRU(checkpoint["input_size"], checkpoint["output_size"],
                             checkpoint["hidden_size"], checkpoint["num_layers"],
                             checkpoint["dropout_p"])
        model.load_state_dict(checkpoint["state_dict"])
        model = model.to(device)
        return model


import torch
import torch.nn as nn
import torch.nn.functional as F

from ..layers.neural_process import fc_layer

class NeuralProcess(nn.Module):

    def __init__(self, hidden_dim=256):
        super(NeuralProcess, self).__init__()
        self.hidden_dim = 30
        self.h = nn.Sequential(
                fc_layer(3, 400),
                fc_layer(400, 400),
                fc_layer(400, 400),
                )

        self.mean_aggregater = nn.Sequential(
                nn.Linear(400, 400),
                nn.LeakyReLU(),
                nn.Linear(400, self.hidden_dim),
                )
        self.variance_aggregater = nn.Sequential(
                nn.Linear(400, 400),
                nn.LeakyReLU(),
                nn.Linear(400, self.hidden_dim),
                )

        self.g = nn.Sequential(
                fc_layer(2 + self.hidden_dim, 400),
                fc_layer(400, 400),
                fc_layer(400, 400),
                fc_layer(400, 400),
                nn.Linear(400, 1)
                )

    def forward(self, context_x, context_y, target_x, target_y):
        context = torch.cat((context_x, context_y), dim=1)
        context_representations = self.h(context).mean(0)

        context_mean = self.mean_aggregater(context_representations)
        context_std = self.variance_aggregater(context_representations)
        context_var = torch.exp(0.5 * context_std)
        context_z = torch.rand_like(context_var) * context_var + context_mean

        all_x = torch.cat((context_x, target_x), dim=0)
        all_y = torch.cat((context_y, target_y), dim=0)

        all_representations = self.h(torch.cat((all_x, all_y), dim=1)).mean(0)
        all_mean = self.mean_aggregater(all_representations)
        all_std = self.variance_aggregater(all_representations)
        all_var = torch.exp(0.5 * all_std)
        all_z = torch.rand_like(all_var) * all_var + all_mean

        x_domain = all_x.size(0)
        target_x_domain = target_x.size(0)
        context_x_domain = context_x.size(0)

        expanded_all_z = torch.stack([all_z for i in range(target_x_domain)])
        target_logits = F.sigmoid(self.g(
            torch.cat((expanded_all_z, target_x), dim=1)))

        expanded_all_z = torch.stack([all_z for i in range(context_x_domain)])
        context_logits = F.sigmoid(self.g(
            torch.cat((expanded_all_z, context_x), dim=1)))

        expanded_context_z = torch.stack([context_z for i in range(context_x_domain)])
        context_preds_logits = F.sigmoid(self.g(
            torch.cat((expanded_context_z, context_x), dim=1)))

        expanded_target_z = torch.stack([context_z for i in range(target_x_domain)])
        target_preds_logits = F.sigmoid(self.g(
            torch.cat((expanded_target_z, target_x), dim=1)))

        negative_kl_divergence = self._calc_negative_kl_divergence(
                context_mean, context_std, all_mean, all_std).sum()

        return target_logits, context_logits, context_preds_logits, \
            target_preds_logits, negative_kl_divergence

    def _calc_negative_kl_divergence(self, c_mu, c_std, a_mu, a_std):
        c_std = torch.exp(0.5 * c_std)
        a_std = torch.exp(0.5 * a_std)
        return  torch.log(c_std / a_std) + \
            ((a_std ** 2 + (a_mu - c_mu) ** 2) / (2 * c_std ** 2)) - 0.5


#! /usr/bin/env python2
# -*- coding: utf-8 -*-

import pygame
import sys
from pygame.locals import *

# inicjacja modułu pygame
pygame.init()

# szerokość i wysokość okna gry
OKNOGRY_SZER = 800
OKNOGRY_WYS = 400
# kolor okna gry, składowe RGB zapisane w tupli
LT_BLUE = (230, 255, 255)

# powierzchnia do rysowania, czyli inicjacja okna gry
oknogry = pygame.display.set_mode((OKNOGRY_SZER, OKNOGRY_WYS), 0, 32)
# tytuł okna gry
pygame.display.set_caption('Prosty Pong')

# paletka gracza #########################################################
PALETKA_SZER = 100  # szerokość
PALETKA_WYS = 20  # wysokość
BLUE = (0, 0, 255)  # kolor wypełnienia
PALETKA_1_POZ = (350, 360)  # początkowa pozycja zapisana w tupli
# utworzenie powierzchni paletki, wypełnienie jej kolorem,
paletka1 = pygame.Surface([PALETKA_SZER, PALETKA_WYS])
paletka1.fill(BLUE)
# ustawienie prostokąta zawierającego paletkę w początkowej pozycji
paletka1_prost = paletka1.get_rect()
paletka1_prost.x = PALETKA_1_POZ[0]
paletka1_prost.y = PALETKA_1_POZ[1]

# pętla główna programu
while True:
    # obsługa zdarzeń generowanych przez gracza
    for event in pygame.event.get():
        # przechwyć zamknięcie okna
        if event.type == QUIT:
            pygame.quit()
            sys.exit()

        # przechwyć ruch myszy
        if event.type == MOUSEMOTION:
            myszaX, myszaY = event.pos  # współrzędne x, y kursora myszy

            # oblicz przesunięcie paletki gracza
            przesuniecie = myszaX - (PALETKA_SZER / 2)

            # jeżeli wykraczamy poza okno gry w prawo
            if przesuniecie > OKNOGRY_SZER - PALETKA_SZER:
                przesuniecie = OKNOGRY_SZER - PALETKA_SZER
            # jeżeli wykraczamy poza okno gry w lewo
            if przesuniecie < 0:
                przesuniecie = 0
            # zaktualizuj położenie paletki w poziomie
            paletka1_prost.x = przesuniecie

    # rysowanie obiektów
    oknogry.fill(LT_BLUE)  # kolor okna gry

    # narysuj w oknie gry paletki
    oknogry.blit(paletka1, paletka1_prost)

    # zaktualizuj okno i wyświetl
    pygame.display.update()

# KONIEC


from django import forms
from p_library.models import Author, Book, BorrowBook, Friend
from django.utils.translation import gettext_lazy as _

class AuthorForm(forms.ModelForm):
    full_name = forms.CharField(widget=forms.TextInput, label="Имя")
#    required_css_class = 'list-group-item'
    class Meta:
        model = Author
        fields = '__all__'
        labels = {
            "birth_year":_("Год рождения"),
            "country":_("Страна")
        }

class BookForm(forms.ModelForm):
    title = forms.CharField(widget=forms.TextInput, label="Название")
    class Meta:
        model = Book
        fields = '__all__'
        labels = {
            'ISBN':_("Код ISBN"),
            'author':_("Автор"),
            'publisher':_("Издательство"),
            'description':_('Описание'),
            'year_release':_('Год издания'),
            'copy_count':_('Экз.(шт.)'),
            'price':_('Цена'),
            'book_img':_('Обложка'),
        } #подписи полей на форме вывода
        exclude = ('friend',)

class BookFormEdit(forms.ModelForm):
    title = forms.CharField(widget=forms.TextInput, label="Название")
    class Meta:
        model = Book
        fields = '__all__'
        labels = {
            'ISBN':_("Код ISBN"),
            'author':_("Автор"),
            'friend':_('Друг'),
            'publisher':_("Издательство"),
            'description':_('Описание'),
            'year_release':_('Год издания'),
            'copy_count':_('Экз.(шт.)'),
            'price':_('Цена'),
            'book_img':_('Обложка'),
        } #подписи полей на форме вывода

class BorrowBookFormGive(forms.ModelForm):
    class Meta:
        model = BorrowBook
        fields = ['book', 'friend','borrow_date',
        'state_before']
        labels = {
            'book':_("Книга"),
            'friend':_("Должник"),
            'borrow_date':_("Дата выдачи"),
            'state_before':_("Состояние книги"),
        } #подписи полей на форме вывода
        widgets = {
            #'borrow_date': forms.DateInput(format='%Y-%m-%d', attrs={'type':'date','placeholder':'mm/dd/yyyy' }),
            'borrow_date': forms.SelectDateWidget(empty_label="Nothing"),
            'state_before': forms.TextInput(),
        } #виджеты на форме

class BorrowBookFormBack(forms.ModelForm):
    class Meta:
        model = BorrowBook
        fields = '__all__'

class BorrowBookFormReturn(forms.Form):
    return_date = forms.DateField(required=True, label="Дата возврата", widget=forms.SelectDateWidget(empty_label="Nothing"))
    #return_date = forms.DateField(required=True, label="Дата возврата", widget=forms.DateInput(format='%m/%d/%Y',attrs={'type':'date','placeholder':'yyyy-mm-dd' }))
    state_after = forms.CharField(required=True, label="Состояние книги")
    returned_flad = forms.BooleanField(initial=True, required=True, label="Принято")
    class Meta:
        pass

class FriendFormAdd(forms.ModelForm):
    full_name = forms.CharField(widget=forms.TextInput, label="Полное имя")
    class Meta:
        model = Friend
        fields = '__all__'
        labels = {
            "phone":_("Телефон"),
            "rating":_("Рейтинг"),
        }
class FriendFormEdit(forms.ModelForm):
    full_name = forms.CharField(widget=forms.TextInput, label="Полное имя")
    class Meta:
        model = Friend
        fields = "__all__"
        labels = {
            'phone':_("Телефон"),
            'rating':_("Рейтинг"),
        }

BorrowBookFormSet = forms.modelformset_factory(BorrowBook, fields=('book','friend','borrow_date'),
        widgets={'borrow_date':forms.TextInput()})


from demoparser2 import DemoParser

parser = DemoParser("path/to/demo.dem")
player_hurt_events = parser.parse_event("player_hurt")
df = parser.parse_ticks(["pitch", "yaw"])

for (idx, event) in player_hurt_events.iterrows():
    start_tick = event["tick"] - 300
    end_tick = event["tick"]
    attacker = event["attacker_steamid"]
    # attacker can be none when player gets hurt by c4 etc.
    if attacker != None:
        subdf = df[(df["tick"].between(start_tick, end_tick)) & (df["steamid"] == int(attacker))]
        print(subdf)