import numpy as np
##For matrix addition and multiplication

import itertools
##For cartesian products and such

import math
##For math stuff

from ast import literal_eval

# import time

#I'm keeping the next two as global variables since it is a pain to put them through multiple functions.
#######
## Probability configuration
## probabilities = [transitioning in specialty 1, transitioning in specialty 2]
trans_probs = np.array([[[0.3, 0.1, 0.6], [0.1, 0.2, 0.7], [0.0, 0.0, 1.0]], [[0.4, 0.1, 0.5], [0.1, 0.2, 0.7], [0.0, 0.0, 1.0]]])
##Entering probs = [entering in specialty 1, entering in specialty 2]
enter_probs = np.array([[0.3, 0.7, 0.0], [0.4, 0.6, 0.0]])
#######

#Resource class L
class Resources:
    def __init__(self,num_res, max_L, avg_ut, cap_L, cap_cost):
        ##How many resources are there
        self.amount = num_res
        ## How many units per resource available
        self.max = max_L
        ## What's the average usage per treatment pattern per period
        self.avgMatrix = avg_ut
        ## What's the desired amount of usage
        self.cap = cap_L
        ## What's the cost for going over this cap?
        self.cost = cap_cost

    def isOverUt(self, state):
        ##We have to group all the treatment patterns together
        ## len(self.avgMatrix[0]) = n
        n= len(self.avgMatrix[0])
        for res_ind in range(self.amount):
            som = 0
            for i in range(len(self.avgMatrix[0])):
                P=0
                for d in range((len(state)//(n))):
                    ##avg_ut = np.array([[2.2,2.6],[2.6,2.2]])
                    for l in range(len(self.avgMatrix[0])):
                        P += (state[l+d*n]*trans_probs[d][l][i])
                som += P*self.avgMatrix[res_ind][i]

            if som > self.max[res_ind]:
                return True
            else:
                pass
        return False

class Patterns:
    def __init__(self, num_patterns):
        ##How many treatment patterns are there
        self.amount = num_patterns
        self.transitions = dict()
        self.states = set()

    def addTransitions(self, specialty, treatmentPattern, size, transitions):
        self.transitions[(specialty, treatmentPattern, size)] = transitions

class Specialties:
    def __init__(self, num_sp, max_spec):
        ## number of specialties
        self.amount = num_sp
        ## Maximum admissions per period
        self.max = max_spec

        #Since self.max exists there is only a limited amount of actions possible
        self.actions = []
        self.actionDistributions = []
        self.__Actions(num_sp, max_spec)

    def Actions(self, amount, max):
        B=[]
        for i in range(len(max)):
            B.append([*range(max[i]+1)])
        A = itertools.product(*B)
        for action in A:
            self.actions.append(list(action))
    __Actions = Actions

    def addActionDistributions(self, actionDistributions):
        self.actionDistributions = actionDistributions

def outputFile(lijst, name):
    f = open(name+'.txt', "w+")
    for i in lijst:
        f.write(str(i)+"\n")
    f.close()

##ATTENTION: PARTITION IS AN AUXILIARY FUNCTION, IT DOESN'T ACTUALLY OUTPUT A PARTITION OF N IN D PARTS LIKE THIS:
##E.g. n=3 in d=3, [[3,0,0],[2,1,0],[2,0,1],[1,2,0],[1,1,1],[1,0,2],[0,3,0],[0,2,1],[0,1,2],[0,0,3]]

##Recursive algorithm for creating partitions of each integer UP TO n in d parts.
##E.g. n=3 in d=3: [0, 0, 0], [1, 0, 0], [2, 0, 0], [3, 0, 0], [0, 1, 0], [1, 1, 0], [2, 1, 0], [0, 2, 0], [1, 2, 0], [0, 3, 0], [0, 0, 1], [1, 0, 1], [2, 0, 1], [0, 1, 1], [1, 1, 1], [0, 2, 1], [0, 0, 2], [1, 0, 2], [0, 1, 2], [0, 0, 3]
def partition(n, d, depth=0):
    if d == depth:
        return [[]]
    return [
        item + [i]
        for i in range(n+1)
        for item in partition(n-i, d, depth=depth+1)
        ]

#makes lists unique and preserves order!
#I am not using set() since it doesn't keep the order I use
def f7(seq):
    seen = set()
    seen_add = seen.add
    return [x for x in seq if not (x in seen or seen_add(x))]

def EnterProb(admstate, specialty):
    P = math.factorial(sum(admstate))
    for i in range(len(admstate)-1):
        if i == len(admstate)-1:
            P = P*(enter_probs[specialty][i])**(sum(admstate)-sum(admstate[:len(admstate)-2]))/math.factorial(sum(admstate)-sum(admstate[:len(admstate)-2]))
        else:
            P = P*(enter_probs[specialty][i])**(admstate[i])/(math.factorial(admstate[i]))
    return P

##Calculates the transition probability given the PARTICULAR(!) transition tuple over the treatment patterns and the specialty
##E.g. Say we want to go to state_(t+1): [0,2,2] from state_t [3,1,1] with action 0, then we have a two-tuple ([0,2,1],[0,0,1])
##Where the first component of the tuple represents x_11 = 0, x_12 = 2, x_13=1 and the second tuple: x_21=0, x_22=0 and x_23=1
def TransProbs(comptuples, specialty):
    ## i will signify in which treatment pattern we're at on time period t
    P=1
    for i in range(len(comptuples)):
        P = P*math.factorial(sum(comptuples[i]))
        for j in range(len(comptuples[i])):
            P = P * (trans_probs[specialty][i][j]) ** (comptuples[i][j]) / (math.factorial(comptuples[i][j]))
    return P

##ATTENTION, IN CASE THERE ARE MORE THAN ONE SPECIALTY DO NOT ATTEMPT TO CALCULATE THE PROBABILITY OF AN ENTIRE STATE
##YOU NEED TO SPLIT IT UP IN PARTS FOR EACH SPECIALTY. E.G. CalcProb IN CASE OF:
##first state [1,0,1,0,2,3], next state [2,3,0,0,2,2], action (4,2), WITH E.amount = 3 AND S.amount = 2
##MUST BE CALCULATED LIKE: CalcProb([2,3,0], [1,0,1], 4, 0)*CalcProb([0,2,2], [0,2,3], 2, 1)

##Given state_(t+1) (state2), state_t (state1), action and specialty this will calculate the probability of going from one state to another
##It will obviously use the functions given above (EnterProb() and TransProbs())
def CalcProb(state2, state1, actionnr, specialty):
    ##If the patients in the next state do not match the patients in each effective treatment pattern + actionnr return probability 0
    if sum(state2) != sum(state1[:len(state1)-1])+actionnr:
        return 0

    ## Else continue
    else:

        ##If state2 = (8,11,3), then B = [[0,1,2,3,4,5,6,7,8],[0,1,2,3,4,5,6,7,8,9,10,11]]
        B=[]

        ##List of all possible action distributions given an action
        ##E.g. if the action is 2, then C = [[2,0],[1,1],[0,2]]
        C=[]

        ##Targetlist is a list for all targets I want to reach by applying only transitions and their respective entering probability.
        ##One target is defined as state2 - (minus) action distribution
        ##E.g. if state2 = [2,2,2] and action = 1, then the possible action distributions are:
        ##[1,0,0] and [0,1,0] and thus the possible targets are: [1,2,2] and [2,1,2]
        ##So targetlist = [[[1,2,2],P([1,0,0])], [[2,1,2],P([0,1,0])]]
        targetlist=[]

        ##Make B from description I gave above
        ##If state2 = (8,11,3), then B = [[0,1,2,3,4,5,6,7,8],[0,1,2,3,4,5,6,7,8,9,10,11]] and nothing for discharge pattern
        for i in range(len(state2)-1):
            B.append([*range(state2[i]+1)])
        ##A is a cartesian product of the sets in B
        ##E.g. B = [[1,2],[7,8]], then A = {(1,7),(1,8),(2,7),(2,8)}
        A = itertools.product(*B)

        ##this creates the list of action distributions C of above
        for action in A:
            ##We add the discharge pattern which is 0
            action = list(action)+[0]
            ##If the sum of the amount of patients equal the action then add the action distribution to C
            if sum(action)== actionnr:
                C.append(action)

        ##Note that we could have created C by means of the partition() function above, but then we would have to trim
        ##C by taking out all the action distributions where there are too many people being added.

        ##Let us now begin to
        for action in C:
            ##Target is the target we want to hit by doing only transitions
            target = np.array(state2)-np.array(action)
            ##Now we add the target and its action probability to targetlist
            targetlist.append([target.tolist(),EnterProb(action, specialty)])

        ##Above we got all the probabilities you could obtain by adding patients in the next period (entering prob).
        ##Now we calculate the probabilities you can get by moving patients around.
        transition_prob_sums = []
        ##Now we want to work for each target alone
        for targ in targetlist:
            ##Create a list for transitions
            trans = []
            ##Create a list for the sums of transitions
            transsums = []

            ##Let us iterate over the elements in target, except for the last element since that is the discharge
            for j in range(len(targ[0])-1):
                ##list of transitions created by going over the partition created by varying component j
                lst3=[]
                ## To partition sumlist in a component for every j, we must create a new list. sumlist = [[...], [...], ..., [...]]
                for i in range(targ[0][j]+1):
                    ##Since i stands for how many patients we keep in a treatment pattern we need to calculate how many we have to distribute
                    n = state1[j]-i
                    d = len(state1)-1

                    ##This will partition the number n in d parts
                    ##E.g. n= 4, d=3, then [[n - sum(p)] + p for p in partition(n, d - 1)] gives:
                    ##[4,0,0],[3,1,0],[3,0,1],[2,2,0],[2,1,1],[2,0,2],[1,3,0],[1,2,1],[1,1,2],[1,0,3],[0,4,0],[0,3,1],
                    ##[0,2,2],[0,1,3],[0,0,4].
                    lst = [[n - sum(p)] + p for p in partition(n, d - 1)]
                    ##What we have done up until this point is, given a target distribution and initial state: E.g. Targ = [2,3,4,5] and InitState = [10,4,0,1]
                    ##We have taken a single non-discharge component j in Targ and let i be iterated in the range of Targ[j]
                    ##So, take j=1, then Targ[j]=3 and thus i is iterated in the range [0,1,2,3].
                    ##Next, we took n = InitState[j]-i and d the number of patterns-1, ! THIS IS NOT BECAUSE OF THE DISCHARGE PATTERN, BUT RATHER BECAUSE WE ARE NOT USING COMPONENT j!
                    ##So in our example, let us say we are in the 3rd iteration, then i = 2 and thus
                    ##n = 4-2 and d = 3. (I realise that recalculating the same number is inefficient/sloppy, but I do not
                    ##think it is very expensive or time consuming.) Now we have created lst, which in our case is:
                    ##lst = [[2,0,0],[1,1,0],[1,0,1],[0,2,0],[0,1,1],[0,0,2]]
                    ##What will we do with lst? Well essentially we have distributed n people over the other treatment patterns,
                    ##While we kept i people in treatment pattern j, therefore we want to add component j in each list in lst.
                    ##lst2 will therefore be:
                    ##lst2 = [[2,2,0,0],[1,2,1,0],[1,2,0,1],[0,2,2,0],[0,2,1,1],[0,2,0,2]]
                    lst2=[]
                    for lijst in lst:
                        lijst = lijst[0:j] + [i] + lijst[j:]
                        lst2.append(lijst)

                    ##lst3 is a superlist of lst2 for all components. this means that it has all of the elements of lst2 for all components i.
                    ##We need this list so we can can make the following cartesian product:
                    ##[transitions by varying component 0]x[trans component 1]x...x[trans component E.amount-1]
                    ##This is equivalent with
                    ##[lst3, j=0]x[lst3, j=1]x...x[lst3, j=len(targ[0])-1]
                    lst3 = lst3 + [element for element in lst2]
                ##trans = [[lst3, j=0],[lst3, j=1],...,[lst3, j=len(targ[0])-1]]
                trans.append(lst3)
            ##transprods is exactly the cartesian product set I wanted, its class is not accessible by the usual operation
            ##transprods[k], but you can iterate over transprods like we do in our for-loop
            transprods = itertools.product(*trans)
            for prod in transprods:
                ##This is where the inefficiency of CalcProb() comes in. I am manually checking if the sum of the
                ##distributions equal the sum of the people in the first state (that are not in the discharge state).
                ##For example: take my above Targ = [2,3,4,5] and InitState [10,4,0,1].
                ##Then a prod in transprods looks like: ([0,1,5,2],[0,2,0,0],[0,0,0,0]), what I want is that
                ##[0,1,5,2]+[0,2,0,0]+[0,0,0,0] equals Targ = [2,3,4,5], but in this case it does not, because
                ##[0,1,5,2]+[0,2,0,0]+[0,0,0,0] = [0,3,5,2] != [2,3,4,5]

                #som in our example is [0,3,5,2]
                som = sum([np.array(list(prod)[u]) for u in range(len(state2)-1)])

                #[0,3,5,2] != [2,3,4,5] and therefore, pass.
                if som.tolist() == targ[0]:
                    #If it accepts, then add the probability we were able to calculate through transprobs.
                    transsums.append([prod, TransProbs(prod, specialty)])
            ##transsums is the list of all possible combinations of transitions ! NOT POSSIBLE ACTION DISTRIBUTIONS !
            ##with their respective transition probability

            ##([1, 3, 4, 2], [1, 0, 0, 3], [0, 0, 0, 0]) is a prod where the if-statement is TRUE, therefore
            ##transsums is definitely not empty.

            #Initial probability is 0
            kans=0

            #For v in the range of transsums
            for v in range(len(transsums)):
                ##P(transsums[v][0])=transsums[v][1], now add it to the total probability
                kans = kans + transsums[v][1]

            ##Now transition_prob_sums is the total probability to go from state1 to Targ
            transition_prob_sums.append(kans)
            #Iterate until the end of targetlist

        #Finalsum will be our final probability and we start it at 0
        finalProb = 0
        for i in range(len(transition_prob_sums)):
            ##distrProb is the probability of going from state2 to targ times the probability of going from Targ to state1.
            ##essentially, the probability given a distribution of the action.
            distrProb = transition_prob_sums[i]*targetlist[i][1]
            ##Since we add up all the above probabilities, finalsum will be the total probability of reaching a state given an action
            finalProb = finalProb+distrProb
        return finalProb

##Actionchecker: given a state, it will see if it's overutilized, and if it isn't then we return a tuple of all possible actions
##and a tuple of possible action distributions.
##E.g. suppose S.amount = 1 and E.amount = 3, S.max = 2
##If [1,2,3] is not overutilized, then actionChecker([1,2,3],L,E,S) = (((0,),(1,),(2,)),((0,0,0),(1,0,0),(0,1,0),(2,0,0),(1,1,0),(0,2,0)))
def actionChecker(state0, L, E, S):
    ##number of treatment patterns in a specialty
    n = E.amount
    ##All possible admission distributions given a state
    actionstatess = ()

    ##All possible actions
    allpossibleactions = ()
    ##If the initial state is overutilizing the resources, then do not add new people
    if L.isOverUt(state0) == True:
        allpossibleactions = ((0,) * S.amount,)
        actionstatess = ((0,) * n * S.amount,)
        return allpossibleactions, actionstatess
    else:
        if len(S.actionDistributions)==0:
            for action in S.actions:
                A = [0]*int(S.amount)
                for i in range(S.amount):
                    ##n-2 should actually be (n-1)-1 , that first n-1 because we do not want to add to the discharge pattern,
                    ##And the second because making a partition of the number k in n parts is calculated by
                    ##[[k - sum(p)] + p for p in partition(k, n - 1)]
                    A[i] = tuple([p +[0] for p in partition(action[i], n - 1)])

                # ##Suppose action = (2,3), and n = E.amount= 3 then A will equal
                # ##[([0,0,0],[1,0,0],[0,1,0],[2,0,0],[1,1,0],[0,2,0]),([0,0,0],[1,0,0],[0,1,0],[2,0,0],[1,1,0],[0,2,0],[3,0,0],[2,1,0],[1,2,0],[0,3,0])]
                # ##A will be a list of tuples of lists like above
                # A = [0]*int(S.amount)
                # actionstates=()
                # for i in range(S.amount):
                #     ##n-2 should actually be (n-1)-1 , that first n-1 because we do not want to add to the discharge pattern,
                #     ##And the second because making a partition of the number k in n parts is calculated by
                #     ##[[k - sum(p)] + p for p in partition(k, n - 1)]
                #     A[i] = tuple([[action[i] - sum(p)] + p +[0] for p in partition(action[i], n - 2)])
                #     print(i, action, action[i], A[i])
                # # ##B is a cartesian product of the tuples in A
                # # ##so A[0]xA[1]x...xA[S.amount]
                # # B = itertools.product(*A)
                # #
                # # ##statecouple could look like this: ([0,1], [1,2]) where [0,1] comes from A[0] and [1,2] comes from A[1]
                # # for statecouple in B:
                # #     ##Given a statecouple we are going to put them in one entire state, called state[]
                # #     state=[]
                # #     ##Let us add the discharge pattern (we add 0)
                # #     for statenr in range(S.amount):
                # #         state=state+statecouple[statenr]+[0]
                # #     ##We add state[] in the tuple actionstates as a tuple
                # #     actionstates = actionstates + (tuple(state),)
                # #
                # # action = tuple(action)
                # #
                # # #Tuple of possible states we can get to given an action
                # # actionstatess = actionstatess + actionstates
                actionstatess = A
                #
                # #Tuple of actions we can take
                # allpossibleactions = allpossibleactions + (action,)

            S.addActionDistributions(actionstatess)
    return tuple(S.actions), tuple(S.actionDistributions)

##Given a state, transitioner will give all possible next states by only transitioning fron one treatment pattern to another,
##So we do not add patients. NO ACTION USED.
def transitioner(state, L, E, S):
    #n is amount of treatment patterns
    n = E.amount

    #d is amount of specialties
    d = S.amount

    # print(11111, state)
    possibletransitions = []
    for specialty in range(d):
        #state_specialty takes the treatment patterns from one specialty
        state_specialty = state[n*specialty:n*(specialty+1)]

        ##This will be a list(lists obtained by varying component from 0 to n-1)
        trans_per_trans_i = []
        #for treatment pattern that is not the discharge pattern
        if (specialty, state_specialty, -1) in E.transitions.keys():
            poss_combs_of_transitions = E.transitions[(specialty,state_specialty, -1)]
        else:
            for treatment in range(n-1):
                trans_i = []
                if (specialty, treatment, state_specialty[treatment]) in E.transitions.keys():
                    trans_i = E.transitions[(specialty, treatment, state_specialty[treatment])]
                else:
                    ##i stands for how many patients we leave in a treatment pattern
                    ##And thus trans_i is the possible transitions we get if we vary i from 0 to state_specialty[treatment]
                    for i in range(state_specialty[treatment] + 1):
                        ##if i stands for how many people we leave, then state_specialty[treatment]-i stands for how many people we redistribute
                        lst = [[state_specialty[treatment]-i-sum(p)] + p for p in partition(state_specialty[treatment]-i, n - 2)]
                        ##suppose state_specialty = [2,50,1], treatment = 0, i=1, then lst = [[51,1],[50,2]]
                        lst2=[]
                        ##Take the same example, lst2 would be: [[1,51,1],[1,50,2]]
                        for lijst in lst:
                            lijst = lijst[0:treatment] + [i] + lijst[treatment:]
                            lst2.append(lijst)
                        trans_i = trans_i+lst2
                    E.addTransitions(specialty, treatment, state_specialty[treatment], trans_i)
                trans_per_trans_i.append(trans_i)

            ##All the possible combinations of redistributions from all possible components
            ##Cartesian product
            poss_combs_of_transitions_tuples = itertools.product(*trans_per_trans_i)

            ##The difference between this and poss_combs_of_transitions_tuples, is that this one will have the states added up to each other
            poss_combs_of_transitions = []

            for comb in poss_combs_of_transitions_tuples:
                #np.array() + np.array() returns a single np.array() by componentwise addition
                sumarray = np.array([0]*n)
                for treatment in range(n-1):
                    sumarray = sumarray + np.array(comb[treatment])
                    sumarray = sumarray.tolist()
                poss_combs_of_transitions.append(sumarray)
            E.addTransitions(specialty, state_specialty, -1, poss_combs_of_transitions)
        ##Start gluing the specialties together by putting all the combinations in a list
        possibletransitions.append(tuple(poss_combs_of_transitions))

    # ##Since each specialty has a different list for combinations, we will make a cartesian product of all the specialties
    # possibletransition_comb_tuples = itertools.product(*possibletransitions)
    #
    # ##Let us start putting all the different combinations in one entire state of dimension n*d
    # semiposs_trans = []
    # for comb in possibletransition_comb_tuples:
    #     sumlist = ()
    #     for i in range(len(possibletransitions)):
    #         sumlist = sumlist+tuple(comb[i])
    #     semiposs_trans.append(sumlist)
    return possibletransitions

##Since transitioner only transitions and actionChecker only adds them lets combine those two
def posStates(state0, L, E, S):
    ##Remember that the second result in actionChecker is all the distributions of the action. so actions() here is a misnomer
    ##and does not mean the amount of people we add in each specialty, but rather the amount of people we add in each treatment pattern.
    ##Or in other words, actions() here actually means action distributions
    actions = actionChecker(state0, L, E, S)[1]
    ##len(actions) == 1 if and only if the only action distribution possible is (0,)*E.amount and thus we only need transitioner(state)
    if len(actions) == 1:
        possibletransitions = tuple(transitioner(state0, L, E, S))
        ##Since each specialty has a different list for combinations, we will make a cartesian product of all the specialties
        possibletransition_comb_tuples = itertools.product(*possibletransitions)

        ##Let us start putting all the different combinations in one entire state of dimension n*d
        semiposs_trans = []
        for comb in possibletransition_comb_tuples:
            sumlist = ()
            for i in range(len(possibletransitions)):
                sumlist = sumlist+tuple(comb[i])
            semiposs_trans.append(sumlist)
        everyposstate = semiposs_trans

    ##in the other case there exist non-zero action distributions
    else:
        #first we calculate all the possible states you can get by transitioning
        transitionstates = transitioner(state0, L, E, S)
        # print(444444)
        everyposstate_combs = []
        for specialty in range(len(transitionstates)):
            pos_states_specialty = []

            #This is the cartesian product of the set of transitions with the set of admissions for a specific specialty
            trans_and_act_per_specialty = itertools.product(transitionstates[specialty],actions[specialty])
            #now we add the two together component-wise
            for comb_prod in trans_and_act_per_specialty:
                # add transition up with an action distribution
                combination = np.array(comb_prod[0])+np.array(comb_prod[1])
                combination = combination.tolist()
                combination = tuple(combination)
                pos_states_specialty.append(combination)
            everyposstate_combs.append(pos_states_specialty)

        everyposstate_prod = itertools.product(*everyposstate_combs)
        everyposstate = ()
        for posstate_comb in everyposstate_prod:
            if posstate_comb not in E.states:
                E.states.add(posstate_comb)
                posstate = itertools.chain(*posstate_comb)
                everyposstate = everyposstate + (tuple(posstate),)
    everyposstate = set(everyposstate)
    # print(555555, everyposstate)
    return everyposstate

##We are also interested in what the possible states are if we are given the action, I will not go into full detail here
##since all the methods were already used once
def posStatesGivenAction(state0, action, L, E, S):
    possTrans = tuple(transitioner(state0, L, E, S))
    n = E.amount
    A = [0] * int(S.amount)
    possActionDistrs = []
    for i in range(S.amount):
        A[i] = [[action[i]-sum(p)] + p for p in partition(action[i], n-2)]
    B = itertools.product(*A)
    for distr in B:
        distrib = []
        for d in range(S.amount):
            distrib = distrib + distr[d] + [0]
        possActionDistrs.append(tuple(distrib))
    possStates = []
    for trans in possTrans:
        nptrans = np.array(trans)
        for act in possActionDistrs:
            npact = np.array(act)
            som = nptrans + npact
            som2 = tuple(som)
            possStates.append(som2)
    return possStates

def stateSpace(state0, L, E, S):
    statelist = list(posStates(state0, L, E, S))
    nextstates = []
    for state in statelist:
        nextstates = nextstates + list(posStates(state, L, E, S))
    nextstates = statelist + nextstates
    nextstates = f7(tuple(nextstates))
    print(nextstates)

    while len(nextstates) > len(statelist):
        length = len(statelist)
        uniquestates = nextstates[len(statelist):]
        statelist = nextstates
        nextstates = []
        # i = 0
        for uniquestate in uniquestates:
            # i += 1
            nextstates = nextstates + list(posStates(uniquestate, L, E, S))
        nextstates = statelist + nextstates
        nextstates = f7(tuple(nextstates))
        print(nextstates)
    return nextstates

def costFunction(state, action, L, E, S):
    ##We don't need to find all the states reachable with the action, since we're only looking for the worst case scenario
    allPossStates = posStatesGivenAction(state, action, L, E, S)
    ##And now for the probabilities:
    totaalsom = 0
    for nextstate in allPossStates:
        n = E.amount
        ##We need to calculate how many people use resource Lj in nextstate
        total_avgusage_per_resource = []
        som=0
        for res_numbr in range(L.amount):
            total_avgusage=0
            for specialty in range(len(action)):
                for tr_pat in range(n-1):
                    ##avg_ut = [[avg util. of res. 1 by E_1,avg ut of res. 1 by E_2], [avg ut of res. 2 by E_1,avg ut of res. 2 by E_2]]
                    ##avg_ut = np.array([[2.2,2.6],[2.6,2.2]])
                    total_avgusage += nextstate[tr_pat+(specialty*n)]*L.avgMatrix[res_numbr][tr_pat]
            Oj = L.cost[0][res_numbr]*max(0, L.cap[res_numbr]-total_avgusage)
            Bj = L.cost[1][res_numbr]*max(0, total_avgusage-L.cap[res_numbr])
            Cj = L.cost[2][res_numbr]*max(0, total_avgusage-L.max[res_numbr])
            som += Oj+Bj+Cj
        P=1
        for specialty in range(len(action)):
             P = P*CalcProb(nextstate[n*specialty:n*(specialty+1)], state[n*specialty:n*(specialty+1)], action[specialty], specialty)
        totaalsom+=P*som
    return totaalsom

def main():
    ##Number of specialties d
    d= 2
    ## maximum admission per period max_S = [max in specialty 1, max in specialty 2]
    max_S= np.array([2,3])

    ## NUMBER OF RESOURCES
    num_res = 2
    ## max(L_j)
    max_L = np.array([6,5])

    ## Number of treatment patterns
    n=3
    ## avg resource utilization per period
    ## avg_ut = [[avg util. of res. 1 by E_1,avg ut of res. 1 by E_2], [avg ut of res. 2 by E_1,avg ut of res. 2 by E_2]]
    avg_ut = np.array([[1.2,3.0,0],[3.6,2.8,0]])
    ## Utilization cap
    cap_L = np.array([4,5])

    ##Cost for resource deviation idleness, excess and over
    cap_cost = np.array([[1.0,1.6],[1.5,1.0],[2.0,2.0]])

    ## We can put this in our resource, specialty and treatment pattern class
    L = Resources(num_res, max_L, avg_ut, cap_L, cap_cost)
    E = Patterns(n)
    S = Specialties(d, max_S)

    #Creates state space
    # start = time.time()
    A = stateSpace((0,0,0,0,0,0),L,E,S)
    # end = time.time()
    # print(end - start)
    # print(len(A))
    print(len(A), A)
    # outputFile(A, "statenspatie5")

    ##Since we do not want to recalculate the state space all the time we just save it externally and call it up
    # statenspatie = open("statenspatie5.txt", 'r')
    # statespace = []
    # for state in statenspatie:
    #     listContent = state.rstrip()
    #     tupel = literal_eval(listContent)
    #     statespace.append(tupel)
    # statenspatie.close()

    ##Time for action space
    # statewithactions = []
    # for state in statespace:
    #     actions = actionChecker(state, L, E, S)[0]
    #     statewithactions.append((state, actions))
    # outputFile(statewithactions, "actionspace")

    # print(costFunction((1,2,5,0,0,0),(2,0), L, E, S))
    # print(costFunction((0,0,0,1,2,5),(0,2),L,E,S))
    # som = 0
    # alreadydone = []
    # for state in statespace:
    #     if state[0:3] not in alreadydone:
    #         B=CalcProb(state[0:3], (1, 0, 0), 1, 0)
    #         print(state, B)
    #         som += B
    #         alreadydone.append(state[0:3])
    #
    # print(som)
main()


# coding=utf-8
from io import BytesIO
from django.http import HttpResponse
import simplejson as json
from reportlab.pdfgen import canvas
from reportlab.graphics.barcode import code39, code128, createBarcodeDrawing
from directions.models import Napravleniya, Issledovaniya, IstochnikiFinansirovaniya, TubesRegistration
from django.contrib.auth.decorators import login_required
import directory.models as directory
from reportlab.pdfbase import pdfmetrics
from reportlab.pdfbase.ttfonts import TTFont
from reportlab.lib.units import inch, mm
import os.path
import re
from reportlab.pdfbase import pdfdoc
from django.conf import settings
import datetime
from django.utils import dateformat

pdfdoc.PDFCatalog.OpenAction = '<</S/JavaScript/JS(this.print\({bUI:true,bSilent:false,bShrinkToFit:false}\);)>>'

PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
pdfmetrics.registerFont(
    TTFont('OpenSans', PROJECT_ROOT + '/../static/fonts/OpenSans.ttf'))
pdfmetrics.registerFont(
    TTFont('clacon', PROJECT_ROOT + '/../static/fonts/clacon.ttf'))

@login_required
def tubes(request):
    """
    Barcodes view
    :param request:
    :return:
    """
    pw, ph = 44, 25  # длина, ширина листа
    direction_id = []
    tubes_id = set()
    istubes = False
    if "napr_id" in request.GET.keys():
        direction_id = json.loads(request.GET["napr_id"])
    elif "tubes_id" in request.GET.keys():
        tubes_id = set(json.loads(request.GET["tubes_id"]))
        istubes = True
    doctitle = "Штрих-коды (%s)" % (("направления " + ", ".join(str(v) for v in direction_id)) if not istubes else ("пробирки " + ", ".join(str(v) for v in tubes_id)))

    response = HttpResponse(content_type='application/pdf')
    response['Content-Type'] = 'application/pdf'
    symbols = (u"абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ",
               u"abvgdeejzijklmnoprstufhzcss_y_euaABVGDEEJZIJKLMNOPRSTUFHZCSS_Y_EUA")
    response['Content-Disposition'] = str.translate('inline; filename="%s.pdf"' % doctitle, {ord(a): ord(b) for a, b in zip(*symbols)})

    buffer = BytesIO()
    pdfdoc.PDFInfo.title = 'Barcodes'
    c = canvas.Canvas(buffer, pagesize=(pw * mm, ph * mm))
    c.setTitle(doctitle)
    dt = {"poli": "Поликлиника", "stat": "Стационар", "poli_stom": "Поликлиника-стом."}
    if istubes:
        direction_id = set([x.napravleniye.pk for x in Issledovaniya.objects.filter(tubes__id__in=tubes_id)])
    for d in direction_id:
        tmp2 = Napravleniya.objects.get(pk=int(d))
        tmp = Issledovaniya.objects.filter(napravleniye=tmp2).order_by("research__title")
        tubes_buffer = {}

        fresearches = set()
        fuppers = set()
        flowers = set()

        for iss in Issledovaniya.objects.filter(napravleniye=tmp2):
            for fr in iss.research.fractions_set.all():
                absor = directory.Absorption.objects.filter(fupper=fr)
                if absor.exists():
                    fuppers.add(fr.pk)
                    fresearches.add(fr.research.pk)
                    for absor_obj in absor:
                        flowers.add(absor_obj.flower.pk)
                        fresearches.add(absor_obj.flower.research.pk)

        for v in tmp:
            for val in directory.Fractions.objects.filter(research=v.research):
                vrpk = val.relation.pk
                rel = val.relation
                if val.research.pk in fresearches and val.pk in flowers:
                    absor = directory.Absorption.objects.filter(flower__pk=val.pk).first()
                    if absor.fupper.pk in fuppers:
                        vrpk = absor.fupper.relation.pk
                        rel = absor.fupper.relation

                if vrpk not in tubes_buffer.keys():
                    if not v.tubes.filter(type=rel).exists():
                        ntube = TubesRegistration(type=rel)
                        ntube.save()
                    else:
                        ntube = v.tubes.get(type=rel)
                    v.tubes.add(ntube)
                    tubes_buffer[vrpk] = {"pk": ntube.pk, "researches": set(), "title": ntube.type.tube.title}
                    if not istubes:
                        tubes_id.add(ntube.pk)
                else:
                    ntube = TubesRegistration.objects.get(pk=tubes_buffer[vrpk]["pk"])
                    v.tubes.add(ntube)

                tubes_buffer[vrpk]["researches"].add(v.research.title)
        for tube_k in tubes_buffer.keys():
            tube = tubes_buffer[tube_k]["pk"]
            if tube not in tubes_id:
                continue
            # c.setFont('OpenSans', 8)
            c.setFont('clacon', 12)
            c.drawString(2 * mm, ph * mm - 3 * mm, "№" + str(d) + "," + dt[tmp2.istochnik_f.istype][0])
            otd = list(tmp2.doc.podrazileniye.title.split(" "))
            st = ""

            if len(otd) > 1:
                if "отделение" in otd[0].lower():
                    st = otd[1][:3] + "/о"
                else:
                    st = otd[0][:3] + "/" + otd[1][:1]
            elif len(otd) == 1:
                st = otd[0][:3]
            st += u"=>" + Issledovaniya.objects.filter(tubes__pk=tube).first().research.subgroup.podrazdeleniye.title[
                          :3]
            c.drawRightString(pw * mm - 2 * mm, ph * mm - 3 * mm, st.lower())

            c.drawRightString(pw * mm - 2 * mm, ph * mm - 6 * mm, "л/в: " + tmp2.doc.get_fio(False))

            # c.setFont('OpenSans', 11)

            c.setFont('clacon', 18)
            fam = tmp2.client.family
            if len(fam) > 12:
                c.setFont('clacon', 18 - len(fam) * 0.7 + 12 * 0.7)
            if tmp2.client.twoname and tmp2.client.twoname != "":
                c.drawRightString(pw * mm - 2 * mm, ph * mm - 10 * mm,
                                  fam + " " + tmp2.client.name[0] + tmp2.client.twoname[0])
            else:
                c.drawRightString(pw * mm - 2 * mm, ph * mm - 10 * mm, fam + " " + tmp2.client.name[0])

            # c.setFont('OpenSans', 10)
            c.setFont('clacon', 12)
            types = ["фиолет", "красн", "стекло", "черн", "белая", "серая", "фильтро", "чашка", "голубая", "зеленая",
                     "зелёная", "контейнер", "зонд", "п ф", "л ф", "синяя"]
            tb_t = tubes_buffer[tube_k]["title"].lower()
            pr = ""
            for s in types:
                if s in tb_t:
                    pr = s[0]
            pr = pr.upper()
            r = re.search(u"(\d+\.\d|\d+,\d+|\d+)\s(мл)", tb_t)
            if r:
                pr += r.group(1) + r.group(2)
            pr += " " + Issledovaniya.objects.filter(tubes__pk=tube).first().comment[:9]
            tdt = str(dateformat.format(datetime.date.today(), settings.DATE_FORMAT))
            tdt = tdt.split(".")[0] + "." + tdt.split(".")[1]
            c.drawString(2 * mm, mm, pr)
            #tube = 4523667
            c.drawRightString(pw * mm - 2 * mm, mm, str(tube))
            m = 0.0245
            if tube >= 10000:
                m = 0.018
            if tube >= 100000:
                m = 0.0212
            if tube >= 1000000:
                m = 0.016
            #m *= 0.9
            # barcode = code39.Standard39(str(tube), barHeight=10*mm, barWidth=inch * m, checksum=0, ratio=2, gap=inch * m*1.5)
            #barcode2 = createBarcodeDrawing("code128", value=str(tube), barHeight=10 * mm, barWidth=inch * m,
            #                                checksum=0, ratio=2, gap=inch * m * 1.8)

            # barcode2.drawOn(c, -4*mm, barcode2.height - 6*mm)
            #barcode2.drawOn(c, 0, barcode2.height - 6 * mm) '''barWidth=inch * m,'''
            barcode = code128.Code128(str(tube), barHeight=10*mm, barWidth=inch * m)
            barcode.drawOn(c, -2*mm, barcode.height - 6 * mm)
            c.showPage()
    c.save()
    pdf = buffer.getvalue()
    buffer.close()
    response.write(pdf)
    return response


from __future__ import print_function
from __future__ import unicode_literals
from future.builtins import str
from future.builtins import object
from past.builtins import basestring

import logging
import signal
import sys
import threading
import traceback
import os.path

from datetime import datetime, timedelta
from optparse import make_option, OptionParser
from time import sleep

from dateutil.parser import parse
from setproctitle import setproctitle

from limpyd import EXACT_VERSION as limpyd_version
from limpyd.exceptions import DoesNotExist

from limpyd_jobs.version import EXACT_VERSION as limpyd_jobs_version
from limpyd_jobs import STATUSES, LimpydJobsException, ConfigurationException
from limpyd_jobs.models import Queue, Job, Error
from limpyd_jobs.utils import import_class, compute_delayed_until

LOGGER_NAME = 'limpyd-jobs'
logger = logging.getLogger(LOGGER_NAME)

class Worker(object):
    # names of queues to use
    queues = None

    # models to use (for all queues/errors !)
    queue_model = Queue
    error_model = Error

    # logging information
    logger_name = LOGGER_NAME
    logger_level = logging.INFO
    save_errors = True
    save_tracebacks = True

    # maximum number of loops to run
    max_loops = 1000
    # max total duration of the worker
    max_duration = None
    # max delay for blpop
    timeout = 30
    # minimum time in seconds between the update of keys to fetch
    # by default < timeout to be sure to fetch them after a timeout occured
    fetch_priorities_delay = 25
    # minimum time in seconds between the update of delayed jobs
    # by default < timeout to be sure to fetch them after a timeout occured
    fetch_delayed_delay = 25
    # number of time to requeue a job after a failure
    requeue_times = 0
    # delta to add to the current priority of a job to be requeued
    requeue_priority_delta = -1
    # how much time to delay a job to be requeued
    requeue_delay_delta = 30

    # we want to intercept SIGTERM and SIGINT signals, to stop gracefuly
    terminate_gracefuly = True

    # callback to run for each job, will be set to self.execute if None
    callback = None

    # all parameters that can be passed to the constructor
    parameters = ('queues', 'callback', 'queue_model', 'error_model',
                  'logger_name', 'logger_level', 'save_errors',
                  'save_tracebacks', 'max_loops', 'max_duration',
                  'terminate_gracefuly', 'timeout', 'fetch_priorities_delay',
                  'fetch_delayed_delay', 'requeue_times',
                  'requeue_priority_delta', 'requeue_delay_delta')

    @staticmethod
    def _parse_queues(queues):
        """
        Parse the given parameter and return a list of queues.
        The parameter must be a list/tuple of strings, or a string with queues
        separated by a comma.
        """
        if not queues:
            raise ConfigurationException('The queue(s) to use are not defined')

        if isinstance(queues, (list, tuple)):
            for queue_name in queues:
                if not isinstance(queue_name, str):
                    raise ConfigurationException('Queue name "%s" is not a (base)string')

        elif isinstance(queues, basestring):
            queues = str(queues).split(',')

        else:
            raise ConfigurationException('Invalid format for queues names')

        return list(queues)

    def __init__(self, queues=None, **kwargs):
        """
        Create the worker by saving arguments, doing some checks, preparing
        logger and signals management, and getting queues keys.
        """
        if queues is not None:
            self.queues = queues
        self.queues = self._parse_queues(self.queues)

        for parameter in self.parameters:
            if parameter in kwargs:
                setattr(self, parameter, kwargs[parameter])

        # save and check models to use
        self._assert_correct_model(self.queue_model, Queue, 'queue')
        self._assert_correct_model(self.error_model, Error, 'error')

        # process other special arguments
        if not self.callback:
            self.callback = self.execute
        if self.max_duration:
            self.max_duration = timedelta(seconds=self.max_duration)

        # prepare logging
        self.set_logger()

        if self.terminate_gracefuly:
            self.handle_end_signal()

        self.keys = []  # list of redis keys to listen
        self.num_loops = 0  # loops counter
        self.start_date = None  # set when run is called
        self.end_date = None  # set when run ends
        self.wanted_end_date = None  # will be the end after which the worker must stop
        self.end_forced = False  # set it to True in "execute" to force stop just after
        self.status = None  # is set to None/waiting/running by the worker
        self.end_signal_caught = False  # internaly set to True if end signal caught
        self.last_fetch_priorities = None  # last time the keys to fetch were updated
        self.last_requeue_delayed = None  # last time the ready delayed jobs where requeued

        self._update_status_callbacks = []  # callbacks to call when status is updated

    @staticmethod
    def _assert_correct_model(model_to_check, model_reference, obj_name):
        """
        Helper that asserts the model_to_check is the model_reference or one of
        its subclasses. If not, raise an ImplementationError, using "obj_name"
        to describe the name of the argument.
        """
        if not issubclass(model_to_check, model_reference):
            raise ConfigurationException('The %s model must be a subclass of %s'
                                         % (obj_name, model_reference.__name__))

    def handle_end_signal(self):
        """
        Catch some system signals to handle them internaly
        """
        try:
            signal.signal(signal.SIGTERM, self.catch_end_signal)
            signal.signal(signal.SIGINT, self.catch_end_signal)
        except ValueError:
            self.log('Signals cannot be caught in a Thread', level='warning')

    def stop_handling_end_signal(self):
        """
        Stop handling the SIGINT and SIGTERM signals
        """
        try:
            signal.signal(signal.SIGTERM, signal.SIG_DFL)
            signal.signal(signal.SIGINT, signal.SIG_DFL)
        except ValueError:
            self.log('Signals cannot be caught in a Thread', level='warning')

    def set_logger(self):
        """
        Prepare the logger, using self.logger_name and self.logger_level
        """
        self.logger = logging.getLogger(self.logger_name)
        self.logger.setLevel(self.logger_level)

    @property
    def id(self):
        """
        Return an identifier for the worker to use in logging
        """
        if not hasattr(self, '_id'):
            self._id = str(threading.current_thread().ident + id(self))[-6:]
        return self._id

    def log(self, message, level='info'):
        """
        Call self.logger with the given level (default to "info") and message
        """
        getattr(self.logger, level)('[%s] %s' % (self.id, message))

    @property
    def connection(self):
        """
        Return the redis connection to use.
        """
        return self.queue_model.get_connection()

    def must_stop(self):
        """
        Return True if the worker must stop when the current loop is over.
        """
        return bool(self.terminate_gracefuly and self.end_signal_caught
                 or self.num_loops >= self.max_loops or self.end_forced
                 or self.wanted_end_date and datetime.utcnow() >= self.wanted_end_date)

    def set_status(self, status):
        """
        Save the new status and call all defined callbacks
        """
        self.status = status
        for callback in self._update_status_callbacks:
            callback(self)

    def _add_update_status_callback(self, callback):
        """
        Add a callback to call when the status is updated
        """
        self._update_status_callbacks.append(callback)

    def wait_for_job(self):
        """
        Use a redis blocking list call to wait for a job, and return it.
        """
        blpop_result = self.connection.blpop(self.keys, self.timeout)
        if blpop_result is None:
            return None
        queue_redis_key, job_ident = blpop_result
        self.set_status('running')
        return self.get_queue(queue_redis_key), self.get_job(job_ident)

    def get_job(self, job_ident):
        """
        Return a job based on its representation (class + pk).
        """
        return Job.get_from_ident(job_ident)

    def get_queue(self, queue_redis_key):
        """
        Return a queue based on the key used in redis to store the list
        """
        try:
            queue_pk = int(queue_redis_key.split(':')[-2])
        except:
            raise DoesNotExist('Unable to get the queue from the key %s' % queue_redis_key)
        return self.queue_model.get(queue_pk)

    def catch_end_signal(self, signum, frame):
        """
        When a SIGINT/SIGTERM signal is caught, this method is called, asking
        for the worker to terminate as soon as possible.
        """
        if self.end_signal_caught:
            self.log('Previous signal caught, will end soon')
            return

        signal_name = dict((getattr(signal, n), n) for n in dir(signal)
                        if n.startswith('SIG') and '_' not in n).get(signum, signum)

        if self.status == 'running':
            self.log('Catched %s signal: stopping after current job' % signal_name,
                     level='critical')
        else:
            delay = self.timeout if self.status == 'waiting' else self.fetch_priorities_delay
            self.log('Catched %s signal: stopping in max %d seconds' % (
                     signal_name, delay), level='critical')

        self.end_signal_caught = self.end_forced = True

    def execute(self, job, queue):
        """
        The method to run for a job when got bak from the queue if no callback
        was defined on __init__.
        The optional return value of this function will be passed to the
        job_success method.
        """
        return job.run(queue)

    def update_keys(self):
        """
        Update the redis keys to listen for new jobs priorities.
        """
        self.keys = self.queue_model.get_waiting_keys(self.queues)

        if not self.keys:
            self.log('No queues yet', level='warning')
        self.last_update_keys = datetime.utcnow()

    def count_waiting_jobs(self):
        """
        Return the number of all jobs waiting in queues managed by the worker
        """
        return self.queue_model.count_waiting_jobs(self.queues)

    def count_delayed_jobs(self):
        """
        Return the number of all delayed jobs in queues managed by the worker
        """
        return self.queue_model.count_delayed_jobs(self.queues)

    def run_started(self):
        """
        Called just before starting to wait for jobs. Actually only do logging.
        """
        self.log('Run started.')

    def run(self):
        """
        The main method of the worker. Will ask redis for list items via
        blocking calls, get jobs from them, try to execute these jobs, and end
        when needed.
        """
        # if status is not None, we already had a run !
        if self.status:
            self.set_status('aborted')
            raise LimpydJobsException('This worker run is already terminated')

        self.set_status('starting')

        self.start_date = datetime.utcnow()
        if self.max_duration:
            self.wanted_end_date = self.start_date + self.max_duration

        must_stop = self.must_stop()

        if not must_stop:

            # get keys
            while not self.keys and not must_stop:
                self.update_keys()
                if not self.keys:
                    sleep(self.fetch_priorities_delay)
                must_stop = self.must_stop()

            if not must_stop:
                # wait for queues available if no ones are yet
                self.requeue_delayed_jobs()
                self.run_started()
                self._main_loop()

        self.set_status('terminated')
        self.end_date = datetime.utcnow()
        self.run_ended()
        if self.terminate_gracefuly:
            self.stop_handling_end_signal()

    @property
    def elapsed(self):
        """
        Return a timedelta representation of the time passed sine the worker
        was running.
        """
        if not self.start_date:
            return None
        return (self.end_date or datetime.utcnow()) - self.start_date

    def requeue_delayed_jobs(self):
        """
        Requeue each delayed job that are now ready to be executed
        """
        failures = []
        for queue in self.queue_model.get_all_by_priority(self.queues):
            failures.extend(queue.requeue_delayed_jobs())

        self.last_requeue_delayed = datetime.utcnow()

        for failure in failures:
            self.log('Unable to requeue %s: %s' % failure)

    def _main_loop(self):
        """
        Run jobs until must_stop returns True
        """
        fetch_priorities_delay = timedelta(seconds=self.fetch_priorities_delay)
        fetch_delayed_delay = timedelta(seconds=self.fetch_delayed_delay)

        while not self.must_stop():
            self.set_status('waiting')

            if self.last_update_keys + fetch_priorities_delay < datetime.utcnow():
                self.update_keys()

            if self.last_requeue_delayed + fetch_delayed_delay < datetime.utcnow():
                self.requeue_delayed_jobs()

            try:
                queue_and_job = self.wait_for_job()
                if queue_and_job is None:
                    # timeout for blpop
                    continue
                queue, job = queue_and_job
            except Exception as e:
                self.log('Unable to get job: %s\n%s'
                    % (str(e), traceback.format_exc()), level='error')
            else:
                self.num_loops += 1
                try:
                    identifier = 'pk:%s' % job.pk.get()
                except Exception as e:
                    identifier = '??'
                try:
                    self.set_status('running')
                    identifier, status = job.hmget('identifier', 'status')
                     # some cache, don't count on it on subclasses
                    job._cached_identifier = identifier
                    job._cached_status = status
                    queue._cached_name = queue.name.hget()

                    if status == STATUSES.DELAYED:
                        self.job_delayed(job, queue)
                    elif status != STATUSES.WAITING:
                        self.job_skipped(job, queue)
                    else:
                        try:
                            self.job_started(job, queue)
                            job_result = self.callback(job, queue)
                        except Exception as e:
                            trace = None
                            if self.save_tracebacks:
                                trace = traceback.format_exc()
                            self.job_error(job, queue, e, trace)
                        else:
                            job._cached_status = job.status.hget()
                            if job._cached_status == STATUSES.DELAYED:
                                self.job_delayed(job, queue)
                            elif job._cached_status == STATUSES.CANCELED:
                                self.job_skipped(job, queue)
                            else:
                                self.job_success(job, queue, job_result)
                except Exception as e:
                    self.log('[%s] unexpected error: %s\n%s'
                        % (identifier, str(e), traceback.format_exc()), level='error')
                    try:
                        queue.errors.rpush(job.ident)
                    except Exception as e:
                        self.log('[%s] unable to add the error in the queue: %s\n%s'
                            % (identifier, str(e), traceback.format_exc()), level='error')

    def run_ended(self):
        """
        Called just after ending the run loop. Actually only do logging.
        """
        self.log('Run terminated, with %d loops (duration=%s)' % (
                                                self.num_loops, self.elapsed))

    def additional_error_fields(self, job, queue, exception, trace=None):
        """
        Return a dict with additional fields to set to a new Error subclass object
        """
        return {}

    def job_error(self, job, queue, exception, trace=None):
        """
        Called when an exception was raised during the execute call for a job.
        """
        to_be_requeued = not job.must_be_cancelled_on_error and self.requeue_times and self.requeue_times >= int(job.tries.hget() or 0)

        if not to_be_requeued:
            job.queued.delete()

        job.hmset(end=str(datetime.utcnow()), status=STATUSES.ERROR)
        queue.errors.rpush(job.ident)

        if self.save_errors:
            additional_fields = self.additional_error_fields(job, queue, exception)
            self.error_model.add_error(queue_name=queue._cached_name,
                                       job=job,
                                       error=exception,
                                       trace=trace,
                                       **additional_fields)

        self.log(self.job_error_message(job, queue, to_be_requeued, exception, trace), level='error')

        if hasattr(job, 'on_error'):
            job.on_error(queue, exception, trace)

        # requeue the job if needed
        if to_be_requeued:
            priority = queue.priority.hget()

            if self.requeue_priority_delta:
                priority = int(priority) + self.requeue_priority_delta

            self.requeue_job(job, queue, priority, delayed_for=self.requeue_delay_delta)

    def requeue_job(self, job, queue, priority, delayed_for=None):
        """
        Requeue a job in a queue with the given priority, possibly delayed
        """
        job.requeue(queue_name=queue._cached_name,
                    priority=priority,
                    delayed_for=delayed_for,
                    queue_model=self.queue_model)

        if hasattr(job, 'on_requeued'):
            job.on_requeued(queue)

        self.log(self.job_requeue_message(job, queue))

    def job_error_message(self, job, queue, to_be_requeued, exception, trace=None):
        """
        Return the message to log when a job raised an error
        """
        return '[%s|%s|%s] error: %s [%s]' % (queue._cached_name,
                             job.pk.get(),
                             job._cached_identifier,
                             str(exception),
                             'requeued' if to_be_requeued else 'NOT requeued')

    def job_requeue_message(self, job, queue):
        """
        Return the message to log when a job is requeued
        """
        priority, delayed_until = job.hmget('priority', 'delayed_until')

        msg = '[%s|%s|%s] requeued with priority %s'
        args = [queue._cached_name, job.pk.get(), job._cached_identifier, priority]

        if delayed_until:
            msg += ', delayed until %s'
            args.append(delayed_until)

        return msg % tuple(args)

    def job_success(self, job, queue, job_result):
        """
        Called just after an execute call was successful.
        job_result is the value returned by the callback, if any.
        """
        job.queued.delete()
        job.hmset(end=str(datetime.utcnow()), status=STATUSES.SUCCESS)
        queue.success.rpush(job.ident)
        self.log(self.job_success_message(job, queue, job_result))
        if hasattr(job, 'on_success'):
            job.on_success(queue, job_result)

    def job_success_message(self, job, queue, job_result):
        """
        Return the message to log when a job is successful
        """
        return '[%s|%s|%s] success, in %s' % (queue._cached_name, job.pk.get(),
                                           job._cached_identifier, job.duration)

    def job_started(self, job, queue):
        """
        Called just before the execution of the job
        """
        job.hmset(start=str(datetime.utcnow()), status=STATUSES.RUNNING)
        job.tries.hincrby(1)
        self.log(self.job_started_message(job, queue))
        if hasattr(job, 'on_started'):
            job.on_started(queue)

    def job_started_message(self, job, queue):
        """
        Return the message to log just befre the execution of the job
        """
        return '[%s|%s|%s] starting' % (queue._cached_name, job.pk.get(),
                                        job._cached_identifier)

    def job_skipped(self, job, queue):
        """
        Called if a job, before trying to run it, has not the "waiting" status,
        or, after run, if its status was set to "canceled"
        """
        job.queued.delete()
        self.log(self.job_skipped_message(job, queue), level='warning')
        if hasattr(job, 'on_skipped'):
            job.on_skipped(queue)

    def job_skipped_message(self, job, queue):
        """
        Return the message to log when a job was skipped
        """
        return '[%s|%s|%s] job skipped (current status: %s)' % (
                queue._cached_name,
                job.pk.get(),
                job._cached_identifier,
                STATUSES.by_value(job._cached_status, 'UNKNOWN'))

    def job_delayed(self, job, queue):
        """
        Called if a job, before trying to run it, has the "delayed" status, or,
        after run, if its status was set to "delayed"
        If delayed_until was not set, or is invalid, set it to 60sec in the future
        """
        delayed_until = job.delayed_until.hget()
        if delayed_until:
            try:
                delayed_until = compute_delayed_until(delayed_until=parse(delayed_until))
            except (ValueError, TypeError):
                delayed_until = None

        if not delayed_until:
            # by default delay it for 60 seconds
            delayed_until = compute_delayed_until(delayed_for=60)

        job.enqueue_or_delay(
            queue_name=queue._cached_name,
            delayed_until=delayed_until,
            queue_model=queue.__class__,
        )

        self.log(self.job_delayed_message(job, queue), level='warning')

        if hasattr(job, 'on_delayed'):
            job.on_delayed(queue)

    def job_delayed_message(self, job, queue):
        """
        Return the message to log when a job was delayed just before or during
        its execution
        """
        return '[%s|%s|%s] job delayed until %s' % (
                queue._cached_name,
                job.pk.get(),
                job._cached_identifier,
                job.delayed_until.hget())

class WorkerConfig(object):
    """
    The WorkerConfig class is aimed to be used in a script to run a worker.
    All options which can be accepted by the Worker class can be passed as
    arguments to the script (they are managed by optparse)
    In a script, simply instantiate a WorkerConfig object then call its
    "execute" method.
    """

    help = "Run a worker using redis-limpyd-jobs"

    option_list = (
        # pythonpath and worker-config are managed in the script, here just to
        # not throw error if found on command line but not in defined options
        make_option('--pythonpath', action='append',
            help='A directory to add to the Python path, e.g. --pythonpath=/my/module'),
        make_option('--worker-config', dest='worker_config',
            help='The worker config class to use, e.g. --worker-config=my.module.MyWorkerConfig, '
                  'default to limpyd_jobs.workers.WorkerConfig'),

        make_option('--print-options', action='store_true', dest='print_options',
            help='Print options used by the worker, e.g. --print-options'),
        make_option('--dry-run', action='store_true', dest='dry_run',
            help='Won\'t execute any job, just starts the worker and finish it immediatly, e.g. --dry-run'),

        make_option('--queues', action='store', dest='queues',
            help='Name of the Queues to handle, comma separated e.g. --queues=queue1,queue2'),

        make_option('--queue-model', action='store', dest='queue_model',
            help='Name of the Queue model to use (for all queues !), e.g. --queue-model=my.module.QueueModel'),
        make_option('--error-model', action='store', dest='error_model',
            help='Name of the Error model to use (for all errors !), e.g. --queue-model=my.module.ErrorModel'),

        make_option('--worker-class', action='store', dest='worker_class',
            help='Name of the Worker class to use, e.g. --worker-class=my.module.WorkerClass'),

        make_option('--callback', action='store', dest='callback',
            help='The callback to call for each job, e.g. --worker-class=my.module.callback'),

        make_option('--logger-name', action='store', dest='logger_name',
            help='The base name to use for logging, e.g. --logger-name="limpyd-jobs.my-job"'),
        make_option('--logger-level', action='store', dest='logger_level',
            help='The level to use for logging, e.g. --worker-class=INFO'),

        make_option('--save-errors', action='store_true', dest='save_errors',
            help='Save job errors in the Error model, e.g. --save-errors'),
        make_option('--no-save-errors', action='store_false', dest='save_errors',
            help='Do not save job errors in the Error model, e.g. --no-save-errors'),
        make_option('--save-tracebacks', action='store_true', dest='save_tracebacks',
            help='Save exception tracebacks on job error in the Error model, e.g. --save-tracebacks'),
        make_option('--no-save-tracebacks', action='store_false', dest='save_tracebacks',
            help='Do not save exception tracebacks on job error in the Error model, e.g. --no-save-tracebacks'),

        make_option('--max-loops', type='int', dest='max_loops',
            help='Max number of jobs to run, e.g. --max-loops=100'),
        make_option('--max-duration', type='int', dest='max_duration',
            help='Max duration of the worker, in seconds (None by default), e.g. --max-duration=3600'),

        make_option('--terminate-gracefuly', action='store_true', dest='terminate_gracefuly',
            help='Intercept SIGTERM and SIGINT signals to stop gracefuly, e.g. --terminate-gracefuly'),
        make_option('--no-terminate-gracefuly', action='store_false', dest='terminate_gracefuly',
            help='Do NOT intercept SIGTERM and SIGINT signals, so don\'t stop gracefuly, e.g. --no-terminate-gracefuly'),

        make_option('--timeout', type='int', dest='timeout',
            help='Max delay (seconds) to wait for a redis BLPOP call (0 for no timeout), e.g. --timeout=30'),

        make_option('--fetch-priorities-delay', type='int', dest='fetch_priorities_delay',
            help='Min delay (seconds) to wait before fetching new priority queues, e.g. --fetch-priorities-delay=20'),

        make_option('--fetch-delayed-delay', type='int', dest='fetch_delayed_delay',
            help='Min delay (seconds) to wait before updating delayed jobs, e.g. --fetch-delayed-delay=20'),

        make_option('--requeue-times', type='int', dest='requeue_times',
            help='Number of time to requeue a failing job (default to 0), e.g. --requeue-times=5'),

        make_option('--requeue-priority-delta', type='int', dest='requeue_priority_delta',
            help='Delta to add to the actual priority of a failing job to be requeued (default to -1, ie one level lower), e.g. --requeue-priority-delta=-2'),

        make_option('--requeue-delay-delta', type='int', dest='requeue_delay_delta',
            help='How much time (seconds) to delay a job to be requeued (default to 30), e.g. --requeue-delay-delta=15'),

        make_option('--database', action='store', dest='database',
            help='Redis database to use (host:port:db), e.g. --database=localhost:6379:15'),

        make_option('--no-title', action='store_false', dest='update_title', default=True,
            help="Do not update the title of the worker's process, e.g. --no-title"),
    )

    def __init__(self, argv=None):
        """
        Save arguments and program name
        """
        self.argv = argv or sys.argv[:]
        self.prog_name = os.path.basename(self.argv[0])
        self.manage_options()
        self.update_proc_title()

    def get_version(self):
        """
        Return the limpyd_jobs version. May be overriden for subclasses

        """
        return '(redis-limpyd-jobs %s, redis-limpyd %s)' % (
                                            limpyd_jobs_version, limpyd_version)

    def usage(self):
        """
        Return a brief description of how to use the worker based on self.help

        """
        usage = '%prog [options]'
        return '%s\n\n%s' % (usage, self.help)

    def create_parser(self):
        """
        Create and return the ``OptionParser`` which will be used to
        parse the arguments to the worker.

        """
        return OptionParser(prog=self.prog_name,
                            usage=self.usage(),
                            version='%%prog %s' % self.get_version(),
                            option_list=self.option_list)

    def manage_options(self):
        """
        Create a parser given the command-line arguments, creates a parser
        Return True if the programme must exit.
        """
        self.parser = self.create_parser()
        self.options, self.args = self.parser.parse_args(self.argv)

        self.do_imports()

        if self.options.callback and not callable(self.options.callback):
            self.parser.error('The callback is not callable')

        self.logger_level = None
        if self.options.logger_level:
            if self.options.logger_level.isdigit():
                self.options.logger_level = int(self.options.logger_level)
            else:
                try:
                    self.options.logger_level = getattr(logging, self.options.logger_level.upper())
                except:
                    self.parser.error('Invalid logger-level %s' % self.options.logger_level)

        if self.options.max_loops is not None and self.options.max_loops < 0:
            self.parser.error('The max-loops argument (%s) must be a <positive></positive> integer' % self.options.max_loops)

        if self.options.max_duration is not None and self.options.max_duration < 0:
            self.parser.error('The max-duration argument (%s) must be a positive integer' % self.options.max_duration)

        if self.options.timeout is not None and self.options.timeout < 0:
            self.parser.error('The timeout argument (%s) must be a positive integer (including 0)' % self.options.timeout)

        if self.options.fetch_priorities_delay is not None and self.options.fetch_priorities_delay <= 0:
            self.parser.error('The fetch-priorities-delay argument (%s) must be a positive integer' % self.options.fetch_priorities_delay)

        if self.options.fetch_delayed_delay is not None and self.options.fetch_delayed_delay <= 0:
            self.parser.error('The fetch-delayed-delay argument (%s) must be a positive integer' % self.options.fetch_delayed_delay)

        if self.options.requeue_times is not None and self.options.requeue_times < 0:
            self.parser.error('The requeue-times argument (%s) must be a positive integer (including 0)' % self.options.requeue_times)

        if self.options.requeue_delay_delta is not None and self.options.requeue_delay_delta < 0:
            self.parser.error('The rrequeue-delay-delta argument (%s) must be a positive integer (including 0)' % self.options.requeue_delay_delta)

        self.database_config = None
        if self.options.database:
            host, port, db = self.options.database.split(':')
            self.database_config = dict(host=host, port=int(port), db=int(db))

        self.update_title = self.options.update_title

    def do_import(self, name, default):
        """
        Import the given option (use default values if not defined on command line)
        """
        try:
            option = getattr(self.options, name)
            if option:
                klass = import_class(option)
            else:
                klass = default
            setattr(self.options, name, klass)
        except Exception as e:
            self.parser.error('Unable to import "%s": %s' % (name, e))

    def do_imports(self):
        """
        Import all importable options
        """
        self.do_import('worker_class', Worker)
        self.do_import('queue_model', self.options.worker_class.queue_model)
        self.do_import('error_model', self.options.worker_class.error_model)
        self.do_import('callback', self.options.worker_class.callback)

    def print_options(self):
        """
        Print all options as parsed by the script
        """
        options = []

        print("The script is running with the following options:")

        options.append(("dry_run", self.options.dry_run))

        options.append(("worker_config", self.__class__))

        database_config = self.database_config or \
                          self.options.queue_model.database.connection_settings
        options.append(("database", '%s:%s:%s' % (database_config['host'],
                                                  database_config['port'],
                                                  database_config['db'])))

        if self.options.worker_class is not None:
            options.append(("worker-class", self.options.worker_class))

        for name, value in options:
            print(" - %s = %s" % (name.replace('_', '-'), value))

        print("The worker will run with the following options:")
        for name in self.options.worker_class.parameters:
            option = getattr(self.worker, name)
            if name == 'callback' and \
                self.options.worker_class.execute == Worker.execute:
                option = '<jobs "run" method>'
            elif isinstance(option, (list, tuple, set)):
                option = ','.join(option)
            print(" - %s = %s" % (name.replace('_', '-'), option))

    def execute(self):
        """
        Main method to call to run the worker
        """
        self.prepare_models()
        self.prepare_worker()
        if self.options.print_options:
            self.print_options()
        self.run()

    def prepare_models(self):
        """
        If a database config ws given as argument, apply it to our models
        """
        if self.database_config:
            for model in (self.options.queue_model, self.options.error_model):
                model.database.reset(**self.database_config)

    def prepare_worker_options(self):
        """
        Prepare (and return as a dict) all options to be passed to the worker
        """
        worker_options = dict()
        for option_name in (self.options.worker_class.parameters):
            option = getattr(self.options, option_name)
            if option is not None:
                worker_options[option_name] = option
        return worker_options

    def prepare_worker(self):
        """
        Prepare the worker, ready to be launched: prepare options, create a
        log handler if none, and manage dry_run options
        """
        worker_options = self.prepare_worker_options()
        self.worker = self.options.worker_class(**worker_options)
        if self.update_title:
            self.worker._add_update_status_callback(self.update_proc_title)
            self.update_proc_title()

        if not self.worker.logger.handlers:
            handler = logging.StreamHandler()
            handler.setFormatter(logging.Formatter(
                                        ' '.join(['[%(process)d]',
                                                  # '%(asctime)s,%(msecs).03d',
                                                  '%(asctime)s',
                                                  '(%(name)s)',
                                                  '%(levelname)-8s',
                                                  '%(message)s',
                                                 ])
                                        # , '%y.%m.%d:%H.%M.%S'
                                    ))
            self.worker.logger.addHandler(handler)

        if self.options.dry_run:
            self.worker.end_forced = True

    def run(self):
        """
        Simply run the worker by calling its "run" method
        """
        self.worker.run()

    def get_proc_title(self):
        """
        Create the title for the current process (set by `update_proc_title`)
        """
        has_worker = bool(getattr(self, 'worker', None))

        title_parts = [self.prog_name.replace('.py', ('#%s' % self.worker.id) if has_worker else '')]

        status = 'init'
        if has_worker and self.worker.status:
            status = self.worker.status
            if self.worker.end_forced:
                status += ' - ending'
        title_parts.append('[%s]' % status)

        if has_worker and self.worker.queues:
            title_parts.append('queues=%s' % ','.join(self.worker.queues))

        if has_worker and self.worker.status:
            # add infos about the main loop
            title_parts.append('loop=%s/%s' % (self.worker.num_loops, self.worker.max_loops))

            # and about the number of jobs to run
            title_parts.append('waiting=%s' % self.worker.count_waiting_jobs())

            # and about the number of delayed jobs
            title_parts.append('delayed=%s' % self.worker.count_delayed_jobs())

            # and about elapsed time
            if self.worker.start_date:
                duraiton_message = 'duration=%s'
                duration_args = (timedelta(seconds=int(round(self.worker.elapsed.total_seconds()))), )
                if self.worker.max_duration:
                    duraiton_message += '/%s'
                    duration_args += (self.worker.max_duration, )
                title_parts.append(duraiton_message % duration_args)

        return ' '.join(title_parts)

    def update_proc_title(self, worker=None):
        """
        Update the title of the process with useful informations
        """
        if not self.update_title:
            return
        setproctitle(self.get_proc_title())


# -*- coding: utf-8 -*-
"""
Predict test image for Kaggle HPA competition.
Predicting methods: Average ensemble, Test Time Augmentation(TTA)

Author: Minghao Liu
Affliate: Mechanical Engineering, Arizona State University
Date: Dec/13/2018
"""

import numpy as np
from PIL import Image
import pandas as pd

import keras
from keras.models import Model, load_model, Input
from keras.layers import Average
from keras import backend as K

import model
from keras_tta import TTA_ModelWrapper

from tqdm import tqdm

K.set_image_data_format('channels_last')

def load_test_image(data_path, input_img_shape, idx):
    colors = ['red', 'green', 'blue']
    img = np.zeros(input_img_shape)
    for i, c in enumerate(colors):
        im = Image.open(data_path + '/' + idx + '_' + c + '.png')
        im_res = im.resize((input_img_shape[0], input_img_shape[1]), Image.ANTIALIAS)
        image_array = np.array(im_res.getdata()).reshape((input_img_shape[0], input_img_shape[1]))
        img[:,:,i] = image_array / 255.
        im.close()
    return np.array(img[:,:,:3])

def ensemble_predict(model_paths, input_shape, test_path, sample_submit,
                    ensemble_weight, threshold=[0.3], method='hard_voting'):
    def weighted_average(models, image, ensemble_weight):
        ensemble_weight = np.array(ensemble_weight) / np.sum(ensemble_weight)
        for i,base_model in enumerate(models):
            predict_prob += ensemble_weight[i]*base_model.predict(image[np.newaxis])[0]
        #predict_prob = predict_prob / len(models)
        label_predict = np.arange(28)[predict_prob >= np.array(threshold)]
        return label_predict

    def mean_average(models, image):
        for i,base_model in enumerate(models):
            predict_prob += base_model.predict(image[np.newaxis])[0]
        predict_prob = predict_prob / len(models)
        label_predict = np.arange(28)[predict_prob >= np.array(threshold)]
        return label_predict

    def hard_voting(models, image):
        voting_threshold = 0.5*np.ones(28)*len(models)
        preds = np.zeros(28)
        for i,base_model in enumerate(models):
            predict_prob = base_model.predict(image[np.newaxis])[0]
            mask = predict_prob >= np.array(threshold)
            preds[mask] += np.ones(28)[mask]
        label_predict = preds >= voting_threshold
        return label_predict

    models = []
    for mp in model_paths:
        base_model = load_model(mp, custom_objects={'combine_loss':model.combine_loss, 'f1':model.f1})
        tta_model = TTA_ModelWrapper(base_model)
        models.append(base_model)
    predicted = []

    for name in tqdm(sample_submit['Id']):
        predict_prob = np.zeros(28)
        image = load_test_image(test_path, input_shape, name)
        if method == 'mean_average':
            label_predict = mean_average(models, image)
        elif method == 'weighted_average':
            label_predict = weighted_average(models, image, ensemble_weight)
        elif method == 'hard_voting':
            label_predict = hard_voting(models, image)
        str_predict_label = ' '.join(str(l) for l in label_predict)
        predicted.append(str_predict_label)

    sample_submit['Predicted'] = predicted
    sample_submit.to_csv('submission.csv', index=False)

def base_model_predict(model_path, input_shape, test_path, sample_submit, threshold=[0.3]):
    predict_model = load_model(model_path, custom_objects={'combine_loss':model.combine_loss,'f1':model.f1})
    threshold = 0.3
    #model.load_weights('pretrained_model/inceptionv3_weights_60epoch.h5')

    tta_model = TTA_ModelWrapper(predict_model)

    predicted = []
    for name in tqdm(sample_submit['Id']):
        image = load_test_image(test_path, input_shape, name)
        score_predict = tta_model.predict(image[np.newaxis])[0]
        label_predict = np.arange(28)[score_predict[0] >= np.array(threshold)]
        str_predict_label = ' '.join(str(l) for l in label_predict)
        predicted.append(str_predict_label)
        print(str_predict_label)

    sample_submit['Predicted'] = predicted
    sample_submit.to_csv('submission.csv', index=False)

def main():
    ensemble = True

    input_img_shape = (512, 512, 3)

    threshold = [0.3]

    submit = pd.read_csv('sample_submission.csv')

    test_path = 'test'

    model_paths = ['pretrained_model/inceptionv3.h5','pretrained_model/densenet121.h5', 'pretrained_model/inceptionresnetv2.h5']
    #model_paths = 'pretrained_model/dense169_512_30epoch.h5'
    if ensemble:
        ensemble_predict(model_paths, input_img_shape, test_path, submit, [0.45,0.45,0.42], threshold)
    else:
        base_model_predict(model_paths, input_img_shape, test_path, submit, threshold)

if __name__ == '__main__':
    main()


from sqlalchemy import create_engine
from env import env
import os
import datetime
import pygsheets

class Redshift() :
    """
    SQL Alchemy engine
    """
    def __init__(self) :
        self.engine = create_engine("postgresql+psycopg2://{user}:{contr}@{host}:{port}/{base}".format( user = os.environ['REDSHIFT_USER'],
                                                                                contr= os.environ["REDSHIFT_PASSWORD"],
                                                                                port= os.environ["REDSHIFT_PORT"],
                                                                                base= os.environ["REDSHIFT_DB"],
                                                                                host= os.environ["REDSHIFT_HOST"] ),
                               connect_args={'options': '-csearch_path={schema}'.format( schema = os.environ["REDSHIFT_SCHEMA"] )}, echo = False)

class Sheets() :
    """
    Methods for google sheets using pygsheets
    """
    def Insert(self, dat, s) :
        con = pygsheets.authorize(service_account_file= "client_secret.json")
        spdsheet = con.open_by_url(os.environ["gsheet_url"])
        sheet = spdsheet.worksheet_by_title(s)
        sheet.clear(start = "A1")
        #sheet.add_rows(len(dat) - sheet.rows)
        sheet.set_dataframe(dat, start = "A1", nan = '', extend = True)
    def Get_DF(self, s) :
        con = pygsheets.authorize(service_account_file= "client_secret.json")
        spdsheet = con.open_by_url(os.environ["gsheet_url"])
        sheet = spdsheet.worksheet_by_title(s)
        df = sheet.get_as_df()
        return df
    def update_cells(self, s, lista) :
        con = pygsheets.authorize(service_account_file= "client_secret.json")
        spdsheet = con.open_by_url(os.environ["gsheet_url"])
        sheet = spdsheet.worksheet_by_title(s)
        sheet.clear(start = "A1")
        sheet.update_col(1, lista)
    def update_value(self, sheet, addr, val) :
        con = pygsheets.authorize(service_account_file= "client_secret.json")
        spdsheet = con.open_by_url(os.environ["gsheet_url"])
        sheet = spdsheet.worksheet_by_title(sheet)
        sheet.update_value(addr,val)

def payroll_days(date) :
    """
    Return a list of days (1,...,31) which can be considered payroll days,
    depending on weekdays
    """
    l = {15,30,31,29}
    mm = datetime.date(year=int(str(date)[:4]), month =int(str(date)[5:7]), day=15)
    if date.month == 2 :
        em = datetime.date(year=int(str(date)[:4]), month =int(str(date)[5:7]), day=28)
        l.add(28)
    else :
        em = datetime.date(year=int(str(date)[:4]), month =int(str(date)[5:7]), day=30)

    wd = mm
    i = 2
    while i > 0 :
        wd = wd - datetime.timedelta(days=1)
        if wd.isoweekday() not in [6,7] :
            l.add(wd.day)
            i = i-1

    wd = em
    i = 2
    while i > 0 :
        wd = wd - datetime.timedelta(days=1)
        if wd.isoweekday() not in [6,7] :
            l.add(wd.day)
            i = i-1
    return l

class add_days() :
    def __init__(self, date, days, payroll_mode) :
        """
        Parameters
        ----------
        date : TYPE datetime.date
            Fecha a partir de la cual se sumaran los dias.
        days : TYPE int
            Cantidad de dias a agregar.
        payroll_mode : int
            Entero con valores 1, 2 y 3:
                0. Se sumaran dias ignorando las fechas de nomina.
                1. Se sumaran dias siempre y cuando estos no sean fechas nomina.
                2. Se sumaran dias solo cuando estos se encuentren dentro de un periodo de nomina
        Returns
        -------
        None.
        """
        self.weekends = [6,7]
        self.holidays = ["01-01", "02-03", "03-21", "05-01", "09-16", "11-20", "12-25"]
        if payroll_mode == 0 :
            self.new_date = self.__adding(date, days)
        elif payroll_mode == 1 :
            self.new_date = self.__adding_1(date, days)
        elif payroll_mode == 2 :
            self.new_date = self.__adding_2(date, days)
        else :
            self.new_date = self.__adding_3(date, days)
    def __adding(self, date, days):
        if days > 0 :
            aux_date = date + datetime.timedelta(days=1)
            if aux_date.isoweekday() not in self.weekends and str(aux_date)[-5:] not in self.holidays  :
                return self.__adding(aux_date, days-1)
            else :
                return self.__adding(aux_date, days)
        else :
            return date

    def __adding_1(self, date, days):
        if days > 0 :
            aux_date = date + datetime.timedelta(days=1)
            if aux_date.isoweekday() not in self.weekends and str(aux_date)[-5:] not in self.holidays and  aux_date.day not in payroll_days(aux_date) :
                return self.__adding_1(aux_date, days-1)
            else :
                return self.__adding_1(aux_date, days)
        else :
            return date

    def __adding_2(self, date, days):
        if days > 0 :
            aux_date = date + datetime.timedelta(days=1)
            if aux_date.isoweekday() not in self.weekends and str(aux_date)[-5:] not in self.holidays and  aux_date.day in payroll_days(aux_date) :
                aux_date_2 = self.__adding(aux_date, 1)
                if aux_date_2.isoweekday() not in self.weekends and str(aux_date_2)[-5:] not in self.holidays and  aux_date_2.day in payroll_days(aux_date_2) :
                    return self.__adding(aux_date_2, days-2)
                else :
                    return self.__adding_2(aux_date_2,days)
            else :
                return self.__adding_2(aux_date, days)
        else :
            return date
    def __adding_3(self, date, days):
        if days > 0 :
            aux_date = date + datetime.timedelta(days=1)
            if aux_date.isoweekday() not in self.weekends and str(aux_date)[-5:] not in self.holidays and  aux_date.day in payroll_days(aux_date) :
                return self.__adding(aux_date, days-1)
            else :
                return self.__adding_2(aux_date, days)
        else :
            return date


# basic parsing and plotting script

import matplotlib.pyplot as plt

modelStats = open('modelStats.txt', 'r')
samplingInterval = 0
numModels = 0

energyVals = []
maxEnergyVals = []
minEnergyVals = []
meanEnergyVals = []
biomassVals = []
meanBiomassVals = []
fitnessVals = []
nGeneVals = []
nSiteVals = []

shapeStats = open('shapeStats.txt', 'r')
numEntriesVals = []
mcShapeVals = []
lcShapeVals = []
highestFreqVals = []
popVals = []
unpopVals = []
puRatioVals = []
meanShapeVals = []
meanFreqVals = []
sdShapeVals = []
sdFreqVals = []

times = []

for line in modelStats:
    if '#' in line:
        tokens = line.split(' ')
        if 'si' in tokens[0]:
            samplingInterval = int(tokens[1])
        else:
            numModels = int(tokens[1])

    else:
        pairs = line.split(';')
        for pair in pairs:
            vals = pair.split('=')
            if vals[0] == 'energy':
                energyVals.append(int(vals[1]))
            elif vals[0] == 'biomass':
                biomassVals.append(int(vals[1]))
            elif vals[0] == 'fitness':
                fitnessVals.append(int(vals[1]))
            elif vals[0] == 'lowestEnergy':
                minEnergyVals.append(int(vals[1]))
            elif vals[0] == 'highestEnergy':
                maxEnergyVals.append(int(vals[1]))
            elif vals[0] == 'meanEnergy':
                meanEnergyVals.append(float(vals[1]))
            elif vals[0] == 'meanBiomass':
                meanBiomassVals.append(float(vals[1]))
            elif vals[0] == 'nGenes':
                nGeneVals.append(int(vals[1]))
            elif vals[0] == 'nSites':
                nSiteVals.append(int(vals[1]))

for line in shapeStats:
    pairs = line.split(';')
    for pair in pairs:
        vals = pair.split('=')
        if vals[0] == 'numEntries':
            numEntriesVals.append(int(vals[1]))
        elif vals[0] == 'mcShape':
            mcShapeVals.append(int(vals[1]))
        elif vals[0] == 'lcShape':
            lcShapeVals.append(int(vals[1]))
        elif vals[0] == 'higestFreq':
            highestFreqVals.append(int(vals[1]))
        elif vals[0] == 'numPopulated':
            popVals.append(int(vals[1]))
        elif vals[0] == 'numUnpopulated':
            unpopVals.append(int(vals[1]))
        elif vals[0] == 'popUnpopRatio':
            puRatioVals.append(float(vals[1]))
        elif vals[0] == 'meanShape':
            meanShapeVals.append(float(vals[1]))
        elif vals[0] == 'meanFreq':
            meanFreqVals.append(float(vals[1]))
        elif vals[0] == 'sdShape':
            sdShapeVals.append(float(vals[1]))
        elif vals[0] == 'sdFreq':
            sdFreqVals.append(float(vals[1]))

count = 0

while count < numModels:
    times.append(count * samplingInterval)
    count = count + 1

plt.plot(times, fitnessVals, 'ro-', label='Fitness')
plt.plot(times, energyVals, 'mh-', label = 'Energy')
plt.plot(times, biomassVals, 'b*-', label = 'Biomass')
plt.legend(loc=0)
plt.xlabel('Generation')
#plt.savefig('fitnessEnergyBiomass.png')

plt.figure()
plt.plot(times, energyVals, 'mh-', label = 'Energy')
plt.legend(loc=0)
plt.xlabel('Generation')
#plt.savefig('energy.png')

plt.figure()
plt.plot(times, nSiteVals, 'yD-', label = '# of binding sites')
plt.plot(times, nGeneVals, 'cx-', label = '# of genes')
plt.legend(loc=0)
plt.xlabel('Generation')
#plt.savefig('nGenesVnSites.png')

plt.figure()
plt.plot(times, numEntriesVals, 'ro-', label='# of entries')
plt.plot(times, popVals, 'b*-', label = '# of populated shapes')
plt.plot(times, mcShapeVals, 'g^:', label = 'most common shape')
plt.legend(loc=0)
plt.xlabel('Generation')
#plt.savefig('shapes1.png')

plt.figure()

plt.plot(times, meanShapeVals, 'ro-', label='mean shape')
plt.plot(times, sdShapeVals, 'mh-', label = 'standard deviation of shapes')
plt.legend(loc=0)
plt.xlabel('Generation')
#plt.savefig('shapes2.png')

plt.figure()
plt.plot(times, puRatioVals, 'ro-', label='populated-unpopulated ratio')
plt.legend(loc=0)
plt.xlabel('Generation')

plt.show()


from common import Config, VocabType
from argparse import ArgumentParser
from interactive_pharo_predict import InteractivePredictor
from predictor_server import PredictorServer
from model import Model
import sys

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument("-te", "--test", dest="test_path",
                        help="path to test file", metavar="FILE", required=True)
    parser.add_argument("-l", "--load", dest="load_path",
                        help="path to save file", metavar="FILE", required=True)
    parser.add_argument("-o", "--output", dest="output_file",
                        help="path to save file", metavar="FILE", required=True)
    args = parser.parse_args()

    config = Config.get_default_config(args)

    model = Model(config)
    print('Created model')
    model.predict_dataset()
    model.close_session()