import torch
import torch.nn as nn
from torch.nn import init
import numpy as np
import functools
import torch.nn.functional as F
from torch.optim import lr_scheduler
import util.util as util
from .InnerShiftTriple import InnerShiftTriple
from .InnerCos import InnerCos

###############################################################################
# Functions
###############################################################################
def get_norm_layer(norm_type='instance'):
    if norm_type == 'batch':
        norm_layer = functools.partial(nn.BatchNorm2d, affine=True)
    elif norm_type == 'instance':
        norm_layer = functools.partial(nn.InstanceNorm2d, affine=True)
    elif norm_type == 'none':
        norm_layer = None
    else:
        raise NotImplementedError('normalization layer [%s] is not found' % norm_type)
    return norm_layer

def get_scheduler(optimizer, opt):
    if opt.lr_policy == 'lambda':
        def lambda_rule(epoch):
            lr_l = 1.0 - max(0, epoch + 1 + opt.epoch_count - opt.niter) / float(opt.niter_decay + 1)
            return lr_l
        scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_rule)
    elif opt.lr_policy == 'step':
        scheduler = lr_scheduler.StepLR(optimizer, step_size=opt.lr_decay_iters, gamma=0.1)
    elif opt.lr_policy == 'plateau':
        scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.2, threshold=0.01, patience=5)
    elif opt.lr_policy == 'cosine':
        scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=opt.niter, eta_min=0)
    else:
        return NotImplementedError('learning rate policy [%s] is not implemented', opt.lr_policy)
    return scheduler

def init_weights(net, init_type='normal', gain=0.02):
    def init_func(m):
        classname = m.__class__.__name__
        if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1):
            if init_type == 'normal':
                init.normal_(m.weight.data, 0.0, gain)
            elif init_type == 'xavier':
                init.xavier_normal_(m.weight.data, gain=gain)
            elif init_type == 'kaiming':
                init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
            elif init_type == 'orthogonal':
                init.orthogonal_(m.weight.data, gain=gain)
            else:
                raise NotImplementedError('initialization method [%s] is not implemented' % init_type)
            if hasattr(m, 'bias') and m.bias is not None:
                init.constant_(m.bias.data, 0.0)
        elif classname.find('BatchNorm2d') != -1:
            init.normal_(m.weight.data, 1.0, gain)
            init.constant_(m.bias.data, 0.0)

    print('initialize network with %s' % init_type)
    net.apply(init_func)

def init_net(net, init_type='normal', init_gain=0.02, gpu_ids=[]):
    if len(gpu_ids) > 0:
        assert(torch.cuda.is_available())
        net.to(gpu_ids[0])
        net = torch.nn.DataParallel(net, gpu_ids)
    init_weights(net, init_type, gain=init_gain)
    return net

def define_G(input_nc, output_nc, ngf, which_model_netG, opt, mask_global, norm='batch', use_dropout=False, init_type='normal', gpu_ids=[], init_gain=0.02):
    netG = None
    norm_layer = get_norm_layer(norm_type=norm)

    innerCos_list = []
    shift_list = []

    # Here we need to initlize an artificial mask_global to construct the init model.
    # When training, we need to set mask for special layers(mostly for Shift layers) first.
    # If mask is fixed during training, we only need to set mask for these layers once,
    # else we need to set the masks each iteration, generating new random masks and mask the input
    # as well as setting masks for these special layers.

    if which_model_netG == 'unet_256':
        netG = UnetGenerator(input_nc, output_nc, 8, ngf, norm_layer=norm_layer, use_dropout=use_dropout)
    elif which_model_netG == 'unet_shift_triple':
        netG = UnetGeneratorShiftTriple(input_nc, output_nc, 8, opt, innerCos_list, shift_list, mask_global, \
                                                         ngf, norm_layer=norm_layer, use_dropout=use_dropout)
    else:
        raise NotImplementedError('Generator model name [%s] is not recognized' % which_model_netG)

    print('Constraint in netG:')
    print(innerCos_list)

    print('Shift in netG:')
    print(shift_list)

    return init_net(netG, init_type, init_gain, gpu_ids), innerCos_list, shift_list

def define_D(input_nc, ndf, which_model_netD,
             n_layers_D=3, norm='batch', use_sigmoid=False, init_type='normal', gpu_ids=[], init_gain=0.02):
    netD = None
    norm_layer = get_norm_layer(norm_type=norm)

    if which_model_netD == 'basic':
        netD = NLayerDiscriminator(input_nc, ndf, n_layers=3, norm_layer=norm_layer, use_sigmoid=use_sigmoid)
    elif which_model_netD == 'n_layers':
        netD = NLayerDiscriminator(input_nc, ndf, n_layers_D, norm_layer=norm_layer, use_sigmoid=use_sigmoid)
    else:
        print('Discriminator model name [%s] is not recognized' %
              which_model_netD)
    return init_net(netD, init_type, init_gain, gpu_ids)

##############################################################################
# Classes
##############################################################################

# Defines the GAN loss which uses either LSGAN or the regular GAN.
# When LSGAN is used, it is basically same as MSELoss,
# but it abstracts away the need to create the target label tensor
# that has the same size as the input
class GANLoss(nn.Module):
    def __init__(self, gan_type='wgan_gp', target_real_label=1.0, target_fake_label=0.0):
        super(GANLoss, self).__init__()
        self.register_buffer('real_label', torch.tensor(target_real_label))
        self.register_buffer('fake_label', torch.tensor(target_fake_label))
        if gan_type == 'wgan_gp':
            self.loss = nn.MSELoss()
        elif gan_type == 'lsgan':
            self.loss = nn.MSELoss()
        elif gan_type == 'vanilla':
            self.loss = nn.BCELoss()
        else:
            raise ValueError("GAN type [%s] not recognized." % gan_type)

    def get_target_tensor(self, input, target_is_real):
        if target_is_real:
            target_tensor = self.real_label
        else:
            target_tensor = self.fake_label
        return target_tensor.expand_as(input)

    def __call__(self, input, target_is_real):
        target_tensor = self.get_target_tensor(input, target_is_real)
        return self.loss(input, target_tensor)

################################### ***************************  #####################################
################################### This the original Shift_net  #####################################
################################### ***************************  #####################################
# Defines the Unet generator.
# |num_downs|: number of downsamplings in UNet. For example,
# if |num_downs| == 7, image of size 128x128 will become of size 1x1
# at the bottleneck
class UnetGeneratorShiftTriple(nn.Module):
    def __init__(self, input_nc, output_nc,  num_downs, opt, innerCos_list, shift_list, mask_global, ngf=64,
                 norm_layer=nn.BatchNorm2d, use_dropout=False):
        super(UnetGeneratorShiftTriple, self).__init__()

        # construct unet structure
        unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=None, norm_layer=norm_layer, innermost=True)
        print(unet_block)
        for i in range(num_downs - 5):  # The innner layers number is 3 (sptial size:512*512), if unet_256.
            unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer, use_dropout=use_dropout)
        unet_block = UnetSkipConnectionBlock(ngf * 4, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer)

        unet_shift_block = UnetSkipConnectionShiftTriple(ngf * 2, ngf * 4, opt, innerCos_list, shift_list, mask_global, input_nc=None, \
                                                         submodule=unet_block, norm_layer=norm_layer)  # passing in unet_shift_block
        unet_block = UnetSkipConnectionBlock(ngf, ngf * 2, input_nc=None, submodule=unet_shift_block, norm_layer=norm_layer)
        unet_block = UnetSkipConnectionBlock(output_nc, ngf, input_nc=input_nc, submodule=unet_block, outermost=True, norm_layer=norm_layer)

        self.model = unet_block

    def forward(self, input):
        return self.model(input)

# Mention: the TripleBlock differs in `upconv` defination.
# 'cos' means that we add a `innerCos` layer in the block.
class UnetSkipConnectionShiftTriple(nn.Module):
    def __init__(self, outer_nc, inner_nc, opt, innerCos_list, shift_list, mask_global, input_nc, \
                 submodule=None, shift_layer=None, outermost=False, innermost=False, norm_layer=nn.BatchNorm2d, use_dropout=False):
        super(UnetSkipConnectionShiftTriple, self).__init__()
        self.outermost = outermost
        if input_nc is None:
            input_nc = outer_nc

        downconv = nn.Conv2d(input_nc, inner_nc, kernel_size=4,
                             stride=2, padding=1)
        downrelu = nn.LeakyReLU(0.2, True)
        downnorm = norm_layer(inner_nc, affine=True)
        uprelu = nn.ReLU(True)
        upnorm = norm_layer(outer_nc, affine=True)

        # As the downconv layer is outer_nc in and inner_nc out.
        # So the shift define like this:
        shift = InnerShiftTriple(opt.threshold, opt.fixed_mask, opt.shift_sz, opt.stride, opt.mask_thred, opt.triple_weight)

        shift.set_mask(mask_global, 3, opt.threshold)
        shift_list.append(shift)

        # Add latent constraint
        # Then add the constraint to the constrain layer list!
        innerCos = InnerCos(strength=opt.strength, skip=opt.skip)
        innerCos.set_mask(mask_global, opt)  # Here we need to set mask for innerCos layer too.
        innerCos_list.append(innerCos)

        # Different position only has differences in `upconv`
            # for the outermost, the special is `tanh`
        if outermost:
            upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1)
            down = [downconv]
            up = [uprelu, upconv, nn.Tanh()]
            model = down + [submodule] + up
            # for the innermost, the special is `inner_nc` instead of `inner_nc*2`
        elif innermost:
            upconv = nn.ConvTranspose2d(inner_nc, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1)
            down = [downrelu, downconv]  # for the innermost, no submodule, and delete the bn
            up = [uprelu, upconv, upnorm]
            model = down + up
            # else, the normal
        else:
            # shift triple differs in here. It is `*3` not `*2`.
            upconv = nn.ConvTranspose2d(inner_nc * 3, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1)
            down = [downrelu, downconv, downnorm]
            # shift should be placed after uprelu
            # NB: innerCos are placed before shift. So need to add the latent gredient to
            # to former part.
            up = [uprelu, innerCos, shift, upconv, upnorm]

            if use_dropout:
                model = down + [submodule] + up + [nn.Dropout(0.5)]
            else:
                model = down + [submodule] + up

        self.model = nn.Sequential(*model)

    def forward(self, x):
        if self.outermost:  # if it is the outermost, directly pass the input in.
            return self.model(x)
        else:
            x_latter = self.model(x)
            _, _, h, w = x.size()
            if h != x_latter.size(2) or w != x_latter.size(3):
                x_latter = F.interpolate(x_latter, (h, w), mode='bilinear')
            return torch.cat([x_latter, x], 1)  # cat in the C channel

# Defines the Unet generator.
# |num_downs|: number of downsamplings in UNet. For example,
# if |num_downs| == 7, image of size 128x128 will become of size 1x1
# at the bottleneck
class UnetGenerator(nn.Module):
    def __init__(self, input_nc, output_nc, num_downs, ngf=64,
                 norm_layer=nn.BatchNorm2d, use_dropout=False):
        super(UnetGenerator, self).__init__()

        # construct unet structure
        unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=None, norm_layer=norm_layer, innermost=True)
        for i in range(num_downs - 5):
            unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer, use_dropout=use_dropout)
        unet_block = UnetSkipConnectionBlock(ngf * 4, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer)
        unet_block = UnetSkipConnectionBlock(ngf * 2, ngf * 4, input_nc=None, submodule=unet_block, norm_layer=norm_layer)
        unet_block = UnetSkipConnectionBlock(ngf, ngf * 2, input_nc=None, submodule=unet_block, norm_layer=norm_layer)
        unet_block = UnetSkipConnectionBlock(output_nc, ngf, input_nc=input_nc, submodule=unet_block, outermost=True, norm_layer=norm_layer)

        self.model = unet_block

    def forward(self, input):
        return self.model(input)

# construct network from the inside to the outside.
# Defines the submodule with skip connection.
# X -------------------identity---------------------- X
#   |-- downsampling -- |submodule| -- upsampling --|
class UnetSkipConnectionBlock(nn.Module):
    def __init__(self, outer_nc, inner_nc, input_nc,
                 submodule=None, outermost=False, innermost=False, norm_layer=nn.BatchNorm2d, use_dropout=False):
        super(UnetSkipConnectionBlock, self).__init__()
        self.outermost = outermost

        if input_nc is None:
            input_nc = outer_nc

        downconv = nn.Conv2d(input_nc, inner_nc, kernel_size=4,
                             stride=2, padding=1)
        downrelu = nn.LeakyReLU(0.2, True)
        downnorm = norm_layer(inner_nc, affine=True)
        uprelu = nn.ReLU(True)
        upnorm = norm_layer(outer_nc, affine=True)

        # Different position only has differences in `upconv`
            # for the outermost, the special is `tanh`
        if outermost:
            upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1)
            down = [downconv]
            up = [uprelu, upconv, nn.Tanh()]
            model = down + [submodule] + up
            # for the innermost, the special is `inner_nc` instead of `inner_nc*2`
        elif innermost:
            upconv = nn.ConvTranspose2d(inner_nc, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1)
            down = [downrelu, downconv]  # for the innermost, no submodule, and delete the bn
            up = [uprelu, upconv, upnorm]
            model = down + up
            # else, the normal
        else:
            upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc,
                                        kernel_size=4, stride=2,
                                        padding=1)
            down = [downrelu, downconv, downnorm]
            up = [uprelu, upconv, upnorm]

            if use_dropout:
                model = down + [submodule] + up + [nn.Dropout(0.5)]
            else:
                model = down + [submodule] + up

        self.model = nn.Sequential(*model)

    def forward(self, x):
        if self.outermost:  # if it is the outermost, directly pass the input in.
            return self.model(x)
        else:
            x_latter = self.model(x)
            _, _, h, w = x.size()
            if h != x_latter.size(2) or w != x_latter.size(3):
                x_latter = F.interpolate(x_latter, (h, w), mode='bilinear')
            return torch.cat([x_latter, x], 1)  # cat in the C channel

################################### This is for D ###################################
# Defines the PatchGAN discriminator with the specified arguments.
class NLayerDiscriminator(nn.Module):
    def __init__(self, input_nc, ndf=64, n_layers=3, norm_layer=nn.BatchNorm2d, use_sigmoid=False):
        super(NLayerDiscriminator, self).__init__()
        if type(norm_layer) == functools.partial:
            use_bias = norm_layer.func == nn.InstanceNorm2d
        else:
            use_bias = norm_layer == nn.InstanceNorm2d

        kw = 4
        padw = 1
        sequence = [
            nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw),
            nn.LeakyReLU(0.2, True)
        ]

        nf_mult = 1
        nf_mult_prev = 1
        for n in range(1, n_layers):
            nf_mult_prev = nf_mult
            nf_mult = min(2**n, 8)
            sequence += [
                nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult,
                          kernel_size=kw, stride=2, padding=padw, bias=use_bias),
                norm_layer(ndf * nf_mult),
                nn.LeakyReLU(0.2, True)
            ]

        nf_mult_prev = nf_mult
        nf_mult = min(2**n_layers, 8)
        sequence += [
            nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult,
                      kernel_size=kw, stride=1, padding=padw, bias=use_bias),
            norm_layer(ndf * nf_mult),
            nn.LeakyReLU(0.2, True)
        ]

        sequence += [nn.Conv2d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)]

        if use_sigmoid:
            sequence += [nn.Sigmoid()]

        self.model = nn.Sequential(*sequence)

    def forward(self, input):
        return self.model(input)


from functools import partial
import numpy as np
from auxiliary import make_individual, to_phenotype, local_search_knowledge_compilation_based
from crossover import clause_crossover_1x, uniform_crossover, matched_uniform_crossover, uniform_clause_crossover, \
    scramble_clause_crossover, avoid_duplicate_clauses_scramble_clause_crossover, smart_clause_crossover_dispatch
from evaluation import evaluate, evaluate_use_infeasibility
from hassle_gen import HassleGen
from mutation import mutate_hardness, mutate_weight, mutate_clause, mutate_to_neighbour,\
    mutate_clause_smart, mutate_clause_smarter

def learn_max_sat_model(n,
                        k,
                        examples,
                        population_size,
                        tournament_size=None,
                        use_crowding=False,
                        crowding_variant="semantic_relative",
                        generations=10,
                        prob_crossover=0.9,
                        crossover_operators=None,
                        mutation_operators=None,
                        use_local_search=False,
                        cutoff_time=None,
                        cutoff_time_flags=None,
                        use_knowledge_compilation_caching=True,
                        knowledge_compilation_variant=4,
                        use_diagram_for_instance_evaluation=True,
                        conjunctive_contexts=0,
                        variable_absence_bias=1,
                        use_infeasibility=False,
                        use_clause_bitvector_cache=True,
                        observers=None):
    """
    Configures HASSLE-GEN as desired and subsequently runs the algorithm.
    :param n: The number of variables
    :param k: The number of constraints
    :param examples: A list of examples, where each example is a context-instance-label tuple
    :param population_size: The population size
    :param tournament_size: The tournament size - only relevant when not using crowding
    :param use_crowding: A Boolean that denotes whether to use crowding
    :param crowding_variant: What crowding variant to use: "semantic_relative", "semantic_absolute", "syntactic". Only
    relevant when use_crowding is True.
    :param generations: The maximum number of generations
    :param prob_crossover: The probability of applying a crossover operator on a pair of parents
    :param crossover_operators: A list of crossover operators to use. Each operator is itself a list of which the first
    element denotes the name of the crossover operator, and optional additional elements are used to configure the
    operator
    :param mutation_operators: A list of mutation operators to use. Each operator is itself a list of which the first
    element denotes the name of the mutation operator, and optional additional elements are used to configure the
    operator.
    :param use_local_search: Whether to apply a HASSLE-SLS local search step to each individual in each iteration
    :param cutoff_time: The maximum number of seconds to run the algorithm for
    :param cutoff_time_flags: A list of strings that denotes what operations should be counted in the runtime of the
    algorithm. Options are: "selection", "crossover", "mutation", "local search" and "evaluation". By default all
    are included.
    :param use_knowledge_compilation_caching: A Boolean that denotes whether to use knowledge-compilation-based caching
    :param knowledge_compilation_variant: What variant of knowledge-compilation-based caching to use. Options are:
    1, 2, 3 and 4. Only relevant when use_knowledge_compilation_caching is True.
    :param use_diagram_for_instance_evaluation: A Boolean that denotes whether to use the knowledge-compiled
    representation not just for computing the optimal value in a context, but also to evaluate the instances themselves.
    Only relevant when use_knowledge_compilation_caching is True.
    :param conjunctive_contexts: A Boolean that denotes whether the contexts occurring in the examples should be
    interpreted as conjunctions. If False, the contexts are interpreted as disjunctions.
    :param variable_absence_bias: A float that can be used to determine the length of constraints in the initial
    population. In initializing a model, in each constraint, each variable can occur as a negative
    literal, as a positive literal, or be absent from the constraint. With variable_absence_bias = 1, any variable
    has the same probability of occurring (either positively or negatively) as being absent in any constraint. With
    variable_absence_bias = 2, any variable is twice as likely to be absent as to occur, in any constraint.
    :param use_infeasibility: A Boolean that denotes whether to make use of the distinction between infeasible and
    suboptimal negative examples. If False, a negative example is simply interpreted as being negative, and no
    distinction between infeasibility and suboptimality is made.
    :param use_clause_bitvector_cache: A Boolean that denotes whether to cache coverage bitvectors computed during
    execution. Relevant when using smart mutation, smarter mutation or smart crossover.
    :param observers: A list of Observer objects to report aggregate data to in every generation.
    :return: A tuple consisting of (i) The best training set accuracy achieved, (ii) the phenotype that achieved said
    training set accuracy and (iii) the runtime of the algorithm (including only the runtime of the operations
    specified in cutoff_time_flags).
    """

    if not use_crowding and tournament_size is None:
        raise ValueError("Specify a tournament size when not using crowding")

    ga = HassleGen(
        partial(make_individual, n, k, variable_absence_bias=variable_absence_bias),
        partial(evaluate, examples=examples, use_knowledge_compilation_caching=use_knowledge_compilation_caching,
                knowledge_compilation_variant=knowledge_compilation_variant,
                use_diagram_for_instance_evaluation=use_diagram_for_instance_evaluation,
                conjunctive_contexts=conjunctive_contexts) if use_infeasibility == False else
        partial(evaluate_use_infeasibility, examples=examples,
                use_knowledge_compilation_caching=use_knowledge_compilation_caching,
                knowledge_compilation_variant=knowledge_compilation_variant,
                use_diagram_for_instance_evaluation=use_diagram_for_instance_evaluation,
                conjunctive_contexts=conjunctive_contexts),
        population_size=population_size,
        prob_crossover=prob_crossover,
        use_crowding=use_crowding,
        crowding_variant=crowding_variant,
        tournament_size=tournament_size
    )

    # Make smart clause mutation and smart clause crossover more efficient by caching
    if use_clause_bitvector_cache:
        clause_bitvector_cache = dict()
    else:
        clause_bitvector_cache = None

    # Add crossover operators
    if crossover_operators is None:
        # Use defaults
        ga.add_crossover(scramble_clause_crossover)
        ga.add_crossover(uniform_crossover)
    else:
        # Use the requested operators
        for crossover_operator in crossover_operators:
            if crossover_operator[0] == "clause_crossover_1x":
                ga.add_crossover(clause_crossover_1x)
            elif crossover_operator[0] == "uniform_crossover":
                ga.add_crossover(uniform_crossover)
            elif crossover_operator[0] == "matched_uniform_crossover":
                ga.add_crossover(matched_uniform_crossover)
            elif crossover_operator[0] == "uniform_clause_crossover":
                ga.add_crossover(uniform_clause_crossover)
            elif crossover_operator[0] == "scramble_clause_crossover":
                ga.add_crossover(scramble_clause_crossover)
            elif crossover_operator[0] == "avoid_duplicate_clauses_scramble_clause_crossover":
                ga.add_crossover(avoid_duplicate_clauses_scramble_clause_crossover)
            elif crossover_operator[0] == "smart_clause_crossover":
                greedy = crossover_operator[1]
                probability_variant = crossover_operator[2]
                temperature = crossover_operator[3]
                ga.add_crossover(partial(smart_clause_crossover_dispatch, examples=examples, greedy=greedy,
                                         probability_variant=probability_variant, temperature=temperature,
                                         clause_bitvector_cache=clause_bitvector_cache,
                                         use_infeasibility=use_infeasibility))
                # With smart clause crossover, 2 parents produce only a single offspring
                ga.single_offspring_per_parent_couple = True
            else:
                raise Exception("A valid crossover operator should be chosen")

    # Add mutation operators
    if mutation_operators is None:
        # Use defaults
        ga.add_mutation(mutate_hardness, trigger_prob=1, inner_prob=0.05)
        ga.add_mutation(mutate_clause, trigger_prob=1, inner_prob=0.05)
        ga.add_mutation(mutate_weight, trigger_prob=1, inner_prob=0.05)
    else:
        # Use the requested operators
        for mutation_operator in mutation_operators:
            if mutation_operator[0] == "mutate_hardness":
                trigger_prob = mutation_operator[1]
                inner_prob = mutation_operator[2]
                ga.add_mutation(partial(mutate_hardness, trigger_prob=trigger_prob, inner_prob=inner_prob))
            elif mutation_operator[0] == "mutate_clause":
                trigger_prob = mutation_operator[1]
                inner_prob = mutation_operator[2]
                ga.add_mutation(partial(mutate_clause, trigger_prob=trigger_prob, inner_prob=inner_prob))
            elif mutation_operator[0] == "mutate_weight":
                trigger_prob = mutation_operator[1]
                inner_prob = mutation_operator[2]
                ga.add_mutation(partial(mutate_weight, trigger_prob=trigger_prob, inner_prob=inner_prob))
            elif mutation_operator[0] == "mutate_to_neighbour":
                trigger_prob = mutation_operator[1]
                contexts = [e[0] for e in examples]
                data = np.array([e[1] for e in examples])
                labels = [e[2] for e in examples]
                boolean_labels = np.array([True if l == 1 else False for l in labels])
                if use_infeasibility:
                    inf = [True if l == -1 else False for l in labels]
                else:
                    inf = None
                ga.add_mutation(partial(mutate_to_neighbour, trigger_prob=trigger_prob, contexts=contexts, data=data,
                                        boolean_labels=boolean_labels, inf=inf))
            elif mutation_operator[0] == "mutate_clause_smart":
                trigger_prob = mutation_operator[1]
                temperature = None
                if len(mutation_operator) > 2:
                    temperature = mutation_operator[2]
                ga.add_mutation(partial(mutate_clause_smart, trigger_prob=trigger_prob, examples=examples,
                                        clause_bitvector_cache=clause_bitvector_cache,
                                        use_infeasibility=use_infeasibility,
                                        temperature=temperature))
            elif mutation_operator[0] == "mutate_clause_smarter":
                trigger_prob = mutation_operator[1]
                temperature = None
                if len(mutation_operator) > 2:
                    temperature = mutation_operator[2]
                ga.add_mutation(partial(mutate_clause_smarter, trigger_prob=trigger_prob, examples=examples,
                                        clause_bitvector_cache=clause_bitvector_cache,
                                        use_infeasibility=use_infeasibility,
                                        temperature=temperature))
            else:
                raise Exception("A valid mutation operator should be chosen")

    # Add method that converts genotype to phenotype representation
    ga.to_phenotype = to_phenotype

    if use_local_search:
        # Perform a local search step to every individual in every generation
        if use_infeasibility:
            contexts = [e[0] for e in examples]
            data = np.array([e[1] for e in examples])
            labels = [e[2] for e in examples]
            inf = [True if l == -1 else False for l in labels]
        else:
            inf = None
        ga.local_search_function = partial(local_search_knowledge_compilation_based, examples,
                                           conjunctive_contexts=conjunctive_contexts, inf=inf)

    if observers is not None:
        # Add observers
        for observer in observers:
            ga.observers.append(observer)

    # Run HASSLE-GEN
    return ga.run(generations, 1, cutoff_time, cutoff_time_flags)


# -*- coding: utf-8 -*-
"""
Created on Mon Oct 14 10:12:36 2019
@author: alexl
"""

import FeatureEngineerer as fe
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sklearn.pipeline import make_pipeline
from sklearn.linear_model import LinearRegression, Ridge
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.preprocessing import PolynomialFeatures, MinMaxScaler, PowerTransformer

#Supress ALL Warnings
import warnings
warnings.filterwarnings("ignore")

data = pd.read_csv('Data/train.csv', index_col=0, parse_dates = True)
X_pred = pd.read_csv('Data/test.csv', index_col=0, parse_dates = True)
#randomise the order of data
data = data.sample(len(data))

def share_results(a,b):
    iterater = 0
    for i in a:
        print(f'coefficient of {X.columns[iterater]}: {i}')
        iterater +=1
    print(f'constant is: {b}')

def XFeatures(data):
    #----------- Feature Engineering and Selection -----------#
    if 'count' in data.columns:
        X = data.drop(['count','registered','casual'], axis = 1)
    else:
        X = data

    #----------- Add year, month, day, hour to df -----------#
    X = fe.FeatureAdder(X)

    #----------- windspeed = 0 to RFclassifier -----------#
    #X = fe.FillWindspeed(X)

    #----------- buckets -----------#
    X['temp'] = pd.qcut(X['temp'], 3, labels = [0, 1, 2])
    #X = fe.OneHotEncoder(X,'temp').rename(columns = {0:'Cold',1:'Nice',2:'Hot'})

    #----------- one hot encode -----------#
    X = fe.OneHotEncoder(X,'season').rename(columns = {1:'Spring',2:'Summer',3:'Autumn',4:'Winter'})
    X = fe.OneHotEncoder(X,'weekday_hour').rename(columns = {0:'wh0',1:'wh1',2:'wh2',3:'wh3',4:'wh4',5:'wh5',
                        6:'wh6',7:'wh7',8:'wh8',9:'wh9',10:'wh10',11:'wh11',12:'wh12',13:'wh13',14:'wh14',15:'wh15',
                        16:'wh16',17:'wh17',18:'wh18',19:'wh19',20:'wh20',21:'wh21',22:'wh22',23:'wh23',24:'wh24'})

    X = fe.OneHotEncoder(X,'weekend_hour').rename(columns = {0:'we0',1:'we1',2:'we2',3:'we3',4:'we4',5:'we5',
                        6:'we6',7:'we7',8:'we8',9:'we9',10:'we10',11:'we11',12:'we12',13:'we13',14:'we14',15:'we15',
                        16:'we16',17:'we17',18:'we18',19:'we19',20:'we20',21:'we21',22:'we22',23:'we23',24:'we24'})
    X = fe.OneHotEncoder(X,'weather')

    #----------- Feature Selection -----------#
#    print(X.columns)
    X = X.drop([1,'we0','wh0', 'Autumn','atemp','humidity',
            'hour','month','year','day',
            'workingday','holiday'], axis = 1)

    return X

#y = data['count']
y = np.log10(data['count'])

X = XFeatures(data)
X_test = XFeatures(X_pred)

pipeline = make_pipeline(
        PolynomialFeatures(),
        MinMaxScaler(),
        PowerTransformer(copy=True, method = 'yeo-johnson',standardize=True),
        Ridge()
        )

g = GridSearchCV(pipeline, cv = 2, param_grid = {
        'polynomialfeatures__degree':[2],
        'ridge__alpha':[0.01,0.1,0.09,0.11]
        })

g.fit(X,y)

#predict test data
ypred = g.predict(X) #data from training set
y_pred = g.predict(X_test) #kaggle predictions

print(f'the best params are: g.best_params_')
print(f'Model score was: {g.best_score_}')
share_results(g.best_estimator_.named_steps['ridge'].coef_,
              g.best_estimator_.named_steps['ridge'].intercept_)

#re-evaluate negatives to 0
print(f'Sum of ypredictions < 0: {ypred[ypred < 0].sum()}')
print(f'Sum of kaggle ys < 0: {y_pred[y_pred < 0].sum()}')
ypred[ypred < 0] = 0
y_pred[y_pred < 0] = 0

#validate results
fe.Validator(X, y, ypred)

#prediction compare graph
#fe.PredictionCompare(data,ypred)

#unlog and export predictions for Kaggle
y_pred = 10**y_pred
export = pd.DataFrame(y_pred, columns = ['count'], index = X_test.index) ##################
export.to_csv(f'Data/count_predictions2.csv')

#dir(g)
#g.best_params_
#g.best_score_

#Gsearch coef and intercept of best fit
#g.best_estimator_.named_steps['ridge'].coef_
#g.best_estimator_.named_steps['ridge'].intercept_


sugu = input("Sisesta enda sugu: ")
sugu = sugu.strip() # eemaldame sõna eest ja tagant üleliigsed tühikud. Võid eraldi konsoolis katsetada nt: " abc   ".strip() väljastab: "abc"
sugu = sugu.capitalize() # Teeme esimese tähe suureks ja muu tähed väikseks.

if sugu == "Mees":
    print("Sa oled mees.")
elif sugu == "Naine":
    print("Sa oled naine.")
elif sugu== "Laps":
    print("Sa oled laps.")
else:
    print("Sa oled mingi imelik elukas.")


from smac.env import StarCraft2Env
import numpy as np
import random
import pickle
import matplotlib.pyplot as plt
from tqdm import tqdm

def choose_action(states, possible_ja):
    if random.uniform(0, 1) < (1 - epsilon):
        action = np.random.choice(possible_ja)
    else:
        qt_arr1 = np.zeros(len(possible_ja))
        qt_arr2 = np.zeros(len(possible_ja))

        keys = np.arange(len(possible_ja))
        act_ind_decode = dict(zip(keys, possible_ja))

        for act_ind in range(len(possible_ja)):
            qt_arr1[act_ind] = q_table[states[0], act_ind_decode[act_ind]]
            qt_arr2[act_ind] = q_table[states[1], act_ind_decode[act_ind]]

        action1 = act_ind_decode[np.argmax(qt_arr1)]
        q_value1 = max(qt_arr1)

        action2 = act_ind_decode[np.argmax(qt_arr2)]
        q_value2 = max(qt_arr2)

        if q_value1 > q_value2:
            action = action1
        else:
            action = action2

    return action

def learn(state, state2, reward, action):
    q_table[state, action] = q_table[state, action] + alpha * \
                             (reward + gamma * np.max(q_table[state2, :]) - q_table[state, action])

def my_get_state(agent_id):
    unit = env.get_unit_by_id(agent_id)
    target_items = env.enemies.items()
    shoot_range = env.unit_shoot_range(agent_id)
    can_attack = False

    for t_id, t_unit in target_items:
        if t_unit.health > 0:
            dist = env.distance(
                unit.pos.x, unit.pos.y, t_unit.pos.x, t_unit.pos.y
            )
            if dist <= shoot_range:
                can_attack = True

    if not can_attack:
        state = (unit.pos.x - map_size[0]) * env_states_range[1] + (unit.pos.y - map_size[2])
    else:
        state = (unit.pos.x - map_size[0]) * env_states_range[1] + (unit.pos.y - map_size[2]) + \
                n_states // 2 - 1

    if state > q_table.shape[0] - 1:
        print('Agent position when out of bounds with state', unit.pos.x, unit.pos.y, state)
        state = q_table.shape[0] - 1

    return int(state)

def jal_encoder(action1, action2):
    return action1 * n_actions + action2

def jal_decoder(action):
    return [action // n_actions, action % n_actions]

def avail_joint_actions(avail_actions_array):
    """
    Takes array of shape [[agent0_avail_actions], [agent1_avail_actions]]

    Returns all available joint actions
    """

    if avail_actions_array[0][0] == 0:
        return avail_actions_array[1]

    if avail_actions_array[1][0] == 0:
        return avail_actions_array[0] * n_actions

    all_actions = []

    for agent0_act in range(len(avail_actions_array[0])):
        for agent1_act in range(len(avail_actions_array[1])):
            all_actions.append(jal_encoder(avail_actions_array[0][agent0_act],
                                           avail_actions_array[1][agent1_act]))
    return all_actions

method_name = "JAL"
env = StarCraft2Env(map_name="2m_vs_2zg")
env_info = env.get_env_info()

map_size = (8, 24, 25, 28)  # x1, x2, y1, y2
env_states_range = (map_size[1] - map_size[0], map_size[3] - map_size[2])
n_actions = env_info["n_actions"]
n_agents = env_info["n_agents"]
episode_limit = env_info["episode_limit"]

n_episodes = 10_200
n_states = env_states_range[0] * env_states_range[1] * 2

alpha = 0.2
gamma = 0.95
EPSILON = 0.6
q_table = np.zeros([n_states, n_actions * n_actions])

epsilon_values = []
total_steps = []
ep_reward = []
mean_total_steps = []
mean_ep_reward = []
ep_reward_agent1 = []
ep_reward_agent2 = []
mean_ep_reward_agent1 = []
mean_ep_reward_agent2 = []

epsilon = EPSILON

for e in tqdm(range(1, n_episodes + 1), ascii=True, unit="episode"):
    env.reset()
    terminated = False
    episode_reward = 0
    episode_reward1 = 0
    episode_reward2 = 0
    n_steps = 1

    if e < 500:
        epsilon = EPSILON
    elif e < 2000:
        epsilon += ((1 - epsilon) ** 0.5) * 0.8 / 1000
    elif e < 3000:
        epsilon -= ((1 - epsilon) ** 0.5) * 1.03 / 1000
    elif e < 3500:
        epsilon = EPSILON + 0.1
    elif e < 5000:
        epsilon += ((1 - epsilon) ** 0.5) * 0.7 / 1000
    elif e < 6000:
        epsilon -= ((1 - epsilon) ** 0.5) * 0.85 / 1000
    elif e < 6500:
        epsilon = EPSILON + 0.2
    elif e < 8000:
        epsilon += ((1 - epsilon) ** 0.5) * 0.58 / 1000
    elif e < 8500:
        epsilon -= ((1 - epsilon) ** 0.5) * 1.2 / 1000
    elif e < 9000:
        epsilon = EPSILON + 0.3
    elif e < 10000:
        epsilon += ((1 - epsilon) ** 0.5) * 0.6 / 1000
    else:
        epsilon = 1

    while not terminated:
        states = []
        all_avail_actions = []
        next_states = []
        actions = []
        rewards = []

        for agent_id in range(n_agents):
            state = my_get_state(agent_id)
            states.append(state)

            avail_actions = env.get_avail_agent_actions(agent_id)
            avail_actions[1] = 0
            avail_actions_ind = np.nonzero(avail_actions)[0]
            all_avail_actions.append(avail_actions_ind)

        possible_joint_actions = avail_joint_actions(all_avail_actions)

        joint_action = choose_action(states, possible_joint_actions)

        actions_pair = jal_decoder(joint_action)

        for agent_id in range(n_agents):
            action = actions_pair[agent_id]
            if action > 5:
                rewards.append(10)
            else:
                rewards.append(-2)
            actions.append(action)

        _, terminated, _ = env.step(actions)

        for i in range(len(rewards)):
            rewards[i] = rewards[i] * ((episode_limit - n_steps + 50) / episode_limit) ** 6

        for agent_id in range(n_agents):
            next_state = my_get_state(agent_id)
            next_states.append(next_state)

        for agent_id in range(n_agents):
            learn(states[agent_id], next_states[agent_id], sum(rewards), joint_action)

            if (states[0] == states[1]) and (next_states[0] == next_states[1]):
                break

        episode_reward1 += rewards[0]
        episode_reward2 += rewards[1]
        episode_reward += sum(rewards)
        n_steps += 1

    ep_reward.append(episode_reward)
    total_steps.append(n_steps)
    ep_reward_agent1.append(episode_reward1)
    ep_reward_agent2.append(episode_reward2)

    if not e % 10:
        epsilon_values.append(epsilon)
        mean_ep_reward.append(np.mean(ep_reward[-10:]))
        mean_total_steps.append(np.mean(total_steps[-10:]))
        mean_ep_reward_agent1.append(np.mean(ep_reward_agent1[-10:]))
        mean_ep_reward_agent2.append(np.mean(ep_reward_agent2[-10:]))

    if (not e % 100) and (e != 0):
        with open(f'2v2_jal_ep{e}.pkl', 'wb') as f:
            pickle.dump(q_table, f)

    game_stats = env.get_stats()
    print()
    print('Episode ', e)
    print('Steps: {}   Reward: {}'.format(n_steps, round(episode_reward, 3)))
    print('Won: {}    Played: {}    Win rate: {}'.format(game_stats['battles_won'],
                                                         game_stats['battles_game'],
                                                         round(game_stats['win_rate'], 3)))

with open("2v2_jal_final.pkl", 'wb') as f:
    pickle.dump(q_table, f)

x = np.linspace(0, n_episodes, n_episodes // 10)

with open(f"{method_name}_plot_mean_ep_reward_agent1", 'wb') as f:
    pickle.dump(mean_ep_reward_agent1, f)

with open(f"{method_name}_plot_mean_ep_reward_agent2", 'wb') as f:
    pickle.dump(mean_ep_reward_agent2, f)

with open(f"{method_name}_plot_mean_ep_reward", 'wb') as f:
    pickle.dump(mean_ep_reward, f)

with open(f"{method_name}_plot_mean_total_steps", 'wb') as f:
    pickle.dump(mean_total_steps, f)

with open(f"{method_name}_plot_x", 'wb') as f:
    pickle.dump(x, f)

fig, ax = plt.subplots(1, 3)
ax1, ax2, ax3 = ax.flatten()

ax1.plot(x, epsilon_values)
ax1.set_title('Epsilon')
ax2.plot(x, mean_ep_reward)
ax2.set_title('Rewards')
ax3.plot(x, mean_total_steps)
ax3.set_title('Steps')
fig.set_size_inches(15, 4)
fig.subplots_adjust(hspace=0.2, wspace=0.2)
plt.show()

env.close()


class Solution:
    def containsNearbyAlmostDuplicate(self, nums: List[int], k: int, t: int) -> bool:
        n = len(nums)
        if t == 0 and n == len(set(nums)):
            return False

        for i in range(n):
            a = nums[i]
            for j in range(i + 1, i + 1 + k):
                if j >= n:
                    break
                if abs(a - nums[j]) <= t:
                    return True
        return False


from django.urls import path

from . import views

app_name="panels"

urlpatterns = [
    path('<slug:slug>/<article_type>/', views.PanelArticleView.as_view(), name='article_detail'),
    path('<slug:slug>/', views.PanelDetailView.as_view(), name='panel_detail'),
]


'''
'''
from datetime import date
class Employee:
	def __init__(self, name, age, salary, employment_year):
		self.name = name
		self.age = age
		self.salary = salary
		self.employment_year = employment_year

	def get_working_years(self):
		current_year = date.today().year
		working_years = current_year - int(self.employment_year)
		return working_years
		# return date.today().year - self.employment_year

	def __str__(self):
		return f"Name: {self.name}, Age:{self.age}, Salary:{self.salary}, Working Years: {self.get_working_years()}"

class Manager(Employee):
	def __init__(self, name, age, salary, employment_year, bonus):
		super().__init__(name, age, salary, employment_year)
		self.bonus = bonus

	def get_bounus(self):
		return float(self.bonus) * float(self.salary)

	def __str__(self):
		return f"Name: {self.name}, Age:{self.age}, Salary:{self.salary}, Working Years{self.get_working_years()}, Bounus: {self.get_bounus()}"

employees_list = []
managers_list = []

print("Welcome to HR Pro 2020")
print("Options:")
print("\t1. Show Employees")
print("\t2. Show Managers")
print("\t3. Add An Employee")
print("\t4.Add A Manager")
print("\t5.Exit")
print("\t6. Go crazy")
print()
choice = int(input("What would you like to do:"))

while choice != 5:
	if choice == 1:
		for x in employees_list:
			print(x)
	elif choice == 2:
		for y in managers_list:
			print(y)
	elif choice == 3:
		name = input("Name: ")
		age = input("Age: ")
		salary = input("Salary: ")
		employment_year = input("Employment Year: ")
		employee_obj = Employee(name , age, salary, employment_year)
		employees_list.append(employee_obj)

	elif choice == 4:
		name = input("Name: ")
		age = input("Age: ")
		salary = input("Salary: ")
		employment_year = input("Employment Year: ")
		bonus = input("Bounus Percentage:")
		manager_obj = Manager(name , age, salary, employment_year, bonus)
		managers_list.append(manager_obj)
	print("Welcome to HR Pro 2020")
	print("Options:")
	print("\t1. Show Employees")
	print("\t2. Show Managers")
	print("\t3. Add An Employee")
	print("\t4.Add A Manager")
	print("\t5.Exit")
	print("\t6. Go crazy")
	print()
	choice = int(input("What would you like to do:"))


import requests
from flightsio import proxies, constants
from unittest.mock import MagicMock

# For testing purposes set the number of seconds to wait between retries to 0.
# there's no reason to wait while testing.
proxies.DEFAULT_RETRY_INTERVAL = 0

def test_proxy_request_200_status_code(mocker):

    mock_proxy_gen, mock_get = setup_mocks(mocker, requests.codes.ok)

    proxy_request = proxies.ProxyRequest()

    target_url = 'https://find.me'
    response = proxy_request.get(target_url)

    mock_get.assert_called_once()

def test_proxy_request_200_includes_http_arg(mocker):

    mock_proxy_gen, mock_get = setup_mocks(mocker, requests.codes.ok)

    proxy_request = proxies.ProxyRequest()
    proxy_request._current_ip = '94.45.153.229:60784'

    target_url = 'https://find.me'
    response = proxy_request.get(target_url)

    args, kwargs = mock_get.call_args
    assert kwargs['proxies']['http'] == proxy_request._current_ip

def test_proxy_request_200_includes_user_agen(mocker):

    mock_proxy_gen, mock_get = setup_mocks(mocker, requests.codes.ok)

    mock_user_agent = mocker.patch('flightsio.proxies.ua')

    proxy_request = proxies.ProxyRequest()
    proxy_request._current_ip = '94.45.153.229:60784'

    target_url = 'https://find.me'
    response = proxy_request.get(target_url)

    args, kwargs = mock_get.call_args
    assert kwargs['headers']['User-Agent'] == mock_user_agent.random

def test_proxy_request_403_retries_request(mocker):
    """
    A 403 forbidden, means that the site has completely block an ip. The url should
    be retried a specific number of times before giving up.
    """

    mock_proxy_gen, mock_get = setup_mocks(mocker, requests.codes.forbidden)

    proxy_request = proxies.ProxyRequest()

    target_url = 'https://find.me'
    response = proxy_request.get(target_url, 3)

    # The total call count should be the max number of retries, plus the original.
    assert mock_get.call_count == 4

def test_proxy_request_403_generates_new_ip(mocker):
    """
    With 403 forbidden, generate a new IP for the subsequent request.
    """

    mock_proxy_gen, mock_get = setup_mocks(mocker, requests.codes.forbidden)

    proxy_request = proxies.ProxyRequest()
    proxy_request._reset_ip = MagicMock()

    target_url = 'https://find.me'
    response = proxy_request.get(target_url, 3)

    # The number of ips generated should equal the max number of retires.
    assert proxy_request._reset_ip.call_count == 3

def test_proxy_request_reset_ip(mocker):
    """
    Validates that a new ip address is requested when the reset_ip method is invoked.
    This is an internal method and it will be called whenever a request responds with
    a 403 status code.
    """

    new_ip = '165.90.66.230:58667'
    mock_proxy_gen = mocker.patch('flightsio.proxies.SslIpGenerator')
    mock_proxy_gen.return_value.get_ip = MagicMock(return_value=new_ip)

    request = proxies.ProxyRequest()
    request._current_ip = 'old:ip'

    request._reset_ip()

    assert request._current_ip == new_ip

def setup_mocks(mocker, status_code):

    mock_proxy_gen = mocker.patch('flightsio.proxies.SslIpGenerator')
    mock_get = mocker.patch('flightsio.proxies.requests.get')
    mock_get.return_value = MagicMock(status_code=status_code)

    return (mock_proxy_gen, mock_get)

def test_proxy_gen(mocker):

    mock_content = read_artifact('sample_proxies.html')
    mock_get = mocker.patch('flightsio.proxies.requests.get')
    mock_get.return_value = MagicMock(status_code=requests.codes.ok, content=mock_content)

    proxy_generator = proxies.SslIpGenerator()
    proxy_generator.load_proxies()

    ips = set([proxy_generator.get_ip() for i in range(100)])
    # The first three ip addresses in the sample proxies file are in the set of all ips
    assert {'87.249.19.154:48996', '1.20.100.183:40424', '125.26.108.12:30540'} < ips

def read_artifact(name):

    with open(f'./html/{name}', 'r') as f:
        return f.read()


from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import random
import numpy as np

a=np.arange(-10, 10, 0.1)
b=np.arange(-10, 10, 0.1)
X=np.arange(-10, 10, 0.1)

s=np.std(X)
N=np.var(s)
u=np.random.normal(0,1,200)
Y=2*X+3+(N)*(u)

n=100

for i in range(0,200):
    E1=((Y-a*X[i]+b)**2)
    i+=1
    E2=E1/n

fig = plt.figure()
ax=fig.gca(projection='3d')

a, b = np.meshgrid(a, b)
E2 = np.array([E1 for X,Y in zip(np.ravel(X), np.ravel(Y))])
E = E2.reshape(a.shape)

ax.plot_surface(a, b, E,cmap='plasma')

ax.set_xlabel('X-AXIS')
ax.set_ylabel('Y-AXIS')
ax.set_zlabel('ERROR')


# See https://stackoverflow.com/a/19521297/3187068
import matplotlib
matplotlib.use('pdf')
font = {'size': 16}
matplotlib.rc('font', **font)

import argparse
import itertools
import matplotlib.pyplot as plt
import os.path

MARKERS = itertools.cycle(['o', '*', '^', 's', 'P', 'x', '1'])

def plot_one_throughput_vs_num_replicas(args) -> None:
    fig, ax = plt.subplots(1, 1, figsize=(1 * 6.4, 4.8))

    for fw in [0, 0.01, 0.02, 0.05, 0.1, 0.25, 1]:
        fr = 1 - fw
        ns = list(range(2, 31))
        throughputs = [(n * args.alpha) / (n*fw + fr) / 1000000 for n in ns]
        ax.plot(ns, throughputs, '.-', marker=next(MARKERS),
                label=f'{int(fr * 100)}% reads')

    ax.set_xlabel('Number of replicas')
    ax.grid()
    ax.set_ylabel('Peak throughput\n(millions of commands per second)')
    ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
    output_filename = os.path.join(args.output_dir,
                                   'theory_one_tput_vs_replicas.pdf')
    fig.savefig(output_filename, bbox_inches='tight')
    print(f'Wrote plot to {output_filename}.')

def plot_throughput_vs_num_replicas(args) -> None:
    fig, ax = plt.subplots(1, 3, figsize=(3 * 6.4, 4.8))

    for fw in [0, 0.01, 0.02, 0.05, 0.1, 0.25, 0.5, 1]:
        fr = 1 - fw
        ns = list(range(2, 31))
        throughputs = [(n * args.alpha) / (n*fw + fr) for n in ns]
        write_throughputs = [fw * t for t in throughputs]
        read_throughputs = [fr * t for t in throughputs]
        ax[0].plot(ns, throughputs, '.-', label=f'{int(fw * 100)}% writes')
        ax[1].plot(ns, write_throughputs, '.-')
        ax[2].plot(ns, read_throughputs, '.-')

    for a in ax:
        a.set_xlabel('Number of replicas')
        a.grid()
    ax[0].set_ylabel('Peak throughput')
    ax[1].set_ylabel('Peak write throughput')
    ax[2].set_ylabel('Peak read throughput')
    ax[0].legend(loc='best')
    output_filename = os.path.join(args.output_dir, 'theory_tput_vs_replicas.pdf')
    fig.savefig(output_filename, bbox_inches='tight')
    print(f'Wrote plot to {output_filename}.')

def plot_throughput_vs_write_ratio(args) -> None:
    fig, ax = plt.subplots(1, 3, figsize=(3 * 6.4, 4.8))

    fws = [i / 100 for i in range(0, 100, 2)] + [1]
    frs = [1 - fw for fw in fws]
    for n in [1, 2, 3, 4, 5, 6, 7, 8, 9]:
        throughputs = [(n * args.alpha) / (n*fw + fr) for (fw, fr) in zip(fws, frs)]
        write_throughputs = [fw * t for (fw, t) in zip(fws, throughputs)]
        read_throughputs = [fr * t for (fr, t) in zip(frs, throughputs)]
        ax[0].plot(frs, throughputs, '.-', label=f'{n} replicas')
        ax[1].plot(frs, write_throughputs, '.-')
        ax[2].plot(frs, read_throughputs, '.-')

    fws = [i / 100 for i in range(10, 100, 2)] + [1]
    frs = [1 - fw for fw in fws]
    throughputs = [args.alpha / fw for fw in fws]
    write_throughputs = [fw * t for (fw, t) in zip(fws, throughputs)]
    read_throughputs = [fr * t for (fr, t) in zip(frs, throughputs)]
    ax[0].plot(frs, throughputs, '.-', label='infinite replicas')
    ax[1].plot(frs, write_throughputs, '.-')
    ax[2].plot(frs, read_throughputs, '.-')

    for a in ax:
        a.set_xlabel('Read fraction')
        a.grid()
    ax[0].set_ylabel('Peak throughput')
    ax[1].set_ylabel('Peak write throughput')
    ax[2].set_ylabel('Peak read throughput')
    ax[0].legend(loc='best')
    output_filename = os.path.join(args.output_dir, 'theory_tput_vs_fraction.pdf')
    fig.savefig(output_filename, bbox_inches='tight')
    print(f'Wrote plot to {output_filename}.')

def plot_nice_throughput_vs_num_replicas(args) -> None:
    fig, ax = plt.subplots(2, 1, figsize=(6.4, 2 * 4.8))

    for alpha_w in [f*args.alpha for f in [0, 0.1, 0.2, 0.3, 0.4, 0.5,
                                      0.75, 0.9, 0.95, 1]]:
        ns = list(range(2, 31))
        throughputs = [alpha_w + n*(args.alpha - alpha_w) for n in ns]
        write_throughputs = [alpha_w] * len(throughputs)
        read_throughputs = [t - alpha_w for t in throughputs]
        frs = [r / (r + w)
               for (r, w) in zip(read_throughputs, write_throughputs)]
        ax[0].plot(ns, throughputs, '.-', label=f'{int(alpha_w)} writes')
        ax[1].plot(ns, frs, '.-')

    for a in ax:
        a.set_xlabel('Number of replicas')
        a.grid()
    ax[0].set_ylabel('Peak throughput')
    ax[1].set_ylabel('Read fraction')
    ax[0].legend(loc='best')
    output_filename = os.path.join(args.output_dir,
                                   'theory_nice_tput_vs_replicas.pdf')
    fig.savefig(output_filename, bbox_inches='tight')
    print(f'Wrote plot to {output_filename}.')

def plot_nice_throughput_vs_writes(args) -> None:
    fig, ax = plt.subplots(1, 1, figsize=(6.4, 4.8))

    alpha_ws = [args.alpha * i / 50 for i in range(0, 51)]
    for n in [1, 2, 3, 4, 5, 6, 7, 8, 9]:
        throughputs = [w + n*(args.alpha - w) for w in alpha_ws]
        read_throughputs = [t - w for (t, w) in zip(throughputs, alpha_ws)]
        ax.plot(alpha_ws, throughputs, '.-', label=f'{n} replicas')

    ax.set_xlabel('Writes')
    ax.grid()
    ax.set_ylabel('Peak throughput')
    ax.legend(loc='best')
    output_filename = os.path.join(args.output_dir,
                                   'theory_nice_tput_vs_writes.pdf')
    fig.savefig(output_filename, bbox_inches='tight')
    print(f'Wrote plot to {output_filename}.')

def main(args) -> None:
    plot_one_throughput_vs_num_replicas(args)
    # plot_throughput_vs_num_replicas(args)
    # plot_throughput_vs_write_ratio(args)
    # plot_nice_throughput_vs_num_replicas(args)
    # plot_nice_throughput_vs_writes(args)

def get_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser()
    parser.add_argument('--output_dir',
                        type=str,
                        default='.',
                        help='Output directory.')
    parser.add_argument('--alpha',
                        type=int,
                        default=100000,
                        help='Peak throughput of a single server.')
    return parser

if __name__ == '__main__':
    parser = get_parser()
    main(parser.parse_args())


#!/usr/bin/python3

import getopt
import sys
import paho.mqtt.client as mqtt

from time import sleep

def usage():
    print("Usage: scan.py -h|-t <topic> -m <msg>")

def main():

    topic = ""
    msg = ""
    verbose = True
    host = "127.0.0.1"
    port = 1883

    try:
        opts, args = getopt.getopt(sys.argv[1:], "?t:m:p:h:",["help","topic","message","port","host"])
    except getopt.GetoptError as err:
        print(err)
        sys.exit(2)

    for o,a in opts:
        if o in ("-?","--help"):
            usage()
            sys.exit(0)
        elif o in ("-t","--topic"):
            topic = a
        elif o in ("-m","--message"):
            msg = a
        elif o in ("-h","--host"):
            host = a
        elif o in ("-p","--port"):
            port = int(a)

    if topic == "" or msg == "":
        usage()
        sys.exit(1)

    if verbose:
        print("Topic:" + topic)
        print("Msg  :" + msg)
        print("Host :" + host)
        print("Port :", port)

    client = mqtt.Client()
    client.connect(host,port,60)

    for t in topic.split():
        print(t)
        print(msg)

        client.publish(t,msg,qos=0,retain=True)
        sleep(0.1)

main()


def queue1(command):
    global first1
    global last1
    global size1
    if command.startswith('push'):
        value = int(command[5:])
        if size1 == 0:
            first1 = [value, None, None]
            last1 = first1
        elif size1 == 1:
            item = [value, first1, None]
            first1[2] = item
            last1 = item
        else:
            item = [value, last1, None]
            last1[2] = item
            last1 = item
        size1 += 1
    if command.startswith('pop'):
        if size1 == 1:
            item = None
            first1 = item
            last1 = item
            size1 = 0
        elif size1 == 2:
            first1[2][1] = None
            first1 = first1[2]
            last1 = first1
            size1 = 1
        else:
            first1[2][1] = None
            first1 = first1[2]
            size1 -= 1
    if command.startswith('front'):
        return first1[0]

def queue2(command):
    global first2
    global last2
    global size2
    if command.startswith('push'):
        value = int(command[5:])
        if size2 == 0:
            first2 = [value, None, None]
            last2 = first2
        elif size2 == 1:
            item = [value, first2, None]
            first2[2] = item
            last2 = item
        else:
            item = [value, last2, None]
            last2[2] = item
            last2 = item
        size2 += 1
    if command.startswith('pop'):
        if size2 == 1:
            item = None
            first2 = item
            last2 = item
            size2 = 0
        elif size2 == 2:
            first2[2][1] = None
            first2 = first2[2]
            last2 = first2
            size2 = 1
        else:
            first2[2][1] = None
            first2 = first2[2]
            size2 -= 1
    if command.startswith('front'):
        return first2[0]

first1 = None
last1 = None
size1 = 0
first2 = None
last2 = None
size2 = 0
a = list(map(int, input().split()))
b = list(map(int, input().split()))
for i in range(len(a)):
    queue1('push ' + str(a[i]))
for j in range(len(b)):
    queue2('push ' + str(b[j]))
k = 0
while size1 != 0 and size2 != 0:
    k += 1
    x = queue1('front')
    y = queue2('front')
    queue1('pop')
    queue2('pop')
    if x > y and (y, x) != (0, 9) or (x, y) == (0, 9):
        queue1('push ' + str(x))
        queue1('push ' + str(y))
    else:
        queue2('push ' + str(x))
        queue2('push ' + str(y))
    if k == 1000000:
        print('botva')
        break

if size1 != 0 and size2 == 0:
    print('first', k)
if size2 != 0 and size1 == 0:
    print('second', k)


#!/usr/bin/env python2.7 -W ignore::DeprecationWarning
# -*- coding: utf-8 -*-
import os
import commands
from os.path import join, exists, isdir, dirname, basename, split
import shutil
from multiprocessing import Process
import pandas as pd
import nibabel as nib
import numpy as np
import yaml

from utils import tools
from utils import resources as rsc

def create_simset_simplepet_detector(params_file, output):

    with open(params_file, 'rb') as f:
        params = yaml.load(f.read(), Loader=yaml.FullLoader)

    energy_resolution = params.get("energy_resolution")

    new_file = open(output, "w")

    new_file.write(
        "ENUM detector_type = simple_pet \n\n" +
        "REAL    reference_energy_keV = 511.0 \n" +
        "REAL    energy_resolution_percentage = %s \n" % energy_resolution
        )

    new_file.close()

def create_stir_hs_from_detparams(params_file,output_file, create_projdata_bin, output_format="SimSET"):

    with open(params_file, 'rb') as f:
        params = yaml.load(f.read(), Loader=yaml.FullLoader)

    num_rings = params.get("num_rings")
    max_segment = params.get("max_segment")
    max_z = params.get("max_z")
    min_z = params.get("min_z")
    z_crystal_size = params.get("z_crystal_size")
    gap_size = (max_z-min_z-z_crystal_size*num_rings)/(num_rings-1)
    ring_spacing = z_crystal_size + gap_size

    max_td = params.get("max_td")
    min_td = params.get("min_td")
    td_bins = params.get("num_td_bins")
    bin_size = (max_td-min_td)/float(td_bins)
    matrix_size, ring_difference = generate_segments_lists_stir(num_rings, max_segment)

    if output_format=="SimSET":
        views_coordinate = 2
        axial_coordinate = 3
    else:
        views_coordinate = 3
        axial_coordinate = 2

    new_file = open(output_file, "w")
    new_file.write(
        "!INTERFILE  :=\n" +
        "!imaging modality := PT\n" +
        "name of data file := " + output_file[0:-2] + "s" + "\n" +
        "originating system := " + params.get("scanner_name") + "\n" +
        "!version of keys := STIR3.0\n" +
        "!GENERAL DATA :=\n" +
        "!GENERAL IMAGE DATA :=\n" +
        "!type of data := PET\n" +
        "imagedata byte order := LITTLEENDIAN\n" +
        "!PET STUDY (General) :=\n" +
        "!PET data type := Emission\n" +
        "applied corrections := {None}\n" +
        "!number format := float\n" +
        "!number of bytes per pixel := 4\n" +
        "number of dimensions := 4\n" +
        "matrix axis label [4] := segment\n"
        "!matrix size [4] := " + str(2*max_segment+1) + "\n" +
        "matrix axis label [" + str(views_coordinate) +"] := view\n"
        "!matrix size [" + str(views_coordinate) +"] :=" + str(params.get("num_aa_bins")) + "\n" +
        "matrix axis label ["+ str(axial_coordinate) +"] := axial coordinate\n" +
        "!matrix size [" + str(axial_coordinate) + "] := " + matrix_size + "\n" +
        "matrix axis label [1] := tangential coordinate\n" +
        "!matrix size [1] :=" + str(td_bins) + "\n" +
        "minimum ring difference per segment := " + ring_difference + "\n" +
        "maximum ring difference per segment := " + ring_difference + "\n" +
        "Scanner parameters:= \n" +
        "Scanner type := " + params.get("scanner_name") + "\n" +
        "Number of rings := " + str(num_rings) + "\n" +
        "Number of detectors per ring := " + str(params.get("num_aa_bins")*2) + "\n" +
        "Inner ring diameter (cm) := " + str(params.get("radio_scanner")*2) + "\n" +
        "Average depth of interaction (cm) := " + str(params.get("average_doi")) + "\n" +
        "Distance between rings (cm) := " + str(ring_spacing) + "\n" +
        "Default bin size (cm) := " + str(bin_size) + "\n" +
        "View offset (degrees) := 0\n" +
        "Maximum number of non-arc-corrected bins := " + str(td_bins) + "\n" +
        "Default number of arc-corrected bins := " + str(td_bins) + "\n" +
        "Energy_resolution := " + str(params.get("energy_resolution")*5.11) + "\n" +
        "Reference energy (in keV) := 511\n" +
        "Number of blocks per bucket in transaxial direction := 1\n" +
        "Number of blocks per bucket in axial direction := 1\n" +
        "Number of crystals per block in axial direction := 1\n" +
        "Number of crystals per block in transaxial direction := 1\n" +
        "Number of crystals per singles unit in axial direction := 1\n" +
        "Number of crystals per singles unit in transaxial direction := 1\n" +
        "end scanner parameters:=\n" +
        "effective central bin size (cm) := " + str(bin_size) + "\n" +
        "number of time frames := 1\n" +
        "start vertical bed position (mm) := 0\n" +
        "start horizontal bed position (mm) := 0\n" +
        "!END OF INTERFILE :=\n"
        )

def generate_segments_lists_stir(nrings, max_segment):

    last_segment_sinograms = nrings-max_segment
    my_matrix_size = " "
    my_matrix_ring_difference = " "
    for i in range(last_segment_sinograms, nrings):
        my_matrix_size = my_matrix_size + str(i) + ","
        my_matrix_ring_difference = my_matrix_ring_difference + str(i-nrings) + ","
    for i in range(nrings, last_segment_sinograms-1,-1):
        my_matrix_size = my_matrix_size + str(i) + ","
        my_matrix_ring_difference = my_matrix_ring_difference + str(nrings-i) + ","

    my_matrix_size = "{" + my_matrix_size [0:-1] + "}"
    my_matrix_ring_difference = "{" + my_matrix_ring_difference [0:-1] + "}"

    return my_matrix_size, my_matrix_ring_difference


class Solution:
    def exist(self, board, word):
        rows = len(board)
        cols = len(board[0])
        # print(board[1][0])
        def dfs(i,j,board, word):
            if len(word) == 0:  # all the characters are checked
                return True
            if i < 0 or j < 0 or i >= rows or j >= cols or board[i][j] != word[0]:
                return False
            tmp = board[i][j]
            board[i][j] = "#"   # 用1个变量，来标记已访问
            res = dfs(i+1,j,board,word[1:]) or dfs(i-1,j,board,word[1:]) or dfs(i,j+1,board,word[1:]) or dfs(i,j-1,board,word[1:])
            board[i][j] = tmp  # 回溯，恢复状态
            return res

        for row in range(rows):
            for col in range(cols):
                res = dfs(row, col, board, word)
                if res:
                    return True
        return False

matrix = [['A','B','C','E'],
          ['S','F','C','S'],
          ['A','D','E','E']]

print(Solution().exist(matrix, "ABC"))


# coding: utf-8

# In[126]:

from graphviz import Digraph
import tempfile
import collections, sys
from Bio import Seq, SeqIO, SeqRecord
from copy import deepcopy

# In[127]:

def read_data(filename):
    seq = []
    reads = SeqIO.parse(filename,'fastq')
    for read in reads:
        seq_s = str(read.seq)
        seq_l = seq_s.split('N')
        seq.append(seq_s)
    return seq

# In[128]:

def dummy_data():
    return ["ACGTCCGTAA"]

# In[129]:

def build_graph(nodeA, nodeB, edges_dict, edges_data, has_parent_nodes, has_child_nodes, child_dict, parent_dict):

    if nodeA in edges_dict:
        edges_dict[nodeA].add(nodeB)
    else:
        edges_dict[nodeA] = set([nodeB])

    data_key = (nodeA, nodeB)
    if data_key in edges_data:
        data = edges_data[data_key]
        data[0] += 1
        edges_data[data_key] = data
    else:
        edges_data[data_key] = [1.0, 1.0]

    has_parent_nodes.add(nodeB)
    has_child_nodes.add(nodeA)
    if nodeA in child_dict:
        child_dict[nodeA].add(nodeB)
    else:
        child_dict[nodeA] = set([nodeB])
    if nodeB in parent_dict:
        parent_dict[nodeB].add(nodeA)
    else:
        parent_dict[nodeB] = set([nodeA])

# In[130]:

def run_de_bruijn(k, seq, nodes, edges_dict, edges_data, has_parent_nodes, has_child_nodes, child_dict, parent_dict):
    for reading in seq:
        for i in range(len(reading) - k + 1):
            nodeA = reading[i:i+k-1]
            nodeB = reading[i+1:i+k]
            nodes.add(nodeA)
            nodes.add(nodeB)
            build_graph(nodeA, nodeB, edges_dict, edges_data, has_parent_nodes, has_child_nodes, child_dict, parent_dict)

# In[131]:

def show_graph(edges_data, name, should_save = False):
    dot = Digraph(comment=name)
    dot.graph_attr['rankdir'] = 'LR'
    for edge, data in edges_data.items():
        label_value = str(data[0]) + ", " + str(data[1])
        dot.edge(edge[0], edge[1], label=label_value)
    dot.view(tempfile.mktemp('.gv'))
    if should_save:
        filename = name + ".gv"
        dot.render(filename, view=False)

# In[169]:

def save_graph(edges_data, name):
    dot = Digraph(comment=name)
    dot.graph_attr['rankdir'] = 'LR'
    for edge, data in edges_data.items():
        label_value = str(data[0]) + ", " + str(data[1])
        dot.edge(edge[0], edge[1], label=label_value)
    file_name = name + "-graph.dot"
    dot.save(filename=file_name)
    dot.render(file_name, view=False)
    print("Result saved as: " + str(file_name))

# In[170]:

def save_edges(edges_data, name):
    file_name = name + "-edges.fastq"
    with open(file_name, 'w') as file:
        for edge, data in edges_data.items():
            line = edge[0] + ", " + edge[1] + "\n"
            file.write(line)
    print("Edges saved as: " + str(file_name))

# In[133]:

def is_condense(child, child_dict, parent_dict):
    return child in child_dict and len(child_dict[child]) == 1 and child in parent_dict and len(parent_dict[child]) == 1

# In[134]:

def merge(head, end, merged, visited, edges_data, edges_dict, child_dict, parent_dict):
    nodes = [head] + merged + [end]
    visited.update(merged)

    length_sum = 0
    for i in range(len(nodes) - 1):
        edge_key = (nodes[i], nodes[i+1])
        if edge_key in edges_data:
            data = edges_data.pop(edge_key)
            length_sum += data[0]

    edges_data[(head, end)] = [1.0, length_sum / len(merged)]
    edges_dict[head].remove(merged[0])
    edges_dict[head].add(end)

    child_dict[head].remove(merged[0])
    child_dict[head].add(end)
    parent_dict[end].remove(merged[-1])
    parent_dict[end].add(head)

    for node in merged:
        edges_dict.pop(node)
        child_dict.pop(node)
        parent_dict.pop(node)

# In[135]:

def do_condense(nodes, edges_data, edges_dict, child_dict, parent_dict, has_parent_nodes):
    visited = set()
    to_be_visit = nodes - has_parent_nodes

    while (len(to_be_visit) != 0):
        node = to_be_visit.pop()
        if node in visited:
            continue;
        visited.add(node)
#         print("visiting: " + str(node))
        if node not in edges_dict:
#             print("reached end!")
            continue;
        childs = deepcopy(edges_dict[node])
        for child in childs:
#             print("child: " + str(child))
            if child in visited:
                continue
            should_merge = False
            to_be_merged = []
            while (is_condense(child, child_dict, parent_dict)):
#                 print("i am here")
                should_merge = True
                to_be_merged.append(child)
                child = list(child_dict[child])[0]
            if should_merge:
                head = node
                end = child
#                 print(to_be_merged)
                merge(head, end, to_be_merged, visited, edges_data, edges_dict, child_dict, parent_dict)
#             else:
#                 print("dont merge")
            to_be_visit.add(child)

# In[164]:

def main(seq, k, need_display=False, result_filename="test"):
    edges_dict = {}
    edges_data = {}
    edges = []
    nodes = set()
    parent_dict = {}
    has_parent_nodes = set()
    child_dict = {}
    has_child_nodes = set()

    print("Building De Bruijn Graph ...")
    run_de_bruijn(k, seq, nodes, edges_dict, edges_data, has_parent_nodes, has_child_nodes, child_dict, parent_dict)
    if need_display:
        show_graph(edges_data, "before")

    print("Condensing De Bruijn Graph ...")
    do_condense(nodes, edges_data, edges_dict, child_dict, parent_dict, has_parent_nodes)
    if need_display:
        show_graph(edges_data, "after")

    print("Saving Edges ...")
    save_edges(edges_data, result_filename)

    print("Saving Graph ...")
    save_graph(edges_data, result_filename)

# In[165]:

def run_simple_example():
    k = 3
    seq = dummy_data()
    main(seq, k, True)

# In[151]:

def run_real_data():
    k = 55
    seq = read_data("s_6.first100000.fastq")
    main(seq, k, False)

# In[152]:

def run():
    filename = sys.argv[1]
    k = int(sys.argv[2])
    print("Reading Data ...")
    seq = read_data(filename)
    main(seq, k, False, filename)

# In[167]:

# run_simple_example()

# In[168]:

# run_real_data()

# In[ ]:

if __name__ == "__main__":
    run()


import argparse
import typing
from os import path
from pathlib import Path, PurePath
from zipfile import ZipFile, is_zipfile

import shtab
import yaml
from yaml import YAMLError

from opera.error import DataError, ParseError, OperaError
from opera.parser import tosca
from opera.parser.tosca.csar import CloudServiceArchive
from opera.storage import Storage

def add_parser(subparsers):
    parser = subparsers.add_parser(
        "init",
        help="Initialize the deployment environment for the TOSCA service template or CSAR"
    )
    parser.add_argument(
        "--instance-path", "-p",
        help="Storage folder location (instead of default .opera)"
    ).complete = shtab.DIR
    parser.add_argument(
        "--inputs", "-i", type=argparse.FileType("r"),
        help="YAML or JSON file with inputs",
    ).complete = shtab.FILE
    parser.add_argument(
        "--clean", "-c", action="store_true",
        help="Clean storage by removing previously initialized TOSCA service template or CSAR",
    )
    parser.add_argument(
        "--verbose", "-v", action="store_true",
        help="Turns on verbose mode",
    )
    parser.add_argument(
        "csar", type=argparse.FileType("r"),
        help="TOSCA YAML service template file or CSAR"
    ).complete = shtab.FILE
    parser.set_defaults(func=_parser_callback)

def _parser_callback(args):
    print("Warning: 'opera init' command is deprecated and will probably be removed within one of the next releases. "
          "Please use 'opera deploy' to initialize and deploy service templates or compressed CSARs.")
    if args.instance_path and not path.isdir(args.instance_path):
        raise argparse.ArgumentTypeError("Directory {} is not a valid path!".format(args.instance_path))

    storage = Storage.create(args.instance_path)
    try:
        inputs = yaml.safe_load(args.inputs) if args.inputs else {}
    except YAMLError as e:
        print("Invalid inputs: {}".format(e))
        return 1

    try:
        if is_zipfile(args.csar.name):
            init_compressed_csar(args.csar.name, inputs, storage, args.clean)
            print("CSAR was initialized")
        else:
            init_service_template(args.csar.name, inputs, storage, args.clean)
            print("Service template was initialized")
    except ParseError as e:
        print("{}: {}".format(e.loc, e))
        return 1
    except DataError as e:
        print(str(e))
        return 1
    except OperaError as e:
        print("Invalid CSAR: {}".format(e))
        return 1

    return 0

def init_compressed_csar(csar_name: str, inputs: typing.Optional[dict], storage: Storage, clean_storage: bool):
    if storage.exists("root_file"):
        if clean_storage:
            storage.remove_all()
        else:
            print("Looks like service template or CSAR has already been initialized. "
                  "Use the --clean/-c flag to clear the storage.")
            return

    if inputs is None:
        inputs = {}
    storage.write_json(inputs, "inputs")

    csars_dir = Path(storage.path) / "csars"
    csars_dir.mkdir(exist_ok=True)

    csar = CloudServiceArchive.create(PurePath(csar_name))
    csar.validate_csar()
    tosca_service_template = csar.get_entrypoint()

    # unzip csar and save the path to storage
    csar_dir = csars_dir / Path("csar")
    ZipFile(csar_name, "r").extractall(csar_dir)
    csar_tosca_service_template_path = csar_dir / tosca_service_template
    storage.write(str(csar_tosca_service_template_path), "root_file")

    # try to initiate service template from csar
    ast = tosca.load(Path(csar_dir), Path(tosca_service_template))
    template = ast.get_template(inputs)
    template.instantiate(storage)

def init_service_template(service_template: str, inputs: typing.Optional[dict],
                          storage: Storage, clean_storage: bool):
    if storage.exists("root_file"):
        if clean_storage:
            storage.remove_all()
        else:
            print("Looks like service template or CSAR has already been initialized. "
                  "Use --clean/-c flag to clear the storage.")
            return

    if inputs is None:
        inputs = {}
    storage.write_json(inputs, "inputs")
    storage.write(service_template, "root_file")

    ast = tosca.load(Path.cwd(), PurePath(service_template))
    template = ast.get_template(inputs)
    template.instantiate(storage)


import sqlite3

from exceptions.zipcode import Zip_codesCreationError

class ZipcodesService:
    """Сервис взаимодействия с таблицей zipcode"""
    def __init__(self, connection):
        self.connection = connection

    def _check_existence(self, zip_code: str) -> bool:
        """Проверка существования индекса в базе"""
        query = (
            """
            SELECT *
            FROM zipcode
            WHERE zip_code = ?
            """
        )
        params = (zip_code,)

        cursor = self.connection.execute(query, params)
        if cursor.fetchone() is not None:
            return True
        return False

    def create(self, zip_code_data: dict):
        """Создание новой связи в базе"""
        zip_code = zip_code_data["zip_code"]
        city_id = zip_code_data["city_id"]

        query = (
            """
            INSERT INTO zipcode (zip_code, city_id) VALUES (?, ?)
            """
        )

        params = (
            zip_code,
            city_id,
        )

        if self._check_existence(zip_code):
            return

        try:
            self.connection.execute(query, params)
            self.connection.commit()
        except sqlite3.IntegrityError:
            raise Zip_codesCreationError

    def update(self, data: dict):
        """Частичное редактирование записи в базе"""
        zip_code = data.get("zip_code")
        city_id = data.get("city_id")

        if not all((zip_code, city_id)):
            return

        if self._check_existence(zip_code):
            self.connection.execute(
                'UPDATE zipcode SET city_id = ? WHERE zip_code = ?',
                (city_id, zip_code)
            )

        else:
            self.create(
                {
                    "zip_code": zip_code,
                    "city_id": city_id,
                }
            )

Lesson 13/final v 2.0/src/services/zipcode.py


from dynabuffers.api.ISerializable import ISerializable, ByteBuffer
from dynabuffers.ast import FieldType
from dynabuffers.ast.structural import ClassOptions

class ClassTypeOptions:
    def __init__(self, name, fields:[FieldType], options:ClassOptions):
        self.name = name
        self.fields = fields
        self.options = options

class ClassType(ISerializable):

    def __init__(self, options: ClassTypeOptions):
        self.options = options

    def size(self, value, registry):
        byName = lambda x: (x.options.name in value) or x.options.defaultVal is not None
        mapper = lambda x: x.size(value[x.options.name] if x.options.name in value else x.options.defaultVal, registry)

        return sum(map(mapper, filter(byName, self.options.fields)))

    def serialize(self, value, buffer: ByteBuffer, registry):
        if self.options.options.isDeprecated():
            registry.addNotification("deprecated class " + self.options.name + " used")

        for field in self.options.fields:
            if (field.options.name not in value) and field.options.defaultVal is None:
                raise ValueError("field " + str(field.options.name) + " is missing")

        for element in list(filter(lambda x: (x.options.name in value) or x.options.defaultVal is not None, self.options.fields)):
            element.serialize(value[element.options.name] if element.options.name in value else element.options.defaultVal, buffer, registry)

    def deserialize(self, buffer, registry):
        map = {}
        for field in self.options.fields:
            map[field.options.name] = field.deserialize(buffer, registry)

        return map


from Fichas.Ficha8.Node import Node

class LinkedList:
    def __init__(self):
        self.head = None
        self.lenght=0

    def isEmpty(self):
        self.head = None

    def add(self, item):
        temp = Node(item)

        temp.setNext(self.head)
        self.head = temp

    def traversal(self):
        current = self.head
        while current != None:
            current = current.getNext()

    # def size(self):
    #     current=self.head
    #     count=0
    #     while current!=None:
    #         count+=1
    #         current=current.getNext()
    #     return count

    # ficha 8, exer 1:
    def size(self):
        return self.lenght

    def search(self, item):
        current = self.head
        found = False
        while current != None and not found:
            if current.getData() == item:
                found = True
            else:
                current = current.getNext()
        return found

    def remove(self, item):
        current = self.head
        previous = None
        found = False
        while not found:
            if current.getData() == item:
                found = True
            else:
                previous = current
                current = current.getNext()
        if previous == None:
            self.head = current.getNext()
        else:
            previous.setNext(current.getNext())

    def __str__(self):  # return string representation of a linked list
        current = self.head
        s = ""
        while current != None:
            s = s + str(current.getData())
            current = current.getNext()
            if current:
                s = s + " -> "
        return s

# lista=LinkedList()
# print(lista)
#
# lista.add(17)
# print(lista)
#
# lista.add(93)
# print(lista)
#
# lista.add(26)
# print(lista)
#
# print(lista.search(93))
# lista.remove(93)
# print(lista)


8# -*- coding: utf-8 -*-
"""
Created on Tue 26 Feb 16:05:01 2019

@author: heyu1
"""

import numpy as np, pandas as pd
from collections import deque
import re, requests
from bs4 import BeautifulSoup
isCSVinput = False
WIKI_LEN = len("/wiki/")
while True:
    choice = input("Import search terms from a .CSV (Y/N)?: ")
    if re.match(re.compile("^([YN]|Yes|No)$", re.I), choice) != None:
        if re.match(re.compile("^Y$", re.I), choice[0]) != None:
            isCSVinput = True
        break

enWikiRoot="https://en.wikipedia.org/wiki/"
def hasArticle(searchTerm):
    if searchTerm.strip() == "":
        return False
    reqAttempt = requests.get(enWikiRoot + searchTerm)
    return(reqAttempt.status_code != 404)

def findActualTitleHelper(soupFromPage):
    linkTitles=soupFromPage.find_all('a', title=True)
    titleText=[elem['title'] for elem in linkTitles]
    hasPerm=[re.search(re.compile("Permanent link"),x) != None for
             x in titleText]
    idxPerm=np.where(hasPerm)
#    assert(len(idxPerm) == 1)
    if len(idxPerm[0]) != 1:
        return None
    permLinkSuffix = linkTitles[idxPerm[0][0]]['href']
    charIdxTitle = re.search(re.compile("title="), permLinkSuffix).end()
    charIdxOldid = re.search(re.compile("&oldid="), permLinkSuffix).start()
    return permLinkSuffix[charIdxTitle:charIdxOldid]

def findActualTitle(s):
    page = requests.get(enWikiRoot + s)
    soupFromPage = BeautifulSoup(page.text, 'html.parser')
    res = findActualTitleHelper(soupFromPage)
    if res == None:
        print("THE FOLLOWING HAD NO PERMANENT REVISION LINK:", s)
    return res

def linksOnPage(searchTerm):
    page = requests.get(enWikiRoot + searchTerm)
    soupFromPage = BeautifulSoup(page.text, 'html.parser')
    aTags=soupFromPage.find_all('a',href=True)
    hrefs=[elem['href'] for elem in aTags]
    wikiRe = re.compile("^/wiki/")
    # exclude non-mainspace article pages on Wiki, i.e. those with the
    # following prefixes:
    prefixes = ['User','Wikipedia','File','MediaWiki','Template','Help',
                'Category','Portal','Book','Draft','Education Program',
                'TimedText','Module','Gadget','Gadget definition']
    prefixTalk = [s + ' talk' for s in prefixes]
    prefixesComplete = prefixes + prefixTalk
    prefixesComplete += ['Special','Talk','MOS']
    pipe = "|"
    regexPrefixes = pipe.join(prefixesComplete)
    # also exclude the Main Page, which changes daily and will lead to
    # unstable results
    nonArticleRe = re.compile("^/wiki/((" + regexPrefixes +
                                        ":)|Main_Page)", re.I)

    articleLinks = list(filter(lambda x: re.search(wikiRe, x) != None and
                               re.search(nonArticleRe, x) == None, hrefs))
    # "/wiki/" is 6 characters
    articles = [lk[WIKI_LEN:] for lk in articleLinks]
    uniqueArticles = set(articles)

    thisSearchTermActualTitle = findActualTitleHelper(soupFromPage)
    if thisSearchTermActualTitle in uniqueArticles:
        uniqueArticles.remove(thisSearchTermActualTitle)
    res = dict(zip(list(uniqueArticles), ["Linked from " + searchTerm
                                    for s in uniqueArticles]))
    return res

# Reasoning adapted from Wikipedia's article on Breadth First Search
def bfs(originTerm, targetArticle, verbose=False):
    futureVertices = deque()
    visited = set()
    pathDict = dict()
    distDict = dict()
    root = originTerm
    # key stores the parent node
    pathDict[root] = None
    distDict[root] = 0
    maxDist = 0
    futureVertices.appendleft(root)
    levels = [1E4, 1E5, 2.5E5, 5E5, 1E6, 1.5E6, 2E6, 2.5E6]
#    pd.DataFrame({'levels': levels, 'visited'})
    attained_levels = set()
#    b10K = False; b100K = False; b250K = False; b500K = False; b1M = False
    while len(futureVertices) > 0:
        subtreeRoot = futureVertices.pop()
        if verbose:
            for size_level in levels:
                if not(size_level in attained_levels) and len(
                        pathDict) >= size_level:
                    print("Built up", str(size_level), "entries in path",
                          "dictionary, having visited", str(len(visited)),
                          "articles with", str(len(futureVertices)),
                          "to be explored.")
                    attained_levels.add(size_level)

        currentDist = distDict[subtreeRoot]
        if verbose and currentDist > maxDist:
            maxDist = currentDist
            print("Reached depth ", currentDist, " with term ", subtreeRoot,
                  ", having visited ", str(len(visited)), " articles and ",
                  "with ", str(len(futureVertices)), " to be explored",
                  sep="")
            # too many lines (684 for depth 1 for Coffee)
#        if verbose and currentDist < 2:
#            print("Examining", subtreeRoot, "at depth =", currentDist)
        if subtreeRoot == targetArticle:
            if verbose:
                print("Found path")
            return constructPath(subtreeRoot, pathDict)
        links = linksOnPage(subtreeRoot)
        if targetArticle in set(links.keys()):
            pathDict[targetArticle] = subtreeRoot
            if verbose:
                print("1st degree connection:", targetArticle, "-->",
                      links[targetArticle])
                print("Visited", str(len(visited)), "articles;", "Added",
                      str(len(pathDict)), "entries to path dictionary")
            return constructPath(targetArticle, pathDict)
        linkChildren = set(links.keys())
        # remove articles that have been previously visited
        linkChildren.difference_update(visited)
        # remove articles that have already been marked for visitation
        linkChildren.difference_update(set(futureVertices))
        for child in linkChildren:
            pathDict[child] = subtreeRoot
            futureVertices.appendleft(child)
            distDict[child] = currentDist+1
        visited.add(subtreeRoot)

def constructPath(destination, pathDict):
    articleList = list()
    node = destination
    while pathDict[node] != None:
        newNode = pathDict[node]
        node = newNode
        articleList.append(newNode)
    articleList.reverse()
    articleList.append(destination)
    return articleList

def process(s1, s2):
    valid1 = hasArticle(s1); valid2 = hasArticle(s2)
    if not(valid1):
        print("No such path exists. English Wikipedia does not have the" +
              "following term: ", s1)
    if not(valid2):
        print("No such path exists. English Wikipedia does not have the" +
              "following term: ", s2)
    if valid1 and valid2:
        article1 = findActualTitle(s1)
        article2 = findActualTitle(s2)
        if article1 == article2:
            print("\'" + s1 + "\'" + " and " + "\'" + s2 + "\'" +
                  " redirect to " + article1)
        else:
            print("PROCESSING", s1, "and", s2)
            resultantPath = bfs(s1, article2, True)
            print(resultantPath)

if isCSVinput:
    dfInput = pd.read_csv("Sample Wikipedia inputs.csv")
    dfInput['concat'] = dfInput['Article1'] + ' ' + dfInput['Article2']
    dfInput['Answer No.'] = list(dfInput.index)
    distinctPairs = dfInput.groupby('concat').agg({'Answer No.': np.min})
    allArticle1 = list(dfInput['Article1'])
    allArticle2 = list(dfInput['Article2'])
    idxs=list(distinctPairs['Answer No.'])
    idxs.sort()
    for k in idxs:
        process(allArticle1[k], allArticle2[k])
else:
    term1 = input("Enter search term 1: ")
    term2 = input("Enter search term 2: ")
    process(term1, term2)


from volttron.platform.vip.agent import core
import random

class Fault(object):
    def __init__(self,state = "suspected"):
        self.state = state
        self.owners = []
        self.uid = random.getrandbits(32)

    def remALLEXCEPT(self,keep):
        if keep in self.owners:
            for owner in self.owners:
                if owner is not keep:
                    self.owners.remove(owner)
        else:
            pass

#fault has been cleared, restore nodes and unlink fault object
    def cleared(self):
        for owner in self.owners:
            if owner in self.isolatednodes or owner in self.faultednodes:
                if owner.__class__.__name__ == "Node":
                    self.restorenode(owner)
            if self in owner.faults:
                owner.faults.remove(self)

            self.owners.remove(owner)

class GroundFault(object):
    def __init__(self,state,zone):
        super(GroundFault,self).__init__(state)
        self.reclose = True
        self.isolatednodes = []
        self.faultednodes = []
        self.reclosecounter = 0
        self.reclosemax = 2

    def isolatenode(self,node):
        if node in self.owners:
            node.isolatenode()
            self.isolatednodes.append(node)

    def restorenode(self,node):
        if node in self.owners:
            node.restore()
            self.isolatednodes.remove(node)

            self.faultednodes.remove(node)

    def reclosenode(self,node):
        self.reclosecounter += 1
        self.restorenode(node)
        if self.reclosecounter == self.reclosemax:
            self.reclose = False

    #initiate procedure to clear a persistent fault
    def clearfault(self):
        self.reclosecounter = 0
        #send message to SG PLC

    def printInfo(self,depth = 0):
        tab = "    "
        print(tab*depth + "FAULT: {state}".format(state = self.state))
        print(tab*depth + "-RECLOSES LEFT: {amt} of {max}".format(amt = self.reclosemax-self.reclosecounter, max = self.reclosemax))
        print(tab*depth + "-AFFECTED NODES and ZONES")
        for owner in self.owners:
            print(tab*depth + "--" + owner.name)
            if owner in self.isolatednodes:
                print("... CURRENTLY ISOLATED ...")
            if owner in self.faultednodes:
                print("!!! FAULT LOCATED HERE!!!")