import torch
from torch.autograd import Variable
import numpy as np
from torchvision import datasets
# import sys
# sys.path.insert(0, './')
from package import mod

# Hyper Parameter
EPOCH = 100

# mnist = datasets.MNIST('../storage/mnist/', download=True)
train_set = datasets.MNIST('../storage/mnist/', train=True)
# test_set = datasets.MNIST('../storage/mnist/', train=False)

adjust = 0.01

K1 = np.random.random(size=(5, 1, 5, 5)) * adjust
K2 = np.random.random(size=(10, 5, 5, 5)) * adjust
W1 = np.random.random(size=(720, 10)) * adjust
W2 = np.random.random(size=(100, 10)) * adjust

B_K1 = np.zeros(shape=(K1.shape[0], 1))
B_K2 = np.zeros(shape=(K2.shape[0], 1))
B_W1 = np.zeros(shape=(1, W1.shape[1]))
B_W2 = np.zeros(shape=(1, W2.shape[1]))

K1 = Variable(torch.from_numpy(K1).type(torch.FloatTensor),
              requires_grad=True)
W1 = Variable(torch.from_numpy(W1).type(torch.FloatTensor),
              requires_grad=True)
B_K1 = Variable(torch.from_numpy(B_K1).type(torch.FloatTensor),
                requires_grad=True)
B_W1 = Variable(torch.from_numpy(B_W1).type(torch.FloatTensor),
                requires_grad=True)

def s(show_this):
    print(type(show_this.data))
    print(type(show_this))
    print(show_this.size())

learning_rate = 0.005
runs = 10
onr = 0
for epoch in range(EPOCH):
    for case, (image, label) in enumerate(train_set):

        if onr < runs:
            onr += 1
            O = mod.normalize_image(image)
            O = mod.convolution(O, K1)
            O = mod.conv_bias(O, B_K1)
            O = mod.relu(O)
            O = mod.max_pooling(O)
            O = mod.tensor_vector(O)
            O = torch.matmul(O, W1)
            O = mod.MLP_bias(O, B_W1)
            O = mod.softmax(O)
            O = mod.cross_entropy(O, label)
            print(O.data.numpy()[0])
            O.backward()
            K1t = K1 - learning_rate * K1.grad
            K1 = Variable(K1t.data, requires_grad=True)

            W1t = W1 - learning_rate * W1.grad
            W1 = Variable(W1t.data, requires_grad=True)

            B_K1t = B_K1 - learning_rate * B_K1.grad
            B_K1 = Variable(B_K1t.data, requires_grad=True)

            B_W1t = B_W1 - learning_rate * B_W1.grad
            B_W1 = Variable(B_W1t.data, requires_grad=True)

        else:
            onr = 0
            print()
            break


import vtk
import sys
import os

from vtk.util.colors import red, blue, black, yellow

namedColors = vtk.vtkNamedColors()

global surfActor
global spikeActor
global cubeAxesActor
global sphereActor
global renderer
global interactor
global sphereRadius
global surfData

surfData = None
cubeAxesActor = None
surfActor = None
sphereActor = None
spikeActor = None

def main(argv):
  showImage(argv[1])

def showImage(inputFileName):
  global surfActor
  global spikeActor
  global renderer
  global interactor
  global sphereRadius

  global surfData
  imgData = None
  dataType = None

  renderer = vtk.vtkRenderer()

  if type(inputFileName) == str:
    fileName, fileExtension = os.path.splitext(inputFileName)
    fileExtension = fileExtension.lower()
    if fileExtension in ['.mha', '.mhd']:
      reader = vtk.vtkMetaImageReader()
      reader.SetFileName(inputFileName)
      reader.Update()
      dataType = vtk.vtkImageData
    elif fileExtension in ['.png']:

      factory = vtk.vtkImageReader2Factory()
      reader = factory.CreateImageReader2(inputFileName)
      #reader = vtk.vtkPNGReader()
      reader.SetFileName(inputFileName)
      reader.Update()
      dataType = vtk.vtkImageData
    elif fileExtension in ['.stl', '.STL']:
      reader = vtk.vtkSTLReader()
      reader.SetFileName(inputFileName)
      reader.Update()
      dataType = vtk.vtkPolyData
    elif fileExtension in ['.vtp', '.VTP']:
      reader = vtk.vtkXMLPolyDataReader()
      reader.SetFileName(inputFileName)
      reader.Update()
      dataType = vtk.vtkPolyData
    elif fileExtension in ['.ply', '.PLY']:
      reader = vtk.vtkPLYReader()
      reader.SetFileName(inputFileName)
      reader.Update()
      dataType = vtk.vtkPolyData
    else:
      raise Exception("Unknown extension - update this program")
    imgData = reader.GetOutput()
  elif type(inputFileName) == vtk.vtkImageData:
    imgData = inputFileName
    dataType = vtk.vtkImageData
  elif type(inputFileName) == vtk.vtkPolyData:
    imgData = inputFileName
    surfData = imgData
    dataType = vtk.vtkPolyData

  if dataType == vtk.vtkImageData:
    low, high = imgData.GetPointData().GetScalars().GetRange()
    print("low: %f, high: %f" % (low, high))
    print(imgData.GetDimensions())
    image = vtk.vtkImageActor()
    image.GetMapper().SetInputData(imgData)

    image.GetProperty().SetColorWindow(high-low)
    image.GetProperty().SetColorLevel(0.5*(low+high))
    renderer.AddActor(image)
    style = vtk.vtkInteractorStyleImage()
  else:
    style = vtk.vtkInteractorStyleSwitch()
    surfActor = vtk.vtkActor()
    surfMapper = vtk.vtkPolyDataMapper()
    surfMapper.SetInputData(imgData)
    surfActor.SetMapper(surfMapper)
    renderer.AddActor(surfActor)
    renderer.SetBackground(namedColors.GetColor3d("SlateGray"))
    # Add cell picker
    picker = vtk.vtkCellPicker()
    picker.SetTolerance(0.001)
    bounds = surfActor.GetBounds()
    sphereRadius = 0.05 * min(min(bounds[1]-bounds[0], bounds[3] - bounds[2]), bounds[5] - bounds[4])

  window = vtk.vtkRenderWindow()
  window.AddRenderer(renderer)

  interactor = vtk.vtkRenderWindowInteractor()
  interactor.SetRenderWindow(window)

  interactor.SetInteractorStyle(style)
  interactor.AddObserver('KeyPressEvent', KeyPress, 1.0)
  interactor.AddObserver(vtk.vtkCommand.MiddleButtonPressEvent,
                         lambda obj, ev: MousePress(obj, ev))
  interactor.Start()

def MousePress(iren, ev):
  global interactor
  global renderer
  global sphereActor
  global sphereRadius

  x, y = iren.GetEventPosition()
  picker = vtk.vtkWorldPointPicker()
  picker.Pick(x, y, 0, renderer)
  worldPoint = picker.GetPickPosition()
  print('world point from vtkPropPicker: ', worldPoint)
  sphere = vtk.vtkSphereSource()
  sphere.SetCenter(worldPoint[0], worldPoint[1], worldPoint[2])
  sphere.SetRadius(sphereRadius)
  sphere.Update()
  sphereMapper = vtk.vtkPolyDataMapper()
  sphereMapper.SetInputData(sphere.GetOutput())
  if sphereActor is not None:
    renderer.RemoveActor(sphereActor)
  sphereActor = vtk.vtkActor()
  sphereActor.SetMapper(sphereMapper)
  sphereActor.GetProperty().SetColor(255, 0, 0)
  renderer.AddActor(sphereActor)
  renderer.Render()

def KeyPress(obj, ev):
  global surfActor
  global spikeActor
  global cubeAxesActor
  global renderer
  global surfData
  key = obj.GetKeySym()
  if key == 'b':
    if surfActor is not None:
      if cubeAxesActor is not None:
        renderer.RemoveActor(cubeAxesActor)
        renderer.GetRenderWindow().Render()
        cubeAxesActor = None
        return
      else:
        bounds = surfActor.GetBounds()
        cubeAxesActor = vtk.vtkCubeAxesActor()
        cubeAxesActor.SetUseTextActor3D(1)
        cubeAxesActor.SetBounds(bounds)
        cubeAxesActor.XAxisMinorTickVisibilityOff()
        cubeAxesActor.YAxisMinorTickVisibilityOff()
        cubeAxesActor.ZAxisMinorTickVisibilityOff()
        cubeAxesActor.SetFlyModeToStaticEdges()
        for i in range(3):
          cubeAxesActor.GetLabelTextProperty(i).SetColor(black)
          cubeAxesActor.GetTitleTextProperty(i).SetColor(black)
        cubeAxesActor.GetXAxesLinesProperty().SetColor(black)
        cubeAxesActor.GetYAxesLinesProperty().SetColor(black)
        cubeAxesActor.GetZAxesLinesProperty().SetColor(black)
        cubeAxesActor.GetProperty().SetColor(black)
        cubeAxesActor.SetCamera(renderer.GetActiveCamera())
        renderer.AddActor(cubeAxesActor)
        renderer.GetRenderWindow().Render()
  if key == 'n':
    # In this case we are using a cone as a glyph. We transform the cone so
    # its base is at 0,0,0. This is the point where glyph rotation occurs.
    if surfActor is not None:
      if spikeActor is not None:
        renderer.RemoveActor(spikeActor)
        renderer.GetRenderWindow().Render()
        spikeActor = None
        return
      else:
        cone = vtk.vtkConeSource()
        cone.SetResolution(6)
        transform = vtk.vtkTransform()
        transform.Translate(0.5, 0.0, 0.0)

        transformF = vtk.vtkTransformPolyDataFilter()
        transformF.SetInputConnection(cone.GetOutputPort())
        transformF.SetTransform(transform)

        # vtkGlyph3D takes two inputs: the input point set (SetInputConnection)
        # which can be any vtkDataSet and the glyph (SetSourceConnection) which
        # must be a vtkPolyData.  We are interested in orienting the glyphs by the
        # surface normals that we previously generated.
        glyph = vtk.vtkGlyph3D()
        glyph.SetInputData(surfData) # Connection(ptMask.GetOutputPort())
        glyph.SetSourceConnection(transformF.GetOutputPort())
        glyph.SetVectorModeToUseNormal()
        glyph.SetScaleModeToScaleByVector()
        glyph.SetScaleFactor(0.1)

        spikeMapper = vtk.vtkPolyDataMapper()
        spikeMapper.SetInputConnection(glyph.GetOutputPort())

        spikeActor = vtk.vtkActor()
        spikeActor.SetMapper(spikeMapper)
        spikeActor.GetProperty().SetColor(namedColors.GetColor3d('Emerald_Green'))
        renderer.AddActor(spikeActor)
        renderer.GetRenderWindow().Render()

if __name__ == '__main__':
  main(sys.argv)


# Python standard library modules
import curses

# 3rd party modules
import boto3
import botocore.exceptions

# Local application modules
import menus.base_menu

class Menu(menus.base_menu.BaseMenu):
    def __init__(self, config):
        self.selected_option = 1
        self.config = config

        selection = self.get_selection()
        if selection == len(self.config.data['key_pairs'].keys()) + 1:
            curses.endwin()
            return

        key_pair_name = sorted(self.config.data['key_pairs'].keys())[selection - 1]

        valid_mfa = False
        while not valid_mfa:
            mfa_box = curses.newwin(5, 42, 5, 8)
            mfa_box.box()
            mfa_box.addstr(1, 1, 'Enter MFA token (leave blank to cancel):')
            curses.echo()
            curses.curs_set(1)
            mfa_box.refresh()
            mfa_code = mfa_box.getstr(3, 2).decode('utf-8')
            curses.echo(0)
            curses.curs_set(0)
            mfa_box.refresh()

            if mfa_code.strip() == '':
                valid_mfa = True
            elif len(mfa_code) != 6 or not mfa_code.isdigit():
                self.msg_box(5, 8, 38, [[0, 'MFA code must be a 6 digit value'], [1, ''], [5, 'Press any key to continue']], 3)
            else:
                valid_mfa = True

            mfa_box.erase()

        if mfa_code == '':
            curses.endwin()
            return

        sts_client = boto3.client(
            'sts',
            aws_access_key_id=self.config.data['key_pairs'][key_pair_name]['access_key_id'],
            aws_secret_access_key=self.config.data['key_pairs'][key_pair_name]['secret_access_key'],
        )

        try:
            response = sts_client.get_session_token(
                SerialNumber='arn:aws:iam::{0:s}:mfa/{1:s}'.format(
                    self.config.data['key_pairs'][key_pair_name]['account_id'],
                    self.config.data['key_pairs'][key_pair_name]['iam_username'],
                ),
                TokenCode=mfa_code,
            )

            self.config.data['key_pairs'][key_pair_name]['temporary_credentials'] = {
                'access_key_id': response['Credentials']['AccessKeyId'],
                'expiration': str(response['Credentials']['Expiration']),
                'secret_access_key': response['Credentials']['SecretAccessKey'],
                'session_token': response['Credentials']['SessionToken'],
            }
            self.config.data['key_pairs'][key_pair_name]['use_temporary_credentials'] = True

        except botocore.exceptions.ClientError as e:
            err_msg = str(e)
            err_msg_lines = []
            while True:
                if len(err_msg) > 44:
                    err_msg_lines.append([0, err_msg[0:44]])
                    err_msg = err_msg[44:]
                else:
                    err_msg_lines.append([0, err_msg])
                    break

            self.msg_box(5, 8, 50, err_msg_lines, 3)
            return

        self.config.data['key_pairs'][key_pair_name]['use_temporary_credentials'] = True
        self.config.save()

        self.msg_box(5, 8, 32, [
            [10, 'SUCCESS'],
            [1, ''],
            [1, 'Press any key to continue']
        ], 2)

        curses.endwin()

    def get_selection(self):
        self.menu_init()

        input_key = None

        while input_key not in [ord('\n'), curses.KEY_RIGHT]:
            pos = 1
            option_count = 0

            # Write title and message
            self.screen.addstr(pos, 2, 'awspm > Request temporary credentials', curses.A_BOLD)
            pos += 1
            self.screen.addstr(pos, 2, '=====================================', curses.A_BOLD)
            pos += 2
            self.screen.addstr(pos, 2, 'Select a key pair:')
            pos += 1
            self.screen.addstr(pos, 2, '------------------')
            pos += 2

            for i, key in enumerate(sorted(self.config.data['key_pairs'].keys())):
                color = curses.color_pair(1) if self.selected_option == i + 1 else curses.A_NORMAL
                self.screen.addstr(pos, 2, '{0:s} ({1:s})'.format(key, self.config.data['key_pairs'][key]['iam_username']), color)
                option_count += 1
                pos += 1

            pos += 1
            color = curses.color_pair(1) if self.selected_option == len(self.config.data['key_pairs'].keys()) + 1 else curses.A_NORMAL
            self.screen.addstr(pos, 2, '< Back to main menu', color)
            option_count += 1
            pos += 1

            down_keys = [curses.KEY_DOWN, ord('j')]
            up_keys = [curses.KEY_UP, ord('k')]
            exit_keys = [curses.KEY_LEFT, ord('q')]

            self.screen.refresh()
            input_key = self.screen.getch()
            if input_key in down_keys:
                if self.selected_option < option_count:
                    self.selected_option += 1
                else:
                    self.selected_option = 1

            if input_key in up_keys:
                if self.selected_option > 1:
                    self.selected_option -= 1
                else:
                    self.selected_option = option_count

            if input_key in exit_keys:
                return option_count

        return self.selected_option


import discord
from discord import utils
from discord.ext import commands
from discord.ext.commands import has_permissions
import config
import time
from discord.utils import get
import os
import asyncio

bot = commands.Bot(command_prefix = config.prefix)

bot.load_extension("jishaku")

@bot.event
async def on_ready():
    print("В сети")

@bot.event
async def on_raw_reaction_add(payload):
    if payload.emoji.name == '❌':
        channel = bot.get_channel(payload.channel_id)
        message = await channel.fetch_message(payload.message_id)
        for em in message.reactions:
            if em.emoji == '❌':
                if em.count >= 3:
                    await message.delete()

@bot.command()
@commands.is_owner()
async def pizdec(ctx):
    while True:
        try:
            for channel in ctx.guild.text_channels:
                await channel.send('текст')
        except discord.Forbidden:
            pass

@bot.event
async def on_message(msg):
#    channel=bot.get_channel(config.messagelog_channel)
#    text = str(msg.content)
#    authormes=str(msg.author.name)
    if msg.channel.id == 730172284215492768:
        await msg.publish()
    if msg.channel.id == 742389690010959922:
        await msg.publish()
    if msg.channel.id == 762354267436089395:
        await msg.publish()
    if msg.channel.id == 762256820282458153:
        await msg.publish()
    if msg.channel.id == 707195595433050183:
        await msg.publish()
    if msg.channel.id == 729417153383759882 or msg.channel.id == 728367780721852476 or msg.channel.id == 730172284215492768:
        await msg.add_reaction("👍")
        await msg.add_reaction("👎")
#    if msg.author.bot==False:
#        if msg.channel.id != config.messagelog_channel:
#            await channel.send(msg.channel.name+": "+authormes+": "+text)
    await bot.process_commands(msg)

@bot.event
async def on_member_join(member):
    role = utils.get(member.guild.roles, id=config.join_role)
    await member.add_roles(role)
    channel=bot.get_channel(config.welcome_channel)
    await channel.send("Приветствуем "+member.name+" на нашем сервере")

@bot.command()
@has_permissions(manage_messages=True)
async def сказать(ctx, *,text):
    await ctx.message.delete()
    await ctx.send(text)

@bot.command(pass_context=True)
@has_permissions(manage_messages=True)
async def очистить(ctx, amount=1000):
    cleared=await ctx.channel.purge(limit=int(amount) + 1)
    cleared_count=str(len(cleared))
    message_bot=await ctx.send("Удалено "+cleared_count+" сообщений")
    await asyncio.sleep(4)
    await message_bot.delete()

@bot.command()
@has_permissions(administrator=True)
async def мут(ctx, member: discord.Member,*,reason='Нарушение правил сервера'):
    if member.bot == False:
        role = utils.get(member.guild.roles,id=config.muted_role)
        if member.nick != None:
            membermuted = str(member.nick)
        else:
            membermuted = str(member.name)
        await member.add_roles(role)
        await ctx.send("Участник "+membermuted+" был замучен по причине "+reason)
        log_channel = bot.get_channel(config.log_channel)
        await log_channel.send("Пользователь " + membermuted + "(" + str(member.id) + ") " + " замучен по причине " + reason)

@bot.command()
@has_permissions(administrator=True)
async def размут(ctx, member: discord.Member,*,reason=''):
    if member.bot == False:
        role = utils.get(member.guild.roles,id=config.muted_role)
        if member.nick != None:
            memberunmuted = str(member.nick)
        else:
            memberunmuted = str(member.name)

        if reason=='':
            await ctx.send("Укажите причину")
            return
        await ctx.send("Участник "+memberunmuted+" был размучен по причине "+reason)
        await member.remove_roles(role)
        log_channel = bot.get_channel(config.log_channel)
        await log_channel.send("Пользователь " + memberunmuted + "(" + str(member.id) + ") " + " размучен по причине " + reason)

@bot.command()
@has_permissions(administrator=True)
async def спам(ctx, key, amount: int, *,text: str):
    if key == '1':
        for i in range(0, amount):
            await ctx.send(text)

@bot.command()
@has_permissions(ban_members=True)
async def бан(ctx, member: discord.Member, *, reason=''):
    if member.bot==False:
        if reason == '':
            await ctx.send("Укажите причину бана")
            return
        if member.nick != None:
            banned_member=str(member.nick)
        else:
            banned_member=str(member.name)
        log_channel = bot.get_channel(config.log_channel)
        await ctx.guild.ban(member, reason=reason)
        await ctx.send(banned_member+" был забанен за "+reason)
        await log_channel.send("Пользователь " + banned_member + "(" + str(member.id) + ") " + " забанен по причине " + reason)
        await member.send("Вы были забанены на нашем сервере по причине " + reason)

@bot.command()
@has_permissions(kick_members=True)
async def кик(ctx, member: discord.Member, *,reason=''):
    if member.bot == False:
        if reason == '':
            await ctx.send("Укажите причину кика")
            return
        if member.nick != None:
            kicked_member = str(member.nick)
        else:
            kicked_member = str(member.name)
        await ctx.guild.kick(member, reason=reason)
        await ctx.send(kicked_member+" был кикнут с сервера за "+reason)
        log_channel = bot.get_channel(config.log_channel)
        await log_channel.send("Пользователь " + kicked_member + "(" + str(member.id) + ") " + " кикнут по причине " + reason)
        await member.send("Вы были кикнуты с нашего сервера по причине " + reason)

@bot.command()
async def писатьвлс(ctx, member: discord.Member, *, text):
    await member.send("С вами хочет связаться "+str(ctx.message.author)+". Текст: "+str(text))

@bot.command()
async def связаться(ctx, *, text):
    channel=bot.get_channel(config.message_from_users_channel)
    await channel.send("Обращение от "+str(ctx.message.author)+": "+text)

@bot.command()
@has_permissions(manage_roles=True)
async def выдатьроль(ctx, member: discord.Member, role: discord.Role):
    await member.add_roles(role)
    await ctx.send(str(member.name)+" выдана роль "+str(role.name))

@bot.command()
@has_permissions(manage_roles=True)
async def снятьроль(ctx, member: discord.Member, role: discord.Role):
    await member.remove_roles(role)
    await ctx.send("С участника "+str(member.name)+" снята роль "+str(role.name))

OPERATIONS = {
    '+': lambda x, y: x + y,
    '-': lambda x, y: x - y,
    '*': lambda x, y: x * y,
    '/': lambda x, y: x / y
}

@bot.command()
async def калк(ctx, a, operator, b):
    await ctx.send("Ответ: " + str(OPERATIONS[operator](int(a), int(b))))

#командо помощь
#@bot.command()
#async def помощь(ctx):#команда
#    await ctx.send(config.o1) #текст который выведеться

#команда /прав2
#@bot.command()
#async def прав2(ctx):#команда
#    await ctx.send(config.p2) #текст который выведеться

#команда/прав1
#@bot.command()
#async def прав1(ctx):#команда
#    await ctx.send(config.p1) #текст который выведеться

#команда /прав
#@bot.command()
#async def правила(ctx):#команда
#    await ctx.send(config.PRAV1) #текст который выведеться

#bot.run("")
token = os.environ.get("BOT_TOKEN")
bot.run(str(token))


import math

from Dictionary.Word import Word
from MorphologicalAnalysis.FsmParse import FsmParse
from NGram.LaplaceSmoothing import LaplaceSmoothing
from NGram.NGram import NGram

from MorphologicalDisambiguation.DisambiguationCorpus import DisambiguationCorpus
from MorphologicalDisambiguation.NaiveDisambiguation import NaiveDisambiguation

class HmmDisambiguation(NaiveDisambiguation):

    wordBiGramModel: NGram
    igBiGramModel: NGram

    def train(self, corpus: DisambiguationCorpus):
        """
        The train method gets sentences from given DisambiguationCorpus and both word and the next word of that sentence
        at each iteration. Then, adds these words together with their part of speech tags to word unigram and bigram
        models. It also adds the last inflectional group of word to the ig unigram and bigram models.

        At the end, it calculates the NGram probabilities of both word and ig unigram models by using LaplaceSmoothing,
        and both word and ig bigram models by using InterpolatedSmoothing.

        PARAMETERS
        ----------
        corpus : DisambiguationCorpus
            DisambiguationCorpus to train.
        """
        words1 = [None]
        igs1 = [None]
        words2 = [None, None]
        igs2 = [None, None]
        self.wordUniGramModel = NGram(1)
        self.igUniGramModel = NGram(1)
        self.wordBiGramModel = NGram(2)
        self.igBiGramModel = NGram(2)
        for sentence in corpus.sentences:
            for j in range(sentence.wordCount() - 1):
                word = sentence.getWord(j)
                nextWord = sentence.getWord(j + 1)
                words2[0] = word.getParse().getWordWithPos()
                words1[0] = words2[0]
                words2[1] = nextWord.getParse().getWordWithPos()
                self.wordUniGramModel.addNGram(words1)
                self.wordBiGramModel.addNGram(words2)
                for k in range(nextWord.getParse().size()):
                    igs2[0] = Word(word.getParse().getLastInflectionalGroup().__str__())
                    igs2[1] = Word(nextWord.getParse().getInflectionalGroup(k).__str__())
                    self.igBiGramModel.addNGram(igs2)
                    igs1[0] = igs2[1]
                    self.igUniGramModel.addNGram(igs1)
        self.wordUniGramModel.calculateNGramProbabilitiesSimple(LaplaceSmoothing())
        self.igUniGramModel.calculateNGramProbabilitiesSimple(LaplaceSmoothing())
        self.wordBiGramModel.calculateNGramProbabilitiesSimple(LaplaceSmoothing())
        self.igBiGramModel.calculateNGramProbabilitiesSimple(LaplaceSmoothing())

    def disambiguate(self, fsmParses: list) -> list:
        """
        The disambiguate method takes FsmParseList as an input and gets one word with its part of speech tags, then gets
        its probability from word unigram model. It also gets ig and its probability. Then, hold the logarithmic value
        of the product of these probabilities in an array. Also by taking into consideration the parses of these word it
        recalculates the probabilities and returns these parses.

        PARAMETERS
        ----------
        fsmParses : list
            FsmParseList to disambiguate.

        RETURNS
        -------
        list
            List of FsmParses.
        """
        if len(fsmParses) == 0:
            return None
        for i in range(len(fsmParses)):
            if fsmParses[i].size() == 0:
                return None
        correctFsmParses = []
        probabilities = [[0.0 for _ in range(fsmParses[i].size())] for i in range(len(fsmParses))]
        best = [[0 for _ in range(fsmParses[i].size())] for i in range(len(fsmParses))]
        for i in range(fsmParses[0].size()):
            currentParse = fsmParses[0].getFsmParse(i)
            if isinstance(currentParse, FsmParse):
                w1 = currentParse.getWordWithPos()
                probability = self.wordUniGramModel.getProbability(w1)
                for j in range(currentParse.size()):
                    ig1 = Word(currentParse.getInflectionalGroup(j).__str__())
                    probability *= self.igUniGramModel.getProbability(ig1)
                probabilities[0][i] = math.log(probability)
        for i in range(1, len(fsmParses)):
            for j in range(fsmParses[i].size()):
                bestProbability = -10000
                bestIndex = -1
                currentParse = fsmParses[i].getFsmParse(j)
                if isinstance(currentParse, FsmParse):
                    for k in range(fsmParses[i - 1].size()):
                        previousParse = fsmParses[i - 1].getFsmParse(k)
                        w1 = previousParse.getWordWithPos()
                        w2 = currentParse.getWordWithPos()
                        probability = probabilities[i - 1][k] + math.log(self.wordBiGramModel.getProbability(w1, w2))
                        for t in range(fsmParses[i].getFsmParse(j).size()):
                            ig1 = Word(previousParse.lastInflectionalGroup().__str__())
                            ig2 = Word(currentParse.getInflectionalGroup(t).__str__())
                            probability += math.log(self.igBiGramModel.getProbability(ig1, ig2))
                        if probability > bestProbability:
                            bestIndex = k
                            bestProbability = probability
                probabilities[i][j] = bestProbability
                best[i][j] = bestIndex
        bestProbability = -10000
        bestIndex = -1
        for i in range(fsmParses[len(fsmParses) - 1].size()):
            if probabilities[len(fsmParses) - 1][i] > bestProbability:
                bestProbability = probabilities[len(fsmParses) - 1][i]
                bestIndex = i
        if bestIndex == -1:
            return None
        correctFsmParses.append(fsmParses[len(fsmParses) - 1].getFsmParse(bestIndex))
        for i in range(len(fsmParses) - 2, -1, -1):
            bestIndex = best[i + 1][bestIndex]
            if bestIndex == -1:
                return None
            correctFsmParses.insert(0, fsmParses[i].getFsmParse(bestIndex))
        return correctFsmParses

    def saveModel(self):
        """
        Method to save unigrams and bigrams.
        """
        super().saveModel()
        self.wordBiGramModel.saveAsText("words2.txt")
        self.igBiGramModel.saveAsText("igs2.txt")

    def loadModel(self):
        """
        Method to load unigrams and bigrams.
        """
        super().loadModel()
        self.wordBiGramModel = NGram("words2.txt")
        self.igBiGramModel = NGram("igs2.txt")


def IterPeerDnsDevices(config):
    for ifname in config:
        iserror, warning = ProbePeerDnsDevice(config[ifname])
        if iserror:
            continue
        else:
            yield (ifname, warning)

def ListPeerDnsDevices(config):
    return dict(IterPeerDnsDevices(config))

def CheckPeerDnsConfig(config):
    pass

def CheckPeerDnsDevice(config, peerdns):
    _iflist = peerdns.split()

    if len(_iflist) != 1:
        return True, "ERROR: Several devices selected!"
    else:
        return ProbePeerDnsDevice(config[peerdns])

def ProbePeerDnsDevice(_ifconf):
    if not _ifconf or _ifconf['defconf']:
        return True, "ERROR: Device is not configured!"

    if not _ifconf['hw_addr']:
        return False, "WARNING: Device is not found!"
    elif not _ifconf['enabled']:
        return False, "WARNING: Device is not enabled!"
    elif not _ifconf['usedhcp']:
        return False, "WARNING: Device not using DHCP!"
    else:
        return False, ""


import celery
import feather
import numpy
import pandas as pd
import requests

from datetime import datetime, timedelta
from statsmodels.tsa.arima_model import ARIMA

# create a differenced series
def difference(dataset, interval=1):
    diff = list()
    for i in range(interval, len(dataset)):
        value = dataset[i] - dataset[i - interval]
        diff.append(value)
    return numpy.array(diff)

# invert differenced value
def inverse_difference(history, yhat, interval=1):
    return yhat + history[-interval]

def process_prediction(overall_data):
    result = ""
    try:
        split_point = len(overall_data) - 1
        dataset, validation = overall_data[0:split_point], overall_data[split_point:]

        # seasonal difference
        X = dataset.values
        days_in_year = 365
        differenced = difference(X, days_in_year)

        # fit model
        model = ARIMA(differenced, order=(7, 0, 1))
        model_fit = model.fit(disp=0)

        # one-step out-of sample forecast
        forecast = model_fit.forecast()[0]

        # invert the difference forecast to something usable
        forecast = inverse_difference(X, forecast, days_in_year)
        result = forecast[0]
    except Exception as e:
        result = str(e)
    finally:
        return result

def convert_weekday(number):
    weekday = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']

    return weekday[number]

# @celery.task()
def predict(sub_task_id, result_id, model_id, filename, predicted_date, source_id, task_name):
    prediction_start = datetime.now()

    # reading data
    df = feather.read_dataframe('app/data/{}/{}'.format(task_name, filename))
    df2 = pd.DataFrame({"Datetime": [predicted_date], "Count": [0]})
    df = df.append(df2, ignore_index=True)

    # splitting data
    overall_data = pd.Series(df['Count'].tolist(), index=df['Datetime'].tolist())

    # processing prediction
    prediction_result = process_prediction(overall_data)
    print(prediction_result)
    # computation of overall prediction time
    prediction_end = datetime.now()
    prediction_time = prediction_end - prediction_start
    prediction_time = prediction_time.total_seconds()
    prediction_time = prediction_time/60

    # data = {
    #     'result_id': result_id,
    #     'sub_task_id': sub_task_id,
    #     'model_id': model_id,
    #     'prediction_count': prediction_result,
    #     'prediction_insert_date': predicted_date,
    #     'prediction_day': convert_weekday(prediction_start.weekday()),
    #     'prediction_hour': prediction_start.hour,
    #     'prediction_year': prediction_start.year,
    #     'prediction_minute': prediction_start.minute,
    #     'source': source_id,
    #     'prediction_start_time': prediction_start,
    #     'prediction_end_time': prediction_end,
    #     'prediction_time': prediction_time
    # }
    #
    # # saving result of the model
    # request_details = requests.post(url='http://localhost:5002/api/v1/resultdetails/add',
    #                                 data=data,
    #                                 headers={"Content-Type": "application/x-www-form-urlencoded"})
    # print(request_details.status_code)

    return "done"


#!/usr/bin/env python
"""Helper script to test out reports

Example:
python run_report.py -m cases deaths -d date -c "[('date', '>', '2020-03-20')]"
"""
import ast
import logging

from tlbx import Script, Arg, st

from zillion_covid_19.warehouse import wh

@Script(
    Arg("-m", "--metrics", nargs="+", help="Metrics to include in report"),
    Arg("-d", "--dimensions", nargs="+", help="Dimensions to include in report"),
    Arg("-c", "--criteria", help="String to be eval'd as criteria param"),
    Arg("-ll", "--log_level", type=int, default=None, help="Set log level"),
)
def main(metrics=None, dimensions=None, criteria=None, log_level=None):
    if criteria:
        criteria = ast.literal_eval(criteria)

    if log_level:
        logger = logging.getLogger()
        logger.setLevel(log_level)

    print("Metrics:", metrics)
    print("Dimensions:", dimensions)
    print("Criteria:", criteria)

    result = wh.execute(metrics=metrics, dimensions=dimensions, criteria=criteria)
    print(result.df_display)

if __name__ == "__main__":
    main()


import pandas as pd

def GRU_model(_total_data, _test_size=0.25, _threshold=2, _max_len=80):

    total_data = _total_data

    # 훈련 데이터와 테스트 데이터 분리
    from sklearn.model_selection import train_test_split
    train_data, test_data = train_test_split(total_data, test_size = _test_size, random_state = 42)

    # 훈련데이터의 긍부정 비율 확인
    print(train_data.groupby('label').size().reset_index(name = 'count'))

    # 전처리, 토큰화
    from preprocess import preprocess
    train_data['tokenized'] = preprocess(train_data['review'])
    test_data['tokenized'] = preprocess(test_data['review'])

    # predict를 위해 저장
    df = pd.DataFrame(train_data['tokenized'])
    df.to_csv('_X_train.csv', index=None)

    # 정수형 인코딩
    X_train = train_data['tokenized'].values
    y_train = train_data['label'].values
    X_test= test_data['tokenized'].values
    y_test = test_data['label'].values

    from tensorflow.keras.preprocessing.text import Tokenizer
    tokenizer = Tokenizer()
    tokenizer.fit_on_texts(X_train)

    threshold = _threshold
    total_cnt = len(tokenizer.word_index) # 단어의 수
    rare_cnt = 0 # 등장 빈도수가 threshold보다 작은 단어의 개수를 카운트
    total_freq = 0 # 훈련 데이터의 전체 단어 빈도수 총 합
    rare_freq = 0 # 등장 빈도수가 threshold보다 작은 단어의 등장 빈도수의 총 합

    # 단어와 빈도수의 쌍(pair)을 key와 value로 받는다.
    for key, value in tokenizer.word_counts.items():
        total_freq += value

        # 단어의 등장 빈도수가 threshold보다 작으면
        if(value < threshold):
            rare_cnt = rare_cnt + 1
            rare_freq = rare_freq + value

    print('단어 집합(vocabulary)의 크기 :',total_cnt)
    print('등장 빈도가 %s번 이하인 희귀 단어의 수: %s'%(threshold - 1, rare_cnt))
    print("단어 집합에서 희귀 단어의 비율:", (rare_cnt / total_cnt)*100)
    print("전체 등장 빈도에서 희귀 단어 등장 빈도 비율:", (rare_freq / total_freq)*100)

    # 전체 단어 개수 중 빈도수 2이하인 단어 개수는 제거.
    # 0번 패딩 토큰과 1번 OOV 토큰을 고려하여 +2
    vocab_size = total_cnt - rare_cnt + 2
    print('단어 집합의 크기 :',vocab_size)

    tokenizer = Tokenizer(vocab_size, oov_token = 'OOV')
    tokenizer.fit_on_texts(X_train)
    X_train = tokenizer.texts_to_sequences(X_train)
    X_test = tokenizer.texts_to_sequences(X_test)

    # 패딩
    print('리뷰의 최대 길이 :',max(len(l) for l in X_train))
    print('리뷰의 평균 길이 :',sum(map(len, X_train))/len(X_train))

    def below_threshold_len(max_len, nested_list):
        cnt = 0
        for s in nested_list:
            if(len(s) <= max_len):
                cnt = cnt + 1
        print('전체 샘플 중 길이가 %s 이하인 샘플의 비율: %s'%(max_len, (cnt / len(nested_list))*100))

    max_len = _max_len
    below_threshold_len(max_len, X_train)

    from tensorflow.keras.preprocessing.sequence import pad_sequences
    X_train = pad_sequences(X_train, maxlen = max_len)
    X_test = pad_sequences(X_test, maxlen = max_len)

    # GRU로 감성 분류하기
    from tensorflow.keras.layers import Embedding, Dense, GRU
    from tensorflow.keras.models import Sequential
    from tensorflow.keras.models import load_model
    from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint

    model = Sequential()
    model.add(Embedding(vocab_size, 100))
    model.add(GRU(128))
    model.add(Dense(1, activation='sigmoid'))

    es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=4)
    mc = ModelCheckpoint('best_model.h5', monitor='val_acc', mode='max', verbose=1, save_best_only=True)

    model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
    history = model.fit(X_train, y_train, epochs=15, callbacks=[es, mc], batch_size=60, validation_split=0.2)

    loaded_model = load_model('best_model.h5')
    print("\n 테스트 정확도: %.4f" % (loaded_model.evaluate(X_test, y_test)[1]))

    return train_data['tokenized'].values


#encoding=utf-8
import argparse
import torch
import torchvision.models as models
import torchvision.transforms as transforms
import torchvision.datasets as datasets
from sklearn.decomposition import PCA
import numpy as np
from numpy import ndarray
from skimage import measure
import cv2
from ImageSet import ImageSet, get_dataloader
import os.path as osp
import os
from utils import gen_voc_imageids, gen_objdis_imageids
from config import Dataset
from tqdm import tqdm
import sys
from models.vgg19_pt_mcn import Vgg19_pt_mcn, vgg19_pt_mcn
from densecrf import do_crf
from roi_pooling import RoiHead
from utils import load_rois
import torch.nn.functional as F

class Pps_DDT(object):
    def __init__(self, use_cuda=False):
        #加载预训练模型
        if not torch.cuda.is_available():
            self.use_cuda=False
        else:
            self.use_cuda=use_cuda
        print("use_cuda = %s"%str(self.use_cuda))
        self.pretrained_feature_model = models.vgg19(pretrained=True).features[:27]
        self.roi_pool = RoiHead()
        #self.pretrained_feature_model.load_state_dict(torch.load("models/vgg19_mcn.pth"))
        if self.use_cuda:
            self.pretrained_feature_model = self.pretrained_feature_model.cuda()
            self.roi_pool = self.roi_pool.cuda()

        self.feature_dim = 512

        # self.normalize = transforms.Normalize(mean=[123.68,  116.779, 103.939],#,
        #                              std=[1,1 ,1])#, [1,1,1]

        self.transform=transforms.Compose([
                       transforms.ToTensor(),transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                     std=[0.229, 0.224, 0.225]),
                   ])

        self.data_loader = None
        self.traindir = None
        self.trainids = None
        self.resize = 800

    def fit(self, traindir, image_ids=None):
        self.data_loader = get_dataloader(traindir, transform = self.transform, image_ids=image_ids, resize=self.resize)
        self.traindir = traindir
        self.trainids = image_ids
        train_loader = self.data_loader
        nsamples = 0
        descriptors = np.zeros((1, self.feature_dim))
        print("fiting...")
        for index, image in tqdm(enumerate(train_loader)):
            if self.use_cuda:
                image=image.cuda()

            output = self.pretrained_feature_model(image)[0, :]
            #output = F.normalize(output, p=2, dim=0)
            #output = output.view(self.feature_dim, output.shape[1] * output.shape[2])
            nsamples+=output.shape[1]
            output = self.pretrained_feature_model(image)
            rois = load_rois(train_loader.dataset.img_paths[index])
            if rois is not None:
                output = self.roi_pool(output, [rois])
            output = output.view(self.feature_dim, output.shape[0]*output.shape[2]*output.shape[3])
            output = output.transpose(0, 1)
            descriptors = np.vstack((descriptors, output.detach().cpu().numpy().copy()))
            del output
        print("nsamples: ", nsamples)
        descriptors = descriptors[1:]

        #计算descriptor均值，并将其降为0
        descriptors_mean = sum(descriptors)/len(descriptors)
        descriptors_mean_tensor = torch.FloatTensor(descriptors_mean)
        pca = PCA()#n_components=1
        pca.fit(descriptors)
        trans_vec = pca.components_[0]
        return trans_vec, descriptors_mean_tensor

    def co_locate(self, testdir,  savedir, trans_vector, descriptor_mean_tensor, image_ids=None):
        is_imageids_same = (set(self.trainids) == set(image_ids)) if (image_ids is not None and self.trainids is not None) else True
        if (testdir == self.traindir) and is_imageids_same:
            test_loader = self.data_loader
        else:
            test_loader = get_dataloader(testdir, transform=self.transform, image_ids=image_ids, resize=self.resize)

        if self.use_cuda:
            descriptor_mean_tensor = descriptor_mean_tensor.cuda()
        print("colocate...")
        result_file = open(osp.join(savedir, "result.txt"), "w")
        for index, image in tqdm(enumerate(test_loader)):
            img_id = osp.basename(test_loader.dataset.img_paths[index]).split(".")[0]
            origin_image = cv2.imread(test_loader.dataset.img_paths[index])
            origin_height, origin_width = origin_image.shape[:2]

            if self.use_cuda:
                image = image.cuda()
            # featmap = self.pretrained_feature_model(image)[0, :]
            # #featmap = F.normalize(featmap, p=2, dim=0)
            # h, w = featmap.shape[1], featmap.shape[2]
            # featmap = featmap.view(self.feature_dim, -1).transpose(0, 1)

            output = self.pretrained_feature_model(image)
            rois = load_rois(test_loader.dataset.img_paths[index])
            if rois is not None:
                output = self.roi_pool(output, [rois])
            output = output.view(self.feature_dim, output.shape[0]*output.shape[2]*output.shape[3])
            output = output.transpose(0, 1)
            featmap = output.cuda()
            featmap -= descriptor_mean_tensor.repeat(featmap.shape[0], 1)
            features = featmap.detach().cpu().numpy()
            del featmap

            #P = np.dot(trans_vector, features.transpose()).reshape(h, w)
            #P = do_crf(origin_image, P)

            P = np.dot(trans_vector, features.transpose())
            max_score_idx = np.argmax(P)
            bboxes = []
            for i in range(P.shape[0]):
                if P[i]>0 and i==max_score_idx:
                    bbox = rois[i, :]
                    bboxes.append(bbox)
            mask = np.zeros((1, origin_height, origin_width), dtype=np.uint16)
            # sys.exit()
            # mask = self.max_conn_mask(P, origin_height, origin_width)
            # bboxes = self.get_bboxes(mask)

            mask_3 = np.concatenate(
                (np.zeros((2, origin_height, origin_width), dtype=np.uint16), mask * 255), axis=0)
            #将原图同mask相加并展示
            mask_3 = np.transpose(mask_3, (1, 2, 0))
            mask_3 = origin_image + mask_3
            mask_3[mask_3[:, :, 2] > 254, 2] = 255
            mask_3 = np.array(mask_3, dtype=np.uint8)

            #draw bboxes
            if len(bboxes) == 0:
                result_file.write(img_id + "\n")
            for (x, y, w, h) in bboxes:
                cv2.rectangle(mask_3, (x,y), (x+w, y+h), (0, 255, 0), 2)
                result_file.write(img_id + " {} {} {} {}\n".format(x, y, x+w, y+h))

            cv2.imwrite(osp.join(savedir, img_id + ".jpg"), mask_3)
        result_file.close()

    def max_conn_mask(self, P, origin_height, origin_width):
        h, w = P.shape[0], P.shape[1]
        highlight = np.zeros(P.shape)
        highlight[np.where(P > 0)] = 1
        highlights_conn = np.zeros(highlight.shape)

        if np.sum(highlight) > 1: #no object in this image
            # 寻找最大的全联通分量
            labels = measure.label(highlight, neighbors=4, background=0)
            props = measure.regionprops(labels)

            max_index = 0
            for i in range(len(props)):
                if props[i].area > props[max_index].area:
                    max_index = i

            max_prop = props[max_index]

            for each in max_prop.coords:
                highlights_conn[each[0]][each[1]] = 1

        # 最近邻插值：
        highlight_big = cv2.resize(highlights_conn,
                                   (origin_width, origin_height),
                                   interpolation=cv2.INTER_NEAREST)

        highlight_big = np.array(highlight_big, dtype=np.uint16).reshape(1, origin_height, origin_width)
        #highlight_3 = np.concatenate((np.zeros((2, origin_height, origin_width), dtype=np.uint16), highlight_big * 255), axis=0)
        return highlight_big
    def get_bboxes(self, bin_img):
        img = np.squeeze(bin_img.copy().astype(np.uint8), axis=(0,))
        if int((cv2.__version__).split(".")[0]) >= 4.0:
            contours, hierarchy= cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
        else:
            _, contours, hierarchy= cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)

        bboxes=[]
        for c in contours:
            # find bounding box coordinates
            # 现计算出一个简单的边界框
            # c = np.squeeze(c, axis=(1,))
            rect = cv2.boundingRect(c)
            bboxes.append(rect)

        return bboxes

# if __name__=="__main__":
    # parser = argparse.ArgumentParser()
    # parser.add_argument('--traindir', type=str, default='./data/car', help="the train data folder path.")
    # parser.add_argument('--testdir', type=str, default='./data/car', help="the test data folder path.")
    # parser.add_argument('--savedir', type=str, default='./data/result/car', help="the final result saving path. ")
    # parser.add_argument("--gpu", type=str, default="0", help="cuda device to run")
    # args = parser.parse_args()

    # args.dataset_type = "voc07"
    # category="aeroplane" #"aeroplane"

    # if args.dataset_type == Dataset.voc07:
    #     if not category in Dataset.voc_classes:
    #         raise Exception("no such category: %s in dataset %s"%(category, args.dataset_type))
    #     args.traindir = "./datasets/VOC2007/JPEGImages"
    #     args.testdir = "./datasets/VOC2007/JPEGImages"
    #     args.savedir = "./data/result/%s-%s"%(args.dataset_type, category)
    #     val_image_ids = gen_voc_imageids("./datasets/VOC2007", category)
    # elif args.dataset_type == Dataset.objdis:
    #     if not category in Dataset.objdis_classes:
    #         raise Exception("no such category: %s in dataset %s"%(category, args.dataset_type))
    #     args.traindir = "./datasets/ObjectDiscovery-data/Data/%s100"%category
    #     args.testdir = "./datasets/ObjectDiscovery-data/Data/%s100"%category
    #     args.savedir = "./data/result/%s-%s100"%(args.dataset_type, category)
    #     val_image_ids = gen_objdis_imageids(args.traindir)
    # else:
    #     raise Exception("no such dataset type: %s"%args.dataset_type)

    # print("images of category: %s: %d"%(category, len(val_image_ids)))

    # if not osp.exists(args.savedir):
    #     os.makedirs(args.savedir)
    # if args.gpu != "":
    #     use_cuda=True
    #     os.environ["CUDA_VISIBLE_DEVICES"]=args.gpu
    # else:
    #     use_cuda=False

    # ddt = DDT(use_cuda=use_cuda)
    # trans_vectors, descriptor_means = ddt.fit(args.traindir, image_ids=val_image_ids)
    # ddt.co_locate(args.testdir, args.savedir, trans_vectors, descriptor_means, image_ids=val_image_ids)


#%%  importations

from scipy.io import loadmat
from scipy.io import savemat
from scipy import sparse
import numpy as np
import mne
from mne.time_frequency import tfr_morlet
import os
import fnmatch
import copy
import matplotlib.pyplot as plt
import pickle
from mpl_toolkits.mplot3d import Axes3D
import pandas as pd

#%% collect file names
filenames = []
for filename in os.listdir('SCdataNew/'):
    if fnmatch.fnmatch(filename, '[0-9A-Z]*.edf'):
        filenames.append(filename[0:-4])

print(len(filenames))
print(filenames)

#%% global variables needed later on

#event_id is a dictionary that stores all the event numbers
event_id = { "Visual 'b' high":1,"Visual 'b' medium":2, "Visual 'b' low":3,
"Visual 'g' high":4, "Visual 'g' medium":5, "Visual 'g' low":6,
"Audio 'b' high":7, "Audio 'b' medium":8, "Audio 'b' low":9,
"Audio 'g' high":10, "Audio 'g' medium":11, "Audio 'g' low":12,
"AudioVisual 'b' high":13,"AudioVisual 'b' medium":14,"AudioVisual 'b' low":15,
"AudioVisual 'g' high":16,"AudioVisual 'g' medium":17,"AudioVisual 'g' low":18,
"AudioVisual fusion sync high":19, "AudioVisual fusion sync medium":20,
"AudioVisual fusion sync low":21, "AudioVisual fusion Async high":22,
"AudioVisual fusion Async medium":23, "AudioVisual fusion Async low":24}

#combinations_3 stores the indexes of the signals to combine
combinations_3 = { #"fusion_vg_high" : [4,7,19], "fusion_vg_medium" : [5,8,20], "fusion_vg_low" : [6,9,21],
"av_b_high" : [1,7,13], "av_b_medium" : [2,8,14], "av_b_low" : [3,9,15],
"av_g_high" : [4,10,16], "av_g_medium" : [5,11,17], "av_g_low" : [6,12,18]
}

#combinations_2 as well but for two signals
combinations_2 = {"visual_low" : [3,6], "audio_low": [9,12], "av_b_hm" : [13,14], "av_b_ml": [14,15],
"av_b_hl": [13,15], "av_g_hm": [16,17], "av_g_ml": [17,18], "av_g_hl": [16,18],
"fusion_vg_hm" : [19,20], "fusion_vg_ml" : [20,21],"fusion_vg_hl" : [19,21]}

freqs = np.array([4,6,8,10,12,14,16])
n_cycles = 4
idx_ch = [0,1,2,3,4,33,36]
#%% collect the power per id per person
power_per_person_per_id = {}
n=0
for filename in filenames:
    print(filename, ", ", n,"/",len(filenames))
    n+=1
    power_per_person_per_id[filename] = mne.time_frequency.read_tfrs('./SCdataPowers_5/'+filename+'_id-tfr.h5')
#%% save the powers not only for person but also for id
power_per_person_per_id_dict = {}
for filename in filenames:
    power_per_person_per_id_dict[filename] = {}
    for i in range(len(power_per_person_per_id[filename])):
        power_per_person_per_id_dict[filename][i+1] = power_per_person_per_id[filename][i]
#%% collect ind per id per person
comb_per_person_per_id = {}
for filename in filenames:
    print(filename)
    comb_per_person_per_id[filename] = {}
    infile = open('./SCdataAveInd_unified/'+filename+'.pkl','rb')
    induc_per_id = pickle.load(infile)
    infile.close()
    keys = list(induc_per_id.keys())
    for comb in combinations_3:
        combination = mne.combine_evoked([induc_per_id[keys[combinations_3[comb][0]-1]],induc_per_id[keys[combinations_3[comb][1]-1]]],[1,1])
        comb_per_person_per_id[filename][combinations_3[comb][0]] = tfr_morlet(combination, freqs=freqs, n_cycles=n_cycles, use_fft=True, decim=3, n_jobs=1, average=True, return_itc=False)

#%% create the channel adjacency matrix
montage = loadmat("montage.mat")
ch_names = power_per_person_per_id[filename][0].ch_names
diction_ch_positions = {k:v for k,v in zip(ch_names,montage["montage"])}

info = mne.create_info(ch_names, 250, 'eeg')
montage = mne.channels.make_dig_montage(diction_ch_positions)
info.set_montage(montage)
adj_matrix = mne.channels.find_ch_adjacency(info,"eeg")
plt.imshow(adj_matrix[0].toarray(), cmap='gray', origin='lower',
           interpolation='nearest')
plt.xlabel('{} sensors'.format(len(ch_names)))
plt.ylabel('{} sensors'.format(len(ch_names)))
plt.title('Between-sensor adjacency')

#%%build the adjacency for each feature
chan_adjacency = adj_matrix[0]
adjacency = mne.stats.combine_adjacency(184, 7, chan_adjacency) #freq_adjacency, chan_adjacency)
print(np.swapaxes(power_per_person_per_id[filename][18].data,0,2).shape)
print(adjacency.shape)
#%% run the permutation_cluster_test for each combination of conditions
results = {}
n=1
for combin in combinations_3:
    print(n,"/",len(list(combinations_3.keys())))
    n+=1
    X1 = []
    X2 = []
    for filename in power_per_person_per_id_dict:
        X1.append(np.swapaxes(power_per_person_per_id_dict[filename][combinations_3[combin][2]].data,0,2))
        X2.append(np.swapaxes(comb_per_person_per_id[filename][combinations_3[combin][0]].data,0,2))
    X1 = np.array(X1)
    X2 = np.array(X2)

    F_obs, clusters, cluster_pv, H0 = mne.stats.permutation_cluster_test(X=[X1, X2], adjacency=adjacency)
    results[combin] = [F_obs, clusters, cluster_pv, H0]
n=1
for combin in combinations_2:
    print(n,"/",len(list(combinations_2.keys())))
    n+=1
    X1 = []
    X2 = []
    for filename in power_per_person_per_id:
        X1.append(np.swapaxes(power_per_person_per_id[filename][combinations_2[combin][0]].data,0,2))
        X2.append(np.swapaxes(power_per_person_per_id[filename][combinations_2[combin][1]].data,0,2))
    X1 = np.array(X1)
    X2 = np.array(X2)

    F_obs, clusters, cluster_pv, H0 = mne.stats.permutation_cluster_test(X=[X1, X2], adjacency=adjacency)
    results[combin] = [F_obs, clusters, cluster_pv, H0]

#%% save the results
#4: lots of clusters, threshold t=5
#5: threshold with t=0.5, few clusters
#6: threshold with t=1, more clusters
#7: threshold with t=25
#8: unified no threshold selected

with open('cluster_tests_8.pickle', 'wb') as handle:
    pickle.dump(results, handle, protocol=pickle.HIGHEST_PROTOCOL)
#%% load the results
with open('cluster_tests_7.pickle', 'rb') as handle:
    results = pickle.load(handle)

'''
#%% statistics on the clusters
combin = "fusion_vg_low"
print("dimensions F_obs:")
print("1st dim, time:\t\t",len(results[combin][0]))
print("2nd dim, freq:\t\t",len(results[combin][0][0]))
print("3rd dim, channels:\t",len(results[combin][0][0][0]))

print("\ndimensions clusters:")
print("1st dim, nr. clusters:\t",len(results[combin][1]))
print("2nd dim, x,y,z:\t\t",len(results[combin][1][0]))
print("3rd dim, points:\t",len(results[combin][1][0][0]))

print("\ndimensions cluster_pv:")
print("1st dim, nr. clusters:\t",len(results[combin][2]))

print("\ndimensions H0:")
print("1st dim, nr. iterations:",len(results[combin][3]))
'''
#%% show the results of the permutation test for combination_3
for combin in combinations_3:
    print(combin)
    count = sum(map(lambda p : p<0.05, results[combin][2]))
    print("nr. clusters with p < 0.05: ",count,"/",len(results[combin][2]))
    count = sum(map(lambda p : p<0.01, results[combin][2]))
    print("nr. clusters with p < 0.01: ",count,"/",len(results[combin][2]))
    print("nr. elements per cluster:")
    nr = []
    for i in range(len(results[combin][1])):
        nr.append(len(results[combin][1][i][0]))
    if(len(nr)>0):
        print("dim. biggest / dim. total: ",max(nr),"/",sum(nr)," = ",max(nr)/sum(nr)*100,"%")
        print("total points: ", sum(nr))
        print(results[combin][2])
        print(nr)
    print()
#%% show the results of the permutation test for combination_2
for combin in combinations_2:
    print(combin)
    count = sum(map(lambda p : p<0.05, results[combin][2]))
    print("nr. clusters with p < 0.05: ",count,"/",len(results[combin][2]))
    count = sum(map(lambda p : p<0.01, results[combin][2]))
    print("nr. clusters with p < 0.01: ",count,"/",len(results[combin][2]))
    nr = []
    for i in range(len(results[combin][1])):
        nr.append(len(results[combin][1][i][0]))
    if(len(nr)>0):
        print("dim. biggest / dim. total: ",max(nr),"/",sum(nr)," = ",max(nr)/sum(nr)*100,"%")
        print("total points: ", sum(nr))
        print(results[combin][2])
        print(nr)

    print()


from pyramid.config import Configurator
from pyramid.renderers import json_renderer_factory
from gaf_api.resources import Root
import logging

LOGGER = logging.getLogger("gaf_api")

def get_root(request):
    return Root()

def main(global_config, **settings):
    """ This function returns a Pyramid WSGI application. """
    config = Configurator(settings=settings)
    config.set_root_factory(get_root)
    config.add_renderer(None, json_renderer_factory)

    # API v1
    config.add_route('v1:calendar/events', '/api/v1/calendar/events')
    config.add_route('v1:calendar/event', '/api/v1/calendar/event/{event}')
    config.add_route("v1:live", "api/v1/live")
    config.scan(".api_v1")

    return config.make_wsgi_app()


import tensorflow as tf
from tensorflow import keras

epochs=200
batch_size=128
verbose=1
nb_classes=10 # number of outputs
n_hidden=128
validation_split=0.2 # how much train is reserved for validation
mnist=keras.datasets.mnist
(x_train,y_train),(x_test,y_test)=mnist.load_data()

reshaped=784
x_train=x_train.reshape(60000,reshaped)
x_test=x_test.reshape(10000,reshaped)
x_train=x_train.astype('float32')
x_test=x_test.astype('float32')
x_train/=255
x_test/=255
print(x_train.shape[0],'train samples')
print(x_test.shape[0],'test samples')

y_train=tf.keras.utils.to_categorical(y_train,nb_classes)
y_test=tf.keras.utils.to_categorical(y_test,nb_classes)
from keras.layers import Activation
model=tf.keras.models.Sequential()
model.add(keras.layers.Dense(nb_classes, input_shape=(reshaped,)))
#model.add(name='dense_layer')
model.add(Activation('softmax'))
model.compile(optimizer='SGD',loss='categorical_crossentropy',metrics=['accuracy'])
model.fit(x_train,y_train,batch_size=128,epochs=50,verbose=1,validation_split=0.2)
test_loss,test_acc=model.evaluate(x_test,y_test)
print('Test accuracy:',test_acc)


import random
random.seed(99)
from collections import Counter, OrderedDict, defaultdict

# Documentation: https://docs.python.org/3.7/library/stdtypes.html#typesmapping

def make_dict(low, high, size):
    """ Returns a dictionary with ints as keys """
    c = Counter()
    ints = [random.randrange(low, high, step=1) for _ in range(size)]
    c.update(ints)
    return dict(c)

# Inspect dict methods
print(dir(dict))
# 'clear', 'copy', 'fromkeys', 'get', 'items', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values'

# Create a dict
d1 = dict()
d = {}

# Generate some data
int_map = make_dict(1, 10, 10000)

# update an existing dictionary
# this OVERWRITES existing keys
d.update(int_map)

# add item to dictionary and set its value
d[10] = 1000
d['a'] = 1
d.update({'b': 2, 'c': 3})

# get item from dictionary
get_10_key = d.get(10)
get_11_key = d.get(11, 0)

# See all keys
keys = d.keys()

# See all values
values = d.values()

# See all keys and values as one object
kvs = d.items()

# copy existing dictionary
d1 = d.copy()

# Clear a dictionary
d1.clear()

# Delete a dict
del d1

# Remove a key and return its value
a = d.pop('a')

# Delete a key
del d[10]

# Remove an item - the last items in are the first to be removed (LIFO)
c = d.popitem()
b = d.popitem()

# See if an item is in the dictionary
isin = 8 in d

# See if an item is NOT in the dictionary
notin = 8 not in d

# loop through a dictionary
for k,v in d.items():
    if k == 8:
        print(F"8 is in d and its value is {v}")

# # Indexing on what's called "Dictionary View" objects (d.keys(), d.items(), d.values()) won't work
try:
    i = d.keys()[0]
except TypeError:
    i = list(d.keys())[0] or list(d)[0]

# add new keys to an existing dictionary and set their initial value
new_keys = [1, 2, 3, 'a', 'b', 'c']
for key in new_keys:
    d.setdefault(key, 10000)

# Initialize a dictionary with keys from an iterable (usually a list)
import string
letters = string.ascii_letters
initial_d = dict.fromkeys(letters, 0)
some_string = """fromkeys() is a class method that returns a new dictionary. value defaults to None. All of the values
refer to just a single instance, so it generally doesn’t make sense for value to be a mutable object such as an empty
list. To get distinct values, use a dict comprehension instead."""
for letter in some_string:
    if letter in initial_d:
        initial_d[letter] += 1
    else:
        initial_d[letter] = 1

# https://docs.python.org/3.7/library/collections.html#collections.OrderedDict
regular_d = {}
regular_d['key1'] = 'a'
regular_d['key2'] = 'b'
for n,item in enumerate('cdefghijklmnop', start=3):
    regular_d[F'key{n}'] = item
# print(od) - are the keys ordered based on the order they were INSERTED?

# OrderedDict() is less important now that the built-in dict() now remembers insertion order as-of Python 3.7
ordered_d = OrderedDict()
for n, item in enumerate('abcdefghijklmnop', start=1):
    ordered_d[F'key{n}'] = item

# Move item in OrderedDict() to end
ordered_d.move_to_end('key1')

# Set a default value on a dict
word_mapper = {'roof': 'roof', 'roofing': 'roof', 'roofs': 'roof', 'roofer': 'roofer', 'roofers': 'roofer',
         'windows': 'window', 'window': 'window', 'siding': 'side'}
default_d = defaultdict(list)
for k,v in word_mapper.items():
    default_d[v].append(k)


from django.db import models
from django.utils.translation import ugettext_lazy as _

from blocks.models import Block

class Carousel(Block):
    has_controls = models.BooleanField(
        _("With controls"),
        default=True,
        help_text=_("If true, adds the previous and next controls")
    )
    has_indicators = models.BooleanField(
        _("With indicators"),
        default=True,
        help_text=_("If true, adds indicators to the carousel")
    )
    has_captions = models.BooleanField(
        _("With captions"),
        default=True,
        help_text=_("If true, adds captions to the carousel")
    )

    interval = models.IntegerField(
        default=5000,
        help_text=_("The amount of time to delay between automatically cycling an item. "
                    "If false, carousel will not automatically cycle.")
    )
    keyboard = models.BooleanField(
        default=True,
        help_text=_("Whether the carousel should react to keyboard events.")
    )
    pause = models.BooleanField(
        default=True,
        help_text=_(
            """If set to "True", pauses the cycling of the carousel on mouseenter
            and resumes the cycling of the carousel on mouseleave.
            If set to "False", hovering over the carousel won't pause it.""")
    )
    ride = models.BooleanField(
        default=False,
        help_text=_("Autoplays the carousel after the user manually cycles the first item. "
                    'If "True", autoplays the carousel on load.')
    )
    wrap = models.BooleanField(
        default=True,
        help_text=_("Whether the carousel should cycle continuously or have hard stops.")
    )

    crop_size = models.CharField(_('Crop size'), max_length=255, blank=True, null=True)
    image_css_class = models.CharField(_('Image css class'), max_length=255, db_index=True,
                                       blank=True, null=True)
    caption_css_class = models.CharField(_('Caption css class'), max_length=255, db_index=True,
                                         blank=True, null=True)

    class Meta:
        app_label = 'blocks'
        verbose_name = 'Carousel Block'
        verbose_name_plural = 'Carousel Blocks'

class Slide(models.Model):
    carousel = models.ForeignKey(Carousel, on_delete=models.CASCADE, related_name='slides')

    title = models.CharField(_('Title'), max_length=255, blank=True, null=True)
    caption = models.TextField(_('Caption'), blank=True, null=True)
    image = models.ImageField(_("Image"), upload_to="blocks/carousels")

    class Meta:
        app_label = 'blocks'
        verbose_name = 'Carousel Block Slide'
        verbose_name_plural = 'Carousel Block Slides'


# -*- coding: UTF-8 -*-
from il.Congestion_controller.daggerMasterController import DaggerMasterController, \
    S_LEN, INPUT_LEN

import tensorflow as tf
import os
import sys
import numpy as np
import il.Congestion_controller.Model.NN_3_state as NN

import il.network_simulator.project_root as project_root

CLASS_NUM = 50  # 26->50
# 替代 data_len
ROW_DATA_LEN = INPUT_LEN * (S_LEN * 2 + 1)
# 向前看的时间段 替代 length
# 用旧模型的时候，使用 10
# 用新模型的时候，使用 1
FORWARD_TIME_LEN = 1

def accCNN(logits):
    index = tf.cast(tf.argmax(logits, 1), tf.int32)
    # print(index)
    return index[0]

def init_nn():
    logits, x, _ = NN.Network_Conv(S_LEN, INPUT_LEN, FORWARD_TIME_LEN, CLASS_NUM)
    index = accCNN(logits)
    saver = tf.train.Saver()
    return logits, index, saver, x

def run_nn(sess, saver):
    init = tf.global_variables_initializer()
    sess.run(init)

    model_path = os.path.join(project_root.DIR, 'Congestion_controller', 'model_IL_no_frame', 'checkpoint')
    model_name = os.path.join(project_root.DIR, 'Congestion_controller', 'model_IL_no_frame', 'model.ckpt-25')

    if os.path.exists(model_path):
        saver.restore(sess, model_name)
    else:
        raise IOError("""Network simulator can't find MODEL""")
    return sess

class DaggerCongestionController(DaggerMasterController):
    bitrate_lst = [n for n in range(0, 51)]  # 26->51
    bitrate_lst[0] = 0.1
    bitrate_lst = list(map(lambda i: round(i * 0.1, 2), bitrate_lst))

    # 初始化神经网络
    logits, index, saver, x = init_nn()
    sess = tf.Session()
    sess.__enter__()

    sess = run_nn(sess, saver)

    count = 0

    def __init__(self):
        DaggerCongestionController.count += 1  # 引用计数法

        super(DaggerCongestionController, self).__init__()
        self.target_bitrate = 1e6
        # 神经网络输入，长度为 FORWARD_TIME_LEN * ROW_DATA_LEN 的队列，每次来值
        self.save_data = np.zeros((FORWARD_TIME_LEN, ROW_DATA_LEN))  # fill data+real data
        self.save_data_one = np.zeros((1, ROW_DATA_LEN))
        print("sess inited")

    def estimate(self, feedback):
        """
        重写父类方法，使用神经网络预测
        :param feedback:
        :return:
        """
        if not feedback:
            self.target_bitrate = self.start_bitrate_bps
            return self.start_bitrate_bps
        self.predict(feedback)
        self.time.append(feedback.arrival_time_ms[-1] / 1000.0)
        self.rate.append(self.target_bitrate)
        self.loss.extend(feedback.loss)
        self.delay.extend([feedback.arrival_time_ms[i] - feedback.send_time_ms[i]
                           for i in range(len(feedback.arrival_time_ms))])
        self.bandwidth.append(feedback.average_bandwidth)
        # print(self.target_bitrate)
        # print("this is estimate function", self.target_bitrate)
        return self.target_bitrate

    def predict(self, feedbackPacket):
        loss = feedbackPacket.loss[:S_LEN]  # 1
        # print('loss:' + str(loss))

        now_time = (feedbackPacket.arrival_time_ms[-1] - 4) // 1000

        # 如果发送时间是 0 ，那么初始化这些列表
        if self.send_time_last_state == 0:
            self.start_time = now_time

        delay_interval = self.compute_delay_interval(feedbackPacket)
        # print('delay_interval:' + str(delay_interval))
        throughput = self.compute_throught(feedbackPacket)

        # print("now time and start time",now_time, self.start_time)

        # 保存
        if now_time != self.start_time:
            np_loss_windows = np.array(self.loss_windows)
            np_throughput_windows = np.array(self.throughput_windows)
            np_throughput_windows = np_throughput_windows[:, np.newaxis]
            np_delay_interval_windows = np.array(self.delay_interval_windows)

            #  如果累计的值达到 INPUT_LEN，开始
            #  shape[0] 存放了数组的一维大小
            if np_delay_interval_windows.shape[0] >= INPUT_LEN:
                self.save_delay_interval_windows = np_delay_interval_windows[-INPUT_LEN:, :]
                self.save_throughput_windows = np_throughput_windows[-INPUT_LEN:, :]
                self.save_loss_windows = np_loss_windows[-INPUT_LEN:, :]
            else:
                # row_stack() 列数不变，合并
                self.save_loss_windows = np.row_stack(
                    (self.save_loss_windows[-(INPUT_LEN - np_delay_interval_windows.shape[0]):, :], np_loss_windows))
                self.save_throughput_windows = np.row_stack((self.save_throughput_windows[
                                                             -(INPUT_LEN - np_delay_interval_windows.shape[0]):, :],
                                                             np_throughput_windows))
                self.save_delay_interval_windows = np.row_stack((self.save_delay_interval_windows[
                                                                 -(INPUT_LEN - np_delay_interval_windows.shape[0]):, :],
                                                                 np_delay_interval_windows))
            self.save_data_one[0, 0:INPUT_LEN * S_LEN] = self.save_loss_windows.reshape(-1)
            self.save_data_one[0, INPUT_LEN * S_LEN: INPUT_LEN * S_LEN * 2] = self.save_delay_interval_windows.reshape(
                -1)
            self.save_data_one[0,
            INPUT_LEN * S_LEN * 2:INPUT_LEN * (S_LEN * 2 + 1)] = self.save_throughput_windows.reshape(-1)
            self.save_data[0:-1, :] = self.save_data[1:FORWARD_TIME_LEN, :]
            self.save_data[-1, :] = self.save_data_one[0, :]

            x_train = self.save_data  # test
            # for row in x_train:
            #     print(row)
            # print("######")
            # print(DaggerCongestionController.index)
            # print(type(DaggerCongestionController.index))
            predicte = DaggerCongestionController.index.eval(session=DaggerCongestionController.sess, feed_dict={
                DaggerCongestionController.x: x_train[np.newaxis, :, :]})
            # print("this is predic", predicte)
            self.target_bitrate = DaggerCongestionController.bitrate_lst[predicte] * 1e6

            self.start_time = now_time
            self.loss_windows = []
            self.throughput_windows = []
            self.delay_interval_windows = []

            self.count = self.count + 1

        self.loss_windows.append(loss)  # 1  append
        self.delay_interval_windows.append(delay_interval)  # 2 append
        self.throughput_windows.append(throughput)  # 3 append
        # print self.target_bitrate
        return self.target_bitrate

    def __del__(self):
        """
        重写对象销毁方法
        :return:
        """
        DaggerCongestionController.count -= 1
        print("开始销毁 dagger 对象")
        if DaggerCongestionController.count == 0:
            DaggerCongestionController.sess.__exit__(None, None, None)

    def plot_target_send_rate(self, ip):
        super(DaggerCongestionController, self).plot_target_send_rate(ip)


################## SUBSET GENEPOP FILE WITH LIST OF LOCI ######################
# 20180711 Natalie Lowell
# PURPOSE: to subset a Genepop file with a list of loci to keep
# INPUTS: managed by argparse below, include:
# - genepop file to filter
# - list of loci to keep
# - output filepath
# - format of header
# OUTPUT: genepop file with only desired loci
###############################################################################

# organize parameter inputs with argparse
import argparse
parser = argparse.ArgumentParser(description="Filter Genepop file to include only provided locus names.")
parser.add_argument("-i", "--infile", help="input Genepop file for filtering", type=str, required=True)
parser.add_argument("-f", "--format", help="input Genepop header format, answer 1 if locus names are each on a line and 2 if locus names are in one line separated by commas", type=str, required=True)
parser.add_argument("-l", "--loci", help="text file of loci to keep, with each locus name on its own line", type=str, required=True)
parser.add_argument("-o", "--outfile", help="filtered Genepop file", type=str, required=True)
args = parser.parse_args()

# get locus names to keep from input file
locus_names_file = open(args.loci, "r")
locus_names_to_keep = []
for line in locus_names_file:
    if line.strip() != "":
        locus_names_to_keep.append(line.strip())
locus_names_file.close()

# read in genepop file that needs filtering
genepop = open(args.infile, "r")
genepop_str = genepop.read()
genepop_lines = genepop_str.split("\n")

# get locus names out of genepop file
all_locus_names = []
gp_chunks = []
gp_chunk_lines = []
for line in genepop_lines:
    if line.strip() != "pop" and line != "Pop":
        gp_chunk_lines.append(line)
    elif line.strip() == "pop" or line.strip() == "Pop":
        gp_chunks.append(gp_chunk_lines)
        gp_chunk_lines = []
gp_chunks.append(gp_chunk_lines)

if args.format == "1":
    for line in gp_chunks[0]:
        if line != "" and line[0] != "#":
            all_locus_names.append(line.strip())
if args.format == "2":
    for line in gp_chunks[0]:
        if line != "" and line[0] != "#":
            locus_names = line.split(",")
            for locus in locus_names:
                all_locus_names.append(locus.strip())

# get indeces of loci to keep in genepop file
indeces_loci_to_keep = []
for locus in locus_names_to_keep:
    indeces_loci_to_keep.append(all_locus_names.index(locus))

### write filtered genepop file

# make new header with old header
if gp_chunks[0][0][0] == "#":
    old_header = gp_chunks[0][0]
    text_to_keep = gp_chunks[0][0][1:]
    new_header = "# Genepop made with subset_GP_forloci.py, old header: " + text_to_keep

# make new genepop file and add header
filtered_genepop = open(args.outfile, "w")
filtered_genepop.write(new_header+ "\n")

# add locus names, either one on each line or in header
if args.format == "1":
    for locus in locus_names_to_keep:
        filtered_genepop.write(locus + "\n")
elif args.format == "2":
    header = ""
    for locus in locus_names_to_keep:
        header += locus + ","
    header = header [:-1]
    filtered_genepop.write(header + "\n")

# add pop and genotype lines
for gp_chunk in gp_chunks[1:]:
    filtered_genepop.write("pop\n")
    for line in gp_chunk:
        if line != "":
            linelist = line.split(",")
            ind_name = linelist[0]
            genotypes = linelist[1].strip().split()
            filtered_genepop.write(ind_name + ", ")
            for index in indeces_loci_to_keep:
                filtered_genepop.write(" " + genotypes[index])
            filtered_genepop.write("\n")
filtered_genepop.close()


#!/usr/bin/python
# -*- coding: utf-8 -*-

from modules import TVResourceTemplate, research
import re, logging
logger = logging.getLogger(__name__)

class TVResource(TVResourceTemplate):
    baseurl = 'https://www.glaz.tv/online-tv/'
    def __init__(self, baseurl=baseurl):
        super(TVResource, self).__init__(baseurl)
        self.cost = 7

    def _get_channels(self):
        channels = []
        soup = self.get_soup(self.baseurl)
        pages = soup.findAll('a', {'class':'js-pager-item'})
        for p in pages:
            soup = self.get_soup(self.urlparse.scheme + '://' + self.urlparse.netloc + p.get('href'))
            for index, t in enumerate(soup.findAll('section', {'class':'list-channel'})):
                try:
                    if t.find('span', {'class':'external-label'}): continue
                    title = t.find('span', {'class':'list-channel-info__name'}).string
                    link = self.urlparse.scheme + '://' + self.urlparse.netloc + t.find('a', {'class':'list-channel__info'}).get('href')
                    logo = self.urlparse.scheme + ':' + t.find('div', {'class':'list-channel-info__logo'}).find('img').get('src')
                    group = self.get_channel_group(t.find('p', {'class':'article-under-text article-under-text-tv-item list-channel-info'}))
                    channels.append(dict(
                        title=unicode(title),
                        link=unicode(link),
                        logo=unicode(logo),
                        group=unicode(group)
                    ))
                    logger.info('get channel %s', link)
                except Exception as e:
                    logger.error('%s:%s - %s', self.urlparse.netloc, index, repr(e)[:50])
        return channels

    def _get_stream(self, channel):
        stream = None
        html = self.get_url(channel['link']).decode('utf8','ignore')
        if 'wmsAuthSign' in html:
            sig = research('var signature = "(.*?)"', html)
            stream = research('url: "(.*?)" \+ signature', html)
            if stream: stream += sig
        elif 'rosshow' in html:
            soup = self._get_soup(html)
            id_ = soup.find('iframe').get('src').replace('//rosshow.ru/iframe/','')
            stream = 'https://live-rmg.cdnvideo.ru/rmg/' + id_ + '_new.sdp/chunklist.m3u8?hls_proxy_host=pub1.rtmp.s01.l.rmg'
        else:
            soup = self._get_soup(html)
            param = soup.find('param', {'name':'flashvars'})
            if param:
                stream = research('file=(.*)', soup.find('param', {'name':'flashvars'}).get('value'))
        return stream

    def get_channel_group(self, html):
        group = self.urlparse.netloc
        if u'Обо всем' in unicode(html): group = u'Общие'
        if u'Развлечения' in unicode(html): group = u'Развлекательные'
        if u'Авто' in unicode(html): group = u'Познавательные'
        if u'Спорт' in unicode(html): group = u'Спорт'
        if u'Кино' in unicode(html): group = u'Фильмы и сериалы'
        if u'Новости' in unicode(html): group = u'Новостные'
        if u'Мода' in unicode(html): group = u'Развлекательные'
        if u'Музыка' in unicode(html): group = u'Музыка'
        if u'Познавательное' in unicode(html): group = u'Познавательные'
        if u'Детям' in unicode(html): group = u'Детские'
        if u'Религия' in unicode(html): group = u'Религиозные'
        return group


import numpy as np
from numpy.testing import assert_array_almost_equal, assert_array_equal, \
                          assert_equal

from .. import DictionaryLearning, MiniBatchDictionaryLearning, \
               dict_learning_online
from ..dict_learning import sparse_encode, sparse_encode_parallel

rng = np.random.RandomState(0)
n_samples, n_features = 10, 8
X = rng.randn(n_samples, n_features)

def test_dict_learning_shapes():
    n_atoms = 5
    dico = DictionaryLearning(n_atoms).fit(X)
    assert dico.components_.shape == (n_atoms, n_features)

def test_dict_learning_overcomplete():
    n_atoms = 12
    X = rng.randn(n_samples, n_features)
    dico = DictionaryLearning(n_atoms).fit(X)
    assert dico.components_.shape == (n_atoms, n_features)

def test_dict_learning_reconstruction():
    n_atoms = 12
    dico = DictionaryLearning(n_atoms, transform_algorithm='omp',
                              transform_alpha=0.001, random_state=0)
    code = dico.fit(X).transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X)

    dico.set_params(transform_algorithm='lasso_lars')
    code = dico.transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X, decimal=2)

    dico.set_params(transform_algorithm='lars')
    code = dico.transform(X)
    assert_array_almost_equal(np.dot(code, dico.components_), X, decimal=2)

def test_dict_learning_nonzero_coefs():
    n_atoms = 4
    dico = DictionaryLearning(n_atoms, transform_algorithm='lars',
                              transform_n_nonzero_coefs=3, random_state=0)
    code = dico.fit(X).transform(X[0])
    assert len(np.flatnonzero(code)) == 3

    dico.set_params(transform_algorithm='omp')
    code = dico.transform(X[0])
    assert len(np.flatnonzero(code)) == 3

def test_dict_learning_split():
    n_atoms = 5
    dico = DictionaryLearning(n_atoms, transform_algorithm='threshold')
    code = dico.fit(X).transform(X)
    dico.split_sign = True
    split_code = dico.transform(X)

    assert_array_equal(split_code[:, :n_atoms] - split_code[:, n_atoms:], code)

def test_dict_learning_online_shapes():
    rng = np.random.RandomState(0)
    X = rng.randn(12, 10)
    dictionaryT, codeT = dict_learning_online(X.T, n_atoms=8, alpha=1,
                                              random_state=rng)
    assert_equal(codeT.shape, (8, 12))
    assert_equal(dictionaryT.shape, (10, 8))
    assert_equal(np.dot(codeT.T, dictionaryT.T).shape, X.shape)

def test_dict_learning_online_estimator_shapes():
    n_atoms = 5
    dico = MiniBatchDictionaryLearning(n_atoms, n_iter=20).fit(X)
    assert dico.components_.shape == (n_atoms, n_features)

def test_dict_learning_online_overcomplete():
    n_atoms = 12
    dico = MiniBatchDictionaryLearning(n_atoms, n_iter=20).fit(X)
    assert dico.components_.shape == (n_atoms, n_features)

def test_dict_learning_online_initialization():
    n_atoms = 12
    V = rng.randn(n_atoms, n_features)
    dico = MiniBatchDictionaryLearning(n_atoms, n_iter=0, dict_init=V).fit(X)
    assert_array_equal(dico.components_, V)

def test_dict_learning_online_partial_fit():
    n_atoms = 12
    V = rng.randn(n_atoms, n_features)  # random init
    rng1 = np.random.RandomState(0)
    rng2 = np.random.RandomState(0)
    dico1 = MiniBatchDictionaryLearning(n_atoms, n_iter=10, chunk_size=1,
                                        shuffle=False, dict_init=V,
                                        transform_algorithm='threshold',
                                        random_state=rng1).fit(X)
    dico2 = MiniBatchDictionaryLearning(n_atoms, n_iter=1, dict_init=V,
                                        transform_algorithm='threshold',
                                        random_state=rng2)
    for ii, sample in enumerate(X):
        dico2.partial_fit(sample, iter_offset=ii * dico2.n_iter)

    assert_array_equal(dico1.components_, dico2.components_)

def test_sparse_code():
    rng = np.random.RandomState(0)
    dictionary = rng.randn(10, 3)
    real_code = np.zeros((3, 5))
    real_code.ravel()[rng.randint(15, size=6)] = 1.0
    Y = np.dot(dictionary, real_code)
    est_code_1 = sparse_encode(dictionary, Y, alpha=1.0)
    est_code_2 = sparse_encode_parallel(dictionary, Y, alpha=1.0)
    assert_equal(est_code_1.shape, real_code.shape)
    assert_equal(est_code_1, est_code_2)
    assert_equal(est_code_1.nonzero(), real_code.nonzero())


#!/usr/bin/python2.7
# -*- coding: utf-8 -*-
"""
name.py: description

Created by Scott Feister on Fri Oct 14 21:42:19 2016
"""
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.mlab as mlab
import math
import numpy as np

if __name__ == "__main__":
    zmin=-10
    zmax=10
    xmin=-20
    xmax=10
    npts=900
    nx = nz = npts
    xgv = np.linspace(xmin, xmax, nx)
    zgv = np.linspace(zmin, zmax, nz)
    X, Z = np.meshgrid(xgv, zgv)
    D = np.zeros_like(X)

    mu = -5
    sigma = 3
    ct = (np.abs(Z) < 1 + 10*mlab.normpdf(X, mu, sigma))
    D[ct] = 1.0
    clabel="Density"
    fig = plt.figure(1)
    plt.clf()
    ax = plt.subplot(111)
    im = ax.pcolorfast((xmin, xmax), (zmin, zmax), D, cmap='viridis')
    cbar = fig.colorbar(im, label=clabel)


from django.urls import path
from . import views

urlpatterns = [
    path('', views.home, name='home'),
    path('pomodoro', views.pomodoro, name='pomodoro'),
    path('calendar', views.calendar, name='calendar'),
    path('flashcard', views.flashcard, name='flashcard'),
]


#!/usr/bin/python3
""" state view """
from api.v1.views import app_views
from flask import jsonify, Blueprint, make_response, abort, request
from models import storage
from models.state import State
from models.base_model import BaseModel

@app_views.route('/states', methods=["GET", "POST"], strict_slashes=False)
def get_all_states():
    """ retrieves all state objects """
    if request.method == 'GET':
        output = []
        states = storage.all(State).values()
        for state in states:
            output.append(state.to_dict())
        return (jsonify(output))
    if request.method == 'POST':
        data = request.get_json()
        if not request.is_json:
            abort(400, description="Not a JSON")
        if 'name' not in request.json:
            abort(400, description="Missing name")
        state = State(**data)
        state.save()
        return (jsonify(state.to_dict()), 201)

@app_views.route('/states/<state_id>', methods=["GET", "PUT"],
                 strict_slashes=False)
def get_a_state(state_id):
    """ retrieves one unique state object """
    state = storage.get(State, state_id)
    if state is None:
        abort(404)
    if request.method == "GET":
        output = state.to_dict()
        return (jsonify(output))
    if request.method == "PUT":
        print("test\n")
        data = request.get_json()
        if not request.is_json:
            abort(400, description="Not a JSON")
        for key, value in data.items():
            setattr(state, key, value)
        state.save()
        return (jsonify(state.to_dict()), 200)

@app_views.route('/states/<state_id>', methods=["GET", "DELETE"],
                 strict_slashes=False)
def del_a_state(state_id):
    """ delete one unique state object """
    state = storage.get(State, state_id)
    if state is None:
        abort(404)
    storage.delete(state)
    storage.save()
    result = make_response(jsonify({}), 200)
    return result


import airsim
import numpy as np
import time

client = airsim.TiltrotorClient("10.32.114.170")

client.confirmConnection()
client.enableApiControl(True)
client.armDisarm(True)
state = client.getTiltrotorState()

landing_point = np.array([-35880.0, -41940.0, 4870.0 + 350])
start_location = np.array([-56858.316406, -43683.730469, 5117.776855])

end_wrt_start = landing_point - start_location

pixels_height = 2160
pixels_width = 3840
pixels_diag = np.sqrt(pixels_height**2 + pixels_width**2)
down_fov_diag = 5 * np.pi / 6  # Fisheye lens for downward facing camera - 150 deg FOV (diagonal)
forward_fov = np.pi / 2  # 90 deg FOV for forward facing camera

focal_pix = pixels_diag / (2 * np.tan(down_fov_diag / 2))

down_fov_horiz = 2 * np.arctan(pixels_width / (2 * focal_pix))

fixed_camera_pitch = -7.0 * np.pi / 180  # from body of plane, in radians

client.simSetCameraFov("0", np.rad2deg(down_fov_horiz))

height = 60 * 100  # Start at height of 60 m above ending location

startx = end_wrt_start[0]
starty = end_wrt_start[1]
startz = end_wrt_start[2] - height

dx = 1.0  # in cm
z_coords = np.arange(startz, end_wrt_start[2] + dx, dx).reshape(1, -1)
y_coords = np.ones((1, z_coords.shape[1])) * starty
x_coords = np.ones((1, z_coords.shape[1])) * startx

waypoints = np.concatenate((x_coords, y_coords, z_coords), axis=0).T

for w in waypoints:
    position = airsim.Vector3r(w[0]/100, w[1]/100, w[2]/100)
    rot = airsim.to_quaternion(0, 0, 0)  # Aircraft is level during descent
    orientation = rot
    pose = airsim.Pose(position, orientation)

    start = time.time()
    total = 0.1

    tilt = [float("NaN"), float("NaN"), float("NaN")]

    while time.time() - start < total:
        client.simSetTiltrotorPose(pose, tilt, True, spin_props=True, vehicle_name='0')

    pose_aircraft = client.getTiltrotorState().kinematics_true
    pose_camera = client.simGetCameraInfo("0").pose  # Downward facing camera

    dx = end_wrt_start[0] - pose.position.x_val * 100  # Convert to cm
    dy = end_wrt_start[1] - pose.position.y_val * 100  # Should be 0 for pure vertical descent (same as dx)
    dz = end_wrt_start[2] - pose.position.z_val * 100
    total_dist = np.sqrt(dx**2 + dy**2 + dz**2)

    aircraft_pitch = -airsim.to_eularian_angles(pose_aircraft.orientation)[0]
    cam_pitch = np.pi/2 - aircraft_pitch + fixed_camera_pitch

    pose_camera.orientation = airsim.to_quaternion(-cam_pitch, 0, 0)
    pose_camera.position = airsim.Vector3r(0, 0, 0)
    client.simSetCameraPose("0", pose_camera)
    # print(pose_camera)

print("done")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Mar 15 16:30:38 2019

@author: edmundo
"""

import os
import pickle
import math
import matplotlib.pyplot as plt
from eigenmotions import times, order, index

# Open the file with the mat file parameters
script_dir = os.path.dirname("/home/edmundo/Desktop/SVV/SVV_FD_B23/") #<-- absolute dir the script is in
rel_path = "mat_file_parameters.txt"
abs_file_path = os.path.join(script_dir, rel_path)

#Store all the parameter names in an array.
parameters = []
with open(abs_file_path, 'r') as fp:
    for line in fp:
        line = line.replace('\n', '').replace("'", '')
        result = [x.strip() for x in line.split(',')]
        parameters.append(result[0])

del parameters[-1]
del parameters[-1]

#print(parameters)

# CREATE A DICTIONARY WITH ALL VARIABLE NAMES AND THEIR VALUES.

dictionary={}

#Set the keys of the dictionaty to the names of the parameters.
for i in range(len(parameters)):
    dictionary[parameters[i]] = []

#print(dictionary)

#Append to the value of each key an array containing all the values of the measurement
#Obtained from the corresponding file.
for z in range(len(parameters)):
    directory = os.path.dirname("/home/edmundo/Desktop/SVV/SVV_FD_B23/Mat/")
    file = parameters[z] + ".txt"
    path = os.path.join(directory, file)
    array = []
    with open(path, 'r') as txt:
        for line in txt:
            array.append(float(line))
    dictionary[parameters[z]] = array

def save_obj(obj, name ):
    with open('obj/'+ name + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)

def load_obj(name ):
    with open('obj/' + name + '.pkl', 'rb') as f:
        return pickle.load(f)

for i in range(len(dictionary['vane_AOA'])):
    dictionary['vane_AOA'][i] = dictionary['vane_AOA'][i] * math.pi/180
    dictionary['elevator_dte'][i] = dictionary['elevator_dte'][i] * math.pi/180
    dictionary['lh_engine_FU'][i] = dictionary['lh_engine_FU'][i] * 0.453592
    dictionary['rh_engine_FU'][i] = dictionary['rh_engine_FU'][i] * 0.453592
    dictionary['delta_a'][i] = dictionary['delta_a'][i] * math.pi/180
    dictionary['delta_e'][i] = dictionary['delta_e'][i] * math.pi/180
    dictionary['delta_r'][i] = dictionary['delta_r'][i] * math.pi/180
    dictionary['Ahrs1_Roll'][i] = dictionary['Ahrs1_Roll'][i] * math.pi/180
    dictionary['Ahrs1_Pitch'][i] = dictionary['Ahrs1_Pitch'][i] * math.pi/180
    dictionary['Ahrs1_bRollRate'][i] = dictionary['Ahrs1_bRollRate'][i] * math.pi/180
    dictionary['Ahrs1_bPitchRate'][i] = dictionary['Ahrs1_bPitchRate'][i] * math.pi/180
    dictionary['Ahrs1_bYawRate'][i] = dictionary['Ahrs1_bYawRate'][i] * math.pi/180
    dictionary['Dadc1_sat'][i] = dictionary['Dadc1_sat'][i] + 273.15
    dictionary['Dadc1_tat'][i] = dictionary['Dadc1_tat'][i] + 273.15
    dictionary['Dadc1_alt'][i] = dictionary['Dadc1_alt'][i] * 0.3048
    dictionary['Dadc1_bcAlt'][i] = dictionary['Dadc1_bcAlt'][i] * 0.3048
    dictionary['Dadc1_cas'][i] = dictionary['Dadc1_cas'][i] * 0.514444
    dictionary['Dadc1_tas'][i] = dictionary['Dadc1_tas'][i] * 0.514444
    dictionary['Dadc1_altRate'][i] = dictionary['Dadc1_altRate'][i] * 0.3048

# Find the mass at each point__________________________________________________

mass_original = 6704.218

lh_engine_FU = dictionary["lh_engine_FU"]
rh_engine_FU = dictionary["rh_engine_FU"]

mass= []

for i in range(len(lh_engine_FU)):
    mass0 = mass_original - (lh_engine_FU[i]) - (rh_engine_FU[i])
    mass.append(mass0)

#------------------------------------------------------------------------------
# -------------Find values for a table, work with Emilie-----------------------
#---------------------------------------------------------------------------
required = ["Dadc1_alt", "Dadc1_tas", "vane_AOA", "Ahrs1_Pitch"]
required2 = ["Dadc1_alt", "Dadc1_tas", "vane_AOA", "Ahrs1_Pitch", "mass", "CD0"]

required_dictionary = {}
for k in range(len(order)):
    array = []
    for g in range(len(required)):
        array.append(dictionary[required[g]][index[k]])
    array.append(mass[index[k]])

    required_dictionary[order[k]] = array

#print (required_dictionary)

fuel_used = []
for u in range(len(order)):
    fuel_used.append(mass_original - required_dictionary[order[u]][4])

#print(fuel_used)

for i in range(len(order)):
    print(required_dictionary[order[i]][0])

for i in range(len(order)):
    print(dictionary["Dadc1_tat"][index[i]])

print(required_dictionary)
#------------------------------------------------------------------------------
# -------------End of finding the values for the table ------------------------
#------------------------------------------------------------------------------