import numpy as np
from numpy.linalg import inv
from numpy.linalg import eig
import matplotlib.pyplot as plt
import time
'''
README: Run the file, uncomment lines 21, 25, 29(the lines running extended_kalman_filter code)
one after the other to see the plots, error values and coordinates. Type main() and run.
Change lambda at your own peril!
'''
def main():
  beacon_positions = read_position_from_csv("beacons.csv")

  # TODO find a random point inside a polygon as the initial position
  receiver_position = np.array([-3, 10])
  lamda = 1.3

  stationary_distances = read_distance_from_csv("stationary_z.csv")
  ## Following are three scenarios for the extended kalman filter.
  max_iterations = stationary_distances.shape[0]

  # Stationary receiver - stationary model
  extended_kalman_filter(beacon_positions, receiver_position, stationary_distances, max_iterations, lamda, "static")

  # Moving receiver - stationary model
  moving_distances = read_distance_from_csv("moving_z.csv")
  #extended_kalman_filter(beacon_positions, receiver_position, moving_distances, max_iterations, lamda, "static")

  # moving receiver - moving model
  m_state_vector = np.array([-3, 10, 0, 0]) #x, y, ux, uy
  #extended_kalman_filter(beacon_positions, m_state_vector, moving_distances, max_iterations, lamda, "dynamic")
  pass

def calculate_position(m_state_vector, t):
  position = []
  position[0] = m_state_vector[0] + t*(m_state_vector[2])
  position[1] = m_state_vector[1] + t*(m_state_vector[3])
  return position

def calculate_distance(bp_x, bp_y, receiver_position):
  distance = np.sqrt(((receiver_position[0] - bp_x)**2) + ((receiver_position[1] - bp_y)**2))
  #print("distance", distance)
  return distance

def compute_g(beacon_positions, receiver_position, actual_distances):
  g = np.zeros((beacon_positions.shape[1]))
  #print("g-shape",g.shape[0])
  for index in range(g.shape[0]):
    g[index] = actual_distances[index] - calculate_distance(beacon_positions[0][index], beacon_positions[1][index], receiver_position)
    #print("g",index, g[index])
  return g

def compute_dg(beacon_positions, receiver_position):
  dg = np.zeros(beacon_positions.shape)
  for index in range(beacon_positions.shape[1]):
    t = calculate_distance(beacon_positions[0][index], beacon_positions[1][index], receiver_position)
    #print(beacon_positions)
    dg[0][index] = 1/t*(beacon_positions[0][index] - receiver_position[0])
    dg[1][index] = 1/t*(beacon_positions[1][index] - receiver_position[1])
    #print("dg-shapes", index, dg[0][index], dg[1][index])
  return dg

def read_position_from_csv(p_csv_file):
  beacon_positions = np.loadtxt(p_csv_file, delimiter=',')
  #beacon_positions_transpose = np.transpose(beacon_positions)
  #print(beacon_positions_transpose.shape)
  return beacon_positions

def read_distance_from_csv(d_csv_file):
  beacon_distances = np.loadtxt(d_csv_file, delimiter=',')
  #print(beacon_distances.shape)
  return beacon_distances

def calculate_moving_distance(bp_x, bp_y, receiver_state_vector, time):
  distance = np.sqrt(((receiver_state_vector[0] + receiver_state_vector[2]*time - bp_x)**2) + ((receiver_state_vector[1] + receiver_state_vector[3]*time - bp_y)**2))

  return distance
def g_moving(beacon_positions, receiver_state_vector, actual_distances, time):
  #print("BEACON HSHAPE", beacon_positions.shape)
  g = np.zeros(beacon_positions.shape[1])
  #print("G HSHAPE", g.shape)
  for index in range(g.shape[0]):
    g[index] = actual_distances[index] - calculate_moving_distance(beacon_positions[0][index], beacon_positions[1][index], receiver_state_vector, time)
  return g

def dg_moving(beacon_positions, receiver_state_vector, time):
  dg = np.zeros((beacon_positions.shape[1], receiver_state_vector.shape[0]))
  for index in range(beacon_positions.shape[1]):
    d = calculate_distance(beacon_positions[0][index], beacon_positions[1][index], receiver_state_vector)
    #print(beacon_positions)
    dg[0][index] = 1/d*(beacon_positions[0][index] - receiver_state_vector[0] - time*receiver_state_vector[2])
    dg[1][index] = 1/d*(beacon_positions[1][index] - receiver_state_vector[1] - time*receiver_state_vector[3])
    dg[2][index] = 1/d*(beacon_positions[0][index] - receiver_state_vector[0] - time*receiver_state_vector[2])*time
    dg[3][index] = 1/d*(beacon_positions[1][index] - receiver_state_vector[1] - time*receiver_state_vector[3])*time
  return dg

def extended_kalman_filter(beacon_positions, receiver_position, actual_distances_list, iterations, lamda, movement_type):
  if movement_type == "static":
    H = np.zeros((2,2))
    g_e =[]
    ite = []
    phi = [receiver_position]

    # generate 2D grid
    plt.figure(1, figsize=(7,7))
    plt.ylabel('Y coordinate ')
    plt.xlabel('X coordinate')

    x_grid, y_grid = np.mgrid[-5:25:0.01, -5:25:0.01]
    plt.xlim([-5, 25])
    plt.ylim([-5, 25])

    # draw contour lines of g
    p_1,p_2 = beacon_positions[0, :], beacon_positions[1, :]
    #x, y = phi[-1][0], phi[-1][1]
    f_all = np.zeros((4, *x_grid.shape))
    for i in range(0, 4):
        f_all[i, :, :] += actual_distances_list[99][i] - np.sqrt((x_grid - p_1[i])**2 + (y_grid - p_2[i])**2)

    f_norm = np.linalg.norm(f_all, axis=0)
    levels = 15
    plt.contour(x_grid, y_grid, f_norm, levels, label="Error")
    plt.plot(phi[-1][0], phi[-1][1], '*', markersize=10, color='red', label="phi coordinates")
    fig1 = plt.gcf()
    plt.legend()
    for i in range(0,beacon_positions.shape[1]):
      plt.plot(beacon_positions[0][i], beacon_positions[1][i], '+', markersize=10, color='blue', label="beacon-positions")
      plt.legend()
    for index in range(iterations):
      actual_distances = actual_distances_list[index]
      #print(actual_distances)
      for i in range(beacon_positions.shape[1]):
        phi_prev = phi[-1]
        g = compute_g(beacon_positions, phi_prev, actual_distances)
        dg = compute_dg(beacon_positions, phi_prev)
        #print("dg-shape", dg.shape)
        C = -dg.T
        #print("C-shape", C.shape)
        H = lamda*(H) + C.T@C
        #print("H-shape", H.shape)
        z = g.reshape(4, 1) - (dg.T@receiver_position).reshape(4, 1)
        #print("z-shape", z.shape)
        #print(phi[-1].shape, (z.reshape(4,1) - (C@(phi[-1])).reshape(4,1)).shape )
        #phi.append((phi_prev.reshape(2,1) + np.linalg.pinv( H )@( C.T )@( z.reshape(4,1) - (C@(phi_prev)).reshape(4,1) )).reshape(2,))
        phi.append((phi_prev.reshape(2,1) - np.linalg.pinv( H )@dg.reshape(2, 4)@g.reshape(4, 1)).reshape(2,))
        g_e.append(mean_error(g))
        ite.append(index)
        plt.plot(phi[-1][0], phi[-1][1], '*', markersize=10, color='red')
        fig1.canvas.draw()

        print(i, index, phi_prev)
        print("Error", mean_error(g))

    plt.figure(2, figsize=(7,7))
    #print(ite)
    plt.ylabel('Mean error ')
    plt.xlabel('Iterations')
    plt.plot(ite, g_e)

  elif movement_type == "dynamic":
    H = np.zeros((4,4))
    g_e =[]
    ite = []
    phi = [receiver_position]
    # generate 2D grid
    plt.figure(1, figsize=(7,7))
    plt.ylabel('Y coordinate ')
    plt.xlabel('X coordinate')

    x_grid, y_grid = np.mgrid[-5:25:0.01, -5:25:0.01]
    plt.xlim([-5, 25])
    plt.ylim([-5, 25])

    # draw contour lines of g
    p_1,p_2 = beacon_positions[0, :], beacon_positions[1, :]
    #x, y = phi[-1][0], phi[-1][1]
    f_all = np.zeros((4, *x_grid.shape))
    for i in range(0, 4):
        f_all[i, :, :] += actual_distances_list[99][i] - np.sqrt((x_grid - p_1[i])**2 + (y_grid - p_2[i])**2)

    f_norm = np.linalg.norm(f_all, axis=0)
    levels = 15
    plt.contour(x_grid, y_grid, f_norm, levels, label="Error")
    plt.plot(phi[-1][0], phi[-1][1], '*', markersize=10, color='red', label="phi coordinates")
    fig1 = plt.gcf()
    plt.legend()

    for index in range(iterations):
      actual_distances = actual_distances_list[index]
      print(actual_distances)
      for i in range(beacon_positions.shape[1]):
        phi_prev = phi[-1]
        g = g_moving(beacon_positions, phi_prev, actual_distances, index)
        dg = dg_moving(beacon_positions, phi_prev, index)
        #print("dg-shape", dg.shape)
        C = -dg.T
        #print("C-shape", C.shape)
        H = lamda*(H) + C.T@C
        #print("H-shape", H.shape)
        z = g.reshape(4, 1) - (dg.T@receiver_position).reshape(4, 1)
        #print("z-shape", z.shape)
        #print(phi[-1].shape, (z.reshape(4,1) - (C@(phi[-1])).reshape(4,1)).shape )
        #phi.append((phi_prev.reshape(2,1) + np.linalg.pinv( H )@( C.T )@( z.reshape(4,1) - (C@(phi_prev)).reshape(4,1) )).reshape(2,))
        phi.append((phi_prev.reshape(4,1) - np.linalg.pinv( H )@dg.reshape(4, 4)@g.reshape(4, 1)).reshape(4,))
        g_e.append(mean_error(g))
        ite.append(index)
        plt.plot((phi[-1][0] + index*phi[-1][2]), (phi[-1][1] + index*phi[-1][3]), '*', markersize=10, color='red')
        fig1.canvas.draw()
        print(i, index, phi_prev)
        print("Error", mean_error(g))

    plt.figure(2, figsize=(7,7))
    plt.ylabel('Mean error ')
    plt.xlabel('Iterations')
    #print(ite)
    plt.plot(ite, g_e)

  pass

def mean_error(g):
  mean = 0
  for i in g:
    mean = mean + abs(i/4)
  return mean

main()


# -*- encoding: utf-8 -*-

'''
把当前目录下的图片文件按拍摄日期进行重命名
如果没有拍摄日期，按修改日期重命名

如果遇到同一时刻的两张图，不覆盖，给同时刻的后面的图片的名字里加上时间戳，
然后人工处理。
'''

import os
import datetime
import shutil
import exifread
import time

def get_file_modification_date(file_name):
    '''
    获得文件修改时间
    '''
    # 获得文件创建时间戳
    file_modification_timestamp = os.path.getmtime(file_name)
    # 转换为创建时间
    file_modification_time = datetime.datetime.fromtimestamp(file_modification_timestamp)
    # 转换为创建日期
    file_modification_date = file_modification_time.strftime('%Y-%m-%d %H.%M.%S')
    return file_modification_date

def get_file_date(file_name):
    '''
    获得文件拍摄时间，如果没有拍摄时间，用修改时间代替
    '''
    try:
        with open(file_name, 'rb') as file:
            original_data = exifread.process_file(file)
    except:
        return "000"

    try:
        file_origin_date = original_data['EXIF DateTimeOriginal']
        file_date = str(file_origin_date)[:10].replace(':', '-') # 文件日期
        file_time = str(file_origin_date)[11:19].replace(':', '.') # 文件时间
        final_file_datetime = file_date + " " + file_time # 文件日期+时间

        return str(final_file_datetime)
    except:
        return get_file_modification_date(file_name)

def get_current_dir_files():
    '''
    获得当前目录的所有文件，排除.py文件
    '''
    # 获得当前目录的所有文件和文件夹
    files_and_dirs = os.listdir()
    # 过滤文件夹
    files = [file for file in files_and_dirs if (os.path.isfile(file) and file[-3:] != '.py')]
    return files

def create_dir(dir_name):
    # 文件夹不存在是创建文件夹
    if not os.path.isdir(dir_name):
        os.mkdir(dir_name)

if __name__ == '__main__':
    files = get_current_dir_files()
    print('共%d个文件' % len(files))

    for index, file in enumerate(files):
        file_date = get_file_date(file)

        # 跳过异常文件
        if file_date == "000":
            continue

        # 源文件名和目标文件名
        source_file_name = file
        file_type = os.path.splitext(file)[-1]
        target_file_name = file_date + file_type

        # 跳过命名正确的文件
        if source_file_name == target_file_name:
            continue

        try:
            os.rename(source_file_name, target_file_name)
        except:
            # 遇到同时刻的文件，名字里加上现在的时间戳
            target_file_name = file_date + "(" + str(time.time()) + ")" + file_type
            os.rename(source_file_name, target_file_name)

        print('第%d个' % (index + 1))
    print('完成!!!')


import itertools
import os
import re
import xml.etree.ElementTree as ET

# pattern to filiter only amharic words
pattern  = re.compile(r'[ሀ-ፚ\s]+', re.DEBUG)
input_dir = "dataset/text/AA"
output_path = "dataset/amwiki.txt"
texts = []
for file in os.listdir(input_dir):
    with open(os.path.join(input_dir, file)) as f:
        it = itertools.chain('<root>', f, '</root>')
        root = ET.fromstringlist(it)
    for doc in root.findall("doc"):
        doc_txt = ""
        for match in pattern.findall(doc.text.strip()):
            doc_txt+= match.strip().replace("\n"," ")
        texts.append(doc_txt)

all_texts = "\n".join(texts)

with open (output_path,"w+") as f:
    f.write(all_texts + "\n")


from datetime import datetime

class Error(Exception):
    pass

class RequestError(Error):
    def __init__(self, func, module, status):
        message = error_msg(f"RequestError: {func} - {module}", 200, status)
        add_log(message)
        self.message = message

class TimeRequestError(Error):
    def __init__(self, func, module, tries):
        message = error_msg(f"TimeRequestError: {func} - {module}", "", "Exceeded the limit of tries ({})".format(tries))
        add_log(message)
        self.message = message

class NotSubset(Error):
    def __init__(self, func, module, exp, rec):
        message = error_msg(f"NotSubset: {func} - {module}", exp, rec)
        add_log(message)
        self.message = message

def writeError(errorType, func, module, exp, rec):
        message = error_msg(str(errorType) + ": " + str(func) + " - " + str(module), exp, rec)
        add_log(message)
        return message

def error_msg(where, expected, received):
    where = str(where)
    expected = str(expected)
    received = str(received)

    date = datetime.now()
    date = date.strftime("%d/%m/%Y %H:%M:%S")
    msg = date + " Error local: " + where + "\n\tExpected: " + expected + "\n\tReceived: " + received + "\n"

    return msg

def add_log(msg):
    with open('log_file', 'a') as log:
        log.write(msg)


"""Unit tests for netbox-netisp ."""

from django.test import TestCase
from netbox_netisp.models import Customer, Address, Account
from datetime import datetime

class CustomerTestCase(TestCase):
    """Test the Customer"""

    def setUp(self):
        """Create a customer object with sample data"""
        Customer.objects.create(
            first_name="Schylar",
            middle_name="S",
            last_name="Utley",
            birthdate=datetime(1997, 2, 10, 16, 13, 49, 71549),
        )

    def test_customer_name(self):
        """Verify that the name is being joined correctly"""
        sutley = Customer.objects.get(first_name="Schylar")
        self.assertEqual(sutley.name(), "Schylar S Utley")

    def test_customer_age(self):
        """Verify that the customer has a valid age"""
        sutley = Customer.objects.get(first_name="Schylar")
        self.assertGreaterEqual(sutley.age(), 1)

class AddressTestCase(TestCase):
    def setUp(self):
        addr1 = Address.objects.create(
            street_number=1502,
            street_name="Main",
            street_suffix="Ave",
            city="Webb City",
            state_code="MO",
            zip="64870-1234",
            slug="f",
        )

    def test_address_name_without_ord(self):
        address = Address.objects.get(slug="f")
        self.assertEqual("1502 Main Ave, Webb City MO, 64870-1234", str(address))

    def test_address_name_with_ord(self):
        address = Address.objects.get(slug="f")
        address.street_ordinance = "S"
        self.assertEqual("1502 S. Main Ave, Webb City MO, 64870-1234", str(address))

class AccountTestCase(TestCase):
    """Test the Account"""

    def setUp(self):
        """Create an account object with sample data"""
        customer = Customer.objects.create(
            first_name="Tom",
            middle_name="S",
            last_name="Riddle",
            birthdate=datetime(1997, 2, 10, 16, 13, 49, 71549),
        )

        Account.objects.create(primary_applicant=customer)

    def test_applicant_name(self):
        account = Account.objects.get(pk=1)
        self.assertEqual("Tom S Riddle", account.primary_applicant.name())


#   1. R2를 음수가 아닌 0.5 이하로 줄이기
#   2. 데이터는 건들지 말것
#   3. 레이어는 인풋 아웃풋 포함 7개 이상
#   4. batch_size=1
#   5. 히든레이어의 노드는 각각 10개 이상 100개 이하
#   6. train 70%
#   7. epoch 100번 이상
#   8. loss지표는 mse 또는 mae
#   9. activation 사용 금지

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
import numpy as np
from sklearn.model_selection import train_test_split

#1. 데이터
x = np.array(range(1,21))
y = np.array(range(1,21))

x_train, x_test, y_train, y_test = train_test_split(x, y,
    train_size=0.7, shuffle=True, random_state=123
)

#2. 모델구성
model = Sequential()
model.add(Dense(10, input_dim = 1))
model.add(Dense(10))
model.add(Dense(10))
model.add(Dense(10))
model.add(Dense(10))
model.add(Dense(1))

#3. 컴파일, 훈련
model.compile(loss='mse', optimizer='adam',
              metrics=['mae'])                                  #   metrics는 훈련에 영향을 미치진 않지만 값은 나온다.
                                                                #   acc와 accuracy는 동일
                                                                #   error가 그 다음 가중치 갱신을 할때 영향을 미친다.(훈련에 영향을 미친다.)
model.fit(x_train, y_train, epochs=100, batch_size=1)           #   가중치 생성

#4. 평가, 예측
loss = model.evaluate(x_test, y_test)                           #   test 데이터로 평가
print('loss : ', loss)

y_predict = model.predict(x_test)                               #   최적의 가중치

print("=============")
print(y_test)
print(y_predict)
print("=============")

from sklearn.metrics import mean_squared_error, r2_score
def RMSE(y_test, y_predict):
    return np.sqrt(mean_squared_error(y_test, y_predict))
print("RMSE : ", RMSE(y_test, y_predict))

r2 = r2_score(y_test, y_predict)
print("R2 : ", r2)


#!/usr/bin/python3

import datetime

try:
    from Crypto.Hash import keccak
    sha3_256 = lambda x: keccak.new(digest_bits=256, data=x).digest()
except:
    import _sha3
    sha3_256 = lambda x: _sha3.sha3_256(x).digest()

import sys
import binascii

registryStarted = 1493895600
launchLength = 4838400

def sha3(seed):
    return sha3_256(str(seed).encode('ascii'))

def getAllowedTime(name):
    hash = binascii.hexlify(sha3(name))
    return registryStarted + ((launchLength * (int(hash, 16) >> 128)) >> 128)

def main(args):
    utime = getAllowedTime(args[0])
    now = datetime.datetime.now()
    release = datetime.datetime.fromtimestamp(utime)
    if (release < now):
        print("AVAILABLE NOW")
    else:
        print("Available", release.strftime("%B %d %H:%M:%S"))
        print("Remaining:", str(release - now))

if __name__ == "__main__":
    main(sys.argv[1:])


#! /usr/bin/env python
# -*- coding: utf-8 -*-
# __author__ = "Q1mi"
# Date: 2018/5/24
from django import template
from blog import models

register = template.Library()

@register.inclusion_tag("panel.html")
def show_panel(username):
    user = models.UserInfo.objects.filter(username=username).first()
    # 拿到当前用户的博客对象
    blog = user.blog
    # 方式一：查询当前站点的文章分类，在前端用连表查询取分类对应的文章数
    # category_list = models.Category.objects.filter(blog=blog)
    # 方式二：利用分组查询出每一个分类以及分类对应的文章数
    from django.db.models import Count
    category_list = models.Category.objects.filter(blog=blog).annotate(c=Count("article")).values("title", "c")
    print(category_list)
    # 查询当前站点的文章标签
    tag_list = models.Tag.objects.filter(blog=blog).annotate(c=Count("article")).values("title", "c")
    # 按年月归档文章
    archive_list = models.Article.objects.filter(user=user).extra(
        select={"create_ym": "DATE_FORMAT(create_time, '%%Y-%%m')"}
    ).values("create_ym").annotate(c=Count("nid")).values("create_ym", "c")

    return {
        "username": username,
        "category_list": category_list,
        "tag_list": tag_list,
        "archive_list": archive_list
    }


'''Controllers used by AR PoC Server'''
from flask import Flask, request
from ar_poc_server.api import routes

APP = Flask(__name__)

@APP.route("/")
def hello():
    '''Hello Controller'''
    return "Hello World"

@APP.route(routes.AR_OBJECT_DETECTION_V1, methods=['POST'])
def ar_object_detection():
    '''This controller accepts POST request with type multipart/form-data.
    Image file comes as form input field named "file"'''
    files = request.files['file']
    APP.logger.error(files.read())
    return "OK"

if __name__ == "__main__":
    APP.run()


import networkx as nx
import matplotlib.pyplot as plt

def disassemble_bytecode(bytecode):
    instructions = []
    index = 0
    length = len(bytecode)
    graph = nx.DiGraph()

    while index < length:
        opcode = bytecode[index]
        instruction = ""

        if opcode < 0x60:
            instruction = f"PUSH{opcode - 0x5f}"
            index += opcode - 0x5e

        elif opcode == 0x60:
            instruction = "PUSH1"
            index += 2

        elif opcode == 0x61:
            instruction = "PUSH2"
            index += 3

        # ... continue handling existing opcodes ...

        elif opcode == 0xfa:
            instruction = "STATICCALL"
            index += 1

        elif opcode == 0xfd:
            instruction = "REVERT"
            index += 1

        elif opcode == 0xfe:
            instruction = "INVALID"
            index += 1

        elif opcode == 0xff:
            instruction = "SELFDESTRUCT"
            index += 1

        else:
            instruction = "UNKNOWN"
            index += 1

        instructions.append(instruction)
        graph.add_edge(instruction, index)

    return instructions, graph

# Example usage
bytecode = b'\x60\x60\x80\xfd\xff'
instructions, graph = disassemble_bytecode(bytecode)

# Print instructions
for instruction in instructions:
    print(instruction)

# Visualize the graph
pos = nx.spring_layout(graph)
nx.draw_networkx(graph, pos, with_labels=True)
plt.show()


"""Definition of the WhoSaid model"""
import torch
from torch import nn
import torch.nn.functional as F

DEVICE = "cuda" if torch.cuda.is_available() else "cpu"

EMBEDDING_DIM = 256

def to_tensors(datasets):
    """Convert dataset indices to torch tensors and move them to target device"""
    train = []
    val = []
    for sample in datasets["train"]:
        index = torch.tensor(datasets["speakers"].index(sample[1]),dtype=torch.long).view(1)
        tensors = [torch.tensor(sample[0]).to(DEVICE), index.to(DEVICE)]
        train.append(tensors)

    for sample in datasets["validation"]:
        index = torch.tensor(datasets["speakers"].index(sample[1]),dtype=torch.long).view(1)
        tensors = [torch.tensor(sample[0]).to(DEVICE), index.to(DEVICE)]
        val.append(tensors)

    datasets["train"] = train
    datasets["validation"] = val

    return datasets

class WhoSaidModel(nn.Module):
    """The WhoSaid Model"""
    def __init__(self, vocab_size, nr_speakers):
        super(WhoSaidModel, self).__init__()
        self.hidden_size = EMBEDDING_DIM * 2
        self.embedding = nn.Embedding(vocab_size, EMBEDDING_DIM).to(DEVICE)
        self.gru = nn.GRU(input_size = EMBEDDING_DIM,
                          num_layers=3,
                          hidden_size = self.hidden_size,
                          dropout=0.5).to(DEVICE)
        self.linear = nn.Linear(self.hidden_size, nr_speakers).to(DEVICE)
        self.relu = nn.ReLU()

    def forward(self, token):
        """forward pass"""
        state, hidden = token
        state = self.embedding(state)
        state, hidden = self.gru(state, hidden)
        state = self.linear(state)
        state = F.log_softmax(state, dim=2)
        return state, hidden

    def init(self):
        """weight initialization"""
        weight = next(self.parameters()).data
        hidden = weight.new(3, 1, self.hidden_size).zero_()
        return hidden


"""Object Dict from tornado.util
Reduce contained code to it's absolute minimum and import what you need.
"""

from pprint import pprint
from typing import Dict, List, Any, AnyStr, NamedTuple, Tuple, Union, Sequence
import json
import os, sys

__all__ = [
    'ObjectDict'
]

class ObjectDict(Dict[str, Any]):
    """Makes a dictionary behave like an object, with attribute-style access.
    Analogous to how javascript objects work.
    From tornado.util
    """

    _version = "0.2"

    def __getattr__(self, name: str) -> Any:
        try:
            return self[name]
        except KeyError:
            raise AttributeError(name)

    def __setattr__(self, name: str, value: Any) -> None:
        self[name] = value

    def printmyjson(self):
        """Print a json string representation of self"""
        return json.dumps(self)

    @staticmethod
    def printjson(val):
        return json.dumps(val)

d = ObjectDict({"test": True})

def test():
    '''This will only execute if executed as a script'''
    d = ObjectDict()
    d.one = 1
    d.two = 2
    print(json.dumps(d))

if __name__ == "__main__":
    test()


import torch
from helper import get_formatted_time
import helper

def train_model(model, criterion, optimizer, train_loader, valid_loader, epochs, run_on_gpu):
    """
    Train a model on a dataset.

    :param model: The model to train.
    :param criterion: The loss function.
    :param optimizer: The optimizer.
    :param train_loader: The training data loader.
    :param valid_loader: The validation data loader.
    :param epochs: The number of epochs to train for.
    :param run_on_gpu: If true, process function on GPU
    :return: The trained model.
    """

    print(helper.get_formatted_time(), "train_model start. run_on_gpu:", run_on_gpu)
    # Move the model to the GPU if available.
    if run_on_gpu and torch.cuda.is_available():
        print(helper.get_formatted_time(), "train_model before: device = torch.device('cuda')")
        device = torch.device("cuda")
        print(helper.get_formatted_time(), "train_model before: model.to(device)")
        model.to(device)
        print(helper.get_formatted_time(), "GPU is available and being used")
    else:
        device = torch.device("cpu")
        if run_on_gpu:
            print("GPU is not available, using CPU instead")
        else:
            print("using CPU as requested")

    print("\n*** train_model: epochs: {} ***\n".format(epochs))

    # Start training
    print("Start training time:", get_formatted_time())
    running_loss = 0
    for epoch in range(epochs):
        print(f"{get_formatted_time()} epoch {epoch + 1}/{epochs} start")
        model.train()
        for inputs, labels in train_loader:
            # Move the input and label tensors to the GPU.
            inputs, labels = inputs.to(device), labels.to(device)

            # Forward pass
            outputs = model(inputs)
            loss = criterion(outputs, labels)

            # Backward pass
            optimizer.zero_grad()
            # print("before loss.backward()")
            loss.backward()
            # print("after  loss.backward()")
            optimizer.step()
            running_loss += loss.item()

        # Evaluate the model on the validation set
        print(f"{get_formatted_time()} validation start")
        model.eval()
        with torch.no_grad():
            val_loss = 0
            val_accuracy = 0
            for inputs, labels in valid_loader:
                inputs, labels = inputs.to(device), labels.to(device)
                outputs = model(inputs)
                val_loss += criterion(outputs, labels).item()

                # Calculate accuracy
                ps = torch.exp(outputs)
                top_p, top_class = ps.topk(1, dim=1)
                equals = top_class == labels.view(*top_class.shape)
                val_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()

        print(f"{get_formatted_time()} Epoch {epoch + 1}/{epochs}.. "
              f"Train loss: {running_loss / len(train_loader):.3f}.. "
              f"Validation loss: {val_loss / len(valid_loader):.3f}")
        print(f"Validation accuracy: {val_accuracy / len(valid_loader):.3f}")

    device = torch.device("cpu")
    model.to(device)
    print(f"{get_formatted_time()} End training time:")
    print(helper.get_formatted_time(), "train_model end")
    return model


import torch
import math
import h5py
import cv2
import numpy as np
import matplotlib.pyplot as plt
from face_swapping import *
from torch.autograd import Variable
import torch.nn as nn
from tqdm import tqdm

def find_g(bfm, alpha=None, delta=None, save_G=False, name="morphable2.obj"):
    '''
    Obtain point cloud, triangles and colors of BFM.
    '''

    # PCA for shape
    shape_mean = np.asarray(bfm['shape/model/mean'], dtype=np.float32)
    shape_mean = np.reshape(shape_mean, (-1, 3))

    shape_base = np.asarray(bfm['shape/model/pcaBasis'], dtype=np.float32)
    shape_base = np.reshape(shape_base, (shape_base.shape[0] // 3, 3, -1))[:,:,:30]

    shape_var = np.asarray(bfm['shape/model/pcaVariance'], dtype=np.float32)[:30]
    shape_var = np.sqrt(shape_var)

    # PCA for expression
    exp_mean = np.asarray(bfm['expression/model/mean'], dtype=np.float32)
    exp_mean = np.reshape(exp_mean, (-1, 3))

    exp_base = np.asarray(bfm['expression/model/pcaBasis'], dtype=np.float32)
    exp_base = np.reshape(exp_base, (exp_base.shape[0] // 3, 3, -1))[:,:,:20]

    exp_var = np.asarray(bfm['expression/model/pcaVariance'], dtype=np.float32)[:20]
    exp_var = np.sqrt(exp_var)

    if alpha is None:
        alpha = torch.FloatTensor(30).uniform_(-1, 1)
    if delta is None:
        delta = torch.FloatTensor(20).uniform_(-1, 1)

    shape_mean = torch.from_numpy(shape_mean)
    shape_base = torch.from_numpy(shape_base)
    shape_var = torch.from_numpy(shape_var)
    exp_mean = torch.from_numpy(exp_mean)
    exp_base = torch.from_numpy(exp_base)
    exp_var = torch.from_numpy(exp_var)

    G = shape_mean + shape_base @ (shape_var * alpha) + exp_mean + exp_base @ (exp_var * delta)

    color = np.asarray(bfm['color/model/mean'], dtype=np.float32)
    color = np.reshape(color, (color.shape[0] // 3, 3))

    shape_rep = np.asarray(bfm['shape/representer/cells'], dtype=np.int32).T
    if save_G:
        save_obj(name, G, color, shape_rep)

    return G, color, shape_rep

def pinhole(G, Rots, t, h, w, fov=0.5, save_model=False, name="rotation.obj", shape_rep=None):
    '''
    Calcualte and apply the pinhole camera of a point cloud with rotation and
    translation.
    '''

    # Create rotation matrix
    Rx, Ry, Rz = Rots
    Rx, Ry, Rz = math.radians(Rx), math.radians(Ry), math.radians(Rz)
    Rotx = np.array([[1, 0, 0],
                    [0, np.cos(Rx), -np.sin(Rx)],
                    [0, np.sin(Rx), np.cos(Rx)]])
    Roty = np.array([[np.cos(Ry), 0, np.sin(Ry)],
                    [0, 1, 0],
                    [-np.sin(Ry), 0, np.cos(Ry)]])
    Rotz = np.array([[np.cos(Rz), -np.sin(Rz), 0],
                    [np.sin(Rz), np.cos(Rz), 0],
                    [0, 0, 1]])

    R = (torch.from_numpy(Rotx) @ torch.from_numpy(Roty) @ torch.from_numpy(Rotz)).float()

    bfm = h5py.File("model2017-1_face12_nomouth.h5", 'r')
    color = np.asarray(bfm['color/model/mean'], dtype=np.float32)
    color = np.reshape(color, (color.shape[0] // 3, 3))

    # Roto translate the model
    G = G @ R.T + t

    # Save model if wanted
    if save_model:
        save_obj(name, G, color, shape_rep)

    # Convert to homogeneous coordinates
    G_ = torch.cat((G, torch.ones(G.shape[0],1)), dim=1)

    # Find ranges
    aspect_ratio = w/h
    vn = 600
    vf = 1000
    vt = np.tan(fov/2) * vn
    vb = - vt
    vr = vt * aspect_ratio
    vl = -vt * aspect_ratio

    # Create viewport matrix
    V = np.array([[(vr-vl)/2, 0, 0, (vr+vl)/2],
                  [0, (vt-vb)/2, 0, (vt+vb)/2],
                  [0, 0, 1/2, 1/2],
                  [0, 0, 0, 1]])
    # Create perspective projection matrix
    P = np.array([[2*vn/(vr-vl), 0, (vr+vl)/(vr-vl), 0],
                 [0, 2*vn/(vt-vb), (vt+vb)/(vt-vb), 0],
                 [0, 0, -(vf+vn)/(vf-vn), -2*vf*vn/(vf-vn)],
                 [0, 0, -1, 0]])

    # Create the pinhole camera and apply.
    Pi = torch.from_numpy(V @ P).float()

    G_3D = (G_ @ Pi.T)

    # Dividing by homogenous coordinate.
    G_3D = (G_3D / G_3D[:, 3][:, None])

    return G_3D[:, :3]

def get_landmarks(G_3D, plot=False):
    '''
    Obtain and possibly plot the landmark points of 3D point cloud.
    '''
    with open("Landmarks68_model2017-1_face12_nomouth.anl", 'r') as f:
        idxs = f.read().split('\n')
        idxs = [int(x) for x in idxs]
        landmarks = G_3D[idxs, :2]

        if plot:
            plt.scatter(landmarks[:, 0], landmarks[:, 1])
            plt.show()

        return landmarks

class EnergyOptim(nn.Module):
    '''
    Neural net for the parameter optimization.
    '''
    def __init__(self):

        super(EnergyOptim, self).__init__()

        self.alpha = nn.Parameter(torch.FloatTensor(30).uniform_(-1, 1), requires_grad=True)
        self.delta = nn.Parameter(torch.FloatTensor(20).uniform_(-1, 1), requires_grad=True)
        self.Rots = nn.Parameter(torch.FloatTensor([0, 0, 180]), requires_grad=True)
        self.t = nn.Parameter(torch.FloatTensor([0, 0, -500]), requires_grad=True)

    def forward(self, bfm, h, w):
        # Find 3D face representation.
        face_3D, _, shape_rep = find_g(bfm, alpha=self.alpha, delta=self.delta)
        # Project onto 2D using pinhole and predict landmark points.
        face_2D = pinhole(face_3D, self.Rots, self.t, h, w)
        pred = get_landmarks(face_2D)

        return (face_3D, face_2D, pred, shape_rep)

def loss_c(landmarks, pred, alpha, delta, l_alpha=0, l_delta=0):
    '''
    Calculate Energy minimization loss.
    '''
    Llan = torch.sum(torch.square(pred - landmarks)) / 68
    Lreg = l_alpha * torch.sum(torch.square(alpha)) + l_delta * torch.sum(torch.square(delta))
    return (Llan + Lreg)

def train_model(model, bfm, gt, h, w, iters=10000):
    '''
    Return a trained model.
    '''
    # Set the optimizer.
    optimizer = torch.optim.Adam(model.parameters(), lr=0.05)

    losses = []
    # Training loop.
    for i in range(iters):
        # Predict and calculate loss.
        _, _, pred, _ = model(bfm, h, w)
        loss = loss_c(gt, pred, model.alpha, model.delta)
        losses.append(loss.item())

        # Model step.
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # Print every 1000 iterations.
        if i % 1000 == 0:
            print(f"Finished iter {i}")
            print('loss:', loss)

    return(model, losses)

if __name__ == '__main__':
    '''
    Run the training of the parameters and plot first and final prediction
    vs the ground truth.
    '''
    # Load bfm
    bfm = h5py.File('model2017-1_face12_nomouth.h5', 'r')

    img = cv2.imread('beyonce.jpg')[:, :, ::-1]  # Convert from BGR to RGB
    h, w, _ = img.shape

    # Get gt landmarks
    gt_landmarks = detect_landmark(img)
    gt_landmarks[:,0] = gt_landmarks[:,0] - w/2
    gt_landmarks[:,1] = gt_landmarks[:,1] - h/2

    # Create and train model.
    model = EnergyOptim()
    model, losses = train_model(model, bfm, torch.from_numpy(gt_landmarks), h, w)
    print(model.alpha, "\n")
    print(model.delta, "\n")
    print(model.Rots, "\n")
    print(model.t, "\n")

    # Plot final prediction.
    _, _, pred_landmarks = model(bfm, h, w).detach().numpy()
    print(pred_landmarks.shape)
    plt.scatter(pred_landmarks[:,0], pred_landmarks[:,1], color='red', label='prediction')
    plt.scatter(gt_landmarks[:,0], gt_landmarks[:,1], color='blue', label='ground truth')
    plt.legend()
    plt.show()


import pyodbc as po
import pandas as pd
#import sqlite3
#import mysql.connector
#import mysql.connector
#from mysql.connector import Error
#from mysql.connector import Error
#import re, nltk
import pyodbc as po
import pandas as pd
from datetime import datetime, timedelta

#import sqlite3
#import mysql.connector
#from mysql.connector import Error
#import re, nltk
import nltk
import numpy as np
from nltk.stem.porter import PorterStemmer
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
import nltk
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
#from sklearn.naive_bayes import MultinomialNB
#from sklearn.linear_model import LogisticRegression
#from sklearn import metrics
# Machine Learning
from sklearn.model_selection import train_test_split
#from sklearn.model_selection import cross_val_score
#from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
#from sklearn.naive_bayes import GaussianNB
#from sklearn.svm import LinearSVC
#from sklearn.model_selection import cross_val_predict
#from sklearn.metrics import confusion_matrix
#from sklearn.metrics import precision_score, recall_score
#from sklearn.metrics import f1_score
#from sklearn.metrics import precision_recall_curve
#from sklearn.metrics import roc_curve
#from sklearn.metrics import roc_auc_score
#from sklearn.metrics import classification_report
nltk.download('stopwords')
lemmatizer = WordNetLemmatizer()
stemmer = PorterStemmer()
english_stops = stopwords.words('english')
import numpy as np
# Connection variables

    # Connection string
cnxn = po.connect('Driver={SQL Server Native Client 11.0};SERVER=' +
server+';DATABASE='+database+';UID='+username+';PWD=' + password)
cursor = cnxn.cursor()

storedProc = " EXEC EMS.SpGetAccountIdListIdTimeDifferenceForAnalysis"
result=pd.read_sql_query(storedProc,cnxn)
#cursor.execute(storedProc)
df2=pd.DataFrame(result,columns=['ACCOUNTID','LISTID'])
print(df2)

#median
def my_median(sample):
    n = len(sample)
    index = n // 2
     # Sample with an odd number of observations
    if n % 2:
         return sorted(sample)[index]
     # Sample with an even number of observations
    return sum(sorted(sample)[index - 1:index + 1]) / 2

for index,row in df2.iterrows():
    storedProc1 = "EXEC EMS.SpGetAggregateCampaignDataForAnalysis @AccountId="+str(row['ACCOUNTID'])+",@ListId="+str(row['LISTID'])+""
    result1=pd.read_sql_query(storedProc1,cnxn)
    #cursor.execute(storedProc1)
    df1=pd.DataFrame(result1,columns=['ListID',	'SegmentID',	'CampaignName','AccountID',	'CampaignSubject'	,'List'	,'Segment','Sent'	,'Delivered',	'Open',	'Click',	'Bounced',	'UniqueOpen',	'UniqueClick',	'Unsubscribe',	'Spam'	,'VirtualSpam',	'SendDate'])
    print(df1)
    data=df1
    data["Percentage_open"]=(pd.to_numeric(data['UniqueOpen']/data['Delivered']))*100
    data=data.sort_values("Percentage_open",axis=0,ascending=False)
    value_median=my_median(data["Percentage_open"])
#print(value_median)
    final_median=(20/100)*value_median
#final_median
    sd_median=value_median-final_median
    f=np.percentile(data["Percentage_open"], sd_median)
    data["Open_Percentage"]=np.where(data['Percentage_open']>=f,'1','0')

    from sklearn.feature_extraction.text import TfidfTransformer
    cv=TfidfVectorizer(analyzer='word',min_df=.005, max_df=.9,ngram_range=(2,3))
    review_tf = cv.fit_transform(data['CampaignSubject'])
    review_tf_nd = review_tf.toarray()
    df = pd.DataFrame(review_tf_nd, columns=cv.get_feature_names())
    print(df)
    model = pd.merge(data, df, left_index=True, right_index=True)
#file=model.to_csv('D:/new/result.csv')
    ml_model=model.drop(['ListID',	'SegmentID',	'CampaignName','AccountID',	'CampaignSubject'	,'List'	,'Segment','Sent'	,'Delivered',	'Open',	'Click',	'Bounced',	'UniqueOpen',	'UniqueClick',	'Unsubscribe',	'Spam'	,'VirtualSpam',	'SendDate','Percentage_open'],axis=1)
    print('hi')
#for i in model.columns:
    #print([i])
#ml_model=model.drop(["CampaignName"])
# Create X & y variables for Machine Learning
    X = ml_model.drop('Open_Percentage', axis=1)
    y = ml_model['Open_Percentage']
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state=0)
    def model(mod, model_name, x_train, y_train, x_test, y_test):
        mod.fit(x_train, y_train)
        print(model_name)

        #acc = cross_val_score(mod, X_train, y_train, scoring = "accuracy", cv = 5)
        #predictions = cross_val_predict(mod, X_train, y_train, cv = 5)
        #print("Accuracy:", round(acc.mean(),3))
        #cm = confusion_matrix(predictions, y_train)
        #print("Confusion Matrix:  \n", cm)
        #print("                    Classification Report \n",classification_report(predictions, y_train))

# 2. Random Forest Classifier
    ran = RandomForestClassifier(random_state=0)
    model(ran, "Random Forest Classifier", X_train, y_train, X_test, y_test)
# Compile arrays of columns (words) and feature importances
    fi = {'Words':ml_model.drop('Open_Percentage',axis=1).columns.tolist(),'Importance':ran.feature_importances_}

# Bung these into a dataframe, rank highest to lowest then slice top 20
    Importances = pd.DataFrame(fi).sort_values('Importance',ascending=False).head(20)
    data['wordsCount'] = data['CampaignSubject'].apply(lambda x: len(str(x).split(" ")))
    data['Date']=pd.to_datetime(data['SendDate'])
    data["SendDay"] = pd.to_datetime(data.Date, format="%Y/%m/%d").dt.day_name()
    data["SendMonth"] = pd.to_datetime(data.Date, format="%Y/%m/%d").dt.month
    data['hour'] = data['Date'].dt.hour
    group_words=data.groupby('wordsCount',as_index=False)["Percentage_open"].count()

    group_words=pd.DataFrame(group_words).sort_values(by=['Percentage_open'],ascending=False).head(10)
    group_day=data.groupby('SendDay',as_index=False)["Percentage_open"].count()
    group_day=pd.DataFrame(group_day).head(10)
    group_hour=data.groupby('hour',as_index=False)["Percentage_open"].count()

    group_hour=pd.DataFrame(group_hour).head(10)

    h=group_hour['hour'].to_list()
    h = str(h)[1:-1]
    h="".join(h.split())
    h="'{}'".format(h)

    w=group_words['wordsCount'].to_list()
    w=str(w)[1:-1]
    w="".join(w.split())
    w="'{}'".format(w)

    d=",".join(group_day['SendDay'])
    d="'{}'".format(d)
    words=','.join(Importances['Words'])
    words="'{}'".format(words)
    #store in sp
    hh=group_hour.values.tolist()
    hh=str(hh)
    import re
    import json
    dd=group_day[['SendDay','Percentage_open']]
    dd=dd.values.tolist()
    dd=json.dumps(dd)
    dd=re.sub(r'(?<!: )"(\S*?)"', '\\1', dd)
    dd="'{}'".format(dd)
    ff=group_hour.values.tolist()
    ff=str(ff)
    ff="'{}'".format(ff)

    ww=group_words.values.tolist()
    ww=str(ww)
    Importances=Importances['Words'].astype(str)

    sql="EXEC EMS.SpSaveCampaignAggregateAnalysisReport @AccountID="+str(row['ACCOUNTID'])+",@ListId="+str(row['LISTID'])+",@Days="+dd+",@Words="+words+",@Hours="+ff+",@WordCount="+w+""
    #cursor.execute(sql).commit()
    print(sql)

    #sql="EXEC EMS.SpSaveCampaignAggregateAnalysisReport @AccountID=?,@ListId=?,@Days=?,@Words=?,@Hours=?,WordCount=?"
    #params=(str(row['ACCOUNTID']),str(row['LISTID']),str(group_day),str(group_words),str(group_hour),str(Importances))


"""Cluster by attributions in a way that allows for comparison of clusters
by label or TP/TN status.
"""

import os
import copy
from datetime import datetime

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.cluster import DBSCAN, dbscan
import hdbscan

import logomaker

def reformat_attribution_pkl(attr_data):
	"""Reformat attribution data from as read from pkl to numpy arrays
	or pandas dataframes.
	"""
	attr_data['pre'] = np.stack(attr_data['pre'])
	attr_data['post'] = np.stack(attr_data['post'])
	attr_data['pre_strings'] = np.stack(attr_data['pre_strings'])
	attr_data['post_strings'] = np.stack(attr_data['post_strings'])

	# fix a typo
	if 'predicitons' in attr_data.keys():
		attr_data['predictions'] = attr_data.pop('predicitons')

	attr_data['predictions'] = np.array(attr_data['predictions'])
	attr_data['labels'] = np.array(attr_data['labels'])
	attr_data['sample_data'] = pd.DataFrame(attr_data['sample_data'])

	return attr_data

def add_prediction_features(attr_data, cutoff=.5):
	"""Add binary prediction features and their correctness to attr_data."""
	attr_data['binary_pred'] = (attr_data['predictions'] > cutoff).astype(int)
	attr_data['correct_pred'] = (attr_data['binary_pred'] == attr_data['labels'])
	attr_data['true_pos'] = (attr_data['binary_pred'] == 1) & (attr_data['labels'] == 1)
	attr_data['true_neg'] = (attr_data['binary_pred'] == 0) & (attr_data['labels'] == 0)
	return attr_data

def subset_attr_data(attr_data, subset_mask):
	attr_data = copy.deepcopy(attr_data)

	for k,v in attr_data.items():
		if isinstance(v, np.ndarray):
			attr_data[k] = v[subset_mask]
		elif isinstance(v, pd.DataFrame):
			attr_data[k] = v.loc[subset_mask]

	return attr_data

def plot_cluster_attn_and_ppm(clust_labels, subset_data, n_per_side,
		str_pad_size, plt_height=1.0, plt_width=6.5, label_desc=''):
	"""Plot cluster attribution and ppm for each cluster.

	TODO: handle post STR seqs

	Args:
		clust_labels: array of cluster labels
		subset_data: attribution data in the normal dict form
		n_per_side: number of samples per side STR used
		str_pad_size: size of STR padding
		plt_height: height of total plot is plt_height * num_clusters
	"""
	sorted_labels = np.sort(np.unique(clust_labels))

	fig, axs = plt.subplots(len(sorted_labels), 2,
		figsize=(plt_width,len(sorted_labels) * plt_height + int(len(sorted_labels)==1)*.25),
		squeeze=False,
		sharex=True,
		sharey='col'
	)

	# fig.suptitle("Logo before {} {}".format(start_pattern, label_desc))
	axs[0,0].set_title('Median Attribution Score')
	axs[0,1].set_title('Position Probability Matrix')

	max_attr_mag = 0
	min_attr_mag = 0

	for i,cluster in enumerate(sorted_labels):
		clust_mask = clust_labels == cluster
		clust_strings = np.array(
			[np.array(list(c[-n_per_side-str_pad_size:-str_pad_size])) for c in subset_data['pre_strings'][clust_mask]]
		)

		# get median attributions
		mean_ignore_0 = True
		if mean_ignore_0:
			full_attrs = subset_data['pre'][clust_mask][:,:4:,-n_per_side-str_pad_size:-str_pad_size]
			full_attrs[full_attrs == 0] = np.nan
			with np.errstate(all='ignore'):
				med_attrs = np.nan_to_num(np.nanmedian(full_attrs, 0))
		else:
			med_attrs = subset_data['pre'][clust_mask][
				:,:4:,-n_per_side-str_pad_size:-str_pad_size].median(0)
		med_attr_df = pd.DataFrame(
			med_attrs.T,
			columns=['A', 'C', 'G', 'T']
		)

		# For scaling plot
		max_attr_mag = max(
			max_attr_mag,
			np.where(med_attr_df.values > 0, med_attr_df.values, 0).sum(1).max()
		)
		min_attr_mag = min(
			min_attr_mag,
			np.where(med_attr_df.values < 0, med_attr_df.values, 0).sum(1).min()
		)

		logomaker.Logo(med_attr_df, ax=axs[i, 0], center_values=True)
		axs[i, 0].set_ybound(min_attr_mag, max_attr_mag)

		# get position probability matrix
		counts_mat = np.stack([
			(clust_strings == 'A').sum(0),
			(clust_strings == 'C').sum(0),
			(clust_strings == 'G').sum(0),
			(clust_strings == 'T').sum(0)
		])
		ppm = counts_mat / counts_mat.sum(0, keepdims=True)
		ppm_df = pd.DataFrame(ppm.T, columns=['A', 'C', 'G', 'T'])

		logomaker.Logo(ppm_df, ax=axs[i, 1])
		axs[i, 1].set_yticklabels([])

	# Format plot
	plt.setp(
		axs,
		xticks=np.array(list(range(0, ppm.shape[1], 5))),
		xticklabels=list(range(-ppm.shape[1], 0, 5))
	)
	min_y = min(a.dataLim.get_points()[0,1] for a in axs[:,0])
	max_y = max(a.dataLim.get_points()[1,1] for a in axs[:,0])
	plt.setp(axs[:,0], ybound=(min_y, max_y))

	plt.tight_layout(w_pad=.21, h_pad=.1)

	return fig, axs, sorted_labels

if __name__ == '__main__':
	"""Args:

		str_motif_len: length of STR motif(s) (e.g. 2 for {'CA', 'TG',
			'GA'} motifs)
		str_pad_size: number of positions of attributions that are STR padding.
		n_per_side: number of loci to use in clustering.
		use_TP_TN: whether to use TP/TN status (True) or label (False) when
			seperating groups before clustering.
	"""
	# Options
	attrs_dir = '../prediction/training_output'
	label_version = [
		'v1-mfr0_005_mnc2000-m6_5',
		'v1-mfr0_0025_mnc2000-m7_5'
	][0]
	model_version = 'tscc_version_0'
	attr_file = 'ig_global_val_test.pkl'

	str_motif_len = 2
	str_pad_size = 4

	n_per_side = 15

	use_TP_TN = False

	cluster_metric = 'l1'
	cluster_method = 'hdbscan'
	dbscan_params = {
		'eps': .2,
		'min_samples': 40,
	}
	hdbscan_params = {
		'min_cluster_size': 10,
		'min_samples': 3,
		'cluster_selection_epsilon': 0.0,
		'alpha': 1.
	}

	clip_attrs = True
	clip_thresh = [10, 90]

	save_plots = True
	show_plots = False

	# Load single attr type
	attr_data = pd.read_pickle(os.path.join(
		attrs_dir, label_version, model_version, 'attributions', attr_file
	))

	# Reformat data and add prediction features
	attr_data = reformat_attribution_pkl(attr_data)
	attr_data = add_prediction_features(attr_data)
	attr_data['sample_data']['alpha_motif'] = attr_data['sample_data'].motif.apply(
		lambda x: ''.join(sorted([*x])) # motif name with letters sorted alphabetically
	)

	if use_TP_TN:
		pos_class_data = subset_attr_data(
			attr_data,
			(attr_data['binary_pred'] == 1) & (attr_data['labels'] == 1)
		)
		neg_class_data = subset_attr_data(
			attr_data,
			(attr_data['binary_pred'] == 0) & (attr_data['labels'] == 0)
		)
	else:
		pos_class_data = subset_attr_data(
			attr_data,
			(attr_data['labels'] == 1)
		)
		neg_class_data = subset_attr_data(
			attr_data,
			(attr_data['labels'] == 0)
		)

	# Add data on start/end of STR, since clustering will be done
	#	independantly for each type
	min_count = 100 # must be at least this many examples to cluster for STR start/end

	pos_STR_starts = np.array([
		s[-str_pad_size : -str_pad_size+str_motif_len] for s in pos_class_data['pre_strings']
	])
	pos_STR_ends = np.array([
		s[str_pad_size-str_motif_len : str_pad_size] for s in pos_class_data['post_strings']
	])
	neg_STR_starts = np.array([
		s[-str_pad_size : -str_pad_size+str_motif_len] for s in neg_class_data['pre_strings']
	])
	neg_STR_ends = np.array([
		s[str_pad_size-str_motif_len : str_pad_size] for s in neg_class_data['post_strings']
	])

	print("Positive class data: {} examples".format(len(pos_class_data['sample_data'])))
	print(*zip(*np.unique(pos_STR_starts, return_counts=True)))
	print(*zip(*np.unique(pos_STR_ends, return_counts=True)))

	print("Negative class data: {} examples".format(len(neg_class_data['sample_data'])))
	print(*zip(*np.unique(neg_STR_starts, return_counts=True)))
	print(*zip(*np.unique(neg_STR_ends, return_counts=True)))

	pos_starts_to_cluster = {
		s for s,c in zip(*np.unique(pos_STR_starts, return_counts=True)) if c >= min_count
	}
	neg_starts_to_cluster = {
		s for s,c in zip(*np.unique(neg_STR_starts, return_counts=True)) if c >= min_count
	}
	starts_to_cluster = pos_starts_to_cluster & neg_starts_to_cluster
	# ends_to_cluster = [
	# 	s for s,c in zip(*np.unique(STR_ends, return_counts=True)) if c >= min_count
	# ]

	# For each STR start pattern subset, cluster seqs by attribution weights
	if cluster_method == 'dbscan':
		clusterer = DBSCAN(
			eps=dbscan_params['eps'],
			min_samples=dbscan_params['min_samples'],
			metric=cluster_metric,
			n_jobs=-1
		)
	elif cluster_method == 'hdbscan':# and not use_weights:
		clusterer = hdbscan.HDBSCAN(
			min_cluster_size=hdbscan_params['min_cluster_size'],
			min_samples=hdbscan_params['min_samples'],
			cluster_selection_epsilon=hdbscan_params['cluster_selection_epsilon'],
			alpha=hdbscan_params['alpha'],
			metric=cluster_metric,
			core_dist_n_jobs=-1
		)
	else:
		raise ValueError("Invalid cluster_method: {}".format(cluster_method))

	# Create dir to save cluster results
	if cluster_method == 'dbscan':
		cluster_details_str = '_'.join(
			str(v) for v in [dbscan_params['eps'], dbscan_params['min_samples']]
		)
	elif cluster_method == 'hdbscan':
		cluster_details_str = '_'.join([
			str(hdbscan_params['min_cluster_size']),
			str(hdbscan_params['min_samples']),
			str(hdbscan_params['cluster_selection_epsilon']),
			str(hdbscan_params['alpha'])
		])

	if save_plots:
		cluster_res_dir = '_'.join([
			'TPTN' if use_TP_TN else 'label',
			cluster_method,
			cluster_metric,
			cluster_details_str,
			str(n_per_side),
			'clip{}-{}'.format(*clip_thresh) if clip_attrs else 'no_clip',
		])
		cluster_res_dir = os.path.join(
			'cba_2_plots', label_version, model_version, cluster_res_dir
		)

		if os.path.exists(cluster_res_dir):
			i = 1
			while os.path.exists(cluster_res_dir + '_' + str(i)):
				i += 1

			cluster_res_dir = cluster_res_dir + '_' + str(i)

		os.makedirs(cluster_res_dir)

	for start_pattern in sorted(starts_to_cluster):
		print(start_pattern)

		# subset data
		pos_subset_data = subset_attr_data(
			pos_class_data,
			pos_STR_starts == start_pattern
		)
		neg_subset_data = subset_attr_data(
			neg_class_data,
			neg_STR_starts == start_pattern
		)

		# get data in useful format and cluster
		X_pos = pos_subset_data['pre'][:,:4:,-n_per_side-str_pad_size:-str_pad_size]
		X_pos = X_pos.reshape(X_pos.shape[0], -1)

		if clip_attrs:
			cutoffs = np.percentile(X_pos, [clip_thresh[0], clip_thresh[1]])
			X_pos = np.clip(X_pos, cutoffs[0], cutoffs[1])

		pos_clust_labels = clusterer.fit_predict(X_pos)
		print("Positive class: {} clusters".format(len(np.unique(pos_clust_labels))))
		print(*zip(*np.unique(pos_clust_labels, return_counts=True)))

		X_neg = neg_subset_data['pre'][:,:4:,-n_per_side-str_pad_size:-str_pad_size]
		X_neg = X_neg.reshape(X_neg.shape[0], -1)

		if clip_attrs:
			cutoffs = np.percentile(X_neg, [clip_thresh[0], clip_thresh[1]])
			X_neg = np.clip(X_neg, cutoffs[0], cutoffs[1])

		neg_clust_labels = clusterer.fit_predict(X_neg)
		print("Negative class: {} clusters".format(len(np.unique(neg_clust_labels))))
		print(*zip(*np.unique(neg_clust_labels, return_counts=True)))

		# Plot clusters for both classes
		pos_fig, _, pos_sorted_labels = plot_cluster_attn_and_ppm(
			pos_clust_labels,
			pos_subset_data,
			n_per_side,
			str_pad_size,
			label_desc='TP' if use_TP_TN else 'heterozygous',
			plt_height=.8
		)
		neg_fig, _, neg_sorted_labels = plot_cluster_attn_and_ppm(
			neg_clust_labels,
			neg_subset_data,
			n_per_side,
			str_pad_size,
			label_desc='TN' if use_TP_TN else 'non-heterozygous',
			plt_height=.8
		)

		# plot prediction confidence dists and attrs by cluster and class
		pos_conf_df = pd.DataFrame({
			'pred_conf': pos_subset_data['predictions'],
			'cluster': pos_clust_labels,
			'heterozygosity_score': pos_subset_data['sample_data'].heterozygosity.values
		})

		pos_conf_attr_fig, axs = plt.subplots(1, 3,
			figsize=(6.4, pos_fig.get_figheight())
		)
		sns.violinplot(
			x='pred_conf',
			y='cluster',
			orient='h',
			# scale='count',
			order=pos_sorted_labels,
			cut=0,
			data=pos_conf_df,
			ax=axs[0]
		)
		axs[0].set_title("Predicted Likelihood")
		axs[0].set_xlabel(None)
		sns.violinplot(
			x='heterozygosity_score',
			y='cluster',
			orient='h',
			order=pos_sorted_labels,
			cut=0,
			data=pos_conf_df,
			ax=axs[1]
		)
		axs[1].set_ylabel(None)
		axs[1].set_title('Heterozygosity')
		axs[1].set_xlabel(None)
		sns.countplot(
			y='cluster',
			data=pos_conf_df,
			order=pos_sorted_labels,
			ax=axs[2]
		)
		axs[2].set_title('Cluster size')
		axs[2].set_xlabel(None)
		axs[2].set_ylabel(None)

		abs_values = pos_conf_df['cluster'].value_counts(ascending=False)
		rel_values = pos_conf_df['cluster'].value_counts(ascending=False, normalize=True) * 100
		count_labels = [
			f'{abs_values[l]} ({rel_values[l]:.1f}%)' for l in pos_sorted_labels
		]
		axs[2].bar_label(container=axs[2].containers[0], labels=count_labels)
		plt.tight_layout(w_pad=.21)

		# Negative equivalent plot (plot 4/4)
		neg_conf_df = pd.DataFrame({
			'pred_conf': neg_subset_data['predictions'],
			'cluster': neg_clust_labels,
			'heterozygosity_score': neg_subset_data['sample_data'].heterozygosity.values
		})

		neg_conf_attr_fig, axs = plt.subplots(1, 2,
			figsize=(6.4, neg_fig.get_figheight())
		)
		sns.violinplot(
			x='pred_conf',
			y='cluster',
			orient='h',
			# scale='count',
			order=neg_sorted_labels,
			cut=0,
			data=neg_conf_df,
			ax=axs[0]
		)
		# sns.violinplot(
		# 	x='heterozygosity_score',
		# 	y='cluster',
		# 	orient='h',
		# 	order=neg_sorted_labels,
		# 	cut=0,
		# 	data=neg_conf_df,
		# 	ax=axs[1]
		# )
		sns.countplot(
			y='cluster',
			data=neg_conf_df,
			order=neg_sorted_labels,
			ax=axs[1]
		)
		# axs[1].set_xscale('log')
		axs[1].set_xlabel(None)
		axs[1].set_ylabel(None)
		# plt.tight_layout()

		abs_values = neg_conf_df['cluster'].value_counts(ascending=False)
		rel_values = neg_conf_df['cluster'].value_counts(ascending=False, normalize=True) * 100
		count_labels = [
			f'{abs_values[l]} ({rel_values[l]:.1f}%)' for l in neg_sorted_labels
		]
		axs[1].bar_label(container=axs[1].containers[0], labels=count_labels)

		if save_plots:
			pos_fig.savefig(os.path.join(
				cluster_res_dir,
				'pre_{}_pos_by_base.png'.format(start_pattern)
			))
			neg_fig.savefig(os.path.join(
				cluster_res_dir,
				'pre_{}_neg_by_base.png'.format(start_pattern)
			))
			pos_conf_attr_fig.savefig(os.path.join(
				cluster_res_dir,
				'pre_{}_pos_by_cluster.png'.format(start_pattern)
			))
			neg_conf_attr_fig.savefig(os.path.join(
				cluster_res_dir,
				'pre_{}_neg_by_cluster.png'.format(start_pattern)
			))
		if show_plots:
			plt.show()

		plt.close()


#!/usr/bin/env python
# -*- coding:utf-8 -*-
import json
import importlib
from django.views import View
from django.http import JsonResponse
from utils.response import BaseResponse
from django.shortcuts import HttpResponse
from django.views.decorators.csrf import csrf_exempt
from django.utils.decorators import method_decorator

from utils import auth
from api import config
from repository import models
from api.service import asset

class LimitView(View):
    def dispatch(self, request, *args, **kwargs):
        return super(LimitView, self).dispatch(request, *args, **kwargs)

    def get(self, request, *args, **kwargs):
        response = BaseResponse()

        name = kwargs.get('n1', None)
        if name:
            try:
                result = models.AuthInfo.objects.filter(username=name, status=2).values('username', 'ip', 'hostname', 'rank')
                response.data = list(result)
                response.status = True
                return JsonResponse(response.__dict__)
            except Exception as e:
                response.error = "Didn't find %s" % name
                response.status = False
                return JsonResponse(response.__dict__)

        result = models.AuthInfo.objects.all().values('username', 'ip', 'hostname', 'rank', 'status')
        # result = models.AuthInfo.objects.all()
        response.data = list(result)
        response.status = True
        return JsonResponse(response.__dict__)

class AssetView(View):
    # @method_decorator(csrf_exempt)
    def dispatch(self, request, *args, **kwargs):
        return super(AssetView, self).dispatch(request, *args, **kwargs)

    # @method_decorator(auth.api_auth)
    def get(self, request, *args, **kwargs):
        """
        :param request:
        :param args:
        :param kwargs:
        :return:
        """
        # appid = request.META.get(['HTTP_APPID'], None)
        # print(appid)
        # test = {'user': '用户名', 'pwd': '密码'}
        # result = json.dumps(test,ensure_ascii=True)
        # result = json.dumps(test,ensure_ascii=False)
        # return HttpResponse(result)

        # test = {'user': '用户名', 'pwd': '密码'}
        # result = json.dumps(test,ensure_ascii=True)
        # result = json.dumps(test,ensure_ascii=False)
        # return HttpResponse(result,content_type='application/json')

        # test = {'user': '用户名', 'pwd': '密码'}
        # return JsonResponse(test, json_dumps_params={"ensure_ascii": False})
        from django.db.models import Q

        # response = asset.get_untreated_servers()
        response = BaseResponse()

        # print(kwargs)
        b1 = kwargs.get('b1', None)
        b2 = kwargs.get('b2', None)
        b3 = kwargs.get('b3', None)
        if b1:
            try:
                b1_id = models.BusinessOne.objects.filter(name=kwargs['b1']).first().id
                if b2:
                    try:
                        b2_id = models.BusinessTwo.objects.filter(name=kwargs['b2']).first().id
                        if b3:
                            try:
                                b3_id = models.BusinessThree.objects.filter(name=kwargs['b3']).first().id

                                condition = Q()
                                con_business_1 = Q()
                                con_business_1.children.append(('business_1', b1_id))
                                con_business_2 = Q()
                                con_business_2.children.append(('business_2', b2_id))
                                con_business_3 = Q()
                                con_business_3.children.append(('business_3', b3_id))

                                condition.add(con_business_1, 'AND')
                                condition.add(con_business_2, 'AND')
                                condition.add(con_business_3, 'AND')

                                result = models.Asset.objects.filter(condition).values('host_ip', 'host_name', 'business_2__name')
                                response.data = list(result)
                                response.status = True
                                return JsonResponse(response.__dict__)
                            except Exception as e:
                                response.error = "Didn't find %s" % kwargs['b3']
                                response.status = False
                                return JsonResponse(response.__dict__)
                        print(b2_id)
                        condition = Q()
                        con_business_1 = Q()
                        con_business_1.children.append(('business_1', b1_id))
                        con_business_2 = Q()
                        con_business_2.children.append(('business_2', b2_id))

                        condition.add(con_business_1, 'AND')
                        condition.add(con_business_2, 'AND')

                        result = models.Asset.objects.filter(condition).values('host_ip', 'host_name', 'business_2__name')
                        response.data = list(result)
                        response.status = True
                        return JsonResponse(response.__dict__)
                    except Exception as e:
                        response.error = "Didn't find %s" % kwargs['b2']
                        response.status = False
                        return JsonResponse(response.__dict__)

                print(b1_id)
                condition = Q()
                con_business_1 = Q()
                con_business_1.children.append(('business_1', b1_id))

                condition.add(con_business_1, 'AND')

                result = models.Asset.objects.filter(condition).values('host_ip')
                response.data = list(result)
                response.status = True
                return JsonResponse(response.__dict__)

            except Exception as e:
                response.error = "Didn't find %s" % kwargs['b1']
                response.status = False
                return JsonResponse(response.__dict__)
        result = models.Asset.objects.all().values('host_ip')
        response.data = list(result)
        response.status = True

        return JsonResponse(response.__dict__)

    @method_decorator(auth.api_auths)
    def post(self, request, *args, **kwargs):
        """
        更新或者添加资产信息
        :param request:
        :param args:
        :param kwargs:
        :return: 1000 成功;1001 接口授权失败;1002 数据库中资产不存在
        """

        server_info = json.loads(request.body.decode('utf-8'))
        server_info = json.loads(server_info)
        new_ip = server_info['docker']

        host_ip = models.Asset.objects.filter(host_ip=new_ip).count()
        if host_ip:
            try:
                models.Asset.objects.create(host_ip=new_ip, host_name='a0-docker-%s' % new_ip, host_status=2,
                                            host_item=1, host_type=5, host_cpu=2, host_memory=2, business_1_id=1,
                                            bussiness_2_id=1, business_3_id=1)
            except Exception as e:
                ret = {'code': 1000, 'message': '[%s]添加失败...%s' % (new_ip, e)}
                return JsonResponse(ret)
            ret = {'code': 1002, 'message': '[%s]IP已经存在！' % new_ip}
        else:
            ret = {'code': 1000, 'message': '[%s]添加成功！' % new_ip}
        #
        # server_obj = models.Server.objects.filter(hostname=hostname).select_related('asset').first()
        # if not server_obj:
        #     ret['code'] = 1002
        #     ret['message'] = '[%s]资产不存在' % hostname
        #     return JsonResponse(ret)
        #
        # for k, v in config.PLUGINS_DICT.items():
        #     module_path, cls_name = v.rsplit('.', 1)
        #     cls = getattr(importlib.import_module(module_path), cls_name)
        #     response = cls.process(server_obj, server_info, None)
        #     if not response.status:
        #         ret['code'] = 1003
        #         ret['message'] = "[%s]资产更新异常" % hostname
        #     if hasattr(cls, 'update_last_time'):
        #         cls.update_last_time(server_obj, None)

        return JsonResponse(ret)

class ReleaseView(View):
    def dispatch(self, request, *args, **kwargs):
        return super(ReleaseView, self).dispatch(request, *args, **kwargs)

    def get(self, request, *args, **kwargs):
        response = BaseResponse()

        name = kwargs.get('n1', None)
        if name:
            try:
                result = models.AuthInfo.objects.filter(username=name, status=2).values('username', 'ip', 'hostname', 'rank')
                response.data = list(result)
                response.status = True
                return JsonResponse(response.__dict__)
            except Exception as e:
                response.error = "Didn't find %s" % name
                response.status = False
                return JsonResponse(response.__dict__)

        result = models.ReleaseTask.objects.all().values('username', 'ip', 'hostname', 'rank', 'status')
        # result = models.AuthInfo.objects.all()
        response.data = list(result)
        response.status = True
        return JsonResponse(response.__dict__)

    def post(self, request, *args, **kwargs):
        response = BaseResponse()

        n1 = json.loads(request.body.decode('utf-8'))
        n1 = json.loads(n1)

        r_id = n1.get('id', None)
        msg = n1.get('msg', None)

        # 上传包md5值
        md5 = n1.get('md5', None)
        if md5:
            print(md5)
            models.ReleaseTask.objects.filter(id=r_id).update(release_md5=md5)
            models.ReleaseLog.objects.create(release_id=r_id, release_msg=md5)
            # 发布任务成功上传
            response.status = True
            return JsonResponse(response.__dict__)

        print(r_id, msg)
        try:
            models.ReleaseLog.objects.create(release_id=r_id, release_msg=msg)
            response.status = True
        except Exception as e:
            response.status = False

        obj = models.ReleaseLog.objects.filter(release_time__gt='2017-05-19 05:00').first()
        print(obj.release_time)

        return JsonResponse(response.__dict__)


import torch
import math
import time
import numpy as np

class TrainNhppNm(torch.nn.Module):

    def __init__(self, trainSet, testSet, numOfepochs, lr, numOfNodes, dim, order, numOfSamples=10, timeInterval=0.01, seed=0):
        super().__init__()

        # Set the seed value
        self.__seed = seed
        torch.manual_seed(self.__seed)

        self.__numOfNodes = numOfNodes
        self.__dim = dim
        self.__order = order

        self.__numOfepochs = numOfepochs
        self.__lr = lr

        self.__numOfSamples = numOfSamples

        # Set the models parameters
        self.__gamma = torch.rand(size=(self.__numOfNodes, ))
        #self.__gamma = torch.zeros(size=(self.__numOfNodes, 1))
        self.__z0 = []
        self.__z0_nongrad = [torch.randn(self.__numOfNodes, self.__dim) for _ in range(self.__order)]
        #self.__z0_nongrad = [torch.zeros(self.__numOfNodes, self.__dim) for _ in range(self.__order)]

        self.__timeInterval = timeInterval

        # Get the node indices
        self.__nodePairs = torch.triu_indices(row=self.__numOfNodes, col=self.__numOfNodes, offset=1)

        # self.tn_train = tn_train # last time point on time axis in simul
        # self.tn_test = tn_test
        # self.pdist = nn.PairwiseDistance(p=2) # euclidean
        self.__trainSet = trainSet
        self.__testSet = testSet

        self.__factorials = [float(math.factorial(o)) for o in range(self.__order+1)]

        self.__pdist = torch.nn.PairwiseDistance(p=2,  keepdim=False)

    def initializeModelParams(self, gamma=None, order=None):

        if gamma is not None:
            self.__gamma = torch.nn.Parameter(gamma)
        if order is not None:
            # self.__z0[order] = torch.nn.Parameter(self.__z0[order])
            #print([self.__z0[o] for o in range(self.__order)])
            self.__z0 = torch.nn.ParameterList([self.__z0[o] for o in range(len(self.__z0))] + [torch.nn.Parameter(self.__z0_nongrad[order])])

    def get_gamma(self):

        return self.__gamma

    def getLatentVariableAt(self, i, orderIndex, t):

        z = torch.zeros(size=(self.__dim,), dtype=torch.float)
        for o in range(self.__order - orderIndex):
            if orderIndex + o < len(self.__z0):
                z += self.__z0[orderIndex + o][i, :] * (t ** o) / math.factorial(o)
            else:
                z += self.__z0_nongrad[orderIndex + o][i, :] * (t ** o) / math.factorial(o)

        return z

    def getPositionOf(self, i, t):

        return self.getLatentVariableAt(i=i, orderIndex=0, t=t)

    def getAllPositions(self, t):

        z = torch.zeros(size=(self.__numOfNodes, self.__dim,), dtype=torch.float)
        for o in range(self.__order):
            to = t ** o
            if o < len(self.__z0):
                z += self.__z0[o] * to / self.__factorials[o]
            else:
                z += self.__z0_nongrad[o] * to / self.__factorials[o]

        return z

    def getDistanceBetween(self, i, j, t, norm=2):

        if norm != 2:
            raise ValueError("Invalid norm value!")

        xi = self.getPositionOf(i=i, t=t)
        xj = self.getPositionOf(i=j, t=t)

        #return sum([(xi[d] - xj[d]) ** norm for d in range(self.__dim)]) ** (1. / norm)
        diff = xi - xj
        return torch.sqrt(torch.dot(diff, diff))

    def __approxIntensityIntegral(self, i, j, tInit, tLast):

        if tInit >= tLast:
            raise ValueError("The initial time {} cannot be greater than the last time step {}!".format(tInit, tLast))

        value = 0.
        timePoints, stepSize = np.linspace(start=tInit, stop=tLast, num=self.__numOfSamples, endpoint=False, retstep=True)
        for t in timePoints:
            value += self.computeIntensityForPair(i=i, j=j, t=t) * stepSize
        # timePoints = [tInit, tLast] #torch.linspace(start=tInit, end=tLast, steps=self.__numOfSamples+1)
        # stepSize = timePoints[1] - timePoints[0]
        # for t in timePoints[:-1]:
        #     value += self.computeIntensityForPair(i=i, j=j, t=t) * stepSize

        return value

    def computeIntensityForPair(self, i, j, t):

        return torch.exp( self.__gamma[i] + self.__gamma[j] - self.getDistanceBetween(i=i, j=j, t=t) )

    def __computeLogIntensityForPair(self, i, j, t):

        return self.__gamma[i] + self.__gamma[j] - self.getDistanceBetween(i=i, j=j, t=t)

    def meanNormalization(self):

        for o in range(len(self.__z0)):
            m = torch.mean(self.__z0[o], dim=0)
            for i in range(self.__numOfNodes):
                self.__z0[o].data[i, :] = self.__z0[o].data[i, :] - m

    def getAllDistances(self, t, norm=2):

        if norm != 2:
            raise ValueError("Invalid norm value!")

        x = self.getAllPositions(t=t)

        #return sum([(xi[d] - xj[d]) ** norm for d in range(self.__dim)]) ** (1. / norm)
        # diff = xi - xj

        tt = torch.nn.functional.pdist(x)
        return tt

    def computeSumOfIntensityForAllPairs(self, t):

        # temp = self.getAllDistances(t=t)
        # for i, j in zip(self.__nodePairs):
        #     temp[i, j] = torch.exp(self.__gamma[i] + self.__gamma[j] - temp[i, j])
        #
        # return temp
        temp = torch.exp(-self.getAllDistances(t=t))
        expGamma = torch.exp(self.__gamma)
        # temp = expGamma[1:] * temp
        # temp = expGamma[:-1] * temp.T

        tri = torch.triu(torch.outer(expGamma, expGamma), diagonal=1)
        tri_inx = torch.triu_indices(row=self.__numOfNodes, col=self.__numOfNodes, offset=1)
        # temp = temp * torch.reshape(tri[tri_inx[0], tri_inx[1]], (-1,))
        temp = temp * tri[tri_inx[0], tri_inx[1]]

        temp = torch.sum( temp )

        return temp

    def computeAllNonEventsIntegral(self, tInit, tLast):

        totalValue = 0.
        timePoints, stepSize = np.linspace(start=tInit, stop=tLast, num=self.__numOfSamples,
                                           endpoint=False, retstep=True)
        for t in timePoints:
            totalValue += self.computeSumOfIntensityForAllPairs(t=t) * stepSize

        return totalValue

    def __computeLogIntensitySumForANodePairThroughTime(self, i, j, timeList):

        gamma_sum = self.__gamma[i] + self.__gamma[j]

        dist_coeff = torch.zeros(size=(self.__order, self.__dim), dtype=torch.float)
        for o in range(self.__order):
            if o < len(self.__z0):
                dist_coeff[o, :] = ( self.__z0[o][i, :] - self.__z0[o][j, :] ) / self.__factorials[o]
            else:
                dist_coeff[o, :] = ( self.__z0_nongrad[o][i, :] - self.__z0_nongrad[o][j, :] ) / self.__factorials[o]

        dist_sum = 0.0
        for t in timeList:

            dist_vect = torch.zeros(size=(self.__dim, ), dtype=torch.float)
            dist_vect += dist_coeff[0, :]  # Initial coefficient
            power_of_t = 1.0
            for o in range(1, self.__order):
                power_of_t = power_of_t * t

                # dist_vect += dist_coeff[o, :] * power_of_t
                if o < len(self.__z0):
                    dist_vect += power_of_t * (self.__z0[o][i, :] - self.__z0[o][j, :]) / self.__factorials[o]
                else:
                    dist_vect += power_of_t * (self.__z0_nongrad[o][i, :] - self.__z0_nongrad[o][j, :]) / self.__factorials[o]

            dist_sum += torch.sqrt(torch.dot(dist_vect, dist_vect))

        return (len(timeList) * gamma_sum) - dist_sum

    def forward(self, data):

        # neg_logLikelihood = 0.
        # for tInit, tLast, eventTimes in data:
        #     for i, j in zip(self.__nodePairs[0], self.__nodePairs[1]):
        #         ci = self.__approxIntensityIntegral(i=i, j=j, tInit=tInit, tLast=tLast)
        #         neg_logLikelihood += -ci
        #         for e in eventTimes:
        #             neg_logLikelihood += self.__computeLogIntensityForPair(i=i, j=j, t=e)
        #
        # return -neg_logLikelihood

        neg_logLikelihood = 0.
        for tInit, tLast, eventsList in data:

            t0 = time.time()
            # # Non-events
            # if e < 0:
            #     ci = self.__approxIntensityIntegral(i=i, j=j, tInit=tInit, tLast=tLast)
            #     neg_logLikelihood += -ci
            neg_logLikelihood += -self.computeAllNonEventsIntegral(tInit=tInit, tLast=tLast)
            # print("Non-event: {}".format(time.time() - t0))
            # print(neg_logLikelihood)
            t0 = time.time()
            # Events
            for ij_list in eventsList:
                ij_pair = ij_list[0]
                ij_events = ij_list[1]
                neg_logLikelihood += self.__computeLogIntensitySumForANodePairThroughTime(i=ij_pair[0], j=ij_pair[1],timeList=ij_events)
            # for e in ij_events:
            #     s = self.__computeLogIntensityForPair(i=i, j=j, t=e)
            #     neg_logLikelihood += s

            # print("# of events: {}".format(sum([len(el[1]) for el in eventsList])))
            # print("Event: {}".format(time.time() - t0))

        return -neg_logLikelihood

    def learn(self, type="seq"):

        # List for storing the training and testing set losses
        trainLoss, testLoss = [], []

        # Learns the parameters sequentially
        if type == "seq":

            for inx in list(range(1, self.__order + 1)) + [0]:  # range(order+1):

                print("Index: {}".format(inx))

                if inx == 0:
                    self.initializeModelParams(gamma=torch.rand(size=(self.__numOfNodes, )))
                    # nhppNmmModel.initializeModelParams(gamma=torch.zeros(size=(numOfNodes, 1)))
                else:
                    self.initializeModelParams(order=inx - 1)
                # if inx == 1:
                #     nhppNmmModel.initializeModelParams(x0=torch.rand(size=(numOfNodes, dim)))
                # if inx == 2:
                #     nhppNmmModel.initializeModelParams(v0=torch.rand(size=(numOfNodes, dim)))
                # if inx == 3:
                #     nhppNmmModel.initializeModelParams(a0=torch.rand(size=(numOfNodes, dim)))

                # print(len(list(self.parameters())))
                # print(list(self.parameters()))

                # Define the optimizer
                optimizer = torch.optim.Adam(self.parameters(), lr=self.__lr)

                for epoch in range(self.__numOfepochs):
                    t0 = time.time()
                    # running_loss = 0.
                    # print(f"Batch {idx+1} of {len(training_batches)}")
                    self.train()
                    #optimizer.zero_grad()
                    for param in self.parameters():
                        param.grad = None
                    train_loss = self(data=self.__trainSet)
                    t1 = time.time()
                    # print("Forward Time: {}".format(t1 - t0))
                    train_loss.backward()
                    # print("Backward Time: {}".format(time.time() - t1))
                    optimizer.step()
                    # running_loss += loss.item()
                    trainLoss.append(train_loss.item() / len(self.__trainSet))

                    # print("Time: {}".format(time.time() - t0))
                    self.eval()
                    with torch.no_grad():

                        test_loss = self(data=self.__testSet)
                        testLoss.append(test_loss / len(self.__testSet))

                    if epoch % 50 == 0:
                        print(f"Epoch {epoch + 1} train loss: {train_loss.item() / len(self.__trainSet)}")
                        print(f"Epoch {epoch + 1} test loss: {test_loss / len(self.__testSet)}")
                        # if inx == 0:
                        #     print(list(self.parameters()))

        return trainLoss, testLoss


import collections
import math

from PyGenTools import makeGen
from PyArrTools import insort

def coordRotatedCCW(coord):
  return (-coord[1],coord[0])

def coordRotatedCW(coord):
  return (coord[1],-coord[0])

def coordSum(coords):
  result = (0,0)
  for coord in coords:
    result = (result[0] + coord[0], result[1] + coord[1])
  return result

def scaledCoord(coord,scalar):
  return (coord[0]*scalar,coord[1]*scalar)

def directionIndexToVector(index):
  return [(0,1),(1,0),(0,-1),(-1,0)][index]

def coordsAreNeighbors(coord0,coord1):
  assert len(coord0) == len(coord1)
  return all((abs(coord0[i]-coord1[i]) <= 1) for i in range(len(coord0)))

def distance(coord0,coord1):
  assert len(coord0) == len(coord1)
  return float(sum((coord0[i]-coord1[i])**2 for i in range(len(coord0))))**0.5

def magnitude(coord0):
  return float(sum(coord0Element**2 for coord0Element in coord0))**0.5

def sign(x):
  return int(math.copysign(1,x))

def signs(coord):
  return tuple([sign(value) for value in coord])

def modularMinimumDistance(value0,value1,modulus):
  distances = [item for item in [value0 - value1, value1 - value0, value0 + modulus - value1, value1 + modulus - value0] if item >= 0]
  return min(distances)

def spiralRingSideLength(ringIndex,includedCorners):
  assert includedCorners in [0,1,2]
  assert ringIndex > 0
  #if ringIndex == 0:
  #  result = spiralRingSideLength(ringIndex+1,includedCorners) - 4
  #  print("SpiralMath.spiralRingSideLength: returning {} instead of an error.".format(result))
  #  return result
  #  return Undefined("spiralRingSideLength is undefined for ringIndex=0")
  return ((2*ringIndex)-1)+includedCorners

def spiralRingLength(ringIndex):
  assert ringIndex >= 0
  if ringIndex == 0:
    return 1
  return 4 * spiralRingSideLength(ringIndex,1)

def spiralRingArea(ringIndex):
  if ringIndex == 0:
    return 1
  if ringIndex == 1:
    return 9
  return spiralRingSideLength(ringIndex,2)**2

def spiralRingInnerArea(ringIndex):
  #return sum(spiralRingLength(i) for i in range(ringIndex))
  if ringIndex == 0:
    return 0
  return spiralRingArea(ringIndex-1)

def spiralRingSideStartCoords(ringIndex):
  assert ringIndex > 0
  if ringIndex == 1:
    return [(0,1),(1,0),(0,-1),(-1,0)]
  return [(0-(ringIndex-1),ringIndex),(ringIndex,ringIndex-1),(ringIndex-1,-ringIndex),(-ringIndex,0-(ringIndex-1))]

def spiralRingSideMatchableCoords(ringIndex):
  assert ringIndex > 0
  if ringIndex == 1:
    return [(None,1),(1,None),(None,-1),(-1,None)]
  return [(None,ringIndex),(ringIndex,None),(None,-ringIndex),(-ringIndex,None)]

def spiralCoordEncode(coord, **customizationKwargs):
  basicCoord = decustomizedSpiralCoord(coord, **customizationKwargs)
  result = basicSpiralCoordEncode(basicCoord)
  return result

def spiralCoordDecode(t, **customizationKwargs):
  basicResult = basicSpiralCoordDecode(t)
  result = customizedSpiralCoord(basicResult, **customizationKwargs)
  return result

def decustomizedSpiralCoord(coord, home=None, rot=None, spin=None):
  assert len(coord) == 2 and len(home) == 2
  assert spin in [1,-1] #1 is CW, -1 is CCW.
  result = coordSum([coord,scaledCoord(home,-1)])
  for i in range(rot%4): #0 1 2 3 -> first step is N E S W
    result = coordRotatedCCW(result)
  result = (result[0]*spin, result[1])
  return result

def customizedSpiralCoord(coord, home=None, rot=None, spin=None):
  assert len(coord) == 2 and len(home) == 2
  assert spin in [1,-1] #1 is CW, -1 is CCW.
  result = (coord[0]*spin, coord[1])
  for i in range(rot%4): #0 1 2 3 -> first step is N E S W
    result = coordRotatedCW(result)
  result = coordSum([result, home])
  return result

def basicSpiralCoordDecode(t):
  assert t >= 0
  if t == 0: #because spiralRingSideLength can't handle ring index 0.
    return (0,0)

  ringIndex, tInRing = findSubdivisionAndInnerLocation(0, t, subdivisionSizeFun=spiralRingLength)

  sideLen = spiralRingSideLength(ringIndex, 1)
  sideInRing, tInSide = findSubdivisionAndInnerLocation(0, tInRing, subdivisionSize=sideLen)

  sideDirection = directionIndexToVector((sideInRing+1)%4)
  sideStartCoord = spiralRingSideStartCoords(ringIndex)[sideInRing]
  result = coordSum([sideStartCoord,scaledCoord(sideDirection,tInSide)])
  return result

def basicSpiralCoordEncode(coord):
  ringIndex = max(abs(value) for value in coord)
  if ringIndex == 0:
    return 0
  if ringIndex == 1: #because the side index finding loop can't handle ring 1.
    return {(0,1):1,(1,1):2,(1,0):3,(1,-1):4,(0,-1):5,(-1,-1):6,(-1,0):7,(-1,1):8}[coord]
  ringSideStartCoords = spiralRingSideStartCoords(ringIndex)
  ringSideMatchableCoords = spiralRingSideMatchableCoords(ringIndex)
  sideInRing = None
  sideMatches = []
  for sideIndex,testCoord in enumerate(ringSideMatchableCoords):
    if (coord[0] == testCoord[0] or coord[1] == testCoord[1]):
      sideMatches.append(sideIndex)
  if len(sideMatches) == 1:
    sideInRing = sideMatches[0]
  else:
    assert len(sideMatches) == 2
    offset = modularMinimumDistance(sideMatches[0],sideMatches[1],4)
    assert offset == 1
    if sideMatches == [0,3]:
      sideInRing = 3
    else:
      assert sideMatches in [[0,1],[1,2],[2,3]]
      sideInRing = min(sideMatches)
  assert sideInRing != None
  sideStartCoord = ringSideStartCoords[sideInRing]
  tPassedRings = spiralRingInnerArea(ringIndex)
  tPassedSides = (sideInRing*spiralRingSideLength(ringIndex,1))
  tWithinSide = int(round(distance(coord,sideStartCoord)))
  #print("(ringIndex,sideInRing,tPassedRings,tPassedSides,tWithinSide)="+str((ringIndex,sideInRing,tPassedRings,tPassedSides,tWithinSide))+".")
  t = tPassedRings + tPassedSides + tWithinSide
  return t

def findSubdivisionAndInnerLocation(startingSubdivisionIndex, absoluteLocation, subdivisionSize=None, subdivisionSizeFun=None): #this can be modified to work in higher dimensions.
  assert not (subdivisionSize == None and subdivisionSizeFun == None)
  if subdivisionSizeFun == None:
    subdivisionSizeFun = (lambda ignore: subdivisionSize)

  currentSubdivisionIndex = startingSubdivisionIndex
  relativeLocation = absoluteLocation
  while True:
    currentSubdivisionSize = subdivisionSizeFun(currentSubdivisionIndex)
    if currentSubdivisionSize > relativeLocation:
      break
    relativeLocation -= currentSubdivisionSize
    currentSubdivisionIndex += 1
  return currentSubdivisionIndex, relativeLocation

def genSpiralCoords(*args, **customizationKwargs):
  for basicCoord in genBasicSpiralCoords(*args):
    yield customizedSpiralCoord(basicCoord, **customizationKwargs)

def genBasicSpiralCoords(*args):
  #4 to 8 times faster for large spirals.
  startT = 0
  endTExclusive = None
  direction = 1

  if len(args) == 1:
    endTExclusive = args[0]
    assert endTExclusive >= -1
  if len(args) >= 2:
    startT, endTExclusive = args[0], args[1]
    assert startT >= 0
    assert endTExclusive >= -1 #do stricter checks later.
  if len(args) >= 3:
    direction = args[2]
    assert direction in [1, -1]
  if len(args) > 3:
    raise ValueError("wrong number of arguments.")

  if endTExclusive != None:
    endT = endTExclusive - direction
    assert endT >= 0
  else:
    endT = None

  startRingIndex, startTInStartRing = findSubdivisionAndInnerLocation(0, startT, subdivisionSizeFun=spiralRingLength)
  if startRingIndex > 0:
    startSideLen = spiralRingSideLength(startRingIndex, 1)
    startSideInStartRing, startTInStartSide = findSubdivisionAndInnerLocation(0, startTInStartRing, subdivisionSize=startSideLen)

  if endT == None:
    endRingIndex, endTInEndRing = None, None
    endSideInEndRing, endTInEndSide = None, None
  else:
    endRingIndex, endTInEndRing = findSubdivisionAndInnerLocation(0, endT, subdivisionSizeFun=spiralRingLength)
    if endRingIndex > 0:
      endSideLen = spiralRingSideLength(endRingIndex, 1)
      endSideInEndRing, endTInEndSide = findSubdivisionAndInnerLocation(0, endTInEndRing, subdivisionSize=endSideLen)

  ringIndex = startRingIndex

  if ringIndex == endRingIndex:
    for outputItem in genBasicSpiralRingCoords(ringIndex,startTInStartRing,endTInEndRing,direction):
      yield outputItem
    return
  else:
    for outputItem in genBasicSpiralRingCoords(ringIndex,startTInStartRing,None,direction):
      yield outputItem
    ringIndex += direction

  while ringIndex != endRingIndex: #iterate ringIndex
    #print("ringIndex",ringIndex)
    if ringIndex == 0:
      yield (0,0)
      assert direction < 0
    for outputItem in genBasicSpiralRingCoords(ringIndex,None,None,direction):
      yield outputItem
    ringIndex += direction

  assert ringIndex == endRingIndex
  if ringIndex == 0:
    yield (0,0)
    return

  for outputItem in genBasicSpiralRingCoords(ringIndex,None,endTInEndRing,direction):
    yield outputItem
  return

def genBasicSpiralRingCoords(ringIndex,startTInRing,endTInRing,direction):
  #print("genBasicSpiralRingCoords: ringIndex={}, startTInRing={}, endTInRing={}, direction={}.".format(ringIndex,startTInRing,endTInRing,direction))
  if ringIndex == 0:
    if startTInRing in [0,None] and endTInRing in [0,None]:
      yield (0,0)
      return
    else:
      raise ValueError("the conditions to work with ringIndex=0 are not met. startTInRing and endTInRing each must be either 0 or None.")
  sideLen = spiralRingSideLength(ringIndex, 1)

  if startTInRing != None:
    startSideInRing, startTInStartSide = findSubdivisionAndInnerLocation(0, startTInRing, subdivisionSize=sideLen)
    sideInRingRangeStart = startSideInRing
  else:
    startSideInRing, startTInStartSide = (None, None)
    sideInRingRangeStart = 0 if direction > 0 else 3

  if endTInRing != None:
    endSideInRing, endTInEndSide = findSubdivisionAndInnerLocation(0, endTInRing, subdivisionSize=sideLen)
    sideInRingSeqEnd = endSideInRing
  else:
    endSideInRing, endTInEndSide = (None,None)
    sideInRingSeqEnd = 3 if direction > 0 else 0

  sideInRingSeq = range(sideInRingRangeStart, sideInRingSeqEnd + sign(direction), direction)

  basicTInSideRangeStart = 0 if direction > 0 else sideLen-1
  basicTInSideRangeEnd = sideLen if direction > 0 else -1

  #print("genBasicSpiralRingCoords: (startSideInRing,startTInStartSide,sideInRingRangeStart)="+str((startSideInRing, startTInStartSide, sideInRingRangeStart)))
  #print("genBasicSpiralRingCoords: (endSideInRing,endTInEndSide,sideInRingSeqEnd)="+str((endSideInRing, endTInEndSide, sideInRingSeqEnd)))
  #print("genBasicSpiralRingCoords: (basicTInSideRangeStart,basicTInSideRangeEnd)="+str((basicTInSideRangeStart, basicTInSideRangeEnd)))

  #print("genBasicSpiralRingCoords: sideInRingSeq={}".format(sideInRingSeq)) #THIS LINE NOT PYTHON3 COMPATIBLE.

  for sideInRing in sideInRingSeq: #iterate sideInRing
    tInSideSeq = range(startTInStartSide if sideInRing==startSideInRing else basicTInSideRangeStart, endTInEndSide+direction if sideInRing==endSideInRing else basicTInSideRangeEnd, direction)
    sideDirection = directionIndexToVector((sideInRing+1)%4)
    sideStartCoord = spiralRingSideStartCoords(ringIndex)[sideInRing]
    #print("genBasicSpiralRingCoords: sideInRing={}, tInSideSeq={}.".format(sideInRing,tInSideSeq))
    for tInSide in tInSideSeq:
      result = coordSum([sideStartCoord,scaledCoord(sideDirection,tInSide)])
      #print("genBasicSpiralRingCoords: sideInRing={}, tInSide={}, yielding {}.".format(sideInRing,tInSide,result))
      yield result
  return

def genCircularSpiralCoords(**customizationKwargs):
  for basicCoord in genBasicCircularSpiralCoords():
    yield customizedSpiralCoord(basicCoord, **customizationKwargs)

def genBasicCircularSpiralCoords():
  return genDezigged(genBasicSpiralCoords(), keyFun=magnitude)

def genDezigged(inputSeq, forbidLocalMinimaDecrease=True, forbidLocalMaximaDecrease=True, keyFun=None):
  if keyFun == None:
    keyFun = (lambda x: x)
  inputGen = makeGen(inputSeq)

  rememberedLocalMinima = collections.deque()
  rememberedLocalMaxima = collections.deque()
  history = []
  currentTriplet = collections.deque()

  sortKeyFun = (lambda itemA: itemA[0])

  #getStatus = (lambda: "status:(history={}, rlmn={}, rlmx={}, triplet={}, canOutputNow()={})".format(history, rememberedLocalMinima, rememberedLocalMaxima, currentTriplet, canOutputNow()))

  if forbidLocalMinimaDecrease and forbidLocalMaximaDecrease:
    def localExtremaAllowOutputNow():
      testResult = sortKeyFun(rememberedLocalMinima[-1]) > sortKeyFun(rememberedLocalMaxima[0])
      #maybe they could be popped now?
      return testResult
  else:
    def localExtremaAllowOutputNow():
      return sortKeyFun(max(rememberedLocalMinima, key=sortKeyFun)) > sortKeyFun(min(rememberedLocalMaxima, key=sortKeyFun))

  def canOutputNow():
    if len(history) == 0:
      return False
    if len(rememberedLocalMinima) == 0 or len(rememberedLocalMaxima) == 0:
      return False
    return localExtremaAllowOutputNow()

  def detectLocalExtrema(suspendRules=False):
    assert len(currentTriplet) in [2,3]
    if sortKeyFun(min(currentTriplet ,key=sortKeyFun)) == sortKeyFun(currentTriplet[1]):
      if forbidLocalMinimaDecrease and not suspendRules:
        if len(rememberedLocalMinima) > 0:
          if not sortKeyFun(rememberedLocalMinima[-1]) <= sortKeyFun(currentTriplet[1]):
            raise ValueError("forbidLocalMinimaDecrease violated by inputSeq!")
      #print("SpiralMath.genDezigged.detectLocalExtrema: detected local minimum {}. before registering, {}.".format(currentTriplet[1], getStatus()))
      rememberedLocalMinima.append(currentTriplet[1])
      #print("SpiralMath.genDezigged.detectLocalExtrema: detected local minimum {}. after registering, {}.\n".format(currentTriplet[1], getStatus()))
    if sortKeyFun(max(currentTriplet, key=sortKeyFun)) == sortKeyFun(currentTriplet[1]):
      if forbidLocalMaximaDecrease and not suspendRules:
        if len(rememberedLocalMaxima) > 0:
          if not sortKeyFun(rememberedLocalMaxima[-1]) <= sortKeyFun(currentTriplet[1]):
            raise ValueError("forbidLocalMaximaDecrease violated by inputSeq!")
      #print("SpiralMath.genDezigged.detectLocalExtrema: detected local maximum {}. before registering, {}.".format(currentTriplet[1], getStatus()))
      rememberedLocalMaxima.append(currentTriplet[1])
      #print("SpiralMath.genDezigged.detectLocalExtrema: detected local maximum {}. after registering, {}.\n".format(currentTriplet[1], getStatus()))

  currentTripletFilledBefore = False
  inputItemsExhausted = False

  while True:
    assert len(currentTriplet) <= 3
    if len(currentTriplet) == 3:
      currentTripletFilledBefore = True

    if canOutputNow():
      #print("SpiralMath.genDezigged: normal out: {}, yielding {}.\n".format(getStatus(), history[0][1]))
      yield history[0][1]
      if forbidLocalMinimaDecrease:
        if history[0] == rememberedLocalMinima[0]:
          rememberedLocalMinima.popleft()
      else:
        raise NotImplementedError("There is currently no way to remove items from rememberedLocalMaxima when they are not sorted. Try forbidding decreases in them.")
      if forbidLocalMaximaDecrease:
        if history[0] == rememberedLocalMaxima[0]:
          rememberedLocalMaxima.popleft()
      else:
        raise NotImplementedError("There is currently no way to remove items from rememberedLocalMaxima when they are not sorted. Try forbidding decreases in them.")
      del history[0] #@ slow!
      continue

    try:
      nextValue = next(inputGen)
      nextKey = keyFun(nextValue)
      nextItem = (nextKey, nextValue)
      #print("SpiralMath.genDezigged: notice: {}, ingesting {}.".format(getStatus(), nextItem))
      assert len(currentTriplet) <= 3
      currentTriplet.append(nextItem)
      insort(history, nextItem, stable=True, keyFun=sortKeyFun)
      #print("SpiralMath.genDezigged: notice: {}, ingested {}.\n".format(getStatus(), nextItem))

    except StopIteration:
      inputItemsExhausted = True

    if currentTripletFilledBefore:
      currentTriplet.popleft() #to make up for not doing it earlier in the try block above.
      assert len(currentTriplet) <= 3

      if len(currentTriplet) == 3:
        detectLocalExtrema()
        continue
      elif len(currentTriplet) == 2:
        assert inputItemsExhausted, "if not, then why is currentTriplet less than full?"
        detectLocalExtrema(suspendRules=True)
        continue
      elif len(currentTriplet) == 1:
        pass #allow endgame.
      elif len(currentTriplet) == 0:
        raise ValueError()
      else:
        assert False
    else:
      continue #don't reach the endgame before the right time.

    assert len(currentTriplet) == 1
    assert inputItemsExhausted

    for endgameItem in sorted(history):

      #print("SpiralMath.genDezigged: endgame out: {}, yielding {}.\n".format(getStatus(), history[0][1]))
      yield endgameItem[1]
    return

  assert False, "unreachable statement."

for rotationDirection in [-1,1]:
  for startDirection in [0,1,2,3]:
    for startPosition in [(0,0),(20,100)]:
      assert [spiralCoordEncode(spiralCoordDecode(i,home=startPosition,rot=startDirection,spin=rotationDirection),home=startPosition,rot=startDirection,spin=rotationDirection) for i in range(50)] == [i for i in range(50)]

assert [basicSpiralCoordDecode(t) for t in range(16)] == [(0, 0), (0, 1), (1, 1), (1, 0), (1, -1), (0, -1), (-1, -1), (-1, 0), (-1, 1), (-1, 2), (0, 2), (1, 2), (2, 2), (2, 1), (2, 0), (2, -1)]


# -*- coding: utf-8 -*-

"""
Created on Thu Apr  2 12:11:58 2020

@author: carlo
"""

import PyPDF2

def watermark_pages(input_pdf,watermark,output_pdf):

    pdf = PyPDF2.PdfFileReader(input_pdf)
    watermark = PyPDF2.PdfFileReader(watermark)
    output = PyPDF2.PdfFileWriter()

    for page_number in range( pdf.getNumPages()):
        pages = pdf.getPage(page_number)
        pages.mergePage(watermark.getPage(0))
        output.addPage(pages)

    with open(output_pdf, "wb") as final_pdf:
       output.write(final_pdf)

    return 'all done'


import flask
from pymongo import MongoClient
from bson.objectid import ObjectId
from bson.json_util import dumps, RELAXED_JSON_OPTIONS

from backend import app

def get_db():
    """Open a new database connection."""
    if not hasattr(flask.g, 'mongodb'):
        client = MongoClient()
        flask.g.mongodb = client[app.config['DATABASE_NAME']]
    return flask.g.mongodb

def jsonify(data):
    return dumps(data, json_options=RELAXED_JSON_OPTIONS)

def db_get_user_groups(uid):
    db = get_db()
    pipeline = [{
        "$match": {
            "uid": ObjectId(uid)
        }
    }, {
        "$lookup": {
            "from": "groups",
            "localField": "gid",
            "foreignField": "_id",
            "as": "group"
        }
    }, {
        "$group": {
            "_id": "$uid",
            "groups": {"$push": {"$arrayElemAt": ["$group", 0]}}
        }
    }]
    data = list(db.group_user.aggregate(pipeline))
    if data:
        data = data[0]["groups"]
    return data


from flask import Flask, request, redirect, render_template, jsonify, Response
from datetime import date, datetime
import requests

app = Flask(__name__)

@app.route("/")
def index():
    return render_template('api.html')

def hora_sistema():
    url = f'http://127.0.0.1:5000/hora'
    response = requests.get(url)

    if response.status_code == 200:
        dados = response.json()
        print('Hora recebida com sucesso!')
        return dados
    else:
        print('Falha ao enviar a mensagem.')

@app.route('/validar/<int:id>/<int:valorRem>/<int:valorTrans>/<string:horario>', methods=['POST'])
def receberTransacao(id,valorRem, valorTrans,horario):
    if request.method != 'POST' and id == '':
        return jsonify(['Method Not Allowed'])

    return jsonify({'id': id, 'status': 2})

@app.errorhandler(404)
def page_not_found(error):
    return render_template('page_not_found.html'), 404

if __name__ == "__main__":
    app.run(host='0.0.0.0',port=50010, debug=True)