import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Output, Input
import dash_bootstrap_components as dbc
import pandas as pd
import plotly.express as px
from plotly.graph_objs import *

def run_dash(bales_graph_file_path, pie_chart_file_path, bar_chart_file_path, leader_chart_file_path, leader):
    df_bales = pd.read_csv(bales_graph_file_path)
    df_word = pd.read_csv(pie_chart_file_path)
    df_pos_neg = pd.read_csv(bar_chart_file_path)
    df_leader = pd.read_csv(leader_chart_file_path)

    leader_title = "Potential leader: Player " + str(leader)
    leadership_style = "There is a task-oriented positive leadership style"
    player_colors = ['#E06C75', '#98C379', '#E5C07B', '#C678DD', '#56B6C2']
    leader_color = player_colors[leader-1]

    app = dash.Dash(
        external_stylesheets=[dbc.themes.SUPERHERO]
    )

    layout_set = Layout(
        paper_bgcolor='rgba(0,0,0,0)',
        plot_bgcolor='rgba(0,0,0,0)',
        font=dict(
            family="Verdana",
            color="Black"
        )
    )

    df_line_graph = df_bales
    fig_bales = px.line(df_line_graph, x="Category", y="Amount", color="Player",
                        color_discrete_map={'Player 1': '#E06C75', 'Player 2': '#98C379', 'Player 3': '#E5C07B',
                                            'Player 4': '#C678DD', 'Player 5': '#56B6C2'},
                        title="Bales Interaction Process Analysis Profiles")

    df_pie_chart = df_word
    fig_pie_chart = px.pie(df_pie_chart, values="Words", names='Player', hole=.5, color='Player',
                           color_discrete_map={'Player 1': '#E06C75', 'Player 2': '#98C379', 'Player 3': '#E5C07B',
                                               'Player 4': '#C678DD', 'Player 5': '#56B6C2'},
                           title="Percentage of words "
                                 "said")

    df_bar_chart = df_pos_neg
    fig_bar_chart = px.bar(df_bar_chart, x="Amount", y="Polarity", color="Player", orientation="h",
                           color_discrete_map={'Player 1': '#E06C75', 'Player 2': '#98C379', 'Player 3': '#E5C07B',
                                               'Player 4': '#C678DD', 'Player 5': '#56B6C2'}, title="Amount of "
                                                                                "positive and "
                                                                                "negative "
                                                                                "statements")

    df_leader_chart = df_leader
    fig_leader_chart = px.bar(df_leader_chart, x="Category", y="Amount (%)", color='Category', color_discrete_map={'Sentences': leader_color, 'Positive': leader_color, 'Negative': leader_color}, title=leader_title)

    fig_bales.update_layout(layout_set)
    fig_pie_chart.update_layout(layout_set)
    fig_bar_chart.update_layout(layout_set)
    fig_leader_chart.update_layout(layout_set)
    fig_leader_chart.update_layout(showlegend=False)

    app.layout = html.Div([
        dbc.Row([
            html.Div([html.Img(
                src='https://upload.wikimedia.org/wikipedia/commons/thumb/f/f4/Logo_marine.svg/1200px-Logo_marine.svg.png',
                style={'margin-left': '70%', 'margin-right': 'auto', 'margin-bottom': '1%'}),
                html.Img(src='https://www.kiemt.nl/wp-content/uploads/2018/06/logos-partners_0000s_0017_UTwente.png',
                         style={'margin-left': 'auto', 'margin-right': '10%', 'margin-bottom': '1%'})],
                style={'background-color': '#ffffff', 'width': '100%', 'height': '90px', 'display': 'flex'}),
        ]),
        dbc.Row([
            html.Div(
                dbc.Col(html.H1("Communication and Leadership Competency Dashboard",
                                style={'textAlign': 'left', 'font-family': 'Verdana', 'background-color': '#0E61AA',
                                       'font-size': '30px', 'margin': '5px'}), width=12))
        ], style={'background-color': '#0E61AA', 'padding-left': '1%', 'padding-top': '10px', 'padding-bottom': '10px',
                  'box-shadow': '0px 5px 2px #888888'}),
        dbc.Row([
            html.Div(
                dbc.Col(dcc.Graph(
                    id="test",
                    figure=fig_bales)
                ),
                style={'background-color': 'white', 'width': '50%', 'border-radius': '25px', 'margin-left': '3%',
                       'margin-top': '2%', 'margin-bottom': '2%', 'box-shadow': '5px 10px 5px'}),
            html.Div(
                dbc.Col(dbc.Col(dcc.Graph(
                             id="test4",
                             figure=fig_leader_chart)
                         )),
                style={'background-color': 'white', 'width': '18.5%', 'border-radius': '25px', 'margin-left': '3%',
                       'margin-top': '2%', 'margin-bottom': '2%', 'box-shadow': '5px 10px 5px'}),
            html.Div(
                dbc.Col([html.H1("What type of leadership has the group leader?",
                                 style={'textAlign': 'center', 'color': 'black', 'margin': '5px', 'margin-top': '20px',
                                        'font-size': '25px'}),
                         html.Hr(style={'border': '1px solid black'}),
                         html.H2(leadership_style,
                                 style={'textAlign': 'center', 'color': 'black', 'margin': '5px', 'margin-top': '20px',
                                        'font-size': '20px'}),
                         html.H3("Source: Butler et al.",
                                 style={'textAlign': 'right', 'color': 'black', 'margin': '5px', 'margin-top': '75%',
                                        'font-size': '13px'})]),
                style={'background-color': 'white', 'width': '18.5%', 'border-radius': '25px', 'margin-left': '3%',
                       'margin-top': '2%', 'margin-bottom': '2%', 'box-shadow': '5px 10px 5px'})
        ]),
        dbc.Row([
            html.Div(
                dbc.Col(dcc.Graph(
                    id="test2",
                    figure=fig_pie_chart)
                ), style={'background-color': 'white', 'width': '35%', 'border-radius': '25px', 'margin-left': '3%',
                          'margin-bottom': '2%', 'box-shadow': '5px 10px 5px'}),
            html.Div(
                dbc.Col(dcc.Graph(
                    id="test3",
                    figure=fig_bar_chart)
                ),
                style={'background-color': 'white', 'box-shadow': '5px 10px 5px', 'width': '55%',
                       'border-radius': '25px',
                       'margin-left': '3%', 'margin-bottom': '2%'})
        ])
    ], style={'background-color': '#f3f3f3'})

    return app


from django.urls import path

from .views import (
    BasicInfoView,
    ContactFormView,
    FindProviderMedicationView,
    GetFormOptionsView,
)

public_api_urlpatterns = [
    path(
        'options/',
        GetFormOptionsView.as_view(),
    ),
    path(
        'find_providers/',
        FindProviderMedicationView.as_view(),
    ),
    path(
        'basic_info/',
        BasicInfoView.as_view(),
    ),
    path(
        'contact_form/',
        ContactFormView.as_view(),
    ),
]


from flask import Flask, request, jsonify
from urllib.request import urlopen
import json

app = Flask(__name__, static_folder='static')

@app.route('/')
def index():
    return app.send_static_file('index.html')

@app.route('/api/submit', methods=['POST'])
def submit():
    data = request.get_json()
    message = data.get('message')

    message = input("response")

    response = {'status': 'success', 'message': message}

    return jsonify(response)

@app.route('/api/submit/setword', methods=['POST'])
def setWordOfTheDay():
    data = request.get_json()

    addWordToList(data.get('word'), data.get('definition'))

    return jsonify({ 'status': 'success' })

@app.route('/api/submit/getword', methods=['POST'])
def getWordOfTheDay():
    data = request.get_json()

    current = getCurrentWord()

    word = current.get('word')
    definition = current.get('definition')

    url = 'https://api.dictionaryapi.dev/api/v2/entries/en/' + word

    return jsonify({ 'status': 'success', 'word': word, 'definition': definition })

def addWordToList(word, definition):
    text = ''
    with open(r'wordOfTheDayList.txt', 'r') as file:
        text = file.read()
        text += ', ' + word + ':' + definition

    with open(r'wordOfTheDayList.txt', 'w') as file:
        file.write(text)

def getCurrentWord():
    with open(r'wordOfTheDayList.txt', 'r') as file:
        text = file.read()
        try:
            list = text.split(", ")
            list = list[len(list) - 1].split(":")
            word = list[0]
            definition = list[1]
            return { 'word': word, 'definition': definition }
        except:
            print(Exception('Parsing error'))
            return { 'word': 'error', 'definition': 'error' }

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000)


# coding=utf-8

import datetime
from OnlineClassroom.app.ext.plugins import db
from .curriculums import *
from .account import *
from .catalog import *
"""
购买记录
CREATE TABLE `shopping_carts` (
  `aid` int DEFAULT NULL COMMENT '外键 用户id',
  `cid` int DEFAULT NULL COMMENT '外键 课程id',
  `number` int DEFAULT '1' COMMENT '课程数量,但是是网易云课堂类似的,默认就是1买把...',
  `price` float(10,2) DEFAULT '0.00' COMMENT '购买时价格',
  `create_at` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '创建时间',
  KEY `uid` (`aid`),
  KEY `cid` (`cid`),
  CONSTRAINT `shopping_carts_ibfk_1` FOREIGN KEY (`aid`) REFERENCES `accounts` (`aid`),
  CONSTRAINT `shopping_carts_ibfk_2` FOREIGN KEY (`cid`) REFERENCES `curriculums` (`cid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
"""

class ShoppingCarts(db.Model):
    __tablename__ = "shopping_carts"

    aid = db.Column(db.Integer,db.ForeignKey("accounts.aid"),primary_key=True, comment="外键 用户id")
    cid = db.Column(db.Integer,db.ForeignKey("curriculums.cid"),primary_key=True,comment="外键 课程id")
    number = db.Column(db.Integer,default=1)
    price = db.Column(db.Float(10,2),default=0.00)
    create_at = db.Column(db.DateTime,default=datetime.datetime.utcnow(),comment="创建时间")

    _curriculum = db.relationship(Curriculums,backref="shop")
    # user = db.relationship(Account,backref="shops")

    def __repr__(self):
        return "数据库{} {}_{}_{}_{}".format(self.__tablename__,self.cid,self.aid,self.number,self.price)

    def __init__(self,aid=None,cid=None,price=None):
        self.aid =aid
        self.cid = cid
        self.number = 1
        self.price = price
        self.create_at = datetime.datetime.now()

    # 查询是否购买或者价格为0
    def is_record(self):
        shop = self.query.filter_by(aid=self.aid,cid=self.cid).first()
        if shop == None:
            cu = Curriculums.query.filter_by(cid=self.cid).first()
            if cu.price <= float(0):
                return True
        else:
            if shop.number != 1:
                if not shop.curriculum.price <= float(0):
                    return False
                return True
            elif shop.number == 1:
                return True

    def get_curriculum__catalog(self):

        _catalogs = Catalog.query.filter_by(cat_id=self.cid).all()

        items = {}
        list_time = []

        for catalog in _catalogs:
            list_time.append(catalog.serializetion_itme())

        items["datas"] = list_time
        items["len"] = len(_catalogs)
        return items

    def is_purchase(self):
        shop = self.query.filter(ShoppingCarts.aid==self.aid,ShoppingCarts.cid==self.cid).first()
        if shop == None:
            return False
        return shop.number == 1

    def save(self):
        self.number = 1
        self.create_at = datetime.datetime.now()
        return self.is_commit()

    # commit
    def is_commit(self):
        try:
            db.session.add(self)
            db.session.commit()
            return True
        except Exception as e:
            db.session.rollback()
            return False

    def serialize_item(self):
        item = {
            "aid":self.aid,
            "nickname":self.aid.user.nickname,
            "cid":self.cid,
            "number":self.number,
            "price":self.price,
            "create_at":self.create_at
        }
        return item

    def get_purchase_curriculums(self,page=1,number=10):
        if page ==None:
            page = 1
        if number ==None:
            number = 10

        items = {}
        list_item = []

        shops = self.query.filter_by(aid=self.aid).paginate(int(page),int(number),False)
        for shop in shops.items:
            list_item.append(shop._curriculum.serialize_item())

        items["datas"] = list_item
        items["len"] = len(shops.items)
        items["nexts"] = shops.pages
        items["total"] = shops.total

        return items

    def get_days_shop_effective_sum(self,day):
        sql = """
        select
        date_format(s.create_at,'%Y-%m-%d %H:%i:%S')as dateDay,sum(s.price)as moneyDay
        FROM shopping_carts as s
        group by dateDay
        order by dateDay desc limit 0,{}
        """.format(day)

        results = db.session.execute(sql).fetchall()
        items = sql_result_to_dict(results)
        return items

    def get_month_shop_effective_sum(self,month):
        sql = """
        select
        date_format(s.create_at,'%Y-%m')as dateMonth,sum(s.price)as moneyMonth
        FROM shopping_carts as s
        group by dateMonth
        order by dateMonth desc limit 0,{}
        """.format(month)

        results = db.session.execute(sql).fetchall()
        items = sql_result_to_dict(results)
        return items

    def get_monerys(self):
        sql = """
        select sum(s.price)as moneys FROM shopping_carts as s
        """
        results = db.session.execute(sql).fetchall()
        items = sql_result_to_dict(results)
        return items[0]


# -*- coding: utf-8 -*-
import os
import shutil
import torch
# from torch.utils.data import *
from torch.utils import data
from imutils import paths
import numpy as np
import random
from PIL import Image
from torchvision.transforms import transforms
import cv2

def cv_imread(path):
    img = cv2.imdecode(np.fromfile(path, dtype=np.uint8),-1)
    return img
def cv_imwrite(path,img):
    cv2.imencode('.jpg', img)[1].tofile(path)

def checkfiles(root):
    if(not os.path.exists(root)):
        os.mkdir(root)
    else:
        shutil.rmtree(root)
        os.mkdir(root)
def ensurefiles(root):
    if(not os.path.exists(root)):
        os.mkdir(root)
    else:
        pass

class crack_loader(data.Dataset):
    def __init__(self, img_file, imgSize, PreprocFun=None):
        with open(img_file,'r',encoding='utf-8') as f:
            lines = f.readlines()
        self.img_paths = [line.strip().split(',') for line in lines]
        self.img_size = imgSize
        if PreprocFun is not None:
            self.PreprocFun = PreprocFun
        else:
            self.PreprocFun = self.transform
    def __len__(self):
        return len(self.img_paths)

    def __getitem__(self, index):
        raw_img_path,label_img_path,mask_img_path = self.img_paths[index]
        raw_img = Image.open(raw_img_path)
        # raw_img = cv_imread(raw_img_path)
        label_img = cv_imread(label_img_path)
        mask_img = cv_imread(mask_img_path)
        try:
            height1, width1 = label_img.shape
        except:
            label_img = label_img.mean(2)
            height1, width1 = label_img.shape
        if height1 != self.img_size or width1 != self.img_size:
            label_img = cv2.resize(label_img, (self.img_size,self.img_size))
        raw_img = self.PreprocFun(raw_img)

        return raw_img, torch.tensor(label_img/255.0,dtype=torch.long), torch.tensor(mask_img,dtype=torch.long)#.unsqueeze(0)#torch.tensor(raw_img,dtype=torch.float32)

    def transform(self, img):
        transform_pre = transforms.Compose(
            [
                transforms.Resize((512,512)),
                # transforms.RandomHorizontalFlip(),
                transforms.ToTensor(),
                transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
            ]
        )
        img = transform_pre(img)
        return img

if __name__ == "__main__":
    train_set = r'./scheme_set/train.txt'
    train_loader = crack_loader(train_set,512)
    for i in train_loader:
        print(i)


import urllib.request, urllib.parse, urllib.error
import twurl
import json
import ssl

# https://apps.twitter.com/
# Create App and get the four strings, put them in hidden.py

TWITTER_URL = 'https://api.twitter.com/1.1/friends/list.json'

# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

def get_data(acct, info):
    """
    (string, string) ->

    Function prints on the screen information from json file that user wants to get
    """
    url = twurl.augment(TWITTER_URL,
                        {'screen_name': acct, 'count': '10'})
    connection = urllib.request.urlopen(url, context=ctx)
    headers = dict(connection.getheaders())
    data = connection.read().decode()

    js = json.loads(data)

    for i in js['users']:
        print(i['name'])
        if info == "followers_count":
            print(i["followers_count"])
        elif info == "friends_count":
            print(i["friends_count"])
        elif info == "date of creation":
            print(i["created_at"])
        elif info == "id":
            print(i["id"])
        elif info == "screen_name":
            print(i["screen_name"])
        elif info == "location":
            print(i["location"])
        elif info == "url":
            print(i["url"])

if __name__ == "__main__":
    acct = input('Enter Twitter Account:')
    print(" ")
    print(["id", "location", "url", "followers_count", "date of creation", "screen_name"])
    info = input('Enter where you want to get:')
    print(get_data(acct, info))


import networkx as nx
import numpy as np

from params import args

class JobDAG(object):

    def __init__(self, nodes, adj_mat, name):
        # nodes: list of N nodes
        # adj_mat: N by N 0-1 adjacency matrix, e_ij = 1 -> edge from i to j
        assert len(nodes) == adj_mat.shape[0]
        assert adj_mat.shape[0] == adj_mat.shape[1]

        self.name = name

        self.nodes = nodes
        self.adj_mat = adj_mat

        self.num_nodes = len(self.nodes)
        self.num_nodes_done = 0

        # set of executors currently running on the job
        self.executors = set()

        # the computation graph needs to be a DAG
        assert is_dag(self.num_nodes, self.adj_mat)

        # get the set of schedule nodes
        self.frontier_nodes = set()
        for node in self.nodes:
            if node.is_schedulable():
                self.frontier_nodes.add(node)

        # assign job dag to node
        self.assign_job_dag_to_node()

        # dag is arrived
        self.arrived = False

        # dag is completed
        self.completed = False

        # dag start ime
        self.start_time = None

        # dag completion time
        self.completion_time = np.inf

        # map a executor number to an interval
        self.executor_interval_map = self.get_executor_interval_map()

    def assign_job_dag_to_node(self):
        for node in self.nodes:
            node.job_dag = self

    def get_executor_interval_map(self):
        executor_interval_map = {}
        entry_pt = 0

        # get the left most map
        for e in range(args.executor_data_point[0] + 1):
            executor_interval_map[e] = \
                (args.executor_data_point[0],
                 args.executor_data_point[0])

        # get the center map
        for i in range(len(args.executor_data_point) - 1):
            for e in range(args.executor_data_point[i] + 1,
                            args.executor_data_point[i + 1]):
                executor_interval_map[e] = \
                    (args.executor_data_point[i],
                     args.executor_data_point[i + 1])
            # at the data point
            e = args.executor_data_point[i + 1]
            executor_interval_map[e] = \
                (args.executor_data_point[i + 1],
                 args.executor_data_point[i + 1])

        # get the residual map
        if args.exec_cap > args.executor_data_point[-1]:
            for e in range(args.executor_data_point[-1] + 1,
                            args.exec_cap + 1):
                executor_interval_map[e] = \
                    (args.executor_data_point[-1],
                     args.executor_data_point[-1])

        return executor_interval_map

    def reset(self):
        for node in self.nodes:
            node.reset()
        self.num_nodes_done = 0
        self.executors = set()
        self.frontier_nodes = set()
        for node in self.nodes:
            if node.is_schedulable():
                self.frontier_nodes.add(node)
        self.arrived = False
        self.completed = False
        self.completion_time = np.inf

    def update_frontier_nodes(self, node):
        frontier_nodes_changed = False
        for child in node.child_nodes:
            if child.is_schedulable():
                if child.idx not in self.frontier_nodes:
                    self.frontier_nodes.add(child)
                    frontier_nodes_changed = True
        return frontier_nodes_changed

def is_dag(num_nodes, adj_mat):
    G = nx.DiGraph()
    G.add_nodes_from(range(num_nodes))
    for i in range(num_nodes):
        for j in range(num_nodes):
            if adj_mat[i, j] == 1:
                G.add_edge(i, j)
    return nx.is_directed_acyclic_graph(G)


import tensorflow as tf
gpus = tf.config.list_physical_devices('GPU')
if gpus:
    tf.config.set_logical_device_configuration(gpus[0], [tf.config.LogicalDeviceConfiguration(memory_limit=5292)])

import keras
import matplotlib.pyplot as plt
import numpy as np
from keras import layers, optimizers, losses, metrics, Model
from PIL import Image
from sklearn import preprocessing as pre

# Construct the discriminator.
def create_D():
    input_img = keras.Input(shape=(256,256, 1))

    conv1 = layers.Conv2D(8, (2,2), (2,2), padding='valid',
                                activation=layers.ReLU())(input_img)
    drop1 = layers.Dropout(rate=0.5)(conv1)

    down1 = layers.MaxPool2D(pool_size=(2,2), strides=(2,2), padding="valid")(drop1)

    flat1 = layers.Flatten()(down1)

    score = layers.Dense(1, activation='sigmoid')(flat1)

    D: Model = Model(inputs=input_img, outputs=score)
    D.compile(optimizer=optimizers.Nadam(learning_rate=0.002), loss=losses.BinaryCrossentropy(), metrics=metrics.MeanAbsoluteError())
    return D

# Construct the generator.
def create_G():
    random_input = keras.Input(shape=(100, ))

    Dense1 = layers.Dense(units = 16*16)(random_input)

    B_Norm1 = layers.BatchNormalization()(Dense1)

    Relu1 = layers.LeakyReLU()(B_Norm1)

    reshape1 = layers.Reshape(target_shape=(16, 16, 1))(Relu1)

    DeConv1 = layers.Conv2DTranspose(filters=4, kernel_size=(2, 2), strides=(1,1), padding='same')(reshape1)

    B_Norm2 = layers.BatchNormalization()(DeConv1)

    Relu2 = layers.LeakyReLU()(B_Norm2)

    DeConv2 = layers.Conv2DTranspose(filters=16, kernel_size=(3,3), strides=(1,1), padding='same')(Relu2)

    re = layers.Reshape((16,16,16,1))(DeConv2)

    maxi0 = layers.MaxPool3D(pool_size=(4,4,2))(re)

    B_Norm3 = layers.BatchNormalization()(maxi0)

    Relu3 = layers.LeakyReLU()(B_Norm3)

    flat0 = layers.Flatten()(Relu3)

    dense00 = layers.Dense(units=256*256, activation='tanh')(flat0)

    output_img = layers.Reshape((256, 256, 1))(dense00)

    G = Model(inputs=random_input, outputs=output_img)
    return G

# Join the discriminator and generator, forming the final GAN model.
def create_GAN():
    D = create_D()
    G = create_G()
    latent_input = keras.Input(shape=(100, ))
    img = G(latent_input)
    D.trainable = False
    score = D(img)
    GAN = Model(inputs = latent_input, outputs=score)
    GAN.compile(loss=losses.BinaryCrossentropy(),
            optimizer=optimizers.Nadam(learning_rate=0.002), metrics=metrics.MeanAbsoluteError())
    return D, G, GAN

# Function returns a sample of latent points from a normal distribution.
def getLatentSamples(num_samples) -> np.ndarray:
    latent = np.random.normal(size=100*int(num_samples))
    latent = np.reshape(a=latent, newshape=(int(num_samples), 100))
    return latent

# Function imputs latent data into the generator and returns the fake image samples outputted
# by the generator.
def getRealSamples(batch_size: int, train_img: np.ndarray):
    X_real, y_real = np.array(train_img.tolist() * batch_size, dtype=np.float32).reshape((batch_size, train_img.shape[0], train_img.shape[1])), np.ones(shape=batch_size, dtype=np.int8)
    assert X_real.shape == (batch_size, train_img.shape[0], train_img.shape[1])
    assert y_real.shape == (batch_size, )
    return X_real, y_real

# Function imputs latent data into the generator and returns the fake image samples outputted
# by the generator.
def getFakeSamples(G:Model, batch_size:int):
    latent = getLatentSamples(batch_size)
    assert latent.shape == (batch_size, 100)
    fake_imgs = G(latent)
    fake_imgs = np.asarray(fake_imgs, dtype=np.float32)
    assert fake_imgs.shape[0] == batch_size
    fake_imgs = np.squeeze(fake_imgs)
    output_labels = np.zeros(shape=batch_size, dtype=np.int8)
    assert output_labels.shape == (batch_size, )
    return fake_imgs, output_labels

# Function plots a sample of 16 'predicted' images from the trained generator.
def pred_plot(G: Model, epoch:int, train_img_shape: tuple[int, int]):
    shape = train_img_shape
    latent = getLatentSamples(1)
    gen_image = G(latent)
    gen_image = np.asarray(gen_image, dtype=np.float32)
    gen_image = np.squeeze(gen_image)
    assert gen_image.shape == shape
    filename = str('images/gen3-%d.png' %epoch)
    plt.imsave(filename, gen_image, cmap='gray')
    if epoch % 100 == 0:
        im=Image.open(filename)
        im.load()
        im.convert('L').show()

# Function plots the networks loss values with respect to epoch.
def loss_plot(dLossIn, gLossIn):
    fig, ax = plt.subplots(figsize=(12, 8))
    ax.plot(dLossIn, label='Discriminator Loss')
    ax.plot(gLossIn, label='GAN (generator) Loss')
    ax.legend()
    ax.set_xlim(xmin=-5, xmax=305)
    ax.set_ylim(ymin=-0.2, ymax=1.2)
    plt.xlabel('Epoch')
    plt.ylabel('Loss value')
    plt.title("Loss value vs Epoch for Generator and Discriminator")

# Function trains the GAN (generator) model.
def trainGAN(D:Model, G:Model, GAN:Model, num_epochs: int, batchSize: int, train_img: np.ndarray, train_img_shape: tuple[int, int]):
    d_loss, g_loss = [], []
    d_mae, g_mae = [], []
    pred_plot(G, 0, train_img_shape)
    for epoch in range(1, num_epochs+1):
        X_real, y_real = getRealSamples(int(batchSize/2), train_img)
        X_fake, y_fake = getFakeSamples(G, int(batchSize / 2))
        assert X_real.shape == X_fake.shape
        assert X_real.shape[0] == int(batchSize / 2)
        assert y_real.shape == y_fake.shape
        assert y_real.shape[0] == int(batchSize / 2)
        D.trainable = True
        d_loss_fake, mae_fake = D.train_on_batch(X_fake, y_fake)
        d_loss_real, mae_real = D.train_on_batch(X_real, y_real)
        D.trainable = False
        X_latent, y_latent = getLatentSamples(batchSize), np.ones(shape=batchSize, dtype=np.int8)
        loss, mae = GAN.train_on_batch(X_latent, y_latent)

        d_loss_res = float((d_loss_fake + d_loss_real) / 2)
        d_mae_res = float((mae_fake + mae_real) / 2)
        d_loss.append(d_loss_res)
        d_mae.append(d_mae_res)
        g_loss.append(loss)
        g_mae.append(mae)
        if (epoch % 50 == 0) and epoch != 0:
            pred_plot(G, epoch, train_img_shape)
            models = (D, G, GAN)
            filepaths = ('D_final.h5', 'G_final.h5', 'GAN_final.h5')
            for model, name in zip(models, filepaths):
                model.save(name)
            print('Epoch: %d\nDiscriminator Loss: %.3f, Generator Loss: %.3f\nDiscriminator MAE: %.3f, Generator MAE: %.3f' % (epoch, d_loss[-1], g_loss[-1], d_mae[-1], g_mae[-1]))

    fig, ax = plt.subplots(2,2, figsize=(11,11))
    ax[0][0].plot(list(range(len(d_loss))), d_loss, label='D_loss', color='red', linestyle='dashed', linewidth=1)
    ax[0][1].plot(list(range(len(g_loss))), g_loss, label='G_loss', color='red', linestyle='solid', linewidth=1)
    ax[1][0].plot(list(range(len(d_mae))), d_mae, label='D_mae', color='blue', linestyle='dashed', linewidth=1)
    ax[1][1].plot(list(range(len(g_mae))), g_mae, label='G_mae', color='blue', linestyle='solid', linewidth=1)
    fig.legend()
    fig.tight_layout()
    plt.savefig("loss_plots")
    plt.show()

if __name__ == '__main__':

    img = Image.open("orig_img_color.png")
    img.load()
    img = img.convert(mode='RGB').convert(mode='L')
    img = np.asarray(img, dtype=np.float32)
    try:
        assert img.shape == (256, 256)
    except:
        img = np.reshape(a=img, newshape=(256, 256))
    training_img = pre.minmax_scale(img.flatten(), (-1,1))
    training_img = training_img.reshape(img.shape)

    D, G, GAN = create_GAN()

    def show_summary(D, G, GAN):
                D.summary()
                G.summary()
                GAN.summary()

    verbose = input('Enter T/F for verbose output: ')
    if verbose == 'T':
        show_summary(D, G, GAN)

    trainGAN(D, G, GAN, 10000, 16, training_img, training_img.shape)


#encoding:utf-8

import os
import requests

class MyRequests():
    def get_url(self, url, headers):
        re = self.request(url, headers)
        return re

    def request(self, url, headers):
        re = requests.get(url, headers=headers)
        return re

def main():
    url = "https://www.baidu.com"
    headers = {
        'Accept': 'text/html, application/xhtml+xml, image/jxr, */*',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN',
        'Connection': 'Keep-Alive',
        'Cookie': 'BD_CK_SAM=1; BD_HOME=1; BD_UPN=1d314753; ispeed_lsm=2; delPer=0; H_PS_645EC=d5bbSBejmXxPM1%2BsCmVEyoFLm39JwsP3AbAlAoSW0b0%2BplUfviS57%2FLdlE4b6KwhNKa176%2BUEcGq; H_PS_PSSID=; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; PSINO=5; BDRCVFR[4r8LXJfwh-6]=I67x6TjHwwYf0; BAIDUID=79468E69F21EBFAC3378FA10DD014840:FG=1; BIDUPSID=79468E69F21EBFAC3378FA10DD014840; PSTM=1529030279; MCITY=-%3A; BDUSS=DRWYnNvNHh2ZnJJdEdFT1BQTndGa3pnYUF4ZUhoLUwwaUVYZU5JNlJvcHZERlJiQVFBQUFBJCQAAAAAAAAAAAEAAACZ9sEfcGVuZ2ZhbjEyMzc4MwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAG9~LFtvfyxbd; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598',
        'Host': 'www.baidu.com',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063'
    }
    req = MyRequests()
    re = req.get_url(url, headers=headers)
    print("retype:%s\n" %(type(re)))
    print("status_code:%s\n" %(re.status_code))
    print("texttype:%s\n" %(type(re.text)))
    print("text:%s\n" %(re.text))
    print("cookies:%s\n" %(re.cookies))
    print("content:%s\n" %(re.content))
    print("decode:%s\n" %(re.content.decode("utf-8")))

if ("__main__"==__name__):
    main()


import logging
from math import sqrt
from typing import Optional

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
import torch
from tqdm import trange

from ..common import HistoricalData, get_device
from ..exceptions import SimulationException
from . import Simulations

logger = logging.getLogger("pyfmc.simulations.gbm")

class Trajectory:
    def __init__(self, dist: torch.Tensor, label: str = "Trajectory") -> None:
        self.dist = dist.numpy()
        self.label = label

    def value(self):
        return self.dist

    def plot(self, title=None, xlabel=None, ylabel=None):
        fig, ax = plt.subplots()
        sns.lineplot(data=self.dist, legend=False)
        ax.set_xlabel(xlabel or "time")
        ax.set_ylabel(ylabel or self.label)
        ax.set_title(title or self.label)
        return fig, ax

class Distribution:
    def __init__(self, dist: torch.Tensor, label: str = "Distribution"):
        self.dist = dist.numpy()
        self.label = label

    def value(self):
        return self.dist

    def plot(self, bins=10, kde=False, title=None, xlabel=None, ylabel=None):
        fig, ax = plt.subplots()
        if kde:
            sns.kdeplot(data=self.dist, color="blue", fill=True, ax=ax)
        else:
            sns.histplot(self.dist, bins=bins, ax=ax)
        ax.set_xlabel(xlabel or self.label)
        ax.set_ylabel(ylabel or ("Density" if kde else "Counts"))
        ax.set_title(title or self.label)
        return fig, ax

    def __str__(self) -> str:
        return str(self.dist)

class GBMResult:
    def __init__(
        self, init_dist: torch.Tensor, final_dist: torch.Tensor, trajectories: Optional[torch.Tensor] = None
    ) -> None:
        self.init_dist = init_dist.cpu()
        self.final_dist = final_dist.cpu()
        self._trajectories = trajectories.cpu() if trajectories is not None else trajectories

    def price_distribution(self):
        return Distribution(self.final_dist, label="Price Distribution")

    def trajectories(self):
        if self._trajectories is None:
            logger.warning("No trajectories")
            return
        return Trajectory(self._trajectories)

    def return_distribution(self):
        if self.final_dist.size() != self.init_dist.size():
            logger.warning("Size mismatch due to MPS mode, fixing ...")
            init_dist = self.init_dist[: self.final_dist.size(0)]
        else:
            init_dist = self.init_dist
        return Distribution((self.final_dist - init_dist) / init_dist, label="Return Distribution")

    def VaR(self, alpha: float):
        return np.percentile(self.return_distribution().value(), alpha)

class GBM(Simulations):
    def __init__(
        self,
        df: pd.DataFrame,
        n_walkers: int,
        n_steps: int,
        n_trajectories: int = 0,
        step_size: float = 1,
        open_index: str = "Open",
        close_index: str = "Close",
        device_acc: bool = False,
    ) -> None:
        super().__init__(n_walkers, n_steps, device_acc)
        self.df = df
        self.step_size = step_size
        self.open_index = open_index
        self.close_index = close_index
        self.n_trajectories = n_trajectories
        if (open_index and close_index) not in df.columns:
            raise SimulationException("Wrong open_index or close_index")

    def simulate(self):
        hist_data = HistoricalData(self.df, self.open_index, self.close_index)
        device = get_device() if self.device_acc else torch.device("cpu")
        logger.info("Using %s for calculation ...", device)
        dtype = torch.float32 if device == torch.device("mps") else torch.float64
        logger.info("Using device: %s", device)

        exp_return = torch.tensor(hist_data.return_mean, device=device, dtype=dtype)
        std_return = torch.tensor(hist_data.return_std, device=device, dtype=dtype)
        last_price = hist_data.get_latest_close_price()

        s0 = torch.tensor([last_price] * self.n_walkers, device=device, dtype=dtype)
        init_dist = torch.clone(s0)
        trajectories = init_dist[: self.n_trajectories] if self.n_trajectories > 0 else None

        s1 = torch.zeros(self.n_walkers, device=device, dtype=dtype)
        ds = torch.zeros(self.n_walkers, device=device, dtype=dtype)
        dt = torch.tensor(self.step_size)

        for _ in trange(self.n_steps):
            epsilon = torch.randn(self.n_walkers, device=device, dtype=dtype)
            shock = (std_return * sqrt(dt)) * epsilon
            drift = dt * exp_return
            ds = (shock + drift) * s0
            s1 = s0 + ds
            s0 = torch.clone(s1)
            if self.n_trajectories > 0:
                trajectories = torch.concat((trajectories, s0[: self.n_trajectories]))

        s0 = s0[torch.isfinite(s0)]
        return GBMResult(
            init_dist,
            s0,
            trajectories.reshape([self.n_steps + 1, self.n_trajectories]) if self.n_trajectories > 0 else None,
        )


import math
import warnings
from typing import Sequence

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torchmetrics.functional import pairwise_cosine_similarity
from mmcv.cnn import (build_activation_layer, build_conv_layer,
                      build_norm_layer, xavier_init)
from mmcv.cnn.bricks.registry import (TRANSFORMER_LAYER,
                                      TRANSFORMER_LAYER_SEQUENCE)
from mmcv.cnn.bricks.transformer import (BaseTransformerLayer,
                                         TransformerLayerSequence,
                                         build_transformer_layer_sequence)
from torch.nn.init import normal_

from .builder import ROTATED_TRANSFORMER
from mmdet.models.utils import Transformer
from mmdet.models.utils.transformer import inverse_sigmoid
from mmcv.ops import diff_iou_rotated_2d
from fast_pytorch_kmeans import KMeans, MultiKMeans
# from mmrotate.core import obb2poly, poly2obb
# # from mmrotate.core import obb2xyxy
# from mmdet.core import bbox_cxcywh_to_xyxy

try:
    from mmcv.ops.multi_scale_deform_attn import MultiScaleDeformableAttention

except ImportError:
    warnings.warn(
        '`MultiScaleDeformableAttention` in MMCV has been moved to '
        '`mmcv.ops.multi_scale_deform_attn`, please update your MMCV')
    from mmcv.cnn.bricks.transformer import MultiScaleDeformableAttention
torch.autograd.set_detect_anomaly(True)

kmeans = KMeans(n_clusters=15, mode='cosine',verbose=1)

def obb2poly_tr(rboxes):
    """Convert oriented bounding boxes to polygons.

    Args:
        obbs (torch.Tensor): [x_ctr,y_ctr,w,h,angle]

    Returns:
        polys (torch.Tensor): [x0,y0,x1,y1,x2,y2,x3,y3]
    """
    x = rboxes[..., 0]
    y = rboxes[..., 1]
    w = rboxes[..., 2]
    h = rboxes[..., 3]
    a = rboxes[..., 4]
    cosa = torch.cos(a)
    sina = torch.sin(a)
    wx, wy = w / 2 * cosa, w / 2 * sina
    hx, hy = -h / 2 * sina, h / 2 * cosa
    p1x, p1y = x - wx - hx, y - wy - hy
    p2x, p2y = x + wx - hx, y + wy - hy
    p3x, p3y = x + wx + hx, y + wy + hy
    p4x, p4y = x - wx + hx, y - wy + hy
    return torch.stack([p1x, p1y, p2x, p2y, p3x, p3y, p4x, p4y], dim=2)

def bbox_cxcywh_to_xyxy_tr(bbox):
    """Convert bbox coordinates from (cx, cy, w, h) to (x1, y1, x2, y2).

    Args:
        bbox (Tensor): Shape (n, 4) for bboxes.

    Returns:
        Tensor: Converted bboxes.
    """
    cx, cy, w, h = bbox.split((1, 1, 1, 1), dim=-1)
    bbox_new = [(cx - 0.5 * w), (cy - 0.5 * h), (cx + 0.5 * w), (cy - 0.5 * h),
                (cx - 0.5 * w), (cy + 0.5 * h), (cx + 0.5 * w), (cy + 0.5 * h)]
    return torch.cat(bbox_new, dim=-1)

@ROTATED_TRANSFORMER.register_module()
class RotatedDeformableDetrTransformer(Transformer):
    """Implements the DeformableDETR transformer.

    Args:
        as_two_stage (bool): Generate query from encoder features.
            Default: False.
        num_feature_levels (int): Number of feature maps from FPN:
            Default: 4.
        two_stage_num_proposals (int): Number of proposals when set
            `as_two_stage` as True. Default: 300.
    """

    def __init__(self,
                 as_two_stage=False,
                 num_feature_levels=5,
                 two_stage_num_proposals=300,
                 mixed_selection = True,
                 **kwargs):
        super(RotatedDeformableDetrTransformer, self).__init__(**kwargs)
        self.as_two_stage = as_two_stage
        self.num_feature_levels = num_feature_levels
        self.two_stage_num_proposals = two_stage_num_proposals
        self.mixed_selection = mixed_selection
        self.embed_dims = self.encoder.embed_dims
        self.init_layers()

    def init_layers(self):
        """Initialize layers of the DeformableDetrTransformer."""
        self.level_embeds = nn.Parameter(
            torch.Tensor(self.num_feature_levels, self.embed_dims))

        if self.as_two_stage:
            self.enc_output = nn.Linear(self.embed_dims, self.embed_dims)
            self.enc_output_cls = nn.Linear(8, 256)
            self.enc_output_reg = nn.Linear(5, 256)
            self.enc_output_pro = nn.Linear(5, 256)
            self.enc_output_cls_norm = nn.LayerNorm(256)
            self.enc_output_reg_norm = nn.LayerNorm(256)
            self.enc_output_pro_norm = nn.LayerNorm(256)
            self.enc_output_norm = nn.LayerNorm(self.embed_dims)
            self.pos_trans = nn.Linear(self.embed_dims*2,
                                       self.embed_dims*2)
            self.pos_trans_xy = nn.Linear(self.embed_dims,
                                       self.embed_dims)
            self.pos_trans_whacls = nn.Linear(self.embed_dims,
                                       self.embed_dims)
            self.pos_trans_norm = nn.LayerNorm(self.embed_dims*2)
            self.pos_trans_xy_norm = nn.LayerNorm(self.embed_dims)
            self.pos_trans_whacls_norm = nn.LayerNorm(self.embed_dims)

            self.tgt_embed = nn.Embedding(self.two_stage_num_proposals, self.embed_dims)
            self.two_stage_wh_embedding = nn.Embedding(1, 2)
            # self.two_stage_wh1_embedding = nn.Embedding(1,512)
            # self.two_stage_wh2_embedding = nn.Embedding(1,2048)
            # self.two_stage_wh3_embedding = nn.Embedding(1,8192)
            # self.two_stage_wh4_embedding = nn.Embedding(1,32768)
            self.two_stage_theta1_embedding = nn.Embedding(1,256)
            self.two_stage_theta2_embedding = nn.Embedding(1,1024)
            self.two_stage_theta3_embedding = nn.Embedding(1,4096)
            self.two_stage_theta4_embedding = nn.Embedding(1,16384)
            nn.init.normal_(self.tgt_embed.weight.data)
        else:
            self.reference_points = nn.Linear(self.embed_dims, 2)

    def init_weights(self):
        """Initialize the transformer weights."""
        for p in self.parameters():
            if p.dim() > 1:
                nn.init.xavier_uniform_(p)
        for m in self.modules():
            if isinstance(m, MultiScaleDeformableAttention):
                m.init_weights()
        if not self.as_two_stage:
            xavier_init(self.reference_points, distribution='uniform', bias=0.)
        normal_(self.level_embeds)

        nn.init.constant_(self.two_stage_wh_embedding.weight, math.log(0.05 / (1 - 0.05)))
        # nn.init.uniform_(self.two_stage_wh1_embedding.weight, a=math.log(0.05 / (1 - 0.05)),b=math.log(0.08 / (1 - 0.08)))
        # nn.init.uniform_(self.two_stage_wh2_embedding.weight, a=math.log(0.05 / (1 - 0.05)),b=math.log(0.08 / (1 - 0.08)))
        # nn.init.uniform_(self.two_stage_wh3_embedding.weight, a=math.log(0.04 / (1 - 0.04)),b=math.log(0.05 / (1 - 0.05)))
        # nn.init.uniform_(self.two_stage_wh4_embedding.weight, a=math.log(0.04 / (1 - 0.04)),b=math.log(0.05 / (1 - 0.05)))
        # nn.init.uniform_(self.two_stage_wh1_embedding.weight, a=0.36,b=0.44)
        # nn.init.uniform_(self.two_stage_wh2_embedding.weight, a=0.17,b=0.23)
        # nn.init.uniform_(self.two_stage_wh3_embedding.weight, a=0.08,b=0.12)
        # nn.init.uniform_(self.two_stage_wh4_embedding.weight, a=0.03,b=0.07)
        nn.init.uniform_(self.two_stage_theta1_embedding.weight, a=0.0,b=(np.pi/2))
        nn.init.uniform_(self.two_stage_theta2_embedding.weight, a=0.0,b=(np.pi/2))
        nn.init.uniform_(self.two_stage_theta3_embedding.weight, a=0.0,b=(np.pi/2))
        nn.init.uniform_(self.two_stage_theta4_embedding.weight, a=0.0,b=(np.pi/2))

    # def gen_encoder_output_proposals(self, memory, memory_padding_mask,
    #                                  spatial_shapes, inputwh=None, learnedwh1=None, learnedwh2=None, learnedwh3=None,
    #                                  learnedwh4=None, learnedtheta1=None ,learnedtheta2=None,
    #                                  learnedtheta3=None, learnedtheta4=None):

    def gen_encoder_output_proposals(self, memory, memory_padding_mask,
                                     spatial_shapes, inputwh=None, learnedtheta1=None ,learnedtheta2=None,
                                     learnedtheta3=None, learnedtheta4=None):
        """Generate proposals from encoded memory.

        Args:
            memory (Tensor) : The output of encoder,
                has shape (bs, num_key, embed_dim).  num_key is
                equal the number of points on feature map from
                all level.
            memory_padding_mask (Tensor): Padding mask for memory.
                has shape (bs, num_key).
            spatial_shapes (Tensor): The shape of all feature maps.
                has shape (num_level, 2).

        Returns:
            tuple: A tuple of feature map and bbox prediction.

                - output_memory (Tensor): The input of decoder,  \
                    has shape (bs, num_key, embed_dim).  num_key is \
                    equal the number of points on feature map from \
                    all levels.
                - output_proposals (Tensor): The normalized proposal \
                    after a inverse sigmoid, has shape \
                    (bs, num_keys, 4).
        """

        N, S, C = memory.shape
        proposals = []
        _cur = 0
        # 生成网格一样的proposals
        # spatial_shapes = 16x16, 32x32, 64x64, 128x128
        for lvl, (H, W) in enumerate(spatial_shapes):

            mask_flatten_ = memory_padding_mask[:, _cur:(_cur + H * W)].view(
                N, H, W, 1)
            valid_H = torch.sum(~mask_flatten_[:, :, 0, 0], 1)
            valid_W = torch.sum(~mask_flatten_[:, 0, :, 0], 1)

            grid_y, grid_x = torch.meshgrid(
                torch.linspace(
                    0, H - 1, H, dtype=torch.float32, device=memory.device),
                torch.linspace(
                    0, W - 1, W, dtype=torch.float32, device=memory.device))
            grid = torch.cat([grid_x.unsqueeze(-1), grid_y.unsqueeze(-1)], -1)

            scale = torch.cat([valid_W.unsqueeze(-1),
                               valid_H.unsqueeze(-1)], 1).view(N, 1, 1, 2)
            grid = (grid.unsqueeze(0).expand(N, -1, -1, -1) + 0.5) / scale
            # wh = torch.ones_like(grid) * 0.05 * (2.0**lvl)
            # grid point 마다 wh를 learn할 수 있도록 생성
            # 각 layer level 별로 embedding을 줘서 학습?
            # wh를 hxx로 줘서 학습가능하게?

            # if H == 16:

            # #     learnedwh1 = torch.reshape(learnedwh1,(16,16,2))

            # #     learnedwh1 = learnedwh1.unsqueeze(0)
            # #     learnedwh1 = learnedwh1.repeat(N,1,1,1)

            #     learnedtheta1 = torch.reshape(learnedtheta1,(16,16,1))
            #     learnedtheta1 = learnedtheta1.unsqueeze(0)
            #     learnedtheta1 = learnedtheta1.repeat(N,1,1,1)

            # # #     # wh  = torch.mul(torch.ones_like(grid),(learnedwh1.sigmoid() * (2.0**lvl)))
            # #     wh  = torch.mul(torch.ones_like(grid),(learnedwh1))

            #     angle = learnedtheta1
            # # #     # print(wh[4][15])

            # if H == 32:
            # # #     # learnedwh2 = self.two_stage_wh2_embedding.weight[0]

            # #     learnedwh2 = torch.reshape(learnedwh2,(32,32,2))
            # #     learnedwh2 = learnedwh2.unsqueeze(0)
            # #     learnedwh2 = learnedwh2.repeat(N,1,1,1)

            #     learnedtheta2 = torch.reshape(learnedtheta2,(32,32,1))
            #     learnedtheta2 = learnedtheta2.unsqueeze(0)
            #     learnedtheta2 = learnedtheta2.repeat(N,1,1,1)

            # # #     # wh  = torch.mul(torch.ones_like(grid),(learnedwh2.sigmoid() * (2.0**lvl)))
            # #     wh  = torch.mul(torch.ones_like(grid),(learnedwh2))
            #     angle = learnedtheta2
            # # #     # print('------------------------------')
            # # #     # print(wh[4][15])
            # # #     # print('------------------------------')
            # if H == 64:
            # # #     # learnedwh3 = self.two_stage_wh3_embedding.weight[0]

            # #     learnedwh3 = torch.reshape(learnedwh3,(64,64,2))
            # #     learnedwh3 = learnedwh3.unsqueeze(0)
            # #     learnedwh3 = learnedwh3.repeat(N,1,1,1)

            #     learnedtheta3 = torch.reshape(learnedtheta3,(64,64,1))
            #     learnedtheta3 = learnedtheta3.unsqueeze(0)
            #     learnedtheta3 = learnedtheta3.repeat(N,1,1,1)

            # # #     # wh  = torch.mul(torch.ones_like(grid),(learnedwh3.sigmoid() * (2.0**lvl)))
            # #     wh  = torch.mul(torch.ones_like(grid),(learnedwh3))
            #     angle = learnedtheta3

            # if H == 128:
            # # #     # learnedwh4 = self.two_stage_wh4_embedding.weight[0]

            # #     learnedwh4 = torch.reshape(learnedwh4,(128,128,2))

            # #     learnedwh4 = learnedwh4.unsqueeze(0)
            # #     learnedwh4 = learnedwh4.repeat(N,1,1,1)

            #     learnedtheta4 = torch.reshape(learnedtheta4,(128,128,1))
            #     learnedtheta4 = learnedtheta4.unsqueeze(0)
            #     learnedtheta4 = learnedtheta4.repeat(N,1,1,1)

            # # #     # wh  = torch.mul(torch.ones_like(grid),(learnedwh4.sigmoid() * (2.0**lvl)))
            # #     wh  = torch.mul(torch.ones_like(grid),(learnedwh4))

            #     angle = learnedtheta4

            # # if H != 16:
            # #     wh = torch.ones_like(grid) * learnedwh.sigmoid() * (2.0**lvl)
            if inputwh is not None:
            # import ipdb; ipdb.set_trace()
                wh = torch.ones_like(grid) * inputwh.sigmoid() * (2.0 ** lvl)

            # wh = torch.ones_like(grid) * 0.05 * (2.0**lvl)
            angle = torch.zeros_like(mask_flatten_)
            proposal = torch.cat((grid, wh, angle), -1).view(N, -1, 5)
            # proposal = torch.cat((grid, wh), -1).view(N, -1, 4)
            proposals.append(proposal)
            _cur += (H * W)
        output_proposals = torch.cat(proposals, 1)

        output_proposals_valid = ((output_proposals[..., :4] > 0.01) &
                                  (output_proposals[..., :4] < 0.99)
                                  ).all(
            -1, keepdim=True)

        # output_proposals_valid = ((output_proposals[..., :4] > 0.01) &
        #                           (output_proposals[..., :4] < 0.99)&
        #                           (output_proposals[..., 4:5] > 0.00)&
        #                           (output_proposals[..., 4:5] < (np.pi/2))
        #                           ).all(
        #     -1, keepdim=True)

        # output_proposals_valid1 = ((output_proposals[...,4:5] > 0.00) &
        #                            (output_proposals[...,4:5] < np.pi/2)).all(
        #     -1, keepdim=True)
        # 反sigmoid函数 inversigmoid
        output_proposals[..., :4] = torch.log(output_proposals[..., :4].clone() / (1 - output_proposals[..., :4].clone()))

        # output_proposals[..., :4] = torch.log(output_proposals[..., :4] / (1 - output_proposals[..., :4]))
        # output_proposals = output_proposals.masked_fill(
        #     memory_padding_mask.unsqueeze(-1), float('inf'))
        # output_proposals = output_proposals.masked_fill(
        #     ~output_proposals_valid, float('inf'))

        output_proposals[..., :4] = output_proposals[..., :4].masked_fill(
            memory_padding_mask.unsqueeze(-1), 10000)
        output_proposals[..., :4] = output_proposals[..., :4].masked_fill(
            ~output_proposals_valid, 10000)

        # output_proposals[..., 4:5] = output_proposals[..., 4:5].masked_fill(
        #     memory_padding_mask.unsqueeze(-1), 0)
        # output_proposals[..., 4:5] = output_proposals[...,4:5].masked_fill(
        #     ~output_proposals_valid, 0)

        output_memory = memory
        output_memory = output_memory.masked_fill(
            memory_padding_mask.unsqueeze(-1), float(0))
        output_memory = output_memory.masked_fill(~output_proposals_valid,
                                                  float(0))
        output_memory = self.enc_output_norm(self.enc_output(output_memory))
        return output_memory, output_proposals

    @staticmethod
    def get_reference_points(spatial_shapes, valid_ratios, device):
        """Get the reference points used in decoder.

        Args:
            spatial_shapes (Tensor): The shape of all
                feature maps, has shape (num_level, 2).
            valid_ratios (Tensor): The radios of valid
                points on the feature map, has shape
                (bs, num_levels, 2)
            device (obj:`device`): The device where
                reference_points should be.

        Returns:
            Tensor: reference points used in decoder, has \
                shape (bs, num_keys, num_levels, 2).
        """
        reference_points_list = []
        for lvl, (H, W) in enumerate(spatial_shapes):
            #  TODO  check this 0.5
            ref_y, ref_x = torch.meshgrid(
                torch.linspace(
                    0.5, H - 0.5, H, dtype=torch.float32, device=device),
                torch.linspace(
                    0.5, W - 0.5, W, dtype=torch.float32, device=device))
            ref_y = ref_y.reshape(-1)[None] / (
                    valid_ratios[:, None, lvl, 1] * H)
            ref_x = ref_x.reshape(-1)[None] / (
                    valid_ratios[:, None, lvl, 0] * W)
            ref = torch.stack((ref_x, ref_y), -1)
            reference_points_list.append(ref)
        reference_points = torch.cat(reference_points_list, 1)
        reference_points = reference_points[:, :, None] * valid_ratios[:, None]
        return reference_points

    def get_valid_ratio(self, mask):
        """Get the valid radios of feature maps of all  level."""
        _, H, W = mask.shape
        valid_H = torch.sum(~mask[:, :, 0], 1)
        valid_W = torch.sum(~mask[:, 0, :], 1)
        valid_ratio_h = valid_H.float() / H
        valid_ratio_w = valid_W.float() / W
        valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1)
        return valid_ratio

    ##### theta 고려해서 position encoding 처리필요
    def get_proposal_pos_embed(self,
                               proposals,
                               num_pos_feats=128,
                               temperature=10000):
        """Get the position embedding of proposal."""
        scale = 2 * math.pi
        dim_t = torch.arange(
            num_pos_feats, dtype=torch.float32, device=proposals.device)
        dim_t = temperature**(2 * (dim_t // 2) / num_pos_feats)
        # N, L, 4
        proposals[:,:,:4] = proposals[:,:,:4].sigmoid() * scale
        # N, L, 4, 128
        pos = proposals[:, :, :, None] / dim_t
        # N, L, 4, 64, 2
        pos = torch.stack((pos[:, :, :, 0::2].sin(), pos[:, :, :, 1::2].cos()),
                          dim=4).flatten(2)
        return pos

    def get_proposal_pos_xy_embed(self,
                               proposals,
                               num_pos_feats=128,
                               temperature=10000):
        """Get the position embedding of proposal."""
        scale = 2 * math.pi
        dim_t = torch.arange(
            num_pos_feats, dtype=torch.float32, device=proposals.device)
        dim_t = temperature**(2 * (dim_t // 2) / num_pos_feats)
        # N, L, 2
        proposals[:,:,:2] = proposals[:,:,:2].sigmoid() * scale
        # N, L, 2, 128
        pos_xy = proposals[:, :, :, None] / dim_t

        # N, L, 256
        pos_xy = torch.stack((pos_xy[:, :, :, 0::2].sin(), pos_xy[:, :, :, 1::2].cos()),
                          dim=4).flatten(2)
        return pos_xy

    def get_proposal_pos_wha_embed(self,
                               proposals,
                               num_pos_feats=64,
                               temperature=10000):
        """Get the position embedding of proposal."""
        scale = 2 * math.pi
        dim_t = torch.arange(
            num_pos_feats, dtype=torch.float32, device=proposals.device)
        dim_t = temperature**(2 * (dim_t // 2) / num_pos_feats)
        # N, L, 3
        proposals[:,:,2:5] = proposals[:,:,2:5].sigmoid() * scale
        # N, L, 3, 64
        pos_wha = proposals[:, :, :, None] / dim_t
        # N, L, 192
        pos_wha = torch.stack((pos_wha[:, :, :, 0::2].sin(), pos_wha[:, :, :, 1::2].cos()),
                          dim=4).flatten(2)
        return pos_wha

    def get_proposal_pos_cls_embed(self,
                               proposal_cls,
                               num_pos_feats=64,
                               temperature=10000):
        """Get the position embedding of proposal."""
        scale = 2 * math.pi
        dim_t = torch.arange(
            num_pos_feats, dtype=torch.float32, device=proposal_cls.device)
        dim_t = temperature**(2 * (dim_t // 2) / num_pos_feats)
        # N, L, 1
        proposal_cls[:,:,:1] = proposal_cls[:,:,:1].sigmoid() * scale
        # N, L, 1, 64
        pos_cls = proposal_cls[:, :, :, None] / dim_t
        # N, num_q, 64
        pos_cls = torch.stack((pos_cls[:, :, :, 0::2].sin(), pos_cls[:, :, :, 1::2].cos()),
                          dim=4).flatten(2)

        return pos_cls

    # query position만 처리 query content는
    # def get_proposal_pos_embed(self,
    #                            proposals,
    #                            num_pos_feats=64,
    #                            temperature=10000):
    #     """Get the position embedding of proposal."""
    #     scale = 2 * math.pi
    #     dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=proposals.device)
    #     dim_t = 10000 ** (2 * (dim_t // 2) / num_pos_feats)
    #     x_embed = proposals[:, :, 0] * scale
    #     y_embed = proposals[:, :, 1] * scale
    #     pos_x = x_embed[:, :, None] / dim_t
    #     pos_y = y_embed[:, :, None] / dim_t
    #     pos_x = torch.stack((pos_x[:, :, 0::2].sin(), pos_x[:, :, 1::2].cos()), dim=3).flatten(2)
    #     pos_y = torch.stack((pos_y[:, :, 0::2].sin(), pos_y[:, :, 1::2].cos()), dim=3).flatten(2)
    #     if proposals.size(-1) == 2:
    #         pos = torch.cat((pos_y, pos_x), dim=2)
    #     elif proposals.size(-1) == 4:
    #         w_embed = proposals[:, :, 2] * scale
    #         pos_w = w_embed[:, :, None] / dim_t
    #         pos_w = torch.stack((pos_w[:, :, 0::2].sin(), pos_w[:, :, 1::2].cos()), dim=3).flatten(2)

    #         h_embed = proposals[:, :, 3] * scale
    #         pos_h = h_embed[:, :, None] / dim_t
    #         pos_h = torch.stack((pos_h[:, :, 0::2].sin(), pos_h[:, :, 1::2].cos()), dim=3).flatten(2)

    #         pos = torch.cat((pos_y, pos_x, pos_w, pos_h), dim=2)
    #     else:
    #         raise ValueError("Unknown pos_tensor shape(-1):{}".format(proposals.size(-1)))
    #     return pos

    def forward(self,
                mlvl_feats,
                mlvl_masks,
                query_embed,
                mlvl_pos_embeds,
                bbox_coder=None,
                reg_branches=None,
                cls_branches=None,
                first_stage=False,
                **kwargs):
        """Forward function for `Transformer`.

        Args:
            mlvl_feats (list(Tensor)): Input queries from
                different level. Each element has shape
                [bs, embed_dims, h, w].
            mlvl_masks (list(Tensor)): The key_padding_mask from
                different level used for encoder and decoder,
                each element has shape  [bs, h, w].
            query_embed (Tensor): The query embedding for decoder,
                with shape [num_query, c].
            mlvl_pos_embeds (list(Tensor)): The positional encoding
                of feats from different level, has the shape
                 [bs, embed_dims, h, w].
            reg_branches (obj:`nn.ModuleList`): Regression heads for
                feature maps from each decoder layer. Only would
                be passed when
                `with_box_refine` is True. Default to None.
            cls_branches (obj:`nn.ModuleList`): Classification heads
                for feature maps from each decoder layer. Only would
                 be passed when `as_two_stage`
                 is True. Default to None.

        Returns:
            tuple[Tensor]: results of decoder containing the following tensor.

                - inter_states: Outputs from decoder. If
                    return_intermediate_dec is True output has shape \
                      (num_dec_layers, bs, num_query, embed_dims), else has \
                      shape (1, bs, num_query, embed_dims).
                - init_reference_out: The initial value of reference \
                    points, has shape (bs, num_queries, 4).
                - inter_references_out: The internal value of reference \
                    points in decoder, has shape \
                    (num_dec_layers, bs,num_query, embed_dims)
                - enc_outputs_class: The classification score of \
                    proposals generated from \
                    encoder's feature maps, has shape \
                    (batch, h*w, num_classes). \
                    Only would be returned when `as_two_stage` is True, \
                    otherwise None.
                - enc_outputs_coord_unact: The regression results \
                    generated from encoder's feature maps., has shape \
                    (batch, h*w, 4). Only would \
                    be returned when `as_two_stage` is True, \
                    otherwise None.
        """
        # assert self.as_two_stage or query_embed is not None

        feat_flatten = []
        mask_flatten = []
        lvl_pos_embed_flatten = []
        spatial_shapes = []
        for lvl, (feat, mask, pos_embed) in enumerate(
                zip(mlvl_feats, mlvl_masks, mlvl_pos_embeds)):
            bs, c, h, w = feat.shape
            # pos_embed.shape = [2, 256, 128, 128]
            spatial_shape = (h, w)
            spatial_shapes.append(spatial_shape)

            # [bs, w*h, c]
            feat = feat.flatten(2).transpose(1, 2)
            # [bs, w*h]
            mask = mask.flatten(1)
            # [bs, w*h]
            pos_embed = pos_embed.flatten(2).transpose(1, 2)
            lvl_pos_embed = pos_embed + self.level_embeds[lvl].view(1, 1, -1)
            lvl_pos_embed_flatten.append(lvl_pos_embed)
            feat_flatten.append(feat)
            mask_flatten.append(mask)
        feat_flatten = torch.cat(feat_flatten, 1)
        mask_flatten = torch.cat(mask_flatten, 1)
        lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1)
        spatial_shapes = torch.as_tensor(
            spatial_shapes, dtype=torch.long, device=feat_flatten.device)
        level_start_index = torch.cat((spatial_shapes.new_zeros(
            (1, )), spatial_shapes.prod(1).cumsum(0)[:-1]))
        valid_ratios = torch.stack(
            [self.get_valid_ratio(m) for m in mlvl_masks], 1)
        # multi-scale reference points
        reference_points = \
            self.get_reference_points(spatial_shapes,
                                      valid_ratios,
                                      device=feat.device)

        feat_flatten = feat_flatten.permute(1, 0, 2)  # (H*W, bs, embed_dims)
        lvl_pos_embed_flatten = lvl_pos_embed_flatten.permute(
            1, 0, 2)  # (H*W, bs, embed_dims)
        # 21760 = 128*128+64*64+32*32+16*16 query的个数
        # memory是编码后的每个query和keys在多层featuremap中对应的特征 一维特征 256

        memory = self.encoder(
            query=feat_flatten,
            key=None,
            value=None,
            query_pos=lvl_pos_embed_flatten,
            query_key_padding_mask=mask_flatten,
            spatial_shapes=spatial_shapes,
            reference_points=reference_points,
            level_start_index=level_start_index,
            valid_ratios=valid_ratios,
            **kwargs)

        memory = memory.permute(1, 0, 2)
        bs, _, c = memory.shape

        if self.as_two_stage:
            input_hw = self.two_stage_wh_embedding.weight[0]
            # learnedwh1 = self.two_stage_wh1_embedding.weight[0]
            # learnedwh2 = self.two_stage_wh2_embedding.weight[0]
            # learnedwh3 = self.two_stage_wh3_embedding.weight[0]
            # learnedwh4 = self.two_stage_wh4_embedding.weight[0]
            # learnedtheta1 = self.two_stage_theta1_embedding.weight[0]
            # learnedtheta2 = self.two_stage_theta2_embedding.weight[0]
            # learnedtheta3 = self.two_stage_theta3_embedding.weight[0]
            # learnedtheta4 = self.two_stage_theta4_embedding.weight[0]
            # output_memory, output_proposals = \
            #     self.gen_encoder_output_proposals(
            #         memory, mask_flatten, spatial_shapes, learnedwh1, learnedwh2,
            #         learnedwh3,learnedwh4,learnedtheta1,learnedtheta2,learnedtheta3,learnedtheta4)

            output_memory, output_proposals = \
                self.gen_encoder_output_proposals(
                    memory, mask_flatten, spatial_shapes, input_hw)

            # cls score,reg output feature map 별로 짜르기
            enc_outputs_class = cls_branches[self.decoder.num_layers](
                output_memory)
            cls_wieght = cls_branches[self.decoder.num_layers].weight
            enc_outputs_coord_unact_angle= \
                reg_branches[
                    self.decoder.num_layers](output_memory) + output_proposals
            # print(enc_outputs_coord_unact_angle)

            if first_stage:
                return enc_outputs_coord_unact_angle
            #############################
            # if first_stage:
            #     # enc_ouputs_class_f0 = enc_outputs_class[..., 0]

            #     enc_ouputs_class_f0 = enc_outputs_class.clone().detach()
            #     enc_ouputs_class_f0[:,16384:-1,:] = -99

            #     enc_ouputs_class_f1 = enc_outputs_class.clone().detach()
            #     enc_ouputs_class_f1[:,0:16384,:] = -99
            #     enc_ouputs_class_f1[:,20480:-1,:] = -99

            #     enc_ouputs_class_f2 = enc_outputs_class.clone().detach()
            #     enc_ouputs_class_f2[:,0:20480,:] = -99
            #     enc_ouputs_class_f2[:,21504:-1,:] = -99

            #     enc_ouputs_class_f3 = enc_outputs_class.clone().detach()
            #     enc_ouputs_class_f3[:,0:21504,:] = -99

            #     topk_proposals0 =  torch.topk(
            #         enc_ouputs_class_f0.max(dim=2)[0], 180, dim=1)[1]

            #     topk_coords_unact0 = torch.gather(
            #         enc_outputs_coord_unact_angle, 1,
            #         topk_proposals0.unsqueeze(-1).repeat(1, 1, 5))
            #     topk_coords_unact0 = topk_coords_unact0.detach()

            #     topk_proposals1 =  torch.topk(
            #         enc_ouputs_class_f1.max(dim=2)[0], 70, dim=1)[1]
            #     topk_coords_unact1 = torch.gather(
            #         enc_outputs_coord_unact_angle, 1,
            #         topk_proposals1.unsqueeze(-1).repeat(1, 1, 5))
            #     topk_coords_unact1 = topk_coords_unact1.detach()

            #     topk_proposals2 =  torch.topk(
            #         enc_ouputs_class_f2.max(dim=2)[0], 25, dim=1)[1]
            #     topk_coords_unact2 = torch.gather(
            #         enc_outputs_coord_unact_angle, 1,
            #         topk_proposals2.unsqueeze(-1).repeat(1, 1, 5))
            #     topk_coords_unact2 = topk_coords_unact2.detach()

            #     topk_proposals3 =  torch.topk(
            #         enc_ouputs_class_f3.max(dim=2)[0], 25, dim=1)[1]
            #     topk_coords_unact3 = torch.gather(
            #         enc_outputs_coord_unact_angle, 1,
            #         topk_proposals3.unsqueeze(-1).repeat(1, 1, 5))
            #     topk_coords_unact3 = topk_coords_unact3.detach()
            #     topk_enc_outputs_coord_unact_angle = torch.cat([topk_coords_unact0, topk_coords_unact1, topk_coords_unact2, topk_coords_unact3],dim=1)

            #     return topk_enc_outputs_coord_unact_angle
            ######## clustering
            # topk_1 = self.two_stage_num_proposals * 3
            # topk_2 = self.two_stage_num_proposals

            # for i in range(bs):
            #     kmeans_inx, kmeans_v = kmeans.fit_predict(memory[i])
            #     b_kmeans_v = kmeans_v.unsqueeze(0)
            #     if i==0:
            #         ub_kmeans_v = b_kmeans_v
            #     else:
            #         ub_kmeans_v = torch.cat([ub_kmeans_v,b_kmeans_v],dim=0)
            # # print(kmeans_in.shape)

            # topk_test=  torch.topk(ub_kmeans_v,topk_1,dim=1)[1]

            # topk_confidence = torch.gather(
            #     enc_outputs_class, 1,
            #     topk_test.unsqueeze(-1).repeat(1, 1,15))
            # topk_confidence = topk_confidence.detach()

            # topk_confidence_ind =  torch.topk(
            #     topk_confidence.max(dim=2)[0], topk_2, dim=1)[1]

            # # 4x300
            # topk_proposals = torch.gather(topk_test,1,topk_confidence_ind.repeat(1, 1))

            ##### previous
            ############################
            ## topk_1을 다이나믹하게 0.5이상 다뽑기 if 0.5 넘는게 400개 안넘으면 400으로 설정
            enc_out_topk_mask = enc_outputs_class.max(dim=2)[0].clone().detach()
            enc_out_topk_mask = enc_out_topk_mask.sigmoid()
            enc_m = nn.Threshold(0.3, 0)
            enc_m1 = nn.Threshold(0.2, 0)
            enc_m2 = nn.Threshold(0.1, 0)
            enc_out_topk_mask1 = enc_m(enc_out_topk_mask)
            enc_out_topk_mask2 = enc_m1(enc_out_topk_mask)
            enc_out_topk_mask3 = enc_m2(enc_out_topk_mask)
            enc_m_topk1 = torch.count_nonzero(enc_out_topk_mask1)
            enc_m_topk2 = torch.count_nonzero(enc_out_topk_mask2)
            enc_m_topk3 = torch.count_nonzero(enc_out_topk_mask3)

            # 1500

            # filter_1 = 2000
            # filter_2 = 3000
            # filter_3 = 4000
            # filter_4 = 5000

            # 1200
            # filter_1 = 1800
            # filter_2 = 2400
            # filter_3 = 3000
            # filter_4 = 4000

            ## 900
            filter_1 = 1200
            filter_2 = 1800
            filter_3 = 2000
            filter_4 = 3000

             # 300
            # filter_1 = 400
            # filter_2 = 600
            # filter_3 = 900
            # filter_4 = 1200

            # 100
            # filter_1 = 150
            # filter_2 = 250
            # filter_3 = 350
            # filter_4 = 450

            # 200
            # filter_1 = 250
            # filter_2 = 400
            # filter_3 = 600
            # filter_4 = 800

            if enc_m_topk3 >= filter_1:
                if enc_m_topk2 >= filter_1:
                    if filter_1<=enc_m_topk1 <= filter_2:
                        topk_1 = int(enc_m_topk1)
                    elif enc_m_topk1 > filter_2:
                        topk_1 = filter_2
                    elif (enc_m_topk1 < filter_1) & (enc_m_topk2 <= filter_3):
                        topk_1 = int(enc_m_topk2)
                    elif (enc_m_topk1 < filter_1) & (enc_m_topk2 > filter_3):
                        topk_1 = filter_3

                elif (enc_m_topk2 < filter_1) & (enc_m_topk3 <= filter_4):
                    topk_1 = int(enc_m_topk3)
                elif (enc_m_topk2 < filter_1) & (enc_m_topk3 > filter_4):
                    topk_1 = filter_4

            else:
                topk_1 = filter_1

            # print("-------------------------------")
            # print(enc_m_topk1)
            # print(enc_m_topk2)
            # print(enc_m_topk3)
            # print(topk_1)
            # print("-------------------------------")
            # topk_1 = self.two_stage_num_proposals * 2
            topk_2 = self.two_stage_num_proposals
            bs, fm_num, cls_num =enc_outputs_class.shape

            enc_ouputs_class_topk1 = enc_outputs_class.clone().detach()
            enc_outputs_class_sigmoid_topk1 = enc_ouputs_class_topk1.sigmoid()

            # print(enc_ouputs_class_topk1[0][0])
            # print(enc_outputs_class_sigmoid_topk1[0][0])
            # exit()

             # 4x900
            topk_test=  torch.topk(enc_ouputs_class_topk1.max(dim=2)[0],topk_1,dim=1)[1]

             # 1차 proposal box 필터링 900개 중 iou(proposal, bbox)를 고려하여 confidence score 높은거 top 300 선정

            # 4x21760x15
            # enc_outputs_class_softmax =  F.softmax(enc_ouputs_class_topk1, dim=2)

            # 4x21760

            enc_outputs_coord_unact_angle_sig = enc_outputs_coord_unact_angle.clone().detach()
            output_proposals_sig = output_proposals.clone().detach()

            enc_outputs_coord_unact_angle_sim = enc_outputs_coord_unact_angle.clone().detach()
            output_proposals_sim = output_proposals.clone().detach()

            # enc_outputs_coord_unact_angle_sig[...,:4] = enc_outputs_coord_unact_angle_sig[...,:4].sigmoid()
            # output_proposals_sig[...,:4] = output_proposals_sig[...,:4].sigmoid()

            enc_outputs_coord_unact_angle_sig = enc_outputs_coord_unact_angle_sig.sigmoid()
            output_proposals_sig = output_proposals_sig.sigmoid()

            ## test_a = topk_test[0][0]
            # # print(enc_outputs_coord_unact_angle_sig[0][test_a])
            # # print(output_proposals_sig[0][test_a])
            # # print(topk_test[0])

            ## 4x21760
            ious = diff_iou_rotated_2d(enc_outputs_coord_unact_angle_sig, output_proposals_sig)
            # # print(ious[0][test_a])
            # # iouss = diff_iou_rotated_2d(enc_outputs_coord_unact_angle.sigmoid(), output_proposals.sigmoid())

            ## 4x900
            top_ious = torch.gather(
                ious, 1,
                topk_test.repeat(1, 1))

            ## 4x21760x256
            # enc_outputs_coord_unact_angle_vec = self.enc_output_reg_norm(self.enc_output_reg(enc_outputs_coord_unact_angle_sim))
            # output_proposals_vec = self.enc_output_pro_norm(self.enc_output_pro(output_proposals_sim))

            # ## 4x21760
            # cos_sim = torch.nn.CosineSimilarity(dim=2)
            # # cos_sim_out = cos_sim(enc_outputs_coord_unact_angle_vec, output_proposals_vec)
            # cos_sim_out = cos_sim(output_memory, enc_outputs_coord_unact_angle_vec)
            # # nom_cos_sim = (cos_sim_out + 1) / 2
            # torch.pi = torch.acos(torch.zeros(1)).item() * 2
            # nom_cos_sim = torch.arccos(cos_sim_out) / torch.pi

            # # 4x900
            # top_sim = torch.gather(
            #     nom_cos_sim, 1,
            #     topk_test.repeat(1, 1))

            # bb = enc_outputs_class_sigmoid_topk1.max(dim=2)[0]
            # test_a = topk_test[0][0]
            # print(bb[0][test_a])
            # print(nom_cos_sim[0][test_a])

            #### topk_test unsqeeze?,  topk_confidence.max(dim=2) shape check

            ## class 개수 만큼
            # 4x900x15
            # topk_confidence = torch.gather(
            #     enc_outputs_class_softmax, 1,
            #     topk_test.unsqueeze(-1).repeat(1, 1,8))
            # topk_confidence = topk_confidence.detach()

            topk_confidence = torch.gather(
                enc_outputs_class_sigmoid_topk1, 1,
                topk_test.unsqueeze(-1).repeat(1, 1,8))
            topk_confidence = topk_confidence.detach()

            # topk_memory

            topk_memory = torch.gather(
                output_memory, 1,
                topk_test.unsqueeze(-1).repeat(1, 1, 256))
            topk_memory = topk_memory.detach()

            ############################
            ## topk 1차 필터링에서 해당하는 memory 가져와서 sim 계산 후 score에 곱해서 topk300 선정

            ### 4 x topk
            for i in range (bs):

                cos_sim = pairwise_cosine_similarity(topk_memory[i])
                top_cos_sim = cos_sim.max(dim=1)[0].unsqueeze(0)
                if i ==0:
                    top_cos_totoal_sim = top_cos_sim
                else:
                    top_cos_totoal_sim = torch.cat([top_cos_totoal_sim,top_cos_sim],dim=0)

            ############################

            # # 4x300  iou
            topk_confidence_ind =  torch.topk(
                topk_confidence.max(dim=2)[0]+top_cos_totoal_sim, topk_2, dim=1)[1]

             # 4x300  cos_sim
            # topk_confidence_ind =  torch.topk(
            #     (topk_confidence.max(dim=2)[0]+(top_sim*0.5)+(top_ious*0.3)), topk_2, dim=1)[1]
            # topk_confidence_ind =  torch.topk(
            #     topk_confidence.max(dim=2)[0]*top_ious, topk_2, dim=1)[1]

            # 4x300
            topk_proposals = torch.gather(topk_test,1,topk_confidence_ind.repeat(1, 1))

            # ###########################aa
            ###### previous
            # topk = self.two_stage_num_proposals

            # topk 어떤 클래스 비중 높은지 파악
            # cls score top 900을 먼저 뽑고 어던 방법 사용해서 300개로 다시 필터링?

            # topk_proposals = torch.topk(
            #     enc_outputs_class[..., 0], topk, dim=1)[1]

            # enc_outputs_class split, topk_proposals

            # topk_proposals = torch.topk(
                # enc_outputs_class.max(dim=2)[0], topk, dim=1)[1]

            ##################
            ####################################

            # topk_proposals_0 = torch.topk(
            #     enc_outputs_class[..., 0], 20, dim=1)[1]
            # topk_proposals_00 = topk_proposals_0.split(1,dim=0)
            # a0 = enc_outputs_class.max(dim=2)[1]
            # check_0 = torch.numel(a0[a0 ==0])
            # check_1 = torch.numel(a0[a0==1])
            # check_2 = torch.numel(a0[a0==2])
            # check_3 = torch.numel(a0[a0==3])
            # check_4 = torch.numel(a0[a0==4])
            # check_5 = torch.numel(a0[a0==5])
            # check_6 = torch.numel(a0[a0==6])
            # check_7 = torch.numel(a0[a0==7])
            # check_8 = torch.numel(a0[a0==8])
            # check_9 = torch.numel(a0[a0==9])
            # check_10 = torch.numel(a0[a0==10])
            # check_11 = torch.numel(a0[a0==11])
            # check_12 = torch.numel(a0[a0==12])
            # check_13 = torch.numel(a0[a0==13])
            # check_14 = torch.numel(a0[a0==14])

            # print(check_0,check_1,check_2,check_3,check_4,check_5,check_6,check_7,check_8,check_9,check_10,check_11,check_12,check_13,check_14)

            # for i in range(bs):
            #     topk_proposals_0_inx = topk_proposals_00[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals_0_inx,:] = -99

            # topk_proposals_1 = torch.topk(
            #     enc_outputs_class[..., 1], 40, dim=1)[1]
            # # topk_proposalsss = torch.cat([topk_proposals_0,topk_proposals_1],dim=1)
            # topk_proposals_01 = topk_proposals_1.split(1,dim=0)

            # for i in range(bs):
            #     topk_proposals_1_inx = topk_proposals_01[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals_1_inx,:] = -99

            # topk_proposals_2 = torch.topk(
            #     enc_outputs_class[..., 2], 20, dim=1)[1]
            # topk_proposals_02 = topk_proposals_2.split(1,dim=0)

            # for i in range(bs):
            #     topk_proposals_2_inx = topk_proposals_02[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals_2_inx,:] = -99

            # topk_proposals_3 = torch.topk(
            #     enc_outputs_class[..., 3], 20, dim=1)[1]
            # topk_proposals03 = topk_proposals_3.split(1,dim=0)

            # for i in range(bs):
            #     topk_proposals3_inx = topk_proposals03[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals3_inx,:] = -99

            # topk_proposals_4 = torch.topk(
            #     enc_outputs_class[..., 4], 40, dim=1)[1]
            # topk_proposals_04 = topk_proposals_4.split(1,dim=0)

            # for i in range(bs):
            #     topk_proposals_4_inx = topk_proposals_04[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals_4_inx,:] = -99

            # topk_proposals_5 = torch.topk(
            #     enc_outputs_class[..., 5], 40, dim=1)[1]
            # topk_proposals_05 = topk_proposals_5.split(1,dim=0)

            # for i in range(bs):
            #     topk_proposals_5_inx = topk_proposals_05[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals_5_inx,:] = -99

            # topk_proposals_6 = torch.topk(
            #     enc_outputs_class[..., 6], 40, dim=1)[1]
            # topk_proposals_06 = topk_proposals_6.split(1,dim=0)

            # for i in range(bs):
            #     topk_proposals_6_inx = topk_proposals_06[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals_6_inx,:] = -99

            # topk_proposals_7 = torch.topk(
            #     enc_outputs_class[..., 7], 40, dim=1)[1]
            # topk_proposals_07 = topk_proposals_7.split(1,dim=0)

            # for i in range(bs):
            #     topk_proposals_7_inx = topk_proposals_07[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals_7_inx,:] = -99

            # topk_proposals_8 = torch.topk(
            #     enc_outputs_class[..., 8], 40, dim=1)[1]
            # topk_proposals_08 = topk_proposals_8.split(1,dim=0)

            # for i in range(bs):
            #     topk_proposals_8_inx = topk_proposals_08[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals_8_inx,:] = -99

            # topk_proposals_9 = torch.topk(
            #     enc_outputs_class[..., 9], 40, dim=1)[1]
            # topk_proposals_09 = topk_proposals_9.split(1,dim=0)

            # for i in range(bs):
            #     topk_proposals_9_inx = topk_proposals_09[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals_9_inx,:] = -99

            # topk_proposals_10 = torch.topk(
            #     enc_outputs_class[..., 10], 20, dim=1)[1]
            # topk_proposals_010 = topk_proposals_10.split(1,dim=0)

            # for i in range(bs):
            #     topk_proposals_10_inx = topk_proposals_010[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals_10_inx,:] = -99

            # topk_proposals_11 = torch.topk(
            #     enc_outputs_class[..., 11], 40, dim=1)[1]
            # topk_proposals_011 = topk_proposals_11.split(1,dim=0)

            # for i in range(bs):
            #     topk_proposals_11_inx = topk_proposals_011[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals_11_inx,:] = -99

            # topk_proposals_12 = torch.topk(
            #     enc_outputs_class[..., 12], 40, dim=1)[1]
            # topk_proposals_012 = topk_proposals_12.split(1,dim=0)

            # for i in range(bs):
            #     topk_proposals_12_inx = topk_proposals_012[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals_12_inx,:] = -99

            # topk_proposals_13 = torch.topk(
            #     enc_outputs_class[..., 13], 6, dim=1)[1]
            # topk_proposals_013 = topk_proposals_13.split(1,dim=0)

            # for i in range(bs):
            #     topk_proposals_13_inx = topk_proposals_013[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals_13_inx,:] = -99

            # topk_proposals_14 = torch.topk(
            #     enc_outputs_class[..., 14], 7, dim=1)[1]
            # topk_proposals_014 = topk_proposals_14.split(1,dim=0)

            # for i in range(bs):
            #     topk_proposals_14_inx = topk_proposals_014[i].squeeze(0)
            #     enc_outputs_class[i,topk_proposals_14_inx,:] = -99

            # topk_proposals = torch.cat([topk_proposals_0,topk_proposals_1,topk_proposals_2, topk_proposals_3,topk_proposals_4,topk_proposals_5,topk_proposals_6,topk_proposals_7,topk_proposals_8,
            #                             topk_proposals_9,topk_proposals_10,topk_proposals_11,topk_proposals_12,topk_proposals_13,topk_proposals_14], dim=1)

            topk_coords_unact = torch.gather(
                enc_outputs_coord_unact_angle, 1,
                topk_proposals.unsqueeze(-1).repeat(1, 1, 5))
            topk_coords_unact = topk_coords_unact.detach()

            topk_cls_score = torch.gather(
            enc_outputs_class, 1,
            topk_proposals.unsqueeze(-1).repeat(1, 1, 8))
            topk_cls_score = topk_cls_score.detach()

            # topk_cls_score = topk_cls_score.max(dim=2)[0]
            # topk_cls_score = topk_cls_score.unsqueeze(-1)

            # topk_class_argmax = enc_outputs_class.argmax(2)
            # # topk_class_argmax = torch.as_tensor(
            # # topk_class_argmax, dtype=torch.long, device=enc_outputs_class.device)
            # topk_class_onehot = torch.zeros(enc_outputs_class.shape, device = enc_outputs_class.device)
            # topk_class_onehot = topk_class_onehot.scatter(2,topk_class_argmax.unsqueeze(2), 1.0)

            # topk_class_onehot = torch.gather(
            #     topk_class_onehot, 1,
            #     topk_proposals.unsqueeze(-1).repeat(1, 1, 15))
            # topk_class_onehot = topk_class_onehot.detach()

            # topk_class = torch.gather(
            #     output_memory, 1,
            #     topk_proposals.unsqueeze(-1).repeat(1, 1, 256))
            # topk_class = topk_class.detach()

            # quantized, indices, commit_loss = vq(topk_class, topk_class_onehot)
            # quantized = quantized.detach()
            # obb2xyxy
            # reference_points = obb2poly_tr(topk_coords_unact).sigmoid()

            reference_points = topk_coords_unact[..., :4].sigmoid()

            ##### theta 고려해서 reference points를 5개 줘야함
            # reference_points = torch.cat(topk_coords_unact[...,:4].sigmoid(),topk_coords_unact[4])

            init_reference_out = reference_points
            # obb2xywh

            ##### theta 고려해서 norm 필요
            pos_trans_out = self.pos_trans_norm(
                self.pos_trans(self.get_proposal_pos_embed(topk_coords_unact[..., :4])))

            # query_embed here is the content embed for deformable DETR
            if not self.mixed_selection:
                query_pos, query = torch.split(pos_trans_out, c, dim=2)
            else:
                query = query_embed.unsqueeze(0).expand(bs, -1, -1)
                query_pos, _ = torch.split(pos_trans_out, c, dim=2)

            # query = torch.gather(memory, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, memory.size(-1)))
            # query_pos = pos_trans_out

            #  # #################### angle, cls positioning해서 add
            # ### 4x300x256
            # pos_trans_xy_out = self.get_proposal_pos_xy_embed(topk_coords_unact[..., :2])

            # ### 4x300x192
            # pos_trans_wha_out = self.get_proposal_pos_wha_embed(topk_coords_unact[..., 2:5])

            # ### 4x300x64
            # pos_trans_cls_out = self.get_proposal_pos_cls_embed(topk_cls_score[...,:1])

            # ### 4x300x256
            # pos_trans_whacls_out = torch.cat([pos_trans_wha_out,pos_trans_cls_out],dim=2)

            # pos_trans_xy_out = self.pos_trans_xy_norm(
            #     self.pos_trans_xy(pos_trans_xy_out))

            # pos_trans_whacls_out = self.pos_trans_whacls_norm(
            #     self.pos_trans_whacls(pos_trans_whacls_out))

            # query_pos = pos_trans_xy_out
            # query = pos_trans_whacls_out
            # # ######################################

            ## 단순 query + cls embedding
            # enc_ouputs_class_embedding = topk_cls_score
            # enc_ouputs_class_embedding = self.enc_output_cls_norm(self.enc_output_cls(enc_ouputs_class_embedding))
            # query = query + enc_ouputs_class_embedding

            ## cluster
            # topk_class = self.tgt_embed.weight[:, None, :].repeat(1, bs, 1)
            # query = topk_class
            # query_pos = self.pos_trans_norm(
            #     self.pos_trans(self.get_proposal_pos_embed(topk_coords_unact[..., :4])))

            # query = query+quantized
        else:
            query_pos, query = torch.split(query_embed, c, dim=1)
            query_pos = query_pos.unsqueeze(0).expand(bs, -1, -1)
            query = query.unsqueeze(0).expand(bs, -1, -1)
            reference_points = self.reference_points(query_pos).sigmoid()
            init_reference_out = reference_points

        # decoder
        # if first_stage:
        # vq = VectorQuantize(
        #     dim=256,
        #     codebook_size = 15,
        #     codebook_dim = 256,
        #     # orthogonal_reg_weight = 10,
        #     # kmeans_init= True,
        #     sync_codebook = False,
        # )
        # query = topk_memory.permute(1, 0, 2)

        # if not first_stage:
        # quantized, indices, commit_loss = vq(topk_class, topk_class_onehot)
        # print(commit_loss)
        # quantized = quantized.permute(1, 0, 2)
        # query = torch.add(query,quantized)

        # if first_stage:
        #     return enc_outputs_coord_unact_angle

        query = query.permute(1, 0, 2)
        memory = memory.permute(1, 0, 2)
        query_pos = query_pos.permute(1, 0, 2)
        inter_states, inter_references = self.decoder(
            query=query,
            key=None,
            value=memory,
            query_pos=query_pos,
            key_padding_mask=mask_flatten,
            reference_points=reference_points,
            spatial_shapes=spatial_shapes,
            level_start_index=level_start_index,
            valid_ratios=valid_ratios,
            reg_branches=reg_branches,
            bbox_coder=bbox_coder,
            **kwargs)

        inter_references_out = inter_references
        if self.as_two_stage:
            return inter_states, init_reference_out, \
                   inter_references_out, enc_outputs_class, \
                   enc_outputs_coord_unact_angle

        return inter_states, init_reference_out, \
               inter_references_out, None, None

@TRANSFORMER_LAYER_SEQUENCE.register_module()
class RotatedDeformableDetrTransformerDecoder(TransformerLayerSequence):
    """Implements the decoder in DETR transformer.

    Args:
        return_intermediate (bool): Whether to return intermediate outputs.
        coder_norm_cfg (dict): Config of last normalization layer. Default：
            `LN`.
    """

    def __init__(self, *args, return_intermediate=False, **kwargs):

        super(RotatedDeformableDetrTransformerDecoder, self).__init__(*args, **kwargs)
        self.return_intermediate = return_intermediate

    def forward(self,
                query,
                *args,
                reference_points=None,
                valid_ratios=None,
                reg_branches=None,
                bbox_coder=None,
                **kwargs):
        """Forward function for `TransformerDecoder`.

        Args:
            query (Tensor): Input query with shape
                `(num_query, bs, embed_dims)`.
            reference_points (Tensor): The reference
                points of offset. has shape
                (bs, num_query, 4) when as_two_stage,
                otherwise has shape ((bs, num_query, 2).
            valid_ratios (Tensor): The radios of valid
                points on the feature map, has shape
                (bs, num_levels, 2)
            reg_branch: (obj:`nn.ModuleList`): Used for
                refining the regression results. Only would
                be passed when with_box_refine is True,
                otherwise would be passed a `None`.

        Returns:
            Tensor: Results with shape [1, num_query, bs, embed_dims] when
                return_intermediate is `False`, otherwise it has shape
                [num_layers, num_query, bs, embed_dims].
        """
        output = query
        intermediate = []
        intermediate_reference_points = []
        ##### theta 고려해서 reference points 5개로 각도는 valid ratios 고려 x
        for lid, layer in enumerate(self.layers):
            if reference_points.shape[-1] == 4:
                reference_points_input = reference_points[:, :, None] * \
                                         torch.cat([valid_ratios, valid_ratios], -1)[:, None]
            else:
                assert reference_points.shape[-1] == 2
                reference_points_input = reference_points[:, :, None] * \
                                         valid_ratios[:, None]
            output = layer(
                output,
                *args,
                reference_points=reference_points_input,
                **kwargs)
            output = output.permute(1, 0, 2)

            if reg_branches is not None:
                # tmp = obb2xyxy(reg_branches[lid](output), version='le90')
                tmp = reg_branches[lid](output)
                ###### reference points 5개로 theta고려 필요
                if reference_points.shape[-1] == 4:
                    new_reference_points = tmp[..., :4] + inverse_sigmoid(
                        reference_points)
                    new_reference_points = new_reference_points.sigmoid()
                else:
                    assert reference_points.shape[-1] == 2
                    new_reference_points = tmp
                    new_reference_points[..., :2] = tmp[
                                                    ..., :2] + inverse_sigmoid(reference_points)
                    new_reference_points = new_reference_points.sigmoid()
                reference_points = new_reference_points.detach()

            output = output.permute(1, 0, 2)
            if self.return_intermediate:
                intermediate.append(output)
                # intermediate_reference_points.append(reference_points)
                intermediate_reference_points.append(new_reference_points)

        if self.return_intermediate:
            return torch.stack(intermediate), torch.stack(
                intermediate_reference_points)

        return output, reference_points


# app imports
import random

from django.utils.translation import ugettext_lazy as _
from django.conf import settings

from utils.faker_factory import faker

from ..mails import BaseMailView

class InvitationConsultantProjectMailView(BaseMailView):
    template_name = 'mails/invitation_consultant_project.html'
    mandatory_mail_args = [
        'user', 'project_name', 'relation_name',
        'is_coach',
    ]
    optional_mail_args = ['team_name', ]
    subject = _(settings.BRAND_NAME + ' - %(name)s invite')
    section = 'project'

    def get_subject(self, **kwargs):
        is_coach = kwargs.get('is_coach')
        if is_coach:
            return _(settings.BRAND_NAME + ' - Team coach invite')
        else:
            relation_name = kwargs.get('relation_name')
            return self.subject % ({'name': relation_name})

    def get_mock_data(self, optional=True):
        project_name = faker.word()
        team_name = faker.word()
        mock_data = {
            'team_name': team_name,
            'project_name': project_name,
            'relation_name': '[relation_name]',
            'user': faker.first_name(),
            'is_coach': random.randint(0, 1),
            'public_url': '/{}'.format(faker.uri_path()),
        }

        return mock_data


import json
import shutil
import unittest
from src import SDF3dData, train_sdf
from src.train_utils import *

class SDFTrainTest(unittest.TestCase):
    @staticmethod
    def get_abs_path():
        path = os.path.abspath(__file__)
        parent_dir = os.path.split(path)[0]
        return parent_dir

    def get_dataloaders(self):
        parent_dir = self.get_abs_path()
        data_configs = os.path.join(parent_dir, "configs", "test_data_configs.json")
        with open(data_configs, "rb") as fid:
            configs = json.load(fid)
        configs = configs['SDF3dData']
        data_handler = SDF3dData(**configs)
        train_dl, test_dl = data_handler.mesh_to_dataloader()
        return train_dl, test_dl

    def get_network(self):
        abs_path = os.path.abspath(__file__)
        parent_folder = os.path.split(os.path.split(os.path.split(abs_path)[0])[0])[0]
        sys.path.append(parent_folder)
        from graph_networks.src import EncodeProcessDecode

        parent_dir = self.get_abs_path()
        network_configs = os.path.join(parent_dir, "configs", "test_network_configs.json")
        with open(network_configs, "rb") as fid:
            config = json.load(fid)
        model_params = config['encode_process_decode']
        model = EncodeProcessDecode(**model_params)
        return model

    def test_graph_loss(self):
        loss_func = {'sdf_loss': {'loss_func_aggr': 'l1'}}
        data_parallel = False
        func = get_loss_funcs(loss_func, data_parallel)
        train_dl, _ = self.get_dataloaders()
        data = next(iter(train_dl))
        data.x = torch.norm(data.x, dim=1, keepdim=True) - 0.5
        func(data)

    def test_train(self):
        parent_dir = self.get_abs_path()
        training_configs = os.path.join(parent_dir, "configs", "test_training_configs.json")
        with open(training_configs, "rb") as fid:
            configs = json.load(fid)
        training_params = configs['train']
        model = self.get_network()
        train_dataloader, test_dataloader = self.get_dataloaders()
        train_sdf(model, train_dataloader, test_dataloader, **training_params)
        save_dir = os.path.join(os.getcwd(), training_params['save_folder_name'])
        self.assertTrue(os.path.isdir(save_dir))
        shutil.rmtree(save_dir)

if __name__ == '__main__':
    unittest.main()


import os
import torch
import cflearn

import numpy as np

# for reproduction
np.random.seed(142857)
torch.manual_seed(142857)

# preparation
data_config = {"label_name": "Survived"}
file_folder = os.path.dirname(__file__)
train_file = os.path.join(file_folder, "train.csv")
test_file = os.path.join(file_folder, "test.csv")

def write_submissions(name: str, predictions_: np.ndarray) -> None:
    with open(test_file, "r") as f:
        f.readline()
        id_list = [line.strip().split(",")[0] for line in f]
    with open(name, "w") as f:
        f.write("PassengerId,Survived\n")
        for test_id, prediction in zip(id_list, predictions_.ravel()):
            f.write(f"{test_id},{prediction}\n")

# wide and deep
m = cflearn.make("wnd", data_config=data_config)
m.fit(train_file)

cflearn.evaluate(train_file, pipelines=m, contains_labels=True)

predictions = m.predict(test_file, contains_labels=False)
write_submissions("submissions.csv", predictions)  # type: ignore

# tree linear
m = cflearn.make("tree_linear", data_config=data_config).fit(train_file)
predictions = m.predict(test_file, contains_labels=False)
write_submissions("submissions_tree_linear.csv", predictions)  # type: ignore

# save & load
cflearn.save(m)
loaded = cflearn.load()["tree_linear"][0]
predictions = m.predict(test_file, contains_labels=False)
write_submissions("submissions_loaded_tree_linear.csv", predictions)  # type: ignore