import unicodedata
from typing import List

import requests
from bs4 import BeautifulSoup

def target_page(page):
    request = requests.get(page).text
    soup = BeautifulSoup(request, 'html.parser')

    return soup

def get_notice_articles(page_num: int = 1) -> List[List[str]]:
    page = f"https://www.bible.ac.kr/ko/life/notice/list/{page_num}"
    table_list = target_page(page).select('ul li.tbody')
    outline = []

    for element in table_list:
        url = element.select_one('div ul.black a')['href']
        url = "https://www.bible.ac.kr" + url

        title = element.select_one('.title').get_text()
        title = title.strip()

        name = element.select_one('.name').get_text()
        date = element.select_one('.reg_date').get_text()

        article = target_page(url).select_one('.content').get_text()
        article = article.strip()
        article = unicodedata.normalize("NFKD", article)

        outline.append([url, title, name, date, article])

    return outline

if __name__ == '__main__':
    for item in get_notice_articles(3):
        print(item)


import random
import json

import torch

from model import NeuralNet
from nltk_utils import bag_of_words, tokenize

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# Load file data
with open('intents.json', 'r',encoding='utf8') as json_data:
    intents = json.load(json_data)

FILE = "data.pth"
data = torch.load(FILE)
# Khai báo size
all_words = data['all_words']
tags = data['tags']
model_state = data["model_state"]
input_size = data["input_size"]
hidden_size = data["hidden_size"]
output_size = data["output_size"]

# Khai báo model
model = NeuralNet(input_size, hidden_size, output_size).to(device)
model.load_state_dict(model_state)
model.eval()

bot_name = "Senki"
# Get response from message
def get_response(msg):
    sentence = tokenize(msg)
    X = bag_of_words(sentence, all_words)
    X = X.reshape(1, X.shape[0])
    X = torch.from_numpy(X).to(device)
    # Lấy output từ model
    output = model(X)
    _, predicted = torch.max(output, dim=1)
    # Lấy tag từ predicted
    tag = tags[predicted.item()]
    probs = torch.softmax(output, dim=1)
    prob = probs[0][predicted.item()]
    # Nếu prob > 0.75 thì lấy câu trả lời từ file intents.json
    if prob.item() > 0.75:
        for intent in intents['intents']:
            if tag == intent["tag"]:
                return random.choice(intent['responses'])

    return "Xin lỗi, tôi không hiểu những gì bạn đang nói..."

# Test
if __name__ == "__main__":
    print("Hãy chat với tôi ! (Gõ 'exit' để thoát)")
    while True:
        # sentence = "Xin chào"
        sentence = input("You: ")
        if sentence == "exit":
            break
        resp = get_response(sentence)
        print(resp)


from PyQt5.QtWidgets import QOpenGLWidget, QApplication, QHBoxLayout, QSlider, QWidget
from PyQt5.QtGui import QColor
from PyQt5.QtCore import pyqtSignal, QPoint, QSize, Qt
import OpenGL.GL as gl
import math

class Paint(QOpenGLWidget):

    xRotationChanged = pyqtSignal(int)
    yRotationChanged = pyqtSignal(int)
    zRotationChanged = pyqtSignal(int)

    ini_i = 0
    paint_i = 0
    res_i = 0

    def __init__(self, *args):
        super().__init__(args[0])
        print(args[0])
        self.object = 0
        self.xRot = 0
        self.yRot = 0
        self.zRot = 0

        self.lastPos = QPoint()

        self.trolltechGreen = QColor.fromCmykF(0.40, 0.0, 1.0, 0.0)
        self.trolltechGreen.setNamedColor('#C239B3')
        self.trolltechPurple = QColor.fromCmykF(0.39, 0.39, 0.0, 0.0)
        self.trolltechPurple.setNamedColor('#EDCAE3')

    def minimumSizeHint(self):
        return QSize(50, 50)

    def sizeHint(self):
        return QSize(400, 400)

    def setXRotation(self, angle):
        angle = self.normalizeAngle(angle)
        if angle != self.xRot:
            self.xRot = angle
            self.xRotationChanged.emit(angle)
            self.update()

    def setYRotation(self, angle):
        angle = self.normalizeAngle(angle)
        if angle != self.yRot:
            self.yRot = angle
            self.yRotationChanged.emit(angle)
            self.update()

    def setZRotation(self, angle):
        angle = self.normalizeAngle(angle)
        if angle != self.zRot:
            self.zRot = angle
            self.zRotationChanged.emit(angle)
            self.update()

    def initializeGL(self):
        # self.ini_i += 1
        # print('i_i is ' + str(self.ini_i))
        self.setClearColor(self.trolltechPurple)
        self.object = self.makeObject()
        gl.glShadeModel(gl.GL_FLAT)
        gl.glEnable(gl.GL_DEPTH_TEST)
        gl.glEnable(gl.GL_CULL_FACE)

    def paintGL(self):
        # self.paint_i += 1
        # print('paint_i is ' + str(self.paint_i))
        gl.glClear(gl.GL_COLOR_BUFFER_BIT | gl.GL_DEPTH_BUFFER_BIT)
        gl.glLoadIdentity()
        gl.glTranslated(0.0, 0.0, -10.0)
        gl.glRotated(self.xRot / 16.0, 1.0, 0.0, 0.0)
        gl.glRotated(self.yRot / 16.0, 0.0, 1.0, 0.0)
        gl.glRotated(self.zRot / 16.0, 0.0, 0.0, 1.0)
        gl.glCallList(self.object)

    def resizeGL(self, width, height):
        # self.res_i += 1
        # print('res_i is ' + str(self.res_i))
        side = min(width, height)
        if side < 0:
            return
        gl.glViewport((width - side) // 2, (height - side) // 2, side, side)
        gl.glMatrixMode(gl.GL_PROJECTION)
        gl.glLoadIdentity()
        gl.glOrtho(-0.5, +0.5, +0.5, -0.5, 4.0, 15.0)
        gl.glMatrixMode(gl.GL_MODELVIEW)

    def mousePressEvent(self, event):
        self.lastPos = event.pos()

    def mouseMoveEvent(self, event):
        dx = event.x() - self.lastPos.x()
        dy = event.y() - self.lastPos.y()

        if event.buttons() & Qt.LeftButton:
            self.setXRotation(self.xRot - 8 * dy)
            self.setYRotation(self.yRot + 8 * dx)
        elif event.buttons() & Qt.RightButton:
            self.setXRotation(self.xRot + 8 * dy)
            self.setZRotation(self.zRot + 8 * dx)

        self.lastPos = event.pos()

    def makeObject(self):
        genList = gl.glGenLists(1)
        gl.glNewList(genList, gl.GL_COMPILE)

        gl.glBegin(gl.GL_QUADS)

        x1 = +0.06
        y1 = -0.14
        x2 = +0.14
        y2 = -0.06
        x3 = +0.08
        y3 = +0.00
        x4 = +0.30
        y4 = +0.22

        # two squads
        self.quad(x1, y1, x2, y2, y2, x2, y1, x1)
        self.quad(x3, y3, x4, y4, y4, x4, y3, x3)

        self.extrude(x1, y1, x2, y2)
        self.extrude(x2, y2, y2, x2)
        self.extrude(y2, x2, y1, x1)
        self.extrude(y1, x1, x1, y1)
        self.extrude(x3, y3, x4, y4)
        self.extrude(x4, y4, y4, x4)
        self.extrude(y4, x4, y3, x3)

        NumSectors = 200

        for i in range(NumSectors):
            angle1 = (i * 2 * math.pi) / NumSectors
            x5 = 0.30 * math.sin(angle1)
            y5 = 0.30 * math.cos(angle1)
            x6 = 0.20 * math.sin(angle1)
            y6 = 0.20 * math.cos(angle1)

            angle2 = ((i + 1) * 2 * math.pi) / NumSectors
            x7 = 0.20 * math.sin(angle2)
            y7 = 0.20 * math.cos(angle2)
            x8 = 0.30 * math.sin(angle2)
            y8 = 0.30 * math.cos(angle2)

            self.quad(x5, y5, x6, y6, x7, y7, x8, y8)

            self.extrude(x6, y6, x7, y7)
            self.extrude(x8, y8, x5, y5)

        gl.glEnd()
        gl.glEndList()

        return genList

    def quad(self, x1, y1, x2, y2, x3, y3, x4, y4):
        self.setColor(self.trolltechGreen)

        gl.glVertex3d(x1, y1, -0.05)
        gl.glVertex3d(x2, y2, -0.05)
        gl.glVertex3d(x3, y3, -0.05)
        gl.glVertex3d(x4, y4, -0.05)

        gl.glVertex3d(x4, y4, +0.05)
        gl.glVertex3d(x3, y3, +0.05)
        gl.glVertex3d(x2, y2, +0.05)
        gl.glVertex3d(x1, y1, +0.05)

    def extrude(self, x1, y1, x2, y2):
        self.setColor(self.trolltechGreen.darker(250 + int(100 * x1)))

        gl.glVertex3d(x1, y1, +0.05)
        gl.glVertex3d(x2, y2, +0.05)
        gl.glVertex3d(x2, y2, -0.05)
        gl.glVertex3d(x1, y1, -0.05)

    def normalizeAngle(self, angle):
        while angle < 0:
            angle += 360 * 16
        while angle > 360 * 16:
            angle -= 360 * 16
        return angle

    def setClearColor(self, c):
        gl.glClearColor(c.redF(), c.greenF(), c.blueF(), c.alphaF())

    def setColor(self, c):
            gl.glColor4f(c.redF(), c.greenF(), c.blueF(), c.alphaF())


# @File: 4.py
# @Author: Kevin Huo
# @Date: 2020/4/18

from typing import List

def minJump(jump: List[int]) -> int:
    res = 0
    N = len(jump)

    opts = []
    # 求每一个 i 的最小步数
    for i in range(0, N):
        curr_idx = i
        curr_opt = 0
        while (curr_idx + jump[curr_idx]) < N:
            curr_opt += 1
            # 先往左
            if curr_idx > 0:
                curr_opt += min(opts[:i])
                continue
            # 再往右
            curr_idx += 1
        # 若curr_idx 能跳出去, 则准备写入 opts, 并计算下一个i

        opts.append(curr_opt)
    return res


import argparse
import seaborn as sns  # noqa
from matplotlib import pyplot as plt
from pathlib import Path
from dask import dataframe as dd

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input', required=True)
    args = parser.parse_args()

    data_dir = Path(args.input)
    name = data_dir.parent.name
    path = str(data_dir/'*.csv')
    df = dd.read_csv(path).compute()
    df = df.groupby(['model', 'n_trials', 'model_train_frac']).agg({
        'CV AUC': ['mean', 'std'],
        'duration[s]': ['mean', 'std'],
    })
    df.columns = ['%s_%s' % (a, b) for a, b in df.columns]
    df = df.reset_index()
    print(df)

    # plot
    fracs = sorted(df['model_train_frac'].unique())
    plt.figure(figsize=(12, 5))
    for j, frac in enumerate(fracs):
        for i, n_trials in enumerate([1, 10, 20, 30]):
            idx = (df['model'] == 'auto') &\
                  (df['n_trials'] == n_trials) &\
                  (df['model_train_frac'] == frac)
            x = df.loc[idx, 'duration[s]_mean']
            y = df.loc[idx, 'CV AUC_mean']
            xerr = df.loc[idx, 'duration[s]_std']
            yerr = df.loc[idx, 'CV AUC_std']
            fmt = '%sC%d' % (['x', 'o', 's', 'D'][i], j)
            label = 'n_trials=%d, model_train_frac=%.2f' % (n_trials, frac)  # noqa
            plt.errorbar(x, y, xerr=xerr, yerr=yerr, fmt=fmt, label=label)

    plt.title('Parameter Comparison (dataset=%s)' % (name))
    plt.xlabel('Training Time[s]')
    plt.ylabel('CV AUC')
    plt.legend()
    plt.savefig(str(data_dir/'frac-and-n_trials.png'))

if __name__ == '__main__':
    main()


import argparse
import glob
import json

import gradio as gr
import numpy as np
import torch

import torch.nn.functional as F
import tqdm

import MIDI
from midi_model import MIDIModel
from midi_tokenizer import MIDITokenizer
from midi_synthesizer import synthesis
from huggingface_hub import hf_hub_download

@torch.inference_mode()
def generate(prompt=None, max_len=512, temp=1.0, top_p=0.98, top_k=20,
             disable_patch_change=False, disable_control_change=False, disable_channels=None, amp=True):
    if disable_channels is not None:
        disable_channels = [tokenizer.parameter_ids["channel"][c] for c in disable_channels]
    else:
        disable_channels = []
    max_token_seq = tokenizer.max_token_seq
    if prompt is None:
        input_tensor = torch.full((1, max_token_seq), tokenizer.pad_id, dtype=torch.long, device=model.device)
        input_tensor[0, 0] = tokenizer.bos_id  # bos
    else:
        prompt = prompt[:, :max_token_seq]
        if prompt.shape[-1] < max_token_seq:
            prompt = np.pad(prompt, ((0, 0), (0, max_token_seq - prompt.shape[-1])),
                            mode="constant", constant_values=tokenizer.pad_id)
        input_tensor = torch.from_numpy(prompt).to(dtype=torch.long, device=model.device)
    input_tensor = input_tensor.unsqueeze(0)
    cur_len = input_tensor.shape[1]
    bar = tqdm.tqdm(desc="generating", total=max_len - cur_len)
    with bar, torch.cuda.amp.autocast(enabled=amp):
        while cur_len < max_len:
            end = False
            hidden = model.forward(input_tensor)[0, -1].unsqueeze(0)
            next_token_seq = None
            event_name = ""
            for i in range(max_token_seq):
                mask = torch.zeros(tokenizer.vocab_size, dtype=torch.int64, device=model.device)
                if i == 0:
                    mask_ids = list(tokenizer.event_ids.values()) + [tokenizer.eos_id]
                    if disable_patch_change:
                        mask_ids.remove(tokenizer.event_ids["patch_change"])
                    if disable_control_change:
                        mask_ids.remove(tokenizer.event_ids["control_change"])
                    mask[mask_ids] = 1
                else:
                    param_name = tokenizer.events[event_name][i - 1]
                    mask_ids = tokenizer.parameter_ids[param_name]
                    if param_name == "channel":
                        mask_ids = [i for i in mask_ids if i not in disable_channels]
                    mask[mask_ids] = 1
                logits = model.forward_token(hidden, next_token_seq)[:, -1:]
                scores = torch.softmax(logits / temp, dim=-1) * mask
                sample = model.sample_top_p_k(scores, top_p, top_k)
                if i == 0:
                    next_token_seq = sample
                    eid = sample.item()
                    if eid == tokenizer.eos_id:
                        end = True
                        break
                    event_name = tokenizer.id_events[eid]
                else:
                    next_token_seq = torch.cat([next_token_seq, sample], dim=1)
                    if len(tokenizer.events[event_name]) == i:
                        break
            if next_token_seq.shape[1] < max_token_seq:
                next_token_seq = F.pad(next_token_seq, (0, max_token_seq - next_token_seq.shape[1]),
                                       "constant", value=tokenizer.pad_id)
            next_token_seq = next_token_seq.unsqueeze(1)
            input_tensor = torch.cat([input_tensor, next_token_seq], dim=1)
            cur_len += 1
            bar.update(1)
            yield next_token_seq.reshape(-1).cpu().numpy()
            if end:
                break

def create_msg(name, data):
    return {"name": name, "data": data}

def run(tab, instruments, drum_kit, mid, midi_events, gen_events, temp, top_p, top_k, allow_cc, amp):
    mid_seq = []
    gen_events = int(gen_events)
    max_len = gen_events

    disable_patch_change = False
    disable_channels = None
    if tab == 0:
        i = 0
        mid = [[tokenizer.bos_id] + [tokenizer.pad_id] * (tokenizer.max_token_seq - 1)]
        patches = {}
        for instr in instruments:
            patches[i] = patch2number[instr]
            i = (i + 1) if i != 8 else 10
        if drum_kit != "None":
            patches[9] = drum_kits2number[drum_kit]
        for i, (c, p) in enumerate(patches.items()):
            mid.append(tokenizer.event2tokens(["patch_change", 0, 0, i, c, p]))
        mid_seq = mid
        mid = np.asarray(mid, dtype=np.int64)
        if len(instruments) > 0:
            disable_patch_change = True
            disable_channels = [i for i in range(16) if i not in patches]
    elif mid is not None:
        mid = tokenizer.tokenize(MIDI.midi2score(mid))
        mid = np.asarray(mid, dtype=np.int64)
        mid = mid[:int(midi_events)]
        max_len += len(mid)
        for token_seq in mid:
            mid_seq.append(token_seq.tolist())

    init_msgs = [create_msg("visualizer_clear", None)]
    for tokens in mid_seq:
        init_msgs.append(create_msg("visualizer_append", tokenizer.tokens2event(tokens)))
    yield mid_seq, None, None, init_msgs
    generator = generate(mid, max_len=max_len, temp=temp, top_p=top_p, top_k=top_k,
                         disable_patch_change=disable_patch_change, disable_control_change=not allow_cc,
                         disable_channels=disable_channels, amp=amp)
    for i, token_seq in enumerate(generator):
        mid_seq.append(token_seq)
        event = tokenizer.tokens2event(token_seq.tolist())
        yield mid_seq, None, None, [create_msg("visualizer_append", event), create_msg("progress", [i + 1, gen_events])]
    mid = tokenizer.detokenize(mid_seq)
    with open(f"output.mid", 'wb') as f:
        f.write(MIDI.score2midi(mid))
    audio = synthesis(MIDI.score2opus(mid), soundfont_path)
    yield mid_seq, "output.mid", (44100, audio), [create_msg("visualizer_end", None)]

def cancel_run(mid_seq):
    mid = tokenizer.detokenize(mid_seq)
    with open(f"output.mid", 'wb') as f:
        f.write(MIDI.score2midi(mid))
    audio = synthesis(MIDI.score2opus(mid), soundfont_path)
    return "output.mid", (44100, audio), [create_msg("visualizer_end", None)]

def load_model(path):
    ckpt = torch.load(path, map_location="cpu")
    state_dict = ckpt.get("state_dict", ckpt)
    model.load_state_dict(state_dict, strict=False)
    model.eval()
    return "success"

def get_model_path():
    model_paths = sorted(glob.glob("**/*.ckpt", recursive=True))
    return model_path_input.update(choices=model_paths)

def load_javascript(dir="javascript"):
    scripts_list = glob.glob(f"{dir}/*.js")
    javascript = ""
    for path in scripts_list:
        with open(path, "r", encoding="utf8") as jsfile:
            javascript += f"\n"
    template_response_ori = gr.routes.templates.TemplateResponse

    def template_response(*args, **kwargs):
        res = template_response_ori(*args, **kwargs)
        res.body = res.body.replace(
            b'', f'{javascript}'.encode("utf8"))
        res.init_headers()
        return res

    gr.routes.templates.TemplateResponse = template_response

class JSMsgReceiver(gr.HTML):

    def __init__(self, **kwargs):
        super().__init__(elem_id="msg_receiver", visible=False, **kwargs)

    def postprocess(self, y):
        if y:
            y = f"
{json.dumps(y)}
"
        return super().postprocess(y)

    def get_block_name(self) -> str:
        return "html"

number2drum_kits = {-1: "None", 0: "Standard", 8: "Room", 16: "Power", 24: "Electric", 25: "TR-808", 32: "Jazz",
                    40: "Blush", 48: "Orchestra"}
patch2number = {v: k for k, v in MIDI.Number2patch.items()}
drum_kits2number = {v: k for k, v in number2drum_kits.items()}

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--port", type=int, default=7860, help="gradio server port")
    parser.add_argument("--device", type=str, default="cuda", help="device to run model")
    soundfont_path = hf_hub_download(repo_id="skytnt/midi-model", filename="soundfont.sf2")
    opt = parser.parse_args()
    tokenizer = MIDITokenizer()
    model = MIDIModel(tokenizer).to(device=opt.device)

    load_javascript()
    app = gr.Blocks()
    with app:
        js_msg = JSMsgReceiver()
        with gr.Accordion(label="Model option", open=False):
            load_model_path_btn = gr.Button("Get Models")
            model_path_input = gr.Dropdown(label="model")
            load_model_path_btn.click(get_model_path, [], model_path_input)
            load_model_btn = gr.Button("Load")
            model_msg = gr.Textbox()
            load_model_btn.click(
                load_model, model_path_input, model_msg
            )
        tab_select = gr.Variable(value=0)
        with gr.Tabs():
            with gr.TabItem("instrument prompt") as tab1:
                input_instruments = gr.Dropdown(label="instruments (auto if empty)", choices=list(patch2number.keys()),
                                                multiselect=True, max_choices=15, type="value")
                input_drum_kit = gr.Dropdown(label="drum kit", choices=list(drum_kits2number.keys()), type="value",
                                             value="None")
                example1 = gr.Examples([
                    [[], "None"],
                    [["Acoustic Grand"], "None"],
                    [["Acoustic Grand", "Violin", "Viola", "Cello", "Contrabass"], "Orchestra"],
                    [["Flute", "Cello", "Bassoon", "Tuba"], "None"],
                    [["Violin", "Viola", "Cello", "Contrabass", "Trumpet", "French Horn", "Brass Section",
                      "Flute", "Piccolo", "Tuba", "Trombone", "Timpani"], "Orchestra"],
                    [["Acoustic Guitar(nylon)", "Acoustic Guitar(steel)", "Electric Guitar(jazz)",
                      "Electric Guitar(clean)", "Electric Guitar(muted)", "Overdriven Guitar", "Distortion Guitar",
                      "Electric Bass(finger)"], "Standard"]
                ], [input_instruments, input_drum_kit])
            with gr.TabItem("midi prompt") as tab2:
                input_midi = gr.File(label="input midi", file_types=[".midi", ".mid"], type="binary")
                input_midi_events = gr.Slider(label="use first n midi events as prompt", minimum=1, maximum=512,
                                              step=1,
                                              value=128)

        tab1.select(lambda: 0, None, tab_select, queue=False)
        tab2.select(lambda: 1, None, tab_select, queue=False)
        input_gen_events = gr.Slider(label="generate n midi events", minimum=1, maximum=4096, step=1, value=512)
        with gr.Accordion("options", open=False):
            input_temp = gr.Slider(label="temperature", minimum=0.1, maximum=1.2, step=0.01, value=1)
            input_top_p = gr.Slider(label="top p", minimum=0.1, maximum=1, step=0.01, value=0.98)
            input_top_k = gr.Slider(label="top k", minimum=1, maximum=20, step=1, value=12)
            input_allow_cc = gr.Checkbox(label="allow midi cc event", value=True)
            input_amp = gr.Checkbox(label="enable amp", value=True)
            example3 = gr.Examples([[1, 0.98, 12], [1.2, 0.95, 8]], [input_temp, input_top_p, input_top_k])
        run_btn = gr.Button("generate", variant="primary")
        stop_btn = gr.Button("stop and output")
        output_midi_seq = gr.Variable()
        output_midi_visualizer = gr.HTML(elem_id="midi_visualizer_container")
        output_audio = gr.Audio(label="output audio", format="mp3", elem_id="midi_audio")
        output_midi = gr.File(label="output midi", file_types=[".mid"])
        run_event = run_btn.click(run, [tab_select, input_instruments, input_drum_kit, input_midi, input_midi_events,
                                        input_gen_events, input_temp, input_top_p, input_top_k,
                                        input_allow_cc, input_amp],
                                  [output_midi_seq, output_midi, output_audio, js_msg])
        stop_btn.click(cancel_run, output_midi_seq, [output_midi, output_audio, js_msg], cancels=run_event, queue=False)
    app.queue(1).launch(server_port=opt.port)


# coding: utf-8

"""
    TKG Kubernetes API

    This API provides to vCD tenants the means to provision (create and update) Tanzu Kubernetes Grid clusters. This is complementary to the defined-entity APIs:    GET /cloudapi/1.0.0/entities/urn:vcloud:entity:vmware.tkgcluster:1.0.0:{id} which allows to retrieve the clusters created by the API presented here. This is why you will not find here a GET operation for the corresponding entity.   # noqa: E501

    OpenAPI spec version: 1.0.0

    Generated by: https://github.com/swagger-api/swagger-codegen.git
"""

import pprint
import re  # noqa: F401

import six

from container_service_extension.client.tkgclient.models.tkg_cluster_spec_distribution import TkgClusterSpecDistribution  # noqa: F401,E501
from container_service_extension.client.tkgclient.models.tkg_cluster_spec_settings import TkgClusterSpecSettings  # noqa: F401,E501
from container_service_extension.client.tkgclient.models.tkg_cluster_spec_topology import TkgClusterSpecTopology  # noqa: F401,E501

class TkgClusterSpec(object):
    """NOTE: This class is auto generated by the swagger code generator program.

    Do not edit the class manually.
    """

    """
    Attributes:
      swagger_types (dict): The key is attribute name
                            and the value is attribute type.
      attribute_map (dict): The key is attribute name
                            and the value is json key in definition.
    """
    swagger_types = {
        'distribution': 'TkgClusterSpecDistribution',
        'settings': 'TkgClusterSpecSettings',
        'topology': 'TkgClusterSpecTopology'
    }

    attribute_map = {
        'distribution': 'distribution',
        'settings': 'settings',
        'topology': 'topology'
    }

    def __init__(self, distribution=None, settings=None, topology=None):  # noqa: E501
        """TkgClusterSpec - a model defined in Swagger"""  # noqa: E501

        self._distribution = None
        self._settings = None
        self._topology = None
        self.discriminator = None

        self.distribution = distribution
        if settings is not None:
            self.settings = settings
        self.topology = topology

    @property
    def distribution(self):
        """Gets the distribution of this TkgClusterSpec.  # noqa: E501

        :return: The distribution of this TkgClusterSpec.  # noqa: E501
        :rtype: TkgClusterSpecDistribution
        """
        return self._distribution

    @distribution.setter
    def distribution(self, distribution):
        """Sets the distribution of this TkgClusterSpec.

        :param distribution: The distribution of this TkgClusterSpec.  # noqa: E501
        :type: TkgClusterSpecDistribution
        """
        if distribution is None:
            raise ValueError("Invalid value for `distribution`, must not be `None`")  # noqa: E501

        self._distribution = distribution

    @property
    def settings(self):
        """Gets the settings of this TkgClusterSpec.  # noqa: E501

        :return: The settings of this TkgClusterSpec.  # noqa: E501
        :rtype: TkgClusterSpecSettings
        """
        return self._settings

    @settings.setter
    def settings(self, settings):
        """Sets the settings of this TkgClusterSpec.

        :param settings: The settings of this TkgClusterSpec.  # noqa: E501
        :type: TkgClusterSpecSettings
        """

        self._settings = settings

    @property
    def topology(self):
        """Gets the topology of this TkgClusterSpec.  # noqa: E501

        :return: The topology of this TkgClusterSpec.  # noqa: E501
        :rtype: TkgClusterSpecTopology
        """
        return self._topology

    @topology.setter
    def topology(self, topology):
        """Sets the topology of this TkgClusterSpec.

        :param topology: The topology of this TkgClusterSpec.  # noqa: E501
        :type: TkgClusterSpecTopology
        """
        if topology is None:
            raise ValueError("Invalid value for `topology`, must not be `None`")  # noqa: E501

        self._topology = topology

    def to_dict(self):
        """Returns the model properties as a dict"""
        result = {}

        for attr, _ in six.iteritems(self.swagger_types):
            value = getattr(self, attr)
            if isinstance(value, list):
                result[attr] = list(map(
                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
                    value
                ))
            elif hasattr(value, "to_dict"):
                result[attr] = value.to_dict()
            elif isinstance(value, dict):
                result[attr] = dict(map(
                    lambda item: (item[0], item[1].to_dict())
                    if hasattr(item[1], "to_dict") else item,
                    value.items()
                ))
            else:
                result[attr] = value

        return result

    def to_str(self):
        """Returns the string representation of the model"""
        return pprint.pformat(self.to_dict())

    def __repr__(self):
        """For `print` and `pprint`"""
        return self.to_str()

    def __eq__(self, other):
        """Returns true if both objects are equal"""
        if not isinstance(other, TkgClusterSpec):
            return False

        return self.__dict__ == other.__dict__

    def __ne__(self, other):
        """Returns true if both objects are not equal"""
        return not self == other


import sys
from PySide2 import QtWidgets
from lone import Ui_Dialog

class mydilog(QtWidgets.QDialog,Ui_Dialog):
    def __init__(self):
        super(mydilog,self).__init__()
        self.setupUi(self)

#main
app = QtWidgets.QApplication(sys.argv)

lone = mydilog()
lone.show()
lone.exec_()


# -*- coding=UTF-8 -*-
# pyright: strict
# spell-checker: words chara inout
""".  """

if True:
    import sys
    import os

    sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))

from typing import Iterator, Text
import sqlite3
import argparse
import os
import contextlib
from auto_derby.single_mode.item import Item, Effect, g
import json
import pathlib

def _get_effects(db: sqlite3.Connection, group: int) -> Iterator[Effect]:

    with contextlib.closing(
        db.execute(
            """
SELECT
  t1.id,
  t1.effect_group_id,
  t1.effect_type,
  t1.effect_value_1,
  t1.effect_value_2,
  t1.effect_value_3,
  t1.effect_value_4,
  t1.turn
  FROM single_mode_free_shop_effect AS t1
  WHERE t1.effect_group_id = ?
""",
            (group,),
        )
    ) as cur:
        for i in cur:
            assert len(i) == 8, i
            e = Effect()
            (
                e.id,
                e.group,
                e.type,
                v1,
                v2,
                v3,
                v4,
                e.turn_count,
            ) = i
            e.values = (v1, v2, v3, v4)
            yield e

def _read_master_mdb(path: Text) -> Iterator[Item]:

    db = sqlite3.connect(path)
    cur = db.execute(
        """
SELECT
  t1.id,
  t2.text,
  t3.text,
  t1.coin_num,
  t1.limit_num,
  t1.effect_priority,
  t1.effect_group_id
  FROM single_mode_free_shop_item AS t1
  LEFT JOIN text_data AS t2 ON t2.category = 225 AND t2."index" = t1.item_id
  LEFT JOIN text_data AS t3 ON t3.category = 238 AND t3."index" = t1.item_id
;
    """
    )

    with contextlib.closing(cur):
        for i in cur:
            assert len(i) == 7, i
            v = Item()
            (
                v.id,
                v.name,
                v.description,
                v.original_price,
                v.max_quantity,
                v.effect_priority,
                effect_group,
            ) = i
            v.effects = tuple(_get_effects(db, effect_group))
            yield v

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "path",
        nargs="?",
        default=os.path.expandvars(
            "${LocalAppData}Low/cygames/umamusume/master/master.mdb"
        ),
    )
    args = parser.parse_args()
    path: Text = args.path

    with pathlib.Path(g.data_path).open("w", encoding="utf-8") as f:
        for item in _read_master_mdb(path):
            json.dump(item.to_dict(), f, ensure_ascii=False)
            f.write("\n")

if __name__ == "__main__":
    main()


# def log(func):
#     def wrapper(*args, **kw):
#         print('call %s():' % func.__name__)
#         return func(*args, **kw)
#     return wrapper

# @log
# def now():
#     print('2015-3-25')

# now=log(now)

# f=now

# print(f())

# import functools

# def log(text):
#     def decorator(func):
#         @functools.wraps(func)
#         def wrapper(*args, **kw):
#             print('%s %s():' % (text, func.__name__))
#             return func(*args, **kw)
#         return wrapper
#     return decorator

# @log('execute')
# def now():
#     print('2015-3-25')

# now=log("execute")(now)

# f=now

# print(f.__name__)

import  time, functools

def metric(fn):
    @functools.wraps(fn)
    def wrapper(*args, **kw):
        print('begin call %s():' % fn.__name__)
        temp = fn(*args, **kw)
        print('begin call %s():' % fn.__name__)
        return temp
    return wrapper

# 测试
@metric
def fast(x, y):
    time.sleep(0.0012)
    print(x+y)
    return x + y;

@metric
def slow(x, y, z):
    time.sleep(0.1234)
    print(x * y * z)
    return x * y * z;

f = fast(11, 22)
s = slow(11, 22, 33)
if f != 33:
    print('测试失败!')
elif s != 7986:
    print('测试失败!')


"""Uses keras RNN architecture.

"""
import keras
from keras.layers import *
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical

import numpy as np

def create_tokens2(filename):
    sequences = np.load(filename)
    num_sequences = len(sequences)
    tokenizer = Tokenizer()
    tokenizer.fit_on_texts(sequences)
    sequences = tokenizer.texts_to_sequences(sequences)
    sequences = pad_sequences(sequences, 300)
    vocab_size = len(tokenizer.word_index) + 1
    X, y = sequences[:][:-1], sequences[:][-1]
    X = np.reshape(X, (num_sequences-1, 300, 1))
    y = to_categorical(y, num_classes=vocab_size)
    seq_length = 300
    return vocab_size, seq_length, X, y

def create_tokens(filename):
    sequences = np.load(filename)
    tokenizer = Tokenizer()
    tokenizer.fit_on_texts(sequences)
    X = list()
    y = list()

    for seq in sequences:
        seq_arr = seq.split()
        for length in range(300-1):
            X.append(seq_arr[:length + 1])
            y.append(seq_arr[length + 1])

    X = tokenizer.texts_to_sequences(X)
    y = tokenizer.texts_to_sequences(y)
    X = pad_sequences(X, 300)
    vocab_size = len(tokenizer.word_index) + 1
    X = np.reshape(X, (len(X), 300, 1))
    y = to_categorical(y, num_classes=vocab_size)
    seq_length = 300
    return vocab_size, seq_length, X, y

def create_train_model(vocab_size, seq_length):
    model = keras.Sequential()
    #model.add(Embedding(input_dim=vocab_size, output_dim=512, input_shape=(300, 1)))
    model.add(LSTM(256, input_shape=(seq_length, 1), return_sequences=True))
    model.add(LSTM(units=256, return_sequences=True, stateful=False))
    #model.add(Dropout(rate=.2))
    model.add(LSTM(units=256, stateful=False))
    model.add(Dense(vocab_size))
    model.add(Activation("softmax"))
    return model

def create_prediction_model(vocab_size):
    model = keras.Sequential()
    #model.add(Embedding(input_dim=vocab_size, output_dim=512, input_shape=(300, 1)))
    model.add(LSTM(256, return_sequences=True))
    model.add(LSTM(units=256, return_sequences=True, stateful=False))
    #model.add(Dropout(rate=.2))
    model.add(LSTM(units=256, stateful=False))
    model.add(Dense(vocab_size))
    model.add(Activation("softmax"))
    return model

def train(X, y, vocab_size, num_epochs, seq_length):
    model = create_train_model(vocab_size, seq_length)
    model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
    model.fit(X, y, batch_size=32, epochs=num_epochs)

    model.save_weights("data/keras_rnn.h5")

def generate_sequence(seq_length, start_note, model_weights_file):
    model = create_prediction_model()
    model.load_weights(model_weights_file)
    seq = [60]

    for _ in range(seq_length - 1):
        batch = np.zeros((1, 1))
        batch[0, 0] = start_note
        predicted_probs = model.predict_on_batch(batch).ravel()
        sample = np.random.choice(range(128), size=1, p=predicted_probs)
        seq.append(sample[0])

    return seq

def train_and_generate(seq_file, num_epochs):
    vocab_size, seq_length, X, y = create_tokens("data/train_sequences/all_note_sequences.npy")
    train(X, y, vocab_size, num_epochs, seq_length)
    #generated = generate_sequence(seq_length, 48, "models/keras_only_notes_model.h5")
    #print(generated)

def main():
    train_and_generate("data/all_note_sequences.npy", 50)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-

"""
Created on Thu Aug 14 13:28:09 2018

@author: HareeshRavi
"""

import keras
from keras.models import Model
from keras.layers import TimeDistributed, Dense, GRU, Dropout, Embedding
from keras.layers import Input, concatenate
from keras import optimizers
from keras import backend as K
import tensorflow as tf
import utils_vist
from keras.initializers import Constant

# network architecture for baseline experiment
def baseline(modconfig, num_words, embedding_matrix):

    # read config
    MAX_SEQUENCE_LENGTH = modconfig['MAX_SEQUENCE_LENGTH']
    word_embd_dim = modconfig['wd_embd_dim']
    sent_feat_dim = modconfig['sent_fea_dim']
    img_feat_dim = modconfig['img_fea_dim']
    embedding_layer = Embedding(num_words,
                                word_embd_dim,
                                embeddings_initializer=Constant(embedding_matrix),
                                input_length=MAX_SEQUENCE_LENGTH,
                                name='wd_embedding_layer',
                                trainable=False)

    input_sent = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32',
                       name='input1')
    input_img = Input(shape=(img_feat_dim,), dtype='float32', name='input2')

    # Encode sentence
    # This embedding layer will encode the input sequence
    # into a sequence of dense 512-dimensional vectors.
    x1 = embedding_layer(input_sent)

    # using GRU instead of LSTM
    Encode_sent = keras.layers.recurrent.GRU(sent_feat_dim, name='gru')(x1)

    Encode_sent_normed = keras.layers.Lambda(lambda x: K.abs(K.l2_normalize(
            x, axis=1)), name='sentFeaNorm')(Encode_sent)

    # encoding image feat
    Encode_img = Dense(sent_feat_dim, activation='linear',
                       name='imgEncode')(input_img)
    Encode_img_normed = keras.layers.Lambda(lambda x: K.abs(K.l2_normalize(
            x, axis=1)), name='imgFeaNorm')(Encode_img)

    # define a Lambda merge layer
    lambda1 = keras.layers.Lambda(utils_vist.contrastive_loss,
                                  output_shape=utils_vist.edis_outputshape,
                                  name = 'orderEmbd')
    main_output = lambda1([Encode_sent_normed, Encode_img_normed])

    lambda2 = keras.layers.Lambda(utils_vist.retriv_acc,
                                  output_shape=utils_vist.edis_outputshape,
                                  name = 'Recall_1')
    acc_output = lambda2([Encode_sent_normed, Encode_img_normed])

#    main_output = keras.layers.merge([Encode_sent_normed, Encode_img_normed],
#                                     mode=utils_vist.contrastive_loss,
#                                     output_shape=utils_vist.edis_outputshape,
#                                     name='orderEmbd')
#
#    acc_output = keras.layers.merge([Encode_sent_normed, Encode_img_normed],
#                                    mode=utils_vist.retriv_acc,
#                                    output_shape=utils_vist.edis_outputshape,
#                                    name='Recall_1')

    baselinemodel = Model(inputs=[input_sent, input_img],
                          outputs=[main_output, acc_output])
    baselinemodel.compile(loss=['mean_absolute_error',
                                 utils_vist.MyCustomLoss],
                           optimizer='adam',
                           loss_weights=[1,0])
    baselinemodel.summary()
    return baselinemodel

# sequential order embedding loss function
def orderEmb_loss(y_true, y_pred):
    y_true = K.l2_normalize(K.abs(y_true), axis=2)
    y_pred = K.l2_normalize(K.abs(y_pred), axis=2)
    y_truemod = K.expand_dims(y_true, axis=0)
    y_predmod = K.expand_dims(y_pred, axis=1)
    order_viola = K.permute_dimensions(K.sum(K.pow(K.maximum(
            K.cast(0, 'float32'), y_predmod - y_truemod), 2), axis=3),
            (2, 0, 1))
    diagErr_im = K.expand_dims(tf.matrix_diag_part(order_viola), axis=2)
    diagErr_s = K.expand_dims(tf.matrix_diag_part(order_viola), axis=1)
    margin = K.cast(0.05, 'float32')

    # negative samples shuffling code
    def body_func(idx, bool_in):
        idx = K.variable([0, 1, 2, 3, 4], dtype='int32')
        shuff_idx = tf.random_shuffle(idx)
        bool_in = tf.reduce_any(tf.equal(shuff_idx, idx))
        return shuff_idx, bool_in

    cond_func = lambda idx, bool_in: bool_in

    idx = K.variable([0, 1, 2, 3, 4], dtype='int32')
    bool_in = True

    def shuffle_tensoridx(arr_inp):
        result = tf.while_loop(cond_func, body_func, [idx, bool_in])
        arr_b = tf.gather(arr_inp, result[0])
        return arr_b

    ypred_shuffle = tf.map_fn(shuffle_tensoridx, y_pred)
    ypredmod_shuffle = K.expand_dims(ypred_shuffle, axis=1)

    # loss calculation according to objective function
    order_violb = K.permute_dimensions(K.sum(K.pow(K.maximum(
            K.cast(0, 'float32'), ypredmod_shuffle - y_truemod), 2),
            axis=3), (2, 0, 1))

    cost_im = K.maximum(K.cast(0, 'float32'),
                        diagErr_im - order_violb + margin)
    cost_s = K.maximum(K.cast(0, 'float32'),
                       diagErr_s - order_violb + margin)
    #    mask = tf.reverse(K.eye(32),[0])
    temp = tf.ones([32, 32], tf.float32)
    mask = tf.matrix_set_diag(temp, tf.zeros([32], tf.float32))
    tot_cost = K.sum(K.sum(K.sum(tf.multiply(cost_im + cost_s, mask),
                                 axis=2), axis=1), axis=0)
    # print(tot_cost)
    return tot_cost / 32

# stage 1 of proposed model
def stage1(config, num_words, embedding_matrix):

    modconfig = config['stage1']
    # read config
    MAX_SEQUENCE_LENGTH = modconfig['MAX_SEQUENCE_LENGTH']
    word_embed_dim = modconfig['wd_embd_dim']
    sent_feat_dim = modconfig['sent_fea_dim']
    img_feat_dim = modconfig['img_fea_dim']

    embedding_layer = Embedding(num_words,
                                word_embed_dim,
                                embeddings_initializer=Constant(embedding_matrix),
                                input_length=MAX_SEQUENCE_LENGTH,
                                name='vist_wd_embedding_layer',
                                trainable=False)

    input_sent = Input(shape=(MAX_SEQUENCE_LENGTH,),
                       dtype='int32', name='input1')

    x1 = embedding_layer(input_sent)

    # encoding each sentence GRU over words
    Encode_sent = keras.layers.recurrent.GRU(sent_feat_dim, name='gru')(x1)
    Encode_sent_normed = keras.layers.Lambda(lambda x: K.abs(
            K.l2_normalize(x, axis=1)), name='sentFeaNorm')(Encode_sent)

    input_img = Input(shape=(img_feat_dim,), dtype='float32', name='input2')

    # encoding image feat
    Encode_img = Dense(sent_feat_dim, activation='linear',
                       name='imgEncode')(input_img)
    Encode_img_normed = keras.layers.Lambda(lambda x: K.abs(
            K.l2_normalize(x, axis=1)), name='imgFeaNorm')(Encode_img)

    sentence_model = Model(inputs=[input_sent, input_img], outputs=[
            Encode_sent_normed, Encode_img_normed])

    sentence_model.get_layer('vist_wd_embedding_layer').trainable = False

    sentence_model.compile(loss=['mean_absolute_error',
                                 utils_vist.MyCustomLoss],
                           optimizer='adam',
                           loss_weights=[1,0])
    sentence_model.summary()
    return sentence_model

# stage 2 of proposed model
def stage2(config, num_words, embedding_matrix):

    # read config
    modconfig = config['stage2']
    hidden_size1 = modconfig['hidden_size1']
    hidden_size2 = modconfig['hidden_size2']
    hidden_size3 = modconfig['hidden_size3']
    learningrate = modconfig['learningrate']
    x_len = modconfig['x_len']
    x_dim = modconfig['x_dim']
    y_dim = modconfig['y_dim']
    cohfeat_dim = modconfig['cohfeat_dim']

    opt = optimizers.adam(lr=learningrate)

    # input from sentence encoder and image encoder
    i11 = Input(shape=(x_len, x_dim), name='txt_input')
    d11 = Dropout(0.2, input_shape=(x_len, x_dim), name='layer_1_drop')(i11)

    # story encoder
    g11 = GRU(hidden_size1, return_sequences=True, name='layer_1_gru')(d11)
    g12 = GRU(hidden_size2, return_sequences=True, name='layer_2_gru')(g11)
    g13 = GRU(hidden_size3, return_sequences=True, name='layer_3_gru')(g12)

    if cohfeat_dim:
        # concatenate coherence vector as input
        i21 = Input(shape=(x_len, cohfeat_dim), name='coh_input')
        m1 = concatenate([g13, i21], axis=2, name='concatlayer1')
        # final dense layer
        td11 = TimeDistributed(Dense(y_dim), name='layer_4_timedist')(m1)
        # final model
        story_model = Model(inputs=[i11, i21], outputs=td11)
    else:
        # final dense layer
        td11 = TimeDistributed(Dense(y_dim), name='layer_4_timedist')(g13)
        # final model
        story_model = Model(inputs=i11, outputs=td11)

    # order embedding loss
    story_model.compile(loss=orderEmb_loss, optimizer=opt,
                        metrics=['accuracy'])

    story_model.summary()

    return story_model


from django.contrib import messages
from django.contrib.auth.views import LoginView
from django.shortcuts import redirect
from django.urls import reverse
from django.views.generic import TemplateView, FormView

from dashboard.forms import FetchComicsForm
from .models import Comic
from .scrapers import NewComicsParser

class DashboardHomeView(TemplateView):
    template_name = 'dashboard/index.html'

class DashboardLoginView(LoginView):
    template_name = 'dashboard/login.html'

    def get_success_url(self):
        return reverse('dashboard:home')

class FetchComics(FormView):
    form_class = FetchComicsForm
    template_name = 'dashboard/fetch_comics.html'

    def get_context_data(self, **kwargs):
        ctx = super(FetchComics, self).get_context_data(**kwargs)
        date = self.request.GET.get('date')
        ctx['comic_list'] = Comic.objects.nearest_wednesday_comics(date)
        return ctx

    def form_valid(self, form):
        date = form.cleaned_data.get('date')
        parser = NewComicsParser()
        parser.parse(date=date)
        messages.add_message(self.request, messages.INFO, 'Comics have been fetched for the `{}`'.format(date))
        return redirect('{url}?date={date}'.format(url=reverse('dashboard:fetch_comics'), date=date))

fetch_comics = FetchComics.as_view()
login = DashboardLoginView.as_view()
home = DashboardHomeView.as_view()


from django.contrib import admin

from .models import BibliographicRecord, WorkRecord, ISBN

class BibliographicRecordAdmin(admin.ModelAdmin):
    model = BibliographicRecord
    list_display = ["ils_id",
                    "author",
                    "browse_title",
                    "provider",
                    "provider_id",
                    "abs_mat_code"]
    search_fields = ["ils_id", "browse_title", "browse_author", "provider"]
    list_filter = ["abs_mat_code", "provider"]
    raw_id_fields = ["isbns"]

class BibliographicRecordInline(admin.TabularInline):
    model = BibliographicRecord
    raw_id_fields = ("work_record", "isbns")

class WorkRecordAdmin(admin.ModelAdmin):
    model = WorkRecord
    list_display = ["id", "title", "author"]
    search_fields = ["title", "author"]
    raw_id_fields = ["primary_bib_record"]
    inlines = [
        BibliographicRecordInline,
    ]

class ISBNAdmin(admin.ModelAdmin):
    model = ISBN
    list_display = ["isbn"]

admin.site.register(BibliographicRecord, BibliographicRecordAdmin)
admin.site.register(WorkRecord, WorkRecordAdmin)
admin.site.register(ISBN, ISBNAdmin)


# -------------------------------
# PipeBit
# 'setup.py'
# Author: Juan Carlos Juárez.
# Licensed under MPL 2.0.
# All rights reserved.
# -------------------------------

from setuptools import setup, find_packages

VERSION = '0.0.41'
DESCRIPTION = 'PipeBit'
LONG_DESCRIPTION = 'PipeBit is a Quick Pipeline Management System for Local Data Pipelines.'

# Setting up
setup(
    name="pipebit",
    version=VERSION,
    author="Juan Carlos Juárez",
    author_email="<jc.juarezgarcia@outlook.com>",
    description=DESCRIPTION,
    long_description_content_type="text/markdown",
    long_description=LONG_DESCRIPTION,
    packages=find_packages(),
    install_requires=[],
    keywords=['Pipelines', 'Python', 'Tool'],
    classifiers=[
        "Development Status :: 1 - Planning",
        "Intended Audience :: Developers",
        "Programming Language :: Python :: 3",
        "Operating System :: Unix",
        "Operating System :: MacOS :: MacOS X",
        "Operating System :: Microsoft :: Windows",
    ]
)


from tkinter import *
from PIL import ImageTk, Image

screen = Tk()
screen.geometry("300x450")
screen.configure(bg='powderblue')

screen.title("Loan Interest Software")

def interest_calculator():
    loan_interest = (float(property_price.get())*float(spinboxvar.get())*float(interest_rate.get()))/100
    total_payment = float(property_price.get())+loan_interest
    result.set(f'Your total payment with interest is AED {total_payment}')

Label(text="", bg='powderblue').pack()
Label(text="Loan Interest Calculator", bg='gold').pack()

Label(text="", bg='powderblue').pack()
Label(text="Enter Property Price (AED)", bg='powderblue').pack()
property_price = Entry(screen)
property_price.pack()

Label(text="", bg='powderblue').pack()
Label(text="Enter Loan Period (Years)", bg='powderblue').pack()
spinboxvar = StringVar(screen)
SpinBox1 = Spinbox(screen,width=18, from_=1, to=30, increment=1, textvariable=spinboxvar)
SpinBox1.pack()

Label(text="", bg='powderblue').pack()
Label(text="Enter Interest Rate (%)", bg='powderblue').pack()
interest_rate = Entry(screen)
interest_rate.pack()

Label(text="", bg='powderblue').pack()
Button(text="Calculate", command=interest_calculator, bg='RosyBrown1').pack()

Label(text="", bg='powderblue').pack()
result = IntVar()
Label(text="", textvariable=result).pack()

def show_image():

    canvas = Canvas(screen, width=150, height=150, bg='powderblue', highlightthickness=0)
    canvas.pack()

    img = Image.open("5a46848ab4df7b9cbf90d7dd73c4a855-simple-house-black-silhouette-by-vexels.png")
    img = img.resize((150, 150), Image.ANTIALIAS)
    img = ImageTk.PhotoImage(img)
    canvas.create_image(78, 150, anchor=S, image=img)
    screen.mainloop()

if __name__ == '__main__':
    show_image()


from setuptools import setup, Extension, find_packages
import os
import numpy as np
from Cython.Build import cythonize

requirements = [
        'emcee',
        'scipy',
        'numpy',
        'h5py',
        'pandas',
        'future',
        'lru-dict'
        ]

have_mkl = False
try:
    mklinfo = np.__config__.blas_mkl_info
    if mklinfo != {}:
        have_mkl = True
except:
    False
have_openblas = False
try:
    openblasinfo = np.__config__.openblas_info
    if openblasinfo != {}:
        have_openblas = True
except:
    pass
openblas_link_args = ["-lgsl", "-lopenblas"]
mkl_link_args = ["-lgsl", "-lmkl_sequential", "-lmkl_core", "-lmkl_intel_lp64"]
cblas_link_args = ["-lgsl", "-lblas"]
if have_mkl:
    link_args = mkl_link_args
elif have_openblas:
    link_args = openblas_link_args
else:
    link_args = cblas_link_args
compile_args = ['-march=native', '-O3']
try:
    library_dirs = os.environ['LIBRARY_PATH'].split(':')
except KeyError:
    library_dirs = []
try:
    include_dirs = [np.get_include()] + os.environ['CPATH'].split(':')
except KeyError:
    include_dirs = [np.get_include()]

extensions = [
        Extension(
            'mope._wf',
            ['mope/_wf.pyx'],
            include_dirs = include_dirs,
            extra_link_args = link_args,
            extra_compile_args = compile_args,
            library_dirs = library_dirs),
        Extension(
            'mope._util',
            ['mope/_util.pyx'],
            include_dirs = include_dirs,
            library_dirs = library_dirs),
        Extension(
            'mope._binom',
            ['mope/_binom.pyx'],
            include_dirs = include_dirs,
            extra_link_args = link_args,
            extra_compile_args = compile_args,
            library_dirs = library_dirs),
        Extension(
            'mope._interp',
            ['mope/_interp.pyx'],
            include_dirs = include_dirs,
            extra_compile_args = compile_args,
            library_dirs = library_dirs),
        Extension(
            'mope._likes',
            ['mope/_likes.pyx'],
            include_dirs = include_dirs,
            extra_link_args = link_args,
            library_dirs = library_dirs,
            extra_compile_args = compile_args),
        Extension(
            'mope._poisson_binom',
            ['mope/_poisson_binom.pyx'],
            include_dirs = include_dirs,
            library_dirs = library_dirs,
            extra_link_args = link_args,
            extra_compile_args = compile_args),
        Extension(
            'mope._transition',
            ['mope/_transition.pyx'],
            include_dirs = include_dirs,
            library_dirs = library_dirs,
            extra_link_args = link_args,
            extra_compile_args = compile_args)
        ]

entry_points = {'console_scripts': ['mope = mope.cli:main']}

setup(
        name='mope',
        version = '0.5.1',
        description='Molecular ontogenetic phylogeny estimation',
        author='Peter Wilton',
        author_email='pwilton@berkeley.edu',
        license='GPLv3',
        install_requires = requirements,
        python_requires = '>=2.7',
        packages = find_packages(),
        entry_points = entry_points,
        ext_modules=cythonize(extensions))


from django.contrib import admin
from django.urls import path
from .views import(
    ################ User ########################
    UserListAPIVeiw,
    UserCreateAPIVeiw,
    UserUpdateListAPIVeiw,
    UserRetieveAPIVeiw,
    UserDestroyAPIView,
)

urlpatterns = [
    ################ User ########################
    path('User/' , UserListAPIVeiw.as_view() ) ,
    path('User/create' , UserCreateAPIVeiw.as_view() ) ,
    path('User/<pk>' , UserRetieveAPIVeiw.as_view() ) ,
    path('User/<pk>/edit' , UserUpdateListAPIVeiw.as_view() ) ,
    path('User/<pk>/delete' , UserDestroyAPIView.as_view() ) ,
]


import random

from mesa import Agent

class SingleSpread(Agent):
    '''
    A simple tree cell, where the fire only advances 1 cell at a time.

    Attributes:
        x, y: Grid coordinates
        condition: Can be "Empty", "Fine", "On Fire", or "Burned Out"
        unique_id: (x,y) tuple.
        forest_density: density of the forest.
        protection: density of the protection systems in the forest.

    unique_id isn't strictly necessary here, but it's good practice to give one to each agent anyway.
    '''
    def __init__(self, model, pos, forest_density, protection, p, f):
        '''
        Create a new tree.
        Args:
            pos: The tree's coordinates on the grid. Used as the unique_id
        '''
        super().__init__(pos, model)
        self.pos = pos
        self.unique_id = pos

        random_value = random.random()
        self.condition = "Empty"
        if random_value < forest_density:
            self.condition = "Fine"
        if random_value > 1 - protection:
            self.condition = "Protected"

        # Set all trees in the first column on fire.
        if self.pos[0] == 0:
            self.set_on_fire()
        self.last_condition = self.condition
        self.advance = True

    def set_on_fire(self):
        if self.condition == "Fine":
            self.condition = "On Fire"

    def step(self):
        '''
        Advance 1 step in the simulation.
        '''
        if self.advance:
            self.advance_step()
        else:
            self.estabilize_step()

    def advance_step(self):
        '''
        If the tree is on fire, spread it to fine trees nearby.
        Only spread the fire if the tree has caught fire the last "step" (advance step).
        '''
        if self.last_condition == "On Fire":
            neighbors = self.model.grid.get_neighbors(self.pos, moore=False)
            for neighbor in neighbors:
                neighbor.set_on_fire()
            self.condition = "Burned Out"
        self.advance = False

    def estabilize_step(self):
        '''
        Estabilize tree condition so the fire only spreads once every "step" (in this case, it will be once every 2 steps).
        '''
        self.last_condition = self.condition
        self.advance = True


import io
import math
import random
import unittest

import numpy as np
import onnx
import onnx.checker
import onnxruntime
import torch
import torch.onnx
from torchvision.ops.deform_conv import DeformConv2d

import deform_conv2d_onnx_exporter

deform_conv2d_onnx_exporter.register_deform_conv2d_onnx_op()

def tonumpy(tensor):
    return tensor.to('cpu').detach().numpy().copy()

class DeformConv2dOpTestCase(unittest.TestCase):
    OPSET_VERSION = 12

    def create_input_params(self, dcn_params):
        size = [
            dcn_params["batch"],
            dcn_params["input_ch"],
            dcn_params["input_h"],
            dcn_params["input_w"],
        ]
        input = torch.rand(size, dtype=torch.float)

        size = [
            dcn_params["batch"],
            2 * dcn_params["n_offset_grps"] * dcn_params["kernel_h"] *
            dcn_params["kernel_w"],
            dcn_params["output_h"],
            dcn_params["output_w"],
        ]
        offset = torch.randn(size, dtype=torch.float) * dcn_params["kernel_w"]

        size = [
            dcn_params["batch"],
            dcn_params["n_offset_grps"] * dcn_params["kernel_h"] *
            dcn_params["kernel_w"],
            dcn_params["output_h"],
            dcn_params["output_w"],
        ]
        mask = torch.rand(size, dtype=torch.float)

        return input, offset, mask

    def create_pytorch_model(self, dcn_params):
        model = DeformConv2d(
            in_channels=dcn_params["input_ch"],
            out_channels=dcn_params["output_ch"],
            kernel_size=(dcn_params["kernel_h"], dcn_params["kernel_w"]),
            stride=(dcn_params["stride_h"], dcn_params["stride_w"]),
            padding=(dcn_params["padding_h"], dcn_params["padding_w"]),
            dilation=(dcn_params["dilation_h"], dcn_params["dilation_w"]),
            groups=dcn_params["groups"],
            bias=dcn_params["bias"],
        )
        return model

    def check_onnx_model(self, model_data):
        try:
            onnx.checker.check_model(model_data, full_check=True)
            self.assertTrue("Valid ONNX model")
        except onnx.checker.ValidationError as e:
            self.fail(f"Invalid ONNX model: {e}")
        except Exception as e:
            self.fail(f"Unknown exception: {e}")

    def convert_to_onnx_model(self, pytorch_model, input, offset, mask=None):
        if mask is not None:
            input_params = (input, offset, mask)
            input_names = ["input", "offset", "mask"]
        else:
            input_params = (input, offset)
            input_names = ["input", "offset"]
        buffer = io.BytesIO()
        torch.onnx.export(pytorch_model,
                          input_params,
                          buffer,
                          input_names=input_names,
                          output_names=["output"],
                          opset_version=self.OPSET_VERSION)
        onnx_model_data = buffer.getvalue()
        self.check_onnx_model(onnx_model_data)

        onnx_model = onnxruntime.InferenceSession(onnx_model_data)
        return onnx_model

    def run_pytorch_model(self, model, input, offset, mask=None):
        model.eval()
        return model(input, offset, mask)

    def run_onnx_model(self, model, input, offset, mask=None):
        input_params = {
            "input": tonumpy(input),
            "offset": tonumpy(offset),
        }
        if mask is not None:
            input_params["mask"] = tonumpy(mask)
        return model.run(["output"], input_params)[0]

    def run_with_dcn_params(self, dcn_params, message=""):
        input, offset, mask = self.create_input_params(dcn_params)
        pytorch_model = self.create_pytorch_model(dcn_params)
        if not dcn_params["use_mask"]:
            mask = None
        pytorch_output = self.run_pytorch_model(pytorch_model, input, offset,
                                                mask)
        onnx_model = self.convert_to_onnx_model(pytorch_model, input, offset,
                                                mask)
        onnx_output = self.run_onnx_model(onnx_model, input, offset, mask)

        return pytorch_output, onnx_output

    def assert_result(self, pytorch_result, onnx_result, message=""):
        pytorch_result = tonumpy(pytorch_result)
        self.assertTrue(
            np.allclose(pytorch_result, onnx_result, rtol=1e-03, atol=1e-05),
            message)

    def generate_dcn_parameters(self, base_dcn_params={}):
        dcn_params = {
            "batch": random.randrange(1, 6),
            # "input_ch": 0,
            "input_h": random.randrange(100, 201),
            "input_w": random.randrange(100, 201),
            # "output_ch": 0,
            # "output_h": 0,
            # "output_w": 0,
            "kernel_h": random.randrange(1, 8),
            "kernel_w": random.randrange(1, 8),
            "stride_h": random.randrange(1, 5),
            "stride_w": random.randrange(1, 5),
            "padding_h": random.randrange(0, 5),
            "padding_w": random.randrange(0, 5),
            "dilation_h": random.randrange(1, 4),
            "dilation_w": random.randrange(1, 4),
            "groups": random.randrange(1, 4),
            "n_offset_grps": random.randrange(1, 4),
            "bias": random.choice([True, False]),
            "use_mask": random.choice([True, False]),
        }
        dcn_params.update(base_dcn_params)

        if "input_ch" not in dcn_params:
            lcm = (dcn_params["groups"] * dcn_params["n_offset_grps"] //
                   math.gcd(dcn_params["groups"], dcn_params["n_offset_grps"]))
            dcn_params["input_ch"] = lcm * random.randrange(1, 17)
        if "output_ch" not in dcn_params:
            dcn_params["output_ch"] = dcn_params["groups"] * random.randrange(
                1, 17)
        ker_h = dcn_params["dilation_h"] * (dcn_params["kernel_h"] - 1) + 1
        if "output_h" not in dcn_params:
            dcn_params["output_h"] = (
                (dcn_params["input_h"] + 2 * dcn_params["padding_h"] - ker_h)
                // dcn_params["stride_h"]) + 1
        ker_w = dcn_params["dilation_w"] * (dcn_params["kernel_w"] - 1) + 1
        if "output_w" not in dcn_params:
            dcn_params["output_w"] = (
                (dcn_params["input_w"] + 2 * dcn_params["padding_w"] - ker_w)
                // dcn_params["stride_w"]) + 1
        return dcn_params

    def test_no_padding(self):
        dcn_params = {"padding_h": 0, "padding_w": 2}
        dcn_params = self.generate_dcn_parameters(dcn_params)
        pytorch_result, onnx_result = self.run_with_dcn_params(dcn_params)
        self.assert_result(pytorch_result, onnx_result,
                           f"no padding_h: {dcn_params}")

        dcn_params = {"padding_h": 1, "padding_w": 0}
        dcn_params = self.generate_dcn_parameters(dcn_params)
        pytorch_result, onnx_result = self.run_with_dcn_params(dcn_params)
        self.assert_result(pytorch_result, onnx_result,
                           f"no padding_w: {dcn_params}")

        dcn_params = {"padding_h": 0, "padding_w": 0}
        dcn_params = self.generate_dcn_parameters(dcn_params)
        pytorch_result, onnx_result = self.run_with_dcn_params(dcn_params)
        self.assert_result(pytorch_result, onnx_result,
                           f"no paddings: {dcn_params}")

    def test_full_parameters(self):
        dcn_params = {
            "batch": 8,
            "input_ch": 64,
            "input_h": 300,
            "input_w": 200,
            "output_w": 66,
            "kernel_h": 3,
            "kernel_w": 4,
            "stride_h": 2,
            "stride_w": 3,
            "padding_h": 0,
            "padding_w": 2,
            "dilation_h": 1,
            "dilation_w": 2,
            "groups": 2,
            "n_offset_grps": 2,
            "bias": True,
            "use_mask": True,
        }
        dcn_params = self.generate_dcn_parameters(dcn_params)
        pytorch_result, onnx_result = self.run_with_dcn_params(dcn_params)
        self.assert_result(pytorch_result, onnx_result,
                           f"full test: {dcn_params}")

    def test_random_parameters(self):
        test_count = 10
        for _ in range(test_count):
            dcn_params = self.generate_dcn_parameters()
            pytorch_result, onnx_result = self.run_with_dcn_params(dcn_params)
            self.assert_result(pytorch_result, onnx_result,
                               f"random parameters: {dcn_params}")

    def test_options_for_register_deform_conv2d_onnx_op(self):
        try:
            option_patterns = [
                (False, False),
                (True, False),
                (False, True),
                (True, True),
            ]
            for use_gathernd, enable_openvino_patch in option_patterns:
                deform_conv2d_onnx_exporter.register_deform_conv2d_onnx_op(
                    use_gathernd=use_gathernd,
                    enable_openvino_patch=enable_openvino_patch)
                dcn_params = self.generate_dcn_parameters()
                pytorch_result, onnx_result = self.run_with_dcn_params(
                    dcn_params)
                self.assert_result(pytorch_result, onnx_result,
                                   f"parameters: {dcn_params}")
        finally:
            deform_conv2d_onnx_exporter.register_deform_conv2d_onnx_op()


import itertools

def permutations(class_count = 3):
    return itertools.permutations([ i for i in range(1,class_count+1)])

def calc(A,perm,init_pos,k):
    best_recursive_solution = 0
    current_len = 0
    last = init_pos
    was_valid = True
    for class_index,current_class in enumerate(perm):
        total = 0
        last_last_position = 0
        for i in range(last,len(A)):
            if A[i] == current_class:
                total+=1
                last_last_position = last
                last = i
                if total == k:
                    break
        if total==k:
            val, is_valid = calc(A,perm[class_index+1:],last_last_position,k)
            if is_valid:
                best_recursive_solution = max(best_recursive_solution, total-1+val)
            current_len += total
        elif total==k-1:
            current_len+=total
        else:
            was_valid = False
            break

    return max(
        current_len if was_valid else 0,
        best_recursive_solution
    ), was_valid or best_recursive_solution !=0

def solution( A: list[int], class_count=3):
    k= len(A) // class_count +1
    better_solution = 0

    while k > 0:
        for p in permutations(class_count):
            sol,was_valid=calc(A,p,0,k)
            better_solution = max(sol,better_solution) if was_valid else better_solution
        k-=1
    return better_solution

if __name__ =='__main__':
    import tester
    #case = tester.gen_random_case(300,8)
    case = [i for i in range(1,9) for _ in range(10)]
    #print(case)
    #print(solution(case,8))
    print(solution(case,8))


import time
import network
import machine
from umqtt.robust import MQTTClient
import ubinascii
import ujson as json
import socket
import sys

last_send_time = time.time()

machineid = int.from_bytes(machine.unique_id(), 4)

print('init %d' % machineid)

ssid = 'Area51'
psk = 'badsector99'
CONFIG = {
    'broker': '13.250.41.251',
    'client_id': b'esp8266_' + ubinascii.hexlify(machine.unique_id())
    # 'client_id': machineid
}

# missing pins 6,7,8,11

analog_control_pins = [
    16, # D0 deepsleep
    5,  # D1
    4,  # D2
]

digital_out_pins = [
    0,  # D3 flash
    2,  # D4 tied to vcc
    14, # D5
    12, # D6
    13, # D7
    15, # D8 tied to gnd
    # 1,  # TX dont touch
    # 3,  # RX
    # 9,  # SD2
    # 10, # SD3
]

intervals = {
    'data': 5 * 60 * 60 * 1000
}

sta = network.WLAN(network.STA_IF)
sta.active(True)
sta.connect(ssid, psk)
while not sta.isconnected():
    pass

print('wifi connected')

try:
    socket.getaddrinfo('micropython.org', 80)
    print('internet connected')
except Exception as e:
    # no internet
    machine.reset()

client = MQTTClient(CONFIG['client_id'], CONFIG['broker'])
try:
    client.connect()
except Exception as e:
    # mqtt lib error
    machine.reset()

def read_from(pin):
    pin_val = '{0:03b}'.format(pin)
    for i in range(len(analog_control_pins)):
        machine.Pin(analog_control_pins[i], machine.Pin.OUT).value(int(pin_val[i]))
    return machine.ADC(0).read()

def check_data_send():
    global last_send_time
    # threshold = last_send_time + 5 * 60 * 60
    threshold = last_send_time + 5
    if time.time() > threshold:
        print('sending data')
        data = {}
        for i in range(8):
            data[str(i)] = read_from(i)
        print(data)
        payload = bytearray(json.dumps({'data':data,'type':'sensorvalue'}))
        client.publish('{}/{}'.format('/hydroots/client', machineid), payload)
        last_send_time = time.time()

def send_pin_status():
    data = {}
    for i in range(max(6, len(digital_out_pins))):
        data[str(i)] = machine.Pin(digital_out_pins[i]).value()
    payload = {'data':data,'type':'pinstatus'}
    client.publish('{}/{}'.format('/hydroots/client', machineid), bytearray(json.dumps(payload)))

def on_mqtt_msg(topic, msg):
    print('mqtt received')
    if msg == b'':
        print('empty received')
        return
    payload = {}
    try:
        payload = json.loads(msg)
    except Exception as e:
        print(msg)
        print('invalid received')
        return
    print(payload)

    if 'type' not in payload:
        print('unknown command')
        return

    if payload['type'] == 'pinstatus':
        send_pin_status()
    elif payload['type'] == 'changepin':
        if 'data' in payload and type(payload['data']) == dict:
            pins_changed = 0
            for pin in payload['data']:
                if type(pin) == str and not str.isdigit(pin): continue
                ipin = int(pin)
                if ipin < len(digital_out_pins):
                    pin_val = int(payload['data'][pin])
                    machine.Pin(digital_out_pins[ipin], machine.Pin.OUT).value(pin_val)
                    pins_changed += 1
            if pins_changed: send_pin_status()

send_pin_status()
client.set_callback(on_mqtt_msg)
channel = '{}/{}'.format('/hydroots/server', machineid)
client.subscribe(channel)
print('subscribed', channel)

if __name__ == '__main__':
    while True:
        print('looping')
        check_data_send()
        client.check_msg()
        time.sleep(1)


### ECB.py ###
# The purpose of this script is to scrape metadata from the most recent ECB working papers. This script uses
# the ECB website.
# Lorae Stojanovic
# Special thanks to ChatGPT for coding assistance in this project.
# LE: 1 September 2023

from bs4 import Tag, BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from datetime import datetime
import pandas as pd

def get_soup(url):
    # Create a new instance of the Firefox driver
    driver = webdriver.Firefox()

    # Go to the page
    driver.get(url)
    time.sleep(5)

    # Use WebDriverWait to wait for the element to become clickable
    try:
        element = WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.ID, "snippet1"))
        )
        # Scroll to the element
        driver.execute_script("arguments[0].scrollIntoView();", element)
    except Exception as e:
        print(f"An error occurred: {e}")
    time.sleep(5) # It still needs some extra time to load

    # Get the page source and parse it
    soup = BeautifulSoup(driver.page_source, 'html.parser')

    # Close the driver
    driver.quit()

    return soup

# I define the function "scrape" in every webscraper. That way, in runall.py, it is easy to call BOE.scrape()
# or NBER.scrape(), for instance, knowing that they all do the same thing - namely, navigate to their respective
# websites and extract the data.
def scrape():
    url = "https://www.ecb.europa.eu/pub/research/working-papers/html/index.en.html"

    soup = get_soup(url)

    elements = soup.find('dl', {'class': 'ecb-basicList wpSeries ecb-lazyload pub-list-filter'})

    # Filter so that we keep only the <dt> tags that have the 'isodate' attribute (if they do not
    # have this attribute, then they do not correspond with data we care about, so we discard them).
    filtered_dt_elements = [dt for dt in elements.find_all('dt') if dt.has_attr('isodate')]

    # Initialize lists to store data
    Date = []
    Title = []
    Author = []
    Abstract = []
    Link = []
    Number = []

    # Loop through the filtered <dt> and corresponding <dd> tags.  <dt> tags contain the date.
    # All other information is in the <dd> tags.
    for dt in filtered_dt_elements[:20]: # only selecting first 20 elements - we don't need them all
        dd = dt.find_next_sibling('dd')
        if not dd:
            continue

        # Date (from dt tag)
        date_div = dt.find('div', class_='date')
        date = date_div.text if date_div else "No date"
        Date.append(date)

        # Title
        title_div = dd.find('div', class_='title')
        title = title_div.text if title_div else "No title"
        Title.append(title)

        # Author
        author_list = dd.find_all('li')
        author = ', '.join([li.text for li in author_list]) if author_list else "No authors"
        Author.append(author)

        # Abstract
        abstract_dd = dd.find('dd')
        abstract = abstract_dd.text if abstract_dd else "No abstract"
        Abstract.append(abstract)

        # Link
        if title_div:
            link_a = title_div.find('a')
            link = "https://www.ecb.europa.eu"+ link_a['href'] if link_a else "No link"
        else:
            link = "No link"
        Link.append(link)

        # Number
        number_div = dd.find('div', {'class': 'category'})
        number = number_div.text.replace("No. ", "") if number_div else "No number"
        Number.append(number)

    # Create a dictionary of the six lists, where the keys are the column names.
    data = {'Title': Title,
            'Link': Link,
            'Date': Date,
            'Author': Author,
            'Number': Number,
            'Abstract': Abstract}

    # Create a DataFrame from the dictionary.
    df = pd.DataFrame(data)

    # Instead of the data frame having row names (indices) equalling 1, 2, etc,
    # we set them to be an identifier that is unique. In the case of Chicago, we combine
    # Chicago with the number of the paper (eg. 999) to get an identifier Chicago999 that
    # is completely unique across all papers scraped.
    df["Source"] = "ECB"
    df.index = df["Source"] + df['Number'].astype(str)
    df.index.name = None

    print(df)
    return(df)


from django.contrib import admin
from .models import (
    News,
    Category,
    NewsLikes,
    Comment,
    FavoriteUserNews,
)

# Register your models here.

class NewsAdmin(admin.ModelAdmin):
    list_display = (
        'id', 'title', 'description', 'created_at',
        'updated_at', 'is_public',
        'photo', 'category', 'owner', 'offer',
    )
    list_display_links = ('id', 'title')
    search_fields = ('id', 'title', 'owner')
    list_editable = ('is_public', 'offer')
    list_filter = ('is_public', 'category', 'offer')
    fields = (
        'title', 'description', 'category', 'content', 'photo',
        'is_public', 'created_at', 'updated_at', 'owner', 'liked',
        'offer', 'favorite_news',
    )
    readonly_fields = ('created_at',
                       'updated_at',)

class CategoryAdmin(admin.ModelAdmin):
    list_display_links = ('id', 'title',)
    list_display = ('id', 'title',)
    search_fields = ('id', 'title')
    list_filter = ('title',)

class NewsLikesAdmin(admin.ModelAdmin):
    list_display_links = ('id', 'user', 'news')
    list_display = ('id', 'user', 'news')
    search_fields = ('id', 'user', 'news')
    fields = (
        'user', 'news', 'value',
    )

class CommentAdmin(admin.ModelAdmin):
    list_display = (
        'id', 'user', 'news',
        'created_data', 'status'
    )
    list_display_links = ('id', 'news', 'user')
    search_fields = ('id', 'user', 'news')
    list_editable = ('status',)
    list_filter = ('status',)
    fields = (
        'user', 'news', 'text', 'status',
    )
    readonly_fields = ('created_data',)

class FavoriteUserNewsAdmin(admin.ModelAdmin):
    list_display = (
        'id', 'news', 'user',
    )
    list_display_links = ('id', 'news',)
    fields = (
        'news', 'user',
    )

admin.site.register(News, NewsAdmin)
admin.site.register(Comment, CommentAdmin)
admin.site.register(Category, CategoryAdmin)
admin.site.register(NewsLikes, NewsLikesAdmin)
admin.site.register(FavoriteUserNews, FavoriteUserNewsAdmin)
admin.site.site_title = 'Управление новостями'
admin.site.site_header = 'Управление новостями'


import logging
import sys
logger = logging.getLogger('root')
FORMAT = "[%(asctime)s] %(message)s"
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG, format=FORMAT)
logger.setLevel(logging.INFO)
logger.info('logger created')
logger.info('try to importing the others')
import argparse
import json
import os
import random
import time
import numpy as np
import torch
from warpctc_pytorch import CTCLoss

from loader.asr_data_loader import AudioDataLoader, SpectrogramDataset, BucketingSampler
from utility.decoder import GreedyDecoder
from utility import build_roberta_model
from models.ammodel import DeepSpeech, supported_rnns
from utility.util import check_loss
from models import CrossBert

parser = argparse.ArgumentParser(description='DeepSpeech training')
parser.add_argument('--train-manifest', metavar='DIR',
                    help='path to train manifest csv', default='data/train_manifest.csv')
parser.add_argument('--val-manifest', metavar='DIR',
                    help='path to validation manifest csv', default='data/val_manifest.csv')
parser.add_argument('--num-workers', default=4, type=int, help='Number of workers used in data-loading')
parser.add_argument('--epochs', default=70, type=int, help='Number of training epochs')
parser.add_argument('--momentum', default=0.9, type=float, help='momentum')
parser.add_argument('--silent', dest='silent', action='store_true', help='Turn off progress tracking per iteration')
parser.add_argument('--config', type=str)

def to_np(x):
    return x.cpu().numpy()

class AverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def evaluate(test_loader, device, model, decoder, target_decoder, save_output=False, verbose=True, half=False):
    model.eval()
    total_cer, total_wer, num_tokens, num_chars = 0, 0, 0, 0
    output_data = []
    for i, (data) in enumerate(test_loader):

        inputs, targets, input_percentages, target_sizes = data

        input_sizes = input_percentages.mul_(int(inputs.size(2))).int()
        inputs = inputs.type(torch.LongTensor).to(device)
        if half:
            inputs = inputs.half()
        # unflatten targets
        split_targets = []
        offset = 0
        for size in target_sizes:
            split_targets.append(targets[offset:offset + size])
            offset += size

        out, output_sizes = model(inputs, input_sizes)

        decoded_output, _ = decoder.decode(out, output_sizes)
        target_strings = target_decoder.convert_to_strings(split_targets)

        if save_output is not None:
            # add output to data array, and continue
            output_data.append((out.cpu().numpy(), output_sizes.numpy(), target_strings))
        for x in range(len(target_strings)):
            transcript, reference = decoded_output[x][0], target_strings[x][0]
            wer_inst = decoder.wer(transcript, reference)
            cer_inst = decoder.cer(transcript, reference)
            total_wer += wer_inst
            total_cer += cer_inst
            num_tokens += len(reference.split())
            num_chars += len(reference.replace(' ', ''))
            if verbose and i%300 == 0:
                logger.info('Ref:%s', reference.lower())
                logger.info('Hyp:%s', transcript.lower())
                logger.info('WER:%.4f\t'
                            'CER:%.4f',(float(wer_inst) / len(reference.split())), (float(cer_inst) / len(reference.replace(' ', ''))))
    wer = float(total_wer) / num_tokens
    cer = float(total_cer) / num_chars
    return wer * 100, cer * 100, output_data

if __name__ == '__main__':
    args = parser.parse_args()

    ##load config file
    with open(args.config, 'r') as config_file:
        config = json.load(config_file)

    # Set seeds for determinism
    torch.manual_seed(config['seed'])
    torch.cuda.manual_seed_all(config['seed'])
    np.random.seed(config['seed'])
    random.seed(config['seed'])

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    main_proc = True
    save_folder = config['save_folder']
    os.makedirs(save_folder, exist_ok=True)  # Ensure save folder exists
    loss_results, cer_results, wer_results = torch.Tensor(config['epochs']), torch.Tensor(config['epochs']), torch.Tensor(
        config['epochs'])
    best_cer = None
    avg_loss, start_epoch, start_iter = 0, 0, 0

    # vq_bert = RobertaModel.from_pretrained(config['vq-bert_dir'],
    #                                        checkpoint_file=config['vq-bert'])
    # text_bert = RobertaModel.from_pretrained(config['text-bert_dir'], checkpoint_file=config['text-bert'])

    vq_arg_path = 'configs/args/vq_roberta.args'
    vq_dict_path = config["vq_bert_dict"]
    text_arg_path = 'configs/args/text_roberta.args'
    text_dict_path = config["text_bert_dict"]
    vq_bert = build_roberta_model(vq_dict_path, vq_arg_path)
    text_bert = build_roberta_model(text_dict_path, text_arg_path)

    rnn_type = config['rnn-type'].lower()

    with open(config['label']) as label_file:
        labels = str(''.join(json.load(label_file)))

    premodel = CrossBert(vq_bert=vq_bert, text_bert=text_bert)
    model_dict = torch.load(config['pre_kd_model'])
    from collections import OrderedDict

    new_state_dict = OrderedDict()

    for k, v in model_dict.items():
        if 'module' in k:
            k = k.replace('module.', '')
        new_state_dict[k] = v
    premodel.load_state_dict(new_state_dict)
    model_dict = None
    new_state_dict = None
    del model_dict, new_state_dict

    premodel = premodel.to(device)
    model = DeepSpeech(rnn_hidden_size=config['hidden_size'],
                       nb_layers=config['hidden_layers'],
                       labels=labels,
                       rnn_type=supported_rnns[rnn_type],
                       bidirectional=True,
                       vq_bert=premodel)

    decoder = GreedyDecoder(labels)
    train_dataset = SpectrogramDataset(manifest_filepath=args.train_manifest, labels=labels, vq_bert=vq_bert)
    test_dataset = SpectrogramDataset(manifest_filepath=args.val_manifest, labels=labels, vq_bert=vq_bert)

    train_sampler = BucketingSampler(train_dataset, batch_size=config['batch_size'])
    train_loader = AudioDataLoader(train_dataset,
                                   num_workers=args.num_workers, batch_sampler=train_sampler)
    test_loader = AudioDataLoader(test_dataset, batch_size=config['batch_size'],
                                  num_workers=args.num_workers)
    model = torch.nn.DataParallel(model)
    model = model.to(device)
    parameters = model.parameters()
    optimizer = torch.optim.SGD(parameters, lr=config['lr'],
                                momentum=args.momentum, nesterov=True, weight_decay=1e-5)

    criterion = CTCLoss()
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()

    for epoch in range(start_epoch, config['epochs']):
        model.train()
        end = time.time()
        start_epoch_time = time.time()
        for i, (data) in enumerate(train_loader, start=start_iter):
            if i == len(train_sampler):
                break
            inputs, targets, input_percentages, target_sizes = data

            input_sizes = input_percentages.mul_(int(inputs.size(2))).int()
            # measure data loading time
            data_time.update(time.time() - end)
            inputs = inputs.type(torch.LongTensor).to(device)

            out, output_sizes = model(inputs, input_sizes)
            out = out.transpose(0, 1)  # TxNxH

            float_out = out.float()  # ensure float32 for loss
            loss = criterion(float_out, targets, output_sizes, target_sizes).to(device)
            loss = loss / inputs.size(0)  # average the loss by minibatch

            loss_value = loss.item()

            # Check to ensure valid loss was calculated
            valid_loss, error = check_loss(loss, loss_value)
            if valid_loss:
                optimizer.zero_grad()
                # compute gradient
                loss.backward()
                optimizer.step()
            else:
                logger.info(error)
                logger.info('Skipping grad update')
                loss_value = 0

            avg_loss += loss_value
            losses.update(loss_value, inputs.size(0))

            # measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()
            if not args.silent:
                logger.info('Epoch: [%d][%d/%d]\t'
                            'Time %.3f (%.3f)\t'
                            'Data %.3f (%.3f)\t'
                            'Loss %.4f (%.4f)\t',
                            (epoch + 1), (i + 1), len(train_sampler),
                            batch_time.val, batch_time.avg,
                            data_time.val, data_time.avg,
                            losses.val, losses.avg)

            del loss, out, float_out

        avg_loss /= len(train_sampler)

        epoch_time = time.time() - start_epoch_time
        logger.info('Training Summary Epoch: [%d]\t'
                    'Time taken (s): %.0f\t'
                    'Average Loss %.3f\t',
                    epoch + 1,
                    epoch_time,
                    avg_loss)

        start_iter = 0  # Reset start iteration for next epoch
        with torch.no_grad():
            wer, cer, output_data = evaluate(test_loader=test_loader,
                                             device=device,
                                             model=model,
                                             decoder=decoder,
                                             target_decoder=decoder)
        loss_results[epoch] = avg_loss
        wer_results[epoch] = wer
        cer_results[epoch] = cer
        logger.info('Validation Summary Epoch: [%d]\t'
                    'Average WER %.3f\t'
                    'Average CER %.3f\t',
                    epoch + 1,
                    wer,
                    cer)

        values = {
            'loss_results': loss_results,
            'cer_results': cer_results,
            'wer_results': wer_results
        }

        for g in optimizer.param_groups:
            g['lr'] = g['lr'] / config['learning_anneal']
        logger.info('Learning rate annealed to: %.6f', g['lr'])

        if main_proc and (best_cer is None or best_cer > cer):
            logger.info('Found better validated model, saving to %s', config['save_path'])
            torch.save(DeepSpeech.serialize(model, optimizer=optimizer, epoch=epoch, loss_results=loss_results,
                                            wer_results=wer_results, cer_results=cer_results)
                       , config['save_path'])
            best_cer = cer
            avg_loss = 0

        logger.info('Shuffling batches...')
        train_sampler.shuffle(epoch)


"""Redo the builtin repr() (representation) but with limits on most sizes."""

__all__ = ["Repr","repr"]

import __builtin__
import itertools

class Repr(object):

    def __init__(self):
        self.maxlevel = 6
        self.maxtuple = 6
        self.maxlist = 6
        self.maxarray = 5
        self.maxdict = 4
        self.maxset = 6
        self.maxfrozenset = 6
        self.maxdeque = 6
        self.maxstring = 30
        self.maxlong = 40
        self.maxother = 20

    def repr(self, x):
        return self.repr1(x, self.maxlevel)

    def repr1(self, x, level):
        typename = type(x).__name__
        if ' ' in typename:
            parts = typename.split()
            typename = '_'.join(parts)
        if hasattr(self, 'repr_' + typename):
            return getattr(self, 'repr_' + typename)(x, level)
        else:
            s = __builtin__.repr(x)
            if len(s) > self.maxother:
                i = max(0, (self.maxother-3)//2)
                j = max(0, self.maxother-3-i)
                s = s[:i] + '...' + s[len(s)-j:]
            return s

    def _repr_iterable(self, x, level, left, right, maxiter, trail=''):
        n = len(x)
        if level <= 0 and n:
            s = '...'
        else:
            newlevel = level - 1
            repr1 = self.repr1
            pieces = [repr1(elem, newlevel) for elem in itertools.islice(x, maxiter)]
            if n > maxiter:  pieces.append('...')
            s = ', '.join(pieces)
            if n == 1 and trail:  right = trail + right
        return '%s%s%s' % (left, s, right)

    def repr_tuple(self, x, level):
        return self._repr_iterable(x, level, '(', ')', self.maxtuple, ',')

    def repr_list(self, x, level):
        return self._repr_iterable(x, level, '[', ']', self.maxlist)

    def repr_array(self, x, level):
        header = "array('%s', [" % x.typecode
        return self._repr_iterable(x, level, header, '])', self.maxarray)

    def repr_set(self, x, level):
        x = _possibly_sorted(x)
        return self._repr_iterable(x, level, 'set([', '])', self.maxset)

    def repr_frozenset(self, x, level):
        x = _possibly_sorted(x)
        return self._repr_iterable(x, level, 'frozenset([', '])',
                                   self.maxfrozenset)

    def repr_deque(self, x, level):
        return self._repr_iterable(x, level, 'deque([', '])', self.maxdeque)

    def repr_dict(self, x, level):
        n = len(x)
        if n == 0: return '{}'
        if level <= 0: return '{...}'
        newlevel = level - 1
        repr1 = self.repr1
        pieces = []
        for key in itertools.islice(_possibly_sorted(x), self.maxdict):
            keyrepr = repr1(key, newlevel)
            valrepr = repr1(x[key], newlevel)
            pieces.append('%s: %s' % (keyrepr, valrepr))
        if n > self.maxdict: pieces.append('...')
        s = ', '.join(pieces)
        return '{%s}' % (s,)

    def repr_str(self, x, level):
        s = __builtin__.repr(x[:self.maxstring])
        if len(s) > self.maxstring:
            i = max(0, (self.maxstring-3)//2)
            j = max(0, self.maxstring-3-i)
            s = __builtin__.repr(x[:i] + x[len(x)-j:])
            s = s[:i] + '...' + s[len(s)-j:]
        return s

    def repr_long(self, x, level):
        s = __builtin__.repr(x) # XXX Hope this isn't too slow...
        if len(s) > self.maxlong:
            i = max(0, (self.maxlong-3)//2)
            j = max(0, self.maxlong-3-i)
            s = s[:i] + '...' + s[len(s)-j:]
        return s

    def repr_instance(self, x, level):
        try:
            s = __builtin__.repr(x)
            # Bugs in x.__repr__() can cause arbitrary
            # exceptions -- then make up something
        except Exception:
            return '<%s instance at %x>' % (x.__class__.__name__, id(x))
        if len(s) > self.maxstring:
            i = max(0, (self.maxstring-3)//2)
            j = max(0, self.maxstring-3-i)
            s = s[:i] + '...' + s[len(s)-j:]
        return s

def _possibly_sorted(x):
    # Since not all sequences of items can be sorted and comparison
    # functions may raise arbitrary exceptions, return an unsorted
    # sequence in that case.
    try:
        return sorted(x)
    except Exception:
        return list(x)

aRepr = Repr()
repr = aRepr.repr


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import json
import os
import threading
from collections import Counter
from collections import namedtuple
from datetime import datetime

import numpy as np
import paths
from vocabulary import Vocabulary

FEATURES_DEF = False
VERBOSE_DEF = True
VOCAB_FILE = 'vocab.txt'
TEST_SIZE = 20000
TRAIN_SIZE = 60000
MIN_WORD_DEFAULT = 4
ImageMetadata = namedtuple("ImageMetadata",
						   ["index", "img_url", "captions"])

def _process_metadata(captions_file):
	with open(captions_file, "r") as f:
		caption_data = json.load(f)

	image_metadata = []
	num_captions = 0

	for index, image in enumerate(caption_data):
		image_id = image[0]
		caption_list = image[1]
		for caption in caption_list:
			for i, word in enumerate(caption):
				caption[i] = word
		image_metadata.append(ImageMetadata(index, image_id, caption_list))
		num_captions += len(caption_list)

	print("Finished processing %d captions for %d images in %s" %
		  (num_captions, len(image_metadata), captions_file.split('/')[-1]))

	return image_metadata

def _create_vocab(captions):
	counter = Counter()
	for caption in captions:
		for word in caption:
			counter.update([word])

	print("Total words:", len(counter))

	# Filter uncommon words and sort by descending count.
	word_counts = [word for word in counter.items() if word[1] >= FLAGS.min_words]
	word_counts.sort(key=lambda x: x[1], reverse=True)
	print("Words in vocabulary:", len(word_counts))

	# Create the vocabulary dictionary.
	reverse_vocab = [item[0] for item in word_counts]
	unk_id = len(reverse_vocab)
	vocab_dict = dict([(x, y) for (y, x) in enumerate(reverse_vocab)])
	print(len(vocab_dict.keys()))
	reverse_vocab.append("#START#")
	reverse_vocab.append("#END#")
	vocab = Vocabulary(FLAGS.vocab_file, reverse_vocab, unk_id)

	return vocab

def _process_dataset(name, images, vocab, feature_file):
	"""Processes a complete data set and saves it as a TFRecord.
	Args:
	  name: Unique identifier specifying the dataset.
	  images: List of ImageMetadata.
	  vocab: A Vocabulary object.
	  num_shards: Integer number of shards for the output files.
	"""

	# Break up each image into a separate entity for each caption.
	images = [ImageMetadata(image.index, image.img_url, [caption])
			  for image in images for caption in image.captions]

	# Shuffle the ordering of images. Make the randomization repeatable.
	np.random.seed(42)
	np.random.shuffle(images)
	features = np.load(feature_file)

	dataset = []
	# Create a mechanism for monitoring when all threads are finished.
	j = 0
	k = 0
	for image in images:
		k += 1
		for caption in image.captions:
			encoded_caption = []
			for i, word in enumerate(caption):
				# Encode word to index
				encoded_caption.append(vocab.word_to_id(word))
			# Append encoded captions to
			encoded_caption = np.array(encoded_caption)
			input_vec = np.array([features[image.index], encoded_caption, image.img_url])
			j += 1
			if j % 100 == 0:
				print("%d/%d Appending to %s dataset image %d with url %s and caption %s" % (
				k + 1, len(images), name, image.index, image.img_url, str(caption)))
				j = 0

			dataset.append(input_vec)

	dataset = np.array(dataset)
	if name == 'training':
		size = TRAIN_SIZE
	else:
		size = TEST_SIZE
	indices = np.arange(dataset.shape[0])
	np.random.shuffle(indices)
	dataset = dataset[indices[0:size]]
	# Save processed data

	if not os.path.exists(paths.PROCESSED_FOLDER):
		os.makedirs(paths.PROCESSED_FOLDER)
	feature_file = str(feature_file).split('/')[-1]
	file_to_save = paths.PROCESSED_FOLDER + feature_file

	np.save(file_to_save, dataset)
	print("Final dataset size:\t%s" % (str(dataset.shape)), 'in ', file_to_save)

def initialize():
	'''
	Initialize the data.
	'''
	features = bool(FLAGS.features)

	print("----------------------------- Parsing targets -----------------------------")
	train_dataset = _process_metadata(paths.TRAIN_CAPTIONS_DEF)
	val_dataset = _process_metadata(paths.VALIDATION_CAPTIONS_DEF)

	if FLAGS.verbose:
		print("Training Dataset  length: %d" % (len(train_dataset)))
		print("Test     Dataset  length: %d" % (len(val_dataset)))

	print("--------------------------- Creating vocabulary ---------------------------")
	train_captions = [caption for image in train_dataset for caption in image.captions]
	vocab = _create_vocab(train_captions)

	if FLAGS.verbose:
		print("Vocabulary length:%d " % (len(vocab._vocab)))

	print("---------------------------- Parsing  features ----------------------------")
	_process_dataset('training', train_dataset, vocab, paths.TRAIN_FEATURES_DEF)
	_process_dataset('test', val_dataset, vocab, paths.VALIDATION_FEATURES_DEF)

def print_flags():
	"""
	Prints all entries in FLAGS variable.
	"""
	for key, value in vars(FLAGS).items():
		print(key + ' \t:\t ' + str(value))

def main(_):
	print("Welcome to our show and tell version")
	print_flags()
	initialize()

if __name__ == '__main__':
	# Command line arguments
	parser = argparse.ArgumentParser()
	parser.add_argument('--features', type=str, default=FEATURES_DEF, help='Parse features (true/false)')
	parser.add_argument('--verbose', type=str, default=VERBOSE_DEF, help='Show further output (true/false)')
	parser.add_argument("--vocab_file", type=str, default=VOCAB_FILE, help="Output vocabulary file of word counts.")
	parser.add_argument("--min_words", type=str, default=MIN_WORD_DEFAULT,
						help="The minimum number of occurrences of each word " +
							 "in the training set for inclusion in the vocabulary.")
	FLAGS, unparsed = parser.parse_known_args()
	main(None)


import boto3
from botocore.client import Config

from config import (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, BUCKET_NAME,
                    KEY_NAME, REGION_NAME, SIGNATURE_VERSION)

s3 = boto3.resource(
    "s3",
    aws_access_key_id=AWS_ACCESS_KEY_ID,
    aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
    config=Config(signature_version=SIGNATURE_VERSION),
    region_name=REGION_NAME,
)

multipart_uploads = s3.meta.client.list_multipart_uploads(
    Bucket=BUCKET_NAME,
)

aborted = []
print("Aborting", len(multipart_uploads), "uploads")
if "Uploads" in multipart_uploads:
    for u in multipart_uploads["Uploads"]:
        upload_id = u["UploadId"]
        aborted.append(
            s3.meta.client.abort_multipart_upload(
                Bucket=BUCKET_NAME, Key=KEY_NAME, UploadId=upload_id
            )
        )
print(aborted)