# -*- coding: utf-8 -*-
"""
Created on Fri Mar 02 15:36:19 2018

@author: zrssch
"""
from __future__ import division
from __future__ import print_function

import os
import sys
import utils.data_utils as data_utils
from seq2seq import Encoder, Decoder, Seq2Seq
import torch
from torch import optim
from torch.autograd import Variable
from torch.nn.utils import clip_grad_norm
from torch.nn import functional as F
import random
import numpy as np
import math
import pickle

sys.path.append('../utils')

def read_data(config, source_path, target_path, max_size=None):
    data_set = []
    source_file = open(source_path, 'r')
    target_file = open(target_path, 'r')
    source, target = source_file.readline(), target_file.readline()
    counter = 0
    while source and target and (not max_size or counter < max_size):
        counter += 1
        if counter % 100000 == 0:
            print("  reading disc_data line %d" % counter)
            sys.stdout.flush()
        source_ids = [int(x) for x in source.strip().split()]
        target_ids = [int(x) for x in target.strip().split()]
        target_ids.append(data_utils.EOS_ID)
        if max(len(source_ids), len(target_ids)) < config.maxlen:
            data_set.append([source_ids, target_ids])
        source, target = source_file.readline(), target_file.readline()
    source_file.close()
    target_file.close()
    return data_set

def prepare_data(gen_config):
    if os.path.exists('vocab') and os.path.exists('rev_vocab') and os.path.exists('dev_set') and os.path.exists('train_set'):
        fr_vocab = open('vocab','rb')
        fr_rev_vocab = open('rev_vocab','rb')
        fr_dev_set = open('dev_set','rb')
        fr_train_set = open('train_set','rb')
        vocab = pickle.load(fr_vocab)
        rev_vocab = pickle.load(fr_rev_vocab)
        dev_set = pickle.load(fr_dev_set)
        train_set = pickle.load(fr_train_set)
        fr_vocab.close()
        fr_rev_vocab.close()
        fr_dev_set.close()
        fr_train_set.close()
    else:
        train_path = os.path.join(gen_config.train_dir, "chitchat.train")
        voc_file_path = [train_path+".answer", train_path+".query"]
        vocab_path = os.path.join(gen_config.train_dir, "vocab%d.all" % gen_config.vocab_size)
        data_utils.create_vocabulary(vocab_path, voc_file_path, gen_config.vocab_size)
        vocab, rev_vocab = data_utils.initialize_vocabulary(vocab_path)

        print("Preparing Chitchat gen_data in %s" % gen_config.train_dir)
        train_query, train_answer, dev_query, dev_answer = data_utils.prepare_chitchat_data(
            gen_config.train_dir, vocab, gen_config.vocab_size)

        # Read disc_data into buckets and compute their sizes.
        print ("Reading development and training gen_data (limit: %d)."
                   % gen_config.max_train_data_size)
        dev_set = read_data(gen_config, dev_query, dev_answer)
        train_set = read_data(gen_config, train_query, train_answer, gen_config.max_train_data_size)

        fw_vocab = open('vocab','wb')
        fw_rev_vocab = open('rev_vocab','wb')
        fw_dev_set = open('dev_set','wb')
        fw_train_set = open('train_set','wb')
        pickle.dump(vocab, fw_vocab)
        pickle.dump(rev_vocab, fw_rev_vocab)
        pickle.dump(dev_set, fw_dev_set)
        pickle.dump(train_set, fw_train_set)
        fw_vocab.close()
        fw_rev_vocab.close()
        fw_dev_set.close()
        fw_train_set.close()
    return vocab, rev_vocab, dev_set, train_set

def prepare_data_new(gen_config):
    if os.path.exists('vocab') and os.path.exists('rev_vocab') and os.path.exists('dev_set') and os.path.exists('train_set'):
        fr_vocab = open('vocab','rb')
        fr_rev_vocab = open('rev_vocab','rb')
        fr_dev_set = open('dev_set','rb')
        fr_train_set = open('train_set','rb')
        vocab = pickle.load(fr_vocab)
        rev_vocab = pickle.load(fr_rev_vocab)
        dev_set = pickle.load(fr_dev_set)
        train_set = pickle.load(fr_train_set)
        fr_vocab.close()
        fr_rev_vocab.close()
        fr_dev_set.close()
        fr_train_set.close()
    else:
        '''
        a pair: the vocabulary (a dictionary mapping string to integers), and
        the reversed vocabulary (a list, which reverses the vocabulary mapping).
        '''
        train_set = []
        dev_set = []
        train_path = os.path.join(gen_config.train_dir, "t_given_s_dialogue_length2_6.txt")
        vocab_path = os.path.join(gen_config.train_dir, "movie_25000")
        fr1 = open(train_path, 'r')
        fr2 = open(vocab_path, 'r')
        line = fr1.readline()
        counter = 0
        while line:
            counter += 1
            if counter % 100000 == 0:
                print("  reading disc_data line %d" % counter)
                sys.stdout.flush()
            source_target = line.strip().split('|')
            source = source_target[0]
            target = source_target[1]
            source_ids = [int(x) for x in source.strip().split()]
            target_ids = [int(x) for x in target.strip().split()]
            target_ids.append(data_utils.EOS_ID)
            if max(len(source_ids), len(target_ids)) < gen_config.maxlen:
                if counter%100==0:
                    dev_set.append([source_ids, target_ids])
                else:
                    train_set.append([source_ids, target_ids])
            line = fr1.readline()
        fr1.close()
        print('read over')

        rev_vocab = []
        rev_vocab.extend(data_utils._START_VOCAB)
        rev_vocab.extend(fr2.readlines())
        rev_vocab.extend(data_utils._END_VOCAB)
        fr2.close()
        rev_vocab = [i.strip() for i in rev_vocab]
        vocab = dict([(x, y) for (y, x) in enumerate(rev_vocab)])
        print('write pickle start')
        fw_vocab = open('vocab','wb')
        fw_rev_vocab = open('rev_vocab','wb')
        fw_dev_set = open('dev_set','wb')
        fw_train_set = open('train_set','wb')
        pickle.dump(vocab, fw_vocab)
        pickle.dump(rev_vocab, fw_rev_vocab)
        pickle.dump(dev_set, fw_dev_set)
        pickle.dump(train_set, fw_train_set)
        fw_vocab.close()
        fw_rev_vocab.close()
        fw_dev_set.close()
        fw_train_set.close()
        print('write pickle done')
    return vocab, rev_vocab, dev_set, train_set

def create_model(gen_config):
    encoder = Encoder(gen_config.vocab_size, gen_config.emb_dim, gen_config.hidden_size, n_layers=2, dropout=0.5)
    decoder = Decoder(gen_config.emb_dim, gen_config.hidden_size, gen_config.vocab_size, n_layers=1, dropout=0.5)
    seq2seq = Seq2Seq(encoder, decoder).cuda()
    optimizer = optim.Adam(seq2seq.parameters(), lr=gen_config.lr)
    return seq2seq, optimizer

def getbatch(dataset, batch_size, maxlen):
    '''
    input:
    datase: [[source,target]...]
    output:
    encoder_inputs: double list, padded input and reversee, (batch_size, maxlen)
    decoder_inputs: double list, padded output and add 'go' flag, (batch_size, maxlen), ('go' and 'end')
    source_inputs: double list, origin input,
    target_inputs: double list, origin output, ('end')
    '''
    encoder_inputs = []
    decoder_inputs = []
    source_inputs = []
    target_inputs = []
    for i in range(batch_size):
        temp = random.choice(dataset)
        source = temp[0]
        source_inputs.append(source)
        target = temp[1]
        target_inputs.append(target)
        encoder_pad = [data_utils.PAD_ID] * (maxlen - len(source))
        encoder_inputs.append(list(reversed(source + encoder_pad)))
        decoder_pad = [data_utils.PAD_ID] * (maxlen - len(target) - 1)
        decoder_inputs.append([data_utils.GO_ID] + target + decoder_pad)
    return encoder_inputs, decoder_inputs, source_inputs, target_inputs

def train_with_reward(gen_config, seq2seq, optim_seq2seq, reward, train_query_neg, train_answer_gen):
    seq2seq.train()
    reward = reward*(gen_config.maxlen-1)
    src = torch.from_numpy(np.array(train_query_neg).T)
    trg = torch.from_numpy(np.array(train_answer_gen).T)
    src = Variable(src).cuda()
    trg = Variable(trg).cuda()
    optim_seq2seq.zero_grad()
    output = seq2seq(src, trg)
    loss = F.cross_entropy(output[1:].view(-1, gen_config.vocab_size),
                               trg[1:].contiguous().view(-1),
                               ignore_index=data_utils.PAD_ID, reduce=False)
    print('train with reward loss is:', loss)
    # print('len reward:', len(reward))
    loss_reward = torch.dot(Variable(torch.Tensor(reward)).cuda(), loss)/len(reward)
    loss_reward.backward()
    clip_grad_norm(seq2seq.parameters(), gen_config.grad_clip)
    optim_seq2seq.step()
    return loss_reward.data[0]

def teacher_forcing(gen_config, seq2seq, optim_seq2seq, encoder_inputs, decoder_inputs):
    seq2seq.train()
    src = torch.from_numpy(np.array(encoder_inputs).T)
    trg = torch.from_numpy(np.array(decoder_inputs).T)
    src = Variable(src).cuda()
    trg = Variable(trg).cuda()
    optim_seq2seq.zero_grad()
    output = seq2seq(src, trg)
    loss = F.cross_entropy(output[1:].view(-1, gen_config.vocab_size),
                               trg[1:].contiguous().view(-1),
                               ignore_index=data_utils.PAD_ID)
    print('teacher forcing loss is: ', loss)
    loss.backward()
    clip_grad_norm(seq2seq.parameters(), gen_config.grad_clip)
    optim_seq2seq.step()
    return loss.data[0]

def train(gen_config):
    print("begin pretrain gen model ...")
    vocab, rev_vocab, dev_set, train_set = prepare_data_new(gen_config)
    #seqq2seq model
    seq2seq, optimizer = create_model(gen_config)
    seq2seq.train()
    pad = data_utils.PAD_ID
    total_loss = 0
    b = 0
    while True:
        encoder_inputs, decoder_inputs, source_inputs, target_inputs = getbatch(train_set, gen_config.batch_size, gen_config.maxlen)
        b += 1
        src = torch.from_numpy(np.array(encoder_inputs).T)
        trg = torch.from_numpy(np.array(decoder_inputs).T)
        src = Variable(src).cuda()
        trg = Variable(trg).cuda()
        # src, trg = src.cuda(), trg.cuda()
        optimizer.zero_grad()
        output = seq2seq(src, trg)
        loss = F.cross_entropy(output[1:].view(-1, gen_config.vocab_size),
                               trg[1:].contiguous().view(-1),
                               ignore_index=pad)
        loss.backward()
        clip_grad_norm(seq2seq.parameters(), gen_config.grad_clip)
        optimizer.step()

        total_loss += loss.data[0]
        print("single step loss", loss.data[0])
        if b % 100 == 0 and b != 0:
            total_loss = total_loss / 100
            print("[%d][loss:%5.2f][pp:%5.2f]" %
                  (b, total_loss, math.exp(total_loss)))
            total_loss = 0
        if b % 1000 == 0:
            seq2seq.eval()
            total_val_loss = 0
            for i in range(100):
                val_encoder_inputs, val_decoder_inputs, val_source_inputs, val_target_inputs = getbatch(train_set, gen_config.batch_size, gen_config.maxlen)
                val_src = torch.from_numpy(np.array(val_encoder_inputs).T)
                val_trg = torch.from_numpy(np.array(val_decoder_inputs).T)
                val_src = Variable(val_src).cuda()
                val_trg = Variable(val_trg).cuda()
                # val_src, val_trg = val_src.cuda(), val_trg.cuda()
                val_output = seq2seq(val_src, val_trg)
                val_loss = F.cross_entropy(val_output[1:].view(-1, gen_config.vocab_size),
                               val_trg[1:].contiguous().view(-1),
                               ignore_index=pad)
                total_val_loss += val_loss.data[0]
            total_val_loss = total_val_loss/100.0
            print("[%d][val_loss:%5.2f][pp:%5.2f]" %
                  (b//1000, total_val_loss, math.exp(total_val_loss)))
            seq2seq.train()
            torch.save(seq2seq, './pre_seq2seq.pth')
            print("save model at step %d"%b)

def test_decoder(gen_config):
    # vocab_path = os.path.join(gen_config.train_dir, "vocab%d.all" % gen_config.vocab_size)
    # vocab, rev_vocab = data_utils.initialize_vocabulary(vocab_path)
    fr_vocab = open('vocab','rb')
    fr_rev_vocab = open('rev_vocab','rb')
    vocab = pickle.load(fr_vocab)
    rev_vocab = pickle.load(fr_rev_vocab)
    fr_vocab.close()
    fr_rev_vocab.close()
    # seq2seq, optimizer = create_model(gen_config, vocab)
    seq2seq = torch.load('./pre_seq2seq.pth')
    sys.stdout.write("> ")
    sys.stdout.flush()
    sentence = sys.stdin.readline()
    while sentence:
        source = data_utils.sentence_to_token_ids(sentence, vocab) # list
        encoder_pad = [data_utils.PAD_ID] * (gen_config.maxlen - len(source))
        encoder_inputs = [list(reversed(source + encoder_pad))]
        src = torch.from_numpy(np.array(encoder_inputs).T)
        src = Variable(src).cuda()
        probs, result = seq2seq.decode(src) # maxlen*1*vocab, a list
        ans = []
        for idx in result:
            if idx == data_utils.EOS_ID:
                break
            if idx != data_utils.PAD_ID:
                ans.append(rev_vocab[idx])
        print(" ".join(ans))
        print("> ", end="")
        sys.stdout.flush()
        sentence = sys.stdin.readline()

def gen_disc_data(gen_config):
    print("start gen disc data ...")
    vocab, rev_vocab, dev_set, train_set = prepare_data(gen_config)
    disc_train_query = open("train.query", "w")
    disc_train_answer = open("train.answer", "w")
    disc_train_gen = open("train.gen", "w")
    seq2seq  = torch.load('./pre_seq2seq.pth')
    num_steps = 0
    while num_steps<10000:
        encoder_inputs, decoder_inputs, source_inputs, target_inputs = getbatch(train_set, gen_config.batch_size, gen_config.maxlen)
        src = torch.from_numpy(np.array(encoder_inputs).T)
        trg = torch.from_numpy(np.array(decoder_inputs).T)
        src = Variable(src).cuda()
        trg = Variable(trg).cuda()
        probs, result = seq2seq.gen_data(src, trg)
        result = [list(i) for i in result]
        result = np.array(result).T.tolist()
        resps = []
        for each in result:
            temp = []
            for idx in each:
                if idx == data_utils.EOS_ID:
                    break
                if idx != data_utils.PAD_ID:
                    temp.append(idx)
            resps.append(temp)
        for query, answer, resp in zip(source_inputs, target_inputs, resps):
            answer_str = " ".join([str(rev_vocab[an]) for an in answer][:-1])
            disc_train_answer.write(answer_str)
            disc_train_answer.write("\n")

            query_str = " ".join([str(rev_vocab[qu]) for qu in query])
            disc_train_query.write(query_str)
            disc_train_query.write("\n")

            resp_str = " ".join([str(rev_vocab[output]) for output in resp])

            disc_train_gen.write(resp_str)
            disc_train_gen.write("\n")
        num_steps += 1
        if num_steps%5==0:
            print("generate disc data at %d step"%num_steps)
    disc_train_gen.close()
    disc_train_query.close()
    disc_train_answer.close()
    print("end gen disc data.")


from utils import read_file

def get_views(pos, grid):
    x, y = pos

    right = grid[x][y + 1 :]
    left = grid[x][:y][::-1]

    grid_t = list(map(list, zip(*grid)))
    up = grid_t[y][:x][::-1]
    down = grid_t[y][x + 1 :]

    return up, down, left, right

def is_visible(pos, grid):
    x, y = pos
    for v in get_views(pos, grid):
        if max(v) < grid[x][y]:
            return True

    return False

def takeuntil(pred, lst):
    for l in lst:
        yield l
        if pred(l):
            break

def visible_score(pos, grid):
    x, y = pos

    pred = lambda h: h >= grid[x][y]
    score = 1
    for v in get_views(pos, grid):
        nvis = len(list(takeuntil(pred, v)))
        score *= nvis

    return score

def check_interiors(grid):
    bounds = ((1, len(grid[0]) - 1), (1, len(grid) - 1))

    count = 0
    for x in range(*bounds[0]):
        for y in range(*bounds[1]):
            if is_visible((x, y), grid):
                count += 1

    return count

def view_interiors(grid):
    bounds = ((1, len(grid[0]) - 1), (1, len(grid) - 1))

    return [
        visible_score((x, y), grid)
        for x in range(*bounds[0])
        for y in range(*bounds[1])
    ]

if __name__ == "__main__":
    grid = [list(map(int, x)) for x in read_file("data/08.txt")[:-1]]

    print(check_interiors(grid) + len(grid) * 2 + (len(grid) - 2) * 2)

    print(max(view_interiors(grid)))


'''
Created on Aug 5, 2020

@author: gsq
'''

import sys
import time
import math
from shutil import get_terminal_size

from scipy.ndimage import filters as filters
from scipy.signal import convolve2d
import cv2
import numpy as np
import tensorflow as tf

def generate_gaussian_kernel(k, sigma1=1.6, sigma2=1.6, angle=0):
    """Generate Gaussian kernel`.

    Args:
        k (int): Kernel size.
        sigma1 (float): the first deviation of the Gaussian kernel. Default: 1.6.
        sigma2 (float): the second deviation of the Gaussian kernel. Default: 1.6.
        angle (int): the rotation angle between 0 to 180
    Returns:
        np.array: The Gaussian kernel.
    """
    kernel = np.zeros((k, k))
    # set element at the middle to one, a dirac delta
    kernel[k // 2, k // 2] = 1
    # gaussian-smooth the dirac, resulting in a gaussian filter
    kernel = filters.gaussian_filter(kernel, (sigma1, sigma2))
    kernel = cv2.warpAffine(kernel, cv2.getRotationMatrix2D((k / 2 - 0.5, k / 2 - 0.5), angle, 1.0), (k, k))
    return kernel / kernel.sum()

def generate_simple_motionblur_kernel(k, l=13, angle=0):
    """Generate motion kernel`.

    Args:
        k (int): Kernel size.
        l (int): the length of motion kernel, ranges in [5, k]
        angle (int): the rotation angle between 0 to 180
    Returns:
        np.array: The motion kernel.
    """
    kernel = np.zeros((k, k))
    kernel[(k - 1) // 2, (k-l) // 2:(k - l) // 2 + l] = np.ones(l, dtype=np.float32)
    kernel = cv2.warpAffine(kernel, cv2.getRotationMatrix2D((k / 2 - 0.5, k / 2 - 0.5), angle, 1.0), (k, k))
    return kernel / kernel.sum()

def random_gaussian_kernel(k, scale, minval=0.175, maxval=1.0, minangle=0, maxangle=180):
    """generate random gaussian kernel
    Args:
        k (int): Kernel size.
        scale (int): upscaling factor
        minval (float): the minimal value of sigma
        maxval (float): the maximal value of sigma
        minangle (int): the minimal angle of rotation
        maxangle (int): the maximal angle of rotation
    Returns:
        np.array: a random Gaussian kernel.
    """
    sigma1 = np.random.uniform(scale * minval, scale * maxval)
    sigma2 = np.random.uniform(scale * minval, sigma1)
    angle = np.random.randint(minangle, maxangle)
    kernel = generate_gaussian_kernel(k, sigma1, sigma2, angle)
    return kernel

def random_simple_motionblur_kernel(k, scale, minlen=1, maxlen=5, minangle=0, maxangle=180):
    """generate random motion blur kernel
    Args:
        k (int): Kernel size..
        minlen (int): the minimal length of motion
        maxlen (int): the maximal length of motion
        minangle (int): the minimal angle of rotation
        maxangle (int): the maximal angle of rotation
    Returns:
        np.array: a random motion blur kernel.
    """
    assert k >= scale * maxlen
    length = np.random.randint(scale * minlen, scale * maxlen)
    angle = np.random.randint(minangle, maxangle)
    kernel = generate_simple_motionblur_kernel(k, length, angle)
    return kernel

#----------------------------------------------------------------------------------------------
def random_general_motionblur_kernel(h, w=None):
    """generate random motion blur kernel follows:
       Boracchi et al, Modeling the performance of image restoration from motion blur, TIP 2012.
    Args:
        h (int): height of kernel
        w (int): width of kernel
    Returns:
        k (np.array): a random motion blur kernel of size [h, w].
    """
    w = h if w is None else w
    kdims = [h, w]
    x = randomTrajectory(250)
    k = None
    while k is None:
        k = kernelFromTrajectory(x)

    # center pad to kdims
    pad_width = ((kdims[0] - k.shape[0]) // 2, (kdims[1] - k.shape[1]) // 2)
    pad_width = [(pad_width[0],), (pad_width[1],)]

    if pad_width[0][0] < 0 or pad_width[1][0] < 0:
        k = k[0 : h, 0 : h]
    else:
        k = np.pad(k, pad_width, "constant")
    x1, x2 = k.shape
    if np.random.randint(0, 4) == 1:
        k = cv2.resize(k, (np.random.randint(x1, 5*x1), np.random.randint(x2, 5*x2)), interpolation=cv2.INTER_LINEAR)
        y1, y2 = k.shape
        k = k[(y1 - x1) // 2: (y1 - x1) // 2 + x1, (y2 - x2) // 2: (y2 - x2) // 2 + x2]

    if np.sum(k) < 0.1:
        k = fspecial_gauss(h, 0.1 + 6 * np.random.rand(1))
    k = k / np.sum(k)
    return k

def fspecial_gauss(size, sigma):
    """generate an isotropic gaussian kernel
    Args:
        size (int): kernel size
        sigma (float): standard deviation
    Returns:
        np.array: a gaussian kernel
    """
    x, y = np.mgrid[-size // 2 + 1 : size // 2 + 1, -size // 2 + 1 : size // 2 + 1]
    g = np.exp(-((x ** 2 + y ** 2) / (2.0 * sigma ** 2)))
    return g / g.sum()

def kernelFromTrajectory(x):
    """generate motion blur kernel from given trajectory
    Args:
       x (np.array): given trajectory
    Returns:
       np.array: a motion blur kernel
    """
    h = 5 - np.log(np.random.rand()) / 0.15
    h = np.round(np.min([h, 27])).astype(int)
    h = h + 1 - h % 2
    w = h
    k = np.zeros((h, w))

    xmin = np.min(x[0])
    xmax = np.max(x[0])
    ymin = np.min(x[1])
    ymax = np.max(x[1])
    xthr = np.arange(xmin, xmax, (xmax - xmin) / w)
    ythr = np.arange(ymin, ymax, (ymax - ymin) / h)

    for i in range(1, xthr.size):
        for j in range(1, ythr.size):
            idx = (
                (x[0, :] >= xthr[i - 1])
                & (x[0, :] < xthr[i])
                & (x[1, :] >= ythr[j - 1])
                & (x[1, :] < ythr[j])
            )
            k[i - 1, j - 1] = np.sum(idx)
    if np.sum(k) == 0:
        return
    k = k / np.sum(k)
    k = convolve2d(k, fspecial_gauss(3, 1), "same")
    k = k / np.sum(k)
    return k

def randomTrajectory(T):
    """generate random trajactory
    Args:
       T (int): length of trajectory
    Returns:
       np.array: a random trajectory
    """
    x = np.zeros((3, T))
    v = np.random.randn(3, T)
    r = np.zeros((3, T))
    trv = 1 / 1
    trr = 2 * np.pi / T
    for t in range(1, T):
        F_rot = np.random.randn(3) / (t + 1) + r[:, t - 1]
        F_trans = np.random.randn(3) / (t + 1)
        r[:, t] = r[:, t - 1] + trr * F_rot
        v[:, t] = v[:, t - 1] + trv * F_trans
        st = v[:, t]
        st = rot3D(st, r[:, t])
        x[:, t] = x[:, t - 1] + st
    return x

def rot3D(x, r):
    """perform 3D rotation
    Args:
        x (np.array): input data
        r (float): rotation angle
    Returns:
        np.array: data after rotation
    """
    Rx = np.array([[1, 0, 0], [0, math.cos(r[0]), -math.sin(r[0])], [0, math.sin(r[0]), math.cos(r[0])]])
    Ry = np.array([[math.cos(r[1]), 0, math.sin(r[1])], [0, 1, 0], [-math.sin(r[1]), 0, math.cos(r[1])]])
    Rz = np.array([[math.cos(r[2]), -math.sin(r[2]), 0], [math.sin(r[2]), math.cos(r[2]), 0], [0, 0, 1]])
    R = Rz @ Ry @ Rx
    x = R @ x
    return x
#----------------------------------------------------------------------------------------

def generate_random_kernel(k, gauss=False, motion=False, scale=2):
    """generate random blur kernel
    Args:
        k (int): Kernel size.
        gauss (bool): whether to generate gaussian kernel.
        motion (bool): whether to generate motion blur kernel.
        scale (int): upscaling factor
    Returns:
        np.array: a random kernel.
    """
    if not gauss and not motion:
        kernel = np.zeros([k, k])
        kernel[k // 2, k // 2] = 1.
    elif gauss and motion:
        randnum = np.random.random()
        if randnum < 0.33:
            kernel = random_gaussian_kernel(k, scale)
        elif randnum > 0.66:
            kernel = random_simple_motionblur_kernel(k, scale)
        else:
            kernel = random_general_motionblur_kernel(k)
    elif gauss and not motion:
        kernel = random_gaussian_kernel(k, scale)
    else:
        kernel = random_motionblur_kernel(k, scale)
    return np.expand_dims(kernel.astype(np.float32), axis=2)

def generate_batch_random_kernel(k, b, gauss=False, motion=False, scale=2):
    """generate a batch of random blur kernels
    Args:
        k (int): Kernel size.
        b (int): batch size.
        gauss (bool): whether to generate gaussian kernel.
        motion (bool): whether to generate motion blur kernel.
        scale (int): upscaling factor
    Returns:
        np.array: a batch of random kernels.
    """
    batch = [generate_random_kernel(k, gauss, motion, scale) for _ in range(b)]
    return np.stack(batch)

def generate_read_shot_noise(narray, read=10, shot=0, range_in=False):
    """generate gaussian noise
    Args:
        narray (np.float32): input numpy array.
        read (float): read noise level.
        shot (float): shot noise leve.
        range_in (bool): whether to range betwen 0 and noise level
    Returns:
        np.array: a gaussian noise.
    """
    if range_in:
        read = np.random.uniform(0, read)
        shot = np.random.uniform(0, shot)
    sigma = np.sqrt((read / 255.)**2 + (shot / 255.) * narray)
    noise = np.random.normal(size=narray.shape) * sigma
    return noise

def tensor_degradation(tensor, kernel, scale=2, padding=False):
    """blurring and downscaling tensor
    Args:
        tensor (tf.float32): input image of shape [h, w, c].
        kernel (tf.float32): blurring kernel of shape [k, k, 1].
        scale (int): downscaling factor.
    Returns:
        tf.float32: degraded image.
    """
    tensor = tf.expand_dims(tf.transpose(tensor, [2, 0, 1]), 3)
    weight = tf.expand_dims(kernel, 3)
    pad = kernel.shape[0] // 2 if padding else 0
    tensor = tf.pad(tensor, paddings=[[0,0], [pad,pad], [pad,pad], [0,0]], mode='SYMMETRIC')
    tensor = tf.nn.conv2d(tensor, weight, strides=(1, scale, scale, 1), padding='VALID', data_format='NHWC')
    tensor = tf.transpose(tensor[..., 0], [1, 2, 0])
    return tensor

def tensor_batch_degradation(tensor, kernel, scale=2, padding=False):
    """blurring and downscaling tensor
    Args:
        tensor (tf.float32): a batch of images of shape [b, h, w, c].
        kernel (tf.float32): blurring kernel of shape [b, k, k, 1].
        scale (int): downscaling factor.
    Returns:
        tf.float32: a batch of degraded images.
    """
    b, h, w, c = tensor.shape.as_list()
    batch = [tensor_degradation(tensor[i], kernel[i], scale, padding) for i in range(b)]
    return tf.stack(batch)

class ProgressBar(object):
    """A progress bar which can print the progress.

    Modified from:
    https://github.com/hellock/cvbase/blob/master/cvbase/progress.py
    """

    def __init__(self, task_num=0, bar_width=50, start=True):
        self.task_num = task_num
        max_bar_width = self._get_max_bar_width()
        self.bar_width = (
            bar_width if bar_width <= max_bar_width else max_bar_width)
        self.completed = 0
        if start:
            self.start()

    def _get_max_bar_width(self):
        terminal_width, _ = get_terminal_size()
        max_bar_width = min(int(terminal_width * 0.6), terminal_width - 50)
        if max_bar_width < 10:
            print(f'terminal width is too small ({terminal_width}), '
                  'please consider widen the terminal for better '
                  'progressbar visualization')
            max_bar_width = 10
        return max_bar_width

    def start(self):
        if self.task_num > 0:
            sys.stdout.write(f"[{' ' * self.bar_width}] 0/{self.task_num}, "
                             f'elapsed: 0s, ETA:\nStart...\n')
        else:
            sys.stdout.write('completed: 0, elapsed: 0s')
        sys.stdout.flush()
        self.start_time = time.time()

    def update(self, msg='In progress...'):
        self.completed += 1
        elapsed = time.time() - self.start_time + 1e-8
        fps = self.completed / elapsed
        if self.task_num > 0:
            percentage = self.completed / float(self.task_num)
            eta = int(elapsed * (1 - percentage) / percentage + 0.5)
            mark_width = int(self.bar_width * percentage)
            bar_chars = '>' * mark_width + '-' * (self.bar_width - mark_width)
            sys.stdout.write('\033[2F')  # cursor up 2 lines
            sys.stdout.write(
                '\033[J'
            )  # clean the output (remove extra chars since last display)
            sys.stdout.write(
                f'[{bar_chars}] {self.completed}/{self.task_num}, '
                f'{fps:.1f} task/s, elapsed: {int(elapsed + 0.5)}s, '
                f'ETA: {eta:5}s\n{msg}\n')
        else:
            sys.stdout.write(
                f'completed: {self.completed}, elapsed: {int(elapsed + 0.5)}s, '
                f'{fps:.1f} tasks/s')
        sys.stdout.flush()


from pydub import AudioSegment
from tqdm import tqdm
import os

def create_translated_audio(result_diarize, audio_files, Output_name_file):
    total_duration = result_diarize['segments'][-1]['end'] # in seconds

    # silent audio with total_duration
    combined_audio = AudioSegment.silent(duration=int(total_duration * 1000))
    print(round((total_duration / 60),2), 'minutes of video')

    for line, audio_file in tqdm(zip(result_diarize['segments'], audio_files)):
        start = float(line['start'])

        # Overlay each audio at the corresponding time
        try:
          audio = AudioSegment.from_file(audio_file)
          ###audio_a = audio.speedup(playback_speed=1.5)
          start_time = start * 1000  # to ms
          combined_audio = combined_audio.overlay(audio, position=start_time)
        except:
          print(f'ERROR AUDIO FILE {audio_file}')

    os.system("rm -rf audio/*")

    # combined audio as a file
    combined_audio.export(Output_name_file, format="wav") # best than ogg, change if the audio is anomalous


import requests
import pandas as pd
import numpy as np
import datetime as dt
from pandas_datareader import data as pdr

def backtest():
    pair = input("Enter a pair symbol: ")

    finalDate = dt.datetime.now()
    initialDate = finalDate - pd.DateOffset(months=1)

    URL = "https://fxmarketapi.com/apipandas"
    params = {'currency' : pair,
    'start_date' : initialDate.strftime("%Y-%m-%d"),
    'end_date': finalDate.strftime("%Y-%m-%d"),
    'api_key':'tpLtcSkwFVK_JUbJbYlT',
    'interval':'hourly',
    'format':'ohlc'
    }

    response = requests.get("https://fxmarketapi.com/apipandas", params=params)
    df= pd.read_json(response.text)

    for i in df.index:
        time = dt.datetime.strptime(str(i), '%Y-%m-%d %H:%M:%S')

        if(str(time.hour) == '6'):
            high = 0
            low = 0
            high = df.iloc[:,2][i]
            low = df.iloc[:,3][i]
        elif(str(time.hour) == '7'):
            if(float(high) < float(df.iloc[:,2][i])):
                high = df.iloc[:,2][i]
            if(float(low) > float(df.iloc[:,3][i])):
                low = df.iloc[:,3][i]

            print('High: ' + str(high))
            print('Low: ' + str(low))
            print('Datetime: ' + str(time.day) + '-' + str(time.month))

backtest()


#참고 : https://velog.io/@waoderboy/알고리즘-백준-16236-아기상어-파이썬
from collections import deque
import sys

n = int(sys.stdin.readline())
graph = [list(map(int, sys.stdin.readline().split())) for _ in range(n)]
x,y = 0,0
cnt,t = 0,0
move = ((1,0),(-1,0),(0,1),(0,-1))
size = 2
for i in range(n):
    for j in range(n):
        if graph[i][j] == 9:
            x,y = j, i
            graph[y][x] = 0

def bfs(x, y):
    visited = [[0]*n for _ in range(n)]
    queue = deque([(x,y)])
    fishs = []
    visited[y][x] = 1

    while queue:
        x, y = queue.popleft()

        for dx,dy in move:
            nx = x + dx
            ny = y + dy

            if 0<=nx<n and 0<=ny<n and visited[ny][nx] == 0:
                if size > graph[ny][nx] and graph[ny][nx] != 0:
                    visited[ny][nx] =  visited[y][x] + 1
                    fishs.append((nx, ny,visited[ny][nx] - 1))
                elif size == graph[ny][nx] or graph[ny][nx] == 0:
                    visited[ny][nx] =  visited[y][x] + 1
                    queue.append((nx,ny))

    return sorted(fishs, key = lambda x: (x[2], x[1], x[0]))

while True:
    fishs = deque(bfs(x,y))

    if not fishs:
        break

    x, y, dist = fishs.popleft()
    t += dist
    cnt += 1
    if size == cnt:
        size += 1
        cnt = 0

    graph[y][x] = 0

print(t)


from fastapi import FastAPI, Request, status
from fastapi.middleware.cors import CORSMiddleware
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from typing import Union
from pydantic import BaseModel
import requests
import random
from package1 import poke_api

class Quiz(BaseModel):
    name: str
    ans_img: str
    shadow_img: str

app = FastAPI()
URL = "https://pokeapi.co/api/v2/pokemon/"

origins = [
    "http://localhost:5173",
    "http://127.0.0.1:5173",
    "http://localhost:8080",
]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers={"*"},
)

@app.exception_handler(RequestValidationError)
async def handler(request:Request, exc:RequestValidationError):
    print(exc)
    return JSONResponse(content={}, status_code=status.HTTP_422_UNPROCESSABLE_ENTITY)

@app.get("/")
def hello():
    return {
        "message": "Hello World!"
    }

@app.get("/{id}")
def name(id: int):
    res = requests.get(URL+str(id))
    name = poke_api.get_name(res)
    return {
        "name": name
    }

@app.get("/quiz/")
def quiz():
    id: int = random.randint(1,1010)
    res = requests.get(URL+str(id))
    name = poke_api.get_name(res)
    ans_img = poke_api.get_image(res)
    ans_img_encoded = poke_api.decode_image(ans_img)
    shadow_img = poke_api.fill_image_to_black(ans_img)
    shadow_img_encoded = poke_api.decode_shadow_image(shadow_img)
    return {
        "id": id,
        "name": name,
        "ans_img": ans_img_encoded,
        "shadow_img": shadow_img_encoded
    }

@app.get("/all/")
def name_all():
    name_list = poke_api.get_name_all()
    return {
        "result": name_list
    }


import pandas as p
import numpy as n
import  matplotlib.pyplot as plot
from sklearn import metrics

data1=p.read_csv("ww.csv")

num = list(n.array(list(range(0,105,5)))/100)
num1=[]
for i in num:
    data1["data1_pr_ran"] = (data1["y_pred_random_forest"]>=i).astype(int)

    tp=sum((data1["y_act"]==1) & (data1["data1_pr_ran"]==1))
    tn=sum((data1["y_act"]==0) & (data1["data1_pr_ran"]==0))
    fp=sum((data1["y_act"]==0) & (data1["data1_pr_ran"]==1))
    fn=sum((data1["y_act"]==1) & (data1["data1_pr_ran"]==0))

    tpr=tp/(tp+fn)
    fpr=fp/(fp+tn)

    num1.append([tpr,fpr])

graph=p.DataFrame(num1,columns=["x","y"])

from numpy import trapz
ds=trapz(graph.x,graph.y)
print(abs(round(ds)))

plot.scatter(graph.y,graph.x)
plot.show()

#character error

from nltk.metrics import edit_distance

letter1="this is a boy"
letter2="this is a girl"

print("character_error:" , edit_distance(letter1,letter2)/len(letter1))

#word error

word=letter1.split()
word1=letter2.split()

print("word_error:",edit_distance(word,word1)/len(word))

#prc

x_kaar=n.array([1,0,0,0,0,1])

y_pro=n.array([0.4,0.7,0.3,0.8,0.2,0.9])

print('prc rate:', metrics.average_precision_score(x_kaar,y_pro))

#mean absoulte error

value=0
len_val=len(data1)

for i,val in data1.iterrows():
    result = val["y_act"]-val["y_pred_random_forest"]
    value+=result

value=value/len_val

print("man absoulte value:",value)

#mean squre error
to=0
for i,val in data1.iterrows():
    result = val["y_act"]-val["y_pred_random_forest"]
    result*=result
    to+=result

print("mean squre error:",to)


"""Modified from Gluon CV"""
"""Prepare ImageNet Sketch datasets: https://github.com/HaohanWang/ImageNet-Sketch"""
import os
import shutil
import argparse
import tarfile
import pathlib

from download import download
from filesystem import makedirs

try:
  from google_drive_downloader import GoogleDriveDownloader as gdd
except ImportError:
    raise ImportError("Please: pip install googledrivedownloader")

where_am_I = pathlib.Path(__file__).parent.absolute()
_TARGET_DIR = os.path.expanduser(os.path.join(where_am_I, '../data/'))

def parse_args():
  parser = argparse.ArgumentParser(
      description='Initialize ImageNet V2 dataset.',
      epilog='Example: python imagenet_v2.py --download-dir path_to_your_dir',
      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
  parser.add_argument('--download-dir', type=str, required=True,
                      default='~/datasets/ImageNet-Sketch/',
                      help='dataset directory on disk')
  parser.add_argument('--overwrite', action='store_true',
                      help='overwrite downloaded files if set, in case they are corrupted')
  args = parser.parse_args()
  return args

#####################################################################################
# Download and extract the file into ``path``

def download_me(path, overwrite=False):
  # https://drive.google.com/open?id=1Mj0i5HBthqH1p_yeXzsg22gZduvgoNeA
  dest_path=os.path.join(path, 'imagenetsketch.zip')
  if not os.path.exists(dest_path) or args.overwrite:
    gdd.download_file_from_google_drive(file_id='1Mj0i5HBthqH1p_yeXzsg22gZduvgoNeA',
                                      dest_path=os.path.join(path, 'imagenetsketch.zip'),
                                      unzip=True)

if __name__ == '__main__':
  args = parse_args()
  path = os.path.expanduser(args.download_dir)
  makedirs(path)
  download_me(path, overwrite=args.overwrite)

  # make symlink
  os.symlink(os.path.join(path, 'sketch'),
              os.path.join(_TARGET_DIR, 'imagenet-sketch'))


import discord
from discord.ext import commands
from random import randint as rand
import time
import json

intents=discord.Intents.all()
allowed_mentions=discord.AllowedMentions(everyone=False, users=True, roles=False)

bot = commands.Bot(command_prefix='=', intents=intents, allowed_mentions=allowed_mentions)
token = open('token.txt', "r").readline()
startTime = time.time()

def isMod(ctx):
    try:
        return ctx.author.guild_permissions.manage_channels
    except AttributeError:
        return False

@bot.event
async def on_ready():
    game = discord.Game("=help")
    await bot.change_presence(activity=game, status=discord.Status.idle)
    print(f'logged in as {bot.user}')

@bot.event
async def on_message(message):
    if message.author.bot or not message.guild:
        return

    with open('phrases.json', 'r') as f:
        data = json.load(f)

    if str(message.guild.id) not in data:
        pass
    else:
        for phrase in data[str(message.guild.id)]:
            if phrase in message.content.lower():
                await message.channel.send(data[str(message.guild.id)][phrase])

    await bot.process_commands(message)

@bot.event
async def on_command_error(ctx, exception):
	error = str(exception).lower().replace("\"","`")
	await ctx.send(f'{error}.\npro tip: ```you can use =help to view a list of commands or get extra information.```')

@bot.command()
async def ping(ctx):
    await ctx.send(f"{round(bot.latency * 1000)}ms")

@bot.command()
async def info(ctx):
    seconds = time.time() - startTime
    m, s = divmod(seconds, 60)
    h, m = divmod(m, 60)
    d, h = divmod(h, 24)
    w, d = divmod(d, 7)
    uptime_str = f'{int(w)}w : {int(d)}d : {int(h)}h : {int(m)}m : {int(s)}s'

    e = discord.Embed(title="Information about me",
                      url="https://github.com/RKREEE/FaithBot",
                      colour=discord.Colour(0xaa7bff))
    e.set_thumbnail(url=bot.user.avatar_url_as(format="png"))
    e.add_field(name="Info", value=f"Name: {bot.user}\nID: {bot.user.id}", inline=False)
    e.add_field(name="Uptime", value=uptime_str, inline=True)
    e.add_field(name="Guild Count", value=f"{len(bot.guilds)} guilds with {len(bot.users)} users", inline=True)

    await ctx.send(embed=e)

@bot.command()
async def findseed(ctx):
    eyes = 0
    for i in range(12):
        if rand(1,10) == 10:
            eyes += 1
    await ctx.send(f'{ctx.author.mention} your seed is a {eyes} eye')

    with open('findseedstats.json', 'r') as f:
        data = json.load(f)

    if str(ctx.guild.id) not in data or data[str(ctx.guild.id)]["best"] < eyes:
        data[str(ctx.guild.id)] = {"best": eyes, "user": str(ctx.author.id)}

    with open("findseedstats.json", 'w') as f:
        json.dump(data, f, indent=4)

    with open("findseedUsers.json", "r") as f:
        data = json.load(f)

    if str(ctx.author.id) not in data:
        data[str(ctx.author.id)] = {f"{str(eyes)} eyes": "1"}
    elif f"{str(eyes)} eyes" not in data[str(ctx.author.id)]:
        data[str(ctx.author.id)][f"{str(eyes)} eyes"] = "1"
    else:
        data[str(ctx.author.id)][f"{str(eyes)} eyes"] = str(int(data[str(ctx.author.id)][f"{str(eyes)} eyes"]) + 1)

    with open("findseedUsers.json", "w") as f:
        json.dump(data, f, indent=4)

@bot.command()
async def bestseed(ctx):
    with open('findseedstats.json', 'r') as f:
        data = json.load(f)

    if str(ctx.guild.id) not in data:
        await ctx.send(f"**{ctx.guild.name}** doesnt have a seed history")
    else:
        num = data[str(ctx.guild.id)]["best"]
        member = bot.get_user(int(data[str(ctx.guild.id)]["user"]))
        await ctx.send(f'**{ctx.guild.name}** has a best seed of {num} eyes, set by **{member.display_name}#{member.discriminator}**')

@bot.command()
async def findseedstats(ctx, user: discord.User = None):
    if user == None:
        user = ctx.author
    user = bot.get_user(user.id)

    with open('findseedUsers.json', 'r') as f:
        data = json.load(f)

    if str(user.id) not in data:
        await ctx.send(f"{user.display_name}#{user.discriminator} hasnt used findseed yet")
    else:
        total = 0
        for i in range(13):
            try:
                total += int(data[str(user.id)][f'{i} eyes'])
            except KeyError:
                continue

        e = discord.Embed(title=f"{user.display_name}\'s findseed stats from {total:,} seeds", colour=discord.Colour(0xaa7bff))

        for x in data[str(user.id)]:
            try:
                e.add_field(name=f"{str(x)}:", value=f"{int(data[str(user.id)][f'{x} eyes']):,}", inline=True)
            except KeyError:
                e.add_field(name=f"{str(x)} eyes:", value="0", inline=True)

        await ctx.send(embed=e)

@bot.command()
async def addphrase(ctx, phrase :str = None, reaction :str = None):
    if isMod(ctx) != False:
        with open('phrases.json', 'r') as f:
            data = json.load(f)

        if str(ctx.guild.id) not in data:
            data[str(ctx.guild.id)] = {}
        if phrase != None and reaction != None:
            data[str(ctx.guild.id)][phrase] = reaction
        elif phrase != None and reaction == None:
            await ctx.send('what do you want me to reply with?')
            return
        elif phrase == None and reaction == None:
            await ctx.send(f'hint:\n```=addphrase \"the phrase\" \"the reply\"```')
            return

        with open('phrases.json', 'w') as f:
            json.dump(data, f, indent=4)

        await ctx.send(f'{phrase} has been added as a phrase for this server')
    else:
        await ctx.send(f'you dont have permission to use this command')

@bot.command()
async def removephrase(ctx, phrase: str):
    if isMod(ctx) != False:
        with open('phrases.json', 'r') as f:
            data = json.load(f)

        try:
            data[str(ctx.guild.id)].pop(phrase)
        except KeyError:
            await ctx.send(f'i cant find `{phrase}`, are you sure you spelt it correctly?')
            return

        with open('phrases.json', 'w') as f:
            json.dump(data, f, indent=4)

        await ctx.send(f'{phrase} has been removed from this server')
    else:
        await ctx.send('you dont have permission to use this command')
bot.run(token)


import re
import json
from loader import bot, headers
import rapid_api

def city_api(message, hotel_dest, loc):
    querystring_search = {"query": loc, "locale": "en_US", "currency": "USD"}
    url_search = "https://hotels4.p.rapidapi.com/locations/v2/search"

    response = rapid_api.request_api.request_to_api(url_search, headers, querystring_search, message)
    if response != None:
        try:
            pattern = r'(?<="CITY_GROUP",).+?[\]]'
            find = re.search(pattern, response.text)
            if find:
                data_search = json.loads(f"{{{find[0]}}}")
            else:
                bot.send_message(message.from_user.id, "Название города введено не коррестно!")
                return
            for entitle in data_search['entities']:
                hotel_dest.append(entitle['destinationId'])
            return hotel_dest
        except NameError:
            bot.send_message(message.from_user.id, "Название города введено не коррестно!")
            return
    else:
        return


import pygame

class Counter:
    def __init__(self, screen, pos, nTotalItems, nCurrentItems, itemImage, spacerX, directionRight) -> None:
        self.screen = screen
        self.x, self.y = pos
        self.nTotalItems = nTotalItems
        self.nCurrentItems = nCurrentItems
        self.itemImage = itemImage
        self.spacerX = spacerX
        self.directionRight = directionRight

        self.imageWidth = self.itemImage.get_width()

        self.transparentItemImage = self.itemImage.copy()
        self.transparentItemImage.fill((255, 255, 255, 75), None, pygame.BLEND_RGBA_MULT)

    def drawImage(self, transparent, pos):
        self.screen.blit(self.transparentItemImage if transparent else self.itemImage, pos)

    def draw(self):
        if self.directionRight:
            for i in range(self.nTotalItems):
                x, y = self.x + i*self.imageWidth + i*self.spacerX, self.y
                if i < self.nCurrentItems:
                    self.drawImage(False, (x, y))

                else:
                    self.drawImage(True, (x, y))

        else:
            for i in range(self.nTotalItems):
                x, y = self.x+i*self.imageWidth+i*self.spacerX, self. y
                if i+1 > self.nTotalItems - self.nCurrentItems:
                    self.drawImage(False, (x, y))
                else:
                    self.drawImage(True, (x, y))

    def update(self):
        self.draw()


# $Id: tns.py 23 2006-11-08 15:45:33Z dugsong $

"""Transparent Network Substrate."""

import dpkt

class TNS(dpkt.Packet):
    __hdr__ = (
    ('length', 'H', 0),
    ('pktsum', 'H', 0),
    ('type', 'B', 0),
    ('rsvd', 'B', 0),
    ('hdrsum', 'H', 0),
    ('msg', '0s', ''),
    )
    def unpack(self, buf):
        dpkt.Packet.unpack(self, buf)
        n = self.length - self.__hdr_len__
        if n > len(self.data):
            raise dpkt.NeedData('short message (missing %d bytes)' %
                                (n - len(self.data)))
        self.msg = self.data[:n]
        self.data = self.data[n:]


'''
1/3/2018
Send t parameter by url, just change first two letters, ex:
5E9AF59D2467
# http://www.tjfdc.com.cn/Pages/fcdt/HouseInfo.aspx?t=7D4E9AF59D2467&loupan=经纬城市绿洲武清二期&projectname=学知华庭31号楼

Error:UnicodeEncodeError: 'gbk' codec can't encode character '\xa0'
change from
f1.write(myNote.encode('gbk'))
to
f1.write(myNote.encode('gbk','ignore'))
or
str2.replace(u'\xa0', u'')

Error:python write() argument must be str, not bytes
change from
open("temp.txt", "a")
to
open("temp.txt", "wb+")  or ab+
'''
from splinter.browser import Browser
from bs4 import BeautifulSoup
import time
import os

class singleprice:

    fileName='single.csv'

    chromeDriverpath = 'C:\Program Files\Java\webdriver\chromedriver.exe'  # Web driver path
    executable_path = {'executable_path': chromeDriverpath}  # DO NOT change it
    browser = Browser('chrome', **executable_path)  # DO NOT change it

    def __init__(self):
        #self.browser.visit(self.url)
        a=1

    def getinfo(self,url):
        time.sleep(2)
        try:
            self.browser.visit(url)

            innerhtml = self.browser.html
            soup = BeautifulSoup(innerhtml, "lxml")
            soup.replaceWith
            strTotal = ''
            for elem in soup.find_all('td'):
                str1 = elem.text
                str2 = str1.replace('\n', '')  # remove \n
                str3 = str2.strip()  # remove space
                strTotal = strTotal + str3 + ','
            strTotal = strTotal + '\n'
            self.writeLog(strTotal)
        except Exception:
            None

    def writeLog(self,myNote):
        #time1=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        # Open and rewite, add file from beginning with 'w', add file from the end with 'a'
        f1 = open(self.fileName, 'ab+')
        #f1.write(str(myNote))
        #f1.write(str(myNote).encode('utf-8'))
        #string.replace(u’\xa0’, u’ ‘)
        f1.write(str(myNote).encode('gbk','ignore'))
        f1.write(str(myNote).encode('gbk', 'ignore'))
        f1.close()

    def getByCode(self,urlStart,urlEnd):
        url=''
        # Number+Letter
        # Range from 0 to 9
        for i in range(0,10):
            url=str(i)
            for j in range(65, 91):
                print("getting id "+str(i)+chr(j))
                url=urlStart+str(i)+chr(j)+urlEnd
                self.getinfo(url)
        # Letter+Number
        for i in range(65, 91):
            url=str(i)
            for j in range(0, 10):
                print("getting id "+chr(i)+str(j))
                url=urlStart+chr(i)+str(j)+urlEnd
                self.getinfo(url)

        # Letter+Letter
        for i in range(65, 91):
            url=str(i)
            for j in range(65, 91):
                print("getting id "+chr(i)+chr(j))
                url=urlStart+chr(i)+chr(j)+urlEnd
                self.getinfo(url)

        # Number+Number
        for i in range(0, 10):
            url = str(i)
            for j in range(0, 10):
                print("getting id " + str(i) + str(j))
                url = urlStart + str(i) + str(j) + urlEnd
                self.getinfo(url)


import unittest

from unittest.mock import patch
import tempfile
import os

from rhsmlib.facts import kpatch

class TestKPatchCollector(unittest.TestCase):
    def setUp(self):
        self.dir_installed = tempfile.TemporaryDirectory()
        self.DIR_WITH_INSTALLED_KPATCH_MODULES = self.dir_installed.name
        os.mkdir(os.path.join(self.DIR_WITH_INSTALLED_KPATCH_MODULES, "3.10.0-1062.el7.x86_64"))
        os.mkdir(os.path.join(self.DIR_WITH_INSTALLED_KPATCH_MODULES, "3.10.0-1062.1.1.el7.x86_64"))
        os.mkdir(os.path.join(self.DIR_WITH_INSTALLED_KPATCH_MODULES, "3.10.0-1062.1.2.el7.x86_64"))

        self.dir_loaded = tempfile.TemporaryDirectory()
        self.DIRS_WITH_LOADED_MODULE = ["/path/to/not-existing-directory", self.dir_loaded.name]
        os.mkdir(os.path.join(self.DIRS_WITH_LOADED_MODULE[1], "3.10.0-1062.el7.x86_64"))

    @patch("shutil.which")
    def test_kpatch_is_not_installed(self, which):
        which.return_value = None
        collector = kpatch.KPatchCollector()
        kpatch_facts = collector.get_all()
        self.assertEqual(kpatch_facts, {})

    @patch("shutil.which")
    def test_get_kpatch_facts(self, which):
        which.return_value = "/usr/sbin/kpatch"
        collector = kpatch.KPatchCollector()
        collector.DIR_WITH_INSTALLED_KPATCH_MODULES = self.DIR_WITH_INSTALLED_KPATCH_MODULES
        collector.DIRS_WITH_LOADED_MODULE = self.DIRS_WITH_LOADED_MODULE
        kpatch_facts = collector.get_all()
        self.assertIn("kpatch.loaded", kpatch_facts)
        self.assertEqual(kpatch_facts["kpatch.loaded"], "3.10.0-1062.el7.x86_64")
        self.assertIn("kpatch.installed", kpatch_facts)
        installed_kpatches = sorted(kpatch_facts["kpatch.installed"].split())
        self.assertEqual(len(installed_kpatches), 3)
        self.assertEqual(
            installed_kpatches,
            ["3.10.0-1062.1.1.el7.x86_64", "3.10.0-1062.1.2.el7.x86_64", "3.10.0-1062.el7.x86_64"],
        )


from product import Product
import random
from datetime import datetime
from user import Customer

class Orders:
    def __init__(self):
        self.__item = {}

    @property
    def item(self):
        return self.__item

    @item.setter
    def item(self, item):
        self.__item = item

    def add_menu(self, choose):
        """
        This function will add item into customer cart. it will get input if product's id ,
        and then check whether that product is available or not
        :param choose:
        :return:
        """
        for data in Product().readfile():
            if choose == data["id"]:
                amount = int(input(f"How many {data['menu']} do you want?: "))
                if Product().check_stock(int(data["stock"]), amount):  # check whether menu available or not
                    Product().update_stock(int(data["stock"]), amount)  # update stock
                    self.__item.update({data["menu"]: {"id": data["id"],  # update cart
                                                       "quantity": amount,
                                                       "subtotal": int(data["price"]) * amount}})
                    print(f"Add {amount} {data['menu']} to the cart.")
                    return self.__item
                else:
                    return False

    def delete(self):
        """
        This function will delete unwanted item from customer cart
        :return:
        """
        choose = input("Which one do you want to delete?: ")
        for menu, info in self.__item.copy().items():
            # check whether product that customer chooses is in the cart or not
            if choose == info["id"]:
                # check whether customer really want to remove item
                sure = input(f"Are you sure do you want to remove {menu}?(y/n) ")
                if sure == "y":
                    self.__item.pop(menu)  # remove unwanted menu from cart
                    print(f'{menu} has been removed.')
                    break
                elif sure == "n":
                    break

    def cancel(self):
        self.__item.clear()  # delete all orders
        print("Cancel successfully.")

    def subtotal(self):
        total_price = 0
        for menu, info in self.__item.items():
            total_price += int(info["subtotal"])
        return total_price

    def discount(self):
        if self.subtotal() >= 300:  # check whether customer purchase more than 300 or not.
            dis = self.subtotal() * 0.1
            return dis
        return 00.00

    def total(self):
        if self.subtotal() >= 300:
            total = self.subtotal() - self.discount()
            return total
        return self.subtotal()

    def get_id_order(self):
        upper_case = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
        number = '1234567890'
        use_for = upper_case + number
        length = 15
        id_order = ''.join(random.sample(use_for, length))  # create order id's for customer.
        return id_order

    def receipt(self):
        """
        This function will print receipt for customer
        :return:
        """
        now = datetime.now()
        day = now.date()
        time = now.strftime("%H:%M")
        with open("order_receipt.txt", "w") as f:  # write receipt in text file
            f.write(f"\n")
            f.write(" " * 28 + "B A K E  M Y  D A Y\n\n")
            f.write(" " * 21 + "Thank you for ordering our product!\n\n")
            f.write(f"Order #{self.get_id_order()}\n\n")
            f.write("Menu" + " " * 30 + "Quantity" + " " * 30 + "Price\n")
            f.write("-" * 80 + "\n")
            for k, v in self.__item.items():
                f.write(f"{k}".ljust(37, " ") + f"{v['quantity']}".ljust(35, " ") + f"{v['subtotal']}.-\n")
            f.write("-" * 80 + "\n")
            pay = input("Do you want to pay by Bank Transfer(b) or Credit Card(c)?: ")
            while True:
                if pay == 'b':
                    f.write(f"Subtotal: {self.subtotal():.2f}฿".ljust(58, " ") + f"Payment: Bank Transfer\n")
                    break
                elif pay == 'c':
                    f.write(f"Subtotal: {self.subtotal():.2f}฿".ljust(58, " ") + f"Payment: Credit Card\n")
                    break
                elif not pay == 'b' or 'c':
                    print("Please enter b or c only")
                    pay = input("Do you want to pay by Bank Transfer(b) or Credit Card(c)?: ")
            f.write(f"Shipping fee: 0.00฿" + " ".ljust(39, " ") + f"Order date: {day}\n")
            f.write(f"Discount: {self.discount():.2f}฿".ljust(58, " ") + f"Time: {time}\n")
            f.write(f"Total: {self.total():.2f}฿\n\n")
            f.write(f"Shipping to:\n")
            c = Customer()
            c.customer_detail()
            f.write(f"{c.name.title()}\n")
            f.write(f"{c.phone}\n")
            f.write(f"{c.address.title()}\n")
            f.write("-" * 80 + "\n")
            f.write(" " * 24 + "H A V E  A  N I C E  D A Y!")


import requests
from bs4 import BeautifulSoup
import jieba
from wordcloud import WordCloud, ImageColorGenerator
import matplotlib.pyplot as plt
import numpy as np
from PIL import Image

# 爬取整页新闻函数
def news_get(base_url):
    # 发起请求
    req = requests.get(base_url)
    # 设置编码
    req.encoding = 'utf-8'
    # 获取网页内容
    html = req.text
    # 创建BeautifulSoup对象
    soup = BeautifulSoup(html, 'html.parser')
    # 存储结果的列表
    news_list = []
    # 提取新闻标题
    sou = soup.find_all('span', attrs={'class': 'news_title'})
    for new in sou:
        news_list.append(new.a.string)
    # 返回新闻标题列表
    return news_list

# 提取指定数量新闻
news = []
num = 100
page = 1
while len(news) < num:
    url = 'https://xc.hfut.edu.cn/1955/list' + str(page) + '.htm'
    news.extend(news_get(url))
    page += 1
print(len(news))

# 保留100条
if len(news) > num:
    news = news[:100]
print(len(news))

# 使用jieba进行分词
seg_list = []

# 遍历数组并进行分词
for title in news:
    seg_list.append(jieba.lcut(title))

# 将分词结果拼接成字符串
word_str = ''
for i in range(len(seg_list)):
    word_str += ' '.join(seg_list[i])

# 读取背景图片
background_image = np.array(Image.open("school.jpg"))

# 设置词云图的参数,并构建词云
wordcloud = WordCloud(
    mask=background_image,
    background_color="white",
    width=800,
    height=600,
    max_words=1000,
    max_font_size=1000,
    relative_scaling=0.6,
    random_state=50,
    scale=2,
    font_path=r'C:\Windows\Fonts\simfang.ttf',
    ).generate(word_str)

# 根据图片生成颜色
image_colors = ImageColorGenerator(background_image)

# 绘制词云图
plt.imshow(wordcloud.recolor(color_func=image_colors), interpolation="bilinear")
plt.axis('off')
plt.show()


import helpFunctions as hf
import matplotlib.pyplot as plt
import numpy as np
from scipy import misc
from PIL import Image

dirIn = '/home/mehmet/Desktop/DTU_Spring/Math_Modelling/cn/Exercises/Exercise_1/ex1_campusnet/data/'

multiIm, annotationIm = hf.loadMulti('multispectral_day20.mat' , 'annotation_day20.png', dirIn)

my_array = annotationIm[:,:,0]
im = Image.fromarray(annotationIm[:,:,0].reshape((514,514)).astype('uint8')*255)
im.save("grey_scale.png")

my_array2 = annotationIm[:,:,1]
im = Image.fromarray(my_array2.reshape((514,514)).astype('uint8')*255)
im.save("second.png")

my_array3 = annotationIm[:,:,2]
im = Image.fromarray(my_array3.reshape((514,514)).astype('uint8')*255)
im.save("third.png")

##############################################################################################################################
#1. Calculate the threshold value t for all spectral bands for day 1
#to do: for day 1.
multiIm, annotationIm = hf.loadMulti('multispectral_day01.mat' , 'annotation_day01.png', dirIn)
[fatPix, fatR, fatC] = hf.getPix(multiIm, annotationIm[:,:,1])
[meatPix, meatR, meatC] = hf.getPix(multiIm, annotationIm[:,:,2])
thresholds_day20 = np.array([20, 13, 19, 22, 22, 22, 22, 25, 41, 52, 57, 64, 70, 69, 67, 59, 59, 49, 2.5])
thresholds_day01 = np.array([19, 13, 20, 23, 23, 27, 25, 29, 48, 54, 58, 64, 69, 69, 67, 63, 61, 53, 4 ])
def error_rate_fat(fatPix,layerNo):
        #returns the failure rate of the assumption
        return (fatPix[:,0].shape[0]-fatPix[fatPix[:,layerNo]>=20,0].shape[0])/fatPix[:,0].shape[0]
##############################################################################################################################
#2. Calculate the error rate for each spectral band.
for i in range(19):
        error_rate_fat(fatPix,i)
        h = hf.showHistograms(multiIm, annotationIm[:,:,1:3], i, 1)
##############################################################################################################################
#3. Identify the spectral band, which has the best discriminative properties for meat and fat
[fatPix, fatR, fatC] = hf.getPix(multiIm, annotationIm[:,:,1]);
[meatPix, meatR, meatC] = hf.getPix(multiIm, annotationIm[:,:,2]);
plt.plot(np.mean(meatPix,0),'b')
plt.plot(np.mean(fatPix,0),'r')
plt.show()#spectral distrubution
"""
We guess that the third mean difference is the biggest one. So it is a good candidate for training day.
"""

"""
We can see the biggest difference between the means is seen at day 3. However, we can do another better
determination via both increasing the difference between means and decreasing the variance in the groups.
Because the whole idea behind the LDA is this 2 principles. (mu1 - mu2)²/(s1² + s2²). The largest spectrum
is defined as our training set.
"""
def best_band(meatPix, fatPix):
	numerator = (np.mean(meatPix,0) - np.mean(fatPix,0))**2
	denomunator = np.std(meatPix,0)**2 + np.std(fatPix,0)**2

	values = numerator/denomunator
	return np.argmax(values)

"""
We were right. It is the third day has the best indicators.
"""
##############################################################################################################################
#4. Classify the entire image of the salami for day 1, and visualise it

"""
SIMPLEST MODEL (thresholding)
We implement our simplest model to day1. While doing this we do it on the third picture(450 nm).

"""
middle = (annotationIm[:,:,2]*1 + annotationIm[:,:,1]*1 + annotationIm[:,:,0]*1)

def classify(threshold, multiIm):
	from scipy.misc import toimage
	meat_anno = np.zeros((514,514))
	fat_anno =  np.zeros((514,514))
	for x in range(514):
		for y in range(514):
			if middle[x][y] ==1:
				if multiIm[:,:,2][x][y] >=20:
					meat_anno[x][y] = 0
					fat_anno[x][y] = 1
				else:
					meat_anno[x][y] = 1
					fat_anno[x][y] = 0

thresholds_day01 = np.array([19, 13, 20, 23, 23, 27, 25, 29, 48, 54, 58, 64, 69, 69, 67, 63, 61, 53, 4 ])
for value in thresholds_day01:
	classify(value, multiIm, )

from scipy.misc import toimage
img = toimage(fat_anno)
img.save("third.png")

img2 = toimage(meat_anno)
img2.save("meatDay1Layer3.png")
##############################################################################################################################
#1. Calculate the multivariate linear discriminant function as described in (23) for day 1.

import numpy as np
from scipy import misc
from PIL import Image

dirIn = '/home/mehmet/Desktop/DTU_Spring/Math_Modelling/cn/Exercises/Exercise_1/ex1_campusnet/data/'

multiIm, annotationIm = hf.loadMulti('multispectral_day01.mat' , 'annotation_day01.png', dirIn)
[fatPix, fatR, fatC] = hf.getPix(multiIm, annotationIm[:,:,1]);
[meatPix, meatR, meatC] = hf.getPix(multiIm, annotationIm[:,:,2]);

imageFat = np.zeros((multiIm.shape[0], multiIm.shape[1]))
imageMeat = np.zeros((multiIm.shape[0], multiIm.shape[1]))
middle = (annotationIm[:,:,2]*1 + annotationIm[:,:,1]*1 + annotationIm[:,:,0]*1)

def lda_amount(typePix, meatPix, fatPix, multiIm, x, y):
	from numpy.linalg import inv
	covMatrixFat = np.cov(np.transpose(fatPix)) #cov matrix
	covMatrixMeat = np.cov(np.transpose(meatPix))

	numo = ( (fatPix.shape[0]-1)*covMatrixFat+(meatPix.shape[0]-1)*covMatrixMeat)
	deno = fatPix.shape[0] + meatPix.shape[0]-2
	covMatrix = numo/deno

	mean = np.mean(typePix,0)
	prior = typePix.shape[0]/(meatPix.shape[0] + fatPix.shape[0])

	first1 = np.transpose(multiIm[x,y,:])
	first2 = inv(covMatrix)
	first = np.dot(np.dot( first1, first2  ),mean)

	second = np.dot(1/2*np.dot(np.transpose(mean), inv(covMatrix)), mean)

	third = np.log(prior)

	return  (first - second +third)

for i in range(multiIm.shape[0]):
	for j in range(multiIm.shape[1]):

		if not middle[i,j] ==1:
			print(i,j)
			continue

		valueFat = lda_amount(fatPix, meatPix, fatPix, multiIm, i, j)
		valueMeat = lda_amount(meatPix, meatPix, fatPix, multiIm, i, j)
		print(valueMeat , valueFat)
		if middle[i,j] ==1:
			if valueFat > valueMeat :
				imageFat[i,j] = 1
			else:
				imageMeat[i,j] = 1

from scipy.misc import toimage
img = toimage(imageFat)
img.save("day1Fat.png")
img2 = toimage(imageMeat)
img2.save("day1Meat.png")

##############################################################################################################################
#2. Calculate the error rate (disagreement between the model and the annotations) for the training set.

def error_rate():
	import numpy as np
	from scipy import misc
	from PIL import Image
	import helpFunctions as hf
	import matplotlib.pyplot as plt
	dirIn = '/home/mehmet/Desktop/DTU_Spring/Math_Modelling/cn/Exercises/Exercise_1/ex1_campusnet/data/'

	multiIm, annotationIm = hf.loadMulti('multispectral_day01.mat' , 'annotation_day01.png', dirIn)
	[fatPix, fatR, fatC] = hf.getPix(multiIm, annotationIm[:,:,1]);
	[meatPix, meatR, meatC] = hf.getPix(multiIm, annotationIm[:,:,2]);

	imageFat = np.zeros((multiIm.shape[0], multiIm.shape[1]))
	imageMeat = np.zeros((multiIm.shape[0], multiIm.shape[1]))

	annoFat = annotationIm[:,:,1]
	annoMeat = annotationIm[:,:,2]
	for i in range(multiIm.shape[0]):
			for j in range(multiIm.shape[1]):
				if not middle[i,j] ==1:
					print(i,j)
					continue

				valueFat = lda_amount(fatPix, meatPix, fatPix, multiIm, i, j)
				valueMeat = lda_amount(meatPix, meatPix, fatPix, multiIm, i, j)
				print(valueMeat , valueFat)
				if middle[i,j] ==1:
					if valueFat > valueMeat :
						imageFat[i,j] = 1

					else:
						imageMeat[i,j] = 1

	errorFatArray = annoFat[(imageFat ==0) & (annoFat==1)]

	return errorFatArray.shape


import json, argparse
import logging, sys, datetime, os
from shutil import copy
from pydal.helpers.classes import RecordDeleter, RecordUpdater
from pydal.objects import LazySet
from applications.vidjil.models.VidjilAuth import PermissionEnum

class MigrateLogger():

    def __init__(self, level=logging.INFO):
        log = logging.getLogger('vidjil_migrate')
        log.setLevel(level)
        log.propagate = False
        #formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        formatter = logging.Formatter('%(levelname)s\t- %(message)s')
        ch = logging.StreamHandler(sys.stdout)
        ch.setFormatter(formatter)
        log.addHandler(ch)
        self.log = log

    def info(self, text):
       self.log.info(text)

    def debug(self, text):
        self.log.debug(text)

    def error(self, text):
        self.log.error(text)

    def getLogger(self):
        return self.log

def get_dict_from_row(row):
    '''
    Create a dict element from a Row element
    while filtering out some key pydal helper fields
    '''
    ex_fields = ['id', 'uuid']
    ex_types = [RecordDeleter, RecordUpdater, LazySet]

    my_dict = {}
    for key in row.keys():
        if ((key not in ex_fields) and (type(row[key]) not in ex_types)):
            tmp = row[key]
            if isinstance(tmp, datetime.datetime) or isinstance(tmp, datetime.date):
                tmp = tmp.__str__()
            my_dict[key] = tmp
    return my_dict

def reencode_dict(data):
    '''
    Recursively reencode the values and keys of a dict to utf-8.
    Takes a dict loaded from json, assumes all keys are strings.
    Values may be any type, only unicode values are reencoded
    '''
    if type(data) == dict:
        tmp = {}
        for key in data:
            val = data[key]
            tmp[key.encode('utf-8')] = reencode_dict(val)
        return tmp
    elif type(data) == unicode:
        return data.encode('utf-8')
    else:
        return data

class IdMapper():

    def __init__(self, log):
        self.log = log
        self.mapping = {}

    def getMatchingId(self, oid):
        if oid not in self.mapping:
            self.log.debug('id %d not in mapping, returning it' % oid)
            self.log.debug("mapping: " + str(self.mapping.keys()))
            return oid
        return self.mapping[oid]

    def setMatchingId(self, old_id, new_id):
        self.mapping[old_id] = new_id
        self.log.debug("mapped: %d to %d" % (old_id, new_id))

class ConfigMapper(IdMapper):

    def __init__(self, log, cfile=None):
        IdMapper.__init__(self, log)
        if cfile is not None:
            self.load(cfile)

    def load(self, cfile):
        self.mapping = {}
        with open(cfile, 'r') as cfg:
            config_map = json.load(cfg, encoding='utf-8')
            for key in config_map:
                self.mapping[long(key)] = config_map[key]
            self.log.info("mapping loaded")
            self.log.debug("mapping: " + str(self.mapping.keys()))

class Extractor():

    def __init__(self, db, log):
        self.log = log
        self.log.info("initialising extractor")
        self.db = db

    def populateSets(self, rows):
        self.log.debug("populate sets")
        sets = {}
        sample_set_ids = []
        for row in rows:
            self.log.debug("populating : %d, sample_set: %d" % (row.id, row.sample_set_id))
            sets[row.id] = get_dict_from_row(row)
            sample_set_ids.append(row.sample_set_id)
        return sets, sample_set_ids

    def getSequenceFiles(self, sample_set_ids):
        db = self.db
        rows = db((db.sample_set_membership.sample_set_id.belongs(sample_set_ids))
                & (db.sequence_file.id == db.sample_set_membership.sequence_file_id)
               ).select(db.sample_set_membership.ALL, db.sequence_file.ALL)
        return rows

    def populateSequenceFiles(self, rows):
        self.log.debug("populate sequence_files")
        memberships = {}
        sequence_files = {}
        for row in rows:
            ssm_id = row.sample_set_membership.id
            sf_id = row.sequence_file.id
            self.log.debug("populating sequence file: %d, membership: %d" % (sf_id, ssm_id))
            memberships[ssm_id] = get_dict_from_row(row.sample_set_membership)
            sequence_files[sf_id] = get_dict_from_row(row.sequence_file)
        return sequence_files, memberships

    def getTableEntries(self, table, ref_field, values):
        db = self.db
        rows = db(db[table][ref_field].belongs(values)).select(db[table].ALL)
        return rows

    def populateEntries(self, rows, etype=''):
        self.log.debug("populate %ss" % etype)
        data = {}
        for row in rows:
            my_dict = get_dict_from_row(row)
            self.log.debug("populating entry: %s" % str(my_dict))
            data[row.id] = my_dict
        return data

class GroupExtractor(Extractor):

    def __init__(self, db, log):
        Extractor.__init__(self, db, log)

    def getAccessible(self, table, groupid):
        db = self.db
        rows = db((db[table].id == db.auth_permission.record_id)
                & (db.auth_permission.table_name == table)
                & (db.auth_permission.name == PermissionEnum.access.value)
                & (db.auth_permission.group_id == groupid)
               ).select(db[table].ALL)
        return rows

class SampleSetExtractor(Extractor):

    def __init__(self, db, log):
        Extractor.__init__(self, db, log)

    def getAccessible(self, table, ids):
        db = self.db
        rows = db(db[table].id.belongs(ids)).select(db[table].ALL)
        return rows

class Importer():

    def __init__(self, groupid, db, log, config_mapper):
        self.log = log
        self.log.info("initialising importer")
        self.groupid = groupid
        self.db = db
        self.mappings = {'config': config_mapper}
        self.mappings['sample_set'] = IdMapper(self.log)

    def importSampleSets(self, stype, sets):
        db = self.db
        self.log.debug("import sets")
        for sid in sets:
            self.log.debug("importing set: %s" % sid)
            sset = sets[sid]
            ssid = db.sample_set.insert(sample_type = stype)
            self.log.debug("new sample_set %d" % ssid)
            self.mappings['sample_set'].setMatchingId(sset['sample_set_id'], ssid)
            sset['sample_set_id'] = ssid
            nid = db[stype].insert(**sset)
            self.log.debug("new %s: %d" % (stype, nid))
            db.auth_permission.insert(group_id=self.groupid,
                                      name=PermissionEnum.access.value,
                                      table_name=stype,
                                      record_id=nid)
            self.log.debug("associated set %d to group %d" % (nid, self.groupid))

    def importTable(self, table, values, ref_fields={}, map_val=False):
        db = self.db
        self.log.debug("import %ss" % table)
        for vid in values:
            self.log.debug("importing %s: %s" % (table, vid))
            val = values[vid]
            for key in ref_fields:
                ref_key = ref_fields[key]
                matching_id = self.mappings[key].getMatchingId(val[ref_key])
                self.log.debug("%s replacing %s: %d with %d" % (table, ref_key, val[ref_key], matching_id))
                val[ref_key] = matching_id
            oid = db[table].insert(**val)
            self.log.debug("new %s: %d" % (table, oid))
            if map_val:
                if table not in self.mappings:
                    self.mappings[table] = IdMapper(self.log)
                self.mappings[table].setMatchingId(long(vid), oid)

def copy_files(data, src, dest, log=MigrateLogger()):
    file_fields = {'results_file': 'data_file',
            'analysis_file': 'analysis_file',
            'fused_file': 'fused_file'}

    if not os.path.exists(dest):
        os.makedirs(dest)

    for t in file_fields:
        for entry in data[t]:
            if (data[t][entry][file_fields[t]] is not None):
                log.debug("Copying %s" % data[t][entry][file_fields[t]])
                copy(src + '/' + data[t][entry][file_fields[t]], dest + '/' + data[t][entry][file_fields[t]])

def export_peripheral_data(extractor, data_dict, sample_set_ids, log=MigrateLogger()):
    sequence_rows = extractor.getSequenceFiles(sample_set_ids)
    data_dict['sequence_file'], data_dict['membership'] = extractor.populateSequenceFiles(sequence_rows)

    results_rows = extractor.getTableEntries('results_file', 'sequence_file_id', data_dict['sequence_file'].keys())
    data_dict['results_file'] = extractor.populateEntries(results_rows, 'results_file')

    task_ids = [r['scheduler_task_id'] for k, r in data_dict['results_file'].iteritems()]
    task_rows = extractor.getTableEntries('scheduler_task', 'id', task_ids)
    data_dict['scheduler_task'] = extractor.populateEntries(task_rows, 'scheduler_task')

    run_rows = extractor.getTableEntries('scheduler_run', 'task_id', task_ids)
    data_dict['scheduler_run'] = extractor.populateEntries(run_rows, 'scheduler_run')

    analysis_rows = extractor.getTableEntries('analysis_file', 'sample_set_id', sample_set_ids)
    data_dict['analysis_file'] = extractor.populateEntries(analysis_rows, 'analysis_file')

    fused_rows = extractor.getTableEntries('fused_file', 'sample_set_id', sample_set_ids)
    data_dict['fused_file'] = extractor.populateEntries(fused_rows, 'fused_file')

    return data_dict

def export_group_data(filesrc, filepath, groupid, log=MigrateLogger()):
    log.info("exporting group data")
    ext = GroupExtractor(db, log)

    tables = {}

    patient_rows = ext.getAccessible('patient', groupid)
    tables['patient'], patient_ssids = ext.populateSets(patient_rows)

    run_rows = ext.getAccessible('run', groupid)
    tables['run'], run_ssids = ext.populateSets(run_rows)

    generic_rows = ext.getAccessible('generic', groupid)
    tables['generic'], generic_ssids = ext.populateSets(generic_rows)

    sample_set_ids = patient_ssids + run_ssids + generic_ssids

    tables = export_peripheral_data(ext, tables, sample_set_ids, log=log)

    if not os.path.exists(filepath):
        os.makedirs(filepath)

    with open(filepath + '/export.json', 'w') as outfile:
        json.dump(tables, outfile, ensure_ascii=False, encoding='utf-8')

    log.info("copying files from %s to %s" % (filesrc, filepath))
    copy_files(tables, filesrc, filepath + '/files', log=log)
    log.info("done")

def export_sample_set_data(filesrc, filepath, sample_type, sample_ids, log=MigrateLogger()):
    log.info("exporting sample set data")
    ext = SampleSetExtractor(db, log)

    tables = {}

    rows = ext.getAccessible(sample_type, sample_ids)
    tables[sample_type], sample_set_ids = ext.populateSets(rows)

    tables = export_peripheral_data(ext, tables, sample_set_ids, log=log)

    if not os.path.exists(filepath):
        os.makedirs(filepath)

    with open(filepath + '/export.json', 'w') as outfile:
        json.dump(tables, outfile, ensure_ascii=False, encoding='utf-8')

    log.info("copying files from %s to %s" % (filesrc, filepath))
    copy_files(tables, filesrc, filepath + '/files')

    log.info("done")

def import_data(filesrc, filedest, groupid, config=None, dry_run=False, log=MigrateLogger()):
    log.info("importing data")
    data = {}
    with open(filesrc + '/export.json', 'r') as infile:
        tmp = json.load(infile, encoding='utf-8')
        data = reencode_dict(tmp)

    config_mapper = ConfigMapper(log)
    if config:
        config_mapper.load(config)
    imp = Importer(groupid, db, log, config_mapper)

    try:
        set_types = ['patient', 'run', 'generic']
        for stype in set_types:
            if stype in data:
                imp.importSampleSets(stype, data[stype])

        imp.importTable('sequence_file', data['sequence_file'], map_val=True)
        imp.importTable('sample_set_membership', data['membership'], {'sample_set': 'sample_set_id', 'sequence_file': 'sequence_file_id'})
        imp.importTable('scheduler_task', data['scheduler_task'], map_val=True)
        imp.importTable('scheduler_run', data['scheduler_run'], {'scheduler_task': 'task_id'})
        imp.importTable('results_file', data['results_file'], {'sequence_file': 'sequence_file_id', 'scheduler_task': 'scheduler_task_id', 'config': 'config_id'})
        imp.importTable('analysis_file', data['analysis_file'], {'sample_set': 'sample_set_id'})
        imp.importTable('fused_file', data['fused_file'], {'sample_set': 'sample_set_id', 'config': 'config_id'})

        if dry_run:
            db.rollback()
            log.info("dry run successful, no data saved")
        else:
            db.commit()
            log.info("copying files from %s to %s" % (filesrc, filedest))
            copy_files(data, filesrc + '/files', filedest, log=log)
            log.info("done")
    except:
        log.error("something went wrong, rolling back")
        db.rollback()
        log.error("rollback was successful")
        raise

def main():
    parser = argparse.ArgumentParser(description='Export and import data')
    subparsers = parser.add_subparsers(help="Select operation mode", dest='command')

    exp_parser = subparsers.add_parser('export', help='Export data from the DB into a JSON file')
    exp_subparser = exp_parser.add_subparsers(dest='mode', help='Select data selection method')

    ss_parser = exp_subparser.add_parser('sample_set', help='Export data by sample-set ids')
    ss_parser.add_argument('sample_type', type=str, choices=['patient', 'run', 'generic'], help='Type of sample')
    ss_parser.add_argument('ssids', metavar='ID', type=long, nargs='+', help='Ids of sample sets to be extracted')

    group_parser = exp_subparser.add_parser('group', help='Extract data by groupid')
    group_parser.add_argument('groupid', type=long, help='The long ID of the group')

    import_parser = subparsers.add_parser('import', help='Import data from JSON into the DB')
    import_parser.add_argument('--dry-run', dest='dry', action='store_true', help='With a dry run, the data will not be saved to the database')
    import_parser.add_argument('--config', type=str, dest='config', help='Select the config mapping file')
    import_parser.add_argument('groupid', type=long, help='The long ID of the group')

    parser.add_argument('-p', type=str, dest='filepath', default='./', help='Select the file destination')
    parser.add_argument('-s', type=str, dest='filesrc', default='./', help='Select the file source')
    parser.add_argument('--debug', dest='debug', action='store_true', help='Output debug information')

    args = parser.parse_args()

    log = MigrateLogger()
    if args.debug:
        log.log.setLevel(logging.DEBUG)

    if args.command == 'export':
        if args.mode == 'group':
            export_group_data(args.filesrc, args.filepath, args.groupid, log)
        elif args.mode == 'sample_set':
            export_sample_set_data(args.filesrc, args.filepath, args.sample_type, args.ssids, log)
    elif args.command == 'import':
        if args.dry:
            log.log.setLevel(logging.DEBUG)
        import_data(args.filesrc, args.filepath, args.groupid, args.config, args.dry, log)

if __name__ == '__main__':
    main()


from django.shortcuts import render

from rest_framework import viewsets
from django.views.decorators.csrf import csrf_exempt
from .serializers import CompanySerializer, HeadlineSerializer
from .models import Company, Headlines
from .views import *
import requests
from django.http import JsonResponse
from django.http import HttpResponse
import subprocess
from django.shortcuts import render
from django.http import HttpResponseRedirect
# from background_tasks.models import ScheduledTask

import csv
# Create your views here.

def index(request):
    return render(request, 'index.html')

class CompanyViewSet(viewsets.ModelViewSet):
    serializer_class = CompanySerializer
    queryset = Company.objects.all()

class HeadlineViewSet(viewsets.ModelViewSet):
    serializer_class = HeadlineSerializer
    queryset = Headlines.objects.all()

def get_company(request, company_id):
    company = Company.objects.get(pk=company_id)
    serialized = CompanySerializer(company).data

    return JsonResponse(serialized, safe=False)

def get_headlines(request, company_id):
    try:
        # Attempt to convert company_id to an integer
        company_id = int(company_id)
        headlines = Headlines.objects.filter(company_id=company_id)
        # Your code to serialize headlines to JSON
        serialized = HeadlineSerializer(headlines, many=True).data
        return JsonResponse(serialized, safe=False)
    except ValueError:
        # Handle the case where company_id is not a valid integer
        return JsonResponse({'error': 'Invalid companyId'}, status=400)

def inputData(request):
    company = CompanyViewSet

def delete_resource(request, resource_id):
    # Make the DELETE request to the API endpoint
    api_endpoint = f'http://localhost:8000/company/{resource_id}/'
    response = requests.delete(api_endpoint)

    if response.status_code == 204:
        return JsonResponse({'message': 'Deletion successful'})
    else:
        return JsonResponse({'error': 'Failed to delete'}, status=500)

    # views.py

@csrf_exempt
def process_csv(request):
    print("processing")
    try:
        request.FILES.get('csv_file')
        csv_file = request.FILES['csv_file']
        csv_bytes = csv_file.read()

            # Decode the bytes to text (assuming it's UTF-8 encoded)
        csv_text = csv_bytes.decode('utf-8')

        # Now you can work with the CSV content as a string
        csv_reader = csv.DictReader(csv_text.splitlines(), delimiter='|')
        if csv_reader:
            print("something inside")
        for result in csv_reader:
            print(result['company'], result['date'], result['headline'])
            companyObj, createdCompany = Company.objects.get_or_create(
                name = result['company'],
                home_url = result['homeurl']
            )
            if not createdCompany:
                companyObj.name= result['company']
                companyObj.home_url = result['homeurl']
                companyObj.save()

            if result['date']:
                try:
                    headlineObj, createdHeadline = Headlines.objects.get_or_create(
                        company = companyObj.id,
                        headlines = result['headline'],
                        link = result['link'],
                        date = result['date'],
                        seen = False
                    )
                    if not createdHeadline:
                        headlineObj.headlines = result['headline']
                        headlineObj.link = result['link']
                        headlineObj.date = result['date']
                        headlineObj.seen = True
                        headlineObj.save()
                except ValueError:
                    print(f'Invalid date format: {result["date"]}')

            else:
                print("sadge")

        print("done")
        return HttpResponseRedirect('/success/')
    except Exception as e:
        print(e)
        return HttpResponse(f'Error: {str(e)}')

def run_custom_command(request):
    try:
        print("custom")
        command_name = 'scrapeNews'

        # Execute the management command using subprocess
        subprocess.run(['python', 'manage.py', command_name])
        # result will have:
        # fieldnames = ["company", "headline", "link", 'homeurl', 'date']
        return HttpResponse('Command executed successfully.')
    except Exception as e:
        return HttpResponse(f'Error: {str(e)}')

def check_updates():
    companyChanges = CompanySerializer.save()
    headlineChanges = HeadlineSerializer.save()
    print(companyChanges)
    print(headlineChanges)
    # if companyChanges or headlineChanges:
    #     subprocess.run(['python', 'manage.py', 'sendEmail'])


tup = ('hello', object(), 42)
tup[2]
# This errors, tuples do not support item assignment
# tup[2] = 23

# NamedTuples are immutable containers, they are "write once, read many"
# Different that tuples because all atrributes are named and can be accessed
# through a human-readable identifier
from collections import namedtuple
# Define named tuple Car with two attributes - color and mileage, and we have
# to assign it to the classname that we want. The first parameter is actually
# the typename
Car = namedtuple('Car', 'color mileage')
# Can also pass a normal list as the second argument
# Car = namedtuple('Car', ['color', 'mileage'])

my_car = Car('red', 3000)
print(my_car.color)
print(my_car.mileage)
# Can also use indices, so namedtuples can be a drop in replacement for regular
# tuples
assert my_car[0] == my_car.color
assert my_car[1] == my_car.mileage
print(tuple(my_car))
# Tuple unpacking works too (in Python3)
# print(*my_car)
# This fails, because the attributes are immutable
# my_car.color = 'blue'

# "nampedtuples are a memory-efficient shortcut to defining an immutable class
# in Python manually.

# Can be subclassed, because they are implemented internally as regular classes

class MyCarWithMethods(Car):
    def hexcolor(self):
        if self.color == 'red':
            return '#ff0000'
        else:
            return '#000000'

c = MyCarWithMethods('red', 1234)
print(c.hexcolor())

# This can get messy, though. Adding new immutable fields is weird.
ElectricCar = namedtuple('ElectricCar', Car._fields + ('charge',))

# In nampedtuples, the preprended _ doesn't signal private, they are just named
# as such to be helper methods that don't have name collisions with user
# defined tuples
print(repr(my_car._asdict()))
#  print(json.dumps(my_car._asdict()))

# Creates a shallow copy where you can change some fields
blue_car = my_car._replace(color='blue')
print(repr(blue_car))

# Makes new instances of namedtuple from a sequence or iterable
another_car = Car._make(['red', 999])
print(repr(another_car))