### Finds the ground state of a trapped BEC with rigid body + nonlinear roation
### with the imaginary time method. See Phys. Rev A 102, 011303(R) (2020).
import numpy as np
import sys
import time
#
def kinm(psi):
    #
    kin[1:-1,1:-1] = psi[:-2,1:-1] + psi[2:,1:-1] + psi[1:-1,:-2] + psi[1:-1,2:] - 4*psi[1:-1,1:-1]
    kin[:,0] = kin[:,-1] = kin[0,:] = kin[-1,:] = 0.0
    #
    return kin
#
def momm(psi):
    #
    px[1:-1,1:-1] = psi[:-2,1:-1] - psi[2:,1:-1]
    py[1:-1,1:-1] = psi[1:-1,:-2] - psi[1:-1,2:]
    #
    px[:,0] = px[:,-1] = px[0,:] = px[-1,:] = 0.0
    py[:,0] = py[:,-1] = py[0,:] = py[-1,:] = 0.0
    #
    return px, py
#
def phi(xx,yy,r):
    #
    phi_out = np.arctan2(np.float64(yy)-r[1],np.float64(xx)-r[0])
    #
    return phi_out

def get_en(psi,x,y,dx,dy,U,g,Om):
    #
    ke = 0
    te = 0
    ie = 0
    am = 0
    #
    for ii in xrange(1,psi.shape[1]-1):
        #
        for jj in xrange(1,psi.shape[0]-1):
            #
            mom_x = (psi[jj,ii+1] - psi[jj,ii-1])/(2.0*dx)
            mom_y = (psi[jj+1,ii] - psi[jj-1,ii])/(2.0*dy)
            #
            ke = ke + 0.5*(np.abs(mom_x)**2 + np.abs(mom_y)**2)
            #
            te = te + U[jj][ii]*np.abs(psi[jj][ii])**2
            ie = ie + 0.5*g*np.abs(psi[jj][ii])**4
            #
            am = am + 1j*np.conj(psi[jj][ii])*(Om[0] + Om[1]*np.abs(psi[jj][ii])**2)*(x[jj]*mom_y - y[ii]*mom_x)
            #
        #
    #
    nm = (dx*dy)*(psi*np.conj(psi)).sum()
    en = (dx*dy)*(ke + te + ie + np.real(am))/nm
    #
    return en
#
def track(psi,x,y,dx,dy,g,ep):
    #
    ry, rx = (np.sqrt(2)*pow(g*ep/np.pi,0.25),(np.sqrt(2)/ep)*pow(g*ep/np.pi,0.25))
    #
    den = np.abs(psi)**2
    vort_pos = [[],[]]
    #
    for ii in range(0,psi.shape[0]):
        #
        for jj in range(0,psi.shape[1]):
            #
            if (x[ii]/rx)**2 + (y[jj]/ry)**2 <= 1:
                #
                if (den[ii-1][jj] > den[ii][jj] < den[ii+1][jj]) and (den[ii][jj-1] > den[ii][jj] < den[ii][jj+1]):
                    #
                    vort_pos = np.append(vort_pos,[[x[ii]],[y[jj]]],axis=1)
                    #
                #
            #
        #
    #
    return vort_pos
#
def imt(Dt,Dx,Dy,tol,g,Om,ep):
    #
    x = np.arange(-Dx[0],Dx[0],Dx[1])
    y = np.arange(-Dy[0],Dy[0],Dx[1])
    #
    yy, xx = np.complex128(np.meshgrid(x, y))
    dx = Dx[1]; dy = Dy[1];
    #
    Nx = x.shape[0]
    Ny = y.shape[0]
    #
    psi0 = np.ones((Nx,Ny)) + 1e-3*np.random.rand(Nx,Ny) + 1j*1e-3*np.random.rand(Nx,Ny)
    psi = psi0/np.sqrt(dx*dy*(psi0*np.conj(psi0)).sum())
    psi_new = np.zeros((Nx,Ny),dtype=psi.dtype)
    #
    en_er = np.random.rand(1)
    en_old = np.random.rand(1)
    #
    en_store = []
    am_store = []
    ee_store = []
    er_store = []
    nv_store = []
    #
    U = 0.5*(xx**2 + (ep*yy)**2).T
    #
    tt = 0
    tt_min = 5e5
    run_flag = True
    t0 = time.time()
    dd = 0.5/dx**2
    xxT = xx.T
    yyT = yy.T
    psi_phi = np.exp(1j*(phi(xx,yy,[1,0])+phi(xx,yy,[-2,0])))
    #
    while run_flag:
        #
        px, py = momm(psi)
        Lz = 1j*(yyT*py - xxT*px)/(2*dx)
        #
        psi_new = psi -Dt*(-dd*kinm(psi) + U*psi - Om[0]*Lz + (g*psi - Om[1]*Lz)*psi*np.conj(psi))
        #
        if divmod(tt,5e4)[1]==0 and tt != 0:
            #
            en = get_en(psi=psi_new,x=x,y=y,dx=Dx[1],dy=Dy[1],U=U,g=g,Om=Om)
            en_er = np.abs((en-en_old)/en)
            en_old = en
            #
            if tt <= tt_min:
                #
                run_flag = True
                #
            elif tt > tt_min and en_er < tol:
                #
                run_flag = False
                #
            #
            px, py = momm(psi_new)
            #
            Lz_new = 1j*(yyT*py - xxT*px)/(2*dx)
            am = dx*dy*(np.conj(psi_new)*Lz_new).sum()
            vort_pos = track(psi=psi_new,x=x,y=y,dx=Dx[1],dy=Dy[1],g=g,ep=ep)
            #
            sig_x = np.real(np.sqrt(dx*dy*((yyT**2)*psi_new*np.conj(psi_new)).sum()))
            sig_y = np.real(np.sqrt(dx*dy*((xxT**2)*psi_new*np.conj(psi_new)).sum()))
            #
            en_store.append(en)
            am_store.append(am)
            ee_store.append(sig_y/sig_x)
            er_store.append(en_er)
            #
            if (len(vort_pos[0]) != 0) and (len(vort_pos[1]) != 0):
                #
                nv_store.append(vort_pos.shape[1])
                nv_out = vort_pos.shape[1]
                #
            else:
                #
                nv_store.append(0)
                nv_out = 0
                #
            #
            print('-----------------------')
            print('Energy: ' + repr(en))
            print('Error: ' + repr(en_er))
            print('Ang. Mom. :' + repr(am))
            print('There are ' + repr(nv_out) + ' vortices')
            print('Asp. rat. sy/sx: ' + repr(sig_y/sig_x))
            print('Itterations: ' + repr(tt))
            #
        #
        if divmod(tt,1e5)[1]==0 and tt != 0:
            #
            if np.float(sys.argv[2]) >= 0:
                #
                np.savez('data/run_2/om_'+repr(int(sys.argv[1]))+'_c_'+repr(int(sys.argv[2]))+'_ep_'+repr(int(sys.argv[3]))+'/it_'+repr(tt)+'.npz',gnd=psi_new,x=x,y=y,g=g,Om=Om,ep=ep,en=en,vort_pos=vort_pos,en_store=en_store,am_store=am_store,ee_store=ee_store,er_store=er_store,nv_store=nv_store)
                #
            elif np.float(sys.argv[2]) < 0:
                #
                np.savez('data/run_2/om_'+repr(int(sys.argv[1]))+'_c_m'+repr(abs(int(sys.argv[2])))+'_ep_'+repr(int(sys.argv[3]))+'/it_'+repr(tt)+'.npz',gnd=psi_new,x=x,y=y,g=g,Om=Om,ep=ep,en=en,vort_pos=vort_pos,en_store=en_store,am_store=am_store,ee_store=ee_store,er_store=er_store,nv_store=nv_store)
                #
            #
            if tt == 1e5:
                #
                print('-----------------------')
                print('First 1e5 Itterations in: ' + repr(time.time()-t0) + 's')
                #
            else:
                #
                print('-----------------------')
                print('Last 1e5 Itterations in: ' + repr(time.time()-tn) + 's')
                #
            #
            tn = time.time()
            #
        #
        psi_new = psi_new/np.sqrt(dx*dy*(psi_new*np.conj(psi_new)).sum())
        psi_new = np.abs(psi_new)*psi_phi
        #
        psi = psi_new
        #
        tt += 1
        #
    #
    print('Energy: ' + repr(en))
    print('Error: ' + repr(en_er))
    print('Itterations: ' + repr(tt))
    #
    return psi, en, vort_pos, en_store, am_store, ee_store, er_store, nv_store, x, y
#
Dx = (np.float(sys.argv[4]),np.float(sys.argv[6]))
Dy = (np.float(sys.argv[5]),np.float(sys.argv[6]))
Dt = np.float(sys.argv[7])
tol = 1e-10
g = 300
#
Om = (np.float(sys.argv[1])/100.0,np.float(sys.argv[2])) #Om,C
ep = np.float(sys.argv[3])/100.0
#
kin = np.zeros((int(2*Dx[0]/Dx[1]),int(2*Dy[0]/Dy[1])),dtype='complex128')
px = np.zeros((int(2*Dx[0]/Dx[1]),int(2*Dy[0]/Dy[1])),dtype='complex128')
py = np.zeros((int(2*Dx[0]/Dx[1]),int(2*Dy[0]/Dy[1])),dtype='complex128')
#
t0 = time.time()
gnd, en, vort_pos, en_store, am_store, ee_store, er_store, nv_store, x, y = imt(Dt=Dt,Dx=Dx,Dy=Dy,tol=tol,g=g,Om=Om,ep=ep)
print(repr(time.time()-t0))
#
if np.float(sys.argv[2]) >= 0:
    #
    np.savez('data/run_2/dyn_om_'+repr(int(sys.argv[1]))+'_c_'+repr(int(sys.argv[2]))+'_ep_'+repr(int(sys.argv[3]))+'.npz', Dt=Dt, Dx=Dx, Dy=Dy, tol=tol, g=g, Om=Om, ep=ep, gnd=gnd, en=en, en_store=en_store, am_store=am_store, er_store=er_store, nv_store=nv_store, x=x, y=y)
    #
elif np.float(sys.argv[2]) < 0:
    #
    np.savez('data/run_2/dyn_om_'+repr(int(sys.argv[1]))+'_c_m'+repr(abs(int(sys.argv[2])))+'_ep_'+repr(int(sys.argv[3]))+'.npz', Dt=Dt, Dx=Dx, Dy=Dy, tol=tol, g=g, Om=Om, ep=ep, gnd=gnd, en=en, en_store=en_store, am_store=am_store, er_store=er_store, nv_store=nv_store, x=x, y=y)
    #
#


import tensorflow as tf
import os
from matplotlib import pyplot as plt
import numpy as np
from sklearn.utils import shuffle
from skimage import transform

# get all samples into memory
def load_sample(sample_dir, shuffle_flag):
    print("loading sample dataset ...")
    lfilenames = []
    labelsnames = []
    for dirpath, dirnames, filenames in os.walk(sample_dir):
        for filename in filenames:
            filename_path = os.sep.join([dirpath, filename])
            lfilenames.append(filename_path)
            labelsnames.append(dirpath.split('\\')[-1])

    # create label list
    lab = list(sorted(set(labelsnames)))
    labdict = dict(zip(lab, list(range(len(lab)))))
    labels = [labdict[i] for i in labelsnames]
    if shuffle_flag:
        return shuffle(
            np.asarray(lfilenames),
            np.asarray(labels)
        ), np.asarray(lab)
    else:
        return (np.asarray(lfilenames), np.asarray(labels)), np.asarray(lab)

def _distorted_image(image, size, ch=1, shuffleflag=False, cropflag=False,
                     brightnessflag=False, contrastflag=False):
    # flip image left and right
    distorted_images = tf.image.random_flip_left_right(image)
    # random cropping
    if cropflag is True:
        distorted_images = tf.image.random_crop(distorted_images, [size[0], size[1], ch])

    # flip image up and down
    distorted_images = tf.image.random_flip_up_down(distorted_images)

    if brightnessflag is True:
        tf.image.random_brightness(distorted_images, max_delta=10)

    if contrastflag is True:
        distorted_images = tf.image.random_contrast(distorted_images, lower=0.2, upper=1.8)

    if shuffleflag is True:
        distorted_images = tf.random.shuffle(distorted_images)

    return distorted_images

def _norm_image(image, size, ch=1, flattenflag=False):
    image_decoded = image / 255.0
    if flattenflag is True:
        image_decoded = tf.reshape(image_decoded, [size[0] * size[1] * ch])
    return image_decoded

def _random_rotated30(image, label):
    def _rotated(image):
        shift_y, shift_x = np.array(image.shape.as_list[:2], np.float32)
        tf_rotate = transform.SimilarityTransform(rotation=np.deg2rad((30)))
        tf_shift = transform.SimilarityTransform(translation=[-shift_x, -shift_y])
        tf_shift_inv, image.size = transform.SimilarityTransform(
            translation=[shift_x, shift_y]
        ), image.shape
        image_rotated = transform.warp(image, (tf_shift + (tf_rotate + tf_shift_inv)).inverse())
        return image_rotated

    def _rotatedwrap():
        image_rotated = tf.py_function(_rotated, [image][tf.float64])
        return tf.cast(image_rotated, tf.float32)[0]

    a = tf.random.uniform([1], 0, 2, tf.int32)
    image_decoded = tf.cond(tf.equal(tf.constant(0), a[0]), lambda: image, _rotatedwrap)
    return image_decoded, label

def dataset(directory, size, batch_size, random_rotated=False):  #
    (filenames, labels), _ = load_sample(directory, shuffle_flag=False)

    def _parseone(filename, label):
        image_string = tf.io.read_file(filename)
        image_decoded = tf.image.decode_image(image_string)
        image_decoded.set_shape([None, None, None])
        image_decoded = _distorted_image(image_decoded, size)
        image_decoded = tf.image.resize(image_decoded, size)
        image_decoded = _norm_image(image_decoded, size)
        image_decoded = tf.cast(image_decoded, dtype=tf.float32)
        label = tf.cast(tf.reshape(label, []), dtype=tf.int32)
        return image_decoded, label

    datasets = tf.data.Dataset.from_tensor_slices(
        (filenames, labels)
    )
    datasets = datasets.map(_parseone)

    if random_rotated is True:
        datasets = dataset.map(_random_rotated30)

    datasets = datasets.batch(batch_size)
    return datasets

def showresult(subplot, title, thisimg):
    p = plt.subplot(subplot)
    p.axis('off')
    p.imshow(thisimg)
    p.set_title(title)

def showimg(index, label, img, ntop):
    plt.figure(figsize=(20, 10))
    plt.axis('off')
    ntop = min(ntop,9)
    print(index)
    for i in range(ntop):
        showresult(100+10*ntop+1+i,label[i],img[i])
    plt.show()

if __name__ == "__main__":
    sample_dir = "samples/man_woman"
    size = [96,96]
    batchsize = 10
    datasets = dataset(sample_dir, size=size, batch_size=batchsize)
    datasets2 = dataset(sample_dir,size,batchsize)
    for step,value in enumerate(datasets):
        showimg(step,value[1].numpy(),np.asarray(value[0]*255,np.uint8),10)


import time

import torch
import torch.backends.cudnn as cudnn
from sklearn.metrics.ranking import roc_auc_score
from tensorboardX import SummaryWriter
from torch.utils.data import DataLoader
from torchvision.datasets import ImageFolder

from convnet_models import ConvnetModel

CPU = torch.device("cpu")

EMA_ALPHA_TRAIN_LOSS = 0.99

def train(*, path_data_train, path_data_valid, path_log, path_model, model_name, model_pretrained, batch_size,
          epoch_num, checkpoint, device, transform_train, transform_valid, optimizer_fn):
    model = ConvnetModel(model_name, class_count=1, is_trained=model_pretrained).to(device)
    data_loader_train = DataLoader(
        ImageFolder(path_data_train, transform=transform_train),
        batch_size=batch_size, shuffle=True, num_workers=20, pin_memory=True)
    data_loader_valid = DataLoader(
        ImageFolder(path_data_valid, transform=transform_valid),
        batch_size=batch_size, shuffle=False, num_workers=20, pin_memory=True)
    optimizer, scheduler = optimizer_fn(model.parameters())
    # -------------------- SETTINGS: LOSS
    loss_fn = torch.nn.BCEWithLogitsLoss(size_average=True)
    # ---- Load checkpoint
    if checkpoint:
        model_checkpoint = torch.load(checkpoint)
        model.load_state_dict(model_checkpoint['state_dict'])
        optimizer.load_state_dict(model_checkpoint['optimizer'])
    # Write Tensorboard
    writer = SummaryWriter(log_dir=path_log)
    # ---- TRAIN THE NETWORK
    loss_min, auroc_max = epoch_valid(model, data_loader_valid, device=device)
    print('Initial: valid-loss= ' + str(loss_min) + "  auroc= " + str(auroc_max))
    writer.add_scalar(tag="valid-loss", scalar_value=loss_min, global_step=0)
    writer.add_scalar(tag="valid-auroc", scalar_value=auroc_max, global_step=0)
    for epoch in range(0, epoch_num):
        train_loss = epoch_train(model, data_loader_train, optimizer, loss_fn, device=device)
        writer.add_scalar(tag="train-loss", scalar_value=train_loss, global_step=epoch + 1)
        loss, auroc = epoch_valid(model, data_loader_valid, device=device)
        timestamp_now = time.strftime("%Y%m%d") + '-' + time.strftime("%H%M%S")
        scheduler.step(loss)
        writer.add_scalar(tag="valid-loss", scalar_value=loss, global_step=epoch + 1)
        writer.add_scalar(tag="valid-auroc", scalar_value=auroc, global_step=epoch + 1)
        save_flag, save_flag_l, save_flag_a = '----', '-', '-'
        if loss < loss_min:
            save_flag, save_flag_l = 'save', 'L'
            loss_min = loss
            torch.save({
                'epoch': epoch + 1,
                'state_dict': model.state_dict(),
                'best_loss': loss_min,
                'optimizer': optimizer.state_dict()
            }, path_model + "-L.pth.tar")
        if auroc > auroc_max:
            save_flag, save_flag_a = 'save', 'A'
            auroc_max = auroc
            torch.save({
                'epoch': epoch + 1,
                'state_dict': model.state_dict(),
                'best_loss': loss_min,
                'optimizer': optimizer.state_dict()
            }, path_model + "-A.pth.tar")
        print('Epoch [{0}] [{1}] [{2}] train-loss={3}  valid-loss={4}  auroc={5}'.format(
            epoch + 1,
            save_flag + save_flag_l + save_flag_a,
            timestamp_now,
            train_loss,
            loss,
            auroc
        ))

        # if loss < loss_min:
        #     loss_min = loss
        #     torch.save({'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_loss': loss_min,
        #                 'optimizer': optimizer.state_dict()}, path_model)
        #     print('Epoch [' + str(epoch + 1) + '] [save] [' + timestamp_now + '] train-loss= ' + str(
        #         train_loss) + '  valid-loss= ' + str(loss) + '  auroc= ' + str(auroc))
        #     writer.add_scalar(tag="valid-loss", scalar_value=loss, global_step=epoch + 1)
        #     writer.add_scalar(tag="valid-auroc", scalar_value=auroc, global_step=epoch + 1)
        # else:
        #     print('Epoch [' + str(epoch + 1) + '] [----] [' + timestamp_now + '] train-loss= ' + str(
        #         train_loss) + '  valid-loss= ' + str(loss) + '  auroc= ' + str(auroc))
        #     writer.add_scalar(tag="valid-loss", scalar_value=loss, global_step=epoch + 1)
        #     writer.add_scalar(tag="valid-auroc", scalar_value=auroc, global_step=epoch + 1)
    writer.close()

def epoch_train(model, data_loader, optimizer, loss_fn, device):
    model.train()
    running_loss, running_norm = 0.0, 0.0
    for (x, y) in data_loader:
        x = x.to(device)
        y = y.to(torch.float32).to(device)
        loss = loss_fn(model(x).view(-1), y)
        running_loss = running_loss * EMA_ALPHA_TRAIN_LOSS + loss.item()
        running_norm = running_norm * EMA_ALPHA_TRAIN_LOSS + 1.0
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    return running_loss / running_norm

def epoch_valid(model, data_loader, device):
    true = torch.LongTensor()
    score = torch.FloatTensor()
    bce = []
    model.eval()
    for (x, y) in data_loader:
        bs, n_crops, c, h, w = x.size()
        x = x.to(device)
        true = torch.cat((true, y), 0)
        with torch.no_grad():
            y_hat = torch.nn.Sigmoid()(model(x.view(-1, c, h, w)))
        y_hat = y_hat.to(CPU)
        y_hat = y_hat.view(bs, n_crops).mean(1)
        score = torch.cat((score, y_hat), 0)
        loss_fn = torch.nn.BCELoss(size_average=True)
        bce.append(loss_fn(y_hat, y.to(torch.float32)))
    return torch.Tensor(bce).mean().item(), compute_auroc(true, score)

def compute_auroc(true, score):
    true = true.cpu().numpy()
    score = score.cpu().numpy()
    # roc_auc_score(true, score)
    return roc_auc_score(true, score)

def test(*, path_data, path_model, model_name, model_pretrained, batch_size, device, transform):
    cudnn.benchmark = True
    model = ConvnetModel(model_name, class_count=1, is_trained=model_pretrained).to(device)
    model_checkpoint = torch.load(path_model + ".pth.tar")
    model.load_state_dict(model_checkpoint['state_dict'])
    data_loader_test = DataLoader(
        ImageFolder(path_data, transform=transform),
        batch_size=batch_size, shuffle=False, num_workers=10, pin_memory=True)
    true = torch.LongTensor()
    score = torch.FloatTensor()
    bce = []
    model.eval()
    for (x, y) in data_loader_test:
        bs, n_crops, c, h, w = x.size()
        x = x.to(device)
        true = torch.cat((true, y), 0)
        with torch.no_grad():
            y_hat = torch.nn.Sigmoid()(model(x.view(-1, c, h, w)))
        y_hat = y_hat.to(CPU)
        y_hat = y_hat.view(bs, n_crops).mean(1)
        score = torch.cat((score, y_hat), 0)
        loss_fn = torch.nn.BCELoss(size_average=True)
        bce.append(loss_fn(y_hat, y.to(torch.float32)))
    loss, auroc = torch.Tensor(bce).mean().item(), compute_auroc(true, score)
    auroc = compute_auroc(true, score)
    print('loss = ' + str(loss) + '  AUROC = ' + str(auroc))
    return


from os.path import dirname, abspath
from pathlib import Path
from typing import Any
from ntpath import split
from os import makedirs
from json import dumps, load
from paraview.simple import *

SRC_DIR = Path(dirname(abspath(__file__)))
PROJ_DIR = SRC_DIR.parent

class SharedState(object):
    case_list = []
    m_widget = None
    cam_props_for_del = []
    slices_for_del = []
    cases_for_del = []

class FileHandling(object):
    @classmethod
    def write_json(cls, obj, path: str) -> None:
        def get_object_dict(d):
            return d.__dict__

        filepath, _ = split(path)
        if filepath:
            makedirs(filepath, exist_ok=True)
        with open(path, "w") as out:
            json_string = dumps(obj, default=get_object_dict)
            out.write(json_string)

    @classmethod
    def read_json(cls, inp: Path, object_hook_=None) -> Any:
        with open(inp, "r") as fin:
            data = load(fin, object_hook=object_hook_)

        return data

    @classmethod
    def read_foamcase(cls, inp: Path):
        foamcase_path = inp / "temp.foam"
        cls.write_file(foamcase_path)
        foamcase = OpenFOAMReader(FileName=str(foamcase_path))
        Path.unlink(foamcase_path)
        return foamcase

    @classmethod
    def write_file(cls, file_path: Path, text: str = "", mode: str = 'w'):
        dir_path = file_path.parent
        dir_path.mkdir(parents=True, exist_ok=True)
        with open(file_path, mode) as out:
            out.write(text)

if __name__ == "__main__":
    print(PROJ_DIR)


import pygame
import random

pygame.init()
SCREEN_WIDTH = 700
SCREEN_HEIGHT = 500

# screen = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))

pygame.display.set_caption("Glass Ceiling")

background = pygame.image.load ("C:\Desktop\Adobe-SIP-2016\week6/background.jpg")
screen = pygame.display.set_mode(background_size)
done = False

# clock = pygame.time.clock()

# -------- Main Program Loop -----------
while not done:
	for event in pygame.event.get():
		if event.type == pygame.QUIT:
			done = True

	# screen.fill(WHITE)
	# gameDisplay.blit(bg, (0, 0))

	pygame.display.flip()

	# clock.tick(60)

pygame.quit()
exit()


# Copyright (c) 2018, ITRI.  All rights reserved.
from __future__ import absolute_import

from .tensorflow_train import TensorflowTrainTask
from myelindl.utils import subclass, override
from myelindl.log import logger
import re

@subclass
class BatchSizeEstimateTask(TensorflowTrainTask):

    def __init__(self, start_batch_size=32, **kwargs):
        super(BatchSizeEstimateTask, self).__init__(**kwargs)
        self.start_batch_size = start_batch_size
        self.max_batch_size = None

    def __getstate__(self):
        return super(BatchSizeEstimateTask, self).__getstate__()

    def __setstate__(self, state):
        super(BatchSizeEstimateTask, self).__setstate__(state)

    # Task overrides
    @override
    def name(self):
        return 'Est Tensorflow Model Batch size'

    @override
    def before_run(self):
        super(BatchSizeEstimateTask, self).before_run()

    @override
    def task_arguments(self, resources, env):
        args = super(BatchSizeEstimateTask, self).task_arguments(resources, env)
        args.append('--test_max_batch_size=true')
        args.append('--start_batch_size=%d' % self.start_batch_size)
        return args

    @override
    def send_parameters_update(self):
        pass

    def send_current_batch_size_update(self, current_test_batch):
        from myelindl.webapp import app, socketio
        with app.app_context():
            socketio.emit('task update',
                          {'task': self.html_id(),
                           'update': 'test_batch_size',
                           'value': current_test_batch,
                           },
                          namespace='/jobs',
                          room=self.job_id)

    def send_max_batch_size_update(self, max_batch_size):
        from myelindl.webapp import app, socketio
        with app.app_context():
            socketio.emit('task update',
                          {'task': self.html_id(),
                           'update': 'max_batch_size',
                           'value': max_batch_size,
                           },
                          namespace='/jobs',
                          room=self.job_id)

    @override
    def process_output(self, line):
        self.tensorflow_log.write('%s\n' % line)
        self.tensorflow_log.flush()
        timestamp, level, message = self.preprocess_output_tensorflow(line)
        if not message:
            return True
        match = re.match(r'Batch size: (.*)', message)
        if match:
            self.logger.info(message)
        match = re.match(r'Test Batch Size (.*?),\s*', message)
        if match:
            current_test_batch = int(match.group(1))
            self.logger.info(message)
            self.send_current_batch_size_update(current_test_batch)
            return True

        match = re.match(r'Max Batch Size found (.*).$', message)
        if match:
            self.max_batch_size = int(match.group(1))
            self.logger.info(message)
            return True
        return True

    @override
    def after_runtime_error(self):
        super(BatchSizeEstimateTask, self).after_runtime_error()
        logger.info(" after runtime error")
        from myelindl.webapp import app, socketio
        parameter_list = self.get_parameters()
        with app.app_context():
            socketio.emit('task update',
                          {'task': self.html_id(),
                           'update': 'test_batch_size_err',
                           },
                          namespace='/jobs',
                          room=self.job_id)

    @override
    def after_run(self):
        super(BatchSizeEstimateTask, self).after_run()
        logger.info(" after run")
        if not self.max_batch_size:
            from myelindl.webapp import app, socketio
            parameter_list = self.get_parameters()
            with app.app_context():
                socketio.emit('task update',
                              {'task': self.html_id(),
                               'update': 'test_batch_size_err',
                               },
                              namespace='/jobs',
                              room=self.job_id)
        else:
            self.send_max_batch_size_update(self.max_batch_size)


import numpy as np
import sys
from scipy.special import logsumexp
from scipy.stats import linregress
from scipy.integrate import RK45, solve_ivp
from timeout import *

def choose_denom(P):
    """Pick a denominator for additive log-ratio transformation.
    """
    np.seterr(divide="ignore", invalid="ignore")
    log_change = None
    for p in P:
        s = p.sum(axis=1,keepdims=True)
        s[s==0] = 1
        deltas = np.log( (p/s)[1:] ) - np.log( (p/s)[:-1] )
        if log_change is None:
            log_change = deltas
        else:
            log_change = np.vstack((log_change, deltas))
    np.seterr(divide="warn", invalid="warn")
    # pick taxon with smallest change in log proportion
    min_idx = -1
    min_var = np.inf
    ntaxa = log_change.shape[1]
    for i in range(ntaxa):
        if not np.all(np.isfinite(log_change[:,i])):
            continue
        var = np.var(log_change[:,i])
        if var < min_var:
            min_idx = i
            min_var = var

    if min_idx == -1:
        print("Error: no valid denominator found", file=sys.stderr)
        exit(1)

    return min_idx

def construct_alr(P, denom, pseudo_count=1e-3):
    """Compute the additive log ratio transformation with a given
    choice of denominator. Assumes zeros have been replaced with
    nonzero values.
    """
    ALR = []
    ntaxa = P[0].shape[1]
    numer = np.array([i for i in range(ntaxa) if i != denom])
    for p in P:
        p = np.copy(p)
        p = (p + pseudo_count) / (p + pseudo_count).sum(axis=1,keepdims=True)
        alr = (np.log(p[:,numer]).T - np.log(p[:,denom])).T
        ALR.append(alr)
    return ALR

class LinearALR:
    """Inference for the linear model under the additive log-ratio transformation
    """

    def __init__(self, P=None, T=None, U=None, denom=None, pseudo_count=1e-3):
        """
        Parameters
        ----------
            P : A list of T_x by D dimensional numpy arrays of
                estimated relative abundances.
            T : A list of T_x by 1 dimensional numpy arrays giving
                the times of each observation x.
            U : An optional list of T_x by P numpy arrays of external
                perturbations for each x.
            denom : integer id for taxa in denominator of log ratio
        """
        self.P = P
        self.T = T

        if P is not None and denom is None:
            self.denom = choose_denom(P)
            self.X = construct_alr(P, self.denom, pseudo_count)
        elif P is not None and denom is not None:
            self.denom = denom
            self.X = construct_alr(P, denom, pseudo_count)
        else:
            self.X = None

        if U is None and self.X is not None:
            self.U = [ np.zeros((x.shape[0], 1)) for x in self.X ]
            self.no_effects = True
        else:
            self.U = U
            self.no_effects = False

        # Parameter estimates
        self.A = None
        self.g = None
        self.B = None
        self.Q_inv = np.eye(self.P[0].shape[1]-1) if P is not None else None

        # Regularization parameters
        self.alpha = None
        self.r_A = None
        self.r_g = None
        self.r_B = None

    def get_regularizers(self):
        return self.alpha, self.r_A, self.r_g, self.r_B

    def set_regularizers(self, alpha, r_A, r_g, r_B):
        self.alpha = alpha
        self.r_A = r_A
        self.r_g = r_g
        self.r_B = r_B

    def train(self, verbose=False, folds=10):
        """Estimate regularization parameters and CLV model parameters.
        """
        if self.alpha is None or self.r_A is None or self.r_g is None or self.r_B is None:
            if verbose:
                print("Estimating regularizers...")
            self.alpha, self.r_A, self.r_g, self.r_B = estimate_elastic_net_regularizers_cv(self.X, self.P, self.U, self.T, self.denom, folds=folds, no_effects=self.no_effects, verbose=verbose)

        if verbose:
            print("Estimating model parameters...")
        self.A, self.g, self.B = elastic_net_alr(self.X, self.U, self.T, self.Q_inv, self.alpha, self.r_A, self.r_g, self.r_B)

        if verbose:
            print()

    def predict(self, p0, times, u = None):
        """Predict relative abundances from initial conditions.

        Parameters
        ----------
            p0     : the initial observation, a D-dim numpy array
            times  : a T by 1 numpy array of sample times
            u      : a T by P numpy array of external perturbations

        Returns
        -------
            y_pred : a T by D numpy array of predicted relative
                     abundances. Since we cannot predict initial
                     conditions, the first entry is set to the array
                     of -1.
        """
        if u is None:
            u = np.zeros((times.shape[0], 1))

        if p0.ndim == 1:
            p0 = p0.reshape((1, p0.size))

        X = construct_alr([p0], self.denom)
        x = X[0]

        return predict(x, p0, u, times, self.A, self.g, self.B, self.denom)

    def get_params(self):
        A = np.copy(self.A)
        g = np.copy(self.g)
        B = np.copy(self.B)
        return A, g, B

def elastic_net_alr(X, U, T, Q_inv, alpha, r_A, r_g, r_B, tol=1e-3, verbose=False, max_iter=10000):

    def gradient(AgB, x_stacked, pgu_stacked):
        f = x_stacked - AgB.dot(pgu_stacked.T).T
        grad = Q_inv.dot(f.T.dot(pgu_stacked))

        # l2 regularization terms
        A = AgB[:,:yDim]
        g = AgB[:,yDim:(yDim+1)]
        B = AgB[:,(yDim+1):]
        grad[:,:yDim] += -2*alpha*(1-r_A)*A
        grad[:,yDim:(yDim+1)] += -2*alpha*(1-r_g)*g
        grad[:,(yDim+1):] += -2*alpha*(1-r_B)*B
        return -grad

    def generalized_gradient(AgB, grad, step):
            nxt_AgB = prv_AgB - step*grad

            # threshold A
            A_prox = nxt_AgB[:,:yDim]
            A_prox[A_prox < -step*alpha*r_A] += step*alpha*r_A
            A_prox[A_prox > step*alpha*r_A] -= step*alpha*r_A
            A_prox[np.logical_and(A_prox >= -step*alpha*r_A, A_prox <= step*alpha*r_A)] = 0

            # threshold g
            g_prox = nxt_AgB[:,yDim:(yDim+1)]
            g_prox[g_prox < -step*alpha*r_g] += step*alpha*r_g
            g_prox[g_prox > step*alpha*r_g] -= step*alpha*r_g
            g_prox[np.logical_and(g_prox >= -step*alpha*r_g, g_prox <= step*alpha*r_g)] = 0

            # threshold B
            B_prox = nxt_AgB[:,(yDim+1):]
            B_prox[B_prox < -step*alpha*r_B] += step*alpha*r_B
            B_prox[B_prox > step*alpha*r_B] -= step*alpha*r_B
            B_prox[np.logical_and(B_prox >= -step*alpha*r_B, B_prox <= step*alpha*r_B)] = 0

            AgB_proximal = np.zeros(AgB.shape)
            AgB_proximal[:,:yDim] = A_prox
            AgB_proximal[:,yDim:(yDim+1)] = g_prox
            AgB_proximal[:,(yDim+1):] = B_prox

            return (AgB - AgB_proximal)/step

    def objective(AgB, x_stacked, pgu_stacked):
        f = x_stacked - AgB.dot(pgu_stacked.T).T
        obj = -0.5*(f.dot(Q_inv)*f).sum()

        return -obj

    def stack_observations(X, U, T):
        # number of observations by xDim
        x_stacked = None
        # number of observations by xDim + 1 + uDim
        pgu_stacked = None
        for x, u, times in zip(X, U, T):
            for t in range(1, times.size):
                dt = times[t] - times[t-1]
                xt0 = x[t-1]
                gt0 = np.ones(1)
                ut0 = u[t-1]
                pgu = np.concatenate((xt0, gt0, ut0))

                if x_stacked is None:
                    x_stacked = x[t] - x[t-1]
                    pgu_stacked = dt*pgu

                else:
                    x_stacked = np.vstack((x_stacked, x[t] - x[t-1]))
                    pgu_stacked = np.vstack((pgu_stacked, dt*pgu))

        return x_stacked, pgu_stacked

    xDim = X[0].shape[1]
    yDim = xDim + 1
    uDim = U[0].shape[1]

    A,g,B = ridge_regression_alr(X, U, T, np.max((alpha*(1-r_A), 0.01)), np.max((alpha*(1-r_g), 0.01)), np.max((alpha*(1-r_B), 0.01)))
    AgB = np.zeros(( xDim, xDim + 1 + uDim ))
    AgB[:,:xDim] = A
    AgB[:,xDim:(xDim+1)] = np.expand_dims(g,axis=1)
    AgB[:,(xDim+1):] = B

    x_stacked, pgu_stacked = stack_observations(X, U, T)
    prv_obj = np.inf
    obj = objective(AgB, x_stacked, pgu_stacked)

    it = 0
    while np.abs(obj - prv_obj) > tol:
        np.set_printoptions(suppress=True)
        prv_AgB = np.copy(AgB)
        prv_obj = obj

        # line search
        step = 0.1
        grad = gradient(prv_AgB, x_stacked, pgu_stacked)
        gen_grad = generalized_gradient(prv_AgB, grad, step)
        nxt_AgB = prv_AgB - step*gen_grad
        obj = objective(nxt_AgB, x_stacked, pgu_stacked)
        while obj > prv_obj - step*(grad*gen_grad).sum() + step*0.5*np.square(gen_grad).sum():
            step /= 2
            gen_grad = generalized_gradient(prv_AgB, grad, step)
            nxt_AgB = prv_AgB - step*gen_grad
            obj = objective(nxt_AgB, x_stacked, pgu_stacked)

        A = nxt_AgB[:,:yDim]
        g = nxt_AgB[:,yDim:(yDim+1)]
        B = nxt_AgB[:,(yDim+1):]
        AgB[:,:yDim] = A
        AgB[:,yDim:(yDim+1)] = g
        AgB[:,(yDim+1):] = B

        obj = objective(AgB, x_stacked, pgu_stacked)
        it += 1

        if verbose:
            print("\t", it, obj)

        if it > max_iter:
            print("Warning: maximum number of iterations ({}) reached".format(max_iter), file=sys.stderr)
            break

    A = AgB[:,:xDim]
    g = AgB[:,xDim:(xDim+1)].flatten()
    B = AgB[:,(xDim+1):]

    return A, g, B

def ridge_regression_alr(X, U, T, r_A=0, r_g=0, r_B=0):
    """Computes estimates of A, g, and B using least squares.

    Parameters
    ----------
        X : a list of T x yDim-1 numpy arrays
        U : a list of T x uDim numpy arrays
        T   : a list of T x 1 numpy arrays with the time of each observation

    Returns
    -------
    """
    xDim = X[0].shape[1]
    yDim = xDim + 1
    uDim = U[0].shape[1]
    AgB_term1 = np.zeros(( xDim, xDim + 1 + uDim ))
    AgB_term2 = np.zeros(( xDim + 1 + uDim, xDim + 1 + uDim))

    for idx, (xi, ui) in enumerate(zip(X, U)):
        for t in range(1, xi.shape[0]):
            pt = xi[t]
            pt0 = xi[t-1]

            xt  = xi[t]
            xt0 = xi[t-1]

            gt0 = np.ones(1)
            ut0 = ui[t-1]

            pgu = np.concatenate((pt0, gt0, ut0))

            delT = T[idx][t] - T[idx][t-1]
            AgB_term1 += np.outer( (xt - xt0) / delT, pgu)
            AgB_term2 += np.outer(pgu, pgu)

    reg = np.array([r_A for i in range(xDim)] + [r_g] + [r_B for i in range(uDim)])
    reg = np.diag(reg)
    AgB = AgB_term1.dot(np.linalg.pinv(AgB_term2 + reg))
    A = AgB[:,:xDim]
    g = AgB[:,xDim:(xDim+1)].flatten()
    B = AgB[:,(xDim+1):]

    return A, g, B

def estimate_elastic_net_regularizers_cv(X, P, U, T, denom, folds, no_effects=False, verbose=False):
    if len(X) == 1:
        print("Error: cannot estimate regularization parameters from single sample", file=sys.stderr)
        exit(1)
    elif len(X) < folds:
        folds = len(X)

    rs = [0.1, 0.5, 0.7, 0.9, 1]
    alphas = [0.1, 1, 10]

    alpha_rA_rg_rB = []
    for alpha in alphas:
        for r_A in rs:
            for r_g in rs:
                if no_effects:
                    alpha_rA_rg_rB.append((alpha, r_A, r_g, 0))
                else:
                    for r_B in rs:
                        alpha_rA_rg_rB.append((alpha, r_A, r_g, r_B))

    np.set_printoptions(suppress=True)
    best_r = 0
    best_sqr_err = np.inf
    for i, (alpha, r_A, r_g, r_B) in enumerate(alpha_rA_rg_rB):
        #print("\tTesting regularization parameter set", i+1, "of", len(alpha_rA_rg_rB), file=sys.stderr)
        sqr_err = 0
        for fold in range(folds):
            train_X = []
            train_P = []
            train_U = []
            train_T = []

            test_X = []
            test_P = []
            test_U = []
            test_T = []
            for i in range(len(X)):
                if i % folds == fold:
                    test_X.append(X[i])
                    test_P.append(P[i])
                    test_U.append(U[i])
                    test_T.append(T[i])

                else:
                    train_X.append(X[i])
                    train_P.append(P[i])
                    train_U.append(U[i])
                    train_T.append(T[i])

            Q_inv = np.eye(train_X[0].shape[1])
            A, g, B = elastic_net_alr(train_X, train_U, train_T, Q_inv, alpha, r_A, r_g, r_B, tol=1e-3)
            sqr_err += compute_prediction_error(test_X, test_P, test_U, test_T, A, g, B, denom)

        if sqr_err < best_sqr_err:
            best_r = (alpha, r_A, r_g, r_B)
            best_sqr_err = sqr_err
            print("\tr", (alpha, r_A, r_g, r_B), "sqr error", sqr_err)
    np.set_printoptions(suppress=False)
    return best_r

def compute_rel_abun(x, denom):
    if x.ndim == 1:
        x = np.expand_dims(x, axis=0)
    z = np.hstack((x, np.zeros((x.shape[0], 1))))
    p  = np.exp(z - logsumexp(z, axis=1, keepdims=True))
    for i in range(p.shape[1]-1, denom, -1):
        tmp = np.copy(p[:,i-1])
        p[:,i-1] = np.copy(p[:,i])
        p[:,i] = tmp
    return p

@timeout(5)
def predict(x, p, u, times, A, g, B, denom):
    """Make predictions from initial conditions
    """
    def grad_fn(A, g, B, u):
        def fn(t, x):
            return g + A.dot(x) + B.dot(u)
        return fn

    p_pred = np.zeros((times.shape[0], x[0].size+1))
    pt = p[0]
    xt = x[0]

    for i in range(1, times.shape[0]):
        grad = grad_fn(A, g, B, u[i-1])
        dt = times[i] - times[i-1]
        ivp = solve_ivp(grad, (0,0+dt), xt, method="RK45")
        xt = ivp.y[:,-1]
        pt = compute_rel_abun(xt, denom).flatten()
        p_pred[i] = pt
    return p_pred

def compute_prediction_error(X, P, U, T, A, g, B, denom_ids):
    def compute_err(p, p_pred):
        err = 0
        ntaxa = p.shape[1]
        #err += np.mean(np.square(p[1:] - p_pred[1:]).sum(axis=1))
        err += np.square(p[1:] - p_pred[1:]).sum()
        return err/ntaxa
    err = 0
    for x, p, u, t in zip(X, P, U, T):
        try:
            p_pred = predict(x, p, u, t, A, g, B, denom_ids)
            err += compute_err(p, p_pred)
        except TimeoutError:
            err += np.inf
    return err/len(X)


# -*- coding: utf-8 -*-
import sys
import os
import tempfile
from datetime import datetime
import subprocess

import ipaddress
import requests

LOG_TS_FORMAT = "%d/%b/%Y:%H:%M:%S"

def load_aws_networks():
    try:
        r = requests.get('https://ip-ranges.amazonaws.com/ip-ranges.json')
    except Exception as e:
        print('Error making https request : {}'.format(e))
        sys.exit(1)
    if r.status_code != 200:
        return []
    return [ipaddress.ip_network(item.get('ip_prefix')) for item in r.json().get('prefixes')]

AWS_NETWORKS = load_aws_networks()

def from_aws(ip):
    is_aws = False
    for nw in AWS_NETWORKS:
        is_aws = is_aws or (ipaddress.ip_address(ip) in nw)
    return is_aws

def _scraped_off_part_not_i_need(origin_log):
    # *.gz と分ける
    cat_cmd = "cat"
    decompress = " | gunzip " if os.path.splitext(origin_log)[1] == ".gz" else ""
    no_bot_no_assets = '| grep "GET /.*/ HTTP" | grep -v -i "bot"'
    i_need_column = '| cut -d " " -f 1,4,14-' #IP, 日時, User-Agentだけ
    tmp_path = tempfile.mkstemp(prefix='eachip', suffix='.log')[1]
    cmd = '{0} {1} {2} {3} {4} > {5}'.format(
        cat_cmd,
        os.path.abspath(origin_log),
        decompress,
        no_bot_no_assets,
        i_need_column,
        tmp_path)
    print (cmd)
    subprocess.call(cmd, shell=True)
    return tmp_path

def open_log():
    if len(sys.argv) < 2:
        return []
    _, fname = tuple(sys.argv)
    return open(_scraped_off_part_not_i_need(fname))

def timedeltas_each_ip(stream):
    hits_each_ips = {}
    for line in stream:
        ip, ts = tuple(line.replace("[", "").split(" ")[0:2])
        # IP毎にtsの配列化を行う
        if ip not in hits_each_ips.keys():
            hits_each_ips[ip] = []
        hits_each_ips[ip].append(datetime.strptime(ts, LOG_TS_FORMAT))
    # IP毎のアクセスタイムスタンプが得られた
    # アクセスの間隔をIP毎に計算
    return [(ip, [end - begin for begin, end in zip(times[:-1], times[1:])])
                for ip, times in hits_each_ips.items()]

def list_of_ips(timedeltas_each_ip):
    for ip, deltas in timedeltas_each_ip:
        delta_seconds = [d.seconds for d in deltas]
        aws = 'AWS' if from_aws(ip) else ''
        if len(delta_seconds) > 5:
            yield ('{0} {1:.1f} {2} {3}'.format(
                ip,
                sum(delta_seconds)/len(delta_seconds),
                len(delta_seconds),
                aws))

def main():
    print ("ip_address | avg interval(sec) | counts | AWS?")
    print ("----------------------------------------------")
    for out in list_of_ips(timedeltas_each_ip(open_log())):
        print(out)

if __name__ == '__main__':
    main()


# coding: utf-8

# In[1]:

#getting libraries
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as mlt
get_ipython().magic(u'matplotlib inline')

# In[2]:

#getting file
crd1=pd.read_csv('D:\\V data analytics\\sample dataset\\Dataset by Imarticus\\Data for logistic regression\\R_Module_Day_7.2_Credit_Risk_Train_data.csv')
crd2=pd.read_csv('D:\\V data analytics\\sample dataset\\Dataset by Imarticus\\Data for logistic regression\\R_Module_Day_8.2_Credit_Risk_Validate_data.csv')

# In[11]:

# to do missing value imputation, we will concat both train and validation data

crd=pd.concat([crd1,crd2],axis=0)

# In[4]:

crd.isna().sum()

# In[12]:

#lets reset index just to be sure
crd.reset_index(inplace=True,drop=True)

# In[13]:

crd.head()

# In[14]:

#all good lets check for missing values
crd.isna().sum()

# In[15]:

#missing value of imputation, to get the list of indices of gender missing values we do
genmiss=crd[crd['Gender'].isna()].index.tolist()

# In[16]:

#to find out the mode we do,
crd.Gender.value_counts()

# In[17]:

#lets fill up missing value by 'Male'
crd['Gender'].iloc[genmiss]='Male'

# In[18]:

crd.Gender.isna().sum()

# In[19]:

#great, lets do the missing value imputation for married
crd.Married.value_counts()

# In[20]:

#lets fill up nas by 'Yes'
crd['Married'].iloc[crd[crd['Married'].isna()].index.tolist()]='Yes'

# In[21]:

crd.Married.isna().sum()

# In[22]:

#lets do missing value impputation of dependents
crd.Dependents.value_counts()

# In[29]:

crd.Dependents.isna().sum()

# In[30]:

#lets fill the nas of dependents missing value in case of not married people as zero
# so we need the dataframe where married is no and dependents is na
dmiss1=crd[(crd['Married']=='No') & (crd['Dependents'].isna())].index.tolist()
print(dmiss1)

# In[31]:

#filling it up with zero
crd['Dependents'].iloc[dmiss1]='0'

# In[32]:

crd.Dependents.value_counts()

# In[33]:

crd.Dependents.isna().sum()

# In[28]:

#lets do a crosstab of gender with dependents
pd.crosstab(crd['Gender'],crd['Dependents'].isna())  # out of 16, 15 values corresponds to male

# In[38]:

#lets do the cross tab of dependents and gender
pd.crosstab(crd['Gender'],crd['Dependents'])

# In[39]:

#since most of male have a mode of 0, lets fill the remaining values by 0
crd['Dependents'][(crd['Dependents'].isna())]='0'

# In[40]:

crd.Dependents.isna().sum()

# In[41]:

#missing value impution of selfemployed
crd.Self_Employed.isna().sum()

# In[42]:

crd.Self_Employed.value_counts()

# In[43]:

#replacing nas with 'No'
crd['Self_Employed'].iloc[crd[crd['Self_Employed'].isna()].index.tolist()]='No'

# In[44]:

#mvi of loanamount loanamount_term
#lets compare the missingvalues of loanamount with loanamountterm
pd.crosstab(crd['Loan_Amount_Term'],crd['LoanAmount'].isna())

# In[45]:

#its evident that loan_amount_term pertaining to 360 has highest numbers of nas.
# we will fill the value by the mean of the loanamount corresponding to the loan_amount_term 360.
crd.groupby(crd['Loan_Amount_Term'])['LoanAmount'].mean()

# In[46]:

# lets fill the loanampunt na's in the range of 360 by 144
crd['LoanAmount'].iloc[crd[(crd['Loan_Amount_Term']==360) & (crd['LoanAmount'].isna())].index.tolist()]=144

# In[47]:

#for the rest of nas lets replace the values by 132
crd['LoanAmount'].iloc[crd[crd['LoanAmount'].isna()].index.tolist()]=132

# In[48]:

#missing value imputation of loan_amount_term
crd.Loan_Amount_Term.isnull().sum()

# In[49]:

crd.Loan_Amount_Term.value_counts()

# In[50]:

#lets fill up by mode, ie 360
crd['Loan_Amount_Term'].iloc[crd[crd['Loan_Amount_Term'].isna()].index.tolist()]=360

# In[51]:

#missing value imutation of Credit_history by Logistic regression

crd.Credit_History.isnull().sum()

# In[56]:

#lets separate the dataframe as train and test, lets take all na's in credithistory as test data
crd_testdata=crd.loc[crd['Credit_History'].isna(),:]

# In[58]:

#to get traindata, we will first get incex of all na's of credithistory
crd_credna_indx=crd[crd['Credit_History'].isna()].index.tolist()

# In[61]:

#getting traindata
crd_traindata_index=[x for x in crd.index.tolist() if x not in crd_credna_indx]

# In[62]:

crd_traindata=crd.iloc[crd_traindata_index]

# In[64]:

#Tp do a logistic regression, we will lose all unimo variables and get dummies for all catagorical variables
crd_train1=pd.get_dummies(crd_traindata.drop(['Loan_ID'],axis=1),drop_first=True)

# In[65]:

crd_train1.head()

# In[67]:

#lets do the same for testdata
crd_test1=pd.get_dummies(crd_testdata.drop(['Loan_ID'],axis=1),drop_first=True)

# In[68]:

crd_test1.head()

# In[71]:

#preparing xtrain and y train, xtest and ytest
x_train1=crd_train1.drop(['Credit_History','Loan_Status_Y'],axis=1)
y_train1=crd_train1['Credit_History']

# In[73]:

#similarly for test set
x_test1=crd_test1.drop(['Credit_History','Loan_Status_Y'],axis=1)
y_test1=crd_test1['Credit_History']

# In[74]:

from sklearn.linear_model import LogisticRegression

# In[75]:

m1=LogisticRegression()

# In[76]:

m1.fit(x_train1,y_train1)

# In[77]:

#predicting for xtest
pred1=m1.predict(x_test1)

# In[78]:

print(pred1)

# In[83]:

crd_test1['Credit_History']=pred1

# In[84]:

crd_new=pd.concat([crd_train1,crd_test1],axis=0)

# In[85]:

crd_new.isnull().sum()

# In[86]:

crd_new.shape

# In[87]:

crd.Credit_History.isna().sum()

# In[88]:

crd['Credit_History'].iloc[crd[crd['Credit_History'].isna()].index.tolist()]=pred1

# In[89]:

crd.isnull().sum()

# In[90]:

crd.head()

# In[ ]:

# lets go ahead woth logistic regression now

# In[97]:

#some basic eda's first
sns.barplot(x=crd['Loan_Status'],y=crd['LoanAmount'],hue=crd['Gender'],data=crd)

# In[98]:

sns.barplot(x=crd['Credit_History'],y=crd['Loan_Status'],hue=crd['Married'],data=crd)

# In[105]:

sns.barplot(x=crd['Loan_Status'],y=crd['ApplicantIncome'],hue=crd['Property_Area'],data=crd) #(so rural property got rejected most)

# In[104]:

sns.barplot(x=crd['Loan_Status'],y=crd['ApplicantIncome'],hue=crd['Dependents'],data=crd) #interesting fact is applicant with..
#..3+ dependents were rejected also were given higher no of loans!!

# In[ ]:

# lets split the data into train and validation,just like how it was given previously

# In[111]:

crd_train=crd.head(len(crd1))

# In[113]:

crd_train.to_csv('D:\\V data analytics\\sample dataset\\Dataset by Imarticus\\Data for logistic regression\\Loan_eligibility_estimation_traindata_withmv_imputed\\crd_train.csv')

# In[114]:

crd_val=crd.tail(len(crd2))

# In[115]:

crd_val.to_csv('D:\\V data analytics\\sample dataset\\Dataset by Imarticus\\Data for logistic regression\\Loan_eligibility_estimation_train _test_data_withmv_imputed\\crd_val.csv')

# ### Logistic regression with area under roc curve

# In[117]:

#getting dummies for all catagorical variables
crd.info()

# In[120]:

#seperating dvb
crd_new1=pd.get_dummies(crd.drop(['Loan_ID','Loan_Status'],axis=1),drop_first=True)

# In[121]:

crd_new1.head()

# In[122]:

x_train=crd_new1

# In[123]:

y_train=crd['Loan_Status']

# In[124]:

m2=LogisticRegression()

# In[125]:

m2.fit(x_train,y_train)

# In[128]:

m2.score(x_train,y_train)

# In[129]:

# PREDICTING DATA FOR VALIDATION DATA
crd_val.shape

# In[130]:

crd_val.head()

# In[132]:

#preparing data to predict
crd_val1=crd_val.drop(['Loan_ID','Loan_Status'],axis=1)

# In[133]:

#getting dummyvalues for cvs
crdval=pd.get_dummies(crd_val1,drop_first=True)

# In[134]:

crdval.head()

# In[135]:

#predicting
pred2=m2.predict(crdval)

# In[136]:

#getting crosstab
pd.crosstab(crd_val['Loan_Status'],pred2)

# In[137]:

#to get crosstab we download metrics
from sklearn import metrics

# In[138]:

metrics.confusion_matrix(crd_val['Loan_Status'],pred2) #we can interpret the results as predicted
                                                                                #       N      Y
                                                  #          actual loanstatus  N      55      22
                                                  #                             Y      1       289

# In[ ]:

#sensitivity of the model =tp/tp+fn= 289/289+1=99% highly sensitive
#speificity of the model = tn/tn+fp=55/55+22=55/77= 71% speific

# In[140]:

#lets get the valuecount of loanstatus in validation data
crd_val.Loan_Status.value_counts()

# In[141]:

#accuracy score
metrics.accuracy_score(crd_val['Loan_Status'],pred2) #93% great

# In[172]:

crd_val.head()

# In[179]:

crdval.head()

# In[180]:

crdval['Loan_Status']=crd_val['Loan_Status'] #in crdval, we add loanstatus, since we have to conver it to digits by mapping

# In[181]:

crdval.head()

# In[182]:

crdval['Loan_Status_digit']=crdval.Loan_Status.map({'N':0,'Y':1})

# In[184]:

crdval1=crdval.drop(['Loan_Status'],axis=1)

# In[186]:

crdval1.info()

# In[187]:

crdval.info()

# In[196]:

probvalues_of_dv=m2.predict_proba(crdval1.drop(['Loan_Status_digit'],axis=1))

# In[197]:

Loan_status_prob0=m2.predict_proba(crdval1.drop(['Loan_Status_digit'],axis=1))[:,1]

# In[199]:

#graph
sns.distplot(Loan_status_prob0,kde=False,bins=50)

# In[200]:

#roc curve
fpr,tpr,thresholds=metrics.roc_curve(crd_val['Loan_Status_digit'],Loan_status_prob0)

# In[202]:

mlt.plot(fpr,tpr)

# In[203]:

#getting area under aoc
metrics.roc_auc_score(crd_val['Loan_Status_digit'],Loan_status_prob0)

Logistic regression (Area under ROC curve) with missing value imputation.py


import sys
import random

# Takes: Clean List of words | Returns: Updated Histogram
class Histogram(dict):

    def __init__(self, source_text_list=None):
        super(Histogram, self).__init__()
        self.types = 0
        self.tokens = 0
        if source_text_list:
            self.update(source_text_list)

    def update(self, source_text_list):
        for index, word in enumerate(source_text_list):

            if word in self:
                prev_tup = self[word]
                word_histogram = prev_tup[1]

                if index + 1 < len(source_text_list):
                    word_right = source_text_list[index + 1]

                    if word_right in word_histogram:
                        word_histogram[word_right] += 1
                    else:
                        word_histogram[word_right] = 1

                self.tokens += 1
                self[word] = (prev_tup[0] + 1, word_histogram)
            else:
                word_histogram = {}

                if index + 1 < len(source_text_list):
                    word_right = source_text_list[index + 1]
                    word_histogram[word_right] = 1
                else:
                    word_histogram["[END]"] = 1

                self.types += 1
                self.tokens += 1
                self[word] = (1, word_histogram)

    def count(self, item):
        return self.get(item, 0)[0]

def tokenize(source_text):
    return source_text.split()

def random_word(histogram):
    # Using Algorithm 2 (less code)
    probability = 0
    rand_index = 0

    # Handles getting truly random word from all words in source text histogram
    if hasattr(histogram, 'tokens'):
        rand_index = random.randint(1, histogram.tokens)
        for word in histogram:
            probability += histogram[word][0]
            if probability >= rand_index:
                return word

    # Handles getting truly random word from consecutive words histogram
    else:
        rand_index = random.randint(1, sum(histogram.values()))
        for word in histogram:
            probability += histogram[word]
            if probability >= rand_index:
                return word

def sentence(histogram, word_count):
    word_start = random_word(histogram)
    sentence = next_word(histogram, histogram[word_start][1], [word_start], word_count)
    return sentence.capitalize()

def next_word(og_hist, word_hist, sentence_list, word_count):
    rand_index = random.randint(1, len(word_hist))

    if len(sentence_list) == word_count:
        sentence = " ".join(sentence_list)
        return sentence
    else:
        word = random_word(word_hist)
        if word == "[END]":
            sentence = " ".join(sentence_list)
            return sentence
        sentence_list.append(word)
        return next_word(og_hist, og_hist[word][1], sentence_list, word_count)

if __name__ == "__main__":
    dict = open('/Users/alexaaronpena/Github Repositories/Tweet-Generator/text_files/test-markov-text.txt', 'r')
    text = dict.read()
    dict.close()

    tokens = tokenize(text)
    hist_dict = Histogram(tokens)

    print(hist_dict)
    print(sentence(hist_dict, 50))


# Image manipulation
#
# You'll need Python 2.7 and must install these packages:
#
#   numpy, PyOpenGL, Pillow
#
# Note that file loading and saving (with 'l' and 's') do not work on
# Mac OSX, so you will have to change 'imgFilename' below, instead, if
# you want to work with different images.
#
# Note that images, when loaded, are converted to the YCbCr
# colourspace, and that you should manipulate only the Y component of
# each pixel when doing intensity changes.

#foo - imp.load_source('module.name')

import sys, os, numpy, math
import PIL as gl
import numpy as np
try:  # Pillow
    from PIL import Image
except:
    print('Error: Pillow has not been installed.')
    sys.exit(0)

try:  # PyOpenGL
    from OpenGL.GLUT import *
    from OpenGL.GL import *
    from OpenGL.GLU import *

except:
    print('Error: PyOpenGL has not been installed.')
    sys.exit(0)

# Globals

windowWidth = 600  # window dimensions
windowHeight = 800

localHistoRadius = 5  # distance within which to apply local histogram equalization

# Current image

imgDir = 'images'
imgFilename = 'mandrill.png'

currentImage = Image.open(os.path.join(imgDir, imgFilename)).convert('YCbCr').transpose(Image.FLIP_TOP_BOTTOM)
tempImage = None

# File dialog (doesn't work on Mac OSX)

if sys.platform != 'darwin':
    import Tkinter, tkFileDialog

    root = Tkinter.Tk()
    root.withdraw()

# Apply brightness and contrast to tempImage and store in
# currentImage.  The brightness and constrast changes are always made
# on tempImage, which stores the image when the left mouse button was
# first pressed, and are stored in currentImage, so that the user can
# see the changes immediately.  As long as left mouse button is held
# down, tempImage will not change.

def addToPixel(a, pixel):
    return [x + a for x in pixel]

def applyBrightnessAndContrast(brightness, contrast):
    width = currentImage.size[0]
    height = currentImage.size[1]

    srcPixels = tempImage.load()
    dstPixels = currentImage.load()

    # YOUR CODE HERE

    # loop through pixels in image
    for i in range(0,currentImage.width):
        for j in range(0,currentImage.height):#loop through image
            #get intensity and other tuple values from pixel data
            [y, Cr, Cb] = srcPixels[i, j]
            #equations for adjusting brightness and contrast
            y = contrast*y
            y = y + brightness
            #reassign updated tuple with new intensity to dstPixel
            dstPixels[i,j] = tuple([y, Cr, Cb])

    print("adjust brightness = " + str(brightness) + ", contrast = " + str(contrast))

# Perform local histogram equalization on the current image using the given radius.
#http://watkins.cs.queensu.ca/~jstewart/457/notes/05-yuv-histo-notes.txt
def performHistoEqualization(radius):
    #how does the function get called if nto a mouse click?
    pixels = currentImage.load()
    width = currentImage.size[0]
    height = currentImage.size[1]

    pixelTemp = currentImage.load()

    print("perform local histogram equalization with radius:" + str(radius))
    print("width: " + str(width) + "height: " + str(height))

# Scale the tempImage by the given factor and store it in
# currentImage.  Use backward projection.  This is called when the
# mouse is moved with the right button held down.
    intensityArray = [] #create array to hold intensities of pixels
    numPixels = 0
    # loop through image
    for i in range(0, width):
        for j in range(0, height):
            # at every pixel, get the radius pixels
            radX, radY = getNeighborhood(i,j, radius, height, width)
            for pixX in radX: #for every pixel in the radius, perform histogram equalization
                for pixY in radY:
                    [y, Cr, Cb] = pixels[pixX, pixY] #get pixel values from temp image
                    numPixels = numPixels + 1
                    intensityArray.append(y)

            # create dictionary from array of all intensity. This will get rid of duplicate values
            histDict = dict.fromkeys(intensityArray)

            # fill dictionary with pixel coordinate with that frequency
            for key in histDict:
                histDict[key] = getTofR(numPixels, getN_r(intensityArray,  key))

            #get tuple values from reference image version
            [y, Cr, Cb] = pixels[i, j]
            #get T(r) from dictionary and assign as y
            y = int(histDict[y])
            #assign new pixel tuple to temp image version
            pixelTemp[i, j] = tuple([y, Cr, Cb])

            #go through all pixels in the radius area and replace r with T(r)
            #loop through pixels in nested loop to change them within radius
            numRad = 0
            for pixX in radX:
                for pixY in radY:
                    #get the number of pixels in the radius area
                    pixels[pixX, pixY]
                    numRad = numRad + 1

#compute N_r values which is the sum of the all pixels with AT MOST intensity r
def getN_r(intensityArray, r):
    frequency = 0
    for y in intensityArray:
        if y <= r:
            frequency = frequency+1
    return frequency

def getTofR(N, N_r):
    #function for T(r) as outlined in notes
    N = float(N)
    N_r = float(N_r)
    s = ((256/N*N_r))-1
    return s

def getNeighborhood(x,y, radius, height, width):

    radX = range(x-radius, radius + x)
    radX = [item for item in radX if  0<=item<width] #don't let the radius go out of image range

    radY = range(y-radius, radius + y)
    radY = [item for item in radY if 0<=item<height]

    return radX, radY

def scaleImage(factor):

#consider the image as a matrix and perform a matrix transformation
#use im.transform

    width = currentImage.size[0]
    height = currentImage.size[1]

    srcPixels = tempImage.load()
    dstPixels = currentImage.load()

    # YOUR CODE HERE
# Set up the display and draw the current image

    for i in range(0, currentImage.width):
        for j in range(0, currentImage.height):  # loop through image
            # use back projection to set the destination pixels to the factored source coordinates
            #I would have preferred to use an if-then clause to assign pixel values but I could not figure out
            #the proper boundary conditional so I used a try-except because it also works
            try:
                dstPixels[i, j] = srcPixels[i/factor, j/factor]
            # if the pixel in the reference image is out of range, replace the pixels with white
            except:
                dstPixels[i, j] = (255, 128, 128)

def display():
    # Clear window

    glClearColor(1, 1, 1, 0)
    glClear(GL_COLOR_BUFFER_BIT)

    # rebuild the image

    img = currentImage.convert('RGB')

    width = img.size[0]
    height = img.size[1]

    # Find where to position lower-left corner of image

    baseX = (windowWidth - width) / 2
    baseY = (windowHeight - height) / 2

    glWindowPos2i(baseX, baseY)

    # Get pixels and draw

    imageData = numpy.array(list(img.getdata()), numpy.uint8)

    glDrawPixels(width, height, GL_RGB, GL_UNSIGNED_BYTE, imageData)

    glutSwapBuffers()

# Handle keyboard input

def keyboard(key, x, y):
    global localHistoRadius

    if key == '\033':  # ESC = exit
        sys.exit(0)

    elif key == 'l':
        if sys.platform != 'darwin':
            path = tkFileDialog.askopenfilename(initialdir=imgDir)
            if path:
                loadImage(path)

    elif key == 's':
        if sys.platform != 'darwin':
            outputPath = tkFileDialog.asksaveasfilename(initialdir='.')
            if outputPath:
                saveImage(outputPath)

    elif key == 'h':
        performHistoEqualization(localHistoRadius)

    elif key in ['+', '=']:
        localHistoRadius = localHistoRadius + 1
        print
        'radius =', localHistoRadius

    elif key in ['-', '_']:
        localHistoRadius = localHistoRadius - 1
        if localHistoRadius < 1:
            localHistoRadius = 1
        print
        'radius =', localHistoRadius

    else:
        print
        'key =', key  # DO NOT REMOVE THIS LINE.  It will be used during automated marking.

    glutPostRedisplay()

# Load and save images.
#
# Modify these to load to the current image and to save the current image.
#
# DO NOT CHANGE THE NAMES OR ARGUMENT LISTS OF THESE FUNCTIONS, as
# they will be used in automated marking.

def loadImage(path):
    global currentImage

    currentImage = Image.open(path).convert('YCbCr').transpose(Image.FLIP_TOP_BOTTOM)

def saveImage(path):
    global currentImage

    currentImage.transpose(Image.FLIP_TOP_BOTTOM).convert('RGB').save(path)

# Handle window reshape

def reshape(newWidth, newHeight):
    global windowWidth, windowHeight

    windowWidth = newWidth
    windowHeight = newHeight

    glutPostRedisplay()

# Mouse state on initial click

button = None
initX = 0
initY = 0

# Handle mouse click/release

def mouse(btn, state, x, y):
    global button, initX, initY, tempImage

    if state == GLUT_DOWN:
        tempImage = currentImage.copy()
        button = btn
        initX = x
        initY = y
    elif state == GLUT_UP:
        tempImage = None
        button = None

    glutPostRedisplay()

# Handle mouse motion

def motion(x, y):
    if button == GLUT_LEFT_BUTTON:

        diffX = x - initX
        diffY = y - initY

        applyBrightnessAndContrast(255 * diffX / float(windowWidth), 1 + diffY / float(windowHeight))

    elif button == GLUT_RIGHT_BUTTON:

        initPosX = initX - float(windowWidth) / 2.0
        initPosY = initY - float(windowHeight) / 2.0
        initDist = math.sqrt(initPosX * initPosX + initPosY * initPosY)
        if initDist == 0:
            initDist = 1

        newPosX = x - float(windowWidth) / 2.0
        newPosY = y - float(windowHeight) / 2.0
        newDist = math.sqrt(newPosX * newPosX + newPosY * newPosY)

        scaleImage(newDist / initDist)

    glutPostRedisplay()

# Run OpenGL
glutInit()
glutInitDisplayMode(GLUT_DOUBLE | GLUT_RGB)
glutInitWindowSize(windowWidth, windowHeight)
glutInitWindowPosition(50, 50)
glutCreateWindow('imaging')
glutDisplayFunc(display)
glutKeyboardFunc(keyboard)
glutReshapeFunc(reshape)
glutMouseFunc(mouse)
glutMotionFunc(motion)

glutMainLoop()
scaleImage(5)


#!/usr/bin/env python3
# _*_ encoding: utf-8 _*_

import socket
import struct
import json

tcp_server = socket.socket()
ip_port = ('127.0.0.1', 8080)
tcp_server.bind(ip_port)

tcp_server.listen()
conn, addr = tcp_server.accept()

# 首先接受到的文件信息长度转换出来的4个字节的数据
file_info_stru = conn.recv(4)

#解包文件信息的长度
file_info_len = struct.unpack('i', file_info_stru)[0]

client_file_info = conn.recv(file_info_len).decode('utf-8')

abc_file_info = json.loads(client_file_info)
print('abc_file_infp>>>', abc_file_info)

client_file_size = abc_file_info['file_size']


class Node:
    def __init__(self, data, link=None):
        self.data = data
        self.link = link

def add(data, idx, p, n):
    while n < idx - 1:
        p = p.link
        n += 1
    p.link = Node(data, p.link)

def delete(idx, p, n):
    while n < idx - 1:
        p = p.link
        n += 1
    p.link = p.link.link

def exchange(data, idx, p, n):
    while n < idx:
        p = p.link
        n += 1
    p.data = data

def get(idx, p, n):
    while p and n < idx:
        p = p.link
        n += 1
    if not p : return -1
    return p.data

T = int(input())
for tc in range(1, T + 1):
    N, M, L = map(int, input().split())

    nums = list(map(int, input().split()))
    pHead = Node(nums[0])
    p = pHead
    for i in range(1, N):
        p.link = Node(nums[i])
        p = p.link

    for i in range(M):
        s = list(input().split())
        if   (s[0] == 'I'):  add(s[2], int(s[1]), pHead, 0)
        elif (s[0] == 'D'):  delete(int(s[1]), pHead, 0)
        elif (s[0] == 'C'):  exchange(s[2], int(s[1]), pHead, 0)

    print('#%d'%tc, get(L, pHead, 0))

Algorithm and Data Structure/SW Expert Academy/Learning/Lv1 Programming Intermediate/07 파이썬 SW 문제해결 기본 Linked List/5122번 수열 편집/수열 편집.py


from reportlab.graphics.shapes import Drawing
from reportlab.graphics.charts.barcharts import VerticalBarChart
from reportlab.graphics.charts.linecharts import HorizontalLineChart
from reportlab.graphics.charts.lineplots import ScatterPlot
from reportlab.graphics.charts.piecharts import Pie
from reportlab.graphics.charts.textlabels import Label
from reportlab.graphics.charts.legends import Legend
import math
from reportlab.graphics.charts.axes import XCategoryAxis, YValueAxis
from reportlab.platypus.doctemplate import SimpleDocTemplate
from reportlab.platypus import Paragraph, Table, Spacer, TableStyle
from reportlab.lib.units import inch
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
from reportlab.lib import colors
from reportlab.platypus import Table
from reportlab.lib.pagesizes import letter, landscape
from genericfunctions import *

def set_chart_size(size):

    if size == 'S':
        drawsizew, drawsizeh = 180, 180
        height = 125
        width = 170
        title_y_offset = 175
        title_x_offset = 102
        delta_y = 20
        chart_font_size = 7
    if size == 'SW':
        drawsizew, drawsizeh = 650, 180
        height = 125
        width = 625
        title_y_offset = 175
        title_x_offset = 338
        delta_y = 25
        chart_font_size = 8
    elif size == 'M':
        drawsizew, drawsizeh = 250, 250
        height = 188
        width = 225
        title_y_offset = 220
        title_x_offset = 138
        delta_y = 25
        chart_font_size = 8
    elif size == 'L':
        drawsizew, drawsizeh = 650, 190
        height = 175
        width = 625
        title_y_offset = 185
        title_x_offset = 338
        delta_y = 25
        chart_font_size = 9
    elif size == 'SB':
        drawsizew, drawsizeh = 180, 220
        height = 125
        width = 210
        title_y_offset = 175
        title_x_offset = 125
        delta_y = 25
        chart_font_size = 7

    return {'drawsizew':drawsizew, 'drawsizeh':drawsizeh, 'height':height, 'width':width,
            'title_y_offset':title_y_offset, 'title_x_offset':title_x_offset, 'delta_y':delta_y,
            'chart_font_size':chart_font_size}

def set_chart_labels(label):

    labeldict = {'Collateralized Bonds': 'Collateralized',
                 'Corporate Bonds': 'Corporate',
                 'Sovereign/Agcy Bonds': 'Govt/Agcy',
                 'Municipal Bonds': 'Municipal',
                 'CO': 'Collateralized',
                 'CR': 'Corporate',
                 'MU': 'Municipal',
                 'SG': 'Govt/Agcy',
                 'TG': 'Govt/Agcy'}

    if label in labeldict:
        return labeldict[str(label)]
    else:
        return label

def set_color(label):

    colordict = {'MBS': colors.HexColor('#2E323C'),
                 'CO': colors.HexColor('#2E323C'),
                 'Collateralized': colors.HexColor('#2E323C'),
                 'Collateralized Bonds': colors.HexColor('#2E323C'),
                 'TG': colors.HexColor('#dadada'),
                 'Govt/Agcy': colors.HexColor('#dadada'),
                 'Sovereign/Agcy': colors.HexColor('#dadada'),
                 'Sovereign/Agcy Bonds': colors.HexColor('#dadada'),
                 'MU': colors.HexColor('#79868D'),
                 'Municipal': colors.HexColor('#79868D'),
                 'Municipal Bonds': colors.HexColor('#79868D'),
                 'CR': colors.HexColor('#6D9DB3'),
                 'Corporate': colors.HexColor('#6D9DB3'),
                 'Corporate Bonds': colors.HexColor('#6D9DB3'),
                 'Cash': colors.HexColor('#867F7F'),
                 'Derivative': colors.HexColor('#191C27'),
                 '10yr': colors.HexColor('#d62728'),
                 '15yr': colors.HexColor('#191C27'),
                 '20yr': colors.HexColor('#dadada'),
                 '30yr': colors.HexColor('#7A6F67'),
                 'Other': colors.HexColor('#6D9DB3'),
                 'Other2': colors.HexColor('#7AA5AD')
                 }

    if label in colordict:
        return colordict[str(label)]
    else:
        return colordict['MBS']

class NiceScale:
    def __init__(self, minv, maxv):
        self.maxTicks = 6
        self.tickSpacing = 0
        self.lst = 10
        self.niceMin = 0
        self.niceMax = 0
        self.minPoint = minv
        self.maxPoint = maxv
        self.calculate()

    def calculate(self):
        self.lst = self.niceNum(self.maxPoint - self.minPoint, False)
        self.tickSpacing = self.niceNum(self.lst / (self.maxTicks - 1), True)
        self.niceMin = math.floor(self.minPoint / self.tickSpacing) * self.tickSpacing
        self.niceMax = math.ceil(self.maxPoint / self.tickSpacing) * self.tickSpacing

    def niceNum(self, lst, rround):
        self.lst = lst
        exponent = 0  # exponent of range */
        fraction = 0  # fractional part of range */
        niceFraction = 0  # nice, rounded fraction */

        if self.lst == 0:
            exponent = 1
        else:
            exponent = math.floor(math.log10(self.lst))
        fraction = self.lst / math.pow(10, exponent)

        if self.lst:
            if fraction < 1.5:
                niceFraction = 1
            elif fraction < 3:
                niceFraction = 2
            elif fraction < 7:
                niceFraction = 5;
            else:
                niceFraction = 10;
        else:
            if fraction <= 1:
                niceFraction = 1
            elif fraction <= 2:
                niceFraction = 2
            elif fraction <= 5:
                niceFraction = 5
            else:
                niceFraction = 10

        return niceFraction * math.pow(10, exponent)

    def setMinMaxPoints(self, minPoint, maxPoint):
        self.minPoint = minPoint
        self.maxPoint = maxPoint
        self.calculate()

    def setMaxTicks(self, maxTicks):
        self.maxTicks = maxTicks
        self.calculate()

def vbar_one_series(size, data, xlabels, chart_title, xaxis_text, axismin, axismax, spacing):

    drawing = Drawing(set_chart_size(size)['drawsizew'], set_chart_size(size)['drawsizeh'])

    bc = VerticalBarChart()
    bc.x = 25
    bc.y = set_chart_size(size)['delta_y']
    bc.height = set_chart_size(size)['height']
    bc.width = set_chart_size(size)['width']
    bc.data = [data]
    bc.valueAxis.valueMin = axismin
    bc.valueAxis.valueMax = axismax
    bc.valueAxis.valueStep = spacing

    bc.valueAxis.labels.fontName = 'Helvetica'
    bc.valueAxis.labels.fontSize = set_chart_size(size)['chart_font_size']
    bc.valueAxis.labelTextFormat = '%0.1f'

    fixed_xlabels = []
    for l in xlabels:
        fixed_xlabels.append(set_chart_labels(l))

    for i in range(len(fixed_xlabels)):
        bc.bars[(0,i)].fillColor = set_color(fixed_xlabels[i])
        bc.bars[(0,i)].strokeColor = set_color(fixed_xlabels[i])

    bc.categoryAxis.labels.boxAnchor = 'c'
    bc.categoryAxis.labelAxisMode = 'low'
    bc.categoryAxis.labels.fontName = 'Helvetica'
    bc.categoryAxis.labels.fontSize = set_chart_size(size)['chart_font_size']
    bc.categoryAxis.tickDown = 0
    bc.categoryAxis.labels.dx = 0
    bc.categoryAxis.labels.dy = -13
    bc.categoryAxis.labels.angle = 0

    bc.labelTextFormat = '%.1f'

    bc.categoryAxis.categoryNames = fixed_xlabels

    bc.barLabels.nudge = 5
    bc.barLabelFormat = '%0.1f'
    bc.barLabels.dx = 0
    bc.barLabels.dy = 0
    bc.barLabels.fontName = 'Helvetica'
    bc.barLabels.fontSize = 7

    xlabel = Label()
    xlabel.setText(xaxis_text)
    xlabel.fontSize = set_chart_size(size)['chart_font_size']
    xlabel.fontName = 'Helvetica'
    xlabel.dx = set_chart_size(size)['title_x_offset']
    xlabel.dy = 3

    title = Label()
    title.setText(chart_title)
    title.fontSize = set_chart_size(size)['chart_font_size']
    title.fontName = 'Helvetica'
    title.dx = set_chart_size(size)['title_x_offset']
    title.dy = set_chart_size(size)['title_y_offset']

    drawing.add(bc)
    drawing.add(xlabel)
    drawing.add(title)

    return drawing

def pie_chart(chart_size, data, labels, chart_title):
    pc = Pie()

    if chart_size == 'S':
        piedraw = Drawing(180, 180)
        pc.x = 35
        pc.y = 30
        pc.height = 115
        pc.width = 115
        pie_title_x_offset = 90
        pie_title_y_offset = 175
        pie_font_size = 7
        pie_label_offset = 1.25
    elif chart_size == 'M':
        piedraw = Drawing(250, 250)
        pc.x = 20
        pc.y = 15
        pc.height = 200
        pc.width = 200
        pie_title_x_offset = 125
        pie_title_y_offset = 245
        pie_font_size = 8
        pie_label_offset = 1.15

    fixed_labels = []
    for l in labels:
        fixed_labels.append(set_chart_labels(l))

    pc.data = data
    data_labels = fixed_labels

    for i in range(len(data_labels)):
        pc.slices[i].fillColor = set_color(data_labels[i])

    pc.labels = []
    for i in range(len(pc.data)):
        pc.labels.append(data_labels[i] + ':' + str('{:.0%}'.format(float(pc.data[i])/sum(pc.data))))

    pc.slices.strokeWidth = 0.5
    pc.slices.fontName = 'Helvetica'
    pc.slices.fontSize = pie_font_size
    pc.sideLabels = 1

    title = Label()
    title.setText(chart_title)
    title.fontSize = pie_font_size
    title.fontName = 'Helvetica'
    title.dx = pie_title_x_offset
    title.dy = pie_title_y_offset

    piedraw.add(pc)
    piedraw.add(title)

    return piedraw

def single_line(data, labels):
    linedraw = Drawing(180, 180)

    lc = HorizontalLineChart()

    lc.x = 20
    lc.y = 50
    lc.height = 100
    lc.width = 175
    lc.data = [data]
    lc.joinedLines = 1

    lc.categoryAxis.categoryNames = labels
    lc.categoryAxis.labels.boxAnchor = 'c'
    lc.categoryAxis.labels.fontName = 'Helvetica'
    lc.categoryAxis.labels.fontSize = 8

    lc.valueAxis.labels.fontName = 'Helvetica'
    lc.valueAxis.labels.fontSize = 8

    lc.valueAxis.valueMin = min(lc.data[0] ) * 1.10
    lc.valueAxis.valueMax = max(lc.data[0]) * 1.10
    lc.valueAxis.valueStep = 1
    lc.valueAxis.labelTextFormat = '%0.1f'

    lc.lines[0].strokeWidth = 1
    lc.lines[0].strokeColor = colors.HexColor('#000035')
    linedraw.add(lc)

    return linedraw

def stacked_bar(data, xlabels, series_names, chart_title, axismin, axismax, spacing):

    drawsizew, drawsizeh = 650, 250
    drawing = Drawing(drawsizew, drawsizeh)
    name_pairs = []

    bc = VerticalBarChart()
    bc.categoryAxis.style = "stacked"
    bc.x = 30
    bc.y = 40
    bc.height = 188
    bc.width = 625

    bc.data = data
    bc.categoryAxis.categoryNames = xlabels
#    bc.valueAxis.labelTextFormat = '%0.1f'
    bc.valueAxis.labels.fontName = 'Helvetica'
    bc.valueAxis.labels.fontSize = 7

    for i in range(len(series_names)):
        bc.bars[i].fillColor = set_color(series_names[i])
        bc.bars[i].strokeColor = set_color(series_names[i])
        name_pairs.append((bc.bars[i].fillColor, series_names[i]))

    bc.categoryAxis.labels.fontName = 'Helvetica'
    bc.categoryAxis.labels.fontSize = 7
    bc.categoryAxis.tickDown = 0
    bc.categoryAxis.labels.dx = 0
    bc.categoryAxis.labels.dy = -18
    bc.categoryAxis.labels.angle = 90
    bc.categoryAxis.labels.boxAnchor = 'c'
    bc.valueAxis.valueMin = axismin
    bc.valueAxis.valueMax = axismax
    bc.valueAxis.valueStep = spacing

    title = Label()
    title.setText(chart_title)
    title.fontSize = 7
    title.fontName = 'Helvetica'
    title.dx = 338
    title.dy = 220

    legend = Legend()
    legend.fontName = 'Helvetica'
    legend.fontSize = 7
    legend.colorNamePairs = name_pairs
    legend.columnMaximum = 1
    legend.x = 185
#    legend.y = 100
    legend.dy = 5
    legend.dx = 5
#    legend.alignment = 'right'

    drawing.add(bc)
    drawing.add(title)
    drawing.add(legend)

    return drawing

def quartile_bar_one_asset(df, dataseries, chart_title):

    drawsizew, drawsizeh = 180, 220
    drawing = Drawing(drawsizew, drawsizeh)

    if dataseries == 1:
        sbarxlabels = clean_quartile_categories(df.columns[:4])
        sbardata = [df.iloc[:, 0].tolist(), df.iloc[:, 1].tolist(),
                    df.iloc[:, 2].tolist(), df.iloc[:, 3].tolist()]
    else:
        sbarxlabels = clean_quartile_categories(df.columns[4:])
        sbardata = [df.iloc[:, 4].tolist(), df.iloc[:, 5].tolist(),
                    df.iloc[:, 6].tolist(), df.iloc[:, 7].tolist()]

    pct2flt = quartile_pct_to_float(sbardata)
    bc = VerticalBarChart()

    bc.x = 20
    bc.y = 30
    bc.height = 125
    bc.width = 210

    graphdata = [i[0] for i in pct2flt]

    bc.data = [graphdata]
    bc.categoryAxis.categoryNames = sbarxlabels
    bc.valueAxis.labels.fontName = 'Helvetica'
    bc.valueAxis.labels.fontSize = 7
    bc.valueAxis.valueMin = 0

    bc.categoryAxis.labels.fontName = 'Helvetica'
    bc.categoryAxis.labels.fontSize = 7
    bc.categoryAxis.tickDown = 0
    bc.categoryAxis.labels.dx = 0
    bc.categoryAxis.labels.dy = -18
    bc.categoryAxis.labels.angle = 0
    bc.categoryAxis.labels.boxAnchor = 'c'

    bc.barLabels.nudge = 5
    bc.barLabelFormat = '%0.1f'
    bc.barLabels.dx = 0
    bc.barLabels.dy = 0
    bc.barLabels.fontName = 'Helvetica'
    bc.barLabels.fontSize = 7

    for i in range(len(sbarxlabels)):
        bc.bars[(0,i)].fillColor = set_color(colors.HexColor('#2E323C'))
        bc.bars[(0,i)].strokeColor = set_color(colors.HexColor('#2E323C'))

    title = Label()
    title.setText(chart_title)
    title.fontSize = 7
    title.fontName = 'Helvetica'
    title.dx = 125
    title.dy = 175

    drawing.add(bc)
    drawing.add(title)

    return drawing

def generic_bar(**kwargs):

    size = kwargs['size']
    graphdata = kwargs['graphdata']
    xlabels = kwargs['xlabels']
    chart_title = kwargs['chart_title']

    drawing = Drawing(set_chart_size(size)['drawsizew'], set_chart_size(size)['drawsizeh'])
    bc = VerticalBarChart()

    bc.x = 20
    bc.y = 30
    bc.height = set_chart_size(size)['height']
    bc.width = set_chart_size(size)['width']

    bc.categoryAxis.categoryNames = xlabels
    bc.valueAxis.labels.fontName = 'Helvetica'
    bc.valueAxis.labels.fontSize = set_chart_size(size)['chart_font_size']
    bc.valueAxis.labelTextFormat = '%.1f'
    if 'sectors' in kwargs:
        bc.data = graphdata
        sectors = kwargs['sectors']
        name_pairs = []
        bc.categoryAxis.style = "stacked"
        for i in range(len(sectors)):
            bc.bars[i].fillColor = set_color(sectors[i])
            bc.bars[i].strokeColor = set_color(sectors[i])
            name_pairs.append((bc.bars[i].fillColor, sectors[i]))

        legend = Legend()
        legend.fontName = 'Helvetica'
        legend.fontSize = set_chart_size(size)['chart_font_size']
        legend.colorNamePairs = name_pairs
        legend.columnMaximum = 1
        legend.x = 10
        #    legend.y = 100
        legend.dy = 5
        legend.dx = 5
        legend.alignment = 'right'
        drawing.add(legend)

    else:
        bc.data = [graphdata]
        datarange = NiceScale(min(graphdata), max(graphdata))

        bc.valueAxis.valueMin = datarange.niceMin
        bc.valueAxis.valueMax = datarange.niceMax
        bc.valueAxis.valueStep = datarange.tickSpacing

        for i in range(len(xlabels)):
            bc.bars[(0, i)].fillColor = set_color(xlabels[i])
            bc.bars[(0, i)].strokeColor = set_color(xlabels[i])

        bc.barLabels.nudge = 5
        bc.barLabelFormat = '%0.1f'
        bc.barLabels.dx = 0
        bc.barLabels.dy = 0
        bc.barLabels.fontName = 'Helvetica'
        bc.barLabels.fontSize = 7

    bc.categoryAxis.labels.fontName = 'Helvetica'
    bc.categoryAxis.labels.fontSize = set_chart_size(size)['chart_font_size']
    bc.categoryAxis.tickDown = 0
    bc.categoryAxis.labels.dx = 0
    bc.categoryAxis.labels.dy = -18
    bc.categoryAxis.labels.angle = 0
    bc.categoryAxis.labels.boxAnchor = 'c'

    title = Label()
    title.setText(chart_title)
    title.fontSize = set_chart_size(size)['chart_font_size']
    title.fontName = 'Helvetica'
    title.dx = set_chart_size(size)['title_x_offset']
    title.dy = set_chart_size(size)['title_y_offset']

    drawing.add(bc)
    drawing.add(title)

    return drawing

def bubble_chart(**kwargs):

    size = kwargs['size']
    graphdata = kwargs['graphdata']
    xylabels = kwargs['xylabels']
    chart_title = kwargs['chart_title']
    sectors = kwargs['sectors']
    sector_pct = kwargs['sector_pct']

    spdraw = Drawing(set_chart_size(size)['drawsizew'], set_chart_size(size)['drawsizeh'])
    sp = ScatterPlot()

    sp.width = set_chart_size(size)['width']
    sp.height = set_chart_size(size)['height']
    sp.x = 20
    sp.y = 30

    sp.data = graphdata

    sp.lineLabels.fontName = 'Helvetica'

    sp.xValueAxis.labels.fontName = 'Helvetica'
    sp.xValueAxis.labelTextFormat = '%0.1f'
    sp.xValueAxis.labels.fontSize = set_chart_size(size)['chart_font_size']
    sp.xValueAxis.avoidBoundFrac = 1

    sp.xValueAxis.visibleGrid = 1
    sp.yValueAxis.labels.fontName = 'Helvetica'
    sp.yValueAxis.labels.fontSize = set_chart_size(size)['chart_font_size']
    sp.yValueAxis.visibleGrid = 1

    title = Label()
    title.fontName = 'Helvetica'
    title.fontSize = set_chart_size(size)['chart_font_size']
    title.dx = set_chart_size(size)['title_x_offset']
    title.dy = set_chart_size(size)['title_y_offset'] + 20
    title._text = chart_title
    title.textAnchor = 'middle'

    name_pairs = []
    for i in range(len(sectors)):
        sp.lines[i].symbol.kind = 'Circle'
        sp.lines[i].symbol.size = 100 * 2 * math.sqrt(sector_pct[sectors[i]] / math.pi)
        sp.lines[i].strokeColor = set_color(sectors[i])
        desc_string = ': (MktVal: ' + '{:.1%}'.format(sector_pct[sectors[i]]) +\
                      ' Yield: ' + '{:.1%}'.format(graphdata[i][0][1] / 100) +\
                      ' Total Return: ' + '{:.1%}'.format(graphdata[i][0][0] / 100) + ')'
        name_pairs.append((sp.lines[i].strokeColor, set_chart_labels(sectors[i]) + desc_string))

    legend = Legend()
    legend.colorNamePairs = name_pairs
    if len(sectors) > 4:
        legend.columnMaximum = 3
    else:
        legend.columnMaximum = 2
    legend.fontName = 'Helvetica'
    legend.fontSize = set_chart_size(size)['chart_font_size']
    legend.x = 20
    legend.y = -10
    # legend.dxTextSpace = 5
    legend.dy = 10
    legend.dx = 10
    legend.alignment = 'right'

    sp.lineLabelFormat = None

    sp.xLabel = xylabels[0]
    sp.y = 30
    sp.yLabel = xylabels[1]
    sp.yValueAxis.labelTextFormat = '%0.1f'
    # sp.yValueAxis.forceZero = 1
    # sp.xValueAxis.forceZero = 1

    spdraw.add(sp)
    spdraw.add(title)
    spdraw.add(legend)

    return spdraw

def make_sector_detail_page(df, portsum, pagetitle, stratvar, chart_title):

    portfolio_matrix_table_title = underlined_title(pagetitle)
    portfolio_matrix = mktval_detail_table(df, portsum, stratvar)

    portfolio_distrib = df.groupby([stratvar])['BaseMarketValue'].sum() / portsum[1]
    portfolio_distrib.dropna(inplace=True)
    portfolio_distrib = portfolio_distrib[portfolio_distrib.values != 0]
    summary_table = mktval_by_duration_summary_table(portfolio_distrib, 'Total Portfolio')
    sector_summary_table = sector_byvar_table(df, portsum, stratvar)

    graph_data = df.groupby([stratvar, 'clean_asset_code'])['BaseMarketValue'].sum() * 100 / portsum[1]
    finl_graph = graph_data.unstack(0, fill_value=0)

    stacked_barcht = generic_bar(size='SW', graphdata=finl_graph.values,
                                 xlabels=finl_graph.columns.astype(str).tolist(), sectors=finl_graph.index,
                                 chart_title=chart_title)

    return portfolio_matrix_table_title, summary_table, sector_summary_table, portfolio_matrix, stacked_barcht


from django.urls import path
from . import views

app_name = 'daftar'
urlpatterns = [
    path('', views.index, name='home'),
    # path('karyawan/<int:pk>/delete', views.karyawan_delete, name='karyawan_delete'),
    # path('karyawan/<int:pk>/edit', views.karyawan_edit, name='karyawan_edit'),
    # path('karyawan/<int:pk>/', views.karyawan_detail, name='karyawan_detail'),
    path('cari/', views.cari, name='cari'),
    path('add/<int:pk>', views.daftar_tk, name='add_tk'),
    # path('daftar/', views.karyawan_list, name='karyawan_list'),
    #path('pk_list', views.pk_list, name='pk_list'),

]


import os
import cv2
import pandas as pd
import numpy as np
import torch

from torch.utils.data import Dataset

class FaceDataset(Dataset):
    """
    dataset

    """

    def __init__(self, csv_path, transform=None):
        """
        initiate function
        :param csv_path: anotated data csvfile
        :param transform: itarational transformer
        """
        self.csv_path = csv_path
        self.transform = transform
        self.data = pd.read_csv(self.csv_path, header=None)
        self.image_num = len(self.data)
        self.label_dict = {}
        label_count = 0
        for i in range(self.image_num):
            if not self.data.ix[i, 1] in self.label_dict:
                self.label_dict[self.data.ix[i, 1]] = label_count
                label_count += 1

    def label_num(self):
        return len(self.label_dict)

    def get_label_dict(self):
        return self.label_dict

    def give_label_dict(self, label_dict):
        self.label_dict = label_dict

    def __len__(self):
        """
        return the data_num
        """
        return self.image_num

    def __getitem__(self, idx):
        image_path = self.data.ix[idx, 0]
        label = self.data.ix[idx, 1]
        label_onehot = np.zeros(len(self.label_dict))
        if not label in self.label_dict:
            label_onehot[-1] = 1
        else:
            label_onehot[self.label_dict[label]] = 1
        label_onehot = torch.tensor(label_onehot, dtype=torch.float)
        input_image = cv2.imread(image_path)
        if self.transform:
            image = self.transform(input_image)
        return (image, label_onehot)

class FaceResolutorDataset(FaceDataset):

    def __getitem__(self, idx):
        higher_path = self.data.ix[idx, 0]
        lower_path = self.data.ix[idx, 1]
        higher_image = cv2.imread(higher_path)
        lower_image = cv2.imread(lower_path)
        if self.transform:
            higher_image = self.transform(higher_image)
            lower_image = self.transform(lower_image)
        return(lower_image, higher_image)


import pickle as pk
import os
file_path=os.path.expanduser('~/catkin_ws/src/voice_common_pkg/config')
food=["snack","bread","snack","rice"]
drink=["coke","water","tea","green tea","coffee","milk"]

with open(file_path+"/class_generalization.pkl","wb") as fl:
    dit={"food":set(food),"drink":set(drink)}
    pk.dump(dit,fl)


#-*- coding:utf-8 -*-

import time
import uuid
from os.path import splitext, join

from sqlalchemy.exc import DataError

from distreet.models.photo import Photo
from distreet.setting import PHOTO_BASE_URL, UPLOAD_DIR

def generate_filename():
    ts = str(int(time.time()))
    return '%s-%s' % (ts, uuid.uuid4().get_hex())

class PhotoUploader(object):

    ALLOW_EXTENSIONS = ('.png', '.jpeg', '.jpg')
    UPLOAD_DIR_KEY = 'upload_dir'

    #1M
    MAX_PHOTO_SIZE = 1024000

    def __init__(self, handler, dest_directory=None, **options):
        self.handler = handler
        if not dest_directory:
            dest_directory = UPLOAD_DIR
        self.dest_directory = dest_directory
        self.options = options

    def upload(self):
        files = self.handler.request.files
        if not files:
            return None
        photos_data = []
        for fn, info in files.iteritems():
            try:
                ext_name = splitext(info[0]['filename'])[-1].lower()
                if ext_name not in self.ALLOW_EXTENSIONS:
                    raise TypeError
                body = info[0].pop('body')
            except (KeyError, TypeError, IndexError):
                continue
            body_size = len(body)
            if body_size > self.MAX_PHOTO_SIZE:
                continue
            new_filename = generate_filename() + ext_name
            full_path = join(self.dest_directory, new_filename)
            try:
                with open(full_path, 'w') as fp:
                    fp.write(body)
            except IOError:
                continue
            photos_data.append({
                'fn': full_path,
                'url': PHOTO_BASE_URL + new_filename,
                'ref_type': self.options.pop('ref_type', ''),
                'ref_id': self.options.pop('ref_id', 0)
            })
        if photos_data:
            photos = [Photo(name=p['fn'], ref_type=p['ref_type'],
                            ref_id=p['ref_id'], full_url=p['url'])
                      for p in photos_data]
            session = self.handler.application.backend.get_session()
            try:
                session.add_all(photos)
                session.commit()
                ids = [photo.id for photo in photos]
            except DataError:
                session.rollback()
            finally:
                session.close()
            return ids
        return None


from django.urls import path, reverse_lazy
from . import views

app_name='recipes'
urlpatterns = [
    path('', views. HomePage.as_view(), name='home_page'),
    path('recipes', views.RecipeListView.as_view(), name='all'),
    path('recipes/<int:pk>', views.RecipeDetailView.as_view(), name='recipe_detail'),
    path('recipes/create',
        views.RecipeCreateView.as_view(success_url=reverse_lazy('recipes:all')), name='recipe_create'),
    path('recipes/<int:pk>/update',
        views.RecipeUpdateView.as_view(success_url=reverse_lazy('recipes:all')), name='recipe_update'),
    path('recipes/<int:pk>/delete',
        views.RecipeDeleteView.as_view(success_url=reverse_lazy('recipes:all')), name='recipe_delete'),
    path('your_recipes', views.UserRecipeListView.as_view(), name='user_recipe_list'),

    path('recipe_picture/<int:pk>', views.stream_file, name='recipe_picture'),
    path('recipe/<int:pk>/comment',
        views.CommentCreateView.as_view(), name='recipe_comment_create'),
    path('comment/<int:pk>/delete',
        views.CommentDeleteView.as_view(success_url=reverse_lazy('recipes')), name='recipe_comment_delete'),
    path('recipe/<int:pk>/favorite',
        views.AddFavoriteView.as_view(), name='recipe_favorite'),
    path('recipe/<int:pk>/unfavorite',
        views.DeleteFavoriteView.as_view(), name='recipe_unfavorite'),

    path('recipe/<int:pk>/ingredient', views.IngredientCreateView.as_view(), name='recipe_ingredient_create'),
    path('ingredient/<int:pk>/delete', views.IngredientDeleteView.as_view(), name='recipe_ingredient_delete'),

]


from selenium import webdriver
from selenium.webdriver.remote.webelement import WebElement
from selenium.common.exceptions import StaleElementReferenceException
import time

def waitForLoad(driver):
    elem = driver.find_element_by_tag_name('html')
    print(elem.text)
    count = 0
    while True:
        count += 1
        if count > 10:
            print ('Timing out after 10 seconds and returning')
            return
        time.sleep(0.5)
        try:
            elem == driver.find_element_by_tag_name('html')
        except StaleElementReferenceException:
            return

driver = webdriver.PhantomJS(executable_path=r'D:\03-CS\plantomJS\phantomjs-2.1.1-windows\bin\phantomjs')
driver.get('http://pythonscraping.com/pages/javascript/redirectDemo1.html')
waitForLoad(driver)
print(driver.page_source)


import argparse, pickle
import numpy as np
import os

import torch
from torch import nn, optim
from torchvision import transforms

# Use the util.load() function to load your dataset
from .utils import load
from .models import *

dirname = os.path.dirname(os.path.abspath(__file__))

def accuracy(outputs, labels):
    outputs_idx = outputs.max(1)[1].type_as(outputs)
    return outputs_idx.eq(labels.float()).float().mean()

train_transforms = transforms.Compose([
    transforms.ToPILImage(),

    transforms.RandomApply([

        transforms.ColorJitter(brightness=0.25, contrast=0.35, saturation=0.25, hue=0.28),
        transforms.RandomHorizontalFlip(p=0.5),

    ], 0.5),

    transforms.ToTensor(),
    transforms.Normalize(mean=[0.429, 0.505, 0.517]*255, std=[0.274, 0.283, 0.347]*255)
])

val_transforms = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.429, 0.505, 0.517]*255, std=[0.274, 0.283, 0.347]*255)
])

def augment_train(inputs):
    return torch.stack( [ train_transforms(inp_i) for inp_i in inputs ] )

def augment_val(inputs):
    return torch.stack( [ val_transforms(inp_i) for inp_i in inputs ] )

def train(iterations, batch_size=64, log_dir=None):
    '''
    This is the main training function, feel free to modify some of the code, but it
    should not be required to complete the assignment.
    '''

    """
    Load the training data
    """

    train_inputs, train_labels = load(os.path.join('tux_train.dat'))
    val_inputs, val_labels = load(os.path.join('tux_valid.dat'))

    model = ConvNetModel()

    log = None
    if log_dir is not None:
        from .utils import SummaryWriter
        log = SummaryWriter(log_dir)

    '''
    You can also change your train code here if neccesary
    '''

    # If your model does not train well, you may swap out the optimizer or change the lr
    optimizer = optim.Adam(model.parameters(), lr = 1e-4)
    loss = nn.CrossEntropyLoss()

    for iteration in range(iterations):
        model.train()
        # Construct a mini-batch
        batch = np.random.choice(train_inputs.shape[0], batch_size)
        batch_inputs = augment_train(train_inputs[batch])
        batch_labels = torch.as_tensor(train_labels[batch], dtype=torch.long)

        # zero the gradients (part of pytorch backprop)
        optimizer.zero_grad()

        # Compute the model output and loss (view flattens the input)
        model_outputs = model(batch_inputs)
        t_loss_val = loss(model_outputs, batch_labels)
        t_acc_val = accuracy(model_outputs, batch_labels)

        # Compute the gradient
        t_loss_val.backward()

        # Update the weights
        optimizer.step()

        if iteration % 10 == 0:
            model.eval()
            batch = np.random.choice(val_inputs.shape[0], 256)
            batch_inputs = augment_val(val_inputs[batch])
            batch_labels = torch.as_tensor(val_labels[batch], dtype=torch.long)
            model_outputs = model(batch_inputs)
            v_acc_val = accuracy(model_outputs, batch_labels)

            print('[%5d]'%iteration, 'loss = %f'%t_loss_val, 't_acc = %f'%t_acc_val, 'v_acc = %f'%v_acc_val)
            if log is not None:
                log.add_scalar('train/loss', t_loss_val, iteration)
                log.add_scalar('train/acc', t_acc_val, iteration)
                log.add_scalar('val/acc', v_acc_val, iteration)

    # Save the trained model
    torch.save(model.state_dict(), os.path.join(dirname, 'convnet.th')) # Do NOT modify this line

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--iterations', type=int, default=2000)
    parser.add_argument('-l', '--log_dir')
    args = parser.parse_args()

    print ('[I] Start training')
    train(args.iterations, log_dir=args.log_dir)
    print ('[I] Training finished')