import time

import torch
import torch.backends.cudnn as cudnn
from sklearn.metrics.ranking import roc_auc_score
from tensorboardX import SummaryWriter
from torch.utils.data import DataLoader
from torchvision.datasets import ImageFolder

from convnet_models import ConvnetModel

CPU = torch.device("cpu")

EMA_ALPHA_TRAIN_LOSS = 0.99

def train(*, path_data_train, path_data_valid, path_log, path_model, model_name, model_pretrained, batch_size,
          epoch_num, checkpoint, device, transform_train, transform_valid, optimizer_fn):
    model = ConvnetModel(model_name, class_count=1, is_trained=model_pretrained).to(device)
    data_loader_train = DataLoader(
        ImageFolder(path_data_train, transform=transform_train),
        batch_size=batch_size, shuffle=True, num_workers=20, pin_memory=True)
    data_loader_valid = DataLoader(
        ImageFolder(path_data_valid, transform=transform_valid),
        batch_size=batch_size, shuffle=False, num_workers=20, pin_memory=True)
    optimizer, scheduler = optimizer_fn(model.parameters())
    # -------------------- SETTINGS: LOSS
    loss_fn = torch.nn.BCEWithLogitsLoss(size_average=True)
    # ---- Load checkpoint
    if checkpoint:
        model_checkpoint = torch.load(checkpoint)
        model.load_state_dict(model_checkpoint['state_dict'])
        optimizer.load_state_dict(model_checkpoint['optimizer'])
    # Write Tensorboard
    writer = SummaryWriter(log_dir=path_log)
    # ---- TRAIN THE NETWORK
    loss_min, auroc_max = epoch_valid(model, data_loader_valid, device=device)
    print('Initial: valid-loss= ' + str(loss_min) + "  auroc= " + str(auroc_max))
    writer.add_scalar(tag="valid-loss", scalar_value=loss_min, global_step=0)
    writer.add_scalar(tag="valid-auroc", scalar_value=auroc_max, global_step=0)
    for epoch in range(0, epoch_num):
        train_loss = epoch_train(model, data_loader_train, optimizer, loss_fn, device=device)
        writer.add_scalar(tag="train-loss", scalar_value=train_loss, global_step=epoch + 1)
        loss, auroc = epoch_valid(model, data_loader_valid, device=device)
        timestamp_now = time.strftime("%Y%m%d") + '-' + time.strftime("%H%M%S")
        scheduler.step(loss)
        writer.add_scalar(tag="valid-loss", scalar_value=loss, global_step=epoch + 1)
        writer.add_scalar(tag="valid-auroc", scalar_value=auroc, global_step=epoch + 1)
        save_flag, save_flag_l, save_flag_a = '----', '-', '-'
        if loss < loss_min:
            save_flag, save_flag_l = 'save', 'L'
            loss_min = loss
            torch.save({
                'epoch': epoch + 1,
                'state_dict': model.state_dict(),
                'best_loss': loss_min,
                'optimizer': optimizer.state_dict()
            }, path_model + "-L.pth.tar")
        if auroc > auroc_max:
            save_flag, save_flag_a = 'save', 'A'
            auroc_max = auroc
            torch.save({
                'epoch': epoch + 1,
                'state_dict': model.state_dict(),
                'best_loss': loss_min,
                'optimizer': optimizer.state_dict()
            }, path_model + "-A.pth.tar")
        print('Epoch [{0}] [{1}] [{2}] train-loss={3}  valid-loss={4}  auroc={5}'.format(
            epoch + 1,
            save_flag + save_flag_l + save_flag_a,
            timestamp_now,
            train_loss,
            loss,
            auroc
        ))

        # if loss < loss_min:
        #     loss_min = loss
        #     torch.save({'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_loss': loss_min,
        #                 'optimizer': optimizer.state_dict()}, path_model)
        #     print('Epoch [' + str(epoch + 1) + '] [save] [' + timestamp_now + '] train-loss= ' + str(
        #         train_loss) + '  valid-loss= ' + str(loss) + '  auroc= ' + str(auroc))
        #     writer.add_scalar(tag="valid-loss", scalar_value=loss, global_step=epoch + 1)
        #     writer.add_scalar(tag="valid-auroc", scalar_value=auroc, global_step=epoch + 1)
        # else:
        #     print('Epoch [' + str(epoch + 1) + '] [----] [' + timestamp_now + '] train-loss= ' + str(
        #         train_loss) + '  valid-loss= ' + str(loss) + '  auroc= ' + str(auroc))
        #     writer.add_scalar(tag="valid-loss", scalar_value=loss, global_step=epoch + 1)
        #     writer.add_scalar(tag="valid-auroc", scalar_value=auroc, global_step=epoch + 1)
    writer.close()

def epoch_train(model, data_loader, optimizer, loss_fn, device):
    model.train()
    running_loss, running_norm = 0.0, 0.0
    for (x, y) in data_loader:
        x = x.to(device)
        y = y.to(torch.float32).to(device)
        loss = loss_fn(model(x).view(-1), y)
        running_loss = running_loss * EMA_ALPHA_TRAIN_LOSS + loss.item()
        running_norm = running_norm * EMA_ALPHA_TRAIN_LOSS + 1.0
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    return running_loss / running_norm

def epoch_valid(model, data_loader, device):
    true = torch.LongTensor()
    score = torch.FloatTensor()
    bce = []
    model.eval()
    for (x, y) in data_loader:
        bs, n_crops, c, h, w = x.size()
        x = x.to(device)
        true = torch.cat((true, y), 0)
        with torch.no_grad():
            y_hat = torch.nn.Sigmoid()(model(x.view(-1, c, h, w)))
        y_hat = y_hat.to(CPU)
        y_hat = y_hat.view(bs, n_crops).mean(1)
        score = torch.cat((score, y_hat), 0)
        loss_fn = torch.nn.BCELoss(size_average=True)
        bce.append(loss_fn(y_hat, y.to(torch.float32)))
    return torch.Tensor(bce).mean().item(), compute_auroc(true, score)

def compute_auroc(true, score):
    true = true.cpu().numpy()
    score = score.cpu().numpy()
    # roc_auc_score(true, score)
    return roc_auc_score(true, score)

def test(*, path_data, path_model, model_name, model_pretrained, batch_size, device, transform):
    cudnn.benchmark = True
    model = ConvnetModel(model_name, class_count=1, is_trained=model_pretrained).to(device)
    model_checkpoint = torch.load(path_model + ".pth.tar")
    model.load_state_dict(model_checkpoint['state_dict'])
    data_loader_test = DataLoader(
        ImageFolder(path_data, transform=transform),
        batch_size=batch_size, shuffle=False, num_workers=10, pin_memory=True)
    true = torch.LongTensor()
    score = torch.FloatTensor()
    bce = []
    model.eval()
    for (x, y) in data_loader_test:
        bs, n_crops, c, h, w = x.size()
        x = x.to(device)
        true = torch.cat((true, y), 0)
        with torch.no_grad():
            y_hat = torch.nn.Sigmoid()(model(x.view(-1, c, h, w)))
        y_hat = y_hat.to(CPU)
        y_hat = y_hat.view(bs, n_crops).mean(1)
        score = torch.cat((score, y_hat), 0)
        loss_fn = torch.nn.BCELoss(size_average=True)
        bce.append(loss_fn(y_hat, y.to(torch.float32)))
    loss, auroc = torch.Tensor(bce).mean().item(), compute_auroc(true, score)
    auroc = compute_auroc(true, score)
    print('loss = ' + str(loss) + '  AUROC = ' + str(auroc))
    return


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

__author__ = 'Francis'

from globals.functions import printLn

imgUrl = '/MyData/Python/test/asset/imgs/test.jpg'

printLn('使用Image.thumbnail缩小一张图片')
from PIL import Image

# 打开一个图片文件
im = Image.open('/MyData/Python/test/asset/imgs/test.jpg')
w, h = im.size
print('Original image size: %sx%s' % (w, h))
# 缩放到50%
im.thumbnail((w // 2, h // 2))
print('Resize image to: %sx%s' % (w // 2, h // 2))
# 把缩放后的图像用jped格式保存
im.save('/MyData/Python/test/asset/imgs/thumbtest.jpg', 'jpeg')

printLn('使用ImageFilter模糊一张图片')
from PIL import Image, ImageFilter

# 打开一个图片
img = Image.open(imgUrl)
# 应用模糊路径
img2 = im.filter(ImageFilter.BLUR)
img2.save('/MyData/Python/test/asset/imgs/blurtest.jpg', 'jpeg')

printLn('使用ImageDraw生成一张验证码图片')
from PIL import Image, ImageDraw, ImageFont, ImageFilter

import random

# 随机字母
def rndChar():
    return chr(random.randint(65, 90))

# 随机颜色
def rndColor():
    return (random.randint(64, 255), random.randint(64, 255), random.randint(64, 255))

# 随机颜色2
def rndColor2():
    return (random.randint(32, 127), random.randint(32, 127), random.randint(32, 127))

# 图片大小
width = 60 * 4
height = 60
image = Image.new('RGB', (width, height), (255, 255, 255))
# 创建Font对象
font = ImageFont.truetype('Arial.ttf', 36)
# 创建Draw对象
draw = ImageDraw.Draw(image)
# 填充每个像素
for x in range(width):
    for y in range(height):
        draw.point((x, y), fill=rndColor())

# 输出文字
for t in range(4):
    draw.text((60 * t + 10, 10), rndChar(), font=font, fill=rndColor2())

# 模糊
image = image.filter(ImageFilter.BLUR)
image.save('/MyData/Python/test/asset/imgs/code.jpg', 'jpeg')


from django.urls import path
from . import views

app_name = 'daftar'
urlpatterns = [
    path('', views.index, name='home'),
    # path('karyawan/<int:pk>/delete', views.karyawan_delete, name='karyawan_delete'),
    # path('karyawan/<int:pk>/edit', views.karyawan_edit, name='karyawan_edit'),
    # path('karyawan/<int:pk>/', views.karyawan_detail, name='karyawan_detail'),
    path('cari/', views.cari, name='cari'),
    path('add/<int:pk>', views.daftar_tk, name='add_tk'),
    # path('daftar/', views.karyawan_list, name='karyawan_list'),
    #path('pk_list', views.pk_list, name='pk_list'),

]


"""Provides routes for the external API."""

import json

from flask.json import jsonify
from flask import Blueprint, render_template, redirect, request, url_for, \
    Response, make_response, send_file
from werkzeug.exceptions import NotFound, Forbidden, Unauthorized, \
    InternalServerError, HTTPException, BadRequest
from arxiv.base import routes as base_routes
from arxiv import status
from arxiv.users import domain as auth_domain
from arxiv.users.auth import scopes

from arxiv.users.auth.decorators import scoped

from filemanager.services import uploads
from filemanager.controllers import upload

blueprint = Blueprint('upload_api', __name__, url_prefix='/filemanager/api')

def is_owner(session: auth_domain.Session, upload_id: str, **kwargs) -> bool:
    """User must be the upload owner, or an admin."""
    upload_obj = uploads.retrieve(upload_id)
    if upload_obj is None:
        return True

    return session.user.user_id == uploads.retrieve(upload_id).owner_user_id

@blueprint.route('/status', methods=['GET'])
def service_status() -> tuple:
    """Health check endpoint."""
    return jsonify({'status': 'OK', 'total_uploads': 1}), status.HTTP_200_OK

@blueprint.route('/', methods=['POST'])
@scoped(scopes.WRITE_UPLOAD)
def new_upload() -> tuple:
    """Initial upload where workspace (upload_id) does not yet exist.

    This requests creates a new workspace. Upload package is processed normally.

    Client response include upload_id which is necessary for subsequent requests."""

    # Optional category/archive - this is required to accurately calculate
    # whether submission is oversize.
    archive_arg = request.form.get('archive', None)

    # is this optional??
    archive_arg = request.args.get('archive')

    # Required file payload
    file = request.files.get('file', None)

    # Collect arguments and call main upload controller
    data, status_code, headers = upload.upload(None, file, archive_arg,
                                               request.session.user)

    return jsonify(data), status_code, headers

@blueprint.route('<int:upload_id>', methods=['POST'])
@scoped(scopes.WRITE_UPLOAD, authorizer=is_owner)
def upload_files(upload_id: int) -> tuple:
    """Upload individual files or compressed archive
    and add to existing upload workspace. Multiple uploads accepted."""
    archive_arg = request.form.get('archive')
    ancillary = request.form.get('ancillary', None) == 'True'
    file = request.files.get('file', None)
    # Attempt to process upload
    data, status_code, headers = upload.upload(upload_id, file, archive_arg,
                                               request.session.user,
                                               ancillary=ancillary)
    return jsonify(data), status_code, headers

# Separated this out so that we can support auth granularity. -E
@blueprint.route('<int:upload_id>', methods=['GET'])
@scoped(scopes.READ_UPLOAD, authorizer=is_owner)
def get_upload_files(upload_id: int) -> tuple:
    data, status_code, headers = upload.upload_summary(upload_id)
    return jsonify(data), status_code, headers

@blueprint.route('<int:upload_id>/<path:public_file_path>', methods=['DELETE'])
@scoped(scopes.DELETE_UPLOAD_FILE, authorizer=is_owner)
def delete_file(upload_id: int, public_file_path: str) -> tuple:
    """Delete individual file."""
    data, status_code, headers = upload.client_delete_file(upload_id,
                                                           public_file_path)
    return jsonify(data), status_code, headers

# File and workspace deletion

@blueprint.route('<int:upload_id>/delete_all', methods=['POST'])
@scoped(scopes.WRITE_UPLOAD, authorizer=is_owner)
def delete_all_files(upload_id: int) -> tuple:
    """Delete all files in specified workspace."""
    data, status_code, headers = upload.client_delete_all_files(upload_id)
    return jsonify(data), status_code, headers

@blueprint.route('<int:upload_id>', methods=['DELETE'])
@scoped(scopes.DELETE_UPLOAD_WORKSPACE)
def workspace_delete(upload_id: int) -> tuple:
    """Delete the specified workspace."""
    data, status_code, headers = upload.delete_workspace(upload_id)
    return jsonify(data), status_code, headers

# Lock and unlock upload workspace

@blueprint.route('/<int:upload_id>/lock', methods=['POST'])
@scoped(scopes.WRITE_UPLOAD, authorizer=is_owner)
def lock(upload_id: int) -> tuple:
    """Lock submission (read-only mode) while other services are
    processing (major state transitions are occurring)."""
    data, status_code, headers = upload.upload_lock(upload_id)
    return jsonify(data), status_code, headers

# This could be thaw or release instead of unlock
@blueprint.route('/<int:upload_id>/unlock', methods=['POST'])
@scoped(scopes.WRITE_UPLOAD, authorizer=is_owner)
def unlock(upload_id: int) -> tuple:
    """Unlock submission and enable write mode."""
    data, status_code, headers = upload.upload_unlock(upload_id)
    return jsonify(data), status_code, headers

# This could be remove or delete instead of release
@blueprint.route('/<int:upload_id>/release', methods=['POST'])
@scoped(scopes.WRITE_UPLOAD, authorizer=is_owner)
def release(upload_id: int) -> tuple:
    """Client indicates they are finished with submission.
    File management service is free to remove submissions files,
    or schedule files for removal at later time."""
    data, status_code, headers = upload.upload_release(upload_id)
    return jsonify(data), status_code, headers

# This could be remove or delete instead of release
@blueprint.route('/<int:upload_id>/unrelease', methods=['POST'])
@scoped(scopes.WRITE_UPLOAD, authorizer=is_owner)
def unrelease(upload_id: int) -> tuple:
    """Client indicates they are finished with submission.
    File management service is free to remove submissions files,
    or schedule files for removal at later time."""
    data, status_code, headers = upload.upload_unrelease(upload_id)
    return jsonify(data), status_code, headers

# Get content

@blueprint.route('/<int:upload_id>/content', methods=['HEAD'])
@scoped(scopes.READ_UPLOAD)
def check_upload_content_exists(upload_id: int) -> tuple:
    """
    Verify that upload content exists.

    Returns an ``ETag`` header with the current source package checksum.
    """
    data, status_code, headers = upload.check_upload_content_exists(upload_id)
    return jsonify(data), status_code, headers

@blueprint.route('/<int:upload_id>/content', methods=['GET'])
@scoped(scopes.READ_UPLOAD)
def get_upload_content(upload_id: int) -> tuple:
    """
    Get the upload content as a compressed tarball.

    Returns a stream with mimetype ``application/tar+gzip``, and an ``ETag``
    header with the current source package checksum.
    """
    data, status_code, headers = upload.get_upload_content(upload_id)
    response = send_file(data, mimetype="application/tar+gzip")
    response.set_etag(headers.get('ETag'))
    return response

@blueprint.route('/<int:upload_id>/<path:public_file_path>/content', methods=['HEAD'])
@scoped(scopes.READ_UPLOAD)
def check_file_exists(upload_id: int, public_file_path: str) -> tuple:
    """
    Verify specified file exists.

    Returns an ``ETag`` header with the current source file checksum.
    """
    data, status_code, headers = upload.check_upload_file_content_exists(upload_id, public_file_path)

    return jsonify(data), status_code, headers

@blueprint.route('/<int:upload_id>/<path:public_file_path>/content', methods=['GET'])
@scoped(scopes.READ_UPLOAD)
def get_file_content(upload_id: int, public_file_path: str) -> tuple:
    """
    Return content of specified file.

    """

    data, status_code, headers = upload.get_upload_file_content(upload_id, public_file_path)

    response = send_file(data, mimetype="application/*")
    response.set_etag(headers.get('ETag'))
    return response

# Get logs

@blueprint.route('/<int:upload_id>/log', methods=['HEAD'])
@scoped(scopes.READ_UPLOAD_LOGS)
def check_upload_source_log_exists(upload_id: int) -> tuple:
    """
    Check that upload source log exists.

    Parameters
    ----------
    upload_id: int

    Returns
    -------
    Returns an ``ETag`` header with the current source package checksum.

    """
    data, status_code, headers = upload.check_upload_source_log_exists(upload_id)
    return jsonify(data), status_code, headers

@blueprint.route('/<int:upload_id>/log', methods=['GET'])
@scoped(scopes.READ_UPLOAD_LOGS)
def get_upload_source_log(upload_id: int) -> tuple:
    """
    Get the upload source log for specified upload workspace. This provides details of all
    upload/deletion activity on specified workspace.

    Parameters
    ----------
    upload_id : int

    Returns
    -------

    """
    data, status_code, headers = upload.get_upload_source_log(upload_id)
    response = send_file(data, mimetype="application/tar+gzip")
    response.set_etag(headers.get('ETag'))
    return response

@blueprint.route('/log', methods=['HEAD'])
@scoped(scopes.READ_UPLOAD_SERVICE_LOGS)
def check_upload_service_log_exists() -> tuple:
    """
    Check that upload source log exists.

    Returns
    -------
    Returns an ``ETag`` header with the current source package checksum.

    """
    data, status_code, headers = upload.check_upload_service_log_exists()
    return jsonify(data), status_code, headers

@blueprint.route('/log', methods=['GET'])
@scoped(scopes.READ_UPLOAD_SERVICE_LOGS)
def get_upload_service_log() -> tuple:
    """
    Return the top level file management service log that records high-level requests along with
    important errors/warnings. Details for specific upload workspace are found in workspace
    source log.

    Returns
    -------

    """
    data, status_code, headers = upload.get_upload_service_log()
    response = send_file(data, mimetype="application/tar+gzip")
    response.set_etag(headers.get('ETag'))
    return response

# Exception handling

@blueprint.errorhandler(NotFound)
@blueprint.errorhandler(InternalServerError)
@blueprint.errorhandler(Forbidden)
@blueprint.errorhandler(Unauthorized)
@blueprint.errorhandler(BadRequest)
@blueprint.errorhandler(NotImplementedError)
def handle_exception(error: HTTPException) -> Response:
    """
    JSON-ify the error response.

    This works just like the handlers in zero.routes.ui, but instead of
    rendering a template we are JSON-ifying the response. Note that we are
    registering the same error handler for several different exceptions, since
    we aren't doing anything that is specific to a particular exception.
    """
    content = jsonify({'reason': error.description})

    # Each Werkzeug HTTP exception has a class attribute called ``code``; we
    # can use that to set the status code on the response.
    response = make_response(content, error.code)
    return response


import pandas as pd
import numpy as np
from lmfit import report_fit  # Minimizer, Parameters,
from lmfit.models import GaussianModel
import matplotlib.pyplot as plt
import pickle
import calendar
from scipy import signal

# UGRHI SP
# tagname = '58220000'
# -------------------------
# tagname = '3D-001'
# -------------------------
# tagname = '4C-007'
# -------------------------
# tagname = '4B-015'
# -------------------------
tagname = '5B-011'
# -------------------------

dirInput = '/home/evandro/lcbiag/ProcessoOtimizacaoModelos/' \
    'calibracaoBalagua/InstrucaoModBalagua/dados/ugrhi_sp/'

dirMR = '/home/evandro/lcbiag/ProcessoOtimizacaoModelos/' \
    'calibracaoBalagua/InstrucaoModBalagua/resultados/'

dirplot = '/home/evandro/lcbiag/ProcessoOtimizacaoModelos/' \
    'calibracaoBalagua/InstrucaoModBalagua/plots/'

# CRU
arqvminres = tagname+'_ugrhi_leastsqMinimizerResult.pkl'
input_df = pd.read_pickle(dirInput+tagname+'_ugrhi_sp.pkl')

# Xavier
# arqvminres = tagname+'_xavier_ugrhi_leastsqMinimizerResult.pkl'
# input_df = pd.read_pickle(dirInput+tagname+'_xavier_ugrhi_sp.pkl')

out_leastsq = pickle.load(open(dirMR + arqvminres, "rb"))

"""
Modelagem do balanco de agua com os parametros otimos
"""
a1 = out_leastsq.params['a1']
a2 = out_leastsq.params['a2']
a22 = out_leastsq.params['a22']
a3 = out_leastsq.params['a3']

etp = np.asarray(input_df.etp)
p2 = np.asarray(input_df.p)
q2 = np.asarray(input_df.q)

# where
posval = np.asarray(~np.isnan(etp) &
                    ~np.isnan(p2) &
                    ~np.isnan(q2)).nonzero()
posval = posval[0]

datatempo = input_df.index

nxts = len(input_df)

ts_mt = np.empty(nxts)
ts_dt = np.empty(nxts)
ts_u = np.empty(nxts)
ts_Dm = np.empty(nxts)
ts_r = np.empty(nxts)
ts_mt.fill(np.nan)
ts_dt.fill(np.nan)
ts_u.fill(np.nan)
ts_Dm.fill(np.nan)
ts_r.fill(np.nan)

m1 = np.float64(500.)
r2 = np.float64(0.)
s2 = np.float64(0.)
n2 = np.float64(0.)
d2 = np.float64(0.)
m2 = np.float64(0.)

m_func = len(posval)

for kount in range(0, m_func):
    """
    Modelo de balanço de agua

    r2 => evapotranpiracao real
    s2 => escoamento lento
    n2 => precipitacao ativa
    f2 => escoamento rapido
    d2 => vazao com filtro
    """
    posck = posval[kount]

    r2 = min(
        etp[posck]*(1.-a1**((p2[posck]+max(m1, 0.))/etp[posck])),
        (p2[posck]+max(m1, 0.))
    )
    s2 = a2*(max(m1, 0.)**a22)
    n2 = p2[posck]-etp[posck]*(1-np.exp(-p2[posck]/etp[posck]))
    f2 = a3*max(m1, 0.)*n2
    d2 = s2+f2
    m2 = m1 + p2[posck] - r2 - d2
    # print(d2, s2, f2, m2)
    ts_mt[posck] = m2
    ts_dt[posck] = d2
    ts_u[posck] = (np.sqrt(q2[posck]) - np.sqrt(d2))
    ts_Dm[posck] = m2 - m1
    ts_r[posck] = r2

    m1 = m2

'''
Visualicao dos dados
'''

x_eixo = datatempo

# histograma de u
minbin = -5
maxbin = 5
nx = 30.
step = (maxbin - minbin) / nx

binarr = np.arange(nx+1)*step + minbin
hist_u, edges_u = np.histogram(ts_u, bins=binarr, density='True')

xhist = np.arange(nx)*step + minbin + step

# ajuste gaussiano
gmodel = GaussianModel()
print('parameter names: {}'.format(gmodel.param_names))
print('independent variables: {}'.format(gmodel.independent_vars))

gparams = gmodel.make_params()
gresult = gmodel.fit(hist_u, gparams, x=xhist)
print(gresult.fit_report())

# Para agrupamentos mensais/anuais
# d_mes = dict(enumerate(calendar.month_abbr))
dados = {'ETP': etp, 'ET': ts_r, 'P': p2,
         'Qm': q2, 'Qc': ts_dt, 'S': ts_mt, 'DeltaS': ts_Dm,
         'mes': x_eixo.month}
toanual_df = pd.DataFrame(data=dados, index=x_eixo)
# totanual = toanual_df.resample('Y').sum()
totanual = toanual_df.resample('Y').agg(np.nansum)
medianual = toanual_df.resample('Y').agg(np.nanmean)
nantotanual = toanual_df.isnull().resample('Y').agg(np.sum)

'''
Plota paineis de verificacao
'''

'''
Figura series temporais
'''

# where
posval2 = np.asarray(~np.isnan(etp) &
                     ~np.isnan(p2) &
                     ~np.isnan(q2)).nonzero()
posval2 = posval2[0]

xi = x_eixo[posval2[0]]
xf = x_eixo[posval2[-1]]

# xi = np.datetime64('2014-01-01')
# xf = np.datetime64('2016-08-01')

pngfigplot = dirplot+tagname+'_pltBalagua_SerieTemporal.png'

# report_fit(out_leastsq)
# print(gresult.fit_report())

fig = plt.figure()
fig.set_figwidth(25)
fig.set_figheight(18)
# fig.suptitle(tagname)

plt.style.use('bmh')  # ggplot dark_background classic bmh seaborn-bright

plt.subplot(7, 1, 1)
plt.title(tagname, fontsize=10)
plt.plot(x_eixo, ts_mt, '.-')
plt.ylabel('S\n(mm/mês)')
plt.xlim(xi, xf)

plt.subplot(7, 1, 2)
plt.plot(x_eixo, ts_Dm, '.-')
plt.ylabel('$\Delta$S\n(mm/mês)')
plt.xlim(xi, xf)

# considera os anos com falhas nas medidas menores do que 3 meses
posval9 = np.asarray((nantotanual.DeltaS < 3.)).nonzero()
posval9 = posval9[0]

x_eixo_ano_ts = totanual.index
nxano = len(x_eixo_ano_ts)
DeltaS_ano_ts = np.empty(nxano)
DeltaS_ano_ts.fill(np.nan)

DeltaS_ano_ts[posval9] = totanual.DeltaS[posval9]

plt.subplot(7, 1, 3)
#plt.plot(x_eixo, ts_mt, '.-')
plt.plot(x_eixo_ano_ts, DeltaS_ano_ts, '.-')
plt.ylabel('$\Delta$S\n(mm/ano)')
plt.xlim(xi, xf)

ax1 = plt.subplot(7, 1, 4)
ax1.plot(x_eixo, q2, 'black', label='Q$_m$', linewidth=0.8)
ax1.plot(x_eixo, ts_dt, 'red', label='Q$_c$', linewidth=0.8)
plt.ylabel('Q$_{m,c}$\n(mm/mês)')
plt.xlim(xi, xf)

ax2 = ax1.twinx()
ax2.plot(x_eixo, p2, 'b', label='P', linewidth=0.8)
plt.ylabel('P\n(mm/mês)', axes=ax2)
plt.xlim(xi, xf)

ax2.legend(loc='upper right')
ax1.legend(loc='upper left')

# Anual
qm_anual = np.asarray(medianual.Qm)
qc_anual = np.asarray(medianual.Qc)
p_anual = np.asarray(medianual.P)

ax1 = plt.subplot(7, 1, 5)
ax1.plot(x_eixo_ano_ts, qm_anual, 'black', label='Q$_m$', linewidth=0.8)
ax1.plot(x_eixo_ano_ts, qc_anual, 'red', label='Q$_c$', linewidth=0.8)
plt.ylabel('Q$_{m,c}$\n(mm/ano)')
plt.xlim(xi, xf)

ax2 = ax1.twinx()
ax2.plot(x_eixo_ano_ts, p_anual, 'b', label='P', linewidth=0.8)
plt.ylabel('P\n(mm/ano)', axes=ax2)
plt.xlim(xi, xf)

ax2.legend(loc='upper right')
ax1.legend(loc='upper left')

plt.subplot(7, 1, 6)
plt.plot(x_eixo, ts_r, 'g', label='ET', linewidth=0.8)
plt.plot(x_eixo, etp, 'black', label='ETP', linewidth=0.8)
plt.ylabel('ET,ETP\n(mm/mês)')
plt.xlim(xi, xf)
plt.legend()

# Anual
ET_anual = np.asarray(medianual.ET)
ETP_anual = np.asarray(medianual.ETP)
plt.subplot(7, 1, 7)
plt.plot(x_eixo_ano_ts, ET_anual, 'g', label='ET', linewidth=0.8)
plt.plot(x_eixo_ano_ts, ETP_anual, 'black', label='ETP', linewidth=0.8)
plt.ylabel('ET,ETP\n(mm/ano)')
plt.xlim(xi, xf)
plt.legend()

plt.savefig(pngfigplot, dpi=300, bbox_inches='tight')

'''
Figura - erro do modelamento
'''

pngfigplot = dirplot+tagname+'_pltBalagua_ErroModelo.png'
fig = plt.figure()
fig.set_figwidth(7)
fig.set_figheight(4)

# semivariancia
plt.subplot(2, 1, 1)

ts_u_svar = ts_u[~np.isnan(ts_u)]

semivar = signal.correlate(ts_u_svar, ts_u_svar)
# https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.correlate.html
semivarplt = semivar[np.int(semivar.size/2):]
plt.plot(semivarplt)
plt.ylabel('Autocorrelação')
plt.xlabel('Lag')

plt.subplot(2, 1, 2)
plt.scatter(xhist, hist_u)
plt.plot(xhist, gresult.best_fit, 'r-', label='Gaussiana')
plt.ylabel('(%)')
plt.xlabel('u')
plt.legend()
corrperson = np.corrcoef(hist_u, gresult.best_fit)
corrperson = corrperson[0, 1]
plt.text(2.5, np.max(hist_u)/2.,
         'C(prob, gauss) = '+'{:5.3f}'.format(corrperson), wrap=True)
print('correlacao de pearson')
print(np.corrcoef(hist_u, gresult.best_fit))

plt.savefig(pngfigplot, dpi=300, bbox_inches='tight')

# '''
# Figura anual
# '''

# pngfigplot = dirplot+tagname+'_pltBalagua_ts_anual.png'

# # considera os anos com falhas nas medidas menores do que 3 meses
# posval9 = np.asarray((nantotanual.DeltaS < 3.)).nonzero()
# posval9 = posval9[0]

# x_eixo_ano_ts = totanual.index
# nxano = len(x_eixo_ano_ts)
# DeltaS_ano_ts = np.empty(nxano)
# DeltaS_ano_ts.fill(np.nan)

# DeltaS_ano_ts[posval9] = totanual.DeltaS[posval9]

# fig = plt.figure()
# fig.set_figwidth(25)
# fig.set_figheight(18)
# # fig.suptitle(tagname)

# plt.style.use('bmh')  # ggplot dark_background classic bmh seaborn-bright

# plt.subplot(6, 1, 1)
# plt.title(tagname, fontsize=10)
# #plt.plot(x_eixo, ts_mt, '.-')
# plt.plot(x_eixo_ano_ts, DeltaS_ano_ts, '.-')
# plt.ylabel('S\n(mm/ano)')
# plt.xlim(xi, xf)

# # dados = {'ETP': etp, 'ET': ts_r, 'P': p2,
# #          'Qm': q2, 'Qc': ts_dt, 'S': ts_mt, 'DeltaS': ts_Dm,
# #          'mes': x_eixo.month}
# # toanual_df = pd.DataFrame(data=dados, index=x_eixo)
# # # totanual = toanual_df.resample('Y').sum()
# # totanual = toanual_df.resample('Y').agg(np.nansum)
# # medianual = toanual_df.resample('Y').agg(np.nanmean)
# # nantotanual = toanual_df.isnull().resample('Y').agg(np.sum)

# qm_anual = np.asarray(medianual.Qm)
# qc_anual = np.asarray(medianual.Qc)
# p_anual = np.asarray(medianual.P)

# ax1 = plt.subplot(6, 1, 2)
# ax1.plot(x_eixo_ano_ts, qm_anual, 'black', label='Q$_m$', linewidth=0.8)
# ax1.plot(x_eixo_ano_ts, qc_anual, 'red', label='Q$_c$', linewidth=0.8)
# plt.ylabel('(mm/ano)')
# plt.xlim(xi, xf)

# ax2 = ax1.twinx()
# ax2.plot(x_eixo_ano_ts, p_anual, 'b', label='P', linewidth=0.8)
# plt.ylabel('P (mm/ano)', axes=ax2)
# plt.xlim(xi, xf)

# ax2.legend(loc='upper right')
# ax1.legend(loc='upper left')

# ET_anual = np.asarray(medianual.ET)
# ETP_anual = np.asarray(medianual.ETP)

# plt.subplot(6, 1, 3)
# plt.plot(x_eixo_ano_ts, ET_anual, 'g', label='ET', linewidth=0.8)
# plt.plot(x_eixo_ano_ts, ETP_anual, 'black', label='ETP', linewidth=0.8)
# plt.ylabel('(mm/ano)')
# plt.xlim(xi, xf)
# plt.legend()

# plt.savefig(pngfigplot, dpi=300, bbox_inches='tight')

'''
Figura - ciclo anual
'''

pngfigplot = dirplot+tagname+'_pltBalagua_CicloAnual.png'

canual = toanual_df.groupby('mes').agg(np.nanmean)

d_mes = dict(enumerate(calendar.month_abbr))
canual.index = canual.index.map(d_mes)  # .str.slice(stop=3)

fig = plt.figure()
fig.set_figwidth(6)
fig.set_figheight(6)

plt.style.use('bmh')  # ggplot dark_background classic bmh seaborn-bright
# plt.rcParams.update({'font.size': 10})

ax1 = plt.subplot(311)
ax1.plot(canual.Qm, label='Q$_m$')
ax1.plot(canual.Qc, label='Q$_c$')
plt.ylabel('(mm/mês)')

ax2 = ax1.twinx()
ax2.plot(canual.P, 'black', label='P')
plt.ylabel('(mm/mês)')
ax2.legend(loc='upper right')
ax1.legend(loc='upper left')

ax1 = plt.subplot(312)
ax1.plot(canual.ET, label='ET')
ax1.plot(canual.ETP, label='ETP')
plt.ylabel('(mm/mês)')

ax2 = ax1.twinx()
ax2.plot(canual.P, 'black', label='P')
plt.ylabel('(mm/mês)')
ax2.legend(loc='upper right')
ax1.legend(loc='upper left')

ax1 = plt.subplot(313)
ax1.plot(canual.S, label='S')
plt.ylabel('(mm/mês)')

ax2 = ax1.twinx()
ax2.plot(canual.P, 'black', label='P')
plt.ylabel('(mm/mês)')
ax2.legend(loc='upper right')
ax1.legend(loc='upper left')

plt.savefig(pngfigplot, dpi=300, bbox_inches='tight')

'''
Terceira figura
'''

pngfigplot = dirplot+tagname+'_pltBalagua_Dispersao.png'

fig = plt.figure()
fig.set_figwidth(10)
fig.set_figheight(9)

plt.subplot(221)

#  where
q2 = np.asarray(q2)
posval3 = np.asarray(~np.isnan(q2) |
                     ~np.isnan(ts_dt)).nonzero()
posval3 = posval3[0]
q2_plt = q2[posval3]
ts_dt_plt = ts_dt[posval3]

minv = np.nanmin(np.concatenate((q2_plt, ts_dt_plt)))
maxv = np.nanmax(np.concatenate((q2_plt, ts_dt_plt)))
xx = np.arange(minv, maxv)
yy = xx
plt.plot(xx, yy, c='green', linewidth=0.8)
plt.scatter(q2_plt, ts_dt_plt, c='black', marker='+', linewidth=0.8)
plt.ylabel('Q$_c$ (mm/mês)')
plt.xlabel('Q$_m$ (mm/mês)')

plt.subplot(222)
posval4 = np.asarray(~np.isnan(ts_r) |
                     ~np.isnan(etp)).nonzero()
posval4 = posval4[0]
ts_r_plt = ts_r[posval4]
etp_plt = etp[posval4]

minv = np.nanmin(np.concatenate((ts_r_plt, etp_plt)))
maxv = np.nanmax(np.concatenate((ts_r_plt, etp_plt)))
xx = np.arange(minv, maxv)
yy = xx
plt.plot(xx, yy, c='green', linewidth=0.8)
plt.scatter(etp_plt, ts_r_plt, c='black', marker='+', linewidth=0.8)
plt.xlabel('ETP (mm/mês)')
plt.ylabel('ET (mm/mês)')

plt.subplot(223)
posval5 = np.asarray((nantotanual.Qm < 1.) & (nantotanual.P < 1.)).nonzero()
posval5 = posval5[0]
Qm_ano_plt = totanual.Qm[posval5]
P_ano_plt = totanual.P[posval5]

posval6 = np.asarray((nantotanual.Qc < 1.) & (nantotanual.P < 1.)).nonzero()
posval6 = posval6[0]
Qc_ano_plt = totanual.Qc[posval6]
P_ano_plt = totanual.P[posval6]

posval7 = np.asarray((nantotanual.ET < 1.) & (nantotanual.P < 1.)).nonzero()
posval7 = posval7[0]
ET_ano_plt = totanual.ET[posval7]

minv = np.nanmin(np.concatenate((
    P_ano_plt, Qm_ano_plt, Qc_ano_plt, ET_ano_plt)))
maxv = np.nanmax(np.concatenate((
    P_ano_plt, Qm_ano_plt, Qc_ano_plt, ET_ano_plt)))
xx = np.arange(minv, maxv)
yy = xx

plt.plot(xx, yy, c='green', linewidth=0.8)
plt.scatter(P_ano_plt, ET_ano_plt, c='red', marker='1',
            linewidth=0.8, label='ET')
plt.scatter(P_ano_plt, Qc_ano_plt, c='blue', marker='.',
            linewidth=0.8, label='Q$_c$')
plt.scatter(P_ano_plt, Qm_ano_plt, c='black', marker='+',
            linewidth=0.8, label='Q$_m$')
plt.ylabel('(mm/ano)')
plt.xlabel('P (mm/ano)')
plt.legend()

plt.subplot(224)

posval8 = np.asarray((nantotanual.DeltaS < 1.)
                     & (nantotanual.P < 1.)).nonzero()
posval8 = posval8[0]
DeltaS_ano_plt = totanual.DeltaS[posval8]
P_ano_plt = totanual.P[posval8]

plt.scatter(P_ano_plt, DeltaS_ano_plt, c='black', marker='+', linewidth=0.8)
plt.ylabel('$\Delta$S (mm/ano)')
plt.xlabel('P (mm/ano)')

plt.savefig(pngfigplot, dpi=300, bbox_inches='tight')

plt.show()

# # Salva resultados para plots

# pickle.dump((out_leastsq,
#              x_eixo,
#              ts_mt,
#              ts_dt,
#              ts_u,
#              ts_Dm,
#              ts_r,
#              q2,
#              p2,
#              etp,
#              xhist,
#              hist_u,
#              gresult),
#             # open(dirR+tagname+'_ugrhi_leastsqMinimizerResult.pkl',
#             # 'wb'))
#             open(dirR+tagname+'_xavier_ugrhi_leastsqMinimizerResult.pkl',
#                  'wb'))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Oct  5 10:25:36 2019

@author: khaled
"""

#### Travelling with a Tesla ####

import matplotlib.pyplot as plt
import sys
import psutil
import os
import numpy as np
import pandas as pd
import time
from multiprocessing import Pool, cpu_count

import sklearn
import mlrose

## https://mortada.net/the-traveling-tesla-salesman.html
#https://teslamotorsclub.com/tmc/threads/a-list-of-all-supercharger-coordinates-and-a-script-to-grab-them.37948/

#### Corordinates of the superchargers in California, USA (2014) ####
coordsSupershargersCA= [(47.8803440000, 10.0403420000),
                        (35.4865850000, -120.6663780000),
                        (34.8491090000, -117.0854420000),
                        (34.6145550000, -120.1884320000),
                        (39.9264600000, -122.1984000000),
                        (38.6422910000, -121.1881300000),
                        (37.4924390000, -121.9447250000),
                        (37.0244500000, -121.5653500000),
                        (36.2541430000, -120.2379200000),
                        (33.9210630000, -118.3300740000),
                        (41.3102220000, -122.3173100000),
                        (38.7712080000, -121.2661490000),
                        (33.4985380000, -117.6630900000),
                        (34.9873700000, -118.9462720000),
                        (38.3666450000, -121.9581360000),
                        (39.3274380000, -120.2074100000),
                        (34.2381150000, -119.1780840000),
                        (33.7412910000, -116.2150290000),
                        (34.1135840000, -117.5294270000)]

coordsMayanmar=[(16.47, 96.10),
                (16.47, 94.44),
                (20.09, 92.54),
                (22.39, 93.37),
                (25.23, 97.24),
                (22.00, 96.05),
                (17.20, 96.29),
                (16.30, 97.38),
                (14.05, 98.12),
                (16.53, 97.38),
                (21.52, 95.59),
                (19.41, 97.13),
                (20.09, 94.55)]

dist_list = [(0, 1, 3.1623), (0, 2, 4.1231), (0, 3, 5.8310), (0, 4, 4.2426), \
(0, 5, 5.3852), (0, 6, 4.0000), (0, 7, 2.2361), (1, 2, 1.0000), \
(1, 3, 2.8284), (1, 4, 2.0000), (1, 5, 4.1231), (1, 6, 4.2426), \
(1, 7, 2.2361), (2, 3, 2.2361), (2, 4, 2.2361), (2, 5, 4.4721), \
(2, 6, 5.0000), (2, 7, 3.1623), (3, 4, 2.0000), (3, 5, 3.6056), \
(3, 6, 5.0990), (3, 7, 4.1231), (4, 5, 2.2361), (4, 6, 3.1623), \
(4, 7, 2.2361), (5, 6, 2.2361), (5, 7, 3.1623), (6, 7, 2.2361)]

coords_list= [(1,1), (4,2), (5,2), (6,4), (4,4), (3,6), (1,5), (2,3)]

fitness= mlrose.TravellingSales(coords= coordsSupershargersCA);
#fitness= mlrose.TravellingSales(distances= dist_list);

problem=mlrose.TSPOpt(length=19, fitness_fn=fitness, maximize=False)

#problem=mlrose.TSPOpt(length=8, fitness_fn=fitness, maximize=True)

maxIter=[10, 100, 500, 1000, 1500, 2000, 3000, 5000, 10000]
#maxIter=[10, 100]
maxAttempts=10
popSize=1000
muProb=0.1

print("Simulating Genetic Algorithm.......")
for i in range(len(maxIter)):
    if i==0:
        scoreFitnessGA=[]
        usageTimeGA=[]

    start_time = time.time()
    best_state_ga, best_fitness_ga, best_curve_ga=mlrose.genetic_alg(problem, max_iters=maxIter[i],
                            pop_size=popSize, mutation_prob=muProb, curve=True, max_attempts=maxAttempts, random_state = 3)

    usageTimeGA.append(time.time()-start_time);
    scoreFitnessGA.append(1/best_fitness_ga)

print("Simulating Random Hill Climbing Algorithm.......")
for i in range(len(maxIter)):
    if i==0:
        scoreFitnessRHC=[]
        usageTimeRHC=[]

    start_time = time.time()
    best_state_rhc, best_fitness_rhc, best_curve_rhc=mlrose.random_hill_climb(problem, max_iters=maxIter[i],
                                                                curve=True, max_attempts=maxAttempts, random_state = 3)

    usageTimeRHC.append(time.time()-start_time);
    scoreFitnessRHC.append(1/best_fitness_rhc)

print("Simulating Simulated Annealing Algorithm.......")
for i in range(len(maxIter)):
    if i==0:
        scoreFitnessSA=[]
        usageTimeSA=[]

    start_time = time.time()
    best_state_sa, best_fitness_sa, best_curve_sa=mlrose.simulated_annealing(problem, max_iters=maxIter[i],
                                                                curve=True, max_attempts=maxAttempts, random_state = 3)

    usageTimeSA.append(time.time()-start_time);
    scoreFitnessSA.append(1/best_fitness_sa)

print("Simulating MIMIC Algorithm.......")
for i in range(len(maxIter)):
    if i==0:
        scoreFitnessMM=[]
        usageTimeMM=[]

    start_time = time.time()
    best_state_mm, best_fitness_mm, best_curve_mm=mlrose.mimic(problem, max_iters=maxIter[i],
                                 keep_pct= 0.2, curve=True, max_attempts=maxAttempts, random_state = 3)

    usageTimeMM.append(time.time()-start_time);
    scoreFitnessMM.append(1/best_fitness_mm)

plt.figure('TWTesla_01')
plt.title('Fitness vs Iterations', fontsize=16);
plt.ylabel('Fitness', fontsize=14);
plt.xlabel('Number of Iterations', fontsize=14);
#plt.xlim(0, 100)
plt.ylim(0.003, 0.0035)
plt.plot(maxIter, scoreFitnessGA, 'go', label='Genetic Algorithm')
plt.plot(maxIter, scoreFitnessRHC, 'rs' , label='Random Hill Climbing')
plt.plot(maxIter, scoreFitnessSA, 'b*', label='Simulated Annealing')
plt.plot(maxIter, scoreFitnessMM, 'mD' , label='MIMIC')
plt.legend(loc='lower right', fontsize=12)
plt.savefig('./Plots/TWTesla_Fitness.png')
plt.show(block=False)

plt.figure('TWTesla_02')
plt.title('Run Time vs Iterations', fontsize=16);
plt.ylabel('Run Time (s)', fontsize=14);
plt.xlabel('Number of Iterations', fontsize=14);
#plt.xlim(0, 100)
#plt.ylim(-1, 20)
plt.yscale('log')
plt.plot(maxIter, usageTimeGA, 'go', label='Genetic Algorithm')
plt.plot(maxIter, usageTimeRHC, 'rs' , label='Random Hill Climbing')
plt.plot(maxIter, usageTimeSA, 'b*', label='Simulated Annealing')
plt.plot(maxIter, usageTimeMM, 'mD' , label='MIMIC')
plt.legend(loc='best', fontsize=12)
plt.savefig('./Plots/TWTesla_RunTime.png')
plt.show()


from django.urls import path, reverse_lazy
from . import views

app_name='recipes'
urlpatterns = [
    path('', views. HomePage.as_view(), name='home_page'),
    path('recipes', views.RecipeListView.as_view(), name='all'),
    path('recipes/<int:pk>', views.RecipeDetailView.as_view(), name='recipe_detail'),
    path('recipes/create',
        views.RecipeCreateView.as_view(success_url=reverse_lazy('recipes:all')), name='recipe_create'),
    path('recipes/<int:pk>/update',
        views.RecipeUpdateView.as_view(success_url=reverse_lazy('recipes:all')), name='recipe_update'),
    path('recipes/<int:pk>/delete',
        views.RecipeDeleteView.as_view(success_url=reverse_lazy('recipes:all')), name='recipe_delete'),
    path('your_recipes', views.UserRecipeListView.as_view(), name='user_recipe_list'),

    path('recipe_picture/<int:pk>', views.stream_file, name='recipe_picture'),
    path('recipe/<int:pk>/comment',
        views.CommentCreateView.as_view(), name='recipe_comment_create'),
    path('comment/<int:pk>/delete',
        views.CommentDeleteView.as_view(success_url=reverse_lazy('recipes')), name='recipe_comment_delete'),
    path('recipe/<int:pk>/favorite',
        views.AddFavoriteView.as_view(), name='recipe_favorite'),
    path('recipe/<int:pk>/unfavorite',
        views.DeleteFavoriteView.as_view(), name='recipe_unfavorite'),

    path('recipe/<int:pk>/ingredient', views.IngredientCreateView.as_view(), name='recipe_ingredient_create'),
    path('ingredient/<int:pk>/delete', views.IngredientDeleteView.as_view(), name='recipe_ingredient_delete'),

]


from selenium import webdriver
from selenium.webdriver.remote.webelement import WebElement
from selenium.common.exceptions import StaleElementReferenceException
import time

def waitForLoad(driver):
    elem = driver.find_element_by_tag_name('html')
    print(elem.text)
    count = 0
    while True:
        count += 1
        if count > 10:
            print ('Timing out after 10 seconds and returning')
            return
        time.sleep(0.5)
        try:
            elem == driver.find_element_by_tag_name('html')
        except StaleElementReferenceException:
            return

driver = webdriver.PhantomJS(executable_path=r'D:\03-CS\plantomJS\phantomjs-2.1.1-windows\bin\phantomjs')
driver.get('http://pythonscraping.com/pages/javascript/redirectDemo1.html')
waitForLoad(driver)
print(driver.page_source)


import pickle
import cv2
import face_recognition

class FaceDetector:
    def __init__(self, model = "hog", encodings_file = "../FaceEncodings.pickle"):
        """
        :param model: 'hog' or 'cnn'
        """
        self.frame = None
        self.faces = None
        self.landmarks = None
        self.model = model
        self.encodings_file = encodings_file
        self.data = pickle.loads(open(encodings_file, "rb").read())
        self.r = 1

    def _analyze(self):
        # convert the input frame from BGR to RGB then
        rgb = cv2.cvtColor(self.frame, cv2.COLOR_BGR2RGB)
        # resize it to have a width of 750px (to speedup processing)
        # rgb = imutils.resize(rgb, width=750)
        self.r = self.frame.shape[1] / float(rgb.shape[1])

        # detect the (x, y)-coordinates of the bounding boxes
        # corresponding to each face in the input frame, then compute
        # the facial embeddings for each face
        boxes = face_recognition.face_locations(rgb, model=self.model)
        encodings = face_recognition.face_encodings(rgb, boxes)
        names = []

        # loop over the facial embeddings
        for encoding in encodings:
            # attempt to match each face in the input image to our known
            # encodings
            matches = face_recognition.compare_faces(self.data["encodings"], encoding)
            name = "Unknown"

            # check to see if we have found a match
            if True in matches:
                # find the indexes of all matched faces then initialize a
                # dictionary to count the total number of times each face
                # was matched
                matchedIdxs = [i for (i, b) in enumerate(matches) if b]
                counts = {}

                # loop over the matched indexes and maintain a count for
                # each recognized face face
                for i in matchedIdxs:
                    name = self.data["names"][i]
                    counts[name] = counts.get(name, 0) + 1

                # determine the recognized face with the largest number
                # of votes (note: in the event of an unlikely tie Python
                # will select first entry in the dictionary)
                name = max(counts, key=counts.get)

            # update the list of names
            names.append(name)

        self.faces = zip(boxes, names)

    def refresh(self, frame):
        """Refreshes the frame and analyzes it."""
        self.frame = frame
        self._analyze()

    def annotate_frame(self):
        frame = self.frame.copy()
        # print(frame)
        colors = [(255, 0, 0), (0, 255, 0), (0, 0, 255)]
        # loop over the recognized faces
        for i, ((top, right, bottom, left), name) in enumerate(self.faces):
            color = colors[i%len(colors)]
            # rescale the face coordinates
            top = int(top * self.r)
            right = int(right * self.r)
            bottom = int(bottom * self.r)
            left = int(left * self.r)

            # draw the predicted face name on the image
            cv2.rectangle(frame, (left, top), (right, bottom), color, 2)
            y = top - 15 if top - 15 > 15 else bottom + 20
            cv2.putText(frame, name, (left, y), cv2.FONT_HERSHEY_SIMPLEX, 0.75, color, 2)
        return frame


import os
import cv2
import pandas as pd
import numpy as np
import torch

from torch.utils.data import Dataset

class FaceDataset(Dataset):
    """
    dataset

    """

    def __init__(self, csv_path, transform=None):
        """
        initiate function
        :param csv_path: anotated data csvfile
        :param transform: itarational transformer
        """
        self.csv_path = csv_path
        self.transform = transform
        self.data = pd.read_csv(self.csv_path, header=None)
        self.image_num = len(self.data)
        self.label_dict = {}
        label_count = 0
        for i in range(self.image_num):
            if not self.data.ix[i, 1] in self.label_dict:
                self.label_dict[self.data.ix[i, 1]] = label_count
                label_count += 1

    def label_num(self):
        return len(self.label_dict)

    def get_label_dict(self):
        return self.label_dict

    def give_label_dict(self, label_dict):
        self.label_dict = label_dict

    def __len__(self):
        """
        return the data_num
        """
        return self.image_num

    def __getitem__(self, idx):
        image_path = self.data.ix[idx, 0]
        label = self.data.ix[idx, 1]
        label_onehot = np.zeros(len(self.label_dict))
        if not label in self.label_dict:
            label_onehot[-1] = 1
        else:
            label_onehot[self.label_dict[label]] = 1
        label_onehot = torch.tensor(label_onehot, dtype=torch.float)
        input_image = cv2.imread(image_path)
        if self.transform:
            image = self.transform(input_image)
        return (image, label_onehot)

class FaceResolutorDataset(FaceDataset):

    def __getitem__(self, idx):
        higher_path = self.data.ix[idx, 0]
        lower_path = self.data.ix[idx, 1]
        higher_image = cv2.imread(higher_path)
        lower_image = cv2.imread(lower_path)
        if self.transform:
            higher_image = self.transform(higher_image)
            lower_image = self.transform(lower_image)
        return(lower_image, higher_image)


# -*- coding: utf-8 -*-
"""
Created on Mon Oct 19 20:59:22 2020

@author: SA20149963
"""
import spacy
import PyPDF2

nlp = spacy.load('en_core_web_sm')

def get_pdf_content(filename):
    pdf = open(filename, 'rb')
    # Creating pdf reader object.
    pdf_reader = PyPDF2.PdfFileReader(pdf)
    # Checking total number of pages in a pdf file.
   # print("Document Metadata:", pdf_reader.getDocumentInfo())
    # Creating a page object.
    pages = pdf_reader.numPages
    # Extract data from a specific page number.
    content = ''
   # metadata = pdf_reader.getDocumentInfo()
    for i in range(pages):
        page=pdf_reader.getPage(i)
        content = content + page.extractText().replace("\n","").lower()
    pdf.close()
    return content.encode("ascii",errors='namereplace').decode()

def extract_relations(doc):

    spans = list(doc.ents) + list(doc.noun_chunks)
    for span in spans:
        span.merge()

    triples = []

    for ent in doc.ents:
        preps = [prep for prep in ent.root.head.children if prep.dep_ == "prep"]
        for prep in preps:
            for child in prep.children:
                triples.append((ent.text, "{} {}".format(ent.root.head, prep), child.text))
    return triples
pdf_content = get_pdf_content(r'D:\OSDU\Unstructured_Data\prov22.pdf')
content_from_nlp = nlp(pdf_content)
sentences = list (content_from_nlp.sents)

for text in sentences:
    print("\n" + str(text))
    relations = extract_relations(nlp(str(text)))
    for r1, r2, r3 in relations:
        print('({}, {}, {}) '.format(r1, r2, r3))


#-*- coding:utf-8 -*-

import time
import uuid
from os.path import splitext, join

from sqlalchemy.exc import DataError

from distreet.models.photo import Photo
from distreet.setting import PHOTO_BASE_URL, UPLOAD_DIR

def generate_filename():
    ts = str(int(time.time()))
    return '%s-%s' % (ts, uuid.uuid4().get_hex())

class PhotoUploader(object):

    ALLOW_EXTENSIONS = ('.png', '.jpeg', '.jpg')
    UPLOAD_DIR_KEY = 'upload_dir'

    #1M
    MAX_PHOTO_SIZE = 1024000

    def __init__(self, handler, dest_directory=None, **options):
        self.handler = handler
        if not dest_directory:
            dest_directory = UPLOAD_DIR
        self.dest_directory = dest_directory
        self.options = options

    def upload(self):
        files = self.handler.request.files
        if not files:
            return None
        photos_data = []
        for fn, info in files.iteritems():
            try:
                ext_name = splitext(info[0]['filename'])[-1].lower()
                if ext_name not in self.ALLOW_EXTENSIONS:
                    raise TypeError
                body = info[0].pop('body')
            except (KeyError, TypeError, IndexError):
                continue
            body_size = len(body)
            if body_size > self.MAX_PHOTO_SIZE:
                continue
            new_filename = generate_filename() + ext_name
            full_path = join(self.dest_directory, new_filename)
            try:
                with open(full_path, 'w') as fp:
                    fp.write(body)
            except IOError:
                continue
            photos_data.append({
                'fn': full_path,
                'url': PHOTO_BASE_URL + new_filename,
                'ref_type': self.options.pop('ref_type', ''),
                'ref_id': self.options.pop('ref_id', 0)
            })
        if photos_data:
            photos = [Photo(name=p['fn'], ref_type=p['ref_type'],
                            ref_id=p['ref_id'], full_url=p['url'])
                      for p in photos_data]
            session = self.handler.application.backend.get_session()
            try:
                session.add_all(photos)
                session.commit()
                ids = [photo.id for photo in photos]
            except DataError:
                session.rollback()
            finally:
                session.close()
            return ids
        return None


# Image manipulation
#
# You'll need Python 2.7 and must install these packages:
#
#   numpy, PyOpenGL, Pillow
#
# Note that file loading and saving (with 'l' and 's') do not work on
# Mac OSX, so you will have to change 'imgFilename' below, instead, if
# you want to work with different images.
#
# Note that images, when loaded, are converted to the YCbCr
# colourspace, and that you should manipulate only the Y component of
# each pixel when doing intensity changes.

#foo - imp.load_source('module.name')

import sys, os, numpy, math
import PIL as gl
import numpy as np
try:  # Pillow
    from PIL import Image
except:
    print('Error: Pillow has not been installed.')
    sys.exit(0)

try:  # PyOpenGL
    from OpenGL.GLUT import *
    from OpenGL.GL import *
    from OpenGL.GLU import *

except:
    print('Error: PyOpenGL has not been installed.')
    sys.exit(0)

# Globals

windowWidth = 600  # window dimensions
windowHeight = 800

localHistoRadius = 5  # distance within which to apply local histogram equalization

# Current image

imgDir = 'images'
imgFilename = 'mandrill.png'

currentImage = Image.open(os.path.join(imgDir, imgFilename)).convert('YCbCr').transpose(Image.FLIP_TOP_BOTTOM)
tempImage = None

# File dialog (doesn't work on Mac OSX)

if sys.platform != 'darwin':
    import Tkinter, tkFileDialog

    root = Tkinter.Tk()
    root.withdraw()

# Apply brightness and contrast to tempImage and store in
# currentImage.  The brightness and constrast changes are always made
# on tempImage, which stores the image when the left mouse button was
# first pressed, and are stored in currentImage, so that the user can
# see the changes immediately.  As long as left mouse button is held
# down, tempImage will not change.

def addToPixel(a, pixel):
    return [x + a for x in pixel]

def applyBrightnessAndContrast(brightness, contrast):
    width = currentImage.size[0]
    height = currentImage.size[1]

    srcPixels = tempImage.load()
    dstPixels = currentImage.load()

    # YOUR CODE HERE

    # loop through pixels in image
    for i in range(0,currentImage.width):
        for j in range(0,currentImage.height):#loop through image
            #get intensity and other tuple values from pixel data
            [y, Cr, Cb] = srcPixels[i, j]
            #equations for adjusting brightness and contrast
            y = contrast*y
            y = y + brightness
            #reassign updated tuple with new intensity to dstPixel
            dstPixels[i,j] = tuple([y, Cr, Cb])

    print("adjust brightness = " + str(brightness) + ", contrast = " + str(contrast))

# Perform local histogram equalization on the current image using the given radius.
#http://watkins.cs.queensu.ca/~jstewart/457/notes/05-yuv-histo-notes.txt
def performHistoEqualization(radius):
    #how does the function get called if nto a mouse click?
    pixels = currentImage.load()
    width = currentImage.size[0]
    height = currentImage.size[1]

    pixelTemp = currentImage.load()

    print("perform local histogram equalization with radius:" + str(radius))
    print("width: " + str(width) + "height: " + str(height))

# Scale the tempImage by the given factor and store it in
# currentImage.  Use backward projection.  This is called when the
# mouse is moved with the right button held down.
    intensityArray = [] #create array to hold intensities of pixels
    numPixels = 0
    # loop through image
    for i in range(0, width):
        for j in range(0, height):
            # at every pixel, get the radius pixels
            radX, radY = getNeighborhood(i,j, radius, height, width)
            for pixX in radX: #for every pixel in the radius, perform histogram equalization
                for pixY in radY:
                    [y, Cr, Cb] = pixels[pixX, pixY] #get pixel values from temp image
                    numPixels = numPixels + 1
                    intensityArray.append(y)

            # create dictionary from array of all intensity. This will get rid of duplicate values
            histDict = dict.fromkeys(intensityArray)

            # fill dictionary with pixel coordinate with that frequency
            for key in histDict:
                histDict[key] = getTofR(numPixels, getN_r(intensityArray,  key))

            #get tuple values from reference image version
            [y, Cr, Cb] = pixels[i, j]
            #get T(r) from dictionary and assign as y
            y = int(histDict[y])
            #assign new pixel tuple to temp image version
            pixelTemp[i, j] = tuple([y, Cr, Cb])

            #go through all pixels in the radius area and replace r with T(r)
            #loop through pixels in nested loop to change them within radius
            numRad = 0
            for pixX in radX:
                for pixY in radY:
                    #get the number of pixels in the radius area
                    pixels[pixX, pixY]
                    numRad = numRad + 1

#compute N_r values which is the sum of the all pixels with AT MOST intensity r
def getN_r(intensityArray, r):
    frequency = 0
    for y in intensityArray:
        if y <= r:
            frequency = frequency+1
    return frequency

def getTofR(N, N_r):
    #function for T(r) as outlined in notes
    N = float(N)
    N_r = float(N_r)
    s = ((256/N*N_r))-1
    return s

def getNeighborhood(x,y, radius, height, width):

    radX = range(x-radius, radius + x)
    radX = [item for item in radX if  0<=item<width] #don't let the radius go out of image range

    radY = range(y-radius, radius + y)
    radY = [item for item in radY if 0<=item<height]

    return radX, radY

def scaleImage(factor):

#consider the image as a matrix and perform a matrix transformation
#use im.transform

    width = currentImage.size[0]
    height = currentImage.size[1]

    srcPixels = tempImage.load()
    dstPixels = currentImage.load()

    # YOUR CODE HERE
# Set up the display and draw the current image

    for i in range(0, currentImage.width):
        for j in range(0, currentImage.height):  # loop through image
            # use back projection to set the destination pixels to the factored source coordinates
            #I would have preferred to use an if-then clause to assign pixel values but I could not figure out
            #the proper boundary conditional so I used a try-except because it also works
            try:
                dstPixels[i, j] = srcPixels[i/factor, j/factor]
            # if the pixel in the reference image is out of range, replace the pixels with white
            except:
                dstPixels[i, j] = (255, 128, 128)

def display():
    # Clear window

    glClearColor(1, 1, 1, 0)
    glClear(GL_COLOR_BUFFER_BIT)

    # rebuild the image

    img = currentImage.convert('RGB')

    width = img.size[0]
    height = img.size[1]

    # Find where to position lower-left corner of image

    baseX = (windowWidth - width) / 2
    baseY = (windowHeight - height) / 2

    glWindowPos2i(baseX, baseY)

    # Get pixels and draw

    imageData = numpy.array(list(img.getdata()), numpy.uint8)

    glDrawPixels(width, height, GL_RGB, GL_UNSIGNED_BYTE, imageData)

    glutSwapBuffers()

# Handle keyboard input

def keyboard(key, x, y):
    global localHistoRadius

    if key == '\033':  # ESC = exit
        sys.exit(0)

    elif key == 'l':
        if sys.platform != 'darwin':
            path = tkFileDialog.askopenfilename(initialdir=imgDir)
            if path:
                loadImage(path)

    elif key == 's':
        if sys.platform != 'darwin':
            outputPath = tkFileDialog.asksaveasfilename(initialdir='.')
            if outputPath:
                saveImage(outputPath)

    elif key == 'h':
        performHistoEqualization(localHistoRadius)

    elif key in ['+', '=']:
        localHistoRadius = localHistoRadius + 1
        print
        'radius =', localHistoRadius

    elif key in ['-', '_']:
        localHistoRadius = localHistoRadius - 1
        if localHistoRadius < 1:
            localHistoRadius = 1
        print
        'radius =', localHistoRadius

    else:
        print
        'key =', key  # DO NOT REMOVE THIS LINE.  It will be used during automated marking.

    glutPostRedisplay()

# Load and save images.
#
# Modify these to load to the current image and to save the current image.
#
# DO NOT CHANGE THE NAMES OR ARGUMENT LISTS OF THESE FUNCTIONS, as
# they will be used in automated marking.

def loadImage(path):
    global currentImage

    currentImage = Image.open(path).convert('YCbCr').transpose(Image.FLIP_TOP_BOTTOM)

def saveImage(path):
    global currentImage

    currentImage.transpose(Image.FLIP_TOP_BOTTOM).convert('RGB').save(path)

# Handle window reshape

def reshape(newWidth, newHeight):
    global windowWidth, windowHeight

    windowWidth = newWidth
    windowHeight = newHeight

    glutPostRedisplay()

# Mouse state on initial click

button = None
initX = 0
initY = 0

# Handle mouse click/release

def mouse(btn, state, x, y):
    global button, initX, initY, tempImage

    if state == GLUT_DOWN:
        tempImage = currentImage.copy()
        button = btn
        initX = x
        initY = y
    elif state == GLUT_UP:
        tempImage = None
        button = None

    glutPostRedisplay()

# Handle mouse motion

def motion(x, y):
    if button == GLUT_LEFT_BUTTON:

        diffX = x - initX
        diffY = y - initY

        applyBrightnessAndContrast(255 * diffX / float(windowWidth), 1 + diffY / float(windowHeight))

    elif button == GLUT_RIGHT_BUTTON:

        initPosX = initX - float(windowWidth) / 2.0
        initPosY = initY - float(windowHeight) / 2.0
        initDist = math.sqrt(initPosX * initPosX + initPosY * initPosY)
        if initDist == 0:
            initDist = 1

        newPosX = x - float(windowWidth) / 2.0
        newPosY = y - float(windowHeight) / 2.0
        newDist = math.sqrt(newPosX * newPosX + newPosY * newPosY)

        scaleImage(newDist / initDist)

    glutPostRedisplay()

# Run OpenGL
glutInit()
glutInitDisplayMode(GLUT_DOUBLE | GLUT_RGB)
glutInitWindowSize(windowWidth, windowHeight)
glutInitWindowPosition(50, 50)
glutCreateWindow('imaging')
glutDisplayFunc(display)
glutKeyboardFunc(keyboard)
glutReshapeFunc(reshape)
glutMouseFunc(mouse)
glutMotionFunc(motion)

glutMainLoop()
scaleImage(5)


from enum import Enum, unique
from functools import cached_property, lru_cache, total_ordering
from pathlib import Path
from typing import Any, ClassVar

from models_library.basic_types import PortInt
from models_library.projects import ProjectID
from models_library.projects_nodes_io import NodeID
from models_library.services import VERSION_RE, DynamicServiceKey
from models_library.users import UserID
from pydantic import BaseModel, Field

class CommonServiceDetails(BaseModel):
    key: DynamicServiceKey = Field(
        ...,
        description="distinctive name for the node based on the docker registry path",
        examples=[
            "simcore/services/dynamic/3dviewer",
        ],
        alias="service_key",
    )
    version: str = Field(
        ...,
        description="semantic version number of the node",
        regex=VERSION_RE,
        examples=["1.0.0", "0.0.1"],
        alias="service_version",
    )

    user_id: UserID
    project_id: ProjectID
    node_uuid: NodeID = Field(..., alias="service_uuid")

class ServiceDetails(CommonServiceDetails):
    basepath: Path = Field(
        default=None,
        description="predefined path where the dynamic service should be served. If empty, the service shall use the root endpoint.",
        alias="service_basepath",
    )

    class Config:
        allow_population_by_field_name = True
        schema_extra: ClassVar[dict[str, Any]] = {
            "example": {
                "key": "simcore/services/dynamic/3dviewer",
                "version": "2.4.5",
                "user_id": 234,
                "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe",
                "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa",
                "basepath": "/x/75c7f3f4-18f9-4678-8610-54a2ade78eaa",
            }
        }

@unique
class ServiceBootType(str, Enum):
    V0 = "V0"
    V2 = "V2"

@total_ordering
@unique
class ServiceState(Enum):
    PENDING = "pending"
    PULLING = "pulling"
    STARTING = "starting"
    RUNNING = "running"
    COMPLETE = "complete"
    FAILED = "failed"
    STOPPING = "stopping"

    def __lt__(self, other):
        if self.__class__ is other.__class__:
            comparison_order = ServiceState.comparison_order()
            self_index = comparison_order[self]
            other_index = comparison_order[other]
            return self_index < other_index
        return NotImplemented

    @staticmethod
    @lru_cache(maxsize=2)
    def comparison_order() -> dict["ServiceState", int]:
        """States are comparable to supportmin() on a list of ServiceState"""
        return {
            ServiceState.FAILED: 0,
            ServiceState.PENDING: 1,
            ServiceState.PULLING: 2,
            ServiceState.STARTING: 3,
            ServiceState.RUNNING: 4,
            ServiceState.STOPPING: 5,
            ServiceState.COMPLETE: 6,
        }

class RunningDynamicServiceDetails(ServiceDetails):
    boot_type: ServiceBootType = Field(
        default=ServiceBootType.V0,
        description=(
            "Describes how the dynamic services was started (legacy=V0, modern=V2)."
            "Since legacy services do not have this label it defaults to V0."
        ),
    )

    host: str = Field(
        ..., description="the service swarm internal host name", alias="service_host"
    )
    internal_port: PortInt = Field(
        ..., description="the service swarm internal port", alias="service_port"
    )
    published_port: PortInt = Field(
        default=None,
        description="the service swarm published port if any",
        deprecated=True,
    )

    entry_point: str | None = Field(
        default=None,
        description="if empty the service entrypoint is on the root endpoint.",
        deprecated=True,
    )
    state: ServiceState = Field(
        ..., description="service current state", alias="service_state"
    )
    message: str | None = Field(
        default=None,
        description="additional information related to service state",
        alias="service_message",
    )

    @cached_property
    def legacy_service_url(self) -> str:
        return f"http://{self.host}:{self.internal_port}{self.basepath}"  # NOSONAR

    @classmethod
    def from_scheduler_data(
        cls,
        node_uuid: NodeID,
        scheduler_data: "SchedulerData",  # type: ignore
        service_state: ServiceState,
        service_message: str,
    ) -> "RunningDynamicServiceDetails":
        return cls.parse_obj(
            {
                "boot_type": ServiceBootType.V2,
                "user_id": scheduler_data.user_id,
                "project_id": scheduler_data.project_id,
                "service_uuid": node_uuid,
                "service_key": scheduler_data.key,
                "service_version": scheduler_data.version,
                "service_host": scheduler_data.service_name,
                "service_port": scheduler_data.service_port,
                "service_state": service_state.value,
                "service_message": service_message,
            }
        )

    class Config(ServiceDetails.Config):
        keep_untouched = (cached_property,)
        schema_extra: ClassVar[dict[str, Any]] = {
            "examples": [
                {
                    "boot_type": "V0",
                    "key": "simcore/services/dynamic/3dviewer",
                    "version": "2.4.5",
                    "user_id": 234,
                    "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe",
                    "uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa",
                    "basepath": "/x/75c7f3f4-18f9-4678-8610-54a2ade78eaa",
                    "host": "3dviewer_75c7f3f4-18f9-4678-8610-54a2ade78eaa",
                    "internal_port": 8888,
                    "state": "running",
                    "message": "",
                    "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa",
                },
                {
                    "boot_type": "V2",
                    "key": "simcore/services/dynamic/dy-static-file-viewer-dynamic-sidecar",
                    "version": "1.0.0",
                    "user_id": 234,
                    "project_id": "dd1d04d9-d704-4f7e-8f0f-1ca60cc771fe",
                    "uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa",
                    "host": "dy-sidecar_75c7f3f4-18f9-4678-8610-54a2ade78eaa",
                    "internal_port": 80,
                    "state": "running",
                    "message": "",
                    "node_uuid": "75c7f3f4-18f9-4678-8610-54a2ade78eaa",
                },
            ]
        }


# coding: utf-8

# In[1]:

#getting libraries
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as mlt
get_ipython().magic(u'matplotlib inline')

# In[2]:

#getting file
crd1=pd.read_csv('D:\\V data analytics\\sample dataset\\Dataset by Imarticus\\Data for logistic regression\\R_Module_Day_7.2_Credit_Risk_Train_data.csv')
crd2=pd.read_csv('D:\\V data analytics\\sample dataset\\Dataset by Imarticus\\Data for logistic regression\\R_Module_Day_8.2_Credit_Risk_Validate_data.csv')

# In[11]:

# to do missing value imputation, we will concat both train and validation data

crd=pd.concat([crd1,crd2],axis=0)

# In[4]:

crd.isna().sum()

# In[12]:

#lets reset index just to be sure
crd.reset_index(inplace=True,drop=True)

# In[13]:

crd.head()

# In[14]:

#all good lets check for missing values
crd.isna().sum()

# In[15]:

#missing value of imputation, to get the list of indices of gender missing values we do
genmiss=crd[crd['Gender'].isna()].index.tolist()

# In[16]:

#to find out the mode we do,
crd.Gender.value_counts()

# In[17]:

#lets fill up missing value by 'Male'
crd['Gender'].iloc[genmiss]='Male'

# In[18]:

crd.Gender.isna().sum()

# In[19]:

#great, lets do the missing value imputation for married
crd.Married.value_counts()

# In[20]:

#lets fill up nas by 'Yes'
crd['Married'].iloc[crd[crd['Married'].isna()].index.tolist()]='Yes'

# In[21]:

crd.Married.isna().sum()

# In[22]:

#lets do missing value impputation of dependents
crd.Dependents.value_counts()

# In[29]:

crd.Dependents.isna().sum()

# In[30]:

#lets fill the nas of dependents missing value in case of not married people as zero
# so we need the dataframe where married is no and dependents is na
dmiss1=crd[(crd['Married']=='No') & (crd['Dependents'].isna())].index.tolist()
print(dmiss1)

# In[31]:

#filling it up with zero
crd['Dependents'].iloc[dmiss1]='0'

# In[32]:

crd.Dependents.value_counts()

# In[33]:

crd.Dependents.isna().sum()

# In[28]:

#lets do a crosstab of gender with dependents
pd.crosstab(crd['Gender'],crd['Dependents'].isna())  # out of 16, 15 values corresponds to male

# In[38]:

#lets do the cross tab of dependents and gender
pd.crosstab(crd['Gender'],crd['Dependents'])

# In[39]:

#since most of male have a mode of 0, lets fill the remaining values by 0
crd['Dependents'][(crd['Dependents'].isna())]='0'

# In[40]:

crd.Dependents.isna().sum()

# In[41]:

#missing value impution of selfemployed
crd.Self_Employed.isna().sum()

# In[42]:

crd.Self_Employed.value_counts()

# In[43]:

#replacing nas with 'No'
crd['Self_Employed'].iloc[crd[crd['Self_Employed'].isna()].index.tolist()]='No'

# In[44]:

#mvi of loanamount loanamount_term
#lets compare the missingvalues of loanamount with loanamountterm
pd.crosstab(crd['Loan_Amount_Term'],crd['LoanAmount'].isna())

# In[45]:

#its evident that loan_amount_term pertaining to 360 has highest numbers of nas.
# we will fill the value by the mean of the loanamount corresponding to the loan_amount_term 360.
crd.groupby(crd['Loan_Amount_Term'])['LoanAmount'].mean()

# In[46]:

# lets fill the loanampunt na's in the range of 360 by 144
crd['LoanAmount'].iloc[crd[(crd['Loan_Amount_Term']==360) & (crd['LoanAmount'].isna())].index.tolist()]=144

# In[47]:

#for the rest of nas lets replace the values by 132
crd['LoanAmount'].iloc[crd[crd['LoanAmount'].isna()].index.tolist()]=132

# In[48]:

#missing value imputation of loan_amount_term
crd.Loan_Amount_Term.isnull().sum()

# In[49]:

crd.Loan_Amount_Term.value_counts()

# In[50]:

#lets fill up by mode, ie 360
crd['Loan_Amount_Term'].iloc[crd[crd['Loan_Amount_Term'].isna()].index.tolist()]=360

# In[51]:

#missing value imutation of Credit_history by Logistic regression

crd.Credit_History.isnull().sum()

# In[56]:

#lets separate the dataframe as train and test, lets take all na's in credithistory as test data
crd_testdata=crd.loc[crd['Credit_History'].isna(),:]

# In[58]:

#to get traindata, we will first get incex of all na's of credithistory
crd_credna_indx=crd[crd['Credit_History'].isna()].index.tolist()

# In[61]:

#getting traindata
crd_traindata_index=[x for x in crd.index.tolist() if x not in crd_credna_indx]

# In[62]:

crd_traindata=crd.iloc[crd_traindata_index]

# In[64]:

#Tp do a logistic regression, we will lose all unimo variables and get dummies for all catagorical variables
crd_train1=pd.get_dummies(crd_traindata.drop(['Loan_ID'],axis=1),drop_first=True)

# In[65]:

crd_train1.head()

# In[67]:

#lets do the same for testdata
crd_test1=pd.get_dummies(crd_testdata.drop(['Loan_ID'],axis=1),drop_first=True)

# In[68]:

crd_test1.head()

# In[71]:

#preparing xtrain and y train, xtest and ytest
x_train1=crd_train1.drop(['Credit_History','Loan_Status_Y'],axis=1)
y_train1=crd_train1['Credit_History']

# In[73]:

#similarly for test set
x_test1=crd_test1.drop(['Credit_History','Loan_Status_Y'],axis=1)
y_test1=crd_test1['Credit_History']

# In[74]:

from sklearn.linear_model import LogisticRegression

# In[75]:

m1=LogisticRegression()

# In[76]:

m1.fit(x_train1,y_train1)

# In[77]:

#predicting for xtest
pred1=m1.predict(x_test1)

# In[78]:

print(pred1)

# In[83]:

crd_test1['Credit_History']=pred1

# In[84]:

crd_new=pd.concat([crd_train1,crd_test1],axis=0)

# In[85]:

crd_new.isnull().sum()

# In[86]:

crd_new.shape

# In[87]:

crd.Credit_History.isna().sum()

# In[88]:

crd['Credit_History'].iloc[crd[crd['Credit_History'].isna()].index.tolist()]=pred1

# In[89]:

crd.isnull().sum()

# In[90]:

crd.head()

# In[ ]:

# lets go ahead woth logistic regression now

# In[97]:

#some basic eda's first
sns.barplot(x=crd['Loan_Status'],y=crd['LoanAmount'],hue=crd['Gender'],data=crd)

# In[98]:

sns.barplot(x=crd['Credit_History'],y=crd['Loan_Status'],hue=crd['Married'],data=crd)

# In[105]:

sns.barplot(x=crd['Loan_Status'],y=crd['ApplicantIncome'],hue=crd['Property_Area'],data=crd) #(so rural property got rejected most)

# In[104]:

sns.barplot(x=crd['Loan_Status'],y=crd['ApplicantIncome'],hue=crd['Dependents'],data=crd) #interesting fact is applicant with..
#..3+ dependents were rejected also were given higher no of loans!!

# In[ ]:

# lets split the data into train and validation,just like how it was given previously

# In[111]:

crd_train=crd.head(len(crd1))

# In[113]:

crd_train.to_csv('D:\\V data analytics\\sample dataset\\Dataset by Imarticus\\Data for logistic regression\\Loan_eligibility_estimation_traindata_withmv_imputed\\crd_train.csv')

# In[114]:

crd_val=crd.tail(len(crd2))

# In[115]:

crd_val.to_csv('D:\\V data analytics\\sample dataset\\Dataset by Imarticus\\Data for logistic regression\\Loan_eligibility_estimation_train _test_data_withmv_imputed\\crd_val.csv')

# ### Logistic regression with area under roc curve

# In[117]:

#getting dummies for all catagorical variables
crd.info()

# In[120]:

#seperating dvb
crd_new1=pd.get_dummies(crd.drop(['Loan_ID','Loan_Status'],axis=1),drop_first=True)

# In[121]:

crd_new1.head()

# In[122]:

x_train=crd_new1

# In[123]:

y_train=crd['Loan_Status']

# In[124]:

m2=LogisticRegression()

# In[125]:

m2.fit(x_train,y_train)

# In[128]:

m2.score(x_train,y_train)

# In[129]:

# PREDICTING DATA FOR VALIDATION DATA
crd_val.shape

# In[130]:

crd_val.head()

# In[132]:

#preparing data to predict
crd_val1=crd_val.drop(['Loan_ID','Loan_Status'],axis=1)

# In[133]:

#getting dummyvalues for cvs
crdval=pd.get_dummies(crd_val1,drop_first=True)

# In[134]:

crdval.head()

# In[135]:

#predicting
pred2=m2.predict(crdval)

# In[136]:

#getting crosstab
pd.crosstab(crd_val['Loan_Status'],pred2)

# In[137]:

#to get crosstab we download metrics
from sklearn import metrics

# In[138]:

metrics.confusion_matrix(crd_val['Loan_Status'],pred2) #we can interpret the results as predicted
                                                                                #       N      Y
                                                  #          actual loanstatus  N      55      22
                                                  #                             Y      1       289

# In[ ]:

#sensitivity of the model =tp/tp+fn= 289/289+1=99% highly sensitive
#speificity of the model = tn/tn+fp=55/55+22=55/77= 71% speific

# In[140]:

#lets get the valuecount of loanstatus in validation data
crd_val.Loan_Status.value_counts()

# In[141]:

#accuracy score
metrics.accuracy_score(crd_val['Loan_Status'],pred2) #93% great

# In[172]:

crd_val.head()

# In[179]:

crdval.head()

# In[180]:

crdval['Loan_Status']=crd_val['Loan_Status'] #in crdval, we add loanstatus, since we have to conver it to digits by mapping

# In[181]:

crdval.head()

# In[182]:

crdval['Loan_Status_digit']=crdval.Loan_Status.map({'N':0,'Y':1})

# In[184]:

crdval1=crdval.drop(['Loan_Status'],axis=1)

# In[186]:

crdval1.info()

# In[187]:

crdval.info()

# In[196]:

probvalues_of_dv=m2.predict_proba(crdval1.drop(['Loan_Status_digit'],axis=1))

# In[197]:

Loan_status_prob0=m2.predict_proba(crdval1.drop(['Loan_Status_digit'],axis=1))[:,1]

# In[199]:

#graph
sns.distplot(Loan_status_prob0,kde=False,bins=50)

# In[200]:

#roc curve
fpr,tpr,thresholds=metrics.roc_curve(crd_val['Loan_Status_digit'],Loan_status_prob0)

# In[202]:

mlt.plot(fpr,tpr)

# In[203]:

#getting area under aoc
metrics.roc_auc_score(crd_val['Loan_Status_digit'],Loan_status_prob0)

Logistic regression (Area under ROC curve) with missing value imputation.py


from collections import defaultdict

class Solution:
    def findOrder(self, numCourses: int, prerequisites: List[List[int]]) -> List[int]:
        node_list = defaultdict(set)
        for pair in prerequisites:
            if pair[0] in node_list:
                node_list[pair[0]].add(pair[1])
            else:
                node_list[pair[0]] = set([pair[1]])

        explored = [False] * numCourses
        current_label = numCourses
        top_sort = {}

        def dfs(course: int):
            nonlocal current_label
            explored[course] = True
            if course in node_list:
                for next_course in node_list[course]:
                    if not explored[next_course]:
                        dfs(next_course)
            top_sort[course] = current_label
            current_label -= 1

        for course in range(numCourses):
            if not explored[course]:
                dfs(course)

        for pair in prerequisites:
            if top_sort[pair[0]] >= top_sort[pair[1]]:
                return []

        return sorted(top_sort, key=top_sort.get, reverse=True)


import argparse, pickle
import numpy as np
import os

import torch
from torch import nn, optim
from torchvision import transforms

# Use the util.load() function to load your dataset
from .utils import load
from .models import *

dirname = os.path.dirname(os.path.abspath(__file__))

def accuracy(outputs, labels):
    outputs_idx = outputs.max(1)[1].type_as(outputs)
    return outputs_idx.eq(labels.float()).float().mean()

train_transforms = transforms.Compose([
    transforms.ToPILImage(),

    transforms.RandomApply([

        transforms.ColorJitter(brightness=0.25, contrast=0.35, saturation=0.25, hue=0.28),
        transforms.RandomHorizontalFlip(p=0.5),

    ], 0.5),

    transforms.ToTensor(),
    transforms.Normalize(mean=[0.429, 0.505, 0.517]*255, std=[0.274, 0.283, 0.347]*255)
])

val_transforms = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.429, 0.505, 0.517]*255, std=[0.274, 0.283, 0.347]*255)
])

def augment_train(inputs):
    return torch.stack( [ train_transforms(inp_i) for inp_i in inputs ] )

def augment_val(inputs):
    return torch.stack( [ val_transforms(inp_i) for inp_i in inputs ] )

def train(iterations, batch_size=64, log_dir=None):
    '''
    This is the main training function, feel free to modify some of the code, but it
    should not be required to complete the assignment.
    '''

    """
    Load the training data
    """

    train_inputs, train_labels = load(os.path.join('tux_train.dat'))
    val_inputs, val_labels = load(os.path.join('tux_valid.dat'))

    model = ConvNetModel()

    log = None
    if log_dir is not None:
        from .utils import SummaryWriter
        log = SummaryWriter(log_dir)

    '''
    You can also change your train code here if neccesary
    '''

    # If your model does not train well, you may swap out the optimizer or change the lr
    optimizer = optim.Adam(model.parameters(), lr = 1e-4)
    loss = nn.CrossEntropyLoss()

    for iteration in range(iterations):
        model.train()
        # Construct a mini-batch
        batch = np.random.choice(train_inputs.shape[0], batch_size)
        batch_inputs = augment_train(train_inputs[batch])
        batch_labels = torch.as_tensor(train_labels[batch], dtype=torch.long)

        # zero the gradients (part of pytorch backprop)
        optimizer.zero_grad()

        # Compute the model output and loss (view flattens the input)
        model_outputs = model(batch_inputs)
        t_loss_val = loss(model_outputs, batch_labels)
        t_acc_val = accuracy(model_outputs, batch_labels)

        # Compute the gradient
        t_loss_val.backward()

        # Update the weights
        optimizer.step()

        if iteration % 10 == 0:
            model.eval()
            batch = np.random.choice(val_inputs.shape[0], 256)
            batch_inputs = augment_val(val_inputs[batch])
            batch_labels = torch.as_tensor(val_labels[batch], dtype=torch.long)
            model_outputs = model(batch_inputs)
            v_acc_val = accuracy(model_outputs, batch_labels)

            print('[%5d]'%iteration, 'loss = %f'%t_loss_val, 't_acc = %f'%t_acc_val, 'v_acc = %f'%v_acc_val)
            if log is not None:
                log.add_scalar('train/loss', t_loss_val, iteration)
                log.add_scalar('train/acc', t_acc_val, iteration)
                log.add_scalar('val/acc', v_acc_val, iteration)

    # Save the trained model
    torch.save(model.state_dict(), os.path.join(dirname, 'convnet.th')) # Do NOT modify this line

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--iterations', type=int, default=2000)
    parser.add_argument('-l', '--log_dir')
    args = parser.parse_args()

    print ('[I] Start training')
    train(args.iterations, log_dir=args.log_dir)
    print ('[I] Training finished')