#!/usr/bin/python

# Network crazy eyes : Client
#
# A remix of:
#
# Adafruit / Phillip Burgess (Paint Your Dragon)'s Animated Snake Eyes for Raspberry Pi
# https://learn.adafruit.com/animated-snake-eyes-bonnet-for-raspberry-pi/software-installation
# https://github.com/adafruit/Pi_Eyes
#
# Nathan Jennings / Real Python guide to Socket Programming in Python (Guide)
# https://realpython.com/python-sockets/#handling-multiple-connections
# https://github.com/realpython/materials/blob/master/python-sockets-tutorial/multiconn-client.py

# This renders a single rightt eye (centered on screen) and
# assumes you are using HDMI for display.

import pickle
import math
import pi3d
import random
import threading
import time
from svg.path import Path, parse_path
from xml.dom.minidom import parse
from gfxutil import *
import sys
import socket
import selectors
import types

# Get my IP address
hostname = socket.gethostname()
host = socket.gethostbyname(hostname)
host = '127.0.0.1' # Uncomment this line for local testing
#host = '' # Listen on all available interfaces

# Use this port (make sure the eye server is using the same one!)
port = 65432

# Set up display and initialize pi3d ---------------------------------------

#DISPLAY = pi3d.Display.create(samples=4, w=640, h=480)
DISPLAY = pi3d.Display.create(samples=4, w=1280, h=720)
#DISPLAY = pi3d.Display.create(samples=4, w=1920, h=1080)
#DISPLAY = pi3d.Display.create(samples=4)
DISPLAY.set_background(0, 0, 0, 1) # r,g,b,alpha

# Load SVG file, extract paths & convert to point lists --------------------

dom               = parse("graphics/eye.svg")
vb                = get_view_box(dom)
pupilMinPts       = get_points(dom, "pupilMin"      , 32, True , True )
pupilMaxPts       = get_points(dom, "pupilMax"      , 32, True , True )
irisPts           = get_points(dom, "iris"          , 32, True , True )
scleraFrontPts    = get_points(dom, "scleraFront"   ,  0, False, False)
scleraBackPts     = get_points(dom, "scleraBack"    ,  0, False, False)
upperLidClosedPts = get_points(dom, "upperLidClosed", 33, False, True )
upperLidOpenPts   = get_points(dom, "upperLidOpen"  , 33, False, True )
upperLidEdgePts   = get_points(dom, "upperLidEdge"  , 33, False, False)
lowerLidClosedPts = get_points(dom, "lowerLidClosed", 33, False, False)
lowerLidOpenPts   = get_points(dom, "lowerLidOpen"  , 33, False, False)
lowerLidEdgePts   = get_points(dom, "lowerLidEdge"  , 33, False, False)

TRACKING        = True  # If True, eyelid tracks pupil

# eyeRadius is the size, in pixels, at which the whole eye will be rendered.
if DISPLAY.width <= (DISPLAY.height * 2):
	# For WorldEye, eye size is -almost- full screen height
	eyeRadius   = DISPLAY.height / 2.1
else:
	eyeRadius   = DISPLAY.height * 2 / 5

# A 2D camera is used, mostly to allow for pixel-accurate eye placement,
# but also because perspective isn't really helpful or needed here, and
# also this allows eyelids to be handled somewhat easily as 2D planes.
# Line of sight is down Z axis, allowing conventional X/Y cartesion
# coords for 2D positions.
cam    = pi3d.Camera(is_3d=False, at=(0,0,0), eye=(0,0,-1000))
shader = pi3d.Shader("uv_light")
light  = pi3d.Light(lightpos=(0, -500, -500), lightamb=(0.2, 0.2, 0.2))

# Load texture maps --------------------------------------------------------

irisMap   = pi3d.Texture("graphics/iris.jpg"  , mipmap=False,
              filter=pi3d.GL_LINEAR)
scleraMap = pi3d.Texture("graphics/sclera.png", mipmap=False,
              filter=pi3d.GL_LINEAR, blend=True)
lidMap    = pi3d.Texture("graphics/lid.png"   , mipmap=False,
              filter=pi3d.GL_LINEAR, blend=True)
# U/V map may be useful for debugging texture placement; not normally used
#uvMap     = pi3d.Texture("graphics/uv.png"    , mipmap=False,
#              filter=pi3d.GL_LINEAR, blend=False, m_repeat=True)

# Initialize static geometry -----------------------------------------------

# Transform point lists to eye dimensions
scale_points(pupilMinPts      , vb, eyeRadius)
scale_points(pupilMaxPts      , vb, eyeRadius)
scale_points(irisPts          , vb, eyeRadius)
scale_points(scleraFrontPts   , vb, eyeRadius)
scale_points(scleraBackPts    , vb, eyeRadius)
scale_points(upperLidClosedPts, vb, eyeRadius)
scale_points(upperLidOpenPts  , vb, eyeRadius)
scale_points(upperLidEdgePts  , vb, eyeRadius)
scale_points(lowerLidClosedPts, vb, eyeRadius)
scale_points(lowerLidOpenPts  , vb, eyeRadius)
scale_points(lowerLidEdgePts  , vb, eyeRadius)

# Regenerating flexible object geometry (such as eyelids during blinks, or
# iris during pupil dilation) is CPU intensive, can noticably slow things
# down, especially on single-core boards.  To reduce this load somewhat,
# determine a size change threshold below which regeneration will not occur;
# roughly equal to 1/2 pixel, since 2x2 area sampling is used.

# Determine change in pupil size to trigger iris geometry regen
irisRegenThreshold = 0.0
a = points_bounds(pupilMinPts) # Bounds of pupil at min size (in pixels)
b = points_bounds(pupilMaxPts) # " at max size
maxDist = max(abs(a[0] - b[0]), abs(a[1] - b[1]), # Determine distance of max
              abs(a[2] - b[2]), abs(a[3] - b[3])) # variance around each edge
# maxDist is motion range in pixels as pupil scales between 0.0 and 1.0.
# 1.0 / maxDist is one pixel's worth of scale range.  Need 1/2 that...
if maxDist > 0: irisRegenThreshold = 0.5 / maxDist

# Determine change in eyelid values needed to trigger geometry regen.
# This is done a little differently than the pupils...instead of bounds,
# the distance between the middle points of the open and closed eyelid
# paths is evaluated, then similar 1/2 pixel threshold is determined.
upperLidRegenThreshold = 0.0
lowerLidRegenThreshold = 0.0
p1 = upperLidOpenPts[len(upperLidOpenPts) // 2]
p2 = upperLidClosedPts[len(upperLidClosedPts) // 2]
dx = p2[0] - p1[0]
dy = p2[1] - p1[1]
d  = dx * dx + dy * dy
if d > 0: upperLidRegenThreshold = 0.5 / math.sqrt(d)
p1 = lowerLidOpenPts[len(lowerLidOpenPts) // 2]
p2 = lowerLidClosedPts[len(lowerLidClosedPts) // 2]
dx = p2[0] - p1[0]
dy = p2[1] - p1[1]
d  = dx * dx + dy * dy
if d > 0: lowerLidRegenThreshold = 0.5 / math.sqrt(d)

# Generate initial iris mesh; vertex elements will get replaced on
# a per-frame basis in the main loop, this just sets up textures, etc.
iris = mesh_init((32, 4), (0.0, 0.5 / irisMap.iy), True, False) ## LEFT IS 0.5, 0.5
iris.set_textures([irisMap])
iris.set_shader(shader)
irisZ = zangle(irisPts, eyeRadius)[0] * 0.99 # Get iris Z depth, for later

# Eyelid meshes are likewise temporary; texture coordinates are
# assigned here but geometry is dynamically regenerated in main loop.
upperEyelid = mesh_init((33, 5), (0, 0.5 / lidMap.iy), False, True)
upperEyelid.set_textures([lidMap])
upperEyelid.set_shader(shader)
lowerEyelid = mesh_init((33, 5), (0, 0.5 / lidMap.iy), False, True)
lowerEyelid.set_textures([lidMap])
lowerEyelid.set_shader(shader)

# Generate sclera for eye...start with a 2D shape for lathing...
angle1 = zangle(scleraFrontPts, eyeRadius)[1] # Sclera front angle
angle2 = zangle(scleraBackPts , eyeRadius)[1] # " back angle
aRange = 180 - angle1 - angle2
pts    = []
for i in range(24):
	ca, sa = pi3d.Utility.from_polar((90 - angle1) - aRange * i / 23)
	pts.append((ca * eyeRadius, sa * eyeRadius))

eye = pi3d.Lathe(path=pts, sides=64)
eye.set_textures([scleraMap])
eye.set_shader(shader)
re_axis(eye, 0.0) ## LEFT IS 0.5

# Init global stuff --------------------------------------------------------

#mykeys = pi3d.Keyboard() # For capturing key presses

startX       = random.uniform(-30.0, 30.0)
n            = math.sqrt(900.0 - startX * startX)
startY       = random.uniform(-n, n)
destX        = startX
destY        = startY
curX         = startX
curY         = startY
moveDuration = random.uniform(0.075, 0.175)
holdDuration = random.uniform(0.1, 1.1)
startTime    = 0.0
isMoving     = False

frames        = 0
beginningTime = time.time()

eye.positionX(0.0)
iris.positionX(0.0)
upperEyelid.positionX(0.0)
upperEyelid.positionZ(-eyeRadius - 42)
lowerEyelid.positionX(0.0)
lowerEyelid.positionZ(-eyeRadius - 42)

lidWeight    = 0.0
blinkState      = 0

currentPupilScale  =  0.5
prevPupilScale     = -1.0 # Force regen on first frame
prevUpperLidWeight = 0.5
prevLowerLidWeight = 0.5
prevUpperLidPts    = points_interp(upperLidOpenPts, upperLidClosedPts, 0.5)
prevLowerLidPts    = points_interp(lowerLidOpenPts, lowerLidClosedPts, 0.5)

ruRegen = True
rlRegen = True

trackingPos = 0.3

# These are the settings shared by (read from) the server
shared = {"curX":curX, "curY":curY, "pupil":currentPupilScale, "lid":lidWeight, "blink":blinkState}

# Generate one frame of imagery
def frame():

	global startX, startY, destX, destY, curX, curY
	global moveDuration, holdDuration, startTime, isMoving
	global frames
	global iris
	global pupilMinPts, pupilMaxPts, irisPts, irisZ
	global eye
	global upperEyelid, lowerEyelid
	global upperLidOpenPts, upperLidClosedPts, lowerLidOpenPts, lowerLidClosedPts
	global upperLidEdgePts, lowerLidEdgePts
	global prevUpperLidPts, prevLowerLidPts
	global prevUpperLidWeight, prevLowerLidWeight
	global prevPupilScale
	global irisRegenThreshold, upperLidRegenThreshold, lowerLidRegenThreshold
	global luRegen, llRegen, ruRegen, rlRegen
	global timeOfLastBlink, timeToNextBlink
	global blinkState
	global blinkDuration
	global blinkStartTime
	global trackingPos
	global shared

	DISPLAY.loop_running()

	now = time.time()
	dt  = now - startTime

	frames += 1

	curX = shared["curX"]
	curY = shared["curY"]
	p = shared["pupil"]
	lidWeight = shared["lid"]
	blinkState = shared["blink"]

	# Regenerate iris geometry only if size changed by >= 1/2 pixel
	if abs(p - prevPupilScale) >= irisRegenThreshold:
		# Interpolate points between min and max pupil sizes
		interPupil = points_interp(pupilMinPts, pupilMaxPts, p)
		# Generate mesh between interpolated pupil and iris bounds
		mesh = points_mesh((None, interPupil, irisPts), 4, -irisZ, True)
		iris.re_init(pts=mesh)
		prevPupilScale = p

	# Eyelid WIP

	if TRACKING:
		# 0 = fully up, 1 = fully down
		n = 0.4 - curY / 60.0
		if   n < 0.0: n = 0.0
		elif n > 1.0: n = 1.0
		trackingPos = (trackingPos * 3.0 + n) * 0.25

	newUpperLidWeight = trackingPos + (lidWeight * (1.0 - trackingPos))
	newLowerLidWeight = (1.0 - trackingPos) + (lidWeight * trackingPos)

	if (ruRegen or (abs(newUpperLidWeight - prevUpperLidWeight) >=
	  upperLidRegenThreshold)):
		newUpperLidPts = points_interp(upperLidOpenPts,
		  upperLidClosedPts, newUpperLidWeight)
		if newUpperLidWeight > prevUpperLidWeight:
			upperEyelid.re_init(pts=points_mesh(
			  (upperLidEdgePts, prevUpperLidPts,
			  newUpperLidPts), 5, 0, False))
		else:
			upperEyelid.re_init(pts=points_mesh(
			  (upperLidEdgePts, newUpperLidPts,
			  prevUpperLidPts), 5, 0, False))
		prevUpperLidWeight = newUpperLidWeight
		prevUpperLidPts    = newUpperLidPts
		ruRegen = True
	else:
		ruRegen = False

	if (rlRegen or (abs(newLowerLidWeight - prevLowerLidWeight) >=
	  lowerLidRegenThreshold)):
		newLowerLidPts = points_interp(lowerLidOpenPts,
		  lowerLidClosedPts, newLowerLidWeight)
		if newLowerLidWeight > prevLowerLidWeight:
			lowerEyelid.re_init(pts=points_mesh(
			  (lowerLidEdgePts, prevLowerLidPts,
			  newLowerLidPts), 5, 0, False)) ## LEFT IS True
		else:
			lowerEyelid.re_init(pts=points_mesh(
			  (lowerLidEdgePts, newLowerLidPts,
			  prevLowerLidPts), 5, 0, False)) ## LEFT IS True
		prevLowerLidWeight = newLowerLidWeight
		prevLowerLidPts    = newLowerLidPts
		rlRegen = True
	else:
		rlRegen = False

	convergence = 2.0

	# Draw eye

	iris.rotateToX(curY)
	iris.rotateToY(curX-convergence) ## LEFT IS +
	iris.draw()
	eye.rotateToX(curY)
	eye.rotateToY(curX-convergence) ## LEFT IS +
	eye.draw()
	upperEyelid.draw()
	lowerEyelid.draw()

sel = selectors.DefaultSelector()

def start_connections(host, port):
        print("Searching for server")
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.setblocking(False)
        sock.settimeout(0.01)
        ip_LSB = 0
        successful = False
        while (successful == False):
                try:
                        net = host.split('.')
                        try_ip = net[0] + '.' + net[1] + '.' + net[2] + '.' + str(ip_LSB)
                        print("Trying:", try_ip)
                        server_addr = (try_ip, port)
                        if (sock.connect_ex(server_addr) == 0):
                                successful = True
                        else:
                                ip_LSB = ip_LSB + 1
                except:
                        ip_LSB = ip_LSB + 1
                if (ip_LSB == 256):
                        print("Could not find server. Exiting...")
                        sel.close()
                        exit(0)
        print("Connected!")
        events = selectors.EVENT_READ # | selectors.EVENT_WRITE
        data = types.SimpleNamespace(
                inb=b"",
                outb=b"",
        )
        sel.register(sock, events, data=data)

def service_connection(key, mask):
        global shared

        sock = key.fileobj
        data = key.data
        if mask & selectors.EVENT_READ:
                recv_data = sock.recv(102)  # Should be ready to read
                if recv_data:
                        #print("Received", len(recv_data), "bytes")
                        data.inb = recv_data
                        try:
                                shared = pickle.loads(recv_data)
                        except:
                                print("pickle.loads failed. Closing connection.")
                                sel.unregister(sock)
                                sock.close()
                else:
                        pass # We got no data. We should probably do something about this?
#        if mask & selectors.EVENT_WRITE:
#                pass
#                data.outb = b"???"
#                print("sending", repr(data.outb))
#                sent = sock.send(data.outb)  # Should be ready to write

start_connections(host, int(port))

# MAIN LOOP -- runs continuously -------------------------------------------

try:
        while True:
                frame()

                events = sel.select(timeout=0.01)
                if events:
                        for key, mask in events:
                                service_connection(key, mask)
                # Check for a socket being monitored to continue.
                if not sel.get_map():
                        break

except KeyboardInterrupt:
        print("caught keyboard interrupt, exiting")

finally:
        DISPLAY.stop()
        sel.close()


import requests
from bs4 import BeautifulSoup
import datetime
import json, time

borrowerID = list(set([8962, 6403, 3587, 7825, 5267, 7847, 7858, 7738, 7745, 6728, 7760, 7761, 8145, 7635, 8530, 4310, 7256, 7654, 6630, 4075, 7150, 8053, 6775, 8130, 7390, 2047, 3812, 7837, 6276, 8078, 7837, 8327, 8310, 8155, 5360, 8602, 8698, 8763, 8810, 5021, 8310, 8896, 8895, 8435, 7905, 8896, 8602, 7637]))
# borrowerID = [5383]
entireDataList = []
list(set(borrowerID))
#{"username":username,"Loans": [{"lender":x,"borrower":x,"loanAmount":x,"dateLent":x,"datePaid":x, "status":x}] }
userNum = 0
print("There are {} total loans".format(len(borrowerID)))
for userID in borrowerID:
	userNum += 1
	print("Starting Loan {} out of {}".format(userNum,len(borrowerID)))
	extractedUserData = {}
	extractedUserData['username'] = ""
	extractedUserData['loans'] = []
	offset = 0
	while True: # loop to keep increasing pages
		url = "https://redditloans.com/mobile_query.php?checkid={}&limit=25&offset={}".format(userID, offset)
		r = requests.get(url)
		soup = BeautifulSoup(r.text, 'html.parser')
		loans = soup.find_all('tr')
		if len(loans) > 1: # If there are loans on this page continue
			for loan in loans: # iterate through all loans
				dataColumns = loan.find_all('td')
				if len(dataColumns) > 0: # if 'loan' is actually a loan
					loanTuple = {}
					loanTuple['lender'] = dataColumns[2].text
					loanTuple['borrower'] = dataColumns[4].text
					loanTuple['loanAmount'] = dataColumns[5].text
					if dataColumns[7].text == "Yes":
						loanTuple['status'] = 0
					else:
						loanTuple['status'] = 2
					loanTuple['dateLent'] = dataColumns[8].text
					try:
						loanTuple['datePaid'] = dataColumns[9].text
					except: # no date of repayment
						loanTuple['status'] = 1
						loanTuple['datePaid'] = 0

					extractedUserData['loans'].append(loanTuple)

			offset += 25
		else:
			break
	extractedUserData['username'] = loanTuple['borrower']
	entireDataList.append(extractedUserData)
	time.sleep(2)

with open("scrapedData1.txt", "a") as theFile:
	theFile.write(str(entireDataList))


# importing the module
import json
import ase
import numpy as np
from ase import Atoms, Atom
from ase.calculators.emt import EMT
from ase.calculators.singlepoint import SinglePointCalculator
from amptorch.trainer import AtomsTrainer
import pickle
import random
import math
import sys
import torch
import os
from sklearn.linear_model import LinearRegression

def log(log_filename, message):
    f = open(log_filename, "a")
    f.write(message)
    f.close()
    return

def load_data_to_atoms(system_data):
    atoms_list = []
    for i, atom in enumerate(system_data["atoms"]):
        atoms_list.append(Atom(atom, np.array(system_data["coordinates"][i])))
    system = Atoms(atoms_list, cell=[50, 50, 50])
    system.center()
    calc = SinglePointCalculator(system,energy = system_data["corrected_energy"])
#     calc = EMT(energy = system_data['u0'])
    system.set_calculator(calc)
    return system

def count_atoms(data, atom_list):
    atom_dict = {atom:i for i, atom in enumerate(atom_list)}
    atom_dict_rev = {i:atom for i, atom in enumerate(atom_list)}

    atom_count_dict = {}
    num_atom_mat = np.zeros((len(data), len(atom_list)))
    energy_vec = np.zeros((len(data),))

    for i, system in enumerate(data.keys()):
#         print(system)
        temp_atoms = data[system]["atoms"]
#         print(temp_atoms)
        atom_count_dict[system] = {}
        energy_vec[i] = data[system]["u0"]
        for atom in temp_atoms:
            num_atom_mat[i, atom_dict[atom]] += 1.0
#             print(atom, atom_count_dict[system])
            if atom in atom_count_dict[system]:
                atom_count_dict[system][atom] += 1
            else:
                atom_count_dict[system][atom] = 1
    print(num_atom_mat)
    print(energy_vec)
    reg = LinearRegression().fit(num_atom_mat, energy_vec)
    linear_regression_result = reg.coef_
    correction_dict = {atom_dict_rev[i]: correction for i, correction in enumerate(linear_regression_result)}
    print(correction_dict)

    linear_model_result_filename = "linear_model_result.dat"
    linear_model_info_filename = "linear_model_info.dat"

    result1 = ""

    for atom in correction_dict.keys():
        result1 += "{}\t{}\n".format(atom, correction_dict[atom])
    log(linear_model_result_filename,result1)

    corrected_energy_list = []
    atoms_count_list = []
    for i, system in enumerate(data.keys()):
        data[system]["corrected_energy"] = data[system]["u0"]
        num_atoms = 0
        for atom in atom_count_dict[system].keys():
            data[system]["corrected_energy"] -= atom_count_dict[system][atom] * correction_dict[atom]
            num_atoms += atom_count_dict[system][atom]
        corrected_energy_list.append(data[system]["corrected_energy"])
        atoms_count_list.append(num_atoms)

    corrected_energy_list = np.array(corrected_energy_list)
    atoms_count_list = np.array(atoms_count_list)
    corrected_energy_per_atom_list = np.divide(corrected_energy_list, atoms_count_list)

    result2 = "me: {}\n".format(np.mean(corrected_energy_list))
    result2 += "mae: {}\n".format(np.mean(np.abs(corrected_energy_list)))
    result2 += "mse: {}\n".format(np.mean(np.square(corrected_energy_list)))

    result2 += "mepa: {}\n".format(np.mean(corrected_energy_per_atom_list))
    result2 += "maepa: {}\n".format(np.mean(np.abs(corrected_energy_per_atom_list)))
    result2 += "msepa: {}".format(np.mean(np.square(corrected_energy_per_atom_list)))

    log(linear_model_info_filename,result2)

    return

def linear_fit(data_list, atom_list, index_list):
    atom_dict = {atom:i for i, atom in enumerate(atom_list)}
    atom_dict_rev = {i:atom for i, atom in enumerate(atom_list)}

    # atom_count_list = []

    training_list = []
    test_list = []
    training_atom_count_list = []
    test_atom_count_list = []
    overall_atom_count_list = []
    print("start counting atoms")
    for i,system in enumerate(data_list):
        temp_atoms = system["atoms"]
        temp_atom_count_dict = {}
        for atom in temp_atoms:
            temp_atom_count_dict[atom] = temp_atom_count_dict.get(atom, 0) + 1

        overall_atom_count_list.append(temp_atom_count_dict)

        if i in index_list:
            training_list.append(system)
            training_atom_count_list.append(temp_atom_count_dict)
        else:
            test_list.append(system)
            test_atom_count_list.append(temp_atom_count_dict)

    print("start preparing linear system")
    num_atom_mat = np.zeros((len(index_list), len(atom_list)))
    energy_vec = np.zeros((len(index_list),))
    for i, system in enumerate(training_list):
        energy_vec[i] = system["u0_atom"]
        atom_counts = training_atom_count_list[i]
        for atom in atom_counts.keys():
            num_atom_mat[i, atom_dict[atom]] = atom_counts[atom]

    print(num_atom_mat)
    print(energy_vec)
    reg = LinearRegression().fit(num_atom_mat, energy_vec)
    linear_regression_result = reg.coef_
    correction_dict = {atom_dict_rev[i]: correction for i, correction in enumerate(linear_regression_result)}
    print(correction_dict)

    linear_model_result_filename = "linear_model_result.dat"
    linear_model_info_filename = "linear_model_info.dat"

    result1 = ""
    result2 = ""

    for atom in correction_dict.keys():
        result1 += "{}\t{}\n".format(atom, correction_dict[atom])
    log(linear_model_result_filename,result1)

    corrected_energy_list = []
    atoms_count_list = []
    for i, system in enumerate(training_list):
        energy = system["u0_atom"]
        num_atoms = 0
        for atom in training_atom_count_list[i].keys():
            energy -= training_atom_count_list[i][atom] * correction_dict[atom]
            num_atoms += training_atom_count_list[i][atom]
        corrected_energy_list.append(energy)
        atoms_count_list.append(num_atoms)

    corrected_energy_list = np.array(corrected_energy_list)
    atoms_count_list = np.array(atoms_count_list)
    corrected_energy_per_atom_list = np.divide(corrected_energy_list, atoms_count_list)

    result2 += "training me: {}\n".format(np.mean(corrected_energy_list))
    result2 += "training mae: {}\n".format(np.mean(np.abs(corrected_energy_list)))
    result2 += "training mse: {}\n".format(np.mean(np.square(corrected_energy_list)))

    result2 += "training mepa: {}\n".format(np.mean(corrected_energy_per_atom_list))
    result2 += "training maepa: {}\n".format(np.mean(np.abs(corrected_energy_per_atom_list)))
    result2 += "training msepa: {}\n".format(np.mean(np.square(corrected_energy_per_atom_list)))

    corrected_energy_list = []
    atoms_count_list = []
    for i, system in enumerate(test_list):
        energy = system["u0_atom"]
        num_atoms = 0
        for atom in test_atom_count_list[i].keys():
            energy -= test_atom_count_list[i][atom] * correction_dict[atom]
            num_atoms += test_atom_count_list[i][atom]
        corrected_energy_list.append(energy)
        atoms_count_list.append(num_atoms)

    corrected_energy_list = np.array(corrected_energy_list)
    atoms_count_list = np.array(atoms_count_list)
    corrected_energy_per_atom_list = np.divide(corrected_energy_list, atoms_count_list)

    result2 += "test me: {}\n".format(np.mean(corrected_energy_list))
    result2 += "test mae: {}\n".format(np.mean(np.abs(corrected_energy_list)))
    result2 += "test mse: {}\n".format(np.mean(np.square(corrected_energy_list)))

    result2 += "test mepa: {}\n".format(np.mean(corrected_energy_per_atom_list))
    result2 += "test maepa: {}\n".format(np.mean(np.abs(corrected_energy_per_atom_list)))
    result2 += "test msepa: {}\n".format(np.mean(np.square(corrected_energy_per_atom_list)))

    corrected_energy_list = []
    atoms_count_list = []
    for i, system in enumerate(data_list):
        system["corrected_energy"] = system["u0_atom"]
        num_atoms = 0
        for atom in overall_atom_count_list[i].keys():
            system["corrected_energy"] -= overall_atom_count_list[i][atom] * correction_dict[atom]
            num_atoms += overall_atom_count_list[i][atom]
        corrected_energy_list.append(system["corrected_energy"])
        atoms_count_list.append(num_atoms)

    corrected_energy_list = np.array(corrected_energy_list)
    atoms_count_list = np.array(atoms_count_list)
    corrected_energy_per_atom_list = np.divide(corrected_energy_list, atoms_count_list)

    result2 += "overall me: {}\n".format(np.mean(corrected_energy_list))
    result2 += "overall mae: {}\n".format(np.mean(np.abs(corrected_energy_list)))
    result2 += "overall mse: {}\n".format(np.mean(np.square(corrected_energy_list)))

    result2 += "overall mepa: {}\n".format(np.mean(corrected_energy_per_atom_list))
    result2 += "overall maepa: {}\n".format(np.mean(np.abs(corrected_energy_per_atom_list)))
    result2 += "overall msepa: {}\n".format(np.mean(np.square(corrected_energy_per_atom_list)))

    log(linear_model_info_filename,result2)

    return training_list, test_list

def load_training_data(training_filename, test_filename, num_training, atom_list = ["H","C","N","O","F"]):

    try:
        training_list = pickle.load( open( training_filename, "rb" ) )
        test_list = pickle.load( open( test_filename, "rb" ) )
    except:

        with open('../qm9.json') as json_file:
            data = json.load(json_file)

        print(len(data))
        data_list = []
        for i, system in enumerate(data.keys()):
            data_list.append(data[system])

        index_list = random.sample(range(0, len(data_list)-1), num_training)
        #index_list = random.sample(range(0, len(data_list)-1), math.ceil(0.8 * len(data_list)))

        training_list, test_list = linear_fit(data_list, atom_list, index_list)

        training_images = [load_data_to_atoms(system) for system in training_list]
        test_images = [load_data_to_atoms(system) for system in test_list]

        images = []
        for i, system in enumerate(data.keys()):
            image = load_data_to_atoms(data[system])
            images.append(image)

        pickle.dump( training_images, open( training_filename, "wb" ) )
        pickle.dump( test_images, open( test_filename, "wb" ) )

    return training_images, test_images

torch.set_num_threads(1)
trail_num = sys.argv[1]
num_training = int(sys.argv[2])
train_filename = "QM9_train_{}.p".format(trail_num)
test_filename = "QM9_test_{}.p".format(trail_num)
folder_name = "trial_{}".format(trail_num)
if not os.path.exists(folder_name):
    os.makedirs(folder_name)
os.chdir(folder_name)
train_images, test_images = load_training_data(train_filename, test_filename, num_training)


import os
from pathlib import Path
import httplib2

from datetime import datetime
from rest_framework import status
from rest_framework.decorators import api_view
from rest_framework.views import APIView
from rest_framework.response import Response
from django.shortcuts import get_list_or_404, get_object_or_404
import logging

from googleapi.models import *
from googleapi.api.serializers import *
from datetime import datetime

from google.oauth2 import id_token, service_account
from google.auth.transport import requests
from apiclient import discovery

logger = logging.getLogger(__name__)

class UserAccessRecordListAPIView(APIView):
    def get(self, request, email):
        userAccessRecord = get_list_or_404(UserAccessRecord, email=email)
        serializer = UserAccessRecordSerializer(userAccessRecord, many=True)
        return Response(serializer.data, status=status.HTTP_200_OK)

class UserAccessRecordAPIView(APIView):
    scopes = ["https://www.googleapis.com/auth/drive",
              "https://www.googleapis.com/auth/drive.file",
              "https://www.googleapis.com/auth/spreadsheets"]
    secretFile = os.path.join(os.getcwd(), "env", os.getenv(
        "DJANGO_GOOGLE_SERVICES_ACCOUNT_FILENAME"))
    credentials = service_account.Credentials.from_service_account_file(
        secretFile, scopes=scopes)
    service = discovery.build("sheets", "v4", credentials=credentials)
    spreadsheetID = os.getenv("DJANGO_GOOGLE_SPREADSHEET_ID")
    spreatsheetName = "UserInfoForPortal"
    spreatSheetColumnNumber = {
        "EmailAddress": {"colnumNumber": 2, "updateable": False},
        "ChineseName": {"colnumNumber": 8, "updateable": False},
        "FirstName": {"colnumNumber": 9, "updateable": False},
        "LastName": {"colnumNumber": 10, "updateable": False},
        "NickName": {"colnumNumber": 11, "updateable": False},
        "WechatID": {"colnumNumber": 13, "updateable": False},
        "TShirtSize": {"colnumNumber": 14, "updateable": False},
        "DietaryRestriction": {"colnumNumber": 15, "updateable": False},
        "ReimbursementMethod": {"colnumNumber": 16, "updateable": True},
        "ReimbursementAccountType": {"colnumNumber": 17, "updateable": True},
        "ReimbursementAccountEmail": {"colnumNumber": 18, "updateable": True},
        "ReimbursementAccountPhoneNumber": {"colnumNumber": 19, "updateable": True},
        "PhoneNumber": {"colnumNumber": 23, "updateable": True},
        "Zgid": {"colnumNumber": 32, "updateable": False},
        "ZgidEmail": {"colnumNumber": 33, "updateable": False},
        "PromoCode2020": {"colnumNumber": 34, "updateable": False}
    }

    def _checkVaildToken(self, request):
        req = requests.Request()
        id_info = id_token.verify_oauth2_token(
            request.data["id_token"], req, os.getenv('DJANGO_GOOGLE_CLIEND_ID'))
        if id_info["iss"] != "accounts.google.com" or id_info["email"] != request.data["email"]:
            return False
        return True

    def _getGoogleSheetRowNumberFromEmail(self, request):
        ranges = [
            f"{self.spreatsheetName}!B:B",
            f"{self.spreatsheetName}!AG:AG"
        ]
        serviceRequest = self.service.spreadsheets().values().batchGet(
            spreadsheetId=self.spreadsheetID, ranges=ranges, majorDimension='COLUMNS')
        try:
            response = serviceRequest.execute()
        except Exception as e:
            logger.error(e)
            return -1
        userEmail: str = request.data["email"]
        rowList: list
        if userEmail.endswith("@zgzg.io"):
            rowList = response["valueRanges"][1]["values"][0]
        else:
            rowList = response["valueRanges"][0]["values"][0]
            map(str.lower, rowList)
        return rowList.index(userEmail)+1 if userEmail in rowList else -1

    def _updateGoogleSheet(self, request):
        ret = {"returnCode": -1}
        rowNumber = self._getGoogleSheetRowNumberFromEmail(request)
        if rowNumber < 1:
            return ret
        range_ = f"{self.spreatsheetName}!A{rowNumber}:ZZ{rowNumber}"
        valueInputOption = "RAW"
        values = []
        serviceRequest = self.service.spreadsheets().values().get(
            spreadsheetId=self.spreadsheetID, range=range_)
        try:
            response = serviceRequest.execute()
        except Exception as e:
            logger.error(e)
            return ret
        responseValues = response["values"][0]
        try:
            for key in self.spreatSheetColumnNumber:
                if self.spreatSheetColumnNumber[key]["updateable"]:
                    responseValues[self.spreatSheetColumnNumber[key]
                                   ["colnumNumber"]-1] = str(request.data["data"][key])
        except Exception as e:
            print(e, self.spreatSheetColumnNumber[key]
                  ["colnumNumber"]-1, len(responseValues))
        serviceRequest = self.service.spreadsheets().values().update(
            spreadsheetId=self.spreadsheetID,
            range=f"{self.spreatsheetName}!A{rowNumber}:ZZ{rowNumber}",
            valueInputOption=valueInputOption,
            body={
                "values": [responseValues]
            })

        try:
            response = serviceRequest.execute()
        except Exception as e:
            logger.error(e)
            return False
        return True

    def _getGoogleSheetData(self, request):
        ret = {"returnCode": -1}
        rowNumber = self._getGoogleSheetRowNumberFromEmail(request)
        if rowNumber < 1:
            return ret
        range_ = f"{self.spreatsheetName}!A{rowNumber}:ZZ{rowNumber}"
        serviceRequest = self.service.spreadsheets().values().get(
            spreadsheetId=self.spreadsheetID, range=range_)
        try:
            response = serviceRequest.execute()
        except Exception as e:
            logger.error(e)
            return ret
        responseValues = response["values"][0]
        PersonEmail = responseValues[self.spreatSheetColumnNumber["EmailAddress"]["colnumNumber"]-1]
        ZgidEmail = responseValues[self.spreatSheetColumnNumber["ZgidEmail"]["colnumNumber"]-1]
        if PersonEmail == request.data["email"] or ZgidEmail == request.data["email"]:
            ret["returnCode"] = 1
            for key in self.spreatSheetColumnNumber:
                ret[key] = responseValues[self.spreatSheetColumnNumber[key]
                                          ["colnumNumber"]-1]
        return ret

    def get(self, request):
        userAccessRecord = UserAccessRecord.objects.all()
        serializer = UserAccessRecordSerializer(userAccessRecord, many=True)
        return Response(serializer.data, status=status.HTTP_200_OK)

    def post(self, request):
        request.data["record_time"] = datetime.now()
        request.data["action"] = "View"

        serializers = UserAccessRecordSerializer(data=request.data)
        if serializers.is_valid():
            serializers.save()
            if self._checkVaildToken(request):
                ret = self._getGoogleSheetData(request)
                status_ = status.HTTP_201_CREATED if ret["returnCode"] == 1 else status.HTTP_404_NOT_FOUND
                return Response(ret, status=status_)
            return Response(status=status.HTTP_401_UNAUTHORIZED)
        return Response(serializers.errors, status=status.HTTP_400_BAD_REQUEST)

    def put(self, request):
        request.data["record_time"] = datetime.now()
        tokenVaild = self._checkVaildToken(request)
        request.data["action"] = "Unauthorized to update profile"

        if tokenVaild:
            request.data["action"] = "Update profile: "
            ret = self._getGoogleSheetData(request)
            for key in ret:
                if ret[key] != request.data["data"][key]:
                    request.data["action"] += f"{key} from {ret[key]} to {request.data['data'][key]},"
            self._updateGoogleSheet(request)

        serializers = UserAccessRecordSerializer(data=request.data)
        if serializers.is_valid():
            serializers.save()
            if tokenVaild:
                ret = self._getGoogleSheetData(request)
                status_ = status.HTTP_201_CREATED if ret["returnCode"] == 1 else status.HTTP_404_NOT_FOUND
                return Response(ret, status=status_)
            return Response(status=status.HTTP_401_UNAUTHORIZED)
        return Response(serializers.errors, status=status.HTTP_400_BAD_REQUEST)


# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import paho.mqtt.publish as publish

from modules.event.instance.status.events import *
from modules.util.log import *
from modules.util import cartridgeagentutils
import healthstats
import constants
from config import Config

log = LogFactory().get_log(__name__)

started = False
activated = False
ready_to_shutdown = False
maintenance = False

publishers = {}
""" :type : dict[str, EventPublisher] """

def publish_instance_started_event():
    global started, log
    if not started:
        log.info("Publishing instance started event")

        application_id = Config.application_id
        service_name = Config.service_name
        cluster_id = Config.cluster_id
        member_id = Config.member_id
        instance_id = Config.instance_id
        cluster_instance_id = Config.cluster_instance_id
        network_partition_id = Config.network_partition_id
        partition_id = Config.partition_id

        instance_started_event = InstanceStartedEvent(
            application_id,
            service_name,
            cluster_id,
            cluster_instance_id,
            member_id,
            instance_id,
            network_partition_id,
            partition_id)

        publisher = get_publisher(constants.INSTANCE_STATUS_TOPIC + constants.INSTANCE_STARTED_EVENT)
        publisher.publish(instance_started_event)
        started = True
        log.info("Instance started event published")
    else:
        log.warn("Instance already started")

def publish_instance_activated_event(health_stat_plugin):
    global activated, log
    if not activated:
        # Wait for all ports to be active
        listen_address = Config.listen_address
        configuration_ports = Config.ports
        ports_active = cartridgeagentutils.wait_until_ports_active(
            listen_address,
            configuration_ports,
            int(Config.read_property("port.check.timeout", critical=False)))

        if ports_active:
            log.info("Publishing instance activated event")
            service_name = Config.service_name
            cluster_id = Config.cluster_id
            member_id = Config.member_id
            instance_id = Config.instance_id
            cluster_instance_id = Config.cluster_instance_id
            network_partition_id = Config.network_partition_id
            partition_id = Config.partition_id

            instance_activated_event = InstanceActivatedEvent(
                service_name,
                cluster_id,
                cluster_instance_id,
                member_id,
                instance_id,
                network_partition_id,
                partition_id)

            publisher = get_publisher(constants.INSTANCE_STATUS_TOPIC + constants.INSTANCE_ACTIVATED_EVENT)
            publisher.publish(instance_activated_event)

            log.info("Instance activated event published")
            log.info("Starting health statistics notifier")

            health_stat_publishing_enabled = Config.read_property(constants.CEP_PUBLISHER_ENABLED, True)

            if health_stat_publishing_enabled:
                interval_default = 15  # seconds
                interval = Config.read_property("stats.notifier.interval", False)
                if interval is not None and len(interval) > 0:
                    try:
                        interval = int(interval)
                    except ValueError:
                        interval = interval_default
                else:
                    interval = interval_default
                health_stats_publisher = healthstats.HealthStatisticsPublisherManager(interval, health_stat_plugin)
                log.info("Starting Health statistics publisher with interval %r" % interval)
                health_stats_publisher.start()
            else:
                log.warn("Statistics publisher is disabled")

            activated = True
            log.info("Health statistics notifier started")
        else:
            log.error("Ports activation timed out. Aborting InstanceActivatedEvent publishing [IPAddress] %s [Ports] %s"
                      % (listen_address, configuration_ports))
    else:
        log.warn("Instance already activated")

def publish_maintenance_mode_event():
    global maintenance, log
    if not maintenance:
        log.info("Publishing instance maintenance mode event")

        service_name = Config.service_name
        cluster_id = Config.cluster_id
        member_id = Config.member_id
        instance_id = Config.instance_id
        cluster_instance_id = Config.cluster_instance_id
        network_partition_id = Config.network_partition_id
        partition_id = Config.partition_id

        instance_maintenance_mode_event = InstanceMaintenanceModeEvent(
            service_name,
            cluster_id,
            cluster_instance_id,
            member_id,
            instance_id,
            network_partition_id,
            partition_id)

        publisher = get_publisher(constants.INSTANCE_STATUS_TOPIC + constants.INSTANCE_MAINTENANCE_MODE_EVENT)
        publisher.publish(instance_maintenance_mode_event)

        maintenance = True
        log.info("Instance Maintenance mode event published")
    else:
        log.warn("Instance already in a Maintenance mode...")

def publish_instance_ready_to_shutdown_event():
    global ready_to_shutdown, log
    if not ready_to_shutdown:
        log.info("Publishing instance activated event")

        service_name = Config.service_name
        cluster_id = Config.cluster_id
        member_id = Config.member_id
        instance_id = Config.instance_id
        cluster_instance_id = Config.cluster_instance_id
        network_partition_id = Config.network_partition_id
        partition_id = Config.partition_id

        instance_shutdown_event = InstanceReadyToShutdownEvent(
            service_name,
            cluster_id,
            cluster_instance_id,
            member_id,
            instance_id,
            network_partition_id,
            partition_id)

        publisher = get_publisher(constants.INSTANCE_STATUS_TOPIC + constants.INSTANCE_READY_TO_SHUTDOWN_EVENT)
        publisher.publish(instance_shutdown_event)

        ready_to_shutdown = True
        log.info("Instance ReadyToShutDown event published")
    else:
        log.warn("Instance already in a ReadyToShutDown event...")

def get_publisher(topic):
    if topic not in publishers:
        publishers[topic] = EventPublisher(topic)

    return publishers[topic]

class EventPublisher:
    """
    Handles publishing events to topics to the provided message broker
    """
    def __init__(self, topic):
        self.__topic = topic

    def publish(self, event):
        mb_ip = Config.read_property(constants.MB_IP)
        mb_port = Config.read_property(constants.MB_PORT)
        payload = event.to_json()
        publish.single(self.__topic, payload, hostname=mb_ip, port=mb_port)


import numpy as np
import matplotlib
from matplotlib import pyplot as plt

def get_memory_len(kernel_array, ratio):
    """
    Return the number of time bins until kernel_array has decreased by the factor 'ratio'
    'kernel_array' may initially rise, but must be monotonically decreasing from
    the maximum.
    """
    arr = (kernel_array - ratio*kernel_array.max())[::-1]
        # The point where this array reaches zero is the desired memory time
        # We flip the order with [::-1] because np.searchsorted expects increasing order
    memory_len = len(kernel_array) - np.searchsorted(arr, 0)
    return memory_len

def find_first_spike(V, threshold):
    # Based on the equivalent code to py_find_1st (https://github.com/roebel/py_find_1st)
    ind = np.flatnonzero(V >= threshold)
    if len(ind):
        return ind[0]
    else:
        return -1

def calculate_Vt(unweighted_input, theta, omega):
    spike_time = []
    datalen = unweighted_input.shape[1]
    V = (omega[:,np.newaxis] * unweighted_input).sum(axis=0)
    ref_kernel = np.exp(- np.arange(1000) / 20) #tau_mem = 20
    ref_memory_len = get_memory_len(ref_kernel, ratio=0.001)
    new_ref_kernel = ref_kernel[:ref_memory_len]
    done = False
    while not done:
        spike_idx = find_first_spike(V, theta)
        if spike_idx == -1:
            done = True
        else:
            spike_time.append(spike_idx)
            mem_len = min(ref_memory_len, datalen - spike_idx)
            V[spike_idx:spike_idx+mem_len] -= theta * new_ref_kernel[:mem_len]
    return V, spike_time

def plt_V(in_data, omega, cycle):
    input_data = np.load(in_data)
    data, presyn_input, markers, n_fea_occur, fea_time, fea_order = input_data['arr_0'], input_data['arr_1'], input_data['arr_2'], input_data['arr_3'], input_data['arr_4'], input_data['arr_5']

    if cycle != None:
        updated_omega = np.load(omega)[cycle]
    else:
        updated_omega = np.load(omega)
    time = np.arange(0, len(presyn_input[1]), 1)
    V_t, spike_time = calculate_Vt(presyn_input, 1, updated_omega)

    matplotlib.rcParams['figure.figsize'] = (15.0, 3.5)
    plt.xlabel("Time (ms)", fontsize=10)
    plt.ylabel("Voltage", fontsize=10)
    plt.plot(time, V_t)
    for marker in markers:
        plt.gca().add_patch(marker)
    plt.ylim((0, 1.2))

def multi_Vplot(row, col, data, omega, cycle, title, fig_width, fig_height):
    fig, axs = plt.subplots(row, col, figsize=(fig_width, fig_height))

    for n in range(len(cycle)):
        axs = plt.subplot(row, col, n+1)
        plt_V(data, omega[n], cycle[n])
        axs.set_title(title[n], fontsize = 15)

    fig.tight_layout()


import cv2
import numpy as np
import torch
import onnxruntime
import pycuda.driver as cuda
import pycuda.autoinit
import tensorrt as trt
from options.options import parser
from options.config import cfg

def inference_with_onnx(model_file, model_in):
    ort_session = onnxruntime.InferenceSession(model_file)

    # compute ONNX Runtime output prediction
    ort_inputs = {ort_session.get_inputs()[0].name: model_in}
    ort_outs = ort_session.run(None, ort_inputs)
    return ort_outs

def inference_with_engine(model_file, model_in):
    TRT_LOGGER = trt.Logger(trt.Logger.INFO)
    ort_outs = []
    with open(model_file, "rb") as f, trt.Runtime(TRT_LOGGER) as runtime:
        engine = runtime.deserialize_cuda_engine(f.read())
        model_in = model_in.ravel()
        h_input = np.empty(trt.volume(engine.get_binding_shape(0)), np.float32)
        d_input = cuda.mem_alloc(1 * h_input.nbytes)

        if cfg.NUM_CLASSES and cfg.NUM_EGO:
            output_cls_size = engine.get_binding_shape(1)
            output_ego_size = engine.get_binding_shape(2)
            output_exist_size = engine.get_binding_shape(3)
            h_output_cls = np.empty(trt.volume(engine.get_binding_shape(1)), np.float32)
            h_output_ego = np.empty(trt.volume(engine.get_binding_shape(2)), np.float32)
            h_output_exist = np.empty(trt.volume(engine.get_binding_shape(3)), np.float32)
            d_output_cls = cuda.mem_alloc(1 * h_output_cls.nbytes)
            d_output_ego = cuda.mem_alloc(1 * h_output_ego.nbytes)
            d_output_exist = cuda.mem_alloc(1 * h_output_exist.nbytes)
            bindings = [int(d_input), int(d_output_cls), int(d_output_ego), int(d_output_exist)]
        elif cfg.NUM_CLASSES and cfg.NUM_EGO == 0:
            output_cls_size = engine.get_binding_shape(1)
            h_output_cls = np.empty(trt.volume(engine.get_binding_shape(1)), np.float32)
            d_output_cls = cuda.mem_alloc(1 * h_output_cls.nbytes)
            bindings = [int(d_input), int(d_output_cls)]
        elif cfg.NUM_CLASSES == 0 and cfg.NUM_EGO:
            output_ego_size = engine.get_binding_shape(1)
            output_exist_size = engine.get_binding_shape(2)
            h_output_ego = np.empty(trt.volume(engine.get_binding_shape(1)), np.float32)
            h_output_exist = np.empty(trt.volume(engine.get_binding_shape(2)), np.float32)
            d_output_ego = cuda.mem_alloc(1 * h_output_ego.nbytes)
            d_output_exist = cuda.mem_alloc(1 * h_output_exist.nbytes)
            bindings = [int(d_input), int(d_output_ego), int(d_output_exist)]

        stream = cuda.Stream()
        with engine.create_execution_context() as context:
            cuda.memcpy_htod_async(d_input, model_in, stream)
            context.execute_async(bindings=bindings, stream_handle=stream.handle)
            if cfg.NUM_CLASSES:
                cuda.memcpy_dtoh_async(h_output_cls, d_output_cls, stream)
            if cfg.NUM_EGO:
                cuda.memcpy_dtoh_async(h_output_ego, d_output_ego, stream)
                cuda.memcpy_dtoh_async(h_output_exist, d_output_exist, stream)
            stream.synchronize()
            if cfg.NUM_CLASSES:
                h_output_cls = h_output_cls.reshape(output_cls_size)
                ort_outs.append(h_output_cls)
            if cfg.NUM_EGO:
                h_output_ego = h_output_ego.reshape(output_ego_size)
                h_output_exist = h_output_exist.reshape(output_exist_size)
                ort_outs.append(h_output_ego)
                ort_outs.append(h_output_exist)
            return ort_outs

def inference(model_file, image_file):
    image = cv2.imread(image_file)
    h, w, c = image.shape
    image_bgr = cv2.resize(image, (cfg.LOAD_IMAGE_WIDTH, cfg.LOAD_IMAGE_HEIGHT), interpolation=cv2.INTER_NEAREST)
    image_bgr = image_bgr[cfg.VERTICAL_CROP_SIZE:, :, :]
    image_bgr = cv2.resize(image_bgr, (cfg.MODEL_INPUT_WIDTH, cfg.MODEL_INPUT_HEIGHT), interpolation=cv2.INTER_NEAREST)
    image_rgb = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)
  #  image_rgb = (image_rgb - cfg.INPUT_MEAN) * cfg.INPUT_STD
    image_rgb = np.transpose(image_rgb, (2, 0, 1))
    image_rgb = np.ascontiguousarray(image_rgb, dtype=np.float32)
    image_rgb = np.expand_dims(image_rgb, 0)
    ort_outs = []
    if model_file.find('onnx') != -1:
        ort_outs = inference_with_onnx(model_file, image_rgb)
        ort_out = torch.from_numpy(ort_outs[0])
        ort_out = ort_out.data.cpu().numpy()
        ort_out = np.squeeze(ort_out, 0)
    elif model_file.find('engine') != -1:
        ort_outs = inference_with_engine(model_file, image_rgb)
        ort_out = ort_outs[0]
        ort_out = np.transpose(ort_out, (2, 0, 1))
    if len(ort_outs):
        threshold_cls = int(cfg.THRESHOLD_CLS * 255)
        result_cls = np.zeros((h, w)).astype(np.uint8)
        # ort_out = F.softmax(ort_out, dim=1)

        for num in range(cfg.NUM_CLASSES-1):
            prob_map = (ort_out[num + 1] * 255).astype(np.uint8)
            map_bak = np.zeros((cfg.LOAD_IMAGE_HEIGHT, cfg.LOAD_IMAGE_WIDTH))
            map_bak[cfg.VERTICAL_CROP_SIZE:, :] = cv2.resize(prob_map, (cfg.LOAD_IMAGE_WIDTH, cfg.IN_IMAGE_H_AFTER_CROP),
                                                         interpolation=cv2.INTER_NEAREST)
            map_bak = cv2.resize(map_bak, (w, h), interpolation=cv2.INTER_NEAREST)
            result_cls[map_bak >= threshold_cls] = num + 1

        result_cls_color = np.copy(image)
        # cv2.imshow("alllane", result_cls_color)
        # cv2.waitKey()
        for i in range(h):
            for j in range(w):
                if result_cls[i, j] != 0:
                    result_cls_color[i, j, :] = cfg.EG0_POINT_COLORS[result_cls[i, j] - 1]
        # cv2.imshow("alllane", result_cls_color)
        # cv2.waitKey()
        print("src/samples/alllane.png")
        cv2.imwrite("src/samples/alllane.png", result_cls_color)

if __name__ == '__main__':
    args = parser.parse_args()
    cfg.THRESHOLD_CLS = 0.85
    cfg.MODEL_INPUT_WIDTH, cfg.MODEL_INPUT_HEIGHT = 480,224
    image_file = 'src/samples/lane_detection_l4e_input.png'
    image = cv2.imread(image_file)
    h,w,c = image.shape
    image_re = cv2.resize(image, (int(1.2*w), int(1.2*h)), interpolation=cv2.INTER_NEAREST)
    image_re1 = cv2.resize(image, (int(1.2*w), int(1.2*h)), interpolation=cv2.INTER_LINEAR)
    merged = np.concatenate([image_re, image_re1], axis=1)
    cv2.imshow("resize", merged)
    cv2.waitKey()
    inference(args.model_file, image_file)
    print("All Done")


# KidsCanCode - Game Development with Pygame video series
# Tile-based game - Part 1
# Project setup
# Video link: https://youtu.be/3UxnelT9aCo
import pygame
import sys
import settings
from settings import WIDTH, HEIGHT, TITLE, TILESIZE, FPS, PLAYER_IMG, WALL_IMG,\
MOB_IMG, BULLET_IMG, BULLET_DAMAGE, MOB_DAMAGE, MOB_KNOCKBACK, GREEN, YELLOW,\
RED, WHITE, PLAYER_HEALTH, CYAN, MUZZLE_FLASHES, ITEM_IMAGES, HEALTH_PACK_AMOUNT,\
BG_MUSIC, EFFECTS_SOUNDS, WEAPON_SOUNDS_GUN, ZOMBIE_MOAN_SOUNDS, PLAYER_HIT_SOUNDS,\
ZOMBIE_HIT_SOUNDS
from sprites import Player, Mob, collide_hit_rect, Obstacle, Item
from os import path
from tilemap import Camera, TiledMap
from random import random, choice
vec = pygame.math.Vector2

# HUD FUNCTION
def draw_player_health(surf, x, y, pct):
    if pct < 0:
        pct = 0
    BAR_LENGTH = 100
    BAR_HEIGHT = 20
    fill = pct * BAR_LENGTH
    outline_rect = pygame.Rect(x, y, BAR_LENGTH, BAR_HEIGHT)
    fill_rect = pygame.Rect(x, y, fill, BAR_HEIGHT)
    if pct > 0.6:
        col = GREEN
    elif pct > 0.3:
        col = YELLOW
    else:
        col = RED
    pygame.draw.rect(surf, col, fill_rect)
    pygame.draw.rect(surf, WHITE, outline_rect, 2)

class Game:
    def __init__(self):
        # Initialize pygame
        pygame.init()

        # Set screen settings
        self.screen = pygame.display.set_mode([WIDTH, HEIGHT])
        pygame.display.set_caption(TITLE)

        # Set clock to limit FPS
        self.clock = pygame.time.Clock()

        # pygame.key.set_repeat(when, inteval)
        pygame.key.set_repeat(500, 100)

        # Load data
        self.load_data()

    def load_data(self):
        # Specify game folder
        game_folder = path.dirname(__file__)
        img_folder = path.join(game_folder, 'img')
        map_folder = path.join(game_folder, 'maps')
        snd_folder = path.join(game_folder, 'snd')
        music_folder = path.join(game_folder, 'music')

        # Load data from map
        self.map_data = []
        self.map = TiledMap(path.join(map_folder, 'level1.tmx'))
        self.map_img = self.map.make_map()
        self.map_rect = self.map_img.get_rect()

        # Load img data
        self.player_img = pygame.image.load(path.join(img_folder, PLAYER_IMG)).convert_alpha()
        self.bullet_img = pygame.image.load(path.join(img_folder, BULLET_IMG)).convert_alpha()
        self.mob_img = pygame.image.load(path.join(img_folder, MOB_IMG)).convert_alpha()
        self.wall_img = pygame.image.load(path.join(img_folder, WALL_IMG)).convert_alpha()
        # Scale wall img due to its huge dimensions
        self.wall_img = pygame.transform.scale(self.wall_img, (TILESIZE, TILESIZE))

        self.gun_flashes = []
        for img in MUZZLE_FLASHES:
            self.gun_flashes.append(pygame.image.load(path.join(img_folder, img)).convert_alpha())

        self.item_images = {}
        for item in ITEM_IMAGES:
            self.item_images[item] = pygame.image.load(path.join(img_folder, ITEM_IMAGES[item])).convert_alpha()

        # Sound load
        pygame.mixer.music.load(path.join(music_folder, BG_MUSIC))
        self.effects_sounds = {}
        for type in EFFECTS_SOUNDS:
            self.effects_sounds[type] = pygame.mixer.Sound(path.join(snd_folder, EFFECTS_SOUNDS[type]))

        self.weapon_sounds = {}
        self.weapon_sounds['gun'] = []
        for snd in WEAPON_SOUNDS_GUN:
            self.weapon_sounds['gun'].append(pygame.mixer.Sound(path.join(snd_folder, snd)))

        self.zombie_moan_sounds = []
        for snd in ZOMBIE_MOAN_SOUNDS:
            s = pygame.mixer.Sound(path.join(snd_folder, snd))
            s.set_volume(0.2)
            self.zombie_moan_sounds.append(s)

        self.player_hit_sounds = []
        for snd in PLAYER_HIT_SOUNDS:
            self.player_hit_sounds.append(pygame.mixer.Sound(path.join(snd_folder, snd)))

        self.zombie_hit_sounds = []
        for snd in ZOMBIE_HIT_SOUNDS:
            self.zombie_hit_sounds.append(pygame.mixer.Sound(path.join(snd_folder, snd)))

    def new(self):
        # initialize all variables and do all the setup for a new game
        self.all_sprites = pygame.sprite.LayeredUpdates()
        self.walls = pygame.sprite.Group()
        self.mobs = pygame.sprite.Group()
        self.bullets = pygame.sprite.Group()
        self.items = pygame.sprite.Group()

        # Spawning walls
        # enumerate makes row as index, example:
        # l = ['a', 'b', 'c', 'd']
        # for index, item in enumerate(l):
        #   print(index, item)
#        for row, tiles in enumerate(self.map.data):
#            for col, tile in enumerate(tiles):
#                if tile == '1':
#                    Wall(self, col, row)
#                elif tile == 'P':
#                    # Spawn player
#                    self.player = Player(self, col, row)
#                elif tile == 'M':
#                    Mob(self, col, row)
        for tile_object in self.map.tmxdata.objects:
            # To spawn our object in the center of the object in map, not in left upper corner
            obj_center = vec(tile_object.x + tile_object.width // 2, tile_object.y + tile_object.height // 2)
            if tile_object.name == 'player':
                self.player = Player(self, obj_center.x, obj_center.y)
            if tile_object.name == 'zombie':
                Mob(self, obj_center.x, obj_center.y)
            if tile_object.name == 'wall':
                Obstacle(self, tile_object.x, tile_object.y, tile_object.width, tile_object.height)
            if tile_object.name in ['health']:
                Item(self, obj_center, tile_object.name)
        self.camera = Camera(self.map.width, self.map.height)
        self.draw_debug = False
        self.effects_sounds['level_start'].play()

    def run(self):
        # game loop - set self.playing = False to end the game
        self.playing = True

        # Loops=-1 means loop and repeat
        pygame.mixer.music.play(loops=-1)

        while self.playing:
            self.dt = self.clock.tick(FPS) / 1000
            self.events()
            self.update()
            self.draw()

    def quit(self):
        pygame.quit()
        sys.exit()

    def update(self):
        # update portion of the game loop
        self.all_sprites.update()
        self.camera.update(self.player)

        # Playa' hits items
        hits = pygame.sprite.spritecollide(self.player, self.items, False)
        for hit in hits:
            if hit.item_type == 'health' and self.player.health < PLAYER_HEALTH:
                hit.kill()
                self.effects_sounds['health_up'].play()
                self.player.add_health(HEALTH_PACK_AMOUNT)

        # Mobs hits playa'
        hits = pygame.sprite.spritecollide(self.player, self.mobs, False, collide_hit_rect)
        for hit in hits:
            if random() < 0.7:
                choice(self.player_hit_sounds).play()
            self.player.health -= MOB_DAMAGE
            hit.vel = vec(0, 0)
            if self.player.health <= 0:
                self.playing = False
        if hits:
            # Rotate whatever mob hit us
            self.player.pos += vec(MOB_KNOCKBACK, 0).rotate(-hits[0].rot)
        # Bullets hit mobs
        hits = pygame.sprite.groupcollide(self.mobs, self.bullets, False, True)
        for hit in hits:
            hit.health -= BULLET_DAMAGE
            # When we hit, we stop it
            hit.vel = vec(0, 0)

    def draw_grid(self):
        for x in range(0, WIDTH, TILESIZE):
            pygame.draw.line(self.screen, settings.LIGHTGREY, (x, 0), (x, HEIGHT))
        for y in range(0, HEIGHT, TILESIZE):
            pygame.draw.line(self.screen, settings.LIGHTGREY, (0, y), (WIDTH, y))

    def draw(self):
        # Display FPS
        pygame.display.set_caption("{:.2f}".format(self.clock.get_fps()))
        #self.screen.fill(settings.BGCOLOR)
        self.screen.blit(self.map_img, self.camera.apply_rect(self.map_rect))
        #self.draw_grid()
        for sprite in self.all_sprites:
            # If it is a Mob, then draw its health
            if isinstance(sprite, Mob):
                sprite.draw_health()
            #self.screen.blit(sprite.image, sprite.rect)
            self.screen.blit(sprite.image, self.camera.apply(sprite))
            if self.draw_debug:
                pygame.draw.rect(self.screen, CYAN, self.camera.apply_rect(sprite.hit_rect), 1)
        if self.draw_debug:
            for wall in self.walls:
                pygame.draw.rect(self.screen, CYAN, self.camera.apply_rect(wall.rect), 1)

        #used to draw rect showing additional helpful info
        #pygame.draw.rect(self.screen, settings.WHITE, self.player.hit_rect, 2)

        # HUD functions
        draw_player_health(self.screen, 10, 10, self.player.health / PLAYER_HEALTH)

        pygame.display.flip()

    def events(self):
        # catch all events here
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                self.quit()
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_h:
                    self.draw_debug = not self.draw_debug

    def show_start_screen(self):
        pass

    def show_go_screen(self):
        pass

# create the game object
g = Game()
g.show_start_screen()
while True:
    g.new()
    g.run()
    g.show_go_screen()


# coding: utf-8

# In[1]:

import imp
import matplotlib.pyplot as plt
import numpy as np
ctrl = imp.load_package('control', r'C:\Users\gonmo\PycharmProjects\python-control-GDM\control')

# # Simple example that is not OK

# In[2]:

sys1 = ctrl.tf([1, 1], [1, 10, 1])
print(sys1)

# In[3]:

r, k = ctrl.rlocus(sys1)
plt.show()
r, k = ctrl.rlocus(sys1, grid=True)

# However, when I plot the grid the figure looks not so good. The problem are the axis limits. If I redefine the y-axis limits the problem is solved.

# In[4]:

r, k = ctrl.rlocus(sys1, grid=True, ylim=[-10, 10])

# # Excercises from Franklyn

# I gonna plot several root locus in order to test the quality of the plots.

# In[5]:

sys1 = ctrl.tf([1,2],np.poly([0,-1,-5,-10]))
r1,k1=ctrl.rlocus(sys1)
plt.show()
r1,k1=ctrl.rlocus(sys1, grid = True)

# In[6]:

sys2 = ctrl.tf([1],np.poly([0,-1,-5,-10]))
r2,k2=ctrl.rlocus(sys2)
plt.show()
r2,k2=ctrl.rlocus(sys2, grid = True)

# In[7]:

sys3 = ctrl.tf(np.poly([-2,-6]),np.poly([0,-1,-5,-10]))
r3,k3=ctrl.rlocus(sys3)
plt.show()
r3,k3=ctrl.rlocus(sys3, grid = True)

# In[8]:

sys4 = ctrl.tf(np.poly([-2,-4]),np.poly([0,-1,-5,-10]))
r4,k4=ctrl.rlocus(sys4)
plt.show()
r4,k4=ctrl.rlocus(sys4, grid = True)

# In[9]:

sys5 = ctrl.tf([1],np.poly(np.roots([1,3,10])))
r5,k5=ctrl.rlocus(sys5)
plt.show()
r5,k5=ctrl.rlocus(sys5, grid = True)

# Here the plot show problems. This is beacuase the axis limits are not similar.
# I can resolve his problem redefinig the x limits axis.

# In[10]:

r5,k5=ctrl.rlocus(sys5, grid = True, xlim=[-5,0.1])

# In[11]:

sys6 = ctrl.tf([1],np.poly(np.hstack((np.roots([1,3,10]),[0]))))
r6,k6=ctrl.rlocus(sys6)
plt.show()
r6,k6=ctrl.rlocus(sys6, grid = True)

# In[12]:

sys7 = ctrl.tf(np.poly(np.roots([1,2,8])),np.poly(np.hstack((np.roots([1,2,10]),[0]))))
r7,k7=ctrl.rlocus(sys7)
plt.show()
r7,k7=ctrl.rlocus(sys7, grid = True)

# In[13]:

sys8 = ctrl.tf([1,0,1],[1,0,4,0])
r8,k8=ctrl.rlocus(sys8)
plt.show()
r8,k8=ctrl.rlocus(sys8, grid = True)

# In[14]:

sys9 = ctrl.tf(np.poly(np.roots([1,2,12])),np.poly(np.hstack((np.roots([1,2,10]),[0]))))
r9,k9=ctrl.rlocus(sys9)
plt.show()
r9,k9=ctrl.rlocus(sys9, grid = True)

# In[15]:

sys10 = ctrl.tf([1,0,4],[1,0,1,0])
r10,k10=ctrl.rlocus(sys10)
plt.show()
r10,k10=ctrl.rlocus(sys10, grid = True)

# # Simple examples that can cause problems

# In[16]:

sys11 = ctrl.tf([1, 0, -4], [1, 0, -1])
r11,k11=ctrl.rlocus(sys11)
plt.show()
r11,k11=ctrl.rlocus(sys11, grid = True)

# In[17]:

sys12 = ctrl.tf([1, 1], [1, 0])
r12,k12=ctrl.rlocus(sys12)
plt.show()
r12,k12=ctrl.rlocus(sys12, grid = True)


import pandas as pd

from matplotlib import pyplot as p

y=[2,34,5,67,4,3]
x=[2001,2002,2003,2004,2005,2006]

p.plot(x,y)
#p.plot(y,x)
p.title('data')
p.xlabel('food corps')
p.ylabel('years')
p.show()


from flask import Flask
app = Flask(__name__)

@app.route('/')
def main():
    head = open("index0.html").read()
    body = open("laser_macaroni.html").read()
    foot = open("index1.html").read()
    return head+body+foot

if __name__ == "__main__":
    app.run()


from data import DICTIONARY, LETTER_SCORES
import pytest

def load_words():
    """Loads Dictionary into List"""
    return open(DICTIONARY,'r').read().split('\n')

def calc_word_value(word):
    """Returns the Scrabble value of any string
    str => int"""
    value = 0
    for letter in list(word):
        try:
            value += LETTER_SCORES[letter.upper()]
        except(KeyError):
            pass
    return value

def max_word_value(word_list=[]):
    max_word = ["",0]
    if not word_list:
        word_list = load_words()

    for word in word_list:
        value = calc_word_value(word)
        if value > max_word[1]:
            max_word = [word, value]
    return max_word[0]

class test(object):

    def test_load_words(self):
        assert type(load_words()) == list

    def test_calc_word_value(self):
        assert calc_word_value("Green") == 6
        assert calc_word_value("zebra") == 16

    def test_max_word_value(self):
        assert max_word_value() == 'benzalphenylhydrazone'

print(max_word_value(load_words()))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon May 17 16:18:12 2021

@author: konstantinspuler
"""
import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
from sklearn.cluster import KMeans
import matplotlib.ticker as ticker  # import a special package
from sklearn.model_selection import train_test_split
from sklearn import metrics
import plotly.express as px
import plotly.graph_objects as go
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import silhouette_score
from sklearn import  linear_model
from sklearn.feature_selection import RFE
from sklearn.linear_model import LinearRegression
from scipy import stats

#to avoid future warnings:
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)

#___________________________________________________________________________________________
# DATA CLEANING

# Import data files

nt17_raw = pd.read_csv('IST_North_Tower_2017_Ene_Cons.csv')
nt18_raw = pd.read_csv('IST_North_Tower_2018_Ene_Cons.csv')
meteo_data_raw = pd.read_csv('IST_meteo_data_2017_2018_2019.csv')
holidays_pt_raw = pd.read_csv('holiday_17_18_19.csv')

#Transform the date into datetime object and make the date the index:

holidays_pt_raw.Date = pd.to_datetime(holidays_pt_raw.Date)

nt17_raw.rename(columns = {'Date_start': 'datetime'}, inplace = True)
nt17_raw.datetime = pd.to_datetime(nt17_raw.datetime, format='%d-%m-%Y %H:%M')
#nt17_raw['Date'] = nt17_raw.datetime.dt.date
nt17_raw = nt17_raw.set_index('datetime', drop = True)

nt18_raw.rename(columns = {'Date_start': 'datetime'}, inplace = True)
nt18_raw.datetime = pd.to_datetime(nt18_raw.datetime, format='%d-%m-%Y %H:%M')
#nt18_raw['Date'] = nt18_raw.datetime.dt.date
nt18_raw = nt18_raw.set_index('datetime', drop = True)

meteo_data_raw.rename(columns = {'yyyy-mm-dd hh:mm:ss': 'datetime'}, inplace = True) #renames the 'yyyy-mm-dd hh:mm:ss' column
meteo_data_raw.datetime = pd.to_datetime(meteo_data_raw.datetime) #transforming datetime column to datetime object
meteo_data_raw['Date'] = meteo_data_raw.datetime.dt.date #adds column than includes just the date
meteo_data_raw.Date = pd.to_datetime(meteo_data_raw.Date) #transforming the Date column to datetime object
meteo_data_raw['Holiday'] = meteo_data_raw['Date'].isin(holidays_pt_raw['Date']).astype(float) #adds Holiday column marking the holidays with a 1
meteo_data_raw = meteo_data_raw.set_index('datetime', drop = True) #makes datetime column the index
meteo_data_raw['Week Day']=meteo_data_raw.index.dayofweek
meteo_data_raw['Hour']=meteo_data_raw.index.hour
meteo_data_raw = meteo_data_raw.resample('H').mean().dropna(how='all') #takes the hourly average of the different paramters and drops all empty columns
meteo_data_raw['Date']=meteo_data_raw.index.date

#meteo_data_clean = meteo_data_raw.iloc[:15386] #2134 data rows missing
meteo_data_clean = meteo_data_raw

nt17_meteo_clean = pd.merge(meteo_data_clean,nt17_raw, on='datetime',how = 'inner')
nt18_meteo_clean = pd.merge(meteo_data_clean,nt18_raw, on='datetime',how = 'inner')
nt17_meteo_clean.rename(columns = {'temp_C': 'Temperature [°C]', 'HR': 'Relative Humidity [%]', 'windSpeed_m/s': 'Wind Speed [m/s]', 'windGust_m/s': 'Wind Gust [m/s]', 'pres_mbar': 'Ambient Pressure [mbar]', 'solarRad_W/m2': 'Solar Radiation [W/m^2]', 'rain_mm/h': 'Precipitation [mm/h]', 'rain_day': 'Rain Day', 'Power_kW': 'Power [kW]'}, inplace = True)
nt18_meteo_clean.rename(columns = {'temp_C': 'Temperature [°C]', 'HR': 'Relative Humidity [%]', 'windSpeed_m/s': 'Wind Speed [m/s]', 'windGust_m/s': 'Wind Gust [m/s]', 'pres_mbar': 'Ambient Pressure [mbar]', 'solarRad_W/m2': 'Solar Radiation [W/m^2]', 'rain_mm/h': 'Precipitation [mm/h]', 'rain_day': 'Rain Day', 'Power_kW': 'Power [kW]'}, inplace = True)

nt17_18_meteo_clean = pd.concat([nt17_meteo_clean, nt18_meteo_clean], ignore_index=False)

#___________________________________________________________________________________________
## Data Analysis

#plt.plot(nt17_18_meteo_clean['Solar Radiation [W/m^2]'])

#fig, ax = plt.subplots() # create objects of the plot (figure and plot inside)
#fig.set_size_inches(20,10) # define figure size

#ax.xaxis.set_major_locator (ticker.MultipleLocator(60)) # define the interval between ticks on x axis
    # Try changing (the number) to see what it does
#ax.xaxis.set_tick_params (which = 'major', pad = 5, labelrotation = 50)
    # parameters of major labels of x axis: pad = distance to the axis;
    # label rotation = angle of label text (in degrees)

#fig, ax = plt.subplots() # create objects of the plot (figure and plot inside)
#fig.set_size_inches(20,10) # define figure size

#ax.xaxis.set_major_locator (ticker.MultipleLocator(60)) # define the interval between ticks on x axis
    # Try changing (the number) to see what it does
#ax.xaxis.set_tick_params (which = 'major', pad = 5, labelrotation = 50)
    # parameters of major labels of x axis: pad = distance to the axis;
    # label rotation = angle of label text (in degrees)

## Removing outliers: calculate Zcore
z17 = np.abs(stats.zscore(nt17_18_meteo_clean['Power [kW]']))

threshold = 3 # 3 sigma...Includes 99.7% of the data

nt17_18_meteo_clean2=nt17_18_meteo_clean[(z17 < 3)]

## Removing outliers: calculate IQR
Q1 = nt17_18_meteo_clean['Power [kW]'].quantile(0.25)
Q3 = nt17_18_meteo_clean['Power [kW]'].quantile(0.75)
IQR = Q3 - Q1

nt17_18_meteo_clean3 = nt17_18_meteo_clean[((nt17_18_meteo_clean['Power [kW]'] > (Q1 - 1.5 * IQR)) & (nt17_18_meteo_clean['Power [kW]'] < (Q3 + 1.5 * IQR)))]
#### Clean datafrom outliers from EDA

nt17_18_meteo_clean4 = nt17_18_meteo_clean[nt17_18_meteo_clean['Power [kW]'] > nt17_18_meteo_clean['Power [kW]'].quantile(0.05)]
#I set quantile(0.05), bc 0.25 cleaned too much..

# Plots:
    #Power plots (raw)
fig_17_18_P = px.scatter(nt17_18_meteo_clean, x = 'Date', y='Power [kW]', color_discrete_sequence = ["blue"], width=1600, height=500)
fig_17_P = px.scatter(nt17_meteo_clean, x = 'Date', y='Power [kW]', color_discrete_sequence = ["blue"], width=1600, height=500)
fig_18_P = px.scatter(nt18_meteo_clean, x = 'Date', y='Power [kW]', color_discrete_sequence = ["blue"], width=1600, height=500)
    #Power plot 17+18 (outliers removed)
fig_17_18_P_clean = px.scatter(nt17_18_meteo_clean4, x = 'Date', y='Power [kW]', color_discrete_sequence = ["blue"], width=1600, height=500)
    #Temperature plots
fig_17_18_Temp = px.scatter(nt17_18_meteo_clean, x = 'Date', y='Temperature [°C]', color_discrete_sequence = ["blue"], width=1600, height=500)
fig_17_Temp = px.scatter(nt17_meteo_clean, x = 'Date', y='Temperature [°C]', color_discrete_sequence = ["blue"], width=1600, height=500)
fig_18_Temp = px.scatter(nt18_meteo_clean, x = 'Date', y='Temperature [°C]', color_discrete_sequence = ["blue"], width=1600, height=500)
    #Solar radiation plots
fig_17_18_SR = px.scatter(nt17_18_meteo_clean, x = 'Date', y='Solar Radiation [W/m^2]', color_discrete_sequence = ["blue"], width=1600, height=500)
fig_17_SR = px.scatter(nt17_meteo_clean, x = 'Date', y='Solar Radiation [W/m^2]', color_discrete_sequence = ["blue"], width=1600, height=500)
fig_18_SR = px.scatter(nt18_meteo_clean, x = 'Date', y='Solar Radiation [W/m^2]', color_discrete_sequence = ["blue"], width=1600, height=500)

#___________________________________________________________________________________________
# DATA CLUSTERING

nt17_18_meteo_cluster = nt17_18_meteo_clean4.set_index('Date', drop = True)
#removing all unwanted columns
nt17_18_meteo_cluster = nt17_18_meteo_cluster.drop(columns=['Relative Humidity [%]','Wind Speed [m/s]', 'Wind Gust [m/s]', 'Ambient Pressure [mbar]', 'Solar Radiation [W/m^2]', 'Precipitation [mm/h]', 'Rain Day'])
# create kmeans object
model = KMeans(n_clusters=4).fit(nt17_18_meteo_cluster)
pred = model.labels_

Nc = range(1, 20)
kmeans = [KMeans(n_clusters=i) for i in Nc]
score = [kmeans[i].fit(nt17_18_meteo_cluster).score(nt17_18_meteo_cluster) for i in range(len(kmeans))]

nt17_18_meteo_cluster['Cluster']=pred
nt17_18_meteo_cluster['Cluster'] = nt17_18_meteo_cluster['Cluster'].astype(str)
ax1=px.scatter(nt17_18_meteo_cluster, x='Power [kW]',y='Week Day', color='Cluster', color_discrete_map = {"0":"red","1":"green","2":"blue", "3":"orange"}, width=550, height=550)
ax2=px.scatter(nt17_18_meteo_cluster, x='Power [kW]',y='Temperature [°C]', color='Cluster', color_discrete_map = {"0":"red","1":"green","2":"blue", "3":"orange"}, width=550, height=550)
ax3=px.scatter(nt17_18_meteo_cluster, x='Power [kW]',y='Hour', color='Cluster', color_discrete_map = {"0":"red","1":"green","2":"blue", "3":"orange"}, width=550, height=550)
ax4=px.scatter(nt17_18_meteo_cluster, x='Power [kW]',y='Holiday', color='Cluster', color_discrete_map = {"0":"red","1":"green","2":"blue", "3":"orange"}, width=550, height=550)

#3D plots Hour and Day of the week:
cluster0=nt17_18_meteo_cluster[pred==0]
cluster1=nt17_18_meteo_cluster[pred==1]
cluster2=nt17_18_meteo_cluster[pred==2]
cluster3=nt17_18_meteo_cluster[pred==3]
fig_3D_THP = px.scatter_3d(nt17_18_meteo_cluster, x='Temperature [°C]', y='Hour', z='Power [kW]', color='Cluster', color_discrete_map = {"0":"red","1":"green","2":"blue", "3":"orange"}, width=800, height=550)
fig_3D_WTP = px.scatter_3d(nt17_18_meteo_cluster, x='Week Day', y='Temperature [°C]', z='Power [kW]', color='Cluster', color_discrete_map = {"0":"red","1":"green","2":"blue", "3":"orange"}, width=800, height=550)

### Step 5: Identifying daily patterns

#https://towardsdatascience.com/clustering-electricity-profiles-with-k-means-42d6d0644d00
df = nt17_18_meteo_cluster
df = df.drop(columns=['Temperature [°C]','Week Day','Cluster','Holiday'])
df.rename(columns = {'Power [kW]': 'Power'}, inplace = True)

#Create a pivot table
df_pivot = df.pivot(columns='Hour')
df_pivot = df_pivot.dropna()
#df_pivot
#df_pivot.T.plot(figsize=(13,8), legend=False, color='blue', alpha=0.09)

sillhoute_scores = []
n_cluster_list = np.arange(2,10).astype(int)

X = df_pivot.values.copy()

# Very important to scale!
sc = MinMaxScaler()
X = sc.fit_transform(X)

for n_cluster in n_cluster_list:

    kmeans = KMeans(n_clusters=n_cluster)
    cluster_found = kmeans.fit_predict(X)
    sillhoute_scores.append(silhouette_score(X, kmeans.labels_))

#plt.plot(n_cluster_list,sillhoute_scores)

kmeans = KMeans(n_clusters=3)
cluster_found = kmeans.fit_predict(X)
cluster_found_sr = pd.Series(cluster_found, name='cluster')
df_pivot = df_pivot.set_index(cluster_found_sr, append=True )

#df_pivot
#fig, ax= plt.subplots(1,1, figsize=(18,10))
#color_list = ['blue','red','green']
#cluster_values = sorted(df_pivot.index.get_level_values('cluster').unique())
#for cluster, color in zip(cluster_values, color_list):
#    df_pivot.xs(cluster, level=1).T.plot(
#        ax=ax, legend=False, alpha=0.01, color=color, label= f'Cluster {cluster}'
#       )
#   df_pivot.xs(cluster, level=1).median().plot(
#       ax=ax, color=color, alpha=0.9, ls='--'
#    )

#ax.set_xticks(np.arange(1,25))
#ax.set_ylabel('kilowatts')
#ax.set_xlabel('hour')

#___________________________________________________________________________________________
# FEATURE SELECTION AND ENGINEERING

nt17_18_meteo_feature = nt17_18_meteo_cluster.drop(columns=['Cluster'])

nt17_18_meteo_feature['Power (-1h)'] = nt17_18_meteo_feature['Power [kW]'].shift(1)
# adding a column with the previous hour consumption

nt17_18_meteo_feature = nt17_18_meteo_feature.dropna()

# Define input and outputs
X=nt17_18_meteo_feature.values

Y=X[:,4]
X=X[:,[0,1,2,3,5]]

## Feature Selection

### Filter Methods

#### kBest

from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import f_regression

features=SelectKBest(k=2,score_func=f_regression) # Test different k number of features, uses f-test ANOVA
fit=features.fit(X,Y) #calculates the f_regression of the features
#print(fit.scores_)
features_results=fit.transform(X)
#print(features_results) # k=2:Power-1 and Hour k=3: Temperature and Power-1 and Hour

## Wrapper methods
### Recursive Feature Elimination (RFE)
model=LinearRegression() # LinearRegression Model as Estimator
rfe=RFE(model,2)# using 2 features
rfe2=RFE(model,3) # using 3 features

fit=rfe.fit(X,Y)
fit2=rfe2.fit(X,Y)

## Emsemble methods

from sklearn.ensemble import RandomForestRegressor

model = RandomForestRegressor()
model.fit(X, Y)
#print(model.feature_importances_)

## Feature Extraction/Engineering

# featuretools for automated feature engineering
#Log of temperature
nt17_18_meteo_feature['logtemp']=np.log(nt17_18_meteo_feature['Temperature [°C]'])
# Weekday square
nt17_18_meteo_feature['day^2']=np.square(nt17_18_meteo_feature['Week Day'])
# Heating degree.hour
nt17_18_meteo_feature['HDH']=np.maximum(0,nt17_18_meteo_feature['Temperature [°C]']-16)
#Not weekday
nt17_18_meteo_feature['type_of_day'] = 1
nt17_18_meteo_feature.loc[(nt17_18_meteo_feature['Holiday'] == 1) | (nt17_18_meteo_feature['Week Day'] >4 ), 'type_of_day'] = 0
#rounding the temperature values
nt17_18_meteo_feature['Temperature [°C]']  = round(nt17_18_meteo_feature['Temperature [°C]'], 2)

# recurrent
X=nt17_18_meteo_feature.values
Y=X[:,4]
X=X[:,[0,1,2,3,5,6,7,8]]

model = RandomForestRegressor()
model.fit(X, Y)

dataFeature = {'Variable':['Temp' ,'Holiday','Week Day', 'Hour' ,'Power(-1)','logtemp', 'day^2' ,'HDH']}
ResultsFeatureSelect = pd.DataFrame(dataFeature)
ResultsFeatureSelect['Feature Importance'] =(model.feature_importances_)

#___________________________________________________________________________________________
# REGRESSION

## Pre-processing
nt17_18_meteo_reg = nt17_18_meteo_feature

# recurrent
X=nt17_18_meteo_reg.values
Y=X[:,4]
X=X[:,[3,5,6,8,9]] #Temp0, Holiday1, Weekday2, Hour3, Power-1 5, Logtemp6, day^2 7, HDH 8, type_of_day9
#I went through all the features and noted the change in results.
#The ones left as the outputs are the ones that had the biggest impact.
#I can provide the list of impacts, if you would like to see them

### Split Data into training and test data
#by default, it chooses randomly 75% of the data for training and 25% for testing
#I didn't modify the partitioning..
X_train, X_test, y_train, y_test = train_test_split(X,Y)

#----------------------------------------
## LINEAR REGRESSION
regr = linear_model.LinearRegression()
regr.fit(X_train,y_train)
y_pred_LR = regr.predict(X_test)

data_lin=pd.DataFrame(y_test[1:200], columns=['Test'])
data_lin['Prediction'] = y_pred_LR[1:200]
fig_lin_reg = px.line(data_lin, color_discrete_map = {"Prediction":"blue", "Test":"red"}, width=700, height=400, labels=dict(x="Time", y="Power [kW]"))
fig_lin_scat = px.scatter(x = y_test, y = y_pred_LR, color_discrete_sequence = ["orange"], width=700, height=400, labels=dict(x="Real data", y="Regression Results"))
#Evaluate errors
MAE_LR=metrics.mean_absolute_error(y_test,y_pred_LR)
MSE_LR=metrics.mean_squared_error(y_test,y_pred_LR)
RMSE_LR= np.sqrt(metrics.mean_squared_error(y_test,y_pred_LR))
cvRMSE_LR=RMSE_LR/np.mean(y_test)
data = {'Test':['MAE', 'MSE', 'RMSE','cvRMSE']}
Results = pd.DataFrame(data)
Results['LR']=(MAE_LR,MSE_LR,RMSE_LR,cvRMSE_LR)

#----------------------------------------
#Random forest
from sklearn.ensemble import RandomForestRegressor
parameters = {'bootstrap': True,
              'min_samples_leaf': 3,
              'n_estimators': 200,
              'min_samples_split': 15,
              'max_features': 'sqrt',
              'max_depth': 20,
              'max_leaf_nodes': None}
RF_model = RandomForestRegressor(**parameters)
#RF_model = RandomForestRegressor()
RF_model.fit(X_train, y_train)
y_pred_RF = RF_model.predict(X_test)

data_RF=pd.DataFrame(y_test[1:200], columns=['Test'])
data_RF['Prediction'] = y_pred_RF[1:200]
fig_RF_reg = px.line(data_RF, color_discrete_map = {"Prediction":"blue", "Test":"red"}, width=700, height=400, labels=dict(x="Time", y="Power [kW]"))
fig_RF_scat = px.scatter(x = y_test, y = y_pred_RF, color_discrete_sequence = ["orange"], width=700, height=400, labels=dict(x="Real data", y="Regression Results"))
#Evaluate error
MAE_RF=metrics.mean_absolute_error(y_test,y_pred_RF)
MSE_RF=metrics.mean_squared_error(y_test,y_pred_RF)
RMSE_RF= np.sqrt(metrics.mean_squared_error(y_test,y_pred_RF))
cvRMSE_RF=RMSE_RF/np.mean(y_test)
Results['RF']=(MAE_RF,MSE_RF,RMSE_RF,cvRMSE_RF)

#----------------------------------------
#Gradient Boosting
from sklearn.ensemble import GradientBoostingRegressor

GB_model = GradientBoostingRegressor()
GB_model.fit(X_train, y_train)
y_pred_GB =GB_model.predict(X_test)

data_GB=pd.DataFrame(y_test[1:200], columns=['Test'])
data_GB['Prediction'] = y_pred_GB[1:200]
fig_GB = px.line(data_GB, color_discrete_map = {"Prediction":"blue", "Test":"red"}, width=700, height=400, labels=dict(x="Time", y="Power [kW]"))
fig_GB_scat = px.scatter(x = y_test, y = y_pred_GB, color_discrete_sequence = ["orange"], width=700, height=400, labels=dict(x="Real data", y="Regression Results"))

MAE_GB=metrics.mean_absolute_error(y_test,y_pred_GB)
MSE_GB=metrics.mean_squared_error(y_test,y_pred_GB)
RMSE_GB= np.sqrt(metrics.mean_squared_error(y_test,y_pred_GB))
cvRMSE_GB=RMSE_GB/np.mean(y_test)
Results['GB']=(MAE_GB,MSE_GB,RMSE_GB,cvRMSE_GB)

table_LR = go.Figure(data=[go.Table(header=dict(values=['MAE', 'MSE', 'RMSE','cvRMSE'],line_color='gray',
                fill_color='gray',),
                 cells=dict(values= Results['LR'],line_color='blue',
               fill_color='lightblue',
))
                     ])

table_RF = go.Figure(data=[go.Table(header=dict(values=['MAE', 'MSE', 'RMSE','cvRMSE'],line_color='gray',
                fill_color='gray',),
                 cells=dict(values= Results['RF'],line_color='blue',
               fill_color='lightblue',
))
                     ])

table_GB = go.Figure(data=[go.Table(header=dict(values=['MAE', 'MSE', 'RMSE','cvRMSE'],line_color='gray',
                fill_color='gray',),
                 cells=dict(values= Results['GB'],line_color='blue',
               fill_color='lightblue',
))
                     ])

#-__________________________________________________________________________________

external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']

app = dash.Dash(__name__, external_stylesheets=external_stylesheets)

app.layout = html.Div([
    html.Img(src=app.get_asset_url('IST_logo.png')),
    html.H2('POWER CONSUMPTION FORECAST - NORTH TOWER - IST'),
    dcc.Tabs(id='tabs', value='tab-1', children=[
        dcc.Tab(label='Data', value='tab-1'),
        dcc.Tab(label='Clean Power Data', value='tab-2'),
        dcc.Tab(label='Clustering', value='tab-3'),
        dcc.Tab(label='Feature Selection', value='tab-4'),
        dcc.Tab(label='Regression', value='tab-5'),
        dcc.Tab(label='Results', value='tab-6'),

    ]),
    html.Div(id='tabs-content')
])

def generate_table(dataframe, max_rows=10):
    return html.Table([
        html.Thead(
            html.Tr([html.Th(col) for col in dataframe.columns])
        ),
        html.Tbody([
            html.Tr([
                html.Td(dataframe.iloc[i][col]) for col in dataframe.columns
            ]) for i in range(min(len(dataframe), max_rows))
        ])
    ])

@app.callback(Output('tabs-content', 'children'),
              Input('tabs', 'value'))

def render_content(tab):
    if tab == 'tab-1':
        return html.Div([
            html.H3('Power, Temperature and Solar Radiation 2017 & 2018'),
            dcc.Checklist(
        id='radio_year',
        options=[
            {'label': '2017', 'value': 2017},
            {'label': '2018', 'value': 2018},

        ],

        value=[2017,2018]

        ),

            dcc.Dropdown(
        id='dropdown_measurement',
        options=[
            {'label': 'Power [kW]', 'value': 1},
            {'label': 'Temperature [°C]', 'value': 2},
            {'label': 'Solar Radiation [W/m^2]', 'value': 3},
        ],
        value=1
        ),

        html.Div(id='graphyear_png'),

                    ])

    elif tab == 'tab-2':
       return html.H3('Cleaned Power Data'), html.Div([dcc.Graph(figure=fig_17_18_P_clean),])

    elif tab == 'tab-3':
        return html.Div([
                html.H3('Clusters in the Data'),
                html.Div([
                    html.H4('2D-Clusters'),
                    dcc.Dropdown(
                    id='2D_Cluster',
                    options=[
                        {'label': 'Hour', 'value': 1},
                        {'label': 'Temperature [°C]', 'value': 2},
                        {'label': 'Week Day', 'value': 3},
                        {'label': 'Holiday', 'value': 4},
                        ],
                    value= 1 ),
                    html.Div(id='graph_cluster_2D')
                ], className="four columns"),

                html.Div([
                    html.H4('3D-Clusters'),
                    dcc.Dropdown(
                    id='3D_Cluster',
                    options=[
                        {'label': 'Temperature [°C] and Hour', 'value': 1},
                        {'label': 'Weekday and Temperature [°C]', 'value': 2},
                        ],
                    value= 1 ),
                    html.Div(id='graph_cluster_3D')
                ],className="four columns")
        ], className="row")

    elif tab == 'tab-4':
        return html.Div([
            html.Div([
            html.H3('Table of importance'),
                 generate_table(ResultsFeatureSelect),
        ], className="four columns"),

        html.Div([
            html.H3('Graph'),
            html.Div([html.Img(src=app.get_asset_url('FS.png'), width=1000),])

        ], className="eight columns"),
    ], className="row")

    elif tab == 'tab-5':
        return html.Div([
            html.H3('Results of different Regression methods'),
            dcc.RadioItems(
                id='Regression_method',
                options=[
                    {'label': 'Linear Regression', 'value': 1},
                    {'label': 'Gradient boosting', 'value': 2},
                    {'label': 'Random Forest', 'value': 3}
                ],
                value=1),

        html.Div([
            html.H4('Prediction & Test'),
                html.Div(id='graph_regression'),
        ],
        style={'width': '48%', 'display': 'inline-block'}),
        html.Div([
            html.H4('Regression Results vs. Real Data'),
                html.Div(id='graph_regression_scatter'),
        ],
        style={'width': '48%', 'display': 'inline-block'}),
        html.Div([
            html.H4('Errors'),
            html.Div(id='table_Errors'),
            ])

        ])

    elif tab == 'tab-6':
        return html.Div([
            generate_table(Results),

    ])

@app.callback(Output('graphyear_png', 'children'),
              Input('radio_year', 'value'),
              Input('dropdown_measurement', 'value'))

def render_figure_png(radio_year, dropdown_measurement):

    if radio_year == [2017]:
        if dropdown_measurement == 1:
            return  html.Div([dcc.Graph(figure=fig_17_P),])
        elif dropdown_measurement == 2:
            return  html.Div([dcc.Graph(figure=fig_17_Temp),])
        elif dropdown_measurement == 3:
            return  html.Div([dcc.Graph(figure=fig_17_SR),])
    elif radio_year == [2018]:
        if dropdown_measurement == 1:
            return  html.Div([dcc.Graph(figure=fig_18_P),])
        elif dropdown_measurement == 2:
            return  html.Div([dcc.Graph(figure=fig_18_Temp),])
        elif dropdown_measurement == 3:
            return  html.Div([dcc.Graph(figure=fig_18_SR),])
    elif radio_year == [2017,2018]:
        if dropdown_measurement == 1:
            return  html.Div([dcc.Graph(figure=fig_17_18_P),])
        elif dropdown_measurement == 2:
            return  html.Div([dcc.Graph(figure=fig_17_18_Temp),])
        elif dropdown_measurement == 3:
            return  html.Div([dcc.Graph(figure=fig_17_18_SR),])
    elif radio_year == [2018,2017]:
        if dropdown_measurement == 1:
            return  html.Div([dcc.Graph(figure=fig_17_18_P),])
        elif dropdown_measurement == 2:
            return  html.Div([dcc.Graph(figure=fig_17_18_Temp),])
        elif dropdown_measurement == 3:
            return  html.Div([dcc.Graph(figure=fig_17_18_SR),])
    elif radio_year == []:
        return html.Div([html.Img(src='assets/NO_OPTION_CHOSEN.png', width=1600)])

@app.callback(Output('graph_cluster_2D', 'children'),
              Input('2D_Cluster', 'value'))

def figure_cluster_2D(Cluster2D):
    if Cluster2D == 1:
        return  html.Div([dcc.Graph(figure=ax3),])
    if Cluster2D == 2:
        return  html.Div([dcc.Graph(figure=ax2),])
    if Cluster2D == 3:
        return  html.Div([dcc.Graph(figure=ax1),])
    if Cluster2D == 4:
        return  html.Div([dcc.Graph(figure=ax4),])

@app.callback(Output('graph_cluster_3D', 'children'),
              Input('3D_Cluster', 'value'))

def figure_cluster_3D(Cluster3D):
    if Cluster3D == 1:
        return html.Div([dcc.Graph(figure=fig_3D_THP),])
    if Cluster3D == 2:
        return html.Div([dcc.Graph(figure=fig_3D_WTP),])

@app.callback(Output('graph_regression', 'children'),
              Input('Regression_method', 'value'))

def figure_regression(Regression_method):
    if Regression_method == 1:
        return html.Div([dcc.Graph(figure=fig_lin_reg),])
    if Regression_method == 2:
        return html.Div([dcc.Graph(figure=fig_GB),])
    if Regression_method == 3:
        return html.Div([dcc.Graph(figure=fig_RF_reg),])

@app.callback(Output('graph_regression_scatter', 'children'),
              Input('Regression_method', 'value'))
def scatter_regression(Regression_method):
    if Regression_method == 1:
        return html.Div([dcc.Graph(figure=fig_lin_scat),])
    if Regression_method == 2:
        return html.Div([dcc.Graph(figure=fig_GB_scat),])
    if Regression_method == 3:
        return html.Div([dcc.Graph(figure=fig_RF_scat),])

@app.callback(Output('table_Errors', 'children'),
              Input('Regression_method', 'value'))
def table_Regression_error(Regression_method):
    if Regression_method == 1:
        return html.Div([dcc.Graph(figure=table_LR),])
    if Regression_method == 2:
        return html.Div([dcc.Graph(figure=table_GB),])
    if Regression_method == 3:
        return html.Div([dcc.Graph(figure=table_RF),])

if __name__ == '__main__':
    app.run_server(debug=True)


import os
import sqlite3
import re
from subprocess import Popen, check_output

def init_db(db_file):
    conn = sqlite3.connect(db_file)
    c = conn.cursor()
    c.execute("CREATE TABLE models (name string primary key, script text, description string)")
    c.execute("CREATE TABLE trainings (label string primary key, status string, train_dir string, tensorboard integer, submit_at text, start_at text, stop_at text, num_gpu integer)")
    conn.commit()
    conn.close()

def deploy_web():
    db_file = os.environ.get('FLASK_DB')
    if not os.path.isfile(db_file):
        print("Database is not setup, setting up now")
        print(db_file)
        init_db(db_file)

    cmd = "ps -u ai u"
    res = check_output(cmd.split()).decode('ascii').split('\n')
    for line in res:
        if re.search('gunicorn', line):
            pid = line.split()[1]
            os.system('kill {}'.format(pid))
            print('Kill flask process with PID {}'.format(pid))

    cwd = os.getcwd()
    flask_dir = os.path.dirname(os.path.realpath(__file__)) + '/../../../webapp'
    os.chdir(flask_dir)
    cmd = 'gunicorn main:app -b 0.0.0.0:{} -w 4'.format(os.environ.get('FLASK_PORT'))
    Popen(cmd.split())
    os.chdir(cwd)


from django.shortcuts import render, redirect, get_object_or_404
from django.utils import timezone
# from django.http import Http404

from .models import Post
from .forms import PostForm, UpdateForm

# Create your views here.

def board_list(request):
    template = 'hpblog/board_list.html'

    post = Post.objects.order_by('-id')
    context = {'post': post}

    return render(request, template, context)

def board_item(request, p_id):
    template = 'hpblog/board_item.html'

    item = Post.objects.get(id=p_id)

    context = {'item': item}

    return render(request, template, context)

def board_insert(request):
    template = 'hpblog/board_insert.html'

    form = PostForm(request.POST or None)

    if form.is_valid():
        form.save()
        return redirect('board_list')
    else:
        form = PostForm()

    context = {'form': form}

    return render(request, template, context)

def board_update(request, p_id):
    template = 'hpblog/board_update.html'

#     item = Post.objects.get(id=p_id)
    item = get_object_or_404(Post, id=p_id)

    if request.method == "POST":
        form = UpdateForm(request.POST, instance=item)

        if form.is_valid():
            item = form.save(commit=False)
#             item.title = request.title
            print('>>> req: {}'.format(request.user))
            item.published_date = timezone.now()
            item.save()

            return redirect('board_list')
    else:
        form = UpdateForm(instance=item)

    context = {
        'form': form,
        'item' : item,
    }

    return render(request, template, context)

def board_delete(request, p_id):
    template = 'hpblog/board_delete.html'

#     item = Post.objects.get(id=p_id)
    item = get_object_or_404(Post, id=p_id)
    item.delete()

    context = {'item': p_id}

    return render(request, template, context)