from flask import Flask,render_template,url_for,redirect,request,jsonify
from flask_mail import Mail,Message
import Text_summarization
from werkzeug.utils import secure_filename
# from allennlp.predictors.predictor import Predictor
from summarizer import Summarizer
import os
import re
import docx
import PyPDF2
from striprtf.striprtf import rtf_to_text
from pptx import Presentation
import json

# Make a regular expression
# for validating an Email
regex = "^[a-z0-9]+[\\._]?[a-z0-9]+[@]\\w+[.]\\w{2,3}$"

#app configurations
application=app=Flask(__name__)
app.config['UPLOAD_FOLDER']= 'uploads/'
app.config['MAIL_SERVER']='smtp.gmail.com'
app.config['MAIL_PORT']=587
app.config['MAIL_USE_TLS']=True
app.config['MAIL_USE_SSL']=False
app.config['MAIL_USERNAME']='aashima0809@gmail.com'
app.config['MAIL_PASSWORD']=os.environ.get('password')
app.config['MAIL_DEFAULT_SENDER']='aashima0809@gmail.com'

mail=Mail(app)

text=''
text_summary=''
pred_summary=''
name=''

# predictor = Predictor.from_path("static/bidaf-model-2020.03.19.tar.gz")

def readfile(flag):
	global text
	global text_summary
	global name
	File=''

	#loads file from
	if(flag==0):
		File=request.files['filename']
	else:
		File=request.files['file_summary']

	print(File)

	#contains filename
	fn=os.path.basename(File.filename)
	fn=fn.replace(" ", "_")
	print(fn)

	#saving file in specified directory
	File.save(os.path.join(app.config['UPLOAD_FOLDER'],secure_filename(File.filename)))

	#contains path where file is saved
	File=str(os.path.join(app.config['UPLOAD_FOLDER'])+fn)

	#dividing the file location into its filename and extension
	name, ext =os.path.splitext(File)

	#checking whether the file is videofile or not
	allowed_extensions=[".doc",".docx",".pdf",".rtf",".pptx",".txt"]
	if (ext.lower() in allowed_extensions)==False:
		return "extension_error"

	if(ext.lower()=='.txt'):
		try:
			f = open(File, "r")
			data = f.read()
		except Exception:
			return "Something_went_wrong"

	if(ext.lower()=='.docx' or ext.lower()=='.doc'):
		try:
			doc = docx.Document(File)  # Creating word reader object.
			data = ""
			fullText = []
			for para in doc.paragraphs:
				fullText.append(para.text)
				data = '\n'.join(fullText)
		except Exception:
			return "Something_went_wrong"

	if(ext.lower()=='.pptx'):
		try:
			prs = Presentation(File)
			data = ""
			fullText = []
			for slide in prs.slides:
				for shape in slide.shapes:
					if hasattr(shape, "text"):
						fullText.append(shape.text)
				data = '\n'.join(fullText)
		except Exception:
			return "Something_went_wrong"

	if(ext.lower()=='.pdf'):
		try:
			pdfFileObj = open(File, mode='rb')    # creating a pdf file object
			pdfReader = PyPDF2.PdfFileReader(pdfFileObj)    # creating a pdf reader object
			number_of_pages=pdfReader.numPages    #counting number of pages
			data=""
			fullText=[]
			for i in range(0,number_of_pages):
				pageObj = pdfReader.getPage(i)    # a page object
				fullText.append(pageObj.extractText())   # extracting text from pdf
			data=' '.join(fullText)
		except Exception:
			return "Something_went_wrong"

	if(ext.lower()=='.rtf'):
		try:
			f = open(File, 'r')
			rtf_text=f.read()
			data = rtf_to_text(rtf_text)
		except Exception:
			return "Something_went_wrong"

	# flag=0 -> Original_File
	# flag=1 -> Summary_File
	if(flag==0):
		text=data
	else:
		text_summary=data
	return "successfully_read"

@app.route('/',methods=["GET","POST"])
def dashboard():
    return render_template('index.html')

@app.route('/email',methods=["GET","POST"])
def email():
	#retrieves email ID from html
	global pred_summary
	global name

	recipient_id=str(request.form.get("email"))

	#if invalid email is provided
	if not(re.search(regex,recipient_id)):
		return render_template("summarize.html",text="Invalid Email Id")

	name=name[8:]
	print(name)
	#Summary file name
	Summary_file="summary_"+name+".txt"
	file1 = open(Summary_file, "w+")
	file1.write(pred_summary)
	file1.close()

	try:
		#Message Content that is to be emailed
		msg=Message(
			subject='Summary file',
			recipients=[recipient_id],
			body='Below is your required summary'
			)

		#attaching summary file
		with app.open_resource(Summary_file) as output_file:
			msg.attach(Summary_file,'text/plain',output_file.read())

		#sending mail
		mail.send(msg)

		return render_template("summarize.html",text="Successfully mailed summary to your provided email id")

	except Exception:
		return render_template("summarize.html",text="Sorry!!! Unable to send the email")

@app.route('/summarize',methods=["GET","POST"])
def summarize():
	global pred_summary

	if request.method=="POST":
		radio_butt=request.form.get("optradio")
		print(radio_butt)
		output=readfile(0)
		if(output=="extension_error"):
			return render_template("summarize.html",summary_output="Not a text file")

		if(output=="Something_went_wrong"):
			return render_template("summarize.html",summary_output="Sorry!!! Something_went_wrong")

		if(radio_butt=="bert"):
			model = Summarizer()
			pred_summary = model(text)
			return render_template("summarize.html",summary_output=pred_summary)
		else:
			pred_summary=Text_summarization.get_data(text,'',0)
			return render_template("summarize.html",summary_output=pred_summary)

	return render_template('summarize.html')

@app.route('/analyse',methods=["GET","POST"])
def analyse():
	global text
	global text_summary
	if request.method=="POST":

		output_text=readfile(0)
		output_summary=readfile(1)

		#if original file is not a text file
		if(output_text=="extension_error"):
			return render_template("analyse.html",analyse_output="Original file is not a text file")

		if(output_summary=="extension_error"):
			return render_template("analyse.html",analyse_output="summary file is not a text file")

		if(output_text=="Something_went_wrong" or output_summary=="Something_went_wrong"):
			return render_template("analyse.html",analyse_output="Sorry!!! Something_went_wrong")

		accuracy=Text_summarization.get_data(text,text_summary,1)
		return render_template("analyse.html",analyse_output=accuracy)

	return render_template('analyse.html')

@app.route('/quesans',methods=["GET","POST"])
def quesans():
	return render_template('quesans.html')

@app.route('/file',methods=["POST"])
def file():
	global text

	output=readfile(0)

	if(output=="extension_error"):
		return render_template("quesans.html",qa_output="Not a text file")

	if(output=="successfully_read"):
		return render_template("quesans.html",qa_output="File uploaded successfully")

	if(output=="Something_went_wrong"):
		return render_template("quesans.html",qa_output="Sorry!!! Something_went_wrong")

# @app.route('/question',methods=["GET","POST"])
# def question():
# 	global text
# 	question=str(request.form.get('comment'))
# 	try:
# 		result=predictor.predict(
# 		  	passage=text,
# 	        question=question
# 		)
# 		answer=result['best_span_str']
# 		return render_template("quesans.html",question=question,answer=answer)
# 	except Exception:
# 		return render_template("quesans.html",answer="Sorry!!! Currently application is out of service")

@app.route('/MailMe', methods=['POST', 'GET'])
def MailMe():
	data= request.get_json(force = True)
	name=data.get("name")
	email=data.get("email")
	phone=data.get("phone")
	message=data.get("message")
	try:
		msg=Message(
			subject='Text Summarization Website Contact Form: %s'%(name),
			recipients=['aadarshgupta875@gmail.com'],
			body="You have received a new message from your website contact form.\n\n. Here are the details:\n\nName: {}\n\nEmail: {}\n\nPhone: {}\n\nMessage:\n{}".format(name,email,phone,message)
		)
		mail.send(msg)
		return json.dumps({'success':True})
	except Exception:
		return json.dumps({'error':True})

if __name__=="__main__":
	app.run()


from collections import deque

def solution(queue1, queue2):
    nq1 = deque(queue1)
    nq2 = deque(queue2)
    s1 = sum(nq1)
    s2 = sum(nq2)
    cnt = 0

    # len*2 +2 인 이유
    # 한쪽에 값이 쏠려있는 형태가 대표적. 예시) [1,1,1,8,10,9] [1,1,1,1,1,1] (14회 확인필요)
    for i in range(len(queue1)*2 + 2):
        if s1 == s2:
            break

        if s1 > s2:
            temp = nq1.popleft()
            nq2.append(temp)
            s1 -= temp
            s2 += temp
            cnt += 1
        else:
            temp = nq2.popleft()
            nq1.append(temp)
            s1 += temp
            s2 -= temp
            cnt += 1

    if sum(nq1) != sum(nq2):
        cnt = -1

    return cnt

Programmers/2022 KAKAO TECH INTERNSHIP/118667.py


# -*- coding: utf-8 -*-
"""
Created on Mon May 20 14:57:18 2019

@author: Ethan
"""

import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.font_manager as fm
import matplotlib.cm as cmx
import seaborn as sns
import pandas as pd
import statsmodels.api as sm
import el_load_global_plants
import pickle, gzip
import sys, os

import warnings
warnings.filterwarnings('ignore')

#matplotlib.rcParams['font.family'] = 'Helvetica'
#matplotlib.rcParams['font.weight'] = 'normal'

#dataDir = '/dartfs-hpc/rc/lab/C/CMIG'
#dataDir = 'e:/data/'
dataDirDiscovery = '/dartfs-hpc/rc/lab/C/CMIG/ecoffel/data/projects/electricity'

plotFigs = True

rebuild = True

models = ['bcc-csm1-1-m', 'canesm2', \
              'ccsm4', 'cesm1-bgc', 'cesm1-cam5', 'cnrm-cm5', 'csiro-mk3-6-0', \
              'gfdl-esm2g', 'gfdl-esm2m', \
              'inmcm4', 'miroc5', 'miroc-esm', \
              'mpi-esm-mr', 'mri-cgcm3', 'noresm1-m']

# models = [sys.argv[1]]

#grdc or gldas
runoffData = 'grdc'

# world, useu, or entsoe-nuke
plantData = 'useu'

modelPower = 'pow2-noInteraction'

if plantData == 'useu':
    globalPlants = el_load_global_plants.loadGlobalPlants(world=False)
elif plantData == 'world':
    globalPlants = el_load_global_plants.loadGlobalPlants(world=True)

with open('%s/script-data/active-pp-inds-40-%s.dat'%(dataDirDiscovery, plantData), 'rb') as f:
    livingPlantsInds40 = pickle.load(f)

# load temp-runoff outage models
pcModel10 = []
pcModel50 = []
pcModel90 = []
plantIds = []
plantYears = []
with gzip.open('%s/script-data/pPolyData-%s-%s.dat'%(dataDirDiscovery, runoffData, modelPower), 'rb') as f:
    pPolyData = pickle.load(f)
    # these are mislabeled in dict for now (90 is 10, 10 is 90)
    pcModel10 = pPolyData['pcModel10'][0]
    pcModel50 = pPolyData['pcModel50'][0]
    pcModel90 = pPolyData['pcModel90'][0]
    plantIds = pPolyData['plantIds']
    plantYears = pPolyData['plantYears']
    plantCooling = pPolyData['plantCooling']
    plantFuel = pPolyData['plantFuel']
    plantAge = pPolyData['plantAge']

yearRange = [1981, 2018]

monthLen = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]

baseTx = 27
baseQs = 0

dfpred = pd.DataFrame({'T1':[baseTx]*len(plantIds), 'T2':[baseTx**2]*len(plantIds), \
                         'QS1':[baseQs]*len(plantIds), 'QS2':[baseQs**2]*len(plantIds), \
                         'QST':[baseTx*baseQs]*len(plantIds), 'QS2T2':[(baseTx**2)*(baseQs**2)]*len(plantIds), \
                         'PlantIds':plantIds, 'PlantYears':plantYears, 'PlantAge':plantAge, 'PlantCooling':plantCooling, 'PlantFuel':plantFuel})

basePred10 = np.nanmean(pcModel10.predict(dfpred))
basePred50 = np.nanmean(pcModel50.predict(dfpred))
basePred90 = np.nanmean(pcModel90.predict(dfpred))

if not os.path.isfile('%s/agg-outages-%s/aggregated-%s-outages-hist-hourly-%s-10.dat'%(dataDirDiscovery, runoffData, plantData, modelPower)) and not rebuild:

    extra=''
    if not 'globalPCHist50' in locals():

        with open('%s/pc-future-%s/%s-pc-hist-hourly-%s-10%s.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, extra), 'rb') as f:
            globalPCHist = pickle.load(f)
            globalPCHist10 = globalPCHist['globalPCHist10']

        with open('%s/pc-future-%s/%s-pc-hist-hourly-%s-50%s.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, extra), 'rb') as f:
            globalPCHist = pickle.load(f)
            globalPCHist50 = globalPCHist['globalPCHist50']

        with open('%s/pc-future-%s/%s-pc-hist-hourly-%s-90%s.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, extra), 'rb') as f:
            globalPCHist = pickle.load(f)
            globalPCHist90 = globalPCHist['globalPCHist90']

    yearlyOutagesHist10 = np.full([len(livingPlantsInds40[2018]), 12], np.nan)
    yearlyOutagesHist50 = np.full([len(livingPlantsInds40[2018]), 12], np.nan)
    yearlyOutagesHist90 = np.full([len(livingPlantsInds40[2018]), 12], np.nan)

    print('calculating total capacity outage for historical')

    numPlants10 = 0
    numPlants50 = 0
    numPlants90 = 0

    # over all plants that are active in 2018
    for pInd, p in enumerate(livingPlantsInds40[2018]):#range(globalPCHist50.shape[0]):

        if pInd % 500 == 0:
            print('plant %d...'%pInd)

        numYears = 0

        yearlyOutagesHistCurPlant10 = np.full([globalPCHist50.shape[1], 12], np.nan)
        yearlyOutagesHistCurPlant50 = np.full([globalPCHist50.shape[1], 12], np.nan)
        yearlyOutagesHistCurPlant90 = np.full([globalPCHist50.shape[1], 12], np.nan)

        plantHasData10 = False
        plantHasData50 = False
        plantHasData90 = False

        # calculate the total outage (MW) on each day of each year
        for year in range(globalPCHist50.shape[1]):
            yearlyOutagesHistCurYear10 = np.full([12], np.nan)
            yearlyOutagesHistCurYear50 = np.full([12], np.nan)
            yearlyOutagesHistCurYear90 = np.full([12], np.nan)

            numDays10 = 0
            numDays50 = 0
            numDays90 = 0

            for month in range(12):
                curMonthPc = np.reshape(globalPCHist10[p,year,month,:,:], [globalPCHist10[p,year,month,:,:].size])
                monthlyOutages10 = (basePred10-curMonthPc) / 100.0
                monthlyOutages10[monthlyOutages10 < 0] = 0
                monthlyOutages10[monthlyOutages10 > 1] = np.nan
                numHours10 = len(np.where(~np.isnan(monthlyOutages10))[0])
                # this is in MWh
                monthlyTotal10 = np.nansum(globalPlants['caps'][p] * monthlyOutages10)

                # divide by actual # of days with non-nan data in this month, then multiply by full month length
                # this accounts for model/years where there are nans
                if numHours10 > 0:
                    monthlyTotal10 /= numHours10
                    # now this too is MWh
                    monthlyTotal10 *= monthLen[month] * 24
                    yearlyOutagesHistCurYear10[month] = monthlyTotal10

                curMonthPc = np.reshape(globalPCHist50[p,year,month,:,:], [globalPCHist50[p,year,month,:,:].size])
                monthlyOutages50 = (basePred50-curMonthPc) / 100.0
                monthlyOutages50[monthlyOutages50 < 0] = 0
                monthlyOutages50[monthlyOutages50 > 1] = np.nan
                numHours50 = len(np.where(~np.isnan(monthlyOutages50))[0])
                # this is in MWh
                monthlyTotal50 = np.nansum(globalPlants['caps'][p] * monthlyOutages50)

                # divide by actual # of days with non-nan data in this month, then multiply by full month length
                # this accounts for model/years where there are nans
                if numHours50 > 0:
                    monthlyTotal50 /= numHours50
                    # now this too is MWh
                    monthlyTotal50 *= monthLen[month] * 24
                    yearlyOutagesHistCurYear50[month] = monthlyTotal50

                curMonthPc = np.reshape(globalPCHist90[p,year,month,:,:], [globalPCHist90[p,year,month,:,:].size])
                monthlyOutages90 = (basePred90-curMonthPc) / 100.0
                monthlyOutages90[monthlyOutages90 < 0] = 0
                monthlyOutages90[monthlyOutages90 > 1] = np.nan
                numHours90 = len(np.where(~np.isnan(monthlyOutages90))[0])
                # this is in MWh
                monthlyTotal90 = np.nansum(globalPlants['caps'][p] * monthlyOutages90)

                # divide by actual # of days with non-nan data in this month, then multiply by full month length
                # this accounts for model/years where there are nans
                if numHours90 > 0:
                    monthlyTotal90 /= numHours90
                    # now this too is MWh
                    monthlyTotal90 *= monthLen[month] * 24
                    yearlyOutagesHistCurYear90[month] = monthlyTotal90

            if len(np.where(~np.isnan(yearlyOutagesHistCurYear10))[0]) == 12:
                yearlyOutagesHistCurPlant10[year, :] = yearlyOutagesHistCurYear10
                plantHasData10 = True

            if len(np.where(~np.isnan(yearlyOutagesHistCurYear50))[0]) == 12:
                yearlyOutagesHistCurPlant50[year, :] = yearlyOutagesHistCurYear50
                plantHasData50 = True

            if len(np.where(~np.isnan(yearlyOutagesHistCurYear90))[0]) == 12:
                yearlyOutagesHistCurPlant90[year, :] = yearlyOutagesHistCurYear90
                plantHasData90 = True

        # divide by # of years to get total outage per month/year, if there is data
#         if len(yearlyOutagesHistCurPlant10) > 0:
        yearlyOutagesHistCurPlant10 = np.nanmean(yearlyOutagesHistCurPlant10, axis=0)
        if plantHasData10: numPlants10 += 1
        yearlyOutagesHist10[pInd, :] = yearlyOutagesHistCurPlant10

        yearlyOutagesHistCurPlant50 = np.nanmean(yearlyOutagesHistCurPlant50, axis=0)
        if plantHasData50: numPlants50 += 1
        yearlyOutagesHist50[pInd, :] = yearlyOutagesHistCurPlant50

        yearlyOutagesHistCurPlant90 = np.nanmean(yearlyOutagesHistCurPlant90, axis=0)
        if plantHasData90: numPlants90 += 1
        yearlyOutagesHist90[pInd, :] = yearlyOutagesHistCurPlant90

#         if len(yearlyOutagesHistCurPlant50) > 0:
#             yearlyOutagesHistCurPlant50 = np.array(yearlyOutagesHistCurPlant50)
#             yearlyOutagesHistCurPlant50 = np.nanmean(yearlyOutagesHistCurPlant50, axis=0)

#             if plantHasData50: numPlants50 += 1

#             yearlyOutagesHist50.append(yearlyOutagesHistCurPlant50)

#         if len(yearlyOutagesHistCurPlant90) > 0:
#             yearlyOutagesHistCurPlant90 = np.array(yearlyOutagesHistCurPlant90)
#             yearlyOutagesHistCurPlant90 = np.nanmean(yearlyOutagesHistCurPlant90, axis=0)

#             if plantHasData90: numPlants90 += 1

#             yearlyOutagesHist90.append(yearlyOutagesHistCurPlant90)

    # sum over all plants, divide by # plants with data, multiply by total number of plants regardless
    # of whether they have data
    yearlyOutagesHist10 = (np.nansum(yearlyOutagesHist10, axis=0)/numPlants10)*yearlyOutagesHist10.shape[0]
    yearlyOutagesHist50 = (np.nansum(yearlyOutagesHist50, axis=0)/numPlants50)*yearlyOutagesHist50.shape[0]
    yearlyOutagesHist90 = (np.nansum(yearlyOutagesHist90, axis=0)/numPlants90)*yearlyOutagesHist90.shape[0]

    with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-hist-hourly-%s-10.dat'%(dataDirDiscovery, runoffData, plantData, modelPower), 'wb') as f:
        pickle.dump(yearlyOutagesHist10, f)

    with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-hist-hourly-%s-50.dat'%(dataDirDiscovery, runoffData, plantData, modelPower), 'wb') as f:
        pickle.dump(yearlyOutagesHist50, f)

    with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-hist-hourly-%s-90.dat'%(dataDirDiscovery, runoffData, plantData, modelPower), 'wb') as f:
        pickle.dump(yearlyOutagesHist90, f)

for model in range(len(models)):

    yearlyOutagesCurModel10 = np.full([4, 12], np.nan)
    yearlyOutagesCurModel50 = np.full([4, 12], np.nan)
    yearlyOutagesCurModel90 = np.full([4, 12], np.nan)

    if not rebuild and os.path.isfile('%s/agg-outages-%s/aggregated-%s-outages-fut-hourly-%s-%s-10-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, models[model])):
        print('skipping %s...'%models[model])
        continue

    for w in range(1,4+1):

        fileName10 = '%s/pc-future-%s/%s-pc-future-anom-best-dist-10-%ddeg-%s-%s-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, w, modelPower, models[model])
        fileName50 = '%s/pc-future-%s/%s-pc-future-anom-best-dist-50-%ddeg-%s-%s-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, w, modelPower, models[model])
        fileName90 = '%s/pc-future-%s/%s-pc-future-anom-best-dist-90-%ddeg-%s-%s-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, w, modelPower, models[model])

        if os.path.isfile(fileName10):
            with open(fileName10, 'rb') as f:
                globalPC = pickle.load(f)
                globalPCFut10 = globalPC['globalPCFut10']
            with open(fileName50, 'rb') as f:
                globalPC = pickle.load(f)
                globalPCFut50 = globalPC['globalPCFut50']
            with open(fileName90, 'rb') as f:
                globalPC = pickle.load(f)
                globalPCFut90 = globalPC['globalPCFut90']
        else:
            print('%s not found!'%fileName10)
            continue

        yearlyOutagesGMT10 = np.full([len(livingPlantsInds40[2018]), 12], np.nan)
        yearlyOutagesGMT50 = np.full([len(livingPlantsInds40[2018]), 12], np.nan)
        yearlyOutagesGMT90 = np.full([len(livingPlantsInds40[2018]), 12], np.nan)

        print('calculating total capacity outage for %s/+%dC'%(models[model],w))

        # num plants for current model with data
        numPlants10 = 0
        numPlants50 = 0
        numPlants90 = 0

        # over all plants living in 2018
        for pInd, p in enumerate(livingPlantsInds40[2018]):#range(globalPCFut10.shape[0]):
            yearlyOutagesCurPlant10 = np.full([globalPCFut10.shape[1], 12], np.nan)
            yearlyOutagesCurPlant50 = np.full([globalPCFut10.shape[1], 12], np.nan)
            yearlyOutagesCurPlant90 = np.full([globalPCFut10.shape[1], 12], np.nan)

            if pInd%500 == 0:
                print('plant %d...'%pInd)

            plantHasData10 = False
            plantHasData50 = False
            plantHasData90 = False

            # calculate the total outage (MW) on each day of each year
            for year in range(globalPCFut10.shape[1]):
                yearlyOutagesCurYear10 = np.full([12], np.nan)
                yearlyOutagesCurYear50 = np.full([12], np.nan)
                yearlyOutagesCurYear90 = np.full([12], np.nan)

                for month in range(12):

                    curMonthPc = np.reshape(globalPCFut10[p,year,month,:,:], [globalPCFut10[p,year,month,:,:].size])
                    monthlyOutages10 = (basePred10-curMonthPc) / 100.0
                    monthlyOutages10[monthlyOutages10<0] = 0
                    monthlyOutages10[monthlyOutages10>1] = np.nan

                    curMonthPc = np.reshape(globalPCFut50[p,year,month,:,:], [globalPCFut50[p,year,month,:,:].size])
                    monthlyOutages50 = (basePred50-curMonthPc) / 100.0
                    monthlyOutages50[monthlyOutages50<0] = 0
                    monthlyOutages50[monthlyOutages50>1] = np.nan

                    curMonthPc = np.reshape(globalPCFut90[p,year,month,:,:], [globalPCFut90[p,year,month,:,:].size])
                    monthlyOutages90 = (basePred90-curMonthPc) / 100.0
                    monthlyOutages90[monthlyOutages90<0] = 0
                    monthlyOutages90[monthlyOutages90>1] = np.nan
#
#                    indBadData10 = np.where((monthlyOutages10 < 0) | (monthlyOutages10 > 1))[0]
#                    monthlyOutages10[indBadData10] = np.nan
#
#                    indBadData50 = np.where((monthlyOutages50 < 0) | (monthlyOutages50 > 1))[0]
#                    monthlyOutages50[indBadData50] = np.nan
#
#                    indBadData90 = np.where((monthlyOutages90 < 0) | (monthlyOutages90 > 1))[0]
#                    monthlyOutages90[indBadData90] = np.nan

                    numHours10 = len(np.where(~np.isnan(monthlyOutages10))[0])
                    monthlyTotal10 = np.nansum(globalPlants['caps'][p] * monthlyOutages10)

                    # divide by actual # of days in this month, then multiply by full summer (62 days)
                    # this accounts for model/years where there are nans
                    if numHours10 > 0:
                        monthlyTotal10 /= numHours10
                        monthlyTotal10 *= monthLen[month] * 24
                        yearlyOutagesCurYear10[month] = monthlyTotal10

                    numHours50 = len(np.where(~np.isnan(monthlyOutages50))[0])
                    monthlyTotal50 = np.nansum(globalPlants['caps'][p] * monthlyOutages50)

                    if numHours50 > 0:
                        monthlyTotal50 /= numHours50
                        monthlyTotal50 *= monthLen[month] * 24
                        yearlyOutagesCurYear50[month] = monthlyTotal50

                    numHours90 = len(np.where(~np.isnan(monthlyOutages90))[0])
                    monthlyTotal90 = np.nansum(globalPlants['caps'][p] * monthlyOutages90)

                    if numHours90 > 0:
                        monthlyTotal90 /= numHours90
                        monthlyTotal90 *= monthLen[month] * 24
                        yearlyOutagesCurYear90[month] = monthlyTotal90

                if len(np.where(~np.isnan(yearlyOutagesCurYear10))[0]) == 12:
                    yearlyOutagesCurPlant10[year, :] = yearlyOutagesCurYear10
                    plantHasData10 = True

                if len(np.where(~np.isnan(yearlyOutagesCurYear50))[0]) == 12:
                    yearlyOutagesCurPlant50[year, :] = yearlyOutagesCurYear50
                    plantHasData50 = True

                if len(np.where(~np.isnan(yearlyOutagesCurYear90))[0]) == 12:
                    yearlyOutagesCurPlant90[year, :] = yearlyOutagesCurYear90
                    plantHasData90 = True

            yearlyOutagesCurPlant10 = np.nanmean(yearlyOutagesCurPlant10, axis=0)
            if plantHasData10: numPlants10 += 1
            yearlyOutagesGMT10[pInd, :] = yearlyOutagesCurPlant10

            yearlyOutagesCurPlant50 = np.nanmean(yearlyOutagesCurPlant50, axis=0)
            if plantHasData50: numPlants50 += 1
            yearlyOutagesGMT50[pInd, :] = yearlyOutagesCurPlant50

            yearlyOutagesCurPlant90 = np.nanmean(yearlyOutagesCurPlant90, axis=0)
            if plantHasData90: numPlants90 += 1
            yearlyOutagesGMT90[pInd, :] = yearlyOutagesCurPlant90

#             # divide by # of years to get total outage per year, if there is data
#             if len(yearlyOutagesCurPlant10) > 0:
#                 yearlyOutagesCurPlant10 = np.array(yearlyOutagesCurPlant10)
#                 yearlyOutagesCurPlant10 = np.nanmean(yearlyOutagesCurPlant10, axis=0)
#                 yearlyOutagesCurGMT10.append(yearlyOutagesCurPlant10)
#                 if plantHasData10: numPlants10 += 1

#             if len(yearlyOutagesCurPlant50) > 0:
#                 yearlyOutagesCurPlant50 = np.array(yearlyOutagesCurPlant50)
#                 yearlyOutagesCurPlant50 = np.nanmean(yearlyOutagesCurPlant50, axis=0)
#                 yearlyOutagesCurGMT50.append(yearlyOutagesCurPlant50)
#                 if plantHasData50: numPlants50 += 1

#             if len(yearlyOutagesCurPlant90) > 0:
#                 yearlyOutagesCurPlant90 = np.array(yearlyOutagesCurPlant90)
#                 yearlyOutagesCurPlant90 = np.nanmean(yearlyOutagesCurPlant90, axis=0)
#                 yearlyOutagesCurGMT90.append(yearlyOutagesCurPlant90)
#                 if plantHasData90: numPlants90 += 1

        # divide by number of plants
        yearlyOutagesGMT10 = (np.nansum(yearlyOutagesGMT10, axis=0)/numPlants10)*yearlyOutagesGMT10.shape[0]
        yearlyOutagesCurModel10[w-1, :] = yearlyOutagesGMT10

        yearlyOutagesGMT50 = (np.nansum(yearlyOutagesGMT50, axis=0)/numPlants50)*yearlyOutagesGMT50.shape[0]
        yearlyOutagesCurModel50[w-1, :] = yearlyOutagesGMT50

        yearlyOutagesGMT90 = (np.nansum(yearlyOutagesGMT90, axis=0)/numPlants90)*yearlyOutagesGMT90.shape[0]
        yearlyOutagesCurModel90[w-1, :] = yearlyOutagesGMT90

    with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-fut-hourly-%s-%s-10-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, models[model]), 'wb') as f:
        pickle.dump(yearlyOutagesCurModel10, f)

    with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-fut-hourly-%s-%s-50-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, models[model]), 'wb') as f:
        pickle.dump(yearlyOutagesCurModel50, f)

    with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-fut-hourly-%s-%s-90-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, models[model]), 'wb') as f:
        pickle.dump(yearlyOutagesCurModel90, f)

with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-hist-hourly-%s-10.dat'%(dataDirDiscovery, runoffData, plantData, modelPower), 'rb') as f:
    yearlyOutagesHist10 = pickle.load(f)

with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-hist-hourly-%s-50.dat'%(dataDirDiscovery, runoffData, plantData, modelPower), 'rb') as f:
    yearlyOutagesHist50 = pickle.load(f)

with gzip.open('%s/agg-outages-%s/aggregated-%s-outages-hist-hourly-%s-90.dat'%(dataDirDiscovery, runoffData, plantData, modelPower), 'rb') as f:
    yearlyOutagesHist90 = pickle.load(f)

yearlyOutagesFut10 = []
yearlyOutagesFut50 = []
yearlyOutagesFut90 = []

for model in range(len(models[0])):
    yearlyOutagesCurModel10 = []
    yearlyOutagesCurModel50 = []
    yearlyOutagesCurModel90 = []

    fileName10 = '%s/agg-outages-%s/aggregated-%s-outages-fut-hourly-%s-%s-10-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, models[model])
    fileName50 = '%s/agg-outages-%s/aggregated-%s-outages-fut-hourly-%s-%s-50-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, models[model])
    fileName90 = '%s/agg-outages-%s/aggregated-%s-outages-fut-hourly-%s-%s-90-preIndRef.dat'%(dataDirDiscovery, runoffData, plantData, modelPower, models[model])

    if os.path.isfile(fileName10):

        with gzip.open(fileName10) as f:
            yearlyOutagesCurModel10 = pickle.load(f)
        with gzip.open(fileName50) as f:
            yearlyOutagesCurModel50 = pickle.load(f)
        with gzip.open(fileName90) as f:
            yearlyOutagesCurModel90 = pickle.load(f)

        yearlyOutagesFut10.append(yearlyOutagesCurModel10)
        yearlyOutagesFut50.append(yearlyOutagesCurModel50)
        yearlyOutagesFut90.append(yearlyOutagesCurModel90)

# convert all to twh
yearlyOutagesHist10 /= 1e6
yearlyOutagesHist50 /= 1e6
yearlyOutagesHist90 /= 1e6
yearlyOutagesFut10 = np.array(yearlyOutagesFut10)/1e6
yearlyOutagesFut50 = np.array(yearlyOutagesFut50)/1e6
yearlyOutagesFut90 = np.array(yearlyOutagesFut90)/1e6

snsColors = sns.color_palette(["#3498db", "#e74c3c"])

#xd = np.array(list(range(1981, 2018+1)))-1981+1

#z = np.polyfit(xd, mean10[0,:], 1)
#histPolyTx10 = np.poly1d(z)
#z = np.polyfit(xd, mean50[0,:], 1)
#histPolyTx50 = np.poly1d(z)
#z = np.polyfit(xd, mean90[0,:], 1)
#histPolyTx90 = np.poly1d(z)

# in PJ
# yearlyOutagesHist10 = yearlyOutagesHist10/1e18*1e3
# yearlyOutagesHist50 = yearlyOutagesHist50/1e18*1e3
# yearlyOutagesHist90 = yearlyOutagesHist90/1e18*1e3

# yearlyOutagesFut10 = yearlyOutagesFut10/1e18*1e3
# yearlyOutagesFut50 = yearlyOutagesFut50/1e18*1e3
# yearlyOutagesFut90 = yearlyOutagesFut90/1e18*1e3

yearlyOutagesFut10 = np.moveaxis(yearlyOutagesFut10, 1, 0)
yearlyOutagesFut50 = np.moveaxis(yearlyOutagesFut50, 1, 0)
yearlyOutagesFut90 = np.moveaxis(yearlyOutagesFut90, 1, 0)

if plantData == 'world':
    yLimSet = 850
    yTickStep = 200
    yticks = np.arange(0,yLimSet,yTickStep)
else:
    yLimSet = 45
    yTickStep = 10
    yTicks = [0, 10, 20, 30, 40]
    yticks = np.arange(0,45,yTickStep)

# Twh
totalMonthlyEnergy = np.nansum(globalPlants['caps'][livingPlantsInds40[2018]])*30*24/1e6
totalAnnualEnergy = np.nansum(globalPlants['caps'][livingPlantsInds40[2018]])*24*365/1e6
xpos = [1,2,3,4,5,6,7,8,9,10,11,12]

pctEnergyGrid = np.round(yticks/totalAnnualEnergy*100,decimals=1)

outageSumHist = []
outageSum1 = []
outageSum2 = []
outageSum3 = []
outageSum4 = []

for m in range(0,12):
    outageSumHist.append(np.nansum(yearlyOutagesHist50[0:m+1]))
    outageSum1.append(np.nansum(yearlyOutagesFut50[0,:,0:m+1], axis=1))
    outageSum2.append(np.nansum(yearlyOutagesFut50[1,:,0:m+1], axis=1))
    outageSum3.append(np.nansum(yearlyOutagesFut50[2,:,0:m+1], axis=1))
    outageSum4.append(np.nansum(yearlyOutagesFut50[3,:,0:m+1], axis=1))

outageSum1 = np.sort(np.array(outageSum1), axis=1)
outageSum2 = np.sort(np.array(outageSum2), axis=1)
outageSum3 = np.sort(np.array(outageSum3), axis=1)
outageSum4 = np.sort(np.array(outageSum4), axis=1)

#plt.rc('font', family='helvetica', weight='normal')
#plt.rc('axes', labelweight='normal')

plt.figure(figsize=(5,2))
#plt.ylim([0, 320])
plt.ylim([0,yLimSet])
plt.xlim([0,13])
plt.grid(True, alpha = 0.25)
plt.gca().set_axisbelow(True)

plt.plot(xpos, outageSumHist, '-', lw=2, color='black', label='Historical')
plt.plot(xpos, np.nanmean(outageSum2, axis=1), '-', lw=2, color='#ffb835', label='+ 2$\degree$C')
plt.plot(xpos, outageSum2[:,-1], '--', lw=2, color='#ffb835')
plt.plot(xpos, np.nanmean(outageSum4, axis=1), '-', lw=2, color=snsColors[1], label='+ 4$\degree$C')
plt.plot(xpos, outageSum4[:,-1], '--', lw=2, color=snsColors[1])

plt.fill_between(xpos, outageSumHist, [0]*12, facecolor='black', alpha=.5, interpolate=True)
plt.fill_between(xpos, np.nanmean(outageSum2, axis=1), outageSumHist, facecolor='#ffb835', alpha=.5, interpolate=True)
plt.fill_between(xpos, np.nanmean(outageSum4, axis=1), np.nanmean(outageSum2, axis=1), facecolor=snsColors[1], alpha=.5, interpolate=True)

plt.xticks(range(1,13))
plt.gca().set_xticklabels(range(1,13))
plt.yticks(yticks)
plt.xlabel('Month', fontname = 'Helvetica', fontsize=16)
#plt.ylabel('Cumulative US-EU outage (EJ)', fontname = 'Helvetica', fontsize=16)

for tick in plt.gca().xaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)
for tick in plt.gca().yaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)

#plt.tick_params(
#    axis='x',          # changes apply to the x-axis
#    which='both',      # both major and minor ticks are affected
#    bottom=False,      # ticks along the bottom edge are off
#    top=False,         # ticks along the top edge are off
#    labelbottom=False) # labels along the bottom edge are off

ax2 = plt.gca().twinx()
plt.xlim([0,13])
plt.ylim([0,yLimSet])
plt.yticks(yticks)
plt.gca().set_yticklabels(pctEnergyGrid)

for tick in plt.gca().yaxis.get_major_ticks():
    tick.label2.set_fontname('Helvetica')
    tick.label2.set_fontsize(14)

if plotFigs:
    plt.savefig('accumulated-annual-outage-cdf-%s-%s-%s.png'%(plantData,runoffData, modelPower), format='png', dpi=500, bbox_inches = 'tight', pad_inches = 0, transparent=True)

if plantData == 'world':
    yLimSet = 160
    yTickStep = 50
else:
    yLimSet = 11.5
    yTickStep = 2

yticks = np.arange(0,yLimSet,yTickStep)
pctEnergyGrid = np.round(yticks/totalMonthlyEnergy*100,decimals=1)

yearlyOutagesFut50GMT2Sorted = np.sort(yearlyOutagesFut50[1,:,:],axis=0)
yearlyOutagesFut50GMT4Sorted = np.sort(yearlyOutagesFut50[3,:,:],axis=0)

plt.figure(figsize=(5,2))
#plt.xlim([0, 7])
plt.xlim([0,13])
plt.ylim([0, yLimSet])
plt.grid(True, alpha = 0.25)
plt.gca().set_axisbelow(True)

plt.plot(xpos, yearlyOutagesHist50, '-', lw=2, color='black', label='Historical')
plt.plot(xpos, np.nanmean(yearlyOutagesFut50[1],axis=0), '-', lw=2, color='#ffb835', label='+ 2$\degree$C')
plt.plot(xpos, yearlyOutagesFut50GMT2Sorted[-2,:], '--', lw=2, color='#ffb835')
plt.plot(xpos, np.nanmean(yearlyOutagesFut50[3],axis=0), '-', lw=2, color=snsColors[1], label='+ 4$\degree$C')
plt.plot(xpos, yearlyOutagesFut50GMT4Sorted[-3,:], '--', lw=2, color=snsColors[1])

plt.fill_between(xpos, yearlyOutagesHist50, [0]*12, facecolor='black', alpha=.5, interpolate=True)
plt.fill_between(xpos, yearlyOutagesHist50, np.nanmean(yearlyOutagesFut50[1],axis=0), facecolor='#ffb835', alpha=.5, interpolate=True)
plt.fill_between(xpos, np.nanmean(yearlyOutagesFut50[1],axis=0), np.nanmean(yearlyOutagesFut50[3],axis=0), facecolor=snsColors[1], alpha=.5, interpolate=True)

plt.xticks(xpos)
plt.yticks(yticks)
#plt.xlabel('Month', fontname = 'Helvetica', fontsize=16)
#plt.ylabel('Monthly US-EU outage (EJ)', fontname = 'Helvetica', fontsize=16)

for tick in plt.gca().xaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)
for tick in plt.gca().yaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)

plt.tick_params(
    axis='x',          # changes apply to the x-axis
    which='both',      # both major and minor ticks are affected
    bottom=False,      # ticks along the bottom edge are off
    top=False,         # ticks along the top edge are off
    labelbottom=False) # labels along the bottom edge are off

leg = plt.legend(prop = {'size':12, 'family':'Helvetica', 'weight':'normal'}, loc = 'upper left')
leg.get_frame().set_linewidth(0.0)

#leg = plt.legend(prop = {'size':11, 'family':'Helvetica'})
#leg.get_frame().set_linewidth(0.0)

ax2 = plt.gca().twinx()
plt.xlim([0,13])
plt.ylim([0, yLimSet])
plt.yticks(yticks)
plt.gca().set_yticklabels(pctEnergyGrid)
#plt.ylabel('% of US-EU electricity capacity', fontname = 'Helvetica', fontsize=16)

for tick in plt.gca().yaxis.get_major_ticks():
    tick.label2.set_fontname('Helvetica')
    tick.label2.set_fontsize(14)

#x0,x1 = plt.gca().get_xlim()
#y0,y1 = plt.gca().get_ylim()
#plt.gca().set_aspect(abs(x1-x0)/abs(y1-y0))

if plotFigs:
    plt.savefig('accumulated-monthly-outage-%s-%s-%s.png'%(plantData, runoffData, modelPower), format='png', dpi=500, bbox_inches = 'tight', pad_inches = 0, transparent=True)

plt.show()
sys.exit()


from flask import session, jsonify

from server import app
from server.database import *
from server.utils import login_required, json_abort

@app.route('/api/user', methods=['GET'])
@login_required
def get_user():
    return jsonify({
        'id': session.get('userID'),
        'email': session.get('userEmail')
    })

@app.route('/api/users/<string:id>', methods=['GET'])
@login_required
def get_user_by_id(id):
    user = db_get_user_by_id(id)
    if user == None:
        json_abort(404, "User not found.")

    return jsonify({
        'id': user.id,
        'email': user.email
    })


import numpy as np
import imagehash
from PIL import Image

HISTOGRAM_BINS = 256
HISTORGRAM_MAX_DISSIMILARITY = 0.02
PHASH_HASH_SIZE = 20
PHASH_MAX_DISSIMILARITY = 0.06

def hist(frame):
    return np.histogram(frame.ravel(), HISTOGRAM_BINS)[0]

def hist_dissimilarity(hist1, hist2):
    shorter = min(np.linalg.norm(hist1), np.linalg.norm(hist2))
    diff_norm = np.linalg.norm(hist1 - hist2)
    return diff_norm / shorter

def phash(frame):
    return imagehash.phash(Image.fromarray(frame), hash_size=PHASH_HASH_SIZE).hash.ravel()

def phash_dissimilarity(hash1, hash2):
    if isinstance(hash1, list) or isinstance(hash2, list):
        pass
    assert hash1.shape == hash2.shape
    size = len(hash1)
    not_equal_count = np.logical_not(np.equal(hash1, hash2)).sum()
    return not_equal_count / size


import pytz
from django.utils import timezone

from booking_test.settings import TIME_ZONE

def timezone_middleware(get_response):
    def middleware(request):
        """
        Middleware that activates user's timezone for each request
        """
        if request.user.is_authenticated and hasattr(request.user, "profile"):
            # Timezone is recorded in request object so it can be used in views
            request.timezone = request.user.profile.timezone
        else:
            request.timezone = TIME_ZONE

        timezone.activate(pytz.timezone(request.timezone))

        return get_response(request)

    return middleware


"""
1. Clarification
2. Possible solutions
    - bfs, topological sort
    - dfs
3. Coding
4. Tests
"""

# T=O(V+E), S=O(V+E)
from collections import defaultdict, deque
class Solution:

    def findOrder(self, numCourses: int, prerequisites: List[List[int]]) -> List[int]:
        adj_list = defaultdict(list)
        indegree = {}
        for dest, src in prerequisites:
            adj_list[src].append(dest)
            indegree[dest] = indegree.get(dest, 0) + 1

        zero_indegree_queue = deque([k for k in range(numCourses) if k not in indegree])
        topological_sorted_order = []

        while zero_indegree_queue:
            vertex = zero_indegree_queue.popleft()
            topological_sorted_order.append(vertex)
            if vertex in adj_list:
                for neighbor in adj_list[vertex]:
                    indegree[neighbor] -= 1
                    if indegree[neighbor] == 0:
                        zero_indegree_queue.append(neighbor)

        return topological_sorted_order if len(topological_sorted_order) == numCourses else []

# T=O(V+E), S=O(V+E)
from collections import defaultdict
class Solution:

    WHITE = 1
    GRAY = 2
    BLACK = 3

    def findOrder(self, numCourses: int, prerequisites: List[List[int]]) -> List[int]:

        adj_list = defaultdict(list)
        for dest, src in prerequisites:
            adj_list[src].append(dest)

        topological_sorted_order = []
        is_possible = True

        color = {k: Solution.WHITE for k in range(numCourses)}
        def dfs(node):
            nonlocal is_possible
            if not is_possible:
                return

            color[node] = Solution.GRAY

            if node in adj_list:
                for neighbor in adj_list[node]:
                    if color[neighbor] == Solution.WHITE:
                        dfs(neighbor)
                    elif color[neighbor] == Solution.GRAY:
                        is_possible = False

            color[node] = Solution.BLACK
            topological_sorted_order.append(node)

        for vertex in range(numCourses):
            if color[vertex] == Solution.WHITE:
                dfs(vertex)

        return topological_sorted_order[::-1] if is_possible else []


from django.contrib.auth import get_user_model
from rest_framework import serializers

User = get_user_model()

class SignUpSerializer(serializers.ModelSerializer):
    """Регистрация пользователя."""

    class Meta:
        model = User
        fields = ('email', 'username')

class UserSerializer(serializers.ModelSerializer):
    """Сериализатор для пользователя общего назначения."""

    class Meta:
        model = User
        fields = ('username', 'email',
                  'first_name', 'last_name', 'role')

class SafeUserSerializer(serializers.ModelSerializer):
    """Серилазиатор для пользователя с безопасными полями."""

    role = serializers.CharField(read_only=True)

    class Meta:
        model = User
        exclude = ('id', 'confirmation_code')

class ObtainTokenSerializer(serializers.ModelSerializer):
    username = serializers.CharField()

    class Meta:
        model = User
        fields = ('confirmation_code', 'username')

    def validate(self, data):
        username = data.get('username')
        confirmation_code = data.get('confirmation_code')
        if not username and not confirmation_code:
            raise serializers.ValidationError(
                f"Fields are blank {username}, {confirmation_code}"
            )
        return data

    def validate_username(self, value):
        if not value:
            raise serializers.ValidationError(
                "username can't be blank"
            )
        return value


from django.urls import path
from .views import index, register, sign_in, sign_out, logged_in, send_a_post,delete_a_post

app_name = 'usermodel'

#do not forget adding app_name

urlpatterns = [
    path('', index, name = 'index_page'),
    path('registration/', register, name = 'registration_page'),
    path('login/', sign_in, name = 'login_page'),
    path('logout/', sign_out, name = 'logout_page'),
    path('success/', logged_in, name = 'successful_login_page'),
    path('posts/', send_a_post, name = 'post_page'),
    path('posts/delete/<int:post_id>', delete_a_post, name = 'delete_post_page'),
]


import torch
import torch.nn as nn
import torch.nn.functional as F

import dgl
from dgl.nn.pytorch.glob import SumPooling, AvgPooling, MaxPooling

"""
    GIN: Graph Isomorphism Networks
    HOW POWERFUL ARE GRAPH NEURAL NETWORKS? (Keyulu Xu, Weihua Hu, Jure Leskovec and Stefanie Jegelka, ICLR 2019)
    https://arxiv.org/pdf/1810.00826.pdf
"""

from layers.gin_layer import GINLayer, ApplyNodeFunc, MLP

activations = {
    'relu': nn.ReLU(),
    'tanh': nn.Tanh(),
    'identity': nn.Identity()
}

class GINNet(nn.Module):

    def __init__(self, net_params):
        super().__init__()
        in_dim = net_params['in_dim']
        hidden_dim = net_params['hidden_dim']
        out_dim = net_params['out_dim']
        dropout = net_params['dropout']
        self.n_layers = net_params['L']
        n_mlp_layers = net_params['n_mlp_GIN']               # GIN
        learn_eps = net_params['learn_eps_GIN']              # GIN
        neighbor_aggr_type = net_params['neighbor_aggr_GIN'] # GIN
        readout = net_params['readout']                      # this is graph_pooling_type
        batch_norm = net_params['batch_norm']
        residual = net_params['residual']
        activation_name = net_params['activation']

        # List of MLPs
        self.ginlayers = torch.nn.ModuleList()

        self.embedding_h = nn.Linear(in_dim, hidden_dim)

        for layer in range(self.n_layers - 1):
            mlp = MLP(n_mlp_layers, hidden_dim, hidden_dim, hidden_dim, activations[activation_name])

            self.ginlayers.append(GINLayer(ApplyNodeFunc(mlp), neighbor_aggr_type,
                                           dropout, batch_norm, residual, 0, learn_eps, activations[activation_name]))

        self.ginlayers.append(GINLayer(ApplyNodeFunc(MLP(n_mlp_layers, hidden_dim, hidden_dim, out_dim, activations[activation_name])), neighbor_aggr_type,
                                dropout, batch_norm, residual, 0, learn_eps, activations[activation_name]))

    def forward(self, g, h, e, intermediate=None):
        h = self.embedding_h(h)

        for i in range(self.n_layers):
            h = self.ginlayers[i](g, h, intermediate)

            if intermediate is not None:
                intermediate(h, 'out')

        return h

    def loss(self, scores, targets):
        loss = nn.L1Loss()(scores, targets)
        return loss


from __future__ import print_function
import torch
import scipy.io as spio
import numpy as np
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')

from scipy.io import loadmat
import os
from tqdm import tqdm
import copy

import pandas as pd
import matplotlib.pyplot as plt
import os

get_ipython().run_line_magic('tensorflow_version', '1.x')
from torchvision.transforms import Compose, Grayscale
import torchvision.transforms as transforms
from torchvision.transforms.functional import to_grayscale

from PIL import Image, ImageDraw
from sklearn.model_selection import train_test_split, cross_val_score

import keras
from keras.utils import to_categorical
from keras import models
from keras import layers
from keras.datasets import imdb
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D

from sklearn.model_selection import KFold

from numpy.random import seed
from numpy import asarray
from tensorflow import set_random_seed

import tensorflow as tf
import datetime
import glob

keras.applications.resnet_v2.ResNet50V2()
keras.applications.resnet_v2.ResNet101V2()
keras.applications.resnet_v2.ResNet152V2()

keras.applications.resnet.ResNet50()
keras.applications.resnet.ResNet101()
keras.applications.resnet.ResNet152()

# Preprocessings
# List classes  consists of 17 types of actions, including 9 ADLs and 8 Falls:

classes = [
    'StandingUpFS',
    'StandingUpFL',
    'Walking',
    'Running',
    'GoingUpS',
    'Jumping',
    'GoingDownS',
    'LyingDownFS',
    'SittingDown',
    'FallingForw',
    'FallingRight',
    'FallingBack',
    'HittingObstacle',
    'FallingWithPS',
    'FallingBackSC',
    'Syncope',
    'FallingLeft'
]

# Method transforms.Compose combines methods transforms.ToTensor
# (converts image with pixel diapason [0, 255] into tensor with [0, 1] diapason)
# and transforms.Normalize (normalizes images) into a method transform:

transform = transforms.Compose([transforms.ToTensor(),
        transforms.Normalize((0.5,), (0.5,))])

# Dictionary dict_classes assigns a unique index to every class:

dict_classes = {}
i = 0
for activity in classes:
    dict_classes[activity] = i
    i += 1

# Function preprocess_function processes иoriginal images and forms a benchmarked dataset.
# Arguments:
#
# * input_path - input directory pathway,
# * sample_width - width of cropped images,
# * step - step of frame shift,
# * list_of_folders - list of directories with original images,
#
# Returns:
#
# * array - tensor of cropped and preprocessed images,
# * targets - vetor of labels

def preprocess_function(input_path,
                        multiframe=True,
                        sample_width=90, step=15,
                        list_of_folders=[]):

    array = []                             # List of final images
    targets_words = []                     # Vector of labels (consists of action names, for example Running)
    targets = []                           # Encoded vector of labels

    """Here we test if there is at least one directory in the list.
       If not, the function stops running and returns empty arrays"""

    if len(list_of_folders)==0:
        print("No folder is chosen. Chose at least one folder, please")
        return [], []

    path = os.path.join(input_path, list_of_folders[0])

    """Here we test if all directories have the same number of images.
       If not, the function stops running and returns empty arrays"""

    def listdir_nohidden(path):
        return glob.glob(os.path.join(path, '*'))      # ignore hidden files (including '.ipynb_checkpoints')

    if len(list_of_folders) > 1:
        for i in range(len(list_of_folders)-1):
            path_1 = os.path.join(input_path, list_of_folders[i])
            path_2 = os.path.join(input_path, list_of_folders[i+1])
            if len(listdir_nohidden(path_1)) != len(listdir_nohidden(path_2)):
                print("Error!  There are different number of files in folders!")
                return [], []

    """Search for the minimal image size min_width."""

    all_widths = []

    for filename in tqdm(os.listdir(path)):
        if filename==".ipynb_checkpoints":
            continue
        current_image = Image.open(os.path.join(input_path, list_of_folders[0], filename))
        data = asarray(current_image)
        height, width, dimensions = data.shape
        all_widths.append(width)

    min_width = min(all_widths)

    """Here we iterate on images from the chosen directory.
       If any images is not in every directory,
       the function stops running and returns empty arrays.
       It is assumed that images in different directories
       related to the same object are identical in size"""

    for filename in tqdm(os.listdir(path)):

        if filename==".ipynb_checkpoints":
            continue

        try:
            data_array = []
            for folder in list_of_folders:
                current_image = Image.open(os.path.join(input_path, folder, filename))
                data_array.append(current_image)
            data = asarray(current_image)
            height, width, dimensions = data.shape

        except:
            print("Image " + str(filename) + " has not all the chosen axes")
            return [], []

        if multiframe:
            for j in range(int(round((width - sample_width)/step + 1))):

                # Image size
                left = step*j
                right = left + sample_width
                top = 0
                bottom = copy.copy(height)

                # Images along all chosen axes
                images = []

                for data in data_array:
                    im = data.crop((left, top, right, bottom))
                    im = transforms.functional.to_grayscale(im, 1)
                    im = np.array(im)
                    images.append(im)

                images = np.array(transform(np.array(images)))
                array.append(images)
                targets_words.append(filename)

        else:

            # Image size
            left = 0
            right = copy.copy(min_width)
            top = 0
            bottom = copy.copy(height)

            # Images along all chosen axes
            images = []

            for data in data_array:
                im = data.crop((left, top, right, bottom))
                im = transforms.functional.to_grayscale(im, 1)
                im = np.array(im)
                images.append(im)

            images = np.array(transform(np.array(images)))
            array.append(images)
            targets_words.append(filename)

    """Now we have 2 lists: arrays (contains tensors of cropped
       and preprocessed images) and targets_words (vector of labels).
       Let's encode categorical targets_words to targets"""

    for i in range(len(targets_words)):
        targets_words[i] = targets_words[i][:-10]
        if targets_words[i][-1] == '_':
            targets_words[i] = targets_words[i][:-1]

    for word in targets_words:
        targets.append(dict_classes[word])
    targets = np.array(targets)

    targets = to_categorical(targets, num_classes=len(classes))
    targets = np.array(targets)

    return array, targets

# Dataset split

# Function split_function splits dataset into train and test samples.
# Arguments:
#
# * array - original array of features,
# * targets - vector of labels,
# * test_size - test sample fraction (0.2 by default),
# * random_state - random state value (26 by defaut).
#
# Function split_function returns train and test samples and corresponing vector of labels:
#
# * X_train,
# * X_test,
# * y_train,
# * y_test.

def split_function(array, targets, test_size=0.20, random_state=26):

    X_train, X_test, y_train, y_test = train_test_split(array, targets,
                                                        test_size=test_size,
                                                        shuffle=True,
                                                        random_state=random_state)

    X_train = np.array(X_train)
    y_train = np.array(y_train)
    X_test = np.array(X_test)
    y_test = np.array(y_test)

    X_train = np.transpose(X_train, (0, 3, 1, 2))
    X_test = np.transpose(X_test, (0, 3, 1, 2))

    return X_train, X_test, y_train, y_test

# Now we build neural network

# my_callback is a daughter class of the class callbacks:

class my_callback(tf.keras.callbacks.Callback):

    def __init__(self):
        super().__init__()
        self.acc = []
        self.loss = []
        self.rec = []
        self.prec = []

    """After every epoch metrics are counted via cross-validation"""

    def on_epoch_end(self, epoch, logs=None):
        l, a, r, p = self.model.evaluate(self.validation_data[0],
                                         self.validation_data[1])
        self.loss.append(l)
        self.acc.append(a)
        self.rec.append(r)
        self.prec.append(p)

    """After training is complete metrics are saved into metrics.csv"""

    def on_train_end(self, logs=None):

            df = {'loss': self.loss,
                  'accuracy': self.acc,
                  'recall': self.rec,
                  'precision': self.prec}
            df = pd.DataFrame(data=df)
            df.to_csv("metrics.csv", index=False)

# Function create_and_run_model builds and trains a neural network and creates a file metrics.csv containing metrics.
# Arguments:
#
# * X_train, X_test, y_train, y_test - train and test samples and corresponding vectors of labels,
# * res_model - ResNet model from Keras (keras.applications.resnet_v2.ResNet50V2,
#                                        keras.applications.resnet_v2.ResNet101V2,
#                                        keras.applications.resnet_v2.ResNet152V2,
#                                        keras.applications.resnet.ResNet50,
#                                        keras.applications.resnet.ResNet101,
#                                        keras.applications.resnet.ResNet152),
# * number_of_classes - number of different types of activity,
# * height - input image height,
# * width - input image width,
# * dimensions - number of channels of input images,
# * batch_size - batch size,
# * epochs - number of epochs.

def create_and_run_model(X_train, X_test, y_train, y_test,
                         resnet = True,
                         base_model=keras.applications.resnet_v2.ResNet50V2,
                         number_of_classes=len(classes),
                         height=36,
                         width=90,
                         dimensions=3,
                         batch_size=200,
                         epochs=300):

    if resnet:
        model = base_model(weights=None, input_shape=(height, width, dimensions),
                         classes=number_of_classes)
    else:
        model = base_model

    model.compile(optimizer = "adam",
                  loss = "binary_crossentropy",
                  metrics = ["accuracy", tf.keras.metrics.Recall(), tf.keras.metrics.Precision()])

    log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")

    model.fit(X_train, y_train,
              batch_size=batch_size,
              epochs=epochs,
              validation_data=(X_test, y_test),
              callbacks=[my_callback()])


import argparse
import cmd2
import typing as T
from cmd2 import CommandSet, with_argparser, with_category, with_default_category

# Required to register the apps
from .edit.commands import EditCommands
from .edit.interactive_requirement import AppRequirement
from .edit.interactive_products import AppProduct
from .edit.interactive_constraints import AppConstraint
from .edit.interactive_maturity import AppMaturity
from .analyze.app_analyze import AppAnalyze
# from .edit import AppProduct
# from .edit import AppConstraint

from bop.gui.window import BopMainWindow,BopMainWindowThread
from .env import AppEnv

from threading import Thread, Semaphore
from queue import Empty
from time import sleep

class Interactive(cmd2.Cmd):
	def __init__(self, *args, **kwargs):
		# gotta have this or neither the plugin or cmd2 will initialize
		super().__init__(*args, allow_cli_args=False, auto_load_commands=True,
						 persistent_history_file="~/.bop.history",persistent_history_length=100, **kwargs)
		self.prompt = "Bop>"

		self.intro = f"Welcome to the Build Objects Properly (BOP) tool.\n"
		self.gui_thread : BopMainWindowThread = None
		self.monitor_thread : BopMainWindowThread = None

		self.register_postloop_hook(self._ensure_gui_cleanup)
		self.register_precmd_hook(self._get_command_lock)
		self.register_cmdfinalization_hook(self._release_command_lock)

		self._cmd_lock = Semaphore()

	def _start_monitor_thread(self):
		def monitor():
			while True:
				sleep(0.1)
				with AppEnv().cmd_queue_lock :
					try :
						while True:
							elt = AppEnv().cmd_control_queue.get(block=False)
							if elt == "exit" :
								while not AppEnv().cmd_control_queue.empty() :
									AppEnv().cmd_control_queue.get_nowait()
								return
							print(elt)
							self.onecmd_plus_hooks(elt,add_to_history=False)
							print(self.prompt,end="")
					except Empty:
						pass

		if self.monitor_thread is None or not self.monitor_thread.is_alive() :
			self.monitor_thread = Thread(target=monitor)
			self.monitor_thread.start()

	def _get_command_lock(self, _ : cmd2.plugin.PrecommandData) -> cmd2.plugin.PrecommandData:
		self._cmd_lock.acquire()
		return _

	def _release_command_lock(self, _ : cmd2.plugin.CommandFinalizationData) -> cmd2.plugin.CommandFinalizationData:
		self._cmd_lock.release()
		return _

	def _ensure_gui_cleanup(self) -> None:
		if self.gui_thread is not None and self.gui_thread.is_alive():
			self.poutput("Cleanup the GUI")
			self.gui_thread.stop()
			self.gui_thread.join()

	# Add "aliases"
	do_exit = cmd2.Cmd.do_quit
	do_source = cmd2.Cmd.do__relative_run_script

	def do_gui(self,_):
		"""Start the GUI"""
		if self.gui_thread is not None and self.gui_thread.is_alive():
			self.poutput("Await for already opened window to be closed.")
			self.gui_thread.join()

		def gui_function() :
			root = BopMainWindow()
			root.mainloop()

		self.poutput("Starting BOP GUI.")
		self._start_monitor_thread()
		self.gui_thread = BopMainWindowThread()
		self.gui_thread.start()

	def update_prompt(self):
		self.prompt = f"Bop {AppEnv().db.name}> "
		self.intro = f"Welcome to the Build Objects Properly (BOP) tool.\n" \
					 f"Database is {AppEnv().db.name}"

	def _default_subcommand_stub(self, command : str, ns : argparse.Namespace):
		handler = ns.cmd2_handler.get()
		if handler is not None:
			# Call whatever subcommand function was selected
			handler(ns)
		else:
			# No subcommand was provided, so call help
			self.poutput('This command does nothing without sub-parsers registered')
			self.do_help(command)


#!/usr/bin/env python3
import click
import trimesh
import numpy as np

def mesh2arrayCentered(mesh, voxel_size=1, array_length=32):
    # given array length 64, voxel size 2, then output array size is [128,128,128]
    array_size = np.ceil(
        np.array([array_length, array_length, array_length]) / voxel_size
    ).astype(int)
    vox_array = np.zeros(
        array_size, dtype=bool
    )  # tanh: voxel representation [-1,1], sigmoid: [0,1]
    # scale mesh extent to fit array_length
    max_length = np.max(np.array(mesh.extents))
    mesh = mesh.apply_transform(
        trimesh.transformations.scale_matrix((array_length - 1) / max_length)
    )  # now the extent is [array_length**3]
    v = mesh.voxelized(voxel_size)  # max voxel array length = array_length / voxel_size

    # find indices in the v.matrix to center it in vox_array
    indices = ((array_size - v.matrix.shape) / 2).astype(int)
    vox_array[
        indices[0] : indices[0] + v.matrix.shape[0],
        indices[1] : indices[1] + v.matrix.shape[1],
        indices[2] : indices[2] + v.matrix.shape[2],
    ] = v.matrix

    return vox_array

@click.command()
@click.option("--input", "-i", required=True)
@click.option("--task", "-t", type=click.Choice(["check", "voxelize"]))
def cli(input, task):
    print(f"Loading {input}")
    mesh = trimesh.load(input, force="mesh")
    if task == "voxelize":
        array = mesh2arrayCentered(mesh)
        voxelmesh = trimesh.voxel.VoxelGrid(
            trimesh.voxel.encoding.DenseEncoding(array)
        ).marching_cubes
        # voxelmesh.show()
        print("ok")

main = cli

if __name__ == "__main__":
    cli()


import json

# 节点列表，存放病例信息
list_nodes = []
# 边列表，存放传染关系
list_edges = []
# 输出json文件的字典
outdic = {
    "nodes": list_nodes,
    "edges": list_edges
}

def add_a_node(pid, phone, name, gender, diagnosedTime, level, type):
    tmp_dic = {
        "pid": pid,
        "phone": phone,
        "name": name,
        "gender": gender,
        "diagnosedTime": diagnosedTime,
        "level": level,
        "type": type
    }
    list_nodes.append(tmp_dic)

def add_a_edge(id, source, target, relation, isTruth):
    tmp_dic = {
        "id": id,
        "source": source,
        "target": target,
        "relation": relation,
        "isTruth": isTruth
    }
    list_edges.append(tmp_dic)

if __name__ == '__main__':
    with open("reason_result.json", 'r', encoding='utf-8') as fw:
        reason_result = json.load(fw)

    edge_num = 0
    for i in range(len(reason_result)):
        if reason_result[i].get('recorded') == 0:
            list_nodes.append(reason_result[i])
            continue
        pid = reason_result[i].get('pid')
        phone = "12345678901"                # search_database
        name = reason_result[i].get('name')
        gender = "男"                        # search_database
        diagnosedTime = "2021-11-13"         # search_database
        if reason_result[i].get('father') != -1:
            level = str(int(list_nodes[reason_result[i].get('father')].get('level')) + 1)
        else:
            level = "0"
        typee = "确诊"                        # search_database

        add_a_node(pid, phone, name, gender, diagnosedTime, level, typee)

        if reason_result[i].get('father') != -1:
            id = str(edge_num)
            source = str(reason_result[i].get('father'))
            target = str(i)
            relation = reason_result[i].get('relation')
            isTruth = reason_result[i].get('isTruth')

            edge_num += 1
            add_a_edge(id, source, target, relation, isTruth)

    with open("test.json", 'w', encoding='utf-8') as fw:
        json.dump(outdic, fw, indent=4, ensure_ascii=False)


# Solved:
# (E) Search Insert Position
# https://leetcode.com/problems/search-insert-position/
# BINS: Search for leftmost

from typing import List

class Solution:
    def searchInsert(self, nums: List[int], target: int) -> int:
        # We should find leftmost 'target' value in the list of nums
        l = 0
        r = len(nums) - 1
        while r - l > 1:
            m = (l + r) // 2
            if target <= nums[m]:
                r = m
            else:
                l = m + 1
        if nums[r] < target:
            return r + 1
        elif nums[l] < target:
            return l + 1
        else:
            return l

x = Solution()

nums = [1]

for i in range(0, 11):
    print(f"{i} -> {x.searchInsert(nums, i)}")


import cv2
import subprocess

cap = cv2.VideoCapture(0)

while True:
    ret , photo = cap.read()
    resize_vid = cv2.resize(photo, None , fx=0.5 , fy = 0.5 , interpolation = cv2.INTER_AREA)
    photo[0:240,0:320] = resize_vid
    subprocess.getoutput("ret , photo1 = cap.read()")
    subprocess.getoutput("resize_vid1 = cv2.resize(photo1, None , fx=0.5 , fy = 0.5 , interpolation = cv2.INTER_AREA)")
    subprocess.getoutput("")
    cv2.imshow("MyVideoCam",photo)
    if cv2.waitKey(1) == 13:
        break

cv2.destroyAllWindows()
cap.release()


import os
import sys
from functools import partial

# >>>>>>  Package Imports <<<<<<<
import numpy as np

# >>>>>>  Local Imports   <<<<<<<
from barl import BarlException
from .interfaces import StateLessEnvironment

####################################################
# CODE
####################################################

class MultiArmedBandit(StateLessEnvironment):
    """
    MultiArmedbandit - MAB
    """
    #Allowed Distributions
    distributionMapping = {
        "Normal" : np.random.normal,
    }

    def __init__(self, arms=None, means=None, variances=None, distribution="Normal"):
        if arms:
            self.arms = arms
            if not means:
                means = MultiArmedBandit.get_means( arms )

            if not variances:
                variances = MultiArmedBandit.get_variances( arms )
        else:
            if not means:
                raise BarlException("Need to provide Means to Multiarmed Bandit")
            else:
                self.arms = len(means)

            if not variances:
                variances = MultiArmedBandit.get_variances( self.arms )

        self.__means = means
        #to allow single number input
        flag = type(variances)==np.ndarray or type(variances)==list
        self.__variances = variances if flag else np.ones(self.arms) * variances

        self.__distribution = distribution

        self.__set_arms()

    def sample_rewards(self, action=None, N=1):
        """
        samples rewards: np.array ( size[__arms,N] )
        """
        if type(action)==type(None):

            outList = []

            for arm in self.__arms:

                outList.append( arm(N) )

            rewards = np.array( outList )

        else:

            rewards = self.__arms[ int(action) ](N)

        return np.squeeze( rewards )

    def get_true_variances(self):
        """
        """
        return self.__variances

    def get_true_means(self):
        """
        """
        return self.__means

    def __set_arms(self):
        """
        sets the arm properties
        """

        self.__arms = []

        distro = MultiArmedBandit.distributionMapping[self.__distribution]

        for mean, variance in zip(self.__means,self.__variances):

            temp = partial(distro, mean, variance)

            self.__arms.append(temp)

    @staticmethod
    def get_variances(arms = 2):
        """
        returns [v1,v2, ...v___arms] sampled from ~Unif(0,1)
        """
        return np.random.beta(a=1,b=1,size=[arms])

    @staticmethod
    def get_means(arms = 2):
        """
        returns [m1,m2, ...m___arms] sampled from ~Normal(0,1)
        """
        return np.random.normal(loc=0.0, scale=1.0, size=[arms])

####################################################
# MAIN
####################################################

# EOF


"""
Code for accessing the data in the database easily.
"""
import math
import os
import shutil
import patoolib
import requests
import torch
import numpy as np
import pandas as pd
import imageio
from skimage import transform
import sklearn.utils
import gdown
from torch.utils.data import Dataset

from utility import to_normalized_range, seed_all

database_directory = '../Steering Angle Database/Steering Angle Database/driving_dataset/driving_dataset/'

class SteeringAngleDataset(Dataset):
    """The dataset class for the age estimation application."""
    def __init__(self, start=None, end=None, seed=None, batch_size=None):
        seed_all(seed)
        self.dataset_path = database_directory
        #with open(os.path.join(self.dataset_path, 'data.txt')) as f:
        meta = pd.read_csv(os.path.join(self.dataset_path, 'data.txt'),header=None, delimiter =" ")
        #meta = pd.read_pickle(os.path.join(self.dataset_path, 'meta.pkl'))
        meta = sklearn.utils.shuffle(meta, random_state=seed)  # Shuffles only first axis.
        image_names = meta.iloc[:, 0].values
        angles = meta.iloc[:, 1].values
        #print(angles)
        #from sklearn import preprocessing
        #min_max_scaler = preprocessing.MinMaxScaler()
        #angles = min_max_scaler.fit_transform(angles.reshape(-1, 1)).reshape(-1)
        self.image_names = np.array(image_names[start:end])
        self.angles = np.array(angles[start:end], dtype=np.float32)
        # Force full batch sizes
        if self.image_names.shape[0] < batch_size:
            repeats = math.ceil(batch_size / self.image_names.shape[0])
            self.image_names = np.repeat(self.image_names, repeats)
            self.angles = np.repeat(self.angles, repeats)
        self.length = self.angles.shape[0]
        self.image_size = 128

    def __len__(self):
        return self.length

    def __getitem__(self, index):
        image_name = self.image_names[index]
        image = np.load(os.path.join(self.dataset_path, image_name.replace('.jpg', '.npy')))
        image = torch.tensor(image.astype(np.float32))
        image = to_normalized_range(image)
        angle = self.angles[index]
        angle = torch.tensor(angle, dtype=torch.float32)
        return image, angle

class SteeringAngleDatabaseProcessor:
    """A class for preparing the Sully Chen steering angle database."""
    def __init__(self, preprocessed_image_size=128):
        self.preprocessed_image_size = preprocessed_image_size

    def download_and_preprocess(self):
        """Downloads and preprocesses the database."""
        print('Preparing steering angle database.')
        print('Downloading...')
        #self.download()
        print('Preprocessing...')
        self.preprocess()

    @staticmethod
    def download():
        """Downloads the database."""
        if os.path.exists(database_directory):
            shutil.rmtree(database_directory)
        os.makedirs(database_directory)
        SteeringAngleDatabaseProcessor.download_file_from_google_drive('1M1AOX3T5GqYMpU6y13dN24GQ22ILTlJK',
                                                                       os.path.join(database_directory, 'temporary'))
        patoolib.extract_archive(os.path.join(database_directory, 'temporary'), outdir=database_directory)
        default_directory_name = 'driving_dataset'
        files = os.listdir(os.path.join(database_directory, default_directory_name))
        for file_ in files:
            if not file_.startswith('.'):
                shutil.move(os.path.join(database_directory, default_directory_name, file_), database_directory)
        shutil.rmtree(os.path.join(database_directory, default_directory_name))
        os.remove(os.path.join(database_directory, 'temporary'))

    @staticmethod
    def download_file_from_google_drive(id_, destination):
        url = 'https://drive.google.com/uc?id=' + id_
        output = destination
        gdown.download(url, output, quiet=False)

    def preprocess(self):
        """Preprocesses the database to the format needed by the network."""
        meta_file_path = os.path.join(database_directory, 'data.txt')
        meta = pd.read_csv(meta_file_path, delimiter=' ', header=None)
        meta = meta[meta[0] != '45567.jpg']  # Corrupt image.
        meta.to_pickle(os.path.join(database_directory, 'meta.pkl'))
        for file_name in meta.iloc[:, 0].values:
            if file_name.endswith('.jpg'):
                file_path = os.path.join(database_directory, file_name)
                image = imageio.imread(file_path).astype(np.uint8)
                image = transform.resize(image, (self.preprocessed_image_size, self.preprocessed_image_size),
                                         preserve_range=True)
                image = image.transpose((2, 0, 1))
                np.save(file_path.replace('.jpg', '.npy'), image)

if __name__ == '__main__':
    SteeringAngleDatabaseProcessor().download_and_preprocess()


from django.template.loader import render_to_string

from account.models import PublicNetwork

from resource_inventory.models import Vlan

class IDFTemplater:
    """Utility class to create a full Installer Descriptor File (IDF) yaml file."""

    net_names = ["admin", "mgmt", "private", "public"]
    bridge_names = {
        "admin": "br-admin",
        "mgmt": "br-mgmt",
        "private": "br-private",
        "public": "br-public"
    }

    def __init__(self):
        self.networks = {}
        for i, name in enumerate(self.net_names):
            self.networks[name] = {
                "name": name,
                "vlan": -1,
                "interface": i,
                "ip": "10.250." + str(i) + ".0",
                "netmask": 24
            }

    def makeIDF(self, booking):
        """Fill the IDF template with info about the resource."""
        template = "dashboard/idf.yaml"
        info = {}
        info['version'] = "0.1"
        info['net_config'] = self.get_net_config(booking)
        info['fuel'] = self.get_fuel_config(booking)

        return render_to_string(template, context=info)

    def get_net_config(self, booking):
        net_config = {}
        try:
            net_config['oob'] = self.get_oob_net(booking)
        except Exception:
            net_config['oob'] = {}
        try:
            net_config['public'] = self.get_public_net(booking)
        except Exception:
            net_config['public'] = {}

        for net in [net for net in self.net_names if net != "public"]:
            try:
                net_config[net] = self.get_single_net_config(booking, net)
            except Exception:
                net_config[net] = {}

        return net_config

    def get_public_net(self, booking):
        public = {}
        config = booking.opnfv_config
        public_role = config.networks.get(name="public")
        public_vlan = Vlan.objects.filter(network=public_role.network).first()
        public_network = PublicNetwork.objects.get(vlan=public_vlan.vlan_id, lab=booking.lab)
        self.networks['public']['vlan'] = public_vlan.vlan_id
        public['interface'] = self.networks['public']['interface']
        public['vlan'] = public_network.vlan  # untagged??
        public['network'] = public_network.cidr.split("/")[0]
        public['mask'] = public_network.cidr.split("/")[1]
        # public['ip_range'] = 4  # necesary?
        public['gateway'] = public_network.gateway
        public['dns'] = ["1.1.1.1", "8.8.8.8"]

        return public

    def get_oob_net(self, booking):
        net = {}
        hosts = booking.resource.hosts.all()
        addrs = [host.remote_management.address for host in hosts]
        net['ip_range'] = ",".join(addrs)
        net['vlan'] = "native"
        return net

    def get_single_net_config(self, booking, net_name):
        config = booking.opnfv_config
        role = config.networks.get(name=net_name)
        vlan = Vlan.objects.filter(network=role.network).first()
        self.networks[net_name]['vlan'] = vlan.vlan_id
        net = {}
        net['interface'] = self.networks[net_name]['interface']
        net['vlan'] = vlan.vlan_id
        net['network'] = self.networks[net_name]['ip']
        net['mask'] = self.networks[net_name]['netmask']

        return net

    def get_fuel_config(self, booking):
        fuel = {}
        fuel['jumphost'] = {}
        try:
            fuel['jumphost']['bridges'] = self.get_fuel_bridges()
        except Exception:
            fuel['jumphost']['bridges'] = {}

        fuel['network'] = {}
        try:
            fuel['network']['nodes'] = self.get_fuel_nodes(booking)
        except Exception:
            fuel['network']['nodes'] = {}

        return fuel

    def get_fuel_bridges(self):
        return self.bridge_names

    def get_fuel_nodes(self, booking):
        jumphost = booking.opnfv_config.host_opnfv_config.get(
            role__name__iexact="jumphost"
        )
        hosts = booking.resource.hosts.exclude(pk=jumphost.pk)
        nodes = []
        for host in hosts:
            node = {}
            ordered_interfaces = self.get_node_interfaces(host)
            node['interfaces'] = [iface['name'] for iface in ordered_interfaces]
            node['bus_addrs'] = [iface['bus'] for iface in ordered_interfaces]
            nodes.append(node)

        return nodes

    def get_node_interfaces(self, node):
        # TODO: this should sync with pdf ordering
        interfaces = []

        for iface in node.interfaces.all():
            interfaces.append({"name": iface.name, "bus": iface.bus_address})

        return interfaces


"""nine URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.8/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Add an import:  from blog import urls as blog_urls
    2. Add a URL to urlpatterns:  url(r'^blog/', include(blog_urls))
"""
from django.conf.urls import include, url
from django.contrib import admin

urlpatterns = [
    url(r'^aaaaa/', include(admin.site.urls)),  # 防扫描后台入口
    url(r'^$', 'NineCo.views.Index', name='home'),
    url(r'^About/$', 'NineCo.views.summary', name='About us'),
    url(r'^contact/', 'NineCo.views.contact'),
    url(r'^jobs/', 'NineCo.views.jobs'),
    url(r'^games/', 'NineCo.views.gamelist'),
    url(r'^News/$', 'NineCo.views.NewsPage', name='news page'),
    url(r'^news/(?P<newsid>\d)/$', 'NineCo.views.NewsDetail'),
    url(r'^gameclass/', 'NineCo.views.gamecl'),
    url(r'^game/(\d+)/$', 'NineCo.views.gamed'),
    url(r'^login', 'NineCo.views.login'),
    url(r'^logout', 'NineCo.views.logout'),
    url(r'^reg', 'NineCo.views.regist'),
    url(r'^NewsOfBus', 'NineCo.views.NewsOfB'),
    url(r'^gamecenter', 'NineCo.views.gameCenter'),
    url(r'^static/(?P<path>.*)$', 'django.views.static.serve',
        {'document_root': '/static/'}),

    url(r'^Forum/$', 'NineCo.views.ForumCenter'),
    url(r'^forums/(?P<forumsid>\d)/$', 'NineCo.views.ForumDetail'),

    url(r'^balagw/index.html', 'NineCo.views.BalagwIndex'),
    url(r'^mvfenxiang/fenxiang.html', 'NineCo.views.fenxiang'),
    url(r'^fenxiang/fenxiang.html', 'NineCo.views.fenxiang2'),
    url(r'^bb/index.html', 'NineCo.views.bbindex'),
    url(r'^balala2fx/index.html', 'NineCo.views.balala2fx'),
    url(r'^h5fen/index.html', 'NineCo.views.h5fenindex'),
]  # 加上集合tomcat的静态页面


# import
## batteries
import re
import os
import sys
import gzip
import tempfile
import multiprocessing
import argparse
import logging
import urllib
import tarfile
## 3rd party
import pandas as pd
## package
from leylab_pipelines import Utils

# logging
logging.basicConfig(
    level=logging.DEBUG, format='%(asctime)s|%(levelname)s|%(message)s')

# functions
def get_desc():
    desc = 'Convert an NCBI taxonomy (taxdump) into lineages (taxID<-->lineage)'
    return desc

def parse_args(test_args=None, subparsers=None):
    # desc
    desc = get_desc()
    epi = """DESCRIPTION:
    Convert the NCBI taxonomy dump to a lineage table.
    The lineage table can be used to map taxonomy IDs to taxonomy.

    If the specific taxonomy dump files are not provided, this command
    will download the full dump file, uncompress it, and use the needed
    files. By default, the dump files are downloaded to a temporary directory.
    You can keep the taxonomy dump by using the --outdir option.

    TO CONVERT ACCESSION TO TAX_ID:
      see ftp://ftp.ncbi.nih.gov/pub/taxonomy/accession2taxid/nucl_gb.accession2taxid.gz
    """
    if subparsers:
        parser = subparsers.add_parser('taxID2lintbl', description=desc, epilog=epi,
                                       formatter_class=argparse.RawTextHelpFormatter)
    else:
        parser = argparse.ArgumentParser(description=desc, epilog=epi,
                                         formatter_class=argparse.RawTextHelpFormatter)

    # args
    io = parser.add_argument_group('File input/output')
    io.add_argument('--nodes', default=None,
                    help='nodes.dmp file path. (default: %(default)s)')
    io.add_argument('--names', default=None,
                    help='names.dmp file path. (default: %(default)s)')
    io.add_argument('-o', '--outfile', default='NCBI_taxID2lin.txt',
                    help='Output file for lineage table (default: %(default)s)')

    dmp = parser.add_argument_group('Taxonomy dump')
    dmp.add_argument('-u', '--url', default='ftp://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz',
                    help='URL for downloading taxonomy dump. (default: %(default)s)')
    dmp.add_argument('-d', '--outdir', default=None,
                     help='Output directory for the taxonomy dump download. (default: %(default)s)')
    dmp.add_argument('-a', '--all', action='store_true', default=False,
                     help='Output all taxonomy levels instead of just the standard levels. (default: %(default)s)')

    misc = parser.add_argument_group('Misc')
    misc.add_argument('-p', '--procs', default=1,
                      help='Number of processors to use. (default: %(default)s)')

    # running test args
    if test_args:
        args = parser.parse_args(test_args)
        return args

def strip(str_):
    '''
    :param str_: a string
    '''
    return str_.strip()

def load_nodes(nodes_file):
    '''
    load nodes.dmp and convert it into a pandas.DataFrame
    '''
    df = pd.read_csv(nodes_file, sep='|', header=None, index_col=False,
                     names=[
                         'tax_id',
                         'parent_tax_id',
                         'rank',
                         'embl_code',
                         'division_id',
                         'inherited_div_flag',
                         'genetic_code_id',
                         'inherited_GC__flag',
                         'mitochondrial_genetic_code_id',
                         'inherited_MGC_flag',
                         'GenBank_hidden_flag',
                         'hidden_subtree_root_flag',
                         'comments'
                     ])

    # To get rid of flanking tab characters
    df['rank'] = df['rank'].apply(strip)
    df['embl_code'] = df['embl_code'].apply(strip)
    df['comments'] = df['comments'].apply(strip)
    return df

def load_names(names_file):
    '''
    load names.dmp and convert it into a pandas.DataFrame
    '''
    df = pd.read_csv(names_file, sep='|', header=None, index_col=False,
                     names=[
                         'tax_id',
                         'name_txt',
                         'unique_name',
                         'name_class'
                     ])
    df['name_txt'] = df['name_txt'].apply(strip)
    df['unique_name'] = df['unique_name'].apply(strip)
    df['name_class'] = df['name_class'].apply(strip)

    sci_df = df[df['name_class'] == 'scientific name']
    sci_df.reset_index(drop=True, inplace=True)
    return sci_df

def to_dict(lineage):
    """
    convert the lineage into a list of tuples in the form of

    [
        (tax_id1, rank1, name_txt1),
        (tax_id2, rank2, name_txt2),
        ...
    ]

    to a dict
    """
    dd = {}
    num_re = re.compile('[0-9]+')
    len_lineage = len(lineage)
    for k, __ in enumerate(lineage):
        tax_id, rank, name_txt = __
        # use the last rank as the tax_id, whatever it is, genus or species.
        if k == len_lineage - 1:
            dd['tax_id'] = tax_id

        # e.g. there could be multiple 'no rank'
        numbered_rank = rank
        while numbered_rank in dd:
            # print __, numbered_rank
            search = num_re.search(numbered_rank)
            if search is None:
                count = 1
            else:
                count = int(search.group()) + 1
            numbered_rank = '{0}{1}'.format(rank, count)
        dd[numbered_rank] = name_txt
    return dd

def find_lineage(tax_id):
    if tax_id % 50000 == 0:
        logging.debug('working on tax_id: {0}'.format(tax_id))
    lineage = []
    while True:
        rec = TAXONOMY_DICT[tax_id]
        lineage.append((rec['tax_id'], rec['rank'], rec['name_txt']))
        tax_id = rec['parent_tax_id']

        if tax_id == 1:
            break

    # reverse results in lineage of Kingdom => species, this is helpful for
    # to_dict when there are multiple "no rank"s
    lineage.reverse()
    return to_dict(lineage)

def get_taxdump(url, outDir=None):
    """Getting taxdump file from NCBI.
    Saving to a temporary directory.
    """
    logging.info('downloading NCBI taxonomy dump...')

    # downloading
    if outDir:
        # saving taxdump
        dmpFile = os.path.join(outDir, 'taxdump.tar.gz')
    else:
        # taxdump written to temporary directory
        outDir = tempfile.gettempdir()
        dmpFile = None
    if sys.version_info[0] >= 3:
        dmpFile, headers = urllib.request.urlretrieve(url, filename=dmpFile)
    else:
        dmpFile, headers = urllib.urlretrieve(url, filename=dmpFile)

    # uncompressing
    logging.info('uncompressing NCBI taxonomy dump file...')
    tar = tarfile.open(dmpFile, "r:gz")
    tar.extractall(path=outDir)
    tar.close()
    logging.info('files uncompressed to: {}'.format(outDir))

    # target file names
    files = {'nodes' : os.path.join(outDir, 'nodes.dmp'),
             'names' : os.path.join(outDir, 'names.dmp')}
    ## checking for existence
    for k,v in files.items():
        if not os.path.isfile(v):
            raise ValueError('Cannot find file: {}'.format(v))
    # ret
    return files

def main(args=None):
    # Input
    if args is None:
        args = parse_args()

    # data downloaded from ftp://ftp.ncbi.nih.gov/pub/taxonomy/
    if args.outdir is not None or args.nodes is None or args.names is None:
        files = get_taxdump(args.url, args.outdir)
        args.nodes = files['nodes']
        args.names = files['names']

    nodes_df = load_nodes(args.nodes)
    names_df = load_names(args.names)
    df = nodes_df.merge(names_df, on='tax_id')
    df = df[['tax_id', 'parent_tax_id', 'rank', 'name_txt']]
    df.reset_index(drop=True, inplace=True)
    logging.info('# of tax ids: {0}'.format(df.shape[0]))
    # log summary info about the dataframe
    df.info()

    # force to use global variable TAXONOMY_DICT because map doesn't allow
    # passing extra args easily
    global TAXONOMY_DICT
    logging.info('generating TAXONOMY_DICT...')
    TAXONOMY_DICT = dict(zip(df.tax_id.values, df.to_dict('records')))

    ncpus = int(args.procs)
    logging.info('using {0} cpus to find lineages for all tax ids'.format(ncpus))
    pool = multiprocessing.Pool(ncpus)
    lineages_dd = pool.map(find_lineage, df.tax_id.values)
    pool.close()

    logging.info('generating a dictionary of lineages information...')
    dd_for_df = dict(zip(range(len(lineages_dd)), lineages_dd))

    logging.info('generating lineages_df...')
    lineages_df = pd.DataFrame.from_dict(dd_for_df, orient='index')
    lineages_df.sort_values('tax_id', inplace=True)

    logging.info('writing lineages to: {}'.format(args.outfile))
    cols = ['tax_id',
            'superkingdom',
            'phylum',
            'class',
            'order',
            'family',
            'genus',
            'species']

    other_cols = sorted([__ for __ in lineages_df.columns if __ not in cols])
    if args.all:
        output_cols = cols + other_cols
    else:
        output_cols = cols
    lineages_df.to_csv(args.outfile, index=False, columns=output_cols, sep='\t')


import torch
import numpy as np
import torch.nn as nn

x_values = list(range(11))
x_train = np.array(x_values, dtype=np.float32)
print(x_train.shape)

x_train = x_train.reshape(-1, 1)
print(x_train.shape)

y_values = [2 * i + 1 for i in x_values]
print(y_values)

y_train = np.array(y_values, dtype=np.float32)
y_train = y_train.reshape(-1, 1)
print(y_train.shape)

class LinearRegressionModel(nn.Module):
    def __init__(self, input_size, output_size):
        super(LinearRegressionModel, self).__init__()
        self.linear = nn.Linear(input_size, output_size)

    def forward(self, x):
        out = self.linear(x)
        return out

input_dim = 1
out_dim = 1

model = LinearRegressionModel(input_dim, out_dim)
criterion = nn.MSELoss()
if torch.cuda.is_available():
    model.cuda()

learning_rate = 0.01
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

epochs = 10

for epoch in range(epochs):
    inputs = torch.from_numpy(x_train)
    labels = torch.from_numpy(y_train)
    # Clear gradients
    if torch.cuda.is_available():
        inputs.cuda()
        labels.cuda()
    optimizer.zero_grad()
    outputs = model(inputs)

    loss = criterion(outputs, labels)
    loss.backward()
    optimizer.step()

    print('epoch {}, loss {}'.format(epoch + 1, loss.item()))

# torch.save(model.state_dict(), "liner.pk1")

model.load_state_dict(torch.load('liner.pk1'))
print(model(inputs).data.numpy())
print(model.parameters())