import tasktiger
from redis import Redis
from rpy2.robjects.packages import importr
from rpy2 import rinterface
import rpy2.robjects as ro
from email.MIMEMultipart import MIMEMultipart
from email.MIMEText import MIMEText
from email.MIMEBase import MIMEBase
import smtplib
from rpy2.robjects import ListVector
import sys # DELETE
import multiprocessing
import rpy2.robjects as robjects
import subprocess

rmarkdown = importr('rmarkdown')

conn = Redis(host="redis")
tiger = tasktiger.TaskTiger(connection=conn)

NECESSARY_FIELDS = ['dataFilePath', 'classFilePath', 'gmtFilePath']
ACCEPTED_FIELDS = ['outFilePath', 'classificationAlgorithm', 'numCrossValidationFolds', 'numRandomIterations',
                   'numCores', 'removePercentLowestExpr', 'removePercentLowestVar'] + NECESSARY_FIELDS
# takes the stuff from the queque
@tiger.task()
def call_gsoa(request):
    local_buffer = []
    def append_output(line):
        print(line)
        local_buffer.append(line)
    # data from task tiger
    print("request: {}".format(request))
    try:
        rmarkdown = importr('rmarkdown')
        args = request.copy()
        for field in NECESSARY_FIELDS:
            args.pop(field)
        if len(str(request.get('dataFilePath'))) < 2:
            return "no data"
        outFilePath = "/data/{}_{}.txt".format(request.get('email', 'results_txt').replace('.com', '').strip(),request.get('dataFilePath').split(".")[0])
        print("email: {}".format(request.get('email', 'results_txt')))
        cmd = ["/gsoa/scripts/run", request.get('dataFilePath', ''),request.get('classFilePath', ''),
               request.get('gmtFilePath', ''), outFilePath, multiprocessing.cpu_count(), "/dev/null",
               request.get('numCrossValidationFolds', ''), "", request.get('numCrossValidationFolds', ''),
               request.get('classificationAlgorithm', '')]

        process = subprocess.Popen(cmd, shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
        # wait for the process to terminate
        out, err = process.communicate()
        errcode = process.returncode
        if errcode > 0 :
           email_error(request.get('email'), "", out+err)

        print("Writing RMarkdown")
        outFilePath_html=outFilePath.replace('txt', 'html')
        rmarkdown.render('/app/GSOA_Report.Rmd', output_file = outFilePath.replace('txt', 'html'),
            params=ListVector({'data1': outFilePath,  'alg': request.get('classificationAlgorithm', 'svm') ,
            'class': request.get('classFilePath', ''),
            'crossval': request.get('numCrossValidationFolds', ''),
            'data_files' : request.get('dataFilePath', ''),
            'genesets': request.get('gmtFilePath', ''),
            #'hallmarks':
            'iterations': request.get('numRandomIterations', ''),
            'lowexpress' : request.get('removePercentLowestExpr', ''),
            #'results_hallmark' :
            'var': request.get('removePercentLowestVar', '')}))
        email_report(request.get('email'), outFilePath)
    except Exception as e:
        email_error(request.get('email'), e, "")

def email_report(email_address, file_path):
    from_ = 'gsoa.app@gmail.com'
    msgRoot = MIMEMultipart('related')
    msgRoot['Subject'] = 'GSOA Results'
    msgRoot['From'] = from_
    msgRoot['To'] = email_address
    body = "GSOA ran successfully! \n Your raw results are in the '.txt' file. \n The GSOA report is in the 'HTML' file (please downloand HTML file before viewing in web browser) \n Thank you for using GSOA!"
    msgRoot.attach(MIMEText(body, 'plain'))
    #msg = MIMEMultipart('alternative')
    text = MIMEText('Results File', 'plain')
    #msgRoot.attach(msg)
    with open(file_path) as fp:
        attachment = MIMEBase('text', None)
        attachment.set_payload(fp.read())
        attachment.add_header('Content-Disposition', 'attachment', filename=file_path.split('/')[-1])
    msgRoot.attach(attachment)

    with open(file_path.replace('txt', 'html')) as fp:
        attachment1 = MIMEBase('html', None)
        attachment1.set_payload(fp.read())
        attachment1.add_header('Content-Disposition', 'attachment', filename=file_path.split('/')[-1].replace('txt', 'html'))
    msgRoot.attach(attachment1)
    #msg.attach(text)
    mailer = smtplib.SMTP('smtp.gmail.com:587')
    mailer.starttls()
    mailer.login('gsoa.app', 'p@thway@nalysi$')
    mailer.sendmail(from_, email_address, msgRoot.as_string())
    mailer.close()

def email_error(email_address, exception, local_buffer):
    from_ = 'gsoa.app@gmail.com'
    msg = MIMEMultipart()
    msg['From'] = from_
    msg['To'] = email_address
    msg['Subject'] = "GSOA ERROR"
    #msg.preamble = 'GSOA Returned the Following Error:'
    body = "GSOA Returned the Following Error: \n 'Error message: {}: \n {} \n Please email gsoa.app@gmail.com with further questions".format(exception, '\n'.join(local_buffer))
    msg.attach(MIMEText(body, 'plain'))
    #msg.attach(text)
    mailer = smtplib.SMTP('smtp.gmail.com:587')
    mailer.starttls()
    mailer.login('gsoa.app', 'p@thway@nalysi$')
    mailer.sendmail(from_, email_address, msg.as_string())
    mailer.close()


from __future__ import print_function
import numpy as np

np.random.seed(2668)  # for reproducibility
from keras.preprocessing import sequence
from keras.utils import np_utils
from keras.models import Sequential,load_model,Model
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import *
from keras.optimizers import SGD
from random import shuffle
import time
from keras.utils import plot_model
import matplotlib.pyplot as plt
from PIL import Image
import xml.dom.minidom as xmldom
import os
import cv2 as cv
from keras.preprocessing.image import ImageDataGenerator
from segmentation_models.backbones import get_preprocessing
from segmentation_models.losses import bce_jaccard_loss
from segmentation_models.metrics import iou_score
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import EarlyStopping
import sherpa

def load():
  imgList=[]
  labelList=[]
  files = os.listdir('D:/most/processedXR/side/training/') #training image path
  shuffle(files)
  for file in files:
       if file.endswith(".xml"):continue
       fi_d = os.path.join('D:/most/processedXR/side/training/',file) #training image path
       img=Image.open(fi_d)
       img=img.crop((0,0,1520,2047))
       im=np.array(img.resize((1520,2047), Image.ANTIALIAS))
       imgList.append(im)
       filename=file.split('.')[0]
       dom = xmldom.parse("D:/most/processedXR/side/training/"+filename+".xml") #training image path, where also contains ROI labels
       rois=dom.documentElement
       xmin = int(rois.getElementsByTagName("xmin")[0].firstChild.data)
       xmax = int(rois.getElementsByTagName("xmax")[0].firstChild.data)
       ymin = int(rois.getElementsByTagName("ymin")[0].firstChild.data)
       ymax = int(rois.getElementsByTagName("ymax")[0].firstChild.data)
       label= np.zeros((2047,1520))
       for i in range(xmin,xmax):
          for j in range(ymin,ymax):
             label[j][i]=1
       labelList.append(label)
  return np.array(imgList),np.array(labelList)

def load_val():
  imgList=[]
  labelList=[]
  files = os.listdir('D:/most/processedXR/side/predict/') #test path
  for file in files:
     if file.endswith(".xml"):continue
     fi_d = os.path.join('D:/most/processedXR/side/predict/',file) # test path
     img=Image.open(fi_d)
     img=img.crop((0,0,1520,2047))
     im=np.array(img.resize((1520,2047), Image.ANTIALIAS))
     imgList.append(im)
     filename=file.split('.')[0]
     dom = xmldom.parse('D:/most/processedXR/side/predict/'+filename+'.xml') #test path
     rois=dom.documentElement
     xmin = int(rois.getElementsByTagName("xmin")[0].firstChild.data)
     xmax = int(rois.getElementsByTagName("xmax")[0].firstChild.data)
     ymin = int(rois.getElementsByTagName("ymin")[0].firstChild.data)
     ymax = int(rois.getElementsByTagName("ymax")[0].firstChild.data)
     label= np.zeros((2047,1520))
     for i in range(xmin,xmax):
          for j in range(ymin,ymax):
             label[j][i]=1
     labelList.append(label)
     return np.array(imgList),np.array(labelList)

if __name__ == '__main__':
    parameters = [sherpa.Ordinal(name='batch_size', range=[16,32,64,128,256]),
                  sherpa.Ordinal(name='steps', range=[16,32,64,128,256]),
                  sherpa.Ordinal(name='epoch', range=[4096])]
    alg = sherpa.algorithms.BayesianOptimization(max_num_trials=500)
    study = sherpa.core.Study(parameters = parameters,
                         algorithm = alg,
                         lower_is_better = False,
                         disable_dashboard=True)

    for trial in study:
          model = Sequential()
          model.add(Convolution2D(filters=32, kernel_size =(3, 3), border_mode='same', input_shape=(2047,1520,1))) #if the memory is small, just try what we did for U-NET.
          model.add(BatchNormalization())
          model.add(Activation('relu'))
          model.add(MaxPooling2D(pool_size=(2,2)))
          model.add(Convolution2D(filters=32, kernel_size =(3, 3), border_mode='same'))
          model.add(BatchNormalization())
          model.add(Activation('relu'))
          model.add(Convolution2D(filters=32, kernel_size =(3, 3), border_mode='same'))
          model.add(BatchNormalization())
          model.add(Activation('relu'))
          model.add(MaxPooling2D(pool_size=(2,2)))
          model.add(Convolution2D(filters=64, kernel_size =(3, 3), border_mode='same'))
          model.add(BatchNormalization())
          model.add(Activation('relu'))
          model.add(Convolution2D(filters=64, kernel_size =(3, 3), border_mode='same'))
          model.add(BatchNormalization())
          model.add(Activation('relu'))
          model.add(MaxPooling2D(pool_size=(2,2)))
          model.add(Convolution2D(filters=96, kernel_size =(3, 3), border_mode='same'))
          model.add(BatchNormalization())
          model.add(Activation('relu'))
          model.add(Convolution2D(filters=96, kernel_size =(3, 3), border_mode='same'))
          model.add(BatchNormalization())
          model.add(Activation('relu'))
          model.add(UpSampling2D(size=(8, 8),data_format="channels_last"))
          model.add(Convolution2D(filters=1,kernel_size =(1, 1), border_mode='same'))
          model.add(Activation('sigmoid'))
          model.compile('Adam', loss=bce_jaccard_loss, metrics=[iou_score])

          datagen = ImageDataGenerator(
                  featurewise_center=False,
                  samplewise_center=True,  # set each sample mean to 0
                  featurewise_std_normalization=False,
                  samplewise_std_normalization=True)
          X_train, Y_train = load()
          X_test, Y_test = load_val()
          X_train = X_train.reshape( len(X_train), len(X_train[0]), len(X_train[0][0]),1)
          X_test = X_test.reshape( len(X_test), len(X_test[0]), len(X_test[0][0]),1)
          Y_train = Y_train.reshape( len(Y_train),len(Y_train[0]), len(Y_train[0][0]),1)
          Y_test = Y_test.reshape( len(Y_test), len(Y_test[0]), len(Y_test[0][0]),1)
          X_train = X_train.astype('float32')
          X_test = X_test.astype('float32')
          X_train /= 255
          X_test /= 255
          datagen.fit(X_train)
          for i in range(len(X_test)):
                X_test[i] = datagen.standardize(X_test[i])
          earlystop=EarlyStopping(monitor='val_iou_score', min_delta=0, patience=80, verbose=1, mode='max', restore_best_weights=True)
          callbacklist = [earlystop]
          history = model.fit_generator(datagen.flow(X_train, Y_train,batch_size=trial.parameters['batch_size']),steps_per_epoch=trial.parameters['steps'],epochs=trial.parameters['epoch'],shuffle=True,validation_data=(X_test, Y_test), verbose=1,callbacks=callbacklist)

          model.save("FCN_" + str(time.time())+".h5") #models are saved with time stamps


#import importlib ; import nlwTools  ; import nlwTests
#importlib.reload(nlwTools) ; importlib.reload(nlwTests); nlwTests.Test()

#==============================================================================

def Test():

    #test_klo_vs_psi("test/a1",     80, fnum=1)
    #test_klo_vs_psi("test/tmp_m05", 6, fnum=2)

    #test_klo_readavg("test/a1",     80, 256,  fnum=1)
    #test_klo_readavg("test/a1s55", 160, 512,  fnum=124)
    #test_klo_readavg("test/a1s55", 160, 512,  fnum=125)

    #test_interp_klo("test/a1s55", 160, 512,  fnum=124)

    #test_AvgSpectrumKlo("c1s", 60, 128, [14,15], istart = 20,  iend = -1)
    test_CorrelatorsKlo("c1s+03+05", "c1s", ((3,5),), 60, 128, [14,15], istart = 20,  iend = -1)

#==============================================================================

def test_CorrelatorsKlo(fbaseout, fbase, modes, N, No, seeds, istart = 1,  iend = -1):
#
#  reads a sequence of *.klo files and returns 2D array with average spectrum
#  file name in the form "fbase00/fbase00.klo.0000" is assumed
#
#  fbase   string file base
#  modes   tuple of modes
#  N       gridpoints in each direction
#  No      gridpoints in each direction in original simulation
#  seeds   iterable sequence of integer seeds to process
#  istart  first file for each seed
#  iend    first file not to process; last existing if iend=-1
#

    import numpy as np
    import matplotlib.pyplot as plt
    import nlwTools as nlw

    corr = nlw.CorrelatorsKlo(fbaseout, fbase, modes, N, No, seeds, istart = istart,  iend = iend)

    #-- image --

    fig, ax = plt.subplots(ncols=1, nrows=1, constrained_layout=True, figsize=(4,4))

    #a=np.fft.fftshift(np.abs(corr));
    a=np.abs(corr);
    ax.imshow(a, cmap='jet', origin = 'lower', vmin=0, vmax=1);
    print("corr min,max: ",np.min(a), np.max(a))

    plt.show();

    return()

#==============================================================================

def test_AvgSpectrumKlo(fbase, N, No, seeds, istart = 1,  iend = -1):
#
#  reads a sequence of *.klo files and returns 2D array with average spectrum
#  file name in the form "fbase00/fbase00.klo.0000" is assumed
#
#  fbase   string file base
#  N       gridpoints in each direction
#  No      gridpoints in each direction in original simulation
#  seeds   iterable sequence of integer seeds to process
#  istart  first file for each seed
#  iend    first file not to process; last existing if iend=-1
#

    import numpy as np
    import matplotlib.pyplot as plt
    import nlwTools as nlw

    spc = nlw.AvgSpectrumKlo(fbase, N, No, seeds, istart = istart,  iend = iend)

    #return()
    #-- image --

    fig, ax = plt.subplots(ncols=1, nrows=1, constrained_layout=True, figsize=(4,4))

    a=np.log10(np.fft.fftshift(spc));
    ax.imshow(a, cmap='jet', origin = 'lower', vmin=-6, vmax=-3);
    print("log10(klo) min,max: ",np.min(a), np.max(a))

    plt.show();

    return()

#==============================================================================

def test_interp_klo(fname, N, N0, fnum = 0):

    import numpy as np
    import matplotlib.pyplot as plt
    import nlwTools as nlw

    fname_klo = fname + ".klo." + str(fnum).zfill(4)

    klo = nlw.ReadKlo(fname_klo, N, N0, spectrum=False);
    n, N1, N2 = klo.shape

    nlw.KloAverageClear()

    nlw.KloAverageAdd(klo, dNk=4, Nphi=32)

    a, b, count = nlw.KloAverageGet()

    print("Spectrum from klo:  {} x {},  n = {} slices, count = {}".format(N1, N2, n, count))

    #-- image --

    fig, ax = plt.subplots(ncols=3, nrows=1, constrained_layout=True, figsize=(12,4))

    ax[0].imshow(a, cmap='jet', origin = 'lower', vmin=0, vmax=0.002);
    print("min, max: ", np.min(a), np.max(a))

    ax[1].imshow(b, cmap='jet', origin = 'lower');
    print("min, max: ", np.min(b), np.max(b))

    ax[2].plot(np.arange(b.shape[1]), np.sum(b,0), '-o');

    plt.show();

    return()

#==============================================================================

def test_klo_readavg(fname, N, N0, fnum = 0):

    import numpy as np
    import matplotlib.pyplot as plt
    import nlwTools as nlw

    n = 1  # number of snapshots in "klo" file, use n=1 for better comaprison
    L = 2*np.pi

    fname_klo = fname + ".klo." + str(fnum).zfill(4)

    klo, n = nlw.ReadKlo(fname_klo, N, N0, spectrum=True);
    (N1, N2) = klo.shape
    print("Spectrum from klo:  {} x {}, sum = {},  n = {}".format(N1, N2, np.sum(klo), n))

    klo = np.fft.fftshift(klo)

    #-- image --

    fig, ax = plt.subplots(ncols=1, nrows=1, constrained_layout=True, figsize=(4,4))

    a=np.log10(klo);
    ax.imshow(a, cmap='jet', origin = 'lower', vmin=-6, vmax=-3);
    print("log10(klo) min,max: ",np.min(a), np.max(a))

    plt.show();

    return()

#==============================================================================

def test_klo_vs_psi(fname, Nklo, fnum = 0):
#
# On normalization of FFT.
#
# See "*.dat" file for:  8.sumA_sq  9.sumAk_sq
#
#    #8.sumA_sq   = (1/2) * sum(|psi|^2) dxdy
#    #9.sumAk_sq  = sum( |A_k|^2 )
#    (#9) = (2/L^2)*(#8)
#
#    From "test/tmp_m05.dat": (#8) = 1.577169407370e-01,  (#9) = 7.990033557959e-03
#    From "test/a1.dat":      (#8) = 2.825774223384e-01,  (#9) = 1.431553945097e-02

    import numpy as np
    import matplotlib.pyplot as plt
    import nlwTools as nlw

    L = 2*np.pi

    fname_psi = fname + ".psi." + str(fnum).zfill(4)
    fname_klo = fname + ".klo." + str(fnum).zfill(4)

    #-- compare integral quantities --

    psi=0

    psi = nlw.ReadPsi(fname_psi,spectrum=False);
    (N1, N2) = psi.shape
    sumA_sq = 0.5 * np.sum( np.real(psi * np.conjugate(psi)) ) * L/N1 * L/N2
    sumAk_sq = sumA_sq * 2/L/L

    print("Integral of psi:  {} x {}, integral = {} ({})".format(
        N1, N2, sumA_sq, sumAk_sq))

    psi = nlw.ReadPsi(fname_psi, spectrum=True);
    (N1, N2) = psi.shape
    print("Spectrum from psi:  {} x {}, sum = {}".format(N1, N2, np.sum(psi)))

    N0 = N1

    klo, n = nlw.ReadKlo(fname_klo, Nklo, N0, spectrum=True);
    (N1, N2) = klo.shape
    print("Spectrum from klo:  {} x {}, sum = {}".format(N1, N2, np.sum(klo)))

    #-- recenter and trim "psi" to match "klo"

    klo = np.fft.fftshift(klo)
    psi = np.fft.fftshift(psi)

    i1 = int(N0/2 - N1/2)
    i2 = i1 + N1

    psi = psi[i1:i2, i1:i2]
    print("Trimmed psi:  {} x {}, sum = {}".format(N1, N1, np.sum(psi)))

    #-- images --

    fig, ax = plt.subplots(ncols=2, nrows=1, constrained_layout=True, figsize=(6,3))

    a=np.log10(klo);
    ax[0].imshow(a, cmap='jet', origin = 'lower', vmin=-6, vmax=-3);
    print("log10(klo) min,max: ",np.min(a), np.max(a))

    a=np.log10(psi);
    ax[1].imshow(a, cmap='jet', origin = 'lower', vmin=-6, vmax=-3);
    print("log10(psi) min,max: ",np.min(a), np.max(a))

    plt.show();

    return()

#==============================================================================

#===============================================================================


#!/usr/bin/env python
import numpy as np
from keras.models import Sequential
from keras.layers import LSTM
from keras.layers import Dense
import keras
import matplotlib.pyplot as plt
import pandas as pd
import time

from tkinter import ttk
from tkinter import filedialog
from tkinter import *
from PIL import Image, ImageTk
import os

# global parameters1
global timestep
timestep = 5
global batch_size
batch_size = 1
global train_length
train_length = 1300
global epochs
epochs = 5

# global parameters2
global unit1
unit1 = 4
global unit2
unit2 = 4
global unit3
unit3 = 168

# global parameters
global adam_lr
adam_lr = 0.003
global loss_func
loss_func = 'mean_squared_error'
global file_path
file_path = "./dt1.txt"

global data_from
data_from = 0

global data_to
data_to = 2000

global data_from_column
data_from_column = 1

global data_to_column
data_to_column = 1

global dataset
dataset = []

# Windows
root = Tk()
root.title('Deep Learning Platform')
root.resizable(False, False)
windowWidth = 1400
windowHeight = 1400
screenWidth, screenHeight = root.maxsize()

# load data set
def load_dataset():
    global file_path
    file_path = filedialog.askopenfilename()
    if (os.path.splitext(file_path)[-1] == ".txt"):
        for line in open(file_path):
            line = line.strip()
            dataset.append(float(line))
    if (os.path.splitext(file_path)[-1] == ".csv"):
        cvs_reader = pd.read_csv(file_path)
        # label = ttk.Text(monty3, width=80, text=cvs_reader.head(22))
        label = Text(monty5, width=80, height=30)
        label.insert(INSERT, cvs_reader)
        label.grid(row=0, column=0)
        train_set = cvs_reader.values
        for line in train_set:
            # line = line.strip()
            dataset.append(float(line))
    if (os.path.splitext(file_path)[-1] == ".xlsx"):
        xls_reader = pd.read_excel(file_path)
        #  label = ttk.Text(monty3, width=80, text=xls_reader.head(220))
        label = Text(monty5, width=80, height=30)
        label.insert(INSERT, xls_reader)
        label.grid(row=0, column=0)
        train_set = xls_reader.values
        for line in train_set:
            #  line = line.strip()
            dataset.append(float(line))

# Pages
tabControl = ttk.Notebook(root)
tab2 = ttk.Frame(tabControl)
tabControl.add(tab2, text='DATA LOAD')
tabControl.pack(expand=1, fill='both')
tab1 = ttk.Frame(tabControl)
tabControl.add(tab1, text='PARAMETER CONTROL')
tabControl.pack(expand=1, fill='both')

# tags
monty1 = ttk.LabelFrame(tab1, text='BASIC_PARA', width=500, height=100)
monty1.grid(column=0, row=0, padx=5, pady=20)
monty2 = ttk.LabelFrame(tab1, text='OPERATION', width=500, height=100)
monty2.grid(column=0, row=2, columnspan=2, padx=5, pady=20)
monty3 = ttk.LabelFrame(tab1, text='RESULT_SHOW', width=500, height=100)
monty3.grid(column=1, row=0, rowspan=2, padx=5, pady=20)

monty5 = ttk.LabelFrame(tab2, text='DATA_SHOW', width=650, height=550)
monty5.grid(column=1, row=0, rowspan=3, padx=5, pady=20)
monty4 = ttk.LabelFrame(tab2, text='DATA_CHOSEN_ROW', width=200, height=50)
monty4.grid(column=0, row=0, padx=5)
monty6 = ttk.LabelFrame(tab2, text='DATA_CHOSEN_COLUMN', width=200, height=50)
monty6.grid(column=0, row=1, padx=5)
monty7 = ttk.LabelFrame(tab2, text='DATA_LOAD', width=200, height=50)
monty7.grid(column=0, row=2, padx=5)

# Parameters1
var_steptime = StringVar()
var_steptime.set(timestep)
text_timestep_lable = StringVar()

var_batch_size = StringVar()
var_batch_size.set(batch_size)
text_batchsize_lable = StringVar()

var_train_length = StringVar()
var_train_length.set(train_length)
text_train_length_label = StringVar()

var_epochs = StringVar()
var_epochs.set(epochs)
text_epochs_label = StringVar()

# Parameters2
var_unit1 = StringVar()
var_unit1.set(unit1)
text_unit1_lable = StringVar()

var_unit2 = StringVar()
var_unit2.set(unit2)
text_unit2_lable = StringVar()

var_unit3 = StringVar()
var_unit3.set(unit3)
text_unit3_label = StringVar()

# Parameters3
var_adam_lr = StringVar()
var_adam_lr.set(adam_lr)
text_adam_lr_label = StringVar()

var_loss_func = StringVar()

# Parameters4
var_data_from = StringVar()
var_data_from.set(data_from)
text_data_from = StringVar()

var_data_to = StringVar()
var_data_to.set(data_to)
text_data_to = StringVar()

var_data_from_column = StringVar()
var_data_from_column.set(data_from_column)
text_data_from_column = StringVar()

var_data_to_column = StringVar()
var_data_to_column.set(data_to_column)
text_data_to_column = StringVar()

var_adam_lr = StringVar()
var_adam_lr.set(adam_lr)
text_adam_lr_label = StringVar()

def show(content):
    if content.isdigit():
        return True
    else:
        return False

show_com = root.register(show)

# Entry1
entry_steptime = Entry(monty1, textvariable=var_steptime, width=10, validate='key',
                       validatecommand=(show_com, '%P')).grid(row=0, column=2, pady=5)
lb1 = Label(monty1, text='timestep', width=13).grid(row=0, column=1, pady=5)
entry_batch_size = Entry(monty1, textvariable=var_batch_size, width=10, validate='key',
                         validatecommand=(show_com, '%P')).grid(row=1, column=2, pady=5)
lb2 = Label(monty1, width=13, text='batch_size').grid(row=1, column=1, pady=10)
entry_train_length = Entry(monty1, textvariable=var_train_length, width=10, validate='key',
                           validatecommand=(show_com, '%P')).grid(row=2, column=2, pady=5)
lb3 = Label(monty1, width=13, text='train_length').grid(row=2, column=1, pady=5)

entry_epochs = Entry(monty1, textvariable=var_epochs, width=10, validate='key',
                     validatecommand=(show_com, '%P')).grid(row=3, column=2, pady=5)
lb4 = Label(monty1, width=13, text='epochs').grid(row=3, column=1, pady=5)

entry_adam_lr = Entry(monty1, textvariable=var_adam_lr, width=10, validate='key').grid(row=4, column=2, pady=5)
lb3 = Label(monty1, width=13, text='adam_lr').grid(row=4, column=1, pady=5)

loss_funcChosen = ttk.Combobox(monty1, width=20, textvariable=var_loss_func)
loss_funcChosen['values'] = (
    'mean_squared_error', 'mean_absolute_error', 'squared_hinge', 'binary_crossentropy', 'mape', 'msle', 'hinge',
    'categorical_crossentropy')
loss_funcChosen.grid(row=5, column=1, columnspan=2, pady=5)
loss_funcChosen.current(0)
loss_funcChosen.config(state='readonly')
# Entry2
entry_unit1 = Entry(monty1, textvariable=var_unit1, width=10, validate='key',
                    validatecommand=(show_com, '%P')).grid(row=6, column=2, pady=5)
lb1 = Label(monty1, text='Embedding', width=13).grid(row=6, column=1, pady=5)
entry_unit2 = Entry(monty1, textvariable=var_unit2, width=10, validate='key',
                    validatecommand=(show_com, '%P')).grid(row=7, column=2, pady=5)
lb2 = Label(monty1, width=13, text='Hidden').grid(row=7, column=1, pady=5)
entry_unit3 = Entry(monty1, textvariable=var_unit3, width=10, validate='key',
                    validatecommand=(show_com, '%P')).grid(row=8, column=2, pady=5)
lb3 = Label(monty1, width=13, text='Dense').grid(row=8, column=1, pady=5)

# Entry 3
entry_data_from = Entry(monty4, textvariable=var_data_from, width=10, validate='key',
                        validatecommand=(show_com, '%P')).grid(row=0, column=2, pady=5)
lb1 = Label(monty4, text='From', width=13).grid(row=0, column=1, pady=5)

entry_data_to = Entry(monty4, textvariable=var_data_to, width=10, validate='key',
                      validatecommand=(show_com, '%P')).grid(row=1, column=2, pady=5)
lb1 = Label(monty4, text='To', width=13).grid(row=1, column=1, pady=5)

# Entry 4
entry_data_from_column = Entry(monty6, textvariable=var_data_from_column, width=10, validate='key',
                               validatecommand=(show_com, '%P')).grid(row=0, column=2, pady=5)
lb1 = Label(monty6, text='From', width=13).grid(row=0, column=1, pady=5)

entry_data_to_column = Entry(monty6, textvariable=var_data_to_column, width=10, validate='key',
                             validatecommand=(show_com, '%P')).grid(row=1, column=2, pady=5)
lb1 = Label(monty6, text='To', width=13).grid(row=1, column=1, pady=5)

# var_modify
def var_modify():
    global timestep
    timestep = int(var_steptime.get())
    global batch_size
    batch_size = int(var_batch_size.get())
    global train_length
    train_length = int(var_train_length.get())
    global epochs
    epochs = int(var_epochs.get())
    global unit1
    unit1 = int(var_unit1.get())
    global unit2
    unit2 = int(var_unit2.get())
    global unit3
    unit3 = int(var_unit3.get())

def conf_selected():
    global data_from
    data_from = int(var_data_from.get())
    global data_to
    data_to = int(var_data_to.get())

    # global adam_lr
    # adam_lr = float(var_unit3.get())
    # global loss_func
    # loss_func = var_loss_func.get()

# Figure Show
global IMG
IMG = []
global counter
counter = 0
global label
im = Image.open('ai.jpeg')
img = ImageTk.PhotoImage(im)
label = Label(monty3, image=img)
label.grid(row=0, column=0)

# RNN Training Function
def training():
    im = Image.open('ai.jpeg')
    img = ImageTk.PhotoImage(im)
    label = Label(monty3, image=img)
    label.grid(row=0, column=0)

    RNN_units = [unit1, unit2, unit3]  # 442
    data_length = train_length + timestep + RNN_units[2]
    test_length = timestep + RNN_units[2]

    global dataset
    dataset = dataset[data_from:data_to]
    date = []
    for line in open("./dt2.txt", "r"):
        line = line.strip()
        date.append(line)

    len1 = data_length + RNN_units[2] + timestep - 1
    len2 = len(dataset)
    dataset = dataset[(len2 - len1):len2]
    length = len(dataset)
    dataset = np.array(dataset)
    old_dataset = dataset.copy()
    dataset = (dataset - np.min(dataset)) / (max(dataset) - min(dataset))

    values = []
    labels = []
    for i in range(length - timestep - RNN_units[2] + 1):
        temp1 = dataset[i:i + timestep]
        temp2 = dataset[i + timestep:i + timestep + RNN_units[2]]
        values.append(temp1)
        labels.append(temp2)
    print(len(values))
    x_train = values[0:train_length]
    x_test = values[(train_length):train_length + timestep + RNN_units[2]]
    y_train = labels[0:train_length]
    y_test = labels[(train_length):train_length + timestep + RNN_units[2]]

    x_train = np.reshape(x_train, (train_length, timestep, 1))
    y_train = np.reshape(y_train, (train_length, RNN_units[2]))
    x_test = np.reshape(x_test, (timestep + RNN_units[2], timestep, 1))
    y_test = np.reshape(y_test, (timestep + RNN_units[2], RNN_units[2]))

    print(x_train.shape)
    model = Sequential()
    model.add(LSTM(RNN_units[0], input_shape=(timestep, 1), return_sequences=True))
    model.add(LSTM(RNN_units[1], return_sequences=False))
    model.add(Dense(RNN_units[2]))
    model.summary()
    adam = keras.optimizers.adam(lr=adam_lr)

    model.compile(loss=loss_func, optimizer=adam, metrics=['accuracy'])
    history = model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, validation_split=0.2, verbose=1)

    trainpredict = model.predict(x_test, verbose=1, batch_size=batch_size)

    trainpredict = trainpredict * (max(old_dataset) - min(old_dataset)) + min(old_dataset)
    y_test = y_test * (max(old_dataset) - min(old_dataset)) + min(old_dataset)

    y = old_dataset[timestep + train_length:timestep + train_length + RNN_units[2]]
    print(trainpredict.shape)
    print(y_test.shape)
    print(len(y_test[0]))

    plt.clf()
    # plot prediction and real result
    fig = plt.figure(0)
    # plt.plot(range(168),y_test[0],'ro-')
    # plt.plot(range(168),trainpredict[0],'g*:')
    #

    plt.plot(range(unit3), y_test[0], 'ro-')
    plt.plot(range(unit3), trainpredict[0], 'g*:')
    plt.legend(('real', 'pred'), loc='upper right')
    plt.savefig('3.png')
    plt.clf()

    fig = plt.figure(1)
    plt.title("Intend Perception Accuracy", fontproperties="Times New Roman")
    plt.ylabel("Intend Encode", fontproperties="Times New Roman")
    plt.xlabel("Epoch", fontproperties="Times New Roman")
    plt.plot(range(timestep + unit3), y_test[:, 0], 'ro-')
    plt.plot(range(timestep + unit3), trainpredict[:, 0], 'g*:')
    plt.legend(('real', 'pred'), loc='upper right')
    plt.savefig('1.png')
    plt.clf()

    # plot train process
    fig = plt.figure(2)
    plt.plot(history.history['acc'], label='training acc')
    plt.plot(history.history['val_acc'], label='val acc')
    plt.title('model accuracy')
    plt.ylabel('accuracy')
    plt.xlabel('epoch')
    plt.legend(loc='lower right')
    plt.savefig('2.png')

    clearIMG()
    loadFigure()
    dataset = []

def clearIMG():
    global IMG
    IMG = []

def loadFigure():
    for i in range(1, 4):
        im = Image.open(str(i) + '.png')
        img = ImageTk.PhotoImage(im)
        IMG.append(img)
        counter = 0
        label = Label(monty3, image=IMG[counter])
    label.grid(row=0, column=0)

def chimg():
    global label
    global counter
    if counter < 1:
        counter += 1
    else:
        counter = 0
    label.destroy()
    label = Label(monty3, image=IMG[counter])
    label.grid(row=0, column=0)

def figure_config():
    counter = 2
    label = Label(monty3, image=IMG[counter])
    label.grid(row=0, column=0)

# Operation
Button(monty7, text='LOAD DATASET', width=15, command=load_dataset, activeforeground='white',
       activebackground='green').grid(row=0, column=0, padx=20, pady=10)
Button(monty7, text='CONF SELECT', width=15, command=conf_selected, activeforeground='white',
       activebackground='green').grid(row=1, column=0, padx=20, pady=10)

Button(monty2, text='MODIFY PARAMETERS', width=15, command=var_modify, activeforeground='white',
       activebackground='green').grid(row=0, column=1, padx=26, pady=10)
Button(monty2, text='START TRAINING', width=15, command=training, activeforeground='white',
       activebackground='green').grid(row=0, column=2, padx=26, pady=10)
Button(monty2, text='NEXT RESULT', width=15, command=chimg, activeforeground='white', activebackground='green').grid(
    row=0, column=3, padx=26, pady=10)
Button(monty2, text='FIGURE CONFIG', width=15, command=figure_config, activeforeground='white',
       activebackground='green').grid(row=0, column=4, padx=26, pady=10)
mainloop()


# This script runs the Metropolis algorithm for the 2x2 spin system.
# It calculates and plots the energy and magnetisation as a function of
# Monte Carlo cycles.
# It also calculates the mean energy, mean magnetisation, heat capacity
# and susceptibility.
# Optional: This program can also plot the analytical values for the
# 2x2 case.

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import re
import os
import utils as ut

runCppCode = 0
# Set to false if you have the results files and just want to plot the results.
if runCppCode == True:
    # Compile and run the C++ files (this is exactly what is in the makefile):
    os.system("echo compiling C++ codes...")
    os.system("g++ -O3 -o main.out ../cpp_codes/main.cpp ../cpp_codes/utils.cpp ../cpp_codes/ising_solver.cpp -larmadillo -fopenmp")
    os.system("echo executing...")
    os.system("./main.out")

# The results directory:
directory = "../results/4d/"

# Mean values as function of # MC cycles:
filename = "4d_mean_values_vs_MC_cycles.csv"
#filename = "4d_mean_values_vs_1000_cycles_T=1.0_random"
filePath = os.path.join(directory, filename) # The full file path.
data = np.loadtxt(filePath, skiprows=1, delimiter=",")
# Get the columns of data as lists:
data = pd.DataFrame(data, columns=["MC_cycle", "E_mean", "E2_mean", "M_mean", \
    "M_abs_mean", "M2_mean", "C_V", "chi"])
MC_cycle_list = data["MC_cycle"].tolist() # tolist() converts dataframe to a list.
E_mean_list = data["E_mean"].tolist()
E2_mean_list = data["E2_mean"].tolist()
M_mean_list = data["M_mean"].tolist()
M_abs_mean_list = data["M_abs_mean"].tolist()
M2_mean_list = data["M2_mean"].tolist()
C_V_list = data["C_V"].tolist()
chi_list = data["chi"].tolist()

# E_list and M_list:
filename = "4d_E_list_M_list.csv"
#filename = "4d_results_1000_T=1.0_random.csv"
filePath = os.path.join(directory, filename) # The full file path.
data = np.loadtxt(filePath, skiprows=1, delimiter=",")
# Get the columns of data as lists:
data = pd.DataFrame(data, columns=["MC_cycle", "E", "E2", "M", "M_abs", "M2", "flips_accepted"])
MC_cycle_list2 = data["MC_cycle"].tolist() # tolist() converts dataframe to a list.
E_list = data["E"].tolist()
E2_list = data["E2"].tolist()
M_list = data["M"].tolist()
M_abs_list = data["M_abs"].tolist()
M2_list = data["M2"].tolist()
flips_accepted_list = data["flips_accepted"].tolist()

# Plot the results:
# Mean values:
plotCodeWord = "E_mean"
#plotCodeWord = "E2_mean"
#plotCodeWord = "M_mean"
#plotCodeWord = "M_abs_mean"
#plotCodeWord = "M2_mean"
#plotCodeWord = "C_V"
#plotCodeWord = "chi"
# Or plot the lists:
#plotCodeWord = "E_list"
#plotCodeWord = "M_list"

labelSize = 13
titleSize = 12
markerSize = 5

if plotCodeWord == "E_mean":
    plt.plot(MC_cycle_list, E_mean_list, 'r.', label='E_mean', markersize=markerSize)
    #plt.ylabel(r'$\langleE\rangle$', fontsize=labelSize)
    plt.ylabel('E_mean', fontsize=labelSize)
    plt.suptitle('Mean energy of the 20x20 spin system vs. Monte Carlo cycles', fontsize=titleSize)

elif plotCodeWord == "E2_mean":
    plt.plot(MC_cycle_list, E2_mean_list, 'r.', label='E2_mean', markersize=markerSize)
    plt.ylabel('E2_mean', fontsize=labelSize)
    plt.suptitle('Mean of ' + r'$E^2$' + ' for the 20x20 spin system vs. Monte Carlo cycles', fontsize=titleSize)

elif plotCodeWord == "M_mean":
    plt.plot(MC_cycle_list, M_mean_list, 'r.', label='M_mean', markersize=markerSize)
    plt.ylabel('M_mean', fontsize=labelSize)
    plt.suptitle('Mean of ' + r'$M$' + ' for the 20x20 spin system vs. Monte Carlo cycles', fontsize=titleSize)

elif plotCodeWord == "M_abs_mean":
    plt.plot(MC_cycle_list, M_mean_list, 'r.', label='|M|_mean', markersize=markerSize)
    plt.ylabel('|M|_mean', fontsize=labelSize)
    plt.suptitle('Mean of ' + r'$|M|$' + ' for the 20x20 spin system vs. Monte Carlo cycles', fontsize=titleSize)

elif plotCodeWord == "M2_mean":
    plt.plot(MC_cycle_list, M2_mean_list, 'r.', label='M2_mean', markersize=markerSize)
    plt.ylabel('M2_mean', fontsize=labelSize)
    plt.suptitle('Mean of ' + r'$M^2$' + ' for the 20x20 spin system vs. Monte Carlo cycles', fontsize=titleSize)

elif plotCodeWord == "C_V":
    plt.plot(MC_cycle_list, C_V_list, 'r.', label=r'$C_V$', markersize=markerSize)
    plt.ylabel(r'$C_V$', fontsize=labelSize)
    plt.suptitle('Numerical heat capacity for the 20x20 spin system vs. Monte Carlo cycles', fontsize=titleSize)

elif plotCodeWord == "chi":
    plt.plot(MC_cycle_list, chi_list, 'r.', label=r'$\chi$', markersize=markerSize)
    plt.ylabel(r'$\chi$', fontsize=labelSize)
    plt.suptitle('Numerical magnetic susceptibility for the 20x20 spin system vs. Monte Carlo cycles', fontsize=titleSize)

elif plotCodeWord == "E_list":
    # Plot E_list
    plt.plot(MC_cycle_list2, E_list, '.', label=r'$E$', markersize=markerSize)
    # Plot the final mean value as well (as a straight horizontal line):
    plt.plot([MC_cycle_list2[0], MC_cycle_list2[-1]], [E_mean_list[-1], E_mean_list[-1]], 'r-', label='E_mean')
    plt.ylabel(r'$E$', fontsize=labelSize)
    plt.suptitle('The energy of the 20x20 spin system vs. Monte Carlo cycles', fontsize=titleSize)

elif plotCodeWord == "M_list":
    # Plot M_list
    plt.plot(MC_cycle_list2, M_list, '.', label=r'$M$', markersize=markerSize)
    # Plot the final mean value as well:
    plt.plot([MC_cycle_list2[0], MC_cycle_list2[-1]], [M_mean_list[-1], M_mean_list[-1]], 'r-', label='M_mean')
    plt.ylabel(r'$M$', fontsize=labelSize)
    plt.suptitle('The magnetisation of the 20x20 spin system vs. Monte Carlo cycles', fontsize=titleSize)

plt.xlabel(r'Number of Monte Carlo cycles', fontsize=labelSize)
plt.legend()
plt.grid()
plt.show()


import json
import boto3
import logging

client = boto3.client('iot-data', region_name='us-west-2')
s3 = boto3.client('s3')

def function_handler(event, context):
    response = s3.get_object(Bucket='phfromhottubbucket', Key='phlevelfromhottub.json')
    value = response['Body'].read()
    value = float(value)
    OUTPUT_TOPIC = 'phlevelcheck'
    try:
        if (value >= 2.8 and value <= 2.9):
            response_message = "pH Value of water is good"
        elif  value > 2.9:
            response_message = "Water is acidic. Add pH increaser"
        elif  value < 2.8:
            response_message = "Add pH reducer"
    except Exception as e:
        logging.error(e)
    client.publish(topic=OUTPUT_TOPIC, payload=response_message)
    return


import torch
import time
from torch.utils.tensorboard import SummaryWriter
import training.config as config
from training.model import SeqLSTM
from training.data_utils import get_data_loader_batch_chr, save_processed_data

device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

def train_model(cfg, writer):
    """
    train_model(cfg, model_name, writer) -> No return object
    Loads existing model (or creates new one), loads data, trains the model.
    Saves the trained model in a .pth file.
    Specify training and model parameters in the configuration file.
    Args:
        cfg (Config): the configuration to use for the experiment.
        model_name (string): The model name that needs to be used to load model or create new model
        writer (SummaryWriter): tensorboard summary writer
    """

    "Initalize model and load model weights if they exist"
    model = SeqLSTM(cfg, device).to(device)
    model.load_weights()

    "Initalize optimizer"
    optimizer, criterion = model.compile_optimizer()

    "Get data"
    data_loader = get_data_loader_batch_chr(cfg)
    print("%s batches loaded" % str(len(data_loader)))

    "Train model"
    model.train_model(data_loader, criterion, optimizer, writer)

    "Save model"
    torch.save(model.state_dict(), cfg.model_dir + cfg.model_name + '.pth')

if __name__ == '__main__':
    cfg = config.Config()
    cell = cfg.cell
    model_name = cfg.model_name

    "Set up Tensorboard logging"
    timestr = time.strftime("%Y%m%d-%H%M%S")
    writer = SummaryWriter('./tensorboard_logs/' + model_name + timestr)

    if cfg.save_processed_data:
        "Process input data and save the input IDs and Hi-C Values"
        save_processed_data(cfg)

    "Train the model and save the .pth file"
    train_model(cfg, writer)


import functools
import logging
import sys
from logging.handlers import RotatingFileHandler

class LoggerWrapper:
    def __init__(self, stream=None):
        self.logged = set()
        self.logger = logging.Logger(name=__name__, level=logging.DEBUG)

        if not stream:
            stream = sys.stdout

        file_handler = RotatingFileHandler("changetext.log", encoding="utf-8", backupCount=1, maxBytes=1024**2)
        file_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
        stream_handler = logging.StreamHandler(stream)
        self.logger.addHandler(file_handler)
        self.logger.addHandler(stream_handler)

    def write(self, text):
        if text not in self.logged:
            self.logger.debug(text)
            self.logged.add(text)

@functools.lru_cache
def get_logger(stream=None):
    return LoggerWrapper(stream)

def log_exceptions(stream=None):
    def decorator(func):
        @functools.wraps(func)
        def wrapper(text):
            try:
                return func(text)
            except Exception:
                get_logger(stream).logger.exception(f"An exception occurred. Initial string: {text!r}")

        return wrapper

    return decorator


import requests
import pytest

@pytest.mark.parametrize('number', [1, 2, 3, 4, 5, 7, 8, 9, 10])
def test_first_dog(number):
    response = requests.get(f"https://dog.ceo/api/breed/hound/afghan/images/random/{number}")
    jpg = response.json()
    j = len(jpg['message'])
    print(j)
    assert j == number and jpg['status'] == 'success'

@pytest.mark.parametrize('sub_breeds', ["afghan",
        "basset",
        "blood",
        "english",
        "ibizan",
        "plott",
        "walker"])

def test_random_sub_breeds(sub_breeds):
    response = requests.get(f'https://dog.ceo/api/breed/hound/{sub_breeds}/images/random')
    j = response.json()['message']
    # print(len(sub_breeds))
    wait_subbreads = j[36:36+len(sub_breeds)]
    assert response.json()['status'] == 'success' and sub_breeds == wait_subbreads

def test_sub_breds():
    response = requests.get("https://dog.ceo/api/breed/terrier/list")
    assert len(response.json()['message']) == 21

def test_all_breeds():
    response = requests.get("https://dog.ceo/api/breeds/list/all")
    print(response.json())
    assert response.json()['status'] == "success"

@pytest.mark.parametrize('sub_breed', ['spaniel','hound','mastiff',])
def test_success_subbreeds(sub_breed):
    response_all = requests.get(f"https://dog.ceo/api/breeds/list/all")
    response_spain = requests.get(f'https://dog.ceo/api/breed/{sub_breed}/list')
    print(response_all.json()['message'][sub_breed])
    print(response_spain.json()['message'])
    assert response_all.json()['message'][sub_breed] == response_spain.json()['message']


import pandas as pd
import numpy as np
import telebot
from telebot import types
from res.func import do_picture
from res.token_for_bot import Token
import time

bot = telebot.TeleBot(Token)

def out(text):
    print('\033[1m\033[32m{}'.format(text), end='')
    print('\033[1m\033[31m{}'.format(''))

global not_main_rq, alf, df_examiners, df_students, df_exams, examiners_name
global date_of_exam, date_of_exam1, date_of_exam2, mark_by_exam, dict_date_of_exam

not_main_rq = ['Запрос на выборку']
alf = ['А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ё', 'Ж', 'З', 'И', 'Й', 'К', 'Л', 'М',
       'Н', 'О', 'П', 'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч', 'Ш', 'Щ', 'Э', 'Ю', 'Я']
df_students = pd.read_excel(r'data/Data.xlsx', sheet_name='students')
df_examiners = pd.read_excel(r'data/Data.xlsx', sheet_name='examiners')
df_exams = pd.read_excel(r'data/Data.xlsx', sheet_name='exams')
examiners_name = list(df_examiners.examiner_name)
mark_by_exam = {
    'philosophy': 'philosophy_mark',
    'math': 'math_mark',
    'IKT': 'IKT_mark',
    'PE': 'PE_mark',
    'programming': 'programming_mark',
    'history': 'history_mark',
    'cultural_studies': 'cultural_studies_mark',
    'english': 'english_mark'
}
dict_date_of_exam = {
    'philosophy': '2022-06-21',
    'math': '2022-06-22',
    'IKT': '2022-06-22',
    'PE': '2022-06-23',
    'programming': '2022-06-27',
    'history': '2022-06-27',
    'cultural_studies': '2022-06-27'
}

# date_of_exam = sorted(list(set(df_examiners.date.values)))
# date_of_exam = list(map(lambda x: x[:10], list(map(str, np.array(df_exams.values)[0]))))
date_of_exam = sorted(map(lambda x: x[:10], map(lambda x: str(x), list(set(df_examiners.date.values)))))
date_of_exam1 = ''
date_of_exam2 = ''

@bot.message_handler(commands=['start', 'restart'])
def welcome(message):
    """
    Функция приветствия
    """

    out(f'Отправлено сообщение:\n {message.text}\t от\t{message.from_user.username}')
    start = types.ReplyKeyboardMarkup(row_width=1)
    btn_start = 'Начать'
    start.add(btn_start)
    bot.send_message(message.from_user.id, 'Этот бот создан для работы с тестовой базой данных',
                     reply_markup=start)
    time.sleep(1)
    bot.send_sticker(message.from_user.id, 'CAACAgIAAxkBAAEFNCZixXvz226_cCpfszkWsxGjNhmaGQAC-BAAAuO_SEpmmeh30LPWwSkE')

@bot.message_handler(func=lambda message: True)
def other_text(message):
    global not_main_rq, alf, df_exams, df_examiners, df_students, examiners_name
    global date_of_exam, date_of_exam1, date_of_exam2, mark_by_exam

    out(f'Отправлено сообщение:\n {message.text}\t от\t{message.from_user.username}')

    rq = types.ReplyKeyboardMarkup(row_width=1)
    button_1 = 'Автоформа в столбец'  # Done!
    button_2 = 'Форма с Подчиненной формой'  # Done!
    button_3 = 'Запрос на выборку'  # Done!
    button_4 = 'Запрос с параметром'  # Done!
    button_5 = 'Запрос с вычисляемыми полями'  # Done!
    button_6 = 'Итоговый запрос'  # Done!
    button_7 = 'Запрос на создание базовой таблицы'  # Done!
    button_8 = 'Запрос на удаление'  # Done!
    button_9 = 'Запрос на обновление'  # Done!
    button_10 = 'Автоотчет в столбец'  # Done!
    button_11 = 'Отчет, созданный средствами Мастера отчетов'

    rq.add(button_1, button_2, button_3, button_4, button_5, button_6, button_7, button_8, button_9,
           button_10, button_11)
    if message.text not in not_main_rq:
        bot.send_message(message.from_user.id, 'Обновляем запросы', reply_markup=rq)

    if message.text == 'Автоформа в столбец':
        df = df_students
        do_picture(df)
        photo = open(r'data/table.jpg', 'rb')
        time.sleep(1)
        bot.send_photo(message.from_user.id, photo)
        time.sleep(2)
        photo.close
    if message.text == 'Запрос на выборку':
        rq = types.ReplyKeyboardMarkup(row_width=1)
        button_1 = 'Сортировка по первой букве Фамилии'  # Выводит таблицу со студентами, фамилии которых, начинающимеся на "И"
        button_2 = 'Сортировка по стоимости экзамена'  # Выводит таблицу с экзаменаторами, стоимость экзамена у которых от 30 до 50
        button_3 = 'Сортировка по году рождения'  # Выводит таблицу со студентами роившимися с 1980 года по 1990 год
        rq.add(button_1, button_2, button_3)
        bot.send_message(message.from_user.id, 'Выберете тип запроса на выборку:', reply_markup=rq)
    if message.text == 'Сортировка по первой букве Фамилии':
        df = df_students
        df = df[df['Full_Name'].str.find('И') == 0]
        if do_picture(df):
            photo = open(r'data/table.jpg', 'rb')
            time.sleep(1)
            bot.send_photo(message.from_user.id, photo)
            time.sleep(2)
            photo.close
    if message.text == 'Сортировка по году рождения':
        df = df_students
        new_df = df[(df.Year_of_birth >= '1980') & (df.Year_of_birth < '1990')]
        if do_picture(new_df):
            photo = open(r'data/table.jpg', 'rb')
            time.sleep(1)
            bot.send_photo(message.from_user.id, photo)
            time.sleep(2)
            photo.close
        else:
            bot.send_message(message.from_user.id, 'Нет информации по этим данным')
    if message.text == 'Запрос с параметром':
        rq = types.ReplyKeyboardMarkup(row_width=1)
        button_1 = 'Запрос с параметром - 1'
        button_2 = 'Запрос с параметром - 2'
        rq.add(button_1, button_2)
        bot.send_message(message.from_user.id,
                         '''
        Выберете тип запроса с параметром:

        1 - Выбирает из таблицы ЭКЗАМЕНАТОРЫ информацию об экзаменаторе по ФИО

        2 - Выбирает из таблиц информацию обо всех экзаменах в некоторый заданный интервал времени
        ''', reply_markup=rq)
    if message.text == 'Запрос с параметром - 1':  # DONE!
        rq = types.ReplyKeyboardMarkup(row_width=1)
        button_1 = str(examiners_name[0]);
        button_2 = str(examiners_name[1])
        button_3 = str(examiners_name[2]);
        button_4 = str(examiners_name[3])
        button_5 = str(examiners_name[4]);
        button_6 = str(examiners_name[5])
        button_7 = str(examiners_name[6])
        rq.add(button_1, button_2, button_3, button_4, button_5, button_6, button_7)
        bot.send_message(message.from_user.id, 'Выберете ФИО экзаменатора', reply_markup=rq)
    if message.text in examiners_name:
        df = df_examiners[df_examiners.examiner_name == message.text]
        bot.send_message(message.from_user.id,
                         f'Имя: {df.examiner_name.values[0]}\nЭкзамен: {df.exam.values[0]}\nСтоимость экзамена: {df.cost.values[0]}')

    if (message.text == 'Запрос с параметром - 2' or len(message.text) == 21):

        if len(message.text) == 21:
            date_of_exam1, date_of_exam2 = message.text.split('_')

            df = df_examiners[(df_examiners.date >= date_of_exam1) & (df_examiners.date <= date_of_exam2)]
            examiner_name = list(df.examiner_name.values)
            for i in range(5):
                examiner_name.append('')
            dt = list(map(lambda x: [x[:10]], list(map(str, np.array(df_exams.values)[0]))))
            for i in dt:
                for j in range(10):
                    i.append(i[0])
            exam_date = {i: dt[x] for x, i in enumerate(list(
                df_examiners.exam.values[(df_examiners.date >= date_of_exam1) & (df_examiners.date <= date_of_exam2)]))}

            exams = list(
                df_examiners.exam.values[(df_examiners.date >= date_of_exam1) & (df_examiners.date <= date_of_exam2)])
            out_df = pd.DataFrame(
                df_students.Full_Name
            ).join(df_students.Year_of_birth
                   ).join(df_students.Address
                          ).join(df_students.number_of_passport
                                 ).join(pd.DataFrame({mark_by_exam[i]: df_students[mark_by_exam[i]] for i in exams})
                                        ).join(
                df_examiners.examiner_name[
                    (df_examiners.date >= date_of_exam1) & (df_examiners.date <= date_of_exam2)
                    ]
            ).join(
                pd.DataFrame({i: [dict_date_of_exam[i] for _ in range(11)] for i in exams})
            )
            do_picture(out_df)
            if do_picture(out_df):
                photo = open(r'data/table.jpg', 'rb')
                time.sleep(1)
                bot.send_photo(message.from_user.id, photo)
                time.sleep(2)
                photo.close
                # print(f'date_of_exam1 - {date_of_exam1}\ndate_of_exam2 - {date_of_exam2}')
                date_of_exam1 = ''
                date_of_exam2 = ''
            else:
                bot.send_message(message.from_user.id, 'Нет информации по этим данным')

        else:
            bot.send_message(message.from_user.id, 'Введите дату по образцу ГГГГ-ММ-ДД_ГГГГ-ММ-ДД')
            bot.send_message(message.from_user.id,
                             'Где первая дата - начало выборки по дате экзаменов, а вторая - конец')
            bot.send_message(message.from_user.id, f'Даты проведения экзаменов: {str(date_of_exam)[1:-1]}')

    if message.text == 'Форма с Подчиненной формой':
        df = df_students.join(pd.DataFrame(
            {i: [dict_date_of_exam[i] for _ in range(11)] for i in list(df_examiners.exam.values)}
        ))
        do_picture(df)
        if do_picture(df):
            photo = open(r'data/table.jpg', 'rb')
            time.sleep(1)
            bot.send_photo(message.from_user.id, photo)
            time.sleep(2)
            photo.close
            date_of_exam1 = ''
            date_of_exam2 = ''
        else:
            bot.send_message(message.from_user.id, 'Нет информации по этим данным')

    if message.text == 'Сортировка по стоимости экзамена':
        df = df_examiners[(df_examiners.cost >= 30) & (df_examiners.cost <= 50)]
        do_picture(df)
        if do_picture(df):
            photo = open(r'data/table.jpg', 'rb')
            time.sleep(1)
            bot.send_photo(message.from_user.id, photo)
            time.sleep(2)
            photo.close
            date_of_exam1 = ''
            date_of_exam2 = ''
        else:
            bot.send_message(message.from_user.id, 'Нет информации по этим данным')
    if message.text == 'Запрос с вычисляемыми полями':
        df = pd.DataFrame({
            'examiner_name': list(df_examiners.examiner_name.values),
            'cost_for_1_exam': list(df_examiners.cost.values),
            'date_of_exam': list(df_examiners.date.values),
            'exam': list(df_examiners.exam.values),
            'tax': list(map(lambda x: x * 0.13, list(df_examiners.cost.values))),
            'salary': list(
                map(lambda x: (x - x * 0.13) * len(list(df_students.Full_Name.values)), list(df_examiners.cost.values)))
        })
        do_picture(df)
        if do_picture(df):
            photo = open(r'data/table.jpg', 'rb')
            time.sleep(1)
            bot.send_photo(message.from_user.id, photo)
            time.sleep(2)
            photo.close
            date_of_exam1 = ''
            date_of_exam2 = ''
        else:
            bot.send_message(message.from_user.id, 'Нет информации по этим данным')
    if message.text == 'Итоговый запрос':
        rq = types.ReplyKeyboardMarkup(row_width=1)
        button_1 = 'Итоговый запрос - 1'
        button_2 = 'Итоговый запрос - 2'
        rq.add(button_1, button_2)
        bot.send_message(message.from_user.id,
                         '''
        Выберете тип запроса с параметром:

        1 - Выполняет группировку по полю Год рождения. Для каждой группы определяет количество абитуриентов

        2 - Выполняет группировку по полю Наименование дисциплины. Для каждой группы вычисляет среднее значение оценки
        ''', reply_markup=rq)
    if message.text == 'Итоговый запрос - 1':
        df = pd.DataFrame({
            '1981': ['Самсонов Роман Богдановч', '', '', '', '', '', '1 абитуриет'],
            '1983': ['Иванов Степан Фёдорович', '', '', '', '', '', '1 абитуриет'],
            '1988': ['Носков Мариан Игнатович', '', '', '', '', '', '1 абитуриет'],
            '2000': ['Александров Харлампий Вячеславович', 'Аверкий Чеславович Волков',
                     'Шашкова Елизавета Андреевна', 'Логинов Павел Юльевич',
                     'Богданов Федосий Устинович', 'Орехова Валерия Михайловна', '6 абитуриентов'],
            '2001': ['Зыкоа Емельян Тарасович', 'Рубен Антипович Вишняков', '', '', '', '', '2 абитуриента']
        })
        do_picture(df)
        if do_picture(df):
            photo = open(r'data/table.jpg', 'rb')
            time.sleep(1)
            bot.send_photo(message.from_user.id, photo)
            time.sleep(2)
            photo.close
            date_of_exam1 = ''
            date_of_exam2 = ''
        else:
            bot.send_message(message.from_user.id, 'Нет информации по этим данным')
    if message.text == 'Итоговый запрос - 2':
        df = pd.DataFrame({i: [str(round(np.mean((df_students[mark_by_exam[i]].values)), 2))] for i in
                           list(df_examiners.exam.values)})
        do_picture(df)
        if do_picture(df):
            photo = open(r'data/table.jpg', 'rb')
            time.sleep(1)
            bot.send_photo(message.from_user.id, photo)
            time.sleep(2)
            photo.close
            date_of_exam1 = ''
            date_of_exam2 = ''
        else:
            bot.send_message(message.from_user.id, 'Нет информации по этим данным')

    if message.text == 'Запрос на создание базовой таблицы':
        rq = types.ReplyKeyboardMarkup(row_width=1)
        button_1 = 'Запрос на создание базовой таблицы - 1'
        button_2 = 'Запрос на создание базовой таблицы - 2'
        rq.add(button_1, button_2)
        bot.send_message(message.from_user.id,
                         '''
        Выберете тип запроса с параметром:

        1 - Создаёт таблицу "Encrollee_1988", содержащую информацию о абитуриентах 1988 года рождения

        2 - Создает копию таблицы Экзаменаторы с имененм "КОПИЯ_ЭКЗАМЕНАТОРЫ"
        ''', reply_markup=rq)

    if message.text == 'Запрос на создание базовой таблицы - 1':
        t_df = df_students.join(
            pd.DataFrame({'only_year': list(map(lambda x: str(x)[0:4], list(df_students.Year_of_birth.values)))}))
        df = (t_df[t_df.only_year == '1988']).drop(['only_year'], axis=1)
        do_picture(df)
        if do_picture(df):
            photo = open(r'data/table.jpg', 'rb')
            time.sleep(1)
            bot.send_photo(message.from_user.id, photo)
            bot.send_message(message.from_user.id, 'Таблица сохранена под названием "Encrollee_1988"')
            new_data = pd.ExcelWriter(r'data/Encrollee_1988.xlsx')
            df.to_excel(new_data, index=False)
            new_data.save()
            time.sleep(2)
            photo.close

        else:
            bot.send_message(message.from_user.id, 'Нет информации по этим данным')
    if message.text == 'Запрос на создание базовой таблицы - 2':
        new_data = pd.ExcelWriter(r'data/COPY_examiners.xlsx')
        df_examiners.to_excel(new_data, index=False)
        new_data.save()
        bot.send_message(message.from_user.id, 'Таблица сохранена под названием "COPY_examiners"')
        if do_picture(df_examiners):
            photo = open(r'data/table.jpg', 'rb')
            time.sleep(1)
            bot.send_photo(message.from_user.id, photo)
            time.sleep(2)
            photo.close

        else:
            bot.send_message(message.from_user.id, 'Нет информации по этим данным')
    if message.text == 'Запрос на удаление':
        df = pd.read_excel(r'data/COPY_examiners.xlsx')
        df = df.drop(labels=list((df[df.cost > 100]).index), axis=0)
        new_data = pd.ExcelWriter(r'data/COPY_examiners.xlsx')
        df.to_excel(new_data, index=False)
        new_data.save()
        if do_picture(df):
            photo = open(r'data/table.jpg', 'rb')
            time.sleep(1)
            bot.send_photo(message.from_user.id, photo)
            time.sleep(2)
            photo.close

        else:
            bot.send_message(message.from_user.id, 'Нет информации по этим данным')

    if message.text == 'Запрос на обновление':
        df = pd.read_excel('data/COPY_examiners.xlsx')
        df.loc[df.cost <= 50, 'cost'] = 55
        new_data = pd.ExcelWriter(r'data/COPY_examiners.xlsx')
        df.to_excel(new_data, index=False)
        new_data.save()
        bot.send_message(message.from_user.id, 'Таблица обновлена')
        if do_picture(df):
            photo = open(r'data/table.jpg', 'rb')
            time.sleep(1)
            bot.send_photo(message.from_user.id, photo)
            time.sleep(2)
            photo.close

        else:
            bot.send_message(message.from_user.id, 'Нет информации по этим данным')
    if message.text == 'Автоотчет в столбец':
        df = df_examiners
        if do_picture(df):
            photo = open(r'data/table.jpg', 'rb')
            time.sleep(1)
            bot.send_photo(message.from_user.id, photo)
            time.sleep(2)
            photo.close

        else:
            bot.send_message(message.from_user.id, 'Нет информации по этим данным')
    if message.text == 'Отчет, созданный средствами Мастера отчетов':
        df = pd.DataFrame({
            'date': ['21/06/22', '22/06/22', '22/06/22', '24/06/22', '27/06/22', '27/06/22', '27/06/22'],
            'examiner_name': ['Гришина Марина Дмитриевна', 'Михайлов Парфен Устинович',
                              'Полковская Марина Николаевна', 'Сидорова Оксана Николаевна', 'Салина Вера Игоревна',
                              'Дорофеева Элеонора Натановна', 'Селезнев Иван Феодосиевич'],
            'exam': ['philosophy', 'math', 'IKT', 'programming', 'history', 'cultural_studies', 'PE'],
            'cost': ['50', '60', '60', '50', '50', '60', '60']
        })
        if do_picture(df):
            photo = open(r'data/table.jpg', 'rb')
            time.sleep(1)
            bot.send_photo(message.from_user.id, photo)
            time.sleep(2)
            photo.close

        else:
            bot.send_message(message.from_user.id, 'Нет информации по этим данным')

bot.infinity_polling()


import cv2
import torch
import numpy as np
import torch.nn as nn
import torchvision.transforms as tt
import time
import streamlit as st
from streamlit_webrtc import VideoTransformerBase
import os

basename = os.path.abspath(os.path.dirname(__file__))
base_model_file = os.path.join(
    basename, "models", "emotion_detection_model_state.pth")

face_classifier = os.path.join(basename, "haarcascade_frontalface_default.xml")

model_state = torch.load(
    f"{base_model_file}", map_location=torch.device('cpu'))

class_labels = ["Angry", "Happy", "Neutral", "Sad", "Suprise"]

def conv_block(in_channels, out_channels, pool=False):
    layers = [
        nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1),
        nn.BatchNorm2d(out_channels),
        nn.ELU(inplace=True),
    ]
    if pool:
        layers.append(nn.MaxPool2d(2))
    return nn.Sequential(*layers)

class ResNet(nn.Module):
    def __init__(self, in_channels, num_classes):
        super().__init__()

        self.input = conv_block(in_channels, 64)

        self.conv1 = conv_block(64, 64, pool=True)
        self.res1 = nn.Sequential(conv_block(64, 32), conv_block(32, 64))
        self.drop1 = nn.Dropout(0.5)

        self.conv2 = conv_block(64, 64, pool=True)
        self.res2 = nn.Sequential(conv_block(64, 32), conv_block(32, 64))
        self.drop2 = nn.Dropout(0.5)

        self.conv3 = conv_block(64, 64, pool=True)
        self.res3 = nn.Sequential(conv_block(64, 32), conv_block(32, 64))
        self.drop3 = nn.Dropout(0.5)

        self.classifier = nn.Sequential(
            nn.MaxPool2d(6), nn.Flatten(), nn.Linear(64, num_classes)
        )

    def forward(self, xb):
        out = self.input(xb)

        out = self.conv1(out)
        out = self.res1(out) + out
        out = self.drop1(out)

        out = self.conv2(out)
        out = self.res2(out) + out
        out = self.drop2(out)

        out = self.conv3(out)
        out = self.res3(out) + out
        out = self.drop3(out)

        return self.classifier(out)

model = ResNet(1, len(class_labels))
model.load_state_dict(model_state)

# load face
try:
    face_cascade = cv2.CascadeClassifier(face_classifier)
except Exception:
    st.write("Error loading cascade classifiers")

class Faceemotion(VideoTransformerBase):
    def transform(self, frame):
        img = frame.to_ndarray(format="bgr24")

        # image gray
        img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        faces = face_cascade.detectMultiScale(
            image=img_gray, scaleFactor=1.3, minNeighbors=5)

        for (x, y, w, h) in faces:
            cv2.rectangle(img=img, pt1=(x, y), pt2=(
                x + w, y + h), color=(255, 0, 0), thickness=2)
            roi_gray = img_gray[y:y + h, x:x + w]
            roi_gray = cv2.resize(roi_gray, (48, 48),
                                  interpolation=cv2.INTER_AREA)

            if np.sum([roi_gray]) != 0:
                roi = tt.functional.to_pil_image(roi_gray)
                roi = tt.functional.to_grayscale(roi)
                roi = tt.ToTensor()(roi).unsqueeze(0)

                #  make a prediction on the ROI
                t0 = time.time()
                tensor = model(roi)
                inference_time = time.time() - t0

                probs = torch.nn.functional.softmax(tensor, dim=1)
                conf, num_class = torch.max(probs, 1)
                label = class_labels[num_class.item()]

                label_position = (x, y)

                cv2.putText(img, label, label_position,
                            cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)

                cv2.putText(img, f"confidence score : {round(conf.item(), 2)}", (50, 50),
                            cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 1)

                cv2.putText(img, f"inference time : {round(inference_time, 2)}", (50, 85),
                            cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 1)

            else:
                cv2.putText(img, "No Face Found", (20, 60),
                            cv2.FONT_HERSHEY_COMPLEX, 2, (0, 255, 0), 3, )

        return img

week 2/face expression classification/streamlit interface/stream_service/live_inference.py


from collections import deque

def bfs(idx):
    cnt = 1     # 컴퓨터 갯수
    q = deque()
    q.append(idx)
    v = [0 for _ in range(N + 1)]   # 방문 리스트 생성
    v[idx] = 1
    while q:
        ni = q.popleft()
        for i in hak[ni]:
            if not v[i]:
                v[i] = 1
                q.append(i)
                cnt += 1
    return cnt

N, M = map(int, input().split())
hak = [[] for _ in range(N + 1)]
# 접근을 편하게 하기 위해 M만큼 for문을 돌면서 해킹 리스트에 추가
for i in range(M):
    a, b = map(int, input().split())
    hak[b].append(a)

result = []     # 컴퓨터 번호와 갯수를 담을 리스트
max_cnt = 0     # 갯수의 최대값
for si in range(1, N + 1):
    if hak[si]:             # 신뢰하는 컴퓨터가 있을 때
        cnt = bfs(si)
        if max_cnt <= cnt:  # 최대값 갱신
            max_cnt = cnt
        result.append([si, cnt])    # 컴퓨터 번호와 갯수 추가

for i, cnt in result:
    # 컴퓨터가 최대 갯수이면 출력
    if max_cnt == cnt:
        print(i, end=' ')


from django.db import migrations, models

class Migration(migrations.Migration):
    dependencies = [
        ("projects", "0004_add_project_container_image"),
    ]

    operations = [
        migrations.AlterField(
            model_name="project",
            name="documentation_type",
            field=models.CharField(
                default=b"sphinx",
                help_text='Type of documentation you are building. More info.',
                max_length=20,
                verbose_name="Documentation type",
                choices=[
                    (b"auto", "Automatically Choose"),
                    (b"sphinx", "Sphinx Html"),
                    (b"mkdocs", "Mkdocs (Markdown)"),
                    (b"sphinx_htmldir", "Sphinx HtmlDir"),
                    (b"sphinx_singlehtml", "Sphinx Single Page HTML"),
                ],
            ),
        ),
        migrations.AlterField(
            model_name="project",
            name="single_version",
            field=models.BooleanField(
                default=False,
                help_text='A single version site has no translations and only your "latest" version, served at the root of the domain. Use this with caution, only turn it on if you will neverhave multiple versions of your docs.',
                verbose_name="Single version",
            ),
        ),
        migrations.AlterField(
            model_name="project",
            name="use_virtualenv",
            field=models.BooleanField(
                default=False,
                help_text="Install your project inside a virtualenv using setup.py install",
                verbose_name="Install Project",
            ),
        ),
    ]


import datetime
import os
import pickle
import random
from subprocess import PIPE

import click
import librosa
import pandas as pd
from ffmpy import FFmpeg

from ava_asd.annotation import Annotation
from ava_asd.annotation import vid_id_from_filename, get_min_max_timestamp, find_annotation_file
from ava_asd.config import read_config
from ava_asd.mfcc import Mfcc
from ava_asd.utils import create_dir
from ava_asd.utils import get_vid_width_height

@click.group()
def main():
    pass

def save_start_ts(ts, extracted_path, vid_id, start_ts_filename):
    """
    Save the timestamp corresponding to 1.jpg to a file.
    """
    out_path = os.path.join(extracted_path, vid_id, start_ts_filename)
    with open(out_path, 'w') as f:
        f.write(str(ts))

def extract_av(vid, test_ann_dir, train_ann_dir, fps, eps, start_ts_filename, vid_save_path, extracted_path):
    """
    Extract audio and video frames.
    """

    vid_in = os.path.join(vid_save_path, vid)
    vid_id = vid_id_from_filename(vid)

    out_path = os.path.join(extracted_path, vid_id)
    create_dir(out_path)

    audio_out = os.path.join(out_path, 'audio.wav')
    vid_out = os.path.join(out_path, '%d.jpg')

    ann_file = find_annotation_file(vid, train_ann_dir, test_ann_dir)
    min_ts, max_ts = get_min_max_timestamp(ann_file, eps=eps)

    save_start_ts(min_ts, extracted_path, vid_id, start_ts_filename)

    start_ts = str(datetime.timedelta(seconds=min_ts))
    end_ts = str(datetime.timedelta(seconds=max_ts))
    ff = FFmpeg(inputs={vid_in: ['-ss', start_ts, '-to', end_ts]}, outputs={
        audio_out: None, vid_out: ['-filter:v', 'fps=fps=' + str(fps), '-y']})
    ff.run(stderr=PIPE)

@main.command()
@click.argument('config-file', type=click.File('r'))
@click.argument('data-path', type=click.Path(exists=True, file_okay=False, dir_okay=True))
def videos(config_file, data_path):
    """ Extract audio and video frames.

    CONFIG_FILE: the config file with settings for the experiment.
    DATA_PATH: the path to the folder with the data files.
    """

    config = read_config(config_file.name)
    test_ann_dir = os.path.join(data_path, config['test_ann_dir'])
    train_ann_dir = os.path.join(data_path, config['train_ann_dir'])
    vid_save_path = os.path.join(data_path, config['vid_save_path'])
    extracted_path = os.path.join(data_path, config['extracted_path'])
    fps = config['fps']
    eps = config['eps']
    start_ts_filename = config['start_ts']

    if not os.path.exists(vid_save_path):
        raise Exception('video directory ' + vid_save_path + ' does not exist')

    create_dir(extracted_path)

    vids = os.listdir(vid_save_path)
    nvids = str(len(vids))

    for i, vid in enumerate(vids):
        print('[' + str(i + 1) + '/' + nvids + ' @ ' + str(fps) + 'fps] extracting ' + vid)
        extract_av(vid, test_ann_dir, train_ann_dir, fps, eps, start_ts_filename, vid_save_path, extracted_path)

def normalise_mfccs(mfccs, apply_mean=True, apply_stddev=True):
    """
    Apply normalisation some cepstral mean variance normalisation (in part or in whole).
    """
    mean = 0
    stddev = 1

    if apply_mean:
        mean = mfccs.mean(axis=1).reshape(mfccs.shape[0], 1)
    if apply_stddev:
        stddev = mfccs.std(axis=1).reshape(mfccs.shape[0], 1)

    normalised = (mfccs - mean) / stddev
    return normalised, mean, stddev

def gen_mfcc(vid_id, extracted_path, train_ann_dir, test_ann_dir, stride, window_size, nmfcc, eps, apply_mean,
             apply_stddev):
    """
    Generate MFCCs for a video.
    """

    input_path = os.path.join(extracted_path, vid_id, 'audio.wav')
    pcm, sr = librosa.load(input_path, sr=None)
    sample_stride = int(sr * stride)
    window = int(sr * window_size)

    mfccs = librosa.feature.mfcc(pcm, sr, n_mfcc=nmfcc, n_fft=window, hop_length=sample_stride)
    normalised_mfccs, mean, stddev = normalise_mfccs(mfccs, apply_mean=apply_mean, apply_stddev=apply_stddev)

    ann_file = find_annotation_file(vid_id, train_ann_dir, test_ann_dir)
    mints, _ = get_min_max_timestamp(ann_file, eps=eps)

    new_ts = mints + window_size
    result = Mfcc(new_ts, nmfcc, mean, stddev,
                  window_size, stride, normalised_mfccs)

    return result

@main.command()
@click.argument('config-file', type=click.File('r'))
@click.argument('data-path', type=click.Path(exists=True, file_okay=False, dir_okay=True))
def mfccs(config_file, data_path):
    """ Extract MFCCs from the dataset.

    CONFIG_FILE: the config file with settings for the experiment.
    DATA_PATH: the path to the folder with the data files.
    """

    config = read_config(config_file.name)
    extracted_path = os.path.join(data_path, config['extracted_path'])
    train_ann_dir = os.path.join(data_path, config['train_ann_dir'])
    test_ann_dir = os.path.join(data_path, config['test_ann_dir'])
    stride = config['stride']
    window_size = config['mfcc_window_size']
    nmfcc = config['nmfcc']
    eps = config['eps']
    apply_mean = config['apply_mean']
    apply_stddev = config['apply_stddev']

    dirs = os.listdir(extracted_path)
    ndirs = str(len(dirs))

    for i, vid_id in enumerate(dirs):
        print('[' + str(i + 1) + '/' + ndirs + '] generating MFCCs for ' + vid_id)
        mfccs = gen_mfcc(vid_id, extracted_path, train_ann_dir, test_ann_dir, stride, window_size, nmfcc, eps,
                         apply_mean, apply_stddev)
        output_path = os.path.join(extracted_path, vid_id, 'mfcc.pkl')
        pickle.dump(mfccs, open(output_path, 'wb'))

def get_vid_id(row):
    """
    Extract the video id from the annotations.
    """
    return row[0]

def get_bbox(row):
    """
    Extract the bounding box from the annotations.
    """
    return row[2], row[3], row[4], row[5]

def get_timestamp(row):
    """
    Extract the timestamp from the annotations.
    """
    return row[1]

def get_label(row):
    """
    Extract the active speaker label from the annotations.
    """
    return row[6]

def get_face_id(row):
    """
    Extract the face id from the annotations.
    """
    return row[7]

def row_to_annotation(row, width, height):
    """
    Convert row information to an annotation class.
    """
    vid_id = get_vid_id(row)
    timestamp = get_timestamp(row)
    bbox = get_bbox(row)
    label = get_label(row)
    face_id = get_face_id(row)
    face_size = get_face_size(bbox, width, height)
    return Annotation(vid_id, timestamp, bbox, label, face_id, face_size)

def generate_tracks(df, extracted_path):
    """
    Generate candidate face tracks from the annotations.
    """
    eps = 0.1
    firstrow = df.iloc[0]
    lastid = get_face_id(firstrow)
    lastts = get_timestamp(firstrow)

    vid_id = get_vid_id(firstrow)
    width, height = get_vid_width_height(extracted_path, vid_id)

    tracks = []
    track = []

    for i in range(len(df)):
        row = df.iloc[i]
        ann = row_to_annotation(row, width, height)

        if ann.face_id == lastid and ann.timestamp - lastts <= eps:
            track.append(ann)
        else:
            tracks.append(track)
            track = []
        lastid = ann.face_id
        lastts = ann.timestamp

    tracks.append(track)

    return tracks

def get_face_size(bbox, width, height):
    """
    Compute the face size in pixel coordinates.
    """

    x1 = int(bbox[0] * width)
    y1 = int(bbox[1] * height)
    x2 = int(bbox[2] * width)
    y2 = int(bbox[3] * height)
    width = x2 - x1
    height = y2 - y1
    return width, height

def closest_annotation(timestamp, track):
    """
    Find the closest annotation in a track to a given timestamp.
    """
    closest = 0
    closest_delta = float('inf')

    for i in range(len(track)):
        delta = abs(track[i].timestamp - timestamp)
        if delta < closest_delta:
            closest_delta = delta
            closest = i
    return track[closest]

def delete_small_tracks(tracks, window_size, fps):
    """
    Delete tracks that are smaller than the required window size. Use the timestamp to determine size rather than number of frames.
    """
    period = 1 / fps
    keep = []
    for track in tracks:
        filledtrack = []
        if len(track) == 0:
            continue
        ts_segment = track[-1].timestamp - track[0].timestamp
        if ts_segment <= window_size * period:
            continue

        hops = int(ts_segment / period)
        for i in range(hops):
            ca = closest_annotation(i * period + track[0].timestamp, track)
            filledtrack.append(ca)

        keep.append(filledtrack)

    return keep

def get_chunk(tracks, window_size, stride):
    """
    Cut tracks into window_size chunks. Uses a sliding window of stride 1.
    """
    chunks = []

    for track in tracks:
        n = len(track)
        for i in range(0, n - window_size, stride):
            if i + window_size < n:
                chunk = [track[i + j] for j in range(window_size)]
                chunks.append(chunk)

    return chunks

def get_homogenous(data):
    """
    Keep only homogenous data
    """
    keep = []
    for seq in data:
        label = seq[0].label
        hom = True
        for ann in seq:
            if ann.label != label:
                hom = False
                break
        if hom:
            keep.append(seq)

    return keep

def process_anns(ann_dir, output_filename, extracted_path, vid_frame_size, sequence_size, stride, fps):
    """
    Generate training meta data from the annotations in preparation for training.
    """
    window_size = int(vid_frame_size * sequence_size)

    data = []
    dirs = os.listdir(ann_dir)
    ndirs = len(dirs)
    for i, f in enumerate(dirs):
        print('[' + str(i + 1) + '/' + str(ndirs) + '] processing: ' + f)
        ann_file = os.path.join(ann_dir, f)
        df = pd.read_csv(ann_file, header=None)
        tracks = generate_tracks(df, extracted_path)
        tracks = delete_small_tracks(tracks, window_size, fps)
        chunk = get_chunk(tracks, window_size, stride=stride)
        data = data + chunk

    # Filter out inhomogenous sequences
    # data = get_homogenous(data)

    random.shuffle(data)

    serialised_data = pickle.dumps(data)
    with open(output_filename, 'wb') as f:
        f.write(serialised_data)

@main.command()
@click.argument('config-file', type=click.File('r'))
@click.argument('data-path', type=click.Path(exists=True, file_okay=False, dir_okay=True))
def annotations(config_file, data_path):
    """ Generate training meta data from the annotations in preparation for training.

    CONFIG_FILE: the config file with settings for the experiment.
    DATA_PATH: the path to the folder with the data files.
    """

    config = read_config(config_file.name)
    extracted_path = os.path.join(data_path, config['extracted_path'])
    train_ann_dir = os.path.join(data_path, config['train_ann_dir'])
    test_ann_dir = os.path.join(data_path, config['test_ann_dir'])
    train_annotations_file = os.path.join(data_path, config['train_annotations_full'])
    test_annotations_file = os.path.join(data_path, config['test_annotations_full'])
    vid_frame_size = config['vid_frame_size']
    sequence_size = config['sequence_size']
    stride = config['ann_stride']
    fps = config['fps']

    print('Generating training annotations file.')
    process_anns(train_ann_dir, train_annotations_file, extracted_path, vid_frame_size, sequence_size, stride, fps)

    print('Generating test annotations file.')
    process_anns(test_ann_dir, test_annotations_file, extracted_path, vid_frame_size, sequence_size, stride, fps)

if __name__ == "__main__":
    main()


import pytest

from ddtrace.appsec._iast import oce
from ddtrace.appsec._iast._ast.ast_patching import astpatch_module

def _iast_patched_module(module_name):
    module = __import__(module_name, fromlist=[None])
    module_path, patched_source = astpatch_module(module)

    compiled_code = compile(patched_source, module_path, "exec")
    exec(compiled_code, module.__dict__)
    return module

@pytest.fixture(autouse=True, scope="module")
def _enable_oce():
    oce._enabled = True
    yield
    oce._enabled = False


from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework.exceptions import ParseError
from django.http import Http404
from datetime import date

from .models import Poll, Submission, Answer, Option
from .serializers import (TestSerializer, QuestionSerializer,
                          UserOptionSerializer, SubmissionSerializer)

class Tests(APIView):
    def get(self, request):
        today = date.today()
        pollSet = Poll.objects.filter(startDate__lte=today,
                                      finishDate__gt=today)
        return Response(TestSerializer(pollSet, many=True).data)

class WriteInfoTest(APIView):
    def get(self, request, id):
        try:
            today = date.today()
            poll = Poll.objects.get(id=id)
            if poll.startDate > today or poll.finishDate < today:
                raise Poll.DoesNotExist()

            result = TestSerializer(poll).data
            result['questions'] = []
            for question in poll.question_set.all():
                questionDict = QuestionSerializer(question).data
                questionDict['options'] = UserOptionSerializer(
                            question.option_set.all(), many=True).data
                result['questions'].append(questionDict)

            return Response(result)

        except Poll.DoesNotExist:
            raise Http404()
        except Exception as ex:
            raise ParseError(ex)

    def post(self, request, id):
        try:
            poll = Poll.objects.get(id=id)
            user_id = request.data['user_id']

            def makeAnswer(question):
                answer = Answer(
                    question=question
                    )
                return answer

            answerList = [makeAnswer(question) for question in
                          poll.question_set.all()]
            if len(answerList) != poll.question_set.count():
                raise Exception('Not enough answers')

            submis = Submission(user_id=user_id, poll=poll)
            submis.save()
            for answer in answerList:
                answer.submission = submis
                answer.choice = Option(id=id)
                answer.save()

            return Response('Ответ принят')

        except Poll.DoesNotExist:
            raise Http404()
        except Exception as ex:
            raise ParseError(ex)

class ResultTest(APIView):
    def get(self, request, id):
        try:
            result = []
            for submission in Submission.objects.filter(
                                        user_id=id).order_by('submitTime'):
                submissionDict = SubmissionSerializer(submission).data
                submissionDict['poll_id'] = submission.poll_id
                submissionDict['answers'] = []
                for answer in submission.answer_set.all():
                    submissionDict['answers'].append({
                        'question': {
                            'id': answer.question_id,
                            'text': answer.question.text,
                        },
                        'answer': answer.choice.text,
                        'points': answer.choice.points
                    })

                result.append(submissionDict)
            return Response(result)

        except Exception as ex:
            raise ParseError(ex)


import argparse
import h5py
import numpy as np
import hydra

from manimo.environments.single_arm_env import SingleArmEnv

HZ = 20

parser = argparse.ArgumentParser()
parser.add_argument("--replay_file", type=str, required=True)
args = parser.parse_args()
replay_file = args.replay_file

# read the test.h5 file
f = h5py.File(replay_file, "r")

# get position action
ee_pos_desired = f["eef_pos"][:]
ee_quat_desired = f["eef_rot"][:]
ee_joints = f["q_pos"][:]

# get start position
demo_start_pos = ee_joints[0]
print(f'num_obs: {ee_pos_desired.shape[0]}')
print(f'start_pos: {demo_start_pos}')

# initialize agent
hydra.initialize(
        config_path="../conf", job_name="collect_demos_test"
    )

actuators_cfg = hydra.compose(config_name="actuators")
sensors_cfg = hydra.compose(config_name="sensors")

env = SingleArmEnv(sensors_cfg, actuators_cfg, hz=HZ)
hydra.core.global_hydra.GlobalHydra.instance().clear()

print(f'Setting home to demo start pos: {demo_start_pos}')
env.set_home(demo_start_pos)
env.reset()

assert ee_pos_desired.shape[0] == ee_quat_desired.shape[0]
num_actions = ee_pos_desired.shape[0]

print("starting replay")

for i in range(num_actions):
    action = [np.append(ee_pos_desired[i], ee_quat_desired[i])]
    env.step(action)

env.reset()


from datetime import datetime
import csv
import pdb

from race import Race
from horse import Horse
import utilities

MAX = 18

date = '20150429'
track = 'CD'
utilities.set_verbosity('HIGH')
# client = MongoClient()
# db = client.PONIES
# rdb = db.RACES
# hdb = db.HORSES

filename = 'results/singleHorse_{}.csv'.format(datetime.now().strftime('%Y-%m-%d'))
with open(filename,'w') as f:
  races = Race.findRaces()

  first_row = 'race_number,purse,distance,class_rating,num_in_field,h_odds,h_age,h_weight,h_gate_position,h_claim_value,h_odds_index,finish_wps\n'
  f.write(first_row)
  for race in races:
    if int(race.class_rating) <0: continue
    ordered_horses = race.sortedHorseOdds()
    for i,horse in enumerate(ordered_horses):
      race_text = '{},{},{},{},{},'.format(race.race_number, race.purse, race.distance, race.class_rating, len(ordered_horses))
      row = race_text + '{},{},{},{},{},{},'.format(horse.odds, horse.age, horse.weight, horse.gate_position, horse.claim_value, i)
      if horse.finish_position['position'] in ['1','2','3']:
        finish_wps = '1'
      else:
        finish_wps = '0'
      horse_count = 0

      row += finish_wps+'\n'
      f.write(row)


import argparse
import datetime
import json
import os
import re
from collections import defaultdict

parser = argparse.ArgumentParser(description="Script to parce access.log file(s)")
parser.add_argument("-p", "--path", dest="path", action="store", help="Provide path to log file or to folder with logs")
args = parser.parse_args()

def read_file(file):
    request_count = 0
    requests_by_method = {
        "POST": 0,
        "GET": 0,
        "PUT": 0,
        "DELETE": 0,
        "HEAD": 0,
        "CONNECT": 0,
        "OPTIONS": 0,
        "TRACE": 0
    }
    requests_other = 0
    requests_by_ip = defaultdict(int)
    requests_by_time = dict()

    with open(file) as file:
        for line in file:
            request = re.search(r"(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s\S\s\S.*HTTP.*\"\s(.*)", line)
            if request:
                ip = request.group(1)
                requests_by_ip[ip] += 1
                if len(requests_by_time) < 3:
                    requests_by_time[line] = int(request.group(2))
                else:
                    min_time_key = min(requests_by_time, key=requests_by_time.get)
                    if int(request.group(2)) >= requests_by_time[min_time_key]:
                        del requests_by_time[min_time_key]
                        requests_by_time[line] = int(request.group(2))

                method = re.search(r"] \"(POST|GET|PUT|DELETE|HEAD|CONNECT|OPTIONS|TRACE)", line)
                if method:
                    requests_by_method[method.group(1)] += 1
                else:
                    requests_other += 1
                request_count += 1

    return requests_other, request_count, requests_by_time, requests_by_method, requests_by_ip

# get info for longest requests and try to use regex more
def get_data_from_longest_request(requests_by_time):
    longest_requests = dict()
    idx = 1
    data_sorted = {k: v for k, v in sorted(requests_by_time.items(), key=lambda x: x[1])}
    for i in data_sorted:
        ip_duration = re.search(r"(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s\S\s\S.*HTTP.*\"\s(.*)", i)
        ip = ip_duration.group(1)
        duration = ip_duration.group(2)
        method = re.search(r"] \"(POST|GET|PUT|DELETE|HEAD|CONNECT|OPTIONS|TRACE)", i).group(1)
        url = re.search(rf"] \"{method}\s(\S+)", i).group(1)
        date_time = re.search(r"\S\s\S\s\S(\S+)", i).group(1).replace(":", " ", 1)
        longest_requests[f"ID {idx}"] = {"Method": method, "URL": url, "ip": ip, "Duration": duration,
                                         "Date and Time": date_time}
        idx += 1

    return longest_requests

def main(file):
    other_requests, count, r_by_time, r_by_method, r_by_ip = read_file(file)
    longest = get_data_from_longest_request(r_by_time)
    sorted_ips = {k: v for k, v in sorted(r_by_ip.items(), key=lambda x: x[1], reverse=True)[:3]}

    all_together = {
        "Log file": file,
        "Total requests": count,
        "By methods": r_by_method,
        "Other requests": other_requests,
        "Top 3 IP": sorted_ips,
        "Longest": longest
    }

    data_json = json.dumps(all_together, indent=4)

    print(data_json)

    with open(f"{datetime.datetime.now()}.json", mode="w", encoding="UTF8") as f:
        f.write(data_json)

if os.path.isdir(args.path):
    if not args.path.endswith("/"):
        args.path += "/"
    for file in os.listdir(args.path):
        if file.endswith(".log"):
            f = 5
            main(f"{args.path}{file}")
else:
    main(args.path)


import os
import random
import shutil
from shutil import copy2
from tqdm import tqdm

def spilt_datasets(data_root, trainDir, validDir):
    trainfiles = os.listdir(data_root)
    num_train = len(trainfiles)
    print("num_train: " + str(num_train))
    index_list = list(range(num_train))
    # print(index_list)
    random.shuffle(index_list)
    num = 0
    # trainDir = '../data/2023/extra_train/images'
    # validDir = '../data/2023/extra_val/images'
    for i in index_list:
        fileName = os.path.join(data_root, trainfiles[i])
        if num < num_train * 0.8:
            print(str(fileName))
            copy2(fileName, trainDir)
        else:
            copy2(fileName, validDir)
        num += 1

def cp_labels(lable_root, trainDir, validDir):
    # lable_root = '../data/2023/extra/labels'

    # trainDir = '../data/2023/extra_train/images'
    trainfiles = os.listdir(trainDir)

    # validDir = '../data/2023/extra_val/images'
    valfiles = os.listdir(validDir)

    for i in tqdm(trainfiles):
        fileName = os.path.join(lable_root, i)
        fileName = fileName.replace('.png', '.txt')
        copy2(fileName, '/workspace/pycharm_project/mmrotate/data/fair1m_ss/train/annfiles')

    for j in tqdm(valfiles):
        fileName = os.path.join(lable_root, j)
        fileName = fileName.replace('.png', '.txt')
        copy2(fileName, '/workspace/pycharm_project/mmrotate/data/fair1m_ss/val/annfiles')

def main():
    data_root = '/workspace/pycharm_project/mmrotate/data/fair1m_ss/trainval/images/'
    trainDir = '/workspace/pycharm_project/mmrotate/data/fair1m_ss/train/images/'
    validDir = '/workspace/pycharm_project/mmrotate/data/fair1m_ss/val/images/'
    # spilt_datasets(data_root, trainDir, validDir)

    lable_root = '/workspace/pycharm_project/mmrotate/data/fair1m_ss/trainval/annfiles/'
    cp_labels(lable_root, trainDir, validDir)

if __name__ == '__main__':
    main()