#!/usr/bin/env python
# coding: utf-8

# # COURSE: Master statistics and machine learning: Intuition, Math, code
# ##### COURSE URL: udemy.com/course/statsml_x/?couponCode=202006
# ## SECTION: The t-test family
# ### VIDEO: One-sample t-test
# #### TEACHER: Mike X Cohen, sincxpress.com

# In[ ]:

# import libraries
import matplotlib.pyplot as plt
import numpy as np
import scipy.stats as stats

# In[ ]:

## generate the data

# parameters
N = 20  # sample size
popMu = .5 # true population mean
data  = np.random.randn(N) + popMu

# let's see what the data look(s) like
plt.plot(data,'ko-',markerfacecolor='w',markersize=10)
plt.xlabel('Data index')
plt.ylabel('Data value')
plt.show()

### question: Should there be lines in this plot?

# In[ ]:

## "manual" t-test

# the null-hypothesis value
H0val = 0

# compute the t-value
t_num = np.mean(data) - H0val
t_den = np.std(data) / np.sqrt(N)
tval = t_num / t_den

# degrees of freedom
df = N-1

# p-value
pval = 1-stats.t.cdf(abs(tval),df)

# show the H0 parameter distribution and observed t-value
x = np.linspace(-4,4,1001)
tdist = stats.t.pdf(x,df) * np.mean(np.diff(x))

plt.plot(x,tdist,linewidth=2)
plt.plot([tval,tval],[0,max(tdist)],'r--')
plt.legend(('H_0 distribution','Observed t-value'))
plt.xlabel('t-value')
plt.ylabel('pdf(t)')
plt.title('t(%g) = %g, p=%g'%(df,tval,pval))
plt.show()

# In[ ]:

## now using the Python function

t,p = stats.ttest_1samp(data,H0val)

print(t,p)
# do these values match our manually computed values?


from flask import Flask, request , jsonify

from flask import Flask, render_template
from werkzeug.utils import secure_filename
import soundfile
import os
import io
import librosa
import pandas as pd
from operator import add
import numpy as np
import pickle as pk
from scipy.stats import skew
from sklearn.decomposition import PCA
import joblib
from pydub import AudioSegment
import os
import time
from scipy.signal import butter, lfilter

from scipy.io import wavfile

lowcut = 10   #freq filter
highcut = 4000
FRAME_RATE = 16000

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = 'webAudioFiles/'

@app.route('/WordUpload', methods=['POST' ])
def Foodupload():

    # Save the audio file to disk or process it in some way
    #audio_file.save("recorded_audio.wav")
    file = request.files.get('audio')
    if file and file.filename != '':
        dest = os.path.join(
            app.config['UPLOAD_FOLDER'],
            secure_filename(file.filename +".wav")
        )
        # Save the file on the server.
        file.save(dest)
        FileName = request.form.get('fileName')

        #data, samplerate = soundfile.read(dest)

        data,samplerate = librosa.load(dest,sr=16000)

        soundfile.write('webAudioFiles/' + FileName, data, samplerate= 16000, subtype='PCM_32')
        data,samplerate = librosa.load('webAudioFiles/' + FileName,sr=None)

        print("sample rate = ",samplerate)

    return "success"

def get_mfccV2(name, path,SAMPLE_RATE = 16000):
    data, _ = librosa.core.load(path + name, sr = SAMPLE_RATE)
    assert _ == SAMPLE_RATE
    try:
        ft1 = librosa.feature.mfcc(data, sr = SAMPLE_RATE, n_mfcc=30)
        ft2 = librosa.feature.zero_crossing_rate(data)[0]
        ft3 = librosa.feature.spectral_rolloff(data)[0]
        ft4 = librosa.feature.spectral_centroid(data)[0]
        ft5 = librosa.feature.spectral_contrast(data)[0]
        ft6 = librosa.feature.spectral_bandwidth(data)[0]
        ft7 = librosa.feature.spectral_flatness(data)[0]
        #S, phase = librosa.magphase(librosa.stft(data))
        #ft8 = librosa.feature.rms(S=S)[0]
        #hop_length = 512
        #oenv = librosa.onset.onset_strength(y=data, sr=SAMPLE_RATE, hop_length=hop_length)
        #ft9 = librosa.feature.fourier_tempogram(onset_envelope=oenv, sr=SAMPLE_RATE,
                                              #hop_length=hop_length)

        ft1_trunc = np.hstack((np.mean(ft1, axis=1), np.std(ft1, axis=1), skew(ft1, axis = 1), np.max(ft1, axis = 1), np.median(ft1, axis = 1), np.min(ft1, axis = 1))) #180 features
        ft2_trunc = np.hstack((np.mean(ft2), np.std(ft2), skew(ft2), np.max(ft2), np.median(ft2), np.min(ft2)))
        ft3_trunc = np.hstack((np.mean(ft3), np.std(ft3), skew(ft3), np.max(ft3), np.median(ft3), np.min(ft3)))
        ft4_trunc = np.hstack((np.mean(ft4), np.std(ft4), skew(ft4), np.max(ft4), np.median(ft4), np.min(ft4)))
        ft5_trunc = np.hstack((np.mean(ft5), np.std(ft5), skew(ft5), np.max(ft5), np.median(ft5), np.min(ft5)))
        ft6_trunc = np.hstack((np.mean(ft6), np.std(ft6), skew(ft6), np.max(ft6), np.median(ft6), np.max(ft6)))
        ft7_trunc = np.hstack((np.mean(ft7), np.std(ft7), skew(ft7), np.max(ft7), np.median(ft7), np.max(ft7))) #30 features.
        #ft8_trunc = np.hstack((np.mean(ft8), np.std(ft8), skew(ft8), np.max(ft8), np.median(ft8), np.max(ft8)))
        #ft9_trunc = np.hstack((np.mean(ft9), np.std(ft9), skew(ft9), np.max(ft9), np.median(ft9), np.max(ft9)))
      #  ft7_trunc =  np.hstack((np.mean(ft7), np.std(ft7), skew(ft7), np.max(ft7), np.median(ft7), np.max(ft7)))
        return pd.Series(np.hstack((ft1_trunc ,ft2_trunc,ft3_trunc,ft4_trunc, ft5_trunc,ft6_trunc,ft7_trunc)))
    except:
        print('bad file')
        print('file = ',name)
        return pd.Series([0]*210)

def input_feature_food(wav_file):
  with open('models/foodParams.npy', 'rb') as f:
    means_food = np.load(f)
    stds_food = np.load(f)

  TrainPCA = pk.load(open("models/TrainPCAfood.pkl",'rb'))

  train_data = get_mfccV2("",wav_file)
  X = np.array(train_data).reshape(1, -1)
  X_scaled = (X - means_food) / stds_food
  #TrainPCA = 0
  output = TrainPCA.transform(X_scaled)
  return output

def input_feature_num(wav_file):
  with open('models/numberParams.npy', 'rb') as f:
    means_num = np.load(f)
    stds_num = np.load(f)
  TrainPCA_num = pk.load(open("models/TrainPCAnum.pkl",'rb'))

  train_data = get_mfccV2("",wav_file)
  X = np.array(train_data).reshape(1, -1)
  X_scaled = (X - means_num) / stds_num # print value and hardcode
  #TrainPCA = 0
  output = TrainPCA_num.transform(X_scaled)
  return output

def label_int_num(output):
   label2num = {'eight': 8, 'five': 5, 'four': 4, 'nine': 9, 'one': 1, 'seven': 7, 'six': 6, 'ten': 10, 'three': 3, 'two': 2}
   y = label2num[output]
   return y

def per_unit_food(food,units):
  df = pd.read_csv('input/nutritionFeaturesExtractedFormatted.csv')
  if food == "yogurt":
    food = "Yoghurt"
  else:
    food = food.capitalize()
  nutrition_data = list(df[food])
  return [x * units for x in nutrition_data]

def get_data(food_wav,number_wav):

  clf_food = joblib.load("models/svm_food_model.pkl")
  clf_num = joblib.load("models/MLP_num_model.pkl")
  X_food = input_feature_food(food_wav)
  X_num = input_feature_num(number_wav)
  food = clf_food.predict(X_food)
  num = clf_num.predict(X_num)
  #print("predicted nunber = ",num[0])
  #print('Food :'+food[0],'  Quantity :'+num[0])
  num = label_int_num(num[0])
  foodItem = food[0]
  qtyItem = num
  #print("predicted nunber = ",num)
  nutrition_list = per_unit_food(food[0],num)
  return nutrition_list,foodItem,qtyItem

#add food and qty information as dictionary
def getNutrientValues(num_food = 1,food_wav = "input/filteredOutputfood/yogurt_kesavRectrain_9.wav",num_wav  = "input/filteredTestDataNumbers/five_kesavRectrain_16.wav"):
  df = pd.read_csv('input/nutritionFeaturesExtractedFormatted.csv')
  elements = df['Nutritional_groups']

  nutrition_list = [0]*30
  foodList = []
  qtyList = []
  for i in range(num_food):

    if(num_food == 1):
      lis,foodItem,qtyItem = get_data(food_wav,num_wav)
    else:
      lis,foodItem,qtyItem = get_data(food_wav[i],num_wav[i])
    foodList.append(foodItem)
    qtyList.append(qtyItem)
    nutrition_list = list(map(add, nutrition_list, lis))
    #print(nutrition_list)
  res = {list(elements)[i]: nutrition_list[i] for i in range(len(list(elements)))}
  return res,foodList,qtyList

def detect_leading_silence(sound, silence_threshold=-35.0, chunk_size=10):
    '''
    sound is a pydub.AudioSegment
    silence_threshold in dB
    chunk_size in ms

    iterate over chunks until you find the first one with sound
    '''
    trim_ms = 0 # ms

    assert chunk_size > 0 # to avoid infinite loop
    while sound[trim_ms:trim_ms+chunk_size].dBFS < silence_threshold and trim_ms < len(sound):
        trim_ms += chunk_size

    return trim_ms

def trimAndExport(recordPath,outputPath):

    for file in os.listdir(recordPath):
        #print(file)
        sound = AudioSegment.from_file(recordPath + file, format="wav")
        start_trim = detect_leading_silence(sound)
        end_trim = detect_leading_silence(sound.reverse())

        duration = len(sound)
        trimmed_sound = sound[start_trim:duration-end_trim]
        trimmed_sound.export(outputPath + file,format = "wav")

def butter_bandpass(lowcut, highcut, fs, order=5):
        nyq = 0.5 * fs
        low = lowcut / nyq
        high = highcut / nyq
        b, a = butter(order, [low, high], btype='band')
        return b, a

def butter_bandpass_filter(data, lowcut, highcut, fs, order=5):
    b, a = butter_bandpass(lowcut, highcut, fs, order=order)
    y = lfilter(b, a, data)
    return y

def bandpass_filter(buffer):
    return butter_bandpass_filter(buffer, lowcut , highcut , FRAME_RATE , order=3)

def filterTestFiles(test_record_path,test_output_path,FRAME_RATE = 16000,dataType = 'int32'):

    for file in os.listdir(test_output_path):
        os.remove(os.path.join(test_output_path,file))

    WAV_FILE_NAME = sorted(os.listdir(test_record_path), key=len)

    #print(WAV_FILE_NAME)

    for fileName in WAV_FILE_NAME:
        samplerate, data = wavfile.read(os.path.join(test_record_path, fileName))
        assert samplerate == FRAME_RATE
        filtered = np.apply_along_axis(bandpass_filter, 0, data)
        wavfile.write(os.path.join(test_output_path, fileName), samplerate, filtered.astype(dataType))

@app.route("/calculate")
def calculateNutrients():

    originalPath = "webAudioFiles/"
    trimmedPath =  "webAudioFilesFiltered/"
    filtered_Path = "webAudioFilesTrimmed/"

    if (os.path.isfile(originalPath+"blob.wav")):
        os.remove(originalPath+"blob.wav")

    trimAndExport(originalPath,trimmedPath)
    filterTestFiles(trimmedPath,filtered_Path)

    foodsFile = [filtered_Path + "food.wav",filtered_Path + "food2.wav"]
    qtyFile =  [filtered_Path + "number.wav",filtered_Path + "number2.wav"]
    print(foodsFile)
    print(qtyFile)
    num_food = 2
    nurtientRes,foodList,qtyList  = getNutrientValues(num_food, foodsFile, qtyFile)
    print(foodList)
    print(qtyList)

    print("files filtered and output produced.")

    for key, value in nurtientRes.items():
      nurtientRes[key] = round(value, 2)

    print(nurtientRes)
    outputDict = {'food':foodList,'qty':str(qtyList)}
    outputDict = {**outputDict, **nurtientRes}

    return jsonify(outputDict)

@app.route("/")
def index():
    return render_template("soundRecV6.html")

if __name__ == "__main__":
    app.run(debug=True)


#!/usr/bin/env python

import zmq

context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.setsockopt(zmq.SUBSCRIBE, "")
socket.connect('tcp://localhost:5555')

for i in range(10):
    print(socket.recv())


from __future__ import absolute_import, division, unicode_literals

import pickle
import os
import io
import copy
import logging
import numpy as np

from discoeval.tools.validation import SplitClassifier

class RSTEval(object):
    def __init__(self, taskpath, seed=1111):
        logging.debug('***** Transfer task : RST-DT Task at {}*****\n\n'.format(taskpath))
        self.seed = seed
        self.task_name = "RST"

        train_sents, train_labels = self.loadFile(os.path.join(taskpath, 'RST_TRAIN.pkl'))
        valid_sents, valid_labels = self.loadFile(os.path.join(taskpath, 'RST_DEV.pkl'))
        test_sents, test_labels = self.loadFile(os.path.join(taskpath, 'RST_TEST.pkl'))

        logging.debug("#train: {}, #dev: {}, #test: {}".format(len(train_sents), len(valid_sents), len(test_sents)))
        self.labelset = set(train_labels + valid_labels + test_labels)
        self.samples = sum(train_sents, []) + sum(valid_sents, []) + sum(test_sents, [])

        logging.debug('***** Total instances loaded: {}*****'.format(len(train_sents + valid_sents + test_sents)))
        logging.debug('***** Total #label categories: {}****\n\n'.format(len(self.labelset)))
        self.data = {'train': (train_sents, train_labels),
                     'valid': (valid_sents, valid_labels),
                     'test': (test_sents, test_labels)
                     }

    def do_prepare(self, params, prepare):
        return prepare(params, self.samples)

    def loadFile(self, fpath):
        data = pickle.load(open(fpath, "rb"))
        sents = []
        labels = []
        for d in data:
            sents.append([d[1], d[2]])
            labels.append(d[0])
        return sents, labels

    def run(self, params, batcher):
        self.X, self.y = {}, {}
        dico_label = {k: v for v, k in enumerate(self.labelset)}
        for key in self.data:
            x_data_filename = "/tmp/RST-%s-conpono-%s-x.npy" % (self.task_name, key)
            y_data_filename = "/tmp/RST-%s-conpono-%s-y.npy" % (self.task_name, key)
            if os.path.isfile(x_data_filename):
                assert os.path.isfile(y_data_filename), "Labels don't exist"
                self.X[key] = np.load(x_data_filename)
                self.y[key] = np.load(y_data_filename)
            else:
                if key not in self.X:
                    self.X[key] = []
                if key not in self.y:
                    self.y[key] = []

                input, labels = self.data[key]

                logging.debug('split: {}, #data: {}'.format(key, len(input)))
                enc_input = []
                sent2vec = {}
                for ii in range(0, len(input), params.batch_size):
                    batch = input[ii:ii + params.batch_size]
                    seq1 = [b[0] for b in batch]
                    seq2 = [b[1] for b in batch]
                    emb = batcher(params, seq1, seq2)
                    enc_input.append(emb)
                    if ii % (100*params.batch_size) == 0:
                        logging.info("PROGRESS (encoding): %.2f%%" %
                                     (100 * ii / len(input)))
                self.X[key] = np.vstack(enc_input)
                self.y[key] = np.array([dico_label[y] for y in labels])
                np.save(x_data_filename, self.X[key])
                np.save(y_data_filename, self.y[key])

            logging.info("encoding X to be: {}".format(self.X[key].shape))

        config = {'nclasses': len(dico_label), 'seed': self.seed,
                  'usepytorch': params.usepytorch,
                  'cudaEfficient': True,
                  'nhid': params.nhid, 'noreg': True}

        config_classifier = copy.deepcopy(params.classifier)
        config_classifier['max_epoch'] = 15
        config_classifier['epoch_size'] = 1
        config['classifier'] = config_classifier

        clf = SplitClassifier(self.X, self.y, config)
        devacc, testacc = clf.run()
        logging.debug('Dev acc : {0} Test acc : {1} for RST-DT\n'
                      .format(devacc, testacc))
        return {'devacc': devacc, 'acc': testacc,
                'ndev': len(self.data['valid'][0]),
                'ntest': len(self.data['test'][0])}


"""Таблица токен."""
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String

from authorization.src.app.db.models.base import Base

class Token(Base):
    """Таблица токен."""

    __tablename__ = 'token'

    token_id = Column(Integer, primary_key=True)
    user_id = Column(Integer, ForeignKey('user.user_id'))
    token = Column(String(100))
    expires = Column(DateTime())


import numpy as np
from scipy.optimize import minimize

class SVM:
    def __init__(self, trainx: np.ndarray, trainy: np.ndarray, n_epoch: int = 100):
        self.trainx = trainx
        self.trainy = trainy
        self.n_sample, self.n_dim = trainx.shape
        self.n_epoch = n_epoch

    def kernel(self, X1, X2, sig = 1):
        #X1 = np.expand_dims(X1, axis = 0).repeat(self.n_sample, axis = 0)
        #X2 = np.expand_dims(X2, axis = 1).repeat(self.n_sample, axis = 1)
        #return np.exp(np.linalg.norm(X1-X2, axis = 2)/sig)
        return np.exp(-np.square(np.linalg.norm(X1[:,None,:]-X2[None,:,:], axis = 2))/sig)

    def optimal_dual(self, C, sig=None, kernel = False):
        A = np.eye(self.n_sample) #idnetical matrix
        C = C*np.ones(self.n_sample)
        y = self.trainy.reshape([-1,1])
        X = self.trainx
        #'jac' is the jacobian matrix of the derivative on alpha
        # alpha y =0
        # A alpha <= C
        # A alpha >= 0
        self.constraints = ({'type': 'eq',   'fun': lambda alpha: np.dot(alpha.reshape([-1]), y.reshape([-1])), 'jac': lambda alpha: y.reshape([-1])},
                            {'type': 'ineq', 'fun': lambda alpha: C - np.dot(A, alpha.reshape([-1])), 'jac': lambda alpha: -A},
                            {'type': 'ineq', 'fun': lambda alpha: np.dot(A, alpha.reshape([-1])), 'jac': lambda alpha: A})

        def dual(alpha):
            alpha = alpha.reshape([-1])
            #minimize -L = maximize L
            if kernel:
                return 0.5 *  alpha.dot(alpha.dot( self.kernel(X, X, sig) * np.matmul(y, y.transpose()))) - alpha.sum()
            else:
                return 0.5 * alpha.dot(alpha.dot(np.matmul(X, X.transpose()) * np.matmul(y, y.transpose()))) - alpha.sum()

        #Partial derivate of Ld on alpha
        def jac_dual(alpha):
            alpha = alpha.reshape([-1])
            if kernel:
                return alpha.dot( self.kernel(X, X, sig) * np.matmul(y, y.transpose())) - np.ones_like(alpha)
            else:
                return alpha.dot(np.matmul(X, X.transpose()) * np.matmul(y, y.transpose())) - np.ones_like(alpha)

        initial = np.zeros([self.n_sample])
        self.optimal = minimize(dual, initial, method = 'SLSQP', jac=jac_dual, constraints=self.constraints)
        opt_alpha = self.optimal.x
        epsilon = 1e-6
        print("the number of support vectors is: ",(opt_alpha > epsilon).sum())
        if kernel:
            #print("the number of support vectors is: ",(opt_alpha > epsilon).sum())
            predict = ((opt_alpha.reshape(-1) * (y.reshape(-1))).reshape([-1])).dot(self.kernel(X, X, sig))
            support_vectors = predict[opt_alpha > epsilon]
            support_labels = y[opt_alpha > epsilon,0]
            b = support_labels[0] - support_vectors[0]
            return opt_alpha, b
        else:
            w = np.matmul(X.transpose(), (opt_alpha.reshape(-1) * (y.reshape(-1))).reshape([-1,1]))
            support_vectors = X[opt_alpha > epsilon,:]
            support_labels = y[opt_alpha > epsilon,0]
            b = support_labels[0] - np.matmul(support_vectors[0].reshape([1,-1]), w.reshape([-1,1]))
            return np.insert(w,0,b)

    def subGD(self, scheduler, C: float, initial_learning_rate: float = 0.05):
        w = np.zeros([self.n_dim+1])
        for t in range(self.n_epoch):
            learning_rate = scheduler(initial_learning_rate, t)
            shuffle = np.random.choice(np.arange(self.n_sample), self.n_sample, replace=False)
            for i in range(self.n_sample):
                index = shuffle[i]
                x = np.insert(self.trainx[index,:], 0, 1)
                y = self.trainy[index]
                if np.sum(y*w*x)<=1:
                    w = w-learning_rate*w + learning_rate*C*self.n_sample*y*x
                else:
                    w = (1-learning_rate)*w
        return w

    def kernel_perceptron(self, sig = 1):
        c = np.zeros(self.n_sample)
        for t in range(self.n_epoch):
            X = self.trainx
            y = self.trainy
            predict = self.perceptron_predict(X, c, sig = sig)
            c += (predict != y)
        return c

    def perceptron_predict(self, testx, c, sig = 1):
        x1 = np.c_[np.ones(self.n_sample), self.trainx]
        x2 = np.c_[np.ones(testx.shape[0]), testx]
        return 2* (((c.reshape(-1) * (self.trainy.reshape(-1))).reshape([-1])).dot(self.kernel(x1, x2, sig)).reshape(-1)>0)-1

    def predict(self, testx, w = None, alpha = None, b=None, sig = 1, kernal: bool = False):
        if kernal:
            return 2*(((alpha.reshape(-1) * (self.trainy.reshape(-1))).reshape([-1])).dot(self.kernel(self.trainx, testx, sig)).reshape(-1)>0)-1
        else:
            w = w.reshape([-1,1])
            return 2*(np.matmul(np.c_[np.ones(testx.shape[0]), testx], w).reshape(-1)>0)-1


import numpy as np
import pandas
import matplotlib.pyplot as plt

with open("data\\5.6\\srvotst.csv") as f:
    servo_pd = pandas.read_csv(f)
    f.close()

print(servo_pd.head())
plt.title("Y vs U for SG995 Left Hip")
plt.scatter(servo_pd["Time"], servo_pd["Left_Hip"], color = "tab:red", label="Y_Left")
plt.scatter(servo_pd["Time"], servo_pd["Right_Hip"], color = "tab:green", label="Y_Right")
plt.scatter(servo_pd["Time"], servo_pd["U"], color = "tab:blue", label="U")
plt.ylabel("Servo angle Y (degrees)")
plt.xlabel("Time (ms)")

plt.legend()
plt.show()


# pylint: disable=no-self-use,invalid-name
import numpy
import pytest

from allennlp.common.checks import ConfigurationError
from allennlp.common.testing import AllenNlpTestCase
from allennlp.data.fields import MultiLabelField
from allennlp.data.vocabulary import Vocabulary

class TestMultiLabelField(AllenNlpTestCase):
    def test_as_tensor_returns_integer_tensor(self):
        f = MultiLabelField([2, 3], skip_indexing=True, label_namespace="test1", num_labels=5)
        tensor = f.as_tensor(f.get_padding_lengths()).detach().cpu().tolist()
        assert tensor == [0, 0, 1, 1, 0]
        assert {type(item) for item in tensor} == {int}

    def test_multilabel_field_can_index_with_vocab(self):
        vocab = Vocabulary()
        vocab.add_token_to_namespace("rel0", namespace="rel_labels")
        vocab.add_token_to_namespace("rel1", namespace="rel_labels")
        vocab.add_token_to_namespace("rel2", namespace="rel_labels")

        f = MultiLabelField(["rel1", "rel0"], label_namespace="rel_labels")
        f.index(vocab)
        tensor = f.as_tensor(f.get_padding_lengths()).detach().cpu().numpy()
        numpy.testing.assert_array_almost_equal(tensor, numpy.array([1, 1, 0]))

    def test_multilabel_field_raises_with_non_integer_labels_and_no_indexing(self):
        with pytest.raises(ConfigurationError):
            _ = MultiLabelField(["non integer field"], skip_indexing=True)

    def test_multilabel_field_raises_with_no_indexing_and_missing_num_labels(self):
        with pytest.raises(ConfigurationError):
            _ = MultiLabelField([0, 2], skip_indexing=True, num_labels=None)

    def test_multilabel_field_raises_with_no_indexing_and_wrong_num_labels(self):
        with pytest.raises(ConfigurationError):
            _ = MultiLabelField([0, 2, 4], skip_indexing=True, num_labels=3)

    def test_multilabel_field_raises_with_incorrect_label_type(self):
        with pytest.raises(ConfigurationError):
            _ = MultiLabelField([1, 2], skip_indexing=False)

    def test_multilabel_field_raises_with_given_num_labels(self):
        with pytest.raises(ConfigurationError):
            _ = MultiLabelField([1, 2], skip_indexing=False, num_labels=4)

    def test_multilabel_field_empty_field_works(self):
        vocab = Vocabulary()
        vocab.add_token_to_namespace("label1", namespace="test_empty_labels")
        vocab.add_token_to_namespace("label2", namespace="test_empty_labels")

        f = MultiLabelField([], label_namespace="test_empty_labels")
        f.index(vocab)
        tensor = f.as_tensor(f.get_padding_lengths()).detach().cpu().numpy()
        numpy.testing.assert_array_almost_equal(tensor, numpy.array([0, 0]))
        g = f.empty_field()
        g.index(vocab)
        tensor = g.as_tensor(g.get_padding_lengths()).detach().cpu().numpy()
        numpy.testing.assert_array_almost_equal(tensor, numpy.array([0, 0]))

    def test_class_variables_for_namespace_warnings_work_correctly(self):
        # pylint: disable=protected-access
        assert "text" not in MultiLabelField._already_warned_namespaces
        with self.assertLogs(logger="allennlp.data.fields.multilabel_field", level="WARNING"):
            _ = MultiLabelField(["test"], label_namespace="text")

        # We've warned once, so we should have set the class variable to False.
        assert "text" in MultiLabelField._already_warned_namespaces
        with pytest.raises(AssertionError):
            with self.assertLogs(logger="allennlp.data.fields.multilabel_field", level="WARNING"):
                _ = MultiLabelField(["test2"], label_namespace="text")

        # ... but a new namespace should still log a warning.
        assert "text2" not in MultiLabelField._already_warned_namespaces
        with self.assertLogs(logger="allennlp.data.fields.multilabel_field", level="WARNING"):
            _ = MultiLabelField(["test"], label_namespace="text2")

    def test_printing_doesnt_crash(self):
        field = MultiLabelField(["label"], label_namespace="namespace")
        print(field)


#!/usr/bin/env python
# coding: utf-8

#  #  Author: Ayush Kakar
#
#
#

#

# #  Titanic - Machine Learning From Disaster - Dataset (World Happiness Report-2021)

# In[23]:

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

# In[24]:

data=pd.read_csv('world-happiness-report-2021.csv')

# In[25]:

data

# In[26]:

data.columns

# In[27]:

data1=data[['Country name','Regional indicator','Ladder score','Logged GDP per capita',
         'Social support','Healthy life expectancy','Freedom to make life choices','Generosity','Perceptions of corruption',
         'Ladder score in Dystopia']]

# In[28]:

data1.shape

# In[29]:

data1.isnull().sum()

# # Country Name Vs Ladder Score

# In[30]:

plt.figure(figsize=(30,10))
sns.barplot(data=data1,x='Country name',y='Ladder score')
plt.xticks(rotation=90)

# In[31]:

data1.corr()[['Ladder score']].sort_values(by='Ladder score',ascending=False)

# # Country Name Vs Freedom to make life choices

# In[32]:

plt.figure(figsize=(20,6))
sns.barplot(data=data1,x='Country name',y='Freedom to make life choices')
plt.xticks(rotation=120)

# In[33]:

data1.corr()[['Freedom to make life choices']].sort_values(by='Freedom to make life choices',ascending=False)

# # Country Name Vs GDP

# In[34]:

plt.figure(figsize=(30,10))
sns.barplot(data=data1,x='Country name',y='Logged GDP per capita')
plt.xticks(rotation=120)

# In[ ]:

# In[35]:

data1.corr()[['Logged GDP per capita']].sort_values(by='Logged GDP per capita',ascending=False)

# # GDP Vs Ladder score

# In[36]:

plt.figure(figsize=(30,10))
sns.barplot(data=data1,x='Logged GDP per capita',y='Ladder score')
plt.xticks(rotation=120)

# # Country Name Vs Generosity

# In[37]:

plt.figure(figsize=(30,10))
sns.barplot(data=data1,x='Country name',y='Generosity')
plt.xticks(rotation=120)

# In[38]:

data1.corr()[['Generosity']].sort_values(by='Generosity',ascending=False)

# # as we can see that GDP,HEATH,Social Support,Freedom to make life choices are positively correlated to happiness score but nobality(generosity)is negatively correlated with happiness score ie it is inversely proportional

# In[39]:

top_10_happiest=data1.sort_values(by='Ladder score',ascending=False).head(10)
top_15_unhappiest=data1.sort_values(by='Ladder score',ascending=False).tail(15)

# # Most Happiest Country( Country name Vs Ladder Score)

# In[41]:

plt.figure(figsize=(20,6))
sns.barplot(data=data1,x=top_10_happiest['Country name'],y=top_10_happiest['Ladder score'])

# # Thus we can see that Finland is the happiest country in the world¶

# # Most Unhappiest Country( Country name Vs Ladder Score)

# In[42]:

plt.figure(figsize=(20,6))
sns.barplot(data=data1,x=top_15_unhappiest['Country name'],y=top_15_unhappiest['Ladder score'])

# # Thus we can see that Afghanistan is the most unhappiest country in the world
#

# In[ ]:

# In[ ]:

# In[ ]:


'''
Trung Vo
CSE 512 - HW2
problem 3.2 - toydata - Multiclass SVM
'''
import numpy as np
from cvxopt import matrix, solvers
import scipy.io as sio
import itertools
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
import math
from utils import *

data = sio.loadmat('./hw2data/q3_1_data.mat')
x_train = data['trD']
y_train = data['trLb']
x_val = data['valD']
y_val = data['valLb']
'''
x_train: (1984, 362)
y_train: (362, 1)
x_val: (1984, 367)
y_val: (367, 1)
'''
N = np.shape(x_train)[1]
D = np.shape(x_train)[0]
y_train = y_train.T 		# [1, N]
y_val = y_val.T
y_train[y_train == -1] += 1;
y_val[y_val == -1] += 1;

# feed data to train and evaluate
k = 2
W = np.zeros((D, k))
num_epoch = 2000
C = 0.1
W_optimal, best_val_acc, train_loss_history, train_acc_history, val_loss_history, val_acc_history = batch_gd(num_epoch, W, x_train, y_train, x_val, y_val, C=C)
print ('Training ends...\n')
print ('Best validation accuracy: {:.4f} %'.format(best_val_acc*100))
print ('Sum of squares of W:', np.sum(W_optimal*W_optimal))

# visualize result
epochs = np.arange(num_epoch)
plt.subplot(121)
plt.plot(epochs, train_loss_history, 'b', label='train')
plt.plot(epochs, val_loss_history, 'g', label='val')
plt.title('loss')
plt.subplot(122)
plt.plot(epochs, train_acc_history, 'b', label='train')
plt.plot(epochs, val_acc_history, 'g', label='val')
plt.title('accuracy')
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.show()

# Compute confusion matrix
pred_scores = np.dot(W_optimal.T, x_train)
y_pred = np.argmax(pred_scores, axis=0)
cnf_matrix = confusion_matrix(y_train[0], y_pred)
np.set_printoptions(precision=2)

class_names = [1,-1]
# Plot normalized confusion matrix
plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=False,
                      title='Unnormalized confusion matrix')

plt.show()


import os
import sys
sys.path.append(os.getcwd())
import warnings
warnings.simplefilter("ignore", ResourceWarning)
import argparse
# Important Imports
import torch
from torch.utils.tensorboard import SummaryWriter
from torch.utils.data import DataLoader
# Project Imports
from nasrec.supernet.supernet import (
    SuperNet,
    ops_config_lib,
)
from nasrec.torchrec.utils import ParallelReadConcat
from nasrec.utils.config import (
    NUM_EMBEDDINGS_AVAZU,
    NUM_EMBEDDINGS_CRITEO,
    NUM_EMBEDDINGS_AVAZU,
    NUM_EMBEDDINGS_KDD,
)
from nasrec.utils.data_pipes import (
    get_criteo_kaggle_pipes,
    get_avazu_kaggle_pipes,
    get_kdd_kaggle_pipes,
)
from nasrec.utils.io_utils import (
    load_model_checkpoint,
    save_model_checkpoint,
    dump_pickle_data,
    create_dir,
)

# Project Imports
from nasrec.utils.train_utils import (
    get_l2_loss,
    train_and_test_one_epoch,
    get_model_flops_and_params,
    warmup_supernet_model,
    init_weights,
)
from nasrec.utils.lr_schedule import (
    CosineAnnealingWarmupRestarts,
    ConstantWithWarmup,
)

def train_and_eval_one_model(model, args):
    """Evaluate one certain model with the training pipeline and validation pipeline.
    Args:
        :params model: Backbone models used to train.
        :params args: Other hyperparameter settings.
    """
    get_pipes = {
        "criteo-kaggle": lambda: get_criteo_kaggle_pipes(args),
        "avazu": lambda: get_avazu_kaggle_pipes(args),
        'kdd': lambda: get_kdd_kaggle_pipes(args),
    }

    train_datapipes, test_datapipes, num_train_workers, num_test_workers = get_pipes[
        args.dataset
    ]()

    # Wrap up data-loader.5
    train_loader = DataLoader(
        ParallelReadConcat(*train_datapipes),
        batch_size=None,
        num_workers=num_train_workers,
    )
    test_loader = DataLoader(
        ParallelReadConcat(*test_datapipes),
        batch_size=None,
        num_workers=num_test_workers,
    )

    with torch.no_grad():
        model = warmup_supernet_model(model, train_loader, args.gpu)

    flops, params = get_model_flops_and_params(model, train_loader, args.gpu)
    print("FLOPS: {:.4f} M \t Params: {:.4f} M".format(flops / 1e6, params / 1e6))

    model.configure_path_sampling_strategy(args.strategy)

    # Functional headers for training purposes.
    if args.loss_function == "bce":
        loss_fn = torch.nn.BCEWithLogitsLoss()
    else:
        raise NotImplementedError(
            "Loss function {} is not implemented!".format(args.loss_function)
        )

    # L2 loss function.
    def _l2_loss_fn(model):
        """
        Customized Loss function.
        Has an optional choice to discard embedding regularization and disabling bias decay.
        """
        return get_l2_loss(model, args.wd, args.no_reg_param_name, gpu=args.gpu)

    l2_loss_fn = _l2_loss_fn
    # Optimizer
    optimizer_lib = {
        "adagrad": torch.optim.Adagrad(
            model.parameters(), lr=args.learning_rate, eps=1e-2,
        ),
        "adam": torch.optim.Adam(model.parameters(), lr=args.learning_rate, eps=1e-8),
        "sgd": torch.optim.SGD(
            model.parameters(), lr=args.learning_rate, nesterov=True, momentum=0.9
        ),
    }
    optimizer = optimizer_lib[args.optimizer]

    # Scheduler
    num_train_steps_per_epoch = args.train_limit // args.train_batch_size
    num_train_steps = (
        num_train_steps_per_epoch * args.num_epochs
    )
    num_warmup_steps = num_train_steps // 10
    if args.lr_schedule == "cosine":
        lr_scheduler = CosineAnnealingWarmupRestarts(
            optimizer,
            first_cycle_steps=num_train_steps,
            warmup_steps=num_warmup_steps,
            max_lr=args.learning_rate,
            min_lr=1e-8,
        )
    elif args.lr_schedule == "constant":
        lr_scheduler = ConstantWithWarmup(
            optimizer, num_warmup_steps=num_warmup_steps)
    else:
        # Do nothing.
        lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(
            optimizer, milestones=[num_train_steps * 10], gamma=0.1
        )

    epoch_logs = []

    if args.checkpoint_path is not None:
        checkpoint = load_model_checkpoint(args.checkpoint_path)
        model.load_state_dict(checkpoint["model_state_dict"], strict=True)
        if "optimizer_state_dict" in checkpoint:
            optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
    else:
        model.apply(init_weights)

    # Load pretrained embeddings if any.
    print(model)

    # Create logging_dir.
    strategy_name = (
        args.strategy
        if args.strategy == "single-path"
        else args.strategy + "-" + args.anypath_choice
    )
    logging_dir = os.path.join(
        args.logging_dir,
        "supernet_{}blocks_layernorm{:d}_{}_lr{:.2f}_supernetwarmup_{}".format(
            args.num_blocks,
            args.use_layernorm,
            strategy_name,
            args.learning_rate,
            args.supernet_training_steps,
        ),
    )

    print("Logging in directory: {}".format(logging_dir))
    create_dir(logging_dir)

    # Initialize TensorBoard Writer.
    with torch.no_grad():
        int_x, cat_x, _ = next(iter(train_loader))
        int_x, cat_x = int_x.to(args.gpu)[0].unsqueeze_(0), cat_x.to(args.gpu)[0].unsqueeze_(0)
        writer = SummaryWriter(args.logging_dir)
        writer.add_graph(model, (int_x, cat_x))

    for epoch in range(args.num_epochs):
        # Train the model for 1 epoch.
        logs = train_and_test_one_epoch(
                model,
                epoch,
                optimizer,
                lr_scheduler,
                train_loader,
                test_loader,
                loss_fn,
                l2_loss_fn,
                args.train_batch_size,
                args.gpu,
                display_interval=args.display_interval,
                test_interval=args.test_interval,
                max_train_steps=num_train_steps_per_epoch,
                test_only_at_last_step=True,
                grad_clip_value=5.0,
                tb_writer=writer,
        )
        epoch_logs.append(logs)

    print("Dumping logs to {}!".format(logging_dir))
    dump_pickle_data(
        os.path.join(logging_dir, "train_test_logs.pickle"), logs
    )
    # Now, store the trained model weights.
    save_model_checkpoint(
        model, os.path.join(logging_dir, "supernet_checkpoint.pt"), optimizer
    )
    return epoch_logs

_num_sparse_inputs_dict = {
    'criteo-kaggle': 26,
    'avazu': 23,
    'kdd': 10
}
_num_embedding_dict = {
    'criteo-kaggle': NUM_EMBEDDINGS_CRITEO,
    'avazu': NUM_EMBEDDINGS_AVAZU,
    "kdd": NUM_EMBEDDINGS_KDD,
}

# sparse-lr: 0.04, eps=1e-3
# dense-lr: 0.0002
# Opt: Adam (0.9, 0.999), eps=1e-8

def main(args):
    model = SuperNet(
        sparse_input_size=_num_sparse_inputs_dict[args.dataset],
        num_blocks=args.num_blocks,
        ops_config=ops_config_lib[args.config],
        use_layernorm=(args.use_layernorm == 1),
        activation="relu",
        num_embeddings=_num_embedding_dict[args.dataset],
        path_sampling_strategy=args.strategy,
        anypath_choice=args.anypath_choice,
        supernet_training_steps=args.supernet_training_steps,
        candidate_choices=None,
        # use_activation_at_residual_add=(args.enable_add_at_residual == 1),
    )
    # Copy model to GPU.
    model = model.to(args.gpu)
    train_and_eval_one_model(model, args)

if __name__ == "__main__":
    torch.backends.cudnn.benchmark = True
    torch.backends.cudnn.enabled = True

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--dataset",
        type=str,
        default="criteo-kaggle",
        help="Choice of datasets",
        choices=["criteo-kaggle", "avazu", "kdd"],
    )
    parser.add_argument(
        "--root_dir", type=str, default=None, help="Root Directory for dataset."
    )
    parser.add_argument(
        "--logging_dir", type=str, default=None, help="Directory to put loggings."
    )
    # Strategies
    parser.add_argument(
        "--strategy", type=str, default="single-path", help="Path sampling strategy",
        choices=['evo-2shot-path', 'default', 'single-path', 'any-path', 'full-path', 'fixed-path']
    )
    parser.add_argument(
        "--use_layernorm",
        type=int,
        default=0,
        help="Whether use layernorm in the supernet or not.",
    )
    parser.add_argument(
        "--config",
        type=str,
        default="xlarge",
        help="Configuration for the supernet search.",
    )
    parser.add_argument(
        "--num_blocks", type=int, default=7, help="Number of blocks per supernet."
    )
    parser.add_argument(
        "--checkpoint_path",
        type=str,
        default=None,
        help="Checkpoint path to load some pre-configured models.",
    )
    # Specific for 2nd shot only. Load the paths.
    parser.add_argument("--evo_2shot_path_candidates", type=str, default=None, \
        help="2nd shot path sampling candidates.")
    # Hyperparameters.
    parser.add_argument("--wd", type=float, default=0, help="L2 Weight decay")
    parser.add_argument(
        "--learning_rate", type=float, default=0.01, help="Learning rate"
    )
    parser.add_argument(
        "--learning_rate_decay", type=float, default=0, help="Learning rate decay."
    )
    parser.add_argument(
        "--num_epochs", type=int, default=1, help="Number of epochs for training."
    )
    parser.add_argument(
        "--supernet_training_steps",
        type=int,
        default=2000,
        help="Supernet training steps. Supernet training probability will be linearly decayed to 0 when it hits 'supernet_training_steps'.",
    )
    parser.add_argument(
        "--anypath_choice",
        type=str,
        default="uniform",
        help="The way to perform any-path sampling.",
    )
    parser.add_argument(
        "--train_batch_size", type=int, default=200, help="Training batch size."
    )
    parser.add_argument(
        "--test_batch_size", type=int, default=16368, help="Testing batch size."
    )
    #-------------------Criteo------------------
    # Train: 36672495 Val: 4584061 Test: 4584061 Trainval: 41256556
    # ------------------Avazu-------------------
    # Train: 32343175 Val: 4042896 Test: 4042896 Trainval: 36386071
    # ------------------KDD---------------------
    # Train: 119711284 Val: 14963910 Test: 14963910 Trainval: 134675194
    parser.add_argument(
        "--train_limit",
        type=int,
        default=36672495,
        help="Maximum number of training examples.",
    )
    parser.add_argument(
        "--test_limit",
        type=int,
        default=6548659,
        help="Maximum number of testing examples.",
    )
    parser.add_argument(
        "--lr_schedule", default="cosine", help="Learning rate schedule",
        choices=["cosine", "constant", "constant-no-warmup"])
    parser.add_argument(
        "--display_interval",
        type=int,
        default=100,
        help="Interval to display tensorboard curve/training stats.",
    )
    parser.add_argument(
        "--test_interval",
        type=int,
        default=100,
        help="Testing interval when training supernet.",
    )
    # Currently useful in SparseNN-V2.
    parser.add_argument(
        "--activation",
        type=str,
        default="relu",
        help="Activation function to use in this work.",
        choices=["relu", "silu"],
    )
    parser.add_argument(
        "--train_split",
        type=str,
        default="train",
        help="Data split for training. Can be one of ['train', 'trainval']",
        choices=["train", "trainval"],
    )
    parser.add_argument(
        "--validate_split",
        type=str,
        default="test",
        help="Data split for validation (evaluation). Can be one of ['val', test']",
        choices=["val", "test"],
    )
    parser.add_argument(
        "--no-reg-param-name",
        type=str,
        default=None,
        help="Name of the parameters that do not need to be regularized.",
    )
    # loss functions
    parser.add_argument(
        "--loss_function",
        type=str,
        default="bce",
        help="Loss function to perform the task.",
        choices=["bce"],
    )
    parser.add_argument(
        "--optimizer",
        type=str,
        default="adagrad",
        help="Optimizer to choose",
        choices=["adagrad", "sgd", "adam", "rmsprop"],
    )
    # Use embeddings during training.
    parser.add_argument("--pretrained_dlrm_emb_path", type=str, default=None,
        help="Pretrained embedding path from DLRM model.")
    # GPU utils
    parser.add_argument("--gpu", type=int, default=0, help="GPU ID to use.")

    args = parser.parse_args()
    main(args)


# Simon Chu
# Created on Sun Nov  1 18:32:24 EST 2020
# Modified on Wed 2021-01-06 09:30:08
#
# lexer.py
# lexer definition file
#
# reference:
# https://rply.readthedocs.io/en/latest/

from rply import LexerGenerator
import stl.tool as tool

class Lexer:
    """lexical analyzer class

    Attributes:
            self._lexer: the lexer that is built
    """

    def __init__(self):
        """Initialize Lexical Analyzer"""

        lg = LexerGenerator()
        # lg.ignore(r"(\t|\ |\v|\r)+")  # ignore all whitespaces except newline
        lg.ignore(r"\s+")

        # reserved stl operators
        lg.add("GLOBALLY", r"\[\]")
        lg.add("EVENTUALLY", r"\<\>")

        # logical operators
        lg.add("LOGICAL_EQUALS", r"\<\=\>")
        lg.add("LOGICAL_AND", r"\&\&")
        lg.add("LOGICAL_OR", r"\|\|")
        lg.add("LOGICAL_IMPLIES", r"\=\>")

        # binary operators for numerical (Int, Float) operations
        lg.add("GREATER_EQUAL", r"\>\=")
        lg.add("LESS_EQUAL", r"\<\=")
        lg.add("GREATER", r"\>")
        lg.add("LESS", r"\<")

        # binary logical operators for both numerical (Int, Float) and Boolean operations
        lg.add("EQUAL_EQUAL", r"\=\=")
        lg.add("NOT_EQUAL", r"\!\=")

        # additional logic operators
        lg.add("LOGICAL_NOT", r"\!")  # must be placed after NOT_EQUAL (!=) due to conflict

        # content grouping structures
        lg.add("L_PAREN", r"\(")
        lg.add("R_PAREN", r"\)")
        lg.add("L_SQ_BRACE", r"\[")
        lg.add("R_SQ_BRACE", r"\]")

        # primary arithmetic operators
        lg.add("EQUAL", r"\=")
        lg.add("PLUS", r"\+")
        lg.add("MINUS", r"-")
        lg.add("MULTIPLY", r"\*")
        lg.add("DIVIDE", r"/")

        # separators/delimiters
        lg.add("COMMA", r",")
        # lg.add("NEWLINE", r"\n")

        # values
        # support 1.2, 1E7, 1.E7, 1.02E2 (equivalent to 102.0) as floating-point numbers
        lg.add("FLOAT", r"[0-9]*(\.[0-9]+([eE][-+]?[0-9]+)?)|([0-9]+([eE][-+]?[0-9]+))")
        lg.add("INT", r"\d+")
        lg.add("STRING", r"\".*\"")
        lg.add("BOOLEAN", r"true|false")

        # identifiers (variable identifiers/type identifiers/STL operator identifier)
        # note that STL operators are included in identifiers to avoid variable name conflict with the operator name
        lg.add("IDENTIFIER", r"[a-zA-Z_][a-zA-Z0-9_.]*")  # identifier cannot begin with numerical values

        # build the lexer
        self._lexer = lg.build()

    # parse raw program string to token stream
    def lex(self, stl_expr):
        return self._lexer.lex(stl_expr)

def main():
    # create lexer
    lexer = Lexer()

    for expr in tool.repl(header="Please enter STL expressions to be lexically analyzed."):

        token_stream = lexer.lex(expr)

        for token in token_stream:
            print(token)

if __name__ == "__main__":
    main()


from math import pi
import torch
import torch.nn as nn

from pytorch_lightning.core.module import LightningModule
from loc_ndf.models import loss
from loc_ndf.utils import vis, utils
import open3d as o3d
from easydict import EasyDict
import tqdm
from loc_ndf.utils import pytimer

##################################################################
################ Lightning Module ################################
##################################################################
class LocNDF(LightningModule):
    def __init__(self, hparams: dict):
        super().__init__()
       # name you hyperparameter hparams, then it will be saved automagically.
        self.save_hyperparameters(hparams)
        self.update_map_params()  # self.T_local and self.occupancy_mask

        self.model = get_network(hparams['model'])
        self.loss = loss.ProjectedDistanceLoss(**hparams['loss']['params'])

    def update_map_params(self, points=None):
        T_local = utils.compute_T_from_bounding_box(
            self.hparams['bounding_box'])
        self.register_buffer('T_local', T_local)

        mask = utils.get_occ_mask(
            points=points, **self.hparams['occupancy_mask'])
        self.register_buffer('occupancy_mask', mask)

    def forward(self, x: torch.Tensor, in_global_frame=True):
        if in_global_frame:
            x = utils.transform(x, self.T_local)
        distance = self.model(x)
        return distance

    def training_step(self, batch: dict, batch_idx):
        points = self.forward(batch['points'])

        inter_grad, inter_val = self.compute_gradient_dists(batch['inter'])

        rand_grad, _ = self.compute_gradient_dists(batch['random'])

        loss, losses = self.loss(
            points_distance=points,
            points=batch['points'],
            inter_val=inter_val,
            inter_grad=inter_grad,
            inter_pos=batch['inter'],
            ray_dists=batch['dists'],
            rand_grad=rand_grad)

        for k, v in losses.items():
            self.log(f'train/{k}', v)
        return loss

    def compute_gradient_dists(self, position: torch.Tensor):
        position.requires_grad_(True)
        val = self.forward(position)
        grad = utils.compute_gradient(val, position)
        return grad, val

    def configure_optimizers(self):
        optimizer = torch.optim.AdamW(
            self.parameters(), lr=self.hparams['train']['lr'])
        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
            optimizer, T_max=self.hparams['train']['max_epoch'], eta_min=self.hparams['train']['lr']/1e3)
        return [optimizer], [scheduler]

    def get_grid(self, nv=[500, 500, 50]):
        mins = self.hparams['bounding_box'][0]
        maxs = self.hparams['bounding_box'][1]
        x = torch.linspace(mins[0], maxs[0], nv[0],
                           device=self.device, dtype=self.dtype)
        y = torch.linspace(mins[1], maxs[1], nv[1],
                           device=self.device, dtype=self.dtype)
        z = torch.linspace(mins[2], maxs[2], nv[2],
                           device=self.device, dtype=self.dtype)
        xx, yy, zz = torch.meshgrid(x, y, z, indexing='ij')
        # make homogenous
        grid = torch.stack([xx, yy, zz, torch.ones_like(zz)], -1)
        return grid

    def get_occupancy_mask(self, nv=[500, 500, 50]):
        """ Returns the occupancy mask for evaluating the mesh for a given voxelsize.
        Be aware that the stored occupancy might be stored in a lower resolution thus the values are interpolated.
        """
        return torch.nn.functional.interpolate(self.occupancy_mask[None, None, ...].float(), size=nv).squeeze().bool()

    def get_mesh(self, nv=[500, 500, 50], tau=None, file=None, verbose=False, mask=None):
        tau = self.hparams['data']['isosurface'] if tau is None else tau
        pytimer.tic()
        if verbose:
            print('Start meshing')
        grid = self.get_grid(nv)
        pytimer.tocTic('Created grid', verbose=verbose)
        rows = []
        with torch.no_grad():
            for row in tqdm.tqdm(grid, leave=False, desc='Eval Grid'):
                rows.append(self.forward(row))
        pytimer.tocTic('Evaluated model', verbose=verbose)
        dist = torch.stack(rows).squeeze().cpu().numpy()
        mesh = vis.grid_to_mesh(
            dist, tau=tau,
            ascent=self.hparams['data']['gradient_ascent'],
            mins=self.hparams['bounding_box'][0],
            maxs=self.hparams['bounding_box'][1],
            mask=mask)
        pytimer.tocTic('finished meshing', verbose=verbose)

        if file is not None:
            o3d.io.write_triangle_mesh(file, mesh)
        pytimer.tocTic(f'stored to disk: {file}', verbose=verbose)
        return mesh

##################################################################
################ Network Architecture ############################
##################################################################

class LidarNerf(nn.Module):
    def __init__(self,
                 inter_fdim: int,
                 pos_encoding: dict,
                 sigmoid=True):
        super().__init__()
        self.pos_encoding = PositionalEncoder(**pos_encoding['params'])
        in_dim = self.pos_encoding.featureSize()
        self.net1 = nn.Sequential(
            MLP_Block(in_dim, inter_fdim),
            MLP_Block(inter_fdim,
                      inter_fdim),
            MLP_Block(inter_fdim,
                      inter_fdim),
            MLP_Block(inter_fdim,
                      inter_fdim),
            MLP_Block(inter_fdim,
                      inter_fdim))
        self.net2 = nn.Sequential(
            MLP_Block(inter_fdim+in_dim,
                      inter_fdim),
            MLP_Block(inter_fdim,
                      inter_fdim),
            MLP_Block(inter_fdim,
                      inter_fdim),
            nn.Linear(inter_fdim, 1),
            nn.Sigmoid() if sigmoid else nn.Identity()
        )

    def forward(self, x):
        pos = self.pos_encoding(x)

        x = self.net1(pos)
        x = torch.cat([x, pos], -1)
        x = self.net2(x)
        return x

class Siren(nn.Module):
    def __init__(self,
                 inter_fdim: int,
                 pos_encoding: dict,
                 sigmoid=True,
                 sin=True):
        super().__init__()
        self.pos_encoding = PositionalEncoder(**pos_encoding['params'])
        in_dim = self.pos_encoding.featureSize()
        self.net = nn.Sequential(
            MLP_Block(in_dim, inter_fdim, sin=sin),
            MLP_Block(inter_fdim,
                      inter_fdim, sin=sin),
            MLP_Block(inter_fdim,
                      inter_fdim, sin=sin),
            MLP_Block(inter_fdim,
                      inter_fdim, sin=sin),
            MLP_Block(inter_fdim,
                      inter_fdim, sin=sin),
            nn.Linear(inter_fdim, 1),
            nn.Sigmoid() if sigmoid else nn.Identity()
        )

    def forward(self, x):
        pos = self.pos_encoding(x)
        x = self.net(pos)
        return x

class Networks(EasyDict):
    LidarNerf = LidarNerf
    Siren = Siren

def get_network(type_params: dict) -> nn.Module:
    """ dict type_params {
        type: str, specifies which loss
        params: dict, kwargs of the loss
    }
    """
    return Networks()[type_params['type']](**type_params['params'])

################################################################
###################### Blocks ##################################
################################################################

class SinLayer(nn.Module):
    def forward(self, x):
        return torch.sin(x)

class MLP_Block(nn.Module):
    def __init__(self, in_dim: int, out_dim: int, sin=False):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(in_features=in_dim, out_features=out_dim),
            nn.LayerNorm(out_dim),
            SinLayer() if sin else nn.LeakyReLU())

    def forward(self, x):
        return self.net(x)

################################################################
################ Positional Encoding ###########################
################################################################

class PositionalEncoder(nn.Module):
    def __init__(self, freq, num_bands=5, dimensionality=3, base=2):
        super().__init__()
        self.freq, self.num_bands = torch.tensor(freq), num_bands
        self.dimensionality, self.base = dimensionality, torch.tensor(base)

    def forward(self, x):
        x = x[..., :self.dimensionality, None]
        device, dtype, orig_x = x.device, x.dtype, x

        scales = torch.logspace(0., torch.log(
            self.freq / 2) / torch.log(self.base), self.num_bands, base=self.base, device=device, dtype=dtype)
        # Fancy reshaping
        scales = scales[(*((None,) * (len(x.shape) - 1)), Ellipsis)]

        x = x * scales * pi
        x = torch.cat([x.sin(), x.cos()], dim=-1)
        x = torch.cat((x, orig_x), dim=-1)
        x = x.flatten(-2, -1)
        return x

    def featureSize(self):
        return self.dimensionality*(self.num_bands*2+1)
################################################################


import configparser
import textwrap

CONFIG_LOC = r"%LOCALAPPDATA%\GOG.com\Galaxy\Configuration\plugins\ps2\config.ini"

class Config:
    def __init__(self):
        self.cfg = configparser.ConfigParser(allow_no_value=True)
        self.cfg.set("DEFAULT", "; Make sure to use / instead of \ in file paths.")
        self.cfg["DEFAULT"]["roms_path"] = "C:/Games/PS2"
        self.cfg["DEFAULT"]["emu_path"] = "C:/Program Files (x86)/PCSX2 1.4.0/pcsx2.exe"
        self.cfg["DEFAULT"]["config_path"] = "C:/Documents/PCSX2/configs"
        self.cfg["DEFAULT"]["method"] = "default"
        self.cfg["DEFAULT"]["api_key"] = None
        self.cfg["DEFAULT"]["emu_fullscreen"] = False
        self.cfg["DEFAULT"]["emu_no_gui"] = False
        self.cfg["DEFAULT"]["emu_config"] = False

        self.cfg.add_section("Paths")
        self.cfg.set("Paths", textwrap.dedent(
                """\
                ; Set your roms folder, path to your pcsx2.exe and optional config path here
                ; To use the config feature, create a folder named "configs" inside "Documents/PCSX2" (the folder containing your bios, savestates etc.)
                ; Put the path to that folder down below
                ; To create settings for a game, copy the "inis" or "inis_1.4.0" folder into the "configs" folder and rename it to match the rom.
                ; Make sure that the renamed folders have the same name as the corresponding rom without the extension.
                ; eg. ROM: "Shadow of the Colossus.iso" | FOLDER: "Shadow of the Colossus"
                ; Without a specific config folder for a game, the default settings will be used.
                ; If you want to specify that PCSX2 does a full boot for a game follow the above instructions
                ; and simply add an empty file "fullboot.ini" to the rom folder in "configs"\
                """
            )
        )

        self.cfg.add_section("Method")
        self.cfg.set("Method", textwrap.dedent(
                """\
                ; "default": your files have to be named exactly as they are in the PCSX2 database (if the name contains a ":", delete it)
                ; "giant": use the Giant Bomb API to search for the name of your files in their database (requires a Giant Bomb account for an API key)
                ; "iso": reads your iso files directly for the serial of the game and then matches the name to it (only adds all your games if they are all iso files)
                ; Also set your API key here if you are using the Giant Bomb method\
                """
            )
        )

        self.cfg.add_section("EmuSettings")
        self.cfg.set("EmuSettings", textwrap.dedent(
                """\
                ; emu_fullscreen: Set to True if you want to launch in fullscreen by default
                ; emu_no_gui: Set to True if you want to suppress the gui when launching a game
                ; emu_config: Set to True if you want to use game specific configurations\
                """
            )
        )


from ast import parse, walk, FunctionDef, ClassDef, Name
from re import findall

from math import log, sqrt

from pandas import DataFrame
from nltk.stem import PorterStemmer
from sklearn.metrics.pairwise import cosine_similarity

from .Analyzer import Analyzer

class SemanticAnalyzer(Analyzer):

    def __init__(self, url, remove_bulk=-1) -> None:
        super().__init__(url, remove_bulk)

        self.couplings_type = 'semantic'

        self.tf_idf_df = None
        self.file_to_identifiers = None

    def compute_couplings(self):

        self.file_to_identifiers = self.get_corpus()

        self.preprocess_words(self.file_to_identifiers)

        voc_size, voc_to_index = self.compute_voc(self.file_to_identifiers)

        tf = self.compute_tf(voc_to_index, self.file_to_identifiers)
        idf = self.compute_idf(voc_to_index, self.file_to_identifiers)
        tf_idf = self.compute_tf_idf(voc_to_index, tf, idf)

        self.tf_idf_df = DataFrame.from_dict(tf_idf, orient='index')

        self.run_general_analysis(
            get_logical_couplings_df=True,
            get_commit_to_files=True,
            get_dates=True
            )

    def get_distance_matrix(self):

        distance_matrix = cosine_similarity(self.tf_idf_df)
        for i in range(len(distance_matrix)):
            distance_matrix[i][i] = 1
        distance_df = DataFrame(distance_matrix, index=self.tf_idf_df.index, columns=self.tf_idf_df.index)

        self.distance_matrix = distance_df

        return distance_df

    def get_corpus(self):
        """ Get a list of identifiers of each file in a repo.
        """

        file_to_identifiers = {}
        for file_path in self.repo_files_path:

            try :
                with open(self.repo_folder + '\\' + file_path) as data_source:
                    ast_root = parse(data_source.read())

                identifiers = []

                for node in walk(ast_root):
                    if isinstance(node, Name):
                        identifiers.append(node.id)
                    elif isinstance(node, FunctionDef) or isinstance(node, ClassDef):
                        identifiers.append(node.name)

                file_to_identifiers[file_path] = identifiers

            except:
                path = self.repo_folder + "\\" + file_path
                print(f'Could not read {path}')
                pass

        return file_to_identifiers

    def preprocess_words(self, file_to_identifiers):
        """ Preprocess words for further analysis : stems, split and lower words.
        """

        stemmer = PorterStemmer()

        for key in file_to_identifiers.keys():

            old_words = file_to_identifiers[key]
            new_words = []

            for i in range(len(old_words)):

                splitted_sentence = self.split_sentence(old_words[i])
                for word in splitted_sentence:
                    new_words.append(word)

            new_words = [str.lower(word) for word in new_words]
            new_words = [stemmer.stem(word) for word in new_words]

            file_to_identifiers[key] = new_words

    @staticmethod
    def split_sentence(word):
        """ Split a snake or camel case string into its composing words.
        """

        # Snake split
        splitted_snake_sentence = word.split('_')

        # camel_word = re.sub(r'_(.)', lambda m: m.group(1).upper(), word)

        splitted_sentence = []
        for snake_word in splitted_snake_sentence:
            camel_words = findall(r'.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)', snake_word)
            for camel_word in camel_words:
                splitted_sentence.append(camel_word)

        return splitted_sentence

    @staticmethod
    def compute_voc(file_to_identifiers):
        """ Compute the vocabulary of a repo (list of all the words).
        """

        voc = set()

        for _, value in file_to_identifiers.items():

            for word in value:

                if word not in voc:
                    voc.add(word)

        voc_size = len(voc)

        voc_to_index = {}

        i = 0
        for word in voc:
            voc_to_index[word] = i
            i += 1

        return voc_size, voc_to_index

    @staticmethod
    def compute_tf(voc_to_index, file_to_identifiers):
        """ Compute the tf values for each file in repo.
        """

        tf = {}

        for file_path in file_to_identifiers.keys():

            tf[file_path] = [0 for _ in range(len(voc_to_index))]

            for word in file_to_identifiers[file_path]:

                tf[file_path][voc_to_index[word]] += 1

            num_identifiers = len(file_to_identifiers[file_path])

            if num_identifiers > 0:
                tf[file_path] = [value / num_identifiers for value in tf[file_path]]

        return tf

    @staticmethod
    def compute_idf(voc_to_index, file_identifiers):
        """ Compute the idf values for each file in repo.
        """

        idf = {}

        for word in voc_to_index.keys():

            num_doc = 0

            for identifiers in file_identifiers.values():

                if word in identifiers:
                    num_doc += 1

            idf[word] = log(len(file_identifiers) / num_doc)

        return idf

    @staticmethod
    def compute_tf_idf(voc_to_index, tf, idf):
        """ Compute the tf_idf values for each file in repo.
        """

        tf_idf = {}

        for file_path in tf.keys():

            tf_idf[file_path] = [0 for _ in range(len(voc_to_index))]

            for word, index in voc_to_index.items():

                tf_idf[file_path][index] = tf[file_path][index] * idf[word]

        return tf_idf

    @staticmethod
    def compute_cosine_distance(a, b):
        """ Compute the cosine distance between two vectors a and b.
        """

        norm_a = 0
        norm_b = 0

        dot = 0

        for i in range(len(a)):

            dot += a[i] * b[i]

            norm_a += a[i] ** 2
            norm_b += b[i] ** 2

        norm_a = sqrt(norm_a)
        norm_b = sqrt(norm_b)

        return dot / (norm_a * norm_b)


import os
import time
import warnings
warnings.filterwarnings("ignore")
import json
import torch
from transformers import DebertaV2Tokenizer, RobertaTokenizer

from util import form_dataset
from model import RE_Model
from eval import eval_re

def adjust_learning_rate_exp(optimizer, epoch, num_ne_epochs, ne_lr, qg_lr, exp_num=1, decay_every=20):
    if epoch < num_ne_epochs:
        lr = ne_lr * (exp_num ** (epoch // decay_every))
    else:
        lr = qg_lr * (exp_num ** (epoch // decay_every))
    if hasattr(optimizer, 'module'):
        for param_group in optimizer.module.param_groups:
            param_group['lr'] = lr
    else:
        for param_group in optimizer.param_groups:
            param_group['lr'] = lr
    print("adjusted lr to {}".format(lr))

def train_re(model, train_loader, dev_loader, test_loader, model_config, do_eval_only=False):

    best_mrr, test_best_mrr = 0., 0.

    output_model_file = model_config['checkpoint_route']
    latest_model_file = output_model_file[:(output_model_file.rfind('.pth'))] + "_latest.pth"
    log_output_file = model_config['log_dir'] + 'log_'+time.ctime()[4:19].replace(':','.')+'.dat'

    # determine gpu ids and parallel model
    os.environ['CUDA_VISIBLE_DEVICES'] = model_config.get('gpus', '0,1')
    main_gpu = int(os.environ['CUDA_VISIBLE_DEVICES'].split(',')[0])
    print('Running on gpu:', os.environ['CUDA_VISIBLE_DEVICES'])
    model = model.cuda()
    if torch.cuda.device_count() > 1:
        model = torch.nn.DataParallel(model)

    # optimizer defined after transfer to GPU
    re_lr_enc = model_config['re_lr_enc']
    weight_decay = model_config.get('weight_decay', 0)
    print('weight decay:', weight_decay)
    if hasattr(model, 'module'):
        optimizer = torch.optim.AdamW(model.module.parameters(), lr=re_lr_enc, weight_decay=weight_decay)     # 4e-6
    else:
        optimizer = torch.optim.AdamW(model.parameters(), lr=re_lr_enc, weight_decay=weight_decay)     # 4e-6

    # print/eval every how many batches?
    print_every, eval_every, save_latest_every = 20, 400, 400
    topk = model_config.get('pred_topk', 5)
    lr_decay_exp, decay_every = model_config['lr_decay_exp'], model_config['decay_every']

    epoch, batch_i = 0,0
    if do_eval_only:
        print("Do eval only.\nDev:")
        eval_re(model, dev_loader, log_output_file, output_model_file,\
                                epoch, best_mrr, topk=topk,\
                                ranges=model_config['dev_triple_ranges'], v_set=model_config['raw_dev_set'],\
                                save_res=False, output_full_result=False)
        print('\nTest:')
        eval_re(model, test_loader, log_output_file, output_model_file,\
                                epoch, test_best_mrr, topk=topk,\
                                ranges=model_config['test_triple_ranges'], v_set=model_config['raw_test_set'],\
                                save_res=False, output_full_result=True)
        exit(0)

    running_loss, correct, total = 0.,0,0
    # adjust lr every epoch
    adjust_learning_rate_exp(optimizer, epoch, 0, re_lr_enc, re_lr_enc, exp_num=lr_decay_exp, decay_every=decay_every)
    train_iter = iter(train_loader)
    while True:
        try:
            itm = train_iter.next()
            batch_i += 1
        except:
            # show train acc every epoch
            print('Train acc: %.8f' % (correct/total))
            # adjust lr every epoch
            epoch += 1
            if epoch >= 20:
                exit(0)
            adjust_learning_rate_exp(optimizer, epoch, 0, re_lr_enc, re_lr_enc, exp_num=lr_decay_exp, decay_every=decay_every)
            batch_i, running_loss, correct, total = 0,0.,0,0
            train_iter = iter(train_loader)
            itm = train_iter.next()

        sents, type_masks, att_masks, tags = itm
        sents, type_masks, att_masks, tags = sents.cuda(), type_masks.cuda(), att_masks.cuda(), tags.cuda()

        optimizer.zero_grad()
        loss = model(sents,\
                enc_attention_mask=att_masks,\
                token_type_ids=type_masks,\
                re_tags=tags)

        #loss = lossf(re_logits, tags)
        loss = loss.mean()      # support both single & multi-gpu training
        loss.backward()
        optimizer.step()
        running_loss += loss.item()

        #correct += (torch.max(re_logits, 1)[1] == tags).sum().item()
        total += tags.shape[0]

        if batch_i % print_every == print_every-1:  # print every 20 mini-batches
            f = open(log_output_file, 'a', encoding='utf-8')
            print('[%d, %5d] loss: %.5f' %
                    (epoch + 1, batch_i + 1, running_loss / print_every))
            f.write('[%d, %5d] loss: %.5f\n' %
                    (epoch + 1, batch_i + 1, running_loss / print_every))
            f.close()
            running_loss = 0.

        # evaluate every epoch
        if batch_i % eval_every == (eval_every-1):
            best_mrr = eval_re(model, dev_loader, log_output_file, output_model_file,\
                                epoch, best_mrr, topk=topk,\
                                ranges=model_config['dev_triple_ranges'], v_set=model_config['raw_dev_set'],\
                                save_res=True, output_full_result=False)

        # save latest checkpoint every 20 epochs
        if batch_i % save_latest_every == (save_latest_every-1):
            print("Saving latest model...")
            if hasattr(model, 'module'):
                torch.save(model.module.state_dict(), latest_model_file)
            else:
                torch.save(model.state_dict(), latest_model_file)

def main(model_config, do_eval_only=False):

    # Dataloaders
    train_full_fn = '../data/RE_gold/Train_RE_gold_preprocessed_1hop_split.json'
    dev_full_fn = '../data/RE_gold/Dev_RE_gold_preprocessed_1hop_split.json'
    test_full_fn = '../data/RE_gold/Test_RE_gold_preprocessed_1hop_split.json'
    tokenizer_dir = '../data/pretrained/roberta_large_tokenizer/'
    tokenizer_class = RobertaTokenizer
    tokenizer = tokenizer_class.from_pretrained(tokenizer_dir)

    train_batch, test_batch = model_config['train_batch'], model_config['test_batch']
    num_workers = model_config.get('loader_num_workers', 0)
    sample_ratio = model_config.get('sample_ratio', 50)

    train_loader, dev_loader, test_loader, dev_triple_ranges, raw_dev_set, test_triple_ranges, raw_test_set = form_dataset(train_full_fn, dev_full_fn, test_full_fn, train_batch, test_batch, tokenizer, num_workers, sample_ratio)
    model_config['tokenizer'] = tokenizer
    model_config['dev_triple_ranges'] = dev_triple_ranges
    model_config['raw_dev_set'] = raw_dev_set
    model_config['test_triple_ranges'] = test_triple_ranges
    model_config['raw_test_set'] = raw_test_set

    model_config['loss_function'] = torch.nn.CrossEntropyLoss()
    load_chkpt = not model_config['load_pre']
    # initiate a new model
    model = RE_Model(model_config)
    if load_chkpt or do_eval_only:
        # load checkpoints
        output_model_file = model_config['checkpoint_route']
        pretrain_model_file = model_config.get('pretrain_checkpoint_route', output_model_file)
        print("loading: ", pretrain_model_file)
        model.load_state_dict(torch.load(pretrain_model_file, map_location=torch.device('cpu')))

    train_re(model, train_loader, dev_loader, test_loader, model_config, do_eval_only=do_eval_only)

if __name__ == '__main__':
    config_fn = '../data/checkpoint/RE_Roberta_small_samp/config.json'
    with open(config_fn) as fin:
        model_config = json.load(fin)
    main(model_config, do_eval_only=False)


# -*- coding: utf-8 -*-

"""
@Time    : 2022/3/1 4:37 下午
@Author  : hcai
@Email   : hua.cai@unidt.com
"""
import os
import sys
import torch
import logging
from datetime import datetime
from torch.utils.data import DataLoader

sys.path.append(os.path.dirname(__file__))
from gutils.config import get_gpt_args
from gmodels.gpt_model import create_model
from gutils.gpt_data import collate_fn, load_dataset

class Evaluator(object):
    def __init__(self, **kwargs):
        config = get_gpt_args(**kwargs)

        # set cuda and device
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        config.device = device

        # 数据目录
        data_dir = os.path.join(config.data_dir, 'data') if not config.data_dir.endswith('data') else config.data_dir
        self.vocab_path = os.path.join(config.model_path, 'vocab.txt')
        self.validate_dataset = load_dataset(self.vocab_path, data_dir, config.max_len, mode='eval')
        self.validate_dataloader = DataLoader(self.validate_dataset, batch_size=self.config.batch_size, shuffle=True,
                                              num_workers=self.config.num_workers, collate_fn=collate_fn, drop_last=False)
        self.config = config

    def validate_epoch(self):
        logging.info("start validating")
        self.model.eval()

        device = self.config.device
        epoch_start_time = datetime.now()
        total_loss = 0
        # 捕获cuda out of memory exception
        try:
            with torch.no_grad():
                for batch_idx, (input_ids, labels) in enumerate(self.validate_dataloader):
                    input_ids = input_ids.to(device)
                    labels = labels.to(device)
                    outputs = self.model.forward(input_ids, labels=labels)
                    logits = outputs.logits
                    loss = outputs.loss
                    loss = loss.mean()

                    total_loss += loss.item()
                    del input_ids, outputs

                # 记录当前epoch的平均loss
                epoch_mean_loss = total_loss / len(self.validate_dataloader)
                logging.info("validate : loss {}".format(epoch_mean_loss))
                epoch_finish_time = datetime.now()
                logging.info('time for validating one epoch: {}'.format(epoch_finish_time - epoch_start_time))
                return epoch_mean_loss

        except RuntimeError as exception:
            if "out of memory" in str(exception):
                logging.info("WARNING: ran out of memory")
                if hasattr(torch.cuda, 'empty_cache'):
                    torch.cuda.empty_cache()
            else:
                logging.info(str(exception))
                raise exception

    def run_eval(self, model=None):
        if model is None:
            self.model = create_model(self.config)
        else:
            self.model = model
        loss = self.validate_epoch()
        return loss

if __name__ == '__main__':
    pass


import itertools
import time
from pprint import pprint

import numpy as np
import ray, torch
import torch.nn as nn
from ray.util.placement_group import placement_group, PlacementGroupSchedulingStrategy, remove_placement_group
from ray.util.queue import Queue
from copy import deepcopy

from architecture.mcts import MCTS, SearchType
from util.data import print_one_line

def generate_data(s, model_cls, params, num_games):
    bundles = []
    tasks = []

    # Create a placement group.
    gpu_decorator = ray.remote(num_cpus=4, num_gpus=1)
    cpu_decorator = ray.remote(num_cpus=4)
    for node in ray.nodes():
        if node['Alive']:
            if 'GPU' in node['Resources']:
                tasks.append(gpu_decorator(Worker))
                bundles.append({'CPU': 4, 'GPU': 1})
            else:
                bundles.append({'CPU': 4})
                tasks.append(cpu_decorator(Worker))
    print(bundles)
    pg = placement_group(bundles=bundles, strategy="STRICT_SPREAD")
    ray.get(pg.ready())

    task_queue = Queue(maxsize=num_games)
    for i in range(num_games):
        task_queue.put(i)

    for i, t in enumerate(tasks):
        tasks[i] = t\
            .options(scheduling_strategy=PlacementGroupSchedulingStrategy(placement_group=pg))\
            .remote(s, model_cls, params, task_queue)

    start = time.time()
    result = ray.get([t.run.remote() for t in tasks])
    print(time.time() - start)
    remove_placement_group(pg)
    return itertools.chain(*result)

class Worker:
    def __init__(self, s, model_cls, params, task_queue: Queue):
        self.s = deepcopy(s)
        self.task_queue = task_queue

        device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
        self.model: nn.Module = model_cls(device)
        self.model.load_state_dict(params)
        self.model.to(device)

    def run(self):
        self.model.eval()
        with torch.no_grad():
            mcts = MCTS(self.model, 2000, SearchType.AlphaZero)
            result = []

            while not self.task_queue.empty():
                item = self.task_queue.get(block=True)

                train_data = []
                print_moves = []

                s = self.s
                while s.get_reward() is None:
                    root = mcts.search(s, train=True)

                    s = s.take_action(root.action)
                    print_moves.append(str(s))
                    probabilities = root.game_state.get_emptyboard()

                    for child in root.children:
                        prob = child.N_sa / root.N_sa
                        probabilities[child.action] = prob

                    probabilities /= np.sum(probabilities)
                    train_data.append((root.game_state, probabilities))
                    del root

                reward = s.get_reward()
                train_data = [(game.board[None], probs, np.array([reward * game.current_player]))
                              for game, probs in train_data]
                print_one_line(print_moves)

                result.append(train_data)
            return result


# # from validate_email import validate_email

# # from tqdm import tqdm
# # # DNS.defaults['server']=['8.8.8.8', '8.8.4.4']

# # if validate_email('varmaashik@gmail.com'):
# # 	print("Email is Valid")
# # else:
# # 	print("Invalid Email")
# # is_valid = validate_email('varmaashik@gmail.com',check_mx=True)

# # if is_valid==True:
# # 	print("Email has SMTP Server")
# # else:
# # 	print("SMTP not found")
# # if validate_email('varmaashik@gmail.com',verify=True):
# # 	print("Email is Verified")
# # else:
# # 	print("Email not verified")
# import csv
# from emailverifier import Client
# from emailverifier import exceptions

# client = Client('at_9mJPBZyFvOEyDhVwv2tD2xYh7Qoh5')

# try:

# 	data = client.get("varmaashik@gmail.com")
# except exceptions.HttpException:
# # If you get here, it means service returned HTTP error code
# 	pass
# except exceptions.GeneralException:
# # If you get here, it means you cannot connect to the service
# 	pass
# except exceptions.UndefinedVariableException:
# # If you get here, it means you forgot to specify the API key
# 	pass
# except exceptions.InvalidArgumentException:
# # If you get here, it means you specified invalid argument
# # (options should be a dictionary)
# 	pass
# except:
# 	pass
# # Something else happened related. Maybe you hit CTRL-C
# # while the program was running, the kernel is killing your process, or
# # something else all together.

# print(data)

# # Use data.json_string to get raw data in JSON.
# # You can access any response field as a class property
# # by converting field name from "camelCase" to "snake_case"
# print("Email address: " + data.email_address)
# print("Format: " + str(data.format_check))
# print("DNS: " + str(data.dns_check))
# print("SMTP: " + str(data.smtp_check))
# print("Catch all: " + str(data.catch_all_check))
# print("Disposable: " + str(data.disposable_check))
# print("Free: " + str(data.free_check))
# print("Last audit date: " + str(data.audit.audit_updated_date))

import dns.resolver

records = dns.resolver.query('emailhippo.com', 'MX')
mxRecord = records[0].exchange
mxRecord = str(mxRecord)

import socket
import smtplib

# Get local server hostname
host = socket.gethostname()

# SMTP lib setup (use debug level for full output)
server = smtplib.SMTP()
server.set_debuglevel(0)

# SMTP Conversation
server.connect(mxRecord)
server.helo(host)
server.mail('varmaashik@gmail.com.com')
code, message = server.rcpt(str('varmaashik@gmail.com'))
server.quit()

# Assume 250 as Success
if code == 250:
	print('Success')
else:
	print('Bad')


import os
import tempfile
import unittest
import torch
from torch import nn

from towhee.trainer.optimization.adafactor import Adafactor
from towhee.trainer.optimization.adamw import AdamW
from towhee.trainer.scheduler import (
    configure_constant_scheduler,
    configure_constant_scheduler_with_warmup,
    configure_linear_scheduler_with_warmup,
    configure_cosine_scheduler_with_warmup,
    configure_cosine_with_hard_restarts_scheduler_with_warmup,
    configure_polynomial_decay_scheduler_with_warmup
)

def unwrap_scheduler(scheduler, num_steps=10):
    lr_sch = []
    for _ in range(num_steps):
        lr_sch.append(scheduler.get_lr()[0])
        scheduler.step()
    return lr_sch

def unwrap_and_save_reload_scheduler(scheduler, num_steps=10):
    lr_sch = []
    for step in range(num_steps):
        lr_sch.append(scheduler.get_lr()[0])
        scheduler.step()
        if step == num_steps // 2:
            with tempfile.TemporaryDirectory() as tmpdirname:
                file_name = os.path.join(tmpdirname, 'schedule.bin')
                torch.save(scheduler.state_dict(), file_name)

                state_dict = torch.load(file_name)
                scheduler.load_state_dict(state_dict)
    return lr_sch

class ScheduleInitTest(unittest.TestCase):
    mdl = nn.Linear(50, 50)
    optimizer = AdamW(mdl.parameters(), lr=10.0)
    num_steps = 10

    def assertListAlmostEqual(self, list1, list2, tol, msg=None):
        self.assertEqual(len(list1), len(list2))
        for a, b in zip(list1, list2):
            self.assertAlmostEqual(a, b, delta=tol, msg=msg)

    def test_schedulers(self):
        common_kwargs = {'num_warmup_steps': 2, 'num_training_steps': 10}
        # schedulers doct format
        # function: (sched_args_dict, expected_learning_rates)
        scheds = {
            configure_constant_scheduler: ({}, [10.0] * self.num_steps),
            configure_constant_scheduler_with_warmup: (
                {'num_warmup_steps': 4},
                [0.0, 2.5, 5.0, 7.5, 10.0, 10.0, 10.0, 10.0, 10.0, 10.0],
            ),
            configure_linear_scheduler_with_warmup: (
                {**common_kwargs},
                [0.0, 5.0, 10.0, 8.75, 7.5, 6.25, 5.0, 3.75, 2.5, 1.25],
            ),
            configure_cosine_scheduler_with_warmup: (
                {**common_kwargs},
                [0.0, 5.0, 10.0, 9.61, 8.53, 6.91, 5.0, 3.08, 1.46, 0.38],
            ),
            configure_cosine_with_hard_restarts_scheduler_with_warmup: (
                {**common_kwargs, 'num_cycles': 2},
                [0.0, 5.0, 10.0, 8.53, 5.0, 1.46, 10.0, 8.53, 5.0, 1.46],
            ),
            configure_polynomial_decay_scheduler_with_warmup: (
                {**common_kwargs, 'power': 2.0, 'lr_end': 1e-7},
                [0.0, 5.0, 10.0, 7.656, 5.625, 3.906, 2.5, 1.406, 0.625, 0.156],
            ),
        }

        for scheduler_func, data in scheds.items():
            kwargs, expected_learning_rates = data

            scheduler = scheduler_func(self.optimizer, **kwargs)
            self.assertEqual(len([scheduler.get_lr()[0]]), 1)
            lr_sch_1 = unwrap_scheduler(scheduler, self.num_steps)
            self.assertListAlmostEqual(
                lr_sch_1,
                expected_learning_rates,
                tol=1e-2,
                msg=f'failed for {scheduler_func} in normal scheduler',
            )

            scheduler = scheduler_func(self.optimizer, **kwargs)
            lr_sch_2 = unwrap_and_save_reload_scheduler(scheduler, self.num_steps)
            self.assertListEqual(lr_sch_1, lr_sch_2, msg=f'failed for {scheduler_func} in save and reload')

class OptimizationTest(unittest.TestCase):
    def assertListAlmostEqual(self, list1, list2, tol):
        self.assertEqual(len(list1), len(list2))
        for a, b in zip(list1, list2):
            self.assertAlmostEqual(a, b, delta=tol)

    def test_adam_w(self):
        w = torch.tensor([0.1, -0.2, -0.1], requires_grad=True)
        target = torch.tensor([0.4, 0.2, -0.5])
        criterion = nn.MSELoss()
        # No warmup, constant schedule, no gradient clipping
        optimizer = AdamW(params=[w], lr=2e-1, weight_decay=0.0)
        for _ in range(100):
            loss = criterion(w, target)
            loss.backward()
            optimizer.step()
            w.grad.detach_()  # No zero_grad() function on simple tensors. we do it ourselves.
            w.grad.zero_()
        self.assertListAlmostEqual(w.tolist(), [0.4, 0.2, -0.5], tol=1e-2)

    def test_adafactor(self):
        w = torch.tensor([0.1, -0.2, -0.1], requires_grad=True)
        target = torch.tensor([0.4, 0.2, -0.5])
        criterion = nn.MSELoss()
        # No warmup, constant schedule, no gradient clipping
        optimizer = Adafactor(
            params=[w],
            lr=1e-2,
            eps=(1e-30, 1e-3),
            clip_threshold=1.0,
            decay_rate=-0.8,
            beta1=None,
            weight_decay=0.0,
            relative_step=False,
            scale_parameter=False,
            warmup_init=False,
        )
        for _ in range(1000):
            loss = criterion(w, target)
            loss.backward()
            optimizer.step()
            w.grad.detach_()  # No zero_grad() function on simple tensors. we do it ourselves.
            w.grad.zero_()
        self.assertListAlmostEqual(w.tolist(), [0.4, 0.2, -0.5], tol=1e-2)

if __name__ == '__main__':
    unittest.main()


import requests
import time
import os
from dotenv import load_dotenv
from twilio.rest import Client

def get_status(user_id):
    load_dotenv()
    token = os.getenv('token_vk')
    params = {
        'user_ids' : user_id,
        'access_token' : token,
        'v':5.103,
        'fields':'online',
    }
    url = 'https://api.vk.com/method/users.get'
    status = requests.post(url=url, params=params)
    return status.json()['response'][0]['online']

def sms_sender(sms_text):
    load_dotenv()
    account_sid = os.getenv('sid_twilio')
    auth_token = os.getenv('token_twilio')
    NUMBER_FROM = os.getenv('NUMBER_FROM')
    NUMBER_TO = os.getenv('NUMBER_TO')
    client = Client(account_sid, auth_token)
    message = client.messages.create(
                              body=sms_text,
                              from_=NUMBER_FROM,
                              to=NUMBER_TO
                          )
    return message.sid  # Верните sid отправленного сообщения из Twilio 642 153

if __name__ == "__main__":
    vk_id = input("Введите id ")
    while True:
        if get_status(vk_id) == 1:
            sms_sender(f'{vk_id} сейчас онлайн!')
            break
        time.sleep(5)


from apistar import Include, Route, hooks, http, types
from apistar.frameworks.wsgi import WSGIApp as App
from apistar.handlers import docs_urls, static_urls
import pymongo

def AcceptOrigin(method: http.Method, response: types.ReturnValue):
    response.headers.append("Access-Control-Allow-Origin", "*")
    response.headers["Access-Control-Allow-Methods"] = "GET,POST,PUT,DELETE,OPTIONS,HEADERS"
    response.headers["Access-Control-Allow-Headers"] = "Authorization"
    response.headers["Access-Control-Expose-Headers"] = "*"
    if method.lower() == "options":
        response.status = 200
    return response

def welcome(name=None):
    if name is None:
        return {'message': 'Welcome to API Star!'}
    return {'message': 'Welcome to API Star, %s!' % name}

settings = {
    "AFTER_REQUEST": [
        hooks.render_response,
        AcceptOrigin,
    ]
}

__mongo = None
def connect_db():
    global __mongo
    if __mongo is None:
        __mongo = pymongo.MongoClient()
    db = __mongo['mck']
    collection = db['yunniao']
    return collection

def show_task(task_id: int):
    collection = connect_db()
    record = collection.find_one({'task_id': task_id}, {"_id": 0})
    return record

routes = [
    Route('/', 'GET', welcome),
    Route('/task', 'GET', show_task),
    Include('/docs', docs_urls),
    Include('/static', static_urls)
]

app = App(routes=routes, settings=settings)

if __name__ == '__main__':
    app.main()


from alpha_vantage.techindicators import TechIndicators
from alpha_vantage.timeseries import TimeSeries
from src import Constants
from src import AlphaV
from src.Persister import Persister
from src.PersisterSqlite import PersisterSqlite
from src.Utils import Utils
from src.Scraper import Scraper
from src.Recommender import Recommender
from time import time
import os
import argparse
import matplotlib
matplotlib.use('Agg')

class PortfolioManager:
    def main(self):
        print("Portfolio Manager started")
        parser = argparse.ArgumentParser()
        parser.add_argument('-database', action='store', dest='database',
                            help='Either of Sqlite/AWS')
        parser.add_argument('-mode', action='store', dest='mode',
                            help='Test Mode[test/real]')
        parser.add_argument('-image', action='store', dest='image',
                            help='Generate Images[generate]')
        parser.add_argument('-model', action='store', dest='model',
                            help='create machine learning model[create]')
        parser.add_argument('-training_period', action='store', dest='training_period',
                            help='Training period in years')
        self.args = parser.parse_args()
        scraper = Scraper()
        config = Utils.read_properties()
        av = AlphaV.AlphaVantage()
        ti = TechIndicators(key=os.environ["ALPHA_VANTAGE_KEY"], output_format='pandas')
        ts = TimeSeries(key=os.environ["ALPHA_VANTAGE_KEY"], output_format='pandas')
        if self.args.mode == 'test':
            symbol_list = scraper.read_symbols()
        else:
            symbol_list = self.get_exchange_symbol_list(Constants.SP500, scraper)
        if self.args.database == 'sqllite':
            self.dump_market_data(symbol_list, Constants.SP500, av, ti, ts, config)
        elif self.args.database == 'AWS':
            for exchange in Constants.Exchanges:
                symbol_list = self.get_exchange_symbol_list(exchange, scraper)
                self.dump_market_data(symbol_list, exchange, av, ti, ts, config)

        if self.args.model == 'create':
            recommender = Recommender(self.args, symbol_list, True)
        else:
            recommender = Recommender(self.args, symbol_list, False)
        #recommender.generate_recommendation(Constants.LearningModel.IMAGE_BASED_CLASSIFICATION, symbol_list)
        #recommender.generate_recommendation(Constants.LearningModel.DECISION_TREE_CLASSIFICATION, symbol_list)
        recommender.generate_recommendation(Constants.LearningModel.LSTM_CLASIFICATION, symbol_list)

    def get_exchange_symbol_list(self, exchange, scraper):
        print("Loading " + exchange + " stocks..")
        symbol_list = {}
        if exchange == Constants.SP500:
            symbol_list = scraper.get_sp500_symbols()
        else:
            symbol_list = scraper.get_exchange_symbols(exchange)
        print("Loaded " + exchange + " stocks..")
        return symbol_list

    def dump_market_data(self, symbol_list, exchange, av, ti, ts, config):
        start_time = time()
        symbol_ts = {}
        for symbol in symbol_list:
            # Fetch Technical indicators data
            rsi_wide_data, rsi_meta_data = av.call_av(Constants.TechnicalIndicatorType.RSI, ti, symbol, config)
            obv_wide_data, obv_meta_data = av.call_av(Constants.TechnicalIndicatorType.OBV, ti, symbol, config)
            bbands_wide_data, bbands_meta_data = av.call_av(Constants.TechnicalIndicatorType.BBAND, ti, symbol, config)
            macd_wide_data, macd_meta_data = av.call_av(Constants.TechnicalIndicatorType.MACD, ti, symbol, config)

            # Fetch TS data
            ts_data, ts_meta_data = av.call_av_time_series(ts, symbol, config)
            if not rsi_wide_data.empty and not obv_wide_data.empty and not bbands_wide_data.empty and not macd_wide_data.empty and not ts_data.empty:
                joinedData = rsi_wide_data.join(obv_wide_data).join(bbands_wide_data).join(macd_wide_data).join(ts_data);
                symbol_ts[symbol] = joinedData

        end_time = time()
        print("Time Elapsed in fetching technical indicators and time series data for exchange [ " + exchange + " ]: " + str(end_time - start_time))
        start_time = time()
        if self.args.database == 'sqllite':
            persister = PersisterSqlite()
            persister.insert_data(symbol_ts, symbol_list)
        else:
            persister = Persister()
            persister.insert_data(symbol_ts, symbol_list, exchange)

        end_time = time()
        print("Time Elapsed in inserting technical indicators and time series data exchange [ " + exchange + " ]: " + str(end_time - start_time))

    def generate_signal(self, symbol_tis):
        for symbol in symbol_tis:
            signal, rsi_value = self.scan_rsi_indicator(symbol_tis[symbol])
            self.print_signal(self, signal, rsi_value, symbol)

    @staticmethod
    def print_signal(self, signal, rsi_value, stock):
        if signal == Constants.Signal.BUY:
            print("BUY Signal generated for: " + stock + ". RSI Value: " + str(rsi_value))
        elif signal == Constants.Signal.SELL:
            print("SELL Signal generated for: " + stock + ". RSI Value: " + str(rsi_value))

if __name__ == "__main__":
    portfolio_manager = PortfolioManager()
    portfolio_manager.main()


from flask import Flask, render_template, request

import multiprocessing
import utilities
import filePreprocessing
import config
import redis
import netRB
import sys

#Počiatočná inicializácia objektov
logs = utilities.Logs()

try:
    r = redis.Redis(host=config.REDIS_IP, port=config.REDIS_PORT)
except:
    logs.logger.error("Connecting to redis failed.")
    sys.exit(1)

app = Flask(__name__)
model = netRB.Classify(r,logs)
global id
id = 0

#Prijímanie žiadosti na spracovanie CSV súborov
#Spracovávané CSV súbory majú byť uložené v zložke ./csvFiles
@app.route("/capture")
def newCapture():
    #Príjem ID CSV súborov
    if request.args.get("csv"):
        p = multiprocessing.Process(target=filePreprocessing.csvPreprocessing, args=(request.args.get("csv"), logs,))
        p.start()
        return "CSV added to the queue."
    else:
        return "No CSV files specify"

#Prijímanie žiadosti na debugovanie systému spracovaním predvytvorenými obrázkami
#Overované obrázky majú byť uložené v zložke ./images
@app.route("/debug")
def debug():
    global id
    lastID = id
    try:
        id = filePreprocessing.imgsPreprocessing(id,logs)
    except:
        logs.logger.error("Error in debug mode.")
        return "Error in debug mode."
    return f"{id-lastID} images was preprocessed."

#Prijímanie žiadosti na vytvorenie predikcie
#Obrázky musia byť uložené v redise
@app.route("/predictions")
def predictions():
    #Predikcia na obrázkov z csv súborov
    if request.args.get("csv"):
        imgs = request.args.get("csv")
        try:
            model.startDetection(imgs,"csv")
        except:
            logs.logger.error(f"Prediction error. CSV IDs: {imgs}")
            return "Prediction error."
        return "Prediction completed."
    #Predikcia na obrázkoch pre debugovanie
    if request.args.get("imgs"):
        imgs = request.args.get("imgs")
        try:
            model.startDetection(imgs,"image")
        except:
            logs.logger.error(f"Prediction error. Images IDs: {imgs}")
            return "Prediction error."
        return "Prediction completed."
    return "No images for prediction was given."

if __name__ == "__name__":
    app.run(port=config.API_PORT, debug = False)