# -*- coding: utf-8 -*-
# __author__ = 'fwy'
# last_modify:20200609

from biocluster.core.exceptions import OptionError
from biocluster.module import Module
import gevent
import re
import os
import unittest
import gevent.subprocess as subprocess
import json
import glob
from collections import OrderedDict
from biocluster.config import Config
import pandas as pd
import numpy  as np

class DiffGenesetAllModule(Module):
    """
    该Module用于基因融合分析，默认使用方法
    """
    def __init__(self, work_id):
        super(DiffGenesetAllModule, self).__init__(work_id)
        options = [
            # {"name": "geneset_names", "type": "infile", "format": "ref_rna_v2.common"},
            {"name": "diff_path", "type": "string", "default": None},
            {"name": "annot_result", "type": "string", "default": None},
            {"name": "diff_method", "type": "string"},
            {"name": "transcript_exp_file", "type": "string"},
            {"name": "level", "type": "string", "default": "G"},
            {"name": "kegg_version", "type": "string", "default": None},
            {"name": "go_version", "type": "string", "default": '20200628'},
            {"name": "reactome_version", "type": "string", "default": None},
            {"name": "species", "type": "string", "default": "Homo_sapiens"}
        ]
        self.add_option(options)
        self.geneset_prepare_tools=[]
        self.genest_infos = OrderedDict()
        self.total_dict = OrderedDict()
        self.run_genesets_analysis = True
        self.diff_geneset_prepare = ""
        self.selecte_geneset = ""

    def check_options(self):
        return True

    def set_step(self, event):
        if 'start' in event['data'].keys():
            event['data']['start'].start()
        if 'end' in event['data'].keys():
            event['data']['end'].finish()
        self.step.update()

    def get_genesets_info(self):
        diff_dir = self.option("diff_path")
        all_detail = pd.read_table(os.path.join(diff_dir, 'all_detail.txt'), header=0, sep='\t')
        all_detail['type'] = np.where(all_detail['seq_id'].str.contains("sRNA"), 'sRNA', 'mRNA')
        all_detail.fillna('', inplace=True)
        diff_dict_list = all_detail.to_dict('records')
        all_diff_list = list()
        for i in all_detail.groupby('compare'):
            compare = i[0]
            diff_pd = i[1]
            ctrl, test = compare.split('|')
            name = ctrl + '_vs_' + test + '_mRNA'
            sig_seqs = diff_pd[(~diff_pd['seq_id'].str.contains("sRNA")) & (diff_pd['significant'] == 'yes')][
                'seq_id'].tolist()
            sig_regulate = list(diff_pd['regulate'][diff_pd['significant'] == 'yes'])
            all_diff_list += sig_seqs
        all_diff_list = list(set(all_diff_list))
        geneset_name = 'All_Diff_mRNA'
        length = len(all_diff_list)
        self.genest_infos[geneset_name] = {
                "geneset_name": geneset_name,
                "compare": "All",
                "gene_num" :length,
                "regulate": "all",
                "compare_path": os.path.join(diff_dir, 'all_detail.txt'),
                'all_diff_list':all_diff_list
        }
        with open(os.path.join(self.work_dir, "sele_geneset_info"), "w") as f:
            result_infos = self.genest_infos[geneset_name]
            json.dump(result_infos, f)
        self.prepare_common_file()

    def prepare_common_file(self):
        self.logger.info("开始准备差异数据挖掘公共文件")
        self.common = self.add_tool("prok_rna.diffgt4work.annot_prepare")
        opts = {
            "annot_result": self.option("annot_result"),
            "level": self.option("level"),
            "kegg_version": self.option("kegg_version"),
            'go_version': self.option("go_version"),
            "transcript_exp_file": self.option("transcript_exp_file"),
            "species": self.option("species"),
        }
        self.common.set_options(opts)
        self.common.on("end", self.prepare_geneset_infos)
        # self.common.on("end", self.prepare_geneset_infos)
        self.common.run()

    def get_selected_genset_info(self):
        # self.get_genesets_info()
        geneset_name = self.genest_infos.keys()[0]
        if self.genest_infos[geneset_name]["gene_num"] < 50:
            return False
        else:
            return geneset_name

    def prepare_geneset_infos(self):
        self.selecte_geneset= self.get_selected_genset_info()
        if not self.selecte_geneset:
            self.set_output()
        else:
            self.diff_geneset_prepare = self.add_tool("prok_rna.diffgt4work.diff_geneset_prepare")
            opts = {
                "annot_result": self.option("annot_result"),
                "compare": self.genest_infos[self.selecte_geneset]["compare"],
                "regulate": self.genest_infos[self.selecte_geneset]["regulate"],
                "geneset_name": self.genest_infos[self.selecte_geneset]["geneset_name"],
                "compare_path": self.genest_infos[self.selecte_geneset]["compare_path"],
                "level": self.option("level"),
                "species": self.option("species"),
                "geneset_infos" :os.path.join(self.work_dir, "sele_geneset_info"),
            }
            self.diff_geneset_prepare.set_options(opts)
            self.diff_geneset_prepare.on('end', self.set_output)
            self.diff_geneset_prepare.run()

    def set_output(self):
        if not self.run_genesets_analysis :
            self.total_dict["has_geneset_pipline_results"] = False
        else:
            common_file_path = self.common.option("common_file_json").prop["path"]
            common_file_dict = json.load(open(self.common.option("common_file_json").prop["path"]))
            self.total_dict["common_file"] = common_file_dict
            self.total_dict["genesets"] =OrderedDict()
            geneset_json = self.diff_geneset_prepare.option("geneset_json").prop["path"]
            geneset_dict = json.load(open(geneset_json))
            geneset_name = geneset_dict["geneset_name"]
            self.total_dict["genesets"][geneset_name] = geneset_dict
        with open(os.path.join(self.output_dir,"prepare_json"),"w") as f:
            json.dump(self.total_dict,f,indent=2)
        self.end()

    def run(self):
        super(DiffGenesetAllModule, self).run()
        self.logger.info("开始运行差异基因集数据挖掘数据准备")
        self.logger.info("首先解析基因集列表文件")
        self.get_genesets_info()

    def end(self):
        super(DiffGenesetAllModule, self).end()

class TestFunction(unittest.TestCase):
    """
    This is test for the tool. Just run this script to do test.
    """
    def test(self):
        import random
        from mbio.workflows.single import SingleWorkflow
        from biocluster.wsheet import Sheet
        test_dir='/mnt/ilustre/users/sanger-dev/workspace/20190322/Snp_tsg_33538_3123_8568/SnpRna'
        data = {
            "id": "geneset_prepare" + str(random.randint(1, 10000)),
            "type": "module",
            "name": "medical_transcriptome.workflow_diffgt.diff_geneset_all",
            "instant": False,
            "options": dict(
                diff_path= "/mnt/ilustre/users/sanger-dev/workspace/20200923/MedicalTranscriptome_medical_transcriptome_workflow_20200923_153802009/DiffexpBatch/output",
                annot_result ="/mnt/ilustre/users/sanger-dev/workspace/20200923/MedicalTranscriptome_medical_transcriptome_workflow_20200923_153802009/AnnotMerge__1/output",
                diff_method="DESeq2",
                kegg_version ="202007",
                reactome_version="202007",
                level = "G",
                gene_count_file ="/mnt/ilustre/users/sanger-dev/workspace/20200923/MedicalTranscriptome_medical_transcriptome_workflow_20200923_153802009/Quant/output/gene.count.matrix",
            )
           }
        wsheet = Sheet(data=data)
        wf = SingleWorkflow(wsheet)
        wf.run()

if __name__ == '__main__':
    unittest.main()


import ply.lex as lex
import ply.yacc as yacc
syntax_correct = True
tokens = [
    'CASE',
    'WHEN',
    'END',
    'IDENTIFIER',
    'COLON',
    'NUMBER',
    'PLUS',
    'MINUS',
    'TIMES',
    'DIVIDE',
    'LPAREN',
    'RPAREN',
    'EQ',
    'NEQ',
    'LT',
    'LE',
    'GT',
    'GE',
    'ASSIGN'
]
t_IDENTIFIER = r'[a-zA-Z_"][a-zA-Z0-9_"]*'  # Updated regular expression
t_COLON = r':'
t_NUMBER = r'\d+'
t_PLUS = r'\+'
t_MINUS = r'-'
t_TIMES = r'\*'
t_DIVIDE = r'/'
t_LPAREN = r'\('
t_RPAREN = r'\)'
t_EQ = r'=='
t_NEQ = r'!='
t_LT = r'<'
t_LE = r'<='
t_GT = r'>'
t_GE = r'>='
t_ASSIGN = r'='
t_ignore = ' \t\n'
# Special case handling for CASE and END
def t_CASE(t):
    r'case'
    return t
def t_END(t):
    r'end'
    return t
def t_WHEN(t):
    r'when'
    return t
# Parser
def p_switch_statement(p):
    '''
    switch_statement : CASE expression COLON when_clauses END
    '''

def p_when_clauses(p):
    '''
    when_clauses : when_clause when_clauses
                 | empty
    '''

def p_when_clause(p):
    '''
    when_clause : WHEN expression COLON statements
    '''

def p_statements(p):
    '''
    statements : statement statements
               | empty
    '''

def p_statement(p):
    '''
    statement : assignment_statement
                | empty

    '''
def p_assignment_statement(p):
    '''
    assignment_statement : IDENTIFIER ASSIGN expression
    '''
def p_expression(p):
    '''
    expression : NUMBER
               | IDENTIFIER
               | expression PLUS expression
               | expression MINUS expression
               | expression TIMES expression
               | expression DIVIDE expression
               | LPAREN expression RPAREN
               | expression EQ expression
               | expression NEQ expression
               | expression LT expression
               | expression LE expression
               | expression GT expression
               | expression GE expression
    '''

def p_empty(p):
    '''
    empty :
    '''

def p_error(p):
    global syntax_correct
    syntax_correct = False
    print(f"Syntax error at line {p.lineno}, Unexpected token '{p.value}'")
    exit(0)
# Build the lexer and parser
lexer = lex.lex()
parser = yacc.yacc()

# Test the parser with a Ruby switch case statement
ruby_code = """
cse x:
  when :
    a=b
  whe 2:
    b=c
ed
"""

lexer.input(ruby_code)
for token in lexer:
    print(token)
if syntax_correct:
    print("Syntax is correct.")
else:
    print("Incorrect syntax")
parser.parse(ruby_code)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import re
import sys
import json
import uuid
import base64
import socket
import string
import binascii
import datetime

import dpkt, pcap

import argparse
import urllib.parse

def common_tcp_data( ip ) :
    tcp = ip.data
    src = ip.src
    dst = ip.dst
    src_a = socket.inet_ntoa(src)
    dst_a = socket.inet_ntoa(dst)

    # TCP情報収集
    tcp_data = {}
    tcp_data["seq"] = tcp.seq
    tcp_data["seq_raw"] = tcp.seq
    tcp_data["window_size_value"] = tcp.win
    tcp_data["window_size"] = tcp.win
    tcp_data["window_size_scalefactor"] = tcp.win

    return { "uuid": str(uuid.uuid4()), "destination_ip": dst_a, "destination_port": tcp.dport,
            "source_ip": src_a, "source_port": tcp.sport, "tcp":tcp_data }

def parse_tcp_tls( ip, tcp_buffer ) :
    TLS_HANDSHAKE = 22

    records, bytes_used = dpkt.ssl.tls_multi_factory(tcp_buffer)
    results = []
    for record in records:
        if record.type != TLS_HANDSHAKE:
            continue
        if len(record.data) == 0:
            continue
        client_hello = bytearray(record.data)
        if client_hello[0] != 1:
            # We only want client HELLO
            continue
    return ""

def parse_tcp_http( ip, tcp_buffer ) :
    cmd_list = ["wget","curl","rm","chmod","busybox","mv","ps","kill","xargs","iptables","grep","pkill","netstat","pgrep",
            "chattr","service","systemctl","crontab","apt","apt-get","unhide","ufw","userdel","adduser","useradd"]
    pattern = "https?://[\w/:%#\$&\?\(\)~\.=\+\-]+"

    http = dpkt.http.Request(tcp_buffer)

    # HTTP情報収集
    http_data = {}
    http_data["http_http_request_method"] = http.method
    http_data["http_http_request_uri"] = http.uri
    http_data["http_http_request_version"] = "HTTP/"+http.version
    http_data["http_http_request"] = True
    http_data["http_http_request_number"] = "1"
    for key in http.headers :
        http_data["http_http_"+key] = http.headers[key]
    #http_data["http_http_request_full_uri"] = "http://"+http_data["http_http_host"]+"/"+http_data["http_http_request_uri"]
    #http_data["http_http_request_full_uri"] = "http:/"+os.path.join(http_data["http_http_host"],http_data["http_http_request_uri"])
    http_data["http_http_request_full_uri"] = "http://"+http_data["http_http_host"]+"/"+http_data["http_http_request_uri"]

    http_data["http_http_params"] = urllib.parse.urlparse(http_data["http_http_request_full_uri"]).query
    http_data["http_http_body"] = http.body.decode('utf-8')

    # URIの分割
    dirname, basename = os.path.split(http_data["http_http_request_uri"])
    #http_data["http_http_request_uri_detail"] = {"urlpath": dirname, "resourcename": basename}
    http_data["http_http_request_uri_urlpath"]      = dirname
    http_data["http_http_request_uri_resourcename"] = basename

    # 攻撃で利用されるコマンド文字列の検索
    http_data["http_http_cmd"] = []

    check_target = http_data["http_http_params"]+" "+http_data["http_http_body"]
    for cmd in cmd_list :
        if (cmd in check_target) :
            http_data["http_http_cmd"].append( cmd )

    # ダウンロードURLの抽出
    http_data["http_http_download"] = re.findall(pattern, check_target )

    # ダウンロードURLの分解
    http_data["http_http_download_detail"] = []
    for url in http_data["http_http_download"] :
        u_parse = urllib.parse.urlparse(url)
        result = {
                "method"      : u_parse.scheme,
                "domain"      : u_parse.netloc,
                "path"        : u_parse.path,
                "params"      : u_parse.params,
                "query"       : u_parse.query,
                "query_detail": [] }
        result["query_detail"] = urllib.parse.parse_qs(u_parse.query)
        http_data["http_http_download_detail"].append(result["query_detail"])

    return str(http), http_data

def parse_tcp_telnet( ip, tcp_buffer ) :
    datas = dpkt.telnet.strip_options(tcp_buffer)
    tmp = []
    for item in datas[0] :
        tmp.append( item.decode() )
    return "\\n".join(tmp)

def parse_tcp_smb( ip, tcp_buffer ) :
    smb = dpkt.smb.SMB(tcp_buffer)
    return str(repr(smb))

def parse_udp_stun( ip, udp_buffer ) :
    datas = dpkt.stun.parse_attrs(udp_buffer)
    print(datas)

def detect_tcp( ip, tcp_info ) :
    result = common_tcp_data( ip )
    result["timestamp"] = str(datetime.datetime.utcfromtimestamp(tcp_info["start_time"]))
    result["connect_time"] = str(datetime.datetime.utcfromtimestamp(tcp_info["end_time"]))
    result["duration"] = tcp_info["end_time"]-tcp_info["start_time"]
    try :
        result["payload"] = base64.b64encode(tcp_info["buffer"]).decode()
    except :
        result["payload"] = "base64 except"

    try :
        ret_data, http_data = parse_tcp_http( ip, tcp_info["buffer"] )
        result["app_proto"] = "http"
        result["payload_printable"] = ret_data
        result["http"] = http_data
        return result
    except :
        import traceback
        traceback.print_exc()

    try :
        parse_tcp_tls( ip, tcp_info["buffer"] )
        result["app_proto"] = "ssl"
        return result
    except :
        import traceback
        traceback.print_exc()

    try :
        ret_data = parse_tcp_smb( ip, tcp_info["buffer"] )
        result["app_proto"] = "smb"
        result["payload_printable"] = ret_data
        return None
    except :
        import traceback
        traceback.print_exc()

    try :
        ret_data = parse_tcp_telnet( ip, tcp_info["buffer"] )
        result["app_proto"] = "telnet"
        result["payload_printable"] = ret_data
        return result
    except :
        import traceback
        traceback.print_exc()
#        pass

def main( device, pcap_option, pcap_file, output_file ) :
    packet_buffer = {}

    if pcap_file != None :
        print( pcap_file )
        pc = dpkt.pcap.Reader(open(pcap_file,'rb'))
    else :
        print( device )
        #pc = pcap.pcap( )
        pc = pcap.pcap( device, promisc=True, immediate=True )
        if pcap_option != None :
            pc.setfilter( 'host %s'%(pcap_option) )
        print( "Device Name:", pc.name, "|Filter:",pc.filter )

    for t,buf in pc :
        # Packetの取得
        eth = dpkt.ethernet.Ethernet(buf)

        # IPプロトコル確認
        if type(eth.data) != dpkt.ip.IP:
            continue

        ip = eth.data
        #TCPデータ
        if type(ip.data) == dpkt.tcp.TCP:
            tcp = ip.data
            fin_flag = int(( tcp.flags & dpkt.tcp.TH_FIN ) != 0)
            syn_flag = int(( tcp.flags & dpkt.tcp.TH_SYN ) != 0)
            rst_flag = int(( tcp.flags & dpkt.tcp.TH_RST ) != 0)
            psh_flag = int(( tcp.flags & dpkt.tcp.TH_PUSH) != 0)
            ack_flag = int(( tcp.flags & dpkt.tcp.TH_ACK ) != 0)
            urg_flag = int(( tcp.flags & dpkt.tcp.TH_URG ) != 0)

            src = ip.src
            dst = ip.dst
            src_a = socket.inet_ntoa(src)
            dst_a = socket.inet_ntoa(dst)

            source_info = "%s:%d"%(src_a, tcp.sport)

            if syn_flag == 1 and ack_flag == 0 :
                if source_info not in packet_buffer :
                    packet_buffer[source_info] = {"start_time": t, "end_time": None, "buffer": None}
                continue

            if source_info not in packet_buffer :
                continue

            if fin_flag == 0 :
                if packet_buffer[source_info]["buffer"] == None :
                    packet_buffer[source_info]["buffer"] = tcp.data
                else :
                    packet_buffer[source_info]["buffer"] += tcp.data
                continue
            else :
                packet_buffer[source_info]["end_time"] = t
                tcp_info = packet_buffer[source_info]
                del packet_buffer[source_info]

                result = detect_tcp( ip, tcp_info )
                if result == None :
                    continue
                print(result, flush=True)

                dt_now = datetime.datetime.now()
                if output_file != None :
                    filename = dt_now.strftime( output_file )
                    f = open(filename, 'a')
                    json.dump(result, f)
                    f.write( '\n' )
                    f.close()
                else :
                    print( json.dumps(result) )

        elif type(ip.data) == dpkt.udp.UDP:
            udp = ip.data
            if len(udp.data) != 0:
                continue

            print(udp)
            try :
                parse_udp_stun( ip, tcp )
                print("parse_udp_stun")
                continue
            except :
                pass
        elif type(ip.data) == dpkt.icmp.ICMP:
            icmp = ip.data
            if len(icmp.data) != 0:
                continue

            print(icmp)
            print("icmp")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='sniffer')
    parser.add_argument('-i', '--interface', help='target device')
    parser.add_argument('-t', '--target', help='target host 1.1.1.1 or 2.2.2.2')
    parser.add_argument('-r', '--read', help='pcapファイル名')
    parser.add_argument('-o', '--output', help='出力ファイル名 ', default="/iot_honey/logs/rfpf_%Y%m%d.json")

    args = parser.parse_args()

    if args.read != None :
        main( None, None, args.read, args.output )
    else :
        main( args.interface, args.target, None, args.output )
#        main( args.interface, args.target, None, None )


import cv2
import numpy as np
import matplotlib.pyplot as plt

# load and convert to binary
img = cv2.imread('./images/image.jpg', 0)
_, binary = cv2.threshold(img, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)

# disk kernel
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))

# erosion
eroded = cv2.erode(binary, kernel)

# plot
fig, ax = plt.subplots(1, 3, figsize=(10, 10))
ax[0].imshow(img, cmap='gray')
ax[0].set_title('Original')
ax[1].imshow(binary, cmap='gray')
ax[1].set_title('Binary')
ax[2].imshow(eroded, cmap='gray')
ax[2].set_title('Eroded')
plt.show()


import uuid
import pytest
from result import Ok

from core.festival.application.query.get_beer_by_id_query import GetBeerByIdQuery
from core.festival.application.query.get_beer_by_id_query_handler import GetBeerByIdQueryHandler
from core.festival.infrastructure.services.inmemory_beer_repository import InMemoryBeerRepository
from core.festival.tests.beer_builder import BeerBuilder

@pytest.mark.asyncio
async def test_get_beer_by_id_success() -> None:
    # Arrange
    beer = BeerBuilder().build()
    beer_repository = InMemoryBeerRepository(beers=[beer])
    handler = GetBeerByIdQueryHandler(beer_repository)
    query = GetBeerByIdQuery(id=beer.beer_id.value)

    # Act
    result = await handler.handle(query)

    # Assert
    assert isinstance(result, Ok)
    assert result.ok() == beer

@pytest.mark.asyncio
async def test_get_beer_by_id_not_found() -> None:
    # Arrange
    beer_repository = InMemoryBeerRepository(beers=[])
    handler = GetBeerByIdQueryHandler(beer_repository)
    query = GetBeerByIdQuery(id=uuid.uuid4())

    # Act
    result = await handler.handle(query)

    # Assert
    assert isinstance(result, Ok)
    assert result.ok() == None


from pyspark import SparkContext

logFile = r'C:\spark\spark-2.2.0-bin-hadoop2.7\README.md'
logdir = r'C:\spark\spark-2.2.0-bin-hadoop2.7\*.txt'
logdir1 = r'C:\spark\spark-2.2.0-bin-hadoop2.7'
sc = SparkContext(master="local[2]",appName="Simple App")

rdd = sc.parallelize([('a',7),('a',2),('b',2)])
rdd3 = sc.parallelize(range(100))
data = [1, 2, 3, 4, 5]
distData = sc.parallelize(data,)
a = distData.reduce(lambda a, b: a + b)

print(a)

# print(a)
#

def map_function(line):
    return len(line.split(' '))
logData1 = sc.textFile(logFile).cache()
logData = sc.textFile(logFile)
logData2 = sc.textFile(logdir) # đọc nhiều file thì sẽ map các file lại thành 1 ( map các line lại với nhau)
logData3 = sc.wholeTextFiles(logdir1) # đọc nhiều file thì sẽ map các file lại thành 1 ( map các line lại với nhau)

numAs = logData.filter(lambda line: 'a' in line).count()
numbBs = logData.filter(lambda line: 'b' in line).count()
a = logData.map(lambda x: len(x.split())).reduce(lambda a,b: a+b)
word_list = logData.flatMap(lambda x: x.split()) # tạo ra 1 RDD chứa data là các word, trong file
word_list1 = logData.map(lambda x: x.split()) # tạo ra 1 RDD chứa data là các word, trong file

map_for_reduce = word_list.map(lambda x: (x, 1))
word_count = map_for_reduce.reduceByKey(lambda a, b: a + b)
word_count = word_list.map(lambda x: (x,1)).reduceByKey(lambda a,b: a+b)
word_count_tuple = map_for_reduce.reduce(lambda a, b: a + b)
map_for_reduce.persist() # save map_for_reduce after first time computed, later use map_for_reduce don't need compute again.

map_1 = logData.map(map_function).reduce(lambda a,b: a if a >b else b)

counter = 0
rdd = sc.parallelize(data)

# Wrong: Don't do this!!
def increment_counter(x):
    global counter
    counter += x
rdd.foreach(increment_counter)

print("Counter value: ", counter)

x = logData.filter(lambda x: 'a' in x)

sc.stop()

rdd3.groupBy(lambda x: x % 2).mapValues(list).collect()


import argparse
import json
import logging
import os
import shutil
import sys
import tempfile

from datetime import datetime

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(name="wt-util")
logger.setLevel(logging.DEBUG)

currdir = os.path.dirname(os.path.abspath(__file__))
options = {
    "original_settings_filepath": "",
    "backup_dir": os.path.join(currdir, "backup")
}

def parse_cmd_arguments():
    parser = argparse.ArgumentParser(prog="wt-util",
                                     description="Script to update windows terminal settings json",
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    # https://stackoverflow.com/questions/24180527/argparse-required-arguments-listed-under-optional-arguments

    mandatoryArgs = parser.add_argument_group("mandatory arguments")

    mandatoryArgs.add_argument("-s", "--settings",
                        type=str,
                        help="full path to windows terminal settings.json",
                        required=True)
    mandatoryArgs.add_argument("-p", "--profile",
                        type=str,
                        help="wt profile name",
                        required=True)
    mandatoryArgs.add_argument("-k", "--key",
                        type=str,
                        help="settings key in the given profile to be updated",
                        required=True)
    mandatoryArgs.add_argument("-v", "--value",
                        type=str,
                        help="new value to be updated for the given key",
                        required=True)
    parser.add_argument("-o", "--output-file",
                        type=str,
                        help="Updated settings will be written to this file.If not given original settings file will be overwritten",
                        default="")
    return parser.parse_args()

def set_options(cmd_options):
    options["original_settings_filepath"] = cmd_options.settings
    options["profile_name"] = cmd_options.profile
    options["settings_key"] = cmd_options.key
    options["settings_value"] = cmd_options.value
    if (cmd_options.output_file != ""):
        options["output_file"] = cmd_options.output_file
    else:
        options["output_file"] = cmd_options.settings

def create_backup_settings(backup_filepath):
    logger.debug("Creating backup of settings json")
    shutil.copy2(options["original_settings_filepath"],
                 backup_filepath)
    logger.debug("Successfully created backup of settings json")

def test_backup():
    with tempfile.TemporaryDirectory() as tempdir:
        backup_file = os.path.join(tempdir, "settings_bk.json")
        logger.debug(backup_file)
        create_backup_settings(backup_file)
        original_file_lines = 0
        backup_file_lines = 0
        with open(options["original_settings_filepath"], 'r') as f:
            original_file_lines = len(f.readlines())

        with open(backup_file, 'r') as f:
            backup_file_lines = len(f.readlines())

        assert original_file_lines == backup_file_lines

def test():
    test_backup()

def get_timestr():
    now = datetime.today()
    date = now.date()
    time = now.time()
    return "{}_{}_{}_{}_{}_{}".format(date.day, date.month, date.year,
                                      time.hour, time.minute, time.second)

def check_options():
    if (not os.path.isfile(options["original_settings_filepath"])):
        raise FileNotFoundError("Incorrect settings json path")

def get_backup_filepath():
    os.makedirs(options["backup_dir"], exist_ok=True)
    backup_filepath = os.path.join(options["backup_dir"],
                                   "settings_bk_{}.json".format(get_timestr()))
    return backup_filepath

def load_json(filepath: str) -> dict:
    with open(filepath, 'r') as file:
        return json.load(file)

def write_json(d: dict, json_filepath: str):
    with open(json_filepath, 'w') as file:
        json.dump(d, file, indent=4)

def get_profile_idx(profiles_list: list, profile_name: str) -> int:
    for idx, profile in enumerate(profiles_list):
        if (profile["name"] == profile_name):
            return idx
    raise ValueError("There is no profile in settings with name %(name)s" % {
                     "name": profile_name})

def get_updated_settings() -> dict:
    settings_dict = load_json(options["original_settings_filepath"])
    profile_idx = get_profile_idx(
        settings_dict["profiles"]["list"], options["profile_name"])
    sys.stdout.write("Profile: {}\n".format(options["profile_name"]))
    sys.stdout.write("Key: {}\n".format(options["settings_key"]))
    sys.stdout.write("Previous value: {}\n".format(
        settings_dict["profiles"]["list"][profile_idx].get(options["settings_key"], '""')))
    sys.stdout.write("New value: {}\n".format(options["settings_value"]))

    settings_dict["profiles"]["list"][profile_idx][options["settings_key"]] = options["settings_value"]
    return settings_dict

def write_settings(settings_dict: dict):
    write_json(settings_dict, options["output_file"])
    sys.stdout.write("Settings updated successfully\n")

def main():
    cmd_options = parse_cmd_arguments()
    set_options(cmd_options)
    check_options()
    create_backup_settings(get_backup_filepath())
    new_settings = get_updated_settings()
    write_settings(new_settings)

if __name__ == "__main__":
    main()
    # test()


# from crypt import methods
from crypt import methods
from tkinter.messagebox import NO
from flask import Blueprint, request

from app.models import User, Channel, ChannelMessage, db, ChannelMember
from flask_login import current_user
from app.aws import upload_file_to_s3, allowed_file, get_unique_filename

channel_routes = Blueprint('channels', __name__, url_prefix='channels')

@channel_routes.route('/<int:channel_id>', methods=['GET', 'PUT', 'DELETE'])
def get_one_put_delete_channel(channel_id):
    channel = Channel.query.get(channel_id)
    if request.method == 'GET':
        return channel.to_dict()

    if request.method == 'PUT':
        name = request.json['name']
        diss_cord_bot_message = ChannelMessage.query.filter(ChannelMessage.channel_id == channel_id).filter(ChannelMessage.sender_id == 1).first()
        diss_cord_bot_message.content = f'Welcome to {channel.server.name}\'s Channel {name}'
        db.session.commit()

        channel.name = request.json['name']
        db.session.commit()
        return channel.to_dict()

    if request.method == 'DELETE':
        db.session.delete(channel)
        db.session.commit()
        return {'channelId': channel.id}

@channel_routes.route('/<int:channel_id>/messages', methods=['POST'])
def post_channel_message(channel_id):
    url = None
    if "image" in request.files:
        image = request.files["image"]
        if not allowed_file(image.filename):
            return {"errors": "file type not permitted"}, 400
        image.filename = get_unique_filename(image.filename)
        upload = upload_file_to_s3(image)
        url = upload["url"]

    new_message = ChannelMessage(
        channel_id=channel_id,
        sender_id = current_user.id,
        content = request.form['content'],
        picture = url

    )
    db.session.add(new_message)
    db.session.commit()
    return new_message.to_dict()

@channel_routes.route('/<int:channel_id>/members', methods=['GET', 'POST'])
def get_all_or_post_to_channel_members(channel_id):

    server_general_channel = Channel.query.get(channel_id)

    print('after first query',server_general_channel.to_dict())

    if request.method == 'GET':
        print('before second querrrrrry')
        channel_members= ChannelMember.query.filter(ChannelMember.channel_id == channel_id).all()

        return {'channelMembers': {member.id:member.to_dict() for member in channel_members}}
    if request.method == 'POST':
        data = request.json
        joining_member = User.query.get(data['userId'])
        data = request.json
        member = ChannelMember(channel_id=channel_id, user_id=data['userId'])
        db.session.add(member)

        first_message = ChannelMessage(channel_id=channel_id, sender_id=1, content=f'ðŸ‘‹{joining_member.username} has slid into the channel')
        db.session.add(first_message)
        db.session.commit()
        channel = Channel.query.get(channel_id)

        return {'member':member.to_dict(), 'channel': channel.to_dict()}

@channel_routes.route('/<int:channel_id>/members/<int:member_id>', methods=['DELETE'])
def delete_server_member(channel_id, member_id):
    channel = Channel.query.get(channel_id)
    member = ChannelMember.query.get(member_id)
    db.session.delete(member)
    leaving_message = ChannelMessage(channel_id=channel_id, sender_id=1, content=f'ðŸ˜ {member.member.username} has left the channel')
    db.session.add(leaving_message)
    db.session.commit()
    return {'memberId': member_id, 'channelId': channel.id }


import pytest

from zarr.n5 import N5ChunkWrapper, N5FSStore
from zarr.creation import create
from zarr.storage import atexit_rmtree
from numcodecs import GZip
import numpy as np
from typing import Tuple
import json
import atexit

from zarr.tests.util import have_fsspec

def test_make_n5_chunk_wrapper():
    dtype = "uint8"
    chunk_shape = (10,)
    codec = GZip()
    # ValueError when specifying both compressor and compressor_config
    with pytest.raises(ValueError):
        N5ChunkWrapper(
            dtype, chunk_shape=chunk_shape, compressor_config=codec.get_config(), compressor=codec
        )

    wrapper_a = N5ChunkWrapper(dtype, chunk_shape=chunk_shape, compressor_config=codec.get_config())
    wrapper_b = N5ChunkWrapper(dtype, chunk_shape=chunk_shape, compressor=codec)
    assert wrapper_a == wrapper_b

@pytest.mark.parametrize("chunk_shape", ((2,), (4, 4), (8, 8, 8)))
def test_partial_chunk_decode(chunk_shape: Tuple[int, ...]):
    # Test that the N5Chunk wrapper can handle fractional chunks that
    # may be generated by other N5 implementations
    dtype = "uint8"
    codec = GZip()
    codec_wrapped = N5ChunkWrapper(dtype, chunk_shape=chunk_shape, compressor=codec)
    subslices = tuple(slice(0, cs // 2) for cs in chunk_shape)
    chunk = np.zeros(chunk_shape, dtype=dtype)
    chunk[subslices] = 1
    subchunk = np.ascontiguousarray(chunk[subslices])
    assert np.array_equal(codec_wrapped.decode(codec_wrapped.encode(subchunk)), chunk)

@pytest.mark.skipif(have_fsspec is False, reason="needs fsspec")
def test_dtype_decode():
    path = "data/array.n5"
    atexit_rmtree(path)
    atexit.register(atexit_rmtree, path)
    n5_store = N5FSStore(path)
    create(100, store=n5_store)
    dtype_n5 = json.loads(n5_store[".zarray"])["dtype"]
    dtype_zarr = json.loads(create(100).store[".zarray"])["dtype"]
    assert dtype_n5 == dtype_zarr


import glob
from skimage import io
import os
import numpy as np

dirname = '20210711_test_out'
files = glob.glob('%s/**/*.png' % dirname,recursive=True)

testsets = {'showcase1':[],'showcase2':[],'showcase3':[]}

for file in files:
    for k in testsets.keys():
        if k in file:
            testsets[k].append(file)

ordered_testsets = {}

for k in testsets.keys():
    ordered_arr = ['']*len(testsets[k])
    print('ordered array now',len(ordered_arr))

    for f in testsets[k]:
        basename = os.path.basename(f)
        idx = int(basename.split('_')[0]) - 1
        ordered_arr[idx] = f

    ordered_testsets[k] = ordered_arr

# creating stacks
os.makedirs('%s/stacks' % dirname, exist_ok=True)

for k in testsets.keys():

    imgarr = []
    for f in ordered_testsets[k]:
        imgarr.append(io.imread(f))

    io.imsave('%s/stacks/%s.tif' % (dirname, k),np.array(imgarr))
    print('generated stack for',k)


#!/usr/bin/env python
#-*- coding:utf-8 -*-
#Created on 2017-6-10 by Author:GuoLikai

import docker
import socket
import time

from shipman.curl import Curl
from shipman.model.node import NodeInfo

class Myswarm(object):
    def ping_port(self, ip, port):
        cs = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        cs.settimeout(0.2)
        address = (str(ip), int(port))
        try:
            cs.connect((address))
        except socket.error as e:
            print(e)
            return 1
        cs.close()
        return 0

    def delete_con_usage(self, container_id):
        if not container_id:
            return
        else:
            NodeInfo.delete_con_usage(container_id)

    def _container_detail(self, node_ip, node_port, containers_id):
        url = ('http://' + node_ip + ":" + node_port + "/containers/" + containers_id + "/json")
        container_more_url = Curl(url)
        ret_json = container_more_url.get_value()
        return ret_json

    def container_list(self, node_ip, node_port):
        #print(node_ip)
        #print(node_port)
        url = 'http://' + node_ip + ":" + node_port + "/containers/json?all=1"
        container_url = Curl(url)
        ret_json = container_url.get_value()

        con_data = {}
        container_id = []
        if ret_json:
            for i in ret_json:
                container_id.append(i['Id'][0:12])
        else:
            return con_data

        if len(container_id) < 1:
            return con_data
        else:
            con_data = {}
            con_num = 1
            for con_id in container_id:
                tmp_dict = {}
                ret_json = self._container_detail(node_ip, node_port, con_id)
                if len(ret_json) < 1:
                    return con_data
                con_state = ""
                if ('Running' in ret_json['State'].keys()) and (
                    'Status' not in ret_json['State'].keys()):  # for docker 1.7
                    con_state = str(ret_json['State']['Running'])
                elif 'Status' in ret_json['State'].keys():  # for docker 1.9 and higher
                    con_state = str(ret_json['State']['Status'])
                else:  # for else
                    con_state = "Exited"
                tmp_dict['id_num'] = ret_json['Id'][0:12]
                tmp_dict['con_ip'] = ret_json['NetworkSettings']['IPAddress']
                tmp_dict['con_name'] = ret_json['Name'].replace('/','')
                tmp_dict['cpuperiod'] = ret_json['HostConfig']['CpuPeriod']
                tmp_dict['cpuquota'] = ret_json['HostConfig']['CpuQuota']
                tmp_dict['memory'] = ret_json['HostConfig']['Memory']
                tmp_dict['state'] = con_state
                tmp_dict['cmd'] = str(ret_json['Config']['Cmd'])
                tmp_dict['created'] = ret_json['State']['StartedAt']
                con_data[con_num] = tmp_dict
                con_num += 1
        return con_data

    def images_list(self, node_ip, node_port):
        client_ins = docker.Client(base_url='tcp://' + node_ip + ':' + node_port, version='1.20', timeout=5)
        ret_json = client_ins.images()
        images_list = []
        for one in ret_json:
            images_list.append(one['RepoTags'])
        return images_list

    def create_container(self, node_ip, node_port, conf):
        client_ins = docker.Client(base_url='tcp://' + node_ip + ":" + node_port, version='1.20', timeout=5)
        print("      Create the container......")
        container_ret = client_ins.create_container(image=conf['Image'],
                                                    stdin_open=conf['OpenStdin'],
                                                    tty=conf['Tty'],
                                                    command=conf['Cmd'],
                                                    name=conf['Name'],
                                                    hostname=conf['Hostname'],
                                                    host_config=conf['HostConfig'])
        if container_ret:
            time.sleep(0.3)
            return (container_ret['Id'])
        else:
            print("Can not create container")
            return

    def start_container(self,node_ip,node_port,container_id):
        if len(container_id) > 0:
            container_ip = ""
            client_ins = docker.Client(base_url='tcp://' + node_ip + ":" + node_port, version='1.20', timeout=5)
            client_ins.start(container_id)
            time.sleep(0.5)
            con_info = self._container_detail(node_ip, node_port,container_id)
            ret = NodeInfo.get_con_usage_info(container_id,node_ip)
            print(ret)
            if len(ret) == 0:
                NodeInfo.insert_con_usage(container_id[0:12],con_info['NetworkSettings']['IPAddress'],con_info['Name'].replace('/',''),node_ip)
            else:
                #print('update_con_usage')
                #print(con_info['NetworkSettings']['IPAddress'], con_info['Name'].replace('/', ''), node_ip, container_id[0:12])
                NodeInfo.update_con_usage(container_id[0:12],con_info['NetworkSettings']['IPAddress'],con_info['Name'].replace('/',''),node_ip)
            return 0
        else:
            print("Please enter the Container ID")
            return

    def container_info(self, node_ip, node_port, container_id):
        con_data = {}
        tmp_dict = {}
        ip_ret = ""
        ret_json = self._container_detail(node_ip, node_port, container_id)
        if len(ret_json) < 1:
            return con_data

        con_state = ""
        if ('Running' in ret_json['State'].keys()) and ('Status' not in ret_json['State'].keys()):  # for docker 1.7
            con_state = str(ret_json['State']['Running'])
        elif 'Status' in ret_json['State'].keys():  # for docker 1.9 and higher
            con_state = str(ret_json['State']['Status'])
        else:  # for else
            con_state = "Exited"
        tmp_dict['id_num'] = ret_json['Id'][0:12]
        tmp_dict['node_ip'] = node_ip
        tmp_dict['con_ip'] = ret_json['NetworkSettings']['IPAddress']
        tmp_dict['name'] = ret_json['Name'].replace('/','')
        tmp_dict['image'] = ret_json['Image']
        tmp_dict['created'] = ret_json['State']['StartedAt']
        tmp_dict['state'] = con_state
        tmp_dict['memory'] = ret_json['HostConfig']['Memory']
        tmp_dict['cpuperiod'] = ret_json['HostConfig']['CpuPeriod']
        tmp_dict['cpuquota'] = ret_json['HostConfig']['CpuQuota']
        tmp_dict['hostname'] = str(ret_json['Config']['Hostname'])
        tmp_dict['cmd'] = str(ret_json['Config']['Cmd'])
        con_data[1] = tmp_dict
        return con_data

    def stop_container(self, node_ip, node_port, container_id):
        if len(container_id) > 0:
            print("      Stop the container %s ........" % container_id)
            client_ins = docker.Client(base_url='tcp://' + node_ip + ":"
                                                + node_port, version='1.20', timeout=5)
            client_ins.stop(container_id)
        else:
            print("Please enter the Container ID")
            return

    def destroy_container(self, node_ip, node_port, container_id):
        if len(container_id) > 0:
            print("      Destroy the container %s ....... " % container_id)
            client_ins = docker.Client(base_url='tcp://' + node_ip + ':'
                                                + node_port, version='1.20', timeout=5)
            try:
                client_ins.stop(container_id)
                time.sleep(0.3)
                client_ins.remove_container(container_id)
                time.sleep(0.3)
            except docker.errors.NotFound:
                print("      NO Such container id")
            self.delete_con_usage(container_id)
        else:
            print("Please enter the Container ID")
            return 1

    def node_info(self,node_ip, node_port):
        url = ('http://' + node_ip + ":" + node_port + "/info")
        node_more_url = Curl(url)
        ret_json = node_more_url.get_value()
        return ret_json

def main_node_info():
    myswarm = Myswarm()
    #for image in myswarm.images_list("172.16.1.101","2375"):
    #    print(image)
    node_info = myswarm.node_info("172.16.1.101","2375")
    print(node_info["Name"])
    #print(node_ip)
    #print(port)
    print(node_info["NCPU"])
    print(node_info["MemTotal"])
    print(node_info["Images"])
    #print(node_info["state"])
    #print(node_group)
    print(node_info["Containers"])
    print(node_info["OperatingSystem"])
    print(node_info["PkgVersion"])
    print(node_info["KernelVersion"])

def main_con_info():
    myswarm = Myswarm()
    node_ip = "172.16.1.101"
    port = "2375"
    con_data = myswarm.container_list(node_ip,port)
    for con in con_data:
        print(con_data[con]["id_num"],con_data[con]["con_ip"],con_data[con]["con_name"],node_ip)

if __name__ == '__main__':
    main_con_info()


import numpy as np
import os
import json
import matplotlib.pyplot as plt

from typing import Dict, Tuple, List, Any

def plt_tendency(logs: List[Dict[str, Any]], fig_path: str) -> Dict[str, Any]:
    list_epoch = [log['epoch'] for log in logs]
    list_train_time = [log['train_time'] for log in logs]
    list_eval_time = [log['eval_time'] for log in logs]
    list_train_pearson_rho = [log['train_pearson_rho'] for log in logs]
    list_train_spearmanr_rho = [log['train_spearmanr_rho'] for log in logs]
    list_train_kendalltau_rho = [log['train_kendalltau_rho'] for log in logs]
    list_train_mae = [log['train_mae'] for log in logs]
    list_train_rmse = [log['train_rmse'] for log in logs]
    list_validate_pearson_rho = [log['validate_pearson_rho'] for log in logs]
    list_validate_spearmanr_rho = [log['validate_spearmanr_rho'] for log in logs]
    list_validate_kendalltau_rho = [log['validate_kendalltau_rho'] for log in logs]
    list_test_pearson_rho = [log['test_pearson_rho'] for log in logs]
    list_test_spearmanr_rho = [log['test_spearmanr_rho'] for log in logs]
    list_test_kendalltau_rho = [log['test_kendalltau_rho'] for log in logs]
    list_test_mae = [log['test_mae'] for log in logs]
    list_test_rmse = [log['test_rmse'] for log in logs]

    fig = plt.figure(figsize=(6, 4))
    plt.plot(list_epoch, list_test_pearson_rho, color='red', label='pearson')
    plt.plot(list_epoch, list_test_spearmanr_rho, color='green', label='spearmanr')
    plt.plot(list_epoch, list_test_kendalltau_rho, color='blue', label='kendalltau')
    plt.plot(list_epoch, list_test_mae, color='purple', linestyle='--', label='mae')
    plt.plot(list_epoch, list_test_rmse, color='pink', linestyle='--', label='rmse')
    plt.legend()
    if not os.path.isdir('hpwl-figures'):
        os.mkdir('hpwl-figures')
    plt.savefig(fig_path)

    list_total = np.array(list_validate_pearson_rho) + np.array(list_validate_spearmanr_rho) + np.array(
        list_validate_kendalltau_rho)
    best_epoch = np.argmax(list_total)

    return {
        'train_time': list_train_time[best_epoch] / 5,
        'pearson': list_test_pearson_rho[best_epoch],
        'spearmanr': list_test_spearmanr_rho[best_epoch],
        'kendalltau': list_test_kendalltau_rho[best_epoch],
        'mae': list_test_mae[best_epoch],
        'rmse': list_test_rmse[best_epoch],
    }, best_epoch

PLT_TUPLES = [
#     ('MLP', 'hpwl-superblue19/MLP.json'),
#     ('Net2f-topo', 'hpwl-superblue19/Net2f.json'),
#     ('Net2a-topo', 'hpwl-superblue19/Net2a.json'),
#     ('LHNN', 'hpwl-superblue19/LHNN.json'),
#     ('Ours', 'hpwl-superblue19/hyper.json'),
#     ('Ours-topo', 'hpwl-superblue19/hyper-topo.json'),

    ('Ours (evaluate)', 'hpwl-superblue19/hyper-trans.json'),
    ('Ours', 'hpwl-superblue19/hyper.json'),
    ('Ours (fine-tune)', 'hpwl-superblue19/hyper-ft.json'),
]

if __name__ == '__main__':
    for name, path in PLT_TUPLES:
        try:
            with open(path) as fp:
                d = json.load(fp)
            ret, be = plt_tendency(d, f'hpwl-figures/{name}.png')
            print(f'For {name}@{be}:')
            for k, v in ret.items():
                print(f'\t{k}: {v:.3f}')
        except FileNotFoundError:
            print(f'For {name}: not found')


# Loading key libraries
import streamlit as st
import os
import numpy as np
import pandas as pd

from PIL import Image
import matplotlib.pyplot as plt
import seaborn as sns
import requests
import datetime

# get absolute path and goo two levels up
DIRPATH = DIRPATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))

# set api endpoint
URL = 'https://bright1-sales-forecasting-ap1-2.hf.space'
API_ENDPOINT = '/predict'

# get list/choices for inputs
CITIES = ['Accra', 'Aflao', 'Akim Oda', 'Akwatia', 'Bekwai', 'Cape coast', 'Elmina,', 'Gbawe', 'Ho', 'Hohoe', 'intampo', 'Koforidua', 'Kumasi', 'Mampong', 'Obuasi', 'Prestea', 'Suhum', 'Tamale', 'Techiman', 'Tema', 'Teshie', 'Winneba']
CLUSTER = [ i for i in range(0, 17)]
STORE_ID = [ i for i in range(1, 55)]
CATEGORY_ID = [ i for i in range(0, 35)]

# Setting the page configurations
st.set_page_config(page_title = "Prediction Forecasting", layout= "wide", initial_sidebar_state= "auto")

st.title("Grocery Store Forecasting Prediction") # Setting the page title

image1 = Image.open('src/app/images1.jpg') # src\app\images1.jpg

#  create a function to make a request to the api
def make_prediction(store_id, category_id, onpromotion, city, store_type, cluster, date):

    # set paramter for the api
    parameters = {
    'store_id':int(store_id),
    'category_id':int(category_id),
    'onpromotion' :int(onpromotion),
    'city' : city,
    'store_type' : int(store_type),
    'cluster': int(cluster),
    'date_': date,

    }

    response = requests.post(url=f'{URL}{API_ENDPOINT}', params=parameters) # make a post request to the api

    sales_value = response.json()['sales'] #  get the sales value from the response

    sales_value = round(sales_value, 4) #  round up the sales value into 4 decimal places
    return sales_value

# display a head imdage for the app.
st.image(image1, width = 700)

st.sidebar.markdown('User Input Details and Information')

# Create interface
date= st.sidebar.date_input("Enter the Date",datetime.date(2023, 6, 30), min_value=datetime.date(2018, 1, 1))
store_id= st.sidebar.selectbox('Store id', options=STORE_ID)
category_id= st.sidebar.selectbox('categegory_id', options=CATEGORY_ID)
onpromotion= st.sidebar.number_input('onpromotion', step=1)
city =  st.sidebar.selectbox("city:", options= CITIES)
store_type=  st.sidebar.selectbox('type', options=[0, 1, 2, 3, 4])
cluster = st.sidebar.selectbox('cluster', options = CLUSTER )

# get predicted value
if st.sidebar.button('Predict', use_container_width=True, type='primary'):
            # make prediction
            sales_value = make_prediction(store_id, category_id, onpromotion,city, store_type, cluster, date)
            st.success('The predicted target is ' + str(sales_value))


import base64
import json
import os
import subprocess
import time

import tkinter as tk
import cv2 as cv
import numpy as np
from PIL import ImageTk, Image
from tkinter import scrolledtext, font, W, CENTER
from  tkinter import ttk

import paho.mqtt.client as mqtt
from djitellopy import Tello

import math
from tkinter import Menu
import tensorflow as tf
import os
# importar libreria para ver el funcionamiento de la red
from tensorflow.keras.callbacks import TensorBoard
# importar libreria para poder modificar las imagenes
from tensorflow.keras.preprocessing.image import ImageDataGenerator, img_to_array

def calc_pixels(tupla_1, tupla_2):
    pixels = math.sqrt(math.pow((tupla_1[0]-tupla_2[0]), 2) + math.pow((tupla_1[1]-tupla_2[1]), 2))
    return pixels

master = tk.Tk()
client = mqtt.Client('Dashboard')
global_broker_address ="127.0.0.1"
global_broker_port = 1884

# treatment of messages received from gate through the global broker

def on_message(client, userdata, message):
    global panel
    global lbl
    global table

    splited = message.topic.split('/')
    origin = splited[0]
    destination = splited[1]
    command = splited[2]

    if origin == "cameraService":

        if command == "videoFrame":
            img = base64.b64decode(message.payload)
            # converting into numpy array from buffer
            npimg = np.frombuffer(img, dtype=np.uint8)
            # Decode to Original Frame
            img = cv.imdecode(npimg, 1)
            # show stream in a separate opencv window
            cv.imshow("Stream", img)
            cv.waitKey(1)
        if command == 'picture':
            print('show picture')
            img = base64.b64decode(message.payload)
            # converting into numpy array from buffer
            npimg = np.frombuffer(img, dtype=np.uint8)
            # Decode to Original Frame
            cv2image = cv.imdecode(npimg, 1)
            gray = cv.cvtColor(cv2image, cv.COLOR_BGR2GRAY)
            print(detector_ball)
            if detector_ball:
                ball = detector_ball.detectMultiScale(gray, 1.35, 20)
                for (x, y, w, h) in ball:
                    cv.rectangle(cv2image, (x, y), (x + w, y + h), (0, 255, 0), 2)
            elif detector_face and detector_body:
                print('detect face')
                face = detector_face.detectMultiScale(gray, 1.2, 5)
                body = detector_body.detectMultiScale(gray, 1.2, 5)
                for (x, y, w, h) in face:
                    cv.rectangle(cv2image, (x, y), (x + w, y + h), (0, 255, 0), 2)
                for (x, y, w, h) in body:
                    cv.rectangle(cv2image, (x, y), (x + w, y + h), (0, 255, 0), 2)
            elif detector_2:
                gray = cv.resize(gray, (200, 200), interpolation=cv.INTER_CUBIC)
                gray = np.array(gray).astype(float) / 255
                img = img_to_array(gray)
                img = np.expand_dims(img, axis=0)
                predict = detector_2.predict(img)
                predict = predict[0][0]
                if predict > 0.98:
                    cv.putText(cv2image, "Cat or dog detected", (200, 70), cv.FONT_HERSHEY_PLAIN, 3, (0, 0, 255))
            dim = (300, 300)
            # resize image
            cv2image = cv.resize(cv2image, dim, interpolation=cv.INTER_AREA)
            img = Image.fromarray(cv2image)
            imgtk = ImageTk.PhotoImage(image=img)
            panel.imgtk = imgtk
            panel.configure(image=imgtk)
    if origin == "autopilotService":
        if (command == "droneAltitude"):
            answer = str(message.payload.decode("utf-8"))
            lbl['text'] = answer[:5]
        if (command == "droneHeading"):
            answer = str(message.payload.decode("utf-8"))
            lbl['text'] = answer[:5]
        if (command == "droneGroundSpeed"):
            answer = str(message.payload.decode("utf-8"))
            lbl['text'] = answer[:5]

        if (command  == "dronePosition"):
            positionStr = str(message.payload.decode("utf-8"))
            position = positionStr.split('*')
            latLbl['text'] = position[0]
            lonLbl['text'] = position[1]

        if (command  == "droneBattery"):
            answer = str(message.payload.decode("utf-8"))
            lbl['text'] = answer[:5]

    if origin == "dataService":
        if  (command == "storedPositions"):
            # receive the positions stored by the data service
            data = message.payload.decode("utf-8")
            # converts received string to json
            dataJson = json.loads(data)
            cont = 0
            for dataItem in dataJson:
                table.insert(parent='', index='end', iid=cont, text='',
                        values=(dataItem['time'], dataItem['lat'], dataItem['lon']))
                cont = cont + 1

            table.pack()

    if origin == "radiationService":
        if (command == "Radiation"):
            answer = str(message.payload.decode("utf-8"))
            lbl['text'] = answer[:5] + ' mSv'

client.on_message = on_message

# |--DASHBOARD master frame ----------------------------------------------------------------------------------|
# |                                                                                |
# |  |---connection frame--------------------------------------------------------------------------------|    |
# |  |---------------------------------------------------------------------------------------------------|    |
# |                                                                                |
# |  |---top frame---------------------------------------------------------------------------------------|    |
# |  |                                                                                |    |
# |  |   |--Autopilot control label frame ----------------------------|  |--LEDs control label frame--|  |    |
# |  |   |                                                            |  |                            |  |    |
# |  |   |  |--Arm/disarm frame -----------------------------------|  |  |----------------------------|  |    |
# |  |   |  |------------------------------------------------------|  |                                  |    |
# |  |   |                                                            |                                  |    |
# |  |   |  |--bottom frame ---------------------------------------|  |                                  |    |
# |  |   |  |                                                      |  |                                  |    |
# |  |   |  |  |-Autopilot get frame--|  |-Autopilot set frame -|  |  |                                  |    |
# |  |   |  |  |----------------------|  |----------------------|  |  |                                  |    |
# |  |   |  |                                                      |  |                                  |    |
# |  |   |  |------------------------------------------------------|  |                                  |    |
# |  |   |                                                            |                                  |    |
# |  |   |------------------------------------------------------------|                                  |    |
# |  |---------------------------------------------------------------------------------------------------|    |
# |                                                                                |
# |  |---camera control label frame----------------------------------------------------------------------|    |
# |  |                                                                                |    |
# |  |   |--- Take picture frame -----------|            |--- Video stream frame -----------|            |    |
# |  |   |                                  |            |                                  |            |    |
# |  |   |----------------------------------|            |----------------------------------|            |    |
# |  |---------------------------------------------------------------------------------------------------|    |
# |                                                                                |
# |-----------------------------------------------------------------------------------------------------------|

# Connection frame ----------------------
connected = False
connectionFrame = tk.Frame (master)
connectionFrame.pack(fill = tk.X)

def connectionButtonClicked():
    global connected
    global client
    global detector_ball
    detector_ball = False
    global detector_2
    detector_2 = False
    global detector_body
    detector_body = False
    global detector_face
    detector_face = False

    if not connected:
        connectionButton['text'] = "Disconnect"
        connectionButton['bg'] = "green"
        connected = True
        client.connect(global_broker_address,  global_broker_port)
        client.publish("dashBoard/gate/connectPlatform")
        client.loop_start()
        client.subscribe("+/dashBoard/#")
        print('Connected with drone platform')

        topFrame.pack(fill=tk.X)
        cameraControlFrame.pack(padx=20, pady=20);

    else:
        print('Disconnect')
        connectionButton['text'] = "Connect with drone platform"
        connectionButton['bg'] = "red"
        connected = False
        topFrame.pack_forget()
        ledsControlFrame.pack_forget()
        cameraControlFrame.pack_forget()

connectionButton = tk.Button(connectionFrame, text="Connect with drone platform", width = 50, bg='red', fg="white", command=connectionButtonClicked)
connectionButton.grid(row = 0, column = 0, padx=60, pady=20)
# top frame -------------------------------------------
topFrame = tk.Frame (master)

# Autopilot control label frame ----------------------
autopilotControlFrame = tk.LabelFrame(topFrame, text="Autopilot control", padx=5, pady=5)
autopilotControlFrame.pack(padx=20, side = tk.LEFT);

# Arm/disarm frame ----------------------
armDisarmFrame = tk.Frame (autopilotControlFrame)
armDisarmFrame.pack(padx=20)

armed = False
def armDisarmButtonClicked():
    global armed

    if not armed:
            armDisarmButton['text'] = "Disarm drone"
            armDisarmButton['bg'] = "green"
            armed = True
            client.publish("dashBoard/autopilotService/armDrone")

    else:
            armDisarmButton['text'] = "Arm drone"
            armDisarmButton['bg'] = "red"
            armed = False
            client.publish("dashBoard/autopilotService/disarmDrone")

armDisarmButton = tk.Button(armDisarmFrame, text="Arm drone", bg='red', fg="white",  width = 90, command=armDisarmButtonClicked)
armDisarmButton.grid(column=0, row=0,  pady = 5)

# bottomFrame frame ----------------------
bottomFrame = tk.Frame (autopilotControlFrame)
bottomFrame.pack(padx=20)

# Autopilot get frame ----------------------
autopilotGet = tk.Frame (bottomFrame)
autopilotGet.pack(side = tk.LEFT, padx=20)

v1 = tk.StringVar()
s1r1= tk.Radiobutton(autopilotGet,text="Altitude", variable=v1, value=1).grid(column=0, row=0, columnspan = 5, sticky=tk.W)
s1r2= tk.Radiobutton(autopilotGet,text="Heading", variable=v1, value=2).grid(column=0, row=1, columnspan = 5, sticky=tk.W)
s1r3= tk.Radiobutton(autopilotGet,text="Ground Speed", variable=v1, value=3).grid(column=0, row=2, columnspan = 5, sticky=tk.W)
s1r4= tk.Radiobutton(autopilotGet,text="Battery level", variable=v1, value=4).grid(column=0, row=3, columnspan = 5, sticky=tk.W)
v1.set(1)

def autopilotGetButtonClicked():
    if v1.get() == "1":
        client.publish("dashBoard/autopilotService/getDroneAltitude")
    elif v1.get() == "2":
        client.publish("dashBoard/autopilotService/getDroneHeading")
    elif v1.get() == "3":
        client.publish("dashBoard/autopilotService/getDroneGroundSpeed")
    else:
        client.publish("dashBoard/autopilotService/getDroneBattery")

autopilotGetButton = tk.Button(autopilotGet, text="Get", bg='red', fg="white", width = 10, height=5, command=autopilotGetButtonClicked)
autopilotGetButton.grid(column=5, row=0, columnspan=2, rowspan = 3, padx=10)

lbl = tk.Label(autopilotGet, text=" ", width = 10, borderwidth=2, relief="sunken")
lbl.grid(column=7, row=1,  columnspan=2 )

# Autopilot set frame ----------------------
autopilotSet = tk.Frame (bottomFrame)
autopilotSet.pack( padx=20)

def takeOffButtonClicked():
    client.publish("dashBoard/autopilotService/takeOff", metersEntry.get() )

takeOffButton = tk.Button(autopilotSet, text="Take Off", bg='red', fg="white",  width = 10, command=takeOffButtonClicked)
takeOffButton.grid(column=0, row=1, columnspan=2, sticky=tk.W)

to = tk.Label(autopilotSet, text="to")
to.grid(column=2, row=1)
metersEntry = tk.Entry(autopilotSet, width = 10)
metersEntry.grid(column=3, row=1,  columnspan=2 )
meters = tk.Label(autopilotSet, text="meters")
meters.grid(column=5, row=1)

lat = tk.Label(autopilotSet, text="lat")
lat.grid(column=2, row=2,  columnspan=2,padx = 5 )

lon = tk.Label(autopilotSet, text="lon")
lon.grid(column=4, row=2,  columnspan=2,padx = 5 )

def getPositionButtonClicked():
    client.publish("dashBoard/autopilotService/getDronePosition" )

getPositionButton = tk.Button(autopilotSet, text="Get Position", bg='red', fg="white",  width = 10,  command=getPositionButtonClicked)
getPositionButton.grid(column=0, row=3, pady = 5, sticky=tk.W)

latLbl = tk.Label(autopilotSet, text=" ", width = 10, borderwidth=2, relief="sunken")
latLbl.grid(column=2, row=3,  columnspan=2,padx = 5 )

lonLbl = tk.Label(autopilotSet, text=" ", width = 10, borderwidth=2, relief="sunken")
lonLbl.grid(column=4, row=3,  columnspan=2,padx = 5 )

def goToButtonClicked():
    position = str (goTolatEntry.get()) + '*' + str(goTolonEntry.get())
    client.publish("dashBoard/autopilotService/goToPosition", position)

goToButton = tk.Button(autopilotSet, text="Go To", bg='red', fg="white",  width = 10,  command=goToButtonClicked)
goToButton.grid(column=0, row=4, pady = 5, sticky=tk.W)

goTolatEntry = tk.Entry(autopilotSet, width = 10)
goTolatEntry.grid(column=2, row=4,  columnspan=2,padx = 5 )

goTolonEntry = tk.Entry(autopilotSet, width = 10)
goTolonEntry.grid(column=4, row=4,  columnspan=2,padx = 5 )

def returnToLaunchButtonClicked():
    client.publish("dashBoard/autopilotService/returnToLaunch")

returnToLaunchButton = tk.Button(autopilotSet, text="Return To Launch", bg='red', fg="white",  width = 40, command=returnToLaunchButtonClicked)
returnToLaunchButton.grid(column=0, row=5,  pady = 5, columnspan=6, sticky=tk.W)

def openWindowToShowRecordedPositions():
    # Open a new small window to show the positions timestamp to be received from the data service
    global newWindow
    global table
    newWindow = tk.Toplevel(master)

    newWindow.title("Recorded positions")

    newWindow.geometry("400x400")
    table = ttk.Treeview(newWindow)

    table['columns'] = ('time', 'latitude', 'longitude')

    table.column("#0", width=0, stretch=tk.NO)
    table.column("time", anchor=tk.CENTER, width=150)
    table.column("latitude", anchor=tk.CENTER, width=80)
    table.column("longitude", anchor=tk.CENTER, width=80)

    table.heading("#0", text="", anchor=tk.CENTER)
    table.heading("time", text="Time", anchor=tk.CENTER)
    table.heading("latitude", text="Latitude", anchor=tk.CENTER)
    table.heading("longitude", text="Longitude", anchor=tk.CENTER)

    # requiere the stored positions from the data service
    client.publish("dashBoard/dataService/getStoredPositions")

    closeButton = tk.Button(newWindow, text="Close", bg='red', fg="white", command=closeWindowToShowRecordedPositions).pack()

def closeWindowToShowRecordedPositions ():
    global newWindow
    newWindow.destroy()

showRecordedPositionsButton = tk.Button(autopilotSet, text="Show recorded positions", bg='red', fg="white",  width = 40, command=openWindowToShowRecordedPositions)
showRecordedPositionsButton.grid(column=0, row=6,  pady = 5, columnspan=6, sticky=tk.W)

# LEDs control frame ----------------------
ledsControlFrame = tk.LabelFrame(topFrame, text="LEDs control", padx=5, pady=5)
ledsControlFrame.pack(padx=20, pady=20);

v3 = tk.StringVar()
s1r7= tk.Radiobutton(ledsControlFrame,text="LED sequence START/STOP", variable=v3, value=1).grid(column=2, row=2, columnspan = 3)
s1r8= tk.Radiobutton(ledsControlFrame,text="LED sequence for N seconds", variable=v3, value=2).grid(column=2, row=3, columnspan = 3)

seconds = tk.Entry(ledsControlFrame, width = 5)
seconds.grid(column=5, row=3, columnspan = 3)
v3.set(1)

lEDSequence = False;

def LEDControlButtonClicked():
    global E1
    global lEDSequence
    if v3.get() == "1":
        if not lEDSequence:
            ledControlButton['text'] = "Stop"
            ledControlButton['bg'] = "green"
            lEDSequence = True
            print ('Start LEDs sequence')
            client.publish("dashBoard/LEDsService/startLEDsSequence")

        else:
            ledControlButton['text'] = "Start"
            ledControlButton['bg'] = "red"
            lEDSequence = False
            print('Stop LEDs sequence')
            client.publish("dashBoard/LEDsService/stopLEDsSequence")

    if v3.get() == "2":
            print('LEDs sequence for N seconds')
            client.publish("dashBoard/LEDsService/LEDsSequenceForNSeconds", seconds.get())

ledControlButton = tk.Button(ledsControlFrame, text="Start", bg='red', fg="white",  width = 10, height = 3, command=LEDControlButtonClicked)
ledControlButton.grid(column=8, row=1,  padx = 5, columnspan=4, rowspan = 3)

# Radiation control frame ----------------------
radiationControlFrame = tk.LabelFrame(topFrame, text="Radiation control", padx=5, pady=5)
radiationControlFrame.pack(padx=20, pady=20)

v4 = tk.StringVar()
s1r9= tk.Radiobutton(radiationControlFrame,text="Radiation value", variable=v4, value=4).grid(column=0, row=1, columnspan = 5, sticky=tk.W)
v4.set(1)

def radiationGetButtonClicked():
    if v4.get() == "1":
        client.publish("dashBoard/radiationService/getRadiation")

radiationGetButton = tk.Button(radiationControlFrame, text="Get", bg='red', fg="white", width = 10, height=3, command=radiationGetButtonClicked)
radiationGetButton.grid(column=5, row=0, columnspan=2, rowspan = 3, padx=10)

lbl = tk.Label(radiationControlFrame, text=" ", width = 10, borderwidth=2, relief="sunken")
lbl.grid(column=7, row=1,  columnspan=2 )

# flightplan open map frame ----------------------
flightplanControlFrame = tk.LabelFrame(topFrame, text="Flightplan Map", padx=5, pady=5)
flightplanControlFrame.pack(padx=20, pady=20)

def openFlightplanButtonClicked():
    print('Opening Flightplan map')
    client.publish("dashBoard/flightplanService/openFlightplan")
    client.subscribe("flightplanService/#")

flightplanOpenButton = tk.Button(flightplanControlFrame, text="OPEN", bg='red', fg="white", width = 10, height=3, command=openFlightplanButtonClicked)
flightplanOpenButton.grid(column=5, row=0, columnspan=2, rowspan = 3, padx=10)

# Camera control label frame ----------------------
cameraControlFrame = tk.LabelFrame(master, text="Camera control", padx=5, pady=5)

takePictureFrame = tk.Frame (cameraControlFrame)
takePictureFrame.pack(side = tk.LEFT)

def train():
    text_1 = input_1_text.get()
    text_2 = input_2_text.get()
    epoch = input_3_text.get()
    print(text_1, text_2, epoch)
    ## Almacenamiento de imagnes
    train_path = text_1
    vali_path = text_2
    train_list = os.listdir(train_path)
    vali_list = os.listdir(vali_path)

    ## Parametros para el almacenamiento de las imagenes
    width = 200
    heigth = 200
    # Entrenamiento
    labels_train = []
    imag_train = []
    data_train = []
    cont_train = 0
    # Validacion
    labels_val = []
    imag_val = []
    data_val = []
    cont_val = 0

    ## Extraer fotos y etiquetarlas

    for folder_name in train_list:
        name_path = train_path + '/' + folder_name
        folder = os.listdir(name_path)
        for file in folder:
            labels_train.append(cont_train)
            img = cv.imread(name_path + '/' + file, 0)
            img = cv.resize(img, (width, heigth), interpolation=cv.INTER_CUBIC)
            img = img.reshape(width, heigth, 1)
            data_train.append([img, cont_train])
            imag_train.append(img)
        cont_train += 1

    for folder_name in vali_list:
        name_path = vali_path + '/' + folder_name
        folder = os.listdir(name_path)
        for file in folder:
            labels_val.append(cont_val)
            img = cv.imread(name_path + '/' + file, 0)
            img = cv.resize(img, (width, heigth), interpolation=cv.INTER_CUBIC)
            img = img.reshape(width, heigth, 1)
            data_val.append([img, cont_val])
            imag_val.append(img)
        cont_val += 1

    ## Normalizar imagenes
    imag_train = np.array(imag_train).astype(float) / 255
    print(imag_train.shape)
    imag_val = np.array(imag_val).astype(float) / 255
    print(imag_val.shape)
    # Pasar las listas a arrays
    labels_train = np.array(labels_train)
    labels_val = np.array(labels_val)

    ## Dar realismo a las imagenes
    # Modificar las imagenes aleatoriamente

    image_random_gen = ImageDataGenerator(
        rotation_range=40,
        width_shift_range=0.2,
        height_shift_range=0.2,
        shear_range=10,
        zoom_range=[0.2, 1],
        horizontal_flip=True,
        vertical_flip=True,
    )

    image_train_gen = image_random_gen.flow(imag_train, labels_train, batch_size=32)

    ## Creamos el modelo de la red convolucional con Drop Out
    cnn_model = tf.keras.models.Sequential([
        tf.keras.layers.Conv2D(32, (3, 3), activation='relu', input_shape=(200, 200, 1)),
        # capa convolucional con 32 Kernel
        tf.keras.layers.MaxPool2D(2, 2),
        tf.keras.layers.Conv2D(64, (3, 3), activation='relu'),  # capa convolucional con 64 Kernel
        tf.keras.layers.MaxPool2D(2, 2),
        tf.keras.layers.Conv2D(128, (3, 3), activation='relu'),  # capa convolucional con 128 Kernel
        tf.keras.layers.MaxPool2D(2, 2),

        tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Flatten(),
        tf.keras.layers.Dense(256, activation='relu'),
        tf.keras.layers.Dense(1, activation='sigmoid')
    ])

    ## Compilamos el modelo y añadimos una funcion de perdida y el optimizador
    cnn_model.compile(
        optimizer='adam',
        loss='binary_crossentropy',
        metrics=['accuracy']
    )

    ## Entrenar la red neuronal
    # tensorboard --logdir="C:/RedNeuronal/board"

    board = TensorBoard(log_dir='C:/RedNeuronal/board')
    cnn_model.fit(
        image_train_gen,
        batch_size=32,
        validation_data=(imag_val, labels_val),
        epochs=epoch,
        callbacks=[board],
        steps_per_epoch=int(np.ceil(len(imag_train) / float(32))),
        validation_steps=int(np.ceil(len(imag_val) / float(32)))
    )

    cnn_model.save('clasificador1.h5')
    print('Red Nueronal Terminada')

def ball_command():
    global detector_ball
    # detect, use 1,7 and 20
    path_ball = os.path.dirname(__file__) + '/cascade_balls.xml'
    detector_ball = cv.CascadeClassifier(path_ball)
    print(detector_ball)
    pass

def animals_command():
    ## path de nuestro modelo
    cnn_model = os.path.dirname(__file__) + '/CatsDogs.h5'

    ## Leer la red neuronal
    detector_2 = tf.keras.models.load_model(cnn_model)
    print(detector_2)

def human_command():
    path_face = os.path.dirname(__file__) + '/haarcascade_frontalface_default.xml'
    path_body = os.path.dirname(__file__) + '/haarcascade_fullbody.xml'
    detector_face = cv.CascadeClassifier(path_face)
    detector_body = cv.CascadeClassifier(path_body)
    detector_2 = False
    detector_ball = False
    print(detector_body, detector_face)

def use():
    path = object_path_text.get()
    print(path)
    cnn_model = path
    detector_2 = tf.keras.models.load_model(cnn_model)
    print(detector_2)

def useObjectWindow():
    # Toplevel object which will
    newWindow = tk.Toplevel(master)
    # sets the title of the
    newWindow.title("New Window")
    # sets the geometry of toplevel
    newWindow.geometry("400x400")

    input_1_label = tk.Label(newWindow,text="Object detection path").place(x=45, y=60)
    submit_button = tk.Button(newWindow,text="Submit", command=use).place(x=45,y=110)
    text_1 = tk.StringVar()
    global object_path_text
    input_1 = tk.Entry(newWindow, textvariable=text_1, width=50).place(x=45,y=85)
    object_path_text =text_1
    newWindow.destroy()

def trainObjectWindow():
    # Toplevel object which will
    newWindow = tk.Toplevel(master)
    # sets the title of the
    newWindow.title("New Window")
    # sets the geometry of toplevel
    newWindow.geometry("400x400")

    input_1_label = tk.Label(newWindow,text="Train path").place(x=45, y=60)
    input_2_label = tk.Label(newWindow,text="Validation path").place(x=40,y=100)
    input_3_label = tk.Label(newWindow, text="Epocas").place(x=40, y=140)
    submit_button = tk.Button(newWindow,text="Submit", command=train).place(x=40,y=180)
    text_1 = tk.StringVar()
    text_2 = tk.StringVar()
    text_3 = tk.StringVar()
    global input_1_text
    input_1 = tk.Entry(newWindow, textvariable=text_1, width=30).place(x=130,y=60)
    input_1_text =text_1
    global input_2_text
    input_2 = tk.Entry(newWindow, textvariable=text_2, width=30).place(x=130,y=100)
    input_2_text = text_2
    global input_3_text
    input_3 = tk.Entry(newWindow, textvariable=text_3, width=30).place(x=130,y=140)
    input_3_text = text_3
    newWindow.destroy()

my_menu = Menu(master)
master.config(menu=my_menu)
red_menu = Menu(my_menu)
my_menu.add_cascade(menu=red_menu, label='Redes Neuronales')
red_menu.add_command(label='Human detection', command=human_command)
red_menu.add_separator()
red_menu.add_command(label="Ball detection", command=ball_command)
red_menu.add_separator()
red_menu.add_command(label="Cats & dogs detection", command=animals_command)
red_menu.add_separator()
red_menu.add_command(label="Train a new object recognition", command=trainObjectWindow)
red_menu.add_separator()
red_menu.add_command(label="Use the new object recognition", command=useObjectWindow)

def takePictureButtonClicked():
    print ("Take picture")
    client.publish("dashBoard/cameraService/takePicture")

takePictureButton = tk.Button(takePictureFrame, text="Take Picture", width=50, bg='red', fg="white", command=takePictureButtonClicked)
takePictureButton.grid(column=0, row=0, pady = 20, padx = 20)

img = Image.open("image1.jpg")
img = img.resize((350, 350), Image.ANTIALIAS)
img = ImageTk.PhotoImage(img)
panel = tk.Label(takePictureFrame, image=img, borderwidth=2, relief="raised")
panel.image = img
panel.grid(column=0, row=1, columnspan=3, rowspan = 3)

videoStreamFrame = tk.Frame(cameraControlFrame)
videoStreamFrame.pack()

videoStream = False;

def videoStreamButtonClicked():
    global videoStream
    global client
    if not videoStream:
        videoStreamButton['text'] = "Stop video stream"
        videoStreamButton['bg'] = "green"
        videoStream = True
        print ('Start video stream')
        client.publish("dashBoard/cameraService/startVideoStream")

    else:
        videoStreamButton['text'] = "Start video stream on a separaded window"
        videoStreamButton['bg'] = "red"
        videoStream = False
        print ('Stop video stream')
        client.publish("dashBoard/cameraService/stopVideoStream")

        cv.destroyWindow("Stream")

videoStreamButton = tk.Button(videoStreamFrame, text="Start video stream \n on a separaded window", width=50, height = 25, bg='red', fg="white",
                              command=videoStreamButtonClicked)
myFont = font.Font(size=12)
videoStreamButton['font'] = myFont
videoStreamButton.grid(column=0, row=0, pady=20, padx=20, )

master.mainloop()


from sqlalchemy.dialects import postgresql

from pycroft.model import ddl
from sqlalchemy import null, and_, or_, func, Column, Integer, String, union, \
    literal, literal_column, select
from sqlalchemy.orm import Query

from .base import ModelBase
from .ddl import View, DDLManager
from .user import User, Property, Membership, PropertyGroup

manager = DDLManager()

property_query_stmt = union(
    Query([User.id.label('user_id'), Property.name.label('property_name'),
           literal(False).label('denied')])
    .select_from(Membership)
    .join(PropertyGroup)
    .join(User)
    .filter(and_(
        or_(Membership.begins_at == null(), Membership.begins_at <= literal_column('evaluation_time')),
        or_(Membership.ends_at == null(), literal_column('evaluation_time') <= Membership.ends_at),
    ))
    .join(Property)
    .group_by(User.id, Property.name)
    .having(func.every(Property.granted))
    .statement,

    Query([User.id.label('user_id'), Property.name.label('property_name'),
           literal(True).label('denied')])
    .select_from(Membership)
    .join(PropertyGroup)
    .join(User)
    .filter(and_(
        or_(Membership.begins_at == null(), Membership.begins_at <= literal_column('evaluation_time')),
        or_(Membership.ends_at == null(), literal_column('evaluation_time') <= Membership.ends_at),
    ))
    .join(Property)
    .group_by(User.id, Property.name)
    # granted by ≥1 membership, but also denied by ≥1 membership
    .having(and_(func.bool_or(Property.granted), ~func.every(Property.granted)))
    .statement,
)

evaluate_properties_function = ddl.Function(
    'evaluate_properties', ['evaluation_time timestamp with time zone'],
    'TABLE (user_id INT, property_name VARCHAR(255), denied BOOLEAN)',
    str(
        property_query_stmt.compile(
            dialect=postgresql.dialect(),
            compile_kwargs={'literal_binds': True}
        )
    ),
    volatility='stable',
)

manager.add_function(
    Membership.__table__,
    evaluate_properties_function
)

current_property = View(
    name='current_property',
    #metadata=ModelBase.metadata,
    query=(
        select([literal_column('user_id'), literal_column('property_name'),
                literal_column('denied')])
        .select_from(func.evaluate_properties(func.current_timestamp()))
    ),
)
manager.add_view(Membership.__table__, current_property)

class CurrentProperty(ModelBase):
    __table__ = current_property.table
    __mapper_args__ = {
        'primary_key': (current_property.table.c.user_id,
                        current_property.table.c.property_name),
    }

manager.register()


#!/usr/bin/env python

#
# REST server implementation
# author:   L. Saetta
# updated:  18/10/2021
# based on some code from https://github.com/philipperemy/deep-speaker
#
import io
import time
import os
import logging
import json
import uvicorn
import numpy as np
from fastapi import FastAPI, UploadFile, File, HTTPException
from fastapi.responses import StreamingResponse

# needed for the cloud, access to Object Storage
import oci
import ocifs

# global configurations in config.py
import config

# the DL model
from conv_models import DeepSpeakerModel

# code for adding new speakers to DB
from speaker_management import add_speaker

# utilities functions
from utilities import compute_embeddings, distance_cosine_similarity, print_dict
from utilities import load_centroids_newfile, save_wav, swap_centroids_files_local, swap_centroids_files_oss
from utilities import NumpyArrayEncoder, load_centroids_from_oss, load_centroids_from_local
from utilities import create_path, write_new_centroids_to_oss, write_new_centroids_to_local
from utilities import restore_centroids_from_bck_local, restore_centroids_from_bck_oss, get_list_speakers_names

#
# constants for audio processing
#
from constants import SAMPLE_RATE, NUM_FRAMES

#
# log settings
#
log = logging.getLogger("server")
logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s',
    level=os.environ.get("LOGLEVEL", "INFO"), datefmt='%Y-%m-%d %H:%M:%S')

log.info("Starting the server...")

#
# Functions
#

#
# load the pre-trained DL model
#
def load_model():
    log.info("Loading the DL model...")

    CONFIG_TYPE = config.global_settings['CONFIG_TYPE']
    MODEL_FILE = config.global_settings['MODEL_FILE']

    vmodel = DeepSpeakerModel()

    # renamed model file to model.h5 (was: ResCNN_triplet_training_checkpoint_265.h5)

    # continue to load local file
    # but the model is not changing.. do we need a special location?
    # we can continue to ship with the code !
    vmodel.m.load_weights(create_path(MODEL_FILE), by_name=True)

    return vmodel
#
# This loads the centroids file DB and handle the two cases: local, cloud CONFIG
#
def load_centroids():
    log.info("Loading the centroids DB...")

    CONFIG_TYPE = config.global_settings['CONFIG_TYPE']

    # with CONFIG_TYPE we manage the different storage in Cloud (Object storage)
    # and locally (on fs)
    if CONFIG_TYPE == "local":
        new_centroids = load_centroids_from_local()
    else:
        if CONFIG_TYPE == "cloud":
            new_centroids = load_centroids_from_oss()

    log.info("Number of distinct speakers: " + str(len(new_centroids)))

    return new_centroids

#
# swap centroid files
#
def swap_centroids_files():
    log.info("Swapping the centroids DB...")

    CONFIG_TYPE = config.global_settings['CONFIG_TYPE']

    if CONFIG_TYPE == "local":
        swap_centroids_files_local()
    else:
        if CONFIG_TYPE == "cloud":
            swap_centroids_files_oss()

#
# This handle the centroids file restore and handle the two cases: local, cloud CONFIG
#
def restore_centroids_from_bck():
    log.info("Restoring the centroids DB...")

    CONFIG_TYPE = config.global_settings['CONFIG_TYPE']

    # with CONFIG_TYPE we manage the different storage in Cloud (Object storage)
    # and locally (on fs)
    if CONFIG_TYPE == "local":
        restore_centroids_from_bck_local()
    else:
        if CONFIG_TYPE == "cloud":
            restore_centroids_from_bck_oss()

    return

# return the dictionary with all scores
# score is the distance between the current sound vector and the centroid
# current sound vector is in embedding
# list is limited to MAX_RESULTS
def compare_other_centroids(embedding, centroids):
    tmp_dict = {}

    for key in centroids.keys():
        dist = round(distance_cosine_similarity(embedding, centroids[key])[0], 3)
        tmp_dict[key] = dist

    # sort
    tmp_list = sorted(tmp_dict.items(), key=lambda x:x[1])
    # in tmp_list I have tuples like ('Lorenzo_DeMarchis', 0.278)

    # here I limit to MAX_RESULTS
    new_tmp_list = []

    for i, el in enumerate(tmp_list):
        if i < config.global_settings['MAX_RESULTS']:
            new_dict = {}
            # unpack the tuple
            new_dict['name'] = el[0]
            new_dict['result'] = el[1]
            new_tmp_list.append(new_dict)

    out_dict = {}
    out_dict["result"] = new_tmp_list

    return out_dict

def compute_distances(embedding, centroids):
    dict_dist = {}

    for key in centroids.keys():
        dist = round(distance_cosine_similarity(embedding, centroids[key])[0], 3)

        dict_dist[key] = dist

    # sort
    tmp_list = sorted(dict_dist.items(), key=lambda x:x[1])

    # here I limit to MAX_RESULTS
    new_tmp_list = []

    for i, el in enumerate(tmp_list):
        if i < config.global_settings['MAX_RESULTS']:
            new_tmp_list.append(el)

    out_dict = dict(new_tmp_list)

    return out_dict

# load the DL model
model = load_model()

# load the centroids file
# file must be modified every time a new speaker is added
#
centroids = load_centroids()

# create the app
app = FastAPI(title=config.global_settings['TITLE'], version=config.global_settings['VERSION'],
        description=config.global_settings['DESCRIPTION'])

#
# functions handling HTTP requests
#

#
# This is the function that handles the POST request for identification
# it expects a wav file in binary format as input
# and gives as output a JSON with pairs name:distance
# where distance is the distance from the given input and the centroid for name
#
@app.post("/identify", tags=["Identification"])
def identify(file: bytes = File(...)):

    # for timing the request
    tStart = time.time()

    log.info('Identify: Received a request')

    try:
        embedding = compute_embeddings(file, model)

        # compare with all centroids
        out_dict = compare_other_centroids(embedding, centroids)

        print_dict(out_dict)

        tEla = time.time() - tStart
        print()
        print('Elapsed time (sec.)', round(tEla, 3))

    except Exception as e:
        print('The exception is:', e)
        print("Input is not a valid audio file!")
        raise HTTPException(status_code=415, detail="Unsupported file provided.")

    return out_dict

#
# compare name with names of first NUM_CANDIDATES
# if names is in first NUM_CANDIDATES return OK
#
@app.post("/verify", tags=["Identification"])
def verify(name: str, file: bytes = File(...)):

    # for timing the request
    tStart = time.time()

    log.info('Verify: Received a request')

    final_dict_out = {}

    try:
        embedding = compute_embeddings(file, model)

        # compare with all centroids
        # out dict is already in iorder of increasing distance
        out_dict = compare_other_centroids(embedding, centroids)

        # compute the summary result
        final_dict_out["summary"] = "false"
        final_dict_out['result'] = out_dict

        list_pair = out_dict["result"]

        for i, pair in enumerate(list_pair):
            # consider only the first NUM_CANDIDATES
            if i < config.global_settings['NUM_CANDIDATES']:
                # compare ignoring case
                if pair["name"].upper() == name.upper():
                    final_dict_out["summary"] = "true"
                    break

        print_dict(final_dict_out)

        tEla = time.time() - tStart
        print()
        print('Elapsed time (sec.)', round(tEla, 3))

    except Exception as e:
        print('The exception is:', e)
        print("Input is not a valid audio file!")
        raise HTTPException(status_code=415, detail="Unsupported file provided.")

    return final_dict_out

# By using @app.get("/") you are allowing the GET method to work for the / endpoint.
@app.get("/", tags=["Operation"])
def home():
    return "Congratulations! Your API is working. Now head over to http://localhost:8888/docs."

#
# return the list of registered speakers' names
# the list can be sorted
#
@app.get("/list_speakers", tags=["Operation"])
def list_speakers(ordered: str = "false"):
    global centroids

    log.info('List speakers: Received a request')

    list_speakers = get_list_speakers_names(centroids, ordered)

    out_dict = {}
    out_dict['speakers'] = list_speakers

    print_dict(out_dict)

    return out_dict
#
# This function handles the request for adding a new speaker to the DB
#
@app.post("/add_speaker", tags=["Operation"])
def add_new_speaker(name: str, file: bytes = File(...)):
    global centroids

    log.info("Adding speaker: "  + name)

    # for timing the request
    tStart = time.time()

    try:
        # save in a wav file in the local dir
        save_wav(name, file)

        filename = create_path(name + ".wav")

        # create the new_centroids structure
        new_centroids = add_speaker(name, filename, centroids, model)

        # if CONFIG = cloud must copy new centroids to OSS
        if config.global_settings['CONFIG_TYPE'] == "cloud":
            log.info("Writing new centroid to oss")
            write_new_centroids_to_oss(new_centroids)
        else:
            # local
            log.info("Writing new centroid to local")
            write_new_centroids_to_local(new_centroids)

        # swap the files (handles the double config cloud or local)
        log.info("Swap centroids files")
        swap_centroids_files()

        # reload
        log.info("Reload centroids")
        centroids = load_centroids()

        out_dict = {}
        out_dict['result'] = "true"

        tEla = time.time() - tStart
        print()
        print('Elapsed time (sec.)', round(tEla, 3))

    except Exception as e:
        print('The exception is:', e)
        print("Input is not a valid!")
        raise HTTPException(status_code=415, detail="Unsupported request.")

    return out_dict
#
# delete speaker
#
@app.post("/delete_speaker", tags=["Operation"])
def delete_speaker(name: str):
    global centroids

    log.info("Delete a speaker from DB")

    # create the new_centroids without speaker = name
    # check if name is in speakers list
    list_speakers = get_list_speakers_names(centroids, ordered=False)

    if name in list_speakers:
        # Cancel, operate on a copy
        new_centroids = centroids.copy()

        # Remove the speaker from the data structure
        del new_centroids[name]

        # now save
        if config.global_settings['CONFIG_TYPE'] == "cloud":
            log.info("Writing new centroid to oss")
            write_new_centroids_to_oss(new_centroids)
        else:
            # local
            log.info("Writing new centroid to local")
            write_new_centroids_to_local(new_centroids)

        # swap
        log.info("Swap centroids files")
        swap_centroids_files()

        # after creating the new centroids file, reload it
        centroids = load_centroids()
    else:
        # do nothing
        log.info(name + " not in list speakers")

    dict_out = {}
    dict_out["result"] = "true"

    return dict_out
#
# force the reload of the centroids file
#
@app.get("/reload", tags=["Operation"])
def reload():
    global centroids

    centroids = load_centroids()

    out_dict = {}
    out_dict['result'] = "true"

    return out_dict
#
# restore of the centroids file
#
@app.get("/restore_centroids", tags=["Operation"])
def restore_centroids():
    global centroids

    print("Restore centroids")
    restore_centroids_from_bck()

    centroids = load_centroids()

    out_dict = {}
    out_dict['result'] = "true"

    return out_dict

#
# return the list of registered speakers' names reading the file
# the list can be sorted
#
@app.get("/list_speakers_from_newfile", tags=["Operation"])
def list_speakers_from_newfile(ordered: str = "false"):

    new_centroids = load_centroids_newfile()

    list_speakers = []

    for key in new_centroids.keys():
        list_speakers.append(key)

    if ordered == "true":
        list_speakers = sorted(list_speakers)

    out_dict = {}
    out_dict['speakers'] = list_speakers

    print_dict(out_dict)

    return out_dict

#
# Main
#
if __name__ == '__main__':
    # set IP on which is listening, port
    # listen on all IP
    uvicorn.run(app, host=config.global_settings['IP'], port=config.global_settings['PORT'])