import os
import cv2
def count_frames(path, override=False):
	# grab a pointer to the video file and initialize the total
	# number of frames read
	video = cv2.VideoCapture(path)
	total = 0
	# if the override flag is passed in, revert to the manual
	# method of counting frames
	if override:
		total = count_frames_manual(video)
	# otherwise, let's try the fast way first
	else:
		# lets try to determine the number of frames in a video
		# via video properties; this method can be very buggy
		# and might throw an error based on your OpenCV version
		# or may fail entirely based on your which video codecs
		# you have installed
		try:
			# check if we are using OpenCV 3
			if cv2.__version__=="4.4.0":
				total = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
			# otherwise, we are using OpenCV 2.4
			else:
				total = int(video.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT))
		# uh-oh, we got an error -- revert to counting manually
		except:
			total = count_frames_manual(video)
	# release the video file pointer
	video.release()
	# return the total number of frames in the video
	return total

def count_frames_manual(video):
	# initialize the total number of frames read
	total = 0
	# loop over the frames of the video
	while True:
		# grab the current frame
		(grabbed, frame) = video.read()

		# check to see if we have reached the end of the
		# video
		if not grabbed:
			break
		# increment the total number of frames read
		total += 1
	# return the total number of frames in the video file
	return total

def ProcessVideo(videPath):

    countedFrames=count_frames(videPath)
    print("This video has",countedFrames," frames")
    frameCounter=0

    cap= cv2.VideoCapture(videPath)
    while True:
        # Capture frame-by-frame
        ret, frame = cap.read()
        if ret==False:
            break
        # Display the resulting frame
        cv2.imshow('frame',frame)
        cv2.imwrite(videPath[0:-4]+str(frameCounter)+".jpg",frame)
        frameCounter+=1

        #allowing the programmer to quit the process by pressing q
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    # When everything done, release the capture
    cap.release()
    cv2.destroyAllWindows()

def main():
    dataSetPath="../dataSetEstadium"
    peoplesDataSet=os.listdir(dataSetPath)

    for personalDataSet in peoplesDataSet:
        print("Belonging to-->",personalDataSet)
        foldersContainingVideo= os.listdir(dataSetPath+"/"+personalDataSet)
        for videoFolder in foldersContainingVideo:
            print("Working on folder of the video-->",videoFolder)
            video=os.listdir(dataSetPath+"/"+personalDataSet+"/"+videoFolder)[0]
            print("Opening -->",video)
            videoPath=dataSetPath+"/"+personalDataSet+"/"+videoFolder+"/"+video
            ProcessVideo(videoPath)

main()


from typing import cast, TYPE_CHECKING

from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from starlette.requests import HTTPConnection
from starlite import (
    AbstractAuthenticationMiddleware,
    AuthenticationResult,
    NotAuthorizedException,
)

from database.model import User
from security.token import decode_jwt_token

if TYPE_CHECKING:
    from sqlalchemy.ext.asyncio import AsyncEngine

API_KEY_HEADER = "X-API-KEY"

class JWTAuthenticationMiddleware(AbstractAuthenticationMiddleware):
    async def authenticate_request(self, request: HTTPConnection) -> AuthenticationResult:

        auth_header = request.headers.get(API_KEY_HEADER)
        if not auth_header:
            #raise NotAuthorizedException()
            return AuthenticationResult(user=None, auth="")

        token = decode_jwt_token(encoded_token=auth_header)

        engine = cast("AsyncEngine", request.app.state.postgres_connection)
        async with AsyncSession(engine) as async_session:
            async with async_session.begin():
                user = await async_session.execute(
                    select(User).where(User.id == token.sub)
                )
        if not user:
            #raise NotAuthorizedException()
            return AuthenticationResult(user=None, auth="")
        return AuthenticationResult(user=user, auth=token)


import base64
import xml.etree.ElementTree as ET

import gimme_aws_creds.common as commondef
from . import errors

class DefaultResolver(object):
    """
       The Aws Client Class performs post request on AWS sign-in page
       to fetch friendly names/alias for account and IAM roles
    """

    def __init__(self, verify_ssl_certs=True):
        return

    def _enumerate_saml_roles(self, assertion, saml_target_url):
        """ using the assertion to fetch aws sign-in page, parse it and return aws sts creds """
        role_pairs = []
        root = ET.fromstring(base64.b64decode(assertion))
        for saml2_attribute in root.iter('{urn:oasis:names:tc:SAML:2.0:assertion}Attribute'):
            if saml2_attribute.get('Name') == 'https://aws.amazon.com/SAML/Attributes/Role':
                for saml2_attribute_value in saml2_attribute.iter('{urn:oasis:names:tc:SAML:2.0:assertion}AttributeValue'):
                    role_pairs.append(saml2_attribute_value.text)

        # Normalize pieces of string; order may vary per AWS sample
        result = []
        for role_pair in role_pairs:
            idp, role = None, None
            for field in role_pair.split(','):
                if 'saml-provider' in field:
                    idp = field
                elif 'role' in field:
                    role = field
            if not idp or not role:
                raise errors.GimmeAWSCredsError('Parsing error on {}'.format(role_pair))
            else:
                result.append(commondef.RoleSet(idp=idp, role=role, friendly_account_name="", friendly_role_name=""))

        return result

    def _display_role(self, roles):
        """ gets a list of available roles and
        asks the user to select the role they want to assume
        """
        # Gather the roles available to the user.
        role_strs = []
        for i, role in enumerate(roles):
            if not role:
                continue
            role_strs.append('[{}] {}'.format(i, role.role))

        return role_strs


import os
import pandas as pd
from django.shortcuts import render
from django.http import HttpResponseRedirect

source_dates = pd.read_csv("us-covid-dates.csv")

titles = {
    "ct": {
        "title": "CASE RATE REPORTED BY STATE | TOTAL",
        "subtitle": "TOTAL NUMBER OF CASES | AS OF THE DATE INDICATED"
    },
    "ck": {
        "title": "CASE RATE REPORTED BY STATE WITH NATIONWIDE AVERAGE | PER 100K",
        "subtitle": "NUMBER OF CASES PER 100 000 POPULATION | AS OF THE DATE INDICATED"
    },
    "dt": {
        "title": "DEATH RATE REPORTED BY STATE | TOTAL",
        "subtitle": "TOTAL NUMBER OF DEATHS | AS OF THE DATE INDICATED"
    },
    "dk": {
        "title": "DEATH RATE REPORTED BY STATE WITH NATIONWIDE AVERAGE | PER 100K",
        "subtitle": "NUMBER OF DEATHS PER 100 000 POPULATION | AS OF THE DATE INDICATE"
    }
}

def index(request):

    if request.method == "POST":
        log = open("log.txt", "w")
        log.write(request.POST["day"])
        log.close()
        log_p = open("log_p.txt", "w")
        log_p.write(request.POST["page"])
        log_p.close()
        return HttpResponseRedirect("/")

    log = open("log.txt", "r")
    day = log.read()
    log.close()
    wb = db = nd = nw = "active"
    if int(day) < 8:
        wb = "inactive"
    if int(day) == 1:
        db = "inactive"
    if int(day) == 343:
        nd = "inactive"
    if int(day) > 336:
        nw = "inactive"

    log_p = open("log_p.txt", "r")
    page = log_p.read()
    log_p.close()
    ct = ck = dt = dk = "inactive"
    err = ""
    if page == "ct":
        ct = "active"
    if page == "ck":
        ck = "active"
        err = "NO STATES WITH AT LEAST 1 CASE PER 100 000 POPULATION"
    if page == "dt":
        dt = "active"
        err = "NO STATES WITH REPORTED DEATHS"
    if page == "dk":
        dk = "active"
        err = "NO STATES WITH AT LEAST 1 DEATH PER 100 000 POPULATION"

    code = "0" * (3 - len(str(day))) + str(day)
    date_line = list(source_dates[(source_dates.id == int(day))]["date_line"])[0]

    context = {
        "day": day,
        "date_line": date_line,
        "test": code,
        "chart": {"name": page, "code": code, "title": titles[page]["title"], "subtitle": titles[page]["subtitle"]},
        "buttons": {
            "days": {"back": {"day": db, "week": wb}, "next": {"day": nd, "week": nw}},
            "cases": {"total": ct, "100k": ck}, "deaths": {"total": dt, "100k": dk}
        },
        "file": os.path.exists("polls/static/polls/test/" + page + "_" + code + ".svg"),
        "error": err
    }

    return render(request, "polls/index.html", context)

def about(request):
    return render(request, "polls/about.html")


# -*- coding: utf-8 -*-
'''mid
此文件为数据管理程序主界面
用于对历史数据进行检查核对

功能：
    1.展示本地数据KLine图形
    2.下载远程数据到本地
窗口布局：
    1.数据展示主窗口
        左侧为本地数据列表
        右侧为KLine
        左下侧为本地数据操作命令按钮
    2.数据下载管理子窗口
        左侧为代码表
        右侧为待下载代码
程序结构：
    为了能够方便的被各种窗体调用嵌入，各个窗体都定义为layout
'''
from matplotlib.backends.backend_qt4agg import FigureCanvasQTAgg as FigureCanvas
from PyQt4 import QtGui,QtCore
from datetime import datetime
import os,sys
import numpy as np
import pandas as pd
import datetime as dt
import matplotlib.dates as mpd
import matplotlib.pyplot as plt

if sys.version > '3':
    PY3 = True
else:
    PY3 = False
dataRoot = os.path.abspath(os.path.join(os.path.dirname(__file__),os.pardir,os.pardir,'histdata'))
sys.path.append(dataRoot)
import dataCenter as dataCenter
from data.mongodb.DataSourceMongodb import Mongodb

windowsRoot = os.path.abspath(os.path.join(os.path.dirname(__file__),os.pardir))
sys.path.append(windowsRoot)
from Widgets.pgCandleWidgetCross import pgCandleWidgetCross
from Views.HistoryCandleView import HistoryCandleView
from Views.HistoryTableView import HistoryTableView
class dataVisualizerLayout(QtGui.QVBoxLayout):
    def __init__(self,parent=None):
        '''mid
        1)创建combobox组，并初始化值
        2)依据combobox组的值初始化table
        3)依据combobox，table初始化candleview
        '''
        self.parent = None
        super(dataVisualizerLayout,self).__init__()
        #mid data
        self.dataCenter = dataCenter.dataCenter()
        #mid 1) local data selector
        layoutLocalDataSource = self.createDataSourceSelectorLayout()
        layoutLocalDataVisualizer = self.createDataVisualizerLayout()
        # bottom--------------------------------------------------------------------
        self.addLayout(layoutLocalDataSource)
        self.addLayout(layoutLocalDataVisualizer)
        #self.setStretch(0, 1)
        #self.setStretch(1, 4)
    def createDataSourceSelectorLayout(self):
        '''mid
        创建数据来源选择组合框组，横向排列
        '''
        layoutSymbolsSelector = QtGui.QHBoxLayout(self.parent)

        labelSourceType = QtGui.QLabel('source Type:')
        labelStorageType = QtGui.QLabel(self.tr("storage type:"))
        labelPeriodType = QtGui.QLabel(self.tr("period type:"))

        datasourceComboBox = QtGui.QComboBox()
        self.localDatasourceComboBox = datasourceComboBox
        dataProviders = self.dataCenter.getDataProviders()
        for dataProvider in dataProviders:
            datasourceComboBox.insertItem(0,dataProvider)

        storageComboBox = QtGui.QComboBox()
        self.localSymbolsStorageComboBox = storageComboBox
        dataStorages = self.dataCenter.getDataStorages()
        for dataStorage in dataStorages:
            storageComboBox.insertItem(0,dataStorage)

        periodComboBox=QtGui.QComboBox()
        self.localSymbolsPeriodComboBox = periodComboBox
        dataPeriods = self.dataCenter.getDataPeriods('mt5')
        for dataPeriod in sorted(dataPeriods):
            periodComboBox.insertItem(0,dataPeriod)

        self.overlayCheckBox = QtGui.QRadioButton('overlay')   # 创建单选框
        self.aloneCheckBox = QtGui.QRadioButton('stand-alone')
        self.aloneCheckBox.setChecked(True)      # 将Radio1选中

        datasourceComboBox.activated[str].connect(self.onDataSourceTypeComboBoxChanged)
        storageComboBox.activated[str].connect(self.onLocalSymbolSelectorActivate)
        periodComboBox.activated[str].connect(self.onLocalSymbolSelectorActivate)

        layoutSymbolsSelector.addWidget(labelSourceType)
        layoutSymbolsSelector.addWidget(datasourceComboBox)

        layoutSymbolsSelector.addWidget(labelStorageType)
        layoutSymbolsSelector.addWidget(storageComboBox)

        layoutSymbolsSelector.addWidget(labelPeriodType)
        layoutSymbolsSelector.addWidget(periodComboBox)

        layoutSymbolsSelector.addWidget(self.aloneCheckBox)
        layoutSymbolsSelector.addWidget(self.overlayCheckBox)

        layoutSymbolsSelector.addWidget(QtGui.QLabel(' '))

        layoutSymbolsSelector.setStretch(8,1)

        return layoutSymbolsSelector
    def onDataSourceTypeComboBoxChanged(self):
        datasource = str(self.localDatasourceComboBox.currentText())
        dataPeriods = self.dataCenter.getDataPeriods(datasource)
        dataPeriodsSorted = sorted(dataPeriods)
        self.localSymbolsPeriodComboBox.clear()
        for dataPeriod in dataPeriodsSorted:
            self.localSymbolsPeriodComboBox.insertItem(0,dataPeriod)

        self.updateLocalAvailableSymbolsTable()
    def onLocalSymbolSelectorActivate(self,text):
        self.updateLocalAvailableSymbolsTable()
    def updateLocalAvailableSymbolsTable(self):
        """mid
        dfLocalSymbols.index = 'code'
        dfLocalSymbols.columns = ['code','name','c_name',...]
        """
        datasource = str(self.localDatasourceComboBox.currentText())
        storageType = str(self.localSymbolsStorageComboBox.currentText())
        period = str(self.localSymbolsPeriodComboBox.currentText())

        dfLocalSymbols = self.dataCenter.getLocalAvailableDataSymbols(dataType = datasource,storageType = storageType,periodType = period)

        self.tableLocalAvailableSymbols.setSelectionBehavior(QtGui.QAbstractItemView.SelectRows)
        self.tableLocalAvailableSymbols.setEditTriggers(QtGui.QTableWidget.NoEditTriggers)
        self.tableLocalAvailableSymbols.setSelectionBehavior(QtGui.QTableWidget.SelectRows)
        self.tableLocalAvailableSymbols.setSelectionMode(QtGui.QTableWidget.SingleSelection)
        self.tableLocalAvailableSymbols.setAlternatingRowColors(True)

        self.tableLocalAvailableSymbols.clear()
        header = ["code","counts","date from","date to"]
        self.tableLocalAvailableSymbols.setColumnCount(len(header))

        if(dfLocalSymbols is None):
            self.tableLocalAvailableSymbols.setRowCount(0)
            return

        self.tableLocalAvailableSymbols.setRowCount(len(dfLocalSymbols))
        self.tableLocalAvailableSymbols.setHorizontalHeaderLabels(header)     #mid should be after .setColumnCount()

        if(True):
            for row in range(len(dfLocalSymbols.index)):
                for column in range(len(dfLocalSymbols.columns)):
                    self.tableLocalAvailableSymbols.setItem(row,column,QtGui.QTableWidgetItem(str(dfLocalSymbols.iget_value(row, column))))
        else: #mid the above codes have better performance than the below.
            for row in np.arange(0,len(dfLocalSymbols)):
                code = dfLocalSymbols.index[row]

                #symbol = QtGui.QLabel(self.tr(code))
                symbol = str(code)
                codeName = dfLocalSymbols.loc[code,'name']
                codeClass = dfLocalSymbols.loc[code,'c_name']

                #self.tableLocalSymbols.setCellWidget(row,0,symbol)
                self.tableLocalAvailableSymbols.setItem(row,0,QtGui.QTableWidgetItem(symbol))
                self.tableLocalAvailableSymbols.setItem(row,1,QtGui.QTableWidgetItem(codeName))
                self.tableLocalAvailableSymbols.setItem(row,2,QtGui.QTableWidgetItem(codeClass))

    #----------------------------------------------------------------------
    def createDataControlButtonLayout(self):
        bottomLeft03 = QtGui.QHBoxLayout(self.parent)

        DeleteOneFromDBButton=QtGui.QPushButton(self.tr("DeleteOneFromDB"))
        DeleteAllFromDBButton=QtGui.QPushButton(self.tr("DeleteAllFromDB"))
        ShowInTableButton=QtGui.QPushButton(self.tr("ShowInTable"))
        ShowInGraphButton=QtGui.QPushButton(self.tr("ShowInGraph"))

        bottomLeft03.addWidget(DeleteOneFromDBButton)
        bottomLeft03.addWidget(DeleteAllFromDBButton)
        bottomLeft03.addWidget(ShowInTableButton)
        bottomLeft03.addWidget(ShowInGraphButton)

        self.connect(DeleteOneFromDBButton,QtCore.SIGNAL("clicked()"),self.slotDeleteOneFromDB)
        self.connect(DeleteAllFromDBButton,QtCore.SIGNAL("clicked()"),self.slotDeleteAllFromDB)
        self.connect(ShowInTableButton,QtCore.SIGNAL("clicked()"),self.slotShowInTable)
        self.connect(ShowInGraphButton,QtCore.SIGNAL("clicked()"),self.slotShowInCandleGraph)
        return bottomLeft03
    def getDataSourceParams(self):
        rowSelected = self.tableLocalAvailableSymbols.currentRow()
        if((rowSelected<0) and (self.tableLocalAvailableSymbols.rowCount()>0)):
            rowSelected = 0
        datasource = str(self.localDatasourceComboBox.currentText())
        storageType = str(self.localSymbolsStorageComboBox.currentText())
        period = str(self.localSymbolsPeriodComboBox.currentText())

        alone = self.aloneCheckBox.isChecked()
        overlay = self.overlayCheckBox.isChecked()

        dataSource={}
        dataSource['dataProvider'] = datasource
        dataSource['storageFormat']=storageType
        dataSource['dataPeriod']=period
        dataSource['dateStart']=None
        dataSource['dateEnd']=None
        dataSource['alone'] = alone
        dataSource['overlay'] = overlay

        if(rowSelected>=0):   #a row selected or table is not empty.
            symbolToShow = str(self.tableLocalAvailableSymbols.item(rowSelected,0).text())
            dataSource['symbol']=symbolToShow
        else:
            dataSource['symbol']=''
        return dataSource
    def onClicked(self):
        print ('******row : ' , self.tableLocalAvailableSymbols.currentRow(), ' ***********')

        rows = self.tableLocalAvailableSymbols.rowCount()

        #for rows_index in range(rows):
            ##print items[item_index].text()
            #print (self.tableLocalAvailableSymbols.item(rows_index,0).text())
        params = self.getDataSourceParams()
        if(params is not None):
            dataForCandle = self.dataCenter.retriveCandleData(params = params)
            if(params['alone'] is True):       # 判断单选框是否选中
                graphMode = 'alone'
            elif (params['overlay'] is True):
                graphMode = 'overlay'
            self.candleWidget.setCandleData(dataForCandle = dataForCandle,graphMode = graphMode)
    def createTableLayout(self):
        # 06)bottomLeft03
        localSymbolTable = QtGui.QVBoxLayout(self.parent)

        self.tableLocalAvailableSymbols = QtGui.QTableWidget()
        localSymbolTable.addWidget(self.tableLocalAvailableSymbols)

        localSymbolTable.addLayout(self.createDataControlButtonLayout())
        self.tableLocalAvailableSymbols.itemClicked.connect(self.onClicked)
        self.updateLocalAvailableSymbolsTable()
        return localSymbolTable
    def createDataVisualizerLayout(self):
        bottomLeft02 = QtGui.QHBoxLayout(self.parent)
        bottomLeft02.addLayout(self.createTableLayout())

        # 05)bottomLeft02---------------------
        params = self.getDataSourceParams()
        if(params is not None):
            dataForCandle = self.dataCenter.retriveCandleData(params = params)

            self.candleWidget = pgCandleWidgetCross(dataForCandle=dataForCandle)

        bottomLeft02.addWidget(self.candleWidget)

        bottomLeft02.setStretch(0,1)
        bottomLeft02.setStretch(1,3.5)
        return bottomLeft02
    def slotDeleteOneFromDB(self):
        params = self.getDataSourceParams()
        if(params is not None):
            self.dataCenter.removeFromStorage(dataProvider = params['dataProvider'],storageType = params['storageFormat'],
                                              symbols = params['symbol'],period=params['dataPeriod'])
            self.updateLocalAvailableSymbolsTable()
    def slotDeleteAllFromDB(self):
        '''
            dataSource={}
            dataSource['dataProvider'] = datasource
            dataSource['storageFormat']=storageType
            dataSource['dataPeriod']=period
            dataSource['symbol']=symbolToShow
            dataSource['dateStart']='2015-03-19'
            dataSource['dateEnd']='2015-12-31'
            dataSource['alone'] = alone
            dataSource['overlay'] = overlay
        '''
        params = self.getDataSourceParams()
        if(params is not None):
            self.dataCenter.removeFromStorage(dataProvider = params['dataProvider'],storageType = params['storageFormat'],
                                              symbols = None,period=params['dataPeriod'])
        self.updateLocalAvailableSymbolsTable()
    def slotShowInTable(self):
        """
        show currrent selected symbol in table
        """
        params = self.getDataSourceParams()
        if(params is not None):
            data = self.dataCenter.retriveHistData(params = params)
            self.tableHistory=HistoryTableView(rawData=data)
            self.tableHistory.setWindowTitle("history")
            self.tableHistory.showMaximized()
        else:   #none selected and empty table
            symbol = 'none to download.'
            QtGui.QMessageBox.information(self.parent,"Information",self.tr(symbol))

    def slotShowInCandleGraph(self):
        params = self.getDataSourceParams()
        if(params is not None):
            dataForCandle = self.dataCenter.retriveCandleData(params = params)
            self.__showCandle__(dataForCandle)
            #self.myWindowfff = MyDialog(dataForCandle=dataForCandle)
            #self.myWindowfff.show()
        else:   #none selected and empty table
            symbol = 'none to download.'
            QtGui.QMessageBox.information(self.parent,"Information",self.tr(symbol))
    def __showCandle__(self,dataForCandle):
        dialog = QtGui.QDialog()
        self.pgCandleView = dialog
        layout = QtGui.QHBoxLayout()
        layoutLeft = QtGui.QVBoxLayout()
        layout.addLayout(layoutLeft)
        dialog.setLayout(layout)
        dialog.setWindowTitle(('ComboView'))
        # 2) creates widgets
        editor = QtGui.QTextEdit()
        editor.setText("x = %0.1f,y = %0.1f"% (2.0,2.0))

        candle = pgCandleWidgetCross(dataForCandle=dataForCandle)
        #candle = pgCrossAddition()
        # 3)arrange widgets
        #layout.addWidget(editor)
        layout.addWidget(candle)
        dialog.showMaximized()

if __name__ == '__main__':
    #----------------------------------------------------------------------
    class MainWindow(QtGui.QMainWindow):
        def __init__(self):
            super(MainWindow, self).__init__()
            cw = QtGui.QWidget()
            self.setCentralWidget(cw)
            cw.setLayout(dataVisualizerLayout(self))
    app = QtGui.QApplication(sys.argv)
    mainWin = MainWindow()
    mainWin.showMaximized()
    sys.exit(app.exec_())


from tests import unittest

from awscli.customizations.s3.utils import find_bucket_key, find_chunksize
from awscli.customizations.s3.constants import MAX_SINGLE_UPLOAD_SIZE

class FindBucketKey(unittest.TestCase):
    """
    This test ensures the find_bucket_key function works when
    unicode is used.
    """
    def test_unicode(self):
        s3_path = '\u1234' + u'/' + '\u5678'
        bucket, key = find_bucket_key(s3_path)
        self.assertEqual(bucket, '\u1234')
        self.assertEqual(key, '\u5678')

class FindChunksizeTest(unittest.TestCase):
    """
    This test ensures that the ``find_chunksize`` function works
    as expected.
    """
    def test_small_chunk(self):
        """
        This test ensures if the ``chunksize`` is appropriate to begin with,
        it does not change.
        """
        chunksize = 7 * (1024 ** 2)
        size = 8 * (1024 ** 2)
        self.assertEqual(find_chunksize(size, chunksize), chunksize)

    def test_large_chunk(self):
        """
        This test ensures if the ``chunksize`` adapts to an appropriate
        size because the original ``chunksize`` is too small.
        """
        chunksize = 7 * (1024 ** 2)
        size = 8 * (1024 ** 3)
        self.assertEqual(find_chunksize(size, chunksize), chunksize * 2)

    def test_super_chunk(self):
        """
        This tests to ensure that the ``chunksize can never be larger than
        the ``MAX_SINGLE_UPLOAD_SIZE``
        """
        chunksize = MAX_SINGLE_UPLOAD_SIZE + 1
        size = MAX_SINGLE_UPLOAD_SIZE * 2
        self.assertEqual(find_chunksize(size, chunksize),
                         MAX_SINGLE_UPLOAD_SIZE)

if __name__ == "__main__":
    unittest.main()


import paho.mqtt.client as mqtt
import sqlite3
from datetime import datetime, timedelta

# Name of DB file.
db_file = "connected_devices.db"

# Exceptions classes.
class ConnectionToMqttBrokerException(Exception):
   def __init__(self):
      pass

   def __str__(self):
      message = "MQTT Manager could not connect to the broker."
      return message

class DbSetUpException(Exception):
   def __init__(self):
      pass

   def __str__(self):
      message = "MQTT Manager could not set up the DB."
      return message

class DbUpdateException(Exception):
   def __init__(self):
      pass

   def __str__(self):
      message = "MQTT Manager could not update the DB."
      return message

class DbReadException(Exception):
   def __init__(self):
      pass

   def __str__(self):
      message = "MQTT Manager could not read from the DB."
      return message

# Callback function for when a connection is established with the broker.
def on_connect(client, userdata, flags, rc):
    if rc != 0:
        raise ConnectionToMqttBrokerException

    # Subscribe to the topics of interest.
    client.subscribe("mqtt_manager")

# Callback function for when a message is received.
# If message was on 'mqtt_manager' topic, then update the DB.
# TODO
# - Is there message filtering required, so only messages with valid syntax are passed thru?
def on_message(client, userdata, msg):
    if msg.topic == "mqtt_manager":
        msg_elements = msg.payload.decode().split(';')

        try:
            # Connect to the database.
            conn = sqlite3.connect(db_file)

            # Create a cursor.
            cursor = conn.cursor()

            # Get the current timestamp.
            current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

            msg_elements.append(current_time)
            msg_elements.append(msg_elements[1])
            msg_elements.append(msg_elements[2])

            # If message comes from sensor device.
            if msg_elements[3] == "Sensor":
                #print("Sensor")
                sql_update_table = """UPDATE connected_sensors SET name=?, mac=?, device_id=?,
                            type=?, value=?, timestamp=? WHERE mac=? AND device_id=?"""

                sql_insert = """INSERT INTO connected_sensors (name, mac, device_id,
                            type, value, timestamp) VALUES (?, ?, ?, ?, ?, ?)"""

            # If message comes from actuator device.
            elif msg_elements[3] == "Actuator":

                #print("Actuator")
                sql_update_table = """UPDATE connected_actuators SET name=?, mac=?, device_id=?,
                            type=?, value=?, value_to_set_topic=?, timestamp=? WHERE mac=? AND device_id=?"""

                sql_insert = """INSERT INTO connected_actuators (name, mac, device_id,
                            type, value, value_to_set_topic, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?)"""

            # Execute the update statement.

            cursor.execute(sql_update_table, msg_elements)

            # Check if the update affected any rows.
            if cursor.rowcount == 0:

                # Remove last two elements from the list (mac and device_id)
                del msg_elements[-2:]

                # Execute the insert statement.

                cursor.execute(sql_insert, msg_elements)

            conn.commit()

            # Close the cursor and the database connection.
            cursor.close()
            conn.close()
        except:
            raise DbUpdateException

    #print(f"Received message '{msg.payload.decode()}' on topic '{msg.topic}'")

class MqttManager:
    def __init__(self, broker_ip, broker_port=1883, keepalive=60):
        self._set_up_the_db()
        self._set_up_mqtt_client()
        self._connect_to_the_broker(broker_ip, broker_port, keepalive)

    # Get list (list of tuples) of active devices (sensors and actuators).
    # For each device its name, mac, device id, value and type are returned.
    def get_devices_list(self):
        self._remove_inactive_devices_form_db(20)

        try:
            # Connect to the database.
            conn = sqlite3.connect(db_file)

            # Create a cursor.
            cursor = conn.cursor()

            # Select all rows from a table
            cursor.execute('SELECT name, mac, device_id, type, value FROM connected_sensors')

            # Fetch all rows as a list of tuples
            sensors = cursor.fetchall()

            # Select all rows from a table
            cursor.execute('SELECT name, mac, device_id, type, value FROM connected_actuators')

            # Fetch all rows as a list of tuples
            actuators = cursor.fetchall()

            devices = sensors + actuators

            # Close the cursor and the database connection.
            cursor.close()
            conn.close()
        except:
            raise DbReadException

        return devices

    # Set device value. Raises an error for incorrect parameters.
    def set_value(self, mac, device_id, value):
        # Make sure the device is the actuator (so setting value makes sense).
        try:
            # Connect to the database.
            conn = sqlite3.connect(db_file)

            # Create a cursor.
            cursor = conn.cursor()

            # Select.
            cursor.execute('SELECT value_to_set_topic FROM connected_actuators WHERE mac=? AND device_id=?', (mac, device_id))

            # Fetch the topic name.
            topic = cursor.fetchone()[0]

            # Send message in given topic
            self.client.publish(topic, str(value))

            # Close the cursor and the database connection.
            cursor.close()
            conn.close()
        except:
            raise DbReadException

    # Connects to the broker. Blocks until connection is established.
    def _connect_to_the_broker(self, broker_ip, broker_port, keepalive):
        try:
            self.client.connect(broker_ip, broker_port, keepalive)
        except:
            # 'from None' suppresses exceptions inside exception
            raise ConnectionToMqttBrokerException from None

        # Start the network loop to handle incoming messages (it is also needed for connecting to the broker).
        self.client.loop_start()

        # Wait for client to connect to the broker.
        while not self.client.is_connected():
            pass

    def _set_up_mqtt_client(self):
        # Create an MQTT client instance (MqttManager acts as a MQTT client).
        self.client = mqtt.Client()

        # Set up the callback functions.
        self.client.on_connect = on_connect
        self.client.on_message = on_message

    # In case there is no DB or table, it creates them.
    def _set_up_the_db(self):

        try:
            # Connect to the database.
            conn = sqlite3.connect(db_file)

            # Create a cursor.
            cursor = conn.cursor()

            # Create sensors table.
            sql_create_table = """ CREATE TABLE IF NOT EXISTS connected_sensors (
                                            id integer PRIMARY KEY AUTOINCREMENT,
                                            name text NOT NULL,
                                            mac text,
                                            device_id integer,
                                            type text,
                                            value text,
                                            timestamp DATE DEFAULT (datetime('now','localtime'))
                                        ); """

            cursor.execute(sql_create_table)

            # Create actuators table.
            sql_create_table = """ CREATE TABLE IF NOT EXISTS connected_actuators (
                                            id integer PRIMARY KEY AUTOINCREMENT,
                                            name text NOT NULL,
                                            mac text,
                                            device_id integer,
                                            type text,
                                            value text,
                                            value_to_set_topic text,
                                            timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
                                        ); """

            cursor.execute(sql_create_table)

            """sql_drop_table = " DROP TABLE connected_devices; "

            cursor.execute(sql_drop_table)"""

            # Close the cursor and the database connection.
            cursor.close()
            conn.close()
        except:
            raise DbSetUpException

    # Removes inactive devices form DB.
    # Devices inactive for longer than 'max_time_of_inactivity' (in seconds) are removed.
    def _remove_inactive_devices_form_db(self, max_time_of_inactivity):
        try:
            # Connect to the database.
            conn = sqlite3.connect(db_file)

            # Create a cursor.
            cursor = conn.cursor()

            # Define the timestamp threshold for deleting old records.
            timestamp_threshold = datetime.now() - timedelta(seconds=max_time_of_inactivity)

            # Execute the DELETE statement.
            sql_delete = "DELETE FROM connected_sensors WHERE timestamp < ?"
            cursor.execute(sql_delete, (timestamp_threshold,))

            sql_delete = "DELETE FROM connected_actuators WHERE timestamp < ?"
            cursor.execute(sql_delete, (timestamp_threshold,))

            conn.commit()

            # Close the cursor and the database connection.
            cursor.close()
            conn.close()
        except:
            raise DbUpdateException


# -*- coding: utf-8 -*-
"""
Trains and tests a Rolling Bayesian Ridge Regression model on data

@author: Nick
"""

import warnings
import numpy as np
import pandas as pd
from sklearn.pipeline import Pipeline
from sklearn.feature_selection import VarianceThreshold
from sklearn.preprocessing import MinMaxScaler
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import accuracy_score

# load dataset
Y = pd.read_csv('Y elect.csv').iloc[:,[0]]
X = pd.read_csv('X elect.csv')

# size the training and forecasting data sets
train_size = 16 # years
test_size = 1 # years

# define the initial train and test sets
train_idx = np.arange(train_size)
test_idx = np.arange(train_size, train_size + test_size)

# train the model on all the training data
pipeline = Pipeline([
    ('var', VarianceThreshold()),
    ('scale', MinMaxScaler()),
    ('model', GaussianNB()),
])
pipeline.fit(X.iloc[train_idx, :], Y.iloc[train_idx, :])

# produce a rolling forecast on all the testing data
predictions = pd.DataFrame()
actual = pd.DataFrame()
for i in range(X.shape[0] - train_size - test_size):
    # forecast
    pred = pd.DataFrame(pipeline.predict(X.iloc[test_idx, :])).T
    pred.columns = ["t" + str(i + 1) for i in range(pred.shape[1])]
    predictions = pd.concat([predictions, pred], axis=0).reset_index(drop=True)

    # actual
    true = Y.iloc[test_idx, :].copy().T
    true.columns = ["t" + str(i + 1) for i in range(true.shape[1])]
    actual = pd.concat([actual, true], axis=0).reset_index(drop=True)

    # roll forecast forward
    train_idx = train_idx + 1
    test_idx = test_idx + 1

    # train a new model
    with warnings.catch_warnings():
        warnings.simplefilter('ignore')
        pipeline.fit(X.iloc[train_idx, :], Y.iloc[train_idx, :])

    # report forecast
    y_pred = pred["t1"]
    y_true = Y.iloc[train_idx[-1],:][0]
    print('predicted=%f, expected=%f' % (y_pred, y_true))

# compute r2 for the predictions
acc = []
for j in predictions.columns:
    acc.append(accuracy_score(actual[j], predictions[j]))
acc


import numpy as np
import torch
from torch.utils.data import Dataset, DataLoader, ConcatDataset
import mat73

class MatCovData(Dataset):
    def __init__(self, snr=10, mode='train', norm=None):
        super(MatCovData, self).__init__()
        if mode == 'train':
            file_path = '/home/External/xr/SDOAnet/doa_project/experiment/train/train_e_simu_data_snr_{}.mat'.format(snr)
        elif mode == 'validate':
            file_path = '/home/External/xr/SDOAnet/doa_project/experiment/test/test_e_simu_data_snr_{}.mat'.format(snr)
        elif mode == 'test':
            file_path = '/home/External/xr/SDOAnet/doa_project/experiment/a/data_snr_{}.mat'.format(snr)
        mat_data = mat73.loadmat(file_path)
        alpha = mat_data['alpha_ls']
        beta = mat_data['beta_ls']
        x_in_ls = np.transpose(mat_data['X_in_ls'], (2, 0, 1))

        self.total_num = alpha.size

        sin_alpha = np.sin(np.deg2rad(alpha))
        cos_alpha = np.cos(np.deg2rad(alpha))
        sin_beta = np.sin(np.deg2rad(beta))
        cos_beta = np.cos(np.deg2rad(beta))
        if norm == 'norm':
            x_norm = x_in_ls / np.sqrt(np.square(np.linalg.norm(x_in_ls[:, 0,:], 2, -1).reshape(-1, 1, 1)) / 1000)
            self.input = torch.from_numpy(np.stack((np.real(x_norm), np.imag(x_norm)), axis=1)).float()
        elif norm == 'Max':
            x_Max = x_in_ls / np.max(np.abs(x_in_ls), axis=(1, 2)).reshape(-1, 1, 1)
            self.input = torch.from_numpy(np.stack((np.real(x_Max), np.imag(x_Max)), axis=1)).float()
        else:
            self.input = torch.from_numpy(np.stack((np.real(x_in_ls), np.imag(x_in_ls)), axis=1)).float()

        self.output = torch.from_numpy(np.stack((sin_alpha, cos_alpha, sin_beta, cos_beta, alpha, beta), axis=-1)).float()

    def __len__(self):
        return self.total_num

    def __getitem__(self, item):
        return self.input[item], self.output[item]

def gen_loader(batch_size=512, snr=30, norm=None):
    if isinstance(snr, int):
        train_set = MatCovData(snr=snr, mode='train', norm=norm)
        test_set = MatCovData(snr=snr, mode='validate', norm=norm)
    elif isinstance(snr, list):
        train_sets = []
        test_sets = []
        for s in snr:
            train_sets.append(MatCovData(snr=s, mode='train', norm=norm))
            test_sets.append(MatCovData(snr=s, mode='validate', norm=norm))
        train_set = ConcatDataset(train_sets)
        test_set = ConcatDataset(test_sets)
    else:
        train_set = MatCovData(mode='train', norm=norm)
        test_set = MatCovData(mode='validate', norm=norm)
    sampler = torch.utils.data.distributed.DistributedSampler(train_set)
    train_loader = DataLoader(train_set,
                              batch_sampler=sampler,
                              batch_size=batch_size, shuffle=True, num_workers=4)
    test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False, num_workers=4)

    return train_loader, test_loader\
        ,sampler


import pandas as pd
import pickle
import datetime
import sys
from nltk.metrics import edit_distance
from siuba import *
from siuba.dply.vector import row_number

needs_to_be_matched = pickle.load(open("../data/manual_cleaning/needs_to_by_joined_wip.p", "rb"))
new_songs = pickle.load(open("../data/data2.p", "rb"))

# Remove video
needs_to_be_matched = (
    needs_to_be_matched
    >> filter(
        ~(_.kind.str.contains('video') | _.kind.str.contains('movie')),
        _.genre != "Podcast"
    )
)

# Clean columns used for search
new_songs = new_songs >> mutate(
    artist_clean = _.artist.str.lower(),
    name_clean = _.name.str.lower(),
    album_clean = _.album.str.lower(),
)

def calculate_distance(search):
    return (_
        .dropna()
        .apply(lambda x: edit_distance(x, search))
    )

postprocess = (
    # mutate(choice=row_number(_))
    select(_.artist, _.album, _.name, _.persistent_id)
    >> arrange(_.artist, _.album, _.name)
)

def results_from_standard_search(strict=True):
    print("Searching using substrings...")
    if strict:
        results = (
            new_songs
            >> filter(
                _.artist_clean.str.contains(song.artist.lower(), regex=False),
                _.name_clean.str.contains(song['name'].lower(), regex=False),
            )
        )
    else:
        results = (
            new_songs
            >> filter(
                (_.artist_clean.str.contains(song.artist.lower(), regex=False)) |
                (_.name_clean.str.contains(song['name'].lower(), regex=False)),
            )
        )
    return results >> postprocess

def results_from_artist_search(artist):
    print("Searching using artist...")
    return (
        new_songs
        >> filter(_.artist == artist)
        >> postprocess
    )

def results_from_custom_search(query):
    print("Searching using custom query...")
    return (
        new_songs
        >> filter(
            (_.artist_clean.str.contains(query.lower())) |
            (_.name_clean.str.contains(query.lower())) |
            (_.album_clean.str.contains(query.lower()))
        )
        >> postprocess
    )

def results_from_edit_distance():
    print("Searching using edit distance...")
    artist_distance = calculate_distance(song.artist.lower())
    name_distance = calculate_distance(song['name'].lower())

    results = (
        new_songs
        >> mutate(
            artist_distance=artist_distance(_.artist_clean),
            name_distance=name_distance(_.name_clean),
        )
    )

    results['artist_ntile'] = pd.qcut(results.artist_distance, 20, duplicates='drop')
    results['name_ntile'] = pd.qcut(results.name_distance, 20, duplicates='drop')

    results = (
        results
        >> filter(
            _.artist_ntile == _.artist_ntile.cat.categories[0],
            _.name_ntile == _.name_ntile.cat.categories[0],
        )
        >> postprocess
    )

    return results

def display_results(results, current_song, i, length=needs_to_be_matched.shape[0]):
    with pd.option_context(
        'display.max_rows', None,
        'display.max_columns', None,
        'display.width', None):
        print(
            results >> select(-_.persistent_id)
        )
    return input("""
        ({0}/{1}) Find a match for:

        song: {2}
        artist: {3}
        album: {4}

        > """.format(
            i,
            length,
            song['name'],
            song.artist,
            song.album
        )
    )

def write_out(df):
    df.to_csv("../data/manual_cleaning/needs_to_by_joined_wip.csv")
    pickle.dump(df, open("../data/manual_cleaning/needs_to_by_joined_wip.p", "wb"))

def match(i, resp):
    # Use the Index rather than the row number
    match = results.loc[int(resp), ]
    needs_to_be_matched.loc[needs_to_be_matched.index[i], "persistent_id_y"] = match.persistent_id
    write_out(needs_to_be_matched)

def increment_reviewed_at(i):
    needs_to_be_matched.loc[needs_to_be_matched.index[i], 'reviewed_at'] = datetime.datetime.now()

nb_pending_review = (
    needs_to_be_matched
    >> filter(_.reviewed_at.isna())
).shape[0]

# Handle complete case
if nb_pending_review == 0:
    print("No work left to do. Congratulations are in order (hopefully).")
    sys.exit()

for i in range(needs_to_be_matched.shape[0]):
    try:
        # Search for matches
        song = needs_to_be_matched.iloc[i]

        # # Skip any song that has already been matched
        # if song.persistent_id_y:
        #     continue

        # Skip any song that has been reviewed
        if song.reviewed_at:
            continue

        # Skip holiday songs
        if song.genre == "Holiday":
            print("Skipping holiday song: {}".format(song['name']))
            increment_reviewed_at(i)
            continue

        # Skip songs with missing values (cheap shortcut to prevent errors)
        if song.artist is None or song['name'] is None:
            print("Skipping song with missing metadata: {0} by {1}".format(song['name'], song.artist))
            continue

        # Search using contains
        results = results_from_standard_search(strict=True)

        # Automatically match a song if 1 result
        if results.shape[0] == 1:
            print("""
                Automatically matching:

                {0} => {1}
            """.format(song['name'], results.iloc[0]['name']))
            # import pdb; pdb.set_trace()
            increment_reviewed_at(i)
            match(i, results.index[0])
            continue

        # If no results, loosen the filter
        if results.shape[0] == 0:
            results = results_from_standard_search(strict=False)

        # If still no results, take edit_distance out for a spin
        if results.shape[0] == 0:
            results = results_from_edit_distance()

        # Display results
        resp = display_results(results, song, i)

        valid_cmds = ['m', 'a', 's', 'q', 'c', 'e']

        # Search by default
        if (resp.lower() not in valid_cmds) and (not resp.isnumeric()):
            results = results_from_custom_search(resp)
            resp = display_results(results, song, i)

        if resp.lower() == 'm':
            results = results_from_edit_distance()
            resp = display_results(results, song, i)

        # Do an artist search
        if resp.lower() == 'a':
            results = results_from_artist_search(song.artist)
            resp = display_results(results, song, i)

        # Allow custom search
        if resp.lower() == 's' or resp.lower() == 'q':
            query = input("\t> ")
            results = results_from_custom_search(query)
            resp = display_results(results, song, i)

        if resp.lower() == 'c':
            increment_reviewed_at(i)
            continue

        # If error, make sure to leave entries blank to return to it
        if resp.lower() == 'e':
            continue

        # Future music: Include option to search for just the artist or just the song

        # Update needs_to_be_matched
        print("Great! You chose {}.".format(results.loc[int(resp), "name"]))
        increment_reviewed_at(i)
        match(i, resp)

    # Save file on exit (or anything bad happening)
    except:
        e = sys.exc_info()#[0]
        print(e)
        print("\n\nSaving...")
        write_out(needs_to_be_matched)
        print("Goodbye for now, my friend.")
        break


from matplotlib import pyplot
from matplotlib.patches import Rectangle

import imageIO.png
import numpy as np
import math

class Queue:
    def __init__(self):
        self.items = []

    def isEmpty(self):
        return self.items == []

    def enqueue(self, item):
        self.items.insert(0,item)

    def dequeue(self):
        return self.items.pop()

    def size(self):
        return len(self.items)

def createInitializedGreyscalePixelArray(image_width, image_height, initValue = 0):

    new_array = [[initValue for x in range(image_width)] for y in range(image_height)]
    return new_array

# this function reads an RGB color png file and returns width, height, as well as pixel arrays for r,g,b
def readRGBImageToSeparatePixelArrays(input_filename):

    image_reader = imageIO.png.Reader(filename=input_filename)
    # png reader gives us width and height, as well as RGB data in image_rows (a list of rows of RGB triplets)
    (image_width, image_height, rgb_image_rows, rgb_image_info) = image_reader.read()

    print("read image width={}, height={}".format(image_width, image_height))

    # our pixel arrays are lists of lists, where each inner list stores one row of greyscale pixels
    pixel_array_r = []
    pixel_array_g = []
    pixel_array_b = []

    for row in rgb_image_rows:
        pixel_row_r = []
        pixel_row_g = []
        pixel_row_b = []
        r = 0
        g = 0
        b = 0
        for elem in range(len(row)):
            # RGB triplets are stored consecutively in image_rows
            if elem % 3 == 0:
                r = row[elem]
            elif elem % 3 == 1:
                g = row[elem]
            else:
                b = row[elem]
                pixel_row_r.append(r)
                pixel_row_g.append(g)
                pixel_row_b.append(b)

        pixel_array_r.append(pixel_row_r)
        pixel_array_g.append(pixel_row_g)
        pixel_array_b.append(pixel_row_b)

    return (image_width, image_height, pixel_array_r, pixel_array_g, pixel_array_b)

# This method packs together three individual pixel arrays for r, g and b values into a single array that is fit for
# use in matplotlib's imshow method
def prepareRGBImageForImshowFromIndividualArrays(r,g,b,w,h):
    rgbImage = []
    for y in range(h):
        row = []
        for x in range(w):
            triple = []
            triple.append(r[y][x])
            triple.append(g[y][x])
            triple.append(b[y][x])
            row.append(triple)
        rgbImage.append(row)
    return rgbImage

# This method takes a greyscale pixel array and writes it into a png file
def writeGreyscalePixelArraytoPNG(output_filename, pixel_array, image_width, image_height):
    # now write the pixel array as a greyscale png
    file = open(output_filename, 'wb')  # binary mode is important
    writer = imageIO.png.Writer(image_width, image_height, greyscale=True)
    writer.write(file, pixel_array)
    file.close()

def convertToGreyscale(image_width, image_height, px_array_r, px_array_g, px_array_b):
    px_array_gs = []
    for i in range(image_height):
        row = []
        for j in range(image_width):
            #weighted covertion
            row.append(int(px_array_r[i][j]*0.299 + px_array_g[i][j]*0.587 + px_array_b[i][j]*0.114))
        px_array_gs.append(row)
    return px_array_gs

def normalize(image_width, image_height, pixel_array,):
    #print(len(pixel_array),len(pixel_array[0]))
    low = min([min(r) for r in pixel_array])
    high = max([max(r) for r in pixel_array])
    img = []
    if high == low:
        return img
    for i in range(image_height):
        row = []
        for j in range(image_width):
            row.append(int(round((pixel_array[i][j] - low) * (255/(high-low)))))
        img.append(row)
    return img

def convolve(a,b):
    #res =a[0][0]*b[2][2] + a[0][1]*b[2][1]+a[0][2]*b[2][0]+a[1][0]*b[1][2] + a[1][1]*b[1][1]+a[1][2]*b[1][0]+a[2][0]*b[0][2] + a[2][1]*b[0][1]+a[2][2]*b[0][0]
    res = []
    for i in range(3):
        for j in range(3):
            res.append(a[i][j]*b[i][j])
    return sum(res)

def sobelFilter(image_width, image_height, gs):
    xfilter = [[1, 0, -1], [2, 0, -2], [1, 0, -1]]
    yfilter = [[1, 2, 1], [0, 0, 0], [-1, -2, -1]]

    sobel = [[0 for x in range(image_width)] for y in range(image_height)]

    for i in range(image_height-2):
        for j in range(image_width-2):
            x = convolve(xfilter,[gs[i][j:j+3],gs[i+1][j:j+3],gs[i+2][j:j+3]])
            y = convolve(yfilter,[gs[i][j:j+3],gs[i+1][j:j+3],gs[i+2][j:j+3]])
            sobel[i+1][j+1] = math.sqrt(x*x+y*y)
    #print(len(sobel),len(sobel[0]))
    return sobel

def meanFilter(image_width, image_height, p):
    window = 5
    c = int((window-1)/2) #border
    t = createInitializedGreyscalePixelArray(image_width+c*2, image_height+c*2)
    for i in range(c,image_height+c):
        for j in range(c,image_width+c):
            t[i][j] =  p[i-c][j-c]

    res = []
    #border ignore
    for i in range(c,image_height + c):
        row = []
        for j in range(c,image_width + c):
            total = 0
            for k in range(-c,c+1):
                total += sum(t[i+k][j-c:j+c+1])
            row.append(int(round(total/(window*window))))
        res.append(row)
    return res

def thresholdFilter(image_width, image_height, gs):
    t = 70 #threshold
    img = []
    for i in range(image_height):
        row = []
        for j in range(image_width):
            if gs[i][j] > t:
                row.append(255)
            else:
                row.append(0)
        img.append(row)
    return img

def createInitializedGreyscalePixelArray(image_width, image_height):
    p = []
    for i in range(image_height):
        row =[]
        for j in range(image_width):
            row.append(0)
        p.append(row)
    return p

def dilation(p, image_width, image_height):
    window = 3
    c = int((window-1)/2) #border
    t = createInitializedGreyscalePixelArray(image_width+2, image_height+2)
    #t = zero padded array
    for i in range(1,image_height+1):
        for j in range(1,image_width+1):
            t[i][j] =  p[i-1][j-1]
    a = []
    for i in range(c,image_height+c): #+c because array is padded
        row = []
        for j in range(c,image_width+c):
            total = 0
            for k in range(-c,c+1):
                for x in t[i+k][j-c:j+c+1]:
                    if not(x) == False:
                        total = 255
                        break;
            row.append(total)
        a.append(row)
    return a

def erosion(p, image_width, image_height):
    window = 3
    c = int((window-1)/2) #border
    t = createInitializedGreyscalePixelArray(image_width+2, image_height+2)
    #t = zero padded array
    for i in range(1,image_height+1):
        for j in range(1,image_width+1):
            t[i][j] =  p[i-1][j-1]

    a = []
    for i in range(c,image_height+c): #+c because array is padded
        row = []
        for j in range(c,image_width+c):
            total = 255
            for k in range(-c,c+1):
                for x in t[i+k][j-c:j+c+1]:
                    if x == 0:
                        total = 0
                        break;
            row.append(total)
        a.append(row)
    return a

def ccl(p, image_width, image_height):

    current_label = 1
    visited = set()
    objects = {}
    cc = createInitializedGreyscalePixelArray(image_width, image_height)
    for i in range(image_height):
        for j in range(image_width):
            if p[i][j] != 0 and (i,j) not in visited:
                visited.add((i,j))
                q = Queue()
                q.enqueue((i,j))
                objects[current_label] = 0
                while q.isEmpty() == False:
                    n = q.dequeue()
                    objects[current_label] += 1
                    x = n[0]#height
                    y = n[1]#width
                    cc[x][y] = current_label
                    if y-1>=0 and p[x][y-1] != 0 and (x,y-1) not in visited: #left
                        visited.add((x,y-1))
                        q.enqueue((x,y-1))
                    if y+1<=image_width-1 and p[x][y+1] != 0 and (x,y+1) not in visited: #right
                        visited.add((x,y+1))
                        q.enqueue((x,y+1))
                    if x-1>=0 and p[x-1][y] != 0 and (x-1,y) not in visited: #top
                        visited.add((x-1,y))
                        q.enqueue((x-1,y))
                    if x+1<=image_height-1 and p[x+1][y] != 0 and (x+1,y) not in visited: #bottom
                        visited.add((x+1,y))
                        q.enqueue((x+1,y))
                current_label += 1

    comp = max(objects, key = objects.get)

    for i in range(image_height):
        for j in range(image_width):
            if cc[i][j] == comp:
                cc[i][j] = 255
            else:
                cc[i][j] = 0
    return cc

def find_box(p, image_width, image_height):
    x = []
    for i in range(image_height):
        for j in range(image_width):
            if p[i][j] != 0:
                x.append(j)
                break;
    y = []
    for i in range(image_width):
        for j in range(image_height):
            if p[j][i] != 0:
                y.append(j)
                break;
    w = []
    for i in range(image_height-1,-1,-1):
        for j in range(image_width-1,-1,-1):
            if p[i][j] != 0:
                w.append(j)
                break;

    h = []
    for i in range(image_width-1,-1,-1):
        for j in range(image_height-1,-1,-1):
            if p[j][i] != 0:
                h.append(j)
                break;

    return min(x),min(y),max(w),max(h)

def main():
    filename = "./images/covid19QRCode/poster1small.png"

    # we read in the png file, and receive three pixel arrays for red, green and blue components, respectively
    # each pixel array contains 8 bit integer values between 0 and 255 encoding the color values
    (image_width, image_height, px_array_r, px_array_g, px_array_b) = readRGBImageToSeparatePixelArrays(filename)

    #CONVERT RGB ARRAYS TO GREYSCALE ARRAY
    print("CONVERT RGB ARRAYS TO GREYSCALE ARRAY")
    px_array_gs = convertToGreyscale(image_width, image_height, px_array_r, px_array_g, px_array_b)
    px_array_gs = normalize(image_width, image_height,px_array_gs)

    #CREATE GREYSCALE PNG
    #print("CREATE GREYSCALE PNG")
    #writeGreyscalePixelArraytoPNG("1greyimg.png", px_array_gs, image_width, image_height)

    #PERFORM SOBEL FILTER ON GREYSCALE ARRAY
    print("PERFORM SOBEL FILTER ON GREYSCALE ARRAY")
    px_array_sobel = sobelFilter(image_width, image_height, px_array_gs)
    px_array_sobel = normalize(image_width, image_height, px_array_sobel)

    #CREATE GREYSCALE PNG
    #print("CREATE GREYSCALE PNG")
    #writeGreyscalePixelArraytoPNG("2sobelimg.png", px_array_sobel, image_width, image_height)

    #PERFORM MEAN FILTER THRICE ON SOBEL FILTER
    print("PERFORM MEAN FILTER THRICE ON SOBEL FILTER")
    px_array_mean = meanFilter(image_width, image_height, px_array_sobel)
    px_array_mean = normalize(image_width, image_height, px_array_mean)
    px_array_mean = meanFilter(image_width, image_height, px_array_mean)
    px_array_mean = normalize(image_width, image_height, px_array_mean)
    px_array_mean = meanFilter(image_width, image_height, px_array_mean)
    px_array_mean = normalize(image_width, image_height, px_array_mean)

    #CREATE GREYSCALE PNG
    #print("CREATE GREYSCALE PNG")
    #writeGreyscalePixelArraytoPNG("3meanimg.png", px_array_mean, image_width, image_height)

    #PERFORM THRESHOLD FILTER ON MEAN FILTER
    print("PERFORM THRESHOLD FILTER ON MEAN FILTER")
    px_array_thres = thresholdFilter(image_width, image_height, px_array_mean)

    #CREATE GREYSCALE PNG
    #print("CREATE GREYSCALE PNG")
    #writeGreyscalePixelArraytoPNG("4thresimg.png", px_array_thres, image_width, image_height)

    #PERFORM  DILATION FILTER ON THRESHOLD FILTER
    print("PERFORM DILATION FILTER ON THRESHOLD FILTER")
    px_array_dial = dilation(px_array_thres, image_width, image_height)
    px_array_dial = dilation(px_array_dial, image_width, image_height)

    #CREATE GREYSCALE PNG
    #print("CREATE GREYSCALE PNG")
    #writeGreyscalePixelArraytoPNG("5dialimg.png", px_array_dial, image_width, image_height)

    #PERFORM  EROSION FILTER ON DILATION  FILTER
    print("PERFORM EROSION FILTER ON DILATION  FILTER")
    px_array_eros = erosion(px_array_dial, image_width, image_height)
    px_array_eros = erosion(px_array_eros, image_width, image_height)

    #CREATE GREYSCALE PNG
    #print("CREATE GREYSCALE PNG")
    #writeGreyscalePixelArraytoPNG("6erosimg.png", px_array_eros, image_width, image_height)

    #PERFORM  CONNECTED COMPONENT LABELING
    print("PERFORM CONNECTED COMPONENT LABELING")
    px_array_ccl = ccl(px_array_eros, image_width, image_height)

    #CREATE GREYSCALE PNG
    #print("CREATE GREYSCALE PNG")
    #writeGreyscalePixelArraytoPNG("7cclimg.png", px_array_ccl, image_width, image_height)

    #FIND BOUNDING BOX
    print("FIND BOUNDING BOX")
    (x,y,w,h) = find_box(px_array_ccl, image_width, image_height)

    pyplot.imshow(prepareRGBImageForImshowFromIndividualArrays(px_array_r, px_array_g, px_array_b, image_width, image_height))

    # get access to the current pyplot figure
    axes = pyplot.gca()
    # create a 70x50 rectangle that starts at location 10,30, with a line width of 3
    #rect = Rectangle( (10, 30), 70, 50, linewidth=3, edgecolor='g', facecolor='none' )

    rect = Rectangle( (x, y), w-x, h-y, linewidth=3, edgecolor='g', facecolor='none' )
    # paint the rectangle over the current plot
    axes.add_patch(rect)

    # plot the current figure
    pyplot.show()

if __name__ == "__main__":
    main()


import math

from src.VAR import Var
from src.VAR_distance import Var_distance
from scipy.spatial import distance

class Grangercalculator_distance:

    def __init__(self, df, lag):
        self.df = df
        self.lag = lag

    def GC_calculator(self, bi_pairs, W, tau):
        distances = []
        tau_exp = math.exp(tau)
        for pair in bi_pairs:
            formula = []
            second_value = []
            for j in range(len(pair)):
                data = self.df[list(pair[j])].copy(deep=True)
                VAR = Var_distance(data, self.lag)

                params, l2norm = VAR.var_calculation(list(pair[j]))

                # Calculates the value of the past values of the second stock * it's coefficient value in the formula.
                second = data[data.columns[0]].to_numpy()
                second_converted = []
                for i in range(len(second)-1):
                    answer = params[1] * second[i]
                    second_converted.append(answer)

                second_value.append(second_converted)
                formula.append(l2norm)
                # Calculates the variance of the residuals univariate model of Z.
                if j == 0:
                    data_uni = self.df[pair[j][1]].copy(deep=True)
                    VAR_uni = Var(data_uni, self.lag)
                    variance_uni = VAR_uni.var_univariate()
            xy = distance.euclidean(second_value[1], second_value[2])
            lower_bound = (abs(abs(abs(xy - formula[1]) - formula[2])-formula[2]))
            Upperbound = (formula[1] + xy + 2*formula[2])
            # calculates the value for which formula[0]**2/W must be smaller so that GC holds
            GC_Threshold = (variance_uni/tau_exp)
            distances.append([[tau, W, lower_bound, formula[0], Upperbound,  formula[0]**2/W, GC_Threshold], pair])
        return distances


# --------------------------------------------------------------------
# shell.py: Shell command execution tools, including ShellRecipe.
#
# Author: Lain Musgrove (lain.proliant@gmail.com)
# Date: Thursday, January 2 2020
#
# Distributed under terms of the MIT license.
# --------------------------------------------------------------------
import asyncio
import os
import shlex
import subprocess
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union

from ansilog import bg, fg

from .config import CPU_CORES
from .errors import BuildError
from .recipes import FileRecipe
from .reports import Report
from .util import digest_env, format_dt, get_logger, is_iterable, decode

# -------------------------------------------------------------------
LineSinkFunction = Callable[[str], None]
ShellOutputTaskData = Tuple[asyncio.StreamReader, LineSinkFunction]

# --------------------------------------------------------------------
log = get_logger("panifex")

# --------------------------------------------------------------------
@dataclass
class OutputLine:
    stderr: bool
    line: str
    when: datetime = field(default_factory=lambda: datetime.now())

    def json(self):
        result = {"when": format_dt(self.when), "line": self.line}

        if self.stderr:
            result["error"] = True

        return result

    def __str__(self):
        return self.line

# -------------------------------------------------------------------
class OutputSink:
    def out(self, line):
        raise NotImplementedError()

    def err(self, line):
        raise NotImplementedError()

    def output(self) -> Generator[OutputLine, None, None]:
        raise NotImplementedError()

# -------------------------------------------------------------------
class InMemoryOutputSink(OutputSink):
    def __init__(self):
        self._output = []

    def out(self, line):
        self._output.append(OutputLine(False, line))

    def err(self, line):
        self._output.append(OutputLine(True, line))

    def output(self) -> Generator[OutputLine, None, None]:
        yield from sorted(self._output, key=lambda x: x.when)

# -------------------------------------------------------------------
class NullOutputSink(OutputSink):
    def output(self) -> Generator[OutputLine, None, None]:
        yield from ()

# -------------------------------------------------------------------
class PostCommunicateOutputSink(OutputSink):
    def __init__(self, stdout: str, stderr: str):
        self._stdout = stdout
        self._stderr = stderr

    def output(self) -> Generator[OutputLine, None, None]:
        yield from (OutputLine(False, line) for line in self._stdout.splitlines())
        yield from (OutputLine(True, line) for line in self._stderr.splitlines())

# -------------------------------------------------------------------
class OutputCollector:
    async def collect(self, proc: Any) -> OutputSink:
        raise NotImplementedError()

# -------------------------------------------------------------------
class ShellOutputCollector:
    # pylint/issues/1469: pylint doesn't recognize asyncio.subprocess
    # pylint: disable=E1101
    def __init__(self):
        self._readline_tasks: Dict[asyncio.Future[Any], ShellOutputTaskData] = {}

    def _setup_readline_task(
        self, stream: asyncio.StreamReader, sink: Callable[[str], None]
    ):
        if stream is not None:
            self._readline_tasks[asyncio.Task(stream.readline())] = (stream, sink)

    async def collect(self, proc: Any, sink: OutputSink):
        if not isinstance(proc, asyncio.subprocess.Process):
            raise ValueError("`proc` is not an asyncio.subprocess.Process object.")
        if hasattr(proc, "stdout") and proc.stdout is not None:
            self._setup_readline_task(proc.stdout, sink.out)
        if hasattr(proc, "stderr") and proc.stderr is not None:
            self._setup_readline_task(proc.stderr, sink.err)

        while self._readline_tasks:
            done, pending = await asyncio.wait(
                self._readline_tasks, return_when=asyncio.FIRST_COMPLETED
            )

            for future in done:
                stream, sink_f = self._readline_tasks.pop(future)
                line = future.result()
                if line:
                    line = decode(line).strip()
                    sink_f(line)
                    self._setup_readline_task(stream, sink_f)

# -------------------------------------------------------------------
class ShellCommunicateOutputCollector(OutputCollector):
    async def collect(self, proc: Any) -> OutputSink:
        if not isinstance(proc, subprocess.Popen):
            raise ValueError("`proc` is not a subprocess.Popen object.")

        return PostCommunicateOutputSink(*proc.communicate())

# --------------------------------------------------------------------
@dataclass
class ShellReport(Report):
    name: str
    started: Optional[datetime]
    finished: Optional[datetime]
    cmd: str
    sink: Optional[OutputSink]
    returncode: Optional[int]

    def succeeded(self):
        return self.returncode == 0

    def generate(self):
        return {
            **super().generate(),
            "cmd": self.cmd,
            "returncode": self.returncode,
            "out": [line.json() for line in self.sink.output() if not line.stderr]
            if self.sink
            else [],
            "err": [line.json() for line in self.sink.output() if line.stderr]
            if self.sink
            else [],
        }

    def output(self, stdout=True, stderr=False) -> Generator[OutputLine, None, None]:
        if self.sink is None:
            return

        for line in self.sink.output():
            if not line.stderr and stdout:
                yield line
            elif line.stderr and stderr:
                yield line

    def log_output(self, stdout=True, stderr=True, log=log):
        if self.sink is None:
            log.info("<no output>")
            return

        for line in self.sink.output():
            if not line.stderr and stdout:
                log.info(line.line)

        for line in self.sink.output():
            if line.stderr and stderr:
                log.info(fg.red(line.line))

# --------------------------------------------------------------------
class ShellFailed(BuildError):
    def __init__(self, report: ShellReport):
        super().__init__(f"{report.name} failed.")
        self.report = report

# -------------------------------------------------------------------
class ShellRecipe(FileRecipe):
    OUT = "output"
    IN = "input"
    INCLUDES = "includes"
    CWD = "cwd"
    _limiter = asyncio.BoundedSemaphore(CPU_CORES)

    def __init__(self, command, **params):
        super().__init__()

        self._input = params.get(self.IN, None)
        self._includes = params.get(self.INCLUDES, None)
        self._output = params.get(self.OUT, None)
        self._cwd = params.get(self.CWD, os.getcwd())
        self._env = {}
        self._params = {**params}
        self._name = "Shell Command"
        self._sink: Optional[OutputSink] = None
        self._returncode = 0
        self._cmd = shlex.join(command) if isinstance(command, (list, tuple)) else command
        self._user_input: Optional[str] = None
        self._interactive = False
        self._echo = True

    def with_env(self, env: Dict):
        self.merge_env(env)
        return self

    def with_name(self, name: str):
        self._name = name
        return self

    def with_sink(self, sink: OutputSink):
        self._sink = sink

    def with_user_input(self, input: str):
        self._user_input = input
        return self

    def no_echo(self):
        self._echo = False
        return self

    def __repr__(self):
        return f"<panifex.ShellRecipe {self._cmd}, {self._params}>"

    def _check_success(self):
        if not self.succeeded():
            raise ShellFailed(self.report())

    def succeeded(self):
        return self.is_done() and self._returncode == 0

    def merge_env(self, env):
        self._env.update(env)
        return self

    def interactive(self):
        self._interactive = True
        return self

    def user_input(self, input):
        self._user_input = input
        return self

    async def _resolve(self) -> Any:
        await self._run_command(self._cmd)
        return self.output()

    async def _run_command(self, cmd) -> None:
        await self._limiter.acquire()
        try:
            params, args, decorated_args = self._parse_command(cmd)
            if self._echo:
                log.info(fg.blue("[sh]") + decorated_args)

            if self._interactive:
                if self._user_input:
                    raise ValueError(
                        "Interactive shell can't provide input programmatically."
                    )
                self._sink = NullOutputSink()
                self._returncode = subprocess.call(
                    self._cmd, env=digest_env(params), cwd=self._cwd, shell=True
                )

            else:
                proc = await asyncio.create_subprocess_shell(
                    self._cmd,
                    stdin=asyncio.subprocess.PIPE,
                    stdout=asyncio.subprocess.PIPE,
                    stderr=asyncio.subprocess.PIPE,
                    env=digest_env(params),
                    cwd=self._cwd,
                )

                if self._user_input is not None and proc.stdin is not None:
                    proc.stdin.write(self._user_input.encode('utf-8'))
                    await proc.stdin.drain()
                    proc.stdin.close()

                collector = ShellOutputCollector()
                self._sink = InMemoryOutputSink()
                await collector.collect(proc, self._sink)
                await proc.wait()
                self._returncode = proc.returncode

            self.finish()
            if self._echo:
                self._print_run_report(decorated_args)

        finally:
            self._limiter.release()

    def _parse_command(self, cmd):
        params = {**self._params, **self._env}

        for k, v in params.items():
            if is_iterable(v):
                params[k] = " ".join([shlex.quote(str(x)) for x in v])
            else:
                params[k] = shlex.quote(str(v))

        self._cmd = cmd.format(**params)
        args = shlex.split(self._cmd)

        decorated_args = f" {fg.magenta(args[0])} {shlex.join(args[1:])}"

        return params, args, decorated_args

    def _run_command_sync(self, cmd):
        params, args, decorated_args = self._parse_command(cmd)
        if self._echo:
            self._print_run_header(decorated_args)

        if self._interactive:
            if self._user_input:
                raise ValueError(
                    "Interactive shell can't provide input programmatically."
                )
            self._sink = NullOutputSink()
            self._returncode = subprocess.call(
                self._cmd, env=digest_env(params), cwd=self._cwd, shell=True
            )
        else:
            proc = subprocess.Popen(
                shlex.split(self._cmd),
                env=digest_env(params),
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                cwd=self._cwd,
                text=True
            )
            stdout, stderr = proc.communicate(input=self._user_input)
            self._sink = PostCommunicateOutputSink(stdout, stderr)
            self._returncode = proc.returncode

        self.finish()
        if self._echo:
            self._print_run_report(decorated_args)

    @classmethod
    def _print_run_header(cls, decorated_args):
        log.info(fg.blue("[sh]") + decorated_args)

    def _print_run_report(self, decorated_args):
        if self.succeeded():
            if self.config and self.config.verbose:
                self.report().log_output()
        else:
            log.info(fg.white(bg.red("[!!]")) + decorated_args)
            self.report().log_output()

    def input(self) -> Any:
        return [self._input, self._includes]

    def output(self) -> Any:
        return self._output

    def report(self) -> ShellReport:
        return ShellReport(
            name=self._name,
            started=self.started,
            finished=self.finished,
            cmd=self._cmd,
            sink=self._sink,
            returncode=self._returncode,
        )

    def sync(self) -> 'ShellRecipe':
        self._run_command_sync(self._cmd)
        return self

# -------------------------------------------------------------------
class ShellRecipeFactory:
    def __init__(self):
        self._env = {**os.environ}

    def clone(self) -> 'ShellRecipeFactory':
        sh = ShellRecipeFactory()
        sh.env(**self._env)
        return sh

    def env(self, *args, **kwargs) -> Union[str, List[str]]:
        if kwargs:
            self._env.update(kwargs)
        if args is not None:
            if len(args) == 1:
                return self._env[args[0]]
            return [self._env[k] for k in args]
        return {**self.env}

    def __getitem__(self, name):
        return self.env(name)

    def __setitem__(self, name, value):
        self.env(**{name: value})

    def __call__(self, *args, **kwargs):
        recipe = ShellRecipe(*args, **kwargs)
        recipe.merge_env(self._env)
        return recipe

# -------------------------------------------------------------------
sh = ShellRecipeFactory()


from typing import Optional

# Definition for a binary tree node.
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    def insertIntoBST(self, root: Optional[TreeNode], val: int) -> Optional[TreeNode]:
        if not root:
            return TreeNode(val)
        node = root
        while node:
            if val > node.val:
                prev = node
                node = node.right
            elif val < node.val:
                prev = node
                node = node.left

        if val > prev.val:
            prev.right = TreeNode(val)
        if val < prev.val:
            prev.left = TreeNode(val)

        return root

class Solution:
    def insertIntoBST(self, root: Optional[TreeNode], val: int) -> Optional[TreeNode]:
        if not root:
            return TreeNode(val)

        if val > root.val:
            root.right = self.insertIntoBST(root.right, val)
        elif val < root.val:
            root.left = self.insertIntoBST(root.left, val)

        return root


import sys
from PyQt5 import QtWidgets
from PyQt5.QtCore import QThread
import pyqtgraph as pg
import numpy as np

from MainWindow import Ui_MainWindow
#from GraphClass import MultiLine # Supposed to improve perforamnce, but with latest pyqtgraph updates maybe not needed anymore
from WorkerClass import AcquireData
from ParamsClass import SampleParams,DLTSParams,TempParams,LakeshoreParams,MFIAParams

class MainWindow(QtWidgets.QMainWindow, Ui_MainWindow):
    def __init__(self, *args, obj=None, **kwargs):
        super(MainWindow, self).__init__(*args, **kwargs)
        self.setupUi(self)
        self.first_plot = True
        self.curves = []

        ## CREATE PARAM STRUCTS
        self.sample = SampleParams()
        self.dlts = DLTSParams()
        self.temp = TempParams()
        self.lake = LakeshoreParams()
        self.mfia = MFIAParams()

        ## CREATE PRIMARY WORKER THREAD
        self.data = AcquireData()
        self.thread = QThread()
        self.data.moveToThread(self.thread)
        self.data.finished.connect(self.thread.quit)
        self.thread.started.connect(self.data.do_scan)

        ## MAIN SIGNAL->SLOTS
        # Machine States
        self.buttonInit.clicked.connect(self.initialize_hardware)
        self.buttonStart.clicked.connect(self.commence_scan)
        self.buttonStop.clicked.connect(self.stopping_scan)
        self.buttonIdle.clicked.connect(self.goto_idle)
        self.data.init_fail.connect(self.scan_complete)
        self.data.init_fail.connect(self.state_stopped)
        self.data.lakeshore.lakeshore_disconnect.connect(self.stopping_scan)
        self.data.lakeshore.lakeshore_disconnect.connect(self.state_stopwait)
        self.data.graph_update.connect(self.graph_data)
        self.data.finished.connect(self.scan_complete)
        self.data.finished.connect(self.state_stopped)

        # Worker LogObjects may print to log
        self.data.log_signal.connect(self.generate_log)
        self.data.lakeshore.log_signal.connect(self.generate_log)
        self.data.device.log_signal.connect(self.generate_log)
        self.data.file.log_signal.connect(self.generate_log)

    def initialize_hardware(self):
        ## POPULATE PARAMS STRUCTS FROM UI
        self.generate_log("All settings now locked in. You must re-initialize to make changes.")
        # Set sample info
        self.sample.user = self.sampleUser.text()
        self.sample.name = self.sampleName.text()
        self.sample.material = self.sampleMat.text()
        self.sample.area = self.sampleArea.value()
        self.sample.comment = self.sampleComment.text()

        # Set DLTS experiment parameters
        self.dlts.ss_bias = self.dltsBias.value()
        self.dlts.pulse_height = self.dltsPulse.value()
        self.dlts.trns_length = self.dltsTWidth.value()
        self.dlts.pulse_width = self.dltsPWidth.value()
        self.dlts.sample_time = self.dltsSample.value()

        # Set temperature parameters
        self.temp.temp_init = self.tempStart.value()
        self.temp.temp_step = self.tempStep.value()
        self.temp.temp_final = self.tempEnd.value()
        self.temp.temp_idle = self.tempIdle.value()
        self.temp.temp_stability = self.tempStable.value()
        self.temp.time_stability = self.timeStable.value()

        # Set Lakeshore parameters
        if self.radioSampleA.isChecked() == True:
            self.lake.sample = 'A'
        elif self.radioSampleB.isChecked() == True:
            self.lake.sample = 'B'
        if self.radioControlA.isChecked() == True:
            self.lake.control = 'A'
        elif self.radioControlB.isChecked() == True:
            self.lake.control = 'B'
        if self.radioHeaterLow.isChecked() == True:
            self.lake.heatpower = '1'
        elif self.radioHeaterMed.isChecked() == True:
            self.lake.heatpower = '2'
        elif self.radioHeaterHi.isChecked() == True:
            self.lake.heatpower = '3'

        # Set MFIA Parameters
        self.mfia.sample_rate = self.mfiaSample.value()
        self.mfia.ac_freq = self.mfiaFreq.value()
        self.mfia.ac_ampl = self.mfiaAmp.value()
        self.mfia.time_constant = self.mfiaTC.value()
        self.mfia.i_range = float(self.mfiaRange.currentText())*0.001 # conversion from mA to A
        self.mfia.sample_reject = self.mfiaReject.value()

        ## INITIALIZE WORKER
        self.data.reset(self.sample,self.dlts,self.temp,self.lake,self.mfia)

    def graph_data(self):
        #nPlots = 100
        nPlots = self.data.cap_data.shape[1]
        #nSamples = 16000
        nSamples = self.data.cap_data.shape[0]
        #y = np.random.normal(size=(120,20000), scale=0.2) + np.arange(120)[:,np.newaxis]
        #x = np.empty((120,20000))
        #x[:] = np.arange(20000)[np.newaxis,:]
        now = pg.ptime.time()
        if self.first_plot:
            #data = np.random.normal(size=(nPlots,nSamples))
            self.graphWidget.disableAutoRange()
            self.graphWidget.setYRange(np.nanmin(self.data.cap_data), np.nanmax(self.data.cap_data))
            self.graphWidget.setXRange(0, nSamples)

            for idx in range(nPlots):
                curve = pg.PlotCurveItem(pen=(170-idx,nPlots*4))  #7->17 out of 40
                self.graphWidget.addItem(curve)
                self.curves.append(curve)
                #lines = MultiLine(x,y)
                self.curves[idx].setData(self.data.cap_data[:,idx])
            curve = pg.PlotCurveItem(pen='r')
            self.graphWidget.addItem(curve)
            self.curves.append(curve)
            self.curves[nPlots].setData(self.data.file.transient)
            #lines = MultiLine(x,y)
            #self.graphWidget.addItem(lines)

            self.first_plot = False
        else:
            #data = np.random.normal(size=(nPlots,nSamples))
            print('hi')
            self.graphWidget.setYRange(np.nanmin(self.data.cap_data), np.nanmax(self.data.cap_data))
            self.graphWidget.setXRange(0, nSamples)
            for idx in range(nPlots):
                self.curves[idx].setData(self.data.cap_data[:,idx])
            self.curves[nPlots].setData(self.data.file.transient)
        print("Plot time: %0.2f sec" % (pg.ptime.time()-now))
        app.processEvents()

    def commence_scan(self):
        self.thread.start()

    def stopping_scan(self):
        self.generate_log("Stop signal sent. Please wait...","orange")
        self.data.lakeshore.stopped = True

    def scan_complete(self):
        self.first_plot = True
        self.curves = []
        self.generate_log("All done.","green")

    def goto_idle(self):
        self.generate_log("Setpoint set to idle temperature, {:3.2f} K.".format(self.tempIdle.value()),"blue")
        self.data.lakeshore.sampleSetPoint(self.tempIdle.value())

app = QtWidgets.QApplication(sys.argv)

window = MainWindow()
window.show()
app.exec()


'''
通过解析xml文件，批量修改xml文件里的标签名称，比如把标签zero改成num
'''
import os.path
import glob
import xml.etree.ElementTree as ET

path = r'数据集\Annotations'    #存储标签的路径，修改为自己的Annotations标签路径
sum3 = 0
for xml_file in glob.glob(path + '/*.xml'):
    ####### 返回解析树
	tree = ET.parse(xml_file)
	##########获取根节点
	root = tree.getroot()
	#######对所有目标进行解析
	# *******************************这个用来批量改打标文件名称
	# for member in root.findall('object'):
	# 	objectname = member.find('name').text
	# 	if objectname == 'uncovered':      #原来的标签名字
	# 		print(objectname)
	# 		member.find('name').text = str('cover')    #替换的标签名字
	# 		tree.write(xml_file)
	# ****************************************************************
	# ********************************这个用来统计xml文件夹中某个标签的数量
	sum2 = 0
	for member in root.findall('object'):
		objectname = member.find('name').text
		sum = 0
		if objectname == 'playPhone':
			sum+=1
		sum2 = sum2+sum
	sum3 = sum2+sum3
print(objectname,'=',sum3)
# ****************************************************************************


#!/usr/bin/env python
# coding: utf-8
# import
import os
import numpy as np
import tensorflow as tf
from models import simple_CNN
from keras.callbacks import EarlyStopping, ModelCheckpoint, CSVLogger, TensorBoard
import matplotlib.pyplot as plt

init = tf.global_variables_initializer()

config = tf.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
session.run(init)
model_save_path = 'trained_models/simpler_CNN.{epoch:02d}-{val_acc:.2f}.hdf5'
train_x = np.load('train_x.npy')
train_y = np.load('train_y.npy')

image_size = train_x.shape[1:]
print(image_size)
num_classes = 2
batch_size = 64
num_epochs = 10

model = simple_CNN(image_size, num_classes)
#model.load_weights('trained_models/simpler_CNN.22-0.67.hdf5')

# 编译模型，categorical_crossentropy多分类选用
model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['accuracy'])

# 记录日志(Record Log)
csv_logger = CSVLogger('training.log')

# 保存检查点(Save checkpoint)
model_checkpoint = ModelCheckpoint(model_save_path,
                                   monitor='val_acc',
                                   verbose=1,
                                   save_best_only=True)

model_callbacks = [model_checkpoint, csv_logger]

# 训练模型(Training Model)
history = model.fit(train_x, train_y, batch_size, num_epochs,
                    verbose=1,
                    callbacks=model_callbacks,
                    validation_split=.2,
                    shuffle=True)

# 绘制训练的准确率值(Plotting the accuracy of training)
plt.figure(1)
plt.plot(history.history['acc'])#acc最新版keras已经无法使用
plt.title('Model accuracy')#图名
plt.ylabel('Accuracy')#纵坐标名
plt.xlabel('Epoch')#横坐标名
plt.legend(['Train'], loc='upper left')#角标及其位置

# 绘制训练的损失值(Plotting the loss value of training)
plt.figure(2)
plt.plot(history.history['loss'])
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train'], loc='upper left')
plt.show()


import tqdm
import pickle

import torch
from torch.utils.data import DataLoader

import sentencepiece as spm

from custom_dataset import CustomDataset
from models.rnn.model import Encoder, NMTDecoder, StylizedNMT
from loss import ce_loss

# os.environ['CUDA_LAUNCH_BLOCKING'] = "1"

torch.cuda.empty_cache()
torch.autograd.set_detect_anomaly(True)

@torch.no_grad()    #no autograd (backpropagation X)
def nmt_evaluate(tst_encoder, nmt_model, data_loader, device):
    tst_encoder.eval()
    nmt_model.eval()

    test_loss = 0.0
    total = len(data_loader)

    with tqdm.tqdm(total=total) as pbar:
        for _, (src, trg) in enumerate(data_loader):
            src = src.to(device)
            trg = trg.to(device)

            _, nmt_hidden, nmt_cell = tst_encoder(src)

            nmt_hidden = nmt_hidden.detach().to(device)
            nmt_cell = nmt_cell.detach().to(device)
            # nmt_hidden = nmt_hidden.to(device)
            # nmt_cell = nmt_cell.to(device)

            nmt_out, output_list = nmt_model(nmt_hidden, nmt_cell, trg)

            nmt_out = nmt_out[:, 1:].reshape(-1, nmt_out.size(-1))

            trg_trg = trg[:, 1:].reshape(-1)

            CE_loss = ce_loss(nmt_out, trg_trg)
            nmt_loss = CE_loss
            loss_value = nmt_loss.item()
            test_loss += loss_value

            pbar.update(1)

    return test_loss/total, trg[:, 1:].tolist(), output_list

def test(args):
    # Device Setting
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    print(f'Initializing Device: {device}')
    print(f'Count of using GPUs:{torch.cuda.device_count()}')

    # Data Setting
    with open("/HDD/yehoon/data/processed/tokenized/spm_tokenized_data.pkl", "rb") as f:
        data = pickle.load(f)
        f.close()

    em_informal_test = data["gyafc"]["test"]["em_informal"]
    pair_kor_test = data['korpora']['test']['pair_kor']
    # fr_informal_test = data["gyafc"]["test"]["fr_informal"]
    # fr_formal_test = data["gyafc"]["test"]["fr_formal"]

    # TODO argparse
    min_len, max_len = 2, 300
    nmt_vocab_size = 2400
    batch_size = 100
    num_workers = 0

    # tst_test_data = CustomDataset(em_informal_test, em_formal_test, min_len, max_len)
    # nmt_test_data = CustomDataset(pair_eng_test, pair_kor_test, min_len, max_len)
    test_data = CustomDataset(em_informal_test, pair_kor_test, min_len, max_len)

    test_loader = DataLoader(test_data, batch_size=batch_size, drop_last=True, shuffle=True, num_workers=num_workers)

    # Model Load
    # model.load_state_dict(torch.load(save_file_name)['model'])

    encoder = Encoder(input_size=nmt_vocab_size, d_hidden=1024, d_embed=256, n_layers=2, dropout=0.1, device=device)
    encoder.load_state_dict(torch.load("/HDD/yehoon/data/tst_model.pth")['model'])
    print("encoder", encoder)
    encoder = encoder.encoder.eval()

    decoder = NMTDecoder(output_size=nmt_vocab_size, d_hidden=1024, d_embed=256, n_layers=2, dropout=0.1, device=device)
    decoder.load_state_dict((torch.load("/HDD/yehoon/data/nmt_model.pth")['model']))
    decoder = decoder.eval()

    total_latent = torch.load(torch.load("/HDD/yehoon/data/nmt_model.pth")['total_latent'])

    model = StylizedNMT(decoder, d_hidden=1024, total_latent=total_latent, device=device)
    model.load_state_dict(torch.load("/HDD/yehoon/data/nmt_model.pth"))
    model = model.to(device)
    model.eval()
    print("Start Testing..")

    test_tokenizer = spm.SentencePieceProcessor()
    test_tokenizer.Load("/HDD/yehoon/data/tokenizer/test_pair_kor_spm.model")

    test_decode_output = []

    test_loss, test_trg_list, test_out_list = nmt_evaluate(encoder, model, test_loader, device)
    print(f"Validation Loss: {test_loss:.5f}")

    test_out_list = list(map(list, zip(*test_out_list)))

    test_target_decode = [test_tokenizer.DecodeIds(i) for i in test_trg_list]
    test_output_decoder = [test_tokenizer.DecodeIds(j) for j in test_out_list]
    test_decode_output.append((test_target_decode, test_output_decoder))

    with open("/HDD/yehoon/data/test_target_decode.txt", "w", encoding="utf8") as tf, open("/HDD/yehoon/data/test_output_decode.txt", "w", encoding="utf8") as of:
        for t_line, o_line in test_decode_output:
            for tline in t_line:
                tf.write(f"{tline}\n")
            for oline in o_line:
                of.write(f"{oline}\n")
        tf.close()
        of.close()


from django.urls import path
from . import views

app_name = 'blog'

urlpatterns = [
    path('', views.blog_list, name='blog-list'),
    path('article/<int:id>/', views.blog_detail, name='blog-detail'),
    path('add-article/', views.add_article, name='add-article'),
    path('confirm-add-article/', views.confirm_add_article, name='confirm-add-article'),
    path('edit-article/<int:pk>/', views.edit_article, name='edit-article'),
    path('confirm-edit-article/<int:pk>/', views.confirm_edit_article, name='confirm-edit-article'),
    path('confirm-delete-article/<int:pk>/', views.confirm_delete_article, name='confirm-delete-article'),
    path('review/<int:pk>/<int:star_count>/', views.review, name='review'),
]