#!/usr/bin/env python

# TODO: search
# - https://docs.python.org/3/library/json.html
# - https://docs.python.org/3/library/urllib.request.html#module-urllib.request

import json
import os
import sys
import urllib.parse as parse
import urllib.request as req
from urllib.error import HTTPError

from colorama import Fore, Back, Style

url = 'http://www.python.org/'

# with req.urlopen(url) as f:
#    print(f.read(300).decode('utf-8'))

# f = req.urlopen(url= url)
# print(f.read().decode('utf-8'))

# TODO: continuar daqui..
# 1- https://www.google.com/search?q=python+use+config+file&rlz=1C1SQJL_enBR856BR856&oq=python+use+config+file&aqs=chrome..69i57.6758j0j7&sourceid=chrome&ie=UTF-8
# 2- yaml file
# https://martin-thoma.com/configuration-files-in-python/
# 3- configparser
# https://docs.python.org/3.4/library/configparser.html

class Board:

    def __init__(self, idBoard):
        self.idBoard = idBoard

    def get_lists(self):
        if self.lists is None:
            self.lists = get_lists_from_board(self.idBoard)
        return self.lists

class TrelloConfig:
    def __init__(self):

        try:
            self.idBoard = os.environ.get('TRELLO_BOARD')
            self.key = os.environ.get('TRELLO_KEY')
            self.token = os.environ.get('TRELLO_TOKEN')

            assert (self.idBoard is not None), "idBoard environment variable is missing."
            assert (self.key is not None), "key environment variable is missing."
            assert (self.token is not None), "token environment variable is missing."

        except Exception as ex:
            message_red_style(f"Something went wrong when reading environment variables. Error: {ex}")
            sys.exit()

    def __repr__(self):
        return repr((self.idBoard, self.key, self.token))

class ResponseApi:
    def __init__(self, json, status, reason):
        self.json = json
        self.status = status
        self.reason = reason

    def __repr__(self):
        return repr((self.json, self.status, self.reason))

def message_red_style(text):
    print('\n' + Back.RED + Fore.LIGHTWHITE_EX +
          text +
          Style.RESET_ALL)

def message_warning_style(text):
    print('\n' + Back.LIGHTYELLOW_EX + Fore.BLACK +
          text +
          Style.RESET_ALL)

def message_green_style(text):
    print('\n' + Back.LIGHTGREEN_EX + Fore.BLACK +
          text +
          Style.RESET_ALL)

def get_json_from_api(api_url, method):

    try:

        f = req.Request(url=api_url, data=None, method=method)

        with req.urlopen(f) as ff:
            json_data_file = ff.read().decode('utf-8')
            return ResponseApi(json.loads(json_data_file), ff.status, ff.reason)
    except HTTPError as ht:
        print('Error:', ht)
        print('Sorry, something wrong happens.')
        sys.exit()

    return None

def find_pr_in_trello(data_to_search, key, token, idBoard):
    trello_api = f'https://api.trello.com/1/search?query={parse.quote(data_to_search)}&idBoards={idBoard}&modelTypes=all&board_fields=name%2CidOrganization&boards_limit=10&card_fields=all&cards_limit=10&cards_page=0&card_board=true&card_list=false&card_members=false&card_stickers=false&card_attachments=false&organization_fields=name%2CdisplayName&organizations_limit=10&member_fields=avatarHash%2CfullName%2Cinitials%2Cusername%2Cconfirmed&members_limit=10&partial=false&key={key}&token={token}'

    response = get_json_from_api(trello_api, 'GET')

    assert (response is not None), 'Sorry, we could not found your PR number in trello, try again.'
    assert (response.status == 200), message_warning_style(
        'Sorry, there is something wrong with trello api, please try again.')

    if len(response.json['cards']) > 0:
        message_green_style('Good, pr founded in trello board.')

        firstCard = response.json['cards'][0]

        idCard = firstCard['id']
        idList = firstCard['idList']

        listName = get_name_of_list_from_card(idList, idBoard)

        print('> Id\t\t\t:', idCard)
        print('> Name\t\t\t:', firstCard['name'])
        print('> Actual List\t:', listName)

        answer = input('\nIs it your trello\'s card? (Y/N): ')

        if answer.lower() == 'n':
            message_red_style('bye, see you again...')
            sys.exit()
        move_pr_trello(idCard, key, token, idBoard)

def get_valid_numeric_option(max):
    number_template = -1
    while number_template < 0 or number_template > max:
        try:
            number_template = int(input('> '))
        except ValueError as ve:
            print(f'Error: {ve}. Try again...')

    return number_template

def get_lists_from_board(idBoard):
    trello_lists_board_api = f'https://api.trello.com/1/boards/{idBoard}/?fields=name&lists=all&list_fields=all'

    response = get_json_from_api(trello_lists_board_api, 'GET')

    if response is not None and len(response.json['lists']) > 0:
        return response.json['lists']
    return None

def get_name_of_list_from_card(idList, idBoard):
    lists_from_board = get_lists_from_board(idBoard)

    for row in lists_from_board:
        if row['id'] == idList:
            return row['name']
    return None

def move_pr_trello(trello_card_id, key, token, idBoard):
    jsonLists = get_lists_from_board(idBoard)

    total = len(jsonLists)

    print('\nWhich list do you wanna move?')
    for t in range(total):
        print(f"> {t} - {jsonLists[t]['name']} - {jsonLists[t]['id']}")

    choosedNumber = get_valid_numeric_option(total)

    id_list_to_move = jsonLists[choosedNumber]['id']
    name_list_to_move = jsonLists[choosedNumber]['name']

    trello_update_card_api = f'https://api.trello.com/1/cards/{trello_card_id}?idList={id_list_to_move}&key={key}&token={token}'

    response = get_json_from_api(trello_update_card_api, 'PUT')

    assert (response.status == 200), message_red_style('Something wrong happens when moving card...')

    message_warning_style(f'moving card to {name_list_to_move}...')

    new_post = f'AUTOMATICO: Movendo card para {name_list_to_move}'

    trello_add_new_comment = f'https://api.trello.com/1/cards/{trello_card_id}/actions/comments?text={parse.quote(new_post)}&&key={key}&token={token}'
    response = get_json_from_api(trello_add_new_comment, 'POST')

    assert (response.status == 200), message_red_style('Something wrong happens when adding a new comment...')

    message_warning_style('adding new comment...')

def main(use_args):
    pr = 0

    # args
    if use_args:

        total_args = len(sys.argv)
        if total_args == 1:
            print('COMMAND arg1; arg1=pr number;')
            sys.exit()

        pr = sys.argv[1]

    config = TrelloConfig()
    find_pr_in_trello(str(pr), config.key, config.token, config.idBoard)

if __name__ == "__main__":
    try:
        main(use_args=True)
    except Exception as ex:
        print('Error:', ex)
    except:
        pass


#coding: utf-8

#本程序用于显示数据清理和流重组后的数据

import sys
from window_base import *
from PyQt5.QtWidgets import QAbstractItemView, QHeaderView
from PyQt5.QtGui import QFont

class show_Dataclustering_Window(QtWidgets.QMainWindow, Ui_MainWindow):
    def __init__(self):
        super(show_Dataclustering_Window, self).__init__()
        super().setupUi(self)

        # 设置垂直布局
        self.verticalLayout = QtWidgets.QVBoxLayout(self.centralwidget)
        self.verticalLayout.setObjectName("verticalLayout")
        # 设置水平布局
        self.horizontalLayout = QtWidgets.QHBoxLayout()
        self.horizontalLayout.setObjectName("horizontalLayout")

        # 表格控件
        self.tableView = QtWidgets.QTableView(self.centralwidget)
        self.tableView.setGeometry(QtCore.QRect(10, 20, 1001, 421))
        self.tableView.setObjectName("tableView")
        self.tableView.setEditTriggers(QAbstractItemView.NoEditTriggers)  # 设置表格内容不可更改
        self.tableView.setSelectionBehavior(QAbstractItemView.SelectRows)  # 整行选中的方式
            # 下面代码让表格100填满窗口
        self.tableView.horizontalHeader().setStretchLastSection(True)
        self.tableView.horizontalHeader().setSectionResizeMode(QHeaderView.Stretch)
            # 设置表格控件的缩放比
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding)
        sizePolicy.setHeightForWidth(self.tableView.sizePolicy().hasHeightForWidth())
        #sizePolicy.setHorizontalStretch(4)
        #sizePolicy.setVerticalStretch(3)
        self.tableView.setSizePolicy(sizePolicy)
            # 加入布局
        self.verticalLayout.addWidget(self.tableView)

        # 文本控件1
        self.textBrowser = QtWidgets.QTextBrowser(self.centralwidget)
        self.textBrowser.setObjectName("textBrowser")
            # 设置文本控件的缩放比
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding)
        sizePolicy.setHeightForWidth(self.textBrowser.sizePolicy().hasHeightForWidth())
        #sizePolicy.setHorizontalStretch(1)
        #sizePolicy.setVerticalStretch(1)
        self.textBrowser.setSizePolicy(sizePolicy)
        self.textBrowser.setFont(QFont('SansSerif', 15))
            # 加入布局
        self.horizontalLayout.addWidget(self.textBrowser)

        # 文本控件2
        self.textBrowser2 = QtWidgets.QTextBrowser(self.centralwidget)
        self.textBrowser2.setObjectName("textBrowser")
            # 设置文本控件的缩放比
        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding)
        sizePolicy.setHeightForWidth(self.textBrowser2.sizePolicy().hasHeightForWidth())
        #sizePolicy.setHorizontalStretch(1)
        #sizePolicy.setVerticalStretch(1)
        self.textBrowser2.setSizePolicy(sizePolicy)
        self.textBrowser2.setFont(QFont('SansSerif', 15))
            # 加入布局
        self.horizontalLayout.addWidget(self.textBrowser2)

        # 水平布局加入垂直布局中
        self.verticalLayout.addLayout(self.horizontalLayout)
        # 布局初始化
        self.setCentralWidget(self.centralwidget)

        # 加载事件
        self.action_event()

        super().createUi(self)

    def tableView_set(self):
        # 添加表头：
        self.model = QtGui.QStandardItemModel(self.tableView)
        # 设置表头
        self.model.setHorizontalHeaderLabels([ '簇标记', '完整性', '源IP', '目的IP', '源端口', '目的端口',  '流长度', '流数据'])

        clustering_num = 0 # 用于记录簇标记
        row = 0            # 用于记录行数
        # color初始化
        color = QtGui.QColor(0, 0, 0)
        num = 0
        # 遍历簇类
        for clusters_data in self.Packet_clustering.items():
            len_cluster = len(clusters_data[1])
            # 遍历同簇流数据
            for i in range(0,len_cluster):   # range取头不取尾
                # 遍历流数据，tableview列数比clusters_data[i]长度多1，
                for j in range(1,8):
                    self.model.setItem(row, 0, QtGui.QStandardItem(str(clustering_num)))
                    # 设置字符颜色
                    self.model.item(row, 0).setForeground(QtGui.QBrush(color))
                    # 设置字符位置
                    self.model.item(row, 0).setTextAlignment(QtCore.Qt.AlignCenter)

                    self.model.setItem(row, j, QtGui.QStandardItem(clusters_data[1][i][j-1]))
                    # 设置字符颜色
                    self.model.item(row, j).setForeground(QtGui.QBrush(color))
                    # 设置字符位置
                    self.model.item(row, j).setTextAlignment(QtCore.Qt.AlignCenter)
                row += 1
            # 不同簇采用不同的颜色区分
            if num == 0:
                # 绿
                color = QtGui.QColor(50, 255, 50)
                num += 1
            elif num == 1:
                # 蓝
                color = QtGui.QColor(50, 50, 255)
                num += 1
            elif num == 2:
                # 红
                color = QtGui.QColor(255, 50, 50)
                num +=1
            else:
                # 黑
                color = QtGui.QColor(0, 0, 0)
                num = 0
            # 簇标记更新
            clustering_num += 1

        # 显示数据
        self.tableView.setModel(self.model)

    def action_event(self):
        # 设置表格点击事件
        self.tableView.clicked['QModelIndex'].connect(self.textBrowser_set)
        pass

    def textBrowser_set(self):
        indexs = self.tableView.currentIndex()
        text = self.model.item(indexs.row(), 7).text()
        self.textBrowser.clear()
        self.textBrowser2.clear()
        self.textBrowser.insertPlainText(text)
        self.textBrowser2.insertPlainText(text.encode(encoding='utf-8', errors='ignore').hex())


# coding=utf8
from django.conf import settings
from hashlib import md5
from pymogile import Client
from wand.image import Image

class Avatar(object):

    def __init__(self, key):
        self.key = key
        self.__origin_store_key = 'avatar/origin/' + self.__key
        self.__large_store_key = 'avatar/large/' + self.__key
        self.__small_store_key = 'avatar/small/' + self.__key

    def __ensure_datastore_client(self):
        if not hasattr(self, '__entity_obj'):
            self.__datastore = Client(
                domain = settings.MOGILEFS_DOMAIN,
                trackers = settings.MOGILEFS_TRACKERS
            )

    def __crop_square(self, data):
        _img = Image(blob = data)
        _delta = _img.width - _img.height
        if _delta > 0:
            _img.crop(_delta / 2 , 0, width = _img.height, height = _img.height)
        elif _delta < 0:
            _img.crop(0, 0, width = _img.width, height = _img.width)
        return _img.make_blob()

    def __resize(self, data, w, h):
        _img = Image(blob = data)
        _img.resize(w, h)
        return _img.make_blob()

    def get_hash_key(self):
        return self.__key

    @classmethod
    def create(cls, origin_data):
        _key = md5(origin_data).hexdigest()
        _inst = cls(_key)

        _inst.__ensure_datastore_client()
        if len(_inst.__datastore.get_paths(_inst.__origin_store_key)) == 0:
            _inst.write(origin_data)

        return _inst

    def read_large_link(self):
        return settings.IMAGE_SERVER + self.__large_store_key

    def read_small_link(self):
        return settings.IMAGE_SERVER + self.__small_store_key

    def write(self, origin_data):
        self.__ensure_datastore_client()
        _square_data = self.__crop_square(origin_data)

        #_fp = self.__datastore.new_file(self.__origin_store_key)
        #_fp.write(origin_data)
        #_fp.close()

        _large_data = self.__resize(_square_data, 180, 180)
        _fp = self.__datastore.new_file(self.__large_store_key)
        _fp.write(_large_data)
        _fp.close()

        _small_data = self.__resize(_square_data, 50, 50)
        _fp = self.__datastore.new_file(self.__small_store_key)
        _fp.write(_small_data)
        _fp.close()

    @staticmethod
    def read_image_data_by_store_key(store_key):
        _datastore = Client(
            domain = settings.MOGILEFS_DOMAIN,
            trackers = settings.MOGILEFS_TRACKERS
        )
        return _datastore.get_file_data(store_key)


"""
This library does all the complex network analysis required by the program
"""
# from typing import List, Any, Tuple, Dict, Union
import inspect
import csv
import networkx as nx
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import matplotlib.cm as cmx
import json
import random
from pycanvass.all import *
import pycanvass.global_variables as gv
import sys
import pycanvass.utilities as util
import time
import datetime
import os
from pathlib import Path

from networkx import Graph

attempts = 0
settings = ""

def _input_user_pref_file():
    util._banner()
    current_folder_path, current_folder_name = os.path.split(os.getcwd())
    wd = current_folder_path + "\\" + current_folder_name + "\\"
    print("[i] Current working folder: {}".format(wd))
    default_json_file_name = current_folder_path + "\\" + current_folder_name + "\\user_preferences.json"
    default_json_file_name = Path(default_json_file_name)

    if default_json_file_name.exists():
        print("[i] Loaded default user preferences file: {}".format(default_json_file_name))
        # time.sleep(0.5)
        user_preference_path = default_json_file_name
    else:
        print("[x] Default user preferences file not found.")
        user_preference_path = input("\n[i] Please enter the path to your user-preferences file (*.JSON):\n")

    return str(user_preference_path)

while attempts < 4:
    if attempts == 3:
        break
    try:
        user_preference_path = _input_user_pref_file()
        gv.filepaths["user_preferences"] = user_preference_path
        u_settings = open(user_preference_path)
        settings = json.load(u_settings)
        u_settings.close()
        break
    except:
        print("[x] Could not get the user preference file.")
        attempts += 1
        if attempts == 3:
            print("[x] Failed to configure user preferences correctly.")
            sys.exit()
        elif attempts == 2:
            print("[i] One more chance to get the user preference correctly setup.")

# ---------------------------

# # #
# A #
# # #

def add_node_attr(graph):
    """
    Add all the details about the nodes here
    :param graph:
    :return: dictionary containing node features
    """
    all_nodes = graph.nodes()
    n_file = gv.filepaths["nodes"]

    for n in all_nodes:
        node_search_result = _node_search(n)
        try:
            with open(n_file, 'r') as f:
                csvr = csv.reader(f)
                csvr = list(csvr)
                for row in csvr:
                    if row[0].lstrip() == n:
                        if node_search_result is not 0:
                            graph.node[n]['type'] = row[9].lstrip()
                            graph.node[n]['gen'] = row[6].lstrip()
                            graph.node[n]['demand'] = row[5].lstrip()
                            graph.node[n]['lat'] = row[2].lstrip()
                            graph.node[n]['long'] = row[3].lstrip()

        except:
            print("[x] Error in setting up the node attributes. ")
            called_from = inspect.stack()[1]
            called_module = inspect.getmodule(called_from[0])
            logging.error("[x] Called from - {} : Node attributes could not be set.".format(called_module, e))

# # #
# N #
# # #

def _node_search(n):
    n_file = gv.filepaths["nodes"]
    with open(n_file) as f:
        counter = 0
        match_flag = 0
        nodes = csv.reader(f)
        for node in nodes:
            counter += 1
            if node[0].lstrip() == n:
                match_flag = 1
                return counter - 1

        if match_flag == 0:
            called_from = inspect.stack()[1]
            called_module = inspect.getmodule(called_from[0])
            logging.error("[x] Called from - {} : Node {} could not be found.".format(called_module, e))
            return 0

# # #
# B #
# # #

def build_network(node_dict, edge_dict):
    """

    :param node_dict:
    :param edge_dict:
    :return: network_dict: Dictionary containing several network graph objects, and edge details
             network_dict["normal"] : Normal network topology
             network_dict["total"] : Graph containing self-loops, including reconfiguration options
             network_dict["n_open"]: List of all edges which are normally open
             network_dict["n_closed"] : List of all edges which are normally closed
    """

    network_dict = {}

    graph = nx.Graph()
    total_graph = nx.DiGraph()

    gv.obj_nodes = []
    gv.obj_edges = []

    for key, value in node_dict.items():
        gv.obj_nodes.append(value)

    for key, value in edge_dict.items():
        gv.obj_edges.append(value)

    for edge in gv.obj_edges:
        total_graph.add_edge(edge.from_node.lstrip(), edge.to_node.lstrip())
        if edge.availability.lstrip() != "0":
            graph.add_edge(edge.from_node.lstrip(), edge.to_node.lstrip())
            gv.closed_edges_list.append(tuple((edge.from_node.lstrip(), edge.to_node.lstrip())))
        else:
            gv.open_edges_list.append(tuple((edge.from_node.lstrip(), edge.to_node.lstrip())))

    logging.info("Populating nodes and edges with values")
    # print(graph.nodes())
    for n in gv.obj_nodes:
        graph.node[n.name]['name'] = n.name
        graph.node[n.name]['load'] = n.load
        graph.node[n.name]['gen'] = n.gen
        graph.node[n.name]['lat'] = n.lat
        graph.node[n.name]['long'] = n.long
        graph.node[n.name]['kind'] = n.kind
        graph.node[n.name]['critical'] = n.critical

        total_graph.node[n.name]['name'] = n.name
        total_graph.node[n.name]['load'] = n.load
        total_graph.node[n.name]['gen'] = n.gen
        total_graph.node[n.name]['lat'] = n.lat
        total_graph.node[n.name]['long'] = n.long
        total_graph.node[n.name]['kind'] = n.kind
        total_graph.node[n.name]['critical'] = n.critical

        try:
            if n.kind.upper() == "PV" or n.kind.upper() == "SWING":
                gv.all_sources[n.name] = n.gen
            else:
                try:
                    gv.all_loads[n.name] = n.load
                    if int(n.critical) > 5:
                        gv.all_critical_loads[n.name] = n.load
                except ValueError:
                    logging.error("No loads found in the network while making the graph.")
        except ValueError:
            logging.error("No power resource found in the network while making the graph.")

    gv.graph_collection.append(graph)
    gv.graph_collection.append(total_graph)

    for g in gv.graph_collection:
        for e in gv.obj_edges:
            try:
                g[e.from_node.lstrip()][e.to_node.lstrip()]['water_risk'] = e.water_risk
                gv.water_risk_values.append(e.water_risk.lstrip())
            except KeyError:
                logging.info("No edge between %s and %s, so no water risk" % (e.from_node.lstrip(), e.to_node.lstrip()))
                gv.water_risk_values.append(0)

            try:
                g[e.from_node.lstrip()][e.to_node.lstrip()]['wind_risk'] = e.wind_risk
                gv.wind_risk_values.append(e.wind_risk.lstrip())
            except KeyError:
                logging.info(
                    "[i] No edge between %s and %s, so no wind risk " % (e.from_node.lstrip(), e.to_node.lstrip()))
                gv.wind_risk_values.append(0)

            try:
                g[e.from_node.lstrip()][e.to_node.lstrip()]['fire_risk'] = e.fire_risk
                gv.fire_risk_values.append(e.fire_risk.lstrip())
            except KeyError:
                logging.info(
                    "[i] No edge between %s and %s, so no fire risk " % (e.from_node.lstrip(), e.to_node.lstrip()))
                gv.fire_risk_values.append(0)

    network_dict["normal"] = graph
    network_dict["total"] = total_graph
    network_dict["n_open"] = gv.open_edges_list
    network_dict["n_closed"] = gv.closed_edges_list

    return network_dict

def build_network_2(node_dict, edge_dict, timestring):
    """

    :param node_dict:
    :param edge_dict:
    :return: network_dict: Dictionary containing several network graph objects, and edge details
             network_dict["normal"] : Normal network topology
             network_dict["total"] : Graph containing self-loops, including reconfiguration options
             network_dict["n_open"]: List of all edges which are normally open
             network_dict["n_closed"] : List of all edges which are normally closed
    """

    network_dict = {}

    graph = nx.Graph()
    total_graph = nx.DiGraph()

    gv.obj_nodes = []
    gv.obj_edges = []

    for key, value in node_dict.items():
        gv.obj_nodes.append(value)

    for key, value in edge_dict.items():
        gv.obj_edges.append(value)

    for edge in gv.obj_edges:
        total_graph.add_edge(edge.from_node.lstrip(), edge.to_node.lstrip())
        if edge.status.lstrip() != "0" and edge.availability.lstrip() != "0":
            graph.add_edge(edge.from_node.lstrip(), edge.to_node.lstrip())
            gv.closed_edges_list.append(tuple((edge.from_node.lstrip(), edge.to_node.lstrip())))
        else:
            gv.open_edges_list.append(tuple((edge.from_node.lstrip(), edge.to_node.lstrip())))

    logging.info("Building graph for time-stamp {}".format(timestring))
    # print(graph.nodes())

    for n in gv.obj_nodes:
        try:
            graph.node[n.name]['name'] = n.name
        except KeyError:
            print("[!] Encountered a disconnected load {}".format(n.name))
            continue
        graph.node[n.name]['load'] = n.load
        graph.node[n.name]['gen'] = n.gen
        graph.node[n.name]['lat'] = n.lat
        graph.node[n.name]['long'] = n.long
        graph.node[n.name]['kind'] = n.kind
        graph.node[n.name]['critical'] = n.critical

        total_graph.node[n.name]['name'] = n.name
        total_graph.node[n.name]['load'] = n.load
        total_graph.node[n.name]['gen'] = n.gen
        total_graph.node[n.name]['lat'] = n.lat
        total_graph.node[n.name]['long'] = n.long
        total_graph.node[n.name]['kind'] = n.kind
        total_graph.node[n.name]['critical'] = n.critical

        try:
            if n.kind.upper() == "PV" or n.kind.upper() == "SWING":
                gv.all_sources[n.name] = n.gen
            else:
                try:
                    gv.all_loads[n.name] = n.load
                    if int(n.critical) > 5:
                        gv.all_critical_loads[n.name] = n.load
                except ValueError:
                    logging.error("No loads found in the network while making the graph.")
        except ValueError:
            logging.error("No power resource found in the network while making the graph.")

    gv.graph_collection.append(graph)
    gv.graph_collection.append(total_graph)

    for g in gv.graph_collection:
        for e in gv.obj_edges:
            try:
                g[e.from_node.lstrip()][e.to_node.lstrip()]['water_risk'] = e.water_risk
                gv.water_risk_values.append(e.water_risk.lstrip())
            except KeyError:
                logging.info("No edge between %s and %s, so no water risk" % (e.from_node.lstrip(), e.to_node.lstrip()))
                gv.water_risk_values.append(0)

            try:
                g[e.from_node.lstrip()][e.to_node.lstrip()]['wind_risk'] = e.wind_risk
                gv.wind_risk_values.append(e.wind_risk.lstrip())
            except KeyError:
                logging.info(
                    "[i] No edge between %s and %s, so no wind risk " % (e.from_node.lstrip(), e.to_node.lstrip()))
                gv.wind_risk_values.append(0)

            try:
                g[e.from_node.lstrip()][e.to_node.lstrip()]['fire_risk'] = e.fire_risk
                gv.fire_risk_values.append(e.fire_risk.lstrip())
            except KeyError:
                logging.info(
                    "[i] No edge between %s and %s, so no fire risk " % (e.from_node.lstrip(), e.to_node.lstrip()))
                gv.fire_risk_values.append(0)

    network_dict["normal"] = graph
    network_dict["total"] = total_graph
    network_dict["n_open"] = gv.open_edges_list
    network_dict["n_closed"] = gv.closed_edges_list

    return network_dict

# # #
# C #
# # #

def _create_pos_dictionary(graph):
    """
    Internal function, that creates a dictionary that contains position of all nodes in the network
    :param graph:
    :return: dictionary of all nodes with their geo-coordinates,
             dictionary with little shifted position for showing labels
    """
    nodes = graph.nodes()  # list of all nodes <-- keys
    pos_dictionary = {}  # initializing a dictionary to hold the positions
    label_pos_dictionary = {}
    for n in nodes:
        pos_dictionary[n] = tuple((float(graph.node[n]['long']), float(graph.node[n]['lat'])))
        label_pos_dictionary[n] = tuple((float(graph.node[n]['lat']), float(graph.node[n]['long'])))

    return pos_dictionary, label_pos_dictionary

# # #
# F #
# # #

def feeder_path(graph, from_node, to_node):
    """
    :param graph: NetworkX graph object
    :param from_node:
    :param to_node:
    :return: all restoration paths from generator to load nodes
    """
    path_graph = nx.Graph()
    paths = nx.all_simple_paths(graph, from_node, to_node)

    return paths

# # #
# L #
# # #

def lat_long_layout(graph, show=False, save=True, title="Visualization"):
    """
    Help visualize the network graph and some of its resilience properties.
    If your graph does not come with
    :param graph:
    :param show: (Default is False)
    :param save: (Default is True)
    :return:
    """
    focus = "impact_on_edge"
    logging.info(
        "Lat_long_layout function used, focus = {}, trying to display = {}, saving = {}".format(focus, show, save))

    try:
        pos, label_pos = _create_pos_dictionary(graph)
    except ValueError:
        print("[x] Error: Could not create determine where to put the nodes and edges. "
              "... Please check the lat, longs of your nodes. "
              ""
              "... In case your file does not have the lat, longs, generate"
              "... random lat-longs by using the layout_model(<model.glm>, map_random=True) function."
              ""
              "... Please read the documentation on how to do that. ")

        logging.error('Could not create the dictionary for pos, or label_pos')

    color_scheme = settings['visualization']['color_scheme']

    values = []

    for e in graph.edges():
        try:
            values.append(graph[str(e[0])][str(e[1])][focus])
        except:
            values.append(1)

    color_scheme = plt.get_cmap(color_scheme)
    c_norm = colors.Normalize(vmin=0, vmax=7)
    scalar_map = cmx.ScalarMappable(norm=c_norm, cmap=color_scheme)
    color_list = []
    edge_labels = nx.get_edge_attributes(graph, focus)

    for i in range(len(values)):
        color_val = scalar_map.to_rgba(float(values[i]))
        color_list.append(color_val)

    # add a connection to a web visualization

    nx.draw_networkx_edges(graph,
                           pos=pos,
                           edge_color=color_list,
                           # style='dashed',
                           width=2.0)

    nx.draw_networkx_nodes(graph,
                           pos=pos,
                           nodelist=graph.nodes(),
                           node_color='b',
                           node_shape='x')

    nx.draw_networkx_labels(graph,
                            pos=pos,
                            edge_labels=edge_labels)

    if show:
        plt.title(title)
        plt.show()

    if save:
        ts = time.time()
        st = datetime.datetime.fromtimestamp(ts).strftime('%Y_%m_%d_%H_%M_%S')
        filename = "visualization_" + st + ".png"
        plt.savefig(filename)

def loads(critical=False):
    """
    This function lists the loads in the power system network.
    :param critical: Default value is set to False, which means all loads will be displayed in the network.
                     Set this value to True to see only the critical loads
    :return: gv.all_loads or gv.all_critical_loads
    """
    if not critical:
        return gv.all_loads
    else:
        return gv.all_critical_loads

# # #
# M #
# # #

def make_graph(all_edges):
    """

    :param all_edges: List of all instances of Edge objects created by the program
    :return: graph: NetworkX graph object,
             normally_open_edges: number of open edges, that can be closed later if necessary
    """
    graph = nx.Graph()
    total_graph = nx.Graph()

    for edge in all_edges:
        total_graph.add_edge(edge.from_node, edge.to_node)
        if edge.status != "0":
            graph.add_edge(edge.from_node, edge.to_node)
            gv.closed_edges_list.append(tuple((edge.from_node, edge.to_node)))
        else:
            gv.open_edges_list.append(tuple((edge.from_node, edge.to_node)))

    return graph, total_graph, gv.open_edges_list, gv.closed_edges_list

def make_path_graph(nodelist, **kwargs):
    """
    Create a graph based on a found path between a source and a generator
    :param nodelist:
    :return: path_graph: A NetworkX object that is a single path's graph, that can be further used for resiliency
                         analysis

    """

    path_graph = nx.Graph()
    i = 0
    while i < len(nodelist) - 1:
        path_graph.add_edge(nodelist[i], nodelist[i + 1])
        i += 1

    return path_graph

# # #
# P #
# # #

def preprocess(graph, node_dict, edge_dict):
    """

    :param graph:
    :param node_dict:
    :param edge_dict:
    :return:
    """

    gv.obj_nodes = []
    gv.obj_edges = []
    for key, value in node_dict.items():
        gv.obj_nodes.append(value)

    for key, value in edge_dict.items():
        gv.obj_edges.append(value)

    print("[i] Attempting to populate nodes and edges with values")

    for n in gv.obj_nodes:
        graph.node[n.name]['name'] = n.name
        graph.node[n.name]['load'] = n.load
        graph.node[n.name]['gen'] = n.gen
        graph.node[n.name]['lat'] = n.lat
        graph.node[n.name]['long'] = n.long
        graph.node[n.name]['kind'] = n.kind
        graph.node[n.name]['critical'] = n.critical

        try:
            if n.kind.upper() == "PV" or n.kind.upper() == "SWING":
                gv.all_sources[n.name] = n.gen
            else:
                try:
                    gv.all_loads[n.name] = n.load
                    if int(n.critical) > 5:
                        gv.all_critical_loads[n.name] = n.load
                except ValueError:
                    logging.error("No loads found in the network, while making the graph.")
        except ValueError:
            logging.error("No power resource found in the network, while making the graph.")

    for e in gv.obj_edges:
        try:
            graph[e.from_node][e.to_node]['water_risk'] = e.water_risk
            gv.water_risk_values.append(e.water_risk)
        except KeyError:
            print("[i] No edge between %s and %s exist in this topology." % (e.from_node, e.to_node))
            gv.water_risk_values.append(0)

        try:
            graph[e.from_node][e.to_node]['wind_risk'] = e.wind_risk
            gv.water_risk_values.append(e.wind_risk)
        except KeyError:
            print("[i] No edge between %s and %s exist in this topology." % (e.from_node, e.to_node))
            gv.water_risk_values.append(0)

        try:
            graph[e.from_node][e.to_node]['fire_risk'] = e.fire_risk
            gv.water_risk_values.append(e.fire_risk)
        except KeyError:
            print("[i] No edge between %s and %s exist in this topology." % (e.from_node, e.to_node))
            gv.water_risk_values.append(0)

# # #
# R #
# # #

def resiliency_downstream(graph, edgelist, event):
    """
    This function computes the resiliency of all downstream sections,
    mainly looking out for critical loads
    :param graph:
    :param edgelist: All downstream sections of the feeder, from perspective of an edge being analyzed
    :param event: dictionary describing the event magnitude
    :return:
    """

    risk_dict = {}

    for e in edgelist:
        for oe in gv.obj_edges:
            if e[0].lstrip() == oe.from_node.lstrip() and e[1].lstrip() == oe.to_node.lstrip():
                risk_dict[tuple((e[0], e[1]))] = (event["water_risk"] * float(oe.water_risk.lstrip())
                                                  + event["wind_risk"] * float(oe.wind_risk.lstrip())
                                                  + event["fire_risk"] * float(oe.fire_risk.lstrip())) / 30

    print("Risk per edge -->")
    print(risk_dict)

    # loads encountered

    # critical loads encountered

    # generators encountered

    # switches encountered

    pass

def resiliency_upstream(graph, edgelist, event):
    """
    This function computes the resiliency of all upstream sections,
    mainly looking out for generators
    :param edgelist: All upstream sections of the feeder, from persective of a feeder being analyzed
    :return:
    """
    print("Heat your ", edgelist)
    pass

def resiliency(analysis='nodal'):
    """
    This function helps compute the resiliency metric of the network of a
    node, or a network
    :param kwargs:
    :return:
    """

    # 0. what's the event?

    event = gv.event

    # 1. get the graphs

    g1 = gv.graph_collection[0]
    g2 = gv.graph_collection[1]
    nodes = g2.nodes()
    edges = g2.edges()

    # 2. for which node and edge are you doing the analysis?
    wg = nx.DiGraph()  # wg: working graph
    wg.add_edges_from(edges)

    for n in nodes:
        # Finding Downstream Edges
        print("Downstream Edges of %s -->" % n)
        downstream_edges = list(nx.dfs_edges(wg, n))
        resiliency_downstream(g2, downstream_edges, event)

        # Finding Upstream Edges
        print("Upstream Edges of %s -->" % n)
        upstream_edges = list(nx.edge_dfs(wg, n, orientation='reverse'))
        resiliency_upstream(g2, upstream_edges, event)

    # 3. track back from that edge to a source, or multiple sources

# # #
# S #
# # #

def sources():
    """
    This is a querying function that lists the power resources in a network
    :return: list of all power sources within a network
    """
    return gv.all_sources

def simple_show(graph):
    """
    Display the simple network representation of the graph
    :param graph: NetworkX graph object
    :return:
    """
    nx.draw(graph)
    plt.show()

def summary(graph):
    """
    Summarize all the important graph topological properties
    :param graph:
    :return:
    """
    temp_dict = {'degree_centrality': nx.degree_centrality(graph),
                 'betweeness': nx.betweenness_centrality(graph),
                 'resistance_centrality': nx.current_flow_closeness_centrality(graph)}

    return temp_dict

# # #
# V #
# # #


"""MATH96012 2019 Project 1
Alexander John Pinches
CID: 01201653
"""
import numpy as np
import matplotlib.pyplot as plt
import scipy.spatial.distance as ds
import scipy.optimize as op
#--------------------------------
plt.style.use('ggplot') # plot style
plt.rcParams["figure.figsize"] = [8,8] # figure size

def simulate1(N=64,L=8,s0=0.2,r0=1,A=0.2,Nt=100):
    """Part1: Simulate bacterial colony dynamics
    Input:
    N: number of particles
    L: length of side of square domain
    s0: speed of particles
    r0: particles within distance r0 of particle i influence direction of motion
    of particle i
    A: amplitude of noise
    dt: time step
    Nt: number of time steps

    Output:
    X,Y: position of all N particles at Nt+1 times
    alpha: alignment parameter at Nt+1 times

    Do not modify input or return statement without instructor's permission.

    Add brief description of approach to problem here:

    First I tought about what can we caculate and store such as R and index later.
    Then I thought about what it was possible to vectorise over and looked in scipy
    for a way to calculate the distances in compiled c code. There's no way
    at least I could think of to vecorise over t as when applyng the limits this affects everything
    downstream so we have to do it in a for loop. As we know the size of all the outputs
    we create empty arrays to store them. Wedont have to store all the thetas each time we could have two
    variables and use .copy() but assignment is faster than copying but this would use less memory.
    Much memory isn't used though.

    """
    #Set initial condition
    phi_init = np.random.rand(N)*(2*np.pi)
    r_init = np.sqrt(np.random.rand(N))
    Xinit,Yinit = r_init*np.cos(phi_init),r_init*np.sin(phi_init)
    Xinit+=L/2
    Yinit+=L/2

    theta = np.column_stack((np.random.rand(N)*(2*np.pi),np.zeros((N,Nt)))) #initial directions of motion and 0s for future values
    #---------------------
    X = np.column_stack((Xinit,np.zeros((N,Nt)))) # create array to store X with initial X
    Y = np.column_stack((Yinit,np.zeros((N,Nt)))) # create array to store Y with initial Y

    R = np.random.rand(N,Nt)*(2*np.pi) # calculate all random numbers

    alpha = np.zeros(Nt+1) # empty array to store alpha
    alpha[0] = np.abs(np.sum(np.exp(1j*theta[:,0])))/N # put initial alpha in a list
    for t in range(0,Nt): # over t
        distances = ds.squareform(ds.pdist(np.stack((X[:,t],Y[:,t]),axis=-1))) <= r0 # get array of points within r0 of each point

        # calculate theta
        theta[:,t+1] = np.angle(np.dot(np.exp(1j*theta[:,t]),distances)+A*np.exp(1j*R[:,t])*np.sum(distances,axis=0))

        # calculate X and Y and apply bounds
        X[:,t+1] = X[:,t] + s0*np.cos(theta[:,t+1])
        X[X[:,t+1]>L,t+1] = X[X[:,t+1]>L,t+1] - L
        X[X[:,t+1]<0,t+1] = L + X[X[:,t+1]<0,t+1]

        Y[:,t+1] = Y[:,t] + s0*np.sin(theta[:,t+1])
        Y[Y[:,t+1]>L,t+1] = Y[Y[:,t+1]>L,t+1] - L
        Y[Y[:,t+1]<0,t+1] = L + Y[Y[:,t+1]<0,t+1]

        # add alpha value at t+1
        alpha[t+1] = np.abs(np.sum(np.exp(1j*theta[:,t+1])))/N

    return X,Y,alpha

def analyze(A,Nt,diffTest,display=False,**kwargs):
    """Part 2: Add input variables and modify return as needed

    **kwargs passed to simulate2
    """
    out = {} # output dict

    fig = plt.figure()
    # subplots
    ax1 = fig.add_subplot(211)
    ax2 = fig.add_subplot(212)
    # x
    x = np.linspace(0,Nt+1,Nt+1)
    # arrays to store results
    alphaMean16 = np.zeros(len(A))
    alphaMean32 = np.zeros(len(A))
    alphaVar16 = np.zeros(len(A))
    alphaVar32 = np.zeros(len(A))
    for i in range(0,len(A)):
        # calculate alphas
        alpha16=simulate2(A=A[i],Nt=Nt,N=16,**kwargs)[2]
        alpha32=simulate2(A=A[i],Nt=Nt,N=32,**kwargs)[2]
        # add to subplots
        ax1.plot(alpha16)
        ax2.plot(alpha32)
        # store var and mean
        alphaVar16[i] = np.var(alpha16[50:])
        alphaVar32[i] = np.var(alpha32[50:])
        alphaMean16[i] = np.mean(alpha16[50:])
        alphaMean32[i] = np.mean(alpha32[50:])
    # set titles and labels
    ax1.set_title("N=16")
    ax2.set_title("N=32")
    ax1.set_ylabel(r'$\alpha$')
    ax2.set_ylabel(r'$\alpha$')
    ax2.set_xlabel('t')

    # add to output dict
    out["alphaMean16"] = alphaMean16
    out["alphaMean32"] = alphaMean32
    out["alphaVar16"] = alphaVar16
    out["alphaVar32"] = alphaVar32

    # add legend
    fig.legend(A)
    plt.savefig("alphaTime.png") # save

    # new figure
    plt.figure()
    # plot series
    plt.plot(A,alphaMean16,color="blue",label="N=16")
    plt.plot(A,alphaMean16 + 2*np.sqrt(alphaVar16),color="blue",linestyle="dashed",label="_")
    plt.plot(A,alphaMean16 - 2*np.sqrt(alphaVar16),color="blue",linestyle="dashed",label="_")
    plt.plot(A,alphaMean32,color="red",label="N=32")
    plt.plot(A,alphaMean32 + 2*np.sqrt(alphaVar32),color="red",linestyle="dashed",label="_")
    plt.plot(A,alphaMean32 - 2*np.sqrt(alphaVar32),color="red",linestyle="dashed",label="_")
    # legend
    plt.legend()
    # title and labels
    plt.title(r"$\alpha$ Vs A")
    plt.xlabel("A")
    plt.ylabel(r"$\alpha$")
    plt.savefig("alphaMean.png") # save

    # new figure
    plt.figure()
    # plot
    plt.plot(A,alphaVar16)
    plt.plot(A,alphaVar32)
    # legend
    plt.legend(["N=16","N=32"])
    # title and labels
    plt.title(r"$\mathrm{Var}[\alpha$] Vs A")
    plt.xlabel("A")
    plt.ylabel(r"$\mathrm{\alpha}$")
    plt.savefig("alphaVar.png") # save

    # results array
    y = np.zeros((len(diffTest),Nt-49))
    for i in range(0,len(diffTest)): # get alpha
        y[i,:] = simulate2(A=diffTest[i],L=4,N=32,Nt=1000)[2][50:]
    y = np.var(y,axis=1) # calculate variance

    Astar = diffTest[y==y.max()] # get the max

    out["Astar"] = Astar # add to output dictionary

    # new figure
    plt.figure()
    # plot
    plt.plot(diffTest,y)
    plt.plot(Astar,y.max(),marker="o")
    # add title and labels
    plt.title(r"$\mathrm{Var}[\alpha]$ Vs A")
    plt.xlabel("A")
    plt.ylabel(r"$\mathrm{Var}[\alpha]$")
    plt.savefig("rate.png") #save

    # dependence
    A2 = np.linspace(0.2,Astar-0.025)
    x = 1 - A2/Astar
    alphaMean = np.zeros(len(A2))
    alphaVar = np.zeros(len(A2))

    for i in range(0,len(A2)):
        alpha = simulate2(A=A2[i],Nt=Nt,N=32,**kwargs)[2][50:]
        alphaMean[i] = np.mean(alpha)
        alphaVar[i] = np.var(alpha)

    # new fig
    plt.figure()
    # plot
    plt.plot(x,alphaMean,color="red")
    plt.plot(x,alphaMean + 2*np.sqrt(alphaVar),color="blue",linestyle="dashed")
    plt.plot(x,alphaMean - 2*np.sqrt(alphaVar),color="blue",linestyle="dashed")
    # title and labels
    plt.title(r"$\alpha$ Vs $1 - \frac{A}{A^{*}}$")
    plt.xlabel(r"$1 - \frac{A}{A^{*}}$")
    plt.ylabel(r"$\alpha$")
    plt.savefig("alphaDependence.png") # save

    # new fig
    plt.figure()
    # plot
    plt.plot(x,alphaVar)
    # title and labels
    plt.title(r"$\mathrm{Var}[\alpha]$ Vs $1 - \frac{A}{A^{*}}$")
    plt.xlabel(r"$1 - \frac{A}{A^{*}}$")
    plt.ylabel(r"$\mathrm{Var}[\alpha$]")
    plt.savefig("alphaDependenceVar.png")

    # add to out dict
    out["1-A/Astar"] = x
    out["AstarMean"] = alphaMean
    out["AstarVar"] = alphaVar

    if display: # display
        plt.show()

    return out

def simulate2(A,N,Nt,L=4,**kwargs):
    """Part 2: Simulation code for Part 2, add input variables and simulation
    code needed for simulations required by analyze
    """

    return simulate1(A=A,N=N,L=L,Nt=Nt,**kwargs)

#--------------------------------------------------------------
if __name__ == '__main__':
    #The code here should call analyze and
    #generate the figures that you are submitting with your
    #discussion.

    output_a = analyze([0.2,0.4,0.6,0.8],1000,np.linspace(0.2,0.8,1201))


from itertools import chain
import sys

A, B, K = map(int, input().split())
diff = B - A

if diff == 0:
    print(A)
    sys.exit()

if K > diff:
    K = diff

ans = [x for x in chain(range(A, A + K), range(B - K + 1, B + 1))]
ans = sorted(list(set(ans)))

print('\n'.join(map(str, ans)))


import pyodbc
from airflow import DAG
from airflow.operators.python import PythonOperator
from airflow.utils.dates import days_ago

our_dag = DAG(
    "RefreshAnalysisServer",
    description='Обновление модели Analysis Server - Monopoly',
    start_date=days_ago(0, 0, 0, 0, 0),
    tags=['python'],
)

conn_str = (
    'Driver=/opt/microsoft/msodbcsql17/lib64/libmsodbcsql-17.10.so.2.1;'
    'Server=10.22.5.129;'
    'Database=ReportServer;'
    'Trusted_Connection=yes;'
    )

def refresh_model():
        cnxn = pyodbc.connect(conn_str)
        cursor = cnxn.cursor()
        cursor.execute("SELECT MachineName FROM Keys")
        while 1:
            row = cursor.fetchone()
            if not row:
                break
            print(row.MachineName)
        cnxn.close()

refresh = PythonOperator(
    task_id='refresh_model',
    python_callable=refresh_model,
    dag=our_dag
)

# operation

refresh


import re

import logging
_log = logging.getLogger("PingRelay")

class MessageFormatter():
    def __init__(self, message_format, alert_user="@everyone", time_format = "%Y-%m-%d %I:%M:%S %p"):
        self.format = message_format
        self.time_format = time_format
        self.alert_user = alert_user

    def format_message(self, message):
        regex = re.compile(r"%{(\w+)}")
        self.message = message
        result = regex.sub(self.fill_placeholder, self.format)
        _log.debug("Formatted with {0} resulted in {1}".format(self.format, result))
        return result

    def fill_placeholder(self, match):
        placeholder = match.group(1).lower()
        if (placeholder == "server"):
            return self.message.server
        if (placeholder == "channel"):
            return self.message.channel
        if(placeholder == "from"):
            return self.message.sender
        if(placeholder == "message"):
            return self.message.message
        if(placeholder == "alerts"):
            if self.message.has_alert:
                return self.alert_user + ": this message matched alert(s): " + self.message.get_alert_str() + "\n"
            else:
                return ""
        if(placeholder == "time"):
            return self.message.time.strftime(self.time_format)
        return match.group()


import argparse
from pathlib import Path

import numpy as np
from matplotlib import pyplot as plt
from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix, f1_score

from models.hmm.facial_movement import ordinal_from_csv
from models.hmm.prediction_visualisation import load_predictions
from models.hmm.test_hmm import predict_hmm
from models.hmm.train_hmm import fit_hmms
from models.simple.memory_detector import MemoryBasedShakeDetector
from models.simple.random_detector import RandomShakeDetector
from models.simple.rule_detector import RuleBasedShakeDetector, majority_vote
from utils.draw import set_seaborn_theme
from utils.frames_csv import get_splits, load_df, load_all_labels
from utils.io import mkdir_if_not_exists
from validation.event_based import evaluate_events
from models.hmm.filters import majority_filter

def hmm_filter_search(frames_csv,
                      folds_dir,
                      results_dir,
                      method='event',
                      metric='f1'):
    # sizes = [3, 7, 11, 15, 19, 23, 27, 31, 35, 39]
    # sizes = [11, 21, 31, 41, 51, 61, 71, 81, 91, 101]
    sizes = [11, 51, 101, 151, 201]
    results = []

    for size in sizes:
        results.append(validate_hmm_folds(frames_csv,
                                          folds_dir,
                                          filter_size=size,
                                          window_size=36,
                                          method=method,
                                          metric=metric,
                                          results_dir=results_dir,
                                          load_previous=True))

    set_seaborn_theme()
    plt.title('F1 score for head-shake detector on different filter sizes', fontsize=15)
    plt.locator_params(axis='x', nbins=10)
    plt.plot(sizes, results, 'o-')
    plt.xticks(sizes, sizes)
    plt.xlabel('Filter size (frames)')
    plt.ylabel('F1 score')
    plt.show()

def hmm_window_search(frames_csv,
                      folds_dir,
                      method='frame',
                      metric='f1'):
    sizes = [12, 24, 36, 48, 60, 72, 84, 96, 108, 120]
    labels = [0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5]
    results = []

    for size in sizes:
        results.append(validate_hmm_folds(frames_csv, folds_dir, window_size=size, method=method, metric=metric))

    set_seaborn_theme()
    plt.title('F1 score for head-shake detector on different window sizes', fontsize=15)
    plt.locator_params(axis='x', nbins=10)
    plt.plot(sizes, results, 'o-')
    plt.xticks(sizes, labels)
    plt.xlabel('Window size (seconds)')
    plt.ylabel('F1 score')
    plt.show()

def validate_hmm_folds(frames_csv,
                       folds_dir,
                       window_size=36,
                       results_dir=None,
                       filter_size=None,
                       method='frame',
                       load_previous=False,
                       metric=None):
    print(f'Loading samples from {frames_csv}')
    df_frames = load_df(frames_csv)
    splits = get_splits(df_frames)
    all_predictions = []
    label_index = 0

    if metric == 'f1':
        all_labels = []

    if load_previous:
        all_predictions = load_predictions(results_dir/'predictions.npz')

    matrix = np.zeros((2, 2))

    for fold in splits:
        print(f'Validating {fold}/{len(splits)}')
        df_val = df_frames[df_frames['split'] == fold]

        if load_previous:
            labels = load_all_labels(df_val, shift=1, window=window_size)
            predictions = all_predictions[label_index:label_index+len(df_val)]
            label_index += len(df_val)
        else:
            labels, predictions = predict_hmm(df_val, folds_dir / fold, window_size=window_size)

        if filter_size:
            predictions = [majority_filter(prediction, filter_size) for prediction in predictions]

        if method == 'frame':
            matrix = matrix + confusion_matrix(np.concatenate(labels), np.concatenate(predictions), labels=[0, 1])
        elif method == 'event':
            matrix = matrix + evaluate_events(np.concatenate(labels), np.concatenate(predictions))
        else:
            raise ValueError('Invalid method!')

        if (results_dir and not load_previous) or (metric == 'f1'):
            all_predictions.extend(predictions)
        if metric == 'f1':
            all_labels.extend(labels)

    if results_dir and not load_previous:
        mkdir_if_not_exists(results_dir)
        np.savez(results_dir/'predictions.npz', *all_predictions)

    if metric == 'f1':
        if method == 'frame':
            return f1_score(np.concatenate(all_labels), np.concatenate(all_predictions))
        if method == 'event':
            tn, fp, fn, tp = evaluate_events(np.concatenate(all_labels), np.concatenate(all_predictions)).ravel()
            return tp / (tp + (0.5 * (fp + fn)))

    plot_confusion_matrix(matrix, title='Event-level confusion matrix for head-shake detection')

def fit_hmm_folds(frames_csv, folds_dir):
    df_frames = load_df(frames_csv)
    splits = get_splits(df_frames)

    for fold in splits:
        mkdir_if_not_exists(folds_dir / fold)

        df_val = df_frames[df_frames['split'] == fold]
        df_train = df_frames[df_frames['split'].str.contains('fold')].drop(df_val.index)

        fit_hmms(df_train, folds_dir / fold)

def cross_validate_random_preload(frames_csv, window_size=48, movement_threshold=0.5):
    df_frames = load_df(frames_csv)
    vectors = ordinal_from_csv(df_frames, threshold=movement_threshold)

    matrix = cross_validate(
        frames_csv,
        data=vectors,
        model_class=RandomShakeDetector,
        params={
            'window_size': window_size,
            'movement_threshold': movement_threshold
        }
    )
    plot_confusion_matrix(matrix,
                          title='Confusion matrix for random classifier')

def cross_validate_rule_preload(frames_csv, window_size=48, movement_threshold=0.5):
    df_frames = load_df(frames_csv)
    vectors = ordinal_from_csv(df_frames, threshold=movement_threshold)

    matrix = cross_validate(
        frames_csv,
        data=vectors,
        model_class=RuleBasedShakeDetector,
        params={
            'window_size': window_size,
            'movement_threshold': movement_threshold,
            'rule_func': majority_vote
        }
    )
    plot_confusion_matrix(matrix,
                          title=f'Confusion matrix for rule-based classifier with threshold={movement_threshold}')

def cross_validate_memory_preload(frames_csv, window_size=48, movement_threshold=0.5):
    """Cross validation wrapper specifically designed for the memory based classifier using HMM vectors.

    :param frames_csv: Dataframe with frame annotations, labels and any other information used by your model
    :param window_size: Window size to use during prediction of new sequences
    :param movement_threshold: Movement threshold in pixels to determine when a person is considered to move
    """
    df_frames = load_df(frames_csv)
    vectors = ordinal_from_csv(df_frames, threshold=movement_threshold)

    matrix = cross_validate(
        frames_csv,
        data=vectors,
        model_class=MemoryBasedShakeDetector,
        params={
            'window_size': window_size,
            'movement_threshold': movement_threshold
        }
    )
    plot_confusion_matrix(matrix,
                          title=f'Confusion matrix for memory-based classifier with threshold={movement_threshold}')

def cross_validate(frames_csv, data, model_class, params):
    """Do cross validation with an arbitrary model and number of folds.

    The model will be created using any parameters given in params and should be able to fit using a Pandas dataframe
    and index-able collection of data sequences, which length may be variable.

    The code assumes your folds are defined in the Dataframe under a 'split' column, where every fold has a unique
    identifier for the rows to use during validation. This identifier may be anything, so long as it includes the word
    'fold'. Any rows that don't include this word will be ignored (e.g. testing sets or any other collections of
    datapoints you don't wish to use at this time).

    :param frames_csv: Dataframe with frame annotations, labels and any other information used by your model
    :param data: Collection of datapoints, the indices of these datapoints must correspond to the Dataframe
    :param model_class: Reference to the model class to use
    :param params: Parameters for creating models
    :return: Confusion matrix of the results
    """
    df_frames = load_df(frames_csv)
    splits = get_splits(df_frames)
    matrix = np.zeros((2, 2))

    for fold in splits:
        model = model_class(**params)
        matrix = matrix + validate_fold(frames_csv=df_frames, model=model, fold=fold, data=data)

    return matrix

def validate_fold(frames_csv, model, data, fold, classes=None):
    """Validate a single fold. This function assumes a two-class classification problem is being solved, but this can
    be explicitly adjusted using the 'classes' argument.

    :param frames_csv: Dataframe with frame annotations, labels and any other information used by your model
    :param model: Instance of the model that will be trained
    :param data: Collection of datapoints, the indices of these datapoints must correspond to the Dataframe
    :param fold: Label of the fold used for evaluation.
    :param classes: List of integers representing classes.
    :return: Confusion matrix of the validation results.
    """
    if classes is None:
        classes = [0, 1]

    df_frames = load_df(frames_csv)
    df_val = df_frames[df_frames['split'] == fold]
    df_train = df_frames[df_frames['split'].str.contains('fold')].drop(df_val.index)
    assert len(df_val) > 0

    train_indices = list(df_train.index)
    val_indices = list(df_val.index)
    model.fit(df_train, data=[data[index] for index in train_indices])
    val_sequences = [data[index] for index in val_indices]

    cut_size = int((model.window_size - 1) / 2)
    i = 0
    pred_list = []
    label_list = []

    for _, row in df_val.iterrows():
        pred_list.append(model.predict(val_sequences[i]))
        labels = np.load(row['labels_path'])[row['start_frame']:row['end_frame']]
        labels = labels[cut_size + 1:-cut_size]
        label_list.append(labels)

        i += 1

    preds = np.concatenate(pred_list)
    labels = np.concatenate(label_list)

    return confusion_matrix(labels, preds, labels=classes)

def plot_confusion_matrix(matrix, title=None):
    disp = ConfusionMatrixDisplay(matrix, display_labels=['background', 'head-shake'])

    disp.plot()
    plt.subplots_adjust(left=0.25)
    if title:
        plt.title(title, fontsize=12, y=1.1)
    plt.show()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('frames_csv', type=Path)
    parser.add_argument('-w', '--window_size', default=36 , type=int)
    parser.add_argument('-t', '--movement_threshold', default=0.5, type=float)
    subparsers = parser.add_subparsers(dest='subparser')

    hmm_parser = subparsers.add_parser('hmm')
    hmm_parser.add_argument('folds_dir', type=Path)
    hmm_parser.add_argument('-s', '--save_dir', type=Path)

    window_parser = subparsers.add_parser('window')
    window_parser.add_argument('folds_dir', type=Path)

    filter_parser = subparsers.add_parser('filter')
    filter_parser.add_argument('folds_dir', type=Path)
    filter_parser.add_argument('-s', '--save_dir', type=Path)

    hmm_fit_parser = subparsers.add_parser('hmm_fit')
    hmm_fit_parser.add_argument('folds_dir', type=Path)

    memory_parser = subparsers.add_parser('memory')
    rule_parser = subparsers.add_parser('rule')
    random_parser = subparsers.add_parser('random')

    args = parser.parse_args()

    if args.subparser == 'hmm':
        validate_hmm_folds(args.frames_csv,
                           args.folds_dir,
                           window_size=args.window_size,
                           results_dir=args.save_dir,
                           filter_size=None,
                           method='event',
                           load_previous=True)
    elif args.subparser == 'window':
        hmm_window_search(args.frames_csv,
                          args.fols_dir,
                          method='frame',
                          metric='f1')
    elif args.subparser == 'filter':
        hmm_filter_search(args.frames_csv,
                          args.folds_dir,
                          results_dir=args.save_dir,
                          method='event',
                          metric='f1')
    elif args.subparser == 'hmm_fit':
        fit_hmm_folds(args.frames_csv,
                      args.folds_dir)
    elif args.subparser == 'memory':
        cross_validate_memory_preload(args.frames_csv,
                                      window_size=args.window_size,
                                      movement_threshold=args.movement_threshold)
    elif args.subparser == 'rule':
        cross_validate_rule_preload(args.frames_csv,
                                    window_size=args.window_size,
                                    movement_threshold=args.movement_threshold)
    elif args.subparser == 'random':
        cross_validate_random_preload(args.frames_csv,
                                      window_size=args.window_size,
                                      movement_threshold=args.movement_threshold)
    else:
        raise RuntimeError('Please choose a subcommand')


from airflow.hooks.postgres_hook import PostgresHook
from airflow.contrib.hooks.aws_hook import AwsHook
from airflow.hooks.S3_hook import S3Hook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults

class StageToRedshiftOperator(BaseOperator):
    ui_color = '#358140'

    @apply_defaults
    def __init__(self, redshift_conn_id="", aws_credentials="", table="", s3_bucket="", s3_key="", json_path="auto", region="us-west-2", *args, **kwargs):

        super(StageToRedshiftOperator, self).__init__(*args, **kwargs)
        self.redshift_conn_id = redshift_conn_id
        self.aws_credentials = aws_credentials
        self.table = table
        self.s3_bucket = s3_bucket
        self.s3_key = s3_key
        self.json_path = json_path
        self.region = region

    def execute(self, context):
        self.s3 =S3Hook(aws_conn_id="aws_credentials")
        credentials = self.s3.get_credentials()
        redshift = PostgresHook(self.redshift_conn_id)
        #sql format
        rendered_key = self.s3_key.format(**context)
        s3_path = "s3://{}/{}".format(self.s3_bucket, rendered_key)
        sql_format = """COPY public.{} FROM '{}' ACCESS_KEY_ID '{}' SECRET_ACCESS_KEY '{}' REGION '{}' JSON '{}'"""
        #load
        sql = sql_format.format(self.table, s3_path, credentials.access_key, credentials.secret_key, self.region, self.json_path)
        redshift.run(sql)


import sys

from csp.tools.build import util
from csp.tools.build import scons
from subprocess import Popen, PIPE

import pickle
import re

from distutils import sysconfig

GetPythonInclude = sysconfig.get_python_inc
GetPythonLibrary = sysconfig.get_python_lib

def CheckSConsVersion(minimum):
    version = scons.GetVersion()
    if util.CompareVersions(version, minimum) < 0:
        print('SCons version %s is too old.  Please install'
              'version %s or newer.' % (version, minimum))
        sys.exit(1)

def CheckPythonVersion(minimum):
    version = '.'.join(map(str, sys.version_info[:3]))
    if util.CompareVersions(version, minimum) < 0:
        print('Python version %s is too old.  Please install'
              'version %s or newer.' % (version, minimum))
        sys.exit(1)

def CheckWXVersion(minimum):
    try:
        import wx
    except ImportError:
        print('Missing wx python module. Please install wxPython')
        sys.exit(1)

    version = wx.version()
    if util.CompareVersions(version, minimum) < 0:
        print('wxPython version (%s) is too old. Please install %s or newer.' % (version, minimum))
        sys.exit(1)

def CheckLXMLVersion():
    try:
        import lxml
    except ImportError:
        print('Missing lxml python module used by the layout2 tool. Please install.')
        sys.exit(1)

    # Don't know how to test which version that is installed.
    # Just check that it is installed for now.

def GetGCCVersion():
    p = Popen('gcc -dumpfullversion', shell=True, stdout=PIPE)
    out, err = p.communicate()
    version = out.strip()
    try:
        return tuple(map(int, version.split('.')))
    except Exception:
        return None

def CheckSwig(context, min_version, not_versions=[]):
    ok = 0
    _checking(context, 'swig', reset_cached=1)
    p = Popen(['%s' % context.env.get('SWIG', 'swig'), '-version'], stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
    swig_out, swig_err = p.communicate()
    if swig_err is not None:
        output = swig_err.splitlines() + swig_out.splitlines()
        output = " ".join(map(lambda x: x.strip(), output))
        match = re.search(r'SWIG Version (\d+\.\d+.\d+)', output)
        if match is not None:
            swig_version = match.groups()[0]
            if util.CompareVersions(swig_version, min_version) >= 0:
                ok = 1
                for nv in not_versions:
                    if util.CompareVersions(swig_version, nv) == 0:
                        ok = 0
    if ok:
        context.Result("yes (%s)" % swig_version)
        context.env['SWIG_VERSION'] = swig_version
    else:
        context.Result("no")
    return ok

def CheckLibVersion(context, name, source, min_version):
    _checking(context, name, min_version)
    old_LIBS = context.env.get('LIBS', '')
    context.env.Append(LIBS=[name])
    success, output = context.TryRun(source, '.c')
    version = output.strip()
    if success and util.CompareVersions(version, min_version) >= 0:
        context.Result("yes (%s)" % version)
        context.env['%s_VERSION' % name.upper()] = version
    else:
        context.Result("no")
        context.env.Replace(LIBS=old_LIBS)
    return success

def CheckOSGVersion(context, lib, min_version):
    OSG_VERSION_CHECK = '''
    /* Override any gcc2 internal prototype to avoid an error.  */
    #ifdef __cplusplus
    extern "C"
    #endif
    #include <stdio.h>
    #include <stdlib.h>
    const char *%(lib)sGetVersion(); /**/
    int main() {
        printf("%%s", %(lib)sGetVersion());
        exit(0);
        return 0;
    }
    ''' % {'lib': lib}
    return CheckLibVersion(context, lib, OSG_VERSION_CHECK, min_version)

# Lazy check that pkg-config is installed.
HAS_PKG_CONFIG = None

def _CheckPkgConfigWorks(context):
    global HAS_PKG_CONFIG
    if HAS_PKG_CONFIG is None:
        _checking(context, 'pkg-config', reset_cached=1)
        p = Popen('pkg-config --version', shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
        output, error = p.communicate()
        version = output.strip()
        error = error.strip()
        if not error and re.match(r'\d', version) is not None:
            HAS_PKG_CONFIG = 1
            context.Result("yes (%s)" % version)
        else:
            HAS_PKG_CONFIG = 0
            context.Result("no")
            context.Log(error)
    return HAS_PKG_CONFIG

def CheckPkgConfig(context, lib, version=None, lib_name=None, command='pkg-config', version_flag='--modversion', config_flags='--cflags --libs'):
    has_pkg_config = _CheckPkgConfigWorks(context)
    if lib_name is None:
        lib_name = lib
    _checking(context, lib, version, reset_cached=1)
    if not has_pkg_config:
        context.Result("unknown (need pkg-config)")
        return 0
    p = Popen('%s %s %s' % (command, version_flag, lib), shell=True, stdout=PIPE, stderr=PIPE)
    output, error = p.communicate()
    ok = 0
    if output is not None:
        lib_version = output.strip()
        ok = (version is None) or (util.CompareVersions(lib_version, version) >= 0)
    if ok:
        context.Result("yes (%s)" % lib_version)
        context.env['%s_VERSION' % lib_name.upper()] = lib_version
        context.env.ParseConfig('%s %s %s' % (command, config_flags, lib))
    else:
        context.Result("no")
    return ok

def CheckCommandVersion(context, lib, command, min_version=None, lib_name=None):
    if lib_name is None:
        lib_name = lib
    _checking(context, lib, min_version, reset_cached=1)
    p = Popen(command, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
    output, error = p.communicate()
    version = output.strip()
    error = error.strip()
    ok = not error and (util.CompareVersions(version, min_version) >= 0)
    if ok:
        context.Result("yes (%s)" % version)
        context.env['%s_VERSION' % lib_name.upper()] = version
    else:
        context.Result("no")
        context.Log(error)
    return ok

def _checking(context, target, min_version=None, reset_cached=0):
    if min_version is None:
        context.Message('Checking for %s... ' % (target))
    else:
        context.Message('Checking for %s (>= %s)... ' % (target, min_version))
    if reset_cached:
        # hack into scons internals to prevent the "(cached)" message from
        # displaying when the subsequent check doesn't use the SConf
        # infrastructure (e.g., a straight popen test).
        context.sconf.cached = 0

def CustomConfigure(env):
    scons.SetReading()
    scons.DisableQuiet()
    conf = env.Configure(log_file="#/.config.log")
    conf.AddTests({'CheckSwig': CheckSwig})
    conf.AddTests({'CheckLibVersion': CheckLibVersion})
    conf.AddTests({'CheckOSGVersion': CheckOSGVersion})
    conf.AddTests({'CheckCommandVersion': CheckCommandVersion})
    conf.AddTests({'CheckPkgConfig': CheckPkgConfig})
    return conf

def SetConfig(env, config):
    if config:
        if ('config' in scons.GetCommandlineTargets()) or ReadConfig(env):
            if not env.GetOption('clean'):
                config(env)
                WriteConfig(env)

def ReadConfig(env):
    config_file = env.GetBuildPath('.config')
    try:
        config = open(config_file, 'rt')
    except IOError:
        return None
    # hack to rerun the config if the SConstruct file is modified; this can
    # probably be done more cleanly using regular scons dependencies.  also
    # shouldn't hardcode the name 'SConstruct' here.
    t0 = scons.FS.getmtime(scons.File('#/SConstruct').abspath)
    t1 = scons.FS.getmtime(scons.File('#/.config').abspath)
    if t0 > t1:
        return None
    return pickle.load(config)

def SaveConfig(env, config):
    config_file = env.GetBuildPath('.config')
    pickle.dump(config, open(config_file, 'wt'))


#
# @lc app=leetcode id=1 lang=python3
#
# [1] Two Sum
#

# @lc code=start
from typing import List

class Solution:
    def twoSum(self, nums: List[int], target: int) -> List[int]:
        num_to_index = {}
        for i, num in enumerate(nums):
            complement = target - num

            if complement in num_to_index:
                return [num_to_index[complement], i]

            num_to_index[num] = i

        return []

# @lc code=end


from django.contrib.auth.decorators import login_required
from django.utils import timezone
from django.shortcuts import render, get_object_or_404, redirect
from django.contrib import messages

from ..models import Question
from ..forms import QuestionForm

# Create your views here.
@login_required(login_url='common:login')
def question_create(request):
    if request.method == 'POST':
        form = QuestionForm(request.POST)
        if form.is_valid():
            question = form.save(commit=False)
            question.author = request.user
            question.create_date = timezone.now()
            question.save()
            return redirect('pybo:index')
    else:
        form = QuestionForm()
    context = {'form':form}
    return render(request, 'pybo/question_form.html', context)

@login_required(login_url='common:login')
def question_modify(request, question_id):
    question = get_object_or_404(Question, pk=question_id)
    if request.user != question.author:
        messages.error(request, '수정 권한이 없습니다.')
        return redirect('pybo:detail', question_id=question.id)
    if request.method =="POST":
        #messages.error(request, '수정 권한이 없습니다.')
        form = QuestionForm(request.POST, instance=question)
        if form.is_valid():
            question = form.save(commit=False)
            question.author = request.user
            question.modify_date = timezone.now()
            question.save()
            return redirect('pybo:detail', question_id = question.id)
    else:
        form = QuestionForm(instance=question)
    context = {'form':form}
    return render(request,'pybo/question_form.html', context)

@login_required(login_url='common:login')
def question_delete(request, question_id):
    question = get_object_or_404(Question, pk=question_id)
    if request.user != question.author:
        messages.error(request, '삭제 권한이 없습니다.')
        return redirect('pybo:detail', question_id=question.id)
    question.delete()
    return redirect('pybo:index')


from __future__ import annotations

from collections import defaultdict
from collections.abc import Hashable
from functools import wraps
from inspect import signature
from time import monotonic
from typing import Any, Callable, TypeVar

from typing_extensions import ParamSpec

T = TypeVar("T")
P = ParamSpec("P")
_sentinel = object()

def _wrap_func(func, seconds):
    expire_result = (0, _sentinel)

    @wraps(func)
    def inner(*args, **kwargs):
        nonlocal expire_result
        now_seconds = monotonic()
        expire, call_result = expire_result
        if now_seconds < expire and call_result is not _sentinel:
            return call_result
        call_result = func(*args, **kwargs)
        expire_result = (now_seconds + seconds, call_result)
        return call_result

    def clear():
        nonlocal expire_result
        expire_result = (0, _sentinel)

    inner.clear = clear
    return inner

def _wrap_method(
    seconds: float, instance_key: Callable[[T], Hashable], meth: Callable[P, Any]
):
    expire_times: dict[Hashable, tuple[float, Any]] = defaultdict(
        lambda: (0, _sentinel)
    )

    @wraps(meth)
    def inner(self, *args, **kwargs):
        now_seconds = monotonic()
        key = instance_key(self)
        expire, call_result = expire_times[key]
        if now_seconds < expire and call_result is not _sentinel:
            return call_result
        call_result = meth(self, *args, **kwargs)
        expire_times[key] = now_seconds + seconds, call_result
        return call_result

    def clear():
        nonlocal expire_times
        keys = list(expire_times.keys())
        for key in keys:
            expire_times[key] = (0, _sentinel)

    inner.clear = clear  # type: ignore
    return inner

def cache_ttl(seconds: float | int) -> Callable[[Callable[P, T]], Callable[P, T]]:
    """simple decorator if you want to cache the results of a call ignoring arguments
    Warning:
        1. Only caches a 'single value'
        2. Expects it to be a method if 'self' is in parameters
    '"""
    assert isinstance(seconds, (float, int)), "ttl seconds must be int/float"

    def decorator(func: Callable[P, T]) -> Callable[P, T]:
        if "self" in signature(func).parameters:
            return _wrap_method(seconds, id, func)
        return _wrap_func(func, seconds)

    return decorator

def clear_cache(func):
    func.clear()


#!/usr/bin/python
#
# -*- coding: utf-8 -*-
# vim: set ts=2 sw=2 et sts=2 ai:

import copy
import time
try:
  import simplejson
except:
  import json as simplejson
import urllib2
import logging

from cache import CACHE

from helpers import *
from libcloud_gae import ResourceNotFoundError

def parse_time(timestamp_str):
  dt = dateutil.parser.parse(timestamp_str)
  return calendar.timegm(dt.utctimetuple())+1e-6*dt.microsecond

class GCEObject(dict):
  # All attributes are dict entries
  def __setattr__(self, key, value):
    self[key] = value

  def __getattr__(self, key):
    return self[key]

  # For pickle and CACHE
  def __setstate__(self, state):
    self.__dict__ = state

  def __getstate__(self):
    return self.__dict__

  def _cache_key(self):
    return type(self).__name__.lower() + ':' + self.name

  # Functions to get/destroy stuff on gce
  @staticmethod
  def _gce_obj_get(driver, name):
    raise NotImplementedError()

  @staticmethod
  def _gce_obj_destory(driver, gce_obj):
    raise NotImplementedError()

  @staticmethod
  def _gce_obj_status(gce_obj):
    return gce_obj.extra['status']

  def update_from_gce(self, gce_obj):
    assert self.name == gce_obj.name, "%s != %s" % (self.name, gce_obj.name)
    self.name = gce_obj.name
    self.create_time = parse_time(gce_obj.extra['creationTimestamp'])
    self.status = self._gce_obj_status(gce_obj)

  # ---------------------------------
  _sentinal = []

  @classmethod
  def load(cls, name, driver=None, gce_obj=None):
    if driver:
      try:
        return cls.load(name, gce_obj=cls._gce_obj_get(driver, name))
      except ResourceNotFoundError:
        pass

    obj = cls(name, sentinal=cls._sentinal)
    cached = CACHE.get(obj._cache_key())
    if cached:
      obj.update(cached)

    if gce_obj:
      obj.update_from_gce(gce_obj)

    if not cached or gce_obj:
      obj.store()
    return obj

  def store(self):
    CACHE.set(self._cache_key(), self, time=60)

  def __init__(self, name, sentinal=None):
    assert sentinal is self._sentinal
    dict.__init__(self)
    self.name = name
    self.status = "IMAGINARY"
    self.create_time = inf

  def __eq__(self, other):
    return type(self) == type(other) and self.name == other.name

  def exists(self):
    return self.create_time != inf

  def ready(self):
    return self.status == "READY"

  # ---------------------------------

  def destroy(self, driver):
    assert self.exists()
    try:
      self._gce_obj_destory(driver, self._gce_obj_get(driver, self.name))
    except ResourceNotFoundError:
      pass
    CACHE.delete(self._cache_key())

class Disk(GCEObject):
  # Functions to get/destroy stuff on gce
  @staticmethod
  def _gce_obj_get(driver, name):
    return driver.ex_get_volume(name)

  @staticmethod
  def _gce_obj_destory(driver, gce_obj):
    return driver.destroy_volume(gce_obj)

  # ---------------------------------

  def create(self, driver, from_snapshot):
    self.update_from_gce(
        driver.create_volume(size=None, name=self.name, snapshot=from_snapshot, ex_disk_type='pd-ssd'))

class Snapshot(GCEObject):
  # Functions to get/destroy stuff on gce
  @staticmethod
  def _gce_obj_get(driver, name):
    return driver.ex_get_snapshot(name)

  @staticmethod
  def _gce_obj_destory(driver, gce_obj):
    return driver.destroy_volume_snapshot(gce_obj)

  @staticmethod
  def _gce_obj_status(gce_obj):
    # For some reason snapshot status is only available via .status rather than
    # .extra['status']!?
    return gce_obj.status

  # ---------------------------------

  def create(self, driver, from_disk):
    self.update_from_gce(
       driver.create_volume_snapshot(driver.ex_get_volume(from_disk), self.name))

class Instance(GCEObject):
  # Functions to get/destroy stuff on gce
  @staticmethod
  def _gce_obj_get(driver, name):
    return driver.ex_get_node(name)

  @staticmethod
  def _gce_obj_destory(driver, gce_obj):
    return driver.destroy_node(gce_obj)

  def update_from_gce(self, gce_obj):
    GCEObject.update_from_gce(self, gce_obj)
    self.public_ips = gce_obj.public_ips

    metadata = {}
    if 'metadata' in gce_obj.extra and 'items' in gce_obj.extra['metadata']:
      for d in gce_obj.extra['metadata']['items']:
        v = d['value']
        if v.startswith('{') or v.startswith('[') or v.startswith('"'):
          try:
            v = simplejson.loads(v)
          except ImportError:
            pass
        metadata[d['key']] = v
    self.metadata = metadata

    self.disks = []
    for d in gce_obj.extra['disks']:
      if d['kind'] != 'compute#attachedDisk':
        continue

      disk = Disk.load(d['deviceName'])
      self.disks.append(disk)

  # ---------------------------------

  def __init__(self, name, sentinal=None):
    GCEObject.__init__(self, name, sentinal)
    self.public_ips = []
    self.disks = []
    self.metadata = {}

  def ready(self):
    if self.status != "RUNNING":
      return False

    #if not self.fetch():
    #  return False

    return True

  def attached(self, disk):
    return disk in self.disks

  def fetch(self, name=""):
    if name:
      name = '/%s' % name

    if not self.public_ips:
      return None

    try:
      return urllib2.urlopen(
          "http://%s/tmp%s" % (
              self.public_ips[0], name)).read()
    except (urllib2.HTTPError, urllib2.URLError) as e:
      return None

  # ---------------------------------

  MACHINE_TYPE = 'n1-standard-2'
  BOOT_IMAGE = 'boot-image-wip-2'
  STARTUP_SCRIPT_URL = 'https://raw.githubusercontent.com/mithro/chrome-buildbot-sprint/master/metadata_watcher.py'
  WINDOWS_STARTUP_SCRIPT = 'windows-startup-script.ps1'
  TAGS = ('http-server',)

  def create(self, driver, machine_type):
    self.update_from_gce(driver.create_node(
      self.name,
      size=machine_type or self.MACHINE_TYPE,
      image=self.BOOT_IMAGE,
      ex_tags=self.TAGS,
      ex_metadata={'startup-script-url': self.STARTUP_SCRIPT_URL,
                   'windows-startup-script-ps1': self.WINDOWS_STARTUP_SCRIPT},
      ))

  def attach(self, driver, disk, mode):
    assert self.exists()
    assert self.ready()
    assert isinstance(disk, Disk)
    assert disk.exists()
    assert disk.ready()

    driver.attach_volume(
      node=self._gce_obj_get(driver, self.name),
      volume=disk._gce_obj_get(driver, disk.name),
      device=disk.name,
      ex_mode=mode)

  def detach(self, driver, disk):
    assert self.exists()
    assert self.ready()
    assert isinstance(disk, Disk)
    assert disk.exists()
    assert disk.ready()
    assert disk in self.disks

    driver.detach_volume(
      volume=disk._gce_obj_get(driver, disk.name),
      ex_node=self._gce_obj_get(driver, self.name))

  def set_metadata(self, driver, data={}, **kw):
    metadata = copy.deepcopy(self.metadata)
    metadata.update(data)
    metadata.update(kw)
    for key in metadata.keys():
      v = metadata[key]
      if isinstance(v, (int, long, float, str, unicode)):
        continue
      elif isinstance(metadata[key], (list, dict)):
        metadata[key] = simplejson.dumps(metadata[key])
      else:
        raise TypeError("Can't set metadata key %s to %r" % (key, v))
    while not driver.ex_set_node_metadata(self._gce_obj_get(driver, self.name), metadata):
      time.sleep(1)

    gce_obj = self._gce_obj_get(driver, self.name)
    self.update_from_gce(gce_obj)
    self.store()


#!/usr/bin python3

# ----------------------------------------------------------------------------
#  IOTpoll cycles thru the connected pins and returns back to Promethus server
#          the settings,  An API gateway allows grafana pages to switch relays
#          if required
#
#    To Set : View Readme.txt in folder
#
#    V 1.1.0  -  2020 Mar - Tidy up code
#
# ----------------------------------------------------------------------------

import sys
sys.path.append('../Common')

from http.server import BaseHTTPRequestHandler, HTTPServer
from prometheus_client import start_http_server, Gauge

from urllib import parse
import RPi.GPIO as GPIO
import configparser
import time
import http.client
import json
from ZigBee import ZBsensors

import os
import time
import datetime
import glob
import socket
from array import array
#from systemd import journal
import Adafruit_DHT
import logging
import socketserver
import requests

Ddir='/sys/bus/w1/devices/'
configPOLL = configparser.ConfigParser()
configPINS = configparser.ConfigParser()
KillSwitch = True
POLLlst={"DEF":0}
GaugeINT={"DEF":40}     # Internal Sensors List
Vers='1.1.0'
MID='XX'

# -----------------------------------------------------------------
#    Wire-1 : Wire 1 functions and routines
# -----------------------------------------------------------------

def LISTwire1():
    SIDs={}
    devicelist = glob.glob(Ddir+'28*')
    if devicelist=='':
        return SIDs
    else:
        for device in devicelist:
            TT=device.split("/")
            SID = TT[len(TT)-1]
            SIDs['W1_S'+SID[3:]]=GETwire1(TT[len(TT)-1])
    return SIDs

# Get Sensor Reading
def GETwire1(SID):
    devicefile=Ddir+SID+'/w1_slave'
    try:
        fileobj = open(devicefile,'r')
        lines = fileobj.readlines()
        fileobj.close()
    except:
        return devicefile+" Not Found"

    # get the status from the end of line 1
    status = lines[0][-4:-1]

    # is the status is ok, get the temperature from line 2
    if status=="YES":
        equals_pos = lines[1].find('t=')
        temp_string = lines[1][equals_pos+2:]
        tempvalue=float(temp_string)/1000
        return tempvalue
    else:
        return -999

# -----------------------------------------------------------------
#    Switch Sensors (Doors, Motion, Etc)
# -----------------------------------------------------------------

def LISTswitches(SIDs):

    for pin in configPINS:
        if pin != 'DEFAULT':
            if configPINS[pin]["Type"]=='Motion' or configPINS[pin]["Type"]=='Switch':
                SendMSG(str(pin))
                if GPIO.input(int(pin)):
                    SIDs[int(pin)]=1

    return SIDs

# -----------------------------------------------------------------
#    ZigBee Sensors
# -----------------------------------------------------------------

def ZBpoll():
    global ZBconfig, ZBsensors, ZBsensorC, LockSys
    LockSys = datetime.datetime.today()
    ZBsensorC = ZBsensors(configPOLL["ZIGBEE"]["ip"],configPOLL["ZIGBEE"]["key"])
    return ZBsensorC.GetALL()

    #if 'ZIGBEE' not in configPOLL: return SIDzb
    #params = {"words": 10, "paragraphs": 1, "format": "json"}
    #response = requests.get(f"http://"+configPOLL["ZIGBEE"]["ip"]+"/api/"+configPOLL["ZIGBEE"]["key"]+"/sensors/")
    #if response.status_code != 200:
    #    print("ZigBee Return Error : "+str(response.status_code))
    #    return True
    #ZBc=ZB.ZBsensors(response.json())   #  Store Sensors in ZigBee Class
    #return ZBc.GetALL()

# -----------------------------------------------------------------
#   PIN : Controllers
# -----------------------------------------------------------------

def PINpoll():
    # Read Pins
    SID={}
    for Pin in configPINS:
        if Pin != 'DEFAULT':
            Sname=configPINS[Pin]['name'].replace(' ','_')
            if configPINS[Pin]["type"]=='AM2302' or configPINS[Pin]["type"]=='DHT22':
                humidity,temperature = Adafruit_DHT.read_retry(22, Pin)
                if isinstance(temperature, float):
                    SID['PIN_DHT22_'+Pin+'_'+Sname+'_TEMP']=temperature
                if isinstance(humidity, float):
                    SID['PIN_DHT22_'+Pin+'_'+Sname+'_HUM']=temperature

            elif configPINS[Pin]["type"]=='relay':
                SID['PIN_RELAY_'+Pin+'_'+Sname]=RelayGET(Pin)

            elif configPINS[Pin]["type"]=='switch':
                SID['PIN_SWITCH_'+Pin+'_'+Sname]=RelayGET(Pin)

            elif configPINS[Pin]["type"]=='motion':
                SID['PIN_MOTION_'+Pin+'_'+Sname]=RelayGET(Pin)
    return SID

# Return Pin ON/OFF status
def RelayGET(PIN):
    if GPIO.input(int(PIN)):
        return 0
    else:
        return 1
    sys.stdout.flush()

# Toggle Pin
def RelayTOGGLE(PIN):
    print('Toggle:'+str(RelayGET(int(PIN))))
    if RelayGET(PIN):
       GPIO.output(int(PIN), GPIO.LOW)
       return "0"
    else:
       GPIO.output(int(PIN), GPIO.HIGH)
       return "1"
    sys.stdout.flush()

def SETpins():
    GPIO.setwarnings(False)
    GPIO.setmode(GPIO.BCM)
    for pin in configPINS:
        if pin != 'DEFAULT':
            if configPINS[pin]["type"]=='relay':
                SendMSG('TYPE:RELAY'+'-'+str(pin)+' '+configPINS[pin]['name']+' Initialised')
                GPIO.setup(int(pin), GPIO.OUT)
                GPIO.output(int(pin), GPIO.HIGH)

            if configPINS[pin]["type"]=='AM2302' or  configPINS[pin]["Type"]=='DHT22':
                SendMSG('TYPE:HUM/TEMP'+'-'+str(pin)+' '+configPINS[pin]['name'])

            if configPINS[pin]["type"]=='switch':
                SendMSG('TYPE:SWITCH'+'-'+str(pin)+' '+configPINS[pin]['name']+' Initialised')
                GPIO.setup(int(pin), GPIO.IN, pull_up_down=GPIO.PUD_UP)

            if configPINS[pin]["type"]=='motion':
                SendMSG('TYPE:Motion'+'-'+str(pin)+' '+configPINS[pin]['name']+' Initialised')
                GPIO.setup(int(pin), GPIO.IN, pull_up_down=GPIO.PUD_UP)

    #sys.stdout.flush()

# ------------------------------------------------------------------------
#   Promethues : Comms to Promethus Server
# ------------------------------------------------------------------------

def DelPromethues(key):
    global GaugeINT
    Mkey = MID + '_' + key
    if Mkey in GaugeINT:
        GaugeW1.pop(key)

def UpdPromethues(key,val):
    global GaugeINT
    Mkey = MID + '_' + key
    if not isSetPromethues(key):
        AddPromethues(key)
    if Mkey in GaugeINT:
        # print('UPD'+Mkey+'-'+str(val))
        GaugeINT[Mkey].set(val)

def AddPromethues(key):
    global GaugeINT
    Mkey = MID + '_' + key
    if  key not in GaugeINT:
        GaugeINT[Mkey]=Gauge(Mkey,key)

def POLLEDgauge(key):
    SendMSG('Polled Gauge '+str(key))

def isSetPromethues(key):
    Mkey = MID + '_' + key
    if Mkey not in GaugeINT:
        return False
    else:
        return True

# ------------------------------------------------------------------------
#   Intial Parm Load and validation
# ------------------------------------------------------------------------

def ValidatePARMS():
    global MID
    global PortNo
    global POLLgap

    configPOLL.read('POLL.ini')
    configPINS.read('PINS.ini')

    if "location" not in configPOLL["POLL"]:
        SendMSG('location missing from Config - Job Terminated')
        return True
    if "name" not in configPOLL["POLL"]:
        SendMSG('name missing from Config - Job Terminated')
        return True

    print("Location     : ",configPOLL["POLL"]["location"])
    print("Name         : ",configPOLL["POLL"]["name"])
    SendMSG(configPOLL["POLL"]["name"]+' / '+configPOLL["POLL"]["name"])
    MID=configPOLL["POLL"]["location"]+'_'+configPOLL["POLL"]["name"]+'_V2'
    SendMSG('Promethues Prefix : '+MID)

    PortNo=8010
    if "port" in configPOLL["POLL"]:
        PortNo=int(configPOLL['POLL']['port'])

    POLLgap=30
    if "Time" in configPOLL["POLL"]:
        POLLgap=int(configPOLL['POLL']['Time'])

    SendMSG('Poll Intervals : '+str(POLLgap)+' Seconds')
    SETpins()
    return False

# --------------------------------------------------------------------------

def SendMSG(msg):
    print(msg)
    #journal.send(msg)

# ---------------------------------------------------------------------------
#   API Server : Open API server to Port 18100
# ---------------------------------------------------------------------------

class gpioHTTPServer_RequestHandler(BaseHTTPRequestHandler):
    def do_OPTIONS(self):
        self.send_response(200)
        self.send_header('Access-Control-Allow-Origin', '*')
        self.send_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
        self.send_header("Access-Control-Allow-Headers", "X-Requested-With")

    def do_GET(self):
        self.send_response(200)
        self.send_header('Content-type','text/plain')
        self.end_headers()
        Rsn=APIincoming(self.path)
        self.wfile.write(bytes(Rsn, "utf8"))

    def do_POST(self):
        self.send_response(200)
        self.send_header("Content-Type", "text/plain")
        self.end_headers()
        self.wfile.write(bytes("OK","utf8"))

# -----------------------------------------------------------------------
#    Incoming API handler
# -----------------------------------------------------------------------

def APIincoming(url):
    qsp=dict(parse.parse_qsl(parse.urlsplit(url).query))
    #print(qsp)
    if 'API' in qsp:
        if qsp['API']!='A4C3D2E5112':
           return 'Security Volation'
        RtnTxt=''
    else:
        return 'Security Key Missing'

    if 'TYP' in qsp:
        if qsp['TYP']=='RLY':
            return apiRELAY(qsp)
        return 'Support Types (RLY,SWT,MON)'
    else:
        return 'Missing Task'

def apiRELAY(qsp):

    # Toggle Relay
    if 'PID' in qsp:
        if qsp['PID'] in configPINS:
            return RelayTOGGLE(qsp['PID'])

    # Return All Relay Status
    RtnTxt=''
    for pin in configPINS:
        if pin != 'DEFAULT':
            if configPINS[pin]["Type"]=='Relay':
                if RelayGET(pin)==0:
                    RlyStat='OFF'
                else:
                    RlyStat='ON'
                RtnTxt=RtnTxt+str(pin)+'('+configPINS[pin]['NAME']+')='+RlyStat+'  '

    return RtnTxt

# ---------------------------------------------------------------------------

def main():
    SendMSG('Version '+Vers)
    if ValidatePARMS(): return

# Start Promethus Server
    start_http_server(PortNo)
    SendMSG('Prometheus running on '+str(PortNo))

# Start API Server
    server_address = ('', 18100)
    httpd = HTTPServer(server_address, gpioHTTPServer_RequestHandler)
    httpd.socket.settimeout(1)
    httpd.handle_request()
    SendMSG('GPIO API running on 18100')

# Main Loop
    while KillSwitch:
        SIDs={}                      # Activate Sensors
        SIDs=LISTwire1()             # Wire-1 Sensor Controls
        print('SID:',SIDs)

        PNs=PINpoll()                # Pin Poll
        print(PNs)
        SIDs = {**SIDs, **PNs}

        if 'ZIGBEE' in configPOLL:   # ZigBee Poll
            ZBs=ZBpoll()
            print('ZBs:',ZBs)
            SIDs = {**SIDs, **ZBs}

        for ActSid in GaugeINT:      # Remove Dead Sensors
            if ActSid not in SIDs:
                DelPromethues(ActSid)

        for sid in SIDs:             # Add New Sensors
            if not isSetPromethues(sid):
                AddPromethues(sid)

        for sid in SIDs:             # Update Sensors
            UpdPromethues(sid,SIDs[sid])

        GapCnt=0
        while GapCnt <= POLLgap:
            time.sleep(1)
            httpd.handle_request()   # Poll API Getway
            GapCnt+=1

    sys.stdout.flush()

if __name__=="__main__":
    main()


from __future__ import unicode_literals
import logging
import time

from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.db import connection

from multigtfs.models import Feed, Route, Shape, Trip

class Command(BaseCommand):
    help = 'Updates the cached geometry of GTFS feeds'

    def add_arguments(self, parser):
        # Positional arguments
        parser.add_argument('feed_ids',
                            nargs='*',
                            metavar='Feed ID',
                            type=int)

        # Named (optional) arguments
        parser.add_argument('-a', '--all',
                            action='store_true',
                            dest='all',
                            default=False,
                            help='Update all feeds')
        parser.add_argument('-q', '--quiet',
                            action='store_false',
                            dest='verbose',
                            default=True,
                            help="Don't print status messages to stdout")

    def handle(self, *args, **options):
        total_start = time.time()

        # Validate the arguments
        all_feeds = options.get('all')
        feed_ids = options.get('feed_ids')
        if len(feed_ids) == 0 and not all_feeds:
            raise CommandError('You must pass in a feed ID or --all.')
        if len(feed_ids) > 0 and all_feeds:
            raise CommandError("You can't specify a feed and --all.")

        # Setup logging
        verbosity = int(options['verbosity'])
        console = logging.StreamHandler(self.stderr)
        formatter = logging.Formatter('%(levelname)s - %(message)s')
        logger_name = 'multigtfs'
        if verbosity == 0:
            level = logging.WARNING
        elif verbosity == 1:
            level = logging.INFO
        elif verbosity == 2:
            level = logging.DEBUG
        else:
            level = logging.DEBUG
            logger_name = ''
            formatter = logging.Formatter(
                '%(name)s - %(levelname)s - %(message)s')
        console.setLevel(level)
        console.setFormatter(formatter)
        logger = logging.getLogger(logger_name)
        logger.setLevel(level)
        logger.addHandler(console)

        # Disable database query logging
        if settings.DEBUG:
            connection.use_debug_cursor = False

        # Get the feeds
        if all_feeds:
            feeds = Feed.objects.order_by('id')
        else:
            feeds = []
            for feed_id in feed_ids:
                try:
                    feeds.append(Feed.objects.get(id=feed_id))
                except Feed.DoesNotExist:
                    raise CommandError('Feed %s not found' % feed_id)

        # Refresh the geometries
        for feed in feeds:
            logger.info(
                "Updating geometries in Feed %s (ID %s)...",
                feed.name, feed.id)

            start_time = time.time()
            shapes = Shape.objects.in_feed(feed)
            for shape in shapes:
                shape.update_geometry(update_parent=False)
            end_time = time.time()
            logger.debug(
                "Imported %s shape%s in %0.1f seconds",
                shapes.count(), '' if shapes.count() == 1 else 's',
                end_time - start_time)

            start_time = time.time()
            trips = Trip.objects.in_feed(feed)
            for trip in trips:
                trip.update_geometry(update_parent=False)
            end_time = time.time()
            logger.debug(
                "Imported %s trip%s in %0.1f seconds",
                trips.count(), '' if trips.count() == 1 else 's',
                end_time - start_time)

            start_time = time.time()
            routes = Route.objects.in_feed(feed)
            for route in routes:
                route.update_geometry()
            end_time = time.time()
            logger.debug(
                "Imported %s route%s in %0.1f seconds",
                routes.count(), '' if routes.count() == 1 else 's',
                end_time - start_time)

            total_end = time.time()
            logger.info(
                "Feed %d: Updated geometries in %d shape%s, %d trip%s, and"
                " %d route%s %0.1f seconds.",
                feed.id,
                shapes.count(), '' if shapes.count() == 1 else 's',
                trips.count(), '' if trips.count() == 1 else 's',
                routes.count(), '' if routes.count() == 1 else 's',
                total_end - total_start)