# encoding=utf-8
import os
import csv
import numpy as np
import random
import re
from itertools import islice

# 生成开源SVM所需的格式
def read_data(file_path):
    data_matrix = []
    with open(file_path, 'r', encoding='gbk', newline="") as file:
        csv_reader = csv.reader(file)
        # 跳过第一行Head和第一列
        for line in islice(csv_reader, 3, None):
            row = []
            for item in islice(line, 1, None):
                value = re.findall('[-+]?[.]?[\d]+(?:,\d\d\d)*[\.]?\d*(?:[eE][-+]?\d+)?', item)
                value = float(value[0])
                row.append(value)
            data_matrix.append(row)
    return np.array(data_matrix)

def event_split(data):
    feature = data[:, 0:-5]
    thirty_day_readmit = data[:, -5]
    thirty_day_readmit = thirty_day_readmit[:, np.newaxis]
    one_year_readmit = data[:, -4]
    one_year_readmit = one_year_readmit[:, np.newaxis]
    cardio_death = data[:, -3]
    cardio_death = cardio_death[:, np.newaxis]
    all_cause_death = data[:, -2]
    all_cause_death = all_cause_death[:, np.newaxis]

    thirty_day_readmit = np.concatenate([feature, thirty_day_readmit], axis=1)
    one_year_readmit = np.concatenate([feature, one_year_readmit], axis=1)
    cardio_death = np.concatenate([feature, cardio_death], axis=1)
    all_cause_death = np.concatenate([feature, all_cause_death], axis=1)
    return thirty_day_readmit, one_year_readmit, cardio_death, all_cause_death

def five_fold_split(data, skip_zero=False):
    # 数据规范，data是一个 M*N的矩阵，其中，M*(N-1)的部分是特征，最后一列是标签
    # 标签要求正标签为1，负标签为-1，无标签为0
    # 如果skip被设为True，则跳过无标签数据
    pos_data = list()
    neg_data = list()
    no_label_data = list()

    for item in data:
        if item[-1] == 1:
            pos_data.append(item)
        elif item[-1] == -1:
            neg_data.append(item)
        elif item[-1] == 0 and not skip_zero:
            no_label_data.append(item)
        elif item[-1] == 0 and skip_zero:
            pass
        else:
            raise ValueError("")

    # 随机化
    random.shuffle(pos_data)
    random.shuffle(neg_data)
    random.shuffle(no_label_data)

    five_fold = list()
    for i in range(5):
        pos_num = len(pos_data) // 5
        neg_num = len(neg_data) // 5
        no_num = len(no_label_data) // 5
        if i != 4:
            fold_pos = np.array(pos_data[i*pos_num:(i+1)*pos_num])
            fold_neg = np.array(neg_data[i*neg_num:(i+1)*neg_num])
            fold_no = np.array(no_label_data[i*no_num:(i+1)*no_num])
            if no_num != 0:
                five_fold.append(np.concatenate([fold_pos, fold_neg, fold_no], axis=0))
            else:
                five_fold.append(np.concatenate([fold_pos, fold_neg], axis=0))
        else:
            # 由于数据珍贵，不能丢弃任何一个数据
            fold_pos = np.array(pos_data[4*pos_num:])
            fold_neg = np.array(neg_data[4*neg_num:])
            fold_no = np.array(no_label_data[4*no_num:])
            if no_num != 0:
                five_fold.append(np.concatenate([fold_pos, fold_neg, fold_no], axis=0))
            else:
                five_fold.append(np.concatenate([fold_pos, fold_neg], axis=0))
    return five_fold

def data_generate(five_fold_data):
    reconstructed = list()
    for i in range(5):
        train = list()
        for j in range(5):
            if i == j:
                continue
            train.append(five_fold_data[j])
        train = np.concatenate([train[0], train[1], train[2], train[3]], axis=0)
        train_data = train[:, :-1]
        train_label = train[:, -1]

        test = []
        for item in five_fold_data[i]:
            if item[-1] != 0:
                test.append(item)
        test = np.array(test)
        test_label = test[:, -1]
        test_data = test[:, : -1]

        reconstructed.append([train_data, train_label, test_data, test_label])
    return reconstructed

def write_data(data, label, data_path, label_path):
    data_to_write = []
    for case in data:
        line_string = str()
        for index, item in enumerate(case):
            line_string += str(index+1)+':'+str(item)+" "
        line_string = line_string[:-1]+'\n'
        data_to_write.append(line_string)

    label_to_write = []
    for case in label:
        label_to_write.append(str(int(case))+"\n")

    with open(data_path, 'w', encoding='utf-8') as file:
        for item in data_to_write:
            file.write(item)
    with open(label_path, 'w', encoding='utf-8') as file:
        for item in label_to_write:
            file.write(item)

def semi_supervised_data_generate():
    source_data_path = os.path.abspath('..\\resource\\Data\\general_preprocessed.csv')
    data = read_data(source_data_path)
    thirty_day_readmit, one_year_readmit, cardio_death, all_cause_death = event_split(data)
    thirty_day_readmit_5_fold = five_fold_split(thirty_day_readmit)
    one_year_readmit_5_fold = five_fold_split(one_year_readmit)
    cardio_death_5_fold = five_fold_split(cardio_death)
    all_cause_death_5_fold = five_fold_split(all_cause_death)

    thirty_day_readmit_data = data_generate(thirty_day_readmit_5_fold)
    one_year_readmit_data = data_generate(one_year_readmit_5_fold)
    cardio_death_data = data_generate(cardio_death_5_fold)
    all_cause_death_data = data_generate(all_cause_death_5_fold)

    thirty_day_readmit_path = os.path.abspath('..\\resource\\Data\\svm_data\\30_day_readmit')
    one_year_readmit_path = os.path.abspath('..\\resource\\Data\\svm_data\\1_year_readmit')
    cardio_death_path = os.path.abspath('..\\resource\\Data\\svm_data\\cardio_death')
    all_cause_death_path = os.path.abspath('..\\resource\\Data\\svm_data\\all_cause_death')
    data_dict = {'30_day': [thirty_day_readmit_data, thirty_day_readmit_path],
                 '1_year': [one_year_readmit_data, one_year_readmit_path],
                 'all_cause': [all_cause_death_data, all_cause_death_path],
                 'cardio': [cardio_death_data,cardio_death_path]}

    for key in data_dict:
        data, path = data_dict[key]
        for i in range(5):
            train_data, train_label, test_data, test_label = data[i]
            train_data_path = os.path.join(path, str(i)+'trd')
            train_label_path = os.path.join(path, str(i) + 'trl')
            test_data_path = os.path.join(path, str(i) + 'ted')
            test_label_path = os.path.join(path, str(i) + 'tel')
            write_data(train_data, train_label, train_data_path, train_label_path)
            write_data(test_data, test_label, test_data_path, test_label_path)

    print('finish')

def main():
    semi_supervised_data_generate()

if __name__ == "__main__":
    main()


import dlib
import uuid
import numpy as np

class TrackState:
    TENTATIVE = 1
    CONFIRMED = 2
    OCCLUDED = 3
    DELETED = 4

class Track(object):
    def __init__(self,
                 rgb_image,
                 detection,
                 n_init,
                 max_disappeared,
                 max_age,
                 use_correlation_tracker=True,
                 feature=None):

        self.uuid = str(uuid.uuid4())

        self.bbox = dlib.rectangle(int(detection.bbox.left()),
                                   int(detection.bbox.top()),
                                   int(detection.bbox.right()),
                                   int(detection.bbox.bottom()))

        self.class_label = detection.class_label

        self.k_past = dlib.rectangles()

        if use_correlation_tracker is True:
            self.tracker = dlib.correlation_tracker()
            self.tracker.start_track(rgb_image, self.bbox)
        else:
            self.tracker = None

        self.state = TrackState.TENTATIVE

        self.n_init = n_init
        self.max_disappeared = max_disappeared
        self.max_age = max_age

        self.translation = None
        self.rotation = None

        if feature is not None:
            self.feature = np.asarray(feature, dtype=np.float32)
        else:
            self.feature = None

        self.missed = 0
        self.age = 1
        self.hits = 1
        self.since_update = 1

    def update(self, rgb_image, detection):
        if self.tracker is not None:
            self.tracker.update(rgb_image, detection.bbox)
            self.bbox = self.tracker.get_position()
        else:
            self.bbox = detection.bbox
        self.class_label = detection.class_label
        self.hits += 1
        self.age = 0
        self.since_update = 0
        if self.state == TrackState.TENTATIVE and self.hits >= self.n_init:
            self.state = TrackState.CONFIRMED
        if self.state == TrackState.OCCLUDED:
            self.state = TrackState.CONFIRMED

    def predict(self, rgb_image):
        if self.age > self.max_age:
            self.mark_missed()
        else:
            if self.tracker is not None:
                self.tracker.update(rgb_image)
                self.bbox = self.tracker.get_position()
            else:
                pass
            self.age += 1
            self.since_update = 0

    def mark_missed(self):
        self.since_update += 1
        if self.state == TrackState.TENTATIVE:
            if self.since_update > self.max_disappeared:
                self.state = TrackState.DELETED
        if self.state == TrackState.CONFIRMED:
            if self.since_update > self.max_disappeared:
                self.state = TrackState.OCCLUDED
        elif self.state == TrackState.OCCLUDED:
            if self.since_update > self.max_age:
                self.state = TrackState.DELETED

    def is_perceived(self):
        if not self.is_deleted():
            return self.state != TrackState.OCCLUDED
        else:
            return False

    def is_confirmed(self):
        return self.state == TrackState.CONFIRMED or self.state == TrackState.OCCLUDED

    def is_deleted(self):
        return self.state == TrackState.DELETED

    def __str__(self):
        return "rect : {} for class : {}".format(self.bbox, self.class_label)


from django.test import TestCase

from api.apps.problems.models import Category
from api.apps.problems.tests.api.api_helper import ApiHelper
from api.apps.problems.urls import ProblemsAppUrls

class CategoriesTestCase(TestCase):
    def setUp(self) -> None:
        self.api_helper = ApiHelper()

    def test_no_categories_saved_empty_list_should_be_returned(self):
        response = self.client.get(ProblemsAppUrls.categories_list_url())

        self.assertEqual(response.status_code, 200)
        self.assertEqual(len(response.data), 0)

    def test_categories_exist_all_categories_should_be_returned(self):
        algebra = self.api_helper.create_category("Algebra")
        geometry = self.api_helper.create_category("Geometry")

        response = self.client.get(ProblemsAppUrls.categories_list_url())
        algebra_from_response = self.__find_category_in_response(response, algebra.name)
        geometry_from_response = self.__find_category_in_response(response, geometry.name)

        self.assertEqual(response.status_code, 200)
        self.assertEqual(len(response.data), 2)
        self.assertIsNotNone(algebra_from_response)
        self.assertIsNotNone(geometry_from_response)

    def test_find_by_not_existing_category_should_return_404(self):
        response = self.client.get(ProblemsAppUrls.category_details_url(0))

        self.assertEqual(response.status_code, 404)

    def test_find_by_existing_category_should_return_single_category(self):
        algebra = self.api_helper.create_category("Algebra")

        response = self.client.get(ProblemsAppUrls.category_details_url(algebra.id))
        algebra_from_response = Category(**response.json())

        self.assertEqual(response.status_code, 200)
        self.assertEqual(algebra_from_response.name, algebra.name)

    def __find_category_in_response(self, response, name):
        return Category(**[category for category in response.json() if category["name"] == name][0])


from typing import Optional, Tuple

import requests
from youtube_dl import YoutubeDL

from ..classes import Song

def search_youtube_video(
    search: str,
) -> Tuple[str, Optional[str], str, str, Optional[str]]:
    """
    Gets the first youtube video from a search, returns a tuple of (title, artist, url, youtube_url, duration)
    If the top url found is a video it will get the video, if it is a playlist it will turn the playlist into a single track
    """
    if not isinstance(search, str):
        raise TypeError(f"Searchs must be a str, not a {type(search)}")
    with YoutubeDL(
        {
            "format": "bestaudio",
            "noplaylist": True,
            "logtostderr": False,
            "quiet": True,
            "no_warnings": True,
            "source_address": "0.0.0.0",
        }
    ) as ydl:
        try:
            requests.get(search)
        except:
            info = ydl.extract_info(f"ytsearch:{search}", download=False)["entries"][0]
        else:
            info = ydl.extract_info(search, download=False)
    return (
        info["title"],
        info["uploader"],
        info["formats"][0]["url"],
        info["webpage_url"],
        info["duration"],
    )

def get_song_youtube(search) -> Song:
    title, artist, url, youtube_url, duration = search_youtube_video(search)
    return Song(title, artist, url, youtube_url, duration)


# Example:
# python -u h5_downsample.py -i 0_human_instance_seg_pred.h5 -o 0_human_instance_seg_pred_v2.h5

import h5py
import numpy as np
import argparse
import os

parser = argparse.ArgumentParser()
parser.add_argument("-i", "--input", required=True, help="Input h5 file")
parser.add_argument("-o", "--output", required=True, help="Output h5 downsample file")
parser.add_argument("-d", "--down", help="[z,y,x] downsampling to be applied", default=[1,2,2])
args = parser.parse_args()

# Load H5 file
h5f = h5py.File(args.input, 'r')
k = list(h5f.keys())

# Change the key accordingly
print("Keys: {}".format(k))
data = np.array(h5f[k[0]])
print("Loaded data shape: {}".format(data.shape))

data = data[::args.down[0],::args.down[1],::args.down[2]]
print("Downsampled data shape: {}".format(data.shape))

out_dir = os.path.dirname(args.output)
os.makedirs(out_dir, exist_ok=True)

# Create the h5 file (using lzf compression to save space)
h5f = h5py.File(args.output, 'w')
h5f.create_dataset('main', data=data, compression="lzf")
h5f.close()
print("File saved in {}".format(args.output))


import networkx as nx
import matplotlib.pyplot as plt

G = nx.Graph([(1, 2), (1, 3), (1, 5), (3, 4), (4, 5)])

num_nodes = G.number_of_nodes()
print('Number of Nodes {}'.format(num_nodes))
num_edges = G.number_of_edges()
print('Number of Edges {}'.format(num_edges))

subax1 = plt.subplot(121)
nx.draw(G, with_labels=True, font_weight='bold')

degree_c = nx.degree_centrality(G)
print('Degrees of Centrality is {}'.format(degree_c))

closeness_c = nx.closeness_centrality(G)
print('Closeness of Centrality is {}'.format(closeness_c))

# compute centrality
betweenness_centrality = nx.betweenness_centrality(G)
print('Betweenness of Centrality is {}'.format(betweenness_centrality))

plt.show()


#!/usr/bin/python3

import requests, os, sys
import exchangeratekey_bb as exchangeratekey

FILENAME = os.path.dirname(sys.argv[0]) + '/exchangerate.txt'
RESPONSEDATA = requests.get("https://openexchangerates.org/api/latest.json", params={'symbols': 'GBP,EUR,JPY', 'app_id': exchangeratekey.APIKEY}).json()
rate = RESPONSEDATA['rates']['JPY'] / RESPONSEDATA['rates']['GBP']
with open(FILENAME, "w") as f:
    f.write(str(rate))


import cv2
import numpy as np
import matplotlib as plt

img=cv2.imread('cat.jpg',0)
cv2.imshow('yuantu',img)

img2=cv2.Canny(img,200,300)
cv2.imshow('Canny',img2)

ret,img3=cv2.threshold(img,125,255,cv2.THRESH_BINARY)
c,h=cv2.findContours(img3,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)

print('轮廓:',c)
print('轮廓类型:',type(c))
print('轮廓数:',len(c))
print('层次',h)
print('层次类型:',type(h))

# img4 = np.zeros_like(img) + 255
# for n in (range(len(c))):
# 	cv2.polylines(img4,[c[n]],True,(255,0,0),2)
# cv2.imshow('%s'%n,img4)

img5=np.zeros(img.shape,np.uint8)+255
img5=cv2.drawContours(img3,c,-1,(0,0,255),2)
cv2.imshow('Contours',img5)

for n in range(len(c)):
	m=cv2.moments(c[n])
	print('轮廓%s的矩：'%n,m)
	print('轮廓%s的面积：' % n, m['m00'])

img=cv2.imread('cat.jpg')#读取图像
cv2.imshow('original',img)#显示原图像
gray=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)#将其转化为灰度图像
ret,img2=cv2.threshold(gray,125,255,cv2.THRESH_BINARY)#二值化阈值处理
c,h=cv2.findContours(img2,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)#查找轮廓

img3=np.zeros(img.shape,np.uint8)+255
img3=cv2.drawContours(img3,c,-1,(0,0,255),2)
cv2.imshow('Counters',img3)
cv2.waitKey(0)
cv2.destroyAllWindows()

cv2.waitKey(0)
cv2.destroyAllWindows()


import functools
import logging
import socket
import sys
import time
from typing import Any, Callable, Dict, List, TypeVar, cast

import requests
from urllib3.connection import HTTPConnection

_BaseDecoratedFunc = Callable[..., Any]
DecoratedFunc = TypeVar("DecoratedFunc", bound=_BaseDecoratedFunc)

logger = logging.getLogger(__name__)
# Let dead connections stick around for 120 seconds at most
TCP_KEEPALIVE_TIME = 60
TCP_KEEPALIVE_INTERVAL = 10
TCP_KEEPALIVE_PROBE_COUNT = 6

# https://github.com/python/mypy/issues/1927
def time_api(fn: DecoratedFunc) -> DecoratedFunc:
    """A decorator to log the duration of a function call."""
    call_name = fn.__qualname__

    @functools.wraps(fn)
    def _run(*args, **kwargs):
        t1 = time.perf_counter()
        res = fn(*args, **kwargs)
        t2 = time.perf_counter()
        delta = t2 - t1
        # Only log significant function calls (>=10 millis) to avoid unnecessary log lines
        # NOTE: This could hide methods that are called many times and lead to large runtimes
        # on aggregate. In general these aren't common, so we ignore them here. Can always
        # enable finer grained logging later if this doesn't detect perf issues (e.g. overall runtime)
        # is significantly longer than logged runtime)
        if delta < 0.01:
            return res

        logger.info(
            r"TIMING - {\"method\": \"%s\", \"duration\": %.2f}",
            call_name,
            t2 - t1,
        )
        return res

    return cast(DecoratedFunc, _run)

def stats(values: List[float]) -> Dict[str, float]:
    """Calculate percentiles and other summary statistics of a list of values.

    Note: Requires numpy to be installed.
    """
    # Don't require numpy in requirements.txt - not required for main execution and increases image size unnecessarily
    import numpy

    return {
        "25th": numpy.percentile(values, 25),  # type: ignore # https://github.com/Project-MONAI/MONAI/issues/3561
        "50th": numpy.percentile(values, 50),  # type: ignore # https://github.com/Project-MONAI/MONAI/issues/3561
        "75th": numpy.percentile(values, 70),  # type: ignore # https://github.com/Project-MONAI/MONAI/issues/3561
        "90th": numpy.percentile(values, 90),  # type: ignore # https://github.com/Project-MONAI/MONAI/issues/3561
        "95th": numpy.percentile(values, 95),  # type: ignore # https://github.com/Project-MONAI/MONAI/issues/3561
        "99th": numpy.percentile(values, 99),  # type: ignore # https://github.com/Project-MONAI/MONAI/issues/3561
        "99.9th": numpy.percentile(values, 99.9),  # type: ignore # https://github.com/Project-MONAI/MONAI/issues/3561
        "mean": float(numpy.mean(values)),
        "stdev": float(numpy.std(values)),
        "median": numpy.median(values),  # type: ignore # https://github.com/Project-MONAI/MONAI/issues/3561
        "count": len(values),
        "total": sum(values),
        "max": max(values),
        "min": min(values),
    }

def _set_socket_options():
    """Turn on TCP keepalive for the HTTPConnection object."""
    logger.debug("Current socket options: %s", HTTPConnection.default_socket_options)
    for platforms, protocol, constant, value in (
        # Keepalive on Linux
        (("linux",), socket.IPPROTO_TCP, "TCP_KEEPIDLE", TCP_KEEPALIVE_TIME),
        # Shared between Linux and Mac
        (
            ("linux", "darwin"),
            socket.IPPROTO_TCP,
            "TCP_KEEPINTVL",
            TCP_KEEPALIVE_INTERVAL,
        ),
        (
            ("linux", "darwin"),
            socket.IPPROTO_TCP,
            "TCP_KEEPCNT",
            TCP_KEEPALIVE_PROBE_COUNT,
        ),
        # Turns KeepAlive on
        (("linux", "darwin"), socket.SOL_SOCKET, "SO_KEEPALIVE", 1),
        # Keepalive on MacOS
        # Not exposed in socket library, found from tcp.h:
        # https://github.com/apple/darwin-xnu/blob/0a798f6738bc1db01281fc08ae024145e84df927/bsd/netinet/tcp.h#L206
        (("darwin",), socket.IPPROTO_TCP, 0x10, TCP_KEEPALIVE_TIME),
    ):
        if sys.platform not in platforms:
            continue
        if isinstance(constant, str) and not hasattr(socket, constant):
            continue  # pragma: nocover # Windows only case, don't test
        elif isinstance(constant, str):
            constant = getattr(socket, constant)
        opts = (protocol, constant, value)
        logger.debug("Setting socket options: %s", opts)
        HTTPConnection.default_socket_options.append(opts)

@functools.lru_cache(maxsize=1)
def shared_session() -> requests.Session:
    """Get a global requests.Session object shared by all methods, which ensures TCP keepalive is enabled."""
    _set_socket_options()
    return requests.Session()

cookiecutters/python_core/{{ cookiecutter.project_name }}/{{ cookiecutter.project_slug }}/common.py


# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

import os
print(os.listdir("../input"))

# Any results you write to the current directory are saved as output.
# import numpy library
import numpy as np

# numpy array
array = [[1,2,3],[4,5,6]]
first_array = np.array(array) # 2x3 array
print("Array Type: {}".format(type(first_array))) # type
print("Array Shape: {}".format(np.shape(first_array))) # shape
print(first_array)
# import pytorch library
import torch

# pytorch array
tensor = torch.Tensor(array)
print("Array Type: {}".format(tensor.type)) # type
print("Array Shape: {}".format(tensor.shape)) # shape
print(tensor)
# numpy ones
print("Numpy {}\n".format(np.ones((2,3))))

# pytorch ones
print(torch.ones((2,3)))
# numpy random
print("Numpy {}\n".format(np.random.rand(2,3)))

# pytorch random
print(torch.rand(2,3))
# random numpy array
array = np.random.rand(2,2)
print("{} {}\n".format(type(array),array))

# from numpy to tensor
from_numpy_to_tensor = torch.from_numpy(array)
print("{}\n".format(from_numpy_to_tensor))

# from tensor to numpy
tensor = from_numpy_to_tensor
from_tensor_to_numpy = tensor.numpy()
print("{} {}\n".format(type(from_tensor_to_numpy),from_tensor_to_numpy))
# create tensor
tensor = torch.ones(3,3)
print("\n",tensor)

# Resize
print("{}{}\n".format(tensor.view(9).shape,tensor.view(9)))

# Addition
print("Addition: {}\n".format(torch.add(tensor,tensor)))

# Subtraction
print("Subtraction: {}\n".format(tensor.sub(tensor)))

# Element wise multiplication
print("Element wise multiplication: {}\n".format(torch.mul(tensor,tensor)))

# Element wise division
print("Element wise division: {}\n".format(torch.div(tensor,tensor)))

# Mean
tensor = torch.Tensor([1,2,3,4,5])
print("Mean: {}".format(tensor.mean()))

# Standart deviation (std)
print("std: {}".format(tensor.std()))
# import variable from pytorch library
from torch.autograd import Variable

# define variable
var = Variable(torch.ones(3), requires_grad = True)
var
# lets make basic backward propagation
# we have an equation that is y = x^2
array = [2,4]
tensor = torch.Tensor(array)
x = Variable(tensor, requires_grad = True)
y = x**2
print(" y =  ",y)

# recap o equation o = 1/2*sum(y)
o = (1/2)*sum(y)
print(" o =  ",o)

# backward
o.backward() # calculates gradients

# As I defined, variables accumulates gradients. In this part there is only one variable x.
# Therefore variable x should be have gradients
# Lets look at gradients with x.grad
print("gradients: ",x.grad)
# As a car company we collect this data from previous selling
# lets define car prices
car_prices_array = [3,4,5,6,7,8,9]
car_price_np = np.array(car_prices_array,dtype=np.float32)
car_price_np = car_price_np.reshape(-1,1)
car_price_tensor = Variable(torch.from_numpy(car_price_np))

# lets define number of car sell
number_of_car_sell_array = [ 7.5, 7, 6.5, 6.0, 5.5, 5.0, 4.5]
number_of_car_sell_np = np.array(number_of_car_sell_array,dtype=np.float32)
number_of_car_sell_np = number_of_car_sell_np.reshape(-1,1)
number_of_car_sell_tensor = Variable(torch.from_numpy(number_of_car_sell_np))

# lets visualize our data
import matplotlib.pyplot as plt
plt.scatter(car_prices_array,number_of_car_sell_array)
plt.xlabel("Car Price $")
plt.ylabel("Number of Car Sell")
plt.title("Car Price$ VS Number of Car Sell")
plt.show()
# Linear Regression with Pytorch

# libraries
import torch
from torch.autograd import Variable
import torch.nn as nn
import warnings
warnings.filterwarnings("ignore")

# create class
class LinearRegression(nn.Module):
    def __init__(self,input_size,output_size):
        # super function. It inherits from nn.Module and we can access everythink in nn.Module
        super(LinearRegression,self).__init__()
        # Linear function.
        self.linear = nn.Linear(input_dim,output_dim)

    def forward(self,x):
        return self.linear(x)

# define model
input_dim = 1
output_dim = 1
model = LinearRegression(input_dim,output_dim) # input and output size are 1

# MSE
mse = nn.MSELoss()

# Optimization (find parameters that minimize error)
learning_rate = 0.02   # how fast we reach best parameters
optimizer = torch.optim.SGD(model.parameters(),lr = learning_rate)

# train model
loss_list = []
iteration_number = 1001
for iteration in range(iteration_number):

    # optimization
    optimizer.zero_grad()

    # Forward to get output
    results = model(car_price_tensor)

    # Calculate Loss
    loss = mse(results, number_of_car_sell_tensor)

    # backward propagation
    loss.backward()

    # Updating parameters
    optimizer.step()

    # store loss
    loss_list.append(loss.data)

    # print loss
    if(iteration % 50 == 0):
        print('epoch {}, loss {}'.format(iteration, loss.data))

plt.plot(range(iteration_number),loss_list)
plt.xlabel("Number of Iterations")
plt.ylabel("Loss")
plt.show()
# predict our car price
predicted = model(car_price_tensor).data.numpy()
plt.scatter(car_prices_array,number_of_car_sell_array,label = "original data",color ="red")
plt.scatter(car_prices_array,predicted,label = "predicted data",color ="blue")

# predict if car price is 10$, what will be the number of car sell
#predicted_10 = model(torch.from_numpy(np.array([10]))).data.numpy()
#plt.scatter(10,predicted_10.data,label = "car price 10$",color ="green")
plt.legend()
plt.xlabel("Car Price $")
plt.ylabel("Number of Car Sell")
plt.title("Original vs Predicted values")
plt.show()
# Import Libraries
import torch
import torch.nn as nn
import torchvision.transforms as transforms
from torch.autograd import Variable
import pandas as pd
from sklearn.model_selection import train_test_split
# Prepare Dataset
# load data
train = pd.read_csv(r"../input/train.csv",dtype = np.float32)

# split data into features(pixels) and labels(numbers from 0 to 9)
targets_numpy = train.label.values
features_numpy = train.loc[:,train.columns != "label"].values/255 # normalization

# train test split. Size of train data is 80% and size of test data is 20%.
features_train, features_test, targets_train, targets_test = train_test_split(features_numpy,
                                                                             targets_numpy,
                                                                             test_size = 0.2,
                                                                             random_state = 42)

# create feature and targets tensor for train set. As you remember we need variable to accumulate gradients. Therefore first we create tensor, then we will create variable
featuresTrain = torch.from_numpy(features_train)
targetsTrain = torch.from_numpy(targets_train).type(torch.LongTensor) # data type is long

# create feature and targets tensor for test set.
featuresTest = torch.from_numpy(features_test)
targetsTest = torch.from_numpy(targets_test).type(torch.LongTensor) # data type is long

# batch_size, epoch and iteration
batch_size = 100
n_iters = 10000
num_epochs = n_iters / (len(features_train) / batch_size)
num_epochs = int(num_epochs)

# Pytorch train and test sets
train = torch.utils.data.TensorDataset(featuresTrain,targetsTrain)
test = torch.utils.data.TensorDataset(featuresTest,targetsTest)

# data loader
train_loader = torch.utils.data.DataLoader(train, batch_size = batch_size, shuffle = False)
test_loader = torch.utils.data.DataLoader(test, batch_size = batch_size, shuffle = False)

# visualize one of the images in data set
plt.imshow(features_numpy[10].reshape(28,28))
plt.axis("off")
plt.title(str(targets_numpy[10]))
plt.savefig('graph.png')
plt.show()
# Create Logistic Regression Model
class LogisticRegressionModel(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(LogisticRegressionModel, self).__init__()
        # Linear part
        self.linear = nn.Linear(input_dim, output_dim)
        # There should be logistic function right?
        # However logistic function in pytorch is in loss function
        # So actually we do not forget to put it, it is only at next parts

    def forward(self, x):
        out = self.linear(x)
        return out

# Instantiate Model Class
input_dim = 28*28 # size of image px*px
output_dim = 10  # labels 0,1,2,3,4,5,6,7,8,9

# create logistic regression model
model = LogisticRegressionModel(input_dim, output_dim)

# Cross Entropy Loss
error = nn.CrossEntropyLoss()

# SGD Optimizer
learning_rate = 0.001
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
# Traning the Model
count = 0
loss_list = []
iteration_list = []
for epoch in range(num_epochs):
    for i, (images, labels) in enumerate(train_loader):

        # Define variables
        train = Variable(images.view(-1, 28*28))
        labels = Variable(labels)

        # Clear gradients
        optimizer.zero_grad()

        # Forward propagation
        outputs = model(train)

        # Calculate softmax and cross entropy loss
        loss = error(outputs, labels)

        # Calculate gradients
        loss.backward()

        # Update parameters
        optimizer.step()

        count += 1

        # Prediction
        if count % 50 == 0:
            # Calculate Accuracy
            correct = 0
            total = 0
            # Predict test dataset
            for images, labels in test_loader:
                test = Variable(images.view(-1, 28*28))

                # Forward propagation
                outputs = model(test)

                # Get predictions from the maximum value
                predicted = torch.max(outputs.data, 1)[1]

                # Total number of labels
                total += len(labels)

                # Total correct predictions
                correct += (predicted == labels).sum()

            accuracy = 100 * correct / float(total)

            # store loss and iteration
            loss_list.append(loss.data)
            iteration_list.append(count)
        if count % 500 == 0:
            # Print Loss
            print('Iteration: {}  Loss: {}  Accuracy: {}%'.format(count, loss.data, accuracy))
# visualization
plt.plot(iteration_list,loss_list)
plt.xlabel("Number of iteration")
plt.ylabel("Loss")
plt.title("Logistic Regression: Loss vs Number of iteration")
plt.show()
# Import Libraries
import torch
import torch.nn as nn
import torchvision.transforms as transforms
from torch.autograd import Variable
# Create ANN Model
class ANNModel(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(ANNModel, self).__init__()
        # Linear function 1: 784 --> 100
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        # Non-linearity 1
        self.relu1 = nn.ReLU()

        # Linear function 2: 100 --> 100
        self.fc2 = nn.Linear(hidden_dim, hidden_dim)
        # Non-linearity 2
        self.tanh2 = nn.Tanh()

        # Linear function 3: 100 --> 100
        self.fc3 = nn.Linear(hidden_dim, hidden_dim)
        # Non-linearity 3
        self.elu3 = nn.ELU()

        # Linear function 4 (readout): 100 --> 10
        self.fc4 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        # Linear function 1
        out = self.fc1(x)
        # Non-linearity 1
        out = self.relu1(out)

        # Linear function 2
        out = self.fc2(out)
        # Non-linearity 2
        out = self.tanh2(out)

        # Linear function 2
        out = self.fc3(out)
        # Non-linearity 2
        out = self.elu3(out)

        # Linear function 4 (readout)
        out = self.fc4(out)
        return out

# instantiate ANN
input_dim = 28*28
hidden_dim = 150 #hidden layer dim is one of the hyper parameter and it should be chosen and tuned. For now I only say 150 there is no reason.
output_dim = 10

# Create ANN
model = ANNModel(input_dim, hidden_dim, output_dim)

# Cross Entropy Loss
error = nn.CrossEntropyLoss()

# SGD Optimizer
learning_rate = 0.02
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
# ANN model training
count = 0
loss_list = []
iteration_list = []
accuracy_list = []
for epoch in range(num_epochs):
    for i, (images, labels) in enumerate(train_loader):

        train = Variable(images.view(-1, 28*28))
        labels = Variable(labels)

        # Clear gradients
        optimizer.zero_grad()

        # Forward propagation
        outputs = model(train)

        # Calculate softmax and ross entropy loss
        loss = error(outputs, labels)

        # Calculating gradients
        loss.backward()

        # Update parameters
        optimizer.step()

        count += 1

        if count % 50 == 0:
            # Calculate Accuracy
            correct = 0
            total = 0
            # Predict test dataset
            for images, labels in test_loader:

                test = Variable(images.view(-1, 28*28))

                # Forward propagation
                outputs = model(test)

                # Get predictions from the maximum value
                predicted = torch.max(outputs.data, 1)[1]

                # Total number of labels
                total += len(labels)

                # Total correct predictions
                correct += (predicted == labels).sum()

            accuracy = 100 * correct / float(total)

            # store loss and iteration
            loss_list.append(loss.data)
            iteration_list.append(count)
            accuracy_list.append(accuracy)
            if count % 500 == 0:
                # Print Loss
                print('Iteration: {}  Loss: {}  Accuracy: {} %'.format(count, loss.data[0], accuracy))
# visualization loss
plt.plot(iteration_list,loss_list)
plt.xlabel("Number of iteration")
plt.ylabel("Loss")
plt.title("ANN: Loss vs Number of iteration")
plt.show()

# visualization accuracy
plt.plot(iteration_list,accuracy_list,color = "red")
plt.xlabel("Number of iteration")
plt.ylabel("Accuracy")
plt.title("ANN: Accuracy vs Number of iteration")
plt.show()
# Import Libraries
import torch
import torch.nn as nn
import torchvision.transforms as transforms
from torch.autograd import Variable
# Create CNN Model
class CNNModel(nn.Module):
    def __init__(self):
        super(CNNModel, self).__init__()

        # Convolution 1
        self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, stride=1, padding=0)
        self.relu1 = nn.ReLU()

        # Max pool 1
        self.maxpool1 = nn.MaxPool2d(kernel_size=2)

        # Convolution 2
        self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, stride=1, padding=0)
        self.relu2 = nn.ReLU()

        # Max pool 2
        self.maxpool2 = nn.MaxPool2d(kernel_size=2)

        # Fully connected 1
        self.fc1 = nn.Linear(32 * 4 * 4, 10)

    def forward(self, x):
        # Convolution 1
        out = self.cnn1(x)
        out = self.relu1(out)

        # Max pool 1
        out = self.maxpool1(out)

        # Convolution 2
        out = self.cnn2(out)
        out = self.relu2(out)

        # Max pool 2
        out = self.maxpool2(out)
        out = out.view(out.size(0), -1)

        # Linear function (readout)
        out = self.fc1(out)

        return out

# batch_size, epoch and iteration
batch_size = 100
n_iters = 2500
num_epochs = n_iters / (len(features_train) / batch_size)
num_epochs = int(num_epochs)

# Pytorch train and test sets
train = torch.utils.data.TensorDataset(featuresTrain,targetsTrain)
test = torch.utils.data.TensorDataset(featuresTest,targetsTest)

# data loader
train_loader = torch.utils.data.DataLoader(train, batch_size = batch_size, shuffle = False)
test_loader = torch.utils.data.DataLoader(test, batch_size = batch_size, shuffle = False)

# Create ANN
model = CNNModel()

# Cross Entropy Loss
error = nn.CrossEntropyLoss()

# SGD Optimizer
learning_rate = 0.1
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

# CNN model training
count = 0
loss_list = []
iteration_list = []
accuracy_list = []
for epoch in range(num_epochs):
    for i, (images, labels) in enumerate(train_loader):

        train = Variable(images.view(100,1,28,28))
        labels = Variable(labels)

        # Clear gradients
        optimizer.zero_grad()

        # Forward propagation
        outputs = model(train)

        # Calculate softmax and ross entropy loss
        loss = error(outputs, labels)

        # Calculating gradients
        loss.backward()

        # Update parameters
        optimizer.step()
        count += 1
        if count % 50 == 0:
            # Calculate Accuracy
            correct = 0
            total = 0
            # Iterate through test dataset
            for images, labels in test_loader:

                test = Variable(images.view(100,1,28,28))

                # Forward propagation
                outputs = model(test)

                # Get predictions from the maximum value
                predicted = torch.max(outputs.data, 1)[1]

                # Total number of labels
                total += len(labels)

                correct += (predicted == labels).sum()

            accuracy = 100 * correct / float(total)

            # store loss and iteration
            loss_list.append(loss.data)
            iteration_list.append(count)
            accuracy_list.append(accuracy)
            if count % 500 == 0:
                # Print Loss
                print('Iteration: {}  Loss: {}  Accuracy: {} %'.format(count, loss.data[0], accuracy))
# visualization loss
plt.plot(iteration_list,loss_list)
plt.xlabel("Number of iteration")
plt.ylabel("Loss")
plt.title("CNN: Loss vs Number of iteration")
plt.show()

# visualization accuracy
plt.plot(iteration_list,accuracy_list,color = "red")
plt.xlabel("Number of iteration")
plt.ylabel("Accuracy")
plt.title("CNN: Accuracy vs Number of iteration")
plt.show()


""" Compares different experience replay sampling methods. """
from functools import partial
from collections import OrderedDict

import torch
from torch.nn import functional as F
import pandas as pd

import gym_fast_envs  # pylint: disable=unused-import
from liftoff.config import read_config, config_to_string

from utils import create_paths, configure_experiment, get_ground_truth

def learn(estimator, transition, loss_fn, optimizer, gamma):
    """ Computes the TD Error and optimises the model.

    Args:
        model (torch.nn.Module): An ensemble model.
        loss_fn (function): The loss functions.
        optimizer (torch.optim.Optim): An optimizer.
        transition (list): A s, a, r_, s_, d_, meta tuple.
        gamma (float): The discount factor.

    Returns:
        torch.tensor: The TD Error.
    """
    state, action, reward, state_, mask, _ = transition

    with torch.no_grad():
        q_targets = estimator(state_)

    qsa = estimator(state).gather(1, action)

    qsa_target = torch.zeros_like(qsa)
    qsa_target[mask] = q_targets.max(1, keepdim=True)[0][mask]

    qsa_target = (qsa_target * gamma) + reward
    loss = loss_fn(qsa, qsa_target)
    loss.backward()

    optimizer.step()
    optimizer.zero_grad()
    return loss

def learn_ensemble(ensemble, transition, learner, mem):
    """A wrapper over the `learn` function for optimizing an ensemble of
    estimators (`model`).
    """

    if mem.shuffle:
        transition[5]["boot_mask"] = mask = mem.sample_mask()
    else:
        mask = transition[5]["boot_mask"]

    mids = [mid for mid, m in enumerate(mask) if m != 0]
    for mid in mids:
        # optim.step() is called on all the parameters of the
        # ensemble when we learn a single component of the
        # ensemble. therefore we delete the gradients of all the
        # other estimators to avoid unnecessary operations in
        # optim.step().
        for i, estimator in enumerate(ensemble):
            if i != mid:
                estimator.weight.grad = None

        ensemble_ = partial(ensemble, mid=mid)
        learner(ensemble_, transition)

    return ensemble.var(transition[0], transition[1][0].item())

def train(mem, model, learner, tester, update_cb=None):
    """ The training sequence.
    """
    has_converged = False
    step_cnt = 0

    while not has_converged:
        batch = mem.sample()

        # we are not doing mini-batch learning
        for transition in batch:

            priority = learner(model, transition)

            # update priorities
            if update_cb:
                update_cb(mem, transition, priority)

            step_cnt += 1

            # check for convergence
            has_converged = tester(step_cnt)

            if has_converged:
                break

class Logger:
    def __init__(self, metrics):
        self.__df = pd.DataFrame(columns=metrics)
        self.__idx = 0

    def update(self, loss, step, with_vote=0):
        self.__df.loc[self.__idx] = [step, loss, with_vote]
        self.__idx += 1

    def get_results(self):
        return self.__df

class ConvergenceTester:
    """ Checks for convergence on one or two metrics.
    """

    def __init__(self, model, mdp_size, metrics, log=None, threshold=0.001):
        self.__model = model
        self.__threshold = threshold
        gamma = 1 - 1 / mdp_size
        test_states, optim_q_vals = get_ground_truth(mdp_size, gamma)
        self.__test_states, self.__optim_q_vals = test_states, optim_q_vals
        self.__converged = OrderedDict(
            {metric: {"converged": False, "step_cnt": 0} for metric in metrics}
        )
        if log:
            self.__log = log

    def get_results(self):
        steps = [metric["step_cnt"] for metric in self.__converged.values()]
        return steps, self.__log.get_results()

    def __call__(self, step_cnt):
        return self.__has_converged(step_cnt)

    def __has_converged(self, step_cnt):
        losses = []
        with torch.no_grad():
            y = self.__model(self.__test_states)
            if isinstance(y, tuple):
                for yi in y:
                    losses.append(F.mse_loss(yi, self.__optim_q_vals).item())
            else:
                losses.append(F.mse_loss(y, self.__optim_q_vals).item())

        for i, (k, metric) in enumerate(self.__converged.items()):
            if not metric["converged"]:
                self.__converged[k]["converged"] = losses[i] < self.__threshold
                self.__converged[k]["step_cnt"] = step_cnt

        if self.__log is not None:
            if step_cnt % 100 == 0:
                for i, loss in enumerate(losses):
                    self.__log.update(loss, step_cnt, i)

        return all([m["converged"] for m in self.__converged.values()])

def run(opt):
    """ Experiment trial. """
    lr = 0.25
    learners = [learn, learn_ensemble]

    mem, cb, model, learner, tag = configure_experiment(opt, lr, learners)
    n, mem_size = opt.mdp_size, len(mem)

    # configure tester @ logger
    metrics = ["optim_steps"]
    log_metrics = ["step", "loss", "vote"]
    try:
        if opt.experience_replay.bayesian:
            metrics = ["optim_steps", "vote_optim_steps"]
    except AttributeError:
        pass

    log = Logger(log_metrics)
    tester = ConvergenceTester(model, n, metrics, log=log)

    # run training
    train(mem, model, learner, tester, update_cb=cb)

    # do reporting
    pd.set_option("precision", 4)
    # retrieve logged metrics (optims steps and loss curves)
    optim_steps, results_df = tester.get_results()
    loss = opt.loss if "loss" in opt.__dict__ else "mse"

    # add columns to the loss dataframe curves
    additional_cols = [
        *metrics,
        "N",
        "mem",
        "trial",
        "sampling_type",
        "loss_fn",
    ]
    row_vals = [*optim_steps, n, mem_size, opt.run_id, tag, loss]
    for col, val in zip(additional_cols, row_vals):
        results_df[col] = val

    # save panda
    results_df.to_msgpack(f"./{opt.out_dir}/results.msgpack")

    # log results
    print(config_to_string(opt), flush=True)
    header = f"N={n}, tag={tag}, trial={opt.run_id} results:"
    print("-" * (len(header) + 2))
    print(header, flush=True)
    print("-" * (len(header) + 2))
    print(results_df.head(3), flush=True)

def main():
    """ Entry point. """
    opt = read_config()
    opt = create_paths(opt)
    run(opt)

if __name__ == "__main__":
    main()


__version__ = "1.7"

from curses import wrapper
import getpass
import os
import math
import curses
import argparse
import time
import random
import json

from .globals import *

from mazepy import mazepy
from .curses_utils import curses_utils
from .utils import utils
from .highscores import *
from .gameclasses import Player
from .gameclasses import Goal
from .gameclasses import MazingCell
from .gameclasses import GameGrid
from .GameScreen import GameScreen

#command line args
args=None

def parseCommandLineArgs():
    #parse command line args
    parser = argparse.ArgumentParser(description='MazinGame. A game of maze.')
    parser.add_argument('-l','--level', nargs=1, type=int, metavar='LEVELID',help='Maze level. This integer is a random seed to create the maze.')
    parser.add_argument('-r','--replay', nargs=1, type=int, metavar='GAMEID',help='Replay game with specified id.')
    parser.add_argument('-nf','--nofullscreen', action='store_true', help='Do not use full screen. Default is to show entire maze in terminal, but only if terminal size is larger than the maze.')
    parser.add_argument('--showpath', action='store_true', help='Show shortest path. Remember: this is cheating.')
    parser.add_argument('--showmaze', action='store_true', help='Show entire maze. Remember: this is cheating.')
    parser.add_argument('-hs','--highscores', action='store_true', help='Show high scores. Specify --level to select scores for the level and --cheat to incude cheat scores.')
    parser.add_argument('--cheat', action='store_true', help='Show also cheat highscores.')
    parser.add_argument('-v','--version', action='store_true', help='Show version info.')

    global args
    args = parser.parse_args()

def getHelpLine():
    return "Use cursor keys to move and 'q' to quit."

def start(stdscr,textList):

    height,width = stdscr.getmaxyx()
    if height < FULLSCREEN_MIN_ROWS:
        input("Console screen is too small: %d rows. Full screen needs minimum of %d rows. CTRL-C to exit." % (height,FULLSCREEN_MIN_ROWS))
        return
    if width < FULLSCREEN_MIN_COLS:
        input("Console screen is too small: %d columns. Full screen needs minimum of %d columns. CTRL-C to exit." % (width,FULLSCREEN_MIN_COLS))
        return

    try:
        cursorVisibility=curses.curs_set(0)
    except:
        cursorVisibility=-1
    # Clear screen
    stdscr.clear()
    #call refresh before anything so that pads work
    #http://stackoverflow.com/a/26305933
    stdscr.refresh()

    replayGameId=None
    if args.replay:
        replayGameId=args.replay[0]

    #show welcome screen
    line1="Welcome to MazinGame!"
    if replayGameId is not None:
        line3="REPLAY: game %d" % replayGameId
    else:
        line3="'X' marks the spot. Go there."
    curses_utils.infoWindow(stdscr,line1=line1,line3=line3)

    stdscr.clear()

    useFullTerminal=True
    if args.nofullscreen:
        useFullTerminal=False
    gameScreen=GameScreen(stdscr,useFullTerminal,args)
    level=None
    if args.level:
        level=args.level[0]

    gameScreen.updateStatusLine("Welcome to Maze. 'X' marks the spot. Go there.")#" %s" % getHelpLine())
    if replayGameId is not None:
        #replay
        #get game level and moves from db
        (level,moves)=getGameMoves(replayGameId)
        gameScreen.replayInProgress=True
        gameScreen.initGame(level)
        player=gameScreen.player
        currentRow=player.row
        currentColumn=player.column
        stdscr.nodelay(1)
        statusMsg="complete"
        for move in moves[1:]:
            direction=None
            toRow=move[0]
            toColumn=move[1]
            if toRow>currentRow:
                direction='down'
            if toRow<currentRow:
                direction='up'
            if toColumn>currentColumn:
                direction='right'
            if toColumn<currentColumn:
                direction='left'

            gameScreen.movePlayer(direction)
            currentRow=toRow
            currentColumn=toColumn
            time.sleep(0.1)
            c = stdscr.getch()
            if c == ord('q'):
                statusMsg="aborted"
                break
        textList.append("Replay of game %d %s." % (replayGameId,statusMsg))
    else:
        gameScreen.initGame(level)
        while 1:
            if gameScreen.gameover==False:
                c = stdscr.getch()
            else:
                break
            if c == ord('h'):
                gameScreen.updateStatusLine(getHelpLine())
            elif c == curses.KEY_UP or c== ord('w'):
                gameScreen.movePlayer('up')
            elif c == curses.KEY_DOWN or c== ord('s'):
                gameScreen.movePlayer('down')
            elif c == curses.KEY_LEFT or c== ord('a'):
                gameScreen.movePlayer('left')
            elif c == curses.KEY_RIGHT or c== ord('d'):
                gameScreen.movePlayer('right')
            elif c== ord('i'):
                #info
                player=gameScreen.player
                #gameScreen.gamepad.cursyncup()
                (screenRow,screenColumn)=gameScreen.gamepad.getyx()
                #gameScreen.updateStatusLine("%s screen: (%d,%d) pad corner: (%d,%d)" % (player.__str__(),screenRow,screenColumn,gameScreen.padCornerRow,gameScreen.padCornerColumn))
                gameScreen.updateStatusLine("%s" % (player))
            elif c== ord('I'):
                player=gameScreen.player
                (screenRow,screenColumn)=stdscr.getyx()
                gameScreen.updateStatusLine("%s screen: (%d,%d) pad corner: (%d,%d)" % (player.__str__(),screenRow,screenColumn,gameScreen.padCornerRow,gameScreen.padCornerColumn))
                #gameScreen.updateStatusLine("Screen row,col: (%d,%d)" % (screenRow,screenColumn))
            elif c== ord('T'):
                #stdscr.deleteln()
                #stdscr.insdelln(2)
                infoWindow(stdscr,line1="This is line1",line2="line 2",line3="Then line 3",line4="and line4")
                gameScreen.updateStatusLine("Test: %d" % random.randint(1,99999))
            elif c == ord('q'):
                #utils.debug(time.mktime(time.gmtime()))
                break  # Exit the while()

    if cursorVisibility>-1:
        curses.curs_set(cursorVisibility)

    if not (replayGameId is not None):
        if gameScreen.gameover==True:
            cheat=False
            if args.showpath or args.showmaze:
                cheat=True
            gameid=saveScores(args,__version__,gameScreen.grid,gameScreen.player,gameScreen.goal,gameScreen.level,gameScreen.score,gameScreen.totalMoves,gameScreen.shortestPathLength,gameScreen.elapsed,cheat,gameScreen.grid.algorithm_key)
            textList.append("'X' reached:")
            if gameid == -1:
                textList.append("  Game ID  : %d (score not saved, MazinGame is meant to be run as Docker image)" % gameid)
            else:
                textList.append("  Game ID  : %d" % gameid)
            textList.append("  Level    : %d" % gameScreen.level)
            textList.append("  Algorithm: %s" % gameScreen.grid.algorithm)
            #textList.append("  Braiding : %f" % args.braid)
            textList.append("  Moves    : %d/%d" % (gameScreen.totalMoves,gameScreen.shortestPathLength))
            textList.append("  Elapsed  : %.03fsecs" % gameScreen.elapsed)
            textList.append("  Score    : %d" % gameScreen.score)
            if args.showpath:
                textList.append("  --showpath cheat was active.")
            if args.showmaze:
                textList.append("  --showmaze cheat was active.")
        else:
            textList.append("'X' not reached.")
            #textList.append("  Level  : %d" % gameScreen.level)

def main():
    try:
        parseCommandLineArgs()
        if args.highscores:
            listHighScores(args)
            return
        if args.version:
            print("%s v%s" % (NAME,__version__))
            print("")
            print("\n".join(DESCRIPTION))
            print("")
            print(COPYRIGHT)
            print(LICENSE)
            return
        if args.replay:
            replayGameId=args.replay[0]
            mazeInfo=getMazeInfo(replayGameId)
            if mazeInfo is None:
                print("Can not replay game. No game info.")
                return

        textList=[]
        wrapper(start,textList)
        if textList:
            print("\n".join(textList))
    except KeyboardInterrupt:
        #ignore ctrl-c
        pass

if __name__ == "__main__":
    main()


import numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.cm as cmx
from mpl_toolkits.mplot3d import Axes3D
import glob
from tqdm import *

import math

# video init
fourcc = cv2.cv.CV_FOURCC('m', 'p', '4', 'v')
video = cv2.VideoWriter()

# get list of frames
files = glob.glob("test_flow.0.*")
files.sort()

b = np.load(files[0])
shape = b.f.rho.shape

# make video
success = video.open('flow_video.mov', fourcc, 8, ((shape[1]-2), (shape[0]-2)), True)

# get list of frames
files = glob.glob("test_flow.0.*")
files.sort()

# generate video
for f in files:
  x = np.load(f)
  if len(x.f.v.shape) == 4:
    frame = np.sqrt(np.square(x.f.v[0,1:-1,shape[2]/2,1:-1]) + np.square(x.f.v[1,1:-1,shape[2]/2,1:-1]) + np.square(x.f.v[2,1:-1,shape[2]/2,1:-1]))
    #frame = x.f.rho[:,shape[2]/2,:]
    frame[np.where(np.isnan(frame))] = 1.0
    frame = frame - .5
  elif len(x.f.v.shape) == 3:
    frame = np.sqrt(np.square(x.f.v[0,:,:]) + np.square(x.f.v[1,:,:]))

  frame = np.nan_to_num(frame)
  print(np.max(frame))
  frame = np.uint8(255 * (frame - np.min(frame))/(np.max(frame) - np.min(frame)))
  frame = cv2.applyColorMap(frame, 2)
  #frame = cv2.reshape(frame, (frame.shape[1]/2, frame.shape[0]/2))

  video.write(frame)

# release video
video.release()
cv2.destroyAllWindows()


#!/usr/bin/env python

'''
Use SNMPv3 to detect when a router configuration changes, and send an e-mail notfication

Write a class that stores state in a Python Pickle file, a .json file, or a .yml file.
    Pass the state file to be created / updated in the constructor
    Choode the method based on the filename specified.

Write a driver script (could be the main() method in the file) which could be called by a cron job.
    The driver would instantiate the config_changes class and read its state file

Ex 1. Using SNMPv3 create a script that detects router configuration changes.

If the running configuration has changed, then send an email notification to yourself identifying the router that changed and the time that it changed.

Note, the running configuration of pynet-rtr2 is changing every 15 minutes (roughly at 0, 15, 30, and 45 minutes after the hour).  This will allow you to test your script in the lab environment.

In this exercise, you will possibly need to save data to an external file. One way you can accomplish this is by using a pickle file, see:
    http://youtu.be/ZJOJjyhhEvM

A pickle file lets you save native Python data structures (dictionaries, lists, objects) directly to a file.

Here is some additional reference material that you will probably need to solve this problem:

Cisco routers have the following three OIDs:

# Uptime when running config last changed
ccmHistoryRunningLastChanged = '1.3.6.1.4.1.9.9.43.1.1.1.0'

# Uptime when running config last saved (note any 'write' constitutes a save)
ccmHistoryRunningLastSaved = '1.3.6.1.4.1.9.9.43.1.1.2.0'

# Uptime when startup config last saved
ccmHistoryStartupLastChanged = '1.3.6.1.4.1.9.9.43.1.1.3.0'

From the above descriptions, the router will save the sysUptime timestamp (OID sysUptime = 1.3.6.1.2.1.1.3.0) when a running configuration change occurs. The router will also record the sysUptime timestamp when the running configuration is saved to the startup config.
Here is some data on the behavior of these OIDs. Note, sysUptime times are in hundredths of seconds so 317579 equals 3175.79 seconds (i.e. a bit less than one hour)

# After reboot
pynet-rtr2.twb-tech.com
317579        (sysUptime)
2440            (ccmHistoryRunningLastChanged--running-config is changed during boot)
0                  (ccmHistoryRunningLastSaved -- i.e. reset to 0 on reload)
0                  (ccmHistoryStartupLastChanged -- i.e. reset to 0 on reload)

# After config change on router (but no save to startup config)
pynet-rtr2.twb-tech.com
322522        (sysUptime)
322219        (ccmHistoryRunningLastChanged)
0                  (ccmHistoryRunningLastSaved)
0                  (ccmHistoryStartupLastChanged)

# After 'write mem' on router
pynet-rtr2.twb-tech.com
324543        (sysUptime)
322219        (ccmHistoryRunningLastChanged)
323912        (ccmHistoryRunningLastSaved)
323912        (ccmHistoryStartupLastChanged)

# After another configuration change (but no save to startup config)
pynet-rtr2.twb-tech.com
327177        (sysUptime)
326813        (ccmHistoryRunningLastChanged)
323912        (ccmHistoryRunningLastSaved)
323912        (ccmHistoryStartupLastChanged)

# After 'show run' command (note, this causes 'ccmHistoryRunningLastSaved' to
# increase i.e. 'write terminal' causes this OID to be updated)
pynet-rtr2.twb-tech.com
343223        (sysUptime)
326813        (ccmHistoryRunningLastChanged)
342898        (ccmHistoryRunningLastSaved)
323912        (ccmHistoryStartupLastChanged)

Bonus challenge: instead of saving your data in a pickle file, save the data using either a YAML or a JSON file.

My alternate solution supports pickle, YAML, or JSON depending on the name of the file (.pkl, .yml, or .json).

Input files:
    router list file:
        Contains a list of dictionaries, with the following format:
            [
                {
                    'router': 'hostname1',
                    'ip_addr': 'router1_ip_addr',
                    'community': 'snmp_community',
                    'port': snmp_port
                },
                {
                    'router': 'hostname2',
                    'ip_addr': 'router2_ip_addr',
                    'community': 'snmp_community',
                    'port': snmp_port
                }
            ]
    state file:
        Contains a dictionary of two dictionaries: lastRebootTime and lastConfigChangeTime.
        The router name is the key, and the time is the value.
            {
                'lastRebootTime': {
                    'router_name1': 1,
                    'router_name2': 22,
                    'router_name3': 333,
                    'router_name4': 4444,
                    'router_name5': 55555
                },
                'lastConfigChangeTime': {
                    'router_name1': 88888,
                    'router_name2': 77777,
                    'router_name3': 6666,
                    'router_name4': 54444,
                    'router_name5': 555552
                }
            }

Parameters:
    check interval in seconds
    Yaml file or json file containing state
    Yaml or Json file containing router list

Algorithm:
    load routerList from router list file
    Get current date from OS
    For router in routerList:
        SNMP-Get router sysUptime
        routerRebootTime = date - sysUptime
        SNMP-Get current ccmHistoryRunningLastChanged
        runningLastChangedTime = routerRebootTime + ccmHistoryRunningLastChanged
        runningConfigChanged = False
        If state file:
            routerRebootTimeS = stateFile.routerRebootTime
            runningLastChangedTimeS = stateFile.runningLastChangedTime
            If routerRebootTime > routerRebootTimeS
                routerRebooted = True
                routerRebootTimeS = routerRebootTime
            If runningLastChangedTime > runningLastChangedTimeS:
                If !routerRebooted or (ccmHistoryRunningLastChanged > ROUTER_REBOOT_SECONDS):
                    runningConfigChanged = True
                    runningLastChangedTimeS = runningLastChangedTime
            if runningConfigChanged:
                send email
        stateFile.dump(routerRebootTimeS, runningLastChangedTimeS)

'''
import sys
import pickle
import yaml
import json
import time
import snmp_helper
import email_helper
from collections import namedtuple

ROUTER_REBOOT_SECONDS = 300 * 100 # converted to timeticks aka 100ths of a second
PICKLE_STATE_FILE = "Router_Reboot_Check.p"
SYSUPTIME_OID = '1.3.6.1.2.1.1.3.0'
RUNNINGLASTCHANGED_OID = '1.3.6.1.4.1.9.9.43.1.1.1.0'
EMAIL_SUBJECT = "%s configuration change detected"
EMAIL_MESSAGE_BODY = "Configuration on router %s was last changed at %s"
DEBUG = False
NetworkDevice = namedtuple("my_name", "field1 field2 field3")

class Router_Reboot_Check(object):
    def __init__(self, router_list_file, state_file, email_sender, email_recipients,
                 router_reboot_seconds=ROUTER_REBOOT_SECONDS):
        self.router_list_file = router_list_file
        self.state_file = state_file
        self.email_sender = email_sender
        self.email_recipients = email_recipients
        self.router_reboot_seconds = router_reboot_seconds

        try:
            if router_list_file.endswith("yml"):
                self.router_list = yaml.load(open(router_list_file, 'r'))
            elif router_list_file.endswith("json"):
                self.router_list = json.load(open(router_list_file, 'r'))
            else:
                sys.exit("Must specify router list as a .yml or .json file")
        except IOError as ioe:
            sys.exit("Cannot load router list file: %s" % router_list_file)
        if DEBUG:
            print("Router List: " + str(self.router_list))
        try:
            if state_file.endswith("yml"):
                self.state = yaml.load(open(state_file, 'r'))
            elif state_file.endswith("json"):
                self.state = json.load(open(state_file, 'r'))
            else:
                self.state = pickle.load(open(PICKLE_STATE_FILE, 'rb'))
        except IOError as ioe:
            self.state = {}
        if DEBUG:
            print("State length: %s, State: %s" % (len(self.state), str(self.state)))

    def check_routers(self):
        timestamp = int(time.time())
        if DEBUG:
            print("My time is: %s" % time.ctime(timestamp))
        for router in self.router_list:
            routerRebooted = False
            routerTuple = (router['ip_addr'], router['community'], router['port'])
            routerUptimeOIDData = snmp_helper.snmp_get_oid(routerTuple, SYSUPTIME_OID)
            routerUptime = int(float(snmp_helper.snmp_extract(routerUptimeOIDData)) / 100)
            routerRebootTime = timestamp - routerUptime

            lastConfigChangeSecondsOIDData = snmp_helper.snmp_get_oid(routerTuple, RUNNINGLASTCHANGED_OID)
            lastConfigChangeSeconds = int(float(snmp_helper.snmp_extract(lastConfigChangeSecondsOIDData)) / 100)
            runningLastChangedTime = routerRebootTime + lastConfigChangeSeconds
            if DEBUG:
                print("Router: %s     uptime: %s    reboot time: %s    last config change seconds: %s    running last changed time %s" % (router['router'], routerUptime, time.ctime(routerRebootTime), lastConfigChangeSeconds, time.ctime(runningLastChangedTime)))

            if len(self.state) == 0:
                self.state['lastRebootTime'] = {}
                self.state['lastConfigChangeTime'] = {}
            if router['router'] not in self.state['lastRebootTime'].keys():
                self.state['lastRebootTime'][router['router']] = routerRebootTime
                self.state['lastConfigChangeTime'][router['router']] = runningLastChangedTime
            if routerRebootTime > self.state['lastRebootTime'][router['router']]:
                routerRebooted = True
                self.state['lastRebootTime'][router['router']] = routerRebootTime
            if runningLastChangedTime > self.state['lastConfigChangeTime'][router['router']]:
                if not routerRebooted or (lastConfigChangeSeconds > ROUTER_REBOOT_SECONDS):
                    self.__send_email_alert(router['router'], runningLastChangedTime)
                self.state['lastConfigChangeTime'][router['router']] = runningLastChangedTime
        self.__dump_state()

    def __send_email_alert(self, router_name, lastChangedTime):
        for email_recipient in self.email_recipients:
            subject = str(EMAIL_SUBJECT % router_name)
            message = str(EMAIL_MESSAGE_BODY % (router_name, time.ctime(lastChangedTime)))
            print("Sending mail to %s with subject \"%s\" and message \"%s\" from %s" %
                  (email_recipient, subject, message, self.email_sender))
            email_helper.send_mail(email_recipient, subject, message, self.email_sender)

    def __dump_state(self):
        try:
            if self.state_file.endswith("yml"):
                with open(self.state_file, "w") as f:
                    f.write(yaml.dump(self.state, default_flow_style=False))
            elif self.state_file.endswith("json"):
                json.dump(self.state, open(self.state_file, 'w'))
            else:
                pickle.dump(self.state, open(self.state_file, "wb"))
        except IOError:
            sys.exit("Could not write state file %s" % self.state_file)

def main():
    if len(sys.argv) < 4:
        sys.exit("Usage: %s <router_list_file> <email_sender> <email_recipient1> [state_file] [email_recipient2] ..." % sys.argv[0])
    router_list_file = sys.argv[1]
    email_sender = sys.argv[2]
    email_recipients = [sys.argv[3]]
    state_file = ""
    if len(sys.argv) > 4:
        if sys.argv[4].endswith("yml") or sys.argv[4].endswith("json"):
            state_file = sys.argv[4]
        else:
            email_recipients.append(sys.argv[4])
            state_file = PICKLE_STATE_FILE
    for remainingArg in sys.argv[5:len(sys.argv)]:
        email_recipients.append(remainingArg)
    if DEBUG:
        print("Script name: %s, router_list_file: %s, state_file: %s, email_sender: %s, email_recipients: %s"
              % (sys.argv[0], router_list_file, state_file, email_sender, str(email_recipients)))
    my_router_reboot_check = Router_Reboot_Check(router_list_file, state_file, email_sender, email_recipients)
    my_router_reboot_check.check_routers()

if __name__ == "__main__":
    main()


from django.shortcuts import render,get_object_or_404
from django.core.paginator import Paginator
from .models import Listings

# Create your views here.
def index(request):
	listing = Listings.objects.order_by('-list_date').filter(is_published=True)
	paginator = Paginator(listing,6)
	page = request.GET.get('page')
	paged_listings = paginator.get_page(page)

	context = { 'listing': paged_listings }
	return render(request,'listings/listings.html',context)

def listing(request,list_id):

	list_fetch = get_object_or_404(Listings,pk=list_id)

	context={
		'list':list_fetch
	}
	return render(request,'listings/listing.html',context)

def search(request):

	return render(request,'listings/search.html')


from datetime import datetime

import pytest
from freezegun import freeze_time

from hr_core.constants.enums import AttendanceStatusType
from hr_core.models.attendance import Attendance
from hr_core.models.employee import Employee
from hr_core.storages.storage_implementation import StorageImplementation

class TestCreateAndGetClockoutAttendance:
    @pytest.mark.django_db
    @freeze_time(str(datetime.now()))
    def test_create_and_get_clockout_attendance(self, create_employees):
        # Arrange
        sql_storage = StorageImplementation()
        employee_obj = Employee.objects.get(employee_id="1")
        Attendance.objects.create(
            employee=employee_obj,
            clock_in_datetime=datetime.now(),
            status=AttendanceStatusType.PRESENT.value
        )

        # Act
        attendance_dto = sql_storage.create_and_get_clockout_attendance(employee_id="1")
        attendance_obj = Attendance.objects.get(id=attendance_dto.attendance_id)

        # Assert
        assert attendance_obj.clock_out_datetime == datetime.now()
        assert attendance_obj.employee.employee_id == "1"
        assert attendance_dto.attendance_id == attendance_obj.id
        assert attendance_dto.clock_out_date_time == attendance_obj.clock_out_datetime


import os
from pymoo.mcdm.high_tradeoff import HighTradeoffPoints
from sklearn.preprocessing import MinMaxScaler
import pareto
import pandas as pd
from edbo.plus.optimizer_botorch import EDBOplus
from edbo.plus.optimizer import EDBOplus as EDBO
import torch
from botorch.utils.multi_objective.hypervolume import Hypervolume
from scipy.spatial.distance import euclidean
import copy
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

def is_pareto(objectives):
    """
    Find the pareto-efficient points.

    Parameters
    ----------
    objectives: array
        array containing all the objectives values.
    Returns
    -------
    is_efficient
        list of booleans, True: Pareto efficient and False: not Pareto efficient.

    """
    is_efficient = np.ones(objectives.shape[0], dtype=bool)
    for i, c in enumerate(objectives):
        is_efficient[i] = np.all(np.any(objectives[:i]>c, axis=1)) and np.all(np.any(objectives[i+1:]>c, axis=1))
    return is_efficient

class Benchmark():

    """
    Class for Benchmarking HTE datasets.
    """

    def __init__(self,
                 df_ground, index_column,
                 objective_names, objective_modes,
                 features_regression='all',
                 objective_thresholds=None,
                 filename='benchmark.csv',
                 acquisition_function='EHVI',
                 filename_results='benchmark_results.csv'):
        """

        Parameters
        ----------
        df_ground: Pandas dataframe.
            Pandas dataframe containing the features and objectives.
        index_column: string
            String with the name of the index column of the 'df_ground' Pandas dataframe.
        objective_names: list
            list of strings including the names of the columns that will be considered as objectives.
        objective_modes: list
            list of strings for deciding whether the objective has to be minimize ('min') or maximize ('max').
        features_regression: list
            list of strings containing the names of the columns that will be considered as features for the regression model.
        objective_thresholds: list
            list of floats containing the threshold values for each objective.
        acquisition_function: string
            name of the acquisition function to use (implemented are: 'EHVI', 'MOUCB', 'MOGreedy').
        filename: string
            name of the file to run the optimization (temporary file).
        filename_results:
            name of the file to store the results of the benchmark.
        """
        self.df_ground = df_ground
        self.objective_names = objective_names
        self.objective_modes = objective_modes
        self.objective_thresholds = objective_thresholds
        self.features_regressions = features_regression
        self.filename = filename
        self.filename_results = filename_results
        self.index_column = index_column
        self.acquisition_function = acquisition_function

        # Safe check. Check whether there are no duplicates in the regression domain.
        df_check = self.df_ground[self.features_regressions]
        msg = 'There are entries that are degenerate. Check your dataset.'
        assert len(df_check) == len(df_check.drop_duplicates()), msg

        # Calculate data for the ground truth function.
        self.objective_values_ground = self.get_objective_values(df=self.df_ground)
        self.pareto_ground, self.idx_pareto_ground = self.get_pareto_points(objective_values=self.objective_values_ground)
        self.tradeoff_ground = self.get_high_tradeoff_points(
                pareto_points=self.pareto_ground)
        print('High trade-off ground truth:', self.tradeoff_ground)

        # Fit scaler for a fair comparison between objectives.
        self.scaler_ground = MinMaxScaler()
        self.scaler_ground.fit(self.objective_values_ground)

        # Calculate Hypervolume using the ground scaler.
        self.hypervolume_ground = self.get_hypervolume(
                pareto_points=self.pareto_ground,
                thresholds=self.objective_thresholds)

        # Instanciate EDBOplus.
        if self.acquisition_function == 'EHVI':
            self.edbo = EDBOplus()
        else:
            self.edbo = EDBO()

        # Generate DataFrame for storing collected data and predictions.
        if not os.path.exists(filename):
            # Create first scope if filename is not found.
            df_edbo = self.df_ground.copy(deep=True)
            df_edbo = df_edbo.drop(columns=self.objective_names)
            df_edbo.to_csv(filename, index=False)

    def get_hypervolume(self, pareto_points, thresholds):
        """
        Calculate hypervolume using a reference point for each objective
        (if None it takes automatically the minimum of each objective in the ground truth).
        Note 1: It is always scaled w.r.t. the ground truth for a fair comparison between objectives.
        Note 2: Assumes maximization.
        """

        references = copy.deepcopy(thresholds)

        if references is None:
            references = [None] * len(self.objective_names)

        for i in range(0, len(references)):
            if references[i] is None:
                references[i] = np.min(self.objective_values_ground, axis=0)[i]

        references_scaled = self.scaler_ground.transform(np.array([references]))[0]
        pareto_points_scaled = self.scaler_ground.transform(pareto_points)

        pareto_torch = torch.Tensor(pareto_points_scaled)
        hv = Hypervolume(ref_point=torch.Tensor(references_scaled))
        hypervolume = hv.compute(pareto_Y=pareto_torch)

        return hypervolume

    def get_objective_values(self, df):  # Maximizing.
        """ Get a list of objective values from a given dataframe (for instance yields and cost)."""
        predata = []
        for obj in range(0, len(self.objective_names)):
            d = [float(i) for i in df[self.objective_names[obj]]]
            if self.objective_modes[obj] == 'min':
                d = -np.array(d)
            else:
                d = np.array(d)
            predata.append(d.flatten())
        objective_values = np.array(predata).T

        # Calculate also bounds (best and worst samples).
        max_bounds = []
        min_bounds = []
        for i in range(0, len(self.objective_names)):
            if self.objective_modes[i] == 'min':
                max_bounds.append(-np.max(self.df_ground[self.objective_names[i]]))
                min_bounds.append(-np.min(self.df_ground[self.objective_names[i]]))
            if self.objective_modes[i] == 'max':
                max_bounds.append(np.max(self.df_ground[self.objective_names[i]]))
                min_bounds.append(np.min(self.df_ground[self.objective_names[i]]))
        self.max_bounds = max_bounds
        self.min_bounds = min_bounds

        return objective_values

    def get_pareto_points(self, objective_values):
        """ Get pareto for the ground truth function.
        NOTE: Assumes maximization."""
        pareto_ground = pareto.eps_sort(tables=objective_values,
                                        objectives=np.arange(len(self.objective_names)),
                                        maximize_all=True)
        idx_pareto = is_pareto(objectives=-objective_values)
        return np.array(pareto_ground), idx_pareto

    def get_high_tradeoff_points(self, pareto_points):
        """ Pass a numpy array with the pareto points and returns a numpy
            array with the high tradeoff points."""

        scaler_pareto = MinMaxScaler()
        pareto_scaled = scaler_pareto.fit_transform(pareto_points)
        try:
            tradeoff = HighTradeoffPoints()

            tradeoff_args = tradeoff.do(-pareto_scaled)  # Always minimizing.
            tradeoff_points = pareto_points[tradeoff_args]
        except:
            tradeoff_points = []
            pass
        return tradeoff_points

    def get_distance_tradeoff_to_ground(self, set_of_points):
        """
        Max. minimum Euclidean distance between a set of points and
        the ground truth tradeoff point(s).
        """
        if len(self.tradeoff_ground) != 0:
            d_ij = []
            try:
                for tradeoff_point_ground in self.tradeoff_ground:
                    d_i = []
                    for point in set_of_points:
                        d_i.append(euclidean(point, tradeoff_point_ground))
                    d_ij.append(np.min(d_i))
            except:
                for tradeoff_point_ground in self.tradeoff_ground[0]:
                    d_i = []
                    for point in set_of_points:
                        d_i.append(euclidean(point, tradeoff_point_ground))
                    d_ij.append(np.min(d_i))
            return np.max(d_ij)
        else:
            return 'NA'

    def get_maximin_distance_pareto_to_ground(self, pareto_set):
        """
        Calculate maximin distance between a pareto set and the ground truth
        pareto. Also called Hausdorff
        """
        d_ij = []
        for pareto_point_ground in self.pareto_ground:
            d_i = []
            for point in pareto_set:
                d_i.append(euclidean(point, pareto_point_ground))
            d_ij.append(np.min(d_i))
        return np.max(d_ij)

    def get_predictions_errors(self):
        """
        Obtain errors in the predictions.
        """
        dict_pred_errors = {}
        df_ground = self.df_ground.copy(deep=True)

        if os.path.exists(f'pred_{self.filename}'):
            df_preds = pd.read_csv(f'pred_{self.filename}')
            df_preds = df_preds.sort_values(by=self.index_column)
            df_ground = df_ground.sort_values(by=self.index_column)

            # Collect information about the MAE, RMSE, ,
            for objective in self.objective_names:
                y_true = df_ground[objective].values
                y_pred = df_preds[f"{objective}_predicted_mean"].values

                mae = (np.sum(np.abs(y_pred - y_true))) / len(y_pred)
                rmse = np.sqrt((np.sum((y_pred - y_true)**2))/len(y_pred))

                from sklearn.metrics import r2_score

                r2 = r2_score(y_true=y_true, y_pred=y_pred)

                dict_pred_errors.update({
                    f"MAE_{objective}": mae,
                    f"RMSE_{objective}": rmse,
                    f"R2_{objective}": r2,
                    })
            print(dict_pred_errors)
        else:
            for objective in self.objective_names:
                dict_pred_errors.update({
                    f"MAE_{objective}": 'NaN',
                    f"RMSE_{objective}": 'NaN',
                    f"R2_{objective}": 'NaN',
                })

        return dict_pred_errors

    def _store_benchmark(self):
        for bt in range(self.batch):
            dict_i = {'step': self.step,
                      'init_method': self.init_method,
                      'init_sample': self.seed,
                      'batch': self.batch,
                      'n_experiments': self.n_experiments,
                      'thresholds': self.objective_thresholds,
                      'dmaximin_pareto': self.pareto_distance_ground_train,
                      'acquisition': self.acquisition_function,
                      'dmaximin_tradeoff': self.tradeoff_distance_ground_train,
                      'hypervolume_ground': self.hypervolume_ground,
                      'hypervolume_sampled': self.hypervolume_train,
                      'hypervolume completed (%)': (self.hypervolume_train/self.hypervolume_ground) * 100,
                      'sample_index': self.idx_next_samples[bt],
            }

            dict_i.update(self.predicted_errors)

            sample_vals = {}
            for kcol in self.collected_values.keys():
                sample_vals[kcol] = self.collected_values[kcol][bt]
            dict_i.update(sample_vals)
            best_samples_values = {}

            for i in range(0, len(self.objective_names)):
                best_i = self.best_values_found[i]
                best_samples_values.update({f"{self.objective_names[i]}_best": best_i})

            dict_i.update(best_samples_values)

            store_columns = list(dict_i.keys())

            if os.path.exists(self.filename_results):
                df_results = pd.read_csv(self.filename_results)
            else:
                df_results = pd.DataFrame(columns=store_columns)

            df_results = df_results.append(dict_i, ignore_index=True)
            df_results.to_csv(self.filename_results, index=False)

        return df_results

    def run(self, steps, batch, seed=0,
            init_method='seed',
            run_folder='./results', plot_ground=True,
            plot_train=False, plot_predictions=False,
            random_sampling=False):

        self.init_method=init_method
        self.seed = seed
        self.batch = batch
        self.run_folder = run_folder

        if not os.path.exists(run_folder):
            os.mkdir(run_folder)

        if plot_ground is True:
            if len(self.objective_names) == 2:
                self._plot_ground_2d()
            if len(self.objective_names) == 3:
                self._plot_ground_3d()

        for step in range(0, steps):
            self.step = step

            df_run = self.edbo.run(
                    filename=self.filename,
                    batch=batch,
                    columns_features=self.features_regressions,
                    objectives=self.objective_names,
                    objective_mode=self.objective_modes,
                    objective_thresholds=self.objective_thresholds,
                    acquisition_function=self.acquisition_function,
                    init_sampling_method=self.init_method,
                    seed=seed)

            df_next = df_run[df_run['priority'] >= 0.5]
            idx_next = df_next[self.index_column].values

            if len(idx_next) < batch:  # Add random if not enough.
                df_next = df_run[df_run['priority'] >= 0]
                idx_next_2 = df_next[self.index_column].values
                l_choice = np.arange(0, len(idx_next_2))
                r_choices = np.random.choice(l_choice, size=(batch-len(idx_next)))
                random_choice_idx = idx_next_2[r_choices]
                idx_next = np.append(idx_next, random_choice_idx)

            self.idx_next_samples = idx_next
            # Test performance of the model with large training data.
            # idx_next = np.random.choice(len(df_run), int(len(df_run)*0.2))
            if random_sampling is True:
                df_next = df_run[df_run['priority'] >= 0]
                idx_next = df_next[self.index_column].values
                choice_idx = idx_next[np.random.choice(np.arange(0, len(idx_next)), size=batch)]
                idx_next = choice_idx

            # Append solutions.
            self.collected_values = {}
            for id in idx_next:
                argwhere_idx_next_ground = np.argwhere(self.df_ground[self.index_column].values == id)[0][0]
                argwhere_dix_next_run = np.argwhere(df_run[self.index_column].values == id)[0][0]
                for obj in self.objective_names:
                    df_run[obj].values[argwhere_dix_next_run] = self.df_ground[obj].values[argwhere_idx_next_ground]
                    try:
                        self.collected_values[obj + "_collected_values"].append(self.df_ground[obj].values[argwhere_idx_next_ground])
                    except:
                        self.collected_values[obj + "_collected_values"] = [self.df_ground[obj].values[argwhere_idx_next_ground]]

            # Update the DataFrame with the new collected data.
            df_run.to_csv(self.filename, index=False)

            new_df = df_run.copy(deep=True)
            # Print best values for each objective and save sampled objectives.
            cumulative_train_y = []
            self.best_values_found = []
            for i in range(0, len(self.objective_names)):
                if self.objective_modes[i] == 'min':
                    best_value = pd.to_numeric(new_df[self.objective_names[i]], 'coerce').min()
                else:
                    best_value = pd.to_numeric(new_df[self.objective_names[i]], 'coerce').max()
                print(f"Best {self.objective_names[i]} found: {best_value}.")
                self.best_values_found.append(best_value)
                vals = pd.to_numeric(new_df[self.objective_names[i]], 'coerce').dropna().values
                if self.objective_modes[i] == 'min':
                    cumulative_train_y.append(-vals)
                else:
                    cumulative_train_y.append(vals)
            cumulative_train_y = np.reshape(cumulative_train_y, (len(cumulative_train_y), -1)).T

            self.pareto_train, self.idx_pareto_train = self.get_pareto_points(objective_values=cumulative_train_y)
            self.tradeoff_train = self.get_high_tradeoff_points(pareto_points=self.pareto_train)
            self.hypervolume_train = self.get_hypervolume(pareto_points=self.pareto_train,
                                                          thresholds=self.objective_thresholds)

            self.tradeoff_distance_ground_train = \
                self.get_distance_tradeoff_to_ground(
                        set_of_points=cumulative_train_y)
            self.pareto_distance_ground_train = \
                    self.get_maximin_distance_pareto_to_ground(
                            pareto_set=self.pareto_train)

            self.n_experiments = len(cumulative_train_y)

            print('Total number of experiments:', self.n_experiments)

            # Print collected samples.
            print('Collected samples \n', df_run[df_run['priority'] < 0])

            print(f"Hypervolume train (w.r.t to ground truth in %): "
                  f"{(self.hypervolume_train/self.hypervolume_ground) * 100}")

            print(f"Maximin distance any train to any ground truth "
                  f"Pareto: {self.pareto_distance_ground_train}")
            print(f"Maximin distance any train to ground truth "
                  f"Tradeoff: {self.tradeoff_distance_ground_train}")

            # Store all information.

            self.predicted_errors = self.get_predictions_errors()
            self._store_benchmark()
            print("\n \n")

            # Get plots.
            self.cumulative_train_y = cumulative_train_y

            if plot_train:
                if len(self.objective_names) == 2:
                    self._plot_train_pareto_2d()
                if len(self.objective_names) == 3:
                    self._plot_train_pareto_3d()

            if plot_predictions and os.path.exists(f"pred_{self.filename}"):
                self._plot_predictions()

    def _plot_predictions(self):

        df_preds = pd.read_csv(f'pred_{self.filename}')
        df_ground = self.df_ground.copy(deep=True)
        df_preds = df_preds.sort_values(by=self.index_column)
        df_ground = df_ground.sort_values(by=self.index_column)

        total_entries = len(df_preds)
        n_columns = 30
        n_rows = int(total_entries / n_columns) + 1

        for i in self.objective_names:
            ground = [np.nan] * n_columns * n_rows
            pred_mean = [np.nan] * n_columns * n_rows
            ei = [np.nan] * n_columns * n_rows
            ground[0:total_entries] = self.df_ground[i].values
            pred_mean[0:total_entries] = df_preds[f"{i}_predicted_mean"].values
            ei[0:total_entries] = df_preds[f"{i}_expected_improvement"].values
            ground = np.reshape(ground, (n_rows, n_columns))
            pred_mean = np.reshape(pred_mean, (n_rows, n_columns))
            ei = np.reshape(ei, (n_rows, n_columns))

            min_threshold = np.min(df_ground[i].values)
            max_threshold = np.max(df_ground[i].values)

            f, (ax1, ax2, ax3) = plt.subplots(
                1, 3,
                figsize=(n_columns, n_rows),
                gridspec_kw={
                  'width_ratios': [1, 1, 1]
                }
            )

            g1 = sns.heatmap(
                ground, cmap='Spectral_r', square=True,
                xticklabels=[], yticklabels=[],
                linewidths=0.7, cbar=False,
                cbar_kws=dict(use_gridspec=False, location="bottom"),
                vmin=min_threshold, vmax=max_threshold,
                ax=ax1
            )

            g2 = sns.heatmap(
                pred_mean, cmap='Spectral_r', square=True,
                xticklabels=[], yticklabels=[],
                linewidths=0.7, cbar=False,
                cbar_kws=dict(use_gridspec=False, location="bottom"),
                vmin=min_threshold, vmax=max_threshold,
                ax=ax2
            )

            g3 = sns.heatmap(
                ei,
                cmap='Blues', square=True,
                xticklabels=[], yticklabels=[],
                linewidths=0.7, cbar=False,
                cbar_kws=dict(use_gridspec=False, location="bottom"),
                vmin=np.min(df_preds[f"{i}_expected_improvement"].values),
                vmax=np.max(df_preds[f"{i}_expected_improvement"].values),
                ax=ax3
            )

            ax1.set_title(f"Ground truth ({i})")
            ax2.set_title(f"Predicted mean ({i})")
            ax3.set_title(f"Expected improvement ({i})")
            plt.tight_layout()
            plt.show()

    def _plot_ground_2d(self):

        fig, ax = plt.subplots(figsize=(10, 10))
        ax.scatter(self.objective_values_ground[:, 0],
                   self.objective_values_ground[:, 1], s=30, lw=1.,
                   edgecolors='black')
        ax.scatter(self.pareto_ground[:, 0], self.pareto_ground[:, 1],
                   s=100, c='orange', lw=1., edgecolors='black')

        import matplotlib.patches as patches
        ax.add_patch(
            patches.Rectangle(
                tuple(self.min_bounds),
                self.max_bounds[0],
                self.max_bounds[1],
                facecolor="red",
                alpha=0.1
            )
        )

        idx_pareto = np.argwhere(self.idx_pareto_ground)
        print('Indices pareto front (ground truth):', idx_pareto)
        for i in range(0, len(idx_pareto)):
            idxi = idx_pareto[i][0]
            ax.text(x=self.objective_values_ground[idxi][0] +0.,
                    y=self.objective_values_ground[idxi][1] +0.,
                    s=str(idxi),
                    )

        ax.set_xlabel(f"{self.objective_names[0]} ({self.objective_modes[0]})")
        ax.set_ylabel(f"{self.objective_names[1]} ({self.objective_modes[1]})")

        if not os.path.exists(f"{self.run_folder}_plots"):
            os.mkdir(f"{self.run_folder}_plots")
        plt.rc("axes.spines", top=False, right=False)
        plt.savefig(f"{self.run_folder}_plots/{self.filename_results}_ground.svg")
        plt.show()

    def _plot_ground_3d(self):

        plt.rcParams['grid.color'] = "lightgray"
        plt.rcParams['grid.linestyle'] = [6., 2., 6., 2.]
        plt.rcParams['grid.linewidth'] = 1.5

        # Scatter plots 3d.
        fig = plt.figure(figsize=(10, 10))
        ax = fig.add_subplot(projection='3d')

        # Plot properties.
        ax.view_init(elev=25, azim=290)
        fig.set_facecolor('white')
        ax.set_facecolor('white')

        ax.scatter(self.objective_values_ground[:, 0],
                   self.objective_values_ground[:, 1],
                   self.objective_values_ground[:, 2],
                   s=100, lw=1.,
                   color='C0',
                   zorder=1.,
                   edgecolors='gray'
                   )

        ax.scatter(self.pareto_ground[:, 0],
                   self.pareto_ground[:, 1],
                   self.pareto_ground[:, 2],
                   marker='o', s=100, c='C1', lw=1.,
                   alpha=0.8,
                   zorder=10.,
                   edgecolors='gray',
                   )

        xmin = np.min(self.objective_values_ground[:, 0])
        xmax = np.max(self.objective_values_ground[:, 0])
        ymin = np.min(self.objective_values_ground[:, 1])
        ymax = np.max(self.objective_values_ground[:, 1])
        zmin = np.min(self.objective_values_ground[:, 2])
        zmax = np.max(self.objective_values_ground[:, 2])

        ax.axes.set_xlim3d(xmin, xmax)
        ax.axes.set_ylim3d(ymin, ymax)
        ax.axes.set_zlim3d(zmin, zmax)

        ax.set_xlabel(f"{self.objective_names[0]} ({self.objective_modes[0]})")
        ax.set_ylabel(f"{self.objective_names[1]} ({self.objective_modes[1]})")
        ax.set_zlabel(f"{self.objective_names[2]} ({self.objective_modes[2]})")

        if not os.path.exists(f"{self.run_folder}_plots"):
            os.mkdir(f"{self.run_folder}_plots")

        plt.savefig(f"{self.run_folder}_plots/{self.filename_results}_ground.svg")

        plt.show()

    def _plot_train_pareto_2d(self):

        fig, ax = plt.subplots(figsize=(10, 10))
        ax.scatter(self.cumulative_train_y[:, 0],
                   self.cumulative_train_y[:, 1], s=30, lw=1.,
                   edgecolors='black')
        ax.scatter(self.pareto_train[:, 0], self.pareto_train[:, 1],
                   s=100, c='orange', lw=1., edgecolors='black')
        ax.scatter(self.pareto_ground[:, 0], self.pareto_ground[:, 1],
                   marker='x', s=100, c='black', lw=1.)

        ax.set_xlabel(f"{self.objective_names[0]} ({self.objective_modes[0]})")
        ax.set_ylabel(f"{self.objective_names[1]} ({self.objective_modes[1]})")

        if not os.path.exists(f"{self.run_folder}_plots"):
            os.mkdir(f"{self.run_folder}_plots")

        plt.savefig(f"{self.run_folder}_plots/{self.filename_results}_train_{self.step}.svg")

        plt.show()

    def _plot_train_pareto_3d(self):

        plt.rcParams['grid.color'] = "lightgray"
        plt.rcParams['grid.linestyle'] = [6., 2., 6., 2.]
        plt.rcParams['grid.linewidth'] = 1.5

        # Scatter plots 3d.
        fig = plt.figure(figsize=(10, 10))
        ax = fig.add_subplot(projection='3d')

        # Plot properties.
        ax.view_init(elev=25, azim=290)
        fig.set_facecolor('white')
        ax.set_facecolor('white')

        ax.scatter(self.cumulative_train_y[:, 0],
                   self.cumulative_train_y[:, 1],
                   self.cumulative_train_y[:, 2],
                   s=100, lw=1.,
                   color='C0',
                   zorder=1.,
                   edgecolors='gray'
                   )

        ax.scatter(self.pareto_ground[:, 0],
                   self.pareto_ground[:, 1],
                   self.pareto_ground[:, 2],
                   marker='o', s=100, c='C1', lw=1.,
                   alpha=0.8,
                   zorder=10.,
                   edgecolors='gray',
                   )

        xmin = np.min(self.objective_values_ground[:, 0])
        xmax = np.max(self.objective_values_ground[:, 0])
        ymin = np.min(self.objective_values_ground[:, 1])
        ymax = np.max(self.objective_values_ground[:, 1])
        zmin = np.min(self.objective_values_ground[:, 2])
        zmax = np.max(self.objective_values_ground[:, 2])

        ax.axes.set_xlim3d(xmin, xmax)
        ax.axes.set_ylim3d(ymin, ymax)
        ax.axes.set_zlim3d(zmin, zmax)

        ax.set_xlabel(f"{self.objective_names[0]} ({self.objective_modes[0]})")
        ax.set_ylabel(f"{self.objective_names[1]} ({self.objective_modes[1]})")
        ax.set_zlabel(f"{self.objective_names[2]} ({self.objective_modes[2]})")

        if not os.path.exists(f"{self.run_folder}_plots"):
            os.mkdir(f"{self.run_folder}_plots")

        plt.savefig(f"{self.run_folder}_plots/{self.filename_results}_train_{self.step}.svg")

        plt.show()


import cv2
import imutils

img_color = cv2.imread('dogs.jpg', cv2.IMREAD_COLOR)

dst = imutils.translate(img_color, 25, -75)

dst = imutils.rotate(img_color, angle=45, center=(0,0))

dst = imutils.auto_canny(img_color)

cv2.imshow('color', dst)

cv2.waitKey(0)

cv2.destroyAllWindows()


from django.shortcuts import render
from eventos.models import Evento
from django.urls import reverse_lazy
from django.views.generic import ListView, CreateView, UpdateView, DeleteView
from eventos.forms import eventoForm
# Create your views here.

def eventos_lista(request):
    evento = Evento.objects.order_by('fecha_inic')
    return render(request, 'eventos/listar.html',{'eventos': evento})

class eventosList(ListView):
    model = Evento
    template_name = 'eventos/listar.html'

class eventoCreate(CreateView):
    model = Evento
    form_class = eventoForm
    template_name = 'eventos/evento_form.html'
    success_url = reverse_lazy('eventos_listar')

class eventoUpdate(UpdateView):
    model = Evento
    form_class = eventoForm
    template_name = 'eventos/evento_form.html'
    success_url = reverse_lazy('eventos_listar')

class eventoDelete(DeleteView):
    model = Evento
    template_name = 'eventos/evento_delete.html'
    success_url = reverse_lazy('eventos_listar')