import matplotlib.pyplot as plt
import sklearn.datasets
import numpy as np
import sklearn.linear_model
import matplotlib

def initialize(input_num, layer1_num, layer2_num):
    model = dict()
    np.random.seed(12)
    w1 = np.random.randn(input_num, layer1_num)
    b1 = np.random.randn(1, layer1_num)
    w2 = np.random.randn(layer1_num, layer2_num)
    b2 = np.random.randn(1, layer2_num)
    model['w1'] = w1
    model['b1'] = b1
    model['w2'] = w2
    model['b2'] = b2
    return model

def predict(model, x):
    w1, b1, w2, b2 = model['w1'], model['b1'], model['w2'], model['b2']
    z1 = x.dot(w1) + b1
    a1 = np.tanh(z1)
    z2 = a1.dot(w2) + b2
    exp_scores = np.exp(z2)
    probs = exp_scores/np.sum(exp_scores, axis=1, keepdims=True)
    return np.argmax(probs, axis=1)

def forward(model, x, y):
    epsilon = 0.043
    reg_lamda =0.0001
    x = x.reshape(1,2) #为什么要reshape?
    w1, b1, w2, b2 = model['w1'], model['b1'], model['w2'], model['b2']
    z1 = x.dot(w1) + b1
    a1 = np.tanh(z1)
    z2 = a1.dot(w2) + b2
    exp_scores = np.exp(z2)
    probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)#维度问题
    loss = -np.log(probs[0,y])

    #导数信息
    delta3 = probs
    delta3[0, y] -= 1#原来的代码时使用迭代器一起取数
    dw2 = (a1.T).dot(delta3)
    db2 = np.sum(delta3, axis=0, keepdims=True)
    delta2 = delta3.dot(w2.T)*(1-np.power(a1,2))
    dw1 = np.dot(x.T, delta2)

    db1 = np.sum(delta2, axis=0)

    dw2 += reg_lamda*w2
    dw1 += reg_lamda*w1

    w1 += -epsilon*dw1
    b1 += -epsilon * db1
    w2 += -epsilon * dw2
    b2 += -epsilon * db2

    model['w1'] = w1
    model['b1'] = b1
    model['w2'] = w2
    model['b2'] = b2
    return loss, model

def plot_decision_boundary(pred_func, X, y):
    # Set min and max values and give it some padding
    x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
    y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
    h = 0.01
    # Generate a grid of points with distance h between them
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
    # Predict the function value for the whole gid
    Z = pred_func(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    # Plot the contour and training examples
    plt.contourf(xx, yy, Z, cmap=plt.cm.Spectral)
    plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Spectral)

def main():
    epochs = 200
    matplotlib.rcParams['figure.figsize'] = (10.0, 8.0)
    np.random.seed(3)
    X, Y = sklearn.datasets.make_moons(200, noise=0.20)
    print(len(X))
    model = initialize(2, 5, 2)
    for epoch in range(epochs):
        loss_ep = 0
        for i in range(len(X)):
            loss, model = forward(model, X[i], Y[i])
            loss_ep += loss
        loss_ep = loss_ep/len(X)
        if epoch%100==0:
            print('epoch:', epoch, 'loss:', loss_ep, '\n')
    plot_decision_boundary(lambda x: predict(model, x), X, Y)
    plt.scatter(X[:, 0], X[:, 1], s=40, c=Y, cmap=plt.cm.Spectral)
    plt.show()

if __name__ == '__main__':
    main()


#!/usr/bin/env python
"""
Created on 02 15, 2023

model: main.py

@Author: Stormzudi
"""
from Code.GAMCLDA.utils import bulid_graph, buildDataLoder
from Code.GAMCLDA.model import GAMCLDA
import torch
from torch_geometric.utils import negative_sampling
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split
import pandas as pd
import os
import datetime

# ========================== Args ========================
args = {
    "num_edges": 18789
    , "num_features": 233
    , "num_nodes": 828
    , "hidden_channels": [256]
    , "out_channels": 128
    , "n_class_size": 64
    , "lncRNA-miRNA": "../../dataset/lncRNA_miRNA_association.csv"
    , "disease-miRNA": "../../dataset/miRNA_disease_association.csv"
    , "weight": [0.7, 0.3] # [miRNA, node2vec]
    , "dropout": 0.0
    , "epoch": 10
}

path = "../../dataset/valdata/"
index, edges_index, node_attr, label = bulid_graph(path)

# 1 load dataset
train_data, val_data, test_data = buildDataLoder(node_attr, edges_index, y=label)

# 2 define model
gamclda = GAMCLDA(
    num_features=args['num_features'],hidden_channels=args['out_channels'],num_classes=args['n_class_size'])
model = gamclda.model()

# 3 define optimization
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
criterion = torch.nn.BCEWithLogitsLoss()

# 4 train
def train():
    model.train()

    optimizer.zero_grad()
    z = model(train_data.x, train_data.edge_index)

    # We perform a new round of negative sampling for every training epoch:
    neg_edge_index = negative_sampling(
        edge_index=train_data.edge_index, num_nodes=train_data.num_nodes,
        num_neg_samples=train_data.edge_label_index.size(1))

    # add edges (train_edge, neg_edge)
    edge_label_index = torch.cat(
        [train_data.edge_label_index, neg_edge_index],
        dim=-1,
    )

    # label
    edge_label = torch.cat([
        train_data.edge_label,
        train_data.edge_label.new_zeros(neg_edge_index.size(1))
    ], dim=0)

    out = model.decode(z, edge_label_index).view(-1)
    loss = criterion(out, edge_label)
    loss.backward()
    train_auc = roc_auc_score(edge_label.numpy(), out.detach().numpy())
    optimizer.step()

    # save auc score
    data_ = pd.DataFrame({
        'pred': out.detach().numpy(),
        'labels': edge_label.numpy()
    })
    date = datetime.date.today()
    data_.to_csv("{}_GAMCLDA_0.87.csv".format(date))
    return loss, train_auc

@torch.no_grad()
def test(data):
    model.eval()
    z = model(data.x, data.edge_index)
    out = model.decode(z, data.edge_label_index).view(-1).sigmoid()
    return roc_auc_score(data.edge_label.cpu().numpy(), out.cpu().numpy())

# ======================
# ==== Train  =======================
best_val_auc = final_test_auc = 0
for epoch in range(1, args["epoch"]):
    loss, train_auc= train()
    val_auc = test(val_data)
    test_auc = test(test_data)
    if val_auc > best_val_auc:
        best_val_auc = val_auc
        final_test_auc = test_auc

    print(f'Epoch: {epoch:03d}, '
          f'Loss: {loss:.4f}, '
          f'train: {train_auc:.4f}, '
          f'Val: {val_auc:.4f}, '
          f'Test: {test_auc:.4f}')


'''
All models need to return a CSP object, and a list of lists of Variable objects
representing the board. The returned list of lists is used to access the
solution.

For example, after these three lines of code

    csp, var_array = kenken_csp_model(board)
    solver = BT(csp)
    solver.bt_search(prop_FC, var_ord)

var_array[0][0].get_assigned_value() should be the correct value in the top left
cell of the KenKen puzzle.

The grid-only models do not need to encode the cage constraints.

1. binary_ne_grid (worth 10/100 marks)
    - A model of a KenKen grid (without cage constraints) built using only
      binary not-equal constraints for both the row and column constraints.

2. nary_ad_grid (worth 10/100 marks)
    - A model of a KenKen grid (without cage constraints) built using only n-ary
      all-different constraints for both the row and column constraints.

3. kenken_csp_model (worth 20/100 marks)
    - A model built using your choice of (1) binary binary not-equal, or (2)
      n-ary all-different constraints for the grid.
    - Together with KenKen cage constraints.

'''

from cspbase import Variable, Constraint, CSP
from itertools import permutations
import operator

def binary_ne_grid(kenken_grid):
    # Get the size of the grid and buid the domain of each variable
    size = kenken_grid[0][0]
    domain = list(range(1, size + 1))
    # Instantiate all variables in size x size list and instantiate csp
    vars = []
    for i in domain:
        vars.append([Variable(str(i) + str(j), domain) for j in domain])
    csp = CSP('kenken', [var for row in vars for var in row])
    # Create binary constraints
    valid = list(permutations(domain, 2))
    for i in range(size):
        for j in range(size):
            for k in range(j + 1, size):
                cons_x = Constraint('{0}{1}-{0}{2}'.format(i, j, k),
                                    [vars[i][j], vars[i][k]])
                cons_y = Constraint('{1}{0}-{2}{0}'.format(i, j, k),
                                    [vars[j][i], vars[k][i]])
                cons_x.add_satisfying_tuples(valid)
                cons_y.add_satisfying_tuples(valid)
                csp.add_constraint(cons_x)
                csp.add_constraint(cons_y)

    return csp, vars

def nary_ad_grid(kenken_grid):
    # Get the size of the grid and buid the domain of each variable
    size = kenken_grid[0][0]
    domain = list(range(1, size + 1))
    # Instantiate all variables in size x size list and instantiate csp
    vars = []
    for i in domain:
        vars.append([Variable(str(i) + str(j), domain) for j in domain])
    csp = CSP('kenken', [var for row in vars for var in row])
    # Create n-ary constraints
    valid = list(permutations(domain, size))
    for i in range(size):
        cons_x = Constraint('r' + str(i), vars[i])
        cons_y = Constraint('c' + str(i), [v[i] for v in vars])
        cons_x.add_satisfying_tuples(valid)
        cons_y.add_satisfying_tuples(valid)
        csp.add_constraint(cons_x)
        csp.add_constraint(cons_y)

    return csp, vars

def get_var(vars, coord):
    c = tuple([int(i) - 1 for i in str(coord)])
    return vars[c[0]][c[1]]

def augment_valid(valid, valid_i, val):
    for v in valid_i:
        if v:
            v.append(val)
    valid.extend(valid_i)

def valid_given_op(n, targ, dom, op, ordered=False):
    if ordered:
        dom = dom[::-1]
    comp = operator.gt if not ordered else operator.lt
    valid = []
    for i, val in enumerate(dom):
        valid_i = __valid_given_op(dom[i], n - 1, targ, dom[i:], op, comp)
        augment_valid(valid, valid_i, val)
    valid = [tuple(v) for p in valid for v in permutations(p)]
    return list(set(valid))

def __valid_given_op(cur, n, targ, dom, op, comp):
    # print(cur, n, targ, dom)
    if comp(cur, targ):
        return []
    if n == 1:
        for val in dom:
            if op(cur, val) == targ:
                return [[val]]
        return []

    valid = []
    for i, val in enumerate(dom):
        new_cur = op(cur, val)
        valid_i = __valid_given_op(new_cur, n - 1, targ, dom[i:], op, comp)
        augment_valid(valid, valid_i, val)
    return valid

ops = ['+', '-', '/', '*']
def kenken_csp_model(kenken_grid):
    size = kenken_grid[0][0]
    domain = list(range(1, size + 1))
    csp, vars = nary_ad_grid(kenken_grid)
    for i, cage in enumerate(kenken_grid[1:]):
        if len(cage) == 2:
            # Forced value constraint
            cons = Constraint('g{} f'.format(i), [get_var(vars, cage[0])])
            cons.add_satisfying_tuples([(cage[1],)])
        else:
            # Extract relevant data
            n_vars = len(cage) - 2
            target = cage[-2]
            operation = cage[-1]
            # Determine scope of the constraint and initialize
            vars_i = [get_var(vars, c) for c in cage[:-2]]
            cons = Constraint('g{} {}{}'.format(i, target, ops[operation]),
                              vars_i)
            # Determine satisfying tuples
            valid = []
            if operation == 0:  # Addition
                valid = valid_given_op(n_vars, target, domain,
                                       operator.add)
            elif operation == 1:  # Subtraction
                valid = valid_given_op(n_vars, target, domain,
                                       operator.sub, True)
            elif operation == 2:  # Division
                valid = valid_given_op(n_vars, target, domain,
                                       operator.floordiv, True)
            elif operation == 3:  # Multiplication
                valid = valid_given_op(n_vars, target, domain,
                                       operator.mul)
            cons.add_satisfying_tuples(valid)

        csp.add_constraint(cons)

    return csp, vars


# -*- coding:utf-8 -*-
from django.shortcuts import HttpResponse
from web.jsonEncoder import *
import os, time
from pathlib import Path
from django.conf import settings

def size_format(size):
    if size < 1000:
        return '%i' % size + ' Byte'
    elif 1000 <= size < 1000000:
        return '%.1f' % float(size/1000) + ' KB'
    elif 1000000 <= size < 1000000000:
        return '%.1f' % float(size/1000000) + ' MB'
    elif 1000000000 <= size < 1000000000000:
        return '%.1f' % float(size/1000000000) + ' GB'
    elif 1000000000000 <= size:
        return '%.1f' % float(size/1000000000000) + ' TB'

node_count = 0
def retrieve_all(dir):
    global node_count
    p=Path(dir)
    DirTree =[]
    for p in list(p.glob('*')):
        filepath = os.path.join(dir, p.name)
        modifiedTime = time.localtime(os.stat(filepath).st_mtime)
        mTime = time.strftime('%Y-%m-%d %H:%M:%S', modifiedTime)
        if p.is_file():
            file_size = size_format(os.path.getsize(filepath))
            node_count = node_count + 1
            DirTree.append({"id":node_count, "label":p.name, "value":"file", "file_size":file_size, "file_time":mTime})
        else:
            #注释掉递归，只查当前目录下的所有
            #subdir=[]
            #subdir=retrieve_dir(filepath)
            node_count = node_count + 1
            DirTree.append({"id":node_count, "label":p.name, "value":"dir", "file_time":mTime})
    return DirTree

count = 0
def retrieve_dir(dir):
    global count
    p=Path(dir)
    DirTree =[]
    for p in list(p.glob('*')):
        if p.is_file():
            pass
            #忽略文件，只进行目录处理
        else:
            subdir=[]
            subdir=retrieve_dir(os.path.join(dir,p.name))
            count = count + 1
            DirTree.append({"id": count, "label":p.name,"value":"dir","children":subdir})
    return DirTree

def get_floder(request):
    Dir = 'E:\Production\ManageSystem\static\Public'
    DirTree = retrieve_dir(Dir)
    json_data = {"msg": 'ok', "state_code": 200, "items": DirTree}
    return HttpResponse(json.dumps(json_data, cls=DateJSONEncoder), content_type="application/json")

def get_floder_all(request):
    Dir = 'E:\Production\ManageSystem\static\Public'
    dirList = json.loads(request.POST.get('dir'))
    for i in dirList:
        Dir = os.path.join(Dir, i)

    DirTree = retrieve_all(Dir)
    DirTree.sort(key = lambda x:x["value"])     #排序，文件夹优先
    json_data = {"msg": 'ok', "state_code": 200, "items": DirTree}
    return HttpResponse(json.dumps(json_data, cls=DateJSONEncoder), content_type="application/json")

def get_now_path(self):
    return '返回当前路径'

def md(request):
    try:
        path = json.loads(request.POST.get('path'))
        name = request.POST.get('name')

        Dir = 'E:\Production\ManageSystem\static\Public'
        for i in path:
            Dir = os.path.join(Dir, i)

        Dir = os.path.join(Dir, name)

        if not os.path.exists(Dir):
            os.makedirs(Dir)

        json_data = {"msg": 'ok', "state_code": 200}
    except:
        json_data = {"msg": 'error', "state_code": 444}
    return HttpResponse(json.dumps(json_data, cls=DateJSONEncoder), content_type="application/json")

def rd(self):
    return '删除目录及下面所有文件'

def delete_file(request):
    try:
        path = json.loads(request.POST.get('path'))
        type = request.POST.get('type')

        Dir = 'E:\Production\ManageSystem\static\Public'
        for i in path:
            Dir = os.path.join(Dir, i)

        print(Dir)

        if type == 'file':
            os.remove(Dir)
        else:
            os.rmdir(Dir)

        json_data = {"msg": 'ok', "state_code": 200}
    except:
        json_data = {"msg": 'error', "state_code": 444}

    return HttpResponse(json.dumps(json_data, cls=DateJSONEncoder), content_type="application/json")

def delete_many(self):
    return '删除多个文件'

def download(request):
    try:
        filepath = request.POST.get('filepath')
        print(filepath)
        json_data = {"msg": 'ok', "state_code": 200}
    except:
        json_data = {"msg": 'error', "state_code": 444}

    return HttpResponse(json.dumps(json_data, cls=DateJSONEncoder), content_type="application/json")

def ren(request):
    try:
        oldDir = json.loads(request.POST.get('oldDir'))
        newDir = json.loads(request.POST.get('newDir'))
        newName = request.POST.get('newName')

        Dir = 'E:\Production\ManageSystem\static\Public'
        for i in oldDir:
            Dir = os.path.join(Dir, i)

        srcFile = Dir

        nDir = 'E:\Production\ManageSystem\static\Public'
        for n in newDir:
            nDir = os.path.join(nDir, n)

        dstFile = os.path.join(nDir, newName)

        os.rename(srcFile, dstFile)

        json_data = {"msg": 'ok', "state_code": 200}
    except:
        json_data = {"msg": 'error', "state_code": 444}

    return HttpResponse(json.dumps(json_data, cls=DateJSONEncoder), content_type="application/json")

def uploadFile(path, fileobj):
    try:
        #file_path = os.path.join(path, fileobj) #打开这个物理文件
        #file_path = os.path.join(path, unicode(str(fileobj), 'utf-8'))
        file_path = os.path.join(path, fileobj)
        f = open(file_path, mode="wb")  #执行写数据
        for i in fileobj.chunks():
            f.write(i)
        f.close()
        return True
    except:
        return False

def upload(request):
    try:
        path = json.loads(request.POST.get('path'))
        Dir = 'E:\Production\ManageSystem\static\Public'

        upfileForm = request.GET.get('upfileForm', 'fileName')  # 如果一个表单中存在多个上传则需要变改每一个UPLOAD的 NAME名称
        fileName = request.FILES.get(upfileForm)
        for i in path:
            Dir = os.path.join(Dir, i)

        imgurl = uploadFile(Dir, fileName)

        if imgurl == False:
            msg = 'error'
        else:
            msg = 'ok'

        json_data = {"msg": msg, "state_code": 200}
    except:
        json_data = {"msg": 'error', "state_code": 444}

    return HttpResponse(json.dumps(json_data, cls=DateJSONEncoder), content_type="application/json")


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.utils import timezone
from django.core.management.base import BaseCommand
from apps.blog.models.post import Post

class Command(BaseCommand):
    help = ('Command to publish the drafted objects.')

    def add_arguments(self, parser):
        parser.add_argument('--max_objects', type=int, default=1)

    def handle(self, *args, **options):
        max_objects = options.get('max_objects')
        posts = Post.objects.filter(
            publish=False,
            deleted_at__isnull=True
        )[:max_objects]
        for post in posts:
            post.created_at = timezone.now()
            post.publish = True
            post.save()


from modules import input_numbers, game_numbers

def menu_graphic():
    print("[1] Delbarhet")
    print("[2] Gissa numret")
    print("[3] Avsluta")

def menu():
    menu_graphic()
    choice_menu = int(input())

    while choice_menu != 3:
        if choice_menu == 1:
            print()
            input_numbers()
        elif choice_menu == 2:
            print()
            game_numbers()
        else:
            print()
            print("Fel val försök igen. Skriv endast in giltiga nummer. (1-3)")

        print()
        menu_graphic()
        choice_menu = int(input())

menu()


#!/usr/bin/env python3
"""
Drew Patrick
CS231 Advanced Python
Homework 04: Task:
    Write a program that demonstrates a generator yielding the
    number of accesses made in each hour, from the beginning of
    /etc/httpd/logs/access_log
"""

from datetime import datetime
from datetime import timedelta
import os.path
import sys

# If file is not local add hills prefix to path.
def fix_path(path):
    if os.path.exists(path):
        return path
    return r'/etc/httpd/logs/' + path

def safe_open_readlines(trypath):
    try:
        with open(fix_path(trypath), 'r') as file_reader:
            return file_reader.readlines()
    except IOError:
        print('File can not be opened')
        sys.exit()

def count_hits_per_hour(log_file, number_of_hours):
    hour_count = 0
    is_first_line = True
    format_code = '%d/%b/%Y:%H:%M:%S'
    hits = 0
    for line in log_file:
        line = line.partition('[')[2]
        line = line.partition(' -')[0]
        dt_object = datetime.strptime(line, format_code)
        time_tuple = dt_object.timetuple()

        if is_first_line:
            last_time_tuple = time_tuple
            current_hour = time_tuple[3]
            is_first_line = False

        if current_hour == time_tuple[3]:
            hits += 1
        elif time_tuple[2] == last_time_tuple[2] and time_tuple[3] > current_hour:
            print('hour', current_hour, ' total hits', hits)
            hour_count += 1
            hits = 1
            current_hour = time_tuple[3]
        elif time_tuple[2] > last_time_tuple[2]:
            print('hour', current_hour, ' total hits', hits)
            last_time_tuple = (dt_object + timedelta(days=1)).timetuple()
            hour_count += 1
            hits = 1
            current_hour = 0

        if hour_count == number_of_hours:
            break

# Executing Program
file_name = 'access_log'

count_hits_per_hour(safe_open_readlines(fix_path(file_name)), 25)


from django.core.checks import Error, Warning, register

@register(deploy=True)
def schema_check(app_configs, **kwargs):
    """ Perform dummy generation and emit warnings/errors as part of Django's check framework """
    from drf_spectacular.drainage import GENERATOR_STATS
    from drf_spectacular.settings import spectacular_settings

    if not spectacular_settings.ENABLE_DJANGO_DEPLOY_CHECK:
        return []

    errors = []
    try:
        with GENERATOR_STATS.silence():
            spectacular_settings.DEFAULT_GENERATOR_CLASS().get_schema(request=None, public=True)
    except Exception as exc:
        errors.append(
            Error(f'Schema generation threw exception "{exc}"', id='drf_spectacular.E001')
        )
    if GENERATOR_STATS:
        for w in GENERATOR_STATS._warn_cache:
            errors.append(Warning(w, id='drf_spectacular.W001'))
        for e in GENERATOR_STATS._error_cache:
            errors.append(Warning(e, id='drf_spectacular.W002'))
    return errors


import tensorflow as tf
import numpy as np
import cv2

def create_label(image_name):
    name = image_name.split('.')[0]
    if name=="Rabindra":
        return np.array([1,0,0])
    elif name=="Riju":
        return np.array([0,1,0])
    elif name=="Rojina":
        return np.array([0,0,1])

##Create Data
import os
from random import shuffle
from tqdm import tqdm

def mydata():
    data=[]
    for image in tqdm(os.listdir("dataset")):
        path=os.path.join("dataset",image)
        image_data =cv2.imread(path,cv2.IMREAD_GRAYSCALE)
        image_data =cv2.resize(image_data, (50,50))
        data.append([np.array(image_data),create_label(image)])

    shuffle(data)
    return data

data = mydata()

train = data[:2400]
test = data[2400:]
x_train = np.array([i[0]  for i in train]).reshape(-1,50,50,1)
y_train = np.array([i[1] for i in train])
print(x_train.shape)
x_test = np.array([i[0] for i in test]).reshape(-1,50,50,1)
y_test = np.array([i[1] for i in test])
print(x_train.shape)


"""This script converts mean.nii created by 'create_mean_mri.py' to png slices."""

import matplotlib.pyplot as plt

from scripts.common.ProcessingFunctions import ProcessingFunctions as ps

dir = 'outputDirectory'

meanScan = ps.readNiftiFile('mean.nii')

for i in range(120):
    slice = meanScan[:, :, i]
    plt.imshow(slice, cmap='gray')
    plt.axis('off')
    plt.savefig(dir + 'slice' + str(i) + '.jpg', bbox_inches='tight', pad_inches=0)
    plt.clf()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Nov 17 12:32:43 2022
speakerClassifier.py
@author: richard
"""

import torch
import torch.nn as nn
import torch.nn.functional as F

class speakerClassifier(nn.Module):
    def __init__(self, numLearners, learnerIn, learnerMid, learnerOut, extra=0):
        super(speakerClassifier, self).__init__()
        self.speakerIn = learnerIn
        self.speakerMid = learnerMid
        self.speakerOut = learnerOut

        self.activator = nn.ReLU()
        self.numLearners = numLearners
        self.learners = nn.ParameterList()
        self.extra = extra

        for i in range(numLearners):
            self.learners.append(nn.Sequential(
                    nn.Linear(self.speakerIn, self.speakerMid),
                    self.activator,
                    nn.Linear(self.speakerMid, self.speakerOut),
                    nn.Dropout(p=0.2),
                    # nn.BatchNorm1d(self.speakerOut),
                    self.activator)
                )

        if extra != 0:
            self.linear0 = nn.Linear(self.speakerOut*self.numLearners+extra,
                                     self.speakerOut)
            # self.norm0 = nn.BatchNorm1d(self.speakerOut*self.numLearners+extra)
        else:
            self.linear0 = nn.Linear(self.speakerOut*self.numLearners,
                                     self.speakerOut)
            # self.norm0 = nn.BatchNorm1d(self.speakerOut*self.numLearners)

        self.linear1 = nn.Linear(self.speakerOut, int(self.speakerOut/2))
        self.classify = nn.Linear(int(self.speakerOut/2), 5)

    def forward(self, x):

        p = []
        for i in range(len(self.learners)):
            p.append(self.learners[i](x[:, i*self.speakerIn:(i+1)*self.speakerIn]))

        if self.extra != 0:
            p.append(x[:, self.speakerIn*self.numLearners:])

        x = torch.cat((p), dim=1)

        x = F.relu(self.linear0(F.dropout(x)))
        x = F.relu(self.linear1(x))
        x = self.classify(x)
        x = F.softmax(x, 1)

        return x


"""
Albert Ratajczak, 2018
Class LocalLibrary - stores information about path to local library directory
and IDs of tracked albums
"""

# general imports
import os
import json

# local imports
from userdir import user_dir

class LocalLibrary:
    def __init__(self, app_name):
        """
        self.path: path to local library dir
        self.albums: dict with album IDs as keys and sets of item IDs as values
        """
        self.path = os.path.join(os.path.expanduser('~'), app_name)
        self.albums = dict()

    def __str__(self):
        return 'Local library:\n- library directory: {}\n- tracked albums: {}'\
               .format(self.path, len(self.albums))

    def add(self, album_id):
        """
        Adds album ID as a key to dict of tracked albums (albums to download)
        :param album_id: ID of an album in Google Photos
        :return: None
        """
        if not isinstance(album_id, str):
            raise TypeError('add() takes \'str\' object as argument')
        self.albums.update({album_id: set()})

    def remove(self, album_id):
        """
        Removes album ID from dict of tracked albums IDs
        :param album_id: ID of an album in Google Photos
        :return: None
        """
        if not isinstance(album_id, str):
            raise TypeError('add() takes \'str\' object as argument')
        try:
            self.albums.pop(album_id)
        except KeyError:
            pass

    def get_album_ids(self):
        return list(self.albums.keys())

    def add_to_album(self, album_id, item_ids=set()):
        """
        Adds item IDs as values to dict self.albums for key album_id
        """
        self.albums[album_id].update(item_ids)

    def get_album_items(self, album_id):
        """
        :returns: IDs of items in given album
        """
        return self.albums[album_id]

    def get_path(self):
        return self.path

    @user_dir
    def load(self, **kwargs):
        """
        Loads local library setups from JSON file: path and albums
        :param kwargs: to send user directory between method and decorator
        :return: self
        """
        # Reading JSON file
        json_file = os.path.join(kwargs['user_dir'], 'local_library.json')
        if os.path.exists(json_file):
            with open(json_file) as f:
                storage = json.load(f)
            self.path = storage['path']
            self.albums = storage['albums']
            # Converting lists form JSON file to sets
            for key in self.albums:
                self.albums[key] = set(self.albums[key])
            return self

    @user_dir
    def store(self, **kwargs):
        """
        Stores (saves) local library setups to JSON file: path and albums
        :param kwargs: to send user directory between method and decorator
        :return: None
        """
        albums = dict()
        # Converting sets in albums dict to lists - JSON dosn't accept set
        for key in self.albums:
            albums[key] = list(self.albums[key])
        # Saving to JSON file
        json_file = os.path.join(kwargs['user_dir'], 'local_library.json')
        with open(json_file, 'w') as f:
            json.dump({'path': self.path, 'albums': albums}, f)

    def set_path(self, path):
        """
        Sets path to directory of local library
        :param path: absolute path to local library
        :return: self.path
        """
        assert os.path.isabs(path), 'Argument path has to be absolute path'
        if path != self.path:
            if not os.path.exists(path):
                os.makedirs(path)
            self.path = path
        return self.path


import traceback
from pytube import YouTube
import requests
import json
from datetime import datetime, timedelta
from debug.delegate_logging import DelegateLogging

class MusicDownloader(DelegateLogging):

    def _get_header_for_request_wake_songs(self):
        headers = {}
        headers['Host'] = 'dodam.b1nd.com'
        headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0'
        headers['Accpet'] = 'application/json, text/plain, */*'
        headers['Accpt-Language'] = 'ko-KR,ko;q=0.8,en-US;q=0.5,en;q=0.3'
        headers['Accpet-Encoding'] = 'gzip, deflate'
        headers['Connection'] = 'keep-alive'
        headers['Referer'] = 'http://dodam.b1nd.com/wakesong'
        headers['Pragma'] = 'no-cache'
        headers['Cache-Control'] = 'no-cache'
        return headers

    def _get_music_url_list(self):
        now = datetime.now()
        target = datetime.now()
        if now.hour > 12:
            target = datetime.now() + timedelta(days=1)

        res = requests.request('GET',
                               f'http://10.64.160.190/api/v2/wakeup-song?year={target.year}&month={target.month}&date={target.day}',
                               headers=self._get_header_for_request_wake_songs(), timeout=3)
        if not res.ok:
            self.log('Failed to Get music list because of status code ', res.status_code)
            return
        res = json.loads(res.content.decode('utf-8'))
        return [row['videoUrl'] for k, row in enumerate(res['data']['allow'])]

    def download(self):
        self.log('start to get music list')
        try:
            music_list = self._get_music_url_list()
            self.log('success to get music list')
        except:
            self.log('failed to get music list')
            self.log(traceback.format_exc())
            return
        all_music_count = len(music_list)
        self.log(f'start downloads {all_music_count}')
        try:
            for k, music_URL in enumerate(music_list):
                self.log(f'start download {k + 1}/{all_music_count} {music_URL}')
                try:
                    YouTube(music_URL).streams.filter(only_audio=True, mime_type='audio/mp4').first() \
                        .download(output_path=f'buffer', filename=f'{k}')
                    self.log(f'finish download {k + 1}/{all_music_count} {music_URL}')
                except:
                    self.log(f'failed download {k + 1}/{all_music_count} {music_URL}')

        except:
            self.log('failed downloads')
            self.log(traceback.format_exc())
        self.log('finish downloads')


import http.client
import urllib.parse

class NetworkError(Exception):
    pass

def krnnt_response_lines(text):
    """
    Get KRNNT's analysis of text as a list of lines.
    """
    params = urllib.parse.urlencode({'text': text})
    headers = {'Content-type': 'application/x-www-form-urlencoded',
            'Accept': 'text/plain'}
    conn = http.client.HTTPConnection('localhost:9003') # where KRNNT resides
    conn.request('POST', '', params, headers)
    response = conn.getresponse()
    if response.status != 200:
        raise NetworkError('Cannot connect to KRNNT: {} {}, is the container running?'.format(
            response.status, response.reason))
    resp_html = response.read().decode('utf-8') # we get a HTML page and need to strip tags
    lines = resp_html[resp_html.index('<pre>')+len('<pre>')
            :resp_html.index('</pre>')].strip().split('\n')
    return lines

def tag_nkjp(text):
    """
    Tag text with NKJP tagset. Returns a list of sentences as lists of (form, lemma, interp).
    """
    lines = krnnt_response_lines(text)
    sents = [[]]
    current_token = None
    for line_n, line in enumerate(lines):
        # Next sentence.
        if len(line) == 0:
            sents.append([])
            continue
        # A form line - assign the form.
        if line_n % 2 == (0 if (len(sents) % 2 == 1) else 1):
            current_token = line.split('\t')[0]
        else:
            interp_data = line.split('\t')
            current_token = (current_token, interp_data[1], interp_data[2])
            sents[-1].append(current_token)
    sents = [sent for sent in sents if len(sent) > 0]
    return sents

def split_sents_krnnt(text, strip_sents=True):
    """
    Use KRNNT to split the text into a list of sentence strings.
    """
    lines = krnnt_response_lines(text)
    sents = ['']
    for line_n, line in enumerate(lines):
        # Next sentence.
        if len(line) == 0:
            sents.append('')
            continue
        # A form line - grow the sentence.
        if line_n % 2 == (0 if (len(sents) % 2 == 1) else 1):
            form_data = line.split('\t')
            form = form_data[0]
            if form_data[1] == 'space':
                preceding_sep = ' '
            elif form_data[1] == 'none':
                preceding_sep = ''
            elif form_data[1] == 'newline':
                preceding_sep = '\n'
            sents[-1] += preceding_sep + form
    if strip_sents:
        sents = [sent.strip() for sent in sents]
    sents = [sent for sent in sents if len(sent) > 0]
    return sents


from __future__ import print_function, division
from copy import deepcopy
from random import random as r, choice

import matplotlib.pyplot as plt

from utils import tuple_add, TorusWorld
from enums import *

NONE_STEP = -30
POISON_STEP = -500
FOOD_STEP = 50
FINISH_STEP = FOOD_STEP

def calculate_reward(current, tile, steps, max_steps):
    """
    Given a flatland instance and a tile content returns the score associated with moving onto the tile
    """
    reward = NONE_STEP * (1.0 + (steps / max_steps))

    if current.done:
        reward = FINISH_STEP
    elif tile > 0:
        reward = FOOD_STEP
    elif tile == -1:
        reward = POISON_STEP
    return reward

class Flatland(TorusWorld):
    def __init__(self, dimensions, x, y, n_food, grid):
        TorusWorld.__init__(self, dimensions, grid)

        self.agent_x, self.agent_y = self.agent_start_x, self.agent_start_y = x, y

        self.n_food = n_food
        self.food_eaten = 0
        self.poison_eaten = 0

    @property
    def agent_position(self):
        return self.agent_x, self.agent_y

    @property
    def up_coordinate(self):
        return self.absolute_coordinates(*tuple_add(self.agent_position, DELTAS[NORTH]))

    @property
    def left_coordinate(self):
        return self.absolute_coordinates(*tuple_add(self.agent_position, DELTAS[WEST]))

    @property
    def right_coordinate(self):
        return self.absolute_coordinates(*tuple_add(self.agent_position, DELTAS[EAST]))

    @property
    def down_coordinate(self):
        return self.absolute_coordinates(*tuple_add(self.agent_position, DELTAS[SOUTH]))

    def perform_action(self, action):
        assert action in ACTIONS

        if action == LEFT:
            self.agent_x, self.agent_y = self.left_coordinate
        if action == UP:
            self.agent_x, self.agent_y = self.up_coordinate
        elif action == RIGHT:
            self.agent_x, self.agent_y = self.right_coordinate
        elif action == DOWN:
            self.agent_x, self.agent_y = self.down_coordinate

        tile_contents = self.get_tile(self.agent_x, self.agent_y)

        if tile_contents > 0:
            self.food_eaten += 1
        elif tile_contents == -1:
            self.poison_eaten += 1

        self.set_tile(self.agent_x, self.agent_y, 0)
        return tile_contents

    @property
    def done(self):
        return self.food_eaten == self.n_food and \
            self.agent_position == (self.agent_start_x, self.agent_start_y)

class State(object):
    def __init__(self, agent_position=(0, 0)):
        self.foods_eaten = ()  # Sorted tuple of eaten foods
        self.position = agent_position

    def __hash__(self):
        return hash((self.foods_eaten, self.position))

    def __eq__(self, other):
        return self.foods_eaten == other.foods_eaten and self.position == other.position

    def __repr__(self):
        return "({} : {})".format(self.foods_eaten, self.position)

class FlatlandQLearn(object):
    def __init__(self, k, scenario, learning_rate, discount_rate, temp, x=0, decay_rate=0, max_steps=500):
        self.k = k
        self.scenario = scenario
        self.learning_rate = learning_rate
        self.discount_rate = discount_rate
        self.temp = temp
        self.ts = []
        self.q = {}  # Q :: State, action -> value
        self.decay_rate = decay_rate
        self.x = x
        self.max_steps = max_steps

    def update_q(self, start_state, end_state, action, reward):

        # max_aQ(s_{t+1}, a)
        end_values = self.q.get(end_state, {})
        best_end_value = max(end_values.values()) if end_values else 0

        # Q(at,st)
        start_values = self.q.get(start_state, {})
        old_value = start_values[action] if action in start_values.keys() else 0

        new_value = old_value + self.learning_rate * \
            (reward + (self.discount_rate * best_end_value) - old_value)
        start_values[action] = new_value
        self.q[start_state] = start_values

    def choose_action(self, current_state):
        ret = choice((LEFT, UP, RIGHT, DOWN))

        if r() > self.temp:
            # Attempt to exploit knowledge
            values = self.q.get(current_state, {})
            ret = max(values.iterkeys(), key=(lambda key: values[key])) if values else ret

        return ret

    def learn(self):
        max_steps = self.scenario.w * self.scenario.h * 2
        for i in range(self.k):
            print(i)
            t = 0
            game = deepcopy(self.scenario)
            current_state = State(game.agent_position)
            action_history = []
            state_history = []
            while not game.done and not t > max_steps:
                # Select action to perform
                action = self.choose_action(current_state)

                # Have the agent perform the action
                tile = game.perform_action(action)

                # Calculate reward based on the result of the action
                reward = calculate_reward(game, tile, t, self.max_steps)

                # Update state
                old_state = current_state
                current_state = State(game.agent_position)
                current_state.foods_eaten += old_state.foods_eaten

                if reward == FOOD_STEP:
                    current_state.foods_eaten = tuple(sorted(current_state.foods_eaten + (tile,)))

                # Update Q
                self.update_q(old_state, current_state, action, reward)

                if self.x and self.x < len(state_history):
                    # Backup x-steps
                    for j in range(-1, - self.x, -1):
                        self.update_q(state_history[j-1], state_history[j], action_history[j], reward)

                if self.decay_rate:
                    # Backup trace decay
                    trace = 1.0
                    action_len = len(action_history)
                    for j in range(1, action_len-1):
                        self.update_q(state_history[-(j+1)], state_history[-j], action_history[-j], trace * reward)
                        trace -= self.decay_rate

                action_history.append(action)
                state_history.append(old_state)
                t += 1
            self.ts.append(t)
            self.temp = max(self.temp - 1/self.k, 0)

    def optimal_run(self):
        self.temp = 0
        t = 0
        game = deepcopy(self.scenario)
        current_state = State(game.agent_position)
        action_history = []
        state_history = []
        while not game.done and not t > self.max_steps:
            # Select action to perform
            action = self.choose_action(current_state)
            action_history.append(action)

            # Have the agent perform the action
            tile = game.perform_action(action)

            # Calculate reward based on the result of the action
            reward = calculate_reward(game, tile, t, self.max_steps)

            # Update state
            old_state = current_state
            state_history.append(old_state)
            current_state = State(game.agent_position)
            current_state.foods_eaten += old_state.foods_eaten

            if reward == FOOD_STEP:
                current_state.foods_eaten = tuple(sorted(current_state.foods_eaten + (tile,)))

            t += 1
        self.ts.append(t)

        print('Food eaten: {}'.format(game.food_eaten))
        print('Poison eaten: {}'.format(game.poison_eaten))
        print('Steps: {}'.format(t))
        return action_history, state_history

    def plot_steps(self):
        plt.title("Steps")
        plt.plot(self.ts)
        plt.show()


#!/usr/bin/python3
"""Module with number_of_subscribers function"""
import requests
import sys

def number_of_subscribers(subreddit):
    """function that queries the Reddit API
    and returns the number of subscri    bers for a given subreddit."""
    url = "https://www.reddit.com/r/{}/about.json".format(subreddit)
    json_obj = requests.get(url,
                            headers={'User-Agent': 'My User Agent 1.0'})
    if json_obj.status_code != 404:
        dict_obj = json_obj.json()
        return dict_obj.get('data').get('subscribers')
    else:
        return 0


import unittest
from contextlib import contextmanager
import os
import logging
import errno
from shutil import rmtree
from tempfile import mkdtemp, NamedTemporaryFile

from swift.common import db_replicator
from swift.common import utils
from swift.common.utils import normalize_timestamp
from swift.container import server as container_server

from test.unit import FakeLogger

def teardown_module():
    "clean up my monkey patching"
    reload(db_replicator)

@contextmanager
def lock_parent_directory(filename):
    yield True

class FakeRing:
    class Ring:
        devs = []

        def __init__(self, path, reload_time=15, ring_name=None):
            pass

        def get_part_nodes(self, part):
            return []

        def get_more_nodes(self, *args):
            return []

class FakeRingWithNodes:
    class Ring:
        devs = [dict(
            id=1, weight=10.0, zone=1, ip='1.1.1.1', port=6000, device='sdb',
            meta=''
        ), dict(
            id=2, weight=10.0, zone=2, ip='1.1.1.2', port=6000, device='sdb',
            meta=''
        ), dict(
            id=3, weight=10.0, zone=3, ip='1.1.1.3', port=6000, device='sdb',
            meta=''
        ), dict(
            id=4, weight=10.0, zone=4, ip='1.1.1.4', port=6000, device='sdb',
            meta='')]

        def __init__(self, path, reload_time=15, ring_name=None):
            pass

        def get_part_nodes(self, part):
            return self.devs[:3]

        def get_more_nodes(self, *args):
            return (d for d in self.devs[3:])

class FakeProcess:
    def __init__(self, *codes):
        self.codes = iter(codes)

    def __call__(self, *args, **kwargs):
        class Failure:
            def communicate(innerself):
                next = self.codes.next()
                if isinstance(next, int):
                    innerself.returncode = next
                    return next
                raise next
        return Failure()

@contextmanager
def _mock_process(*args):
    orig_process = db_replicator.subprocess.Popen
    db_replicator.subprocess.Popen = FakeProcess(*args)
    yield
    db_replicator.subprocess.Popen = orig_process

class ReplHttp:
    def __init__(self, response=None):
        self.response = response
    replicated = False
    host = 'localhost'

    def replicate(self, *args):
        self.replicated = True

        class Response:
            status = 200
            data = self.response

            def read(innerself):
                return self.response
        return Response()

class ChangingMtimesOs:
    def __init__(self):
        self.mtime = 0
        self.path = self
        self.basename = os.path.basename

    def getmtime(self, file):
        self.mtime += 1
        return self.mtime

class FakeBroker:
    db_file = __file__
    get_repl_missing_table = False
    stub_replication_info = None
    db_type = 'container'

    def __init__(self, *args, **kwargs):
        return None

    @contextmanager
    def lock(self):
        yield True

    def get_sync(self, *args, **kwargs):
        return 5

    def get_syncs(self):
        return []

    def get_items_since(self, point, *args):
        if point == 0:
            return [{'ROWID': 1}]
        return []

    def merge_syncs(self, *args, **kwargs):
        self.args = args

    def merge_items(self, *args):
        self.args = args

    def get_replication_info(self):
        if self.get_repl_missing_table:
            raise Exception('no such table')
        if self.stub_replication_info:
            return self.stub_replication_info
        return {'delete_timestamp': 0, 'put_timestamp': 1, 'count': 0}

    def reclaim(self, item_timestamp, sync_timestamp):
        pass

    def get_info(self):
        pass

class TestReplicator(db_replicator.Replicator):
    server_type = 'container'
    ring_file = 'container.ring.gz'
    brokerclass = FakeBroker
    datadir = container_server.DATADIR
    default_port = 1000

class TestDBReplicator(unittest.TestCase):
    def setUp(self):
        db_replicator.ring = FakeRing()
        self.delete_db_calls = []

    def stub_delete_db(self, object_file):
        self.delete_db_calls.append(object_file)

    def test_repl_connection(self):
        node = {'ip': '127.0.0.1', 'port': 80, 'device': 'sdb1'}
        conn = db_replicator.ReplConnection(node, '1234567890', 'abcdefg',
                    logging.getLogger())

        def req(method, path, body, headers):
            self.assertEquals(method, 'REPLICATE')
            self.assertEquals(headers['Content-Type'], 'application/json')

        class Resp:
            def read(self):
                return 'data'
        resp = Resp()
        conn.request = req
        conn.getresponse = lambda *args: resp
        self.assertEquals(conn.replicate(1, 2, 3), resp)

        def other_req(method, path, body, headers):
            raise Exception('blah')
        conn.request = other_req
        self.assertEquals(conn.replicate(1, 2, 3), None)

    def test_rsync_file(self):
        replicator = TestReplicator({})
        with _mock_process(-1):
            fake_device = {'ip': '127.0.0.1', 'device': 'sda1'}
            self.assertEquals(False,
                    replicator._rsync_file('/some/file', 'remote:/some/file'))
        with _mock_process(0):
            fake_device = {'ip': '127.0.0.1', 'device': 'sda1'}
            self.assertEquals(True,
                    replicator._rsync_file('/some/file', 'remote:/some/file'))

    def test_rsync_db(self):
        replicator = TestReplicator({})
        replicator._rsync_file = lambda *args: True
        fake_device = {'ip': '127.0.0.1', 'device': 'sda1'}
        replicator._rsync_db(FakeBroker(), fake_device, ReplHttp(), 'abcd')

    def test_in_sync(self):
        replicator = TestReplicator({})
        self.assertEquals(replicator._in_sync(
            {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'},
            {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'},
            FakeBroker(), -1), True)
        self.assertEquals(replicator._in_sync(
            {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b'},
            {'id': 'a', 'point': -1, 'max_row': 10, 'hash': 'b'},
            FakeBroker(), -1), True)
        self.assertEquals(bool(replicator._in_sync(
            {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'c'},
            {'id': 'a', 'point': -1, 'max_row': 10, 'hash': 'd'},
            FakeBroker(), -1)), False)

    def test_run_once(self):
        replicator = TestReplicator({})
        replicator.run_once()

    def test_usync(self):
        fake_http = ReplHttp()
        replicator = TestReplicator({})
        replicator._usync_db(0, FakeBroker(), fake_http, '12345', '67890')

    def test_repl_to_node(self):
        replicator = TestReplicator({})
        fake_node = {'ip': '127.0.0.1', 'device': 'sda1', 'port': 1000}
        fake_info = {'id': 'a', 'point': -1, 'max_row': 0, 'hash': 'b',
                    'created_at': 100, 'put_timestamp': 0,
                    'delete_timestamp': 0,
                    'metadata': {'Test': ('Value', normalize_timestamp(1))}}
        replicator._http_connect = lambda *args: ReplHttp(
                                                   '{"id": 3, "point": -1}')
        self.assertEquals(replicator._repl_to_node(
            fake_node, FakeBroker(), '0', fake_info), True)

    def test_stats(self):
        # I'm not sure how to test that this logs the right thing,
        # but we can at least make sure it gets covered.
        replicator = TestReplicator({})
        replicator._zero_stats()
        replicator._report_stats()

    def test_replicate_object(self):
        db_replicator.ring = FakeRingWithNodes()
        replicator = TestReplicator({})
        replicator.delete_db = self.stub_delete_db
        replicator._replicate_object('0', '/path/to/file', 'node_id')
        self.assertEquals([], self.delete_db_calls)

    def test_replicate_object_quarantine(self):
        replicator = TestReplicator({})
        was_db_file = replicator.brokerclass.db_file
        try:

            def mock_renamer(was, new, cause_colision=False):
                if cause_colision and '-' not in new:
                    raise OSError(errno.EEXIST, "File already exists")
                self.assertEquals('/a/b/c/d/e', was)
                if '-' in new:
                    self.assert_(
                        new.startswith('/a/quarantined/containers/e-'))
                else:
                    self.assertEquals('/a/quarantined/containers/e', new)

            def mock_renamer_error(was, new):
                return mock_renamer(was, new, cause_colision=True)
            was_renamer = db_replicator.renamer
            db_replicator.renamer = mock_renamer
            replicator.brokerclass.get_repl_missing_table = True
            replicator.brokerclass.db_file = '/a/b/c/d/e/hey'
            replicator._replicate_object('0', 'file', 'node_id')
            # try the double quarantine
            db_replicator.renamer = mock_renamer_error
            replicator._replicate_object('0', 'file', 'node_id')
        finally:
            replicator.brokerclass.db_file = was_db_file
            db_replicator.renamer = was_renamer

    def test_replicate_object_delete_because_deleted(self):
        replicator = TestReplicator({})
        try:
            replicator.delete_db = self.stub_delete_db
            replicator.brokerclass.stub_replication_info = {
                'delete_timestamp': 2, 'put_timestamp': 1, 'count': 0}
            replicator._replicate_object('0', '/path/to/file', 'node_id')
        finally:
            replicator.brokerclass.stub_replication_info = None
        self.assertEquals(['/path/to/file'], self.delete_db_calls)

    def test_replicate_object_delete_because_not_shouldbehere(self):
        replicator = TestReplicator({})
        replicator.delete_db = self.stub_delete_db
        replicator._replicate_object('0', '/path/to/file', 'node_id')
        self.assertEquals(['/path/to/file'], self.delete_db_calls)

    def test_delete_db(self):
        db_replicator.lock_parent_directory = lock_parent_directory
        replicator = TestReplicator({})
        replicator._zero_stats()
        replicator.extract_device = lambda _: 'some_device'
        replicator.logger = FakeLogger()

        temp_dir = mkdtemp()
        try:
            temp_suf_dir = os.path.join(temp_dir, '16e')
            os.mkdir(temp_suf_dir)
            temp_hash_dir = os.path.join(temp_suf_dir,
                                         '166e33924a08ede4204871468c11e16e')
            os.mkdir(temp_hash_dir)
            temp_file = NamedTemporaryFile(dir=temp_hash_dir, delete=False)
            temp_hash_dir2 = os.path.join(temp_suf_dir,
                                         '266e33924a08ede4204871468c11e16e')
            os.mkdir(temp_hash_dir2)
            temp_file2 = NamedTemporaryFile(dir=temp_hash_dir2, delete=False)

            # sanity-checks
            self.assertTrue(os.path.exists(temp_dir))
            self.assertTrue(os.path.exists(temp_suf_dir))
            self.assertTrue(os.path.exists(temp_hash_dir))
            self.assertTrue(os.path.exists(temp_file.name))
            self.assertTrue(os.path.exists(temp_hash_dir2))
            self.assertTrue(os.path.exists(temp_file2.name))
            self.assertEqual(0, replicator.stats['remove'])

            replicator.delete_db(temp_file.name)

            self.assertTrue(os.path.exists(temp_dir))
            self.assertTrue(os.path.exists(temp_suf_dir))
            self.assertFalse(os.path.exists(temp_hash_dir))
            self.assertFalse(os.path.exists(temp_file.name))
            self.assertTrue(os.path.exists(temp_hash_dir2))
            self.assertTrue(os.path.exists(temp_file2.name))
            self.assertEqual([(('removes.some_device',), {})],
                             replicator.logger.log_dict['increment'])
            self.assertEqual(1, replicator.stats['remove'])

            replicator.delete_db(temp_file2.name)

            self.assertTrue(os.path.exists(temp_dir))
            self.assertFalse(os.path.exists(temp_suf_dir))
            self.assertFalse(os.path.exists(temp_hash_dir))
            self.assertFalse(os.path.exists(temp_file.name))
            self.assertFalse(os.path.exists(temp_hash_dir2))
            self.assertFalse(os.path.exists(temp_file2.name))
            self.assertEqual([(('removes.some_device',), {})] * 2,
                             replicator.logger.log_dict['increment'])
            self.assertEqual(2, replicator.stats['remove'])
        finally:
            rmtree(temp_dir)

    def test_extract_device(self):
        replicator = TestReplicator({'devices': '/some/root'})
        self.assertEqual('some_device', replicator.extract_device(
            '/some/root/some_device/deeper/and/deeper'))

#    def test_dispatch(self):
#        rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False)
#        no_op = lambda *args, **kwargs: True
#        self.assertEquals(rpc.dispatch(('drv', 'part', 'hash'), ('op',)
#                ).status_int, 400)
#        rpc.mount_check = True
#        self.assertEquals(rpc.dispatch(('drv', 'part', 'hash'), ['op',]
#                ).status_int, 507)
#        rpc.mount_check = False
#        rpc.rsync_then_merge = lambda drive, db_file,
#                                      args: self.assertEquals(args, ['test1'])
#        rpc.complete_rsync = lambda drive, db_file,
#                                      args: self.assertEquals(args, ['test2'])
#        rpc.dispatch(('drv', 'part', 'hash'), ['rsync_then_merge','test1'])
#        rpc.dispatch(('drv', 'part', 'hash'), ['complete_rsync','test2'])
#        rpc.dispatch(('drv', 'part', 'hash'), ['other_op',])

    def test_rsync_then_merge(self):
        rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False)
        rpc.rsync_then_merge('sda1', '/srv/swift/blah', ('a', 'b'))

    def test_merge_items(self):
        rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False)
        fake_broker = FakeBroker()
        args = ('a', 'b')
        rpc.merge_items(fake_broker, args)
        self.assertEquals(fake_broker.args, args)

    def test_merge_syncs(self):
        rpc = db_replicator.ReplicatorRpc('/', '/', FakeBroker, False)
        fake_broker = FakeBroker()
        args = ('a', 'b')
        rpc.merge_syncs(fake_broker, args)
        self.assertEquals(fake_broker.args, (args[0],))

    def test_roundrobin_datadirs(self):
        listdir_calls = []
        isdir_calls = []
        exists_calls = []
        shuffle_calls = []

        def _listdir(path):
            listdir_calls.append(path)
            if not path.startswith('/srv/node/sda/containers') and \
                    not path.startswith('/srv/node/sdb/containers'):
                return []
            path = path[len('/srv/node/sdx/containers'):]
            if path == '':
                return ['123', '456', '789']  # 456 will pretend to be a file
            elif path == '/123':
                return ['abc', 'def.db']  # def.db will pretend to be a file
            elif path == '/123/abc':
                # 11111111111111111111111111111abc will pretend to be a file
                return ['00000000000000000000000000000abc',
                        '11111111111111111111111111111abc']
            elif path == '/123/abc/00000000000000000000000000000abc':
                return ['00000000000000000000000000000abc.db',
                        # This other.db isn't in the right place, so should be
                        # ignored later.
                        '000000000000000000000000000other.db',
                        'weird1']  # weird1 will pretend to be a dir, if asked
            elif path == '/789':
                return ['ghi', 'jkl']  # jkl will pretend to be a file
            elif path == '/789/ghi':
                # 33333333333333333333333333333ghi will pretend to be a file
                return ['22222222222222222222222222222ghi',
                        '33333333333333333333333333333ghi']
            elif path == '/789/ghi/22222222222222222222222222222ghi':
                return ['22222222222222222222222222222ghi.db',
                        'weird2']  # weird2 will pretend to be a dir, if asked
            return []

        def _isdir(path):
            isdir_calls.append(path)
            if not path.startswith('/srv/node/sda/containers') and \
                    not path.startswith('/srv/node/sdb/containers'):
                return False
            path = path[len('/srv/node/sdx/containers'):]
            if path in ('/123', '/123/abc',
                        '/123/abc/00000000000000000000000000000abc',
                        '/123/abc/00000000000000000000000000000abc/weird1',
                        '/789', '/789/ghi',
                        '/789/ghi/22222222222222222222222222222ghi',
                        '/789/ghi/22222222222222222222222222222ghi/weird2'):
                return True
            return False

        def _exists(arg):
            exists_calls.append(arg)
            return True

        def _shuffle(arg):
            shuffle_calls.append(arg)

        orig_listdir = db_replicator.os.listdir
        orig_isdir = db_replicator.os.path.isdir
        orig_exists = db_replicator.os.path.exists
        orig_shuffle = db_replicator.random.shuffle
        try:
            db_replicator.os.listdir = _listdir
            db_replicator.os.path.isdir = _isdir
            db_replicator.os.path.exists = _exists
            db_replicator.random.shuffle = _shuffle
            datadirs = [('/srv/node/sda/containers', 1),
                        ('/srv/node/sdb/containers', 2)]
            results = list(db_replicator.roundrobin_datadirs(datadirs))
            # The results show that the .db files are returned, the devices
            # interleaved.
            self.assertEquals(results, [
                ('123', '/srv/node/sda/containers/123/abc/'
                        '00000000000000000000000000000abc/'
                        '00000000000000000000000000000abc.db', 1),
                ('123', '/srv/node/sdb/containers/123/abc/'
                        '00000000000000000000000000000abc/'
                        '00000000000000000000000000000abc.db', 2),
                ('789', '/srv/node/sda/containers/789/ghi/'
                        '22222222222222222222222222222ghi/'
                        '22222222222222222222222222222ghi.db', 1),
                ('789', '/srv/node/sdb/containers/789/ghi/'
                        '22222222222222222222222222222ghi/'
                        '22222222222222222222222222222ghi.db', 2)])
            # The listdir calls show that we only listdir the dirs
            self.assertEquals(listdir_calls, [
                '/srv/node/sda/containers',
                '/srv/node/sda/containers/123',
                '/srv/node/sda/containers/123/abc',
                '/srv/node/sdb/containers',
                '/srv/node/sdb/containers/123',
                '/srv/node/sdb/containers/123/abc',
                '/srv/node/sda/containers/789',
                '/srv/node/sda/containers/789/ghi',
                '/srv/node/sdb/containers/789',
                '/srv/node/sdb/containers/789/ghi'])
            # The isdir calls show that we did ask about the things pretending
            # to be files at various levels.
            self.assertEquals(isdir_calls, [
                '/srv/node/sda/containers/123',
                '/srv/node/sda/containers/123/abc',
                ('/srv/node/sda/containers/123/abc/'
                 '00000000000000000000000000000abc'),
                '/srv/node/sdb/containers/123',
                '/srv/node/sdb/containers/123/abc',
                ('/srv/node/sdb/containers/123/abc/'
                 '00000000000000000000000000000abc'),
                ('/srv/node/sda/containers/123/abc/'
                 '11111111111111111111111111111abc'),
                '/srv/node/sda/containers/123/def.db',
                '/srv/node/sda/containers/456',
                '/srv/node/sda/containers/789',
                '/srv/node/sda/containers/789/ghi',
                ('/srv/node/sda/containers/789/ghi/'
                 '22222222222222222222222222222ghi'),
                ('/srv/node/sdb/containers/123/abc/'
                 '11111111111111111111111111111abc'),
                '/srv/node/sdb/containers/123/def.db',
                '/srv/node/sdb/containers/456',
                '/srv/node/sdb/containers/789',
                '/srv/node/sdb/containers/789/ghi',
                ('/srv/node/sdb/containers/789/ghi/'
                 '22222222222222222222222222222ghi'),
                ('/srv/node/sda/containers/789/ghi/'
                 '33333333333333333333333333333ghi'),
                '/srv/node/sda/containers/789/jkl',
                ('/srv/node/sdb/containers/789/ghi/'
                 '33333333333333333333333333333ghi'),
                '/srv/node/sdb/containers/789/jkl'])
            # The exists calls are the .db files we looked for as we walked the
            # structure.
            self.assertEquals(exists_calls, [
                ('/srv/node/sda/containers/123/abc/'
                 '00000000000000000000000000000abc/'
                 '00000000000000000000000000000abc.db'),
                ('/srv/node/sdb/containers/123/abc/'
                 '00000000000000000000000000000abc/'
                 '00000000000000000000000000000abc.db'),
                ('/srv/node/sda/containers/789/ghi/'
                 '22222222222222222222222222222ghi/'
                 '22222222222222222222222222222ghi.db'),
                ('/srv/node/sdb/containers/789/ghi/'
                 '22222222222222222222222222222ghi/'
                 '22222222222222222222222222222ghi.db')])
            # Shows that we called shuffle twice, once for each device.
            self.assertEquals(
                shuffle_calls, [['123', '456', '789'], ['123', '456', '789']])
        finally:
            db_replicator.os.listdir = orig_listdir
            db_replicator.os.path.isdir = orig_isdir
            db_replicator.os.path.exists = orig_exists
            db_replicator.random.shuffle = orig_shuffle

if __name__ == '__main__':
    unittest.main()


from kivy.app import App
from kivy.uix.label import Label
from kivy.lang import Builder
from kivy.uix.widget import Widget
from kivy.uix.floatlayout import FloatLayout
from kivy.uix.gridlayout import GridLayout
from kivy.properties import StringProperty
from kivy.properties import ObjectProperty
from kivy.clock import Clock
import japanize_kivy
from kivy.uix.textinput import TextInput
import subprocess
from Record import myaudio
from Sample import Sample
import threading
import concurrent.futures
from functools import partial
from multiprocessing import Process
import asyncio
from concurrent.futures import ThreadPoolExecutor
from kivy.uix.button import Button

#size_hint_y : y座標(0~2)
# rgba : r(赤色),g（黄色）,b（青色）,a（透明度）
#self.pos : Labelのxy座標
#self.size : Labelの幅と高さの値
#予約語 : self ・・・ そのWidgetを取得
#        root ・・・ rootWidgetを取得
#        app ・・・ Appを取得
Builder.load_string("""
<MyCanvas>
    name: name
    year: year
    month: month
    day: day
    voice_word: voice_word
    c_display: c_display
    word_buttons: word_buttons

    BoxLayout:
        orientation: 'vertical' # 'horizontal'だと列分割 verticalは行分割

        Label:
            id: c_display
            text: root.text
            font_size: 96
            size_hint_y: 1

        BoxLayout:
            orientation: 'horizontal'

            GridLayout:
                rows:3
                GridLayout:
                    rows:3
                    cols:2
                    size_hint_y: 0.3

                    Label:
                        text: "名前(ローマ字入力)"
                    TextInput:
                        id: name
                        text: ''
                    Label:
                        text: "年"
                    Spinner:    # 年のリスト表示
                        id: year
                        size_hint: .1,1
                        halign: 'center'
                        valign: 'middle'
                        text_size: self.size
                        text:'2020'
                        values: [str(y) for y in range(2020, 2022) ]
                    BoxLayout:
                        orientation: 'horizontal'
                        Label:
                            text: "月"
                        Spinner:
                            id: month   # 月のリスト表示
                            text:'01'
                            values: ['{0:02d}'.format(x)  for x in range(1,13)]
                    BoxLayout:
                        orientation: 'horizontal'
                        Label:
                            text: "日"
                        Spinner:    # 日のリスト表示
                            id: day
                            text:'01'
                            values: ['{0:02d}'.format(x)  for x in range(1,30)]
                Label:
                    size_hint_y: 0.2
                    id: voice_word
                    text: "発音言語"
                StackLayout:
                    size_hint_y: 0.5
                    id: word_buttons
                    orientation: 'lr-tb'

            Button:
                size_hint_x: 0.5
                id: button1_b
                text: "録音START"
                font_size: 24
                on_press: root.buttonClicked1_a()
                # on_press: root.buttonClicked1_b()

            Button:
                size_hint_x: 0.4
                id: button2
                text: "測定"
                font_size: 48
                on_press: root.buttonClicked2() #ボタンをクリックした時にpython側の関数を呼ぶ

""")

class MyCanvas(FloatLayout):
    global display_word1
    text = StringProperty()  # プロパティの追加
    name = ObjectProperty(None)
    year = ObjectProperty(None)
    month = ObjectProperty(None)
    day = ObjectProperty(None)
    voice_word = ObjectProperty(None)
    c_display = ObjectProperty(None)
    word_buttons = ObjectProperty()

    def __init__(self, **kwargs):
        super(MyCanvas, self).__init__(**kwargs)
        display_word1 = "STARTで録音開始"
        self.text = display_word1
        word_type = "2"  # 発音音声タイプを入力してください　1:モーラ　2:音声記号表
        self.panda = False
        if word_type == "1":
            self.word_dict = {"モーラ（あ）": "あ い う え お や ゆ よ", "モーラ（か）": "か き く け こ きゃ きゅ きょ",
                              "モーラ（が）": "が ぎ ぐ げ ご ぎゃ ぎゅ ぎょ",
                              "モーラ（さ）": "さ し す せ そ しゃ しゅ しょ", "モーラ（ざ）": "ざ じ ず ぜ ぞ じゃ じゅ じょ",
                              "モーラ（た）": "た ち つ て と ちゃ ちゅ ちょ", "モーラ（だ）": "だ で ど",
                              "モーラ（な）": "な に ぬ ね の にゃ にゅ にょ ", "モーラ（は）": "は ひ ふ へ ほ ひゃ ひゅ ひょ ",
                              "モーラ（ぱ）": "ぱ ぴ ぷ ぺ ぽ ぴゃ ぴゅ ぴょ",
                              "モーラ（ま）": "ま み む め も みゃ みゅ みょ", "モーラ（ら）": "ら り る れ ろ りゃ りゅ りょ", "モーラ（わ）": "わ"}
        elif word_type == "2":
            self.word_dict = {"音記（あ）": "あ い う え お ", "音記（か）": "か き く け こ きゃ きゅ きょ",
                              "音記（さ）": "さ し す せ そ しゃ しゅ しょ", "音記（た）": "た ち つ て と ちゃ ちゅ ちょ",
                              "音記（な）": "な に ぬ ね の にゃ にゅ にょ ", "音記（は）": "は ひ ふ へ ほ ひゃ ひゅ ひょ ",
                              "音記（ま）": "ま み む め も みゃ みゅ みょ", "音記（や）": "や　ゆ　よ", "音記（ら）": "ら り る れ ろ りゃ りゅ りょ",
                              "音記（わ）": "わ", "音記（が）": "が ぎ ぐ げ ご ぎゃ ぎゅ ぎょ", "音記（ざ）": "ざ じ ず ぜ ぞ じゃ じゅ じょ",
                              "音記（だ）": "だ で ど", "音記（ば）": "ば び ぶ べ ぼ びゃ びゅ びょ", "音記（ぱ）": "ぱ　ぴ　ぷ　ぺ　ぽ　ぴゃ　ぴゅ　ぴょ",
                              }
        word_list = list(self.word_dict.keys())
        for v_word in word_list:
            button = Button(text=v_word, size_hint=(.2, .35))
            button.bind(on_press=self.press_word)
            self.word_buttons.add_widget(button)

    def buttonClicked1_a(self):        # ボタンをクリック時
        name = self.name.text
        date = f"{self.year.text}{self.month.text}{self.day.text}"
        word = self.voice_word.text
        if self.panda == True:
            send_word = f"{self.voice_word.text}(文字あり)"
        else:
            send_word = f"{self.voice_word.text}(文字なし)"

        print(f"name is {self.name.text}")
        print(f"date is {date}")
        print(len(list(self.word_dict[word])))
        if len(list(self.word_dict[word])) == 5:
            recotime = 25
        elif len(list(self.word_dict[word])) == 1:
            recotime = 15
        else:
            recotime = 45
        ma = myaudio(name, date, send_word, recotime)
        ma.pyrecord()

    def buttonClicked1_b(self):
        word_list = self.voice_word.text.split()
        num = 2
        for w in word_list:
            Clock.schedule_once(partial(self.my_callback, w), num)
            num += 2
        Clock.schedule_once(partial(self.my_callback, "あ"), 12)

    def buttonClicked2(self):        # ボタンをクリック時
        subprocess.Popen(r".\WS151\WS.EXE")

    def press_word(self,value):
        word = value.text
        self.voice_word.text = word

    def my_callback(self,num,dt):
        global display_word1
        print("起動")
        display_word1 = f"{num}"
        print(f"display_word1 is {display_word1}")
        self.text = display_word1

class TestApp(App):
    def __init__(self, **kwargs):
        super(TestApp, self).__init__(**kwargs)
        self.title = 'Record Support'  # ウィンドウの名前を変更

    def build(self):
        return MyCanvas()

if __name__ == '__main__':
    TestApp().run()


"""
Given two integer arrays nums1 and nums2, return an array of their
intersection. Each element in the result must appear as many times
as it shows in both arrays and you may return the result in any order.
"""

__name__ = 'Narissa Tsuboi'

import collections

def intersect(nums1, nums2):
    """
    :type nums1: List[int]
    :type nums2: List[int]
    :rtype: List[int]
    """
    counter1 = collections.Counter(nums1)
    counter2 = collections.Counter(nums2)

    return (counter1 & counter2).elements()

if __name__ == '__main__':
    nums1 = [1, 2, 2, 1]
    nums2 = [2, 2]
    print(intersect(nums1, nums2))

350. Intersection of Two Arrays II/main.py


"""
This file contains all the utilities used in other scripts.
"""
import os
import sys
import subprocess
import numpy as np
import cv2
from natsort import natsorted
import scipy.io as scio
from tqdm import tqdm
import datetime
import inspect
import matplotlib.pyplot as plt
from scipy.ndimage import gaussian_filter1d
from src import defines as ds

def return_data_dict(data_dict, data_size, num_frames=None, frame_index=None, bounding_box=None, label=None):
    """
    Returns a dict containing all given data.
    :param data_dict:
    :param data_size:
    :param num_frames:
    :param frame_index:
    :param bounding_box:
    :param label:
    :return:
    """
    dict_subset = dict()
    dict_subset['data'] = data_dict
    dict_subset['data_size'] = data_size
    if num_frames is not None:
        dict_subset['num_frames'] = num_frames
    if frame_index is not None:
        dict_subset['frame_index'] = frame_index
    if bounding_box is not None:
        dict_subset['bounding_box'] = bounding_box
    if label is not None:
        dict_subset['label'] = label
    return dict_subset

def return_score_dict(data, scores, bounding_box=None, label=None):
    """
    Returns a dict containing the essential video information and the corresponding scores.
    :param data:
    :param scores:
    :param bounding_box:
    :param label
    :return:
    """
    assert data['data_size'] == len(scores), 'Size of scores mismatch.'

    dict_scores = dict()
    dict_scores['data_size'] = data['data_size']
    dict_scores['num_frames'] = data['num_frames']
    dict_scores['frame_index'] = data['frame_index']
    dict_scores['scores'] = scores
    if bounding_box is not None:
        dict_scores['bounding_box'] = data['bounding_box']
    if label is not None:
        dict_scores['label'] = data['label']
    return dict_scores

def load_detections(src_dir, video_names, det_mdl_name):
    """
    Loads all the detections for all the given videos.
    :param src_dir:
    :param video_names:
    :param det_mdl_name:
    :return:
    """
    # Get all the detections
    return np.array([np.load(os.path.join(os.path.join(src_dir, v), ds.DET_FILENAMES[det_mdl_name]), allow_pickle=True)
                     for v in tqdm(video_names, file=sys.stdout, desc='Loading detections')])

def visualize_data(dict_subset, n_samples_per_viz=ds.DEFAULT_N_SAMPLES_PER_VIZ):
    """
    Produces visualization image with some randomly selected RoI samples.
    :param dict_subset
    :param n_samples_per_viz:
    :return: Viz image.
    """
    assert dict_subset['data_size'] >= n_samples_per_viz, f'The data size must be >= {n_samples_per_viz}.'

    # Show some randomly selected samples
    idx = np.array(list(range(dict_subset['data_size'])))
    np.random.shuffle(idx)
    idx = idx[:n_samples_per_viz]

    # Merge images for nice visualisation
    img_size = len(next(iter(dict_subset['data'].values()))[0])
    mg_h = np.ones((img_size, ds.HORIZONTAL_MARGIN_SIZE), np.float32)

    # stacked images horizontally
    stack_images = []
    for _, data in dict_subset['data'].items():
        stack_images.append(cv2.hconcat([cv2.hconcat([mg_h, d[:, :, 0], mg_h]) for d in data[idx]]))

    # Stack stacked images vertically
    mg_b = np.ones((ds.HORIZONTAL_MARGIN_SIZE, stack_images[0].shape[1]), np.float32)
    mg_v = np.ones((ds.VERTICAL_MARGIN_SIZE, stack_images[0].shape[1]), np.float32)

    stacked_images = cv2.vconcat([mg_b] + intersperse(stack_images, mg_v) + [mg_b])

    # Convert to uint8
    stacked_images *= ds.WHITE_COLOR
    return stacked_images.astype('uint8')

class MAData:
    """
    Structure of the spatio-temporal RoI sample.
    """
    def __init__(self,
                 dataset=ds.DEFAULT_DATASET,
                 frame_diff_gap=ds.FRAME_DIFF_GAP,
                 min_train_det_score=ds.DEFAULT_MIN_TRAIN_DET_SCORE,
                 min_test_det_score=ds.DEFAULT_MIN_TEST_DET_SCORE,
                 n_frames_per_video=ds.DEFAULT_N_FRAMES_PER_VIDEO,
                 normalize_images=ds.NORMALIZE_IMAGES,
                 frames_step=ds.FRAMES_STEP,
                 det_model_name=ds.DEFAULT_DET_MODEL_NAME,
                 estimations_path=ds.ESTIMATIONS_PATH):
        """
        Initializes the data extraction parameters.
        :param dataset:
        :param frame_diff_gap:
        :param min_train_det_score:
        :param min_test_det_score:
        :param n_frames_per_video:
        :param normalize_images:
        :param frames_step:
        :param det_model_name:
        :param estimations_path:
        """
        self.dataset = dataset
        self.frame_diff_gap = frame_diff_gap
        self.n_frames_per_video = n_frames_per_video
        self.normalize_images = normalize_images
        self.frames_step = frames_step
        self.det_model_name = det_model_name
        self.estimations_path = estimations_path

        self.subset_dir_names = [ds.TRAIN_DIRNAME, ds.TEST_DIRNAME]
        self.subset_dirs = {s: os.path.join(self.estimations_path, self.dataset, s) for s in self.subset_dir_names}
        self.subset_videos = {s: natsorted([v for v in os.listdir(self.subset_dirs[s])
                                            if os.path.isdir(os.path.join(self.subset_dirs[s], v))])
                              for s in self.subset_dir_names}
        self.subset_det = {s: None for s in self.subset_dir_names}
        self.min_det_score = {ds.TRAIN_DIRNAME: min_train_det_score,
                              ds.TEST_DIRNAME: min_test_det_score}

        self.frame_size = ds.FRAME_SIZE
        self.roi_size = ds.ROI_SIZE

    def get_roi_subset(self,
                       subset_name,
                       video_name=None,
                       rand_select=False):
        """
        Extracts the RoI data from frames.
        :return:
        """
        assert subset_name in self.subset_dir_names, 'Invalid subset.'
        assert video_name is None or video_name in self.subset_videos[subset_name], 'Invalid video.'

        # Get detections if necessary
        if self.subset_det[subset_name] is None:
            self.subset_det[subset_name] = load_detections(self.subset_dirs[subset_name],
                                                           self.subset_videos[subset_name],
                                                           self.det_model_name)

        # Get samples randomly distributed
        data_curr = []
        data_past = []
        data_next = []
        n_frames = None
        frames_idx = []
        bounding_boxes = []

        if video_name is None:
            videos = tqdm(zip(self.subset_videos[subset_name], self.subset_det[subset_name]),
                          file=sys.stdout,
                          total=len(self.subset_videos[subset_name]),
                          desc='Extracting RoI subset')
        else:
            videos = zip(self.subset_videos[subset_name], self.subset_det[subset_name])

        for video, det_data in videos:
            if video_name is not None and video != video_name:
                continue

            video_dir = os.path.join(self.subset_dirs[subset_name], video)

            frames = np.load(os.path.join(video_dir, ds.FRAMES_FILENAME), mmap_mode='r')

            n_frames = len(frames)

            if video_name is not None or rand_select:
                idx = np.array(list(range(self.frame_diff_gap, n_frames - self.frame_diff_gap)))
            else:
                idx = np.array(list(range(self.frame_diff_gap, n_frames - self.frame_diff_gap, self.frames_step)))

            if rand_select:
                np.random.shuffle(idx)
                idx = idx[:self.n_frames_per_video]

            for i in idx:
                curr_img = frames[i]
                past_img = frames[i - self.frame_diff_gap]
                next_img = frames[i + self.frame_diff_gap]

                # Detections will be a python dict: {category_id : [[x1, y1, x2, y2, score], ...], }
                curr_a_det = det_data[i]
                # Remove empty detections
                curr_a_det = {k: curr_a_det[k] for k in curr_a_det if curr_a_det[k].size}
                # transform ret into [[x1, y1, x2, y2, score], ...]
                curr_a_det = np.vstack(list(curr_a_det.values()))

                # Exclude detections that have lower score
                curr_a_det = curr_a_det[curr_a_det[:, 4] >= self.min_det_score[subset_name], :]

                if not len(curr_a_det):
                    continue

                max_det_y_pixels = int(ds.MAX_DET_SIZE_RATIO * curr_img.shape[0])
                max_det_x_pixels = int(ds.MAX_DET_SIZE_RATIO * curr_img.shape[1])

                for box in curr_a_det:
                    (x1, y1, x2, y2) = box[:-1].astype(int)

                    # Bug fixed
                    if x1 < 0:
                        x1 = 0
                    if y1 < 0:
                        y1 = 0
                    if x2 > curr_img.shape[1]:
                        x2 = curr_img.shape[1]
                    if y2 > curr_img.shape[0]:
                        y2 = curr_img.shape[0]

                    if (x2 - x1) < ds.MIN_DET_PIXELS or (y2 - y1) < ds.MIN_DET_PIXELS:
                        continue

                    if self.dataset == 'ped1' or self.dataset == 'ped2':
                        # Fix whole frame detections on ped2
                        if (x2 - x1) > max_det_x_pixels and (y2 - y1) > max_det_y_pixels:
                            continue

                    # Resize RoIs
                    curr_resized = cv2.resize(curr_img[y1:y2, x1:x2], (self.roi_size, self.roi_size))
                    past_resized = cv2.resize(past_img[y1:y2, x1:x2], (self.roi_size, self.roi_size))
                    next_resized = cv2.resize(next_img[y1:y2, x1:x2], (self.roi_size, self.roi_size))

                    if self.normalize_images:
                        # Normalize images
                        curr_resized = cv2.normalize(curr_resized, None,
                                                     alpha=ds.MIN_NORM, beta=ds.MAX_NORM,
                                                     norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F)
                        past_resized = cv2.normalize(past_resized, None,
                                                     alpha=ds.MIN_NORM, beta=ds.MAX_NORM,
                                                     norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F)
                        next_resized = cv2.normalize(next_resized, None,
                                                     alpha=ds.MIN_NORM, beta=ds.MAX_NORM,
                                                     norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F)

                    # Store results
                    data_curr.append(curr_resized)
                    data_past.append(past_resized)
                    data_next.append(next_resized)

                    frames_idx.append(i)
                    bounding_boxes.append({'x1': x1, 'y1': y1, 'x2': x2, 'y2': y2})

        # Convert to arrays
        data_curr = np.array(data_curr)
        data_past = np.array(data_past)
        data_next = np.array(data_next)

        # Reshape and transform to float
        data_curr = data_curr.reshape((*data_curr.shape, 1))
        data_past = data_past.reshape((*data_past.shape, 1))
        data_next = data_next.reshape((*data_next.shape, 1))
        if not self.normalize_images:
            data_curr = (ds.MAX_NORM - ds.MIN_NORM) * data_curr.astype('float32') / ds.WHITE_COLOR + ds.MIN_NORM
            data_past = (ds.MAX_NORM - ds.MIN_NORM) * data_past.astype('float32') / ds.WHITE_COLOR + ds.MIN_NORM
            data_next = (ds.MAX_NORM - ds.MIN_NORM) * data_next.astype('float32') / ds.WHITE_COLOR + ds.MIN_NORM

        # Get data size
        data_size = len(data_curr)
        assert data_size == len(data_past) and data_size == len(data_next), 'Size of subset mismatch.'

        # Normalize data and return a dict containing data
        dict_data = dict()
        dict_data['curr'] = data_curr
        dict_data['past'] = data_past
        dict_data['next'] = data_next

        if video_name is None:
            return return_data_dict(dict_data, data_size)
        else:
            return return_data_dict(dict_data, data_size,
                                    frame_index=np.array(frames_idx),
                                    bounding_box=np.array(bounding_boxes), num_frames=n_frames)

class EST5EDT(datetime.tzinfo):

    def utcoffset(self, dt):
        return datetime.timedelta(hours=-5) + self.dst(dt)

    def dst(self, dt):
        d = datetime.datetime(dt.year, 3, 8)        # 2nd Sunday in March
        self.dston = d + datetime.timedelta(days=6-d.weekday())
        d = datetime.datetime(dt.year, 11, 1)       # 1st Sunday in Nov
        self.dstoff = d + datetime.timedelta(days=6-d.weekday())
        if self.dston <= dt.replace(tzinfo=None) < self.dstoff:
            return datetime.timedelta(hours=1)
        else:
            return datetime.timedelta(0)

    def tzname(self, dt):
        return 'EST5EDT'

def subprocess_cmd(command):
    process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
    proc_stdout = process.communicate()[0].strip()
    print(proc_stdout.decode())

def get_results(scores_data,
                smoothing_factor=ds.DEFAULT_SMOOTHING_FACTOR,
                normalize=ds.DEFAULT_NORMALIZE_SCORES,
                binarize=ds.DEFAULT_BINARIZE_SCORES):
    """
    Gets the frame-level reconstruction error scores for all test sequences.
    :param scores_data:
    :param smoothing_factor:
    :param normalize:
    :param binarize:
    :return:
    """
    min_score = scores_data['scores'].min()
    max_score = scores_data['scores'].max()

    if normalize:
        scores_data['scores'] = np.interp(scores_data['scores'], (min_score, max_score), (0, 1))

    full_frames = list(range(1, scores_data['num_frames'] + 1))

    # get all fame number used in the test data
    test_frames = list(set(scores_data['frame_index']))
    test_frames.sort()

    max_scores = []
    for f in test_frames:
        frame_scores = scores_data['scores'][scores_data['frame_index'] == f]
        max_scores.append(np.max(frame_scores))

    if binarize:
        threshold = np.mean(max_scores)
        test_scores = [1 if x > threshold else 0 for x in max_scores]
    else:
        test_scores = max_scores

    # Put missing scores in frames to 0 (normal)
    test_frames = np.array(test_frames)
    match_index = [np.where(test_frames == f)[0] for f in range(scores_data['num_frames'])]
    full_scores = np.array([test_scores[i[0]] if i.size > 0 else min_score for i in match_index])

    # Smooth scores using Gaussian 1d filter
    if smoothing_factor:
        final_scores = gaussian_filter1d(full_scores, smoothing_factor)
    else:
        final_scores = full_scores

    return np.array(full_frames), final_scores

def get_gt_results(dataset=ds.DEFAULT_DATASET,
                   dataset_path=ds.DATASETS_PATH):
    """
    Get the frame-level ground-truth results.
    Inspired from:
    https://github.com/StevenLiuWen/ano_pred_cvpr2018/blob/master/Codes/evaluate.py
    :param dataset:
    :param dataset_path:
    :return:
    """
    tests_dir = os.path.join(dataset_path, dataset, ds.TEST_DIRNAME)

    if dataset == 'shanghaitech':
        gt_dir = os.path.join(tests_dir, 'test_frame_mask')

        video_path_list = [f for f in os.listdir(gt_dir)]
        video_path_list.sort()

        gt = []
        for video in video_path_list:
            gt.append(np.load(os.path.join(gt_dir, video)).tolist()[1:])
    else:
        gt_dir = os.path.join(dataset_path, dataset)
        gt_filename = '{}.mat'.format(dataset)

        abnormal_events = scio.loadmat(os.path.join(gt_dir, gt_filename), squeeze_me=True)['gt']

        if abnormal_events.ndim == 2:
            abnormal_events = abnormal_events.reshape(-1, abnormal_events.shape[0], abnormal_events.shape[1])

        num_video = abnormal_events.shape[0]

        frames_dir = os.path.join(tests_dir, ds.FRAMES_DIRNAME)
        video_list = [v for v in os.listdir(frames_dir) if os.path.isdir(os.path.join(frames_dir, v))]
        video_list.sort()

        assert num_video == len(video_list), 'ground truth does not match the number of testing videos. {} != {}' \
            .format(num_video, len(video_list))

        # get the total frames of sub video
        def get_video_length(sub_video_number):
            # video_name = video_name_template.format(sub_video_number)
            video_name = os.path.join(frames_dir, video_list[sub_video_number])
            assert os.path.isdir(video_name), '{} is not directory!'.format(video_name)
            return len(os.listdir(video_name))

        gt = []
        for i in range(num_video):
            length = get_video_length(i)

            sub_video_gt = np.zeros((length,), dtype=np.int8)
            sub_abnormal_events = abnormal_events[i]
            if sub_abnormal_events.ndim == 1:
                sub_abnormal_events = sub_abnormal_events.reshape((sub_abnormal_events.shape[0], -1))

            _, num_abnormal = sub_abnormal_events.shape

            for j in range(num_abnormal):
                # (start - 1, end - 1)
                start = int(sub_abnormal_events[0, j]) - 1
                end = int(sub_abnormal_events[1, j])

                sub_video_gt[start: end] = 1

            gt.append(sub_video_gt.tolist()[1:])  # Skip the first frame

    return gt

def convert_videos_to_frames(videos_dir, frames_dir):
    """
    Converts all videos in the videos dir to frames and saves them to frames dir.
    :param videos_dir:
    :param frames_dir:
    :return:
    """
    videos = [v for v in os.listdir(videos_dir) if os.path.isfile(os.path.join(videos_dir, v)) and
              any(v[v.rfind('.'):] == ext for ext in ds.VIDEO_EXTENSIONS)]
    videos = natsorted(videos)
    for video in tqdm(videos, file=sys.stdout, desc='Extracting frames'):
        video_frames_dir = os.path.join(frames_dir, video[:video.rfind('.')])
        if not os.path.exists(video_frames_dir):
            os.makedirs(video_frames_dir)

        video_cap = cv2.VideoCapture(os.path.join(videos_dir, video))
        success, frame = video_cap.read()
        count = 0
        while success:
            # save frame as JPEG file
            cv2.imwrite(os.path.join(video_frames_dir, '{:04d}.jpg'.format(count)), frame)
            success, frame = video_cap.read()
            count += 1

def get_class_from_name(module, class_name):
    """
    Gets the class from the string name within the given module.
    :param module:
    :param class_name:
    :return: class
    """
    cls_names = [m[0] for m in inspect.getmembers(module, inspect.isclass)]
    assert class_name.lower() in [x.lower() for x in cls_names], 'Class name not found.'
    cls_name = cls_names[[x.lower() for x in cls_names].index(class_name.lower())]
    return getattr(module, cls_name)

def intersperse(lst, item):
    """
    Adds the item between each item in the list.
    :param lst:
    :param item:
    :return:
    """
    result = [item] * (len(lst) * 2 - 1)
    result[0::2] = lst
    return result

def plot_video_anomaly_curve(frame_index, frame_score, true_score, filename):
    """
    Plots the video frame-level anomaly score.
    :param frame_index:
    :param frame_score:
    :param true_score:
    :param filename:
    :return:
    """
    assert len(true_score) == len(frame_score), 'Mismatch between the size of true and predicted scores.'
    # Plot a curve of scores and threshold_value
    fig = plt.figure(figsize=(10, 4))
    plt.plot(frame_index, frame_score, 'r', label='score', lw=3)
    plt.fill_between(frame_index, np.zeros(frame_index.shape), true_score, facecolor='c', label='ground-truth', lw=1)
    plt.xlabel('Frame')
    plt.ylabel('Score')
    plt.yticks(np.arange(0, 1.2, step=0.2))
    plt.legend(loc='best')
    fig.savefig(filename, bbox_inches='tight')
    plt.close(fig)

def add_bool_parser(parser, arg_name, default_arg):
    """
    Adds a boolean parser.
    Ref.: https://stackoverflow.com/questions/15008758/parsing-boolean-values-with-argparse
    :param parser:
    :param arg_name:
    :param default_arg:
    :return:
    """
    group = parser.add_mutually_exclusive_group(required=False)
    group.add_argument('--' + arg_name, dest=arg_name, action='store_true')
    group.add_argument('--no-' + arg_name, dest=arg_name, action='store_false')
    parser.set_defaults(**{arg_name: default_arg})