import numpy as np

from rules import RANK_COUNT, SUIT_COUNT, RANKS, SUITS
from hand_rank import hand_rank, hand_rank_untie

CARDS_SHAPE = (RANK_COUNT, SUIT_COUNT)

class CardSet:
    def __init__(self, cards=None):
        if cards is None:
            self.cards = np.zeros(CARDS_SHAPE, dtype=np.bool)
        else:
            assert cards.shape == CARDS_SHAPE and cards.dtype == np.bool
            self.cards = np.copy(cards)
        self.hand_rank = None

    def copy(self):
        return CardSet(self.cards)

    def __add__(self, other):
        assert isinstance(other, CardSet)
        assert np.any(self.cards & other.cards) == False
        return CardSet(self.cards + other.cards)

    def __iadd__(self, other):
        assert isinstance(other, CardSet)
        assert np.any(self.cards & other.cards) == False
        self.cards += other.cards
        return self

    def rank(self):
        self.hand_rank = hand_rank(self.cards)
        return self.hand_rank

    def rank_untie(self):
        assert self.hand_rank is not None
        return hand_rank_untie(self.cards, self.hand_rank)

    def count(self):
        return np.sum(self.cards)

    def __repr__(self):
        res = []
        res.append("[")
        cards = np.transpose(np.nonzero(self.cards))
        for pos in cards:
            i, j = pos
            res.append(RANKS[i] + SUITS[j])
        res.append("]")
        return " ".join(res)


# coding=utf-8

from django.shortcuts import render, HttpResponse, HttpResponseRedirect
from django.http import HttpResponseBadRequest, HttpResponseRedirect, HttpResponse
from django.contrib.auth.decorators import login_required
from django.core.urlresolvers import reverse
from base.core.dateutils import *
from django.contrib import messages
import json
from django.db import connections
import datetime
from django.views.generic import View
from django.http.response import StreamingHttpResponse
from app.user.utils.check_role import check_role_required
from app.user.models import User
import logging

def my_custom_stat_sql(sql):
    cursor = connections["statstic"].cursor()
    cursor.execute(sql)
    rows = cursor.fetchall()
    cursor.close()
    return rows

def my_custom_sql(sql,db="bskgk_slave"):
    cursor = connections[db].cursor()
    cursor.execute(sql)
    rows = cursor.fetchall()
    cursor.close()
    return rows

def my_custom_one_sql(sql,db="bskgk_slave"):
    cursor = connections[db].cursor()
    cursor.execute(sql)
    rows = cursor.fetchone()
    cursor.close()
    return rows

class SqlExecute(object):

    @classmethod
    def fetch_all(cls, sql, db="bskgk"):
        cursor = connections[db].cursor()
        cursor.execute(sql)
        rows = cursor.fetchall()
        cursor.close()
        return rows

    @classmethod
    def fetch_one(cls, sql, db="bskgk"):
        cursor = connections[db].cursor()
        cursor.execute(sql)
        row = cursor.fetchone()
        cursor.close()
        return row

    @classmethod
    def execute_many(cls, sql, params, db="bskgk"):
        cursor = connections[db].cursor()
        cursor.executemany(sql, params)
        cursor.close()


import xlsxwriter
import time
import shutil
import os
import pandas as pd
import sqlite3

def writer_report():  #officer
    workbook = xlsxwriter.Workbook('name_file.xlsx')
    worksheet = workbook.add_worksheet()

    my_list = [ID_card, Name ,Day, Date, Time,Arrival,Status]  #writehead
    for col_num, data in enumerate(my_list):
        worksheet.write(0, col_num, data)
    #ลูปลงข้อมูล

    #ลูปคำนวนStatus

def writer_daily():

    workbook = xlsxwriter.Workbook('name_file.xlsx')
    worksheet = workbook.add_worksheet()

    my_list = [ID_card, Name ,Day, Date, Time,Arrival,Depart,Office]  #writehead
    for col_num, data in enumerate(my_list):
        worksheet.write(0, col_num, data)

    workbook.close()
def writer_month():
    return

''''
def Excel_Daily(x ,camera_out):
    file_name = x + 'xlsx' #date
    workbook = xlsxwriter.Workbook('file_name')
    worksheet = workbook.add_worksheet()
    my_list = [[1, 1, 1, 1, 1],
           [2, 2, 2, 2, 1],
           [3, 3, 3, 3, 1],
           [4, 4, 4, 4, 1],
           [5, 5, 5, 5, 1]]
    for row_num, row_data in enumerate(my_list):
        for col_num, col_data in enumerate(row_data):
            worksheet.write(row_num, col_num, col_data)
    workbook.close()
    return 0
def Excel_month():
    return 0
'''


import sys
import json

print("\n".join(sys.argv))

sources = [ '/Users/andy/Dropbox/Work/WAP/Data/DC/kafka-report-2018-07-13-11-56.json',
            '/Users/andy/Dropbox/Work/WAP/Data/DC/kafka-report-2018-07-13-12-15.json',
            '/Users/andy/Dropbox/Work/WAP/Data/DC/kafka-report-2018-07-13-13-12.json' ]

for source in sources:
    paritions = {}
    consumed = 0
    with open(source) as f:
        print(source)
        for h in json.load(f):
            kr = h['state']['message']
            counter = 0
            for line in kr.split('\n'):
                if "partition: " in line:
                    line = line.replace("  partition: ","")
                    line = line.replace(" offset: ","")
                    p, o = line.split(',')
                    p = int(p)
                    o = int(o)
                    if p in paritions:
                        print("ACK")
                        exit()
                    paritions[p] = o
                    consumed += o
                    counter += 1
            print("Count: %i" % counter)

    print(json.dumps(paritions, indent=2))
    print("Total Consumed: %i" % consumed)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""

"""

import os
import time
import numpy as np
from importlib import import_module
import torch
from torch.utils.data import DataLoader
from dataload_val import AirwayData
import skimage.measure as measure
import nibabel
from skimage.morphology import skeletonize_3d

def network_prediction(data_path, save_path):
    casemodel = import_module('WingsNet')
    config2, case_net = casemodel.get_model()
    checkpoint = torch.load('WingsNet_GUL.ckpt')
    case_net.load_state_dict(checkpoint['state_dict'])
    val_path = data_path
    dataset = AirwayData(val_path, train = False)
    val_loader_case = DataLoader(dataset, batch_size = 1, shuffle = False)
    case_net = case_net.cuda()
    case_net.eval()
    save_path = save_path
    #sliding window
    cube_size = 128
    step = 64
    for i,(x,y,cb,patient) in enumerate(val_loader_case):
        pred = np.zeros(x.shape)
        pred_num = np.zeros(x.shape)
        x = x.cuda()
        xnum = (x.shape[2]-cube_size)//step + 1 if (x.shape[2]-cube_size)%step==0 else (x.shape[2]-cube_size)//step + 2
        ynum = (x.shape[3]-cube_size)//step + 1 if (x.shape[3]-cube_size)%step==0 else (x.shape[3]-cube_size)//step + 2
        znum = (x.shape[4]-cube_size)//step + 1 if (x.shape[4]-cube_size)%step==0 else (x.shape[4]-cube_size)//step + 2
        for xx in range(xnum):
            xl = step*xx
            xr = step*xx + cube_size
            if xr > x.shape[2]:
                xr = x.shape[2]
                xl = x.shape[2]-cube_size
            for yy in range(ynum):
                yl = step*yy
                yr = step*yy + cube_size
                if yr > x.shape[3]:
                    yr = x.shape[3]
                    yl = x.shape[3] - cube_size
                for zz in range(znum):
                    zl = step*zz
                    zr = step*zz + cube_size
                    if zr > x.shape[4]:
                        zr = x.shape[4]
                        zl = x.shape[4] - cube_size

                    x_input = x[:,:,xl:xr,yl:yr,zl:zr]
                    p0, p = case_net(x_input)
                    p = torch.sigmoid(p)
                    p = p.cpu().detach().numpy()
                    pred[:,:,xl:xr,yl:yr,zl:zr] += p
                    pred_num[:,:,xl:xr,yl:yr,zl:zr] += 1

        pred = pred/pred_num
        pred[pred>=0.5] = 1
        pred[pred<0.5] = 0
        pred = np.squeeze(pred)
        x = x.cpu().detach().numpy()
        x = np.squeeze(x)
        y = y.detach().numpy()
        y = np.squeeze(y)

        save_name_x = save_path + patient[0][0].split('_')[0] + '_img' + '.nii.gz'
        save_name_p = save_path + patient[0][0].split('_')[0] + '_pred' + '.nii.gz'
        save_name_y = save_path + patient[0][0].split('_')[0] + '_label' + '.nii.gz'

        img_nii = nibabel.Nifti1Image(x, np.eye(4))
        nibabel.save(img_nii, save_name_x)
        label_nii = nibabel.Nifti1Image(y, np.eye(4))
        nibabel.save(label_nii, save_name_y)
        pred_nii = nibabel.Nifti1Image(pred, np.eye(4))
        nibabel.save(pred_nii, save_name_p)

def evaluation(data_path, parsing_path):
    file_list = os.listdir(data_path)
    file_list.sort()
    file_list_parse = os.listdir(parsing_path)
    file_list_parse.sort()
    n = 3

    sens = []
    pres = []
    branches = []
    for i in range(len(file_list)//n):
        img = nibabel.load(os.path.join(data_path, file_list[n*i]))
        label = nibabel.load(os.path.join(data_path, file_list[n*i+1]))
        pred = nibabel.load(os.path.join(data_path, file_list[n*i+2]))
        parsing = nibabel.load(os.path.join(parsing_path, file_list_parse[6*i+4])) #please refer to tree_parse.py
        img = img.get_data()
        label = label.get_data()
        pred = pred.get_data()
        parsing = parsing.get_data()

        cd, num = measure.label(pred, return_num = True, connectivity=1)
        volume = np.zeros([num])
        for k in range(num):
            volume[k] = ((cd==(k+1)).astype(np.uint8)).sum()
        volume_sort = np.argsort(volume)
        #print(volume_sort)
        large_cd = (cd==(volume_sort[-1]+1)).astype(np.uint8)

        skeleton = skeletonize_3d(label)
        skeleton = (skeleton > 0)
        skeleton = skeleton.astype('uint8')

        sen = (large_cd*skeleton).sum()/skeleton.sum()
        sens.append(sen)

        pre = (large_cd*label).sum()/large_cd.sum()
        pres.append(pre)

        num_branch = parsing.max()
        detected_num = 0
        for j in range(num_branch):
            branch_label = ((parsing==(j+1)).astype(np.uint8))*skeleton
            if (large_cd*branch_label).sum()/branch_label.sum() >= 0.8:
                detected_num += 1
        branch = detected_num/num_branch
        branches.append(branch)

        print(file_list[n*i].split('_')[0], "Length: %0.4f"%(sen), "Precision: %0.4f"%(pre), "Branch: %0.4f"%(branch))

    sen1_mean = np.mean(sens)
    sen1_std = np.std(sens)
    pre_mean = np.mean(pres)
    pre_std = np.std(pres)
    branch_mean = np.mean(branches)
    branch_std = np.std(branches)
    print("len mean: %0.4f (%0.4f), branch: %0.4f (%0.4f), pre: %0.4f (%0.4f)"%(sen1_mean, sen1_std, branch_mean, branch_std, pre_mean, pre_std))

if __name__=='__main__':
    data_path = "/data_path/"
    save_path = "/save_path/"
    parsing_path = "/parsing_path/"
    network_prediction(data_path, save_path)
    evaluation(save_path, parsing_path)


"""
Given a positive integer n, return the number of all possible attendance records with length n, which will be regarded as rewardable. The
answer may be very large, return it after mod 109 + 7.

A student attendance record is a string that only contains the following three characters:

'A' : Absent.
'L' : Late.
'P' : Present.
A record is regarded as rewardable if it doesn't contain more than one 'A' (absent) or more than two continuous 'L' (late).

Example 1:
Input: n = 2
Output: 8
Explanation:
There are 8 records with length 2 will be regarded as rewardable:
"PP" , "AP", "PA", "LP", "PL", "AL", "LA", "LL"
Only "AA" won't be regarded as rewardable owing to more than one absent times.
Note: The value of n won't exceed 100,000.
"""
import collections
class Solution:
    def checkRecord(self, n):
        # [x, y, z]
        # x - ending with 'P'/'A'
        # y - ending with 'L' excluding 'LL'
        # z - ending with 'LL'
        mod = 10**9 + 7
        cnt_a = [1, 0, 0]
        cnt = [1, 1, 0]
        while n > 1:
            prev = cnt[:]
            prev_a = cnt_a[:]
            cnt_a[0] = (sum(prev_a) + sum(prev)) % mod
            cnt_a[1] = prev_a[0]
            cnt_a[2] = prev_a[1]

            cnt[0] = (prev[0] + prev[1] + prev[2]) % mod
            cnt[1] = prev[0]
            cnt[2] = prev[1]
            n -= 1
        return (sum(cnt_a) + sum(cnt)) % mod

    def checkRecord(self, n):
        '''
        P, L, A: rewardable ways ending with P, L, A
        P[i] = P[i-1] + L[i-1] + A[i-1]
        L[i] = P[i-1] + A[i-1] + P[i-2] + A[i-2]
        A[i] = noAP[i-1] + noAL[i-1]
        noAP[i] = noAP[i-1] + noAL[i-1]
        noAL[i] = noAP[i-1] + noAP[i-2]
        => A[i] = A[i-1] + A[i-2] + A[i-3]
        '''
        if n == 1: return 3
        mod = 10**9 + 7
        P, L, A = [0] * n, [0] * n, [0] * max(n, 3)
        P[0] = L[0] = A[0] = 1
        L[1] = 3
        A[1] = 2
        A[2] = 4
        for i in range(1, n):
            P[i] = (P[i-1] + L[i-1] + A[i-1]) % mod
            if i >= 2: L[i] = (P[i-1] + A[i-1] + P[i-2] + A[i-2]) % mod
            if i >= 3: A[i] = (A[i-1] + A[i-2] + A[i-3]) % mod
        return (P[n-1] + L[n-1] + A[n-1]) % mod

    def checkRecord_with_res(self, n):
        """
        :type n: int
        :rtype: int
        """
        def is_L_allowed(s):
            return len(s) < 2 or s[-1] != 'L' or s[-2] != 'L'

        res_a = collections.deque('A')
        res = collections.deque(['L', 'P'])
        l = 1
        while l < n:
            while res_a and len(res_a[0]) == l:
                record = res_a.popleft()
                res_a.append(record+'P')
                if is_L_allowed(record):
                    res_a.append(record+'L')
            while res and len(res[0]) == l:
                record = res.popleft()
                res_a.append(record+'A')
                res.append(record+'P')
                if is_L_allowed(record):
                    res.append(record+'L')
            l += 1

        return len(res_a) + len(res)

s = Solution()
assert s.checkRecord(1) == 3
assert s.checkRecord(2) == 8
assert s.checkRecord(3) == 19
assert s.checkRecord(100000) == 749184020
print(s.checkRecord(66666))


# -*- coding: utf-8 -*-

import codecs
import random
from typing import List, Tuple, Dict

from docopt import docopt
import progressbar

from FileLoading import file2object
from Relation import Relation

__doc__ = ''' {f}

Usage:
    {f} <FILEp> <FILEn> [-o | --output OUTPUT] [-t | --testdata TEST_DATA] [-e | --extract NUM]
    {f} -h | --help

Options:
    -h, --help                  Show this screen and exit.
    -o, --output=OUTPUT         output file name setting [default: LearnData.txt]
    -t, --testdata=TEST_DATA    test data file name setting [default: TestData.txt]
    -e, --extract=NUM           number of extract data. [default: 100]
'''.format(f=__file__)

def main():
    # option definition
    input_options = docopt(__doc__)
    # print(input_options)
    fin_p_name = input_options['<FILEp>']
    fin_n_name = input_options['<FILEn>']
    fout_name = input_options['--output'][0]
    ftest_name = input_options['--testdata'][0]
    extract = int(input_options['--extract'][0])

    # convert text from Relation list
    relation_p = file2object(fin_p_name)
    test_p, learn_p = separate_data(relation_p, extract)
    print()
    relation_n = file2object(fin_n_name)
    test_n, learn_n = separate_data(relation_n, extract)

    # concat test_p and test_n
    test = test_p
    test.extend(test_n)

    # file open
    fout = codecs.open(fout_name, 'w', 'utf-8')
    ftest = codecs.open(ftest_name, 'w', 'utf-8')

    for rel in test:
        ftest.write('{} {}\n'.format(rel.hyponym, rel.hypernym))

    for rel in learn_p.values():
        fout.write('{}'.format(rel.str_special()))

    fout.close()
    ftest.close()

def separate_data(relation: Dict[str, Relation],extract: int,
                  progressive_rate=10000) -> Tuple[List[Relation], Dict[str, Relation]]:
    extract = int(extract)
    # make all word set dictionary
    wordpair_list = {}

    # convert specific key : WordPair dictionary from Relation list
    print('Convert WordPair Dict from Relation List.', end='')
    count = 0
    for key, item in relation.items():
        parent = key
        for child in item.children:
            specific_key = str(count + 1)
            count += 1
            wp = WordPair(child, parent)
            wordpair_list[specific_key] = wp

            if count % progressive_rate == 0:
                print('.', end='', flush=True)
    print('Complete!')

    print('Test data extracting.', end='', flush=True)

    # random extracting
    sample = random.sample(list(wordpair_list.keys()), extract)
    print('.', end='', flush=True)

    # output samples
    test_data = []
    td_append = test_data.append
    for key in sample:
        td_append(wordpair_list[key])
        del wordpair_list[key]
        print('.', end='', flush=True)
    print('Complete!')

    print('Convert Relation List from WordPair Dict.', end='')
    count = 0

    # convert Relation list from WordPair dictionary
    learn_data = {}
    for value in wordpair_list.values():
        key = value.hypernym
        if key in learn_data:
            learn_data[key].append_child(value.hyponym)
        else:
            learn_data[key] = Relation(key)
            learn_data[key].append_child(value.hyponym)

        count += 1
        if count % progressive_rate == 0:
            print('.', end='', flush=True)
    print('Complete!')

    return test_data, learn_data

class WordPair(object):
    def __init__(self, hypo: str, hype: str) -> None:
        self.hyponym = hypo
        self.hypernym = hype

    def __str__(self):
        ret = ''
        ret += '({}, {})'.format(self.hyponym, self.hypernym)
        return ret

if __name__ == '__main__':
    main()


#!/usr/bin/env python3

import argparse
import logging
from collections import namedtuple, Counter

logging.basicConfig(level=logging.DEBUG)

parser = argparse.ArgumentParser()
parser.add_argument('readtag_tsv', type=argparse.FileType('r'), help="TSV output from readtag script")
parser.add_argument('cd_hit_est_fasta', type=argparse.FileType('r'), help="FASTA output from CD-HIT-EST")
parser.add_argument('cd_hit_est_clstr', type=argparse.FileType('r'), help="CLSTR output from CD-HIT-EST")
args = parser.parse_args()

# read in clstr file
logging.info("Read clusters from clstr file")

clusters = {}
tmp_cluster = ""
tmp_ids = []

# read in tsv file
logging.info("Read in TSV file")

ReadtagInfo = namedtuple('ReadtagInfo', ['sequence', 'origin'])

tsv = {}
line: str
for line in args.readtag_tsv:
    if line.startswith('#'):
        continue
    fields = line.strip().split('\t')
    ident, sequence, origin = fields
    tsv[ident] = ReadtagInfo(sequence=sequence, origin=origin)
logging.debug(tsv)

# read in fasta
id_to_cluster_sequence = {}
logging.info("Read in FASTA")
for line in args.cd_hit_est_fasta:
    if line.startswith(';'):
        continue
    if line.startswith('>'):
        header = line[1:].strip()
        sequence = args.cd_hit_est_fasta.readline().strip()
        id_to_cluster_sequence[header] = sequence
logging.debug(id_to_cluster_sequence)

for line in args.cd_hit_est_clstr:
    if line.startswith('>'):
        if tmp_cluster != "":
            clusters[tmp_cluster] = tmp_ids
            tmp_ids = []
        tmp_cluster = line[1:].strip()
    else:

        sequence_id = line.split('>')[1].split('...')[0]
        tmp_ids.append(sequence_id)
clusters[tmp_cluster] = tmp_ids

logging.debug(clusters)

# generate output
output_tsv_file = args.readtag_tsv.name.replace('.tsv', '.clusters.tsv')
logging.info(f"Writing enriched TSV file to {output_tsv_file}")
cluster_stats_file = args.readtag_tsv.name.replace('.tsv', '.clusters.stats')
logging.info(f"Writing stats file to {cluster_stats_file}")

histogram_counts = Counter()

with open(output_tsv_file, 'w') as out, open(cluster_stats_file, 'w') as stats:
    out.write("#id\t15N sequence\tfound in\tcluster\tcluster_seq\n")
    stats.write("#cluster id\tsequence\tsize\n")
    for cluster_id, cluster_sequence_ids in clusters.items():
        out.write(f"#{cluster_id}\n")

        # find cluster seq
        cluster_seq = ""
        for id_in_cluster in cluster_sequence_ids:
            if not cluster_seq == "":
                break
            cluster_seq = id_to_cluster_sequence.get(id_in_cluster, '')
        if cluster_seq == "":
            logging.warning(f"No cluster sequence found for {cluster_id}")

        cluster_size = len(cluster_sequence_ids)
        stats.write(f"{cluster_id}\t{cluster_seq}\t{cluster_size}\n")
        histogram_counts[cluster_size] += 1

        for sequence_id in cluster_sequence_ids:
            tsv_info = tsv[sequence_id]
            out.write(f"{sequence_id}\t{tsv_info.sequence}\t{tsv_info.origin}\t{cluster_id}\t{cluster_seq}\n")

cluster_hist_file = args.readtag_tsv.name.replace('.tsv', '.clusters.histdata')
logging.info(f"Writing stats file to {cluster_hist_file}")
with open(cluster_hist_file, 'w') as hist_data:
    hist_data.write(f"#cluster size\tcount\n")
    for cls, count in histogram_counts.items():
        hist_data.write(f"{cls}\t{count}\n")

args.readtag_tsv.close()
args.cd_hit_est_fasta.close()
args.cd_hit_est_clstr.close()


from world_ups_proto import world_ups_pb2
from amazon_ups_proto import amazon_ups_pb2
from google.protobuf.internal.encoder import _EncodeVarint
from google.protobuf.internal.decoder import _DecodeVarint32

import socket

HOST = '0.0.0.0'  # Standard loopback interface address (localhost)
PORT = 65432        # Port to listen on (non-privileged ports are > 1023)

def recv_from_ups():
    var_int_buff = []
    while True:
        buf = conn.recv(1)
        var_int_buff += buf
        msg_len, new_pos = _DecodeVarint32(var_int_buff, 0)
        if new_pos != 0:
            break
    whole_message = conn.recv(msg_len)
    print('len is =='+str(msg_len)+'   pos is=='+str( new_pos))
    UConnect = world_ups_pb2.UConnect()
    UConnect.ParseFromString(whole_message)
    print(UConnect)

ups_fd =  socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ups_fd.bind((HOST, PORT))
ups_fd.listen()
conn, addr = ups_fd.accept()
print('Connected by', addr)

def main():
    print("in main")
    recv_from_ups()

if __name__== "__main__":
  main()


# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /Users/chschw/Workspace/django/lab/django-cms/site/statify/utils.py
# Compiled at: 2013-04-24 04:02:44
import requests

def url_is_valid(url):
    valid_codes = [
     200, 301, 302]
    request = requests.get(url)
    if request.status_code in valid_codes:
        return True
    else:
        return False


import operator

import pytest

from pretend import stub, PY3K

class TestPretend(object):
    def test_attribute(self):
        x = stub(attr=3)
        assert hasattr(x, "attr")
        assert x.attr == 3

    def test_function(self):
        x = stub(meth=lambda x, y: x + y)
        assert x.meth(3, 4) == 7

    def test_iter(self):
        x = stub(__iter__=lambda: iter([1, 2, 3]))
        iterator = iter(x)
        assert next(iterator) == 1

    @pytest.mark.skipif("not PY3K")
    def test_next(self):
        x = stub(__next__=lambda: 12)
        assert next(x) == 12

    def test_contains(self):
        x = stub(__contains__=lambda other: True)
        assert "hello world" in x

    @pytest.mark.skipif("PY3K")
    def test_nonzero(self):
        x = stub(__nonzero__=lambda: False)
        assert not bool(x)

    @pytest.mark.skipif("not PY3K")
    def test_bool(self):
        x = stub(__bool__=lambda: False)
        assert not bool(x)

    def test_len(self):
        x = stub(__len__=lambda: 12)
        assert len(x) == 12

    @pytest.mark.parametrize(("func", "op"), [
        (operator.lt, "__lt__"),
        (operator.le, "__le__"),
        (operator.eq, "__eq__"),
        (operator.ne, "__ne__"),
        (operator.gt, "__gt__"),
        (operator.ge, "__ge__"),

        (operator.add, "__add__"),
        (operator.and_, "__and__"),
        (divmod, "__divmod__"),
        (operator.floordiv, "__floordiv__"),
        (operator.lshift, "__lshift__"),
        (operator.mod, "__mod__"),
        (operator.mul, "__mul__"),
        (operator.or_, "__or__"),
        (operator.pow, "__pow__"),
        (operator.rshift, "__rshift__"),
        (operator.sub, "__sub__"),
        (operator.truediv, "__truediv__"),
        (operator.xor, "__xor__"),
    ])
    def test_special_binops(self, func, op):
        x = stub(**{
            op: lambda y: func(2, y)
        })
        assert func(x, 4) == func(2, 4)
        assert func(x, 2) == func(2, 2)

    @pytest.mark.skipif("PY3K")
    def test_div(self):
        x = stub(
            __div__=lambda y: 4
        )
        assert x / 3 == 4

    def test_missing_op_error(self):
        x = stub()
        with pytest.raises(TypeError):
            x + 2


import RPIO
import time
import logging

######################################
# Configuration
######################################

# Use GPIO.BCM mode for pin numbers

# The output pin used to supply the 3.3V power
outPin = 10

# The input pin used to detect events
inPin = 7

######################################
# End of configuration.
######################################

print ("Using following configuration:")
print ("- detection pin: ", inPin)
print ("- power supply pin: ", outPin)

# Setup pin numbering and input/output pins.
def initPins():
	RPIO.setmode(RPIO.BCM)

	RPIO.setup(outPin, RPIO.OUT)
	RPIO.setup(inPin, RPIO.IN)

# Perform simple selftest.
# Checks if the readout works as expected assuming the switch is open.
def selftest():
	RPIO.output(outPin, False)
	r1 = RPIO.input(inPin)

	RPIO.output(outPin, True)
	r2 = RPIO.input(inPin)

	if r1 != False or r2 != True:
		raise Exception("Selftest failed. Readout should have been (False, True), but was (", r1, ", ", r2, ").")
	else:
		print("Selftest ok.")

def inputCallback(gpio_id, value):
	r = RPIO.input(inPin)
	logging.debug("GPIO %s triggered with %s and read %s" % (gpio_id, value, r))

######################################

logging.basicConfig(level=logging.DEBUG)

initPins()
selftest()

RPIO.add_interrupt_callback(inPin, inputCallback, edge='falling')
RPIO.wait_for_interrupts()

print("done.")


from matplotlib import pyplot as plt
import numpy as np

class MyPlot:

	@staticmethod
	def miccheck():
		return '1,2'

	@staticmethod
	def m_by_n(r,c):
		"""fills across then down
		   should be index x[0],x[1] in subplots """
		return map( lambda x: ((x-(x%c))/c, x % c) ,range(0,r*c) )

	@staticmethod
	def all_hist( df, col_list):

		""" for df is pandasdf """

		max_x = 5

		bplt , axs = plt.subplots(nrows = 5, ncols=3, figsize = (20,27))

		for i in range(len(col_list)):
			col = col_list[i]
			vals = filter( lambda x: x == x, df[col] )
			mid = np.mean(vals)
			centers = filter( lambda x: (x < (mid*max_x)),vals )
			bins = mid*max_x if mid*max_x < 30 else 30
			j = MyPlot.m_by_n(5,3)[i]
			axs[j[0],j[1]].hist(centers,bins=bins)
			axs[j[0],j[1]].set_title(col)

		bplt.show()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('thesaurus', '0002_auto_20150722_1708'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='property',
            options={'verbose_name_plural': 'properties'},
        ),
        migrations.RemoveField(
            model_name='resource',
            name='datatype_properties',
        ),
        migrations.RemoveField(
            model_name='resource',
            name='object_properties',
        ),
        migrations.AddField(
            model_name='match',
            name='match_source',
            field=models.ForeignKey(default=0, to='thesaurus.Resource'),
            preserve_default=False,
        ),
        migrations.AddField(
            model_name='property',
            name='source',
            field=models.ForeignKey(default=0, to='thesaurus.Resource'),
            preserve_default=False,
        ),
        migrations.AddField(
            model_name='relationship',
            name='relationship_source',
            field=models.ForeignKey(default=0, to='thesaurus.Resource'),
            preserve_default=False,
        ),
        migrations.AlterField(
            model_name='relationship',
            name='relationship_target',
            field=models.ForeignKey(related_name='target_of', verbose_name=b'Target Resource', to='thesaurus.Resource'),
        ),
    ]


'''
some of these should go into neovim python client
'''
from __future__ import print_function

import os
import time
from subprocess import *
from threading import Thread
import shlex
import string
import random
from functools import wraps
import sched
import _thread as thread
import timeit

from neovim import attach

def attach_socket(path=None):
    '''does it auto'''
    def open_nvim():
        proc = Popen('NVIM_LISTEN_ADDRESS=/tmp/nvim nvim',
                    stdin=PIPE, stdout=PIPE, shell=True)
        proc.communicate()

    # THIS IS DOESNT WORK UNRELIBALE
    #path = os.environ.get('NVIM_LISTEN_ADDRESS')
    if not path:
        print('threading')
        t = Thread(target=open_nvim)
        t.start()
        #todo make this give a random path
        return attach('socket', path='/tmp/nvim')
    else:
        print('attaching socket')
        return attach('socket', path=path)

def attach_child():
    return attach('child', argv=['nvim', '--embed'])

def attach_headless(nvim_args=None, path=None):
    if not path:
        path = '/tmp/nvim' + rand_str(8)
    os.environ['NVIM_LISTEN_ADDRESS'] = path
    dnull = open(os.devnull)
    # TODO WHY USE SHLEX???
    cmd = shlex.split('nvim --headless')
    if nvim_args:
        cmd.extend(nvim_args)
    proc = Popen(cmd,
            stdin=dnull,
            stdout=dnull,
            stderr=dnull)
    dnull.close()
    while proc.poll() or proc.returncode is None:
        try:
            nvim = attach('socket', path=path)
            break
        except IOError:
            # Socket not yet ready
            time.sleep(0.05)

    return nvim

def rand_str(length):
    '''returns a random string of length'''
    chars = []
    for i in range(length):
        chars.append(random.choice(string.ascii_letters))
    return ''.join(char for char in chars)

def _stringify_key(key, state):
    send = []
    if state == 'Shift':
        send.append('S')
    elif state == 'Ctrl':
        send.append('C')
    elif state =='Alt':
        send.append('A')
    send.append(key)
    return '<' + '-'.join(send) + '>'

def _split_color(n):
    return ((n >> 16) & 0xff, (n >> 8) & 0xff, n & 0xff,)

def _invert_color(r, g, b):
    return (255 - r, 255 - g, 255 - b,)

def _stringify_color(r, g, b):
    return '#{0:0{1}x}'.format((r << 16) + (g << 8) + b, 6)

def debug_echo(func):
    '''used on method to simply print the function name and
    parameters if self.debug_echo = True,
    the function will not execute'''
    @wraps(func)
    def deco(*args, **kwargs):
        try:
            debug = args[0].debug_echo
        except AttributeError:
            debug = False
        if debug:
            if len(args) == 1:
                to_print = []
            else:
                to_print = args[1:]
            print(func.__name__, repr(to_print), **kwargs)

        return func(*args, **kwargs)
    return deco

def delay_call(seconds):
    '''Decorator to delay the runtime of your function,
    each succesive call to function will refresh the timer,
    canceling any previous functions
    '''
    _scheduler = sched.scheduler(timeit.default_timer,
                                 time.sleep)
    def delayed_func(func):
        @wraps(func)
        def modded_func(*args, **kwrds):
            if len(_scheduler.queue) == 1:
                _scheduler.enter(seconds, 1, func, args, kwrds)
                _scheduler.cancel(_scheduler.queue[0])
            else:
                _scheduler.enter(seconds, 1, func, args, kwrds)
                thread.start_new_thread(_scheduler.run, ())
        thread.start_new_thread(_scheduler.run, ())
        modded_func.scheduler = _scheduler
        return modded_func
    return delayed_func


from dataclasses import dataclass
from typing import List

@dataclass
class Point:
    x: int
    y: int

    def __add__(self, other):
        return Point(self.x + other.x, self.y + other.y)

    def __str__(self):
        return f"({self.x}, {self.y})"

@dataclass
class Maze:
    inside_maze: list
    is_free: list

class PathFinder:
    @staticmethod
    def generate_random_world():
        max_x, max_y = 4, 4
        maze_points = [Point(x, y) for x in range(max_x) for y in range(max_y)]
        return maze_points, maze_points

    def get_path_to_target(self, maze: Maze, start: Point, target: Point):
        prev = {}
        source = start
        while source:
            if not prev.get(str(source)):
                prev[str(source)] = self.get_next_step_to_target(maze, source, target)
            if prev[str(source)]:
                yield prev[str(source)]
            source = prev[str(source)]

    def get_next_step_to_target(self, maze: Maze, source: Point, target: Point):
        queue = [target, ]
        used = [target, ]
        while len(queue) > 0:
            point = queue.pop(0)
            for neighbour in self.get_neighbours(point, maze):
                if neighbour in used:
                    continue
                if neighbour == source:
                    return point
                queue.append(neighbour)
                used.append(neighbour)

        return None

    @staticmethod
    def get_neighbours(point: Point, maze: Maze) -> List[Point]:
        offsets = [Point(0, 1), Point(1, 0), Point(-1, 0), Point(0, -1)]
        points = [point + offset for offset in offsets]
        return [
            point
            for point in points
            if point in maze.inside_maze and point in maze.is_free
        ]


"""Wobliczuzmian URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include
from django.conf.urls import url
from Wobliczu.views import renderHome, renderSearch, renderKontakt, renderPostCreator, renderBase, renderUserPanel, \
    renderArticles, renderUserArticles, renderSingleArticle, renderSearchResult, renderEditArticle, renderJournalistComments, render_confirm_delete
from django.conf.urls.static import static
from django.conf import settings
from django.views.generic import TemplateView
from Wobliczu.models import Article

urlpatterns = [
    path('nihhal/', admin.site.urls),
    path('', renderHome, name='home'),
    path('kontakt', renderKontakt, name='kontakt'),
    path('articles/search', renderSearch, name='search'),
    path('articles/', renderArticles, name='articles'),
    path('journalist/postCreator', renderPostCreator, name='postCreator'),
    path('base', renderBase, name='base'),
    path('accounts/', include('django.contrib.auth.urls')),
    path('journalist/userPanel', renderUserPanel, name='user-panel'),
    path('journalist/userArticles', renderUserArticles, name='user-articles'),
    path('articles/<slug:slug>', renderSingleArticle, name='single-article'),
    url(r'^search-result', renderSearchResult, name='search-result'),
    path('journalist/editArticle/<slug:slug>', renderEditArticle, name='render-edit'),
    path('journalist/userPanel/<int:journalist_id>', renderJournalistComments, name='journalist-comments'),
    path('ckeditor/', include('ckeditor_uploader.urls')),
    path('articles/confirm-delete/<int:article_id>', render_confirm_delete, name='confirm-delete'),
    path('robots.txt', TemplateView.as_view(template_name='robots.txt', content_type='text/plain')),
    path('sitemap.xml', TemplateView.as_view(template_name='sitemap.xml', content_type='application/xml')),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

handler404 = 'Wobliczu.views.error_404_view'


import argparse

def zero_prepend(array, size_needed):
    zeros = [0] * (size_needed - len(array))
    return zeros + array[:]

def hanoi_towers(n):
    nums = list(range(1,n+1))
    return [ nums,
            [],
            []]

def hanoi_variant_towers(n):
    nums = list(range(1,n+1))
    odds = [n for n in nums if n % 2 == 1]
    evens =  [n for n in nums if n % 2 == 0]
    return [ odds,
            [],
            evens]

def print_towers(towers):
    height = max([len(tw) for tw in towers])
    towers_ = [zero_prepend(tw, height) for tw in towers]
    for a, b, c in zip(*towers_):
        a = ('*' * a).center(15)
        b = ('*' * b).center(15)
        c = ('*' * c).center(15)
        print(a, b, c)

def animate_sol(towers, sol):
    towers_ = towers[:]
    print_towers(towers_)
    print()
    for step in sol:
        towers_= make_move(towers_, step)
        print(step[0]+1, step[1]+1,sep='->')
        print_towers(towers_)
        print()

def make_move(towers, move):
    towers_ = towers[:]
    frm, to = move
    assert len(towers[frm]) > 0
    disc = towers[frm][0]
    if len(towers[to]) == 0:
        towers_[to] = [disc]
        towers_[frm] = towers[frm][1:]
        return towers_
    disc_there = towers[to][0]
    assert disc_there >= disc
    towers_[to] = [disc] + towers_[to]
    towers_[frm] = towers[frm][1:]
    return towers_

def hanoi_steps(start, end, n):
    intermediate = 3-(start+end)
    if n==0:
        return []
    res = []
    res += hanoi_steps(start, intermediate,  n-1)
    res += [(start, end)]
    res += hanoi_steps(intermediate, end,  n-1)
    return res

#here hanoi is synonym of move n ordered pile from, to with an empty stack

def pile(from1, pile_on, k):
    from2 = 3-(pile_on + from1)
    if k == 0:
        return []
    if k == 1:
        return [(from1, pile_on)]
    if k >= 2:
        res = []
        res += pile(from1, pile_on, k-1)
        if k % 2 == 0:
            res +=  hanoi_steps(pile_on, from1, k-1)
            res += [(from2, pile_on)]
            res +=  hanoi_steps(from1, pile_on, k-1)
        else:
            res +=  hanoi_steps(pile_on, from2, k-1)
            res += [(from1, pile_on)]
            res +=  hanoi_steps(from2, pile_on, k-1)
    return res

def depile(piled, start, k):
    another = 3-(piled + start)
    if k == 0:
        return []
    if k == 1:
        return [(piled, start)]
    if k >=2:
        res = []
        if k % 2 == 1:
            res += hanoi_steps(piled, start, k-1)
            res += [(piled, another)]
            res += hanoi_steps(start, piled, k-1)
        else:
            res += hanoi_steps(piled, another , k-1)
            res += [(piled, start)]
            res += hanoi_steps(another, piled, k-1)

    return res +depile(piled, start, k-1)

def hanoi_variant_steps(odds, evens, n):
    intermediate = 3-(odds+evens)
    res = []
    res += pile(odds, intermediate, n-1)
    if n % 2 == 0:
        res += [(evens, odds)]
    else:
        res += [(odds, evens)]
    res += depile(intermediate, odds,n-1)
    res += [(odds,evens)]

    return res

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Solver tower of Hanoi problem')
    parser.add_argument('nb_discs', type=int, help='number of discs')
    args = parser.parse_args()
    nb_discs = args.nb_discs
    # hanoi_(0,2,nb_discs)
    # hanoi_variant(0,2,1,nb_discs)
    print(hanoi_variant_towers(nb_discs))

    # ordinary = hanoi_towers(3)
    # sol = hanoi_steps(0,2,3)
    # animate_sol(ordinary, sol)

    towers = hanoi_variant_towers(nb_discs)
    print_towers(towers)
    piling = pile(0,1,6)
    piling = hanoi_variant_steps(0,2,nb_discs)
    print("piling", piling)
    animate_sol(towers,piling)


#online_consumer.py
import numpy as np
from sklearn.linear_model import SGDClassifier
from kafka import KafkaConsumer
import json
import plotly
import plotly.tools
import plotly.graph_objs as go
from scipy import stats

#############################################
# Create a plotly plot
#############################################

with open('/home/ventum/Syn-Repo-DataScience/Python/KAFKA_Streaming/config2.json') as config_file:
    plotly_user_config = json.load(config_file)

username = plotly_user_config['plotly_username']
api_key = plotly_user_config['plotly_api_key']
stream_token1 = plotly_user_config['plotly_streaming_tokens'][1]
stream_token2 = plotly_user_config['plotly_streaming_tokens'][2]
#stream_token = dict(token=stream_token, maxpoints=60)

print("Starting Streaming")

#############################################
# Initialize your plotly object
#############################################

p = plotly.plotly.sign_in(username, api_key)

x_data          = []
y_data_fit      = []
target      = []
prediction  = []
line  = []
line_history  = []
n_features = 1
count = 0
correct = 0

trace0 = go.Scatter(
    x = [],
    y = [],
    mode = 'markers',
    name = 'train data points'

)
trace1 = go.Scatter(
    x = [],
    y = [],
    mode = 'markers',
    name = 'trend line'
)
data = [trace0,trace1]
plotly.plotly.iplot(data,filename='Support-Vector-Machine-Kafka', fileopt='overwrite')

#plotly.plotly.iplot([{'x': [], 'y': [], 'type': 'scatter', 'mode': 'markers','stream': {'token': stream_token1, 'maxpoints': 100}},
#                     {'x': [], 'y': [], 'type': 'scatter', 'mode': 'line','stream': {'token': stream_token2, 'maxpoints': 100}}],
#                     filename='Support-Vector-Machine-Kafka', fileopt='overwrite')

#############################################
# Open plotly stream
#############################################

s1 = plotly.plotly.Stream(stream_token1)
s1.open()

s2 = plotly.plotly.Stream(stream_token2)
s2.open()

#############################################
# Open Kafka connection and do incremental SVM fit
#############################################

clf = SGDClassifier(loss="hinge", alpha=0.01, n_iter=20, fit_intercept=True)

consumer = KafkaConsumer('simple-test2',
    bootstrap_servers="localhost:9091,localhost:9092")

for binary_message in consumer:

    #############################################
    # encode binary value back to the message
    #############################################
    msg=binary_message.value.decode('utf-8')

    #############################################
    # create a json file
    #############################################

    data = json.loads(msg)
    vec = np.zeros(len(data))
    for key,value in data.items():
        vec[int(key)]=value

    #############################################
    # create a feature vector
    #############################################

    features = vec[:n_features].reshape(1,n_features)

    #############################################
    # Create a target vector
    #############################################

    target = [vec[n_features]]

    # if the stream is initiated we need to create an initial value count
    if count > 1:
        prediction = clf.predict(features)
        prediction =  np.array(prediction)
        correct += int(prediction[0]==int(target[0]))
    if count > 2:
        x_data.extend(features[0])
        y_data_fit.extend(prediction)

        #############################################
        # Generated linear fit
        #############################################

        slope, intercept, r_value, p_value, std_err = stats.linregress(x_data,y_data_fit)
        line = slope*float(x_data[0])+intercept
        line_history.append(float(line))
        print("Score           :   " + str(correct/float(count-1)))
        print("Line of best fit:   " + str(line_history))
        print("xdata[0]        :   " + str(float(x_data[0])))
        print("xdata           :   " + str(x_data))
        print("prediction      :   " + str(prediction))
        print("features        :   " + str(features))
    #############################################
    # Fit the Support vector machine model (Stochastic Gradient Descent)
    #############################################

    clf.partial_fit(features, target,classes=[0,1])

    #############################################
    # Write the data to the plotly stream
    #############################################

    s1.write({'x': x_data, 'y': target[0]})
    s2.write({'x': x_data, 'y': line_history})

    # increment the count
    count += 1


import gym
env = gym.make('MountainCar-v0')
env.reset()
env.render()

EPISODES = 1000

for i_episode in range(EPISODES):
    observation = env.reset()
    for t in range(100):
        env.render()
        action = env.action_space.sample()
        observation, reward, done, info = env.step(action)
        if done:
            print("episode: {}/{}, score: {}".format(i_episode, EPISODES, t))
        else:
            print("Failed at episode: {}/{}".format(i_episode, EPISODES))
            break
env.env.close()


"""import sys
sys.stdin = open('input.txt')
input = sys.stdin.readline
n = int(input())
arr = []
for i in range(n):
    arr.append(int(input()))
arr.sort()
cnt = 1
max = 1
result = arr[0]
for i in range(1,n):
    if (arr[i-1] == arr[i]):
        cnt += 1
    else:
        cnt = 1
    if cnt > max:
        max = cnt
        result = arr[i]
print(result)"""
import sys
from collections import Counter
sys.stdin = open('input.txt')
num = int(sys.stdin.readline())
nlist = [int(x) for x in sys.stdin.read().split()]
ndict = Counter(nlist)
m = max(ndict.values())
for i in sorted(ndict.keys()):
	if ndict[i] == m:
		print(i)
		break

python daily coding/2020.9.27 (자료구조)/11652번 (카드).py


import nltk
import csv
import numpy as np
from hard_score_pub import *
from survey_risk_tolerance_db import*
from tw_for_nl import*
#nltk.download()

###Split, Tokenize & Tag
#must in txt

###score
from nltk.sentiment.vader import SentimentIntensityAnalyzer
#must be in list
#this is individual's
#tweets = [ ]
sa = SentimentIntensityAnalyzer()
scores_ind = []
for twit in tweetxt:
    score = sa.polarity_scores(twit)
#    print(score)
    scores_ind.append(score)

# score = sa.polarity_scores(tweets)
# print(tweets,str(score))

write_to_sentiment = "draft/sentiment_score.csv"
with open(write_to_sentiment,"w") as write_sentiment:
    writer = csv.DictWriter(write_sentiment, fieldnames=["neg","neu","pos","compound"])
    writer.writeheader()
    for i in scores_ind:
        writer.writerow(i)

#avg = 0.0091
diff_ind_list = [ ]
for row in scores_ind:
    pos = row["pos"]
    neg = row["neg"]
    def dif(pos,neg):
        return float(pos-neg)
    diff_ind = dif(pos,neg)
    diff_ind_list.append(diff_ind)

avg_ind = np.mean([diff_ind_list])
def adj_sentiment(avg,avg_ind,std):
    return (avg_ind-avg)/(1.96*std)
adj_sentiment = adj_sentiment(avg,avg_ind,std)

def tot_risk_tolerance(score,adj_sentiment):
    return risk_score+adj_sentiment
tot_risk_tolerance = tot_risk_tolerance(score,adj_sentiment)
#print("The avgerage sentiment score for your tweets is: " + str(round(avg_ind,4)))
print("Your sentiment adjustment to risk tolerance is: " + str(round(adj_sentiment,4)))
print("Your Total Risk Tolerance is: " + str(round(tot_risk_tolerance,4)))


""" Module to for ingest utilities
"""
from __future__ import print_function, absolute_import, division, unicode_literals

import numpy as np
import warnings
import pdb

def chk_meta(meta):
    """ Vettes a meta Table prior to its being ingested into the hdf

    Parameters
    ----------
    meta

    Returns
    -------
    chk : bool

    """
    from igmspec.defs import instruments
    from astropy.time import Time
    from astropy.table import Column
    # Init
    inst_dict = instruments()

    chk = True
    # Required columns
    req_clms = ['IGM_ID', 'RA', 'DEC', 'EPOCH', 'zem', 'R', 'WV_MIN',
                'WV_MAX', 'DATE-OBS', 'SURVEY_ID', 'NPIX', 'SPEC_FILE',
                'INSTR', 'GRATING', 'TELESCOPE']
    meta_keys = meta.keys()
    for clm in req_clms:
        if clm not in meta_keys:
            chk = False
            print("Missing column {:s} in meta".format(clm))
    # Check date formatting
    try:
        tval = Time(list(meta['DATE-OBS'].data), format='iso')
    except:
        print("Bad DATE-OBS formatting")
        chk = False
    # Check instrument
    meta_instr = meta['INSTR'].data
    db_instr = np.array(inst_dict.keys())
    if not np.all(np.in1d(meta_instr, db_instr)):
        print("Bad instrument in meta data")
        chk = False
    # Check for unicode
    for key in meta_keys:
        if 'unicode' in meta[key].dtype.name:
            warnings.warn("unicode in column {:s}.  Will convert to str for hdf5".format(key))
            tmp = Column(meta[key].data.astype(str), name=key)
            meta.remove_column(key)
            meta[key] = tmp
    # Return
    return chk

def set_resolution(head, instr=None):
    """ Sets resolution based on the instrument and header

    Parameters
    ----------
    head : FITS header
    instr : str, optional
      If not provided, attempt to grab from header

    Returns
    -------

    """
    from igmspec import defs
    # Dicts
    Rdicts = defs.get_res_dicts()
    # Grab instrument
    if instr is None:
        if 'CURRINST' in head.keys():  # ESI
            instr = head['CURRINST'].strip()
        elif 'INSTRUME' in head.keys():
            if 'HIRES' in head['INSTRUME']:
                instr = 'HIRES'
            elif 'MagE' in head['INSTRUME']:
                instr = 'MagE'
        else:
            pass
        if instr is None:
            raise ValueError("NEED MORE INFO FOR INSTR")

    # Grab resolution
    if instr == 'ESI':
        try:
            return Rdicts[instr][head['SLMSKNAM']]
        except KeyError:
            pdb.set_trace()
    elif instr == 'HIRES':
        try:
            return Rdicts[instr][head['DECKNAME'].strip()]
        except KeyError:
            print("Need to add {:s}".format(head['DECKNAME']))
            pdb.set_trace()
    elif instr == 'MagE':
        try:
            return Rdicts[instr][head['SLITNAME'].strip()]
        except KeyError:
            print("Need to add {:s}".format(head['SLITNAME']))
            pdb.set_trace()
    else:
        raise IOError("Not read for this instrument")


import logging
import os
import subprocess

import boto3
import click
from retrying import retry

from ..utils.io import IngestStatus
from ..models import Scene
from ..ingest.models import Ingest
from ..ingest import (
    create_landsat8_ingest,
    create_sentinel2_ingest,
    create_ingest_definition
)
from ..uploads.landsat8.settings import datasource_id as landsat_id
from ..utils.exception_reporting import wrap_rollbar
from ..utils.emr import get_cluster_id, wait_for_emr_success

logger = logging.getLogger(__name__)

BATCH_JAR_PATH = os.getenv('BATCH_JAR_PATH', 'rf-batch-761c316.jar')
SENTINEL2_DATASOURCE_IDS = ['4a50cb75-815d-4fe5-8bc1-144729ce5b42', 'c33db82d-afdb-43cb-a6ac-ba899e48638d']

@click.command(name='ingest-scene')
@click.argument('scene_id')
@click.option('--ignore-previous', is_flag=True,
              help='Boolean to ignore scene status for ingestion')
@click.option('--local', is_flag=True,
              help='Boolean to run using a spark local cluster rather than EMR job')
@wrap_rollbar
def ingest_scene(scene_id, ignore_previous, local):
    """Ingest a scene into Raster Foundry

    Args:
        scene_id (str): ID of scene to ingest
        ignore_previous (bool): whether to ignore ingest_status when processing ingest
    """
    logger.info("Ingesting Scene: %s", scene_id)
    s3_uri, ingest_definition_id = save_ingest_def_to_s3(scene_id, ignore_previous)
    launch_spark_ingest_job(s3_uri, ingest_definition_id, scene_id, local)
    wait_for_status(ingest_definition_id)

def save_ingest_def_to_s3(scene_id, ignore_previous=False):
    """Create ingest definition and upload to S3

    Args:
        ignore_previous (bool): boolean to ignore previous ingest status
        scene_id (str): ID of scene to create ingest definition for
    """
    scene = Scene.from_id(scene_id)

    logger.info('Beginning to create ingest definition for scene %s for user %s...',
                scene_id, scene.owner)
    if scene.ingestStatus not in [IngestStatus.TOBEINGESTED, IngestStatus.FAILED] and not ignore_previous:
        raise Exception('Scene is no longer waiting to be ingested, error error')

    scene.ingestStatus = IngestStatus.INGESTING
    logger.info('Updating scene (%s) status to ingesting', scene_id)
    scene.update()
    logger.info('Successfully updated scene (%s) status', scene_id)

    logger.info('Creating ingest definition')
    if scene.datasource == landsat_id:
        ingest_definition = create_landsat8_ingest(scene)
    elif scene.datasource in SENTINEL2_DATASOURCE_IDS:
        ingest_definition = create_sentinel2_ingest(scene)
    else:
        ingest_definition = create_ingest_definition(scene)
    ingest_definition.put_in_s3()
    logger.info('Successfully created and pushed ingest definition for scene %s', scene)

    Ingest.delete_status_from_s3(ingest_definition.id)
    return ingest_definition.s3_uri, ingest_definition.id

def launch_spark_ingest_job(ingest_def_uri, ingest_def_id, scene_id, local=False):
    """Launch ingest job and wait for success/failure

    Args:
        ingest_def_uri (str): S3 URI for location of ingest definition
        ingest_def_id (str): ID for ingest definition
        scene_id (str): ID for scene to be ingested
        local (bool): whether to execute ingest as a spark local job

    Returns:
        bool
    """

    logger.info('Launching Spark ingest job with ingest definition %s for scene %s',
                ingest_def_uri, scene_id)
    if local:
        return execute_local_ingest_job(scene_id, ingest_def_uri)
    else:
        cluster_id = get_cluster_id()
        emr_response = execute_ingest_emr_job(scene_id, ingest_def_uri, ingest_def_id, cluster_id)
        logger.info('Launched Spark ingest job for %s with ingest ID %s. Waiting for status changes.',
                    scene_id, ingest_def_id)
        step_id = emr_response['StepIds'][0]
        is_success = wait_for_emr_success(step_id, cluster_id)
        return is_success

def execute_local_ingest_job(scene_id, ingest_s3_uri):
    """Execute ingest job locally with a spark local cluster

    Args:
        scene_id (str): ID of scene to ingest
        ingest_s3_uri (str): S3 location for ingest definition
    """

    ingest_cores = os.getenv('LOCAL_INGEST_CORES', 32)
    ingest_memory = os.getenv('LOCAL_INGEST_MEM_GB', 48)
    command = ['spark-submit',
               '--master', 'local[{}]'.format(ingest_cores),
               '--driver-memory', '{}g'.format(ingest_memory),
               '--class', 'com.azavea.rf.batch.ingest.spark.Ingest',
               '/opt/raster-foundry/jars/rf-batch.jar',
               '-t', '--overwrite', '-s', scene_id, '-j', ingest_s3_uri
    ]
    subprocess.check_call(command)
    logger.info('Finished Ingesting %s in spark local', scene_id)
    return True

@retry(wait_exponential_multiplier=2000, wait_exponential_max=30000, stop_max_attempt_number=5)
def execute_ingest_emr_job(scene_id, ingest_s3_uri, ingest_def_id, cluster_id):
    """Kick off ingest in AWS EMR

    Args:
        scene_id (str): id of the scene to ingest
        ingest_s3_uri (str): URI for ingest definition
        ingest_def_id (str): ID to namespace ingest job
        cluster_id (str): ID of the cluster to submit work to

    Returns:
        dict
    """
    logger.info('Constructing ingest step request for %s for cluster %s for ingest id %s',
                scene_id, cluster_id, ingest_def_id)
    step = {
        'ActionOnFailure': 'CONTINUE',
        'Name': 'ingest-{}'.format(ingest_def_id),
        'HadoopJarStep': {
            'Args': ['/usr/bin/spark-submit',
                     '--master',
                     'yarn',
                     '--deploy-mode',
                     'cluster',
                     '--conf',
                     'spark.yarn.submit.waitAppCompletion=false',
                     '--class',
                     'com.azavea.rf.batch.ingest.spark.Ingest',
                     's3://rasterfoundry-global-artifacts-us-east-1/batch/{}'.format(BATCH_JAR_PATH),
                     '-t',
                     '--overwrite',
                     '-s',
                     scene_id,
                     '-j',
                     ingest_s3_uri],
            'Jar': 's3://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar'
        }
    }
    logger.info('Submitting step to EMR (%s)', step)
    emr = boto3.client('emr')
    response = emr.add_job_flow_steps(
        JobFlowId=cluster_id,
        Steps=[step]
    )
    logger.info('Received response from EMR API: %s', response)
    return response

@retry(wait_exponential_multiplier=2000, wait_exponential_max=30000, stop_max_attempt_number=2)
def wait_for_status(ingest_def_id):
    """Wait for a result from the Spark job

    Args:
        ingest_def_id (str): ID of ingest definition to check on results for
    """
    ingest_status_dict = Ingest.get_status_from_s3(ingest_def_id)
    scene_id = ingest_status_dict['sceneId']
    status = ingest_status_dict['ingestStatus']

    layer_s3_bucket = os.getenv('TILE_SERVER_BUCKET')
    s3_output_location = 's3://{}/layers'.format(layer_s3_bucket)

    logger.info('Waiting for scene status at %s for scene %s with ingest defintion %s',
                s3_output_location, scene_id, ingest_def_id)

    scene = Scene.from_id(scene_id)
    scene.ingestLocation = s3_output_location
    scene.ingestStatus = status

    if scene.ingestStatus != IngestStatus.FAILED:
        logger.info('Writing scene metadata into postgres.')
        metadata_to_postgres(s3_output_location, scene_id)

    logger.info('Setting scene %s ingest status to %s', scene.id, scene.ingestStatus)
    scene.update()
    logger.info('Successfully updated scene %s\'s ingest status', scene.id)

    notify_for_scene_ingest_status(scene.id)

    if scene.ingestStatus == IngestStatus.FAILED:
        raise Exception('Failed to ingest {} for user {}'.format(scene_id, scene.owner))

def metadata_to_postgres(uri, scene_id):
    """Save metadata of Layer at URI and Scene to Database

    Args:
        uri (str): remote location of layer
        scene_id (str): ID of scene to save metadata for
    """
    bash_cmd = [
        'java', '-cp',
        '/opt/raster-foundry/jars/rf-batch.jar',
        'com.azavea.rf.batch.Main',
        'migration_s3_postgres',
        uri,
        'layer_attributes',
        scene_id
    ]
    logger.debug('Bash command to store metadata: %s', ' '.join(bash_cmd))
    running_cmd = subprocess.Popen(bash_cmd)
    running_cmd.communicate()
    logger.info('Successfully completed metadata postgres write for scene %s', scene_id)
    return True

def notify_for_scene_ingest_status(scene_id):
    """Notify users that are using this scene as well as the scene owner that
    the ingest status of this scene has changed

    Args:
        scene_id (Scene): the scene which has an updated status
    """

    bash_cmd = [
        'java', '-cp',
        '/opt/raster-foundry/jars/rf-batch.jar',
        'com.azavea.rf.batch.Main',
        'notify_ingest_status',
        scene_id
    ]
    running_process = subprocess.Popen(bash_cmd)
    running_process.communicate()
    return True


#!/usr/bin/env python
# coding=utf8

import numpy as np
from scipy.interpolate import interp1d
from scipy.signal import hamming
from . import signals

def make_initial_wave(x, ampl, offset):
    return hamming(x.size) * ampl + offset - ampl / 2

def make_jump(x):
    U = ((-(np.tanh(
        (x - .1 * x[-1]) / 20) + 1) / 2 + 1) * 1 + 1) / 2
    return U

class NoisyHook:
    def __init__(self, freq, turb=False):
        sin_sig = signals.SinusoidalSignal(freq, .1)
        cst_sig = signals.ConstantSignal(offset=1)
        noisy_sig = signals.GaussianBrownNoise(200,
                                               100, std=.02,
                                               seed=74897497)
        self.sig = sin_sig + cst_sig + noisy_sig
        self.turb = turb

    def __call__(self, t, fields, pars):
        fields.h[0] = self.sig(t)
        fields.q[0] = self.sig(t) if self.turb else self.sig(t) ** 3 / 3

        return fields, pars

class TurbCallable:
    def __init__(self, inter):
        self.inter = inter

    def __call__(self, fields, t, pars):
        q = fields.q
        inter = self.inter

        for key in inter.trad:
            fields[key][:] = inter.get(key, q)
        return fields, pars

class Interpolator:
    def __init__(self, data_path, Re):
        data = dict(**np.load(data_path))
        to_interpolate = {key: value for key, value in data.items()
                          if key not in ('grid_q', 'grid_Re')}
        trad = dict(zip(["cK", "cM", "cJ",
                         "cL", "cF", "cG", "cS",
                         "Cthth", "Cthphi", "Cphith", "Cphiphi",
                         "bita", "bita1", "betath", "betaphi",
                         "ffath", "thetaz", "phiz"],
                        ['K(q)', 'M(q)', 'J(q)',
                         'L(q)', 'F(q)', 'G(q)', 'S(q)',
                         'Cthth', 'Cthphi', 'Cphith', 'Cphiphi',
                         'Bita(q)', 'Bita1', 'betath', 'betaphi',
                         'ffath', 'thetaz', 'phiz']))

        self.data = data
        self.interpRe = {key: interp1d(data['grid_Re'],
                                       value,
                                       axis=0,
                                       bounds_error=False)(Re)
                         for key, value in to_interpolate.items()}
        self.interp = {key: interp1d(self.data['grid_q'],
                                     self.interpRe[trad[key]],
                                     bounds_error=False,
                                     fill_value=(
            self.interpRe[trad[key]][0],
            self.interpRe[trad[key]][-1]
        ))
            for key in trad}
        self.interp['Fr'] = interp1d(self.data['grid_q'],
                                     self.interpRe['Fr'],
                                     bounds_error=False,
                                     fill_value=(self.interpRe['Fr'][0],
                                                 self.interpRe['Fr'][-1]))
        self.trad = trad

    def get(self, key, q):
        return self.interp[key](q)


from requests import get
from bs4 import BeautifulSoup
from utils import find, trim, validate_url
from time import sleep
from .model import PronunciationModel, WordReferencesModel

WORD_REFERENCE = "https://www.wordreference.com/"
WORD_REFERENCE_DEFINITION = f"{WORD_REFERENCE}definition/"
WORD_REFERENCE_SYNONYMS = f"{WORD_REFERENCE}synonyms/"
WORD_REFERENCE_USAGE = f"{WORD_REFERENCE}EnglishUsage/"
WORD_REFERENCE_COLLOCATIONS = f"{WORD_REFERENCE}EnglishCollocations/"
WORD_REFERENCE_CONJUG = f"{WORD_REFERENCE}conj/enverbs.aspx?v="

class WordReferences:

    def __init__(self, words, source=None):
        if words == None or len(words) == 0 or words == " ":
            raise AttributeError("Property [words] is required")

        self.source = source
        self._words = []
        words = trim(words)
        for w in words.split(" "):
            if w != None and w != "":
                self._words.append(w.lower())
        # print(self._words)

    def get_search_types(self):
        return {
            1: "DEFINITION",
            2: "SYNONYMS",
            3: "USAGE",
            4: "COLLOCATIONS",
            5: "CONJUG",
        }

    def print_option(self):
        types = self.get_search_types()
        print("\nSelect an options:")
        for t in types:
            print(f"\t {t} for {types[t]}")

    def _get_url_source(self, key):
        sources = {
            "DEFINITION": WORD_REFERENCE_DEFINITION,
            "SYNONYMS": WORD_REFERENCE_SYNONYMS,
            "USAGE": WORD_REFERENCE_USAGE,
            "COLLOCATIONS": WORD_REFERENCE_COLLOCATIONS,
            "CONJUG": WORD_REFERENCE_CONJUG,
        }
        return sources[key]

    def _get_function_by_type(self, key):
        sources = {
            "DEFINITION": self._extract_definition,
            "SYNONYMS": self._extract_definition,
            "USAGE": self._extract_definition,
            "COLLOCATIONS": self._extract_definition,
            "CONJUG": self._extract_definition,
        }
        return sources[key]

    def set_search_type(self, search_type):
        types = self.get_search_types()
        try:
            search = types[int(search_type)]
        except KeyError:
            print(f"invalid type {search_type}")
            self.print_option()
            exit(1)

        self._selected_search_type = search

    def _get_content_from_web(self, url):
        response = get(url)
        if response.status_code != 200:
            raise Exception(f"Error on get resource from web {url}")
        return response

    def _create_bs4_instance(self, source):
        is_url = validate_url(source)
        if is_url == True and self.source == None:
            content = self._get_content_from_web(source)
            source = content.text
        else:
            source = self.source

        if source == None:
            raise AttributeError("url or source required")

        self.html = BeautifulSoup(source, "html.parser")

    def _extract_only_ipa_pronunciation(self, pronunciation):
        prons = pronunciation.split(" ")
        if len(prons) > 1:
            indexes = find(pronunciation, '/')
            new_value = pronunciation[indexes[0]:indexes[1] + 1]
            pronunciation = new_value.strip()
        return pronunciation

    def _extract_types(self, text, term):
        if "strong" not in text and "noun" not in text:
            return [PronunciationModel(
                self._extract_only_ipa_pronunciation(text), term
            )]
        text_split = text.split(",")
        pronounces = []
        for word in text_split:
            splits = word.split(":")
            if len(splits) > 1:
                w_type = splits[0].strip()
                if not w_type.startswith("'"):
                    w_pron = self._extract_only_ipa_pronunciation(
                        splits[1].strip())
                    pronounces.append(PronunciationModel(w_pron, term, w_type))

        return pronounces

    def _extract_definition(self):
        term = self.html.find("h3", class_="headerWord")
        isp = self.html.find("div", class_="pwrapper")
        if isp == None:
            return [PronunciationModel(f"({term.text})", term.text)]
        tag_pron = isp.find("span", class_="pronWR")
        if tag_pron is None:
            return [PronunciationModel(f"({term.text})", term.text)]
        pronun_values = tag_pron.text
        indexs = find(pronun_values, "/")

        if len(indexs) == 0:
            print("Pronunciation ISP not found")
            exit()
        # 36 length to phrase "UK and possibly other pronunciations"
        if pronun_values.startswith("UK and"):
            return self._extract_types(pronun_values[36:], term.text)
        return self._extract_types(pronun_values, term.text)

    def _extract_by_type(self):
        function = self._get_function_by_type(self._selected_search_type)
        return function()

    def _extract_single_word(self, word):
        url_source = self._get_url_source(self._selected_search_type)
        self._create_bs4_instance(url_source + word)
        pronunciation = self._extract_by_type()
        return pronunciation

    def extract_pronunciation(self, search_type):
        if search_type == None:
            raise AttributeError("[search_type] is required")

        if search_type != "1":
            search_type = 1
            print("Only DEFINITION is implemented, change for it")

        search_type = int(search_type)
        if search_type < 1 or search_type > 5:
            raise AttributeError("Invalid option")

        self._selected_search_type = self.get_search_types()[search_type]

        print("extracting, please wait...\n")
        if len(self._words) == "1":
            print(f"\nshowing: {self._selected_search_type}\n")
            return self._extract_single_word(self._words[0])

        pronunciations = []
        for word in self._words:
            extracted = self._extract_single_word(word)
            pronunciations.append(
                extracted[0] if len(extracted) == 1 else extracted
            )
            print(f"{word} OK")
            if self.source == None:
                sleep(3)

        print(f"\nshowing: {self._selected_search_type}\n")
        return pronunciations


# -*- coding: utf-8 -*-

# Copyright (c) 2015 Ericsson AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import json
import string
import glob
try:
    import OpenSSL.crypto
    HAS_OPENSSL = True
except:
    HAS_OPENSSL = False
try:
    import pyrad.packet
    from pyrad.client import Client
    from pyrad.dictionary import Dictionary
    HAS_PYRAD = True
except:
    HAS_PYRAD = False

from calvin.utilities.calvinlogger import get_logger
from calvin.utilities import calvinconfig
from calvin.utilities.utils import get_home

_conf = calvinconfig.get()
_log = get_logger(__name__)
#default timeout
TIMEOUT=5

def security_modules_check():
    if _conf.get("security","security_conf") or _conf.get("security","security_policy"):
        # Want security
        if not HAS_OPENSSL:
            # Miss open ssl
            _log.error("Security: Install openssl to allow verification of signatures and certificates")
            return False
            _conf.get("security","security_conf")['authentication']
        if _conf.get("security","security_conf")['authentication']['procedure'] == "radius" and not HAS_PYRAD:
            _log.error("Security: Install pyrad to use radius server as authentication method.")
            return False
    return True

def security_needed_check():
    if _conf.get("security","security_conf") or _conf.get("security","security_policy"):
        # Want security
        return True
    else:
        return False

class Security(object):
    def __init__(self):
        _log.debug("Security: _init_")
        self.sec_conf = _conf.get("security","security_conf")
        self.sec_policy = _conf.get("security","security_policy")
        if self.sec_conf is not None and not self.sec_conf.get('signature_trust_store', None):
            # Set default directory for trust store
            homefolder = get_home()
            truststore_dir = os.path.join(homefolder, ".calvin", "security", "trustStore")
            self.sec_conf['signature_trust_store'] = truststore_dir
        self.principal = {}
        self.auth = {}

    def __str__(self):
        return "Principal: %s\nAuth: %s" % (self.principal, self.auth)

    def set_principal(self, principal):
        _log.debug("Security: set_principal %s" % principal)
        if not isinstance(principal, dict):
            return False
        # Make sure all principal values are lists
        self.principal = {k: list(v) if isinstance(v, (list, tuple, set)) else [v]
                            for k, v in principal.iteritems()}
        # All default to unauthorized
        self.auth = {k: [False]*len(v) for k, v in self.principal.iteritems()}

    def authenticate_principal(self):
        _log.debug("Security: authenticate_principal")
        if not security_needed_check():
            _log.debug("Security: authenticate_principal no security needed")
            return True

        if self.sec_conf['authentication']['procedure'] == "local_file":
            _log.debug("Security: local file authentication method chosen")
            return self.authenticate_using_local_database()
        if self.sec_conf['authentication']['procedure'] == "radius":
            if not HAS_PYRAD:
                _log.error("Security: Install pyrad to use radius server as authentication method.\n" +
                            "NB! NO AUTHENTICATION USED")
                return False
            _log.info("Security: Radius authtentication method chosen")
            return self.authenticate_using_radius_server()
        _log.info("Security: No security config, so authentication disabled")
        return True

    def authenticate_using_radius_server(self):
        auth = []
        if self.principal['user']:
            srv=Client(server=self.sec_conf['authentication']['server_ip'],
                        secret= bytes(self.sec_conf['authentication']['secret']),
                        dict=Dictionary("extras/pyrad_dicts/dictionary", "extras/pyrad_dicts/dictionary.acc"))
            req=srv.CreateAuthPacket(code=pyrad.packet.AccessRequest,
                        User_Name=self.principal['user'][0],
                        NAS_Identifier="localhost")
            req["User-Password"]=req.PwCrypt(self.principal['password'][0])
            # FIXME is this over socket? then we should not block here
            reply=srv.SendPacket(req)
            _log.debug("Attributes returned by server:")
            for i in reply.keys():
                _log.debug("%s: %s" % (i, reply[i]))
            if reply.code==pyrad.packet.AccessAccept:
                _log.debug("Security:access accepted")
                auth.append(True)
#                return True
            else:
                _log.debug("Security: access denied")
                auth.append(False)
#                return False
        self.auth['user']=auth
        return any(auth)

    def authenticate_using_local_database(self):
        """ Authenticate a principal against config stored information
            This is primarily intended for testing purposes,
            since passwords arn't stored securily.
        """
        if 'local_users' not in self.sec_conf['authentication']:
            _log.debug("local_users not found in security_conf: %s" % self.sec_conf['authentication'])
            return False
        # Verify users against stored passwords
        # TODO expand with other principal types
        d = self.sec_conf['authentication']['local_users']
        if not ('user' in self.principal and 'password' in self.principal):
            return False
        if len(self.principal['user']) != len(self.principal['password']):
            return False
        auth = []
        for user, password in zip(self.principal['user'], self.principal['password']):
            if user in d.keys():
                if d[user] == password:
                    _log.debug("Security: found user: %s",user)
                    auth.append(True)
                else:
                    _log.debug("Security: incorrect username or password")
                    auth.append(False)
            else:
                auth.append(False)
        self.auth['user'] = auth
        return any(auth)

    def check_security_actor_requirements(self, requires):
        _log.debug("Security: check_security_actor_requirements")
        if self.sec_conf and self.sec_conf['access_control_enabled'] == "True":
            for req in requires:
                if not self.check_security_policy_actor(req, "user", self.principal):
                    return False
        #no security config, so access control is disabled
        return True

    def check_security_policy_actor(self, req, principal_type, principal):
        """ Checks that the requirement is allowed by the security policy """
        _log.debug("Security: check_security_policy_actor")
        #Calling function shall already have checked that self.sec_conf exist
        #create list, e.g., ['calvinsys','media','camera','lense']
        temp = req.split(".")
        while len(temp) >0:
            temp2 = '.'.join(temp)
            # Satisfied when one principal match in one policy
            for plcy in [p for p in self.sec_policy.values() if temp2 in p['resource']]:
                if any([principal_name in plcy['principal'][principal_type]
                            for principal_type, principal_names in principal.iteritems()
                                if principal_type in plcy['principal']
                            for principal_name, auth in zip(principal_names, self.auth[principal_type])
                                if auth]):
                    _log.debug("Security: found a match for %s against %s" % (req, temp2))
                    return True
            #Let's go up in hierarchy, e.g. if we found no policy for calvinsys.media.camera
            #let's now try calvinsys.media instead
            temp.pop()
        #The user is not in the list of allowed users for the resource
        _log.debug("Security: the principal does not have access rights to resource: %s" % req)
        return False

    @staticmethod
    def verify_signature_get_files(filename, skip_file=False):
        # Get the data
        sign_filenames = filename + ".sign.*"
        sign_content = {}
        file_content = ""
        sign_files = {os.path.basename(f).split(".sign.")[1]: f for f in glob.glob(sign_filenames)}
        for cert_hash, sign_filename in sign_files.iteritems():
            try:
                with open(sign_filename, 'rt') as f:
                    sign_content[cert_hash] = f.read()
                    _log.debug("Security: found signature for %s" % cert_hash)
            except:
                pass
        if not skip_file:
            try:
                with open(filename, 'rt') as f:
                    file_content = f.read()
            except:
                return None
                _log.debug("Security: file can't be opened")
        return {'sign': sign_content, 'file': file_content}

    def verify_signature(self, file, flag):
        content = Security.verify_signature_get_files(file)
        if content:
            return self.verify_signature_content(content, flag)
        else:
            return False

    def verify_signature_content(self, content, flag):
        _log.debug("Security: verify %s signature of %s" % (flag, content))
        if not self.sec_conf:
            _log.debug("Security: no signature verification required: %s"% content['file'])
            return True

        if flag not in ["application", "actor"]:
            # TODO add component verification
            raise NotImplementedError

        # loop through the policies until one is found that applies to the principal
        # Verification OK if sign and cert OK for any principal matching policy
        # that have previously been authorized
        for plcy in self.sec_policy.values():
            _log.debug("Security: verify_signature policy: %s\nprincipal: %s\nauth:%s" %
                        (plcy, self.principal, self.auth))
            if any([principal_name in plcy['principal'][principal_type]
                        for principal_type, principal_names in self.principal.iteritems()
                            if principal_type in plcy['principal']
                        for principal_name, auth in zip(principal_names, self.auth[principal_type])
                            if auth]):
                _log.debug("Security: found a policy with matching principal")
                if (flag + '_signature') in plcy:
                    if self.verify_signature_and_certificate(content, plcy, flag):
                        _log.debug("Security: signature verification successfull")
                        return True
        _log.error("Security: verification of %s signature failed 1" % flag)
        return False

    def verify_signature_and_certificate(self, content, plcy, flag):
        if "__unsigned__" in plcy[flag + '_signature']:
            _log.debug("Security: %s is allowed unsigned" % flag)
            return True

        if content is None:
            _log.debug("Security: %s need file and signature with certificate hash" % flag)
            return False

        if not content['sign']:
            _log.debug("Security: %s signature information missing" % flag)
            return False

        if not HAS_OPENSSL:
            _log.error("Security: Install openssl to allow verification of signatures and certificates")
            _log.error("Security: verification of %s signature failed 2" % flag)
            return False

        _log.debug("Security:verify_signature_and_certificate")
        for cert_hash, signature in content['sign'].iteritems():
            try:
                trusted_cert = os.path.join(self.sec_conf['signature_trust_store'], cert_hash + ".0")
                with open(trusted_cert, 'rt') as f:
                    string_trusted_cert = f.read()
                    trusted_cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, string_trusted_cert)
                    if self.check_signature_policy(trusted_cert, flag, plcy):
                        try:
                            OpenSSL.crypto.verify(trusted_cert, signature, content['file'], 'sha256')
                            _log.debug("Security: %s signature correct" % flag)
                            return True
                        except Exception as e:
                            _log.debug("Security: OpenSSL verification error", exc_info=True)
                            continue
                    else:
                        _log.debug("Security: signature policy not fulfilled")
                        continue
            except Exception as e:
                _log.debug("Security: error opening one of the needed certificates", exc_info=True)
                continue
        _log.error("Security: verification of %s signature failed 3" % flag)
        return False

    def check_signature_policy(self, cert, flag, plcy):
        """ Checks that if the signer is allowed by the security policy """
        _log.debug("Security:check_signature_policy")
        if flag=="application":
            if 'application_signature' in plcy:
                if cert.get_issuer().CN not in plcy['application_signature']:
                    _log.debug("Security: application signer not allowed")
                    return False
            else:
                _log.debug("Security: no application_signature element, unsigned applications allowed")
        elif flag=="actor":
            if 'actor_signature' in plcy:
                if cert.get_issuer().CN not in plcy['actor_signature']:
                    _log.debug("Security: actor signer not allowed")
                    return False
            else:
                _log.debug("Security: no actor_signature element, unsigned applications allowed")
        return True


"""
A CNN framework uses the tsf data as the input
made with pytorch
"""

import argparse
import numpy as np
import random
from torch.autograd import Variable
import torch.nn as nn
# import matplotlib.pyplot as plt
import torch.nn.functional as F
import torch.autograd as autograd
import torch
import torch.distributed as dist
# from torch.utils.tensorboard import SummaryWriter
# import cv2 as cv
import torchvision

import os
import scipy.io as scio
os.environ["CUDA_VISIBLE_DEVICES"] = '0'
# os.environ["CUDA_VISIBLE_DEVICES"] = '0,6,7'
gpus = [0, 1, 2, 3]
# device = torch.cuda.set_device('cuda:{}'.format(gpus[0]))

device = torch.cuda.set_device(0)

parser = argparse.ArgumentParser()
parser.add_argument("--n_epochs", type=int, default=200, help="number of epochs of training")  # 500 is ok
parser.add_argument("--lr", type=float, default=0.0001, help="adam: learning rate")
parser.add_argument("--b1", type=float, default=0.9, help="adam: decay of first order momentum of gradient")
parser.add_argument("--b2", type=float, default=0.999, help="adam: decay of first order momentum of gradient")
parser.add_argument("--n_cpu", type=int, default=4, help="number of cpu threads to use during batch generation")
parser.add_argument("--channels", type=int, default=1, help="number of image channels")
opt = parser.parse_args()
batch_size = 50  # batch size = 50 at the beginning, 30 is okay, 10 is better

def weights_init(m):

    if isinstance(m, nn.Conv2d):
        torch.nn.init.kaiming_uniform_(m.weight, a=0.2, mode='fan_in', nonlinearity='leaky_relu')
        if m.bias is not None:
            torch.nn.init.zeros_(m.bias)

class CNN(nn.Module):
    def __init__(self):
        super(CNN, self).__init__()
        self.layer1 = nn.Sequential(
            nn.Conv2d(in_channels=1, out_channels=16, kernel_size=(4, 1), stride=(4, 1)),
            nn.BatchNorm2d(16),
            nn.LeakyReLU(0.2)
        )
        self.layer2 = nn.Sequential(
            nn.Conv2d(in_channels=16, out_channels=32, kernel_size=(1, 23), stride=(1, 3)),
            nn.BatchNorm2d(32),
            nn.LeakyReLU(0.2),
            # nn.MaxPool2d(kernel_size=(1, 6), stride=(1, 6))
        )
        self.layer3 = nn.Sequential(
            nn.Conv2d(in_channels=32, out_channels=64, kernel_size=(1, 17), stride=(1, 1)),
            nn.BatchNorm2d(64),
            nn.LeakyReLU(0.2),
            nn.MaxPool2d(kernel_size=(1, 6), stride=(1, 6))
        )

        self.layer4 = nn.Sequential(
            nn.Conv2d(in_channels=64, out_channels=128, kernel_size=(1, 7), stride=(1, 1)),
            nn.BatchNorm2d(128),
            nn.LeakyReLU(0.2),
            nn.MaxPool2d(kernel_size=(1, 2), stride=(1, 2))
        )

        self.dense_layer = nn.Sequential(
            nn.Linear(11264, 2048),
            nn.LeakyReLU(0.2),
            nn.Dropout(0.3),
            nn.Linear(2048, 512),
            nn.LeakyReLU(0.2),
            nn.Dropout(0.3),
            nn.Linear(512, 128),
            nn.LeakyReLU(0.2),
            nn.Dropout(0.3),
            nn.Linear(128, 4),
            nn.Softmax()
        )

    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = out.view(out.size(0), -1)
        out = self.dense_layer(out)
        return out

def cnn(sub_index, data_train, label_train, data_test, label_test):

    classifier = CNN()
    classifier = classifier.cuda()
    classifier = nn.DataParallel(classifier, device_ids=[0, ])
    classifier = classifier.to(device)
    print('classifier')
    print(classifier)

    # set the seed for reproducibility
    seed_n = np.random.randint(500)
    print(seed_n)
    random.seed(seed_n)
    np.random.seed(seed_n)
    torch.manual_seed(seed_n)
    torch.cuda.manual_seed(seed_n)

    data_train = data_train.transpose([0, 2, 1])
    data_test = data_test.transpose([0, 2, 1])
    data_train = np.expand_dims(data_train, axis=1)
    data_test = np.expand_dims(data_test, axis=1)

    data_train = torch.from_numpy(data_train)
    data_test = torch.from_numpy(data_test)
    label_train = torch.from_numpy(label_train)
    label_test = torch.from_numpy(label_test)

    dataset = torch.utils.data.TensorDataset(data_train, label_train)
    dataloader = torch.utils.data.DataLoader(dataset=dataset, batch_size=batch_size, shuffle=True)

    optimizer = torch.optim.Adam(classifier.parameters(), lr=opt.lr)
    loss_func = nn.CrossEntropyLoss()

    Tensor = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor

    data_test = data_test.cuda()
    label_test = label_test.cuda()
    data_test = Variable(data_test.type(Tensor))

    optimizer = torch.optim.Adam(classifier.parameters(), lr=opt.lr)
    loss_func = nn.CrossEntropyLoss()

    test_acc = []
    pre=[]

    for epoch in range(opt.n_epochs):
        for step, data in enumerate(dataloader):
            datatrain, labeltrain = data
            datatrain = datatrain.cuda()
            labeltrain = labeltrain.cuda()

            datatrain = Variable(datatrain.type(Tensor))
            labeltrain = Variable(labeltrain.type(Tensor))

            output = classifier(datatrain)
            loss = loss_func(output, labeltrain.long())
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            if step % 1 == 0:
                test_output = classifier(data_test)
                pred = torch.max(test_output, 1)[1].data.squeeze()
                pre.append(pred)
                accuracy = float((pred == label_test.type(torch.long)).cpu().numpy().astype(int).sum()) / float(label_test.size(0))
                print('Epoch: ', epoch, '| train loss: %.4f' % loss.data, '| test accuracy: %.6f' % accuracy)
                test_acc.append(accuracy)

    test_acc.append(np.mean(test_acc))
    test_acc.append(np.max(test_acc))
    max_index=test_acc.index(np.max(test_acc))
    final_pre=pre[max_index]
    test_acc.append(seed_n)
    save_acc = np.array(test_acc)

    np.savetxt('./result/exp2_collected_data_JSFDSTCNN_' + str(
        sub_index) + '.txt', save_acc, '%.10f')
    scio.savemat('./pre_and_label/exp2_collected_data_JSFDSTCNN_'+str(
        sub_index) + '.mat',{'pre':final_pre,'label':label_test})