from collections import deque

n,k = list(map(int,input().split()))
q = deque([i for i in range(1,n+1)])
res = []

while len(q) > 0 :
    for i in range(1,k+1) :
        if i == k :
            res.append(q.popleft())
        else :
            q.append(q.popleft())

print('<',end='')
print(*res, sep=', ',end='')
# 이렇게 하면 사이에만 ,가 붙고 맨 끝에는 안붙네 개신기
print('>')

자료구조/1158 요세푸스 문제.py


#!/usr/bin/python3

import colorsys
import argparse
import json
import requests
from urllib.parse import urlencode, quote_plus
import pprint
import codecs
import itertools
import math
from igraph import *
import sqlite3
from time import sleep
import os, pickle
import sys
import time

from collections import OrderedDict

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from sklearn.metrics import adjusted_rand_score

today = int(time.time())

def etAl(s, d = None):
    '''format a name string to et al ciation'''
    n = ""
    if s.count(",") > 1:
        # many authors, cant show all
        n = s.split(",")[0].split(" ")[0]
        n = "{} et al.".format(n)
    elif s.count(",") == 1:
        # two authors, show both
        m = s.split(",")
        ns = []
        for i in m:
            ns.append(i.strip().split(" ")[0])
        n = "{} and {}".format(ns[0],ns[1])
    else:
        # one author
        n = s.split(" ")[0]
    if d != None:
        n = "{} ({})".format(n, d)
    return(n)

class epmc:
    def __init__(self, verbose = False, debug = False):
        self.useragent  = "epmc.py"
        self.debug      = debug
        self.verbose    = verbose
        self.limit      = 999 # 1000 is the API limit, we set it one lower just in case

    def search(self, q):
        ''' search via the API'''
        qe = q# urlencode(str(q), quote_via=quote_plus)
        url = "https://www.ebi.ac.uk/europepmc/webservices/rest/search?query={Q}&format=json".format(Q=qe)
        res = requests.get(url =url)
        return(res.json())

    def references(self, id, cl = "MED"):
        ''' fetch the references of the given publication'''
        u = "https://www.ebi.ac.uk/europepmc/webservices/rest/{CL}/{ID}/references?pageSize={S}&format=json"
        urlstring = u.format(CL=cl, ID= id, S = self.limit )
        result = requests.get(url = urlstring).json()

        # if hit count > self.limit,we need a paged loop
        if result["hitCount"] > self.limit:
            page = 1
            maxpages = math.ceil(result["hitCount"]/self.limit)
            while len(result["referenceList"]["reference"]) < result["hitCount"] or page <= maxpages:
                if self.debug:
                    print("on page", page)
                page = page + 1
                newurl = "{}&page={}".format(urlstring, page)
                tmpres = requests.get(url  = newurl).json()
                result["referenceList"]["reference"].extend(tmpres["referenceList"]["reference"])
        return(result)

    def citations(self, id, cl = "MED", paged = True):
        '''fetch al pubilcations that cited this one'''
        urlstring = "https://www.ebi.ac.uk/europepmc/webservices/rest/{}/{}/citations?pageSize={}&format=json".format(cl, id, self.limit)
        result = requests.get(url  = urlstring).json()

        # if hit count > self.limit,we need a paged loop
        if result["hitCount"] > self.limit:
            page = 1
            maxpages = math.ceil(result["hitCount"]/self.limit)
            while (len(result["citationList"]["citation"]) < result["hitCount"] or page <= maxpages) and paged:
                page = page + 1
                if self.debug:
                    print("on page {} of {}".format(page, maxpages))
                newurl = "{}&page={}".format(urlstring, page)
                tmpres = requests.get(url  = newurl).json()
                result["citationList"]["citation"].extend(tmpres["citationList"]["citation"])

        return(result)

class epmcBuffer:
    '''this class is to be used instead of th pure
    epmc class. Here request performed previsously
    will be cached in a database and only be requested again
    if a certain, user defined, number of days has passed'''
    def __init__(self,dbname = ".epmc.db", verbose = False, debug = False):
        self.epmc = epmc(verbose = verbose, debug = debug)
        self.daylimit = 5 # number of days we trust the number of citations did not change
        self.reflimit = 60 # number of days we trust the number of references did not change
        self.verbose = verbose
        self.debug   = debug
        self.DB(dbname)

    def DB(self, dbname = ".epmc.db"):
        # connect to the sqlite db
        self.db  = sqlite3.connect(dbname)
        self.c   = self.db.cursor()
        # create all tables that we need
        self.createTable()

    def updateCitationCount(self, items, paged = True ):
        '''for each paper update the citation count for future reference'''
        sql = "SELECT id, source, citedByDate FROM paper"
        res = self.c.execute(sql).fetchall()
        # remove all items that are not in the item list
        # this should be better than creating an awefull complex sql string
        # make a id-source list:
        idsList = []
        for i in items:
            idsList.append("-".join(i))
        i = 0
        resF = []
        for r in res:
            if "-".join(r[0:2]) in idsList:
                resF.append(r)
            i = i + 1
        total = len(resF)
        del res
        print("Checking citations of {} papers".format(len(resF)))
        i = 0
        j = 0

        for r in resF:
            if self.debug:
                print(r)
            if today - r[2] > (60*60* 24 * self.daylimit):
                # fetch new from the interweb:
                cits = self.citations(r[0], r[1], paged)

                i = i + 1
                if i % 100 == 0:
                    self.db.commit()

            if round((100 * j/total),1) % 1 ==  0 or True:
                percentagedone = round(100* j/total, 1)
                s  =("{}% done ({}/{})".format(percentagedone, j, total))
                sys.stdout.write('\r')
                # the exact output you're looking for:
                sys.stdout.write(s)
                sys.stdout.flush()
            j = j + 1
        self.db.commit()

    def createTable(self):
        # a single table to hold the information

        self.c.execute('''CREATE TABLE IF NOT EXISTS paper
                          (id text , source text,  title text, author TEXT, year INT,citedBy INT DEFAULT 0, refRet INT DEFAULT 0, citRet INT DEFAULT 0, citedByDate INT DEFAULT 0 )''')
        self.c.execute('''CREATE TABLE IF NOT EXISTS `edges`
                          ( `FROM` text, `FSOURCE` text,  `TO`  text, `TSOURCE` text )''')

        # create indices as this speed things up quite a lot
        index1 = ("CREATE INDEX IF NOT EXISTS paper_idx ON paper (id,source);")
        self.c.execute(index1)

        self.db.commit()

    def savePaper(self, res, saveCitations = False):
        # check if paper exists already:
        self.c.execute('SELECT * FROM `paper` WHERE id=? AND source=?', (res['id'], res['source']))
        r = self.c.fetchone()
        if r == None:
            if set(['id','source','title','authorString','pubYear']).issubset(res.keys()):
                self.c.execute('INSERT INTO `paper` (id, source, title, author, year)\
                           VALUES (?, ?, ?, ?, ?)', (res['id'], res['source'], res['title'], res['authorString'], res['pubYear']))
                #self.db.commit()

        # in case the paper comes from a Citations request
        # we can also save and update the citaion count from this data
        if saveCitations:
            self.c.execute("UPDATE `paper` SET citedBy=?, citedByDate=? WHERE id=? AND source=?", (res["citedByCount"], today,res["id"], res['source']))
            #self.db.commit()

        return()

    def references(self, id, source = "MED", verbose = False):
        '''fetch and return a list of reference ids'''
        # check if id class combi is in databank
        self.c.execute('SELECT * FROM `paper` WHERE id=? AND source=?', (id, source))
        r = self.c.fetchone()
        fetchFromWeb = True
        if r != None:
            fetchFromWeb = False
            # we already have the paper in the DB
            # check if its to old
            if today - r[6] > (self.reflimit * 60 * 60 * 24): # the magic number 6 is the field number fo refRet
                # delete this entry and fetch new
                self.c.execute('DELETE FROM `edges` WHERE `FROM` = ? AND `FSOURCE`= ? ',
                        (id, source))
                fetchFromWeb = True

        else:
            # need to insert this paper into the DB
            possres = self.epmc.search(id)
            # save all the results in the database, as there is no reason to discard them here
            if possres['hitCount'] > 0:
                for res in possres['resultList']['result']:
                    if 'id' in res.keys():
                        self.savePaper(res, saveCitations = True)

        # here make a call to the original class to query the server
        if fetchFromWeb:
            if self.verbose:
                print("need to fetch {} from web".format(id))
            res = self.epmc.references(id, source)

            self.c.execute("UPDATE `paper` SET refRet=? WHERE id=? AND source=?", (today, id, source))
            # save the result in the database if interesting:
            if res['hitCount'] > 0:
                for reference in res['referenceList']['reference']:
                    # save metadata of this it if not already
                    # update then the refRet field
                    if 'id' in reference.keys() and 'source' in reference.keys():
                        self.c.execute('INSERT INTO `edges` (`FROM`, `FSOURCE`,  `TO`,  `TSOURCE` )  VALUES (?, ?, ?,?)', (id, source, reference['id'], reference['source'] ))
                        self.savePaper(reference)
        self.db.commit()

        # fetch the data we want from the db
        # this is more simple than maintaining the data over the scope
        # it is also slower
        refresults = self.c.execute("SELECT * FROM `edges` WHERE `FROM`=? AND `FSOURCE`=?",(id, source))
        if fetchFromWeb and self.verbose:
            print("checked ", id, fetchFromWeb)
        return(refresults.fetchall())

    def citations(self, id, source = "MED", paged = True):
        '''fetch and return a list of reference ids'''
        # check if id class combi is in databank
        self.c.execute('SELECT citRet FROM `paper` WHERE id=? AND source=?', (id, source))
        r = self.c.fetchone()
        fetchFromWeb = True
        if r != None:
            fetchFromWeb = False
            # we already have the paper in the DB
            # check if its to old
            if today - r[0] > (self.reflimit * 60 * 60 * 24): # the magic number 6 is the field number fo refRet
                # delete this entry and fetch new
                self.c.execute('DELETE FROM `edges` WHERE `TO` = ? AND `TSOURCE`= ? ',
                        (id, source))
                fetchFromWeb = True

        else:
            # need to insert this paper into the DB
            possres = self.epmc.search(id)
            # save all the results in the database, as there is no reason to discard them here
            if possres['hitCount'] > 0:
                for res in possres['resultList']['result']:
                    if 'id' in res.keys():
                        self.savePaper(res, saveCitations = True)

        # here make a call to the original class to query the server
        if fetchFromWeb:
            if self.verbose:
                print("need to fetch citations of {} from web".format(id))
            res = self.epmc.citations(id, source, paged = paged)

            citedBy = res["hitCount"]

            if self.verbose:
                print("fetched {} citations".format(citedBy))

            self.c.execute("UPDATE `paper` SET citRet=?,citedByDate=?,citedBy=? WHERE id=? AND source=?", (today, today, citedBy, id, source))

            if self.verbose:
                print("Saving edges to these papers and the paper in database")
            # save the result in the database if interesting:
            if res['hitCount'] > 0:
                for reference in res['citationList']['citation']:
                    if self.debug:
                        print("debug: saving ref", reference)
                    # save metadata of this it if not already
                    # update then the refRet field
                    if 'id' in reference.keys() and 'source' in reference.keys():
                        self.c.execute('INSERT INTO `edges` (`FROM`, `FSOURCE`,  `TO`,  `TSOURCE` )  VALUES (?, ?, ?,?)', (reference['id'], reference['source'], id, source ))
                        self.savePaper(reference, saveCitations = True)
        self.db.commit()

        # fetch the data we want from the db
        # this is more simple than maintaining the data over the scope
        # it is also slower
        refresults = self.c.execute("SELECT * FROM `edges` WHERE `TO`=? AND `TSOURCE`=?",(id, source))
        if fetchFromWeb and self.verbose:
            print("fetched citations of {} ".format( id))
        return(refresults.fetchall())

    def nodes(self):
        '''return all nodes with citation values'''
        res = self.c.execute("SELECT id, source, citedBy, author, year, title FROM paper").fetchall()
        n = {}
        for r in res:
            n["{}_{}".format(r[0],r[1])] = {"name": etAl(r[3],r[4]), "cited": r[2],\
                                            "title": r[5], "year" :  r[4], "edges" : 0}

        return(n)

def get_N_HexCol(N=5):
    HSV_tuples = [(x * 1.0 / N, 0.5, 0.5) for x in range(N)]
    hex_out = []
    for rgb in HSV_tuples:
        rgb = map(lambda x: int(x * 255), colorsys.hsv_to_rgb(*rgb))
        hex_out.append('#%02x%02x%02x' % tuple(rgb))
    return hex_out

# from https://pythonprogramminglanguage.com/kmeans-elbow-method/
def clusterByTitle(g, k = 5):

    titles = []
    for v in g.vs:
        titles.append(v["title"])

    vectorizer = TfidfVectorizer(stop_words='english')
    X = vectorizer.fit_transform(titles)
    model = KMeans(n_clusters=k,
                   init='k-means++', max_iter=100, n_init=1)
    model.fit(X)

    # Get the class and assign color hexes to
    # the graph vertices
    classes = model.predict(X)
    cols = get_N_HexCol(k)

    g.vs["color"] = ([cols[i] for i in classes ])
    g.vs["clusterK"] = classes

    order_centroids = model.cluster_centers_.argsort()[:, ::-1]
    terms = vectorizer.get_feature_names()
    for i in range(k):
        print("Cluster %d:" % i),
        for ind in order_centroids[i, :10]:
            print(' %s' % terms[ind]),

    return(g)

class node():
    def __init__(self, name , **kwargs):
        self.name = name
        self.attr = OrderedDict()
        for key in kwargs:
            self.attr[key] = kwargs[key]

class edge():
    def __init__(self, source, target):
        self.source = source
        self.target = target

class graphml():
    def __init__(self):
        ''' class that handles a simple graph but as no
        sanity checks. so what comes in is what goes out'''
        self.nodes = []
        self.edges = []

    def addNode(self, name, **kwargs):
        self.nodes.append(node(name, **kwargs))

    def addEdge(self, source, target):
        self.edges.append(edge(source, target))

    def writeGML(self, filepath):
        '''not working with gephi :('''
        with open(filepath, "w") as f:
            f.write("graph\n[\ncomment \"This is a sample graph\"\ndirected 1\n")
            for node in self.nodes:
                f.write("node\n[\nid {}\n".format(node.name))
                for key in node.attr:
                    f.write("{} {}\n".format(key, node.attr[key]))
                f.write("]\n")
            for edge in self.edges:
                f.write("edge\n[\nsource {}\ntarget {}\n]\n".format(edge.source, edge.target))
            f.write("]\n")
        return
    def write(self, filepath):
        nodedef = False # write node and edge definition once
        edgedef = False
        with open(filepath, "w") as f:

            for node in self.nodes:
                if nodedef == False:
                    nd = "nodedef>name VARCHAR"
                    for key in node.attr:
                        # convert type into gdf definitions
                        tp = "BOOLEAN"
                        if isinstance(node.attr[key],str):
                            tp = "VARCHAR"
                        elif isinstance(node.attr[key], int):
                            tp = "INTEGER"
                        elif isinstance(node.attr[key],float):
                            tp = "DOUBLE"
                        nd = "{}, {} {}".format(nd, key, tp)

                    nd += "\n"
                    f.write(nd)
                    nodedef = True
                f.write(node.name)
                for key in node.attr:
                    if isinstance(node.attr[key],str):
                        f.write(", {}".format(node.attr[key].replace(",","")))
                    else:
                        f.write(", {}".format(node.attr[key]))
                f.write("\n")

            for edge in self.edges:
                if edgedef == False:
                    ed = "edgedef>node1 VARCHAR, node2 VARCHAR\n"
                    edgedef = True
                    f.write(ed)
                f.write("{}, {}\n".format(edge.source, edge.target))
        return

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("id" , type=str,
                            help="The ID of the paper")
    parser.add_argument("source" , type=str,default="MED",
                            help="source of the paper (MED)")

    parser.add_argument("output" , type=str,
                            help="output file")

    parser.add_argument("-c","--count" , type=int,
            help="number of hops, default: 2", default = 2)

    parser.add_argument("-d","--db" , type=str,
            help="Name of the db file", default = "cache.sqlite")

    parser.add_argument("-f", "--future", type = bool,
                            help="instead of references use the citations and show how this publiction was recived", default = False)
    parser.add_argument("-t", "--trim", type = int,
            help = "Minimal Number of edges a nodes needs to be shown", default = 2)
    parser.add_argument("-z", "--cited", type = int,
            help = "Minimal Number of citations nodes needs to be shown", default = 2)

    parser.add_argument("-k", "--kmeans", type = int,
            help = "K for means clustering, 0 for deactivating", default = 10)
    parser.add_argument("-v", "--verbose", type = bool,
                            help="increase output verbosity", default = False)
    parser.add_argument("-D", "--debug", type = bool,
                            help="increase output verbosity", default = False)
    args = parser.parse_args()

    # start a new DB server
    e = epmcBuffer(dbname = args.db, verbose = args.verbose, debug = args.debug)
    # now make hops:
    i = 0
    toVisit = [(args.id, args.source)]
    edges = []
    j = 0
    while i < args.count:
        j = 0
        k = len(toVisit)
        while j < k:
            v = toVisit[j]
            j = j+1
            if args.future:
                newedges = e.citations(v[0], v[1])
            else:
                if args.verbose:
                    print("fetching references now")
                newedges = e.references(v[0], v[1])

            for edge in newedges:
                if args.future:
                    p = (edge[0], edge[1])
                else:
                    p = (edge[2], edge[3])
                toVisit.append(p)
            edges.extend(newedges)
        i = i + 1
        print("At step {} of {}".format(i, args.count))
    print("Database done, will now update citation counts for all papers")
    e.updateCitationCount( items = list(set(toVisit)), paged = False)
    # now that we have the data we can build a graph if we want
    g = Graph()
    G = graphml()

    nodes = e.nodes()

    # for each node, we need to cound the number of related edges
    #for key in nodes:
    #    nodes[key]["edges"] = 0

    for i in edges:
        # count edges
        s = "{}_{}".format(i[0], i[1])
        t = "{}_{}".format(i[2], i[3])
        if s in nodes:
            nodes[s]["edges"] = nodes[s]["edges"] + 1
        if t in nodes:
            nodes[t]["edges"] = nodes[t]["edges"] + 1

    # get all vertices
    keyAskedFor = "{}_{}".format(args.id, args.source) # this was the paper asked for. always show that

    for key in nodes:
        if key == keyAskedFor or (nodes[key]["edges"] >= args.trim and nodes[key]["cited"] >= args.cited):
            g.add_vertex(key, label = nodes[key]["name"], size = nodes[key]["cited"],\
                    title = nodes[key]["title"], citations = nodes[key]["cited"], \
                    year = nodes[key]["year"])
            G.addNode(key, label = nodes[key]["name"], size = nodes[key]["cited"],\
                    title = nodes[key]["title"], citations = nodes[key]["cited"], \
                    year = nodes[key]["year"]) # my own graph class
            if args.debug:
                print(key,nodes[key])
        #else:
        #    print(nodes[key], key)

    for i in edges:
        s = "{}_{}".format(i[0], i[1])
        t = "{}_{}".format(i[2], i[3])
        if args.debug:
            print("adding edge {} {}".format(s,t))

        # Bug that some nodes (minimal number) do not appear
        # this is a workaround
        # remove this check if bug is fixed
        if s not in nodes:
            print("Missing node {}".format(s))
            continue
        if t not in nodes:
            print("Missing node {}".format(s))
            continue
        # END
        if (nodes[s]["edges"] >= args.trim or s == keyAskedFor ) and (nodes[t]["edges"] >= args.trim or t == keyAskedFor ) \
        and (nodes[s]["cited"] >= args.cited or s == keyAskedFor) and (nodes[t]["cited"] >= args.cited or t == keyAskedFor):
            g.add_edge(s,t, S = s, dfg = t)
            G.addEdge(s,t) # my own graph class

    g.simplify()

    if args.kmeans > 0:
        if args.verbose:
            print("dooing Kmeans clustering on titles")
        g = clusterByTitle(g, args.kmeans)

    if args.verbose:
        summary(g)
    # cloding file connection

    G.write(args.output + ".gdf")

    g.save(args.output + ".graphml", format="graphml")

    print(args.trim)

main()


import numpy as np
import process_data as pd
import os
from sklearn.externals import joblib
from sklearn.neural_network import MLPClassifier
from arc_standard import *
from glove_loader import *
from embeddings import tags_labels_dict_onehot, tags_labels_dict_trained9dec, tags_labels_dict_trained11dec

class ModelTest:

    def __init__(self,
                 trained_model_path,
                 word_vect_dict,
                 representation,
                 tags_labels_embedding,
                 amputated_CM_features = [],
                 classes=[i for i in range(-61, 62)]):
        self.trained_classifier = joblib.load(trained_model_path)
        self.word_vect_dict = word_vect_dict
        self.repr = representation
        self.classes = classes
        self.amputated_CM_features = amputated_CM_features
        self.tags_labels_embedding = tags_labels_embedding

    def predict_final_sentence_configurations_from_file(self, conll_in):
        '''
        Uses the trained_classifier to return the predicted transitions.
        Format: list of list of elements in {-1,0,1}.
        This format can be turned into a conll file by pd.file_trans_to_conll.
        This is done in self.predict_conll below.
        '''

        data_triples = pd.analyse_conll(conll_in)
        data_pairs = list(map(lambda triple: (triple[0], triple[1]), data_triples))
        all_final_configurations = []

        sentence_count = 1

        for (indices, word_pos) in data_pairs:
            sentence_count += 1
            if sentence_count%100 == 0:
                    print('Predicting sentence {0}'.format(sentence_count))

            # get initial configurations per sentence
            conf = Configuration('CM', self.word_vect_dict, self.tags_labels_embedding, self.amputated_CM_features)
            conf.load_sentence(indices)
            conf.mapping.extend(word_pos)

            predicted_transitions = []

            shift_count = 0
            reduction_count = 0

            while len(predicted_transitions) < (2 * len(indices)):

                conf_array = np.array(conf.repr()).reshape(1, -1)
                pred = self.trained_classifier.predict(conf_array)[0]

                if pred == 0:
                    # Get second best action if best action is not possible. Same as Chen and Manning.
                    if len(conf.buffer.contents) == 0 or shift_count == len(indices):
                        second_best = sorted(self.trained_classifier.predict_proba(conf_array).tolist()[0], reverse = True)[1]
                        index_second_best = self.trained_classifier.predict_proba(conf_array).tolist()[0].index(second_best)
                        second_best_class = self.trained_classifier.classes_[index_second_best]
                        if second_best_class > 0:
                            label = pd.get_label_transition(second_best_class)
                            conf.right_reduce(label)
                            reduction_count += 1
                            real_prediction = second_best_class

                        elif second_best_class < 0:
                            label = pd.get_label_transition(abs(second_best_class))
                            conf.left_reduce(label)
                            reduction_count += 1
                            real_prediction = second_best_class

                    else:
                        conf.shift()
                        shift_count += 1
                        real_prediction = 0
                elif pred > 0:
                    if len(conf.stack.contents) < 2 or reduction_count == len(indices):
                        # now standardly does shift if cannot right reduce, but perhaps left reduce is sometimes better?
                        # consider using predict_proba again to find best non-right-reduce transition

                        second_best = sorted(self.trained_classifier.predict_proba(conf_array).tolist()[0], reverse = True)[1]
                        index_second_best = self.trained_classifier.predict_proba(conf_array).tolist()[0].index(second_best)
                        second_best_class = self.trained_classifier.classes_[index_second_best]
                        if second_best_class == 0:
                            conf.shift()
                            shift_count += 1
                            real_prediction = 0

                        elif second_best_class < 0:
                            label = pd.get_label_transition(abs(second_best_class))
                            conf.left_reduce(label)
                            reduction_count += 1
                            real_prediction = second_best_class

                    else:
                        label = pd.get_label_transition(pred)
                        conf.right_reduce(label)
                        reduction_count += 1
                        real_prediction = pred
                elif pred < 0:
                    if (len(conf.stack.contents) < 2) or (conf.stack.contents[-2] == '0' and len(conf.buffer.contents) != 0) or (reduction_count == len(indices)):
                        second_best = sorted(self.trained_classifier.predict_proba(conf_array).tolist()[0], reverse = True)[1]
                        index_second_best = self.trained_classifier.predict_proba(conf_array).tolist()[0].index(second_best)
                        second_best_class = self.trained_classifier.classes_[index_second_best]
                        if second_best_class == 0:
                            conf.shift()
                            shift_count += 1
                            real_prediction = 0

                        elif second_best_class > 0:
                            label = pd.get_label_transition(second_best_class)
                            conf.right_reduce(label)
                            reduction_count += 1
                            real_prediction = second_best_class
                    else:
                        label = pd.get_label_transition(abs(pred))
                        conf.left_reduce(label)
                        reduction_count += 1
                        real_prediction = pred

                predicted_transitions += [real_prediction]
            all_final_configurations.append(conf)

        return(all_final_configurations)

    def final_configurations_to_conll(self, final_configurations, output_file):
        output_conll = open(output_file, 'w')

        sentence_count = 0
        for final_config in final_configurations:
            if not sentence_count == 0:
                output_conll.write('\n')
            sentence_count += 1

            all_labeled_arcs = final_config.arcs_left.labeled_arcs + final_config.arcs_right.labeled_arcs
            number_words = len(all_labeled_arcs)
            indices = list(range(number_words + 1))[1:]

            for word_index in indices:
                for arc in all_labeled_arcs:
                    if arc[1] == str(word_index):
                        output_conll.write(str(word_index) + '\t' + 'WORD' + '\t' + '_' + '\t' + 'TAG' + '\t' + 'TAG'+ '\t' + '_' + '\t'+ str(arc[0]) + '\t' + str(arc[2]) + '\t' + '_' + '\t' + '_' + '\n')

    def predict_conll(self, conll_in, conll_out):
        '''
        This does not 'return' anything, instead, it saves the predicted
        conll as a text file at the path given at conll_out.
        '''
        final_configurations = self.predict_final_sentence_configurations_from_file(conll_in)
        self.final_configurations_to_conll(final_configurations, conll_out)

    def eval(self, conll_in, conll_out, result_path):
        '''result_path should be 'results/filename_of_choice'.
        The .txt extension is added automatically.
        Returns results in filename_of_choice.txt.'''
        self.predict_conll(conll_in, conll_out)
        print('Prediction finished, predicted conll-file saved at {}'.format(conll_out))
        os.system('perl eval.pl -g {0} -s {1} > {2}.txt'.format(conll_in, conll_out, result_path))
        print('Evaluation finished, results saved at {}'.format(result_path))
        return(None)

###########################################
####### Looped testing of the model #######
###########################################

#tags_labels_dict = tags_labels_dict_trained9dec

def eval_loop(n_min, n_max):

    '''Performs all the prediction and evaluation (including saving)
    of the test model using a loop. NOTE: all path setttings need
    to be changed manually inside the function-body.'''

    glove_path = '/Users/Daan/Documents/Logic/NLP1/depparser-labeled-final/glove/glove.6B.50d.txt'
    word_vect_dict = load_glove_model(glove_path)
    for n in range(n_min, n_max+1):
        print('Evaluating the epoch {} model'.format(n))
        ### SETTING TEST-MODEL PATHS ###
        #model_path = 'models/one-hot/all-features/fullcorpus_labeled_{0}epochs_trained_model.pkl'.format(n)
        model_path = 'models/trained/trained-9dec/fullcorpus_labeled_{0}epochs_trained_model.pkl'.format(n)

        ### CREATING TEST OBJECT ###
        test_model = ModelTest(model_path, word_vect_dict, representation='CM',
                                                           tags_labels_embedding=tags_labels_dict,
                                                           amputated_CM_features=[])

        ### SETTING TEST PATHS ###
        # The path of the conll file we would like to test on:
        conll_in = 'data/test-stanford-raw.conll'
        # The path where we want to save the predicted conll:
        conll_out = 'output-conll/test-set/trained/trained-9dec/fullcorpus_labeled_onehot_{}epochs_test-stanford-raw'.format(n)
        # The path where we want to save the evaluation txt-file:
        result_path = 'results/test-set/trained/trained-9dec/result_fullcorpus_labeled_onehot_{}epochs_test-stanford-raw'.format(n)

        ### EVALUATING THE MODEL ON TEST PATHS ###
        test_model.eval(conll_in, conll_out, result_path)

#eval_loop(n_min=12, n_max=12)

tags_labels_dict = tags_labels_dict_onehot
amp_feat_worst = [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 44, 45, 46, 47]
amp_feat_best = [0,1,18,19]

def eval_loop2(n_min, n_max):

    '''Performs all the prediction and evaluation (including saving)
    of the test model using a loop. NOTE: all path setttings need
    to be changed manually inside the function-body.'''

    glove_path = '/Users/Daan/Documents/Logic/NLP1/depparser-labeled-final/glove/glove.6B.50d.txt'
    word_vect_dict = load_glove_model(glove_path)
    for n in range(n_min, n_max+1):
        print('Evaluating the epoch {} model'.format(n))
        ### SETTING TEST-MODEL PATHS ###
        #model_path = 'models/one-hot/all-features/fullcorpus_labeled_{0}epochs_trained_model.pkl'.format(n)
        model_path = 'models/one-hot/worst-features-amputated/fullcorpus_labeled_{}epochs_trained_model.pkl'.format(n)

        ### CREATING TEST OBJECT ###
        test_model = ModelTest(model_path, word_vect_dict, representation='CM',
                                                           tags_labels_embedding=tags_labels_dict,
                                                           amputated_CM_features=amp_feat_worst)

        ### SETTING TEST PATHS ###
        # The path of the conll file we would like to test on:
        conll_in = 'data/test-stanford-raw.conll'
        # The path where we want to save the predicted conll:
        conll_out = 'output-conll/test-set/one-hot/best-features-amputated/fullcorpus_labeled_onehot_{}epochs_test-stanford-raw'.format(n)
        # The path where we want to save the evaluation txt-file:
        result_path = 'results/test-set/one-hot/best-features-amputated/result_fullcorpus_labeled_onehot_{}epochs_test-stanford-raw'.format(n)

        ### EVALUATING THE MODEL ON TEST PATHS ###
        test_model.eval(conll_in, conll_out, result_path)

eval_loop2(n_min=12, n_max=12)

def eval_loop3(n_min, n_max):

    '''Performs all the prediction and evaluation (including saving)
    of the test model using a loop. NOTE: all path setttings need
    to be changed manually inside the function-body.'''

    glove_path = '/Users/Daan/Documents/Logic/NLP1/depparser-labeled-final/glove/glove.6B.50d.txt'
    word_vect_dict = load_glove_model(glove_path)
    for n in range(n_min, n_max+1):
        print('Evaluating the epoch {} model'.format(n))
        ### SETTING TEST-MODEL PATHS ###
        #model_path = 'models/one-hot/all-features/fullcorpus_labeled_{0}epochs_trained_model.pkl'.format(n)
        model_path = 'models/one-hot/all-features/fullcorpus_labeled_{}epochs_trained_model.pkl'.format(n)

        ### CREATING TEST OBJECT ###
        test_model = ModelTest(model_path, word_vect_dict, representation='CM',
                                                           tags_labels_embedding=tags_labels_dict,
                                                           amputated_CM_features=[])

        ### SETTING TEST PATHS ###
        # The path of the conll file we would like to test on:
        conll_in = 'data/test-stanford-raw.conll'
        # The path where we want to save the predicted conll:
        conll_out = 'output-conll/test-set/one-hot/all-features/fullcorpus_labeled_onehot_{}epochs_test-stanford-raw'.format(n)
        # The path where we want to save the evaluation txt-file:
        result_path = 'results/test-set/one-hot/all-features/result_fullcorpus_labeled_onehot_{}epochs_test-stanford-raw'.format(n)

        ### EVALUATING THE MODEL ON TEST PATHS ###
        test_model.eval(conll_in, conll_out, result_path)

#eval_loop3(n_min=15, n_max=15)

#############################################
####### 'Manual' testing of the model #######
#############################################

# ### LOADING THE GLOVE FILE ###
# glove_path = '/Users/Daan/Documents/Logic/NLP1/depparser-labeled-final/glove/glove.6B.50d.txt'
# word_vect_dict = load_glove_model(glove_path)

# ### SETTING TEST-MODEL PATHS ###
# model_path = 'models/smallcorpus/smallcorpus_1epochs_trained_model.pkl'
# #model_path = 'models/fullcorpus_2epoch_trained_model.pkl'
# representation = 'CM'

# ### CREATING TEST OBJECT ###
# test_model = ModelTest(model_path, word_vect_dict, representation)

# ### SETTING TEST PATHS ###
# # The path of the conll test we would like to test on:
# conll_in = 'data/disjunct_testset'
# # The path where we want to save the predicted conll:
# conll_out = 'output_conll/smallcorpus/smallcorpus_1epoch_disjunct_testset'
# # The path where we want to save the evaluation txt-file:
# result_path = 'results/smallcorpus/result_smallcorpus_1epoch_disjunct_testset'

# ### EVALUATING THE MODEL ON TEST PATHS ###
# test_model.eval(conll_in, conll_out, result_path)


import requests
import json

URL = 'https://www.sms4india.com/api/v1/sendCampaign'

# get request
def sendPostRequest(reqUrl, apiKey, secretKey, useType, phoneNo, senderId, textMessage):
  req_params = {
  'apikey':apiKey,
  'secret':secretKey,
  'usetype':useType,
  'phone': phoneNo,
  'message':textMessage,
  'senderid':senderId
  }
  return requests.post(reqUrl, req_params)

def sendSms(args):
  number=args['farmerMobile'].lstrip('Mobile:')
  farmername=args['farmerName'].lstrip('Name:')
  message="Agro-Backing\n hey "+farmername+"Looking forward to hear from you. \n name: "+args['name']+"\n"+"mobile: "+args['phno']

  # get response
  response = sendPostRequest(URL, '8HGNN4X47OT6DE6U4TL3W8NN65HEA041', 'HBBZG2AVMS9FS057', 'stage', number, 'active-sender-id', message )

  # print(response.text)
  return

if __name__=='__main__':
  pass


import discord
from discord.ext import commands
import logging
import threading
from settings import TOKEN
from stats import stats as get_stats

description = '''Jeff's Discord Bot'''

bot = commands.Bot(command_prefix='.', description=description)

@bot.event
async def on_ready():
	print('We have logged in as {0.user}'.format(bot))

@bot.command()
async def stats(ctx, ign: str):
	esea_link = get_stats(ign)
	embed = discord.Embed(title="ESEA", description=(ign + " - " + esea_link), url=esea_link, inline=True)
	embed.set_thumbnail(url='https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Ftse4.mm.bing.net%2Fth%3Fid%3DOIP.5NSZleerSox2ncN8uNRYpAHaDd%26pid%3DApi&f=1')
	await ctx.send(embed=embed)
	# await ctx.send(ctx.author.created_at) make .register method

@bot.command()
async def register(ctx):
	# User check if already made
	return

logger = logging.getLogger('discord')
logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(filename='discord.log', encoding='utf-8', mode='w')
handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
logger.addHandler(handler)

bot.run(TOKEN)


from myhdl import block, Signal, intbv, modbv, delay, instance, instances, always_comb, always_seq, always
from gluelogic import varfifo
from random import randrange
import numpy as np
import inspect
from utils import functionStat

#Collection of variables in class
class bus:
    def __init__(self, DATA=8, WORDS=4, ADDR=8, UPPER=2**8-4, LOWER=4, OFFSET=8):
        self.bus   = varfifo.bus(DATA=DATA, WORDS=WORDS, ADDR=ADDR, UPPER=UPPER, LOWER=LOWER, OFFSET=OFFSET)
        self.DATA   = DATA
        self.WORDS  = WORDS
        self.ADDR   = ADDR
        self.UPPER  = UPPER
        self.LOWER  = LOWER
        self.OFFSET = OFFSET

    @block
    def varlane(self, clkS, clkL, rst):
        functionStat(inspect.currentframe().f_code.co_name, __name__)
        varfifo0Inst   = self.bus.one2many(clkS, clkL, rst)
        varfifo1Inst   = self.bus.many2one(clkS, clkL, rst)

        return instances()

@block
def varlane(clkS, clkL, rst, laneBus):
    functionStat(inspect.currentframe().f_code.co_name, __name__)
    laneInst = laneBus.varlane(clkS, clkL, rst)
    return instances()


import matplotlib.pyplot as plt

x_values = list(range(1, 5001))
y_values = [i**3 for i in range(1, 5001)]
plt.scatter(x_values, y_values, c=y_values, cmap=plt.cm.Blues,
            edgecolor='none', s=40)

plt.title("Practice", fontsize=24)
plt.xlabel("X Value", fontsize=14)
plt.ylabel("Y Value", fontsize=14)

plt.tick_params(axis='both', which='major', labelsize=14)

plt.show()


import numpy as np
import matplotlib.pyplot as plt
from hmmlearn import hmm

class Utils:
    def __init__(self, random_state=1):
        self.random_state = random_state
        self.rs = np.random.RandomState(random_state)

    def forward(self, init, transition, emission, o):
        T = len(o)
        N = len(transition)
        alpha = np.zeros(shape=(N, T))
        alpha[:, 0] = init * emission[:, o[0]]
        for t in range(1, T):
            alpha[:, t] = alpha[:, t -
                                1].dot(transition).dot(np.diag(emission[:, o[t]]))
        termination = alpha[:, T - 1].sum()
        return alpha, termination

    def backward(self, init, transition, emission, o):
        T = len(o)
        N = len(transition)
        beta = np.zeros(shape=(N, T))
        beta[:, T - 1] = 1
        for t in range(T - 2, -1, -1):
            beta[:, t] = transition.dot(
                np.diag(emission[:, o[t + 1]])).dot(beta[:, t + 1])
        termination = np.sum(init * emission[:, o[0]] * beta[:, 0])
        return beta, termination

    def forward_backward(self, o, n_iters=10000, threshold=1e-6, verbose=1):
        N = 2
        T = len(o)
        V = np.unique(o)
        p_init, p_transit, p_slip, p_guess = self.rs.rand(4)
        PI = np.array([1 - p_init, p_init])
        A = np.array([[0, 1], [1 - p_transit, p_transit]])
        B = np.array([[p_slip, 1 - p_slip], [1 - p_guess, p_guess]])

        alpha = self.forward(PI, A, B, o)[0]
        beta = self.backward(PI, A, B, o)[0]

        def cal_gamma(t, j):
            return alpha[j, t]*beta[j, t]/(alpha[:, t] * beta[:, t]).sum()

        def cal_xi(t, i, j):
            return alpha[i, t]*A[i, j]*B[j, o[t + 1]]*beta[j, t + 1]/(alpha[:, t]*beta[:, t]).sum()

        for _ in range(n_iters):

            PI_est = np.zeros_like(PI)
            for i in range(N):
                PI_est[i] = cal_gamma(1, i)

            A_est = np.zeros_like(A)
            for i in range(N):
                for j in range(N):
                    foo = 0
                    for t in range(T - 1):
                        A_est[i, j] += cal_xi(t, i, j)
                        for k in range(N):
                            foo += cal_xi(t, i, k)
                    A_est[i, j] /= foo

            B_est = np.zeros_like(B)
            for j in range(N):
                foo = 0
                for v in V:
                    for t in np.argwhere(o == v):
                        bar = cal_gamma(t, j)
                        B_est[j, v] += bar
                        foo += bar
                B_est[j, :] /= foo

            if np.linalg.norm(PI - PI_est) <= threshold and\
                    np.linalg.norm(A - A_est) <= threshold and \
                    np.linalg.norm(B - B_est) <= threshold:
                if verbose:
                    print('Converged at:', _ + 1)
                break

            PI = PI_est
            A = A_est
            B = B_est

        return PI, A, B

class BKT:
    def __init__(self, random_state, algorithm='self', verbose=1):
        self.data_ = []
        self.random_state = random_state
        self.algorithm = algorithm
        self.utils = Utils(random_state)
        self.verbose = verbose

    def fit(self, obs_matrix):
        self.data_ = []
        self.partial_fit(obs_matrix)
        return self

    def partial_fit(self, obs_matrix):
        for obs in obs_matrix:
            if self.algorithm == 'self':
                PI, A, B = self.utils.forward_backward(obs, verbose=self.verbose)
            else:
                model = hmm.MultinomialHMM(n_components=2, init_params='se')
                model.transmat_ = [[0, 1], [0.5, 0.5]]
                model.fit(obs.reshape(-1, 1))
                PI, A, B = model.startprob_, model.transmat_, model.emissionprob_
            L, p_c = self.cal_L(PI, A, B, obs)
            data = dict(PI=PI, A=A, B=B, L=L, p_c=p_c)
            self.data_.append(data)
        return self

    def cal_L(self, PI, A, B, obs):
        p_init = PI[1]
        p_transit = A[1, 1]
        p_slip, p_guess = np.diag(B)
        L = [p_init]
        for o in obs:
            if o:
                p_l = L[-1]*(1-p_slip)/(L[-1]*(1-p_slip)+(1-L[-1])*p_guess)
            else:
                p_l = L[-1]*p_slip/(L[-1]*p_slip+(1-L[-1])*(1-p_guess))
            p_l = p_l + (1 - p_l)*p_transit
            L.append(p_l)
        p_c = L[-1]*(1-p_slip) + (1-L[-1])*p_guess
        return L, p_c

    def plot_posterior(self, idx, y):
        data = self.data_[idx]
        y = np.concatenate((np.array([1]), y))
        x = np.arange(len(data['L']))
        plt.plot(x, data['L'])
        plt.scatter(x, data['L'], c=np.where(y==1, 'red', 'blue'))
        plt.xlabel('Time')
        plt.ylabel('$P(L=1)$')
        plt.ylim([0, 1.2])
        plt.show()


import cv2, numpy as np

image_name = "icons2.png"

# converts any type of data into binary
# will use this to convert the payload and pixel values to binary in the payload hiding and extracting phase
def to_bin(data):
    """Convert 'data' to binary format as string"""
    if isinstance(data, str):
        return ''.join([ format(ord(i), "08b") for i in data])
    elif isinstance(data, bytes) or isinstance(data, np.ndarray):
        return [ format(i, "08b") for i in data ]
    elif isinstance(data, int) or isinstance(data, np.uint8):
        return format(data, "08b")
    else:
        raise TypeError("Type not supported.")

# hide secret_data into the image
def encode(image_name, secret_data):
    image = cv2.imread(image_name) # read the image
    n_bytes=image.shape[0] *image.shape[1] * 3 // 8 # maximum bytes to encode
    print("[*] Maximum bytes to encode:", n_bytes)
    secret_data += "=====" # add stopping criteria
    if len(secret_data) > n_bytes:
        raise ValueError("[!] Insufficient bytes, need bigger image or less data.")
    print("[*] Encoding data...")

    data_index = 0
    binary_secret_data = to_bin(secret_data) # convert data to binary
    data_len = len(binary_secret_data) # size of data to hide
    for row in image:
        for pixel in row:
            r, g, b = to_bin(pixel) # convert RGB values to binary format
            if data_index < data_len: # modify LSB only if there is still data to store
                pixel[0] = int(r[:-1] + binary_secret_data[data_index], 2) # LSB red pixel bit
                data_index += 1
            if data_index < data_len:
                pixel[1] = int(g[:-1] + binary_secret_data[data_index], 2) # LSB blue pixel bit
                data_index += 1
            if data_index < data_len:
                pixel[1] = int(b[:-1] + binary_secret_data[data_index], 2) # LSB green pixel bit
                data_index += 1
            if data_index >= data_len: # if data is encoded, just break out of the loop
                break
    return image

def decode(image_name):
    print("[+] Decoding...")
    # read the image
    image = cv2.imread(image_name)
    binary_data = ""
    for row in image:
        for pixel in row:
            r, g, b = to_bin(pixel)
            binary_data += r[-1]
            binary_data += g[-1]
            binary_data += b[-1]
    # split by 8-bits
    all_bytes = [ binary_data[i: i+8] for i in range(0, len(binary_data), 8) ]
    # convert from bits to characters
    decoded_data = ""
    for byte in all_bytes:
        decoded_data += chr(int(byte, 2))
        if decoded_data[-5:] == "=====":
            break
    return decoded_data[-5:]


import numpy as np
from shapely.geometry import Polygon
import cv2

def iou_rotate(box_a, box_b, method='union'):
    """iou rotate."""
    rect_a = cv2.minAreaRect(box_a)
    rect_b = cv2.minAreaRect(box_b)
    r1 = cv2.rotatedRectangleIntersection(rect_a, rect_b)
    if r1[0] == 0:
        return 0
    else:
        inter_area = cv2.contourArea(r1[1])
        area_a = cv2.contourArea(box_a)
        area_b = cv2.contourArea(box_b)
        union_area = area_a + area_b - inter_area
        if union_area == 0 or inter_area == 0:
            return 0
        if method == 'union':
            iou = inter_area / union_area
        elif method == 'intersection':
            iou = inter_area / min(area_a, area_b)
        else:
            raise NotImplementedError
        return iou

class DetectionIoUEvaluator(object):
    """Define the evaluator.

    The evaluator will
        - Iterate through the ground truth, save the ones which are valid and not_care.
        - Iterate through the predicted polygon, save the ones which are valid and not_care.
        - Calculate the number of valid ground truth and predicted polygons
        - Calculate the number when ground truth and predicted polygons are matched
        - Calculate recall, precision and hmean
    """

    def __init__(self, is_output_polygon=False, iou_constraint=0.5, area_precision_constraint=0.5):
        """Initialize."""
        self.is_output_polygon = is_output_polygon
        self.iou_constraint = iou_constraint
        self.area_precision_constraint = area_precision_constraint

    def evaluate_image(self, gt, pred):
        """evaluate image."""
        def get_union(pD, pG):
            return Polygon(pD).union(Polygon(pG)).area

        def get_intersection_over_union(pD, pG):
            return get_intersection(pD, pG) / get_union(pD, pG)

        def get_intersection(pD, pG):
            return Polygon(pD).intersection(Polygon(pG)).area

        def compute_ap(conf_list, match_list, num_gt_care):
            correct = 0
            AP = 0
            if len(conf_list) > 0:
                conf_list = np.array(conf_list)
                match_list = np.array(match_list)
                sorted_ind = np.argsort(-conf_list)
                conf_list = conf_list[sorted_ind]
                match_list = match_list[sorted_ind]
                for n in range(len(conf_list)):
                    match = match_list[n]
                    if match:
                        correct += 1
                        AP += float(correct) / (n + 1)

                if num_gt_care > 0:
                    AP /= num_gt_care

            return AP

        per_sample_metrics = {}

        matched_sum = 0

        num_globalcare_gt = 0
        num_globalcare_det = 0

        recall = 0
        precision = 0
        hmean = 0

        det_matched = 0

        iou_mat = np.empty([1, 1])

        gt_pols = []
        det_pols = []

        gt_pol_points = []
        det_pol_points = []

        # Array of Ground Truth Polygons' keys marked as don't Care
        gt_dontcare_pols_mum = []
        # Array of Detected Polygons' matched with a don't Care GT
        det_dontcare_pols_num = []

        pairs = []
        det_matched_nums = []

        evaluation_log = ""

        # Iterate through the ground truth
        for n in range(len(gt)):
            points = gt[n]['points']
            dont_care = gt[n]['ignore']

            if not Polygon(points).is_valid or not Polygon(points).is_simple:
                continue

            # Save the valid polygon
            gt_pol = points
            gt_pols.append(gt_pol)
            gt_pol_points.append(points)
            # Save the dont_care polygon
            if dont_care:
                gt_dontcare_pols_mum.append(len(gt_pols) - 1)

        evaluation_log += "GT polygons: " + str(len(gt_pols)) + (" (" + str(len(
            gt_dontcare_pols_mum)) + " don't care)\n" if len(gt_dontcare_pols_mum) > 0 else "\n")

        # Iterate through the predicted polygons
        for n in range(len(pred)):
            points = pred[n]['points']
            if not Polygon(points).is_valid or not Polygon(points).is_simple:
                continue

            # Save the valid polygon
            det_pol = points
            det_pols.append(det_pol)
            det_pol_points.append(points)
            if len(gt_dontcare_pols_mum) > 0:
                # Iterate through the dont_care polygons, calculate the intersection against predicted polygon
                for dontcare_pol in gt_dontcare_pols_mum:
                    # Find the dont_care polygon
                    dontcare_pol = gt_pols[dontcare_pol]
                    # Calculate the intersection between dont_care polygon and predicted polygon
                    intersected_area = get_intersection(dontcare_pol, det_pol)
                    # Calculate the area of predicted polygon
                    pd_dimensions = Polygon(det_pol).area
                    # Calcuate precision
                    precision = 0 if pd_dimensions == 0 else intersected_area / pd_dimensions
                    # Save the polygon number if precision is higher than the constraint
                    if (precision > self.area_precision_constraint):
                        det_dontcare_pols_num.append(len(det_pols) - 1)
                        break

        evaluation_log += "DET polygons: " + str(len(det_pols)) + (" (" + str(len(
            det_dontcare_pols_num)) + " don't care)\n" if len(det_dontcare_pols_num) > 0 else "\n")

        # If both groud truth polygon and predicted polygon are valid and available
        if len(gt_pols) > 0 and len(det_pols) > 0:
            # Calculate IoU and precision matrixs
            output_shape = [len(gt_pols), len(det_pols)]
            iou_mat = np.empty(output_shape)
            gt_rect_mat = np.zeros(len(gt_pols), np.int8)
            det_rect_mat = np.zeros(len(det_pols), np.int8)
            # Iterate through the ground truth and the predicted polygons, then calculate the IOU
            for gt_num in range(len(gt_pols)):
                for det_num in range(len(det_pols)):
                    pG = gt_pols[gt_num]
                    pD = det_pols[det_num]
                    iou_mat[gt_num, det_num] = get_intersection_over_union(pD, pG)

            for gt_num in range(len(gt_pols)):
                for det_num in range(len(det_pols)):
                    if gt_rect_mat[gt_num] == 0 and det_rect_mat[det_num] == 0 and gt_num not in gt_dontcare_pols_mum and det_num not in det_dontcare_pols_num:
                        # Check if ground truth and predicted polygons are matched, and save the number
                        if iou_mat[gt_num, det_num] > self.iou_constraint:
                            gt_rect_mat[gt_num] = 1
                            det_rect_mat[det_num] = 1
                            det_matched += 1
                            pairs.append({'gt': gt_num, 'det': det_num})
                            det_matched_nums.append(det_num)
                            evaluation_log += "Match GT #" + \
                                str(gt_num) + " with Det #" + str(det_num) + "\n"
        # Calcuate number of valid ground truth and predicted polygons
        num_gt_care = (len(gt_pols) - len(gt_dontcare_pols_mum))
        num_det_care = (len(det_pols) - len(det_dontcare_pols_num))
        # Calcuate recall, precision and hmean
        if num_gt_care == 0:
            recall = float(1)
            precision = float(0) if num_det_care > 0 else float(1)
        else:
            recall = float(det_matched) / num_gt_care
            precision = 0 if num_det_care == 0 else float(
                det_matched) / num_det_care

        hmean = 0 if (precision + recall) == 0 else 2.0 * \
            precision * recall / (precision + recall)

        matched_sum += det_matched
        num_globalcare_gt += num_gt_care
        num_globalcare_det += num_det_care

        per_sample_metrics = {
            'precision': precision,
            'recall': recall,
            'hmean': hmean,
            'pairs': pairs,
            'iou_mat': [] if len(det_pols) > 100 else iou_mat.tolist(),
            'gt_pol_points': gt_pol_points,
            'det_pol_points': det_pol_points,
            'gt_care': num_gt_care,
            'det_care': num_det_care,
            'gt_dontcare': gt_dontcare_pols_mum,
            'det_dontcare': det_dontcare_pols_num,
            'det_matched': det_matched,
            'evaluation_log': evaluation_log
        }

        return per_sample_metrics

    def combine_results(self, results):
        """combine results."""
        num_globalcare_gt = 0
        num_globalcare_det = 0
        matched_sum = 0
        for result in results:
            num_globalcare_gt += result['gt_care']
            num_globalcare_det += result['det_care']
            matched_sum += result['det_matched']

        method_recall = 0 if num_globalcare_gt == 0 else float(
            matched_sum) / num_globalcare_gt
        method_precision = 0 if num_globalcare_det == 0 else float(
            matched_sum) / num_globalcare_det
        method_hmean = 0 if method_recall + method_precision == 0 else 2 * \
            method_recall * method_precision / (method_recall + method_precision)

        method_metrics = {'precision': method_precision,
                          'recall': method_recall, 'hmean': method_hmean}

        return method_metrics


import torch
import numpy as np
import random
import os
import os.path as osp
from torch.utils.tensorboard import SummaryWriter
from datetime import datetime

def print_loss_dict(loss_dict):
    plist = []
    for k, v in loss_dict.items():
        if 'loss' in k:
            plist.append(f'{k}:{v:.4f}\t')
        else:
            plist.append(f'loss_{k}:{v:.4f}\t')
    return ''.join(plist)

def get_time(format='%Y-%m-%d %H:%M:%S'):
    return datetime.strftime(datetime.now(), format=format)

def init_logger(cfg):
    """
    initialization the logger part; tensorboard, logger, save_dir
    Args:
        cfg: configs

    Returns:
        writer, logger, save_dir
    """
    cfg.desc = cfg.desc

    os.makedirs(osp.join(cfg.dir, cfg.desc, 'tb'), exist_ok=True)
    os.makedirs(osp.join(cfg.dir, cfg.desc, 'ckpt'), exist_ok=True)

    writer = SummaryWriter(osp.join(cfg.dir, cfg.desc, 'tb'))
    logger = Logger(osp.join(cfg.dir, cfg.desc, f'{cfg.desc}_log'))
    ckpt_dir = osp.join(osp.join(cfg.dir, cfg.desc, 'ckpt'))
    return writer, logger, ckpt_dir

class LossMeter():
    def __init__(self):
        self.loss = 0
        self.num = 0

    def add(self, loss, num):
        self.loss += loss * num
        self.num += num

    def avg(self):
        return self.loss / self.num

    def reset(self):
        self.loss = 0
        self.num = 0

def seed_everything(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    random.seed(seed)
    torch.backends.cudnn.deterministic = True

def z_save_model(model, opt, sch, epoch, save_path):
    save_dict = {
        'model': model.state_dict(),
        'optim': opt.state_dict(),
        'sch': sch.state_dict(),
        'epoch': epoch}
    torch.save(save_dict, save_path)

def z_load_model(model, opt, sch, save_path):
    save_dict = torch.load(save_path)
    model.load_state_dict(save_dict['model'])
    opt.load_state_dict(save_dict['optim'])
    sch.load_state_dict(save_dict['sch'])
    return save_dict['epoch']

if __name__ == '__main__':
    print(get_time())


import sys

from PyQt5 import QtGui,QtCore, QtWidgets
from PyQt5.QtWidgets import QApplication, QPushButton, QMainWindow, QDialog, QGroupBox, QHBoxLayout, QVBoxLayout, QTextEdit, QLineEdit

from PyQt5.QtGui import QPixmap
from metra_conv_bot import get_bot

class window(QDialog):
    def __init__(self):
        super(window,self).__init__()

        self.title = 'bot chat'
        self.left = 50
        self.top = 50
        self.width = 1100
        self.height = 530
        self.initWindow()

    def initWindow(self):
        self.setWindowTitle(self.title)
        self.setGeometry(self.left,self.top,self.width,self.height)

        self.vbox = QVBoxLayout()
        # self.add_buttons()
        self.add_conv_area()
        self.setLayout(self.vbox)
        self.bot = get_bot()
        self.history = []

    def add_conv_area(self):
        conv_group_box = QGroupBox()
        self.output_te = QTextEdit(readOnly=True)
        self.input_le = QLineEdit(returnPressed=self.on_return_pressed)

        conv_layout = QtWidgets.QVBoxLayout()
        conv_layout.addWidget(self.output_te)
        conv_layout.addWidget(self.input_le)

        conv_group_box.setLayout(conv_layout)
        self.vbox.addWidget(conv_group_box)

    def update_chat(self):
        round_text = '\n'.join(self.history)
        self.output_te.setPlainText(round_text)

    @QtCore.pyqtSlot()
    def on_return_pressed(self):
        incoming_msg = self.input_le.text()
        if incoming_msg:
            bot_response = self.bot.run(incoming_msg)
            self.history.append(f'用户：{incoming_msg}')
            self.history.append(f'悠悠：{bot_response}')

            self.update_chat()
            self.input_le.clear()

if __name__ == '__main__':
    app = QApplication(sys.argv)
    window = window()
    window.show()
    app.exit(app.exec_())


import cv2
import numpy as np
import pytesseract

#第一步，创建haar级联器
plate = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_russian_plate_number.xml")
#第二步，导入人脸识别的图片并将其灰度化
img = cv2.imread('3.jpg')
#gray = cv2.cvtCoLor(img,cv2.COLOR_BGR2GRAY)
#第三步，进行人脸识别
#[[x,y,W,h]]
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
plates = plate.detectMultiScale(gray, 1.1, 3)
for (x,y,w,h) in plates:
    cv2.rectangle(img, (x, y), (x+w, y+h), (0, 0, 255), 2)
roi = gray[y:y+h,x:x+w]
ret,roi_bin = cv2.threshold(roi,0,255,cv2.THRESH_BINARY + cv2.THRESH_OTSU)
print(pytesseract.image_to_string(roi,lang="eng",config="--psm 8 --oem 3"))
cv2.imshow("img",img)
cv2.imshow("roi_bin",roi_bin)
cv2.waitKey()


# -*- coding: utf-8 -*-
import scrapy
import scrapy_splash
from scrapy_splash import SplashRequest
from stocks.items import StocksItem

from scrapy.utils.project import get_project_settings

class NSESpider(scrapy.Spider):
  name = "nse"

  def start_requests(self):
    urls = [
            'https://www.nseindia.com/',
            #'http://www.bseindia.com/'
           ]
    for url in urls:
      #yield scrapy.Request(url=url, callback=self.parse)
      yield SplashRequest(url, self.parse,
                          args= {
                            'wait': 0.5,
                            'timeout' : 120, #Timeout to render (default 30 sec)
                            #'url': Prefilled by plugin
                            #'baseurl': Base HTML content, relative resources on page referenced acc. to this.,
                            #'resource_timeout': #Individual resource request timeout value
                            # 'http_method', 'body',
                            #'js', 'js_source', 'filters', 'allowed_domains', 'allowed_content_types',
                            #'forbidden_content_types', 'viewport', 'images', 'headers', 'save_args',
                            #'load_args'
                          },
                          endpoint = 'render.html', #optional; could be render.json, render.png
                          #splash_url = SPLASH_URL, #optional; overrides SPLASH_URL
                          slot_policy=scrapy_splash.SlotPolicy.PER_DOMAIN, #optional
                          )

  def parse(self, response):
    page = response.url
    filename = 'nsepage.html'
    with open(filename, 'wb') as f:
      f.write(response.body)
    self.log("Saved file %s" %filename)


# search.py
# shows artist info for a URN or URL

import spotipy
import sys
import pprint
from spotify_token import Spotify_token

# usage
# $ python search.py [ Keyword you want to search ]
username = "sm75tpoxa0ur9uf8sduo95ktb"
search_str = sys.argv[1]

ST = Spotify_token(username)
token = ST.set()

sp = spotipy.Spotify(auth=token)
result = sp.search(search_str, limit=50)
for item in result['tracks']['items']:
    print(item['name'])
    print(item['id'])
    print('')


'''
Dictionary Merger

'''

import collections
import argparse
import datetime
import os
import sys

from PIL import Image , TiffImagePlugin

def Worker(args):
	print("User Current Version:-", sys.version)
	# -- Arguments Definitions --------------------------------------------------------------------------------------------------------------
	parser = argparse.ArgumentParser(description='FileMan Link Generator')
	parser.add_argument('-d1', '--dictionary', type=str, default={'1':123, '2':234, '3':345, 'X':{'a':'abc', 'b':'bcd', 'c':'cde', }}, help='Dictionary to be updated')
	parser.add_argument('-d2', '--dictnew', type=str, default={'1':1234, '22':2234, '33':3345,  'X':{'aa':'aabc', 'bb':'bbcd', }}, help='New Dictionary to merge into the first one')

	# -- Variable Definitions --------------------------------------------------------------------------------------------------------------
	args = parser.parse_args()
	# -- Initial Setup
	jd1 = args.dictionary
	jd2 = args.dictnew

	#jd = {**jd1, **jd2}

	print(jd1)
	print(jd2)
	jd = DctMrg(jd1, jd2)

	print(dctcln(jd))

#------------------------------------------------------------------------------------
# Dict Clean up ---------------------------------------------------------------------
def dctcln(obj):
	if isinstance(obj, dict):
		for key, val in obj.items():
			obj[key] = dctcln(val)
		return obj
	elif isinstance(obj, tuple):
		return tuple(dctcln(val) for val in obj)
	elif isinstance(obj, TiffImagePlugin.IFDRational):
		return float(obj)
	elif isinstance(obj, bytes):
		#return  obj.decode("utf-8", "ignore")
		return  obj.decode(errors="replace")
	#elif isinstance(obj, list):
	#	return list(dctcln(val) for val in obj)
	else:
		#print("dctcln -> non defined tyte found:", type(obj), str(obj))
		return obj

#------------------------------------------------------------------------------------
# Dict Merger -----------------------------------------------------------------------
def DctMrg(dct='',dctnew=''):
	try:
		d=dct.copy()
	except:
		d=dct
	try:
		d2=dctnew.copy()
	except:
		d2=dctnew
	if isinstance(d, dict) and isinstance(d2, dict):
		for key, val in d2.items():
			if key in d:
				if isinstance(val, dict):
					d[key] = DctMrg(d[key], d2[key])
				else:
					d[key] = val
			else:
				d[key] = d2[key]
		return d
	elif not isinstance(d, dict) and isinstance(d2, dict):
		print('-d1 not a Dictionary: ', d)
		return d2
	elif not isinstance(d2, dict) and isinstance(d, dict):
		print('-d2 not a Dictionary: ', d2)
		return d
	else:
		return {}

#-------------------------------------------------------------------------------------------------------------------------------------------------------------
#-------------------------------------------------------------------------------------------------------------------------------------------------------------
if __name__ == '__main__':
	from sys import argv
	try:
		Worker(argv)
	except KeyboardInterrupt:
		pass
	exit()


from django_filters.rest_framework.backends import DjangoFilterBackend
from rest_framework import mixins
from rest_framework.filters import OrderingFilter, SearchFilter
from rest_framework.permissions import AllowAny
from rest_framework.viewsets import GenericViewSet

from base.auth.permissions.permission import IsSuperStaff, IsManager
from base.common.constant.view_action import BaseViewAction
from base.common.custom.pagination import CustomPagination
from base.common.utils.exceptions import PermissionDenied
from base.menu.models import Menu
from staff.menu.filters.menu import MenuListQueryFields
from staff.menu.serializers.menu import MenuListSlz, MenuRetrieveSlz, MenuCreateSlz, MenuUpdateSlz

class MenuViewSet(mixins.ListModelMixin, mixins.CreateModelMixin, mixins.UpdateModelMixin, mixins.RetrieveModelMixin,
                  mixins.DestroyModelMixin, GenericViewSet):
    permission_classes = (AllowAny,)
    pagination_class = CustomPagination
    filter_backends = (DjangoFilterBackend, OrderingFilter, SearchFilter)
    serializer_class = MenuListSlz
    queryset = Menu.objects.all()
    search_fields = MenuListQueryFields.SEARCH_FIELDS
    ordering_fields = MenuListQueryFields.ORDER_FIELDS
    ordering = MenuListQueryFields.ORDER_DEFAULT_FIELD
    filterset_fields = MenuListQueryFields.FILTERSET_FIELDS

    def get_serializer_class(self):
        slz_switcher = {
            BaseViewAction.LIST: MenuListSlz,
            BaseViewAction.RETRIEVE: MenuRetrieveSlz,
            BaseViewAction.CREATE: MenuCreateSlz,
            BaseViewAction.UPDATE: MenuUpdateSlz,
        }
        slz = slz_switcher.get(self.action, self.serializer_class)
        return slz

    def get_permissions(self):
        perm_switcher = {
            BaseViewAction.LIST: (AllowAny,),
            BaseViewAction.RETRIEVE: (AllowAny,),
            BaseViewAction.CREATE: (IsManager | IsSuperStaff,),
            BaseViewAction.UPDATE: (IsManager | IsSuperStaff,),
            BaseViewAction.DESTROY: (IsSuperStaff,),
        }
        self.permission_classes = perm_switcher.get(self.action, self.permission_classes)
        if self.permission_classes is None:
            raise PermissionDenied()

        return super().get_permissions()


import psychopy.visual
import psychopy.event
import time
import numpy as np
from tqdm import tqdm

def run():
    # PARAMS
    repeats_per_orientation = 15
    orientations = [0, 45, 90]
    spatial_freq = 5.0 / 400.0
    n_frames_on = 45
    n_frames_off = 180
    speed = 5

    # CREATE STUFF
    white, black, gray = [1, 1, 1], [-1, -1, -1], [1, 1, 1]

    win = psychopy.visual.Window(
        pos=[50, 50],
        size=[1920, 1080],
        units="pix",
        fullscr=True,
        screen=1
    )

    square = psychopy.visual.Rect(
        win=win,
        pos=[-835, 415],
        width=250,
        height=250,
        units="pix",
        fillColor=black
    )

    grating_params = dict(
        win=win,
        units="pix",
        size=[8000, 8000]
    )

    grating = psychopy.visual.GratingStim(
        ori=orientations[0],
        **grating_params
    )
    grating.sf = spatial_freq

    # COMPUTE FRAMES
    orientations = np.repeat(orientations, repeats_per_orientation)

    square_frames = np.hstack(np.tile(np.array([np.ones(n_frames_on), np.zeros(n_frames_off)]), len(orientations)))
    ors = np.hstack([np.ones(n_frames_on+n_frames_off)*o for o in orientations])
    phases = np.hstack([np.hstack([np.linspace(0, speed, n_frames_on), np.zeros(n_frames_off)]) for o in orientations])

    if phases.shape != ors.shape or phases.shape != square_frames.shape:
        raise ValueError("Something went wrong with frames calculation")
    else:
        print("Total duration: {} stimuli over {}".format(len(orientations), phases.shape[0]))

    # Animate stimuli
    curr_orientation = orientations[0]
    for s, o, p in tqdm(zip(square_frames, ors, phases)):
        if s:
            square.color = white
        else:
            square.color = black

        if p==0:
            grating.opacity=0
        else:
            grating.opacity = 1

        if o != curr_orientation:
            grating = psychopy.visual.GratingStim(
                ori=o,
            **grating_params
            )
            grating.sf = spatial_freq

            curr_orientation = o
            print("\n\nStarting stim with orientation: {}".format(round(o)))

        grating.phase = p

        grating.draw()
        square.draw()
        win.flip()

    win.close()

if __name__ == "__main__":
    run()


import requests,os,openpyxl
from datetime import date

def dataScraper(u_name):

    headers = {
        'accept-encoding': 'gzip, deflate, br',
        'accept-language': 'en-US,en;q=0.9',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36',
        'accept': '*/*',
        'referer': 'https://www.instagram.com',
        'Connection': 'keep-alive',
    }

    url ='https://www.instagram.com/'+u_name

    params = {
        '__a': '1'

    }

    r = requests.get(url=url, headers=headers,params= params )

    global p_id
    global username
    global fullname
    global bio
    global posts
    global followers
    global following
    global profpic
    global status
    global s_data
    global t_date

    data = r.json()
    p_id = data["graphql"]["user"]["id"]
    username= data["graphql"]["user"]["username"]
    fullname= data["graphql"]["user"]["full_name"]
    bio = data["graphql"]["user"]["biography"]
    posts = data["graphql"]["user"]["edge_owner_to_timeline_media"]["count"]
    followers = data["graphql"]["user"][ "edge_followed_by"]["count"]
    following = data["graphql"]["user"]["edge_follow"]["count"]
    profpic = data["graphql"]["user"]["profile_pic_url_hd"]

    get_status = data["graphql"]["user"]["is_private"]
    if get_status == True:
        status='Private'
    else :
        status ='Public'

    t_day =date.today()

    s_data= [fullname,username,p_id,posts,followers,following,status,profpic,t_day]
    return s_data
    # print(fullname,username,p_id,posts,followers,following,status,profpic)

def extractData(s_data):
    global wb
    title_data=['Full Name','Username','ID','Posts','Followers','Following','Status','Profile Pic']
    os.chdir('C:\\Users\\Shaun\\Desktop\\python practrice') #change the directory to your folder
    wb = openpyxl.load_workbook('data.xlsx')   #devnote create a excel file in the directory you are using
    sheet = wb.active
    sheet.title = 'StalkerData'
    sheet['A1']. value = 'Full Name'
    sheet['B1']. value = 'Username'
    sheet['C1']. value = 'ID'
    sheet['D1']. value = 'Posts'
    sheet['E1']. value = 'Followers'
    sheet['F1']. value = 'Following'
    sheet['G1']. value = 'Status'
    sheet['H1']. value = 'Profile Pic'
    sheet['I1']. value = 'TimeStamp'
    sheet.append(s_data)
    wb.save('data.xlsx')
    wb.close()

ulist = ['_shaunnyboi_'] #add your friends username in this list

print("Data Extracted!...Saving Data Please Wait!...")
for u_name in ulist:
    dataScraper(u_name)

    # print(fullname,username,p_id,posts,followers,following,status,profpic)
    # print(s_data)
    extractData(s_data)
print("Data Saved...")


from os import path as OSPath
import core.Config as CoreConfig
import core.SecretKey as CoreSecretKey
import json

class LoginModel:
    FILE_REMEMBER_ME = 'remember_me.json'

    def __init__(self) -> None:
        self._sdtLogin = {
            'UsrLgn': '',
            'UsrPwd': '',
            'RememberMe': True
        }

    def getSdtLogin(self):
        return self._sdtLogin

    def setUsrLgn(self, data_content):
        self._sdtLogin['UsrLgn'] = data_content

    def setUsrPwd(self, data_content):
        self._sdtLogin['UsrPwd'] = data_content

    def getRememberMe(self):
        src_file = OSPath.join('data', 'cookie', self.FILE_REMEMBER_ME)

        data_content = {}

        if OSPath.isfile(src_file):
            with open(src_file, mode='r', encoding='utf-8') as file:
                data_content = json.load(file)

            if not self._datastr_isvalid(data_content):
                data_content = self._remember_me_default()
        else:
            data_content = self._remember_me_default()

        return data_content

    def _remember_me_default(self):
        data_content = {
            "username": "",
            "password": "",
            "remember_me": True
        }
        return data_content

    def _datastr_isvalid(self, data_content):
        result = True

        if not 'username' in data_content:
            result = False

        if not 'remember_me' in data_content:
            result = False

        return result

    def setRememberMe(self, remember_me=False):
        csCoreConfig = CoreConfig.Config()
        path = csCoreConfig.get_block_item('DAS', 'path')

        csCoreSecretKey = CoreSecretKey.SecretKey()
        hash_pssw = ''
        if (self._sdtLogin['UsrPwd'] != ''):
            hash_pssw = csCoreSecretKey.generate_secret(self._sdtLogin['UsrPwd'])

        self._sdtLogin['RememberMe'] = remember_me
        src_file = OSPath.join(path,'data', 'cookie', self.FILE_REMEMBER_ME)

        data_content = {
            "username": "{}".format(self._sdtLogin['UsrLgn']),
            "password": "{}".format(hash_pssw),
            "remember_me": remember_me
        }
        try:
            with open(src_file, mode='w', encoding='utf-8') as file:
                json.dump(data_content, file, indent=4)
        except:
            pass


import hashlib
import logging
import uuid

from .errors import handle_error
from .models import AuthModel
from .session import BaseManager

class AuthManager(BaseManager):
    def gen_token(self, name):
        token = str(uuid.uuid4())
        salt = hashlib.sha256(str.encode(token)).hexdigest()
        status = self.add_auth_token(name, salt)
        return token

    def find_token(self, salt):
        try:
            self.create_session()
            res = self.session.query(AuthModel).filter(AuthModel.salt == salt).first()
            if res:
                res.touch()
                self.session.commit()
            return res
        except Exception as e:
            handle_error("Failed to find token.", e)

    def delete_token(self, id_):
        try:
            self.create_session()
            token = self.session.query(AuthModel).filter(AuthModel.id == id_).first()
            if not token:
                raise Exception("Token does not exist.")
            name = token.name
            salt = token.salt
            self.session.delete(token)
            self.session.commit()
        except Exception as e:
            handle_error("Unable to delete token.", e)
        return False

    def validate_token(self, token, bearer=""):
        salt = hashlib.sha256(str.encode(token)).hexdigest()
        search_res = self.find_token(salt)
        validity = True if search_res else False
        if validity:
            logging.info(
                f"Authenticated {bearer} with '{search_res.name}' ({search_res.salt[:5]})"
            )
        else:
            logging.warning(f"Failed to authenticate {bearer}")
        return validity

    def get_all_tokens(self):
        self.create_session()
        try:
            return self.session.query(AuthModel).all()
        except Exception as e:
            handle_error("Failed to retrieve tokens", e)
        return []

    def add_auth_token(self, name, salt):
        try:
            self.create_session()
            auth = AuthModel(name, salt)
            self.session.add(auth)
            self.session.commit()
            logging.info(f"Created new auth token '{name}' ({salt[:5]})")
        except Exception as e:
            handle_error("Failed to add new token.", e)
            return False
        return True


from django.http import HttpResponse
from django.template import loader
from django.shortcuts import get_object_or_404, render, redirect
from PIL import Image
import itertools
import os, sys

from django.conf import settings

from .models import Genre, Product

def index(request):

    #this needs to just display all the products
    latest_prod_list = Product.objects.all()
    template = loader.get_template('catalog/index.html')
    context = {
        'latest_prod_list' : latest_prod_list
        }
    return render(request, 'catalog/index.html', context)

def detail(request, prod_id):
    #this will list the hierarchy and the breadcrumbs
    this_product = get_object_or_404(Product, pk=prod_id)

    breadcrumbs=getBreadcrumb(this_product.parent, [])

    hierarchy_list = getFullHierarchy()
    template = loader.get_template('catalog/detail.html')
    context = {
        'breadcrumbs' : breadcrumbs,
        'hierarchy_list' : hierarchy_list,
        'prod' : this_product
        }
    return render(request, 'catalog/detail.html', context)

def add(request):
    template = loader.get_template('catalog/add.html')
    return render(request, 'catalog/add.html',{})

def process(request):
    #check if item already is in database
    #build the adjacency list (genre model)
    media_object = Genre.objects.get(parent=None, genre_name=request.POST['media'])
    genre_objects = Genre.objects.filter(parent=media_object, genre_name=request.POST['genre'])
    if( not genre_objects.exists()):
       genre_objects = [makeGenre(request.POST['genre'], media_object)]
    parentG=genre_objects[0]
    if(request.POST['subgenre'] !=""):
        subgenre_objects = Genre.objects.filter(parent=parentG,
                                                genre_name=request.POST['subgenre'])
        if(not subgenre_objects.exists()):
            subgenre_objects=[makeGenre(request.POST['subgenre'], parentG)]
        parentG=subgenre_objects[0]

    #build the product model

    p = Product(media_type=request.POST['media'], parent=parentG,
                title=request.POST['title'], description=request.POST['description'],
                price=request.POST['price'])
    if 'pic' in request.FILES:
        img_path = handle_uploaded_file(request.FILES['pic'])
        p.image=request.FILES['pic'].name

    p.save()
    return redirect('catalog:index')

#returns relative path of file
def handle_uploaded_file(f):
    path = settings.MEDIA_ROOT+'/'+f.name
    with open(path, 'wb+') as destination:
        for chunk in f.chunks():
            destination.write(chunk)
    im = Image.open(path)
    new_size = 150, 150
    im.thumbnail(new_size, Image.ANTIALIAS)
    im.save(path)

def makeGenre(name, parentObject):
    g=Genre(parent=parentObject, genre_name=name)
    g.save()
    return g

def getBreadcrumb(genre_node, trail):
    if(genre_node.parent is None):
        return trail + [str(genre_node)]
    else:
        return getBreadcrumb(genre_node.parent, trail+[str(genre_node)])

#grabs the expanded hierarchy for a given media type, formatted for Django's unordered list
def getHierarchy(depth, parent_node, hierarchy):
    children_list = Genre.objects.filter(parent=parent_node)
    c_list=[]
    if not children_list:
       return hierarchy + [str(parent_node)]
    for child in children_list:
        c_list += getHierarchy(depth+1, child, hierarchy)
    return hierarchy + [str(parent_node), c_list]

def getFullHierarchy():
    full_list = []
    for choice in Genre.MEDIA_CHOICES:
        media = Genre.objects.get(genre_name=choice[0])
        full_list += getHierarchy(0, media, [])
    return full_list


# -*- coding: utf-8 -*-
import json
import posix_ipc as ipc

from flask import Flask, jsonify, render_template
from flask_socketio import SocketIO, send, emit
from service.chartService import ChartService
from service.brewconfig import BrewConfig
from service.ipchelper import prepare_data, TYPE_CONTROL, CONTROL_NEXT

__author__ = 'oli@fesseler.info'
__version__ = ('0', '0', '1')

app = Flask(__name__)
app.config["JSON_SORT_KEY"] = False
app.config['SECRET_KEY'] = 'secret!braubar!mhh!BIER!'
socketio = SocketIO(app, async_mode='eventlet')
thread = None
cs = ChartService()

@app.route('/')
def index():
    recipe_file = open(BrewConfig.RECIPE_FILE)
    recipe = json.load(recipe_file)
    print(cs.status(brew_id))
    print(recipe)
    return render_template('index.html',
                           brew_id=brew_id,
                           brew_state=cs.status(brew_id),
                           brew_recipe=recipe)

@socketio.on("connected")
def handle_connected(connected_msg):
    emit("fullchart", cs.brew_chart(brew_id=brew_id))
    print(connected_msg)

@socketio.on("update chart")
def handle_update(update_msg):
    socketio.emit("update chart", json.dumps(cs.status(brew_id=brew_id)))
    print("emitted update chart after message: ", update_msg)

@socketio.on("next")
def next_state(next):
    msg = {"ok": False, "status": None}
    try:
        msg["status"] = cs.status(brew_id)
        print("status:", cs.status(brew_id))
        if write_to_queue(CONTROL_NEXT):
            print("write to queue worked")
            msg["ok"] = True
    except Exception as e:
        print("flask_app error:", e)
    finally:
        socketio.emit("next", json.dumps(msg))

@app.route('/start')
def brew_start():
    return "Not Implemented"

@app.route('/status')
def status():
    return jsonify(cs.status(brew_id))

@app.route('/status/brew')
def brew_state():
    return cs.brew_status(brew_id=brew_id)

@app.route('/status/system')
def system_state():
    return cs.system_status(brew_id=brew_id)

@app.route('/next')
def next_state():
    msg = {"ok": False, "status": None}
    try:
        msg["status"] = cs.status(brew_id)
        if write_to_queue(CONTROL_NEXT):
            msg["ok"] = True
    except Exception as e:
        print("error: ", e)

    finally:
        return jsonify(msg)

@app.route('/temp')
def temp():
    return "Not Implemented"

@app.route('/chart/data')
def chart_data():
    a = cs.brew_chart(brew_id=brew_id)
    print(a)
    return a

@app.route('/chart/last_row')
def last_row():
    return jsonify(cs.last_row(brew_id=brew_id))

def write_to_queue(control):
    try:
        msg = {"control": control}
        queue = ipc.MessageQueue(name=BrewConfig.BRAUBAR_QUEUE)
        queue.send(prepare_data(TYPE_CONTROL, msg).encode(encoding=BrewConfig.QUEUE_ENCODING), timeout=5)
    except ipc.ExistentialError as e:
        print("IPC Extential Error happened", e)
        queue.close()
        return False
    except ipc.BusyError as e:
        print("socket busy error: write_to_queue", e)
        queue.close()
        return False
    except Exception as e:
        print("error: ", e)

    return True

if __name__ == "__main__":
    import argparse

    # config.BrewConfig.CONFIG_FILE = "/home/oli/dev/braubar-pi/config/config.json"
    # config.BrewConfig.RECIPE_FILE = "/home/oli/dev/braubar-pi/config/recipe.json"

    parser = argparse.ArgumentParser(description="BrauBar webserver at your service.")
    parser.add_argument('--host', help="IP-Address to listen on. Default is 0.0.0.0", default="0.0.0.0")
    parser.add_argument('-i', '--id', help="brew_id to identify the current brew process. "
                                           "if no id is given, it shall return all brews")
    args = parser.parse_args()
    try:
        host = args.host
        brew_id = args.id
        # start app in debugmode
        app.debug = True
        socketio.run(app, host=host)
    finally:
        print("good beer, see ya")


import os #os module imported here
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support.ui import Select
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
from prettytable import PrettyTable

baseUrl = 'https://www.reddit.com/r/UIUC/'

option = Options()
option.add_argument("--disable-infobars")
option.add_argument("start-maximized")
option.add_argument("--disable-extensions")

# Pass the argument 1 to allow and 2 to block
option.add_experimental_option("prefs", {
    "profile.default_content_setting_values.notifications": 1
})

driver = webdriver.Chrome(options=option)

driver.get(baseUrl)
driver.implicitly_wait(1)
keywords = ['online class']

for keyword in keywords:
    driver.find_element_by_id("header-search-bar").send_keys(keyword)
    driver.find_element_by_id("header-search-bar").send_keys(Keys.RETURN)
    driver.find_elements_by_xpath("//p[text()='r/UIUC']")[0].click()
    driver.find_element_by_id("search-results-time").click()
    driver.find_element_by_xpath("//span[text()='Past Week']").click()
    driver.find_element_by_id("search-results-sort").click()
    driver.find_elements_by_xpath("//text()[.='Top']/ancestor::button[1]")[0].click()
    driver.implicitly_wait(3)

    all_spans = driver.find_elements_by_class_name("SQnoC3ObvgnGjWt90zD9Z._2INHSNB8V5eaWp4P0rY_mE")
    print(all_spans)
    hrefList = []
    for span in all_spans:
        hrefList.append(span.get_attribute('href'))

    x = PrettyTable()
    x.field_names = ['Content', 'Link', 'Reaction']
    for href in hrefList:
        driver.get(href)
        messages = driver.find_elements_by_class_name('_1qeIAgB0cPwnLhDF9XSiJM')
        content = []
        for message in messages:
            content.append(message.text)
        x.add_row([content, href, 'none'])
        print(x)
        break


# author: Ziming Guo
# time: 2020/4/6
'''
    demo01:
    async
'''

import asyncio

# 定义协程函数
async def fun01():
    print("start 1")
    # 用来设置跳转的阻塞点
    await asyncio.sleep(2)
    print("end 1")

async def fun02():
    print("start 2")
    await asyncio.sleep(3)
    print("end 2")

# 即：把上面的两个函数都定义成协程函数，哪个能执行就执行哪个
#   当遇到阻塞的时候，就可以自动跳转
#   也就是说：先执行 fun01，然后当遇到阻塞 await 的时候就跳转执行 fun02
#   然后当 fun01 的两秒等待结束之后，再执行 fun01 的最后一句，然后再跳到 fun02 执行 fun02 的最后一句

# 生成协程对象
cor1 = fun01()
cor2 = fun02()

tasks = [asyncio.ensure_future(cor1),
         asyncio.ensure_future(cor2)]

loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))


import numpy as np
import sys
from random import shuffle
from scipy.misc import imsave

# Determine from a position the next possible steps
def next_possible_steps(position, n, m):
    dpos = [-1, 1]
    steps = []
    for i in dpos:
        if (position[0] + 2 * i < 2*n+1) and (position[0] + 2 * i >= 0):
            steps.append((i, 0))
        if (position[1] + 2 * i < 2*m+1) and (position[1] + 2 * i >= 0):
            steps.append((0, i))
    shuffle(steps)
    return steps

# Create the maze reccursively (this approach doesn't allow large dimensions)
def rec_creation(lab, position, n, m):
    steps = next_possible_steps(position, n, m)
    for step in steps:
        if lab[position[0] + 2 * step[0], position[1] + 2 * step[1]] == 0:
            lab[position[0] + step[0], position[1] + step[1]] = 1
            lab[position[0] + 2 * step[0], position[1] + 2 * step[1]] = 1
            rec_creation(lab, (position[0] + 2*step[0], position[1] + 2*step[1]), n, m)

# Create grid
n = int(sys.argv[1])
m = int(sys.argv[2])
maze = np.zeros((2 * n + 1, 2 * m + 1))
maze[0, 0] = 1
position = (0, 0)
rec_creation(maze, position, n, m)

# Create outside walls
render = np.zeros((2*n+3, 2*m+3))
render[1,0] = 1
render[-2, -1] = 1
render[1:-1, 1:-1] = maze

# Save image
imsave(sys.argv[3], render)


import json
import os

import numpy as np
from scripts.fitting_functions import fit_set
from scripts.plotting_functions import plot_gaussians
from scripts.postprocessing_functions import format_components
from scripts.preprocessor_functions import get_contrasts_from_dir

FILE_DIR = os.path.dirname(__file__)
DATASET_ROOT = os.path.join(FILE_DIR, "..", "Datasets", "GMMDetectorDatasets")
PARAM_ROOT = os.path.join(FILE_DIR, "trained_parameters")
PLOTS_ROOT = os.path.join(FILE_DIR, "plots")

MATERIALS = ["Graphene", "WSe2"]

# define some plotting constants
AXIS_NAMES = ["Blue Contrast", "Green Contrast", "Red Contrast"]
UPPER_BOUNDS = [0, 0, 0]  # B,G,R
LOWER_BOUNDS = [-1, -1, -1]  # B,G,R

# defining material specific constants
PARAMS = {
    "Graphene": {
        "Number of Components": 4,
        "Number of Noise Components": 1,
    },
    "WSe2": {
        "Number of Components": 3,
        "Number of Noise Components": 1,
    },
}

for material in MATERIALS:
    np.random.seed(42)
    data_directory = os.path.join(DATASET_ROOT, material)
    if not os.path.exists(data_directory):
        print(f"Data directory for {material} does not exist. Skipping...")
        continue

    image_directory = os.path.join(data_directory, "train_images")
    mask_directory = os.path.join(data_directory, "train_semantic_masks")

    datapoints_contrast = get_contrasts_from_dir(
        image_directory=image_directory,
        mask_directory=mask_directory,
    )

    # cropping the data so they lie in a certain range
    datapoints_contrast_cropped = datapoints_contrast[
        (datapoints_contrast[:, 0] > LOWER_BOUNDS[0])
        & (datapoints_contrast[:, 0] < UPPER_BOUNDS[0])
        & (datapoints_contrast[:, 1] > LOWER_BOUNDS[1])
        & (datapoints_contrast[:, 1] < UPPER_BOUNDS[1])
        & (datapoints_contrast[:, 2] > LOWER_BOUNDS[2])
        & (datapoints_contrast[:, 2] < UPPER_BOUNDS[2])
    ]

    (
        all_means_gauss,
        all_covariances_gauss,
        all_weights_gauss,
        sampled_data,
        predicted_labels,
    ) = fit_set(
        data=datapoints_contrast_cropped,
        num_components=PARAMS[material]["Number of Components"],
        num_additional_noise_comp=PARAMS[material]["Number of Noise Components"],
        cov_type="full",
        sample_size=30000,
    )

    figures, axes = plot_gaussians(
        data=sampled_data,
        predicted_labels=predicted_labels,
        gauss_means=all_means_gauss,
        gauss_weights=all_weights_gauss,
        gauss_covariances=all_covariances_gauss,
        axis_names=AXIS_NAMES,
        upper_bounds=UPPER_BOUNDS,
        lower_bounds=LOWER_BOUNDS,
        bins=100,
        fig_size=(10, 10),
        plot_type="heatmap",
        heatmap_bins=100,
        heatmap_sigma=1,
    )

    for i, fig in enumerate(figures):
        fig_path = os.path.join(PLOTS_ROOT, f"{material}_GMM_{i}.png")
        fig.savefig(fig_path)

    component_dict = format_components(all_means_gauss, all_covariances_gauss)
    component_path = os.path.join(PARAM_ROOT, f"{material}_GMM.json")
    with open(component_path, "w") as f:
        json.dump(component_dict, f, indent=4, sort_keys=True)


import torch
import torchvision.transforms as transforms
import torch.nn.functional as F

from verification import evaluate

from datetime import datetime
import matplotlib.pyplot as plt

plt.switch_backend('agg')
import numpy as np
from PIL import Image
import bcolz
import io
import os
from tqdm import tqdm
from data.data_pipe import get_test_dataset

def get_time():
    return (str(datetime.now())[:-10]).replace(' ', '-').replace(':', '-')

def l2_norm(input, axis=1):
    norm = torch.norm(input, 2, axis, True)
    output = torch.div(input, norm)

    return output

def make_weights_for_balanced_classes(images, nclasses):
    '''
        Make a vector of weights for each image in the dataset, based
        on class frequency. The returned vector of weights can be used
        to create a WeightedRandomSampler for a DataLoader to have
        class balancing when sampling for a training batch.
            images - torchvisionDataset.imgs
            nclasses - len(torchvisionDataset.classes)
        https://discuss.pytorch.org/t/balanced-sampling-between-classes-with-torchvision-dataloader/2703/3
    '''
    count = [0] * nclasses
    for item in images:
        count[item[1]] += 1  # item is (img-data, label-id)
    weight_per_class = [0.] * nclasses
    N = float(sum(count))  # total number of images
    for i in range(nclasses):
        weight_per_class[i] = N / float(count[i])
    weight = [0] * len(images)
    for idx, val in enumerate(images):
        weight[idx] = weight_per_class[val[1]]

    return weight

def get_val_pair(path, name):
    carray = bcolz.carray(rootdir=os.path.join(path, name), mode='r')
    issame = np.load('{}/{}_list.npy'.format(path, name))

    return carray, issame

def get_val_data(data_path):
    lfw, lfw_issame = get_val_pair(data_path, 'lfw')
    cfp_ff, cfp_ff_issame = get_val_pair(data_path, 'cfp_ff')
    cfp_fp, cfp_fp_issame = get_val_pair(data_path, 'cfp_fp')
    agedb_30, agedb_30_issame = get_val_pair(data_path, 'agedb_30')
    calfw, calfw_issame = get_val_pair(data_path, 'calfw')
    cplfw, cplfw_issame = get_val_pair(data_path, 'cplfw')
    vgg2_fp, vgg2_fp_issame = get_val_pair(data_path, 'vgg2_fp')

    return lfw, cfp_ff, cfp_fp, agedb_30, calfw, cplfw, vgg2_fp, lfw_issame, cfp_ff_issame, cfp_fp_issame, agedb_30_issame, calfw_issame, cplfw_issame, vgg2_fp_issame

def hflip_batch(imgs_tensor):
    hfliped_imgs = torch.empty_like(imgs_tensor)
    for i, img_ten in enumerate(imgs_tensor):
        hfliped_imgs[i] = torch.Tensor((img_ten.cpu().numpy()[:,:,::-1]).copy()).cuda()#.copy()是为了负索引

    return hfliped_imgs

def gen_plot(fpr, tpr):
    """Create a pyplot plot and save to buffer."""
    plt.figure()
    plt.xlabel("FPR", fontsize=14)
    plt.ylabel("TPR", fontsize=14)
    plt.title("ROC Curve", fontsize=14)
    plot = plt.plot(fpr, tpr, linewidth=2)
    buf = io.BytesIO()
    plt.savefig(buf, format='jpeg')
    buf.seek(0)
    plt.close()

    return buf

def perform_val(dataLoader, length,embedding_size, backbone,issame, nrof_folds=10, tta=True):
    idx = 0
    embeddings = np.zeros([length, embedding_size])
    #prefetcher = DataPrefetcher(dataLoader)
    with torch.no_grad():
        begin = 0
        for i,(imgs,labels) in enumerate(tqdm(dataLoader)):
            imgs = imgs.permute(0, 3, 1, 2).float().cuda()
            embedding = backbone(imgs)
            if tta:
                flipImgs = hflip_batch(imgs)
                embedding += backbone(flipImgs)
            embeddings[begin:begin+imgs.shape[0]] = np.copy(l2_norm(embedding).cpu().data.numpy())
            begin = begin+imgs.shape[0]

    tpr, fpr, accuracy, best_thresholds = evaluate(embeddings, issame, nrof_folds)
    return float(accuracy.mean())
def eval_emore_bmk(conf,net,bmk_name):
    test_set = get_test_dataset(conf,bmk_name)
    acc = perform_val(test_set['test_dataset_iter'],test_set['num_class'],512,net,test_set['issame'])
    result = {'acc': acc, 'bmname':bmk_name,'feat_name':'stu_fc1'}
    txt_output = '{:>25s}, {:>15s}, {}'.format(bmk_name, 'stu_fc1',
                                               'acc='+str(acc))
    print(txt_output)
    return acc

def buffer_val(writer, db_name, acc, best_threshold, roc_curve_tensor, epoch):
    writer.add_scalar('{}_Accuracy'.format(db_name), acc, epoch)
    writer.add_scalar('{}_Best_Threshold'.format(db_name), best_threshold, epoch)
    writer.add_image('{}_ROC_Curve'.format(db_name), roc_curve_tensor, epoch)

class AverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].view(-1).float().sum(0)
        res.append(correct_k.mul_(100.0 / batch_size))

    return res