from django.db.models.loading import get_model
from django.core.exceptions import ObjectDoesNotExist,MultipleObjectsReturned
from settings import MEDIA_ROOT,MEDIA_URL
from django import forms
from django.contrib.admin.helpers import ActionForm
from django.contrib import admin
import csv
SIGN2="□"
SIGN1="$$"
class StrForm(ActionForm):
    inputvalue = forms.CharField(label="输入字符串",required=False)
class FieldSet(ActionForm):
    field = forms.CharField(label="字段名",required=False)
    value = forms.CharField(label="字段值",required=False)

def updatefield(modeladmin, request, queryset):
    '''
    更新当前对象的字段值
    '''
    combofield=request.POST["inputvalue"].split("$$")[0]
    value=request.POST["inputvalue"].split("$$")[1]
    for obj in queryset:
        if len(combofield.split(","))==2:
            field1=combofield.split(",")[0]
            field2=combofield.split(",")[1]
            if type(queryset.model._meta.get_field(field1)).__name__ in ["ForeignKey"]:
                try:
                    foreign_obj=queryset.model._meta.get_field(field1).rel.to.objects.get(**{field2:value})
                    setattr(obj,field1,foreign_obj)
                    obj.save()
                except Exception as e:
                    pass
            else:#queryset.model._meta.get_field(field1) in ["ManyToManyField"]:
                try:
                    value_list=value.split(SIGN2)
                    field2=field2+"__in"
                    m2m_list=queryset.model._meta.get_field(field1).rel.to.objects.filter(**{field2:value_list})
                    getattr(obj,field1).clear()
                    for m2mobj in m2m_list:
                        getattr(obj,field1).add(m2mobj)
                except Exception as e:
                    pass
        elif len(combofield.split(","))==1:
            setattr(obj,combofield,value)
        elif len(combofield.split(","))==0:
            pass
updatefield.short_description="更新字段"
def copyitem(modeladmin, request, queryset):
    '''
    对layout节点进行复制
    '''
    for layout_node in queryset:
        layout_node.id=None
        layout_node.save()
copyitem.short_description="复制节点"

def loadM2M(modeladmin, request, queryset):
    '''
    载入m2m关系，但相关模型已经存在数据
    '''
    def get_para(model_tuple):
        '''
        model_tuple 模型的数据tuple(field,value)
        length:字段中的值最多个数
        '''
        length=max([len(value.split(SIGN1)) for key,value in model_tuple])
        para_dict_list=[{} for i in length]
        key_para_dict_list=[{} for i in length]
        for fieldword,fieldvalue in model_tuple:
            value_list=fieldvalue.split(SIGN1)
            if len(value_list)>1:
                for index,value in enumerate(value_list):
                    if "*" in fieldword:
                        key_para_dict_list[index]=key_para_dict_list[index].update({fieldword.replace("*",""):value})
                        para_dict_list[index]=para_dict_list[index].update({fieldword:value})
                    else:
                        para_dict_list[index]=para_dict_list[index].update({fieldword:value})
            else:#如果仅为单个单词
                value=value_list[0]
                for index in length:
                    if "*" in fieldword:
                        key_para_dict_list[index]=key_para_dict_list[index].update({fieldword.replace("*",""):value})
                        para_dict_list[index]=para_dict_list[index].update({fieldword:value})
                    else:
                        para_dict_list[index]=para_dict_list[index].update({fieldword:value})
        return (para_dict_list,key_para_dict_list)
    for obj in queryset:
        model1=get_model(obj.modelname.split('.',1)[0],obj.modelname1.split('.',1)[1])
        model2name=obj.modelname2.split('.',1)[-1]
        model2=get_model(obj.modelname.split('.',1)[0],obj.modelname2.split('.',1)[1])
        field_projection1=[]#模型1映射表(文件字段,数据库字段)
        field_projection2=[]#模型2映射表
        field_list1=[]#模型1导入文件字段列表
        field_list2=[]#模型2导入文件字段列表
        for line in obj.fieldlist1.split("\n"):
            line=line.strip()
            field_list1.append(line.split(":")[0])
            field_projection1.append(dict(line.split(":")))
        for line in obj.fieldlist2.split("\n"):
            line=line.strip()
            field_list2.append(line.split(":")[0])
            field_projection2.append(dict(line.split(":")))
        csvreader=csv.DictReader(open(obj.formatfile.path,"r",encoding="utf8"),delimiter="\t")
        for index, record  in enumerate(csvreader):
            model_tuple1=[(field_projection1[field],record[field])for field in field_list1]
            model_tuple2=[(field_projection2[field],record[field])for field in field_list2]
            para_dict_list1,key_para_dict_list1=get_para(model_tuple1)
            para_dict_list2,key_para_dict_list2=get_para(model_tuple2)
            for i in len(para_dict_list1):
                for j in len(para_dict_list2):
                    modelrecord1=model1.objects.get(**para_dict_list1[i])
                    modelrecord2=model2.objects.get(**para_dict_list2[j])
                    getattr(modelrecord1,model2name).add(modelrecord2)
loadM2M.short_description="导入多对多关系"
def innercmp(item):
    return len(item.split(","))
def field_reduce(model,record_dict):
    '''
    record_dict是模型除了m2m键之外所有键的字典表，key为模型字段名逗号分隔
    该表初始状态是字段引用多个外键,而该程序是对逐次减少外键,直到最终返回一个model内部值的字典
    '''
    field_list=sorted(record_dict.keys(),key=innercmp)#层次越深则在最底层
    if len(field_list[-1].split(","))==1:#如果所有字典key的层次都是一层,则返回该字典表
        return record_dict
    #否则削减最深层的字段
    current_field=",".join(field_list[-1].split(",")[:-1])
    field_list_reducing=[key for key in record_dict if current_field+"," in key]#即将被削减的字段表
    current_model=model
    for field in current_field.split(","):#当前的
        current_model=current_model._meta.get_field(field).rel.to
    #获取被削减字段表的字典数据,作为外键记录的查询或者建立数据
    para_dict={}
    key_para_dict={}
    for fieldword in field_list_reducing:#获取外键表的数据,建立外键表数据字典
        value=record_dict[fieldword]
        field=fieldword.split(",")[-1]
        if value!="":#如果值为空,则忽略
            tempfield=field.strip("*")
            if type(current_model._meta.get_field(tempfield)).__name__ in ["ForeignKey","TreeForeignKey"]:
                para_dict[tempfield]=current_model._meta.get_field(tempfield).rel.to.objects.get(id=int(value))
            else:
                para_dict[tempfield]=value
            if "*" in field:#*表示关键字段
                key_para_dict[tempfield]=para_dict[tempfield]

    modelrecord=None#记录初始化,确保不会误操作以前的数据

    try:#使用外键字典,如果该数据存在则更新并获取id,否则保存外键数据且获取该id,并将该外键id保存到主表字典
        if key_para_dict!={}:
            modelrecord=current_model.objects.get(**key_para_dict)
            current_model.objects.filter(**key_para_dict).update(**para_dict)
        elif para_dict!={}:#期望是不能找到记录
            modelrecord=current_model.objects.get(**para_dict)
        else:#物体为空
            modelrecord=None#外键为空
    except ObjectDoesNotExist:
        try:
            modelrecord=current_model(**para_dict)
            modelrecord.save()
        except Exception as e:
            pass
    except MultipleObjectsReturned:
        print("multiple object get")#外键只能是一个值
    except Exception as e:
        print("loadata,other exception")

    #对record_dict进行削减
    for field in field_list_reducing:
        del record_dict[field]
    #将该获取的外键记录加入到record_dict
    if modelrecord !=None:
        record_dict[current_field]=modelrecord.id
        return field_reduce(model,record_dict)
    else:#如果没有获取合适的外键记录则视current_field 的值允许为空
        record_dict[current_field]=""
        return field_reduce(model,record_dict)
def m2m_para(model_tuple):
    '''
    model_tuple 模型的数据tuple(field,value)：field数据是模型的m2m字段加上引用的对象的对应字段名
    length:字段中的值最多个数
    '''
    m2m_model_dict={}
    m2mfield=set([field.split(",")[0] for field,value in model_tuple])
    for field in m2mfield:
        filter_list=[(combofield,value) for combofield,value in model_tuple if field==combofield.split(",")[0]]
        length=max([len(value.split(SIGN1)) for key,value in filter_list])
        para_dict_list=[{} for i in range(length)]
        key_para_dict_list=[{} for i in range(length)]
        for combofield,fieldvalue in filter_list:
            fieldword=combofield.split(",")[-1]
            value_list=fieldvalue.split(SIGN1)
            if len(value_list)>1:
                for index,value in enumerate(value_list):
                    if "*" in fieldword:
                        key_para_dict_list[index].update({fieldword.replace("*",""):value})
                        para_dict_list[index].update({fieldword.replace("*",""):value})
                    else:
                        para_dict_list[index].update({fieldword:value})
            else:#如果仅为单个单词
                value=value_list[0]
                for index in range(length):
                    if "*" in fieldword:
                        key_para_dict_list[index].update({fieldword.replace("*",""):value})
                        para_dict_list[index].update({fieldword.replace("*",""):value})
                    else:
                        para_dict_list[index]=para_dict_list[index].update({fieldword:value})
        m2m_model_dict[field.replace("*","")]=para_dict_list
    return m2m_model_dict

def recordload(model,field_projection,record_dict):
    '''
    已废弃
    :param model:该模型
    :param field_projection: 字段映射
    :param record_dict: 该模型的记录字典数据
    :return:
    '''

    field_level=field_projection[0][1].split(",")
    if len(field_level)>1:
        foreign_field=field_level[-2]
        filter_word=",".join(field_level[:-1])
        filter_list=[x for x in field_projection if filter_word in x[1]]
        field_projection=[x for x in field_projection if filter_word not in x[1]]
        current_model=model
        for field in field_level:#找到最后的两个字段名
            current_model=current_model._meta.get_field(field).rel.to
            if field==foreign_field:break
        para_dict={}
        key_para_dict={}
        for fieldword in filter_list:#获取外键表的数据,建立外键表数据字典
            value=record_dict[fieldword[0]]
            field=fieldword[1].split(",")[-1]
            if value!="":#如果值为空,则忽略
                #para_list.append(field+"="+value)
                if "*" in field:#*表示关键字段
                    field=field.replace("*","")
                    key_para_dict[field]=value
                    para_dict[field]=value
                else:
                    para_dict[field]=value
        modelrecord=None
        try:#使用外键字典,如果该数据存在则更新并获取id,否则保存外键数据且获取该id,并将该外键id保存到主表字典
            if key_para_dict!={}:
                modelrecord=current_model.objects.get(**key_para_dict)
                current_model.objects.filter(**key_para_dict).update(**para_dict)
            else:#期望是不能找到记录
                modelrecord=current_model.objects.get(**para_dict)
        except ObjectDoesNotExist:
            modelrecord=current_model(**para_dict)
            modelrecord.save()
        except MultipleObjectsReturned:
            print("multiple object get")
        except Exception as e:
            print("loadata,other exception")
            #modelrecord=current_model(",".join(para_list))
        field_projection.append([filter_word,filter_word])
        if modelrecord !=None:
            record_dict[filter_word]=modelrecord.id
            field_projection.sort(key=innercmp)
            recordload(model,field_projection,record_dict)
        else:
            pass
    else:#所有数据的level都是为1
        para_dict={}
        key_para_dict={}
        m2m_dict={}
        for fieldword in field_projection:
            #获取主表的数据,建立主表数据字典
            value=record_dict[fieldword[0]]
            field=fieldword[-1]
            if type(model._meta.get_field(field)).__name__ in ["ForeignKey","TreeForeignKey"] :
                value=model._meta.get_field(field).rel.to.objects.get(id=value)
            if value!="":
                if "*" in field:#*表示关键字段
                    field=field.replace("*","")
                    key_para_dict[field]=value
                    para_dict[field]=value
                else:
                    para_dict[field]=value
        modelrecord=None
        try:#使用主表字典,如果该数据存在则更新数据,否则新建数据并保存
            if key_para_dict!={}:
                modelrecord=model.objects.get(**key_para_dict)
                model.objects.filter(**key_para_dict).update(**para_dict)
            else:#期望是不能找到记录
                modelrecord=model.objects.get(**para_dict)
        except ObjectDoesNotExist:
            modelrecord=model(**para_dict)
            modelrecord.save()
        except MultipleObjectsReturned:
            print("multiple object get")
        except Exception as e:
            print("loadata,other exception")
        if modelrecord!=None:#针对ManyToManyField字段,添加该关系
            pass

def loadModel(modeladmin, request, queryset):
    '''
    针对loadata.LoadSubject 执行该指令
    将对modelname(模块名.模型名)执行从外部文件的导入工作
    fieldlist:格式为 每个字段占一行，每个字段 为 字段名,外键字段名,外外键字段名.....(如果字段标记为*,则表示该字段为数据表关键字)
    该程序检查如果一行具有多个字段,自动将外键数据输入到外键表格.
    程序将文件中的数据存储到数据库当中。
    '''

    for obj in queryset:
        model=get_model(obj.modelname.split('.',1)[0],obj.modelname.split('.',1)[1])
        field_projection=[]
        field_list=[]
        m2m_list=[]
        if obj.fieldlist!="":
            for line in obj.fieldlist.split("\n"):
                line=line.strip()
                field_list.append(line.split(":")[0])
                field_projection.append(tuple(line.split(":")))
        if obj.fieldm2m!="":
            for line in obj.fieldm2m.split("\n"):
                line=line.strip()
                m2m_list.append(line.split(":")[0])
                field_projection.append(tuple(line.split(":")))
        field_projection=dict(field_projection)
        csvreader=csv.DictReader(open(obj.formatfile.path,"r",encoding="utf8"),delimiter="\t")
        first_line=True
        for index, record  in enumerate(csvreader):
            if first_line:
                field_list=[field for field in field_list if field in record]
                m2m_list=[field for field in m2m_list if field in record]
            record_dict=dict([(field_projection[field],record[field]) for field in field_list])
            m2m_tuple=[(field_projection[field],record[field].strip().strip(SIGN1))for field in m2m_list]
            new_record_dict=field_reduce(model,record_dict)
            key_para_dict={}
            para_dict={}
            for field in new_record_dict:
                if new_record_dict[field]=="":continue
                tempfield=field.replace("*","")
                if type(model._meta.get_field(tempfield)).__name__ in ["ForeignKey","TreeForeignKey"]:
                    para_dict[tempfield]=model._meta.get_field(tempfield).rel.to.objects.get(id=int(new_record_dict[field]))
                else:
                    para_dict[tempfield]=new_record_dict[field]
                if "*" in field:
                    key_para_dict[tempfield]=para_dict[tempfield]

            modelrecord=None
            try:#使用主表字典,如果该数据存在则更新数据,否则新建数据并保存
                if key_para_dict!={}:
                    modelrecord=model.objects.get(**key_para_dict)
                    model.objects.filter(**key_para_dict).update(**para_dict)
                else:#期望是不能找到记录,如果找到已经存在一模一样记录则不作任何改变
                    modelrecord=model.objects.get(**para_dict)
            except ObjectDoesNotExist:
                modelrecord=model(**para_dict)
                try:
                    modelrecord.save()
                except Exception as e:
                    pass
            except MultipleObjectsReturned:
                print("multiple object get")
            except Exception as e:
                print("loadata,other exception")
            if modelrecord!=None and m2m_tuple!=[]:#针对ManyToManyField字段,添加该关系
                m2m_model_dict=m2m_para(m2m_tuple)
                for m2mfield in m2m_model_dict:
                    try:
                        m2m_model=model._meta.get_field(m2mfield).rel.to
                    except:
                        pass
                    for m2mPara in m2m_model_dict[m2mfield]:
                        try:
                            m2mRec=m2m_model.objects.get(**m2mPara)
                            getattr(modelrecord,m2mfield).add(m2mRec)
                        except MultipleObjectsReturned:
                            pass
                        except ObjectDoesNotExist:
                            pass
                        except Exception as e:
                            pass
loadModel.short_description="导入模型数据"

def outputModel(modeladmin, request, queryset):
    '''
    对loadata设置的模型进行输出
    '''
    for format_obj in queryset:
        model=get_model(format_obj.modelname.split('.',1)[0],format_obj.modelname.split('.',1)[1])
        field_projection=[]
        field_list=[]
        m2m_list=[]
        if format_obj.fieldlist!="":
            for line in format_obj.fieldlist.split("\n"):
                line=line.strip()
                field_list.append(line.split(":")[0])
                field_projection.append(tuple(line.split(":")))
        if format_obj.fieldm2m!="":
            for line in format_obj.fieldm2m.split("\n"):
                line=line.strip()
                m2m_list.append(line.split(":")[0])
                field_projection.append(tuple(line.split(":")))
        field_projection=dict(field_projection)
        foutput=open(format_obj.formatfile.path,"w")
        foutput.write("\t".join(field_list)+"\t".join(m2m_list))
        foutput.write("\n")
        obj_list=model.objects.all()
        for obj in obj_list:
            record=[]
            for outputfield in field_list:
                combofield=field_projection[outputfield]
                current_obj=obj
                for field in combofield.split(","):
                    current_obj=getattr(current_obj,field.replace("*",""))
                    if current_obj==None:
                        current_obj=" "
                        break
                record.append((outputfield,str(current_obj)))
            for outputfield in m2m_list:
                combofield=field_projection[outputfield]
                m2mfield=combofield.split(",")[0].replace("*","")
                field=combofield.split(",")[1].replace("*","")
                m2mobj_list=getattr(obj,m2mfield).all()
                m2mvalue=SIGN1.join([getattr(m2mobj,field) for m2mobj in m2mobj_list])
                record.append((outputfield,m2mvalue))
            foutput.write("\t".join([value for field,value in record ]))
            foutput.write("\n")
outputModel.short_description="导出模型数据"

def outputitem(modeladmin, request, queryset):
    '''
    对任意模型中的选中节点数据进行输出,输出包括涉及的foreign 字段.将foreign字段的相关实体字段输出
    '''
    field_list=[]
    value_list=[]
    for field in queryset.model._meta.fields:
        if type(field).__name__=="ForeignKey" :
            for foreign_field in field.rel.to._meta.fields:
                if type(foreign_field).__name__!="ForeignKey" and type(foreign_field).__name__!="ManyToManyField":
                    fieldname=field.name+"$"+foreign_field.name
                    field_list.append(fieldname)
        elif  type(field).__name__=="ManyToManyField":
            pass
        else:
            field_list.append(field.name)
    for node in queryset:
        value=[]
        for field in field_list:
            if "$" in field:
                str1=field.split("$")[0]
                str2=field.split("$")[1]
                foreign=getattr(node,str1)
                if getattr(foreign,str2)==None:value.append(" ")
                else:value.append(getattr(foreign,str2))
            else:
                if getattr(node,field)==None:value.append(" ")
                else:value.append(getattr(node,field))
        value_list.append(value)
    foutput=open(MEDIA_ROOT+"/output/"+queryset.model.__name__+".txt","w")
    foutput.write(SIGN2.join(field_list))
    for index in range(len(value_list)):
        foutput.write(SIGN2.join([word.replace("\n","●") for word in value_list[index]]))
outputitem.short_description="导出数据"

def itemlabel(modeladmin, request, queryset):
    '''
    标记对象
    '''
    labels=request.post["x_field"]
    for obj in queryset:
        obj.tags.add(labels)
itemlabel.short_description="标记数据"
admin.site.add_action(copyitem)
#admin.site.add_action(itemlabel)
#admin.site.add_action(updatefield)
#admin.site.add_action(translatefield)
#admin.site.add_action(contentchange)


# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

import pynini
from nemo_text_processing.text_normalization.en.graph_utils import (
    NEMO_WHITE_SPACE,
    GraphFst,
    delete_extra_space,
    delete_space,
    generator_main,
)
from nemo_text_processing.text_normalization.en.taggers.punctuation import PunctuationFst
from nemo_text_processing.text_normalization.en.taggers.whitelist import WhiteListFst
from nemo_text_processing.text_normalization.en.taggers.word import WordFst
from pynini.lib import pynutil

from nemo.utils import logging

class ClassifyFst(GraphFst):
    """
    Final class that composes all other classification grammars. This class can process an entire sentence including punctuation.
    For deployment, this grammar will be compiled and exported to OpenFst Finate State Archiv (FAR) File.
    More details to deployment at NeMo/tools/text_processing_deployment.

    Args:
        input_case: accepting either "lower_cased" or "cased" input.
        deterministic: if True will provide a single transduction option,
            for False multiple options (used for audio-based normalization)
        cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache.
        overwrite_cache: set to True to overwrite .far files
        whitelist: path to a file with whitelist replacements
    """

    def __init__(
        self,
        input_case: str,
        cache_dir: str = None,
        overwrite_cache: bool = False,
        deterministic: bool = True,
        whitelist: str = None,
    ):
        super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic)

        far_file = None
        if cache_dir is not None and cache_dir != "None":
            os.makedirs(cache_dir, exist_ok=True)
            whitelist_file = os.path.basename(whitelist) if whitelist else ""
            far_file = os.path.join(
                cache_dir, f"_{input_case}_en_tn_{deterministic}_deterministic{whitelist_file}.far"
            )
        if not overwrite_cache and far_file and os.path.exists(far_file):
            self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"]
            logging.info(f'ClassifyFst.fst was restored from {far_file}.')
        else:
            logging.info(f"Creating ClassifyFst grammars.")

            punctuation = PunctuationFst(deterministic=deterministic)
            punct_graph = punctuation.fst
            word_graph = WordFst(deterministic=deterministic, punctuation=punctuation).fst
            whitelist_graph = WhiteListFst(input_case=input_case, deterministic=deterministic).fst

            classify = pynutil.add_weight(whitelist_graph, 1) | pynutil.add_weight(word_graph, 100)

            punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }")
            punct = pynini.closure(
                pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space)
                | (pynutil.insert(" ") + punct),
                1,
            )
            token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
            token_plus_punct = (
                pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct)
            )

            graph = (
                token_plus_punct
                + pynini.closure(
                    (
                        pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space)
                        | (pynutil.insert(" ") + punct + pynutil.insert(" "))
                    )
                    + token_plus_punct
                ).optimize()
            )

            graph = delete_space + graph + delete_space
            graph |= punct

            self.fst = graph.optimize()

            if far_file:
                generator_main(far_file, {"tokenize_and_classify": self.fst})
                logging.info(f"ClassifyFst grammars are saved to {far_file}.")


import boto3
import botocore

import json
import urllib2
import re

#Sample event for local testing. Remember to COMMENT OUT before committing
event = {"Records": [{"eventID": "e0993e3697aa2984eba30dfc57f1731c", "eventVersion": "1.0", "dynamodb": {"Keys": {"FeedName": {"S": "nvdcve-2.0-Modified"}}, "SizeBytes": 27, "StreamViewType": "KEYS_ONLY", "SequenceNumber": "3249000000000000791629016"}, "awsRegion": "us-east-1", "eventName": "MODIFY", "eventSourceARN": "arn:aws:dynamodb:us-east-1:459772930601:table/DEV_vulnscape_nvd_feed_list/stream/2015-12-02T14:15:49.196", "eventSource": "aws:dynamodb"}]}

'''Determine the context in which this function is operating - DEV, BETA, or PROD'''
def determine_op_env(event):
    #Determine and capture context
    if event["Records"][0]["eventSource"] == "aws:dynamodb":
        result = re.search("arn:aws:dynamodb:us-.*:[0-9]*:table/([A-Z]{1,10})_.*", event["Records"][0]["eventSourceARN"])
        return result.group(1)

    else:
        print("Unexpected error.")
        raise SystemExit

def check_bucket(s3, bucket_name, logger):
    try:
        bucket_status = s3.Bucket(bucket_name).creation_date
        print(bucket_status)
        return "ACTIVE"
    except botocore.exceptions.ClientError as e:
        if e.response['Error']['Code'] == 'NotFound':
            logger.warning("S3 Bucket does not exist.")
            return "CREATE"
        else:
            logger.critical("Unexpected error: %s" % e)
            raise SystemExit

def create_s3_bucket(op_env):

    bucket = s3.create_bucket(
    ACL='private',
    Bucket='DEV_NvdFeedGzips'
    )

'''The main function, with optional variables'''
def main(event=None, context=None):

    #Determine event operating environment (e.g., DEV, PROD, BETA)
    op_env = determine_op_env(event)
    #print(op_env)

    #Enable logging, with context
    logger = logging_handler(op_env)

    #Write status update to log
    logger.info("Main function loading.")

    #Connect to S3
    s3 = boto3.resource('s3')

    #Establish appropriate name for Bucket
    bucket_name = "%s_NvdFeedGzips" % op_env

    #Check status of DynamoDB table
    bucket_situation = check_bucket(s3, bucket_name, logger)
    print(bucket_situation)

    #React appropriately to status of DynamoDB table, i.e., to continue or to create a new table
    if bucket_situation == "ACTIVE":
        logger.info("S3 Bucket exists and is ACTIVE.")
        pass
    elif bucket_situation == "CREATE":
        logger.info("Creating new S3 Bucket")
        #create_table(dynamodb, table_name)
    else:
        logger.critical("Bucket status is indeterminate.")

    #create_s3_bucket(op_env)

# Main Program
if __name__ == "__main__":
	main(event)


from PyQt5.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QLabel, QPushButton, QSizePolicy, QGridLayout
from PyQt5.QtCore import QRect, pyqtSlot, pyqtSignal
from QtImageViewer import QtImageViewer
import os
from functools import partial

# This widget is responsible for rendering the page where waypoints can be loaded from save files.
class PrevFilesWidget(QWidget):
	changeWidgetSignal = pyqtSignal(int)
	chosenSaveFileSignal = pyqtSignal(str)
	selectTifSignal = pyqtSignal(str,str)
	def __init__(self):
		super().__init__()
		self.savespath = os.fspath('../../saves')
		self.initUI()

	def initUI(self):
		self.l = QLabel("PrevFilesWidget")
		self.btn = QPushButton("< Back",self)

		self.qbly = QVBoxLayout()
		self.qbly.addWidget(self.btn,1)
		self.btn.clicked.connect(self.on_back_clicked)

		self.itemvlist = QVBoxLayout()
		self.createLoadButtons()
		self.setLayout(self.itemvlist)

	def createLoadButtons(self):
		self.tiflist = {}

		# Go through the saves folder and make a button for each one for loading
		# the tif along with its saved waypoints
		if os.path.exists(self.savespath):
			dirlist = os.listdir( self.savespath )
			for item in dirlist:
				if item[0] != '.':
					if item[-4:] == ".txt":
						itemhlist = QHBoxLayout()
						itembtn = QPushButton(item[:-4],self)
						fline = None
						with open( os.fspath(self.savespath + "/" + item) ) as f:
							fline = f.readline().rstrip()
						itembtn.clicked.connect( partial( self.item_button_clicked, fline ) )
						itemhlist.addWidget(itembtn)
						self.tiflist[item] = itembtn
						self.itemvlist.addLayout(itemhlist)

	def getLocations(self, save_name):
		coords = []
		save_path = self.savespath + '/' + save_name + ".txt"
		try:
			with open( os.fspath(save_path) ) as save:
				lines = [ line.rstrip('\n') for line in save ]
				fline = lines[0]
				for i in range(1, len(lines)):
					line = lines[i]
					comma = line.find(',')
					x = line[:comma]
					y = line[comma+1:]
					coords.append( (x, y) )
		except (OSError, IOError, FileNotFoundError) as e:
			return [ "error" ]
		return coords

	@pyqtSlot(str)
	def item_button_clicked(self,itemName):
		self.selectTifSignal.emit(itemName, 'prevfiles')

	# When the back button is clicked, go back to the MainWidget
	@pyqtSlot()
	def on_back_clicked(self):
		self.changeWidgetSignal.emit(0)

	# When user picks a savefile, its path/save filename will be set as a local and passed with this to the overlay.
	@pyqtSlot()
	def on_savefile_chosen(self):
		self.selectTifSignal.emit(self.saveChoice, "prevfiles")


import numpy as np
from typing import Tuple, Optional, Union, Set, Dict, Any
from fractions import Fraction
import random
import pygame
import sys
import os
import json

# from misc import *
from typing import List, Tuple, Union, Dict
from genetic_algorithm.individual import Individual
from neural_network import FeedForwardNetwork, linear, sigmoid, tanh, relu, leaky_relu, ActivationFunction, get_activation_by_name
from settings import settings

BLACK = (0,0,0)
WHITE = (255,255,255)
RED = (255,0,0)
GREEN = (0,180,0)
BLUE = (50,200,255)

class Bird(Individual):
    def __init__(self,
                chromosome: Optional[Dict[str, List[np.ndarray]]] = None,
                x_pos: Optional[int] = settings['init_bird_x_pos'],
                y_pos: Optional[int] = settings['init_bird_y_pos'],
                # x_pos: Optional[int] = 100,
                # y_pos: Optional[int] = 250,
                y_speed: Optional[int] = 0,
                hidden_layer_architecture: Optional[List[int]] = [12, 20],
                hidden_activation: Optional[ActivationFunction] = 'relu',
                output_activation: Optional[ActivationFunction] = 'sigmoid'
                ):

        self.Window_Width = settings['Window_Width']
        self.Window_Height = settings['Window_Height']

        self.x_pos = x_pos
        self.y_pos = y_pos
        self.y_speed = y_speed

        self.bird_surface = pygame.image.load("assets/bluebird-midflap.png").convert_alpha()
        self.bird_surface = pygame.transform.scale2x(self.bird_surface)
        self.bird_rect = self.bird_surface.get_rect(center = (self.x_pos, self.y_pos))

        self._fitness = 0  # Overall fitness
        self.score = 0
        self.x_distance_to_next_pipe_center = 0
        self.y_distance_to_next_pipe_center = 0
        self.is_alive = True

        self.hidden_layer_architecture = hidden_layer_architecture
        self.hidden_activation = hidden_activation
        self.output_activation = output_activation

        # Setting up network architecture
        num_inputs = 5 #@TODO: how many
        self.network_architecture = [num_inputs]                          # Inputs
        self.network_architecture.extend(self.hidden_layer_architecture)  # Hidden layers
        self.network_architecture.append(2)                               # 2 output, jump or not jump
        self.network = FeedForwardNetwork(self.network_architecture,
                                          get_activation_by_name(self.hidden_activation),
                                          get_activation_by_name(self.output_activation)
        )

        # If chromosome is set, take it
        if chromosome:
            # self._chromosome = chromosome
            self.network.params = chromosome
            # self.decode_chromosome()
        else:
            # self._chromosome = {}
            # self.encode_chromosome()
            pass

    @property
    def fitness(self):
        return self._fitness

    def calculate_fitness(self):
        # Give positive minimum fitness for roulette wheel selection
        # self._fitness = (self._frames) + ((2**self.score) + (self.score**2.1)*500) - (((.25 * self._frames)**1.3) * (self.score**1.2))
        # self._fitness = (self._frames) + ((2**self.score) + (self.score**2.1)*500) - (((.25 * self._frames)) * (self.score))
        print(self.y_distance_to_next_pipe_center)
        self._fitness = (2 ** self.score + self.score * 2.1) * 200 + 1000 - (self.x_distance_to_next_pipe_center + abs(self.y_distance_to_next_pipe_center)) * 1.2
        self._fitness = max(self._fitness, .1)

    @property
    def chromosome(self):
        # return self._chromosome
        pass

    def encode_chromosome(self):
        # # L = len(self.network.params) // 2
        # L = len(self.network.layer_nodes)
        # # Encode weights and bias
        # for layer in range(1, L):
        #     l = str(layer)
        #     self._chromosome['W' + l] = self.network.params['W' + l].flatten()
        #     self._chromosome['b' + l] = self.network.params['b' + l].flatten()
        pass

    def decode_chromosome(self):
        # # L = len(self.network.params) // 2
        # L = len(self.network.layer_nodes)
        # # Decode weights and bias
        # for layer in range(1, L):
        #     l = str(layer)
        #     w_shape = (self.network_architecture[layer], self.network_architecture[layer-1])
        #     b_shape = (self.network_architecture[layer], 1)
        #     self.network.params['W' + l] = self._chromosome['W' + l].reshape(w_shape)
        #     self.network.params['b' + l] = self._chromosome['b' + l].reshape(b_shape)
        pass

    # restart the game for this individual
    def reset(self):
        self._fitness = 0
        self.score = 0
        self.x_distance_to_next_pipe_center = 0
        self.y_distance_to_next_pipe_center = 0
        self.x_pos = settings['init_bird_x_pos']
        self.y_pos = settings['init_bird_y_pos']
        self.y_speed = 0
        self.is_alive = True

    def update(self, inputs):
        self.network.feed_forward(inputs)
        if self.network.out == 0:
            # do nothing
            pass
        elif self.network.out == 1:
            # jump
            self.y_speed = settings['bird_jump_speed']

        return True

    def move(self, pipes):
        self.y_pos += self.y_speed
        self.y_speed += settings['gravity']
        self.score += 2/1.2/60
        self.check_collision(pipes)
        # self.distance_travelled += abs(self.xspeed)
        # if self.x_pos < 0:
        #     self.x_pos = 0
        # elif self.x_pos > self.board_size[0]-100:
        #     self.x_pos = self.board_size[0]-100

    def check_collision(self, pipes):
        for pipe in pipes:
            if self.bird_rect.colliderect(pipe):
                self.is_alive = False
        if self.bird_rect.top <= -100 or self.bird_rect.bottom >= self.Window_Height - 100:
            self.is_alive = False

    def rotate_bird(self, bird_surface):
        new_bird = pygame.transform.rotozoom(bird_surface, self.y_speed * -3, 1)
        return new_bird

    # Draw the bird
    def draw(self, screen, winner=False, champion=False):
        if not self.is_alive:
            return
        if champion:
            self.bird_surface = pygame.image.load("assets/redbird-midflap.png").convert_alpha()
            self.bird_surface = pygame.transform.scale2x(self.bird_surface)
        elif winner:
            self.bird_surface = pygame.image.load("assets/yellowbird-midflap.png").convert_alpha()
            self.bird_surface = pygame.transform.scale2x(self.bird_surface)

        self.bird_rect.centery = self.y_pos
        rotated_bird_surface = self.rotate_bird(self.bird_surface)
        screen.blit(rotated_bird_surface, self.bird_rect)
        # TODO: include different color for champion and winner
        # if champion:
        #     pygame.draw.rect(screen,BLACK,[self.x_pos,self.y_pos,100,20])
        #     pygame.draw.rect(screen,GREEN,[self.x_pos+2,self.y_pos+2,100-4,20-4])
        # elif winner:
        #     pygame.draw.rect(screen,BLACK,[self.x_pos,self.y_pos,100,20])
        #     pygame.draw.rect(screen,BLUE,[self.x_pos+2,self.y_pos+2,100-4,20-4])
        # else:
        #     pygame.draw.rect(screen,BLACK,[self.x_pos,self.y_pos,100,20])
        #     pygame.draw.rect(screen,WHITE,[self.x_pos+2,self.y_pos+2,100-4,20-4])

def save_bird(population_folder: str, individual_name: str, bird: Bird, settings: Dict[str, Any]) -> None:
    # Make population folder if it doesn't exist
    if not os.path.exists(population_folder):
        os.makedirs(population_folder)

    # Save off settings
    if 'settings.json' not in os.listdir(population_folder):
        f = os.path.join(population_folder, 'settings.json')
        with open(f, 'w', encoding='utf-8') as out:
            json.dump(settings, out, sort_keys=True, indent=4)

    # Make directory for the individual
    individual_dir = os.path.join(population_folder, individual_name)
    if individual_name in os.listdir(population_folder):
        shutil.rmtree(individual_dir)
    os.makedirs(individual_dir)

    # Save some constructor information for replay
    # @NOTE: No need to save chromosome since that is saved as .npy
    # @NOTE: No need to save board_size or hidden_layer_architecture
    #        since these are taken from settings
    # constructor = {}
    # constructor['start_pos'] = snake.start_pos.to_dict()
    # constructor['apple_seed'] = snake.apple_seed
    # constructor['initial_velocity'] = snake.initial_velocity
    # constructor['starting_direction'] = snake.starting_direction
    # snake_constructor_file = os.path.join(individual_dir, 'constructor_params.json')

    # # Save
    # with open(snake_constructor_file, 'w', encoding='utf-8') as out:
    #     json.dump(constructor, out, sort_keys=True, indent=4)

    L = len(bird.network.layer_nodes)
    for l in range(1, L):
        w_name = 'W' + str(l)
        b_name = 'b' + str(l)

        weights = bird.network.params[w_name]
        bias = bird.network.params[b_name]

        np.save(os.path.join(individual_dir, w_name), weights)
        np.save(os.path.join(individual_dir, b_name), bias)

# def load_snake(population_folder: str, individual_name: str, settings: Optional[Union[Dict[str, Any], str]] = None) -> Snake:
#     if not settings:
#         f = os.path.join(population_folder, 'settings.json')
#         if not os.path.exists(f):
#             raise Exception("settings needs to be passed as an argument if 'settings.json' does not exist under population folder")

#         with open(f, 'r', encoding='utf-8') as fp:
#             settings = json.load(fp)

#     elif isinstance(settings, dict):
#         settings = settings

#     elif isinstance(settings, str):
#         filepath = settings
#         with open(filepath, 'r', encoding='utf-8') as fp:
#             settings = json.load(fp)

#     params = {}
#     for fname in os.listdir(os.path.join(population_folder, individual_name)):
#         extension = fname.rsplit('.npy', 1)
#         if len(extension) == 2:
#             param = extension[0]
#             params[param] = np.load(os.path.join(population_folder, individual_name, fname))
#         else:
#             continue

#     # Load constructor params for the specific snake
#     constructor_params = {}
#     snake_constructor_file = os.path.join(population_folder, individual_name, 'constructor_params.json')
#     with open(snake_constructor_file, 'r', encoding='utf-8') as fp:
#         constructor_params = json.load(fp)

#     snake = Snake(settings['board_size'], chromosome=params,
#                   start_pos=Point.from_dict(constructor_params['start_pos']),
#                   apple_seed=constructor_params['apple_seed'],
#                   initial_velocity=constructor_params['initial_velocity'],
#                   starting_direction=constructor_params['starting_direction'],
#                   hidden_layer_architecture=settings['hidden_network_architecture'],
#                   hidden_activation=settings['hidden_layer_activation'],
#                   output_activation=settings['output_layer_activation'],
#                   lifespan=settings['lifespan'],
#                   apple_and_self_vision=settings['apple_and_self_vision']
#                   )
#     return snake


import json
import numpy as np
import re
import string
from nltk.corpus import stopwords
from collections import Counter
from collections import defaultdict
from datetime import datetime, timedelta
from email.utils import parsedate_tz
import os
import bz2
import bz2file
import time
import scipy.sparse as sps
import pandas as pd
import networkx as nx
import matplotlib.pyplot as plt
from scipy.special import comb
from tqdm import tqdm
import collections

results = re.compile(r'https://[a-zA-Z0-9.?/&=:]*|http://[a-zA-Z0-9.?/&=:]*',re.S)

#read stop words
f = open('stopwords.txt','r')
stop = []
for line in f:
    stop.append(line[:-1])
f.close()

regex_str = [
    r'<[^>]+>', # HTML tags
    r'(?:@[\w_]+)', # @-mentions
    r"(?:\#+[\w_]+[\w\'_\-]*[\w_]+)", # hash-tags
    r'http[s]?://(?:[a-z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-f][0-9a-f]))+', # URLs
    r'(?:(?:\d+,?)+(?:\.?\d+)?)', # numbers
    r"(?:[a-z][a-z'\-_]+[a-z])", # words with - and '
    r'(?:[\w_]+)', # other words
    r'(?:\S)' # anything else
]

tokens_re = re.compile(r'('+'|'.join(regex_str)+')', re.VERBOSE | re.IGNORECASE)

def preprocess(s):
    '''
    tokenize the sentence
    '''
    tokens = tokens_re.findall(s)
    return tokens

def to_datetime(datestring):
    '''
    change a string to datatime format
    '''
    time_tuple = parsedate_tz(datestring.strip())
    dt = datetime(*time_tuple[:6])
    return dt - timedelta(seconds=time_tuple[-1])

def read_bz2(inpath, keyword, word_igonre):
    '''
    read a single bz2 file

    input:
    inpath: the location of the path
    keyword: the entity that you want to filter
    word_igonre: ignore the tweets containing one of the words in word_ignore

    output:
    terms_stop: the list of tokenized tweets after removing stop words, the element is a tweet after tokenization
    dates_: the published time of the tweets
    whole_tweets: the corresponding original tweets
    durre: the time for reading the file
    '''
    infile = bz2.BZ2File(inpath, 'r')
    terms_stop = []
    dates_ = []
    whole_tweets = []
    begin = time.time()
    num_tweet = 0
    for line in infile:
        try:
            tweet = json.loads(line)
        except ValueError:
            continue
        num_tweet = num_tweet + 1
        if 'text' in tweet:
            text = tweet["text"].lower()
            if keyword.lower() not in text:
                continue
            if 'http' in text or 'https' in text:
                continue
            train = results.sub(r'',text)
            train = re.sub('[^A-Za-z#@0-9]+', ' ', train)
            x = [term.lower() for term in preprocess(train)
                               if term.lower() not in stop and not term.startswith(('#', '@'))
                              and not term.isdigit()
                              and len(term)>2]
            flag = 0
            for word in word_igonre:
                if word in x:
                    flag =1
                    break
            if flag == 1:
                continue
            if len(x) < 4:
                continue
            terms_stop.append(x)
            whole_tweets.append(tweet["text"])
            dates_.append(to_datetime(tweet['created_at']))
    end_time =  time.time()
    duree = end_time - begin
    infile.close()
    return terms_stop, dates_, whole_tweets, num_tweet, duree

def read_folder(path, keyword,list_igore, num = False):
    '''
    This is similar to read_bz2. Read all files in a folder
    '''
    files= os.listdir(path)
    if num == False or num > len(files):
        num = len(files)
    terms_stop = []
    dates = []
    whole_tweets = []
    duree = 0
    num_tweet = 0

    for fname in tqdm(range(num)):
        fname = path + "/" + files[fname]
        terms_stop_, dates_, whole_tweets_, num_tweet_, duree_ = read_bz2(fname, keyword, list_igore)
        terms_stop.extend(terms_stop_)
        dates.extend(dates_)
        whole_tweets.extend(whole_tweets_)
        duree = duree + duree_
        num_tweet = num_tweet + num_tweet_
    return terms_stop, dates, whole_tweets, num_tweet, duree

def find_burst(num, accumulated_num, terms_stop, threshold, first_n = False):
    '''
    Given the numbers of tweets for every period, find the burst

    input:
    num:the numbers of tweets for every period by funtion split_by_time
    accumulated_num: accumulated numbers until this period by funtion split_by_time
    terms_stop: the list of tokenized tweets from read_bz2 function
    threshold: only detect burst beyond this threshold, it is multiplied by the average value

    output:
    ind_burst: a dictionnary indicating the burst burst and the burst end
    list_count: a list indicating all words in a given burst
    '''
    ind_burst = search_maxima(num, threshold, first_n)
    begin_burst = ind_burst.keys()
    end_burst = ind_burst.values()
    list_count = []
    for i in range(len(begin_burst)):
        count_all = Counter()
        for j in range(accumulated_num[begin_burst[i]], accumulated_num[end_burst[i]+1]):
            count_all.update(terms_stop[j])
        list_count.append(count_all)

    return ind_burst, list_count

def search_maxima(value_list, threshold,  first_n = False):
    '''
    find local maxima which is bigger than mean value * threshold.
    the neighbors of a maximum is also considred to be in the burst if their difference is very small

    input:
    value_list: the numbers for a period
    threshold: the burst below mean value * threshold is ignored
    first_n: Only find the first N maxima

    output:
    bursts: a dictionnary indicating the burst burst and the burst end
    '''

    threshold = np.mean(value_list) * threshold
    len_ = len(value_list)
    local_max = []

    for i in range(1, len_-1):
        if value_list[i] > threshold and value_list[i] > value_list[i-1] and value_list[i] > value_list[i+1]:
            local_max.append(i)

    bursts = {}
    diff = max(value_list) - min(value_list)

    for pic in local_max:
        for j in range(pic + 1, len_):
            if value_list[j] < threshold or abs(value_list[j] - value_list[pic]) > 0.01 * diff :
                break
        tmp = 0
        for k in range(pic-1,0,-1):
            if value_list[k] < threshold or abs(value_list[k] - value_list[pic]) > 0.01 * diff :
                tmp = k + 1
                break
        bursts[tmp] = j-1

    if 0 not in bursts.keys():
        if value_list[0] > threshold and value_list[0] > value_list[1]:
            bursts[0] = 0
    if len_-1 not in bursts.values():
        if value_list[len_-1] > threshold and value_list[len_-1] > value_list[len_-2]:
            bursts[len_-1] = len_-1

    if first_n:
        ind_n = []
        for i in bursts.keys():
            ind_n.append(i)
        value_max = [value_list[i] for i in ind_n]
        first_n_ind = np.array(value_max).argsort()[-first_n:]
        burst_new = {}
        for i in first_n_ind:
            burst_new[ind_n[i]] = bursts[ind_n[i]]
        return burst_new
    return bursts

def plot_burst(ind_burst, figsize, frequency, threshold):
    '''
    Plot a graph showing the burst

    input:
    ind_burst: a dictionnary indicating the burst burst and the burst end
    figsize: the size of figure
    frequency: the numbers of tweets for every period
    threshold: the burst below mean value * threshold is ignored

    output:
    figure
    '''
    ind_burst_plot = collections.OrderedDict(sorted(ind_burst.items()))

    fig, ax = plt.subplots(figsize = figsize)
    plt.plot(frequency)
    plt.plot([np.mean(frequency) * threshold]*len(frequency))
    c = -1
    for i in ind_burst_plot.keys():
        c = c + 1
        max_tmp = frequency[i]
        for j in range(i, len(frequency)):
            if j<= ind_burst_plot[i]:
                if max_tmp<frequency[j]:
                    max_tmp = frequency[j]
                if c % 2 == 0 :
                    plt.scatter(j, frequency[j],c = 'red')
                else:
                    plt.scatter(j, frequency[j],c = 'green')
        ax.annotate(str(c+1),(i, max_tmp*1.02))

    plt.legend(('change trend', 'thresholod', 'burst'))
    plt.xlabel('period sequence')
    plt.ylabel('number of tweets')
    plt.title('Find burst according to tweets frequency.')

def split_by_time(dates_, periode = 3600):
    '''
    split our data by a given periode

    input:
    dates_: the published time of tweets
    period: the length of period, in seconds

    output:
    accumulated_num: accumulated numbers until this period
    num:the numbers of tweets for every period
    '''
    accumulated_num = [0]
    num = []
    begin = 0
    for i in range(1,len(dates_)):
        dur_tmp = dates_[i] - dates_[begin]
        if dur_tmp.total_seconds() > periode:
            accumulated_num.append(i)
            num.append(i-begin)
            begin = i
    return accumulated_num, num

def build_graph(num_words, list_node, burst_i, ind_burst, accumulated_num, terms_stop, thresh):
    '''
    Need to specify the threshold which delete all values smaller than it.
    The idea is to build the TermDocumentMatrix M then get the co-occurence matrix by M.T.dot(M)

    input:
    num_words: the length of the matrix
    list_node: all considered words to build the TermDocumentMatrix
    burst_i: the ith burst in the dictionnary
    ind_burst: a dictionnary indicating the burst burst and the burst end by funtion find_burst
    accumulated_num: accumulated numbers until this period given by function split_by_time
    terms_stop: the list of tokenized tweets from read_bz2 function
    thresh: ignore the value in comatrix which is smaller than thresh * mean

    output:
    sps_acc: TermDocumentMatrix
    comatrix: square matrix
    '''
    burst_start = ind_burst.keys()[burst_i]
    burst_end = ind_burst[burst_start]
    burst_start= accumulated_num[burst_start]
    burst_end = accumulated_num[burst_end+1]
    num_tweet = burst_end - burst_start

    #build word frequency matrix
    rows, cols = num_tweet, num_words
    sps_acc = sps.lil_matrix((rows, cols)) # empty sparse matrix
    for i in range(burst_start, burst_end):
        for j in range(cols):
            if list_node[j] in terms_stop[i]:
                sps_acc[i - burst_start,j] = 1

    comatrix = sps_acc.T.dot(sps_acc)#.todense()
    mean = np.mean(comatrix)
    comatrix[comatrix<thresh*mean] = 0
    comatrix[range(cols),range(cols)] = 0

    return sps_acc, comatrix

def find_original(burst_i, ind_burst, clique_word, whole_tweets, dates_, accumulated_num, terms_stop):
    '''
    find the original tweets according to the clique words

    input:
    burst_i: the ith burst in the dictionnary
    ind_burst: a dictionnary indicating the burst burst and the burst end by funtion find_burst
    clique_word: the words in the clique
    ...

    output:
    max_count: how many words are matched in the clique and the original tweets
    whole_tweets[index]: the original tweet
    dates_[index]: the published time
    '''
    max_count = 0
    max_ind = 0
    for i in range(accumulated_num[list(ind_burst.keys())[burst_i]],  accumulated_num[list(ind_burst.values())[burst_i]+1]):
        count_appear = 0
        for j in clique_word:
            if j in terms_stop[i]:
                count_appear = count_appear + 1
            if count_appear == len(clique_word):
                max_ind = i
                break
            if count_appear > max_count:
                max_count = count_appear
                max_ind = i
    index = max_ind

    return  max_count, whole_tweets[index], dates_[index]

def find_event(keyword, ignore, ind_burst, list_word, accumulated_num, terms_stop, whole_tweets, dates, num_words= 150, thre = 0, gamma = 0.45, numClique = 12, minClique = 6):
    '''
    Detect events for all bursts.
    If a clique is overlap with previous one, it is ignored.

    input:
    keyword: the root word for quasi clique
    ignore: ignore the word for a better quasi clique
    list_word: all existing words, it is prepared for num_words
    num_words: how many words are candidates for quasi clique
    thre: value for function build graph
    gamma: key parameter, how much a new edge is close to the max edge weight
    numClique: max number of nodes in the clique
    minClique: how many nodes the resulting clique should contain. This is to avoid that a clique only contains 2 or 3 words.

    output:
    list_clique: all resulting cliques
    list_date: corresponding published time
    list_whole: corresponding original tweets
    '''
    list_date = []
    list_whole = []
    list_clique = []
    for burst_i in range(len(ind_burst)):
        list_node = list_word[burst_i].most_common(num_words)
        list_node = dict(list_node).keys()
        count_matrix, comatrix = build_graph(len(list_node), list_node, burst_i, ind_burst, accumulated_num, terms_stop, thre)
        df = pd.DataFrame(data = comatrix.todense(), columns = list_node, index = list_node)
        if ignore in list_node:
            df = df.drop([ignore]).drop(ignore,axis = 1)
        G = df.values
        #find the root word
        for i in range(len(list(df.columns))):
            if list(df.columns)[i] == keyword.lower():
                break
        list_i = findQuasiClique(G, i, gamma, numClique)
        clique_word = []
        for i in list_i:
            clique_word.append(list(df.columns)[i])
        if len(clique_word) > minClique:
            if not isOverlop(list_clique, clique_word):
                count, whole, date = find_original(burst_i, ind_burst, clique_word, whole_tweets, dates, accumulated_num, terms_stop)
                if count < 4:
                    continue
                list_date.append(date)
                list_whole.append(whole)
                list_clique.append(clique_word)
    return list_date, list_whole, list_clique

def isOverlop(list_clique, clique_word):
    '''
    To detect if a new resulting clique has a fraction of same words with previous one

    input:
    list_clique:  all resulting cliques until now
    clique_word: new cliques

    output:
    overlap or not
    '''
    for clique in list_clique:
        overlap = 0
        for word in clique_word:
            if word in clique:
                overlap +=1
        if overlap > 0.6 * len(clique_word):
            return True

    return False

def findQuasiClique(G, v, gamma, s):
    '''
    Given an undirected graph, find a quasiclique.

    inputs:
        G: the undirected graph, the type is np.ndata
        V: the vertices set.
        gamma: the parameter of weighted gamma-quasi-clique.
        s: the maximum number of vertice in the clique.
    outputs:
        S: the set of nodes.
    '''
    V = list(np.arange(len(G)))
    S = []
    x, y = np.meshgrid(v, np.arange(len(G)))
    u = np.argmax(G[x,y])
    loop = 7
    while  (isolate(sorted(G[u,:], reverse=True))):
        for i in range(len(G)):
            G[i,u] = 0
            G[u,i] = 0
        x, y = np.meshgrid(v, np.arange(len(G)))
        u = np.argmax(G[x,y])
        loop = loop -1
        if loop == 0:
            break
    if (G[v,u]) <= 0:
        return S
    else:
        S.append(v)
        S.append(u)
        while True:
            if len(S) >= s:
                return S
            else:
                '''find the neighbors for set S'''
                res = list(set(V) - set(S))
                if len(res)>0:
                    x,y = np.meshgrid(S, res)
                    w = G[x,y]
                    w = w.reshape(len(S), len(res))
                    flag = w > 0
                    flag = np.sum(flag, axis=0)
                    flag = flag > 0
                    index = np.int_(flag*np.arange(len(res)))
                    neighbor = np.array(res)[index]
                    if len(neighbor):
                        qs = []
                        for i in neighbor:
                            qs.append(qG(G,S+[i]))
                        index = np.argmax(qs)
                        qG_max = np.array(qs)[index]
                        if qG_max < gamma:
                            return S
                        else:
                            S = S + [neighbor[index]]
                else:
                    return S

def isolate(l):
    '''
    If a node is only connected to few nodes, it is ignored.
    If a node's weight is much bigger than the next node, it is ignored.
    '''
    if l[0] > l[1] * 2:
        return 1
    l = np.asarray(l)
    if len(l[l> 0]) < 0.1*len(l):
        return 1

def qG(G,S):
    '''
    To calculate a ratio used for verifying if the subgraph S of G is a weighted gamma quasi-clique.

    inputs:
        G: a graph, the type is np.ndata.
        S: a subgraph of G, the type is np.ndata.

    output:
        return the ratio if S is not empty, otherwise return 0.
    '''
    if len(S):
        x, y = np.meshgrid(S,S)
        total_w = np.sum(G[x,y])/2
        max_w = np.max(G[x,y])
        return total_w/(max_w*comb(len(S),2,exact=False))
    else:
        return 0


# -*- coding: utf-8 -*-
import itertools
import os.path
import subprocess
import logging
import re

import click
import requests

_REPO_LIST_URL = 'https://api.github.com/{type}s/{account}/repos?page={page_no}'

def repositories(account, type='org', filter=None):
    for page_no in itertools.count(1):
        click.echo('Page {page_no}'.format(page_no=page_no))
        url = _REPO_LIST_URL.format(type=type,
                                    account=account,
                                    page_no=page_no,
                                    filter=filter)
        logging.info('Getting repo list from {}'.format(url))
        resp = requests.get(url)
        resp.raise_for_status()
        if not resp.json():
            raise StopIteration
        for repo in resp.json():
            if not repo['private']:
                if (not filter) or re.search(filter, repo['name']):
                    yield repo

class Repository(object):
    def __init__(self, data):
        self.__dict__.update(data)

    @classmethod
    def synch_all(cls, account, type, filter=None, *args):
        """
        Clones or updates repositories belonging to `account`.
        """
        for repo_data in repositories(account, type, filter):
            repo = cls(repo_data)
            repo.synch(*args)

    @classmethod
    def synch_present(cls, filter=None, *args):
        """
        Updates repositories already present in child directories.
        """
        for dir in os.listdir():
            if os.path.isdir(dir):
                if filter and not re.search(filter, dir):
                    logging.info('{} ignored; filtered by `{}`'.format(dir,
                                                                       filter))
                else:
                    logging.info('{} exists; pulling changes'.format(dir))
                    subprocess.run(['git', 'pull'], cwd=dir)

    def synch(self, *args):
        if os.path.exists(self.name):
            logging.info('{} exists; pulling changes'.format(self.name))
            subprocess.run(['git', 'pull'], cwd=self.name)
        else:
            logging.info('Cloning '.format(self.name))
            subprocess.run(['git', 'clone', *args, self.clone_url])


# -*- coding: utf-8 -*-

"""
TEXT EXTRACTION FROM IMAGES
"""
import string
import json
import numpy as np
import pytesseract
import cv2 as cv

# Read the image
IMG = cv.imread(r"C:\Users\Internship004\.spyder-py3\orig.jpg", 1)

# Improvise RGB ratio of the image
IMG = cv.cvtColor(IMG, cv.COLOR_BGR2RGB)

def clahe_try(img_rd):
    ''' Return image after applying clahe processing '''
    img_new = cv.cvtColor(img_rd, cv.COLOR_BGR2GRAY)
    clahe = cv.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
    img_new = clahe.apply(img_new)
    return img_new

def unsharp_mask(img_rd, kernel_size=(5, 5), sigma=1.0, amount=1.0, thresh=0):
    '''  Return a sharpened version of the image, using an unsharp mask '''
    blurred = cv.GaussianBlur(img_rd, kernel_size, sigma)
    sharpened = float(amount + 1) * img_rd - float(amount) * blurred
    sharpened = np.maximum(sharpened, np.zeros(sharpened.shape))
    sharpened = np.minimum(sharpened, 255 * np.ones(sharpened.shape))
    sharpened = sharpened.round().astype(np.uint8)
    if thresh > 0:
        low_contrast_mask = np.absolute(img_rd - blurred) < thresh
        np.copyto(sharpened, img_rd, where=low_contrast_mask)
    return sharpened

# Get the width and height of the image
H, W = list(IMG.shape)[0], list(IMG.shape)[1]

# Calculate aspect ratio
ASPECTRATIO = W/H

# Fix a width for passing to tesseract
FIXEDWIDTH = 700

# Resize the read image to the fixed resolution
IMG = cv.resize(IMG, (FIXEDWIDTH, int(FIXEDWIDTH//ASPECTRATIO)))

IMG = unsharp_mask(IMG)
# functon call
IMG = clahe_try(IMG)

# show and save final image
cv.imshow('show', IMG)
cv.imwrite('output.jpg', IMG)

# Pass the image to tesseract and convert it as a dataframe
TEXT = pytesseract.image_to_data(IMG, output_type='data.frame')

# Remove the WORDS with negative CONFidence scores
TEXT = TEXT[TEXT.conf != -1]
LINES = TEXT.groupby('block_num')['text'].apply(list)
CONF = TEXT.groupby(['block_num'])['conf'].mean()

# Remove punctuations from the extracted TEXT and append
# the remaining to a new list
CONFLIST = []
for i in range(len(TEXT['conf'].values)):
    if(TEXT['conf'].values[i] >= 25 and TEXT['text'].values[i] != 'nan' and
       TEXT['text'].values[i] not in string.punctuation and
       TEXT['text'].values[i][0] not in string.whitespace):
        CONFLIST.append(TEXT['text'].values[i])
CONFLIST.append('EOF')

# Pass the image to pytesseract to extract the TEXT from the image
RESULT = pytesseract.image_to_string(IMG)

# Remove white spaces from the extracted TEXT and append the wanted TEXT
NEWLIST = []
EACHLINE = RESULT.split('\n')
for i in EACHLINE:
    NEWSTR = ''
    WORDS = i.split()
    for k in WORDS:
        if k in CONFLIST:
            NEWSTR += k+' '
    NEWLIST.append(NEWSTR[0:-1])

# remove special characters and white spaces from the list
for i in NEWLIST:
    if i in string.whitespace or i in string.punctuation:
        NEWLIST.remove(i)

# Update the dictionary with line number and the line contents
D = {}
k=0
for i in range(len(NEWLIST)):
    if NEWLIST[i] not in string.whitespace:
        k+=1
        D.update({'Line '+str(k): NEWLIST[i]})

# Open a json file and dump the dictionary into it
with open('C:/Users/Internship004/Desktop/newjson.json', 'w') as jsonfile:
    json.dump(D, jsonfile, indent=2)

# Close the json file
jsonfile.close()

# Extract the required field from the json file
CNT = 0
F = open('C:/Users/Internship004/Desktop/newjson.json', 'r', encoding='utf8')
for LINE in F:
    CNT += 1
    if 'Date of Issue' in LINE:
        break
for LINE in F:
    if 'Line '+str(CNT) in LINE:
        print(LINE)
        break

# Close the displayed image after any key is pressed
cv.waitKey()
cv.destroyAllWindows()


import os
import math
import logging
import torch
from tensorboardX import SummaryWriter

from ..model.model import FOTSModel

def save_checkpoint(epoch, model, optimizer, lr_scheduler, best_score, folder, save_as_best):
    if not os.path.exists(folder):
        os.makedirs(folder)
    if save_as_best:
        torch.save(model.state_dict(), os.path.join(folder, 'best_model.pt'))
        print('Updated best_model')
    torch.save({
        'epoch': epoch,
        'model_state_dict': model.state_dict(),
        'optimizer_state_dict': optimizer.state_dict(),
        'lr_scheduler_state_dict': lr_scheduler.state_dict(),
        'best_score': best_score  # not current score
    }, os.path.join(folder, 'last_checkpoint.pt'))

def restore_checkpoint(folder):
    model = FOTSModel().to(torch.device("cuda"))
    optimizer = torch.optim.Adam(model.parameters(), lr=0.001, betas=(0.9, 0.999), eps=1e-08, weight_decay=1e-5,
                                 amsgrad=False)
    lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, 0.94)

    if os.path.isfile(os.path.join(folder, 'last_checkpoint.pt')):
        checkpoint = torch.load(os.path.join(folder, 'last_checkpoint.pt'))
        epoch = checkpoint['epoch'] + 1
        model.load_state_dict(checkpoint['model_state_dict'])

        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
        lr_scheduler.load_state_dict(checkpoint['lr_scheduler_state_dict'])
        best_score = checkpoint['best_score']
        return epoch, model, optimizer, lr_scheduler, best_score
    else:
        return 0, model, optimizer, lr_scheduler, -math.inf

class BaseTrainer:
    """
    Base class for all trainers
    """
    def __init__(self, metrics, config):
        self.config = config
        self.logger = logging.getLogger(self.__class__.__name__)
        self.metrics = metrics
        self.name = config['name']
        self.epochs = config['trainer']['epochs']
        self.summyWriter = SummaryWriter()
        self.checkpoint_dir = os.path.join(config['trainer']['save_dir'], self.name)

        self.start_epoch, self.model, self.optimizer, self.lr_scheduler, self.best_score = \
            restore_checkpoint(self.checkpoint_dir)
        # self.model.summary()

        self.device = torch.device("cuda")
        self.monitor = config['trainer']['monitor']

    def train(self):
        """
        Full training logic
        """
        for epoch in range(self.start_epoch, self.epochs + 1):
            result = self._train_epoch(epoch)

            log = {'epoch': epoch}
            for key, value in result.items():
                if key == 'metrics':
                    for i, metric in enumerate(self.metrics):
                        log[metric.__name__] = result['metrics'][i]
                elif key == 'val_metrics':
                    for i, metric in enumerate(self.metrics):
                        log['val_' + metric.__name__] = result['val_metrics'][i]
                else:
                    log[key] = value

            self.lr_scheduler.step()
            self.logger.info('New Learning Rate: {:.8f}'.format(self.lr_scheduler.get_lr()[0]))

            self.summyWriter.add_scalars('Train', {'train_' + self.monitor: result[self.monitor],
                                                   'val_' + self.monitor: result[self.monitor]}, epoch)

            if self.best_score < result['hmean']:
                self.best_score = result['hmean']
                save_as_best = True
            else:
                save_as_best = False
            save_checkpoint(epoch, self.model, self.optimizer, self.lr_scheduler, self.best_score,
                            self.checkpoint_dir, save_as_best)

        self.summyWriter.close()

    def _train_epoch(self, epoch):
        """
        Training logic for an epoch

        :param epoch: Current epoch number
        """
        raise NotImplementedError


import tensorflow as tf
from keras.models import Sequential
from keras.optimizers import Adam
from keras.layers import Conv2D, ZeroPadding2D, Activation, Input, concatenate
from keras.models import Model

from keras.layers.normalization import BatchNormalization
from keras.layers.pooling import MaxPooling2D
from keras.layers.merge import Concatenate
from keras.layers import Lambda, Flatten, Dense, Input
from keras.initializers import glorot_uniform,RandomNormal

from keras.engine.topology import Layer
from keras.regularizers import l2
from keras import backend as K
from keras import layers
import numpy as np

import time

import random
import numpy as np
from sklearn.utils import shuffle
import random
import numpy as np

import pickle

#
# Methods for defining and initializaing our model
def initialize_weights(shape, name=None):
    """
        The paper, http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf
        suggests to initialize CNN layer weights with mean as 0.0 and standard deviation of 0.01
    """
    return np.random.normal(loc = 0.0, scale = 1e-2, size = shape)
def initialize_bias(shape, name=None):
    """
        The paper, http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf
        suggests to initialize CNN layer bias with mean as 0.5 and standard deviation of 0.01
    """
    return np.random.normal(loc = 0.5, scale = 1e-2, size = shape)

def get_siamese_model_works(input_shape):
    """
        Model architecture based on the one provided in: http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf
    """

    # Define the tensors for the two input images
    left_input = Input(input_shape)
    right_input = Input(input_shape)

    # Convolutional Neural Network
    model = Sequential()
    model.add(Conv2D(64, (10,10), activation='relu', input_shape=input_shape,
                   kernel_initializer=initialize_weights, kernel_regularizer=l2(2e-4)))
    model.add(MaxPooling2D())
    model.add(Conv2D(128, (7,7), activation='relu',
                     kernel_initializer=initialize_weights,
                     bias_initializer=initialize_bias, kernel_regularizer=l2(2e-4)))
    model.add(MaxPooling2D())
    model.add(Conv2D(128, (4,4), activation='relu', kernel_initializer=initialize_weights,
                     bias_initializer=initialize_bias, kernel_regularizer=l2(2e-4)))
    model.add(MaxPooling2D())
    model.add(Conv2D(256, (4,4), activation='relu', kernel_initializer=initialize_weights,
                     bias_initializer=initialize_bias, kernel_regularizer=l2(2e-4)))
    model.add(Flatten())
    model.add(Dense(4096, activation='sigmoid',
                   kernel_regularizer=l2(1e-3),
                   kernel_initializer=initialize_weights,bias_initializer=initialize_bias))

    # Generate the encodings (feature vectors) for the two images
    encoded_l = model(left_input)
    encoded_r = model(right_input)

    # Add a customized layer to compute the absolute difference between the encodings
    L1_layer = Lambda(lambda tensors:K.abs(tensors[0] - tensors[1]))
    L1_distance = L1_layer([encoded_l, encoded_r])

    # Add a dense layer with a sigmoid unit to generate the similarity score
    prediction = Dense(1,activation='sigmoid',bias_initializer=initialize_bias)(L1_distance)

    # Connect the inputs with the outputs
    siamese_net = Model(inputs=[left_input,right_input],outputs=prediction)

    # return the model
    return siamese_net

def get_siamese_model(input_shape):
    """
        Model architecture based on the one provided in: http://www.cs.utoronto.ca/~gkoch/files/msc-thesis.pdf
    """

    # Define the tensors for the two input images
    left_input = Input(input_shape)
    right_input = Input(input_shape)

    # Convolutional Neural Network
    model = Sequential()
    model.add(Conv2D(64, (10,10), activation='relu', input_shape=input_shape,
                   kernel_initializer=RandomNormal(mean=0.0, stddev=0.01),
                   kernel_regularizer=l2(2e-4)))
    model.add(MaxPooling2D())
    model.add(Conv2D(128, (7,7), activation='relu',
                     kernel_initializer=RandomNormal(mean=0.0, stddev=0.01),
                     bias_initializer=RandomNormal(mean=0.5, stddev=0.01),
                     kernel_regularizer=l2(2e-4)))
    model.add(MaxPooling2D())
    model.add(Conv2D(128, (4,4), activation='relu',
                     kernel_initializer=RandomNormal(mean=0.0, stddev=0.01),
                     bias_initializer=RandomNormal(mean=0.5, stddev=0.01),
                     kernel_regularizer=l2(2e-4)))
    model.add(MaxPooling2D())
    model.add(Conv2D(256, (4,4), activation='relu',
                     kernel_initializer=RandomNormal(mean=0.0, stddev=0.01),
                     bias_initializer=RandomNormal(mean=0.5, stddev=0.01),
                     kernel_regularizer=l2(2e-4)))
    model.add(Flatten())
    model.add(Dense(4096, activation='sigmoid',
                   kernel_regularizer=l2(1e-3),
                   kernel_initializer=RandomNormal(mean=0.0, stddev=0.01),
                    bias_initializer=RandomNormal(mean=0.5, stddev=0.01)))

    # Generate the encodings (feature vectors) for the two images
    encoded_l = model(left_input)
    encoded_r = model(right_input)

    # Add a customized layer to compute the absolute difference between the encodings
    L1_layer = Lambda(lambda tensors:K.abs(tensors[0] - tensors[1]))
    L1_distance = L1_layer([encoded_l, encoded_r])

    # Add a dense layer with a sigmoid unit to generate the similarity score
    prediction = Dense(1,activation='sigmoid',bias_initializer=RandomNormal(mean=0.5, stddev=0.01))(L1_distance)

    # Connect the inputs with the outputs
    siamese_net = Model(inputs=[left_input,right_input],outputs=prediction)

    # return the model
    return siamese_net
#
# This make an N-way test sample
#    if use_test_data is True, uses validation data, otherwise uses training data
def make_oneshot(N,use_test_data=True):
#
# For each batch shuffle
    if use_test_data:
        n_classes, n_examples, w, h = X_val.shape
    else:
        n_classes, n_examples, w, h = X_train.shape
    letters = list(range(n_classes))
    examples = list(range(n_examples))
#
# Each trial shuffle our letters and classes
    random.shuffle(letters)
    random.shuffle(examples)
    #print(examples)
#
# Get our support indices
    support_letters_class_indices = letters[0:N]
    support_letters_example_indices = examples[0:N]
#
# Get the letter to test
    test_letter_class_index = letters[0]
    test_letter_example_index = examples[0]
#
# Now get another example (but different) of the test letter
    test_letter_class_index_other = letters[0]
    test_letter_example_index_other = examples[1]
#
# The first letter in our support sample is the correct class
    support_letters_class_indices[0] = test_letter_class_index_other
    support_letters_example_indices[0] = test_letter_example_index_other
    targets = np.zeros((N,))
    targets[0] = 1
#
# Now form our images
    if use_test_data:
        test_images = np.asarray([X_val[test_letter_class_index,test_letter_example_index,:,:]]*N)
        test_images = test_images.reshape(N, w, h,1)
        support_images = X_val[support_letters_class_indices,support_letters_example_indices,:,:]
        support_images = support_images.reshape(N, w, h,1)
    else:
        test_images = np.asarray([X_train[test_letter_class_index,test_letter_example_index,:,:]]*N)
        test_images = test_images.reshape(N, w, h,1)
        support_images = X_train[support_letters_class_indices,support_letters_example_indices,:,:]
        support_images = support_images.reshape(N, w, h,1)

#
# Form return
    pairs = [test_images, support_images]

    return pairs, targets

def get_batch(batch_size):
#
# For each batch shuffle
    n_classes, n_examples, w, h = X_train.shape
    letters = list(range(n_classes))
    examples = list(range(n_examples))
#
# Each trial shuffle our letters and classes
    random.shuffle(letters)
    random.shuffle(examples)
    targets = np.zeros((batch_size,))
    test_images = np.zeros((batch_size,w, h))
    support_images = np.zeros((batch_size,w, h))
#
# Make sure the batch size is < half the classes
    if batch_size < n_classes//2:
        half_batch = batch_size//2
#
# Get the indices for the 1st half - which are pairs from same class
        test_letter_class_indices = letters[0:half_batch]
        test_letter_example_index = examples[0]
        support_letters_class_indices = letters[0:half_batch]
        support_letters_example_index = examples[1]

        test_images[0:half_batch,:,:] = X_train[test_letter_class_indices,test_letter_example_index,:,:]
        support_images[0:half_batch,:,:] = X_train[support_letters_class_indices,support_letters_example_index,:,:]
        targets[0:half_batch] = 1
#
# Get the indices for the 2nd half - which are pairs from different classes
        test_letter_class_indices = letters[half_batch:batch_size]
        test_letter_example_index = examples[0]
        support_letters_class_indices = letters[batch_size:batch_size+half_batch]
        support_letters_example_index = examples[1]

        test_images[half_batch:batch_size,:,:] = X_train[test_letter_class_indices,test_letter_example_index,:,:]
        support_images[half_batch:batch_size,:,:] = X_train[support_letters_class_indices,support_letters_example_index,:,:]
        targets[half_batch:batch_size] = 0
#
# Reshape
        test_images = test_images.reshape(batch_size, w, h,1)
        support_images = support_images.reshape(batch_size, w, h,1)

#
# Now shuffle coherently
    targets, test_images, support_images = shuffle(targets, test_images, support_images)
    pairs = [test_images, support_images]

    return pairs, targets

#
# Get our training and validation data sets
X_train,lang_train = pickle.load(open("data/train.pickle","rb"))
X_val,lang_val = pickle.load(open("data/val.pickle","rb"))
#
# These need to be normalized
X_train = X_train / 255.0
X_val = X_val / 255.0
#
# Some debug printout
print(X_train.shape)
print(X_val.shape)
print(X_val[0,0,:,:])
print("Training:    ",lang_train)
print("Validation:  ",lang_val)
img = X_val[0,1,:,:]
print(img.shape)
#
# Set up the model
model = get_siamese_model((105, 105, 1))
optimizer = Adam(lr = 0.00006)
model.compile(loss="binary_crossentropy",optimizer=optimizer)
model.summary()
#
# Hyper parameters
evaluate_every = 200 # interval for evaluating on one-shot tasks
batch_size = 32
n_iter = 20000 # No. of training iterations
N_way = 20 # how many classes for testing one-shot tasks
n_val = 200 # how many one-shot tasks to validate on
best = -1
#
# Now start training
print("Starting training process!")
print("-------------------------------------")
t_start = time.time()
for i in range(1, n_iter+1):
#
# Get a new batch to test on
    (inputs,targets) = get_batch(batch_size)
    loss = model.train_on_batch(inputs, targets)
#
# Every so many iterations, check the training and validation performance
    if i % evaluate_every == 0:
        print("i=",i)
        print("\n ------------- \n")
        print("Time for {0} iterations: {1} mins".format(i, (time.time()-t_start)/60.0))
        print("Train Loss: {0}".format(loss))
#
# Now get N-way test results for train and validation sets
        n_correct_train = 0
        n_correct_val = 0
        for testTrials in range(n_val):
#
# First check training perfromance
            inputs, targets = make_oneshot(N_way,use_test_data=False)
            probs = model.predict(inputs)
            if np.argmax(probs) == np.argmax(targets):
                n_correct_train += 1
#
# Next check validation performace
            inputs, targets = make_oneshot(N_way,use_test_data=True)
            probs = model.predict(inputs)
            if np.argmax(probs) == np.argmax(targets):
                n_correct_val += 1
        train_acc = (100.0 * n_correct_train / n_val)
        print("     training perf",train_acc)
        val_acc = (100.0 * n_correct_val / n_val)
        print("   validation perf",val_acc)
        if val_acc >= best:
            print("Current best: {0}, previous best: {1}".format(val_acc, best))
            best = val_acc
            model.save('models/siam_model.{}.h5'.format(i))
            model.save_weights('weights/siam_weights.{}.h5'.format(i))

print("Done!")


# CTD9 SVL - M. Nebut - 03/2016
# property-based testing

import unittest

from hypothesis import given, assume, example
from hypothesis.strategies import floats

from banque import *

class TestTomCrediteUnCompte(unittest.TestCase):

    def test_echec_si_somme_negative_classique(self):
        compte = Compte()
        self.assertRaises(ValueError,
                            compte.crediter,
                            -10)

    @given(somme=floats(max_value=0.0))
    @example(somme=0.0)
    def test_echec_si_somme_negative(self, somme):
        compte = Compte()
        self.assertRaises(ValueError,
                            compte.crediter,
                            somme)

    @given(somme=floats(min_value=0.0))
    #@example(somme=float("inf"))
    def test_le_solde_est_augmente(self, somme):
        compte = Compte()

        assume(somme > 0)

        compte.crediter(somme)
        self.assertEqual(compte.montant, somme)

class TestDebiterUnCompte(unittest.TestCase):

    @given(somme=floats(max_value=0.0))
    @example(somme=0.0)
    def test_echec_si_somme_negative(self, somme):
        compte = Compte()
        self.assertRaises(ValueError,
                            compte.debiter,
                            somme)

    @given(somme=floats(min_value=0.0))
    def test_le_solde_est_diminue(self, somme):
        compte = Compte()
        assume(somme > 0)
        compte.debiter(somme)
        self.assertEqual(compte.montant, -somme)


# -*-coding:utf-8-*-
import stomp
import time
import base64
import random
import string
import os

queue_name = '/queue/test'
listener_name = 'SampleListener'

class SampleListener(object):
    def on_error(self, headers, message):
        print('错误：', message)
    def on_message(self, headers, message):
        print('接收：', message)
        #  随机字符串
        ran_str = ''.join(random.sample(string.ascii_letters + string.digits, 8))
        img = base64.b64decode(message)
        file = open(os.getcwd() + '/'+ ran_str +'.jpg', 'wb')
        file.write(img)
        file.close()

# 推送到队列queue
def send_to_queue(msg):
    conn = stomp.Connection10([('127.0.0.1', 61613)])
    conn.start()
    conn.connect()
    print('发送：', msg)
    conn.send(queue_name, msg)
    conn.disconnect()

##从队列接收消息
def receive_from_queue():
    conn = stomp.Connection10([('127.0.0.1', 61613)])
    conn.set_listener(listener_name, SampleListener())
    conn.start()
    conn.connect()
    conn.subscribe(queue_name)
    time.sleep(1)  # secs
    conn.disconnect()

if __name__ == '__main__':
     #send_to_queue('len 123')
     # 无限循环，接收队列中的消息
     var = 1
     while var == 1:
         receive_from_queue()


import logging

from django.db import transaction

from .models import Task, delay
from .task import SerialTask, TaskFunc
from .backends import backend

logger = logging.getLogger('cq')

def run_task(message):
    try:
        task_id = message['task_id']
    except (TypeError, KeyError):
        logger.error('Invalid CQ message.')
        return
    task = Task.objects.get(id=task_id)
    func_name = task.signature['func_name']
    if task.status == Task.STATUS_REVOKED:
        logger.info('Not running revoked task: {}'.format(func_name))
        return
    logger.info('Running task: {}'.format(func_name))
    backend.set_current_task(task_id)
    task.pre_start()
    taskFunc = TaskFunc.get_task(func_name)
    if taskFunc.atomic:
        with transaction.atomic():
            _do_run_task(task)
    else:
        _do_run_task(task)

def _do_run_task(task):
    try:
        result = task.start(pre_start=False)
    except Exception as err:
        task.failure(err)
    else:
        if isinstance(result, Task):
            task.waiting(task=result)
        else:
            if isinstance(result, SerialTask):
                result = result.result
            if task.subtasks.exists():
                task.waiting(result=result)
            else:
                task.success(result)
    finally:
        backend.set_current_task()