import os
import os
import csv
from college.models import College
from django.db.models import Count
import json
from django.conf import settings

file_data = {}
db_data = {}

with open(os.path.join(settings.BASE_DIR, 'scripts/students/data_check_code_dir.json'), "r") as file:
    file_data = json.loads(file.read())
    file.close()

with open(os.path.join(settings.BASE_DIR, 'scripts/students/data_check_db_dir.json'), "r") as file:
    db_data = json.loads(file.read())
    file.close()

final_test_data_not_db = 'code,roll_no'
file_college_codes = file_data.keys()
db_college_codes = db_data.keys()

colleges_codes_not_match = []

colleges_codes_not_match += list(set(file_college_codes) - set(db_college_codes))
colleges_codes_not_match += list(set(db_college_codes) - set(file_college_codes))
colleges_codes_not_match = set(colleges_codes_not_match)

students_not_match = 'college_code,student_roll_no,not_match'
students_not_match_data = {}

# print("flag 3")
# Step 1
final_check_data = {}
for college_code in file_college_codes:
    check_data = []
    file_students = file_data[college_code] if college_code in file_data.keys() else []
    db_students = file_data[college_code] if college_code in db_data.keys() else []

    student_codes_not_match = []
    student_codes_not_match += list(set(file_students) - set(db_students))
    student_codes_not_match += list(set(db_students) - set(file_students))
    student_codes_not_match = set(student_codes_not_match)
    final_check_data[college_code] = list(student_codes_not_match)

# print("flag 4")
final_check_data_codes = final_check_data.keys()
for college_code in db_college_codes:
    check_data = []
    # print("floag - 12")
    print(college_code in file_data.keys())
    print(college_code in db_data.keys())
    file_students = file_data[college_code] if college_code in file_data.keys() else []

    # print("floag - 164")
    db_students = db_data[college_code] if college_code in db_data.keys() else []

    # print("floag - 1641")

    student_codes_not_match = []
    if not db_students:
        student_codes_not_match = set(file_students) if file_students else []
    if not file_students:
        student_codes_not_match = set(db_students) if db_students else []
    else:
        student_codes_not_match += list(set(file_students) - set(db_students))
        student_codes_not_match += list(set(db_students) - set(file_students))
        student_codes_not_match = set(student_codes_not_match)

    # print("floag - 1642")

    # print("floag - 143")
    if college_code in final_check_data_codes:
        final_check_data[college_code] += list(student_codes_not_match)
    else:
        final_check_data[college_code] = list(student_codes_not_match)

final_csv_data = 'college_code,student_roll_no'

for college_code, student_data in final_check_data.items():
    student_data = list(set(student_data))
    for roll_no in student_data:
        final_csv_data += f"\n{college_code},{roll_no}"

with open(os.path.join(settings.BASE_DIR, "scripts/students/student_codes_not_match.csv"), "w") as file:
    file.write(final_csv_data)
    file.close()


#!/usr/bin/env python3
import socket
import string
import re
from multiprocessing.pool import ThreadPool

charset = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ '

def attempt(tag, payload, block_number=0):
    s = socket.socket()
    s.connect(('2018shell2.picoctf.com', 30399))

    # Assert length
    '''
    message = """Agent,
Greetings. My situation report is as follows:
{0}
My agent identifying code is: """.format(payload)
    assert (len(message) - (block_number + 1) * 16) <= 1, len(message)
    # print(payload)
    '''

    # retrieve char
    while True:
        data = s.recv(4096).decode().strip()
        if not data:
            continue
        #print("Received >>", data)

        if 'Welcome, Agent 006!' in data:
            pass
        elif 'Please enter your situation report:' in data:
            s.send(payload.encode())
            s.send(b'\n')
        else:
            return (tag, data[block_number * 32 : block_number * 32 + 32])

def main(progress='', block_number=0):
    # My agent identifying code is:
    block = 'A' * 16
    assert len(block) == 16

    for skip in range(16):
        # padding to round to nearest block
        # 96 bytes rounded up from 84 bytes provided
        padding = "x" * (96-53-31)
        prefix = padding + block[skip+1:]

        # Enter 15 characters and retrieve the first 16 encoded
        _, aim = attempt('', prefix, block_number)

        pool = ThreadPool(processes=25)
        async_results = []

        # Then enter 16 characters changing the last one until you get the
        # same result as with 15 characters for the first 16 encoded bytes
        for ch in charset:
            # dummy to duplicate the uncontrolled blocks
            # 96 bytes rounded up from 84 bytes provided
            dummy = '\nMy agent identifying code is: '.replace('\n', 'n') # server can't receive newline
            payload = prefix + dummy + progress + ch

            async_result = pool.apply_async(attempt, (ch, payload, block_number))
            async_results.append(async_result)

        # That 16th character is the first character of your secret.
        # You can then repeat the process by putting 14 characters and then
        # finding the second secret characters with the same thechnique.
        for async_result in async_results:
            ch, result = async_result.get()
            print(f"Result: {ch}, {result} == {aim}")

            if result == aim:
                progress += ch
                if ch == '}':
                    # end of flag! if we continue we
                    # merely get '0's added forever
                    print(f"Success! {progress}")
                    quit()
                break

        print(f"Progress (Block {block_number} of index {skip}): {progress}")

    # continue to next block
    main(progress, block_number + 1)

if __name__ == '__main__':
    # 96 bytes rounded up from 84 bytes provided
    # so we mask out the first 6 blocks
    main(block_number=6)


from django import forms
from .models import Post, Comment

class PostForm(forms.ModelForm):
    body = forms.CharField(
        label='',
        widget=forms.Textarea(attrs={
            'rows': '3',
            'placeholder': 'Say Something...'
        }))
    picture = forms.ImageField(required=False)  # Add this line for the image field

    class Meta:
        model = Post
        fields = ['body', 'picture']  # Include the 'image' field in the form

class CommentForm(forms.ModelForm):
    comment = forms.CharField(
        label='',
        widget=forms.Textarea(attrs={
            'rows': '3',
            'placeholder': 'Say Something...'
            }))

    class Meta:
        model = Comment
        fields = ['comment']


##
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##

from __future__ import annotations
from typing import Hashable, List, Tuple, Dict, Optional, Sequence, Union, Iterable

import glob
import os

import pandas as pd
import pyarrow as pa
from pyarrow import parquet as pq
from pycylon.frame import CylonEnv
from pycylon.net.comm_ops import allgather_buffer, gather_buffer

def get_files(paths: Union[str, List[str]], ext: str = None) -> List[str]:
    """
    convert a list of glob strings to a list of files
    if directory paths are given, ext string added to the directory path.
    Possible ext values are: "*.parquet", "*.csv", "*.json"

    get all file names as a single list
    sort them alphabetically and return
    """
    if isinstance(paths, str):
        paths = [paths]

    all_files = []
    for p in paths:
        p = os.path.join(p, ext) if (os.path.isdir(p) and ext is not None) else p
        all_files.extend(glob.glob(p))

    return sorted(all_files)

def indices_per_worker(nfiles, my_rank, nworkers):
    """
    calculate indices for a worker by evenly dividing files among them
    """
    files_per_worker = int(nfiles / nworkers)
    workers_with_exra_file = nfiles % nworkers
    start = files_per_worker * my_rank + \
            (my_rank if my_rank < workers_with_exra_file else workers_with_exra_file)
    end = start + files_per_worker + (1 if my_rank < workers_with_exra_file else 0)
    return start, end

def get_worker_files(paths: List[str], env: CylonEnv, ext: str = None) -> List[str]:
    """
    calculate the list of files that this worker will read
    """
    all_files = get_files(paths=paths, ext=ext)
    start, end = indices_per_worker(len(all_files), env.rank, env.world_size)
    return all_files[start:end]

def all_schemas_equal(schema: pyarrow.Schema, env: CylonEnv) -> pyarrow.Schema:
    """
    after reading files, check whether all DataFrame schemas are equal
    return a schema in case the worker has no DataFrame and needs to create an empty table
    """

    # serialize the schema if exist, otherwise allocate empty buffer
    buf = schema.serialize() if schema is not None else pa.allocate_buffer(0)

    buffers = allgather_buffer(buf=buf, context=env.context)

    schemas = []
    for schema_buf in buffers:
        if len(schema_buf) > 0:
            with pa.ipc.open_stream(schema_buf) as reader:
                schemas.append(reader.schema)

    if len(schemas) == 0:
        raise ValueError("No worker has any schema.")

    # make sure all schemas are equal, compare consecutive ones
    for first, second in zip(schemas, schemas[1:]):
        if not first.equals(second):
            raise ValueError("Not all DataFrame schemas are equal.")

    return schemas[0]

def gen_file_name(rank: int, file_ext: str) -> str:
    """
    generate filename to write a DataFrame partition
    """
    if rank < 10:
        numeric_ext = "000" + str(rank)
    elif rank < 100:
        numeric_ext = "00" + str(rank)
    elif rank < 1000:
        numeric_ext = "0" + str(rank)
    else:
        numeric_ext = str(rank)
    return "part_" + numeric_ext + "." + file_ext

def determine_file_name_to_write(dir_path, name_function, file_ext, env):
    """
    Determine the filename to write for a worker
    """
    if not os.path.isdir(dir_path):
        os.makedirs(dir_path)

    file_name = gen_file_name(env.rank, file_ext) if name_function is None else name_function(env.rank)
    return os.path.join(dir_path, file_name)

#####################################################################################
# Parquet io functions
#####################################################################################

def serialize_metadata_file(fmd: pa._parquet.FileMetaData) -> pa.Buffer:
    """
    serialize a FileMetaData object
    return empty buffer if None
    """
    if fmd is None:
        return pa.allocate_buffer(0)
    else:
        _, buf_tuple = fmd.__reduce__()
        return buf_tuple[0]

def deserialize_metadata_buffers(buffers: List[pa.Buffer]) -> pa._parquet.FileMetaData:
    """
    deserialize FileMetaData buffers
    """
    fmd_all = []
    for received_buf in buffers:
        if received_buf.size > 0:
            fmd_all.append(pa._parquet._reconstruct_filemetadata(received_buf))
    return fmd_all

def schemas_equal(fmd_list: List[pa._parquet.FileMetaData], my_rank: int):
    """
    compare schemas of consecutive FileMetaData objects
    """
    for fmd1, fmd2 in zip(fmd_list, fmd_list[1:]):
        if not fmd1.schema.equals(fmd2.schema):
            f1 = fmd1.row_group(0).column(0).file_path
            f2 = fmd2.row_group(0).column(0).file_path
            raise ValueError(my_rank, f"schemas are not equal for the files: {f1} and {f2}")

def all_gather_metadata(worker_files: List[str], env: CylonEnv) -> pa._parquet.FileMetaData:
    """
    allgather FileMetaData objects and check schemas for equality
    """
    fmd_list = []
    for worker_file in worker_files:
        fmd = pq.read_metadata(worker_file)
        fmd.set_file_path(worker_file)
        fmd_list.append(fmd)

    # make sure all schemas are equal for this worker
    schemas_equal(fmd_list, env.rank)

    fmd_single = fmd_list[0] if fmd_list else None
    for fmd in fmd_list[1:]:
        fmd_single.append_row_groups(fmd)

    # serialize and all gather
    buf = serialize_metadata_file(fmd_single)
    buffers = allgather_buffer(buf=buf, context=env.context)

    # deserialize
    fmd_all = deserialize_metadata_buffers(buffers)
    if len(fmd_all) == 0:
        raise ValueError("No worker has any parquet files.")

    # make sure all received schemas are equal
    schemas_equal(fmd_all, env.rank)

    # combine all received schemas into one
    fmd_single = fmd_all[0]
    [fmd_single.append_row_groups(fmd) for fmd in fmd_all[1:]]
    return fmd_single

def row_groups_this_worker(df: pd.DataFrame, env: CylonEnv):
    """
    Determine the row groups this worker will read.
    Try to distribute rows evenly among workers.
    Each row_group will be read once by a single worker.
    Each worker will get consecutive row_groups in files so that reading might be efficient.
    """
    # if the number of workers are more than the number of row_groups,
    # just return a single row_group for each worker corresponding to its rank or an empty df
    if len(df.index) <= env.world_size:
        return df[env.rank:(env.rank + 1)]

    total_rows = df['row_group_size'].sum()
    remaining_workers = env.world_size
    rows_per_worker = total_rows / remaining_workers
    start_index = 0
    end_index = 0
    last_worker = env.rank == env.world_size - 1
    loop_count = env.rank if last_worker else env.rank + 1
    for _ in range(loop_count):
        rows_this_worker = 0
        start_index = end_index
        for i, rg_size in zip(df.index[start_index:], df['row_group_size'][start_index:]):
            rows_this_worker += rg_size
            if rows_this_worker >= rows_per_worker:
                end_index = i + 1
                break
        total_rows -= rows_this_worker
        remaining_workers -= 1
        rows_per_worker = total_rows / remaining_workers

    return df[end_index:] if last_worker else df[start_index:end_index]

def construct_df(fmd: pa._parquet.FileMetaData) -> pd.DataFrame:
    """
    construct a DataFrame with three columns
    for calculating row_groups per worker
    """
    row_group_indices = []
    row_group_sizes = []
    files = []
    rg_index = 0
    for i in range(fmd.num_row_groups):
        rg = fmd.row_group(i)
        worker_file = rg.column(0).file_path
        row_group_sizes.append(rg.num_rows)
        if rg_index > 0 and worker_file != files[len(files) - 1]:
            rg_index = 0
        row_group_indices.append(rg_index)
        rg_index += 1
        files.append(worker_file)

    return pd.DataFrame({'row_group_index': row_group_indices, 'row_group_size': row_group_sizes, 'file': files})

def gather_metadata_save(outfile, env: CylonEnv):
    fmd = pq.read_metadata(outfile)
    fmd.set_file_path(outfile)

    gather_root = 0
    buf = serialize_metadata_file(fmd=fmd)
    buffers = gather_buffer(buf, root=gather_root, context=env.context)
    if gather_root != env.rank:
        return outfile

    fmd_all = deserialize_metadata_buffers(buffers)
    fmd_single = fmd_all[0]
    [fmd_single.append_row_groups(fmd) for fmd in fmd_all[1:]]

    outfile_rp = os.path.realpath(outfile)
    meta_file = os.path.join(os.path.dirname(outfile_rp), "_metadata")
    fmd_single.write_metadata_file(meta_file)


from setuptools import setup

PACKAGE = 'FlashView'
VERSION = '0.1.3'

setup(name=PACKAGE,
      version=VERSION,
      packages=['flashview'],
      author = 'Yuanying',
      author_email = 'yuanying@fraction.jp',
      url = 'http://trac-hacks.org/wiki/FlashViewPlugin',
      description = 'Flash view plugin for Trac',
      license='BSD',
      entry_points={'trac.plugins': '%s = flashview' % PACKAGE},
)


import pandas as pd
import datetime
import win32com.client as win32
import yfinance as yf
from matplotlib import pyplot as plt

# import matplotlib.pyplot as plt
# import matplotlib.style as style

if __name__ == '__main__':
    print("Relatorio mercado financeiro!")

#-----------------BUSCANDO OS DADOS E SENTANDO--------
minhas_acoes = ["MGLU3.SA"]

# Definindo quais ativos vou pegar as cotações do yahoo
codigos_de_negociacao = ["^BVSP", "BRL=X"]

# Pegando a data de hoje
hoje = datetime.datetime.now()

# pegando um perido de 1 ano a partir de hoje
um_ano_atras = hoje - datetime.timedelta(days=365)

# Baixando os dados do yahoo conforme parametros acima
dados_mercado = yf.download(codigos_de_negociacao, um_ano_atras, hoje)
periodo = um_ano_atras.date() - hoje.date()

#-------------MANIPULANDO OS DADOS-------------

# Renomeando os nomes das tabelas
dados_fechamento = dados_mercado['Adj Close']
dados_fechamento.columns = ["dolar", "ibovespa"]

# Excluindo dados onde está vazio e pegando os 3 primeiros dados
exclua_dados_faltante = dados_fechamento.dropna()
primeiros_50_dados = exclua_dados_faltante.head(20)

print("Abaixo consta a cotação em um periodo de", periodo)
print(primeiros_50_dados)

# Pegando e exibindos os dados por ano e por mes e por dia
dados_anuais = dados_fechamento.resample("y").last()
dados_mensais = dados_fechamento.resample("M").last()

retorno_anual = dados_anuais.pct_change().dropna()
retorno_mensal = dados_mensais.pct_change().dropna()
retorno_diario = dados_fechamento.pct_change().dropna()

print("Dados anuais", retorno_anual)
print("Dados mensais", retorno_mensal)
print("Dados diarios", retorno_diario)

#--------------LOCALIZANDO DADOS DENTRO DE UM DATAFRAME---------------------#

# loc -> referencia elementos a partir pelo nome
retorno_fev_17_2022 = retorno_diario.loc["2022-02-17", "ibovespa"]
print(retorno_fev_17_2022)

# iloc -> faz o mesmo mas por uma matriz, como se fosse o procv do excel
retorno_fev_17_2022 = retorno_diario.iloc[2, 1]
print(retorno_fev_17_2022)

# Pegando sempre o fechamento diariamente
retorno_diario_ibov = retorno_diario.iloc[-1, 1]
print("Fechamento do ultimo dia do ibov", retorno_diario_ibov)

# Transformando em %
retorno_diario_ibov_em_perct = round((retorno_diario_ibov * 100), 2)
print(retorno_diario_ibov_em_perct, "%")

#--------------------GERANDO O GRAFICO-------------------#

plt.style.use("cyberpunk")
# Definindo que o eixo vertical será os dados do ibov e os index as datas
dados_fechamento.plot(y = "ibovespa", use_index = True, legend = False)
plt.title("Ibovespa")

plt.savefig("ibovespa.png", dpi = 300)
plt.show()

#--------------------ENVIANDO OS DADOS POR EMAIL-------------------#

# Logando no outlook
outlook = win32.Dispatch("outlook.application")

email = outlook.CreateItem(0)

email.To = "matheushenriquesilva.th@gmail.com"
email.Subject = "Relatório Diário Ibov"
nome = "Matheus Henrique"

# Criando uma string personalizada com quebra de linhas
email.Body = f''' Olá {nome}, o fechamento do Ibov foi de {retorno_diario_ibov_em_perct}.

Obrigado!
'''

# Anexando arquivo e enviando o email
anexo_ibov = r"C:\Development\Python Projects\ibov-fechamento-relatorio\ibov\ibovespa.png"

email.Attachments.Add(anexo_ibov)

email.Send()


#!/usr/bin/env python3
# -*- coding:utf-8 -*-

# 注册蓝本 必须用下列顺序 避免陷入循环依赖
from flask import Blueprint

auth = Blueprint('auth', __name__)

from . import views

from ..models import SecondPageName, WebSetting, FriendLink, User

# 返回数据库内的页面标题以及url
@auth.context_processor
def get_page_names():
    """获得二级页面的内容以及id"""
    data = SecondPageName.query.all()
    names = {}
    for name in data:
        names[name.page_name] = name.id
    return dict(names=names)

@auth.context_processor
def get_setting():
    """获得设置"""
    data = WebSetting.query.get(1)
    if data is not None:
        data = data.to_json()
        return dict(setting=data)
    return dict(setting=None)

@auth.context_processor
def get_links():
    """获得设置"""
    links = FriendLink.query.all()
    return dict(links=[link.to_json() for link in links])


import numpy as np
from sklearn.linear_model import LinearRegression

y = np.array([3, 9, 20, 24, 30, 38])

x = [[1, 10, -19], [2, 15, -13], [3, 20, 0], [4, 25, 10], [5, 30, 14], [6, 35, 20]]
x = np.array(x)

model = LinearRegression().fit(x, y)

# Getting the slope of the regression line.
print('The estimates (b1-b2):', model.coef_)

# Getting the intercept value of the regression line.
print('The intercept (b0):', model.intercept_)

# Predicting the values of y based on the model.
y_Predicted = model.predict(x)
print('Predicted y values:', y_Predicted)

# Specifying a new set of x values to forecast y for.
x_New = [[7, 40, -21], [8, 45, -11], [9, 50, 2], [10, 55, 12], [11, 60, 13], [7, 34, 22]]
x_New = np.array(x_New)
y_Predicted_New = model.predict(x_New)
print('New predicted y values:', y_Predicted_New)


import csv
from flask.cli import FlaskGroup

from project import create_app, db
from project.api.models import Person

app = create_app()
cli = FlaskGroup(create_app=create_app)

@cli.command('recreate_db')
def recreate_db():
    """ Method to recreate a database
    """
    db.drop_all()
    db.create_all()
    db.session.commit()

@cli.command('seed_db')
def seed_db():
    """Command line argument to seed db with titanic csv

    Method loops over a csv file skipping first row and inserting values into
    respective rows.
    O, 1 for survived variable is converted to a boolean type.
    """

    with open('titanic.csv', 'r') as f:
        reader = csv.reader(f)
        next(reader)  # Skip the header row.
        for row in reader:
            survived_bool = None
            if row[0] == '1':
                survived_bool = True

            elif row[0] == '2':
                survived_bool = False

            db.session.add(Person(
                                survived=survived_bool,
                                passengerClass=row[1],
                                name=row[2],
                                sex=row[3],
                                age=int(float(row[4])),
                                siblingsOrSpousesAboard=row[5],
                                parentsOrChildrenAboard=row[6],
                                fare=row[7]
                                ))

    db.session.commit()

if __name__ == '__main__':
    cli()


import os
import json

BASE_DIR = os.path.expanduser('~/.suicarecorder')

FILE_PATH = os.path.join(BASE_DIR, 'config.json')

DEFAULTS = {
    'output_dir': BASE_DIR
}

class Config(dict):
    def __init__(self, file_path=FILE_PATH):
        self.file_path = file_path
        self._dict = None

    @property
    def dict(self):
        if not self._dict:
            if os.path.isfile(self.file_path):
                with open(self.file_path, 'r') as f:
                    self._dict = json.load(f, 'utf-8')
            else:
                self._dict = {}
        return self._dict

    def normalize_value(self, key, value):
        if key == 'output_dir':
            return os.path.expanduser(value)
        else:
            return value

    def load(self):
        for k, v in self.dict.items():
            _v = self.normalize_value(k, v)
            self.__setitem__(k, _v)

        return self

    def get(self, key):
        return dict.get(self, key, DEFAULTS.get(key))

_CONFIGS = {}

def get(key, config_file_path=None):
    file_path = config_file_path or FILE_PATH
    if not _CONFIGS.get(file_path):
        _CONFIGS[file_path] = Config(file_path).load()
    return _CONFIGS[file_path].get(key)


#Written by CRE. 20201013.
import sys
def calcOverlap(B1,E1,B2,E2):
    if B1<=E2 and B1>=B2:
        Overlap=min(E2-B1,E1-B1)
    elif E1>=B2 and E1<=E2:
        Overlap=E1-B2
    elif B2>=B1 and B2<=E1:#1 covers 2
        Overlap=E2-B2
    else:
        Overlap=-1
    return Overlap

def inclusion(In1,In2):#return 0: not overlapped, 1: overlapped, 2: In1 include In2, 3: In2 inlcude In1, 4: identical
    if In1==In2:
        return 4
    elif In1[0]>=In2[0] and In1[1]<=In2[1]:
        return 3
    elif In1[0]<=In2[0] and In1[1]>=In2[1]:
        return 2
    elif In1[0]>=In2[0] and In1[0]<=In2[1] or In1[1]>=In2[0] and In1[1]<=In2[1]:
        return 1
    return 0

class Variant:
    def __init__(self,Sample="",Type="",Chrom="",Start=0,End=0,SC=None,CS=None):
        self.Type=Type
        self.Chrom=Chrom
        self.Start=Start
        self.End=End
        self.Sample=Sample
        self.Length=self.End-self.Start
        self.CS=CS
        self.SC=SC
        self.MatchTo=None

    def __str__(self):
        Scores=""
        if self.SC!=None:
            Scores+=",SC:%s"%self.SC
        if self.CS!=None:
            Scores+=",CS:%s"%self.CS
        return "[%s]%s:%s-%s,%s,%s%s"%(self.Sample,self.Chrom,self.Start,self.End,self.End-self.Start,self.Type,Scores)

    def match(self,Other,Mode=0.8):#if Mode="inclusion" use inclusion, else use Mode percentage overlap
        if self.Type!=Other.Type or self.Chrom!=Other.Chrom:
            return False
        if Mode=="inclusion":
            if inclusion((self.Start,self.End),(Other.Start,Other.End))>1:
                return True
            return False
        else:
            Overlap=calcOverlap(self.Start,self.End,Other.Start,Other.End)
            if Overlap/self.Length>=Mode and Overlap/Other.Length>=Mode:
                return True
            return False

    def __lt__(self,other):
        if self.Chrom<other.Chrom:
            return True
        if self.Chrom>other.Chrom:
            return False
        if self.Start<other.Start:
            return True
        if self.Start>other.Start:
            return False
        if self.End<other.End:
            return True
        return False
    def __gt__(self,other):
        if self<other:
            return False
        if self.Chrom==other.Chrom and self.Start==other.Start and self.End==other.End:
            return False
        return True
    def __eq__(self,other):
        if other==None:
            return False
        if self<other:
            return False
        if self>other:
            return False
        return self.Sample==other.Sample

class Sample:
    def __init__(self,Name):
        self.Name=Name
        self.Variants=[]
        self.IndexParameter=10000
        #self.IndexedLength=0#list length last time indexing taken place
        self.Index={}#by chrom
        self.Chroms=[]
        self.ChromIndex=None

    def addVariant(self,NewVariant):
        self.Variants.append(NewVariant)
        AddChrom=True
        for c in self.Chroms:
            if c==NewVariant.Chrom:
                AddChrom=False
                break
        if AddChrom:
            self.Chroms.append(NewVariant.Chrom)
        self.Chroms.sort()
        self.Variants.sort()

    def addVariantRaw(self,NewVariant):#without sorting, need call force sort afterwards
        self.Variants.append(NewVariant)
        AddChrom=True
        for c in self.Chroms:
            if c==NewVariant.Chrom:
                AddChrom=False
                break
        if AddChrom:
            self.Chroms.append(NewVariant.Chrom)

    def forceSort(self):
        self.Variants.sort()
        self.Chroms.sort()

    def forceIndex(self):#must call after sorted
        self.ChromIndex=set()
        for c in self.Chroms:
            self.ChromIndex.add(c)
        if len(self.Variants)==0:
            self.Index={}
        else:
            self.Index={}
            IndexLength=int(self.Variants[-1].Start/self.IndexParameter)
            FirstStartOccurredAt=0
            if len(self.Chroms)!=0:
                Chrom=self.Chroms[0]
                self.Index[Chrom]=[0]
                i=0
                while FirstStartOccurredAt<len(self.Variants):
                    if self.Variants[FirstStartOccurredAt].Start>=i*self.IndexParameter:
                        self.Index[Chrom].append(FirstStartOccurredAt)
                        i+=1
                    if self.Variants[FirstStartOccurredAt].Chrom!=Chrom:
                        Chrom=self.Variants[FirstStartOccurredAt].Chrom
                        self.Index[Chrom]=[0]
                        i=0
                    FirstStartOccurredAt+=1

    def getStartIndex(self,Chrom,Start):
        if self.ChromIndex!=None:
            if Chrom not in self.ChromIndex:
                return len(self.Variants)
        else:
            NoChrom=True
            for c in self.Chroms:
                if c==Chrom:
                    NoChrom=False
                    break
            if NoChrom:
                return len(self.Variants)
        i=0
        if len(self.Index)!=0:
            if Chrom in self.Index.keys():
                try:
                    i=self.Index[Chrom][int(Start/self.IndexParameter)]
                except:#beyond index
                    i=self.Index[Chrom][-1]
        while i<len(self.Variants) and self.Variants[i].Start<Start:
            i+=1
        return i

    def match(self,other,Mode=0.8):# returns ([All matched self variants],per other matched variant)
        Result=[]
        if Mode!="inclusion":
            ratio=Mode
        for v in other.Variants:
            start=0 if v.Start-v.Length/ratio-1<0 else int(v.Start-v.Length/ratio-1)
            end=v.End+1
            #SI=self.getStartIndex(v.Chrom,start)
            #EI=self.getStartIndex(v.Chrom,end)
            SI=0
            EI=len(self.Variants)
            TempPair=([],v)
            for i in range(SI,EI):
                if self.Variants[i].match(v,Mode):
                    TempPair[0].append(self.Variants[i])
                    self.Variants[i].MatchTo=v
            if len(TempPair[0])!=0:
                Result.append(TempPair)
        return Result

class Record:
    def __init__(self,Chrom="",Start=0,End=0):
        self.Chrom=Chrom
        self.Start=Start
        self.End=End
        self.Length=self.End-self.Start
        self.Variants=[]
        self.Samples={}
    def addVariant(self,NewVariant):
        self.Variants.append(NewVariant)
        if NewVariant.Sample in self.Samples.keys():
            self.Samples[NewVariant.Sample].append(NewVariant)
        else:
            self.Samples[NewVariant.Sample]=[NewVariant]

import pysam

class VariantRecords:
    def __init__(self,Name=""):
        self.Variants=[]
        self.Name=Name
        self.Records=[]
        self.Samples={}

    def getSample(self, SampleName):
        if SampleName not in self.Samples.keys():
            self.Samples[SampleName]=Sample(SampleName)
        return self.Samples[SampleName]

    def matchSample(self,Other,SampleName,Mode=0.8):#other as standard
        try:
            return self.Samples[SampleName].match(Other.Samples[SampleName],Mode)
        except Exception:
            return None

    def printSample(self,SampleName):
        for v in self.Samples[SampleName].Variants:
            print(v)

    def matchAll(self,Other,Mode=0.8):#other as standard
        Result={}
        Result["Samples"]={}
        RSamples=Result["Samples"]
        Result["Self"]=self
        Result["Other"]=Other
        for SN in Other.Samples.keys():
            if SN in self.Samples.keys():
                RSamples[SN]=self.Samples[SN].match(Other.Samples[SN],Mode)
            else:
                RSamples[SN]=None
        return Result

    def interpret(self,maresult,PR=False,Merged=(0,0,0,0)):
        OtherN=0
        SelfN=0
        Matched=0
        SelfMatched=0
        OtherDupN=0
        OtherDelN=0
        MatchedDup=0
        MatchedDel=0
        SelfDupN=0
        SelfDelN=0
        SelfMatchedDup=0
        SelfMatchedDel=0

        RSamples=maresult["Samples"]
        Other=maresult["Other"]

        SelfN=len(self.Variants)
        OtherN=len(Other.Variants)
        for v in self.Variants:
            if v.Type.upper()=="DEL":
                SelfDelN+=1
            elif v.Type.upper()=="DUP":
                SelfDupN+=1
        for v in Other.Variants:
            if v.Type.upper()=="DEL":
                OtherDelN+=1
            elif v.Type.upper()=="DUP":
                OtherDupN+=1
        for SN in Other.Samples.keys():
            if RSamples[SN]!=None:
                Matched+=len(RSamples[SN])
                for i in range(len(RSamples[SN])):
                    SelfMatched+=len(RSamples[SN][i][0])
                    if RSamples[SN][i][1].Type.upper()=="DEL":
                        MatchedDel+=1
                        SelfMatchedDel+=len(RSamples[SN][i][0])
                    elif RSamples[SN][i][1].Type.upper()=="DUP":
                        MatchedDup+=1
                        SelfMatchedDup+=len(RSamples[SN][i][0])
        SelfDelN-=Merged[0]
        SelfDupN-=Merged[1]
        SelfN-=Merged[1]+Merged[0]
        SelfMatchedDel-=Merged[2]
        SelfMatchedDup-=Merged[3]
        SelfMatched-=Merged[2]+Merged[3]

        AS=Matched/OtherN if OtherN!=0 else 0
        AP=SelfMatched/SelfN if SelfN!=0 else 0
        AF1=2*(AS*AP)/(AS+AP) if (AS+AP)!= 0 else 0
        DS=MatchedDel/OtherDelN if OtherDelN!=0 else 0
        DP=SelfMatchedDel/SelfDelN if SelfDelN!=0 else 0
        DF1=2*(DS*DP)/(DS+DP) if (DS+DP)!= 0 else 0
        DuS=MatchedDup/OtherDupN if OtherDupN!=0 else 0
        DuP=SelfMatchedDup/SelfDupN if SelfDupN!=0 else 0
        DuF1=2*(DuS*DuP)/(DuS+DuP) if (DuS+DuP)!= 0 else 0
        Out= "All: Sensitivity: %s (%s/%s), PPV: %s (%s/%s), F1: %s\nDel: Sensitivity: %s (%s/%s), PPV: %s (%s/%s), F1: %s\nDup: Sensitivity: %s (%s/%s), PPV: %s (%s/%s), F1: %s"%(\
            AS,Matched,OtherN,AP,SelfMatched,SelfN,AF1\
                ,DS,MatchedDel,OtherDelN,DP,SelfMatchedDel,SelfDelN,DF1\
                    ,DuS,MatchedDup,OtherDupN,DuP,SelfMatchedDup,SelfDupN,DuF1
                )#PPV:positive predictive value,1-FDR
        if PR:
            for SN in Other.Samples.keys():
                if RSamples[SN]!=None:
                    for i in range(len(RSamples[SN])):
                        Out+="\nGS:%s..."%RSamples[SN][i][1]
                        for j in range(len(RSamples[SN][i][0])):
                            if j!=0:
                                Out+=","
                            Out+="%s"%RSamples[SN][i][0][j]
        Out+="\n%s\t%s\t%s\t%s\n%s\t%s\t%s\t%s"%(OtherDelN,MatchedDel,SelfDelN,SelfMatchedDel,OtherDupN,MatchedDup,SelfDupN,SelfMatchedDup)
        return Out

    def parseVcfCNV(self,filename,contigs=None,samples=None,MinLength=0,MaxAF=1,MinScore=0):
        vcf=pysam.VariantFile(filename,"r")
        result=[]
        included={}
        includedSamples={}
        NoAFInfoWarn=False
        if contigs!=None:
            for c in contigs:
                included[c]=True
        if samples!=None:
            for s in samples:
                includedSamples[s.upper()]=True
        for record in vcf.fetch():
            End=record.pos
            try:
                End=record.pos+abs(record.info["SVLEN"])
            except:
                try:
                    End=record.pos+abs(record.info["SVLEN"][0])
                except:
                    try:
                        End=record.info["END"]
                    except:
                        try:
                            End=record.stop
                        except:
                            pass
            try:
                Length=End-record.pos
                if Length<MinLength:
                    continue
                chrom=record.chrom
                #if len(record.chrom)<3 and (int(record.chrom[:2])<23 or record.chrom.upper()=="X" or record.chrom.upper()=="Y"):
                #        chrom="chr"+record.chrom
                if contigs!=None:
                    if not chrom in included.keys():
                        continue
                CNAlts=[1]
                tempRecord=Record(chrom,record.pos,End)
                SC=None
                try:
                    SC=float(record.info["SC"])
                except:
                    pass
                if MinScore!=0 and SC!=None:
                    if SC<MinScore:
                        continue
                AFs=[1]
                Ai=0
                CNVAlt=False
                for a in record.alts:
                    if "<CNV>" in a:
                        CNVAlt=True
                    elif "<CN" in a:
                        CNAlts.append(int(a[a.find("<CN")+3:-1]))
                    elif "DEL" in a.upper():
                        CNAlts.append(0)
                    elif "DUP" in a.upper():
                        CNAlts.append(2)
                    else:
                        CNAlts.append(1)
                    AF=1
                    try:
                        AF=record.info["AF"][Ai]
                    except:
                        NoAFInfoWarn=True
                    AFs.append(AF)
                    Ai+=1
                try:
                    for s in record.samples:
                        SampleName=record.samples[s].name
                        if samples==None or SampleName in includedSamples.keys():
                            if record.samples[s].allele_indices!=(0,0):
                                CN=2
                                CS=None
                                try:
                                    CS=record.samples[s]["CS"]
                                except:
                                    pass
                                if CNVAlt==True:
                                    CN=record.samples[s]["CN"]
                                elif record.info["SVTYPE"]=="DEL" or record.info["SVTYPE"]=="DUP" or record.info["SVTYPE"]=="CNV":
                                    CN=CNAlts[record.samples[s].allele_indices[0]]+CNAlts[record.samples[s].allele_indices[1]]
                                if CN==2:
                                    continue
                                if not CNVAlt and AFs[record.samples[s].allele_indices[0]]>MaxAF and AFs[record.samples[s].allele_indices[1]]>MaxAF:
                                    continue
                                else:
                                    if CN<2:
                                        SVType="DEL"
                                    else:
                                        SVType="DUP"
                                    self.Variants.append(Variant(SampleName,SVType,chrom,record.pos,End,SC,CS))
                                    self.getSample(SampleName).addVariantRaw(self.Variants[-1])
                                    tempRecord.addVariant(self.Variants[-1])
                except Exception as e:
                    continue
                self.Records.append(tempRecord)
            except:
                continue
        for SN in self.Samples.keys():
            self.Samples[SN].forceSort()
            self.Samples[SN].forceIndex()
        vcf.close()
        if MaxAF!=1 and NoAFInfoWarn:
            print("[WARN] There are alleles don't have AF info!",file=sys.stderr)
        return self.Records


from pymongo import MongoClient

class BookSaver:
    def __init__(self):
        mongo_client = MongoClient('localhost', 27017)
        self.books = mongo_client.book_db

    def insert_chapter(self, book_name, chapter):
        book_chapter = {
            'book_name': book_name,
            'chapter_title': chapter['chapter_title'],
            'index': chapter['index'],
            'url': chapter['url']
        }
        self.books[book_name].insert_one(book_chapter)

    def get_chapter_by_index(self, index, book_name):
        return self.books[book_name].find_one({'index': index})


import pickle
import random
from collections import defaultdict as dd

from keras.layers.core import Dense, Layer, Lambda,Activation
from keras.layers.core import initializations, activations
from keras.layers.embeddings import Embedding
from keras.models import Graph, Sequential
from theano import tensor as T
from keras import backend as K
import logging

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

data = pickle.load(open("../data/person_pub_data.pkl", "rb"))
sorted_names = pickle.load(open("sorted_names.pkl", "rb"))
name_to_idx = pickle.load(open("name_to_idx.pkl", "rb"))

title_df = dd(int)
venue_df = dd(int)
aff_df = dd(int)

class NodeContextProduct(Layer):
    '''
        This layer turns a pair of words (a pivot word + a context word,
        ie. a word from the same context, or a random, out-of-context word),
        indentified by their index in a vocabulary, into two dense reprensentations
        (word representation and context representation).
        Then it returns activation(dot(pivot_embedding, context_embedding)),
        which can be trained to encode the probability
        of finding the context word in the context of the pivot word
        (or reciprocally depending on your training procedure).
        The layer ingests integer tensors of shape:
        (nb_samples, 2)
        and outputs a float tensor of shape
        (nb_samples, 1)
        The 2nd dimension encodes (pivot, context).
        input_dim is the size of the vocabulary.
        For more context, see Mikolov et al.:
            Efficient Estimation of Word reprensentations in Vector Space
            http://arxiv.org/pdf/1301.3781v3.pdf
    '''

    def __init__(self, input_dim, proj_dim=128,
                 init='uniform', activation='sigmoid', weights=None):
        super(NodeContextProduct, self).__init__()
        self.input_dim = input_dim
        self.proj_dim = proj_dim
        self.init = initializations.get(init)
        self.activation = activations.get(activation)

        self.input = T.imatrix()
        # two different embeddings for pivot word and its context
        # because p(w|c) != p(c|w)
        self.W_w = self.init((input_dim, proj_dim))
        self.W_c = self.init((input_dim, proj_dim))

        self.params = [self.W_w, self.W_c]

        if weights is not None:
            self.set_weights(weights)

    def get_output(self, train=False):
        X = self.get_input(train)
        w = self.W_w[X[:, 0]]  # nb_samples, proj_dim
        c = self.W_c[X[:, 1]]  # nb_samples, proj_dim

        dot = T.sum(w * c, axis=1)
        dot = T.reshape(dot, (X.shape[0], 1))
        return self.activation(dot)

    def get_config(self):
        return {
            "name": self.__class__.__name__,
            "input_dim": self.input_dim,
            "proj_dim": self.proj_dim,
            "init": self.init.__name__,
            "activation": self.activation.__name__}

class IntInput(Layer):
    def build(self):
        super(IntInput, self).build()
        self.input = K.placeholder(self._input_shape, dtype='int32')

"""
num_node: number of entity node to be embedded
node_attr_mapping: attributes of each node type
attr_shape: original dimension of each attribute
embedding_dim: dimension of the embedding space
"""
def build_network(num_node, path_type, node_attr_mapping, attr_shape, embedding_dim=100):
    node_type_idx = {}
    attr_type_idx = {}
    num_node_type = 0
    num_attr_type = 0
    for n in node_attr_mapping:
        node_type_idx[n] = num_node_type
        num_node_type += 1
        for a in node_attr_mapping[n]:
            attr_type_idx[a] = num_attr_type
            num_attr_type += 1

    output_loss = []

    model = Graph()

    for i, n in enumerate(path_type):
        for leg in ["left", "right"]:
            model.add_input(name=leg + '_vertex_' + str(i), input_shape=(1,), dtype='int32')
    # shared embedding
    model.add_shared_node(layer=Embedding(input_dim=num_node, output_dim=embedding_dim, input_length=1),
                          name="embedding_pivot",
                          inputs=[(leg + '_vertex_0') for leg in ('left', 'right')],
                          outputs=[(leg + '_embeding_vertex_0') for leg in ('left', 'right')])
    model.add_shared_node(layer=Embedding(input_dim=num_node, output_dim=embedding_dim, input_length=1),
                          name="embedding_context",
                          inputs=[(leg + '_vertex_' + str(i)) for leg in ('left', 'right') for i in range(1, len(path_type))],
                          outputs=[(leg + '_embeding_vertex_' + str(i)) for leg in ('left', 'right') for i in range(1, len(path_type))])

    for i, n in enumerate(path_type):
        attr_layers = []
        for j, a in enumerate(node_attr_mapping[n]):
            for leg in ["left", "right"]:
                model.add_input(name="%s_input_attr_%s_%s" % (leg, i, j), input_shape=(attr_shape[a], ))
            model.add_shared_node(layer=Dense(output_dim=embedding_dim),
                                  name="encoder_attr_%s_%s" % (i, j),
                                  inputs=["%s_input_attr_%s_%s" % (leg, i, j) for leg in ["left", "right"]],
                                  outputs=["%s_encoded_attr_%s_%s" % (leg, i, j) for leg in ["left", "right"]])
            attr_layers.append("_encoded_attr_%s_%s" % (i, j))
            for leg in ["left", "right"]:
                model.add_output("%s_output_vertex_%s_%s" % (leg, i, j),
                                 inputs=["%s_encoded_attr_%s_%s" % (leg, i, j), "%s_embeding_vertex_%s" % (leg, i)],
                                 merge_mode="dot")
                output_loss.append(("%s_output_vertex_%s_%s" % (leg, i, j), "binary_crossentropy"))

        # add vertex attributes loss
    # for i, n in enumerate(path_type):
    #     for leg in ["left", "right"]:
    #         model.add_node(Dense(output_dim=embedding_dim), name="%s_embedding_attr_%s" % (leg, i),
    #                        inputs=[leg+layer_name for layer_name in attr_layers], merge_mode="sum")
    #         model.add_output("%s_output_vertex_%s" % (leg, i),
    #                          inputs=["%s_embedding_attr_%s" % (leg, i), "%s_embeding_vertex_%s" % (leg, i)],
    #                          merge_mode="dot")
    #         output_loss.append(("%s_output_vertex_%s" % (leg, i), "binary_crossentropy"))

    # add local context loss
    for leg in ["left", "right"]:
        for i in range(1, len(path_type)):
            model.add_output("%s_output_edge_%s" % (leg, i),
                             inputs=["%s_embeding_vertex_%s" % (leg, 0), "%s_embeding_vertex_%s" % (leg, i)],
                             merge_mode="dot")
            output_loss.append(("%s_output_edge_%s" % (leg, i), "binary_crossentropy"))

    # add alignment loss
    model.add_output("output_alignment",
                     inputs=["left_embeding_vertex_0", "right_embeding_vertex_0"],
                     merge_mode="dot")
    output_loss.append(("output_alignment", "binary_crossentropy"))

    model.compile('sgd', loss=dict(output_loss))
    return model

vocab = pickle.load(open("vocab.pkl", "rb"))
token_to_idx = pickle.load(open("token_to_idx.pkl", "rb"))
idx_to_token = pickle.load(open("idx_to_token.pkl", "rb"))
fvectors = pickle.load(open("fvectors.pkl", "rb"))

def sample_pair(num_names, num_node, neg_rate):
    current_name = random.randint(0, num_names - 1)
    name = sorted_names[current_name]
    pubs = name_to_idx[name]
    if len(pubs) < 2:
        return []
    true_pair = tuple([data[k[0]]["a"][k[1]]["i"] for k in random.sample(name_to_idx[name], 2)])
    samples = [(true_pair, 1)]
    for i in [(0, 1), (1, 0)]:
        for j in range(neg_rate):
            idx = random.randint(0, num_node - 1)
            false_pair = [-1, -1]
            false_pair[i[0]] = true_pair[i[0]]
            if "a" in data[idx]:
                if len(data[idx]["a"]) == 0:
                    continue
                false_pair[i[1]] = random.sample(data[idx]["a"], 1)[0]["i"]
            else:
                false_pair[i[1]] = idx
            samples.append((tuple(false_pair), 0))
    return samples

def get_path(node, neg_rate, num_vertex):
    pathes = []
    # print(node)
    # print(data[node]["p"])
    pub = data[data[node]["p"]]
    for a in pub["a"]:
        if not a["i"] == data[node]["i"]:
            path = ((node, fvectors['n'][node], fvectors['o'][node]),
                    (pub["i"], fvectors['t'][pub['i']], fvectors['k'][pub['i']], fvectors['v'][pub['i']]),
                    (a['i'], fvectors['n'][a['i']], fvectors['o'][a['i']]), 1, 1)
            # path = ((node, data[node]['n'], data[node]["o"]),
            #         (pub["i"], pub["t"], pub["k"], pub["v"]),
            #         (a["i"], a["n"], a["o"]), 1, 1)
            pathes.append(path)
    for i in range(neg_rate):
        idx = random.randint(0, num_vertex - 1)
        if "a" in data[idx]:
            if len(data[idx]["a"]) == 0:
                continue
            a_idx = random.randint(0, len(data[idx]["a"]) - 1)
            p_id = data[idx]["i"]
            a_id = data[idx]["a"][a_idx]["i"]
            path = ((node, fvectors['n'][node], fvectors["o"][node]),
                    (p_id, fvectors["t"][p_id], fvectors["k"][p_id], fvectors["v"][p_id]),
                    (a_id, fvectors["n"][a_id], fvectors["o"][a_id]),
                    0, 0)
            # path = ((node, data[node]['n'], data[node]["o"]),
            #         (data[idx]["i"], data[idx]["t"], data[idx]["k"], data[idx]["v"]),
            #         (data[idx]["a"][a_idx]["i"], data[idx]["a"][a_idx]["n"], data[idx]["a"][a_idx]["o"]),
            #         0, 0)
            pathes.append(path)
        else:
            p_id = pub["i"]
            a_id = data[idx]["i"]
            path = ((node, fvectors['n'][node], fvectors["o"][node]),
                    (p_id, fvectors["t"][p_id], fvectors["k"][p_id], fvectors["v"][p_id]),
                    (a_id, fvectors["n"][a_id], fvectors["o"][a_id]),
                    1, 0)
            pathes.append(path)
    return pathes

def get_context_pair(pair):
    pathes = []
    for i, leg in enumerate(["left", "right"]):
        pathes.append(get_path(pair[i], 10, 100))
    path_pairs = []
    for p1 in pathes[0]:
        for p2 in pathes[1]:
            path_pairs.append((p1, p2))
    return path_pairs

import numpy as np
from scipy import sparse
def gen_batch():
    num_names = 800000
    num_node = len(data)
    neg_rate = 10
    batch_size = 1
    while True:
        instances = dd(list)
        for _ in range(batch_size):
            samples = sample_pair(num_names, num_node, neg_rate)
            if len(samples) == 0:
                continue
            for s in samples:
                path_pairs = get_context_pair(s[0])
                legs = ["left", "right"]
                for p in path_pairs:
                    instances["output_alignment"].append(s[1])
                    for i, n in enumerate(p):
                        # outputs
                        instances["%s_output_edge_1" % legs[i]].append(n[3])
                        instances["%s_output_edge_2" % legs[i]].append(n[4])

                        # vertex index
                        for k1 in range(3):
                            instances["%s_vertex_%s" % (legs[i], k1)].append(np.array(n[k1][0]))
                        # attribute feature vector
                        for k1 in range(3):
                            for k2 in range(1, len(n[k1])):
                                instances["%s_input_attr_%s_%s" % (legs[i], k1, k2-1)].append(n[k1][k2])
                                instances["%s_output_vertex_%s_%s" % (legs[i], k1, k2-1)].append(1)
        for k in instances:
            instances[k] = sparse.vstack(instances[k]).toarray()
        yield instances

def get_context_graph(idx):
    graph = {}
    graph["vertex_pub"] = [idx]
    graph["vertex_author"] = [a["i"] for a in data[idx]["a"]]
    graph["input_attr_name"] = []
    graph["input_attr_org"] = []
    graph["input_attr_title"] = []
    graph["input_attr_venue"] = []
    return graph

def run():
    num_node = len(data)
    node_attr_mapping = {
        "pub": ["t", "v", "k"],
        "author": ["n", "o"]
    }
    path_type = ["author", "pub", "author"]
    attr_shape = {}
    for t in ["n", "o", "t", "v", "k"]:
        attr_shape[t] = fvectors[t].shape[1]
    model = build_network(num_node, path_type, node_attr_mapping, attr_shape)
    epoch = 0
    for instances in gen_batch():
        print(len(instances["output_alignment"]))
        if len(instances["output_alignment"]) == 0:
            continue
        print(epoch)
        epoch += 1
        loss = model.fit(instances)
        print(loss)


from django.urls import path
from article import views
#
urlpatterns = [
  path('',views.articleIndex , name='article_home'),
  path('createnew',views.createNew , name='create_new'),
  path('delete/<str:id>',views.deteleArticle , name='delete_article'),
  path('edit/<str:id>',views.editArticle , name='edit_article'),
  path('likearticle/<str:article_id>',views.likeArticle , name='like_article'),
  path('likecomment/<str:commentId>',views.likeComment , name='like_comment'),
  path('postacomment/<str:id>',views.postAComment , name='post_a_comment'),
  path('sendreply/<int:id>',views.sendReplay , name='sent_replay'),
  # path('getarticlesof/<str:username>',views.getArticleForASpecificUser , name='get_article_for_username'),
  path('<str:id>',views.previewArticle , name='article_preview'),
]


#!/usr/bin/python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import hashlib
import logging
import os
from optparse import OptionParser

logger = logging.getLogger("checksums.py")

def digest_file(filename, digest, chunk_size=131072):
    """Produce a checksum for the file specified by 'filename'.  'filename'
    is a string path to a file that is opened and read in this function.  The
    checksum algorithm is specified by 'digest' and is a valid OpenSSL
    algorithm.  If the digest used is not valid or Python's hashlib doesn't
    work, the None object will be returned instead.  The size of blocks
    that this function will read from the file object it opens based on
    'filename' can be specified by 'chunk_size', which defaults to 1K"""
    assert not os.path.isdir(filename), "this function only works with files"

    logger.debug("Creating new %s object" % digest)
    h = hashlib.new(digest)
    with open(filename, "rb") as f:
        while True:
            data = f.read(chunk_size)
            if not data:
                logger.debug("Finished reading in file")
                break
            h.update(data)
    hash = h.hexdigest()
    logger.debug("Hash for %s is %s" % (filename, hash))
    return hash

def process_files(dirs, output_filename, digests):
    """This function takes a list of directory names, 'drs'. It will then
    compute the checksum for each of the files in these by by opening the files.
    Once each file is read and its checksum is computed, this function
    will write the information to the file specified by 'output_filename'.
    The path written in the output file will have anything specified by 'strip'
    removed from the path.  The output file is closed before returning nothing
    The algorithm to compute checksums with can be specified by 'digests'
    and needs to be a list of valid OpenSSL algorithms.

    The output file is written in the format:
        <hash> <algorithm> <filesize> <filepath>
    Example:
        d1fa09a<snip>e4220 sha1 14250744 firefox-4.0b6pre.en-US.mac64.dmg
    """

    if os.path.exists(output_filename):
        logger.debug('Overwriting existing checksums file "%s"' % output_filename)
    else:
        logger.debug('Creating a new checksums file "%s"' % output_filename)
    with open(output_filename, "w+") as output:
        for d in dirs:
            for root, dirs, files in os.walk(d):
                for f in files:
                    full = os.path.join(root, f)
                    rel = os.path.relpath(full, d)

                    for digest in digests:
                        hash = digest_file(full, digest)

                        output.write(
                            "%s %s %s %s\n" % (hash, digest, os.path.getsize(full), rel)
                        )

def setup_logging(level=logging.DEBUG):
    """This function sets up the logging module using a speficiable logging
    module logging level.  The default log level is DEBUG.

    The output is in the format:
        <level> - <message>
    Example:
        DEBUG - Finished reading in file"""

    logger = logging.getLogger("checksums.py")
    logger.setLevel(logging.DEBUG)
    handler = logging.StreamHandler()
    handler.setLevel(level)
    formatter = logging.Formatter("%(levelname)s - %(message)s")
    handler.setFormatter(formatter)
    logger.addHandler(handler)

def main():
    """This is a main function that parses arguments, sets up logging
    and generates a checksum file"""
    # Parse command line arguments
    parser = OptionParser()
    parser.add_option(
        "-d",
        "--digest",
        help="checksum algorithm to use",
        action="append",
        dest="digests",
    )
    parser.add_option(
        "-o",
        "--output",
        help="output file to use",
        action="store",
        dest="outfile",
        default="checksums",
    )
    parser.add_option(
        "-v",
        "--verbose",
        help="Be noisy (takes precedence over quiet)",
        action="store_true",
        dest="verbose",
        default=False,
    )
    parser.add_option(
        "-q",
        "--quiet",
        help="Be quiet",
        action="store_true",
        dest="quiet",
        default=False,
    )

    options, args = parser.parse_args()

    # Figure out which logging level to use
    if options.verbose:
        loglevel = logging.DEBUG
    elif options.quiet:
        loglevel = logging.ERROR
    else:
        loglevel = logging.INFO

    # Set up logging
    setup_logging(loglevel)

    # Validate the digest type to use
    if not options.digests:
        options.digests = ["sha1"]

    for i in args:
        if not os.path.isdir(i):
            logger.error("%s is not a directory" % i)
            exit(1)

    process_files(args, options.outfile, options.digests)

if __name__ == "__main__":
    main()


# coding=utf-8
"""
Track modification on SQLAlchemy objects.
"""
from contextlib import contextmanager
from datetime import datetime

from src.constants import CTX_ADMIN
from src.interface_adapter.sql.model.models import Modification, Utilisateur
from src.interface_adapter.sql.model.trackable import RubyHashTrackable

@contextmanager
def track_modifications(ctx, session, obj: RubyHashTrackable):
    """
    Track the modifications of the specified entry and create a new entry in the modification table containing the diff.

    Object must inherit from RubyHashTrackable.
    """
    snap_before = obj.take_snapshot()  # Save the state of the object before actually modifying it.
    try:
        yield
    finally:
        diff = _get_diff(session, snap_before, obj)
        if diff is None:
            return  # No modification.

        now = datetime.now()
        admin = ctx.get(CTX_ADMIN)
        member = obj.get_related_member()

        admin_id = session.query(Utilisateur).filter(Utilisateur.login == admin.login).one().id

        m = Modification(
            adherent_id=member.id,
            action=diff,
            created_at=now,
            updated_at=now,
            utilisateur_id=admin_id,
        )
        session.add(m)

def _get_diff(session, snap_before, obj):
    if obj in session.new:
        return obj.serialize_snapshot_diff(None, obj.take_snapshot())

    if obj in session.deleted:
        return obj.serialize_snapshot_diff(snap_before, None)

    # Not deleted nor created, flush it to the database (not commit!) and compare the current state with the old state.
    #
    # Note: We need to flush here to put the object in a consistent state. For instance if we modify Member.room the
    # foreign key Member.room_id will not be updated until we flush it.
    #
    # Note2: flushing is not commit! The transaction can still be roll-backed if something goes wrong.
    session.flush()
    if snap_before != obj.take_snapshot():
        return obj.serialize_snapshot_diff(snap_before, obj.take_snapshot())


#!/usr/bin/env python
# coding: utf-8

# ---------------------------
# Import modules ------------
# ---------------------------
from splinter import Browser
from bs4 import BeautifulSoup
import pandas as pd
import time

# ---------------------------
# Initialize browser --------
# ---------------------------
def init_browser():
    # @NOTE: Replace the path with your actual path to the chromedriver
    executable_path = {"executable_path": "chromedriver"}
    return Browser("chrome", **executable_path, headless=False)

def scrape():
    browser = init_browser()

    # ---------------------------
    # Retrieve headline & summary
    # ---------------------------
    url = "https://mars.nasa.gov/news/"
    browser.visit(url)
    time.sleep(1)

    # Assistance from Benjamen Alford - change from [] to {}
    mars_info = {}

    html = browser.html
    soup = BeautifulSoup(html, "html.parser")
    results = soup.find("li", class_="slide")
    for r in results:
        try:
            mars_info = {}
            mars_info["news_title"] = r.find("div", class_="content_title").get_text()
            mars_info["news_p"] = r.find("div", class_="article_teaser_body").get_text()

        except:
            break

    # -----------------------
    # Retrieve featured image
    # -----------------------
    img_site = "https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars"
    browser.visit(img_site)
    time.sleep(1)
    browser.links.find_by_partial_text('FULL').click()
    time.sleep(1)
    browser.links.find_by_partial_href('/spaceimages/details').click()
    time.sleep(1)
    browser.links.find_by_partial_href('/spaceimages/images/largesize/').click()

    # https://splinter.readthedocs.io/en/latest/browser.html#verifying-page-url-with-browser-url
    mars_info["featured_image_url"] = browser.url

    # -----------------------
    # Retrieve facts table --
    # -----------------------
    facts_url ='https://space-facts.com/mars/'
    mars_facts = pd.read_html(facts_url)

    facts_table = mars_facts[0]
    mars_info["facts_table"] = facts_table.to_html(header=False, index=False, justify="left", classes="table-striped")

    # ------------------------------------------------
    # Create dictionary for hemispheres and image urls
    # ------------------------------------------------
    hemisphere_image_urls = []
    hemisphere_names = []

    hems_url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
    browser.visit(hems_url)
    html = browser.html
    soup = BeautifulSoup(html, "html.parser")
    titles = soup.find_all("div", class_="description")
    for t in titles:
        try:
            result = []
            result = t.find("h3").get_text()
            hemisphere_names.append(result)
        except:
            pass

    for h in hemisphere_names:
        hems_url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
        browser.visit(hems_url)
        time.sleep(1)
        html = browser.html
        soup = BeautifulSoup(html, "html.parser")
        browser.links.find_by_partial_text(h).click()

        result = {}
        result["title"] = h
        html = browser.html
        soup = BeautifulSoup(html, "html.parser")
        result["img_url"] = soup.find("div", class_="downloads").find("a")["href"]
        hemisphere_image_urls.append(result)
    mars_info["cerberus"] = hemisphere_image_urls[0]
    mars_info["schia"] = hemisphere_image_urls[1]
    mars_info["syr"] = hemisphere_image_urls[2]
    mars_info["valles"] = hemisphere_image_urls[3]

    # mars_info["hemispheres"] = hemisphere_image_urls

    browser.quit()

    return mars_info


from django.contrib.auth.decorators import login_required
from django.shortcuts import render, get_object_or_404
from glide.app.inbox.models import Message, Conversation
from glide.app.inbox.forms import ReplyMessageForm, NewMessageForm
from django.db.models import Q
from django.http import HttpResponsePermanentRedirect
from glide.core.profiles.models import Profile
from glide.core.notifications.models import Notification
@login_required
def inbox(request):
	conversations = Conversation.objects.filter(Q(receiver=request.user) | Q(sender=request.user)).order_by('-last_message')
	return render(request, 'inbox/inbox.html', {'conversations':conversations})

def mark_as_read(user, conversation):
	messages = Message.objects.filter(conversation=conversation).exclude(read=True).exclude(author=user)
	for message in messages:
		message.read = True
		message.save()

@login_required
def view_conversation(request, id):
	conversation = get_object_or_404(Conversation, pk=id)
	mark_as_read(request.user, conversation)
	messages = Message.objects.filter(conversation=conversation).order_by('-date')
	return render(request, 'inbox/conversation.html', {'messages':messages,
														'conversation':conversation})

@login_required
def reply(request):
	if request.method == 'POST':
		form = ReplyMessageForm(request.POST)
		if form.is_valid():
			item = form.save(commit=False)
			item.author = request.user
			item.save()
			conversation = item.conversation
			conversation.save()
			return HttpResponsePermanentRedirect(conversation.get_absolute_url())

@login_required
def new_message(request, id):
	profile = get_object_or_404(Profile, pk=id)
	receiver = profile.user
	conversation = None
	if conversation == None:
		try:
			conversation = Conversation.objects.get(sender=request.user, receiver=receiver)
			return view_conversation(request, conversation.pk)
		except:
			pass
	if conversation == None:
		try:
			conversation = Conversation.objects.get(sender=receiver, receiver=request.user)
			return view_conversation(request, conversation.pk)
		except:
			pass

	form = NewMessageForm()
	if request.method == 'POST':
		form = NewMessageForm(request.POST)
		if form.is_valid():
			conversation = Conversation(sender=request.user, receiver=receiver)
			conversation.save()
			item = form.save(commit=False)
			item.conversation = conversation
			item.author = request.user
			item.save()
			return view_conversation(request, conversation.pk)
	return render(request, 'inbox/form.html', {'form':form})


"""Support for processing input/output."""

from contextlib import contextmanager
import os
import shutil
from tempfile import gettempdir, mkdtemp
from typing import Iterator

from topshelfsoftware_util.log import get_logger
logger = get_logger(__name__, stream=None)

@contextmanager
def cdtmp(tmpdir: str = None, sub_dirname: str = None, cleanup: bool = True) -> Iterator[None]:
    """Context manager to work in temporary directory.

    Provisions directory, changes to it, then removes once context is complete.

    >>> os.chdir('/home')
    >>> with cdtmp():
    >>>     # do stuff or
    >>>     raise Exception("There's no place like home.")
    >>> # Directory is now back to '/home' and the temporary directory no longer exists
    """
    if tmpdir is not None:
        os.environ["TMPDIR"] = tmpdir

    try:
        prev_dir = os.getcwd()
    except FileNotFoundError:
        prev_dir = gettempdir()
        logger.warning(f"Unable to get current working directory, using {prev_dir}")

    if sub_dirname is not None:
        temp_dir = os.path.join(gettempdir(), sub_dirname)
        os.makedirs(temp_dir, exist_ok=True)
    else:
        temp_dir = mkdtemp()
    logger.debug(f"temp directory: {temp_dir}")

    os.chdir(temp_dir)
    try:
        yield
    finally:
        os.chdir(prev_dir)
        if cleanup:
            shutil.rmtree(temp_dir, ignore_errors=True)
            logger.debug(f"deleted temp directory: {temp_dir}")


import os

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.utils.data
import torchvision
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils
import numpy as np
import matplotlib.pyplot as plt
from torch.autograd import Variable

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

class Generator(nn.Module):
    def __init__(self, latent_dim=128, out_channels=3, num_classes=10):
        super(Generator, self).__init__()
        self.latent_dim = latent_dim
        self.num_classes = num_classes
        self.out_channels = out_channels
        self.fc1 = nn.Linear(self.latent_dim,384)

        self.tconv1 = nn.Sequential(
            nn.ConvTranspose2d(in_channels=384, out_channels=192,  kernel_size=4, stride=1, padding=0),
            nn.BatchNorm2d(192),
            nn.LeakyReLU(0.2)
        )
        self.tconv2 = nn.Sequential(
            nn.ConvTranspose2d(in_channels=192, out_channels=96, kernel_size=4, stride=2, padding=1),
            nn.BatchNorm2d(96),
            nn.LeakyReLU(0.2)
        )
        self.tconv3 = nn.Sequential(
            nn.ConvTranspose2d(in_channels=96, out_channels=48, kernel_size=4, stride=2, padding=1),
            nn.BatchNorm2d(48),
            nn.LeakyReLU(0.2)
        )
        self.tconv4 = nn.Sequential(
            nn.ConvTranspose2d(in_channels=48, out_channels=self.out_channels, kernel_size=4, stride=2, padding=1),
            nn.LeakyReLU(0.2)
        )

    def forward(self, z, y):
        z = z.view(-1,self.latent_dim)
        z = self.fc1(z)
        z = z.view(-1,384,1,1)
        z = self.tconv1(z)
        z = self.tconv2(z)
        z = self.tconv3(z)
        z = self.tconv4(z)
        return z

class Discriminator(nn.Module):
    def __init__(self, in_channels=3):
        super(Discriminator, self).__init__()
        self.in_channels = in_channels

        self.conv1 = nn.Sequential(
            nn.Conv2d(in_channels=self.in_channels, out_channels=16, kernel_size=3, stride=2, padding=1),
            nn.LeakyReLU(0.2),
            nn.Dropout(0.5)
            )
        self.conv2 = nn.Sequential(
            nn.Conv2d(in_channels=16,out_channels=32, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(32),
            nn.LeakyReLU(0.2),
            nn.Dropout(0.2)
            )
        self.conv3 = nn.Sequential(
            nn.Conv2d(in_channels=32,out_channels=64, kernel_size=3, stride=2, padding=1),
            nn.BatchNorm2d(64),
            nn.LeakyReLU(0.2),
            nn.Dropout(0.2)
            )
        self.conv4 = nn.Sequential(
            nn.Conv2d(in_channels=64,out_channels=128, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(128),
            nn.LeakyReLU(0.2),
            nn.Dropout(0.2)
            )
        self.conv5 = nn.Sequential(
            nn.Conv2d(in_channels=128,out_channels=256, kernel_size=3, stride=2, padding=1),
            nn.BatchNorm2d(256),
            nn.LeakyReLU(0.2),
            nn.Dropout(0.2)
            )
        self.conv6 = nn.Sequential(
            nn.Conv2d(in_channels=256,out_channels=512, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(512),
            nn.LeakyReLU(0.2),
            nn.Dropout(0.2)
            )
        self.fc1 = nn.Linear(4*4*512,1)
        self.fc2 = nn.Linear(4*4*512,10)
        self.sig = nn.Sigmoid()
        self.soft = nn.Softmax()

    def forward(self, x):

        x = self.conv1(x)
        x = self.conv2(x)
        x = self.conv3(x)
        x = self.conv4(x)
        x = self.conv5(x)
        x = self.conv6(x)
        x = x.view(-1,4*4*512)
        dis = self.sig(self.fc1(x))
        aux = self.soft(self.fc2(x))
        return dis, aux

# -----
# Hyperparameters
classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')

batch_size = 256
workers = 6
latent_dim = 128
lr = 0.001
num_epochs = 300
validate_every = 1
print_every = 100

if not os.path.exists(os.path.join(os.path.curdir, "./visualize", "gan")):
    os.makedirs(os.path.join(os.path.curdir, "./visualize", "gan"))

ckpt_path = 'acgan.pt'
save_path = './visualise'

# -----
# Dataset
tfms = transforms.Compose([
    transforms.ToTensor(),
    ])

train_dataset = torchvision.datasets.CIFAR10(
    root='./data',
    train=True,
    download=True,
    transform=tfms)

train_loader = torch.utils.data.DataLoader(
    train_dataset,
    batch_size=batch_size,
    shuffle=True,
    num_workers=workers)

test_dataset = torchvision.datasets.CIFAR10(
    root='./data',
    train=False,
    download=True,
    transform=tfms)

test_loader = torch.utils.data.DataLoader(
    test_dataset,
    batch_size=batch_size,
    shuffle=False,
    num_workers=workers)

# -----
# Model

generator = Generator(latent_dim, 3)

discriminator = Discriminator()

# -----
# Losses

adv_loss = nn.BCELoss()
aux_loss = nn.NLLLoss()

if torch.cuda.is_available():
    generator = generator.cuda()
    discriminator = discriminator.cuda()
    adv_loss = adv_loss.cuda()
    aux_loss = aux_loss.cuda()

generator = generator.to(device)
discriminator = discriminator.to(device)
# Optimizers for Discriminator and Generator, separate

beta1 = 0.5
optimizer_D = optim.Adam(discriminator.parameters(), lr=lr,betas=(beta1, 0.999))
optimizer_G = optim.Adam(generator.parameters(), lr=lr,betas=(beta1, 0.999))

# -----
# Train loop

def denormalize(x):
    """Denomalize a normalized image back to uint8.
    """

    minimum = torch.min(x)
    maximum = torch.max(x)

    x = x*((255)/(maximum - minimum)) + 0
    x = x.cpu().numpy()
    x = np.transpose(x, (0, 2, 3, 1))
    return x.astype(np.uint8)

# For visualization part
# Generate 20 random sample for visualization

random_z = torch.FloatTensor(20, latent_dim, 1, 1)
random_z = Variable(random_z)
random_z.data.resize_(20, latent_dim, 1, 1).normal_(0, 1)
random_y = np.random.randint(0, 10, 20)
random_z_ = np.random.normal(0, 1, (20, latent_dim))
random_y_oh = np.zeros((20, 10))
random_y_oh[np.arange(20), random_y] = 1
random_z_[np.arange(20), :10] = random_y_oh[np.arange(20)]
random_z_ = (torch.from_numpy(random_z_))
random_z.data.copy_(random_z_.view(20, latent_dim, 1, 1))
random_z = random_z.to(device)

def train_step(x, y):
    """One train step for AC-GAN.
    You should return loss_g, loss_d, acc_d, a.k.a:
        - average train loss over batch for generator
        - average train loss over batch for discriminator
        - auxiliary train accuracy over batch
    """

    real_label = 1
    fake_label = 0

    optimizer_D.zero_grad()

    #optimize discriminator
    real_lab = torch.full(size = (x.shape[0],1), fill_value = real_label, dtype=torch.float16)
    if torch.cuda.is_available():
        real_lab = real_lab.cuda()

    dis_output, aux_output = discriminator(x)
    dis_loss = adv_loss(dis_output.float(), real_lab.float())
    aux_real_loss = aux_loss(aux_output,y)
    total_dis_real_loss = aux_real_loss + dis_loss
    total_dis_real_loss.backward()

    # calculate accuracy
    aux_pred = aux_output.max(1)[1]
    correct = torch.sum(aux_pred == y)
    aux_acc = (float(correct) / float(len(x)))

    noise = torch.FloatTensor(x.shape[0], latent_dim, 1, 1)
    noise = Variable(noise)
    noise.data.resize_(x.shape[0], latent_dim, 1, 1).normal_(0, 1)
    label = np.random.randint(0, 10, x.shape[0])
    noisenp = np.random.normal(0, 1, (x.shape[0], latent_dim))
    class_onehot = np.zeros((x.shape[0], 10))
    class_onehot[np.arange(x.shape[0]), label] = 1
    noisenp[np.arange(x.shape[0]), :10] = class_onehot[np.arange(x.shape[0])]
    noise_and_labels = (torch.from_numpy(noisenp))
    noise.data.copy_(noise_and_labels.view(x.shape[0], latent_dim, 1, 1))
    noise =noise.to(device)

    # fake output
    fake_gen_output = generator(noise, None)

    # discriminator output from fake data
    dis_output, aux_output = discriminator(fake_gen_output.detach())
    fake_lab = torch.full(size = (x.shape[0],1), fill_value = fake_label, dtype=torch.float16).to(device)

    # generator
    dis_error_fake = adv_loss(dis_output.to(device), fake_lab.float())
    aux_error_fake = aux_loss(aux_output, y)

    total_dis_fake_loss = dis_error_fake + aux_error_fake
    total_dis_fake_loss.backward()

    dis_loss = total_dis_real_loss + total_dis_fake_loss

    optimizer_D.step()

    # optimise generator
    optimizer_G.zero_grad()
    real_lab = torch.full((x.shape[0], 1), real_label).to(device)
    dis_output, aux_output = discriminator(fake_gen_output)
    dis_error_fake_G = adv_loss(dis_output.to(device), real_lab.float())
    aux_error_fake_G = aux_loss(aux_output, y)
    generator_loss = dis_error_fake_G + aux_error_fake_G
    generator_loss.backward()

    optimizer_G.step()

    return generator_loss.detach().cpu().numpy()/x.shape[0], dis_loss.detach().cpu().numpy()/ x.shape[0], aux_acc

def test(
    test_loader,
    ):
    """Calculate accuracy over Cifar10 test set.
    """
    size = len(test_loader.dataset)
    corrects = 0

    discriminator.eval()
    with torch.no_grad():
        for inputs, gts in test_loader:
            if torch.cuda.is_available():
                inputs = inputs.cuda()
                gts = gts.cuda()

            # Forward only
            _, outputs = discriminator(inputs)
            _, preds = torch.max(outputs, 1)
            corrects += torch.sum(preds == gts.data)

    acc = corrects / size
    print("Test Acc: {:.4f}".format(acc))
    return acc

g_losses = []
d_losses = []
best_acc_test = 0.0

for epoch in range(1, num_epochs + 1):
    generator.train()
    discriminator.train()

    avg_loss_g, avg_loss_d = 0.0, 0.0
    for i, (x, y) in enumerate(train_loader):
        if torch.cuda.is_available():
            x = x.cuda()
            y = y.cuda()

        # train step
        loss_g, loss_d, acc_d = train_step(x, y)
        avg_loss_g += loss_g * x.shape[0]
        avg_loss_d += loss_d * x.shape[0]

        # Print
        if i % print_every == 0:
            print("Epoch {}, Iter {}: LossD: {:.6f} LossG: {:.6f}, D_acc {:.6f}".format(epoch, i, loss_g, loss_d, acc_d))

    g_losses.append(avg_loss_g / len(train_dataset))
    d_losses.append(avg_loss_d / len(train_dataset))

    # Save
    if epoch % validate_every == 0:
        acc_test = test(test_loader)
        if acc_test > best_acc_test:
            best_acc_test = acc_test
            # Wrap things to a single dict to train multiple model weights
            state_dict = {
                "generator": generator.state_dict(),
                "discriminator": discriminator.state_dict(),
                }
            torch.save(state_dict, ckpt_path)
            print("Best model saved w/ Test Acc of {:.6f}.".format(best_acc_test))

        # Do some reconstruction
        generator.eval()
        with torch.no_grad():
            # Forward

            xg = generator(random_z.detach(), None)
            xg = denormalize(xg)

            # Plot 20 randomly generated images
            plt.figure(figsize=(10, 5))
            for p in range(20):
                plt.subplot(4, 5, p+1)
                plt.imshow(xg[p])
                plt.text(0, 0, "{}".format(classes[random_y[p].item()]), color='black',
                            backgroundcolor='white', fontsize=8)
                plt.axis('off')

            plt.savefig(os.path.join(os.path.join(save_path, "E{:d}.png".format(epoch))), dpi=300)
            plt.clf()
            plt.close('all')

        # Plot losses
        plt.figure(figsize=(10,5))
        plt.title("Generator and Discriminator Loss During Training")
        plt.plot(g_losses, label="G")
        plt.plot(d_losses, label="D")
        plt.xlabel("Epoch")
        plt.ylabel("Loss")
        plt.xlim([1, epoch])
        plt.legend()
        plt.savefig(os.path.join(os.path.join(save_path, "loss.png")), dpi=300)


# -*- coding: utf-8 -*-
import smtplib
import email.mime.text
import settings
import os.path

def send_template(template, info):
    """Send an email based on a template and a dictionary of data.

    Template is a dictionary of mail headers (key is header name,
    value is header value). The values are python format strings, and
    will be formatted with the info dictionary:

    mail_header[key] = template[key] % info

    The special "header" named "Body" is used to supply the body
    content of the message.
    """

    msg = email.mime.text.MIMEText(template['Body'] % info)
    for name, value in template.iteritems():
        if name != "Body":
            msg[name] = value % info

    if settings.MAILSERVER is None:
        f = open(os.path.join(os.path.dirname(__file__), "../mail.txt"), "a")
        f.write(msg.as_string())
        f.write("\n\n")
        f.close()
    else:
        s = smtplib.SMTP(*settings.MAILSERVER)
        s.connect()
        s.sendmail(msg["From"], msg["To"], msg.as_string())
        s.quit()


import pandas as pd
import matplotlib.pyplot as plt
import os
import argparse

def avg_drop(directory, benchmark):
    df = pd.DataFrame()
    file = pd.read_csv(f'{directory}/MLFFR_o1_t.txt', index_col=0)
    df["index"] = list(file.index)
    df.set_index("index", inplace=True)
    versions = ["o1", "o2", "k0", "k1", "k2", "k3", "k4"]
    if benchmark == "xdp2" or benchmark == "xdp_router_ipv4":
        versions.remove("k2")
    for i in versions:
        file = pd.read_csv(f'{directory}/MLFFR_{i}_t.txt', index_col=0)
        file['mean'] = file.mean(axis=1)
        #print(file)
        df[i] = file['mean']

    #print(df)
    fig = plt.figure()
    plt.plot(df, marker="o")
    plt.title(f'Measuring {benchmark} AVG Drop Rate')
    plt.xlabel('TX rate (Mpps)',)
    plt.ylabel('Drop Rate')
    plt.legend(df.columns)
    savePath = f'{directory}/drop/'
    if not os.path.exists(savePath):
        os.makedirs(savePath)
    fig.savefig(f'{savePath}/avg.png', bbox_inches='tight')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Information about Data')
    parser.add_argument('-d', dest="directory", type=str, help='Directory', required=True)
    parser.add_argument('-b', dest="benchmark", type=str, help='Benchmark', required=True)
    args = parser.parse_args()
    avg_drop(args.directory, args.benchmark)


#!/usr/bin/env python
import pika

# basic queue receiver example
credentials = pika.PlainCredentials('rabbitmq', 'rabbitmq')
parameters = pika.ConnectionParameters('queenbee.local',
                                       5673,
                                       '/',
                                       credentials)
connection = pika.BlockingConnection(parameters)

channel = connection.channel()

# declare our queue - this name needs to match value in send.py
channel.queue_declare(queue='hello')

# callback for received message
def callback(ch, method, properties, body):
    print(" [x] Received %r" % body)

# Register listener
# no_ack parameter turns off message acknowledgement
channel.basic_consume(callback,
                      queue='hello',
                      no_ack=True)

print(' [*] Waiting for messages. To exit press CTRL+C')

# loop until canceled
channel.start_consuming()


import bot
from settings import app_secrets
import time
import datetime as dt
from datetime import datetime
import os
import notify
import centre_schedule_scrapper

def current_date():
	now = datetime.now()
	dt_string = now.strftime("%d/%m/%Y")
	return dt_string

def isNowInTimePeriod(startTime, endTime, nowTime):
    if startTime < endTime:
        return nowTime >= startTime and nowTime <= endTime
    else:
        #Over midnight:
        return nowTime >= startTime or nowTime <= endTime

def send_doc(text):
	bot.bot.send_document(app_secrets._18plus_groupid, text , caption = "Vaccine Centre Plan List\nYou can contact ANMs incase you want")
	bot.bot.send_document(app_secrets._45plus_groupid, text , caption = "Vaccine Centre Plan List\nYou can contact ANMs incase you want")

bot.bot.send_message(app_secrets.Group_Owner , "Started...")
print("Started...")
print(dt.datetime.now().time())
num = 0
run_doc = 0
while True:
	if isNowInTimePeriod(dt.time(8,00), dt.time(9,00), dt.datetime.now().time()):
		try:
			if run_doc == 0:
				text = centre_schedule_scrapper.scrape()
				if text[1] == current_date():
					send_doc(text[0])
					run_doc = 1
		except Exception as e:
			bot.bot.send_message(app_secrets.Group_Owner , "Error!!!")
			bot.bot.send_message(app_secrets.Group_Owner , e)

	if isNowInTimePeriod(dt.time(9,00), dt.time(23,00), dt.datetime.now().time()):
		try:
			notify.RUN()
			print(num)
			num+=1
			time.sleep(int(os.environ.get('SLEEP_TIME')))
		except Exception as e:
			if "Max retries exceeded with url:" not in str(e):
				try:
					bot.bot.send_message(app_secrets.Group_Owner , "Error!!!")
					bot.bot.send_message(app_secrets.Group_Owner , e)
				except:
					print("............\n")
					print(e , end = "\n")
			else:
				print("............\n")
				print(e , end = "\n")

			time.sleep(5)

	if isNowInTimePeriod(dt.time(12,00), dt.time(13,00), dt.datetime.now().time()):
		run_doc = 0


#!/usr/bin/python
# -*- coding: utf-8 -*-
# coding=utf-8

import csv
import ldap
import logging
import os

from biblib.services import config_service

config = config_service.config["ldap"]

def ldap_search(ldap_base_dn, ldap_filter, ldap_attrs):
    ldap_con = ldap.initialize(config["host"], config["port"])
    ldap_con.simple_bind_s(config["dn"], config["password"])
    ldap_results = ldap_con.search_s(ldap_base_dn, ldap.SCOPE_SUBTREE, ldap_filter, ldap_attrs)
    results = []
    for dn, entry in ldap_results:
        results.append(entry)
    #logging.debug(results)
    return results

def ldap_search_user_by_identifier(identifier):
    ldap_base_dn = 'ou=Users,o=sciences-po, c=fr'
    ldap_filter = "(|(scpoLibraryNumber=*{0})(scpoBannerID=*{0})(scpoTeacherNumber=*{0})(scpoStudentNumber=*{0})(uid=*{0}))".format(identifier)
    ldap_attrs = ['uid', 'employeeType', 'scpoLibraryNumber', 'scpoBannerID', 'scpoTeacherNumber', 'scpoStudentNumber']
    ldap_search(ldap_base_dn, ldap_filter, ldap_attrs)

def ldap_search_user_by_employeetype(employeetype):
    ldap_base_dn = 'ou=Users,o=sciences-po, c=fr'
    ldap_filter = "(employeeType=*{0})".format(employeetype)
    ldap_attrs = ['uid', 'givenName', 'sn', 'employeeType', 'scpoActifSI', 'scpoLibraryNumber', 'scpoBannerID', 'scpoTeacherNumber', 'scpoStudentNumber', 'scpoDateEntree', 'scpoDateSortie', 'mailForwardingAddress', 'scpoBannerStatus', 'scpoSynchroBanner', 'scpoGestionnairePrincipal', 'scpoEmployeeDSP']
    results = ldap_search(ldap_base_dn, ldap_filter, ldap_attrs)
    csv_file_name = "".join(["ldap-", employeetype, ".csv"])
    csv_file_path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "data", "result", csv_file_name)
    with open(csv_file_path, "wb") as csv_file:
        fieldnames = ldap_attrs
        csvwriter = csv.DictWriter(csv_file, delimiter=',', fieldnames=fieldnames)
        csvwriter.writeheader()
        for item in results:
            for key, value in item.items():
                item[key] = "; ".join(sorted(value))
            csvwriter.writerow(item)

def ldap_search_user_by_scporesearchcommunity():
    ldap_base_dn = 'ou=Users,o=sciences-po, c=fr'
    ldap_filter = "(scpoResearchCommunity=1)"
    ldap_attrs = ['uid', 'givenName', 'sn', 'employeeType', 'scpoActifSI', 'scpoLibraryNumber', 'scpoBannerID', 'scpoTeacherNumber', 'scpoStudentNumber', 'scpoResearchResearchCenter', 'scpoDateEntree', 'scpoDateSortie', 'mailForwardingAddress', 'scpoBannerStatus', 'scpoSynchroBanner', 'scpoGestionnairePrincipal', 'scpoEmployeeDSP']
    results = ldap_search(ldap_base_dn, ldap_filter, ldap_attrs)
    csv_file_name = "".join(["ldap-scporesearchcommunity.csv"])
    csv_file_path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "data", "result", csv_file_name)
    with open(csv_file_path, "wb") as csv_file:
        fieldnames = ldap_attrs
        csvwriter = csv.DictWriter(csv_file, delimiter=',', fieldnames=fieldnames)
        csvwriter.writeheader()
        for item in results:
            for key, value in item.items():
                item[key] = "; ".join(sorted(value))
            csvwriter.writerow(item)

#ldap_search_user_by_identifier("11001829")
#ldap_search_user_by_employeetype("LECTEUR")
ldap_search_user_by_scporesearchcommunity()


import discord

from discord.ext import commands
from random import *

from .server_only_cog import *

vowels = ['a', 'e', 'i', 'o', 'u']
exclude = ["the", "i\'m", "and"]
markdownSymbols = ["*", "**", "***", "__", "~~", "`", "__*", "__**", "__***", "<", ">", ";", "'", "!"]
excludefirsts = ['w']
locFileName = "res/Locations.txt"

# Generic text commands that are not very useful
class FunTextCog(ServerOnlyCog, name = "Fun Text"):

    def __init__(self, bot: commands.Bot):
        self.bot = bot
        self.locations_def = str()
        self.reread_loc_file()

    def reread_loc_file(self):
        with open(locFileName, "r") as locFile:
            self.fortnitelocations_def = locFile.readlines()
            locFile.close()

    def letterify(self, originalStr, replace: str):
        words = originalStr
        finalJimmify = []
        for word in words:
            if len(word) <= 0:
                continue

            if len(word) > 2 and word[0].lower() != replace and word.lower() not in exclude and word[0].lower() not in excludefirsts:

                # Brute-force check for markdown/symbols to skip in the word
                markDownEnd = 0
                wordLen = len(word)
                while not word[markDownEnd].isalpha() and markDownEnd < wordLen:
                    markDownEnd += 1

                # Replace first letter in the string.
                # If it's capitalized, capitalize the replacement letter and lower the old one.
                jRep = replace.upper() if word[markDownEnd].isupper() else replace
                if word[markDownEnd].lower() in vowels:
                    if word[markDownEnd].islower():
                        word = word[0:markDownEnd] + jRep + word[markDownEnd:-1] + word[-1]
                    else:
                        word = word[0:markDownEnd] + jRep + word[markDownEnd].lower() + word[markDownEnd+1:-1] + word[-1]
                else:
                    word = word.replace(word[markDownEnd], jRep, 1)

            finalJimmify.append(word)

        return finalJimmify

    @commands.command(name='Jimmify')
    async def jimmify(self, ctx, *args):

        msg = str(ctx.message.content).split(' ')
        msg.pop(0)
        if len(args) == 0:
            msg = [message async for message in ctx.channel.history(limit=2)]
            msg = msg[1].content.split(' ')

        finalJimmify = self.letterify(msg, 'j')
        await ctx.send(" ".join(finalJimmify))

    @commands.command(name='Mattify')
    async def mattify(self, ctx, *args):

        msg = str(ctx.message.content).split(' ')
        msg.pop(0)
        if len(args) == 0:
            msg = [message async for message in ctx.channel.history(limit=2)]
            msg = msg[1].content.split(' ')

        finalJimmify = self.letterify(msg, 'm')
        await ctx.send(" ".join(finalJimmify))

    @commands.command(name='Droppin')
    async def droppin(self, ctx):
        val = randint(0, len(self.fortnitelocations_def) - 1)
        await ctx.send(self.fortnitelocations_def[val])
        pass

    @commands.command(name='SetDroppin')
    async def set_droppin(self, ctx, *args):

        if args == None:
            await ctx.send("Error: no locations given with command. Locations are unchanged.")
            return

        final = []
        for i in range(len(args)):
            final.append(args[i] + "\n")

        print(final)
        with open (locFileName, "w") as locFile:
            locFile.writelines(final)
            locFile.close()

        self.reread_loc_file()
        await ctx.send("Locations list successfully overwritten and changed to: \n" + ", ".join(args))

    @commands.command(name='AddDroppin')
    async def add_droppin(self, ctx, *args):
        if args == None:
            await ctx.send("Error: no locations given with command. Locations are unchanged.")
            return

        toAdd = " ".join(args).strip()
        with open(locFileName, "a") as locFile:
            locFile.writelines(["\n", f"{toAdd}"])
            locFile.close()

        self.reread_loc_file()
        await ctx.send(f"Added {toAdd} to locations list")

    @commands.command(name='RemoveDroppin')
    async def remove_droppin(self, ctx, *args):

        original = " ".join(args)
        toRem = original.lower()
        ind = None

        locs = self.fortnitelocations_def.copy()
        for i in range(len(locs)):
            locs[i] = locs[i].lower().strip("\n")

        try:
            locs.index(toRem)
        except ValueError as e:
            await ctx.send(f'Could not find: {toRem} in droppin list!')
            return
        else:
            ind = locs.index(toRem)
            self.fortnitelocations_def.pop(ind)

        with open(locFileName, "w") as locFile:
            locFile.writelines(self.fortnitelocations_def)
            locFile.close()

        await ctx.send(f"Successfully removed the location: {original}")

    @commands.command(name='ShowDroppin')
    async def show_droppin(self, ctx):

        toPrint = self.fortnitelocations_def
        if randint(1, 1000) == 1:
            secrets = ["Jimmy Junction", "Matt Mansion"]
            val = randint(0, len(secrets))
            toPrint = secrets[val]

        await ctx.send("Current droppin' locations: \n" + "".join(toPrint))

async def setup(bot):
    await bot.add_cog(FunTextCog(bot))