#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# tf_wgan_tdt4173.py: Rudimentary implementation of a wasserstein GAN that
# is to be used as a base for the second programming task in assignment 4.
#
import sys

import numpy as np
import tensorflow as tf

import helpers

import matplotlib.pyplot as plt

def plot_errors(error_lists, nb_e):
    plt.plot([x[0] for x in error_lists[0]], [x[1] for x in error_lists[0]], 'r')
    plt.plot([x[0] for x in error_lists[1]], [x[1] for x in error_lists[1]], 'b')
    plt.axis([0, 5000, -1.5, 1.5])
    plt.show()

# You might want to alter the learning rate, number of epochs, and batch size
lr = 0.0005
nb_epochs = 20000
batch_size = 200

# Set to `None` if you do not want to write out images
path_to_images = './generated_images'

z_size = 10
x_size = 28*28

# Defined at the top because we need it for initialising weights
def create_weights(shape):
    # See paper by Xavier Glorot and Yoshua Bengio for more information:
    # "Understanding the difficulty of training deep feedforward neural networks"
    # We employ the Caffe version of the initialiser: 1/(in degree)
    return tf.random_normal(shape, stddev=1/shape[0])

#
# Creation of generator and discriminator networks START here
# Task (a) is to improve the generator and discriminator networks as they
# currently do not do very much
#

# Define weight matrices for the generator
# Note: Input of the first layer *must* be `z_size` and the output of the
# *last* layer must be `x_size`
weights_G = {'w1': tf.Variable(create_weights((z_size, x_size))),
             'b1': tf.Variable(tf.zeros(x_size)),
             'w2': tf.Variable(create_weights((x_size, x_size))),
             'b2': tf.Variable(tf.zeros(x_size))}

def generator(z, weights):
    h1 = tf.nn.relu(tf.add(tf.matmul(z, weights['w1']), weights['b1']))
    out = tf.nn.sigmoid(tf.add(tf.matmul(h1, weights['w2']), weights['b2']))

    # Return model and weight matrices
    return out

# Define weight matrices for the discriminator
# Note: Input will always be `x_size` and output will always be 1
weights_D = {'w1': tf.Variable(create_weights((x_size, x_size))),
             'w2': tf.Variable(create_weights((x_size, 1))),
             'b1': tf.Variable(tf.zeros(x_size)),
             'b2': tf.Variable(tf.zeros(1))
             }

def discriminator(x, weights):
    h1 = tf.nn.relu(tf.add(tf.matmul(x, weights['w1']), weights['b1']))
    out = tf.nn.sigmoid(tf.add(tf.matmul(h1, weights['w2']), weights['b2']))

    # Return model and weight matrices
    return out

#
# Creation of generator and discriminator networks END here
#

# Weight clipping (default `c` from the WGAN paper)
c = 0.01
clipped_D = [w.assign(tf.clip_by_value(w, -c, c)) for w in weights_D.values()]

# Definition of how Z samples are generated
z_sampler = lambda nb, dim: np.random.uniform(-1.0, 1.0, size=(nb, dim))

# Load MNIST
mnist = helpers.load_mnist_tf('./mnist')

# Define model entry-points (Z - generator, X - discriminator)
Z = tf.placeholder(tf.float32, shape=(None, z_size))
X = tf.placeholder(tf.float32, shape=(None, x_size))

# Define the different components of a GAN
sample = generator(Z, weights_G)
fake_hat = discriminator(sample, weights_D)
real_hat = discriminator(X, weights_D)

# Define error functions
error_G = - tf.reduce_mean(fake_hat)
error_D = tf.reduce_mean(real_hat) - tf.reduce_mean(fake_hat)

# Specify that we will use RMSProp (one optimiser for each model)
optimiser_G = tf.train.RMSPropOptimizer(lr).minimize(error_G, var_list=weights_G.values())
optimiser_D = tf.train.RMSPropOptimizer(lr).minimize(-error_D, var_list=weights_D.values())

# Generate Op that initialises global variables in the graph
init = tf.global_variables_initializer()

with tf.Session() as sess:
    # Initialise variables and start the session
    sess.run(init)

    if path_to_images:
        helpers.create_dir(path_to_images)

    # Run a set number of epochs (default `n_critic` from the WGAN paper)
    n_critic = 5
    errors = [[], []]
    for epoch in range(nb_epochs):
        for critic in range(n_critic):
            # Retrieve a batch from MNIST
            X_batch, critic = mnist.train.next_batch(batch_size)

            # Clip weights and run one step of the optimiser for D
            sess.run(clipped_D)
            sess.run(optimiser_D, feed_dict={Z: z_sampler(batch_size, z_size), X: X_batch})

        # Run one step of the optimiser for G
        sess.run(optimiser_G, feed_dict={Z: z_sampler(batch_size, z_size)})

        err_G = sess.run(error_G, feed_dict={Z: z_sampler(batch_size, z_size)})
        err_D = sess.run(error_D, feed_dict={Z: z_sampler(batch_size, z_size), X: X_batch})

        errors[0].append([epoch, err_G])
        errors[1].append([epoch, err_D])

        # Print out some information every nth iteration
        if epoch % 20 == 0:

            print('Epoch: ', epoch, '\t Generator error:\t {:.4f}'.format(err_G),
                  '\t Discriminator error:\t {:.4f}'.format(err_D))

        # Plot the image generated from 64 different samples to a directory
        if path_to_images and epoch % 500 == 0:
            samples = sess.run(sample, feed_dict={Z: z_sampler(64, z_size)})

            figure = helpers.plot_samples(samples)
            plt.savefig('{}/{}.png'.format(path_to_images, str(epoch)), bbox_inches='tight')
            plt.close()

    plot_errors(errors, nb_epochs)

del sess
sys.exit(0)


# -*- coding: utf-8 -*-
"""
Created on Fri Nov 23 19:05:39 2018

@author: 10541
"""

from time import strftime
from logging import Formatter,FileHandler,StreamHandler,getLogger,DEBUG
from sys import stdout

def setup_custom_logger(application_name):
    log_file_name = strftime(application_name+"_%d_%m_%Y")+'.log'
    formatter = Formatter('[%(asctime)s]  {%(pathname)s:%(lineno)d} \
                            %(levelname)s - %(message)s','%m-%d %H:%M:%S')
    handler = FileHandler(log_file_name, mode='a')
    handler.setFormatter(formatter)
    screen_handler = StreamHandler(stream=stdout)
    screen_handler.setFormatter(formatter)
    logger = getLogger(application_name)
    logger.setLevel(DEBUG)
    logger.addHandler(handler)
    logger.addHandler(screen_handler)
    logger.removeHandler
    return logger

logger = setup_custom_logger('sample')
logger.info('Sample Info')
logger.error('Sample Error')
logger.warn('Sample Warning')


"""Calculate md5sums and copy files for ChIP-Seq samples."""
from pathlib import Path

import pandas as pd

from ncbi_remap.logging import logger
from ncbi_remap.sample_lists import get_complete_chipseq

from .md5sum import start_cluster, copy_files

DEBUG = False
OUTDIR = Path('../output/geo-wf/justin.fear@nih.gov_chip')
OUTDIR.mkdir(parents=True, exist_ok=True)

if DEBUG:
    import logging
    logger.setLevel(logging.DEBUG)

def main():
    chipseq = get_complete_chipseq()

    if DEBUG:
        chipseq = chipseq[:20]

    logger.info(f'Processing {len(chipseq):,} SRXs')
    work = []
    for srx in chipseq:
        files = {
            'firstFB': {
                'fname': Path(f'../output/aln-wf/samples/{srx}/{srx}.flybase.first.bw'),
                'ftype': 'BigWig',
            },
            'secondFB': {
                'fname': Path(f'../output/aln-wf/samples/{srx}/{srx}.flybase.second.bw'),
                'ftype': 'BigWig',
            },
            'gene': {
                'fname': Path(f'../output/aln-wf/samples/{srx}/{srx}.bam.counts'),
                'ftype': 'abundance measurements',
            },
            'geneJunc': {
                'fname': Path(f'../output/aln-wf/samples/{srx}/{srx}.bam.counts.jcounts'),
                'ftype': 'abundance measurements',
            },
            'inter': {
                'fname': Path(f'../output/aln-wf/samples/{srx}/{srx}.bam.intergenic.counts'),
                'ftype': 'abundance measurements',
            },
            'interJunc': {
                'fname': Path(f'../output/aln-wf/samples/{srx}/{srx}.bam.intergenic.counts.jcounts'),
                'ftype': 'abundance measurements',
            },
        }
        work.append(copy_files(files, OUTDIR, DEBUG))

    try:
        client = start_cluster()
        logger.info('Submitting to Mini Cluster')
        futures = client.compute(work)
        results = client.gather(futures)
    except KeyboardInterrupt as e:
        raise e
    finally:
        client.close()

    hashes = []
    for res in results:
        hashes.extend(res)

    df = pd.DataFrame(hashes, columns=['file name', 'file type', 'file checksum']).set_index('file name')

    if DEBUG:
        logger.debug("\n\n" + df.to_string() + "\n\n")
        return

    logger.info('Writing out hash table')
    df.to_csv(Path(OUTDIR, 'md5sum.tsv'), sep='\t')

if __name__ == '__main__':
    try:
        main()
        logger.info('Script complete')
    except KeyboardInterrupt:
        logger.error('Keyboard interrupted.')


#coding: utf-8

from __future__ import absolute_import

from django.conf.urls import url, patterns

from .views import index, render_form, get_choices, products_search,\
        save_form, render_invoice, invoice_additional_info, total_sum_by_status,\
        total_sum_by_type, prof_into_vat, get_customer_data, get_customer_invoice_data

base_invoice = '^formsave/(?P<invoice_type>\d)/'
urlpatterns = patterns('',
    url('^$', index),
    url('^formtpl/(?P<invoice_type>\d)/$', render_form),
    url('^formtpl/(?P<invoice_type>\d)/(?P<invoice_id>\d+)/$', render_form),
    url('%s$' % base_invoice, save_form),
    url('%s(?P<invoice_id>\d+)/$' % base_invoice, save_form),
    url('^choices/(?P<ct_id>\d+)/$', get_choices),
    url('^customerdata/(?P<ct_id>\d+)/(?P<customer_id>\d+)/$', get_customer_data),
    url('^customerinvoicedata/(?P<ct_id>\d+)/(?P<customer_id>\d+)/$', get_customer_invoice_data),
    url('^profintovat/(?P<inv_id>\d+)/$', prof_into_vat),
    url('^products/$', products_search),
    url('^render/\.(?P<format>\w{2,4})$', render_invoice),
    url('^additionalinfo/(?P<invoice_ids>[0-9,]+)$', invoice_additional_info),
    url('^totalsumstatus/(?P<status>[0-9]+)$', total_sum_by_status),
    url('^totalsumtype/(?P<inv_type>[0-9]+)$', total_sum_by_type),
)


import streamlit as st
import numpy as np
from PIL import Image
from keras.engine.saving import load_model
from keras.preprocessing.image import load_img, img_to_array
from io import BytesIO
import os
import cv2
from keras.utils import normalize
from unet import simple_unet_model
import time
#from pre_processing import my_PreProc
#diabetic retinopathy classifications

def predict(model_name, img):

        results = img_to_array(img)
        arr = results.reshape(-1, 256, 256, 3)
        if model_name == "My_convolution_layer_model":
             new_model = load_model('C:/Users/16189\Documents/hyperspectral/retina_augmented_model__new_mymodel.h5')
        elif model_name == "Transfer learning":
            new_model = load_model('C:/Users/16189\Documents/hyperspectral/retina_augmented_model__new_mymodel_transfer.h5')
        # arr = results.reshape((1,)+results.shape)
        results = new_model.predict(arr)

        # Now predict using the trained RF model.
        # prediction_RF = model1.predict(X_test_features)
        if results == 0:
            x = "diabetic"
        else:
            x = "no_diabetic"

        return x

def get_model():
    return simple_unet_model(patch_size, patch_size, 1)

##Exudates segmentation
def prediction(model, image, patch_size):
    segm_img = np.zeros(image.shape[:2])  # Array with zeros to be filled with segmented values
    patch_num = 1
    my_bar = st.progress(0)
    for i in range(0, image.shape[0], patch_size):  # Steps of 256
        for j in range(0, image.shape[1], patch_size):  # Steps of 256
            # print(i, j)
            single_patch = image[i:i + patch_size, j:j + patch_size]
            single_patch_norm = np.expand_dims(normalize(np.array(single_patch), axis=1), 2)
            single_patch_shape = single_patch_norm.shape[:2]
            single_patch_input = np.expand_dims(single_patch_norm, 0)
            single_patch_prediction = (model.predict(single_patch_input)[0, :, :, 0] > 0.5).astype(np.uint8)
            segm_img[i:i + single_patch_shape[0], j:j + single_patch_shape[1]] += cv2.resize(single_patch_prediction,
                                                                                single_patch_shape[::-1])

            # print("Finished processing patch number ", patch_num, " at position ", i, j)
            patch_num += 1
            # st.write(patch_num)
            time.sleep(0.1)
            # while patch_num <= 100:
            #     my_bar.progress(patch_num + 1)
    return segm_img

def predictions(file, model, patch_size):
    large_image = cv2.imread(file)
    large_image=cv2.cvtColor(large_image,cv2.COLOR_BGR2GRAY)
    large_image = cv2.resize(large_image, (2048, 2048))

    segmented_image = prediction(model, large_image, patch_size)

    return segmented_image

def get_image(model_name, name):
    if name == "Diabetic retinopathy classification":
        image = st.sidebar.file_uploader(label="Select an retinal fundus image", type=['jpg', 'jpeg', 'png'])
        if image is not None:
            image_data = image.read()
            # uploaded_img=load_img(image_data, target_size=(256, 256))
            uploaded_img2 = Image.open(BytesIO(image_data))
            uploaded_img2 = uploaded_img2.resize((256, 256))
            st.write(image.name)

            # uploaded_img3=cv2.resize(uploaded_img2,(256,256))
            st.image(uploaded_img2)
            button = st.button("Click to predict the classification results")
            if button:
                prediction = predict(model_name, uploaded_img2)
                st.write("# The Prediction is:")
                st.write(prediction)
            # predict
    else:
        image = st.sidebar.file_uploader(label="Select an retinal funcdus image", type=['jpg', 'jpeg', 'png'])
        # st.warning("The image should be divisible by the path size")
        if image is not None:
            image_data = image.read()
            # uploaded_img=load_img(image_data, target_size=(256, 256))
            uploaded_img2 = Image.open(BytesIO(image_data))
            st.write(image.name)
            st.image(uploaded_img2)
            new_path = 'C:/Users/16189/Documents/hyperspectral/directory'
            # saving image file
            with open(os.path.join(new_path, image.name, ), "wb") as f:
                f.write(image.getbuffer())
            st.success('File saved')
            button=st.button("Click to predict the segmentation  results")
            if button:
                image_path = 'C:/Users/16189/Documents/hyperspectral/directory/' + image.name
                model = get_model()

                model.load_weights('C:/Users/16189/Documents/hyperspectral/retinal_exudates_segmentation2.h5')
                import time

                st.header("Segmented Image")
                segmented_image = predictions(image_path, model, patch_size)
                with open(os.path.join(new_path, "segmented"+image.name, ), "wb") as f:
                    f.write(image.getbuffer())

                # Large image
                st.balloons()
                st.image(segmented_image)

nav_bar = st.sidebar.radio("Navigation", ["Home", "About"])
if nav_bar == "Home":

    st.title("This is the app of Detecting Diabetic Retinopathy and segmention of the exudates ")
    st.markdown("***")
    st.write("""
    # Model
    Which model you want to work on?
    """)
    name = st.sidebar.selectbox("Select Dataset", ("Diabetic retinopathy classification", "segmentation"))

    st.text(name)
    st.write("Model")

    if name == "Diabetic retinopathy classification":
        model_name = st.sidebar.selectbox("Select the model",
                                          ("Transfer learning", "My_convolution_layer_model", "Eye net"))
    else:
        model_name = st.sidebar.selectbox("Select the model", ("Unet", "Else"))
        patch_size = st.sidebar.selectbox("Select the patch size for segmentation", (256, 512,1024))

    st.text(model_name)
    get_image(model_name, name)
if nav_bar=="About":
    st.header(" Hi, I am Hridoy Biswas and the main developer of the apps.")

    st.markdown("***")
    st.text(" # Ihis is the app for the classification of the diabetic retinopathy."
            "# I used three deep learning model and developed one of my model.You can use all three models and predict ")
    st.text(" The segmentation model is u-net model.YOu can segment the exudates")


from setuptools import setup, find_packages
from setuptools.command.test import test as TestCommand
import sys

class PyTest(TestCommand):
    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = []
        self.test_suite = True

    def run_tests(self):
        import pytest
        errno = pytest.main(self.test_args)
        sys.exit(errno)

setup(
    name='trompe',
    version='0.0',
    description='Acoustics package for Python',
    author='Felipe Raimann',
    author_email='felipeacsi@gmail.com',
    package_dir={'.': 'trompe'},
    packages=find_packages(),
    install_requires=[
        'numpy',
        ],
    tests_require=['pytest'],
    cmdclass={'test': PyTest}
)


from flask import Flask, render_template, request
from classes import PostcodeTravelInfo

app = Flask(__name__)

@app.route("/")
def index():
    return render_template('index.html')

@app.route("/busInfo")
def BusInfo():
    postcode = request.args.get('postcode')
    number_of_stops = int(request.args.get('number_of_stops'))
    number_of_buses = request.args.get('number_of_buses')
    postcode_travel_info = PostcodeTravelInfo(postcode, number_of_stops, number_of_buses)
    return render_template(
        'info.html', postcode=postcode, number_of_stops=number_of_stops, number_of_buses=number_of_buses,
        postcode_travel_info=postcode_travel_info
    )

if __name__ == "__main__": app.run()


import uuid
import json
import xmltodict
from collections import OrderedDict
from ..core import Core
from pyews.utils.exceptions import ObjectType, SoapResponseHasError, SoapAccessDeniedError

class GetItem(Core):

    def __init__(self, userconfiguration):
        super(GetItem, self).__init__(userconfiguration)

    def __camel_to_snake(self, s):
        return ''.join(['_'+c.lower() if c.isupper() else c for c in s]).lstrip('_')

    def __process_keys(self, key):
        return_value = key.replace('t:','')
        if return_value.startswith('@'):
            return_value = return_value.lstrip('@')
        return self.__camel_to_snake(return_value)

    def __process_dict(self, obj):
        if isinstance(obj, dict):
            obj = {
                self.__process_keys(key): self.__process_dict(value) for key, value in obj.items()
                }
        return obj

    def __process_single_calendar_item(self, value):
        ordered_dict = xmltodict.parse(str(value))
        item_dict = json.loads(json.dumps(ordered_dict))
        return self.__process_dict(item_dict)

    def __parse_response(self, value):
        return_dict = {}
        if value.find('ResponseCode').string == 'NoError':
            for item_type in ['CalendarItem', 'Contact', 'Message', 'Item']:
                if value.find(item_type):
                    for item in getattr(value.find('Items'), item_type):
                        return_dict.update(self.__process_single_calendar_item(item))
                    return return_dict

    def run(self, item_id, change_key=None):
        return self.__parse_response(self.invoke(self.soap(item_id, change_key=change_key)))

    def soap(self, item_id, change_key=None):
        '''Creates the SOAP XML message body

        Args:
            email_address (str): A single email addresses you want to GetInboxRules for

        Returns:
            str: Returns the SOAP XML request body
        '''
        if self.userconfiguration.impersonation:
            impersonation_header = self.userconfiguration.impersonation.header
        else:
            impersonation_header = ''

        if change_key:
            item_id_string = '<t:ItemId Id="{item_id}" ChangeKey="{change_key}"/>'.format(
                item_id=item_id,
                change_key=change_key
            )
        else:
            item_id_string = '<t:ItemId Id="{item_id}"/>'.format(
                item_id=item_id
            )

        return '''<?xml version="1.0" encoding="utf-8"?>
<soap:Envelope xmlns:xsd="http://www.w3.org/2001/XMLSchema"
               xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"
               xmlns:t="http://schemas.microsoft.com/exchange/services/2006/types"
               xmlns:m="http://schemas.microsoft.com/exchange/services/2006/messages">
    <soap:Header>
        <t:RequestServerVersion Version="{version}"/>
    </soap:Header>
    <soap:Body>
        <GetItem xmlns="http://schemas.microsoft.com/exchange/services/2006/messages">
            <ItemShape>
                <t:BaseShape>AllProperties</t:BaseShape>
                <t:IncludeMimeContent>true</t:IncludeMimeContent>
                <t:ConvertHtmlCodePageToUTF8>false</t:ConvertHtmlCodePageToUTF8>
            </ItemShape>
            <ItemIds>
                {item_id_string}
            </ItemIds>
        </GetItem>
    </soap:Body>
</soap:Envelope>'''.format(
    version=self.userconfiguration.exchange_version,
    item_id_string=item_id_string)


# File containing helper functions for computing similarity and clustering PDB active sites

import numpy as np
import random
import matplotlib.pyplot as plt

def output_aa_string(residues):
    """
    Convert 3 letter amino acid code to single letter amino acids
    Input: list of residues in three letter codes
    Output: string of residues in single code
    """
    # Dictionary of 3 letter to 1 letter AA conversion
    aa_dict = {'CYS': 'C', 'ASP': 'D', 'SER': 'S', 'GLN': 'Q', 'LYS': 'K',
     'ILE': 'I', 'PRO': 'P', 'THR': 'T', 'PHE': 'F', 'ASN': 'N',
     'GLY': 'G', 'HIS': 'H', 'LEU': 'L', 'ARG': 'R', 'TRP': 'W',
     'ALA': 'A', 'VAL':'V', 'GLU': 'E', 'TYR': 'Y', 'MET': 'M'}

    s = ''
    for res in residues:
        s = s + aa_dict.get(res.type)
    return s

def match(a1, a2):
    """
    Find indexes of matches for both alignment a and alignment b
    Input: two strings after alignment
    Outputs: indexes where match occurs in string 1 and string 2
    """
    inds_1, inds_2 = [], []
    i, j = 0, 0
    for s in range(0, len(a1)):
        increment = lambda a, x, ind: x+1 if a[ind] != "-" else x
        if a1[s] is a2[s] and a1[s] != "-":
            inds_1.append(i)
            inds_2.append(j)
        i = increment(a1,i,s)
        j = increment(a2,j,s)
    return inds_1, inds_2

def create_coord_matrix(site, index):
    """
    Create a matrix of CA atom x,y,z coordinates for every residue contained
    in an active site
    Inputs: active site and list of indices where two alignments match
    Output: matrix with x,y, and z coordinates for every CA atom
    """
    mat = np.empty([0,3])
    for res in np.take(site.residues, index):
        ind = [i for i,x in enumerate(res.atoms) if x.type == "CA" or x.type == "CA A"][0]
        x,y,z = res.atoms[ind].coords
        mat = np.append(mat, [[x,y,z]], axis=0)
    return mat

def initialize_k_mediods(data, k):
    """
    Randomly select k points and set as cluster centers
    Input: similarity matrix and # of clusters
    Output: centers for clusters
    """
    return random.sample(range(len(data)), k)

def assign_k_clusters(data, centers):
    """
    Assign each data point to its nearest cluster center
    Input: matrix and k centers
    Output: cluster assignments for points in matrix
    """
    clusters = []
    center_data = np.take(data, centers, axis=0)
    best_center = np.argmax(center_data, axis=0)
    for i in range(len(centers)):
        inds = [ind for ind in np.where(best_center == i)[0]]
        clusters.append(inds)
    return clusters

def calculate_cost(data, centers, clusters):
    """
    Calculate the sum of similarities of cluster elements to centers
    """
    total = 0
    for i in range(len(centers)):
        total = total + np.sum(data[centers[i]][clusters[i]])
    return total

def recalculate_centers(data, k, clusters):
    """
    Recalculate cluster centers for data
    Input: matrix, number of clusters, and cluster assignments
    Output: new cluster centers
    """
    centers = []
    for k_i in range(k):
        inds = [i for i, j in enumerate(clusters) if j == k_i]
        n = np.take(data, inds, axis=0)
        if len(inds) == 0:
            i = np.random.randint(len(data))
            centers.append((data[i,0], data[i,1]))

        elif len(inds) < 2:
            centers.append((n[0][0], n[0][1]))
        else:
            result = np.sum(n, axis=1)/len(inds)
            centers.append((result[0], result[0]))
    return centers

def dist_HC(active_sites, clusters,c_new, data):
    """
    Output the distance of the new cluster to all other clusters
    by computing the average similarity (average linkage)
    """
    new_arr = np.array([])
    for i in range(c_new):
        if type(clusters.get(i)) == type(None):
            new_arr = np.append(new_arr,float("-inf"))
        else:
            c = clusters.get(i)
            sub = data[np.ix_(clusters.get(c_new), c)]
            d = (np.ma.masked_invalid(sub).sum())/(len(clusters.get(c_new))*len(c))
            new_arr = np.append(new_arr, d)
    return new_arr

def output_cluster_list(active_sites, clusters):
    """
    Output the active site names in list of atoms format
    """
    atoms = []
    for c in clusters:
        clust = []
        for elem in c:
            clust.append(active_sites[elem])
        atoms.append(clust)
    return atoms


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import print_function, absolute_import

import os
import torch
from torch.utils.data import Dataset
import numpy as np

class data_loader(Dataset):
    def __init__(self, data_path, is_train=True, noise=None):
        """
        data_path: path to dataset
        is_train: load train/test dataset
        """
        self.data_path = data_path
        self.is_train = is_train
        self.noise = noise

        self.train_inp, self.train_out, self.test_inp, self.test_out = [], [], [], []
        self.train_meta, self.test_meta = [], []

        if self.is_train:  # loading training data
            self.train_3d = torch.load(os.path.join(data_path, "train_3d.pth.tar"))
            self.train_2d = torch.load(os.path.join(data_path, "train_2d.pth.tar"))
            for key in self.train_2d.keys():
                num_f, _ = self.train_2d[key].shape
                assert (
                    self.train_3d[key].shape[0] == self.train_2d[key].shape[0]
                ), "(training) 3d & 2d shape not matched"
                for i in range(num_f):
                    self.train_inp.append(self.train_2d[key][i])
                    self.train_out.append(self.train_3d[key][i])

        else:  # load test data
            self.test_3d = torch.load(os.path.join(data_path, "test_3d.pth.tar"))
            self.test_2d = torch.load(os.path.join(data_path, "test_2d.pth.tar"))
            for key in self.test_2d.keys():
                num_f, _ = self.test_2d[key].shape
                assert (
                    self.test_2d[key].shape[0] == self.test_3d[key].shape[0]
                ), "(test) 3d & 2d shape not matched"
                for i in range(num_f):
                    self.test_inp.append(self.test_2d[key][i])
                    self.test_out.append(self.test_3d[key][i])

    def __getitem__(self, index):
        # print(len(self.train_inp), len(self.test_inp))
        if self.is_train:
            # print(self.train_inp[index])
            # inputs = torch.from_numpy(self.train_inp[index].reshape(13, 2)).float()
            inputs = torch.from_numpy(self.train_inp[index]).float()
            # outputs = torch.from_numpy(self.train_out[index].reshape(13, 3)).float()
            outputs = torch.from_numpy(self.train_out[index]).float()
        else:
            # inputs = torch.from_numpy(self.test_inp[index].reshape(13, 2)).float()
            inputs = torch.from_numpy(self.test_inp[index]).float()
            # outputs = torch.from_numpy(self.test_out[index].reshape(13, 3)).float()
            outputs = torch.from_numpy(self.test_out[index]).float()

        return inputs, outputs

    def __len__(self):
        if self.is_train:
            return len(self.train_inp)
        else:
            return len(self.test_inp)


#!/usr/bin/env python
"""
_PhEDExInjectorSubscriber_

Poll the DBSBuffer database for unsubscribed datasets, and make subscriptions
associated with these datasets.

The subscription information is stored in the DBSBuffer subscriptions table and specifies the following options
for each dataset:

- site: Site to subscribe the data to
- custodial: 1 if the subscription must be custodial, non custodial otherwise
- auto_approve: 1 if the subscription should be approved automatically, request-only otherwise
- priority: Priority of the subscription, can be Low, Normal or High
- move: 1 if the subscription is a move subscription, 0 otherwise

The usual flow of operation is:

- Find unsuscribed datasets (i.e. dbsbuffer_dataset_subscription.subscribed = 0)
- Check for existing subscription in PhEDEx for such datasets, with the same
  configuration options as registered in the dataset, mark these as already subscribed
- Subscribe the unsubscribed datasets and mark them as such in the database,
  this is done according to the configuration options and aggregated to minimize
  the number of PhEDEx requests.

Additional options are:

- config.PhEDExInjector.subscribeDatasets, if False then this worker doesn't run
"""

import threading
import logging

from WMCore.WorkerThreads.BaseWorkerThread import BaseWorkerThread

from WMCore.Services.PhEDEx import XMLDrop
from WMCore.Services.PhEDEx.DataStructs.PhEDExDeletion import PhEDExDeletion
from WMCore.Services.PhEDEx.DataStructs.SubscriptionList import PhEDExSubscription, SubscriptionList

from WMCore.DAOFactory import DAOFactory

class PhEDExInjectorSubscriber(BaseWorkerThread):
    """
    _PhEDExInjectorSubscriber_

    Poll the DBSBuffer database and subscribe datasets as they are
    created.
    """
    def __init__(self, config, phedex, nodeMappings):
        """
        ___init___

        Initialise class members
        """
        BaseWorkerThread.__init__(self)
        self.phedex = phedex
        self.dbsUrl = config.DBSInterface.globalDBSUrl
        self.group = getattr(config.PhEDExInjector, "group", "DataOps")

        self.phedexNodes = {'MSS':[], 'Disk':[]}
        for node in nodeMappings["phedex"]["node"]:
            if node["kind"] in [ "MSS", "Disk" ]:
                self.phedexNodes[node["kind"]].append(node["name"])

        # initialize the alert framework (if available - config.Alert present)
        #    self.sendAlert will be then be available
        self.initAlerts(compName = "PhEDExInjector")

        return

    def setup(self, parameters):
        """
        _setup_

        Create a DAO Factory for the PhEDExInjector.  Also load the SE names to
        PhEDEx node name mappings from the data service.
        """
        myThread = threading.currentThread()
        daofactory = DAOFactory(package = "WMComponent.PhEDExInjector.Database",
                                logger = self.logger,
                                dbinterface = myThread.dbi)

        self.findDeletableBlocks = daofactory(classname = "GetDeletableBlocks")
        self.markBlocksDeleted = daofactory(classname = "MarkBlocksDeleted")
        self.getUnsubscribed = daofactory(classname = "GetUnsubscribedDatasets")
        self.markSubscribed = daofactory(classname = "MarkDatasetSubscribed")

        return

    def algorithm(self, parameters):
        """
        _algorithm_

        Run the subscription algorithm as configured
        """
        self.deleteBlocks()
        self.subscribeDatasets()
        return

    def deleteBlocks(self):
        """
        _deleteBlocks_

        Find deletable blocks, then decide if to delete based on:

        Is there an active subscription for dataset or block ?
          If yes => set deleted=2
          If no => next check

        Has transfer to all destinations finished ?
          If yes => request block deletion, approve request, set deleted=1
          If no => do nothing (check again next cycle)

        """
        logging.info("Starting deleteBlocks method")

        blockDict = self.findDeletableBlocks.execute(transaction = False)

        if len(blockDict) == 0:
            return

        subscriptions = self.phedex.getSubscriptionMapping(*blockDict.keys())

        skippableBlocks = []
        deletableEntries = {}
        for blockName in blockDict:

            location = blockDict[blockName]['location']

            # should never be triggered, better safe than sorry
            if location.endswith('_MSS'):
                logging.debug("Location %s for block %s is MSS, skip deletion", location, blockName)
                skippableBlocks.append(blockName)
                continue

            dataset = blockDict[blockName]['dataset']
            sites = blockDict[blockName]['sites']

            if blockName in subscriptions and location in subscriptions[blockName]:
                logging.debug("Block %s subscribed to %s, skip deletion",  blockName, location)
                binds = { 'DELETED' : 2,
                          'BLOCKNAME' : blockName }
                self.markBlocksDeleted.execute(binds, transaction = False)
            else:
                blockInfo = []
                try:
                    blockInfo = self.phedex.getReplicaInfoForBlocks(block = blockName, complete = 'y')['phedex']['block']
                except:
                    logging.error("Couldn't get block info from PhEDEx, retry next cycle")
                for entry in blockInfo:
                    if entry['name'] == blockName:
                        nodes = set([x['node'] for x in entry['replica']])
                        if location not in nodes:
                            logging.debug("Block %s not present on %s, mark as deleted", blockName, location)
                            binds = { 'DELETED' : 1,
                                      'BLOCKNAME' : blockName }
                            self.markBlocksDeleted.execute(binds, transaction = False)
                        elif sites.issubset(nodes):
                            logging.debug("Deleting block %s from %s since it is fully transfered", blockName, location)
                            if location not in deletableEntries:
                                deletableEntries[location] = {}
                            if dataset not in deletableEntries[location]:
                                deletableEntries[location][dataset] = set()
                            deletableEntries[location][dataset].add(blockName)

        binds = []
        for blockName in skippableBlocks:
            binds.append( { 'DELETED' : 2,
                            'BLOCKNAME' : blockName } )
        if len(binds) > 0:
            self.markBlocksDeleted.execute(binds, transaction = False)

        for location in deletableEntries:

            chunkSize = 100
            numberOfBlocks = 0
            blocksToDelete = {}
            for dataset in deletableEntries[location]:

                blocksToDelete[dataset] = deletableEntries[location][dataset]
                numberOfBlocks += len(blocksToDelete[dataset])

                if numberOfBlocks > chunkSize:

                    self.deleteBlocksPhEDExCalls(location, blocksToDelete)
                    numberOfBlocks = 0
                    blocksToDelete = {}

            self.deleteBlocksPhEDExCalls(location, blocksToDelete)

        return

    def deleteBlocksPhEDExCalls(self, location, blocksToDelete):
        """
        _deleteBlocksPhEDExCalls_

        actual PhEDEx calls for block deletion
        """
        deletion = PhEDExDeletion(blocksToDelete.keys(), location,
                                  level = 'block',
                                  comments = "WMAgent blocks auto-delete from %s" % location,
                                  blocks = blocksToDelete)

        try:
            xmlData = XMLDrop.makePhEDExXMLForBlocks(self.dbsUrl,
                                                     deletion.getDatasetsAndBlocks())
            logging.debug(str(xmlData))
            response = self.phedex.delete(deletion, xmlData)
            requestId = response['phedex']['request_created'][0]['id']

            # auto-approve deletion request
            self.phedex.updateRequest(requestId, 'approve', location)

            binds = []
            for dataset in blocksToDelete:
                for blockName in blocksToDelete[dataset]:
                    binds.append( { 'DELETED' : 1,
                                    'BLOCKNAME' : blockName } )

            self.markBlocksDeleted.execute(binds, transaction = False)

        except Exception as ex:
            logging.error("Something went wrong when communicating with PhEDEx, will try again later.")
            logging.error("Exception: %s", str(ex))

        return

    def subscribeDatasets(self):
        """
        _subscribeDatasets_

        Poll the database for datasets and subscribe them.
        """
        logging.info("Starting subscribeDatasets method")

        myThread = threading.currentThread()
        myThread.transaction.begin()

        # Check for completely unsubscribed datasets
        unsubscribedDatasets = self.getUnsubscribed.execute(conn = myThread.transaction.conn,
                                                            transaction = True)

        # Keep a list of subscriptions to tick as subscribed in the database
        subscriptionsMade = []

        # Create a list of subscriptions as defined by the PhEDEx data structures
        subs = SubscriptionList()

        # Create the subscription objects and add them to the list
        # The list takes care of the sorting internally
        for subInfo in unsubscribedDatasets:
            site = subInfo['site']

            if site not in self.phedexNodes['MSS'] and site not in self.phedexNodes['Disk']:
                msg = "Site %s doesn't appear to be valid to PhEDEx, " % site
                msg += "skipping subscription: %s" % subInfo['id']
                logging.error(msg)
                self.sendAlert(7, msg = msg)
                continue

            # Avoid custodial subscriptions to disk nodes
            if site not in self.phedexNodes['MSS']:
                subInfo['custodial'] = 'n'
            # Avoid auto approval in T1 sites
            elif site.startswith("T1"):
                subInfo['request_only'] = 'y'

            phedexSub = PhEDExSubscription(subInfo['path'], site,
                                           self.group, priority = subInfo['priority'],
                                           move = subInfo['move'], custodial = subInfo['custodial'],
                                           request_only = subInfo['request_only'], subscriptionId = subInfo['id'])

            # Check if the subscription is a duplicate
            if phedexSub.matchesExistingSubscription(self.phedex) or \
                phedexSub.matchesExistingTransferRequest(self.phedex):
                subscriptionsMade.append(subInfo['id'])
                continue

            # Add it to the list
            subs.addSubscription(phedexSub)

        # Compact the subscriptions
        subs.compact()

        for subscription in subs.getSubscriptionList():
            try:
                xmlData = XMLDrop.makePhEDExXMLForDatasets(self.dbsUrl,
                                                           subscription.getDatasetPaths())
                logging.debug(str(xmlData))
                msg = "Subscribing: %s to %s, with options: " % (subscription.getDatasetPaths(), subscription.getNodes())
                msg += "Move: %s, Custodial: %s, Request Only: %s" % (subscription.move, subscription.custodial, subscription.request_only)
                logging.info(msg)
                self.phedex.subscribe(subscription, xmlData)
            except Exception as ex:
                logging.error("Something went wrong when communicating with PhEDEx, will try again later.")
                logging.error("Exception: %s", str(ex))
            else:
                subscriptionsMade.extend(subscription.getSubscriptionIds())

        # Register the result in DBSBuffer
        if subscriptionsMade:
            self.markSubscribed.execute(subscriptionsMade,
                                        conn = myThread.transaction.conn,
                                        transaction = True)

        myThread.transaction.commit()
        return


# Authors:
#     Sylvain Faure <sylvain.faure@math.u-psud.fr>
#     Bertrand Maury <bertrand.maury@math.u-psud.fr>
#
#      cromosim/examples/micro/social/micro_social.py
#      python micro_social.py --json input.json
#
# License: GPL

import sys, os
import numpy as np
from cromosim import *
from cromosim.micro import *
from optparse import OptionParser
import json
from scipy.spatial import distance
import statistics as stat
from tqdm import tqdm
#plt.ion()

parser = OptionParser(usage="usage: %prog [options] filename",version="%prog 1.0")
parser.add_option('--json',dest="jsonfilename",default="input_catanzaro_social_porta120.json",type="string",
                  action="store",help="Input json filename")
opt, remainder = parser.parse_args()
#print("===> JSON filename = ",opt.jsonfilename)
with open(opt.jsonfilename) as json_file:
    input = json.load(json_file)

"""
    Get parameters from json file :

    name: string
        Domain name
    prefix: string
        Folder name to store the results
    background: string
        Image file used as background
    px: float
        Pixel size in meters (also called space step)
    width: integer
        Domain width (equal to the width of the background image)
    height: integer
        Domain height (equal to the height of the background image)
    wall_lines : list of numpy arrays
        Polylines used to build walls, [ [[x0,x1,x2,...],[y0,y1,y2,...]],... ]
    wall_ellipses : list of numpy arrays
        Ellipses used to build walls, [ [x_center,y_center, width, height, angle_in_degrees_anti-clockwise],... ]
    wall_polygons : list of numpy arrays
        Polygons used to build walls, [ [[x0,x1,x2,...],[y0,y1,y2,...]],... ]
    wall_lines : list of numpy arrays
        Polylines used to build walls, [ [[x0,x1,x2,...],[y0,y1,y2,...]],... ]
    door_lines: list of numpy arrays
        Polylines used to build doors, [ [[x0,x1,x2,...],[y0,y1,y2,...]],... ]
    seed: integer
        Random seed which can be used to reproduce a random selection if >0
    rmin: float
        Minimum radius for people
    rmax: float
        Maximum radius for people
    mass: float
        Mass of one person (typically 80 kg)
    tau: float
        (typically 0.5 s)
    F: float
        Coefficient for the repulsion force between individuals (typically 2000 N)
    kappa: float
        Stiffness constant to handle overlapping (typically 120000 kg s^-2)
    delta: float
        To maintain a certain distance from neighbors (typically 0.08 m)
    Fwall: float
        Coefficient for the repulsion force between individual and walls (typically 2000 N, like for F)
    lambda: float
        Directional dependence (between 0 and 1 = fully isotropic case)
    eta: float
        Friction coefficient (typically 240000 kg m^-1 s^-1)
    N: list
        Number of persons in each boxes
    init_people_box: list
        List of boxes to randomly position people at initialization, \
        [[xmin,xmax,ymin,ymax],...]
    exit_people_box:
        People outside this box will be deleted, [xmin,xmax,ymin,ymax]
    Tf: float
        Final time
    dt: float
        Time step
    drawper: integer
        The results will be displayed every "drawper" iterations
    dmax: float
        Maximum distance used to detect neighbors
    dmin: float
        Minimum distance allowed between individuals
    sensors: list of numpy array
        Segments through which incoming and outgoing flows are measured
        [ [x0,y0,x1,y1],... ]
    plot_people: boolean
        If true, people are drawn
    plot_contacts: boolean
        If true, active contacts between people are drawn
    plot_velocities: boolean
        If true, people velocities are drawn
    plot_paths: boolean
        If true, people paths are drawn
    plot_sensors: boolean
        If true, plot sensor lines on people graph and sensor data graph
"""

name = input["name"]
prefix = input["prefix"]
if not os.path.exists(prefix):
    os.makedirs(prefix)
background = input["background"]
px = input["px"]
width = input["width"]
height = input["height"]
wall_lines = input["wall_lines"]
wall_ellipses = input["wall_ellipses"]
wall_polygons = input["wall_polygons"]
door_lines = input["door_lines"]
seed = input["seed"]
N = sp.array(input["N"]).astype(int)
print("N = ",N)
Np = N.sum()
rmin = input["rmin"]
rmax = input["rmax"]
mass = input["mass"]
tau = input["tau"]
F = input["F"]
kappa = input["kappa"]
delta = input["delta"]
Fwall = input["Fwall"]
lambda_ = input["lambda"]
eta = input["eta"]
init_people_box = input["init_people_box"]
exit_people_box = input["exit_people_box"]
Tf = input["Tf"]
dt = input["dt"]
drawper = input["drawper"]
dmax = input["dmax"]
dmin = input["dmin"]
sensors = input["sensors"]
plot_p = input["plot_people"]
plot_c = input["plot_contacts"]
plot_v = input["plot_velocities"]
plot_pa = input["plot_paths"]
plot_s = input["plot_sensors"]
linewidth = input["line_width"]
filename = input["filename"]
vmax = input["v_max"]
plot = input["plot"]
ferma_prof = input["ferma_prof"]
total = input["total_it"]
'''print("===> Number of persons = ",Np)
print("===> Final time, Tf = ",Tf)
print("===> Time step, dt = ",dt)
print("===> To draw the results each drawper iterations, drawper = ",drawper)
print("===> Maximal distance to find neighbors, dmax = ",dmax,", example : 2*dt")
print("===> Minimal distance between persons, dmin = ",dmin)'''

"""
    Build the Domain
"""

## To create an Domain object
dom = Domain(name=name, pixel_size=px, width=width, height=height, vmax = vmax)
## To add lines : Line2D(xdata, ydata, linewidth)
for xy in wall_lines:
    line = Line2D( xy[0],xy[1], linewidth=linewidth)
    dom.add_wall(line)
## To add doors :
for xy in door_lines:
    line = Line2D( xy[0],xy[1], linewidth=linewidth)
    dom.add_door(line)
## To build the domain : background + shapes
dom.build_domain()
## To compute the distance to the walls
dom.compute_wall_distance()
## To compute the desired velocity
dom.compute_desired_velocity()
## To show the domain dimensions
'''
print("===> Domain : ",dom)
print("===> Wall lines : ",wall_lines)
print("===> Door lines : ",door_lines)

plt.ioff()
dom.plot()
plt.show()
'''
"""
    Initialization
"""
'''
f = open(filename, "w")
m_h = height * px
m_w = width * px
a = m_w * m_h
density = Np / a
f.write("Input: " + str(filename) + "\n")
f.write("Velocità: " + str(vmax) + "\n")
f.write("density: " + str(density) + "\n")
f.write("Np: " + str(Np) + "\n")
f.write("dt: " + str(dt) + "\n\n")
'''
mean = 0
times = []
c_flows = 0
flow_mean_total = 0
it = 1

#door_center = (stat.mean(door_lines[0][0]), stat.mean(door_lines[0][1]))

doors_center = []
for door in door_lines:
    doors_center.append((stat.mean(door[0]), stat.mean(door[1])))

time_single_unit = {}
exit_single_unit = {}
custom_area = [[3.9, 3.9], [4.8, 4.5]]
count_zoneB = []
count_zoneA = []
velocities_iter = []
evacuation_time = []
velocities_mean = []
flow_iter = []
for i in range(1, Np+1):
    time_single_unit[i] = []
    exit_single_unit[i] = []
    for j in range(0, len(door_lines)):
        exit_single_unit[i].append(0)

flows = []
flow_cum = []
people_dist = {}
people_pos = {}
people_door_dist = {}
people_velocities = {}
if (seed<0):
    seed = sp.random.RandomState()
## Current time
t = 0.0
people, people_init_box_id, rng = people_initialization(N, init_people_box, dom,
                                                        dt, rmin, rmax, dmin=dmin,
                                                        seed=seed)
people_ids = people[:,3]
for p in people:
    people_door_dist[p[3]] = [[distance.euclidean((p[0],p[1]), door_center)] for door_center in doors_center]
    people_velocities[p[3]] = [0]
    people_pos[p[3]] = [(p[0], p[1])]
    people_dist[p[3]] = 0
## Array to store the results : all the people coordinates for all times
Np = people.shape[0]
Uold = sp.zeros((Np,2))
Np_init = Np
people_id = sp.arange(Np)
results = sp.copy(people[:,:2]).reshape((Np,2,1))

## Array to store sensor data : time dir pts[2] for each sensor line
if (len(sensors)>0):
    sensor_data = sp.zeros((Np,4,len(sensors)))

count_p_custom_area = []#conta persone nella zona delimitata da custom_area
count_p_cum = []
"""
    Main loop
"""
cc = 0
counter = 0
int_t = int(t)
N_old = Np
tot_vel = []
i = 0
time = {}
ts = {}
people_velocities = {}
velocities = []
#print("Step_" + str(it) + " di " + str(total))
it += 1
for p in people[:,3]:
    ts[p] = [0]
    people_velocities[p] = [0]
while (t<Tf):#continua fino allo scadere del tempo
    contacts = compute_contacts(dom, people, dmax)
    I, J, Vd = compute_desired_velocity(dom, people)

    if int_t < int(t):
        flow = N_old - Np
        #print("===> time = ",int_t," flow = ",flow, "number of persons = ", Np)
        N_old = Np
        int_t = int(t)
        flows.append(flow)
        if(len(flow_cum) > 0):
            flow_cum.append(flow_cum[-1]+flow)
        else:
            flow_cum.append(flow)
        #calcolo la densità vicino alla porta
        count_custom = 0
        for p in people:
            #verifico se la persona è nell'area a scelta custom_area
            if p[0] > custom_area[0][0] and p[0] < custom_area[1][0]:#x
                if p[1] > custom_area[0][1] and p[1] < custom_area[1][1]:#y
                    count_custom += 1

        count_p_custom_area.append(count_custom)
        if(len(count_p_cum) > 0):
            count_p_cum.append(count_p_cum[-1]+count_custom)
        else:
            count_p_cum.append(count_custom)
    if ((cc>=drawper) or (counter==0)):
        #print("===> time = ",t," number of persons = ",Np)
        if(plot == True):
            plot_people(10, dom, people, contacts, Vd, people[:,3], time=t,
                            plot_people=plot_p, plot_contacts=plot_c,
                            plot_velocities=plot_v, plot_paths=False,paths=results,
                            plot_sensors=plot_s, sensors=sensors,
                            savefig=True, filename=prefix+'fig_'+ \
                            str(counter).zfill(6)+ '_' + str(dt) + '.png')
            #plot_grafici(20, people, ts, people_door_dist, t, people_velocities, people_pos)
            plt.pause(0.01)
            if (t>0):
                for i, s in enumerate(sensors):
                    plot_sensor_data(30+i, sensor_data[:,:,i], t, savefig=True,
                            filename=prefix+'sensor_'+str(i)+'_'+str(counter)+'.png')
                    plt.pause(0.01)
        cc = 0
    Forces = compute_forces(F, Fwall, people, contacts, Uold, Vd, lambda_, delta, kappa, eta)
    U = dt*(Vd-Uold)/tau + Uold + dt*Forces/mass[0]

    if ferma_prof:
        if(Np > 1 and people[-1,0] >= doors_center[0][0]-1):
            U[-1, :] = 0
        else:
            U[-1, :] = dt*(Vd[-1,:]-Uold[-1,:])/tau + Uold[-1,:] + dt*Forces[-1,:]/mass[1]

    people = move_people(t, dt, people, U)
    t += dt
    for p in people:
        i = 0
        for door_center in doors_center:
            dist = distance.euclidean((p[0],p[1]), door_center)
            #people_velocities[p[3]].append(people_door_dist[p[3]][-1]-dist)/t)
            people_door_dist[p[3]][i].append(dist)
            i += 1
        ts[p[3]].append(t)
        people_dist[p[3]] += distance.euclidean(people_pos[p[3]][-1], (p[0], p[1]))
        people_pos[p[3]].append((p[0], p[1]))
    t -= dt
    if len(U > 0):
        for p in range(0, len(people_id)):
            #calcolo accellerazione di ogni persona
            vel = pow((U[p][0]*U[p][0]) + (U[p][1]*U[p][1]), 1/2)
            if p in people_velocities.keys():
                people_velocities[people_id[p]].append(vel)
            else:
                people_velocities[people_id[p]] = [vel]

        for u in U:
            vel = pow((u[0]*u[0]) + (u[1]*u[1]), 1/2)
            velocities.append(vel)

        tot_vel.append(sum(velocities)/len(velocities))

    people, U, [people_id], id_exit = exit_door(2*dom.pixel_size, dom, people, U, doors_center, arrays=[ people_id])

    for id in id_exit:
        time_single_unit[id].append(t)
        exit_single_unit[id][id_exit[id]] += 1

    ## Store people positions in the result array (used to draw people paths)
    tmp = 1e99*sp.ones((Np_init,2))
    tmp[people_id,:] = people[:,:2]
    results = sp.concatenate((results,tmp.reshape((Np_init,2,1))), axis=2)
    for id in id_exit:
        time[id] = t
    t += dt
    Uold = U
    cc += 1
    counter += 1
    if(t > 52):
        #f.write("=========> scemo incastrato" )
        plot_people(10, dom, people, contacts, Vd, people[:,3], time=t,
                            plot_people=plot_p, plot_contacts=plot_c,
                            plot_velocities=plot_v, plot_paths=False,paths=results,
                            plot_sensors=plot_s, sensors=sensors,
                            savefig=True, filename="incastrato.png")
        break
    Np = people.shape[0]
    if (Np == 0):
        flow = N_old - Np
        flows.append(flow)
        flow_cum.append(flow_cum[-1]+flow)
        #print("END... Nobody here !")
        break

'''
flow_rate = sum(flows) / len(flows)
flow_mean_total += flow_rate
c_flows += 1
mean = mean + t
f.write("velocity" + str(c_flows) + " = " + str(sum(tot_vel)/len(tot_vel)) + "m/s\n")
print("velocity" + str(c_flows) + " = " + str(sum(tot_vel)/len(tot_vel)) + "m/s")
#velocities_iter.append(sum(tot_vel)/len(tot_vel))
velocities_iter.append(tot_vel)
print("evacuation time = ", t, "s")
print("path length = ", t * (sum(tot_vel)/len(tot_vel)), "m")
f.write("evacuation time_" + str(c_flows) +": " + str(t) + "\n")
evacuation_time.append(t)
velocities_mean.append(sum(tot_vel)/len(tot_vel))
f.write("flow_rate_" + str(c_flows) + ": " + str(flow_rate) + "\n")
f.write("flows" + str(flows) + "\n")
flow_iter.append(flows)
#f.write("persona nell'area a scelta: \n")
#f.write(str(count_p_custom_area) + "\n")
f.write("Media lunghezza cammini: " + str(stat.mean([v for v in people_dist.values()])) + "\n\n")
count_zoneB.append(count_p_custom_area)
count_zoneA.append(flows)
times.append(t)

f.write("\n Flow cumulativo: " + str(flow_cum))
flow_mean_total = flow_mean_total / c_flows
mean = mean/ total
f.write("all times\n")
f.write(str(times))
f.write("\n")
f.write("mean ev: " + str(mean) + "\n")
f.write("std ev: " + str(np.std(times)) + "\n")
#f.write("mean door flow rate: " + str(flow_mean_total) + "\n")
#f.write("mean desk flow rate: " + str(stat.mean(count_p_custom_area)) + "\n")
f.write("\n\n" + "Velocità medie :" + "\n" + str(velocities_mean) + "\n")
f.write("Flussi ad ogni dt :" + "\n" + str(flow_iter))
#print(time_single_unit)
for id in time_single_unit:
    mean = sum(time_single_unit[id]) / len(time_single_unit[id])
    time_single_unit[id] = [mean]
    time_single_unit[id].extend(exit_single_unit[id])
    if len(door_lines) == 1:
        time_single_unit[id].append(0)

f.write("\ntime_single_unit\n")
f.write(str(time_single_unit))
'''
sys.exit()


import sys
sys.stdin = open('input.txt')

def check(i,j,arr):
    global flag
    dx = [-1,-1,-1,0,0,1,1,1]
    dy = [-1,0,1,-1,1,-1,0,1]
    for k in range(8):
        x = dx[k]+i
        y = dy[k]+j
        if 0 <= x < n and 0 <= y < n and arr[x][y] == '*':
            flag = not flag
            break
    if flag:
        bfs(i,j,arr)

def ch(i,j,arr):
    che = True
    dx = [-1,-1,-1,0,0,1,1,1]
    dy = [-1,0,1,-1,1,-1,0,1]
    for k in range(8):
        x = dx[k]+i
        y = dy[k]+j
        if 0 <= x < n and 0 <= y < n and arr[x][y] == '*':
            che = not che
            break
    return che

def bfs(i,j,arr):
    global count
    dx = [-1, -1, -1, 0, 0, 1, 1, 1]
    dy = [-1, 0, 1, -1, 1, -1, 0, 1]
    queue = [[i,j]]
    arr[i][j] = 1
    visit[i][j] = 1
    while queue:
        cnt = 0
        a, b = queue[0][0], queue[0][1]
        del queue[0]
        if ch(a,b,arr):
            for k in range(8):
                x = dx[k] + a
                y = dy[k] + b
                if 0 <= x < n and 0 <= y < n and arr[x][y] != '*' and visit[x][y] == 0:
                    arr[x][y] = 1
                    visit[x][y] = 1
                    queue.append([x,y])
    count += 1

t = int(input())
result = []
for tr in range(t):
    n = int(input())
    arr = []
    for i in range(n):
        arr.append(list(input()))

    visit = [[0]*n for _ in range(n)]
    count = 0
    for i in range(n):
        for j in range(n):
            if arr[i][j] == '.':
                flag = True
                check(i,j,arr)
    for i in range(n):
        for j in range(n):
            if arr[i][j] == '.':
                count += 1
    result.append('#%d %d'%(tr+1,count))
print('\n'.join(result))

python daily coding/2020.7.11 (SW Expert Academy)/1868번 (파핑파핑 지뢰찾기).py

1868번 (파핑파핑 지뢰찾기).py


from glove_split_merge import *

def got_best_alpha():
    Cluster.GAMMA = 0.95
    best_pre, best_coll, best_f1, best_alpha = 0, 0, 0, 0
    truths, predicts = split_phase(flattened_train_data_path)
    split_pre, split_coll, split_f1 = evaluation(truths, predicts)
    print("====split====", split_pre, split_coll, split_f1)
    for alpha in np.arange(0.8, 1, 0.02):
        final_pre = merge_phases(deepcopy(predicts), alpha)
        pre, coll, f1 = evaluation(truths, final_pre)
        print("====merge {}====".format(alpha), pre, coll, f1)
        if f1 > best_f1:
            best_pre, best_coll, best_f1, best_alpha = pre, coll, f1, alpha
        elif f1 < best_f1-0.2:
            break
        if pre < coll and f1 < split_f1:
            print("-----failed-----")
            break

    print("=====best {}======".format(best_alpha), best_pre, best_coll, best_f1)

if __name__ == "__main__":
    got_best_alpha()


import pickle
import gzip
import numpy as np
import os

def load_data():
    current_path = os.getcwd()
    data_path = current_path + "//data"
    f = gzip.open(data_path + '//mnist.pkl.gz', 'rb')
    training_data, validation_data, test_data = pickle.load(f, encoding='latin1')
    f.close()
    return (training_data, validation_data, test_data)

def load_data_wrapper():
    tr_d, va_d, te_d = load_data()
    training_inputs = [np.reshape(x, (784, 1)) for x in tr_d[0]]
    validation_inputs = [np.reshape(x, (784, 1)) for x in va_d[0]]
    test_inputs = [np.reshape(x, (784, 1)) for x in te_d[0]]
    training_data = list(zip(training_inputs, tr_d[1]))
    validation_data = list(zip(validation_inputs, va_d[1]))
    test_data = list(zip(test_inputs, te_d[1]))
    return (training_data, validation_data, test_data)


import smtplib
import time
import imaplib
import email
import pandas as pd
import os

def get_decoded_email_body(message_body):
    raw_email_string = message_body.decode('utf-8')
# converts byte literal to string removing b''
    email_message = email.message_from_string(raw_email_string)
# this will loop through all the available multiparts in mail
    #print(email_message)
    for part in email_message.walk():
       # print(part)
        if part.get_content_type() != "text/plain": # ignore attachments/html
            body = part.get_payload(decode=True)
            #save_string = str("D:Dumpgmailemail_" + str(x) + ".eml")
    print(body.decode('utf-8'))
    return body.decode('utf-8')

def parse_html(body):
    from bs4 import BeautifulSoup
    import re

    p=re.compile(r'(\d{3}[-\.\s]??\d{3}[-\.\s]??\d{4}|\(\d{3}\)\s*\d{3}[-\.\s]??\d{4}|\d{3}[-\.\s]??\d{4})')
    Csoup = BeautifulSoup(body, 'html.parser')
    for order in Csoup.find_all('div', {'id':re.compile('.*cust_service_info')}):

        order=list(order.find_all(text=True))
        order_det=order[2].split(' ')
        order_no=order_det[1]
        if len(order_det)==3:
            order_type=order_det[2]
        else:
            order_type='Delivery'

        cust_info_collect=0
        order_info_collect=0
        cust_info_ilne_count=0
        cust_addr=''
        order_details=''
        order_dict={}
    for j in Csoup.find_all('tr'):
        cols =j.find_all('td')
        vals=([ele.text.strip() for ele in cols])
        #print(vals)
        if vals[0]=='' :
            continue

        if vals[0] in ('Prepare for:','Deliver to:') :
            #print(1)
            cust_info_collect=1
            continue
        if vals[0]=='Qty':
            #print(2)
            order_info_collect=1
            cust_info_collect=2
            continue
        if cust_info_collect==1 and order_info_collect==0:
            #print(3)
            if cust_info_ilne_count==0:
                #print(4)
                cust_name=vals[0]
                order_date=vals[1]
                cust_info_ilne_count=1
            else:
                if(re.match(p,vals[0]) ):
                    cust_num=vals[0]
                else:
                    if len(vals)>=2 :
                        order_date+=vals[1]
                    cust_addr+=vals[0]
        if vals[0]=='Subtotal':
            #print(6)
            order_info_collect=2
            continue
        if order_info_collect==1:
            #print(7)
            order_details+=vals[1]+','
            order_dict[vals[1]]=order_no
    '''
    print('order_type:'+order_type)
    print('order_no:'+order_no)
    print('ordered_date:'+order_date)
    print('cust_name:'+cust_name)
    print('cust_addr:'+cust_addr)
    print('ordered_items:'+order_details)
    print('phone:'+cust_num)'''
    return(order_type,order_no,order_date,order_details,cust_name,cust_addr,cust_num)
    #return(order_type)

def Extract_Orders_4m_mail(SMTP_SERVER,FROM_EMAIL,FROM_PWD,LABEL,SENDER,SUBJECT_SEARCH ):
    #try:
        mail = imaplib.IMAP4_SSL(SMTP_SERVER,PORT)
        mail.login(FROM_EMAIL, FROM_PWD)
        mail.select(LABEL)

        type, data = mail.search(None, '( SINCE "01-Jan-2017" BEFORE "31-dec-2017" )' )
        mail_ids = data[0]

        id_list = mail_ids.split()

        order_type=[]
        order_no=[]
        order_date=[]
        order_details=[]
        item_no=[]
        cust_name=[]
        cust_addr=[]
        cust_num=[]
        order_dict={}
        o_dict={}
        for i in reversed(id_list):

            typ, data = mail.fetch(i, '(RFC822)')

            for response_part in data:

                if isinstance(response_part, tuple):
                    msg = email.message_from_string(response_part[1].decode('utf-8'))
                    email_subject = msg['subject']
                    email_from = msg['from']
#                     print('From : ' + email_from + '\n')
#                     print('Subject : ' + email_subject + '\n')
#                     print('date : ' + msg['date'] + '\n')
                    html_code=get_decoded_email_body (response_part[1])
                    wf.write(html_code)
                    ordertype,orderno,orderdate,orderdetails,custname,custaddr,custnum=parse_html(html_code)

                    order_type.append(ordertype)
                    order_no.append(orderno)
                    order_date.append(orderdate)
                    iph=orderdetails.split(',')[0].split('.')
                    order_details.append(iph[1])
                    item_no.append(iph[0])
                    cust_name.append(custname)
                    cust_addr.append(custaddr)
                    cust_num.append(custnum)
                    for i in range(len(orderdetails.split(','))):
                        if i==0 or i==len(orderdetails.split(','))-1:
                            continue
                        order_type.append('')
                        order_no.append('')
                        order_date.append('')
                        iph=orderdetails.split(',')[i].split('.')
                        order_details.append(iph[1])
                        item_no.append(iph[0])
                        cust_name.append('')
                        cust_addr.append('')
                        cust_num.append('')

        return{'ORDER_TYPE':order_type,'ORDER_NO':order_no,'ORDER_DATE':order_date,'ITEM_NO':item_no,'ITEM_DESC':order_details,'CUSTOMER_NAME':cust_name,'CUSTOMER_ADDR':cust_addr,'CUSTOMER_PHONE':cust_num}

#Enter gmail uid and pwd along with sender ,subject_search
print("Enter your gmail id ex:'myid@gmail.com'")
EMAIL  = input()
print("Enter your GMAIL password")
PWD    = input()
SERVER = "imap.gmail.com"
print("Enter the sender name you are looking for ex:'James Bond'")
SENDER= input()
print("Enter the search string in SUBJECT  you are looking for ex:'Orders'")
SUBJECT_SEARCH=input()
FROM_LABEL='INBOX'
PORT   = 993
path="D:\\"
file="Customer_order_details.xls"
file1="Customer_order_details.html"
ff=os.path.join(path, file)

wf=open(os.path.join(path,file1),'w')

ExWriter = pd.ExcelWriter(ff)
dic1=Extract_Orders_4m_mail(SERVER,EMAIL,PWD,FROM_LABEL,SENDER,SUBJECT_SEARCH)
df_CustData=pd.DataFrame(dic1)
df_CustData.to_excel(ExWriter,'sheet1',index=False,)

ExWriter.save()
print(" here is your file '%s'"%(ff))


#!/usr/bin/env python3
import sys
import os
import pandas as pd
#os.chdir('/Users/jh/Documents/iPythonNB')
import argparse
import re

def main():
    dirdic = {'HI':     '/Users/jh/Documents/iPythonNB/HIforBD/data/dataFrameSPSHI.csv',
              'rand':   '/Users/jh/Documents/iPythonNB/Ranb/data/dataFrameRanbrand.csv',
              'ranU':   '/Users/jh/Documents/iPythonNB/Ranb/data/dataFrameRanbranU.csv',
              'mixU':   '/Users/jh/Documents/iPythonNB/Ranb/data/dataFrameRanbmixU.csv',
              'setPBC': '/Users/jh/Documents/iPythonNB/Ranb/data/dataFrameRanbsetPBC.csv'
                  }
    parser = argparse.ArgumentParser(description='create or update pandas dataframe.')
    parser.add_argument("-sd", "--search_dirs", nargs="+", type=str, required=True, help='search data in these directories.')
    parser.add_argument("-u", "--update", type=str, required=True, help='HI, rand, ranU, mixU, setPBC')
    parser.add_argument("--reset",  action='store_true', required=False, help='set this flag, if you want to reset the dataframe with the data in sd')

    #parser.add_argument("-u", "--update", action='store_true', required=False, help='set -f flag, if you want to force msd and fit')
    args = parser.parse_args()
    csvdir = dirdic[args.update]
    if ( not os.path.exists(csvdir)) or args.reset==True:
        if args.reset==True: print("Reseting data frame ",args.update,"...")
        # if dataframe does not exists: create it!
        reviseDF(pd.DataFrame(makeDict(args.search_dirs[0]))).to_csv(csvdir, encoding='utf-8',index=False)
        return 0
    else:
        dfold = pd.read_csv(csvdir, encoding='utf-8')
    #print(dfold.query('preEwald==True and n==2 and t==200 and p==1'))
    for newdir in args.search_dirs:
        dfnew = pd.DataFrame(makeDict(newdir))
        dfold = updateDF(dfnew,dfold)
    #print(dfold.query('preEwald==True and n==2 and t==200 and p==1'))
    reviseDF(dfold).to_csv(csvdir, encoding='utf-8',index=False)

def getdd0(pathtofitfile):
    with open(pathtofitfile, 'r') as f:
        first_line = f.readline()
        return (float(first_line.split()[1]))/6

def getphiEff(pathtofitfile):
    with open(pathtofitfile, 'r') as f:
        first_line = f.readline()
        return (float(first_line.split()[0]))

def makeDict(directory):
    folder_ignore = ['noreset','Release','sim_data','fixb']
    skipdir = ['gamma','gamma2','pointq','ranRodRel','ranRod']
    dictlist = []
    problemFolders=[]
    for root, dirs, files in os.walk(directory):
        for folderx in dirs:
            if folderx == "InstantValues":
                if os.path.isfile(os.path.join(root,folderx,"linear_fit_parametersMSD.txt")):
                    pathto = root.split('/')
                    pathto = pathto[pathto.index('workspace-cpp')+1:]
                    #print(pathto)
                    datadict = {}
                    datadict['path'] = root
                    for folder in pathto:
                        if folder in folder_ignore: continue
                        elif folder == "SPS": folder = "SPSHI"
                        # extract number with regular expression
                        num = re.findall(r"[+-]?\d+(?:\.\d+)?", folder)
                        if num == []:
                            #print('No number in ',folder)
                            datadict[folder] = True
                        elif len(num) == 1:
                            #print('Number is',num[0],'in folder ',folder)
                            if not folder.endswith(num[0]):
                                #print('!!! Maybe Problem with folder: ', folder)
                                if not folder in problemFolders:
                                    problemFolders.append(folder)
                                datadict['pepType'] = folder
                            else:
                                var = re.sub(num[0], '', folder)
                                datadict[var] = float(num[0])
                        else:
                            print('\n====> ERROR! more than one number in',folder)
                    # add D/D0
                    datadict['dd0'] = getdd0(os.path.join(root,folderx,"linear_fit_parametersMSD.txt"))
                    # add phiEff
                    if os.path.isfile(root+'/'+folderx+'/phiEff.txt'):
                        datadict['phiEff'] = getphiEff(root+'/'+folderx+'/phiEff.txt')
                    dictlist.append(datadict)
    if problemFolders!=[]: print("Possible Problems in:",problemFolders)
    return dictlist

def reviseDF(df):
    # http://stackoverflow.com/questions/13148429/how-to-change-the-order-of-dataframe-columns
    # make path last column and dd0 second last
    cols = df.columns.tolist()
    #print(cols)
    cols.remove("path"); cols.remove("dd0"); cols.remove("dt"); cols.remove("t");
    cols=["t","dt"]+cols+["dd0","path"]
    #print(cols)
    return df[cols]

def updateDF(dfnew,dfold):
    # This bit of code combines the old df with the new one and drops all duplicates in path
    return dfold.append(dfnew).drop_duplicates(subset='path', keep='last').reset_index(drop=True)

if __name__ == '__main__':
    main()


from django.shortcuts import render_to_response, redirect, get_object_or_404
from django.template import RequestContext
from django.contrib import messages
from utils.group_decorator import group_admin_required
from django.core.urlresolvers import reverse
from intranet.caffeine_manager.trays.models import Tray
from intranet.caffeine_manager.trays.forms import TrayForm
from intranet.caffeine_manager.views import fromLocations
import subprocess

def view(request):
    request.session['from']=fromLocations.TRAYS
    trays=Tray.objects.all().order_by('tray_number')
    is_caffeine_admin=request.user.is_group_admin('Caffeine')
    return render_to_response(
       'intranet/caffeine_manager/trays/trays.html',
       {
         'section':'intranet',
         'page':'caffeine',
         'sub_page':'trays',
         'trays':trays,
         'is_caffeine_admin':is_caffeine_admin
       },
       context_instance=RequestContext(request))

@group_admin_required(['Caffeine'])
def add_tray(request):
    tray_form=None;
    if request.method == 'POST':
        tray_form=TrayForm(request.POST)
        if tray_form.is_valid():
            tray_form.save()
            return redirect(reverse('cm_trays_view'))
    else:
        tray_form=TrayForm()

    return render_to_response(
       'intranet/caffeine_manager/trays/edit_tray.html',
       {
         'section':'intranet',
         'page':'caffeine',
         'form':tray_form
       }, context_instance=RequestContext(request))

#@group_admin_required(['Caffeine'])
def edit_tray(request, trayId):
    tray=get_object_or_404(Tray, pk=trayId)
    if request.method == 'POST':
        tray_form=TrayForm(request.POST, instance=tray)
        if tray_form.is_valid():
            tray_form.save()
            return redirect(reverse('cm_trays_view'))
    else:
        tray_form=TrayForm(instance=tray)

    return render_to_response(
       'intranet/caffeine_manager/trays/edit_tray.html',
       {
         'section':'intranet',
         'page':'caffeine',
         'form':tray_form,
         'id':trayId
       }, context_instance=RequestContext(request))

@group_admin_required(['Caffeine'])
def delete_tray(request, trayId):
    get_object_or_404(Tray, pk=trayId).delete()
    return redirect(reverse('cm_trays_view'))

@group_admin_required(['Caffeine'])
def force_vend(request, trayId):
    ret=subprocess.call(['ssh', 'nassri2@acm.illinois.edu', "echo 'not yet implemented'"]) # Requires a valid ssh key
    if ret == 0:
        messages.add_message(request, messages.SUCCESS, 'Force vend successful!')
    else:
        messages.add_message(request, messages.ERROR, 'Force vend failed.')
    return redirect(reverse('cm_trays_view'))


from pandac.PandaModules import *
from toontown.toonbase import ToontownGlobals
from direct.distributed import DistributedObject
from direct.directnotify import DirectNotifyGlobal
import random
from direct.gui.DirectGui import *
from pandac.PandaModules import *
from toontown.toonbase import TTLocalizer
import HouseGlobals

class EstateManager(DistributedObject.DistributedObject):
    notify = DirectNotifyGlobal.directNotify.newCategory("EstateManager")
    neverDisable = 1

    def __init__(self, cr):

        DistributedObject.DistributedObject.__init__(self, cr)

        self.availableZones = 0
        self.popupInfo = None

        import pdb; pdb.set_trace()

    def disable(self):
        self.notify.debug( "i'm disabling EstateManager rightnow.")
        #self.ignore("requestEstateZone")
        self.ignore("getLocalEstateZone")
        self.ignoreAll()
        if self.popupInfo:
            self.popupInfo.destroy()
            self.popupInfo = None
        DistributedObject.DistributedObject.disable(self)

    def allocateMyEstateZone(self):
        # Get a zone for our own estate, i.e. we are going home right now
        self.getLocalEstateZone(base.localAvatar.getDoId())

    def getLocalEstateZone(self, avId):
        # Fullfill client request for estateZone
        name = ""
        if base.localAvatar.doId == avId:
            # going to our own home, provide AI with userName
            name = base.cr.userName
        self.sendUpdate("getEstateZone", [avId, name])

    def setEstateZone(self, ownerId, zoneId):
        # The AI is telling us the zone for this avatars estate
        self.notify.debug("setEstateZone(%s, %s)" % (ownerId, zoneId))

        # Send this to other hooks on the client side
        messenger.send("setLocalEstateZone", [ownerId, zoneId])

    def generate(self):
        self.notify.debug("BASE: generate")
        DistributedObject.DistributedObject.generate(self)

        # register with the cr
        base.cr.estateMgr = self

        # listen for requests
        #self.accept("requestEstateZone", self.allocateMyEstateZone)
        self.accept("getLocalEstateZone", self.getLocalEstateZone)
        #self.__createRandomNumGen()

        # listen for the generate event, which will be thrown after the
        # required fields are filled in
        self.announceGenerateName = self.uniqueName("generate")
        #self.accept(self.announceGenerateName, self.handleAnnounceGenerate)

    def setAvHouseId(self, avId, houseIds):
        self.notify.debug("setAvHouseId %d" % base.localAvatar.doId)
        for av in base.cr.avList:
            if av.id == avId:
                houseId = houseIds[av.position]
                ownerAv = base.cr.doId2do.get(avId)
                if ownerAv:
                    ownerAv.b_setHouseId(houseId)
                return

    def sendAvToPlayground(self, avId, retCode):
        self.notify.debug("sendAvToPlayground: %d" % avId)
        messenger.send("kickToPlayground", [retCode])

    def leaveEstate(self):
        if self.isDisabled():
            self.notify.warning("EstateManager disabled; unable to leave estate.")
            return

        self.sendUpdate("exitEstate")

    def removeFriend(self, ownerId, avId):
        self.notify.debug("removeFriend ownerId = %s, avId = %s" % (ownerId, avId))
        # The estate owner is  removing avId from his friends list.
        # Notify the AI, and kick the ex-friend out of the estate
        self.sendUpdate("removeFriend", [ownerId, avId])


# encoding: utf-8
"""
@version: ??
@author: chenyitao
@license: Apache Licence
@software: PyCharm
@file: binance_protocol_base.py
@time: 2018/4/16 10:19
"""
import json
import logging

from tddc import RecordManager, gevent

from ..base_protocol import WSProtocolBase

log = logging.getLogger(__name__)

class BinanceWSProtocolBase(WSProtocolBase):

    platform = 'binance'

    def start(self):
        raise NotImplementedError

    def _start(self):
        while True:
            if self.pairs:
                pairs = list(self.pairs)
                self.src_currency_pairs = pairs
                self.req_currency_pairs = [pair.replace('_', '') for pair in pairs]
                self.start()
                return
            gevent.sleep(5)
            log.warning('Currency Pairs Not Fount.')

    def message(self, data):
        raise NotImplementedError

    def _message(self, payload, isBinary):
        result = json.loads(payload)
        self.message(result)


import argparse
import os
import pathlib
import h5py
from tqdm import tqdm
parser = argparse.ArgumentParser()
parser.add_argument('--input_dir', default='data/multicoil_train/')
parser.add_argument('--output_dir', default='data/multicoil_splits_train/')

args = parser.parse_args()

input_dir = args.input_dir
output_dir = args.output_dir
pathlib.Path(output_dir).mkdir(exist_ok=True)

for fname in tqdm(os.listdir(input_dir)):
    fpath = os.path.join(input_dir, fname)
    name = fname.split('.')[0]
    with h5py.File(fpath, 'r') as data:
        kspace = data['kspace']
        target = data['reconstruction_rss']
        norm, max_volume = data.attrs['norm'], data.attrs['max']
        acquisition =  data.attrs['acquisition']

        for i, k in enumerate(kspace):
            t = target[i]
            f = h5py.File('%s_%d.h5'%(os.path.join(output_dir, name), i), 'w')
            f['kspace'] = k
            f['target'] = t
            f['norm'] = [norm, max_volume]
            f['acquisition'] = acquisition
            f.close()


import pygame
import numpy as np
import random
import os
import settings
from agents import abstract_conv_agent

class HumanAgent(abstract_conv_agent.AbstractConvAgent):

    def __init__(self, name, input_shape, window_length, nr_actions, learning_rate, gamma, epsilon, min_epsilon,
                 epsilon_decay):

        super().__init__(self, name, input_shape, window_length, nr_actions,
                                                       learning_rate, gamma, epsilon, min_epsilon, epsilon_decay)
        print("Initializing Human Agent witch convolutional network")

    def decay_epsilon(self):
        pass

    def policy_out_of_event(self, event):
        action = settings.MOVE_FORWARD

        # Returns the action
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_RIGHT:
                action = settings.MOVE_RIGHT
            elif event.key == pygame.K_LEFT:
                action = settings.MOVE_LEFT
            elif event.key == pygame.K_SPACE:
                action = settings.BLAST
            else:
                action = settings.MOVE_FORWARD

        elif event.type == pygame.KEYUP:
            action = settings.MOVE_FORWARD

        return action

class SelfLearningAgent(abstract_conv_agent.AbstractConvAgent):

    def __init__(self, name, input_shape, window_length, nr_actions, learning_rate, gamma, epsilon, min_epsilon,
                 epsilon_decay, human_guided):

        super().__init__(self, name, input_shape, window_length, nr_actions,
                                                       learning_rate, gamma, epsilon, min_epsilon, epsilon_decay)
        self.human_guided = human_guided
        if self.human_guided:
            print("Initializing Human Guided Conv Agent")
        else:
            print("Initializing Conv Agent")

    def policy_out_of_event(self, event):
        return None

    def policy_out_of_experience(self, experience):

        actions = []

        if np.random.rand() <= self.epsilon:
            'Exploration'
            return random.randrange(self.nr_actions)
        else:
            'Exploitation'
            actions = self.model.predict(experience)

        # return max action if not a blast
        action = np.argmax(actions)

        return action

    def load_model(self, weights_file):
        """ Loads model architecture, weights, compilation information and optimizer. Doesnt copy the name of the weights
        if the model is human guided."""
        if os.path.isfile(weights_file):
            self.model.load_weights(weights_file)
            if not self.human_guided:
                self.name = os.path.splitext(os.path.basename(weights_file))[0]

class InteractiveAgent(abstract_conv_agent.AbstractConvAgent):

    def __init__(self, name, input_shape, window_length, nr_actions, learning_rate, gamma, epsilon, min_epsilon,
                 epsilon_decay):

        super().__init__(self, name, input_shape, window_length, nr_actions,
                                                       learning_rate, gamma, epsilon, min_epsilon, epsilon_decay)
        print("Initializing Interactive Agent with conv network")

    def policy_out_of_event(self, event):
        action = None

        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_SPACE:
                action = settings.BLAST
        return action

    def policy_out_of_experience(self, experience):

        actions = []

        if np.random.rand() <= self.epsilon:
            'Exploration'
            # Return a non-blast action
            return random.randrange(self.nr_actions - 1)
        else:
            'Exploitation'
            actions = self.model.predict(experience)

        # return max action if not a blast, if blast, return secondbest
        action = np.argmax(actions)

        if action == settings.BLAST:
            action = (np.argsort(actions))[0][self.nr_actions - 2]

        return action


#!/usr/bin/env python
# -*- coding: utf-8 -*-

u"""
### 文档
这里可以写上此产品的长篇描述，
使用markdown语法书写哦~
"""

from airflow import DAG
from airflow import utils
from airflow.operators.bash_operator import BashOperator
from datetime import datetime,timedelta

default_args = {
    'owner': 'chenxianxin',
    'email': ['chenxianxin@sogou-inc.com'],
    'email_on_failure': False,
    'email_on_retry': False,
    'retries': 1,
    'retry_delay': timedelta(minutes=5),
    'execution_timeout': timedelta(hours=12),
    'depends_on_past': True,
    'start_date': utils.dates.days_ago(1)
}

dag = DAG(
    'temp35',
    default_args = default_args,
    description = u'信息流业务统计',
    schedule_interval = "@daily"
)
dag.doc_md = __doc__

env = {'HIVE_CONF_DIR':'daohang','HADOOP_CONF_DIR':'daohang'}

lens = [140,5,5,2,2]

temps = []
for n in range(lens[0]):
    temp = 'temp_%s' % n
    temp = BashOperator(
    dag = dag,
    env = env,
    task_id = temp,
    svn_url = 'http://svn.sogou-inc.com/svn/sogouime/DataAnalysis/platform_java/imedaapptask/newsfeed/temp',
    bash_command ='sleep 120',
    schedule_interval = '0 5 * * *',
    doc_md = u"""""")
    temps.append(temp)

taemps = []
for n in range(lens[1]):
    taemp = 'taemp_%s' % n
    taemp = BashOperator(
    dag = dag,
    env = env,
    task_id = taemp,
    svn_url = 'http://svn.sogou-inc.com/svn/sogouime/DataAnalysis/platform_java/imedaapptask/newsfeed/taemp',
    bash_command ='sleep 120',
    schedule_interval = '0 5 * * *',
    doc_md = u"""""")
    taemps.append(taemp)

tbemps = []
for n in range(lens[2]):
    tbemp = 'tbemp_%s' % n
    tbemp = BashOperator(
    dag = dag,
    env = env,
    task_id = tbemp,
    svn_url = 'http://svn.sogou-inc.com/svn/sogouime/DataAnalysis/platform_java/imedaapptask/newsfeed/tbemp',
    bash_command ='sleep 120',
    schedule_interval = '0 5 * * *',
    doc_md = u"""""")
    tbemps.append(tbemp)

tcemps = []
for n in range(lens[3]):
    tcemp = 'tcemp_%s' % n
    tcemp = BashOperator(
    dag = dag,
    env = env,
    task_id = tcemp,
    svn_url = 'http://svn.sogou-inc.com/svn/sogouime/DataAnalysis/platform_java/imedaapptask/newsfeed/tcemp',
    bash_command ='sleep 120',
    schedule_interval = '0 5 * * *',
    doc_md = u"""""")
    tcemps.append(tcemp)

tdemps = []
for n in range(lens[4]):
    tdemp = 'tdemp_%s' % n
    tdemp = BashOperator(
    dag = dag,
    env = env,
    task_id = tdemp,
    svn_url = 'http://svn.sogou-inc.com/svn/sogouime/DataAnalysis/platform_java/imedaapptask/newsfeed/tdemp',
    bash_command ='sleep 120',
    schedule_interval = '0 5 * * *',
    doc_md = u"""""")
    tdemps.append(tdemp)

for te in temps:
    for ta in taemps:
        for tb in tbemps:
            for tc in tcemps:
                for td in tdemps:
                    ta >> te
                    tb >> ta
                    tc >> tb
                    td >> tc

#for te in temps:
#    for ta in taemps:
#        ta >> te


import tkinter
import random
import queue
import copy
#0 壁
#1 道
#2 進行中

W = 1000
msize = int(1000 / W)
arr = [[0 if i == 0 or j == 0 or i == W-1 or j == W-1 else 1 for i in range(W)] for j in range(W)]
unsearch_list = [[0 for i in range(W)] for j in range(W)]
Evenlist = [[i,j]for i in range(W) for j in range(W) if i % 2 == 0 and j % 2 == 0 and i != 0 and j != 0 and i != W-1 and j != W-1]
cp_Even = copy.deepcopy(Evenlist)
Start = cp_Even[0]
Goal = cp_Even[len(cp_Even)-1]
ans_routelist = []
gowaylist = [[0,-1],[1,0],[0,1],[-1,0]]
s_end = []

root = tkinter.Tk()
root.title("meiro")
root.geometry("1000x1200")   #ウインドウサイズ（「幅x高さ」で指定）

#キャンバスエリア
canvas = tkinter.Canvas(root, width = 810, height = 810)#Canvasの作成

#キャンバスバインド
canvas.place(x=0, y=0)
class maze():
    def mepri(self):
        X = 0
        Y = msize
        canvas.delete('o')
        while True:
            X += msize
            if X >= (W+1)*(msize):
                Y += msize
                X = msize
            if Y >= (W+1)*(msize):
                break
            if int(X/msize)-1 < W and int(Y/msize)-1 < W:
                if arr[int(X/msize)-1][int(Y/msize)-1] == 0:
                    canvas.create_rectangle(X, Y, msize+X, msize+Y, fill = "white", tag = 'o')
                if arr[int(X/msize)-1][int(Y/msize)-1] == 1:
                    canvas.create_rectangle(X, Y, msize+X, msize+Y, fill = "black", tag = 'o')
                if arr[int(X/msize)-1][int(Y/msize)-1] == 2:
                    canvas.create_rectangle(X, Y, msize+X, msize+Y, fill = "green", tag = 'o')
                if int(X/msize)-1 == 2 and int(Y/msize)-1 == 2:
                    canvas.create_rectangle(X, Y, msize+X, msize+Y, fill = "red", tag = 'o')
                if int(X/msize)-1 == Goal[0] and int(Y/msize)-1 == Goal[1]:
                    canvas.create_rectangle(X, Y, msize+X, msize+Y, fill = "blue", tag = 'o')
        canvas.update()
        root.after(0)

    def create_maze(self):
        randxy = Evenlist.pop(random.randint(0,len(Evenlist)-1))
        s_end.append(randxy)
        arr[randxy[0]][randxy[1]] = 0

        while len(Evenlist) > 0:
            random.shuffle(gowaylist)
            for i in range(4):
                x = randxy[0] + gowaylist[i][0] * 2
                y = randxy[1] + gowaylist[i][1] * 2
                if arr[x][y] == 1:
                    Evenlist.remove([x,y])
                    s_end.append([x,y])
                    arr[x][y] = 0
                    arr[x - gowaylist[i][0]][y - gowaylist[i][1]] = 0
                    break
                    if i == 3 :
                        s_end.remove([x,y])
                randxy = s_end[random.randint(0,len(s_end)-1)]
            maze.mepri()

maze = maze()
maze.create_maze()

routelist = [[([0,0],[0,0]) for i in range(W)] for j in range(W)]
Pensearch = queue.Queue()
sin_start = copy.deepcopy(Start)
Pensearch.put(sin_start)
def maze_solve():
    while not Pensearch.empty():
        myplace = Pensearch.get()
        if myplace[0] == Goal[0] and myplace[1] == Goal[1]:
            break
        arr[myplace[0]][myplace[1]] = 2
        for i in range(4):
            if arr[myplace[0] + gowaylist[i][0]][myplace[1] + gowaylist[i][1]] != 0:
                continue
            Pensearch.put([myplace[0] + gowaylist[i][0],myplace[1] + gowaylist[i][1]])
            masu_relation = ((myplace[0] + gowaylist[i][0],myplace[1] + gowaylist[i][1]), (myplace[0],myplace[1]))
            routelist[myplace[0] + gowaylist[i][0]][myplace[1] + gowaylist[i][1]] = masu_relation
        maze.mepri()

def ans_route(ansplace):
    if sin_start == ansplace:
        return
    for i in range(W):
        for j in range(W):
            if routelist[i][j][0][0] == ansplace[0] and routelist[i][j][0][1] == ansplace[1]:
                ans_routelist.append(ansplace)
                ans_route(routelist[i][j][1])

def ttoz():
    for i in range(W):
        for j in range(W):
            if arr[i][j] == 2:
                arr[i][j] = 0
    for masu in ans_routelist:
        arr[masu[0]][masu[1]] = 2

maze_solve()
ans_route([W-3,W-3])
ttoz()
maze.mepri()
print("end")
root.mainloop()
#http://algoful.com/Archive/Algorithm/BFS


"""Manage inbound ON command from device."""
import logging

from ...constants import MessageFlagType
from ...topics import THERMOSTAT_SET_POINT_RESPONSE
from ...utils import build_topic
from .. import inbound_handler
from ..inbound_base import InboundHandlerBase

_LOGGER = logging.getLogger(__name__)

class ThermostatSetPointResponseHandler(InboundHandlerBase):
    """Humidity set point command inbound."""

    def __init__(self, address):
        """Init the ThermostatSetPointResponseHandler class."""
        super().__init__(
            topic=THERMOSTAT_SET_POINT_RESPONSE,
            address=address,
            message_type=MessageFlagType.DIRECT,
        )
        self._subscriber_topic = build_topic(
            prefix="handler.{}".format(self._address.id),  # Force address
            topic=THERMOSTAT_SET_POINT_RESPONSE,
            message_type=MessageFlagType.DIRECT,
        )

    @inbound_handler
    def handle_response(self, cmd1, cmd2, target, user_data, hops_left):
        """Handle the Humidity set point response from a device."""
        stage_1_on_minutes = user_data["d3"]
        humidity_high = user_data["d4"]
        humidity_low = user_data["d5"]
        firmwire = user_data["d6"]
        cool_set_point = user_data["d7"]
        heat_set_point = user_data["d8"]
        rf_offset = user_data["d9"]
        self._call_subscribers(
            stage_1_on_minutes=stage_1_on_minutes,
            humidity_high=humidity_high,
            humidity_low=humidity_low,
            firmwire=firmwire,
            cool_set_point=cool_set_point,
            heat_set_point=heat_set_point,
            rf_offset=rf_offset,
        )