import torch.utils.data as data
from os import listdir
from os.path import join
from PIL import Image
from torchvision.transforms import Compose, ToTensor, Resize, RandomCrop, RandomHorizontalFlip, RandomVerticalFlip, Normalize
import PIL
import matplotlib.pyplot as plt
import numpy as np
import os
import torch
import matplotlib.cm as cm
from torchvision import transforms
import copy
import random

batch_size = 4
patch_size = 96
image_size = 320
upscale_factor = 4
save_on_disk_train = False
save_on_disk_test = False

seed = 324
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)

def is_image_file(filename):
    return any(filename.endswith(extension) for extension in [".png"])

def load_img(filepath):
    img = Image.open(filepath).convert('L')
    return img

class DatasetFromFolder(data.Dataset):
    def __init__(self, image_dir, input_transform=None, target_transform=None, bic_transform = None):
        super(DatasetFromFolder, self).__init__()
        self.image_filenames = [join(image_dir, x) for x in listdir(image_dir) if is_image_file(x)]

        self.input_transform = input_transform
        self.target_transform = target_transform
        self.bic_transform = bic_transform

        self.std_transform = Compose([
            Normalize([0.15], [0.15]),
        ])

    def __getitem__(self, index):
        input = load_img(self.image_filenames[index])
        target = copy.deepcopy(input)

        if self.target_transform:
            target = self.target_transform(target)

        input_lr = copy.deepcopy(target)
        if self.input_transform:
            input_lr = transforms.ToPILImage()(input_lr).convert("L")
            input_lr = self.input_transform(input_lr)

        bic = copy.deepcopy(input_lr)
        if self.bic_transform:
            bic = transforms.ToPILImage()(bic).convert("L")
            bic = self.bic_transform(bic)

        return input_lr, bic, target

    def __len__(self):
        return len(self.image_filenames)

def calculate_valid_crop_size(crop_size, upscale_factor):
    return crop_size - (crop_size % upscale_factor)

def input_transform(crop_size, upscale_factor):
    return Compose([
        Resize(crop_size // upscale_factor, PIL.Image.BICUBIC),
        ToTensor(),
    ])

def target_transform(crop_size):
    return Compose([
        RandomCrop(crop_size),
        RandomHorizontalFlip(0.5),
        RandomVerticalFlip(0.5),
        ToTensor(),
    ])

def bic_transform(crop_size):
    return Compose([
        Resize(crop_size, PIL.Image.BICUBIC),
        ToTensor(),
    ])

def get_training_set(upscale_factor, patch_size):
    train_dir='Subset_images/train_HR/'
    crop_size = calculate_valid_crop_size(patch_size, upscale_factor)
    return DatasetFromFolder(train_dir,
                             input_transform = input_transform(crop_size, upscale_factor),
                             target_transform = target_transform(crop_size),
                             bic_transform  = bic_transform(crop_size))

def get_test_set(upscale_factor, image_size):
    test_dir = 'Subset_images/test_HR/'
    return DatasetFromFolder(test_dir,
                             input_transform = input_transform(image_size, upscale_factor),
                             target_transform = Compose([ToTensor(),]),
                             bic_transform  = bic_transform(image_size))

def saveImage(img, epoch, batch, img_type, model, type_result):
    folder_path = "Results/" + model + "/" + type_result + "/"

    if not os.path.exists(folder_path):
        os.makedirs(folder_path)

    file_path = folder_path + str(epoch) + "_" + str(batch) + "_" + img_type + ".png"
    img = img.squeeze().cpu().detach().numpy()
    plt.imsave(file_path, img, cmap = plt.get_cmap('gray'))

def save_Crops(dataset, type_set):
    save_path = 'Subset_images/'
    lr_crops = type_set + '_LR_crops/'
    hr_crops = type_set + '_HR_crops/'
    bic_crops = type_set + '_bic_crops/'

    lr_folder_path = save_path + lr_crops
    hr_folder_path = save_path + hr_crops
    bic_folder_path = save_path + bic_crops

    if not os.path.exists(lr_folder_path):
        os.makedirs(lr_folder_path)

    if not os.path.exists(hr_folder_path):
        os.makedirs(hr_folder_path)

    if not os.path.exists(bic_folder_path):
        os.makedirs(bic_folder_path)

    num_images = 0

    for input_lr, bic, target in dataset:
        file = str(num_images) + '.png'
        path_lr = os.path.join(lr_folder_path, file)
        path_hr = os.path.join(hr_folder_path, file)
        path_bic = os.path.join(bic_folder_path, file)

        input_lr = np.transpose(input_lr.cpu().detach().numpy(), (1, 2, 0))
        target = np.transpose(target.cpu().detach().numpy(), (1, 2, 0))
        bic = np.transpose(bic.cpu().detach().numpy(), (1, 2, 0))

        plt.imsave(path_lr, input_lr[:, :, 0], cmap = plt.get_cmap('gray'))
        plt.imsave(path_hr, target[:, :, 0], cmap = plt.get_cmap('gray'))
        plt.imsave(path_bic, bic[:, :, 0], cmap = plt.get_cmap('gray'))
        num_images += 1

if save_on_disk_train:
    dataset_train = get_training_set(upscale_factor, patch_size)
    save_Crops(dataset_train, "train")

if save_on_disk_test:
    dataset_test = get_test_set(upscale_factor, image_size)
    save_Crops(dataset_test, "test")


from lppls import lppls
import numpy as np
import pandas as pd
from pandas_datareader import data as pdr
from datetime import timedelta, date
from matplotlib import pyplot as plt
import yfinance as yf
yf.pdr_override()

def main():
    ticker = 'ES=F'
    endDate = date.today()
    startDate = endDate - timedelta(days=4 * 365)
    data = pdr.get_data_yahoo(ticker, start=startDate, end=endDate)
    time = [pd.Timestamp.toordinal(t1) for t1 in data.index]
    price = np.log(data['Adj Close'].values)
    observations = np.array([time, price])
    lppls_model = lppls.LPPLS(observations=observations)

    res = lppls_model.mp_compute_nested_fits(
        workers=8,
        window_size=120,
        smallest_window_size=30,
        outer_increment=1,
        inner_increment=5,
        max_searches=25
    )

    lppls_model.plot_confidence_indicators(res)
    plt.show()

if __name__ == '__main__':
    main()


import scrapy
from scrapy import signals
from fundscraper.items import OnlinedstfirstItem, onlinedstsecondItem
from fundscraper.spiders.send_email import send_email,load_previous_data,save_current_data

# Store previously scraped data in a file (you can use a database as well)
PREVIOUS_DATA_FILE1 = '~/.config/fundscraper/data/onlinedst_data1.csv'
PREVIOUS_DATA_FILE2 = '~/.config/fundscraper/data/onlinedst_data2.csv'
columns1=["serial_number","programme_or_scheme","division","start_date","end_date"]
columns2=["serial_number","programme_or_scheme","division"]
load_previous_data(PREVIOUS_DATA_FILE1,columns1)
load_previous_data(PREVIOUS_DATA_FILE2,columns2)
class OnlinedstSpider(scrapy.Spider):
    name = "onlinedst"
    allowed_domains = ["onlinedst.gov.in"]
    start_urls = ["https://onlinedst.gov.in/Login.aspx"]
    first_scraped_data=[]
    second_scraped_data=[]
    def parse(self, response):
        # Extracting data from the first table
        first_table_rows = response.xpath('//div[@id="divprojectformat"]/table//tr')[1:]  # Skipping the header row

        for row in first_table_rows:
            item = OnlinedstfirstItem()  # Create an instance of the OnlinedstfirstItem
            item['serial_number'] = row.xpath('.//td[1]/text()').get()
            item['programme_or_scheme'] = row.xpath('.//td[2]/a/text()').get()
            item['division'] = row.xpath('.//td[3]/span/text()').get()
            item['start_date'] = row.xpath('.//td[4]/span/text()').get()
            item['end_date'] = row.xpath('.//td[5]/text()').get()
            self.first_scraped_data.append({
                'serial_number': item['serial_number'],
                'programme_or_scheme': item['programme_or_scheme'],
                'division': item['division'],
                'start_date': item['start_date'],
                'end_date': item['end_date']
            })
            yield item

        # Extracting data from the second table
        second_table_rows = response.xpath('//div[@id="divprojectformat1"]//table//tr')[1:]  # Skipping the header row

        for row in second_table_rows:
            item = onlinedstsecondItem()
            item['serial_number']= row.xpath('.//td[1]/text()').get()
            item['programme_or_scheme']= row.xpath('.//td[2]/a/text()').get()
            item['division']= row.xpath('.//td[3]/text()').get()
            self.second_scraped_data.append({
                'serial_number': item['serial_number'],
                'programme_or_scheme': item['programme_or_scheme'],
                'division': item['division'],
            })
            yield item

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(OnlinedstSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider

    def spider_closed(self, reason):
        if reason == 'finished':
            send_email(self.first_scraped_data,columns1,"programme_or_scheme",PREVIOUS_DATA_FILE1,"Onlinedst Updates")
            send_email(self.second_scraped_data,columns2,"programme_or_scheme",PREVIOUS_DATA_FILE2,"Onlinedst Updates")
            save_current_data(self.first_scraped_data,PREVIOUS_DATA_FILE1,columns1)
            save_current_data(self.second_scraped_data,PREVIOUS_DATA_FILE2,columns2)


import functools
import json
import os.path
from pathlib import Path
from typing import Optional, cast

import typer
from pykuda2 import Kuda, APIResponse, Mode
from typer import get_app_dir
from rich import print as rprint

APP_NAME = "kuda-cli"

CACHED_SETTINGS: Optional[dict] = None

def get_settings_path() -> Path:
    app_dir = get_app_dir(APP_NAME)
    if not os.path.isdir(app_dir):
        os.mkdir(app_dir)
    return Path(app_dir) / ".settings.json"

def get_all_settings() -> dict:
    global CACHED_SETTINGS
    if CACHED_SETTINGS:
        return CACHED_SETTINGS
    settings_path = get_settings_path()
    if not settings_path.is_file():
        with settings_path.open("w+") as f:
            f.write(json.dumps({}))
    with settings_path.open("r") as f:
        CACHED_SETTINGS = json.loads(f.read())
        return CACHED_SETTINGS

def get_settings(option: str) -> Optional[str]:
    all_settings = get_all_settings()
    return all_settings.get(option)

def update_settings(option: str, value: str):
    all_settings = get_all_settings()
    all_settings[option] = value
    settings_path = get_settings_path()
    with settings_path.open("w") as f:
        f.write(json.dumps(all_settings))

def reset_settings():
    global CACHED_SETTINGS
    CACHED_SETTINGS = None
    settings_path = get_settings_path()
    with settings_path.open("w+") as f:
        f.write(json.dumps({}))

def get_kuda_wrapper() -> "KudaSingleton":
    email = get_settings("email")
    api_key = get_settings("api_key")
    mode = get_settings("mode")
    mode = cast(Mode, mode)
    if not email:
        rprint(
            "[bold red]Error![/bold red] your email has not been configured. please run `kuda config <email> <api_key>`"
        )
        raise typer.Abort()
    if not api_key:
        rprint(
            "[bold red]Error![/bold red] your api_key as not been configured. please run `kuda config <email> <api_key>`"
        )
        raise typer.Abort()
    if not mode:
        rprint(
            "[bold red]Error![/bold red] your mode as not been configured. please run `kuda config <email> <api_key> --<mode>`"
        )
        raise typer.Abort()
    return KudaSingleton(email=email, api_key=api_key, mode=mode)

def strip_raw(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        response = func(*args, **kwargs)
        response.raw = None
        return response

    return wrapper

def override_output(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        response = func(*args, **kwargs)
        if kwargs["data_only"]:
            return response.data
        return response

    return wrapper

def colorized_print(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        response = func(*args, **kwargs)
        rprint(response)

    return wrapper

class KudaSingleton:
    instance: Optional[Kuda] = None

    def __init__(self, email: str, api_key: str, mode: Mode = Mode.DEVELOPMENT):
        ...

    def __new__(cls, *args, **kwargs):
        if not KudaSingleton.instance:
            KudaSingleton.instance = Kuda(*args, **kwargs)
        return KudaSingleton.instance

    def __getattr__(self, item):
        return getattr(self.instance, item)

    def __setattr__(self, key, value):
        return setattr(self.instance, key, value)


import RPi.GPIO as GPIO
import serial
import time
import thread
from unidecode import unidecode
import json
data = {}
data['AD1'] = []

#from time import sleep
ser = serial.Serial('/dev/ttyAMA0',9600);
def thread_1():
    while True:
        if ser.inWaiting:
            time.sleep(2)
            print("thread 1")
            read = ser.readline().decode().strip('\r\n');
            print("READ : " + read)
            readlist = read
            readlist = list(readlist)
            print(readlist[6])
        if(read == "AD1 + S1"):
            status = read
            print("S : " + status)
        elif(readlist[6] == "D"):
            processes = read
            print("p : " + processes)
        elif(read == "AD1 + S1OFF + S2ON"):
            endprocess = read
            print("ee : " + endprocess)
        elif(read == "AD1 + S1OFF + S2OFF"):
            endprocess_2 = read
            print("ee2 : " + endprocess_2)
        elif(read == "AD1 + END"):
            end = read
            print("e : " + end)
            data['AD1'].append({
                'status' : status,
                'process' : processes,
                'process_2' : endprocess_2,
                'endprocess' : endprocess,
                'end' : end
            })
            with open('data.json','w+') as outfile:
                json.dump(data,outfile)
                print("writefile")
        #thread.start_new_thread(thread_2())

def thread_2():
    while True:
            print("thread 2")
            ser.write("AD1 + ST" + "\r\n")
            time.sleep(3)
            #thread.start_new_thread(thread_1())

def thread_3():
    while True:
            print("thread 3")
            ser.write("AD2 + ST" + "\r\n")
            time.sleep(3)

def thread_4():
    while True:
            print("thread 4")
            ser.write("AD1 + R" + "\r\n")
            time.sleep(3)
def thread_5():
    while True:
            print("thread 5")
            ser.write("AD2 + R" + "\r\n")
            time.sleep(3)

thread.start_new_thread(thread_1,())
thread.start_new_thread(thread_2,())

while True:
    pass


# **************************************************************************** #
#                                                                              #
#                                                         :::      ::::::::    #
#    visu.py                                            :+:      :+:    :+:    #
#                                                     +:+ +:+         +:+      #
#    By: tbrizon <tbrizon@student.42.fr>            +#+  +:+       +#+         #
#                                                 +#+#+#+#+#+   +#+            #
#    Created: 2019/10/25 14:10:24 by tbrizon           #+#    #+#              #
#    Updated: 2019/10/25 14:10:25 by tbrizon          ###   ########.fr        #
#                                                                              #
# **************************************************************************** #

import argparse
import matplotlib.pyplot as plt
import numpy as np
import networkx as nx
import os
from pprint import pprint
from matplotlib.animation import FuncAnimation
from reader import parse
from fourmisses import Ant
from matplotlib.offsetbox import TextArea, DrawingArea, OffsetImage, AnnotationBbox
from matplotlib._png import read_png
import matplotlib.image as mpimg

# Colors
bg_color = '#000000'
edge_color = '#A9A9A9'
text_color = '#d4d1de'
default_node_color = '#540606'
ant_colors = ['#08d141', '#bd21bf', '#ff6f00', '#66B615']

# Node size
node_size = 1000

def highlight_node(node, highlight_color):
    node.set_color(highlight_color)
    node.set_edgecolor(edge_color)

def make_ants(pos, speed_rate, data):
    list_ants = []
    for ant_num in range(1, int(data['ants']) + 1):
        ant = Ant(ant_num)
        ant.set_path(data)
        ant.set_location(pos, data)
        ant.set_trip(pos, speed_rate, data)
        ant.color = ant_colors[ant_num % 4]
        list_ants.append(ant)

    return (list_ants)

def draw_ant(position, color, ant_img, args):

    i = 5
    ax = fig.add_subplot(111)
    ant_img = read_png('./images/Fourmisse.png')
    imagebox = OffsetImage(ant_img, zoom=.03)

    if args.images == True and i == 5:
        xy = [position[0], position[1]]
        ab = AnnotationBbox(imagebox, xy,
            xycoords='data',
            boxcoords="offset points")
        ax.add_artist(ab)
    else:
        ant = plt.plot([position[0]], [position[1]], color=color, marker='.', markersize=40.0)
        return (ant)

def draw_node(node_name, node_color, node_size, edge_color):
    node_list = []
    node_list.append(node_name)
    node = nx.draw_networkx_nodes(
        G,
        pos,
        nodelist=node_list,
        node_color=node_color,
        node_size=node_size)
    node.set_edgecolor(edge_color)
    return (node)

def draw_nodes(data, node_color, node_size, edge_color):
    nodes = {}
    for name in data['nodes']:
        if name == data['start'] or name == data['end']:
            node = draw_node(name, node_color, node_size * 7, edge_color)
        else:
            node = draw_node(name, node_color, node_size, edge_color)
        nodes[name] = node
    return (nodes)

def re_init(num, G, pos, speed_rate, data, ants, args, ant_img):

    fig.clear()
    node_size = 800

    if args.images == True:
        background = plt.imread('./images/Background.png')
        plt.imshow(background, zorder=0, extent=[-1.1, 0, 0, 1.10])
        plt.imshow(background, zorder=0, extent=[0, 1.10, 0, 1.10])
        plt.imshow(background, zorder=0, extent=[-1.1, 0, -1.10, 0])
        plt.imshow(background, zorder=0, extent=[0, 1.10, -1.10, 0])

    # nodes and tunnels
    tunnels = nx.draw_networkx_edges(G, pos, edge_color=edge_color, style = 'dotted',width=2.0)
    nodes = draw_nodes(data, default_node_color,
                       node_size, edge_color)
    # draw the ants
    for ant in ants:
        if num < len(ant.trip):
            draw_ant(ant.trip[num], ant.color, ant_img, args)
    room_names = nx.draw_networkx_labels(G,
                                         pos,
                                         font_size=10,
                                         labels=dict([(data['start'],
                                                       'START'),
                                                      (data['end'],
                                                       'END')]),
                                         font_color=text_color)
    fig.set_facecolor(bg_color)
    plt.axis('off')

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("map", help="Path to map file", type=str)
    parser.add_argument("-s", "--solution",
                        help="File path to solution", type=str)
    parser.add_argument("--images", help="Background activation = 1, ant activation = 2 both = 3", action="store_true")
    args = parser.parse_args()
    if not os.path.exists(args.map):
        raise IOError("Map file not found")

    if args.solution and not os.path.exists(args.solution):
        raise IOError("Solution file not found")
    data = parse(args.map, args.solution)
    # Graph
    G = nx.Graph()
    G.add_edges_from(data['tunnels'])
    # Fruchterman-Reingold force-directed algorithm's to set rooms.
    pos = nx.spring_layout(G)
    # Ants speed
    speed_rate = 7
    # create an array of * on the ants object
    ants = make_ants(pos, speed_rate, data)
    # size of the graph
    fig = plt.figure(figsize = (15, 10))
    ant_img = read_png('./images/Fourmisse.png')
    ani = FuncAnimation(
    fig,
    re_init,
    frames=data['num_turns'] *
    speed_rate,
    fargs=(
        G,
        pos,
        speed_rate,
        data,
        ants,
        args,
        ant_img),
    interval=30,
    repeat=True)
    plt.show()


#  © Shahram Talei @ 2019 The University of Alabama - All rights reserved.
#you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation; either version 3 of the License, or
#(at your option) any later version.
#You should have received a copy of the GNU General Public License
#along with this program.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import division
import yt
import numpy as np
from yt.analysis_modules.halo_finding.api import *
from yt.analysis_modules.halo_analysis.api import *
from os import environ
environ['CFLAGS'] = "-I"+np.get_include()

import pyximport; pyximport.install()
#import particle_ops
import argparse

import tempfile
import shutil
import os
import sys

from scipy.spatial.transform import Rotation as R
from numpy import linalg as LA
from operator import mul
from functools import reduce
import matplotlib.pyplot as plt
from yt.units import parsec, Msun, Mpc
from mpl_toolkits.mplot3d import Axes3D

#define an ellipsoid object in an arbitrary orientation (a,b,c,Alpha,Beta,Yamma)

unit_base0 = {'UnitLength_in_cm'         : 3.08568e+24,
             'UnitMass_in_g'            :   1.989e+43,
             'UnitVelocity_in_cm_per_s' :      100000}

unit_base1 = {'length': (1.0, 'Mpc'),
    'velocity': (1.0, 'km/s'),
    'mass': (1.0e10, 'Msun')}

class halo:
    def __init__(self,pos,R,M,pnum,id):
        self.pos=pos
        self.R=R
        self.M=M
        self.pnum=pnum
        self.id=id
class shape:
    def __init__(self,hid,Rv,Mv,R,ax):
        ax.sort()
        self.hid=hid
        self.Rv=Rv
        self.Mv=Mv
        self.R=R
        self.a=ax[2]
        self.b=ax[1]
        self.c=ax[0]
        self.b_a=ax[1]/ax[2]
        self.c_a=ax[0]/ax[2]
        self.T=(ax[2]**2.-ax[1]**2.)/(ax[2]**2.-ax[0]**2.)
class ellipsoid:
    def __init__(self,axis,orientations):#put in tuple (self,p,angle)..p.x.
        self.a=axis[0]
        self.b=axis[1]
        self.c=axis[2]
        self.axis=axis
        self.orientations=orientations
        self.A=orientations[0]
        self.B=orientations[1]
        self.C=orientations[2]
    def IsInside(self, point,R,rescale):# put in tuple p ... p.x
        #r = R.from_euler('zyx', [self.A, self.B, self.C], degrees=True)
        #point=r.apply(point)
        #print("point in is inside:")
        #print(point)
        #print(R)
        if R==0:
            return False
        t=0
        pNew=np.array([0.0,0.0,0.0])
        for i in range(0,3):
            for j in range(0,3):
                #print(point[j])
                #print(self.orientations[j,i])
                pNew[i]+=point[j]*self.orientations[j,i] # each eig vec is [:,i] comp
        #x=point[0]#-center[0]
        #y=point[1]#-center[1]
        #z=point[2]#-center[2]
        #print("point in is inside:")
        #print(pNew)
        #t=np.sum((pNew/self.axis)**2.)
        if rescale:
            factor=R/np.max(self.axis)
        else:
            factor=1

        for i in range(0,3):
            t+=(pNew[i]/(factor*self.axis[i]))**2.
        #t=(x/self.a)**2.+(y/self.b)**2.+(z/self.c)**2
        #print(t)
        if t<=1:
            return True
        else:
            return False
    def rotate(self, orientation):
        self.orientations=oriantation
    def volume(self):
        return self.axis[0]*self.axis[1]*self.axis[2]
    #    eulerrotation
    def PlotEllipsoid(self,ax,center):
        pNew=np.array([0.0,0.0,0.0])
        #for i in range(0,3):
            #for j in range(0,3):
                #print(point[j])
                #print(self.orientations[j,i])
                #pNew[i]+=point[j]*self.orientations[j,i] # each eig vec is [:,i] comp
        coefs = (np.abs(self.a),np.abs(self.b),np.abs(self.c))
        rx, ry, rz = np.sqrt(coefs)
        # Set of all spherical angles:
        u = np.linspace(0, 2 * np.pi, 100)
        v = np.linspace(0, np.pi, 100)
        x = rx * np.outer(np.cos(u), np.sin(v))
        y = ry * np.outer(np.sin(u), np.sin(v))
        z = rz * np.outer(np.ones_like(u), np.cos(v))

        # Plot:
        ax.plot_surface(x, y, z,  rstride=4, cstride=4, color='b', alpha=0.3)

        # Adjustment of the axes, so that they all have the same span:
        max_radius = max(rx, ry, rz)
        #for axis in 'xyz':
        #    getattr(ax, 'set_{}lim'.format(axis))((-max_radius, max_radius))
    def ShapeTesnsor(self,coords,m, Rin,Rout):
        print("Started extracting the shape tensor")
        i=j=0
        c=0
        #N=2#len() use mass instead, you won't need to count!
        shape=[[0,0,0],[0,0,0],[0,0,0]]#np.array([[0,0,0,],[0,0,0,],[0,0,0]])
        s=0
        #points=coords[(coord[not IsInside()]) && ]
        #for i in range(0,3):
        #    p[i]=coords[:,i]
        #rEll=1#np.sqrt(c)Zemp 10
        #we can use IsInside to find both inside and outside
        for i in range(0,3):
            for j in range(0,3):
                c=s=0
                print("s[%d,%d]"%(i,j))
                for point in coords:
                    #print("Before=Rin:%f, Rout:%f"%(Rin,Rout))
                    #print("point before if:")
                    #print(np.sqrt(point[0]**2.+point[0]**2.+point[0]**2.))
                    #print(not(self.IsInside(point,Rin)))
                    #print(self.IsInside(point,Rout))
                    if(self.IsInside(point,Rout,True) and (not(self.IsInside(point,Rin,True)))):
                        #print("point after if:")
                        #print("After=Rin:%f, Rout:%f"%(Rin,Rout))
                        #print("point:%g"%(m*point[i]*point[j]))
                        s+=point[i]*point[j]
                        c+=1
                #s[i,j]/=(N-1)*m
                Mtot=c*m
                if c>1:
                    shape[i][j]=s/(c-1)
                print("particle count for %g<R<%g=%d"%(Rin,Rout,c))
        return shape
def CompareEllipsoids(ell1,ell2):
    AreSame=False;
    error=1.0e-3
    c=0
    inP=np.array([0,0,0])
    dRatio=np.array([0,0])
    #dax=np.abs(np.divide(ell1.axis/ell2.axis))
    for i in range(1,3):
        if(ell1.axis[0] !=0):
            dRatio[i-1]=np.abs((ell1.axis[i]/ell1.axis[0])/(ell2.axis[i]/ell2.axis[0]))
        #dRatio[i-1]=0
    #dori=np.abs(np.divide(ell1.orientations/ell2.orientations))
    d=np.sum(dRatio[(dRatio<1+error) & (dRatio>1-error)])
    if(d==0):
        for i in range(0,3):
            for j in range(0,3):
                inP[i]+=ell1.orientations[j,i]*ell2.orientations[j,i]
            if(inP[i]<((ell1.axis[i])**2.+error) and inP[i]>((ell1.axis[i])**2.-error)):
                c+=1
        if(c==3):
            AreSame=True
    return AreSame;
def GetShape(h,ds):
    FullShape=[]
    ad = ds.all_data()
    xp=ad[("all","particle_position_x")]
    yp=ad[("all","particle_position_y")]
    zp=ad[("all","particle_position_z")]
    coordsArray=np.array((xp,yp,zp))
    coordinatesDM =coordsArray.T# we need to reshape
    #print(coordinatesDM)
    #dds = halo.halo_catalog.data_ds
    #center = dds.arr([halo.quantities["particle_position_%s" % axis] \
    #for axis in "xyz"])
    c2=r2=0
    #center*=3.24078e-25
    #center=center.in_units('Mpc')
    center =h.pos# dds.arr([halo.quantities["particle_position_%s" % axis] \
    Rvir=h.R
    print("halo center (Mpc):")
    print(center)
    coordsDM=np.array(coordinatesDM)
    for i in range(0,3):
        print(coordsDM[:,i])
        print(center[i])
        coordsDM[:,i]-=center[i]
        print(coordsDM[:,i])
        #print(i)
    #print(coordinatesDM)
    for i in range(0,3):
        r2+=(coordsDM[:,i])**2.
        #c2+=(center[i])**2.
    r=np.sqrt(r2)
    print(r)
    #coords=coordinatesDM[np.abs(r-np.sqrt(c2)<Rvir)]
    coords=coordsDM[r<=Rvir]
    print("# of virialized particles:%d"%len(coords))
    print(coords)
    #coords[:,0]-=center[0]
    #coords[:,1]-=center[1]
    #coords[:,2]-=center[2]
    #print(coords[0,:])
    fig = plt.figure(figsize=plt.figaspect(1))
    ax = fig.add_subplot(111, projection='3d')
    ax.scatter(xp,yp,zp,c='black',alpha=0.1,marker='.',s=1)
    #plt.show()
    #ax.quiver(X, Y, Z, U, V, W)
    #ax.set_xlim([-1, 0.5])
    #ax.set_ylim([-1, 1.5])
    #ax.set_zlim([-1, 8])
    Pmass=ad[("all","particle_mass")].in_units('Msun')
    print("Individual particle mass: %g"%Pmass[0])
    #IDsDM = ad[("Halo","ParticleIDs")]
    #print(len(IDsDM))
    hid =h.id# halo.quantities['particle_identifier']
    #halos = HaloFinder(ds, ptype=Halo, dm_only=False, total_mass=None)
    #ind = halos[0]["particle_index"] # list of particles IDs in this halo
    #print(len(ind))
    # REMOVE
    #Rvir=10
	# Rem
    bins=4
    iteLim=2
    #convLim=5 nor need, we just compare two
    #Rbins=np.logspace(0,Rvir,bins)#(Rvir/bins,Rvir,bins)
    Rbins=np.linspace(0,Rvir,bins+1)
    print("Rbins:")
    print(Rbins)
    axis=np.array([Rbins[1],Rbins[1],Rbins[1]])
    orientation=np.identity(3)#zeros((3,3))#.array([0,0,0],[0,0,0],[0,0,0])
    a=[0]*len(Rbins)
    b=[0]*len(Rbins)
    c=[0]*len(Rbins)
    b_a=[0]*len(Rbins)
    c_a=[0]*len(Rbins)
    v0=1#ell.volume
    for i in range(0,len(Rbins)-1):
        iteration=0
        convergence=False
        conv=0
        while(not(convergence) and iteration<iteLim):
            s=np.array([[1,0,0,],[0,1,0,],[0,0,1]])
            print("halo %d -bin %d -iteration %d"% (hid,i+1, iteration+1))
            ell=ellipsoid(axis,orientation)
            s=ell.ShapeTesnsor(coords,Pmass[0],Rbins[i],Rbins[i+1])
            ell.PlotEllipsoid(ax,center)
            #plt.pause(0.05)
            if (iteration==0):
                v0=ell.volume()
            print("shape tensor:")
            print(np.array(s))
            axisNew,orientationNew=LA.eig(s)
            print("eigenvalues & eigenvectors:")
            print(axisNew)
            print(orientationNew)
            #axisNew=np.array([1,1,1]) #1 just for test
            #orientationNew=np.array([0,0,0],[0,0,0],[0,0,0])#orientationNew # 0 just for test
            #if() what if there was ne eignevalue?
            newell=ellipsoid(axisNew,orientationNew)
            if iteration>0:
                comp=CompareEllipsoids(ell,newell)
                if comp==True:
                    conv+=1
                    if conv>=1:
                        convergence=True
                else:
                    conv=0
                #if conv>=2:
                #    convergence=True#CompareEllipsoids(ell,newell)
                if(not convergence):
                    #iteration+=1  #  new_list = [x+1 for x in my_list]
                    vol=reduce(mul,axisNew)
                    norm=(v0/vol)**(1./3.)
                    axis=[ax*norm for ax in axisNew]
                    print("normalized axis:")
                    print(axis)
                    orientation=orientationNew
                if(iteration>=iteLim):
                    print("S didn't converge for halo %d"%hid)
                    convergence=True
            else:
                vol=reduce(mul,axisNew)
                print(vol)
                norm=(v0/vol)**(1./3.)
                axis=[ax*norm for ax in axisNew]
                orientation=orientationNew
                print("normalized axis:")
                print(axis)
                #if (i==0):
                #    v0=ell.volume()
            iteration+=1
        axis.sort()
        Rave=(Rbins[i]+Rbins[i+1])/2.
        FullShape.append(shape(h.id,h.R,h.M,Rave,axis))
        #a[i]=axis[0] # use max instead
        #b[i]=axis[1]
        #c[i]=axis[2] # use min instead
        #b_a[i]=b[i]/a[i]
        #c_a[i]=c[i]/a[i]
        #use the shape object instead
    #now to save these numbers somewhere
    # save halo id, center, bins, and shapes,
    print(FullShape[0].R)
    return FullShape

    #and let's plot these

#how to run: python ShapeAnalysis.py snapshot_file halo_catalog check_contamination extract_shape
#example: $python ShapeAnalysisTest.py snap_264 halos_0.0.bin 1 1

if __name__ == "__main__":
	p=4000
	UpperMass=1.0e13
	LowerMass=7.0e11

	center=np.zeros(3)
	center[0]=0
	center[1]=0
	center[2]=0
	n_particles = 20000
	h=halo(center,5.0e2,1.2e12,n_particles,0)
	ppx, ppy, ppz =1e2*np.random.normal(size=[3, n_particles])
	#np.shape(ppx))
	#ppx=range(0,10000,n_particles)
	#ppy=range(0,10000,n_particles)
	#ppz=range(0,10000,n_particles)
	ppm = np.ones(n_particles)
	data = {'particle_position_x': ppx,'particle_position_y': ppy,'particle_position_z': ppz,'particle_mass': ppm}
	bbox = 1.1*np.array([[min(ppx), max(ppx)], [min(ppy), max(ppy)], [min(ppz), max(ppz)]])
	#ds = yt.load_particles(data, length_unit=parsec, mass_unit=1e8*Msun, n_ref=256, bbox=bbox)
	ds = yt.load_particles(data, length_unit=Mpc, mass_unit=Msun, n_ref=256, bbox=bbox)
	#print(ds.field_list)
	print(ppx)

	fig2 = plt.figure(2)
	fig2.suptitle('Shapes')
	ax1 = fig2.add_subplot(221)
	ax2 = fig2.add_subplot(222)
	ax3 = fig2.add_subplot(223)
	ax4 = fig2.add_subplot(224)
	ax1.legend(loc=2)
	FinalHaloShape=[]
	print("Let's extract the shape!")
	HShape=GetShape(h,ds)
	ba=np.array(HShape[0].b_a)
	print(ba)
	pR=[]
	pb_a=[]
	pid=[]
	pc_a=[]
	pT=[]
	for s in HShape:
		#ax1.plot(s.R,s.b_a) or ax1.scatter(s.R,s.b_a,c='black', alpha=0.9, marker='.',s=15)
		#ax1.scatter( (s.R)*1000,s.b_a, alpha=0.9, marker='.',s=15)
		pR.append(s.R)
		pb_a.append(s.b_a)
		pc_a.append(s.c_a)
		pid.append(s.hid)
		pT.append(s.T)
	ax1.title.set_text('b/a')
	ax2.title.set_text('c/a')
	ax3.title.set_text('T')
	ax4.title.set_text('c/a - b/a')
	#ax2.plot(pR,pb_a,pR,pb_a,'o',label=str(pid))
	ax1.plot(pR,pb_a,label=str(pid[0]))
	ax2.plot(pR,pc_a,label=str(pid[0]))
	ax3.plot(pR,pT,label=str(pid[0]))
	ax4.plot(pc_a,pb_a,'o',label=str(pid[0]))
	ax1.legend(loc=3)
	ax2.legend(loc=3)
	ax3.legend(loc=3)
	ax4.legend(loc=4)
	fig3=plt.figure(4)
	rp=np.sqrt(ppx**2.+ppy**2.+ppz**2.)
	plt.hist(rp,bins=100)
	plt.show()


import pygame
from pygame.sprite import Sprite

import constants as c

class Mario(Sprite):
    def __init__(self, screen):
        super(Mario, self).__init__()
        self.screen = screen

        # Load Mario standing image
        self.image = pygame.image.load('images/mario/stand_right.png')
        self.walk_left = [pygame.image.load('images/mario/walk_left_1.png'),
                          pygame.image.load('images/mario/walk_left_2.png'),
                          pygame.image.load('images/mario/walk_left_3.png')]
        self.walk_right = [pygame.image.load('images/mario/walk_right_1.png'),
                           pygame.image.load('images/mario/walk_right_2.png'),
                           pygame.image.load('images/mario/walk_right_3.png')]
        self.jump_left = pygame.image.load('images/mario/jump_left.png')
        self.jump_right = pygame.image.load('images/mario/jump_right.png')
        self.rect = self.image.get_rect()
        self.screen_rect = self.screen.get_rect()

        # Position Mario
        self.rect.centerx = self.screen_rect.centerx
        self.rect.bottom = self.screen_rect.bottom - 25

        # Store decimal value of Mario's position
        self.center = float(self.rect.centerx)

        # Movement Flags
        self.moving_right = False
        self.moving_left = False
        self.jumping = False
        self.jump_timer = 0
        self.falling = False

        # Animation Tracker
        self.walk_count = 0
        self.jump_count = 0

    def update(self):
        """Update Mario's position"""
        if self.moving_right and self.rect.right < self.screen_rect.right and self.rect.centerx <= c.SCREEN_WIDTH/2:
            self.rect.centerx += 3
        if self.moving_left and self.rect.left > 0:
            self.rect.centerx -= 3
        if self.falling and self.rect.bottom < self.screen_rect.bottom - 25:
            self.jumping = False
            self.rect.bottom += max(1.0, -0.05 * ((self.jump_timer / 10) ** 2) + 5)
            self.jump_timer -= 1
            if self.rect.bottom >= self.screen_rect.bottom - 25:
                self.rect.bottom = self.screen_rect.bottom - 25
                self.falling = False
        if self.jumping and self.rect.top > 0:
            self.falling = False
            self.rect.bottom -= -0.05 * ((self.jump_timer / 20) ** 2) + 5
            self.jump_timer -= 1
            if self.jump_timer == 0:
                self.jumping = False
                self.reset_jump_timer()
                self.falling = True

    def reset_jump_timer(self):
        self.jump_timer = 25

    def blitme(self, screen):
        """Draw Mario to the screen"""
        if self.walk_count + 1 >= 45:
            self.walk_count = 0

        if self.moving_left:
            if self.jumping:
                screen.blit(self.jump_left, self.rect)
            else:
                screen.blit(self.walk_left[self.walk_count // 15], self.rect)
            self.walk_count += 1
        elif self.moving_right:
            if self.jumping:
                screen.blit(self.jump_right, self.rect)
            else:
                screen.blit(self.walk_right[self.walk_count // 15], self.rect)
            self.walk_count += 1
        else:
            self.screen.blit(self.image, self.rect)
        pass


from __future__ import division

import logbook
import numpy as np
from collections import namedtuple
from math import isnan

from six import iteritems, itervalues

from catalyst.finance.performance.position import Position
from catalyst.finance.transaction import Transaction
from catalyst.utils.input_validation import expect_types
import catalyst.protocol as zp
from catalyst.assets import (
    Future,
    Asset
)
from . position import positiondict

from catalyst.constants import LOG_LEVEL

log = logbook.Logger('Performance', level=LOG_LEVEL)

PositionStats = namedtuple('PositionStats',
                           ['net_exposure',
                            'gross_value',
                            'gross_exposure',
                            'short_value',
                            'short_exposure',
                            'shorts_count',
                            'long_value',
                            'long_exposure',
                            'longs_count',
                            'net_value'])

def calc_position_values(positions):
    values = []

    for position in positions:
        if isinstance(position.asset, Future):
            # Futures don't have an inherent position value.
            values.append(0.0)
        else:
            values.append(position.last_sale_price * position.amount)

    return values

def calc_net(values):
    # Returns 0.0 if there are no values.
    return sum(values, np.float64())

def calc_position_exposures(positions):
    exposures = []

    for position in positions:
        exposure = position.amount * position.last_sale_price

        if isinstance(position.asset, Future):
            exposure *= position.asset.multiplier

        exposures.append(exposure)

    return exposures

def calc_long_value(position_values):
    return sum(i for i in position_values if i > 0)

def calc_short_value(position_values):
    return sum(i for i in position_values if i < 0)

def calc_long_exposure(position_exposures):
    return sum(i for i in position_exposures if i > 0)

def calc_short_exposure(position_exposures):
    return sum(i for i in position_exposures if i < 0)

def calc_longs_count(position_exposures):
    return sum(1 for i in position_exposures if i > 0)

def calc_shorts_count(position_exposures):
    return sum(1 for i in position_exposures if i < 0)

def calc_gross_exposure(long_exposure, short_exposure):
    return long_exposure + abs(short_exposure)

def calc_gross_value(long_value, short_value):
    return long_value + abs(short_value)

class PositionTracker(object):

    def __init__(self, data_frequency):
        # asset => position object
        self.positions = positiondict()
        self._unpaid_dividends = {}
        self._unpaid_stock_dividends = {}
        self._positions_store = zp.Positions()

        self.data_frequency = data_frequency

    @expect_types(asset=Asset)
    def update_position(self, asset, amount=None, last_sale_price=None,
                        last_sale_date=None, cost_basis=None):
        if asset not in self.positions:
            position = Position(asset)
            self.positions[asset] = position
        else:
            position = self.positions[asset]

        if amount is not None:
            position.amount = amount
        if last_sale_price is not None:
            position.last_sale_price = last_sale_price
        if last_sale_date is not None:
            position.last_sale_date = last_sale_date
        if cost_basis is not None:
            position.cost_basis = cost_basis

    def execute_transaction(self, txn):
        # Update Position
        # ----------------
        asset = txn.asset

        if asset not in self.positions:
            position = Position(asset)
            self.positions[asset] = position
        else:
            position = self.positions[asset]

        position.update(txn)

        if position.amount == 0:
            del self.positions[asset]

            try:
                # if this position exists in our user-facing dictionary,
                # remove it as well.
                del self._positions_store[asset]
            except KeyError:
                pass

    @expect_types(asset=Asset)
    def handle_commission(self, asset, cost):
        # Adjust the cost basis of the stock if we own it
        if asset in self.positions:
            self.positions[asset].adjust_commission_cost_basis(asset, cost)

    def handle_splits(self, splits):
        """
        Processes a list of splits by modifying any positions as needed.

        Parameters
        ----------
        splits: list
            A list of splits.  Each split is a tuple of (asset, ratio).

        Returns
        -------
        int: The leftover cash from fractional sahres after modifying each
            position.
        """
        total_leftover_cash = 0

        for asset, ratio in splits:
            if asset in self.positions:
                # Make the position object handle the split. It returns the
                # leftover cash from a fractional share, if there is any.
                position = self.positions[asset]
                leftover_cash = position.handle_split(asset, ratio)
                total_leftover_cash += leftover_cash

        return total_leftover_cash

    def earn_dividends(self, dividends, stock_dividends):
        """
        Given a list of dividends whose ex_dates are all the next trading day,
        calculate and store the cash and/or stock payments to be paid on each
        dividend's pay date.

        Parameters
        ----------
        dividends: iterable of (asset, amount, pay_date) namedtuples

        stock_dividends: iterable of (asset, payment_asset, ratio, pay_date)
            namedtuples.
        """
        for dividend in dividends:
            # Store the earned dividends so that they can be paid on the
            # dividends' pay_dates.
            div_owed = self.positions[dividend.asset].earn_dividend(dividend)
            try:
                self._unpaid_dividends[dividend.pay_date].append(div_owed)
            except KeyError:
                self._unpaid_dividends[dividend.pay_date] = [div_owed]

        for stock_dividend in stock_dividends:
            div_owed = \
                self.positions[stock_dividend.asset].earn_stock_dividend(
                    stock_dividend)
            try:
                self._unpaid_stock_dividends[stock_dividend.pay_date].\
                    append(div_owed)
            except KeyError:
                self._unpaid_stock_dividends[stock_dividend.pay_date] = \
                    [div_owed]

    def pay_dividends(self, next_trading_day):
        """
        Returns a cash payment based on the dividends that should be paid out
        according to the accumulated bookkeeping of earned, unpaid, and stock
        dividends.
        """
        net_cash_payment = 0.0

        try:
            payments = self._unpaid_dividends[next_trading_day]
            # Mark these dividends as paid by dropping them from our unpaid
            del self._unpaid_dividends[next_trading_day]
        except KeyError:
            payments = []

        # representing the fact that we're required to reimburse the owner of
        # the stock for any dividends paid while borrowing.
        for payment in payments:
            net_cash_payment += payment['amount']

        # Add stock for any stock dividends paid.  Again, the values here may
        # be negative in the case of short positions.
        try:
            stock_payments = self._unpaid_stock_dividends[next_trading_day]
        except:
            stock_payments = []

        for stock_payment in stock_payments:
            payment_asset = stock_payment['payment_asset']
            share_count = stock_payment['share_count']
            # note we create a Position for stock dividend if we don't
            # already own the asset
            if payment_asset in self.positions:
                position = self.positions[payment_asset]
            else:
                position = self.positions[payment_asset] = \
                    Position(payment_asset)

            position.amount += share_count

        return net_cash_payment

    def maybe_create_close_position_transaction(self, asset, dt, data_portal):
        if not self.positions.get(asset):
            return None

        amount = self.positions.get(asset).amount
        price = data_portal.get_spot_value(
            asset, 'price', dt, self.data_frequency)

        # Get the last traded price if price is no longer available
        if isnan(price):
            price = self.positions.get(asset).last_sale_price

        txn = Transaction(
            asset=asset,
            amount=(-1 * amount),
            dt=dt,
            price=price,
            commission=0,
            order_id=None,
        )
        return txn

    def get_positions(self):

        positions = self._positions_store

        for asset, pos in iteritems(self.positions):

            if pos.amount == 0:
                # Clear out the position if it has become empty since the last
                # time get_positions was called.  Catching the KeyError is
                # faster than checking `if asset in positions`, and this can be
                # potentially called in a tight inner loop.
                try:
                    del positions[asset]
                except KeyError:
                    pass
                continue

            position = zp.Position(asset)
            position.amount = pos.amount
            position.cost_basis = pos.cost_basis
            position.last_sale_price = pos.last_sale_price
            position.last_sale_date = pos.last_sale_date

            # Adds the new position if we didn't have one before, or overwrite
            # one we have currently
            positions[asset] = position

        return positions

    def get_positions_list(self):
        positions = []
        for asset, pos in iteritems(self.positions):
            if pos.amount != 0:
                positions.append(pos.to_dict())
        return positions

    def sync_last_sale_prices(self, dt, handle_non_market_minutes,
                              data_portal):
        if not handle_non_market_minutes:
            for asset, position in iteritems(self.positions):
                last_sale_price = data_portal.get_spot_value(
                    asset, 'price', dt, self.data_frequency
                )

                if not np.isnan(last_sale_price):
                    position.last_sale_price = last_sale_price
        else:
            for asset, position in iteritems(self.positions):
                last_sale_price = data_portal.get_adjusted_value(
                    asset,
                    'price',
                    data_portal.trading_calendar.previous_minute(dt),
                    dt,
                    self.data_frequency
                )

                if not np.isnan(last_sale_price):
                    position.last_sale_price = last_sale_price

    def stats(self):
        amounts = []
        last_sale_prices = []
        for pos in itervalues(self.positions):
            amounts.append(pos.amount)
            last_sale_prices.append(pos.last_sale_price)

        position_values = calc_position_values(itervalues(self.positions))
        position_exposures = calc_position_exposures(
            itervalues(self.positions)
        )

        long_value = calc_long_value(position_values)
        short_value = calc_short_value(position_values)
        gross_value = calc_gross_value(long_value, short_value)
        long_exposure = calc_long_exposure(position_exposures)
        short_exposure = calc_short_exposure(position_exposures)
        gross_exposure = calc_gross_exposure(long_exposure, short_exposure)
        net_exposure = calc_net(position_exposures)
        longs_count = calc_longs_count(position_exposures)
        shorts_count = calc_shorts_count(position_exposures)
        net_value = calc_net(position_values)

        return PositionStats(
            long_value=long_value,
            gross_value=gross_value,
            short_value=short_value,
            long_exposure=long_exposure,
            short_exposure=short_exposure,
            gross_exposure=gross_exposure,
            net_exposure=net_exposure,
            longs_count=longs_count,
            shorts_count=shorts_count,
            net_value=net_value
        )


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('ggplot')
#%matplotlib inline
from sklearn.model_selection import train_test_split
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
import unicodedata
import string
from nltk.stem.snowball import SnowballStemmer
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.stem.porter import PorterStemmer
from nltk.util import ngrams
from nltk import pos_tag
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics.pairwise import linear_kernel
import operator
from wordcloud import WordCloud, STOPWORDS

#get data
def get_data():
    df_gd = pd.read_csv('data/gratefuldead.csv')
    df_pf = pd.read_csv('data/pinkfloyd.csv')
    df_phish = pd.read_csv('data/phish.csv')
    df_beatles = pd.read_csv('data/beatles.csv')
    #combine to single df
    df_sorted = pd.concat([df_gd, df_pf, df_phish, df_beatles], join  = 'outer', axis = 0, ignore_index = True)
    #shuffle rows so not sorted by band page
    df = df_sorted.sample(frac = 1, random_state=1)

    return df_gd, df_pf, df_phish, df_beatles, df

#get X and y
def get_X_y(df):
    df = df.set_index(df['id']).drop(columns=['id'])
    X = df['title'].to_numpy()
    y = df['subreddit'].to_numpy()
    return X, y

##clean text

#get unicode data
def remove_accents(input_str):
    nfkd_form = unicodedata.normalize('NFKD', input_str)
    only_ascii = nfkd_form.encode('ASCII', 'ignore')
    return only_ascii.decode()

#tokenize and get stem of each word. see which stemmer is preferred
# def tokenize(text):
#     stemmer = SnowballStemmer('english')
#     # stemmer = PorterStemmer()
#     # stemmer = WordNetLemmatizer()
#     return [stemmer.stem(word) for word in word_tokenize(text.lower())]

#get rid of accents, punctuation, make lowercase
# def filter_data(corpus):
#     X = corpus.copy()
#     punctuation = string.punctuation
#     stop_words = set(stopwords.words('english'))
#     stemmer = SnowballStemmer('english')
#     for i in range(len(X)):
#         #get unicode data
#         X[i] = remove_accents(X[i])
#         #get rid of punctuation and make lowercase
#         X[i] = X[i].translate(str.maketrans('', '', punctuation)).lower()
#         #make bag of words, without stop words
#         tokenized_words = [word for word in X[i].split() if word not in stop_words]
#         #get the stem of each word
#         X[i] = [stemmer.stem(word) for word in tokenized_words]
#     return X

def filter_data_text(text):
    X = text
    punctuation = string.punctuation
    stop_words = set(stopwords.words('english'))
    stemmer = SnowballStemmer('english')

    #get unicode data
    X = remove_accents(X)
    #get rid of punctuation and make lowercase
    X = X.translate(str.maketrans('', '', punctuation)).lower()
    #make bag of words, without stop words
    tokenized_words = [word for word in X.split() if word not in stop_words]
    #get the stem of each word
    X = [stemmer.stem(word) for word in tokenized_words]

    return X

def get_cosine_similarities(document_tfidf_matrix):
    cosine_similarities = linear_kernel(document_tfidf_matrix, document_tfidf_matrix)
    output = []
    for i in range(len(document_tfidf_matrix)):
        for j in range(len(document_tfidf_matrix)):
            output.append(i, j, cosine_similarities[i, j])
    return output

def get_top_n_words(corpus, n=100):
    """
    List the top n words in a vocabulary according to occurrence in a text corpus.

    get_top_n_words(["I love Python", "Python is a language programming", "Hello world", "I love the world"]) ->
    [('python', 2),
     ('world', 2),
     ('love', 2),
     ('hello', 1),
     ('is', 1),
     ('programming', 1),
     ('the', 1),
     ('language', 1)]
    """
    vec = CountVectorizer(tokenizer=filter_data_text).fit(corpus)
    bag_of_words = vec.transform(corpus)
    sum_words = bag_of_words.sum(axis=0)
    words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]
    words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
    return words_freq[:n]

def plot_top_words(list_of_tuples, n_documents, xlabel, ylabel, title, string_file_name):
    '''
    takes a list of tuples: (word, count), sorted by top used words

    '''
    word = list(zip(*list_of_tuples))[0][:15]
    count = list(zip(*list_of_tuples))[1][:15]
    freq = tuple(ti/n_documents for ti in count)

    x_pos = np.arange(len(word))

    fig, ax = plt.subplots(figsize=(12,8))
    ax.barh(x_pos, freq, align='center')
    plt.yticks(x_pos, word)
    ax.set_ylabel(xlabel)
    ax.set_xlabel(ylabel)
    ax.set_title(title)
    plt.gca().invert_yaxis()
    plt.savefig(f'images/{string_file_name}.png')
    # plt.show()

def plot_cloud(wordcloud):
    # Set figure size
    plt.figure(figsize=(40, 30))
    # Display image
    plt.imshow(wordcloud)
    # No axis details
    plt.axis("off")

# Generate word cloud from dictionary of 100 top words + their counts
def create_wordcloud(word_freq_dict, file_name_string):

    wordcloud = WordCloud(width = 3000, height = 2000,
                      random_state=1, background_color='black', colormap='Set2',
                      collocations=False, stopwords = STOPWORDS).generate_from_frequencies(word_freq_dict)
    # Plot and save file
    wordcloud.to_file(f'images/{file_name_string}.png')
    # plot_cloud(wordcloud)

if __name__ == "__main__":

    #get dataframes
    df_gd, df_pf, df_phish, df_beatles, df = get_data()

    #get X and y
    X, y = get_X_y(df)

    #get training and testing data
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

    #filter X_train, tokenize words for each document
    # X_train_filtered = filter_data(X_train)
    X_train_tokens = [filter_data_text(doc) for doc in X_train]
    X_test_tokens = [filter_data_text(doc) for doc in X_test]

    #count vectorizer: tf
    #use filter_data_text function as tokenizer
    #fit to X_train
    tf = CountVectorizer(tokenizer=filter_data_text, max_features=5000)
    document_tf_matrix = tf.fit_transform(X_train).todense()
    #feature names
    tf_feature_names = tf.get_feature_names()
    #vocabulary dictionary
    tf_vocabulary_dict = tf.vocabulary_
    #df
    tf_df = pd.DataFrame(document_tf_matrix, columns=tf.get_feature_names())

    #tfidf vectorizer: tf-idf
    #use filter_data_text function as tokenizer
    #fit to X_train
    tfidf = TfidfVectorizer(tokenizer=filter_data_text, max_features=5000)
    document_tfidf_matrix = tfidf.fit_transform(X_train).todense()
    #feature names
    tfidf_feature_names = tfidf.get_feature_names()
    #vocabulary dictionary
    tfidf_vocabulary_dict = tfidf.vocabulary_
    #df
    tf_df = pd.DataFrame(document_tf_matrix, columns=tf.get_feature_names())

    #counts of posts with band mentions
    num_phish_docs = sum(['phish' in doc for doc in X_train_tokens])
    num_gd_docs = sum(['dead' in doc for doc in X_train_tokens])
    num_pf_docs = sum(['floyd' in doc for doc in X_train_tokens])
    num_beatles_docs = sum(['beatl' in doc for doc in X_train_tokens])
    #get top words across all documents/subreddits and plot
    top_words = get_top_n_words(X_train, n=100)
    plot_top_words(top_words, len(X_train), 'Word', 'Word Frequency Per Post', 'Top 15 Words Across Subreddits', 'all_word_freq_graph')

    #get top words for each individual subreddit and plot
    top_gd_words = get_top_n_words(X_train[y_train=='gratefuldead'], n=100)
    top_phish_words = get_top_n_words(X_train[y_train=='phish'], n=100)
    top_pf_words = get_top_n_words(X_train[y_train=='pinkfloyd'], n=100)
    top_beatles_words = get_top_n_words(X_train[y_train=='beatles'], n=100)
    #plot top words for each individual subreddit
    plot_top_words(top_gd_words, len(X_train[y_train=='gratefuldead']), 'Word', 'Word Frequencey Per Post', 'Top 15 Words In Grateful Dead Subreddit', 'gd_word_freq_graph')
    plot_top_words(top_phish_words, len(X_train[y_train=='phish']), 'Word', 'Word Frequency Per Post', 'Top 15 Words In Phish Subreddit', 'phish_word_freq_graph')
    plot_top_words(top_pf_words, len(X_train[y_train=='pinkfloyd']), 'Word', 'Word Frequency Per Post', 'Top 15 Words In Pink Floyd Subreddit', 'pf_word_freq_graph')
    plot_top_words(top_beatles_words, len(X_train[y_train=='beatles']), 'Word', 'Word Frequency Per Post', 'Top 15 Words In Beatles Subreddit', 'beatles_word_freq_graph')

    #create wordclouds
    create_wordcloud(dict(top_words), 'all_wordcloud' )
    create_wordcloud(dict(top_gd_words), 'gd_wordcloud')
    create_wordcloud(dict(top_phish_words), 'phish_wordcloud')
    create_wordcloud(dict(top_pf_words), 'pf_wordcloud')
    create_wordcloud(dict(top_beatles_words), 'beatles_wordcloud')


from openerp.osv import osv, fields
import openerp.tools
from openerp.tools.translate import _
import base64
import xlwt
from cStringIO import StringIO
import datetime
from openerp.tools import DEFAULT_SERVER_DATE_FORMAT
import locale
import openerp.tools
from openerp.addons.sg_account_report.report.financial_report import account_balance_inherit

class bs_pl_xls_report(osv.osv_memory):
    _name = "bs.pl.xls.report"

    def _get_excel_export_data(self, cr, uid, context=None):
        if context is None:
            context = {}
        bank_obj = self.pool.get('hr.bank.details')
        workbook = xlwt.Workbook()
        worksheet = workbook.add_sheet('Sheet 1')
        font = xlwt.Font()
        font.bold = True
        header = xlwt.easyxf('font: bold 1, height 280')
        header2 = xlwt.easyxf('font: bold 1, height 180')
        header1 = xlwt.easyxf('pattern: pattern solid, fore_colour white; borders: top double, bottom double, bottom_color black; font: bold on, height 180, color black; align: wrap off')
        company_name = self.pool.get('res.users').browse(cr, uid, uid).company_id.name
        account_balance_inherit_obj = account_balance_inherit(cr, uid, '', context)
        acc_data = account_balance_inherit_obj.lines(context.get('form'), 0)
        style = xlwt.easyxf('align: wrap yes')
        worksheet.col(0).width = 15000
        worksheet.col(1).width = 5000
        worksheet.col(3).width = 4000
        worksheet.col(4).width = 5000
        worksheet.row(0).height = 500
        worksheet.row(1).height = 500
        worksheet.row(2).height = 500
        row = 6
        if context.get('inf_type') == 'BS':
            worksheet.write(0, 1, company_name , header)
            worksheet.write(1, 1, 'Balance Sheet' , header)
            worksheet.write(2, 1, 'As of ' + datetime.date.today().strftime('%m') + ' ' + context['form'].get('fiscalyear')[1] , header)
        if context.get('inf_type') == 'IS':
            worksheet.write(0, 1, company_name , header)
            worksheet.write(1, 1, 'Profit Loss' , header)
            worksheet.write(2, 1, context['form'].get('period_id')[1] , header)
            worksheet.write(3, 3, 'Current Month' , header2)
            worksheet.write(3, 4, 'Year to Date' , header2)
            worksheet.write(4, 3, 'Amount' , style)
            worksheet.write(4, 4, 'Amount' , style)
        for acc in acc_data:
            if context.get('inf_type') == 'BS':
                if acc['type']=='view' and acc['name'] or acc['name'].title():
                    if acc['total'] and not acc['label']:
                        worksheet.write(row, 0,'    '*(acc['level']-1) + acc['name'],header1)
                        worksheet.write(row, 1,'',header1)
                        worksheet.write(row, 2,'',header1)
                    else:
                        worksheet.write(row, 0,'    '*(acc['level']-1) + acc['name'],style)
                if acc['total']==True:
                    if acc['total'] and not acc['label']:
                        worksheet.write(row, 3, round(acc['balance'] or 0.00 , 2),header1)
                    else:
                        worksheet.write(row, 3, round(acc['balance'] or 0.00, 2),style)
                row += 1
            if context.get('inf_type') == 'IS':
                if acc['type']=='view' and acc['name'] or acc['name'].title():
                    if acc['total'] and not acc['label']:
                        worksheet.write(row, 0,'    '*(acc['level']-1) + acc['name'],header1)
                        worksheet.write(row, 1,'',header1)
                        worksheet.write(row, 2,'',header1)
                    else:
                        worksheet.write(row, 0,'    '*(acc['level']-1) + acc['name'],style)
                if acc['total']==True:
                    if acc['total'] and not acc['label']:
                        worksheet.write(row, 3, round(abs(acc['month']) or 0.00, 2),header1)
                        worksheet.write(row, 4, round(abs(acc['balance']) or 0.00, 2),header1)
                    else:
                        worksheet.write(row, 3, round(abs(acc['month']) or 0.00, 2),style)
                        worksheet.write(row, 4, round(abs(acc['balance']) or 0.00, 2),style)
                row += 1

        borders = xlwt.Borders()
        borders.top = xlwt.Borders.MEDIUM
        borders.bottom = xlwt.Borders.MEDIUM
        border_style = xlwt.XFStyle() # Create Style
        border_style.borders = borders
        row = 2
        payslip_obj = self.pool.get('hr.payslip')
        employee_obj = self.pool.get('hr.employee')
        hr_depart_obj = self.pool.get('hr.department')
        new_employee_ids = []
        style = xlwt.easyxf('align: wrap yes',style)
        fp = StringIO()
        workbook.save(fp)
        fp.seek(0)
        data = fp.read()
        fp.close()
        return base64.b64encode(data)

    _columns = {
        "file":fields.binary("Click On Download Link To Download Xls File", readonly=True),
        "name":fields.char("Name" , size=32, invisible="1")
    }

    _defaults = {
        'name':"BS PL.xls",
        'file': _get_excel_export_data
    }

bs_pl_xls_report()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:


from django.urls import path
from . import views

urlpatterns = [
    path('', views.login, name='login'),
    path('signup', views.signup, name='signup'),
    path('logout', views.signout, name='logout'),
    path('test', views.test, name='test'),
    path('home', views.home, name='home'),
    path('test_result', views.test_result, name='test_result'),

]


# coding:utf-8

import json
from sklearn.externals import joblib
import numpy as np
import numpy.linalg as la

POSITIVE = 1  # 正面词语
NEGATIVE = 2  # 反面词语
NEUTRAL = 3  # 客观词语

def load_key_words(file_path):
    with open(file_path, encoding="utf-8") as fp:
        lines = fp.readlines()
        lines = [line.replace("\n", "") for line in lines]
    return lines

class DateSet:
    def __init__(self, data, label):
        self.data = np.array(data)
        self.label = label

    def Data(self):
        return self.data

    def Label(self):
        return self.label

def load_date_sets(file_path):
    with open(file_path, encoding="utf-8") as f:
        data_list = json.load(f)
    temp = []
    for data in data_list:
        # 前面是特征向量，后面最后一个是标签
        label = data[-1]
        feature = data[:-1]
        d = DateSet(feature, label)
        temp.append(d)
    return temp

# 欧式距离,1表示100%，越接近0表示越不相似
def _ecl_sim(inA, inB):
    return 1.0 / (1.0 + la.norm(inA - inB))

# 皮尔逊相关系数,范围-1->+1， 越大越相似
def _pears_sim(inA, inB):
    if len(inA) < 3:
        return 1.0
    return 0.5 + 0.5 * np.corrcoef(inA, inB, rowvar=0)[0][1]

# 余弦相关范围-1->+1 越大越相似
def _cos_sim(inA, inB):
    num = float(inB * inA.T)
    de_nom = la.norm(inA) * la.norm(inB)
    return 0.5 + 0.5 * (num / de_nom)

def _get_feature(sentence, key_word):
    size = len(key_word)
    feature = [0 for _ in range(size)]
    for index in range(size):
        word = key_word[index]
        value = sentence.find(word)  # 单词最初出现的位置
        if value != -1:
            feature[index] = 1
    return np.array(feature)

def get_mood(sentence, key_word, model_name):
    feature = _get_feature(sentence, key_word)
    gnb = joblib.load(model_name)
    pre_y = gnb.predict([feature])
    result = {
        "positive": 0,
        "negative": 0,
        "neutral": 0
    }
    try:
        if pre_y[0] == POSITIVE:
            result["positive"] = 1
        elif pre_y[0] == NEGATIVE:
            result["negative"] = 1
        elif pre_y[0] == NEUTRAL:
            result["neutral"] = 1
    except:
        pass
    return result


from Simulation_functions import Simulations as Sims
import numpy as np
from multiprocessing import Process, Pool

#Define parameters
nc = 4
nr = 3
ntr = 450
lr = np.arange(0,1.1,0.1)
rep = [0,30]

#Dir = "/Volumes/backupdisc/Modular_learning/Data_Trees/Revision/"+str(res)+"/"
#simulate each model in parallel on separate cores
cp = 8

worker_pool = []

def job(core=0):
    done = False
    if core <4:
        res =12
    else:
        res =24
    mid = int(core%4)
    Dir = "/Volumes/backupdisc/Modular_learning/Data_Revision/Trees/"+ str(res)+"/"
    Sims.Simulation(nc, nr, ntr, res, lr, rep, mid, Dir,False,"Trees")
    done = True
    return done

with Pool(cp) as pool:
    result = pool.map(job, np.arange(cp))
    print(result)


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import KDEFunction
import time

# 计时
start = time.process_time()

# 读取GPS数据
data = pd.read_csv(r'../Data Process/RawGPSData.csv', header=None)
data.columns = ['VehicleNum', 'Stime', 'Lng', 'Lat']
# print(data.shape)
# print(data.head(5))

end = time.process_time()
print("Data Process 2 Step time:", end - start)

# 获取出租车数据经纬度最值
# maxLng = data['Lng'].max()
# maxLat = data['Lat'].max()
# minLng = data['Lng'].min()
# minLat = data['Lat'].min()

# bounds
# print(maxLng, minLng, maxLat, minLat)
# bounds = [117.9963607788086, 115.20406341552734, 40.87900924682617, 39.00091552734375]
# bounds = [115.20406341552734, 39.00091552734375, 117.9963607788086, 40.87900924682617]

# 北京市经纬度边界【最小经度，最小纬度，最大经度，最大纬度】
bounds = [115.7, 39.4, 117.4, 41.6]
MaxX = bounds[2]
MinX = bounds[0]
MaxY = bounds[3]
MinY = bounds[1]

# 坐标采样个数,即每个时点采集100*100个位置的密度信息(测试使用20)
Sample_Num = 20

# 计算采样间隔
Sample_Space_X = (MaxX - MinX) / Sample_Num
Sample_Space_Y = (MaxY - MinY) / Sample_Num

# 采样矩阵，Sample_Num*Sample_Num
Sample_X, Sample_Y = np.mgrid[MinX:MaxX:complex(0, Sample_Num), MinY:MaxY:complex(0, Sample_Num)]  # 步长实数为间隔，虚数为个数
# Sample_X, Sample_Y = np.mgrid[MinX:MaxX:Sample_Space_X, MinY:MaxY:Sample_Space_Y]  # 步长实数为间隔，虚数为个数
Sample_Metric = np.transpose(np.vstack((Sample_X.ravel(), Sample_Y.ravel())))
# print(Sample_Metric)
print(Sample_Metric.shape)

# 计算100秒时的热点数据
data_10 = data[data['Stime'] == 100]
# print(data_10)
Point_Num = len(data_10)
X = np.array(data_10[['Lng', 'Lat']])

# 计算最佳核密度估计最佳带宽
# Method1:
bandwidth1 = np.power(0.68 * Point_Num, -0.2) * np.sqrt((MaxX - MinX) * (MaxY - MinY))
# bandwidth1 = np.power(0.68*Point_Num, -0.2)*np.sqrt((MaxX-MinX)*(MaxY-MinY))/(np.sqrt(111*55))
# print("Best Bandwidth:", bandwidth1)
# Method2:计算中心位置，计算距离方差和中位数

# KDEFunction
kde, dense = KDEFunction.KDE(X, Sample_Metric, bandwidth1)

feature_data = np.c_[Sample_Metric, dense]

print(feature_data.shape)

# 画核密度等高线图
# 作图
Contour_X = Sample_X
Contour_Y = Sample_Y
Contour_Z = np.reshape(dense, Contour_X.shape)

fig = plt.figure()
axis = fig.gca()
axis.set_xlim(MinX, MaxX)
axis.set_ylim(MinY, MaxY)

contourf_set = axis.contourf(Contour_X, Contour_Y, Contour_Z, cmap='Reds')

contour_set = axis.contour(Contour_X, Contour_Y, Contour_Z, colors='k')
axis.clabel(contour_set, inline=1, fontsize=10)

axis.set_xlabel('Longitude')
axis.set_ylabel('Latitude')

plt_name = './KDE_Random'
plt.savefig(plt_name+'.jpg')
plt.show()


from setuptools import find_packages, setup
from pathlib import Path

this_directory = Path(__file__).parent
long_description = (this_directory / "README.md").read_text()

setup(
    name="lifeguard-mongodb",
    version="1.2.0",
    url="https://github.com/LifeguardSystem/lifeguard-mongodb",
    author="Diego Rubin",
    author_email="contact@diegorubin.dev",
    license="GPL2",
    scripts=[],
    include_package_data=True,
    description="Lifeguard integration with MongoDB",
    long_description=long_description,
    long_description_content_type="text/markdown",
    install_requires=["lifeguard", "pymongo"],
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Environment :: Plugins",
        "Intended Audience :: Developers",
        "Intended Audience :: Information Technology",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3",
        "Topic :: System :: Monitoring",
    ],
    packages=find_packages(),
)


import tkinter as tk
from datetime import datetime, time
from tkinter import *
from tkinter import filedialog
import os
from turtle import bgcolor, width
from typing import Dict
from pypylon import pylon
import json
import cv2

# Get the available cameras
tlFactory = pylon.TlFactory.GetInstance()
devices = tlFactory.EnumerateDevices()
if len(devices) == 0:
    raise pylon.RuntimeException("No camera present.")

## Create and attach all Pylon Devices.
cameras = pylon.InstantCameraArray(len(devices))
for i, cam in enumerate(cameras):
    cam.Attach(tlFactory.CreateDevice(devices[i]))

converter = pylon.ImageFormatConverter()
converter.OutputPixelFormat = pylon.PixelType_BGR8packed
converter.OutputBitAlignment = pylon.OutputBitAlignment_MsbAligned

currentCam = cameras[0]
currentCam.Open()
currentCam.StartGrabbing(pylon.GrabStrategy_LatestImageOnly)
cv2.namedWindow(str('Basler Capture ' + str(currentCam.GetDeviceInfo().GetSerialNumber())), cv2.WINDOW_NORMAL)

while currentCam.IsGrabbing():
    # Wait for an image and then retrieve it. A timeout of 5000 ms is used.
    grabResult = currentCam.RetrieveResult(5000, pylon.TimeoutHandling_ThrowException)

    # Image grabbed successfully?
    if grabResult.GrabSucceeded():
        image = converter.Convert(grabResult)
        img = image.GetArray()

        img = cv2.putText(img, datetime.now().strftime("%Y%m%d %H:%M:%S"), (
            15, img.shape[0]-20), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, lineType=cv2.LINE_AA)

        cv2.imshow(str('Basler Capture ' +
                       str(currentCam.GetDeviceInfo().GetSerialNumber())), img)
        k = cv2.waitKey(1)
        if k == 27:
            currentCam.StopGrabbing()
            currentCam.Close()
            cv2.destroyAllWindows()
            break

    else:
        print("Error: ", grabResult.ErrorCode, grabResult.ErrorDescription)

currentCam.Close()


from fastapi import FastAPI
from starlette.middleware.cors import CORSMiddleware

from app.api.api_v1.api import api_router
from app.core.config import settings

import uvicorn

# app = FastAPI()

# @app.get("/")
# def index():
#     return {"Hello": "Worlds"}

app = FastAPI(
    title=settings.PROJECT_NAME, openapi_url=f"{settings.API_V1_STR}/openapi.json"
)

if settings.BACKEND_CORS_ORIGINS:
    app.add_middleware(
        CORSMiddleware,
        allow_origins=["*"],
        allow_credentials=True,
        allow_methods=["*"],
        allow_headers=["*"],
    )

app.include_router(api_router, prefix=settings.API_V1_STR)

if __name__ == "__main__":
    uvicorn.run('app.main:app', host="0.0.0.0", port=8000, reload=True)
    # uvicorn.run(app, host="0.0.0.0", port=8000)


import yaml

device_list = ['leaf1-DC1', 'leaf2-DC1']

config_file = open('config.yml', 'r')

config = yaml.safe_load(config_file)

    # for device in device_list:
    #     print("Interface config for", device)
    #     print("")
    #     for interface in config['devices'][device]['interfaces']:
    #         print("interface", interface)
    #         print("   ip address", config['devices'][device]['interfaces'][interface])
    #     print("")

for pancake in config['devices']['leaf1-DC1']['interfaces']:
    print("interface", pancake)
    ip = config['devices']['leaf1-DC1']['interfaces'][pancake]
    print("ip address", ip)


import difflib as dfl
from functools import reduce
from pybdm_insee.tools.insee import _insee_data

idb_ref = "2341"

idb = _insee_data().get("idbank")
# dfl.get_close_matches(idbank, idb.idbank.unique().tolist(), n=n)

diffs = idb.idbank.map(
    lambda v: reduce(
        lambda acc, x: acc+1 if x[0] in ["+","-"] else acc,
        dfl.ndiff([str(v).zfill(9)], [idb_ref.zfill(9)]),
        0
    )
)

diffs = dfl.ndiff(
    idb.idbank.astype('str').values.tolist(),
    [idb_ref.zfill(9) for _ in range(len(idb.idbank))]
)
print(list(diffs))


import unittest
from pyvcloud.system_test_framework.base_test import BaseTestCase
from pyvcloud.vcd.vapp_firewall import VappFirewall
from pyvcloud.system_test_framework.vapp_constants import VAppConstants
from pyvcloud.system_test_framework.environment import Environment
from pyvcloud.system_test_framework.environment import developerModeAware
from pyvcloud.vcd.client import TaskStatus
from pyvcloud.vcd.client import FenceMode

class TestVappFirewall(BaseTestCase):
    """Test vapp dhcp functionalities implemented in pyvcloud."""
    _vapp_name = VAppConstants.name
    _network_name = VAppConstants.network1_name
    _enable = True
    _disable = False

    def test_0000_setup(self):
        TestVappFirewall._client = Environment.get_sys_admin_client()
        vapp = Environment.get_test_vapp_with_network(TestVappFirewall._client)
        vapp.reload()
        TestVappFirewall._network_name = \
            Environment.get_default_orgvdc_network_name()
        task = vapp.connect_org_vdc_network(
            TestVappFirewall._network_name,
            fence_mode=FenceMode.NAT_ROUTED.value)
        result = TestVappFirewall._client.get_task_monitor().wait_for_success(
            task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

    def test_0012_enable_firewall_service(self):
        vapp_firewall = VappFirewall(
            TestVappFirewall._client,
            vapp_name=TestVappFirewall._vapp_name,
            network_name=TestVappFirewall._network_name)
        # disable firewall service
        task = vapp_firewall.enable_firewall_service(TestVappFirewall._disable)
        TestVappFirewall._client.get_task_monitor().wait_for_success(task=task)
        vapp_firewall._reload()
        firewall_service = \
            vapp_firewall.resource.Configuration.Features.FirewallService
        self.assertFalse(firewall_service.IsEnabled)
        # enable firewall service
        task = vapp_firewall.enable_firewall_service(TestVappFirewall._enable)
        TestVappFirewall._client.get_task_monitor().wait_for_success(task=task)
        vapp_firewall._reload()
        firewall_service = \
            vapp_firewall.resource.Configuration.Features.FirewallService
        self.assertTrue(firewall_service.IsEnabled)

    @developerModeAware
    def test_9998_teardown(self):
        """Test the  method vdc.delete_vapp().

        Invoke the method for all the vApps created by setup.

        This test passes if all the tasks for deleting the vApps succeed.
        """
        vdc = Environment.get_test_vdc(TestVappFirewall._client)
        task = vdc.delete_vapp(name=TestVappFirewall._vapp_name, force=True)
        result = TestVappFirewall._client.get_task_monitor().wait_for_success(
            task)
        self.assertEqual(result.get('status'), TaskStatus.SUCCESS.value)

    def test_9999_cleanup(self):
        """Release all resources held by this object for testing purposes."""
        TestVappFirewall._client.logout()

if __name__ == '__main__':
    unittest.main()


'''
Ian L.
George M.

Task 1.)
this python file will contain the library for our HTTP client.
This library will support these features:
a.) GET operation - Retreives data from the server
b.) POST operation - Submits data to the server
c.) Query parameters
d.) Request headers
e.) Body of the request

Task 2.)
Implementation of a cURL command line that does basic functionalities that
are related to the HTTP

'''
import socket	#for sockets
import sys	#for exit
import time

import argparse
from typing import Text
from xmlrpc.client import Boolean
from urllib.parse import urlparse

def get_request(url, port, verbose=False, headers = None):
    client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    urlparser = urlparse(url)
    #host is 'httpbin.org', a "part" of the url variable/ full url that was passed in
    host = urlparser.hostname
    try:
        #host: 'www.httpbin.org'
        #url: http://httpbin.org/get?course=networking&assignment=1
        client.connect((host, port))

        request = "GET " + url + " HTTP/1.0\r\n" \
        "Host:%s\r\n" % host
        #check if we have passed in a list of headers from our -H arg
        if headers != None:
            #each key:value must end with \r\n (It's essentially the delimeter for lines
            # in a request, look at slides for chapter 2), but that is what is going on below
            for header in headers:
                request += header + "\r\n"

        #every request that is sent to the server must end with one last additional \r\n,
        #that is what is happening below
        request+= "\r\n"
        request = request.encode("utf-8")

        client.sendall(request)
        # MSG_WAITALL waits for full request or error
        #og is response = client.recv(1024)
        response = recv_timeout(client)
        response_details, response_data = response.split("\r\n\r\n")
        #show details with verbose if activated
        if(verbose):
            print(response_details, "\n")
        print(response_data)
    except socket.error:
        #Send failed
        print('Send failed')
        sys.exit()
    except ValueError:
        print(response)
    finally:
        client.close()

def post_request(url, port, verbose=False, headers = None, inline_data= None, file = None):
    client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    urlparser = urlparse(url)
    #host is 'httpbin.org', a "part" of the url variable/ full url that was passed in
    host = urlparser.hostname
    data = None
    try:
        #host: 'www.httpbin.org'
        #url: http://httpbin.org/get?course=networking&assignment=1
        client.connect((host, port))

        request = "POST " + url + " HTTP/1.0\r\n" \
            "Host:%s\r\n" % host

        #supply content-length
        #by default using post request you must supply a content length header
        request+="Content-Length: 17\r\n"
        #check if we have passed in a list of headers from our -H arg
        if headers != None:
            #each key:value must end with \r\n (It's essentially the delimeter for lines
            # in a request, look at slides for chapter 2), but that is what is going on below
            for header in headers:
                request += header + "\r\n"
        key, value = None, None
        # check the combinations of the -d and -f cases
        if (inline_data and not file):
            if(": " in inline_data):
                key, value = inline_data.split(": ")
            else:
                data = inline_data
        elif (not inline_data and file):
            file = open("sample.txt")
            line = file.read()
            try:
                key, value = line.split(": ")
            except ValueError:
                print("Not correct key-value pair format. Try again.")
        else:
            print ("Must have either one of the two valid arguments: -d or -f")
        #must also provided double quotes around the value field like in assignment pdf
        if(data != inline_data):
            data = '''{"%s": %s}",'''%(key,value)
        request += "\r\n"
        request+=data
        request+= "\r\n"
        #print(request)
        request = request.encode("utf-8")
        client.sendall(request)

        response = client.recv(1024)
        full_response = response.decode("utf-8")
        #split() returns a string list that is seperated by what you sent as an arg.
        response_details, response_data = full_response.split("\r\n\r\n")
        #show details with verbose if activated
        if(verbose):
            print(response_details, "\n")
        print(response_data)
    except ValueError:
        print(full_response)
    finally:
        client.close()

def recv_timeout(the_socket,timeout=2):
    #make socket non blocking
    the_socket.setblocking(0)

    #total data partwise in an array
    total_data=[]
    data=''

    #beginning time
    begin=time.time()
    while 1:
        #if you got some data, then break after timeout
        if total_data and time.time()-begin > timeout:
            break

        #if you got no data at all, wait a little longer, twice the timeout
        elif time.time()-begin > timeout*2:
            break

        #recv something
        try:
            data = the_socket.recv(8192)
            if data:
                response = data.decode("utf-8")
                total_data.append(response)
                #change the beginning time for measurement
                begin = time.time()
            else:
                #sleep for sometime to indicate a gap
                time.sleep(0.1)
        except:
            pass

    #join all parts to make final string
    return ''.join(total_data)

#method for help text
def print_help_for_post_or_get(argument):
    if(argument == 'get' or argument == 'GET'):
        print('httpc --HELP get'\
                '\nusage: httpc --get "URL" [-v] [-h key:value]\n' \
                'Get executes a HTTP GET request for a given URL.\n' \
                '-v\t\tPrints the detail of the response such as protocol, status, and headers.\n'\
                "-h key:value\tAssociates headers to HTTP Request with the format 'key:value'.")
    elif(argument =='post' or argument=='POST'):
        print('httpc --HELP post'\
                '\nusage: httpc --post "URL" [-v] [-h key:value] [-d inline-data] [-f file]\n' \
                'Post executes a HTTP POST request for a given URL with inline data or from file.\n' \
                '-v\t\tPrints the detail of the response such as protocol, status, and headers.\n'\
                '-h key:value\t\tAssociates headers to HTTP Request with the format\n'\
                '-d string\t\tAssociates headers to HTTP Request with the format\n'\
                '-f file\t\tAssociates the content of a file to the body HTTP POST request.\n'\
                "Either [-d] or [-f] can be used but not both.")

def main():
    #parser is container to hold our arguments
    #for better understanding of the argparser
    #visit: https://realpython.com/command-line-interfaces-python-argparse/
    parser = argparse.ArgumentParser(prog='httpc',
                                    description='httpc is a curl-like application but supports HTTP protocol only.',
                                    usage="\n\thttpclient.py command [arguments]", allow_abbrev=False,
                                    epilog='Use "httpclient.py help [command]" for more information about a command.')
    '''
    Syntactically, the difference between positional and optional arguments is
    that optional arguments start with - or --, while positional arguments do not.
    We will be using optional for the most part
    '''
    #optional arguments
    parser.add_argument("--host", help="Input the server host ip", default="localhost", action="store")
    parser.add_argument("--port", help="Input the server port number", type=int, default=80, action="store")
    parser.add_argument("--get", help="executes a HTTP GET request and prints the response. For URLs, you must surround it around double quotes.")
    parser.add_argument("--post", help="executes a HTTP POST request and prints the response.")

    #store_true sends a true value when sent once!
    parser.add_argument("--HELP",help="Display the help text for the --get and --post request as well as other options for these requests."
                         , type=str, choices=['GET', 'get', 'POST', 'post'], required=False)
    #more optional arguments that can go with the above, -H, _v for example
    #this one below is -H for the headers "key:value" requirement
    '''
    3. To pass the headers value to your HTTP operation, you could use -H option. The latter means
    setting the header of the request in the format "key: value." Notice that; you can have
    multiple headers by having the -H option before each header parameter.
    WE WILL USE -H TO NOT CONFLICT WITH -h the default help feature argparse uses!
    -h is the default argparse help option that cannot be changed.
    '''
    parser.add_argument('-H', required=False, help='Associates headers to HTTP Request with the format "key:value". You must pass the headers one by one starting with -H followed up by a space then the key:value which should be surrounded by double quotes.' ,action="append")
    parser.add_argument("-v", "--verbose", help="Turns on verbose mode for more details",
                      action="store_true",required=False)
    parser.add_argument('-d', required=False, help="Associates an inline data to the body HTTP POST request.")
    parser.add_argument('-f', required=False, help="Associate the body of the HTTP Request with the data from a given file.")
    #required argument in the params above "required=" can be used to force a user
    #to add an argument, could be useful
    '''
    After you execute .parse_args(), what you get is a Namespace object that contains a simple
    property for each input argument received from the command line.
    //////////////////////////////////////////////
    Example usages in cmd prompt:
    #header option examples: (THEY MUST BE SURROUNDED BY QUOTES)
        "Accept-Language: en us,en;q=0.5"
        "Accept-Encoding: gzip,deflate"
        "Content-Type:application/json"

    httpclient.py --get "http://httpbin.org/headers" -v -H "Accept-Language: en us,en;q=0.5" -H "Content-Type: application/json; charset=utf-8"
    httpclient.py --post "http://httpbin.org/post" -v -d "Assignment: 1" -H "Content-Type: application/json"
    httpclient.py --post "http://httpbin.org/post" -v -f sample.txt -H "Content-Type: application/json"
    httpclient.py --post "http://httpbin.org/post" -v
    httpclient.py --HELP get

    local server communication:

    GET /
    httpclient.py --port 1234 --get "http://localhost/"
    httpclient.py --port 1234 --get "http://localhost/Directory3/"
    httpclient.py --port 1234 --get "http://localhost/Directory2/"

    GET /filename.type
    httpclient.py --port 1234 --get "http://localhost/bar.txt"
    httpclient.py --port 1234 --get "http://localhost/Directory3/bar.txt"
    httpclient.py --port 1234 --get "http://localhost/Directory3/foo_2.txt"
    httpclient.py --port 1234 --get "http://localhost/foo3.txt"
    httpclient.py --port 1234 --get "http://localhost/empty.txt"
    httpclient.py --port 1234 --get "http://localhost/bar"

    httpclient.py --port 1234 --post "http://localhost/bar" -d "Test text"
    httpclient.py --port 1234 --post "http://localhost/Directory3/bar.html" -d "<h>Hello World!</h>"
    if we want to use 'key: value' pairing:
        httpclient.py --port 1234 --post "http://localhost/bar.txt" -d "Assignment: 1"
    //////////////////////////////////////////////
    '''

    args = parser.parse_args()
    if(args.get != None):
        get_request(args.get, args.port, args.verbose, args.H)
    elif(args.post != None):
        post_request(args.post, args.port, args.verbose, args.H, args.d, args.f)
    elif(args.HELP != None):
        print_help_for_post_or_get(args.HELP)

if(__name__=="__main__"):
    main()


from decimal import Decimal
from django.test import TestCase
from .tasks import CalculationState, calculate_summary
from fecfiler.reports.models import Report

class F3XSerializerTestCase(TestCase):
    fixtures = [
        "test_committee_accounts",
        "test_contacts",
        "test_f3x_reports",
        "test_schedulea_summary_transactions",
    ]

    def test_summary_task(self):
        report_id = calculate_summary("not_an_existing_report")
        self.assertIsNone(report_id)

        report_id = calculate_summary("b6d60d2d-d926-4e89-ad4b-c47d152a66ae")
        report = Report.objects.get(id=report_id)
        self.assertEqual(
            report.form_3x.L15_offsets_to_operating_expenditures_refunds_period,
            Decimal("2125.79"),
        )
        self.assertEqual(
            report.form_3x.L37_offsets_to_operating_expenditures_period,
            Decimal("2125.79"),
        )
        self.assertEqual(report.calculation_status, CalculationState.SUCCEEDED.value)

    def test_report_with_no_transactions(self):
        report_id = calculate_summary("a07c8c65-1b2d-4e6e-bcaa-fa8d39e50965")
        report = Report.objects.get(id=report_id)
        self.assertEqual(
            report.form_3x.L15_offsets_to_operating_expenditures_refunds_period,
            Decimal("0"),
        )
        self.assertEqual(
            report.form_3x.L37_offsets_to_operating_expenditures_period, Decimal("0")
        )
        self.assertEqual(report.calculation_status, CalculationState.SUCCEEDED.value)


import pytest

from aws.ec2.helpers import ec2_instance_test_id, ec2_instance_missing_tag_names
from aws.ec2.resources import ec2_instances

@pytest.fixture
def required_tag_names(pytestconfig):
    return frozenset(pytestconfig.custom_config.aws.required_tags)

@pytest.mark.ec2
@pytest.mark.parametrize("ec2_instance", ec2_instances(), ids=ec2_instance_test_id)
def test_ec2_instance_has_required_tags(ec2_instance, required_tag_names):
    """
    Checks that all EC2 instances have the tags with the required names.

    Does not check tag values.
    """
    if len(required_tag_names) == 0:
        pytest.skip("No required tag names were provided")
    missing_tag_names = ec2_instance_missing_tag_names(ec2_instance, required_tag_names)
    assert (
        not missing_tag_names
    ), "EC2 Instance {0[InstanceId]} missing required tags {1!r}".format(
        ec2_instance, missing_tag_names
    )