import altair as alt
import pandas as pd
import streamlit as st
import aws
import driver
import requests
import json
import os
from textblob import TextBlob
import datetime
import time

credentials_path = "credentials.json"

auth_dict = {
    "CONSUMER_KEY": "",
    "CONSUMER_SECRET": "",
    "ACCESS_TOKEN": "",
    "ACCESS_TOKEN_SECRET": "",
    "ENVIRONMENT": ""
}

def get_individual_stock_data(ticker, start="", end="", interval="1d", period="ytd"):
    """get individual stock data"""
    ip_address = get_current_ip()
    if ip_address == None:
        st.warning("IP not valid! Please Create an AWS instance or add valid IP")
        return

    response = requests.get(
        f"http://{ip_address}/stocks/stock?ticker={ticker}&start={start}&end={end}&interval={interval}&period={period}"
    )
    data_dict = response.json()
    # print(data_dict)
    return data_dict

# Streamlit command: pipenv run streamlit run <path_to_file>.py

def preprocess_stock_data(stock_dict):
    stock_df = pd.DataFrame(stock_dict)
    # rename index to date
    stock_df.index.rename("Date", inplace=True)
    stock_df.reset_index(inplace=True)
    stock_df = stock_df.rename(columns={"index": "Date"})
    # convert string time to datetime object
    stock_df["Date"] = pd.to_datetime(
        stock_df["Date"], format="%Y-%m-%dT%H:%M:%S")
    return stock_df

def barplot(df, y, x, title=""):
    """barplot with input of a data frame"""
    bar_plot = (
        alt.Chart(df)
        .mark_bar()
        .encode(
            alt.Y(y),
            alt.X(x),
            opacity=alt.value(0.7),
            color=alt.value("blue"),
        )
        .properties(title=title, width=700, height=300)
    ).interactive()
    return bar_plot

def lineplot(df, y, x):
    """lineplot with input of a data frame"""
    lineplot = (
        alt.Chart(df)
        .mark_line()
        .encode(
            alt.Y(y),
            alt.X(x),
            opacity=alt.value(0.7),
            color=alt.value("blue"),
        )
        .properties(width=700, height=300)
    ).interactive()
    return lineplot

def combined_area_plot(df, y, x):
    brush = alt.selection(type="interval", encodings=["x"])
    base = (
        alt.Chart(df)
        .mark_area()
        .encode(
            x=x,
            y=y,
            opacity=alt.value(0.7),
            color=alt.value("blue"),
        )
        .properties(width=700, height=300)
    ).interactive()
    upper = base.encode(alt.X(x, scale=alt.Scale(domain=brush)))
    lower = base.properties(height=100).add_selection(brush)
    combined = alt.vconcat(upper, lower)
    return combined

def combined_bar_line_plot(df, y, x):
    brush = alt.selection(type="interval", encodings=["x"])
    base = barplot(df, y, x)
    upper = base.encode(alt.X(x, scale=alt.Scale(domain=brush)))
    lower = base.mark_line().properties(height=120).add_selection(brush)
    combined = alt.vconcat(upper, lower)
    return combined

def append_credentials(new_dict):
    """Uses the path to the credentials to add or update dictionary keys"""
    with open(credentials_path, "r") as file:
        old_dict = json.load(file)
    old_dict.update(new_dict)
    with open(credentials_path, "w") as file:
        json.dump(old_dict, file, indent=4)

def get_twitter_credentials():
    """Get and store twitter credentials"""
    st.write("## Please enter your Twitter developer application credentials:")

    consumer_key = st.text_input("Please enter your consumer key",
                                 value=auth_dict["CONSUMER_KEY"])
    consumer_secret = st.text_input("Please enter your consumer secret key",
                                    value=auth_dict["CONSUMER_SECRET"])
    access_token = st.text_input("Please enter your access token",
                                 value=auth_dict["ACCESS_TOKEN"])
    secret_access_token = st.text_input(
        "Please enter your secret access token",
        value=auth_dict["ACCESS_TOKEN_SECRET"])
    environment = st.text_input("Please enter your environment name",
                                value=auth_dict["ENVIRONMENT"])

    if st.button("submit", key="Credentials"):
        twitter_credentials = {"CONSUMER_KEY": consumer_key,
                               "CONSUMER_SECRET": consumer_secret,
                               "ACCESS_TOKEN": access_token,
                               "ACCESS_TOKEN_SECRET": secret_access_token,
                               "ENVIRONMENT": environment}
        st.success("Twitter Credentials Updated!")
        append_credentials({"TWITTER": twitter_credentials})

def get_aws_credentials():
    """Get and store AWS user credentials and create an
    EC2 instance"""
    st.write("## Please enter your AWS credentials:")
    access_key = st.text_input("Please enter your access key")
    secret_key = st.text_input("Please enter your secret key")
    session_token = st.text_input("Please enter your session token")

    if st.button("submit", key="Credentials"):
        aws_credentials = {"access_key": access_key,
                           "secret_key": secret_key, "session_token": session_token}
        new_ip = aws.begin_creation(access_key, secret_key, session_token)
        st.success(f"Instance created. Instance IP {new_ip}")
        append_credentials({"AWS": aws_credentials})
        append_credentials({"ip_address": new_ip})

        upload_and_execute(new_ip)

    st.write("## OR add an accessible IP address:")
    input_ip_address = st.text_input("Please enter your instance's IP address")
    if st.button("submit", key="IP") and (not len(input_ip_address) == 0):
        append_credentials({"ip_address": input_ip_address})
        st.success("New IP Added!")
        # upload_and_execute(input_ip_address)

def upload_and_execute(ip_address):
    """Connect to the AWS instance, upload files, and run the scripts"""
    # NOTE: this assumes that the file is being run from repo root
    stdout, stderr = driver.upload(
        "ec2-auto", ip_address, upload_dir="./code/upload")
    st.warning(stdout)
    st.warning(stderr)
    driver.ssh_execute("ec2-auto", ip_address)

def get_current_ip():
    """Open the credentials file and return IP address"""
    with open(credentials_path, "r") as file:
        credentials_dict = json.load(file)
        if "ip_address" in credentials_dict.keys():
            return credentials_dict["ip_address"]
        else:
            return None

def get_twitter_dict():
    """Open the credentials file and return Twitter dictionary"""
    with open(credentials_path, "r") as file:
        credentials_dict = json.load(file)
        if "TWITTER" in credentials_dict.keys():
            return credentials_dict["TWITTER"]
        else:
            return None

def main():
    st.title("Stock Market and Twitter Sentiment Analysis")
    # Retrieve credentials from local Json
    # create empty json file if file does not exist
    if os.path.exists(credentials_path):
        with open(credentials_path, "r") as file:
            credentials_dict = json.load(file)
    else:
        credentials_dict = {}
        json.dump(credentials_dict, open(credentials_path, "w"))

    # ============= Sidebar handling================
    selection = st.sidebar.selectbox(
        "Please select an operation",
        options=["AWS Authentication", "Twitter Authentication"],
    )
    # Add a placeholder for textboxes and buttons
    placeholder = st.sidebar.empty()
    if selection == "AWS Authentication":
        with placeholder.beta_container():
            get_aws_credentials()

    if selection == "Twitter Authentication":
        with placeholder.beta_container():
            get_twitter_credentials()
    # ==============================================

    # ============== Stock information input ===========
    today = datetime.date.today()
    yesterday = today - datetime.timedelta(days=1)
    stock_ticker = st.text_input("Please enter a stock ticker")
    start_date = st.date_input(
        "Enter Start Date", value=yesterday, max_value=yesterday)
    end_date = st.date_input(
        "Enter End Date", min_value=start_date + datetime.timedelta(days=1), max_value=today)
    if (today - start_date).days < 60:
        interval = st.selectbox(
            'Select a data interval', ("1m", "2m", "5m", "15m", "30m", "60m", "90m", "1h", "1d", "5d", "1wk", "1mo", "3mo"))
    else:
        interval = st.selectbox(
            'Select a data interval', ("1d", "5d", "1wk", "1mo", "3mo"))

    if st.button("get info") and not(stock_ticker == ""):
        ticker_info = get_company_info(stock_ticker)

        stock_df = preprocess_stock_data(
            (get_individual_stock_data(stock_ticker, start=start_date.strftime('%Y-%m-%d'), end=end_date.strftime('%Y-%m-%d'), interval=interval)))

        col1, col2 = st.beta_columns(2)
        company_logo_url = ticker_info["logo_url"]
        company_name = ticker_info["shortName"]
        company_website = ticker_info["website"]
        with col1:
            st.write(f"### About [{company_name}]({company_website})")
        with col2:
            st.markdown(f"![Company Logo]({company_logo_url})")

        st.write(ticker_info["longBusinessSummary"])
        st.write("### Stock Charts:")
        st.write(stock_df)
        # st.altair_chart(barplot(stock_df, "Close", "Date"))
        st.altair_chart(lineplot(stock_df, "Close", "Date"))
        st.altair_chart(combined_area_plot(stock_df, "Close", "Date"))
        # st.altair_chart(combined_bar_line_plot(stock_df, "Close", "Date"))

        st.write("### Twitter Sentiment Charts:")
        # get weekely data if range is less than 7 days
        if (today - start_date).days <= 7:
            sentiment_df = get_weekly_tweet_sentiment(stock_ticker)
        else:
            st.warning("Searching twitter data from over a week ago requires Twitter Premium credentials and could result in an error if maximum monthly requests are reached")
            sentiment_df = get_range_sentiment(
                stock_ticker, start_date, end_date)

        st.altair_chart(lineplot(sentiment_df, "Sentiment", "Date"))
        st.write(sentiment_df)

def get_company_info(ticker):
    """get individual stock general information"""
    ip_address = get_current_ip()
    if ip_address == None:
        st.warning("IP not valid! Please Create an AWS instance or add valid IP")
        return

    response = requests.get(
        f"http://{ip_address}/stocks/info?ticker={ticker}"
    )
    data_dict = response.json()
    return data_dict

def get_range_sentiment(ticker, start_date, end_date):
    """Get weekly tweets from API and compute sentiment"""
    ip_address = get_current_ip()
    if ip_address == None:
        st.warning("IP not valid! Please Create an AWS instance or add valid IP")
        return
    auth_dict = get_twitter_dict()
    if auth_dict == None:
        st.warning("Twitter credentials not found! Please add your credentials")
        return
    num_per_day = 100

    tweets = {}
    for day_since_start in range((end_date - start_date).days):
        start_string = (start_date +
                        datetime.timedelta(days=day_since_start)).strftime("%Y%m%d%H%M")
        end_string = (start_date +
                      datetime.timedelta(days=day_since_start + 1)).strftime("%Y%m%d%H%M")
        request_string = f"http://{ip_address}/tweets?keyword={ticker}&start={start_string}&end={end_string}&num={num_per_day}"
        response = requests.post(
            request_string,
            data=json.dumps(auth_dict),
        )
        response_dict = response.json()
        tweets.update(response_dict)
        time.sleep(5)

    st.write(tweets)
    polarity_dict = {}
    # get average sentiment score of the date
    for k, v in tweets.items():
        score = 0
        for text in v:
            score += compute_sentiment_score(text)
        polarity_dict[k] = score / len(v)
    senti_dict = {"Date": polarity_dict.keys(
    ), "Sentiment": polarity_dict.values()}
    sentiment_df = pd.DataFrame.from_dict(senti_dict)

    # st.write(polarity_dict)

    return sentiment_df

def compute_sentiment_score(text: str):
    """return the polarity score of a sentence"""
    polarity = TextBlob(text).sentiment.polarity
    return polarity

def get_weekly_tweet_sentiment(stock_symbol: str):
    """Get weekly tweets from API and compute sentiment"""
    ip_address = get_current_ip()
    if ip_address == None:
        st.warning("IP not valid! Please Create an AWS instance or add valid IP")
        return
    auth_dict = get_twitter_dict()
    if auth_dict == None:
        st.warning("Twitter credentials not found! Please add your credentials")
        return
    response = requests.post(
        f"http://{ip_address}/weeklyTweets?keyword={stock_symbol}",
        data=json.dumps(auth_dict),
    )
    tweets = response.json()
    polarity_dict = {}
    # get average sentiment score of the date
    for k, v in tweets.items():
        score = 0
        for text in v:
            score += compute_sentiment_score(text)
        polarity_dict[k] = score / len(v)
    senti_dict = {"Date": polarity_dict.keys(
    ), "Sentiment": polarity_dict.values()}
    sentiment_df = pd.DataFrame.from_dict(senti_dict)

    return sentiment_df

if __name__ == "__main__":
    main()


import matplotlib.pyplot as plt
import cv2
import numpy as np
from os.path import join, basename
from collections import deque
from utils import grayscale, canny, gaussian_blur, hough_lines, get_slope, get_bias, draw_lines, weighted_img

# region of interest
def region_of_interest(image):

    height = image.shape[0]
    width = image.shape[1]
    vertices = np.array( [[
                [3*width/4, 3*height/5],
                [width/4, 3*height/5],
                [40, height],
                [width - 40, height]
            ]], dtype=np.int32 )

    #defining a blank mask to start with
    mask = np.zeros_like(image)

    # #defining a 3 channel or 1 channel color to fill the mask with depending on the input image
    if len(image.shape) > 2:
        channel_count = image.shape[2]  # i.e. 3 or 4 depending on your image
        ignore_mask_color = (255,) * channel_count # (255, 255, 255)
    else:
        ignore_mask_color = 255

    #filling pixels inside the polygon defined by "vertices" with the fill color
    cv2.fillPoly(mask, vertices, ignore_mask_color)

    #returning the image only where mask pixels are nonzero
    masked_image = cv2.bitwise_and(image, mask)
    return masked_image

# compute candidate lines
# and return left and right line coordinates
def compute_candidates(canditates_lines, img_shape):
    global left_lane, right_line
    # separate candidate lines according to their slope
    pos_lines = [l for l in canditates_lines if l["slope"] > 0]
    neg_lines = [l for l in canditates_lines if l["slope"] < 0]

    # interpolate biases and slopes to compute equation of line that approximates left lane
    # median is employed to filter outliers
    neg_bias = np.median([l["bias"] for l in neg_lines]).astype(int)
    neg_slope = np.median([l["slope"] for l in neg_lines])
    x1, y1 = 0, neg_bias
    x2, y2 = -np.int32(np.round(neg_bias / neg_slope)), 0
    left_lane = np.array([x1, y1, x2, y2])

    # interpolate biases and slopes to compute equation of line that approximates right lane
    # median is employed to filter outliers
    pos_bias = np.median([l["bias"] for l in pos_lines]).astype(int)
    pos_slope = np.median([l["slope"] for l in pos_lines])
    x1, y1 = 0, pos_bias
    x2, y2 = np.int32(np.round((img_shape[0] - pos_bias) / pos_slope)), img_shape[0]
    right_lane = np.array([x1, y1, x2, y2])

    return [left_lane, right_lane]

def get_lines(img, rho, theta, threshold, min_line_len, max_line_gap):
    # convert to grayscale
    img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # perform gaussian blur
    blur = gaussian_blur(img_gray, kernel_size=17)

    # perform edge detection
    canny_edges = canny(blur, low_threshold=50, high_threshold=70)

    detected_lines = hough_lines(img=canny_edges,
                                 rho=rho,
                                 theta=theta,
                                 threshold=threshold,
                                 min_line_len=min_line_len,
                                 max_line_gap=max_line_gap)

    candidate_lines = []
    for line in detected_lines:
        for x1, y1, x2, y2 in line:
            slope = get_slope(x1, y1, x2, y2)
            if 0.5 <= np.abs(slope) <= 2:
                candidate_lines.append({"slope": slope,
                                        "bias": get_bias(slope, x1, y1)})

    lane_lines = compute_candidates(candidate_lines, img_gray.shape)

    return lane_lines

def last_lines_averages(lane_lines):
    avg_lt = np.zeros((len(lane_lines), 4))
    avg_lr = np.zeros((len(lane_lines), 4))

    for i in range(0, len(lane_lines)):
        # left line
        x1, y1, x2, y2 = lane_lines[i][0]
        avg_lt[i] += np.array([x1, y1, x2, y2])
        # right line
        x1, y1, x2, y2 = lane_lines[i][1]
        avg_lr[i] += np.array([x1, y1, x2, y2])

    return [np.mean(avg_lt, axis=0), np.mean(avg_lr, axis=0)]

def progress(frames):
    detected_lines = []

    # last 10 frames
    for i in range(0, len(frames)):
        # detect lines
        left_right_lines = get_lines(img=frames[i],
                                    rho=2,
                                    theta=np.pi/180,
                                    threshold=1,
                                    min_line_len=15,
                                    max_line_gap=5)

        detected_lines.append(left_right_lines)

    # prepare empty mask on which lines are drawn
    line_img = np.zeros((frames[0].shape[0], frames[0].shape[1], 3), dtype=np.uint8)

    # last 10 frames line averages
    detected_lines = last_lines_averages(detected_lines)

    # draw lines
    for lane in detected_lines:
        draw_lines(line_img, lane)

    # region of interest
    masked_img = region_of_interest(line_img)

    # img_color = frames[-1] if is_videoclip else frames[0]
    result = weighted_img(masked_img, frames[-1])
    return result


from pycocotools.coco import COCO
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
import cv2 as cv

# =======================================
# =======================================
obj_class = 255
dataDir='E:/00_graduation project/DataSet/COCO/' # 数据集根目录
dataType='val2017' # 选择图像类型
annFile='{}2017_Train_Val_annotations/instances_{}.json'.format(dataDir,dataType) # annotation路径

# 蒙版绘制函数
def showAnns(mask, anns, c):
    if len(anns) == 0:  # 判断注释类型
        return 0
    if 'segmentation' in anns[0] or 'keypoints' in anns[0]:
        datasetType = 'instances'  # 蒙版
    else:
        raise Exception('datasetType not supported')  # 错误

    if datasetType == 'instances':
        for ann in anns:
            # c = (np.random.random((1, 3)) * 0.6 + 0.4).tolist()[0] # 随机生成标记颜色
            if 'segmentation' in ann:
                if type(ann['segmentation']) == list:  # polygon多边形
                    for seg in ann['segmentation']:
                        poly = np.array(seg, np.int32).reshape((int(len(seg) / 2), 2))  # 将标记信息转换为坐标数组
                        poly = poly.reshape((-1, 1, 2)) # 修整顶点坐标数组格式
                        cv.fillPoly(mask, [poly], color=c) # 填充形状
        # plt.imshow(mask)
        # plt.show()
        return mask

# 将COCO格式转换为VOC格式
# 图片形状[x,y,3]， COCO标注信息， voc蒙版颜色
def coco2voc(imgShape, anns, color):
    mask = np.zeros(imgShape, np.uint8)  # 绘制纯黑底色

    mask = showAnns(mask, anns, color)  # 绘制mask
    # mask = cv.cvtColor(mask, cv.COLOR_BGR2RGB)

    # 获取图片ID
    imgId = str(anns[0]['image_id']).zfill(12)
    imgId_ = imgId + '.png'
    src = dataDir + 'voc/'+ dataType + '/' + imgId_ # 组成路径
    cv.imwrite(src, mask) # 保存图片
    txt.write(imgId + '\n')

# initialize COCO api for instance annotations
coco=COCO(annFile) # 初始化coco类
''' # 输出所有分类名
# display COCO categories and supercategories
cats = coco.loadCats(coco.getCatIds()) # 获取类别列表,列表中元素是存储着类名、id和亚类的字典
nms=[cat['name'] for cat in cats] # 遍历类别列表，取出所有类的name
print('COCO categories: \n{}\n'.format(' '.join(nms))) # 打印类名

nms = set([cat['supercategory'] for cat in cats]) # set无序不重复序列 取出所有亚类
print('COCO supercategories: \n{}'.format(' '.join(nms))) # 打印类名
'''
# get all images containing given categories
catIds = coco.getCatIds(catNms=['person']) # 获取指定类别的id
imgIds = coco.getImgIds(catIds=catIds) # 获取包含指定类别id的图像id

# ==========================================
# 遍历满足条件的图像
# ==========================================
txt = open(dataDir + 'voc/'+ dataType + '.txt', "w") # 创建txt记录转换的图片id
print("transforming...")
count = 0
for i in tqdm(imgIds):
    img = coco.loadImgs(i)[0] # 获取第一个图像的json
    # load and display image
    I = cv.imread('%s/%s/%s' % (dataDir, dataType, img['file_name']))  # 从本地加载图像
    # I = cv.cvtColor(I, cv.COLOR_BGR2RGB) # 转换颜色格式
    # use url to load image
    # I = cv.imread(img['coco_url']) # 通过网络从远程加载图

    # ==============================================
    # load and display instance annotations
    # ==============================================
    imgShape = I.shape # 得到图片尺寸
    annIds = coco.getAnnIds(imgIds=img['id'], catIds=catIds, iscrowd=None) # 得到指定图像对应类型的annotation id
    anns = coco.loadAnns(annIds) # 根据annotation id加载蒙版json

    color = [obj_class, obj_class, obj_class] # 蒙版颜色
    coco2voc(imgShape, anns, color)
    count = count + 1
txt.close()
print(f"transform completely. total:{count}")


import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

import dgl
from dgl.base import DGLError
from dgl.ops import edge_softmax
import dgl.function as fn

class Identity(nn.Module):
    """A placeholder identity operator that is argument-insensitive.
    (Identity has already been supported by PyTorch 1.2, we will directly
    import torch.nn.Identity in the future)
    """

    def __init__(self):
        super(Identity, self).__init__()

    def forward(self, x):
        """Return input"""
        return x

class MsgLinkPredictor(nn.Module):
    """Predict Pair wise link from pos subg and neg subg
    use message passing.

    Use Two layer MLP on edge to predict the link probability

    Parameters
    ----------
    embed_dim : int
        dimension of each each feature's embedding

    Example
    ----------
    >>> linkpred = MsgLinkPredictor(10)
    >>> pos_g = dgl.graph(([0,1,2,3,4],[1,2,3,4,0]))
    >>> neg_g = dgl.graph(([0,1,2,3,4],[2,1,4,3,0]))
    >>> x = torch.ones(5,10)
    >>> linkpred(x,pos_g,neg_g)
    (tensor([[0.0902],
         [0.0902],
         [0.0902],
         [0.0902],
         [0.0902]], grad_fn=<AddmmBackward>),
    tensor([[0.0902],
         [0.0902],
         [0.0902],
         [0.0902],
         [0.0902]], grad_fn=<AddmmBackward>))
    """

    def __init__(self, emb_dim):
        super(MsgLinkPredictor, self).__init__()
        self.src_fc = nn.Linear(emb_dim, emb_dim)
        self.dst_fc = nn.Linear(emb_dim, emb_dim)
        self.out_fc = nn.Linear(emb_dim, 1)

    def link_pred(self, edges):
        src_hid = self.src_fc(edges.src['embedding'])
        dst_hid = self.dst_fc(edges.dst['embedding'])
        score = F.relu(src_hid+dst_hid)
        score = self.out_fc(score)
        return {'score': score}

    def forward(self, x, pos_g, neg_g):
        # Local Scope?
        pos_g.ndata['embedding'] = x
        neg_g.ndata['embedding'] = x

        pos_g.apply_edges(self.link_pred)
        neg_g.apply_edges(self.link_pred)

        pos_escore = pos_g.edata['score']
        neg_escore = neg_g.edata['score']
        return pos_escore, neg_escore

class TimeEncode(nn.Module):
    """Use finite fourier series with different phase and frequency to encode
    time different between two event

    ..math::
        \Phi(t) = [\cos(\omega_0t+\psi_0),\cos(\omega_1t+\psi_1),...,\cos(\omega_nt+\psi_n)]

    Parameter
    ----------
    dimension : int
        Length of the fourier series. The longer it is ,
        the more timescale information it can capture

    Example
    ----------
    >>> tecd = TimeEncode(10)
    >>> t = torch.tensor([[1]])
    >>> tecd(t)
    tensor([[[0.5403, 0.9950, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,
          1.0000, 1.0000]]], dtype=torch.float64, grad_fn=<CosBackward>)
    """

    def __init__(self, dimension):
        super(TimeEncode, self).__init__()

        self.dimension = dimension
        self.w = torch.nn.Linear(1, dimension)
        self.w.weight = torch.nn.Parameter((torch.from_numpy(1 / 10 ** np.linspace(0, 9, dimension)))
                                           .double().reshape(dimension, -1))
        self.w.bias = torch.nn.Parameter(torch.zeros(dimension).double())

    def forward(self, t):
        t = t.unsqueeze(dim=2)
        output = torch.cos(self.w(t))
        return output

class MemoryModule(nn.Module):
    """Memory module as well as update interface

    The memory module stores both historical representation in last_update_t

    Parameters
    ----------
    n_node : int
        number of node of the entire graph

    hidden_dim : int
        dimension of memory of each node

    Example
    ----------
    Please refers to examples/pytorch/tgn/tgn.py;
                     examples/pytorch/tgn/train.py

    """

    def __init__(self, n_node, hidden_dim):
        super(MemoryModule, self).__init__()
        self.n_node = n_node
        self.hidden_dim = hidden_dim
        self.reset_memory()

    def reset_memory(self):
        self.last_update_t = nn.Parameter(torch.zeros(
            self.n_node).float(), requires_grad=False)
        self.memory = nn.Parameter(torch.zeros(
            (self.n_node, self.hidden_dim)).float(), requires_grad=False)

    def backup_memory(self):
        """
        Return a deep copy of memory state and last_update_t
        For test new node, since new node need to use memory upto validation set
        After validation, memory need to be backed up before run test set without new node
        so finally, we can use backup memory to update the new node test set
        """
        return self.memory.clone(), self.last_update_t.clone()

    def restore_memory(self, memory_backup):
        """Restore the memory from validation set

        Parameters
        ----------
        memory_backup : (memory,last_update_t)
            restore memory based on input tuple
        """
        self.memory = memory_backup[0].clone()
        self.last_update_t = memory_backup[1].clone()

    # Which is used for attach to subgraph
    def get_memory(self, node_idxs):
        return self.memory[node_idxs, :]

    # When the memory need to be updated
    def set_memory(self, node_idxs, values):
        self.memory[node_idxs, :] = values

    def set_last_update_t(self, node_idxs, values):
        self.last_update_t[node_idxs] = values

    # For safety check
    def get_last_update(self, node_idxs):
        return self.last_update_t[node_idxs]

    def detach_memory(self):
        """
        Disconnect the memory from computation graph to prevent gradient be propagated multiple
        times
        """
        self.memory.detach_()

class MemoryOperation(nn.Module):
    """ Memory update using message passing manner, update memory based on positive
    pair graph of each batch with recurrent module GRU or RNN

    Message function
    ..math::
        m_i(t) = concat(memory_i(t^-),TimeEncode(t),v_i(t))

    v_i is node feature at current time stamp

    Aggregation function
    ..math::
        \bar{m}_i(t) = last(m_i(t_1),...,m_i(t_b))

    Update function
    ..math::
        memory_i(t) = GRU(\bar{m}_i(t),memory_i(t-1))

    Parameters
    ----------

    updater_type : str
        indicator string to specify updater

        'rnn' : use Vanilla RNN as updater

        'gru' : use GRU as updater

    memory : MemoryModule
        memory content for update

    e_feat_dim : int
        dimension of edge feature

    temporal_dim : int
        length of fourier series for time encoding

    Example
    ----------
    Please refers to examples/pytorch/tgn/tgn.py
    """

    def __init__(self, updater_type, memory, e_feat_dim, temporal_encoder):
        super(MemoryOperation, self).__init__()
        updater_dict = {'gru': nn.GRUCell, 'rnn': nn.RNNCell}
        self.memory = memory
        memory_dim = self.memory.hidden_dim
        self.temporal_encoder = temporal_encoder
        self.message_dim = memory_dim+memory_dim + \
            e_feat_dim+self.temporal_encoder.dimension
        self.updater = updater_dict[updater_type](input_size=self.message_dim,
                                                  hidden_size=memory_dim)
        self.memory = memory

    # Here assume g is a subgraph from each iteration
    def stick_feat_to_graph(self, g):
        # How can I ensure order of the node ID
        g.ndata['timestamp'] = self.memory.last_update_t[g.ndata[dgl.NID]]
        g.ndata['memory'] = self.memory.memory[g.ndata[dgl.NID]]

    def msg_fn_cat(self, edges):
        src_delta_time = edges.data['timestamp'] - edges.src['timestamp']
        time_encode = self.temporal_encoder(src_delta_time.unsqueeze(
            dim=1)).view(len(edges.data['timestamp']), -1)
        ret = torch.cat([edges.src['memory'], edges.dst['memory'],
                         edges.data['feats'], time_encode], dim=1)
        return {'message': ret, 'timestamp': edges.data['timestamp']}

    def agg_last(self, nodes):
        timestamp, latest_idx = torch.max(nodes.mailbox['timestamp'], dim=1)
        ret = nodes.mailbox['message'].gather(1, latest_idx.repeat(
            self.message_dim).view(-1, 1, self.message_dim)).view(-1, self.message_dim)
        return {'message_bar': ret.reshape(-1, self.message_dim), 'timestamp': timestamp}

    def update_memory(self, nodes):
        # It should pass the feature through RNN
        ret = self.updater(
            nodes.data['message_bar'].float(), nodes.data['memory'].float())
        return {'memory': ret}

    def forward(self, g):
        self.stick_feat_to_graph(g)
        g.update_all(self.msg_fn_cat, self.agg_last, self.update_memory)
        return g

class EdgeGATConv(nn.Module):
    '''Edge Graph attention compute the graph attention from node and edge feature then aggregate both node and
    edge feature.

    Parameter
    ==========
    node_feats : int
        number of node features

    edge_feats : int
        number of edge features

    out_feats : int
        number of output features

    num_heads : int
        number of heads in multihead attention

    feat_drop : float, optional
        drop out rate on the feature

    attn_drop : float, optional
        drop out rate on the attention weight

    negative_slope : float, optional
        LeakyReLU angle of negative slope.

    residual : bool, optional
        whether use residual connection

    allow_zero_in_degree : bool, optional
        If there are 0-in-degree nodes in the graph, output for those nodes will be invalid
        since no message will be passed to those nodes. This is harmful for some applications
        causing silent performance regression. This module will raise a DGLError if it detects
        0-in-degree nodes in input graph. By setting ``True``, it will suppress the check
        and let the users handle it by themselves. Defaults: ``False``.

    '''

    def __init__(self,
                 node_feats,
                 edge_feats,
                 out_feats,
                 num_heads,
                 feat_drop=0.,
                 attn_drop=0.,
                 negative_slope=0.2,
                 residual=False,
                 activation=None,
                 allow_zero_in_degree=False):
        super(EdgeGATConv, self).__init__()
        self._num_heads = num_heads
        self._node_feats = node_feats
        self._edge_feats = edge_feats
        self._out_feats = out_feats
        self._allow_zero_in_degree = allow_zero_in_degree
        self.fc_node = nn.Linear(
            self._node_feats, self._out_feats*self._num_heads)
        self.fc_edge = nn.Linear(
            self._edge_feats, self._out_feats*self._num_heads)
        self.attn_l = nn.Parameter(torch.FloatTensor(
            size=(1, self._num_heads, self._out_feats)))
        self.attn_r = nn.Parameter(torch.FloatTensor(
            size=(1, self._num_heads, self._out_feats)))
        self.attn_e = nn.Parameter(torch.FloatTensor(
            size=(1, self._num_heads, self._out_feats)))
        self.feat_drop = nn.Dropout(feat_drop)
        self.attn_drop = nn.Dropout(attn_drop)
        self.leaky_relu = nn.LeakyReLU(negative_slope)
        self.residual = residual
        if residual:
            if self._node_feats != self._out_feats:
                self.res_fc = nn.Linear(
                    self._node_feats, self._out_feats*self._num_heads, bias=False)
            else:
                self.res_fc = Identity()
        self.reset_parameters()
        self.activation = activation

    def reset_parameters(self):
        gain = nn.init.calculate_gain('relu')
        nn.init.xavier_normal_(self.fc_node.weight, gain=gain)
        nn.init.xavier_normal_(self.fc_edge.weight, gain=gain)
        nn.init.xavier_normal_(self.attn_l, gain=gain)
        nn.init.xavier_normal_(self.attn_r, gain=gain)
        nn.init.xavier_normal_(self.attn_e, gain=gain)
        if self.residual and isinstance(self.res_fc, nn.Linear):
            nn.init.xavier_normal_(self.res_fc.weight, gain=gain)

    def msg_fn(self, edges):
        ret = edges.data['a'].view(-1, self._num_heads,
                                   1)*edges.data['el_prime']
        return {'m': ret}

    def forward(self, graph, nfeat, efeat, get_attention=False):
        with graph.local_scope():
            if not self._allow_zero_in_degree:
                if (graph.in_degrees() == 0).any():
                    raise DGLError('There are 0-in-degree nodes in the graph, '
                                   'output for those nodes will be invalid. '
                                   'This is harmful for some applications, '
                                   'causing silent performance regression. '
                                   'Adding self-loop on the input graph by '
                                   'calling `g = dgl.add_self_loop(g)` will resolve '
                                   'the issue. Setting ``allow_zero_in_degree`` '
                                   'to be `True` when constructing this module will '
                                   'suppress the check and let the code run.')

            nfeat = self.feat_drop(nfeat)
            efeat = self.feat_drop(efeat)

            node_feat = self.fc_node(
                nfeat).view(-1, self._num_heads, self._out_feats)
            edge_feat = self.fc_edge(
                efeat).view(-1, self._num_heads, self._out_feats)

            el = (node_feat*self.attn_l).sum(dim=-1).unsqueeze(-1)
            er = (node_feat*self.attn_r).sum(dim=-1).unsqueeze(-1)
            ee = (edge_feat*self.attn_e).sum(dim=-1).unsqueeze(-1)
            graph.ndata['ft'] = node_feat
            graph.ndata['el'] = el
            graph.ndata['er'] = er
            graph.edata['ee'] = ee
            graph.apply_edges(fn.u_add_e('el', 'ee', 'el_prime'))
            graph.apply_edges(fn.e_add_v('el_prime', 'er', 'e'))
            e = self.leaky_relu(graph.edata['e'])
            graph.edata['a'] = self.attn_drop(edge_softmax(graph, e))
            graph.edata['efeat'] = edge_feat
            graph.update_all(self.msg_fn, fn.sum('m', 'ft'))
            rst = graph.ndata['ft']
            if self.residual:
                resval = self.res_fc(nfeat).view(
                    nfeat.shape[0], -1, self._out_feats)
                rst = rst + resval

            if self.activation:
                rst = self.activation(rst)

            if get_attention:
                return rst, graph.edata['a']
            else:
                return rst

class TemporalEdgePreprocess(nn.Module):
    '''Preprocess layer, which finish time encoding and concatenate
    the time encoding to edge feature.

    Parameter
    ==========
    edge_feats : int
        number of orginal edge feature

    temporal_encoder : torch.nn.Module
        time encoder model
    '''

    def __init__(self, edge_feats, temporal_encoder):
        super(TemporalEdgePreprocess, self).__init__()
        self.edge_feats = edge_feats
        self.temporal_encoder = temporal_encoder

    def edge_fn(self, edges):
        t0 = torch.zeros_like(edges.dst['timestamp'])
        time_diff = edges.data['timestamp'] - edges.src['timestamp']
        time_encode = self.temporal_encoder(
            time_diff.unsqueeze(dim=1)).view(t0.shape[0], -1)
        edge_feat = torch.cat([edges.data['feats'], time_encode], dim=1)
        return {'efeat': edge_feat}

    def forward(self, graph):
        graph.apply_edges(self.edge_fn)
        efeat = graph.edata['efeat']
        return efeat

class TemporalTransformerConv(nn.Module):
    def __init__(self,
                 edge_feats,
                 memory_feats,
                 temporal_encoder,
                 out_feats,
                 num_heads,
                 allow_zero_in_degree=False,
                 layers=1):
        '''Temporal Transformer model for TGN and TGAT

        Parameter
        ==========
        edge_feats : int
            number of edge features

        memory_feats : int
            dimension of memory vector

        temporal_encoder : torch.nn.Module
            compute fourier time encoding

        out_feats : int
            number of out features

        num_heads : int
            number of attention head

        allow_zero_in_degree : bool, optional
            If there are 0-in-degree nodes in the graph, output for those nodes will be invalid
            since no message will be passed to those nodes. This is harmful for some applications
            causing silent performance regression. This module will raise a DGLError if it detects
            0-in-degree nodes in input graph. By setting ``True``, it will suppress the check
            and let the users handle it by themselves. Defaults: ``False``.
        '''
        super(TemporalTransformerConv, self).__init__()
        self._edge_feats = edge_feats
        self._memory_feats = memory_feats
        self.temporal_encoder = temporal_encoder
        self._out_feats = out_feats
        self._allow_zero_in_degree = allow_zero_in_degree
        self._num_heads = num_heads
        self.layers = layers

        self.preprocessor = TemporalEdgePreprocess(
            self._edge_feats, self.temporal_encoder)
        self.layer_list = nn.ModuleList()
        self.layer_list.append(EdgeGATConv(node_feats=self._memory_feats,
                                           edge_feats=self._edge_feats+self.temporal_encoder.dimension,
                                           out_feats=self._out_feats,
                                           num_heads=self._num_heads,
                                           feat_drop=0.6,
                                           attn_drop=0.6,
                                           residual=True,
                                           allow_zero_in_degree=allow_zero_in_degree))
        for i in range(self.layers-1):
            self.layer_list.append(EdgeGATConv(node_feats=self._out_feats*self._num_heads,
                                               edge_feats=self._edge_feats+self.temporal_encoder.dimension,
                                               out_feats=self._out_feats,
                                               num_heads=self._num_heads,
                                               feat_drop=0.6,
                                               attn_drop=0.6,
                                               residual=True,
                                               allow_zero_in_degree=allow_zero_in_degree))

    def forward(self, graph, memory, ts):
        graph = graph.local_var()
        graph.ndata['timestamp'] = ts
        efeat = self.preprocessor(graph).float()
        rst = memory
        for i in range(self.layers-1):
            rst = self.layer_list[i](graph, rst, efeat).flatten(1)
        rst = self.layer_list[-1](graph, rst, efeat).mean(1)
        return rst


import agent.potil_lstm as agent
from expert_dataloader import ExpertLoader, ExpertLoader_each_step
import agent.utils as utils
import sys
sys.path.append('/catkin_workspace/src/ros_kortex/kortex_examples/src/move_it')
import torch
torch.backends.cudnn.benchmark = True
from mpl_toolkits import mplot3d
import numpy as np
import matplotlib.pyplot as plt
import queue

potil_agent = agent.POTILAgent(obs_shape=[4, 1], action_shape=[4, 1], device='cuda', lr=1e-3, feature_dim=512,
                               hidden_dim=128, critic_target_tau=0.01, num_expl_steps=0,
                               update_every_steps=2, stddev_schedule='linear(0.2,0.04,500000)', stddev_clip=0.1, use_tb=True, augment=True,
                               rewards='sinkhorn_cosine', sinkhorn_rew_scale=200, update_target_every=10000,
                               auto_rew_scale=True, auto_rew_scale_factor=10, obs_type='env', bc_weight_type='linear',
                               bc_weight_schedule='linear(0.15,0.03,20000)')

nstep = 32

# expert_loader = ExpertLoader(nstep, 50)
# expert_iter = iter(expert_loader)
# test_data = expert_loader.read_data(0,1)
# state, action = test_data
# seq_queue = queue.Queue(nstep)
# test_data_storage = []
# for i in range(nstep):
#     seq_queue.put(state[0])
# for i in range(state.shape[0]):
#     seq_queue.get()
#     seq_queue.put(state[i])
#     test_data_storage.append((np.stack(seq_queue.queue), action[i]))

# with step
expert_loader = ExpertLoader_each_step(nstep, 50)
expert_iter = iter(expert_loader)
test_data = expert_loader.read_data(0,1)
state, action = test_data
seq_queue = queue.Queue(nstep)
test_data_storage = []
for i in range(nstep):
    seq_queue.put(state[0])
for i in range(state.shape[0]):
    seq_queue.get()
    seq_queue.put(state[i])
    test_data_storage.append((np.stack(seq_queue.queue), action[i]))

cnt = 1
# def test():
#     obs = utils.to_torch(test_data, 'cuda')[0]
#     replay = []
#     for i in range(80):
#         with torch.no_grad():
#             action = potil_agent.act(obs, 0, True)
#         action[0] = (action[0] + 1.3) * 0.2 + 0.2
#         action[1] = (action[1] + 1) * 1.1 * -0.1
#         action[2] = (action[2] + 0.75) * 0.125 + 0.05
#         action[3] = (action[3] + 1) * 0.2
#         obs[:4] = torch.from_numpy(action).cuda()
#         replay.append(obs)

#     fig = plt.figure()
#     #创建3d绘图区域
#     ax = plt.axes(projection='3d')
#     #从三个维度构建
#     z = test_data[0][:,2]
#     x = test_data[0][:,0]
#     y = test_data[0][:,1]
#     #调用 ax.plot3D创建三维线图
#     ax.scatter3D(x, y, z,'gray', s=1)
#     replay = np.stack(replay)
#     z = replay[:,2]
#     x = replay[:,0]
#     y = replay[:,1]
#     #调用 ax.plot3D创建三维线图
#     ax.scatter3D(x, y, z,'gray', s=1)
#     plt.savefig(f'./figs/{cnt}.png')
#     cnt += 1

def scale(action_):
    action = action_.copy()
    action[0] = (action[0] + 1.31) * 0.2 + 0.2
    action[1] = (action[1] + 0.84) * 1.1 * -0.11
    action[2] = (action[2] + 0.83) * 0.15 + 0.05
    action[3] = (action[3] + 1) * 0.25
    return action

def test():
    loss = 0
    for data in test_data_storage:
        data = utils.to_torch(data, 'cuda')
        with torch.no_grad():
            action = potil_agent.act(data[0][:, :4], 0, True)
        loss += np.linalg.norm(scale(action) - scale(data[1].cpu().numpy()), ord=1)
    loss /= state.shape[0]
    print(loss)

# for i in range(100000):
#     batch = next(expert_iter)
#     obs_bc, action_bc = utils.to_torch(batch, 'cuda')
#     metrics = potil_agent.train_bc_norm(obs_bc, action_bc)
#     # if i % 10 == 1:
#         # print(metrics['bc_loss'])
#     if i % 20 == 0:
#         test()
#     if i % 100 == 0:
#         torch.save(potil_agent.actor, 'BC_actor_norm.pkl')

# for i in range(100000):
#     batch = next(expert_iter)
#     obs_bc, action_bc = utils.to_torch(batch, 'cuda')
#     metrics = potil_agent.train_bc_dist(obs_bc, action_bc)
#     # if i % 10 == 1:
#         # print(metrics['bc_loss'])
#     if i % 100 == 0:
#         test()
#     if i % 100 == 0:
#         torch.save(potil_agent.actor, 'BC_actor_dist.pkl')

# potil_agent.actor = torch.load('/catkin_workspace/src/ros_kortex/kortex_examples/src/move_it/BC_actor_dist.pkl').cuda()
potil_agent.actor.train()
potil_agent.actor_opt = torch.optim.Adam(potil_agent.actor.parameters(), lr=potil_agent.lr)
for i in range(100000):
    batch = next(expert_iter)
    obs_bc, action_bc = utils.to_torch(batch, 'cuda')
    metrics = potil_agent.train_bc_norm_step(obs_bc[:,:,:4], action_bc)
    # if i % 10 == 1:
        # print(metrics['bc_loss'])
    if i % 100 == 0:
        test()
    if i % 100 == 0:
        torch.save(potil_agent.actor, 'BC_actor_dist.pkl')


import matplotlib.pyplot as plt

x_list = []
y_list = []

def process(line):

    ### transformation matix 로 궤적 얻기 ##############################################################
    pose = line.strip().split()

    x_list.append(float(pose[3]))  # position
    y_list.append(float(pose[11]))

    ##################################################################################################

file_path = '/media/doyu_now/Linux Workspace/ICSL_Project/Visual SLAM/KITTI_data_odometry_color/data_odometry_poses/dataset/poses/10.txt'
with open(file_path, 'r') as f:
    while True:

        line = f.readline()
        if not line:
            print("End Of File")
            break

        process(line)

plt.plot(x_list, y_list, 'ob')

plt.title('kitti odom - scenario 10')
plt.show()


# Python standard libraries
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import json

import sqlite3

# Third-party libraries
from flask import Flask, redirect, request, url_for, render_template
from flask_login import (
    LoginManager,
    current_user,
    login_required,
    login_user,
    logout_user,
    UserMixin,
    )
from oauthlib.oauth2 import WebApplicationClient
import requests

# Internal imports
from etl.gcp_utils import init_connection_engine, access_secret
from data_viz import journal_calendar

# Configuration
PROJECT_ID = os.getenv("PROJECT_ID")

try:
    GOOGLE_CLIENT_ID = access_secret(project_id=PROJECT_ID,secret_id="CLIENT_ID")
    GOOGLE_CLIENT_SECRET =  access_secret(project_id=PROJECT_ID,secret_id="CLIENT_SECRET")
    EMAIL = access_secret(project_id=PROJECT_ID,secret_id="EMAIL")

except: # google.api_core.exceptions.PermissionDenied
    GOOGLE_CLIENT_ID = os.getenv("CLIENT_ID")
    GOOGLE_CLIENT_SECRET = os.getenv("CLIENT_SECRET")
    LOCAL_DB = os.getenv("LOCAL_DB")
    EMAIL = os.getenv("EMAIL")

finally:
    GOOGLE_DISCOVERY_URL = 'https://accounts.google.com/.well-known/openid-configuration'
    DB_URL = os.getenv('DB_URL')

# TODO: Resolve testing with this initial database declaration.
db = None
db= sqlite3

app = Flask(__name__)
app.secret_key = os.getenv("SECRET_KEY") or os.urandom(24)

# User session management setup
# https://flask-login.readthedocs.io/en/latest
login_manager = LoginManager()
login_manager.init_app(app)

# OAuth 2 client setup
client = WebApplicationClient(GOOGLE_CLIENT_ID)

# User class for flask-login
class User(UserMixin):
    def __init__(self, email):
        self.id = email

    @staticmethod
    def get(user_id):
        user = User(email=user_id)
        return user

# Flask-Login helper to retrieve a user from our db
@login_manager.user_loader
def load_user(user_id):
    return User.get(user_id)

# TODO: Add error handling
def get_google_provider_cfg():
    return requests.get(GOOGLE_DISCOVERY_URL).json()

@app.route("/")
def login():
    # Find out what URL to hit for Google login.
    google_provider_cfg = get_google_provider_cfg()
    authorization_endpoint = google_provider_cfg["authorization_endpoint"]

    # Construct Google login request and retrieve user profile.
    request_uri = client.prepare_request_uri(
        authorization_endpoint,
        redirect_uri=request.base_url + "login/callback",
        scope=["openid", "email", "profile"],
        )
    return redirect(request_uri)

@app.route("/login/callback")
def callback():
    # Get authorization code from Google.
    code = request.args.get("code")

    # Get User token URL.
    google_provider_cfg = get_google_provider_cfg()
    token_endpoint = google_provider_cfg["token_endpoint"]

    # Prepare and send a request to get tokens.
    token_url, headers, body = client.prepare_token_request(
        token_endpoint,
        authorization_response=request.url,
        redirect_url=request.base_url,
        code=code
        )

    token_response = requests.post(
        token_url,
        headers=headers,
        data=body,
        auth=(GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET),
        )

    # Parse tokens.
    client.parse_request_body_response(json.dumps(token_response.json()))

    # Get Google account information,
    userinfo_endpoint = google_provider_cfg["userinfo_endpoint"]
    uri, headers, body = client.add_token(userinfo_endpoint)
    userinfo_response = requests.get(uri, headers=headers, data=body)

    # Verify email address for Google account.
    if userinfo_response.json()["email"] != EMAIL:
        return "User email not available or not verified by Google.", 400

    user = User(email=EMAIL)

    # Begin user session by logging the user in
    login_user(user)

    # Send user to homepage if Google account is verified.
    return redirect(url_for("codex_vitae"))

@login_required
@app.route('/logout')
def logout():
    if current_user.is_authenticated:
        logout_user()
        return redirect(url_for("login"))

@login_required
@app.route('/codex-vitae')
def codex_vitae():
    if current_user.is_authenticated:
        # Connect to DB and plot journal entries.
        global db
        db = db or init_connection_engine()

        with db.connect(LOCAL_DB) as conn:
            journal_tuples = conn.execute(
                                        "select * from journal_view order by date"
                                        ).fetchall()

        journal = journal_calendar(journal_tuples)

        return render_template('index.html', plot=journal)
    return "Please Log In."

if __name__ == '__main__':

    app.run(ssl_context="adhoc",debug=True)


import getopt
import sys

from prisma.calibrate import main_loop

# parse arguments
try:
    opts, args = getopt.getopt(sys.argv[1:], "d:", ["days_to_sync="])

except getopt.GetoptError:
    print("Please provide proper arguments.")
    print("Usage: $python3 sync_fripon.py --d=<days>")
    sys.exit(2)
for opt, arg in opts:
    if opt in ("-d", "--days_to_sync"):
        last_n_days = int(arg)

if __name__ == "__main__":
    main_loop()


#!/usr/bin/env python

""" This module has api-key-specific methods  """

import argparse
from api_request.request import *

class UNIQUE_ID():
    def __init__(self, director_url):
        self.base_url = director_url + "/v3" + "/settings/websocketproxy.unique.identity"
        requestobj = Data('', '')
        self.request = requestobj

    def getId(self, jwt):
        url = self.base_url
        cookie = {'token': jwt,'authProvider': 'localAuthConfig'}
        response = self.request.getByCookie(url, cookie)
        id = response['value']
        print(id)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--url",
            help="director onprem url")
    parser.add_argument("--token",
            help="token")
    args = parser.parse_args()
    director_url = args.url
    token = args.token
    api_key_obj = UNIQUE_ID(director_url)
    api_key_obj.getId(token)

if __name__ == '__main__':
    main()


from __future__ import absolute_import, division, print_function,\
    with_statement

import collections
import errno
import socket
import logging
import ssl
import sys

from tornado import ioloop
from tornado.log import gen_log
from tornado.netutil import ssl_wrap_socket, ssl_match_hostname, \
    SSLCertificateError
from tornado import stack_context

from datetime import timedelta

try:
    from tornado.platform.posix import _set_nonblocking
except ImportError:
    _set_nonblocking = None

# These errnos indicate that a non-blocking operation must be retried
# at a later time. On most platforms they're the same value, but on
# some they differ.
_ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN)

# These errnos indicate that a connection has been abruptly terminated.
# They should be caught and handled less noisily than other errors.
_ERRNO_CONNRESET = (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE)

# Nice constant for enabling debug output
import sys

_SHOULD_LOG_DEBUG_OUTPUT = gen_log.isEnabledFor(logging.DEBUG)

class StreamClosedError(IOError):
    """Exception raised by `IOStream` methods when the stream is closed.

    Note that the close callback is scheduled to run *after* other
    callbacks on the stream (to allow for buffered data to be processed),
    so you may see this error before you see the close callback.
    """
    pass

class WriteQueue(object):

    def __init__(self):
        self._last_send_idx = 0
        self._write_queue = collections.deque()

    def has_next(self):
        return len(self._write_queue) > 0

    def next(self):
        if self.has_next():
            return (self._write_queue[0], self._last_send_idx)
        return None

    def clear(self):
        self._write_queue.clear()

    def append(self, src):
        self._write_queue.append(src)

    def advance(self, bytes_to_advance):
        next_src = self._write_queue[0]

        if bytes_to_advance + self._last_send_idx >= len(next_src):
            self._write_queue.popleft()
            self._last_send_idx = 0
        else:
            self._last_send_idx += bytes_to_advance

class IOHandler(object):

    def __init__(self, io_loop=None):
        # Bind to the eventloop
        self._io_loop = io_loop or ioloop.IOLoop.current()

        # Error tracking
        self.error = 0

        # Callbacks
        self._connect_cb = None
        self._close_cb = None
        self._read_cb = None
        self._write_cb = None

        # TODO:Review - Is this a good idea?
        self._error_cb = None

    def reading(self):
        raise NotImplementedError()

    def writing(self):
        raise NotImplementedError()

    def closed(self):
        raise NotImplementedError()

    def read(self, callback):
        raise NotImplementedError()

    def write(self, src, callback=None):
        raise NotImplementedError()

    def connect(self, address, callback=None):
        raise NotImplementedError()

    def close(self):
        raise NotImplementedError()

class FileDescriptorHandle(object):

    def __init__(self, fd, io_loop):
        assert fd is not None
        assert io_loop is not None

        self.fd = fd
        self._io_loop = io_loop
        self._event_interest = None

    def is_reading(self):
        return self._event_interest & self._io_loop.READ

    def is_writing(self):
        return self._event_interest & self._io_loop.WRITE

    def set_handler(self, event_handler):
        """initialize the ioloop event handler"""
        assert event_handler is not None and callable(event_handler)
        self._event_interest = self._io_loop.ERROR

        with stack_context.NullContext():
            self._io_loop.add_handler(
                self.fd,
                event_handler,
                self._event_interest)

    def remove_handler(self):
        self._io_loop.remove_handler(self.fd)

    def disable_reading(self):
        """
        Alias for removing the read interest from the event handler.
        """
        if _SHOULD_LOG_DEBUG_OUTPUT:
            gen_log.debug('Halting read events for stream(fd:{})'.format(
                self.fd))
        self._drop_event_interest(self._io_loop.READ)

    def disable_writing(self):
        """
        Alias for removing the send interest from the event handler.
        """
        if _SHOULD_LOG_DEBUG_OUTPUT:
            gen_log.debug('Halting write events for stream(fd:{})'.format(
                self.fd))
        self._drop_event_interest(self._io_loop.WRITE)

    def resume_reading(self):
        """
        Alias for adding the read interest to the event handler.
        """
        if _SHOULD_LOG_DEBUG_OUTPUT:
            gen_log.debug('Resuming recv events for stream(fd:{})'.format(
                self.fd))
        self._add_event_interest(self._io_loop.READ)

    def resume_writing(self):
        """
        Alias for adding the send interest to the event handler.
        """
        if _SHOULD_LOG_DEBUG_OUTPUT:
            gen_log.debug('Resuming send events for stream(fd:{})'.format(
                self.fd))
        self._add_event_interest(self._io_loop.WRITE)

    def _add_event_interest(self, event_interest):
        """Add io_state to poller."""
        if not self._event_interest & event_interest:
            self._event_interest = self._event_interest | event_interest
            self._io_loop.update_handler(self.fd, self._event_interest)

    def _drop_event_interest(self, event_interest):
        """Stop poller from watching an io_state."""
        if self._event_interest & event_interest:
            self._event_interest = self._event_interest & (~event_interest)
            self._io_loop.update_handler(self.fd, self._event_interest)

class SocketIOHandler(IOHandler):

    def __init__(self, sock, io_loop=None, recv_chunk_size=4096):
        super(SocketIOHandler, self).__init__(io_loop)

        # Socket init
        self._socket = sock

        # Set socket options
        self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self._socket.setblocking(0)

        # Initialize our handling of the FD events
        self.handle = FileDescriptorHandle(self._socket.fileno(), self._io_loop)
        self.handle.set_handler(self._handle_events)

        # Flow control vars
        self._connecting = False
        self._closing = False

        # Writing and reading management
        self._write_queue = WriteQueue()

        self._recv_chunk_size = recv_chunk_size
        self._recv_buffer = bytearray(self._recv_chunk_size)

    def on_done_writing(self, callback=None):
        """
        Sets a callback for completed send events and then sets the send
        interest on the event handler.
        """
        if not self._closing:
            assert callback is None or callable(callback)
            self._write_cb = stack_context.wrap(callback)

    def on_close(self, callback):
        """
        Sets a callback to be called after this stream closes.
        """
        if not self._closing:
            assert callback is None or callable(callback)
            self._close_cb = stack_context.wrap(callback)

    def on_error(self, callback):
        """
        Sets a callback to be called after this stream closes.
        """
        if not self._closing:
            assert callback is None or callable(callback)
            self._error_cb = stack_context.wrap(callback)

    def reading(self):
        """Returns True if we are currently receiving from the stream."""
        return not self.closed() and self.handle.is_reading()

    def writing(self):
        """Returns True if we are currently writing to the stream."""
        return not self.closed() and self._write_queue.has_next()

    def closed(self):
        return self._closing or self._socket is None

    def read(self, callback):
        """
        Sets a callback for read events and then sets the read interest on
        the event handler.
        """
        self._assert_not_closed()

        assert callback is None or callable(callback)
        self._read_cb = stack_context.wrap(callback)
        self.handle.resume_reading()

    def write(self, msg, callback=None):
        self._assert_not_closed()

        if not isinstance(msg, basestring) and not isinstance(msg, bytearray):
            raise TypeError("bytes/bytearray/unicode/str objects only")

        # Append the data for writing - this should not copy the data
        self._write_queue.append(msg)
        # Enable writing on the FD
        self.handle.resume_writing()
        # Set our callback - writing None to the method below is okay
        self.on_done_writing(callback)

    def connect(self, address, callback=None):
        self._connecting = True

        try:
            self._socket.connect(address)
        except socket.error as e:
            if (e.args[0] != errno.EINPROGRESS and e.args[0] not in _ERRNO_WOULDBLOCK):
                gen_log.warning("Connect error on fd %d: %s", self.handle.fd, e)
                self.close()
                return

        self._on_connect_cb = stack_context.wrap(callback)
        self.handle.resume_writing()

    def close(self):
        if not self._closing:
            if self._write_queue.has_next():
                self.on_done_writing(self._close)
            else:
                self._close()

            self._closing = True

    def _close(self):
        """Close this stream."""
        if self._socket is not None:
            gen_log.debug('Closing stream(fd: {})'.format(self.handle.fd))

            self.handle.remove_handler()

            self._socket.close()
            self._socket = None

            if self._close_cb:
                self._run_callback(self._close_cb)

    def _assert_not_closed(self):
        if self.closed():
            raise StreamClosedError('Stream closing or closed.')

    def _do_read(self, recv_buffer):
        return self._socket.recv_into(recv_buffer, self._recv_chunk_size)

    def _do_write(self, send_buffer):
        return self._socket.send(send_buffer)

    def _handle_events(self, fd, events):
        #gen_log.debug('Handle event for stream(fd: {})'.format(self.handle.fd))

        if self._socket is None:
            gen_log.warning("Got events for closed stream %d", fd)
            return

        try:
            if self._socket is not None and events & self._io_loop.READ:
                self.handle_read()

            if self._socket is not None and events & self._io_loop.WRITE:
                if self._connecting:
                    self.handle_connect()
                else:
                    self.handle_write()

            if not self.closed() and events & self._io_loop.ERROR:
                self.handle_error(
                    self._socket.getsockopt(
                        socket.SOL_SOCKET, socket.SO_ERROR))
        except Exception:
            self.close()
            raise

    def handle_error(self, error):
        try:
            if self._error_cb is not None:
                callback = self._error_cb
                self._error_cb = None
                self._run_callback(callback, error)

            if error not in _ERRNO_CONNRESET:
                gen_log.warning("Error on stream(fd:%d) caught: %s",
                                self.handle.fd, errno.errorcode[error])
        finally:
            # On error, close the FD
            self.close()

    def handle_read(self):
        try:
            read = self._do_read(self._recv_buffer)

            if read is not None:
                if read > 0 and self._read_cb:
                    self._run_callback(self._read_cb, self._recv_buffer[:read])
                elif read == 0:
                    self.close()
        except (socket.error, IOError, OSError) as ex:
                if ex.args[0] not in _ERRNO_WOULDBLOCK:
                    self.handle_error(ex.args[0])

    def handle_write(self):
        if self._write_queue.has_next():
            try:
                msg = None

                while self._write_queue.has_next():
                    msg, offset = self._write_queue.next()
                    sent = self._do_write(msg[offset:])
                    self._write_queue.advance(sent)
            except (socket.error, IOError, OSError) as ex:
                if ex.args[0] in _ERRNO_WOULDBLOCK:
                    self._write_queue.appendleft(msg)
                else:
                    self._write_queue.clear()
                    self.handle_error(ex.args[0])
        else:
            self.handle.disable_writing()

            if self._write_cb:
                callback = self._write_cb
                self._write_cb = None
                self._run_callback(callback)

    def handle_connect(self):
        if self._on_connect_cb is not None:
            callback = self._on_connect_cb
            self._on_connect_cb = None
            self._run_callback(callback)

        self._connecting = False

    def _run_callback(self, callback, *args, **kwargs):
        """Wrap running callbacks in try/except to allow us to
        close our socket."""

        try:
            # Use a NullContext to ensure that all StackContexts are run
            # inside our blanket exception handler rather than outside.
            with stack_context.NullContext():
                callback(*args, **kwargs)
        except Exception as ex:
            gen_log.error("Uncaught exception: %s", ex)

            # Close the socket on an uncaught exception from a user callback
            # (It would eventually get closed when the socket object is
            # gc'd, but we don't want to rely on gc happening before we
            # run out of file descriptors)
            self.close()

            # Re-raise the exception so that IOLoop.handle_callback_exception
            # can see it and log the error
            raise

class SSLSocketIOHandler(SocketIOHandler):
    """A utility class to write to and read from a non-blocking SSL socket.

    If the socket passed to the constructor is already connected,
    it should be wrapped with::

    ssl.wrap_socket(sock, do_handshake_on_connect=False, **kwargs)

    before constructing the `SSLSocketIOHandler`. Unconnected sockets will be
    wrapped when `SSLSocketIOHandler.connect` is finished.
    """
    def __init__(self, *args, **kwargs):
        """The ``ssl_options`` keyword argument may either be a dictionary
        of keywords arguments for `ssl.wrap_socket`, or an `ssl.SSLContext`
        object.
        """
        self._ssl_options = kwargs.pop('ssl_options', {})
        super(SSLSocketIOHandler, self).__init__(*args, **kwargs)
        self._ssl_accepting = True
        self._handshake_reading = False
        self._handshake_writing = False
        self._ssl_on_connect_cb = None
        self._server_hostname = None

        # If the socket is already connected, attempt to start the handshake.
        try:
            self._socket.getpeername()
        except socket.error:
            pass
        else:
            # Indirectly start the handshake, which will run on the next
            # IOLoop iteration and then the real IO event_interest will be set in
            # _handle_events.
            self.handle.resume_writing()

    def reading(self):
        return self._handshake_reading or super(SSLSocketIOHandler, self).reading()

    def writing(self):
        return self._handshake_writing or super(SSLSocketIOHandler, self).writing()

    def _do_ssl_handshake(self):
        # Based on code from test_ssl.py in the python stdlib
        try:
            self._handshake_reading = False
            self._handshake_writing = False
            self._socket.do_handshake()
        except ssl.SSLError as err:
            if err.args[0] == ssl.SSL_ERROR_WANT_READ:
                self._handshake_reading = True
                return
            elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE:
                self._handshake_writing = True
                return
            elif err.args[0] in (ssl.SSL_ERROR_EOF,
                                 ssl.SSL_ERROR_ZERO_RETURN):
                self.close()
                return
            elif err.args[0] == ssl.SSL_ERROR_SSL:
                try:
                    peer = self._socket.getpeername()
                except Exception:
                    peer = '(not connected)'
                gen_log.warning("SSL Error on %d %s: %s",
                                self.handle.fd, peer, err)
                self.close()
                return
            raise
        except socket.error as err:
            if err.args[0] in _ERRNO_CONNRESET:
                self.close()
                return
        except AttributeError:
            # On Linux, if the connection was reset before the call to
            # wrap_socket, do_handshake will fail with an
            # AttributeError.
            self.close()
            return
        else:
            self._ssl_accepting = False
            if not self._verify_cert(self._socket.getpeercert()):
                self.close()
                return
            if self._ssl_on_connect_cb is not None:
                callback = self._ssl_on_connect_cb
                self._ssl_on_connect_cb = None
                self._run_callback(callback)

    def _verify_cert(self, peercert):
        """Returns True if peercert is valid according to the configured
        validation mode and hostname.

        The ssl handshake already tested the certificate for a valid
        CA signature; the only thing that remains is to check
        the hostname.
        """
        if isinstance(self._ssl_options, dict):
            verify_mode = self._ssl_options.get('cert_reqs', ssl.CERT_NONE)
        elif isinstance(self._ssl_options, ssl.SSLContext):
            verify_mode = self._ssl_options.verify_mode

        assert verify_mode in (ssl.CERT_NONE, ssl.CERT_REQUIRED, ssl.CERT_OPTIONAL)

        if verify_mode == ssl.CERT_NONE or self._server_hostname is None:
            return True
        cert = self._socket.getpeercert()
        if cert is None and verify_mode == ssl.CERT_REQUIRED:
            gen_log.warning("No SSL certificate given")
            return False
        try:
            ssl_match_hostname(peercert, self._server_hostname)
        except SSLCertificateError:
            gen_log.warning("Invalid SSL certificate", )
            return False
        else:
            return True

    def handle_read(self):
        if self._ssl_accepting:
            self._do_ssl_handshake()
        else:
            super(SSLSocketIOHandler, self).handle_read()

    def handle_write(self):
        if self._ssl_accepting:
            self._do_ssl_handshake()
        else:
            super(SSLSocketIOHandler, self).handle_write()

    def connect(self, address, callback=None, server_hostname=None):
        # Save the user's callback and run it after the ssl handshake
        # has completed.
        self._ssl_on_connect_cb = stack_context.wrap(callback)
        self._server_hostname = server_hostname

        super(SSLSocketIOHandler, self).connect(address, callback=None)

    def handle_connect(self):
        # When the connection is complete, wrap the socket for SSL
        # traffic. Note that we do this by overriding handle_connect
        # instead of by passing a callback to super().connect because
        # user callbacks are enqueued asynchronously on the IOLoop,
        # but since _handle_events calls handle_connect immediately
        # followed by handle_write we need this to be synchronous.
        self._socket = ssl_wrap_socket(self._socket, self._ssl_options,
                                       server_hostname=self._server_hostname,
                                       do_handshake_on_connect=False)
        super(SSLSocketIOHandler, self).handle_connect()

    def _do_read(self, recv_buffer):
        if self._ssl_accepting:
            # If the handshake hasn't finished yet, there can't be anything
            # to read (attempting to read may or may not raise an exception
            # depending on the SSL version)
            return -1

        try:
            bytes = self._socket.read(self._recv_chunk_size)
            read = len(bytes)
            recv_buffer[:read] = bytes

            return read
        except ssl.SSLError as e:
            # SSLError is a subclass of socket.error, so this except
            # block must come first.
            if e.args[0] == ssl.SSL_ERROR_WANT_READ:
                return -1
            else:
                raise

    def _do_write(self, send_buffer):
        return self._socket.send(send_buffer)


# important: process ids are global, i. e. are preserved (same pid for each instance) ==> <> os-managed pid
import itertools
import json
from datetime import timedelta, datetime
from typing import Sequence, Tuple, Dict
import os
import locale
import sys
import argparse

root, filename = os.path.split(__file__)
proj_root = os.path.realpath(os.path.join(root, "..", ".."))
locale.setlocale(locale.LC_ALL, "de_AT.UTF-8")  # "" for auto, e. g. german: de_AT.UTF-8
sys.path.append(proj_root)

from instances.large.process_log import ProcessLog, FileMode
from instances.large.persistence_helper import persist_summary, save_summary_plot, persist_conflict_report, save_plot

def parse_args():
    parser = argparse.ArgumentParser(description="Reads process traces from json files and analyzes them with respect "
                                                 "to possible sources of resource conflicts. A plot describing the"
                                                 " number of traces per PID and one showing the number of resource "
                                                 "accesses (read, write) as well as possible conflicts are saved. "
                                                 "The underlying data for the plots will be persisted as csv and JSON.")
    parser.add_argument("pid", metavar="pid", type=int, help="ID of the process to analyze")
    parser.add_argument("freqs", metavar="freqs", type=str, help="path frequency plot should be persisted to")
    parser.add_argument("summary_csv", metavar="summary_csv", type=str,
                        help="path resource summary (csv) should be persisted to")
    parser.add_argument("summary_plot", metavar="summary_plot", type=str,
                        help="path resource summary (plot) should be persisted to")
    parser.add_argument("conflicts", metavar="conflicts", type=str, help="path conflict report should be persisted to")
    parser.add_argument("traces", metavar="traces", nargs="+", type=str, help="process trace logs (JSON) to analyze")
    return parser.parse_args()

def load_data(paths: Sequence[str]) -> Sequence[ProcessLog]:
    return_value = []
    for path in paths:
        with open(path) as file:
            data_lst = json.load(file)
            return_value.extend(map(ProcessLog.deserialize, data_lst))

    return return_value

# count of entries, unique pids, all pids (for histogram)
def get_pids(data: Sequence[ProcessLog]) -> (int, Sequence[int], Sequence[int]):
    seen = set()
    count = 0
    pids = []
    for obj in data:
        count += 1
        pids.append(obj.pid)
        if obj.pid not in seen:
            seen.add(obj.pid)

    return count, seen, pids

def get_logs_by_id(logs: Sequence[ProcessLog], pid: int) -> Sequence[ProcessLog]:
    return [log for log in logs if log.pid == pid]

def get_runtime(logs: Sequence[ProcessLog]) -> timedelta:
    runtime = timedelta(0)
    for log in logs:
        runtime += log.end - log.start

    return runtime

def get_overlaps(logs: Sequence[ProcessLog]) -> Tuple[Sequence[Tuple[ProcessLog, ProcessLog]], int]:
    combinations = itertools.combinations(logs, 2)
    overlaps = []
    resource_accesses = 0

    for x, y in combinations:
        resource_accesses += len(x.resources) + len(y.resources)
        if dates_overlap(x.start, x.end, y.start, y.end):
            overlaps.append((x, y))

    return overlaps, resource_accesses

def dates_overlap(start1: datetime, end1: datetime, start2: datetime, end2: datetime):
    return start1 <= end2 and end1 >= start2

def get_conflicts(overlaps: Sequence[Tuple[ProcessLog, ProcessLog]]) -> \
        Sequence[Tuple[ProcessLog, ProcessLog, Sequence[str]]]:
    conflicting_tuples = []
    for x, y, in overlaps:
        intersection = list(set(x.resources).intersection(y.resources))
        # intersection not empty and mode conflict
        if intersection and x.mode == FileMode.WRITE and y.mode == FileMode.WRITE:
            conflicting_tuples.append((x, y, intersection))

    return conflicting_tuples

# returns dict with resource name as key and (read-accesses, write-accesses)-tuple as value
def get_concurrent_accesses_per_resource(concurrent_execs: Sequence[Tuple[ProcessLog, ProcessLog]]) -> \
        Dict[str, Tuple[int, int]]:
    # unique list of executions (equals flattened list of concurrent ProcessLogs without duplicates)
    execs = set([item for t in concurrent_execs for item in t])
    ret_dict = {}

    for execution in execs:
        for res in execution.resources:
            if res not in ret_dict:
                ret_dict[res] = (0, 0)

            current_r, current_w = ret_dict[res]
            if execution.mode == FileMode.READ:
                ret_dict[res] = (current_r + 1, current_w)
            elif execution.mode == FileMode.WRITE:
                ret_dict[res] = (current_r, current_w + 1)

    return ret_dict

# returns dict with resource name as key and number of conflicts as value
def get_conflicts_per_resource(conflicts: Sequence[Tuple[ProcessLog, ProcessLog, Sequence[str]]]) -> Dict[str, int]:
    ret_dict = {}

    for x, y, resources in conflicts:
        for res in resources:
            if res not in ret_dict:
                ret_dict[res] = 0

            ret_dict[res] += 1

    return ret_dict

def get_resource_summary(accesses: Dict[str, Tuple[int, int]], conflicts: Dict[str, int]) -> \
        Dict[str, Tuple[int, int, int]]:
    # dict comprehension: take values of access and add a third component to the tuple by adding (conflicts[k], )
    # (= access int in conflicts dict with key k) -> if key does not exist, take 0 (= no conflicts for this resource)
    return {k: (v + (conflicts.get(k, 0),)) for k, v in accesses.items()}

"""
@begin inst_l.main
@param pid @desc ID of process to analyze
@param freqs @desc persistence path for frequency plot
@param summary_csv @desc persistence path for csv summary
@param summary_plot @desc persistence path for plot summary
@param conflicts @desc persistence path for conflict report
@in traces @desc paths of process trace logs to analyze

@out frequency_plot @uri file:{freqs}
@out conflict_report @uri file:{conflicts}
@out res_summary_csv @uri file:{summary_csv}
@out res_summary_plot @uri file:{summary_plot}
"""

def main():
    args = parse_args()

    """
    @begin load_data @desc read process trace logs
    @in traces
    @out data
    """
    print("Loading data...")
    data = load_data(args.traces)
    print("Done.\n")
    """
    @end load_data
    """

    """
    @begin get_pids @desc determine PIDs stored\nin trace logs
    @in data
    @out count
    @out distinct_pids
    @out all_pids
    """
    print("Initial analysis...")
    count, distinct_pids, all_pids = get_pids(data)
    """
    @end get_pids
    """

    """
    @begin provide_data_info @desc print count and names of\nprocesses; save frequency plot
    @in count
    @in distinct_pids
    @in all_pids
    @in freqs

    @out frequency_plot @uri file:{freqs}
    """
    print("Done. Read {0:n} process log entries from {1} files.\n".format(count, len(args.traces)))
    print("The following {0} processes were captured in the log:".format(len(distinct_pids)))
    print(sorted(distinct_pids))
    save_plot(all_pids, args.freqs)
    print("(See plot {0} for frequency distribution.)".format(os.path.basename(args.freqs)))
    """
    @end provide_data_info
    """

    print("\u2500" * 75)  # unicode character "box drawings light horizontal"
    print("Start gathering information about process with ID {0}...".format(args.pid))
    if args.pid not in distinct_pids:
        print("\nError! PID must be part of the list above (process must be captured in input traces). Abort.")
        exit(1)

    """
    @begin get_logs @desc retrieve trace logs\nof specified process

    @in data
    @in pid

    @out pid_logs
    """
    pid_logs = get_logs_by_id(data, args.pid)  # there are len(pid_logs) instances recorded of that process
    """
    @end get_logs
    """

    """
    @begin get_process_runtime @desc compute total and average runtime

    @in pid_logs
    @out total_runtime
    @out avg_runtime
    """
    total_runtime = get_runtime(pid_logs)
    avg_runtime = total_runtime // len(pid_logs)
    """
    @end get_process_runtime
    """

    """
    @begin get_overlaps @desc determine overlapping\nPID trace logs

    @in pid_logs
    @out concurrent_executions
    @out overall_resource_accesses
    """
    concurrent_executions, overall_resource_accesses = get_overlaps(pid_logs)
    """
    @end get_overlaps
    """

    """
    @begin get_conflicts @desc determine write-write conflicts

    @in concurrent_executions
    @out res_conflicts
    """
    conflicts = get_conflicts(concurrent_executions)
    """
    @end get_conflicts
    """

    """
    @begin get_resource_info @desc determine usage and\nresource conflict infos

    @in concurrent_executions
    @in conflicts
    @out concurrent_accesses_per_resource
    @out conflicts_per_resource
    """
    concurrent_accesses_per_resource = get_concurrent_accesses_per_resource(concurrent_executions)
    conflicts_per_resource = get_conflicts_per_resource(conflicts)
    """
    @end get_resource_info
    """

    """
    @begin get_resource_summary @desc get dict representation\nof resource summary

    @in concurrent_accesses_per_resource
    @in conflicts_per_resource
    @out resource_summary
    """
    resource_summary = get_resource_summary(concurrent_accesses_per_resource, conflicts_per_resource)
    print("Done.\n")
    """
    @end get_resource_summary
    """

    print("Persisting plot and summaries...")

    """
    @begin persist_res_summary @desc save resource summary\nas CSV file and plot

    @in resource_summary
    @in summary_csv
    @in summary_plot
    @in pid

    @out res_summary_csv @uri file:{summary_csv}
    @out res_summary_plot @uri file:{summary_plot}
    """
    persist_summary(resource_summary, args.summary_csv.replace(".csv", "_{0}.csv".format(args.pid)))
    save_summary_plot(resource_summary, args.summary_plot.replace(".jpg", "_{0}.jpg".format(args.pid)), args.pid)
    """
    @end persist_resource_summary
    """

    """
    @begin persist_conflict_report @desc save conflict report\nas JSON file

    @in res_conflicts
    @in total_runtime
    @in avg_runtime
    @in pid
    @in conflicts

    @out conflict_report @uri file:{conflicts}
    """
    persist_conflict_report(conflicts, total_runtime, avg_runtime, args.pid,
                            args.conflicts.replace(".json", "_{0}.json".format(args.pid)))
    print("Done.")
    """
    @end persist_conflict_report
    """

"""
@end inst_l.main
"""

if __name__ == "__main__":
    main()


from iqoptionapi.stable_api import IQ_Option
import iqoptionapi.country_id as Pais
import time, logging, json, configparser
from datetime import datetime, date, timedelta
from dateutil import tz
import sys

#CODIGO PARA DESATIVAR DEBUG OU logging.ERROR
#logging.disable(level=(logging.DEBUG))

# Credenciais
API = IQ_Option('seu_email', 'password')

# Logar
def Login():
    API.connect()

    while True:
        if API.check_connect() == False:
            print('Erro ao se conectar')

            API.connect()
        else:
            print('Conectado com sucesso')
            break

        time.sleep(1)
# carrega sua banca na api
def getBalance():
    # Define a conta de utilização
    conta = 'PRACTICE'#input('\nDigite com qual conta quer entrar "REAL" ou "PRACTICE": ')
    API.change_balance(conta) # PRACTICE / REAL
    return API.get_balance()
#CONVERT TIMESTAMP PARA HORA HUMANA
def timestamp_converter(x): # Função para converter timestamp
	hora = datetime.strptime(datetime.utcfromtimestamp(x).strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')
	hora = hora.replace(tzinfo=tz.gettz('GMT'))

	return str(hora)[:-6]
#PEGA O PERFIL
def perfil():
    return json.loads(json.dumps(API.get_profile_ansyc()))
#PEGA A BANCA
def banca():
    return json.loads(json.dumps(getBalance()))
#PEGA TOP 10 TRADERS DO MUNDO
def topTrades():
    pais='Worldwide'#"BR"
    da_posicao=1
    para_posicao=30
    traders_perto=0
    return json.loads(json.dumps(API.get_leader_board(pais, da_posicao, para_posicao, traders_perto)))
#PEGA PARIDADES ABERTAS
def payout(par, tipo, timeframe = 1):
	if tipo == 'turbo':
		a = API.get_all_profit()
		return int(100 * a[par]['turbo'])

	elif tipo == 'digital':

		API.subscribe_strike_list(par, timeframe)
		while True:
			d = API.get_digital_current_profit(par, timeframe)
			if d != False:
				d = int(d)
				break
			time.sleep(1)
		API.unsubscribe_strike_list(par, timeframe)
		return d

def getCandles(par):
    API.start_candles_stream(par, 60, 1)
    time.sleep(1)

    vela = API.get_realtime_candles(par, 60)

    while True:
        for velas in vela:
            print(vela[velas]['close'])
        time.sleep(1)
    API.stop_candles_stream(par, 60)

def configuracao():
    arquivo = configparser.RawConfigParser()
    arquivo.read('config.txt')

    return { 'paridade': arquivo.get('GERAL', 'paridade'), 'valor_entrada': arquivo.get('GERAL', 'entrada'), 'timeframe': arquivo.get('GERAL', 'timeframe')}

def sinais():
    file = open('sinais.txt', encoding='UTF-8')
    lista = file.read()
    file.close

    lista = lista.split('\n')

    for index,a in enumerate(lista):
        if a == '':
            del lista[index]

    return lista

Login()
print('\n - Inicializando o bot - \n')

# loop infinito para pegar acompanhar velas
# print('\n - Acompanhar velas - \n')
# par = 'EURUSD'
# getCandles(par)

# print meu perfil
data = perfil()
print('\n - Meu perfil - \n')
print(' Apelido: ',data['nickname'])
print(' Email: ',data['email'])
print(' Nome : ',data['name'])
print(' Banca : ', banca())
print(' Criado : ',timestamp_converter(data['created']))

print('\n - Pegando Top Traders - \n')

traders = topTrades()
for colocacao in traders['result']['positional']:
    nome_ranking = traders['result']['positional'][colocacao]['user_name']
    segunda_busca = API.get_user_profile_client(traders['result']['positional'][colocacao]['user_id'])
    print(' Nome:',nome_ranking)
    print(' User ID:',traders['result']['positional'][colocacao]['user_id'])
    print(' Se registrou em:',timestamp_converter(segunda_busca['registration_time']))
    print(' Pais: '+traders['result']['positional'][colocacao]['flag'])
    print(' Faturamento esta semana:',round(traders['result']['positional'][colocacao]['score'], 2))
    print('\n')

print(' - Trader(s) Online - \n')

for index in traders['result']['positional']:
    # Pega o user_id do top 10 traders
    id = traders['result']['positional'][index]['user_id']
    # Quarda o perfil de cada trader
    perfil_trader = API.get_user_profile_client(id)
    # Verifica quem está online
    if perfil_trader['status'] == 'online':
        trade = API.get_users_availability(id)

        try:
            tipo = trade['statuses'][0]['selected_instrument_type']
            par = API.get_name_by_activeId(trade['statuses'][0]['selected_asset_id']).replace('/', '')

            print('\n [',index,'] ',perfil_trader['user_name'])
            print(' Opção: ',tipo)
            print(' Pariedade: ',par)
        except:
            pass

# print('\n - Pegando pariedades abertas - \n')

# par = API.get_all_open_time()

# for paridade in par['turbo']:
	# if par['turbo'][paridade]['open'] == True:
		# print('[ TURBO ]: '+paridade+' | Payout: '+str(payout(paridade, 'turbo')))

# print('\n')

# for paridade in par['digital']:
	# if par['digital'][paridade]['open'] == True:
		# print('[ DIGITAL ]: '+paridade+' | Payout: '+str( payout(paridade, 'digital') ))

# #PEGA OS SINAIS
# print('\n')
# print(json.dumps(sinais(), indent=1))
# lista_sinais = sinais()
# print('\n Quantidade de sinais: ',len(lista_sinais))

# for index, sinal in enumerate(lista_sinais):
    # dados = sinal.split(';')

    # while index <= len(lista_sinais):

        # HORA_ATUAL = time.strftime('%Y-%m-%d %H:%M', time.localtime())

        # if HORA_ATUAL == dados[0]:
            # print('\n - Abrir ordem opções digitais - \n')
            # pareMoeda = dados[2]
            # entradas = 100.00
            # direcoes = dados[3]#call pra cima put pra baixo
            # timesframe = 1#1minuto;5minutos;15minutos

            # status,id = API.buy_digital_spot(pareMoeda, entradas, direcoes, timesframe)

            # if status:
                # print(' Ordem aberta com sucesso')
                # break
            # else:
                # print(' Não executou a ordem!')
                # break

#PEGA AS CONFIGURAÇÕES
# print('\n\n')
# conf = configuracao()
# print(conf['paridade'])

print('\n - Seguir traders - \n')
# # name:
# # "live-deal-binary-option-placed"
# # "live-deal-digital-option"
while_run_time=10
name="live-deal-digital-option"
active="EURAUD"
_type="PT1M" #"PT1M"/"PT5M"/"PT15M"
buffersize=10
print("_____________subscribe_live_deal_______________")
API.subscribe_live_deal(name,active,_type,buffersize)

start_t=time.time()
while True:
    trades = API.get_live_deal(name,active,_type)
    print("__For_digital_option__ data size:"+str(len(trades)))
    print(trades)
    print("\n\n")
    time.sleep(1)
    # if time.time()-start_t>while_run_time:
        # break
    if len(trades) > 0:
        print(json.dumps(trades[0], indent=1))
print("_____________unscribe_live_deal_______________")
API.unscribe_live_deal(name,active,_type)

# Abrir ordem iq option opções digitais
# print('\n - Abrir ordem opções digitais - \n')

# pareMoeda = 'EURCAD'
# entradas = 100.00
# direcoes = 'call'#call pra cima put pra baixo
# timesframe = 1#1minuto;5minutos;15minutos

# status,id = API.buy_digital_spot(pareMoeda, entradas, direcoes, timesframe)

# if status:
    # print(' Ordem aberta com sucesso')
    # print(' Aguarde a ordem ser concluida')
# else:
    # print(' Não executou a ordem!')
    # sys.exit()

# if isinstance(id, int):
    #while True:
        # resultado,lucro = API.check_win_digital_v2(id)

        # if resultado:
            # if lucro > 0:
                # print(' RESULTADO: WIN / LUCRO: '+str(round(lucro,2)))
            # else:
                # print(' RESULTADO: LOSS / PERDA: '+str(entradas))
            # break
#força a saida
#sys.exit()

# Abrir ordem iq option opções binarias
# print('\n - Abrir ordem opções binarias - \n')

# parMoeda = 'AUDCAD-OTC'
# entrada = 100.00
# direcao = 'call'#call pra cima put pra baixo
# timeframe = 1

# status,id = API.buy(entrada, parMoeda, direcao, timeframe)

# if status:
    # resultado,lucro = API.check_win_v3(id)
    # print('RESULTADO: '+resultado+' / LUCRO: '+str(round(lucro, 2)))
    # #Exemplo para pegar resultado
    # # print(API.check_win_v3(id))
    # # print('\n')
    # # print(API.check_win_v4(id))

print('\n - Fim das operações - \n')