# Databricks notebook source
# MAGIC %sql
# MAGIC Create database if not exists f1_processed
# MAGIC location '/mnt/processed'

# COMMAND ----------

# MAGIC %run /Users/Saravana_admin@5njbxz.onmicrosoft.com/Include/Config_File

# COMMAND ----------

Circuit_Schema = StructType(fields = [ StructField("circuitId",IntegerType(),False),
                                   StructField("circuitRef",StringType(),True),
                                   StructField("name",StringType(),True),
                                   StructField("location",StringType(),True),
                                   StructField("country",StringType(),True),
                                   StructField("lat",DoubleType(),True),
                                   StructField("lng",DoubleType(),True),
                                   StructField("alt",IntegerType(),True),
                                   StructField("url",StringType(),True)

])

circuits_df = spark.read\
.option("Header",True)\
.schema(Circuit_Schema)\
.csv('/mnt/raw/circuits.csv')

Circuit_df_select = circuits_df.select(col("circuitId"),col("circuitRef"),col("name"),col("location"),col("country"),col("lat"),col("lng"),col("alt"))

Circuit_df_rename = Circuit_df_select.withColumnRenamed('circuitId','circuit_Id')\
                                        .withColumnRenamed('circuitRef','circuit_Ref')\
                                        .withColumnRenamed("lat","latitude")\
                                        .withColumnRenamed("lng","Longitude")\
                                        .withColumnRenamed("alt","Altitude")

from pyspark.sql.functions import current_timestamp

Circuit_df_timestamp = Circuit_df_rename.withColumn("Dataingestion_time", current_timestamp())

#Circuit_df_timestamp.write.mode("overwrite").parquet("dbfs:/mnt/processed/Circuits")

Circuit_df_timestamp.write.mode("overwrite").format("parquet").saveAsTable("f1_processed.circuits")

# COMMAND ----------

Races_Schema = StructType(fields = [StructField("raceId",IntegerType(),False),
                                    StructField("year",IntegerType(),False),
                                    StructField("round",IntegerType(),True),
                                    StructField("circuitId",IntegerType(),True),
                                    StructField("name",StringType(),False),
                                    StructField("date",StringType(),True),
                                    StructField("time",StringType(),True),
                                    StructField("url",StringType(),True)
                                    ])

Races_df = spark.read.option("Header",True).schema(Races_Schema).csv('/mnt/raw/races.csv')
from pyspark.sql.functions import col
Races_df_renamed = Races_df.select(col("raceId").alias("race_Id"),col("year").alias("Race_year"),col("round"),col("circuitId").alias("circuit_Id"),col("name"),col("date"),col("time"))
from pyspark.sql.functions import current_timestamp,to_timestamp,concat,lit

Races_df_result = Races_df_renamed.withColumn("Dataingestion_time", current_timestamp())\
                                    .withColumn("Race_timestamp",to_timestamp(concat(col("date"),lit(" "),col("time")),'yyyy-MM-dd HH:mm:ss'))

#Races_df_result.write.mode("overwrite").parquet("dbfs:/mnt/processed/races")
Races_df_result.write.mode("overwrite").format("parquet").saveAsTable("f1_processed.races")

# COMMAND ----------

Constructors_Schema  = StructType(fields = [StructField("constructorId",IntegerType(),False),
                                            StructField("constructorRef",StringType(),False),
                                            StructField("name",StringType(),False),
                                            StructField("nationality",StringType(),False),
                                            StructField("url",StringType(),False)])

Constructor_Df = spark.read.schema(Constructors_Schema).json("/mnt/raw/constructors.json")
Constructor_renamedf = Constructor_Df.withColumnRenamed("constructorId","constructor_Id")\
                                        .withColumnRenamed("constructorRef","constructor_Ref")\
                                        .withColumn("Ingestion_time",current_timestamp())\
                                        .drop(col("url"))
#Constructor_renamedf.write.mode("overwrite").parquet("dbfs:/mnt/processed/Constructors")
Constructor_renamedf.write.mode("overwrite").format("parquet").saveAsTable("f1_processed.Constructors")

# COMMAND ----------

Drivers_Schema = StructType(fields = [ StructField ("driverId",IntegerType(),False),
                                        StructField ("driverRef",StringType(),False),
                                        StructField ("number",IntegerType(),True),
                                        StructField ("code",StringType(),False),
                                        StructField ("name",StructType(fields = [ StructField ("forename",StringType(),False),StructField ("surname",StringType(),False)]),False),
                                        StructField ("dob",DateType(),False),
                                        StructField ("nationality",StringType(),False),
                                        StructField ("url",StringType(),False)

])

Drivers_Df = spark.read\
            .schema(Drivers_Schema)\
            .json("/mnt/raw/drivers.json")

Drivers_transform_df = Drivers_Df.withColumnRenamed("driverId","driver_Id")\
                                .withColumnRenamed("driverRef","driver_Ref")\
                                .withColumn("name",concat(col("name.forename"), lit(" "), col("name.surname")))\
                                .withColumn("Ingestion_time",current_timestamp())\
                                .drop(col("url"))
#Drivers_transform_df.write.mode("overwrite").parquet("dbfs:/mnt/processed/Drivers")

Drivers_transform_df.write.mode("overwrite").format("parquet").saveAsTable("f1_processed.Drivers")

# COMMAND ----------

Results_schema = StructType (fields = [StructField("resultId",IntegerType(),True),
                                        StructField("raceId",IntegerType(),True),
                                        StructField("driverId",IntegerType(),True),
                                        StructField("constructorId",IntegerType(),True),
                                        StructField("number",IntegerType(),True),
                                        StructField("grid",IntegerType(),True),
                                        StructField("position",IntegerType(),True),
                                        StructField("positionText",StringType(),True),
                                        StructField("positionOrder",IntegerType(),True),
                                        StructField("points",FloatType(),True),
                                        StructField("laps",IntegerType(),True),
                                        StructField("time",StringType(),True),
                                        StructField("milliseconds",IntegerType(),True),
                                        StructField("fastestLap",IntegerType(),True),
                                        StructField("rank",IntegerType(),True),
                                        StructField("fastestLapTime",StringType(),True),
                                        StructField("fastestLapSpeed",FloatType(),True),
                                        StructField("statusId",IntegerType(),True)
                                        ])

Results_df = spark.read\
         .schema(Results_schema)\
         .json("/mnt/raw/results.json")

Results_Transform_df = Results_df.withColumnRenamed("resultId","result_Id")\
                                .withColumnRenamed("raceId","race_Id")\
                                .withColumnRenamed("driverId","driver_Id")\
                                .withColumnRenamed("driverId","driver_Id")\
                                .withColumnRenamed("constructorId","constructor_Id")\
                                .withColumnRenamed("positionText","position_Text")\
                                .withColumnRenamed("positionOrder","position_Order")\
                                .withColumnRenamed("fastestLap","fastest_Lap")\
                                .withColumnRenamed("fastestLapTime","fastest_Lap_Time")\
                                .withColumnRenamed("fastestLapSpeed","fastest_Lap_Speed")\
                                .withColumn("Ingestion_time",current_timestamp())\
                                .drop(col("statusId"))

#Results_Transform_df.write.mode("overwrite").parquet("dbfs:/mnt/processed/results")

Results_Transform_df.write.mode("overwrite").format("parquet").saveAsTable("f1_processed.Results")

# COMMAND ----------

Pitstops_Schema = StructType(fields = [ StructField ("raceId",IntegerType(),False),
                                        StructField ("driverId",StringType(),False),
                                        StructField ("stop",IntegerType(),True),
                                        StructField ("lap",IntegerType(),False),
                                        StructField ("time",StringType(),False),
                                        StructField ("duration",FloatType(),False),
                                        StructField ("milliseconds",IntegerType(),False)

])

Pitstops_Df = spark.read\
            .schema(Pitstops_Schema)\
            .option("multiline", True)\
            .json("/mnt/raw/pit_stops.json")
Pitstops_transform_df = Pitstops_Df.withColumnRenamed("raceId","race_Id")\
                                .withColumnRenamed("driverId","driver_Id")\
                                .withColumn("Ingestion_time",current_timestamp())
#Pitstops_transform_df.write.mode("overwrite").parquet("dbfs:/mnt/processed/pit_stops")
Pitstops_transform_df.write.mode("overwrite").format("parquet").saveAsTable("f1_processed.Pitstops")

# COMMAND ----------

laptimes_Schema = StructType(fields = [ StructField ("qualifyId",IntegerType(),False),
                                        StructField ("driverId",IntegerType(),False),
                                        StructField ("lap",IntegerType(),False),
                                        StructField ("position",IntegerType(),False),
                                        StructField ("time",StringType(),False),
                                        StructField ("milliseconds",IntegerType(),False)

])

laptimes_Df = spark.read\
            .schema(laptimes_Schema)\
            .json("/mnt/raw/lap_times")
Laptimes_transform_df = laptimes_Df.withColumnRenamed("raceId","race_Id")\
                                .withColumnRenamed("driverId","driver_Id")\
                                .withColumn("Ingestion_time",current_timestamp())
#Laptimes_transform_df.write.mode("overwrite").parquet("dbfs:/mnt/processed/lap_times")
Laptimes_transform_df.write.mode("overwrite").format("parquet").saveAsTable("f1_processed.Laptimes")

# COMMAND ----------

Qualifying_Schema = StructType(fields = [ StructField ("qualifyId",IntegerType(),False),
                                        StructField ("raceId",IntegerType(),False),
                                        StructField ("driverId",IntegerType(),False),
                                        StructField ("constructorId",IntegerType(),False),
                                        StructField ("number",IntegerType(),True),
                                        StructField ("position",IntegerType(),False),
                                        StructField ("q1",StringType(),False),
                                        StructField ("q2",StringType(),False),
                                        StructField ("q3",StringType(),False)

])

Qualifying_Df = spark.read\
            .schema(Qualifying_Schema)\
            .option("multiline", True)\
            .json("/mnt/raw/qualifying")
Qualifying_transform_df = Qualifying_Df.withColumnRenamed("raceId","race_Id")\
                                        .withColumnRenamed("qualifyId","qualify_Id")\
                                .withColumnRenamed("driverId","driver_Id")\
                                .withColumnRenamed("constructorId","constructor_Id")\
                                .withColumn("Ingestion_time",current_timestamp())
#Qualifying_transform_df.write.mode("overwrite").parquet("dbfs:/mnt/processed/Qualifying")
Qualifying_transform_df.write.mode("overwrite").format("parquet").saveAsTable("f1_processed.Qualifying")

# COMMAND ----------

notebooks/Users/Formula1_project/Processed/1. Create Ingestion tables.py


import numpy
import torch
from torchvision import transforms

std = [0.229,0.224,0.225]
mean = [0.485,0.456,0.406]
def encoder(img):
    img = img.cuda()
    for i in range(0,img.size(0)):
        for j in range(0,img.size(1)):
                    img[i][j] = (img[i][j]-112)/112
    return img

def decoder(img):
    img = img.cuda()
    for i in range(0, img.size(0)):
        for j in range(0, img.size(1)):
            img[i][j] = img[i][j]*112 + 112
    return img

def verencode(verify):
    transform_val_list = [
        transforms.Normalize([0.485,0.456,0.406],[0.229,0.224,0.225]),
    ]
    trans_compose = transforms.Compose(transform_val_list)
    re = trans_compose(verify)
    return re


import scrapy
from lxml import etree
from scrapy.http import Request
from movieSpider.items import MovieSpiderItem
# from movieSpider.movieSpider.pipeLine.MySQLPipeline import MySQLPipeline
# from ..pipeLine.MySQLPipeline import MySQLPipeline

class Myspider(scrapy.Spider):
    name = 'movieSpider'
    allowed_domains = ['douban.com']
    headers = {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
    }

    def start_requests(self):
        for id in range(0, 251, 25):
            url = 'https://movie.douban.com/top250?start=' + str(id)
            yield Request(url, headers=self.headers, callback=self.parse)

    def parse(self, response):
        # 将request.content 转化为 Element
        root = etree.HTML(response.text)
        # 选取 ol/li/div[@class="item"] 不管它们在文档中的位置
        items = root.xpath('//ol/li/div[@class="item"]')
        for item in items:
            # 注意可能只有中文名，没有英文名；可能没有quote简评
            rank, name, alias, rating_num, quote, url = "", "", "", "", "", ""
            try:
                save_item = MovieSpiderItem()
                save_item['url'] = item.xpath('./div[@class="pic"]/a/@href')[0]
                save_item['rank'] = item.xpath('./div[@class="pic"]/em/text()')[0]
                title = item.xpath('./div[@class="info"]//a/span[@class="title"]/text()')
                save_item['name'] = title[0].encode('gb2312', 'ignore').decode('gb2312')

                save_item['alias'] = title[1].encode('gb2312', 'ignore').decode('gb2312') if len(title) == 2 else ""
                save_item['rating_num'] = item.xpath('.//div[@class="bd"]//span[@class="rating_num"]/text()')[0]
                quote_tag = item.xpath('.//div[@class="bd"]//span[@class="inq"]')
                if len(quote_tag) is not 0:
                    save_item['quote'] = quote_tag[0].text.encode('gb2312', 'ignore').decode('gb2312').replace('\xa0', '')
                else:
                    save_item['quote'] = ''
                # 输出 排名，评分，简介
                # print(rank, rating_num, quote)
                # 输出 中文名，英文名
                # print(name.encode('gb2312', 'ignore').decode('gb2312'),
                #       alias.encode('gb2312', 'ignore').decode('gb2312').replace('/', ','))
                yield save_item
            except:
                print('faild!')
                pass


import pandas as pd
import pandas as pd
import numpy as np
from numpy import array
import matplotlib.pyplot as plt
from sklearn .model_selection import train_test_split
from sklearn .linear_model import LinearRegression
from sklearn.preprocessing import LabelEncoder

from sklearn .metrics import classification_report,confusion_matrix,accuracy_score

# Step 1 - Load data and understand
ds = pd.read_csv(r"C:\Users\HP\Downloads\weight-height.csv")
print(ds)
print (ds.head(10))
print (ds.tail(10))
print (ds.describe())
print (ds.isnull().sum())

print('##################################################')
# Step 2 - visualize
x=ds.iloc[:,:-1].values
y=ds.iloc[:,2].values
#plt.bar(x,y,color="green")
#plt.grid(True)
#plt.legend
#plt.show
print('##################################################')
#step 3-convert categoricaldata tinto numerical data

ds['Gender'].replace('Female',0, inplace=True)
ds['Gender'].replace('Male',1, inplace=True)
x = ds.iloc[:, :-1].values
y = ds.iloc[:, 2].values
labelEncoder_gender =  LabelEncoder()
x[:,0] = labelEncoder_gender.fit_transform(x[:,0])

X = np.vstack(x[:, :]).astype(np.float)
print(x)
print(y)
print('##################################################')
#sstep 4-splite

x_train,y_test,x_test,y_train =train_test_split(x,y, test_size=(0.2),random_state=0)

print('##################################################')
#step 5 train
model=LinearRegression()

model.fit ( x_train,y_train)

#step 6 preadic

pre=model.predict(x_test)

#step 7 output
print(classification_report(y_test,pre))
print(accuracy_score(y_test,pre))
print(confusion_matrix(y_test,pre))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('core', '0013_auto_20150322_0000'),
    ]

    operations = [
        migrations.AlterField(
            model_name='channel',
            name='chan_id',
            field=models.CharField(help_text='The internal identifier for this channel', unique=True, max_length=64, verbose_name='Channel ID'),
            preserve_default=True,
        ),
        migrations.AlterField(
            model_name='channel',
            name='xmltvid',
            field=models.CharField(default='', help_text='The value of "channel id" in the EPG xmltv file.', max_length=64, verbose_name='XMLTV ID', blank=True),
            preserve_default=True,
        ),
    ]


import sys

input = sys.stdin.readline

# 점화식
# dp[i][j] = max(dp[i-1][j], dp[i][j-1], dp[i-1][j-1] + W[i][j])
# 즉, A대학 i번째 학생까지 고려하고 B대학 j번째 학생까지 고려 했을 때 미팅의 만족도 합의 최대

# 다음과 경우로 나뉜다.
# 1. A대학 i번째 학생이 악수를 못함
# 2. B대학 j번째 학생이 악수를 못함
# 3. A대학 i번째 학생과 B대학 j번째 학생이 악수를 함
# 이중에서 최대값을 구한다.

# 처음 값 들을 0을 추가함으로써 초기값 설정 pass

N, M, C = map(int, input().split())
W = [[0 for _ in range(C + 1)]]
for i in range(C):
    W.append([0] + list(map(int, input().split())))

A_personality = [0] + list(map(int, input().split()))
B_personality = [0] + list(map(int, input().split()))

dp = [[0 for _ in range(M + 1)] for _ in range(N + 1)]

for i in range(1, N + 1):
    for j in range(1, M + 1):
        dp[i][j] = max(dp[i - 1][j], dp[i][j - 1], dp[i - 1][j - 1] + W[A_personality[i]][B_personality[j]])

print(dp[N][M])


# -*- encoding: utf-8 -*-
'''
@File    :   4.chatting_room.py
@Time    :   2020/05/07 16:48:10
@Author  :   Gloria
@Version :   1.0
@Contact :   angelhome21@qq.com
'''

#4 设计一款能实现多人聊天的系统：使用socket和多线程技术，编写全多人聊天室；

import socket
import threading
import os
import getinfo

class Server():
    def __init__(self):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.addr = (getinfo.get_ip(), 10000)
        self.users = {}

    def start_server(self):
        try:
        	self.sock.bind(self.addr)
        except Exception as e:
        	print(e)
        self.sock.listen(5)
        print("服务器已开启，等待连接...")
        print("在空白处输入stop server并回车，来关闭服务器")
        self.accept_cont()

    def accept_cont(self):
        while True:
            s, addr = self.sock.accept()
            self.users[addr] = s
            number = len(self.users)
            print("用户{}连接成功！现在共有{}位用户".format(addr, number))

            threading.Thread(target = self.recv_send, args = (s, addr)).start()

    def recv_send(self, sock, addr):
        while True:
            try:
                response = sock.recv(4096).decode("gbk")
                msg = "{}用户{}发来消息：{}".format(getinfo.get_time(), addr, response)

                for client in self.users.values():
                    client.send(msg.encode("gbk"))
            except ConnectionResetError:
                print("用户{}已经退出聊天！".format(addr))
                self.users.pop(addr)
                break

    def close_server(self):
        for client in self.users.values():
            client.close()
        self.sock.close()
        os._exit(0)

if __name__ == "__main__":
    server = Server()
    server.start_server()
    while True:
        cmd = input()
        if cmd == "stop server":
            server.close_server()
        else:
            print("输入命令无效，请重新输入！")


import cv2
import numpy as np
import urllib.request as urllib
import time

# Goal: from a stream or just a video, cut the silent parts of the input and center the frame if it finds a face

phone_stream_url = 'http://192.168.18.3:8080/shot.jpg'

def main():
    # Load trained face recognition classifier
    face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')

    # Input, for now it is an image instead of a video
    #input_image = cv2.imread('people.jpg')
    #video_stream = cv2.VideoCapture('person_dancing.mp4')
    imgResp = urllib.urlopen(phone_stream_url)
    imgNp = np.array(bytearray(imgResp.read()),dtype=np.uint8)
    img = cv2.imdecode(imgNp,-1)

    video_stream = cv2.VideoCapture(0)

    while True:
        _, input_image = video_stream.read()
        # Convert the input into a grayscale image
        grayscaled_input = cv2.cvtColor(input_image, cv2.COLOR_BGR2GRAY)

        # Detect the faces
        faces = face_cascade.detectMultiScale(grayscaled_input, 1.1, 4)

        # Just drawing a rectangle around it
        for (x, y, w, h) in faces:
            cv2.rectangle(input_image, (x, y), (x+w, y+h), (255, 0, 0), 2)

            #center_coordinates = (x + int(w/2), y + int(y/2))
            #cv2.circle(input_image, center_coordinates, w, (255, 0, 0), 2)
            #cv2.imshow('Cropped frame', input_image[(y-20):(y+h+20), (x-20):(x+w+20)])

        cv2.imshow('Frame', input_image)
        time.sleep(.1)

        # Stop if escape key is pressed
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    video_stream.release()

if __name__ == "__main__":
    main()


"""Main view handler"""
import random
import string
import re
from flask import Blueprint, redirect, url_for, request, render_template
from flask_json import json_response
from ..models import Link, db

page = Blueprint("serve", __name__, static_folder="static")

REGEX = re.compile("http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+") # pylint: disable=anomalous-backslash-in-string

@page.route("/", methods=["GET"])
def index():
    """Render website"""
    return render_template("index.html")

@page.route("/<string:text>")
def redirect_to_page(text):
    """Redirect handler"""
    query = Link.query.filter_by(text=text).first()
    if query is None:
        return redirect(url_for("serve.index"))
    return redirect(query.url)

@page.route("/", methods=["POST"])
def add_new():
    """Add new url to database"""
    # we need two things, the url and optionally text
    data = request.get_json(force=True, silent=True)
    try:
        # check for url, if its not in the posted json then quit to index
        url = data["url"]
        if not REGEX.match(url):
            raise SyntaxError
    except (KeyError, TypeError):
        return json_response(status="URL missing.", error=True)

    except SyntaxError:
        return json_response(status="Illegal URL.", error=True)

    try:
        # we dont need text so just pass the exception
        text = data["text"]
        if not re.match("^[a-zA-Z0-9]*$", text):
            raise SyntaxError
        if len(text) < 1:
            raise KeyError
    except KeyError:
        text = "".join(random.choice(string.ascii_letters) for _ in range(5))
    except SyntaxError:
        return json_response(status="Illegal character used.", error=True)
    # quick check whether text was used before
    query = Link.query.filter_by(text=text).first()
    if query:
        return json_response(status=f"{text} is already taken.", error=True)
    new = Link(url=url, text=text)

    db.session.add(new)
    db.session.commit()

    return json_response(status="ok", text=text, error=False)


import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt

from matplotlib.dates import DateFormatter
import matplotlib.dates as mdates
from datetime import datetime

from matplotlib.ticker import MultipleLocator, FuncFormatter

def dateparse(d):
    return datetime.strptime(d.strip('" '), "%Y-%m-%d")

plt.xkcd()

plt.rcParams["font.family"] = "Humor Sans"

df = pd.read_csv("stats.csv", parse_dates=["date"], date_parser=dateparse)
# print(df.head())
ax = sns.lineplot(
    x=df.date,
    y=df.wc,
    data=df,
    # palette="tab10",
    linewidth=2,
    marker="o",
    color="#2d73b9",
)

# # Set general font size
# plt.rcParams["font.size"] = "10"

# # Set tick font size
# for label in ax.get_xticklabels() + ax.get_yticklabels():
#     label.set_fontsize(10)

ax.set(xlabel="Date", ylabel="Word Count", title="Words per Post")
sns.despine()
plt.setp(ax.get_xticklabels(), rotation=45)

# Define the date format
date_fmt = DateFormatter("%Y")
ax.xaxis.set_major_formatter(date_fmt)
# Ensure a major tick for each week using (interval=1)
# ax.xaxis.set_major_locator(mdates.WeekdayLocator(interval=1))
# ax.xaxis.set_major_locator(mdates.MonthLocator(interval=6))
ax.xaxis.set_major_locator(mdates.YearLocator())
ax.xaxis.set_minor_locator(mdates.MonthLocator(interval=1))

ax.yaxis.set_major_formatter(FuncFormatter(lambda x, pos: f"{int(x):,}"))

# add text annotation
# plt.text(
#     17000,
#     1300,
#     "Can you guess \nwhen we had kids?",
#     horizontalalignment="center",
#     size="medium",
#     color="black",
#     # weight="semibold",
# )

plt.savefig("plot_words_per_post.svg", bbox_inches="tight", transparent=False)


from cmq_rpc import RpcServer, RpcClient, DataPacker
from gui_misc import field2variable, variable2field
import base
import logging
import datetime
import sys
import time
import json

class ProxyRpcServer(RpcServer):
    def __init__(self, rep_address, pub_address, config = {}, tday = datetime.date.today()):
        super(ProxyRpcServer, self).__init__(rep_address, pub_address)
        self.scur_day = tday
        self.name = config.get('name', 'test_agent')
        agent_class = config.get('agent_class', 'agent.Agent')
        cls_str = agent_class.split('.')
        agent_cls = getattr(__import__(str(cls_str[0])), str(cls_str[1]))
        self.agent = agent_cls(config = config, tday = tday)
        self.restart()
        self.register(self.restart)
        self.register(self.get_agent_params)
        self.register(self.set_agent_params)
        self.register(self.set_gateway_params)
        self.register(self.get_strat_params)
        self.register(self.set_strat_params)
        self.register(self.run_agent_func)
        self.register(self.run_gateway_func)
        self.register(self.run_strat_func)
        self.register(self.get_hist_data)
        self.register(self.exit_agent)

    def restart(self):
        if self.agent != None:
            self.scur_day = self.agent.scur_day
        self.agent.restart()

    def get_agent_params(self, fields):
        res = {}
        for f in fields:
            field_list = f.split('.')
            field = field_list[0]
            if field not in res:
                res[field] = {}
            if field == 'Positions':
                positions = {}
                for gway in self.agent.gateways:
                    gateway = self.agent.gateways[gway]
                    positions[gway] = {}
                for inst in gateway.positions:
                    pos = gateway.positions[inst]
                    positions[gway][inst] = {'currlong' : pos.curr_pos.long,
                                       'currshort': pos.curr_pos.short,
                                       'locklong' : pos.locked_pos.long,
                                       'lockshort': pos.locked_pos.short,
                                       'ydaylong':  pos.pos_yday.long,
                                       'ydayshort': pos.pos_yday.short}
                res[field] = positions
            elif field == 'Risk':
                risk_list = field_list[1:]
                sum_risk, risk_dict = self.agent.risk_by_strats(risk_list)
                res[field] = {'total': sum_risk, 'strats': risk_dict }
            elif field == 'Account':
                gateway = self.agent.gateways[field_list[1]]
                res[field][gateway.gatewayName] = dict([(variable2field(var), gateway.account_info[var]) for var in gateway.account_info])
            elif field ==' Orderstats':
                gateway = self.agent.gateways[field_list[1]]
                res[field][gateway.gatewayName] = dict([(variable2field(var), gateway.order_stats[var]) for var in gateway.order_stats])
            elif field == 'Orders':
                order_list = []
                gateway = self.agent.gateways[field_list[1]]
                for o in gateway.id2order.values():
                    inst = o.position.instrument.name
                    order_list.append([o.order_ref, o.sys_id, inst, o.diretion, o.volume, o.filled_volume,  o.limit_price, o.status])
                res[field][gateway.gatewayName] = order_list
            elif field == 'Trades':
                trade_list = []
                for etrade in self.agent.etrades:
                    insts = ' '.join(etrade.instIDs)
                    volumes = ' '.join([str(i) for i in etrade.volumes])
                    filled_vol = ' '.join([str(i) for i in etrade.filled_vol])
                    filled_price = ' '.join([str(i) for i in etrade.filled_price])
                    trade_list.append([etrade.id, insts, volumes, filled_vol, filled_price, etrade.limit_price, etrade.valid_time,
                                  etrade.strategy, etrade.book, etrade.status])
                res[field] = trade_list
            elif field == 'Insts':
                if len(field_list) > 1:
                    insts = field_list[1:]
                else:
                    insts = self.agent.instruments.keys()
                inst_dict = {}
                for inst in insts:
                    inst_obj = self.agent.instruments[inst]
                    inst_dict[inst] = {'Name': inst, 'Price': inst_obj.price, 'MidPrice': inst_obj.mid_price,
                                 'BidPrice': inst_obj.bid_price1, 'BidVol': inst_obj.bid_vol1,
                                 'AskPrice': inst_obj.ask_price1, 'AskVol': inst_obj.ask_vol1,
                                 'PrevClose': inst_obj.prev_close, 'MarginRate': inst_obj.marginrate,
                                 'Updated': inst_obj.last_update, 'Traded': inst_obj.last_traded,
                                 'UpLimit': inst_obj.up_limit, 'DownLimit': inst_obj.down_limit,
                                 'Volume': inst_obj.volume, 'OI': inst_obj.open_interest }
                    if 'Option' in inst_obj.__class__.__name__:
                        inst_dict[inst]['PV'] = inst_obj.pv
                        inst_dict[inst]['Delta'] = inst_obj.delta
                        inst_dict[inst]['Gamma'] = inst_obj.gamma
                        inst_dict[inst]['Vega']  = inst_obj.vega
                        inst_dict[inst]['Theta'] = inst_obj.theta
                        inst_dict[inst]['Otype'] = inst_obj.otype
                        inst_dict[inst]['Strike'] = inst_obj.strike
                        inst_dict[inst]['Underlying'] = inst_obj.underlying
                        inst_dict[inst]['RiskPrice'] = inst_obj.risk_price
                        inst_dict[inst]['RiskUpdated'] = inst_obj.risk_updated
                res[field] = inst_dict
            elif field == 'Volgrids':
                prod = field_list[1]
                vg_param = {}
                if prod in self.agent.volgrids:
                    vg = self.agent.volgrids[prod]
                    for expiry in vg.volnode:
                        vg_param[expiry] = {}
                        vg_param[expiry]['Fwd'] = vg.fwd[expiry]
                        vg_param[expiry]['Updated'] = vg.last_update[expiry]
                        vg_param[expiry]['T2expiry'] = vg.t2expiry[expiry]
                        vg_param[expiry]['Under'] = vg.underlier[expiry]
                        vg_param[expiry]['Df'] = vg.df[expiry]
                        vol_param = vg.volparam[expiry]
                        vg_param[expiry]['Atm'] = vol_param[0]
                        vg_param[expiry]['V90'] = vol_param[1]
                        vg_param[expiry]['V75'] = vol_param[2]
                        vg_param[expiry]['V25'] = vol_param[3]
                        vg_param[expiry]['V10'] = vol_param[4]
                res[field] = {prod: vg_param}
            else:
                var = field2variable(field)
                res[field] = getattr(self.agent, var)
        return res

    def set_agent_params(self, params):
        for field in params:
            var = field2variable(field)
            value = params[field]
            setattr(self.agent, var, value)

    def set_gateway_params(self, gway, params):
        gateway = self.agent.gateways[gway]
        for field in params:
            var = field2variable(field)
            value = params[field]
            setattr(gateway, var, value)

    def get_strat_params(self, strat_name, fields):
        params = {}
        strat = self.agent.strategies[strat_name]
        for field in fields:
            var = field2variable(field)
            params[field] = getattr(strat, var)
        return params

    def set_strat_params(self, strat_name, params):
        strat = self.agent.strategies[strat_name]
        for field in params:
            var = field2variable(field)
            value = params[field]
            setattr(strat, var, value)

    def run_strat_func(self, strat_name, func_name):
        strat = self.agent.strategies[strat_name]
        getattr(strat, func_name)()

    def run_agent_func(self, func_name, params):
        return getattr(self.agent, func_name)(*params)

    def run_gateway_func(self, gway, func_name, params):
        gateway = self.agent.gateways[gway]
        return getattr(gateway, func_name)(*params)

    def get_hist_data(self, inst, freq, nlen = 20):
        data = []
        if freq in ['D', 'd']:
            if inst in self.agent.day_data:
                data = self.agent.day_data[inst].data[-nlen:]
        elif ('m' in freq) or ('M' in freq):
            f = int(freq[:-1])
            if (inst in self.agent.min_data) and (f in self.agent.min_data[inst]):
                data = self.agent.min_data[inst][f].data[-nlen:]
        return data

    def exit_agent(self):
        if self.agent != None:
            self.agent.exit()

def run_proxy_server(config_file, tday):
    with open(config_file, 'r') as infile:
        config = json.load(infile)
    name = config.get('name', 'test_agent')
    base.config_logging(name + "/" + name + ".log", level=logging.DEBUG,
                        format='%(name)s:%(funcName)s:%(lineno)d:%(asctime)s %(levelname)s %(message)s',
                        to_console=True,
                        console_level=logging.INFO)
    scur_day = datetime.datetime.strptime(tday, '%Y%m%d').date()
    rep_address = config.get('rep_address', 'tcp://*:10010')
    pub_address = config.get('pub_address', 'tcp://*:10020')
    proxy_app = ProxyRpcServer(rep_address, pub_address, config, scur_day)
    proxy_app.start()
    try:
        while 1:
            time.sleep(1)
    except KeyboardInterrupt:
        proxy_app.exit_agent()

def run_proxy_client(config_file, tday):
    pass

if __name__=="__main__":
    args = sys.argv[1:]
    app_name = args[0]
    params = (args[1], args[2],)
    getattr(sys.modules[__name__], app_name)(*params)


'''Import all system libraries'''
import time, socket, datetime, os, fcntl, struct, sqlite3	#import time, IP, time and date
from flask import Flask, render_template, request, jsonify, request, send_file, redirect, url_for #import web app
import netifaces as ni
import RPi.GPIO as GPIO
import SystemLibrary as L

GPIO.setmode(GPIO.BCM)             # choose BCM or BOARD
GPIO.setup(14, GPIO.OUT)

os.system('clear')

IP = L.get_IP()
L.displayTime()
#L.shift(str(L.getCurrentBinary), 2)

app = Flask(__name__)

global int_time
int_time = datetime.datetime.now()

@app.route('/') #Sets index
def index():
	return render_template('index.html') #Displays HTML page 'Relay.html'#

@app.route('/relay/<username>', methods=['GET', 'POST'])
def checkSwitch(username):
	timing = float(username)
	GPIO.output(14, 0)
	time.sleep(timing)
	GPIO.output(14, 1)
	time.sleep(timing)
	return render_template('index.html')

if __name__ == '__main__':
	app.run(debug=True, host=(str(IP)), threaded=True, port=80)
GPIO.cleanup()


from collections import deque, Counter
from BinTree import *
from RandomForestBase import *

class RandomForestClassifier(RandomForestBase):
    """
    Klasa wykonujaca klasyfikacje za pomoca lasu losowego.
    """

    def __init__(self, n_features):

        super(RandomForestClassifier, self).__init__(n_features)
        self.classifier_classes = [] #klasy klasyfikacji

    def fit(self, X, y):

        m, n = X.shape

        data_type, classifier_classes = analyse_input_data(X, y)
        self.training_data_type = data_type

        new_tree = True
        last_oob_errors = deque(maxlen=11)

        #zakladamy ze tylko dwie klasy
        #jako pierwsza w liscie ma byc klasa y[0] - wystepujaca w zbiorze treneigowym jako pierwsza
        self.classifier_classes = [y[0]]
        for c in classifier_classes:
            if c != y[0]:
                self.classifier_classes.append(c)

        #klasyfikacje dokonane za pomoca k-1 drzew
        actual_classification_for_training_set = [Counter({self.classifier_classes[0] : 0, self.classifier_classes[1] : 0}) for i in range(m)]

        #proporcje pomiedzy klasamy
        prop = Counter(y)[self.classifier_classes[0]] / float(m)

        while new_tree:

            losuj = True #zmienna odpowiadajaca za losowanie - sprawdzenie proporcji pomiedzy klasami

            while losuj: #losowanie dopoki proporcje pomiedzy klasami nie sa podobne

                #losowanie ze zwracaniem m przykladow ze zbioru treningowego
                rows = np.random.choice(m, m, replace=True)
                training_set = X[rows,:]
                training_set_classes = y[rows]

                #sprawdzenie czy podobne proporcje pomiedzy klasami do tych w pelnym zbiorze treningowym
                act_prop = Counter(training_set_classes)[self.classifier_classes[0]] / float(m)

                if prop - 0.1 <= act_prop <= prop + 0.1:
                    losuj = False

            #wiersze ktore nie sa w zbiorze treningowym - out of bag
            not_rows = list(set(range(m)).difference(rows))
            #testing_set = X[not_rows,:]

            tree = self.create_decision_tree(training_set, training_set_classes, 'C')
            self.forest.append(tree)

            #dla kazdej obserwacji sprawdzamy decyzje utworzone przez dodane drzewo
            for i in not_rows:
                decision = tree.classify(X[i, :])
                actual_classification_for_training_set[i][decision] += 1

            true_sum = 0
            false_sum = 0

            #zliczenie klasyfikacji po dodaniu drzewa
            for i,observation in enumerate(actual_classification_for_training_set):
                forest_decision = observation.most_common(1)[0][0]
                if forest_decision == y[i]:
                    true_sum += 1
                else:
                    false_sum += 1

            #aktualny blad oob po dodaniu do lasu drzewa
            oob_error = true_sum / float(true_sum + false_sum)
            last_oob_errors.append(oob_error)

            #sprawdzenie czy mozna zakonczyc proces uczenia nowych drzew
            #wyliczenie bledu oob
            if len(last_oob_errors) == 11:
                if list(last_oob_errors)[0] - (sum(list(last_oob_errors)[1:]) / 10.) < 0.01:
                    new_tree = False

    def predict(self,X):

        #Sprawdzenie czy regresor zostal wczesniej nauczony
        if self.forest == []:
            raise ValueError('Metoda fit nie zostala wywolana')

        self.check_predict_data(X)

        decisions = self.decisions_made_by_all_trees(X)
        forest_decisions = []

        for dec in decisions:
            forest_decisions.append(dec.most_common(1)[0][0])

        return forest_decisions

    def predict_proba(self,X):
        """
        Zwraca wektor prawdopodobienstw przynaleznosci przykladow do pierwszej klasy - czyli do klasy wystepujacej w zbiorze treningowym jako pierwsza
        """

        self.check_predict_data(X)

        decisions = self.decisions_made_by_all_trees(X)
        forest_proba_decisions = []
        n_trees = float(len(self.forest))

        for dec in decisions:
            forest_proba_decisions.append(dec[self.classifier_classes[0]] / n_trees)

        return forest_proba_decisions

    def decisions_made_by_all_trees(self,X):

        m, n = X.shape

        #dla kazdego wiersza z tabeli X zliczenie decyzji podjetych przez wszystkie drzewa w lesie
        decisions = [Counter({self.classifier_classes[0] : 0, self.classifier_classes[1] : 0}) for i in range(m)]
        for i,row in enumerate(X):
            for tree in self.forest:
                decision = tree.classify(row)
                decisions[i][decision] += 1

        return decisions


import os
import sys
import signal

class Daemon:
    """Base for a UNIX daemon. Only function to be run is required. Doesn't meant to be instantiated, use DaemonBuilder
    instead. Mostly created from this guide http://www.netzmafia.de/skripten/unix/linux-daemon-howto.html

    Args:
        main_function (function): function to be run by the daemon, should accept stdin, stdout, stderr as arguments
        pidfile (str): file containing the process identification number (pid)
        stdin (str): standard input stream fil
        stdout (str): standard output stream file
        stderr (str): standard error stream file

    Attributes:
        main_function (function): function to be run by the daemon, should accept stdin, stdout, stderr as arguments
        pidfile (str): file containing the process identification number (pid)
        stdin (str): standard input stream file
        stdout (str): standard output stream file
        stderr (str): standard error stream file
    """

    def __init__(self, main_function, pidfile, stdin, stdout, stderr):
        self.main_function = main_function
        self.pidfile = pidfile
        self.stdin = stdin
        self.stdout = stdout
        self.stderr = stderr

    def fork(self):
        """Forks the daemon process
        Fork a second child and exit immediately to prevent zombies.  This causes the second child process
        to be orphaned, making the init process responsible for its cleanup.
        """
        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError as error:
            print('Fork failed:', error)
            sys.exit(1)

        os.umask(0)
        os.chdir('/')
        os.setsid()

        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError as error:
            print('Fork failed:', error)
            sys.exit(1)

        # Close the standard file descriptors
        sys.stdin.flush()
        sys.stdout.flush()
        sys.stderr.flush()

        # Duplicate daemon file descriptors to standart
        stdin_file = open(self.stdin, 'r')
        stdout_file = open(self.stdout, 'a')
        stderr_file = open(self.stderr, 'a+')
        os.dup2(stdin_file.fileno(), sys.stdin.fileno())
        os.dup2(stdout_file.fileno(), sys.stdout.fileno())
        os.dup2(stderr_file.fileno(), sys.stdout.fileno())

        pidfile = open(self.pidfile, 'w+')
        pidfile.write('{}\n'.format(os.getpid()))
        pidfile.flush()
        return stdin_file, stdout_file, stderr_file

    def getpid(self):
        """Returns:
            int: pid of the process if it is running. Returns None otherwise"""
        try:
            pidfile = open(self.pidfile)
            pid = int(pidfile.read().strip())
            pidfile.flush()
        except IOError:
            pid = None
        return pid

    def start(self):
        """Starts the daemon process and runs the main function"""
        if self.getpid():
            sys.stderr.write('Daemon is already running')
            sys.exit(1)

        stdin_file, stdout_file, stderr_file = self.fork()
        self.main_function(stdin_file, stdout_file, stderr_file)

    def stop(self):
        """Stops the daemon and performs cleaning up"""
        pid = self.getpid()
        if not pid:
            sys.stderr.write('Daemon is not running')
            sys.exit(1)

        os.kill(pid, signal.SIGTERM)
        os.remove(self.pidfile)
        os.remove(self.stdout)

    def restart(self):
        """Restarts daemon"""
        self.stop()
        self.start()


from django.shortcuts import get_object_or_404
from rest_framework import status
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response

from medicines.models import Medicine
from orders.models import Cart, Order

@api_view(['POST'])
@permission_classes([IsAuthenticated])
def create_order(request):
    order_items = request.data['order_items']  # {medicine_id, quantity}
    address = request.data['address']

    order_items_to_create = []
    not_found_medicines = []

    for order_item in order_items:
        medicine = Medicine.objects.get(id=order_item['medicine_id'])
        if not medicine:
            not_found_medicines.append(order_item['medicine_id'])
            continue

        order_items_to_create.append(
            Cart(
                medicine=medicine,
                quantity=order_item['quantity']
            )
        )

    Cart.objects.bulk_create(order_items_to_create)
    Order.objects.create(
        user=request.user,
        address=address
    )

    return Response({
        'not_found_medicines': not_found_medicines,
    }, status=status.HTTP_200_OK)

@api_view(['DELETE'])
@permission_classes([IsAuthenticated])
def delete_order(request, order_id):
    order = get_object_or_404(Order, id=order_id)
    order.delete()

    return Response({}, status=status.HTTP_200_OK)

@api_view(['GET'])
@permission_classes([IsAuthenticated])
def get_order(request, order_id):
    order = get_object_or_404(Order, id=order_id)

    return Response({}, status=status.HTTP_200_OK)


# Developed by MD-Nazmus Samin Sunbeam
import os
import time

import gym
import numpy as np
import pyautogui
from gym import Env, spaces

class ComputerEnv(gym.Env):
    """Computer Environment that follows the gym template."""
    metadata = {'render.modes': ['human']}

    def __init__(self):
        super(ComputerEnv, self).__init__()
        # define observation space
        (width, height) = pyautogui.size()
        self.observation_shape = (width, height, 3)
        self.observation_space = spaces.Box(low=0, high=255, shape=self.observation_shape, dtype=np.uint8)
        # define action space
        self.action_space = spaces.Discrete(44)

    def step(self, action):
        # Flag that marks the termination of an episode
        done = False

        # Assert that it is a valid action
        assert self.action_space.contains(action), "Invalid Action"
        t = 2
        d = 50
        x_mouse, y_mouse = pyautogui.position()
        # apply actions with mouse and keyboard
        # move horizontally right
        if action == 0:
            x_mouse = d
            pyautogui.move(x_mouse, 0, t)
        # move vertically down
        elif action == 1:
            y_mouse = d
            pyautogui.move(0, y_mouse, t)
        # move horizontally left
        elif action == 2:
            x_mouse = d
            pyautogui.move(x_mouse, 0, t)
        # move vertically down
        elif action == 3:
            y_mouse = d
            pyautogui.move(0, y_mouse, t)
        # move diagonally southeast
        if action == 4:
            x_mouse = d
            y_mouse = d
            pyautogui.move(x_mouse, y_mouse, t)
        # move diagonally southwest
        elif action == 5:
            x_mouse = d
            y_mouse = d
            pyautogui.move(x_mouse, y_mouse, t)
        # move diagonally northeast
        elif action == 6:
            x_mouse = d
            y_mouse = d
            pyautogui.move(x_mouse, y_mouse, t)
        # move diagonally northwest
        elif action == 7:
            x_mouse = d
            y_mouse = d
            pyautogui.move(x_mouse, y_mouse, t)
        # right click
        elif action == 8:
            pyautogui.click(button='right', clicks=1, interval=0.25)
        # left click
        elif action == 9:
            pyautogui.click(button='left', clicks=1, interval=0.25)
        # press a key
        elif action == 10:
            pyautogui.press('a')
        # press b key
        elif action == 11:
            pyautogui.press('b')
        # press c key
        elif action == 12:
            pyautogui.press('c')
        # press d key
        elif action == 13:
            pyautogui.press('d')
        # press e key
        elif action == 14:
            pyautogui.press('e')
        # press f key
        elif action == 15:
            pyautogui.press('f')
        # press g key
        elif action == 16:
            pyautogui.press('g')
        # press h key
        elif action == 17:
            pyautogui.press('h')
        # press j key
        elif action == 18:
            pyautogui.press('j')
        # press k key
        elif action == 19:
            pyautogui.press('k')
        # press l key
        elif action == 20:
            pyautogui.press('l')
        # press m key
        elif action == 21:
            pyautogui.press('m')
        # press n key
        elif action == 22:
            pyautogui.press('n')
        # press o key
        elif action == 23:
            pyautogui.press('o')
        # press p key
        elif action == 24:
            pyautogui.press('p')
        # press q key
        elif action == 25:
            pyautogui.press('q')
        # press r key
        elif action == 26:
            pyautogui.press('r')
        # press s key
        elif action == 27:
            pyautogui.press('s')
        # press t key
        elif action == 28:
            pyautogui.press('t')
        # press u key
        elif action == 29:
            pyautogui.press('u')
        # press v key
        elif action == 30:
            pyautogui.press('v')
        # press x key
        elif action == 31:
            pyautogui.press('x')
        # press y key
        elif action == 32:
            pyautogui.press('y')
        # press z key
        elif action == 33:
            pyautogui.press('z')
        # press 0 key
        elif action == 34:
            pyautogui.press('0')
        # press 1 key
        elif action == 35:
            pyautogui.press('1')
        # press 2 key
        elif action == 36:
            pyautogui.press('2')
        # press 3 key
        elif action == 37:
            pyautogui.press('3')
        # press 4 key
        elif action == 38:
            pyautogui.press('4')
        # press 5 key
        elif action == 39:
            pyautogui.press('5')
        # press 6 key
        elif action == 40:
            pyautogui.press('6')
        # press 7 key
        elif action == 41:
            pyautogui.press('7')
        # press 8 key
        elif action == 42:
            pyautogui.press('8')
        # press 9 key
        elif action == 43:
            pyautogui.press('9')

        obs = pyautogui.screenshot()

        return obs

    def reset(self):
        os.system("killall -9 'Safari'")

    def render(self, mode='human', close=False):
        raise NotImplementedError


import os
import functools
import time
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

from sklearn.base import BaseEstimator, TransformerMixin

from constants import PLOTS_DIRECTORY, DIAGNOSTICS_DIRECTORY, FEATURES_TO_DROP, TARGET

def _plot_correlation_matrix(df):
    """
    Plots a correlation matrix for all numeric features in a dataframe, excluding those that will be dropped for
    modeling, per constants.py. Saves a png of the plot in the nested plots directory, identified in constant.py.
    :param df: pandas dataframe
    """
    df = df.select_dtypes(exclude=[object])
    corr = df.corr()
    mask = np.triu(np.ones_like(corr, dtype=np.bool))
    cmap = sns.diverging_palette(220, 10, as_cmap=True)
    sns.heatmap(corr, mask=mask, cmap=cmap, vmax=.3, center=0, square=True, linewidths=.5, cbar_kws={"shrink": .5})
    plt.title('Correlation Matrix')
    plt.tight_layout()
    plt.savefig(os.path.join(DIAGNOSTICS_DIRECTORY, PLOTS_DIRECTORY, 'correlation_matrix_.png'))
    plt.clf()

def _plot_average_by_category(df, fill_na_value='unknown'):
    """
    Produces a bar plot of the average value of the target, defined in constants.py, for each category in the dataframe.
    The plots are saved as png files in the nested plots directory, identified in constants.py. One file per categorical
    column is produced.
    :param df: pandas dataframe
    :param fill_na_value: the value for which to fill categorical nulls
    """
    features = list(df.select_dtypes(include=[object]))
    for feature in features:
        if feature not in FEATURES_TO_DROP:
            df.fillna({feature: fill_na_value}, inplace=True)
            grouped_df = pd.DataFrame(df.groupby(feature)[TARGET].mean())
            grouped_df.columns = ['mean']
            grouped_df.reset_index(inplace=True)
            sns.barplot(x=feature, y='mean', data=grouped_df)
            plt.title('Class Average for ' + feature)
            plt.xticks(rotation=90)
            plt.savefig(os.path.join(DIAGNOSTICS_DIRECTORY, PLOTS_DIRECTORY, 'class_average_for_' + feature + '.png'))
            plt.clf()

def _plot_histogram(df, column):
    """
    Plots a histogram and saves the result as a png file in the nested plots directory, identified in constants.py.
    :param df: pandas dataframe
    :param column: the column for which to make the histogram
    """
    sns.distplot(df[column], kde=False)
    plt.xlabel(column)
    plt.ylabel('density')
    plt.title('Histogram for {}'.format(column))
    plt.savefig(os.path.join(DIAGNOSTICS_DIRECTORY, PLOTS_DIRECTORY, 'histogram_plot_for_' + column + '.png'))
    plt.clf()

def make_diagnostic_plots(df):
    """
    Runs plotting functions in one call to produce a correlation matrix, average value of the target by category,
    and a histogram of the target.
    :param df: pandas dataframe
    """
    _plot_correlation_matrix(df)
    _plot_average_by_category(df)
    _plot_histogram(df, TARGET)

def timer(func):
    """
    Prints the time in seconds it took to run a function. To be used as a decorator.
    """
    @functools.wraps(func)
    def wrapper_timer(*args, **kwargs):
        start_time = time.perf_counter()
        value = func(*args, **kwargs)
        end_time = time.perf_counter()
        run_time = end_time - start_time
        print(f"finished {func.__name__!r} in {run_time:.4f} secs")
        return value
    return wrapper_timer

def _create_directories_helper(directory, parent):
    """
    Creates a directory if it does not exist.
    :param directory: directory to create
    :param parent: parent directory of the directory argument, which will also be created if necessary
    """
    if not os.path.exists(os.path.join(parent, directory)):
        os.makedirs(os.path.join(parent, directory))

def create_directories(directories_list, parent=os.getcwd()):
    """
    Calls _create_directories_helper to create an arbitrary number of directories
    :param directories_list: list of directories to create
    :param parent: the parent directory for each directory in directories_list
    """
    _create_directories_helper(parent, os.getcwd())
    if parent in directories_list:
        directories_list.remove(parent)
    for directory in directories_list:
        _create_directories_helper(directory, parent)

def drop_features(df, features_drop_list):
    """
    Drops columns from a pandas dataframe.
    :param df: pandas dataframe
    :param features_drop_list: list of features to drop
    :return: pandas dataframe
    """
    return df.drop(features_drop_list, 1)

def get_num_and_cat_feature_names(df):
    """
    Produces lists of numeric and categorical feature names from a dataframe.
    :param df: pandas dataframe
    :return: tuple of two lists, the first containing the names of the numeric features and the second containing the
    names of the categorical features
    """
    numeric_features = list(df.select_dtypes(exclude=[object]))
    categorical_features = list(df.select_dtypes(include=[object]))

    for feature in FEATURES_TO_DROP:
        numeric_features = [f for f in numeric_features if f != feature]
        categorical_features = [f for f in categorical_features if f != feature]
    return numeric_features, categorical_features

class FeaturesToDict(BaseEstimator, TransformerMixin):
    """
    Converts dataframe, or numpy array, into a dictionary oriented by records. This is a necessary pre-processing step
    for DictVectorizer().
    """
    def __int__(self):
        pass

    def fit(self, X, Y=None):
        return self

    def transform(self, X, Y=None):
        if isinstance(X, np.ndarray):
            X = pd.DataFrame(X)
        X = X.to_dict(orient='records')
        return X


import os
import zipfile
import requests
import sys
import time
import argparse
import commands
import urllib3
import concurrent
import logging
from datetime import datetime
from shutil import copy, copytree, rmtree

import cbapi # Used for error handling
from cbapi.response import *

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
parser = argparse.ArgumentParser("Kansa", description="Carbon Black parser for Kansa")

# Send target
parser.add_argument("--targetlist", default="", help="Add a targetlist")
parser.add_argument("--target", default="", help="Run on a single target")
parser.add_argument("--targetlocation", default="C:\\ProgramData\\", help="Location to save on remote host")
parser.add_argument("--targetfoldername", default="targetdata_%s" % (str(datetime.now().timestamp()).split(".")[0]), help="The foldername in the target location")
parser.add_argument("--ModulePath", default="", help="Add modules to be used")
parser.add_argument("--pushbin", default=False, help="Push depencies")
parser.add_argument("--maxsessions", default=9, help="Max concurrent sessions in carbon black")
parser.add_argument("--timeout", default=5, help="Timeout in seconds per request. Default: 5")

logfolder = "./logs"
if not os.path.exists(logfolder):
    print("Made %s for logging" % logfolder)
    os.mkdir(logfolder)

logging.basicConfig(filename="%s/kansa.log" % logfolder, level=logging.INFO)
logging.basicConfig(filename="%s/debug.log" % logfolder, level=logging.DEBUG)
print("Logging to %s/kansa.log" % logfolder)

class Kansa(object):
    def __init__(self):
        self.args = parser.parse_args()
        self.targets = []
        self.modules = []
        self.skipped = []
        self.online_sensors = []
        self.all_sensors = []
        self.jobs = []
        self.cb = CbResponseAPI()
        self.finished = 0
        self.firsttime = 0
        self.starttime = datetime.now().timestamp()
        try:
            self.max_sessions = int(self.args.maxsessions)
            if self.max_sessions <= 0:
                print("Error: maxsessions needs to be > 0")
                logging.error("Error: maxsessions needs to be > 0")
                exit()

            print("Running with %d concurrent sessions" % self.max_sessions)
            logging.info("Running with %d concurrent sessions" % self.max_sessions)

        except ValueError as e:
            print("Error: %s" % e)
            logging.error("Error: %s" % e)
            exit()
        try:
            self.timeout = int(self.args.timeout)
            if self.timeout < 5:
                print("Error: timeout needs to be at least > 5 because of carbon black delay")
                logging.error("Error: timeout needs to be at least > 5 because of carbon black delay")
                exit()

            print("Running with default timeout: %d" % self.timeout)
            logging.info("Running with default timeout: %d" % self.timeout)

        except ValueError as e:
            print("Error: %s" % e)
            logging.error("Error: %s" % e)
            exit()

        # Setting up logging stuff

    # Handles input parameters
    def handle_arguments(self):
        targets = []
        if self.args.targetlist:
            if self.check_target_list():
                self.targets = open(self.args.targetlist, "r").read().splitlines()
            else:
                print("File \"%s\" doesn't exist" % self.args.targetlist)
                logging.error("File \"%s\" doesn't exist" % self.args.targetlist)
                exit()

        # Doesnt overwrite targetlist
        if self.args.target and not self.args.targetlist:
            self.targets.append(self.args.target)

        modulepath = "Modules/Modules.conf"
        configfile = self.get_configuration_paths(modulepath)
        for item in configfile:
            self.modules.append(item)

        if len(configfile) < 5:
            print("Loaded the following modules: %s" % ", ".join(configfile))
            logging.info("Loaded the following modules: %s" % ", ".join(configfile))
        else:
            print("Loaded %d modules")
            logging.info("Loaded %d modules")

        self.modules.append(modulepath)

    # Parses configurationfile
    def get_configuration_paths(self, configpath):
        paths = []
        for line in open(configpath, "r").read().split("\n"):
            if not line:
                continue

            if not line.startswith("#"):
                line = line.replace("\\", "/")
                paths.append("Modules/%s" % line)

        return paths

    # FIX - don't have stuff hardcoded :) - POC
    # Finds the data that should be in the sent zipfile
    def pack_target_data(self):
        foldername = self.args.targetfoldername
        if os.path.exists(foldername):
            rmtree(foldername)

        # Need executables and stuff
        os.mkdir(foldername)
        copy("kansa.ps1", "%s/%s" % (foldername, "kansa.ps1"))

        # Loop over all module files, doesn't push possible depencies
        if not self.args.pushbin:
            os.mkdir("%s/Modules" % foldername)
            for item in self.modules:
                if " " in item:
                    item = item.split(" ")[0]

                filepathsplit = item.split("/")

                filepath = "/".join(filepathsplit[:-1])
                try:
                    os.mkdir("%s/%s" % (foldername, filepath))
                except FileExistsError:
                    pass

                copy(item, "%s/%s" % (foldername, filepath))

            return

        # Finds module binpaths and copies them
        modulepath = "%s/%s" % (foldername, "Modules")

        os.mkdir(modulepath)
        os.mkdir("%s/%s" % (modulepath, "bin"))
        binpaths = []

        # Checks all modules if they have any executable associated
        for item in self.modules:
            binpath = self.get_item(item)

            if not binpath:
                try:
                    copy(item, "%s/%s" % (foldername, item))
                except IOError:
                    # This seems to work specifically for Kansa.
                    # No need to os walk and directory creation magic
                    os.mkdir("%s/%s/%s" % (foldername, item.split("/")[-3], item.split("/")[-2]))
                    copy(item, "%s/%s" % (foldername, item))
                continue

            # FIX - check what it's ran on first
            binpath = binpath.replace("\\", "/")

            # Skip duplicates
            if binpath not in binpaths:
                binpaths.append(binpath)

            # Continue without the module if the binpath doesn't exist (maybe exit?)
            if not os.path.exists(binpath):
                continue

            filename = binpath.split("/")[-1]

            copy(item, "%s/%s" % (foldername, item))
            copy(binpath, "%s/%s" % (foldername, binpath))

    # Gets the requried depencies from a used module
    def get_item(self, filepath):
        for line in open(filepath, "r").read().split("\n"):
            if line.startswith("BINDEP"):
                return line.split(" ")[-1]

    # Zips the data to be sent to the remote host
    def compress_target_data(self):
        foldername = self.args.targetfoldername
        folderzip = zipfile.ZipFile("%s.zip" % foldername, "w", zipfile.ZIP_DEFLATED)
        for root, dirs, files in os.walk(foldername):
            for file in files:
                folderzip.write(os.path.join(root, file))

        folderzip.close()
        return "%s.zip" % foldername

    # Checks if the file exists
    def check_target_list(self):
        return os.path.exists(self.args.targetlist)

            # Does all the hard work
    def loop_targets(self, local_location):
        datafoldername = self.args.targetfoldername

        if not self.targets:
            print("Missing targets - exiting.")
            logging.error("Missing targets - exiting.")
            exit()

        # Manages filenames in a horrible way
        new_filename = ""
        if "/" in local_location:
            new_filename = local_location.split("//")[:-1]
        elif "\\" in local_location:
            new_filename = local_location.split("\\")[:-1]
        else:
            new_filename = local_location

        # Some folder definitions, including save location
        fullfolderlocation = "%s%s\\%s" % (self.args.targetlocation, datafoldername, datafoldername)
        default_remote_location = "%s%s" % (self.args.targetlocation, new_filename)
        output_folder = "result_%s" % (str(datetime.now().timestamp()).split(".")[0])

        # This is where everything on the targets happens
        # http://cbapi.readthedocs.io/en/latest/live-response.html

        # FIX - Not sure if this should loop targets or loop commands
        self.job = commands.handleAllJobs(local_location, default_remote_location, self.args.targetlocation, fullfolderlocation, datafoldername, outputfolder=output_folder)

        # Finds all target sensors
        for target in self.targets:
            sensor = self.cb.select(Sensor).where("hostname:%s" % target)

            # This can prolly be done in the search
            for item in sensor:
                if item.status == "Uninstall Pending":
                    continue

                self.all_sensors.append(item)

                if item.status != "Offline":
                    self.online_sensors.append(item)
                else:
                    self.skipped.append(item)

                break

        if len(self.skipped) > 5:
            print("Skipped %d sensors because they're offline." % len(self.skipped))
            logging.warning("Skipped %d sensors because they're offline." % len(self.skipped))
        elif len(self.skipped) > 0 and len(self.skipped) <= 5:
            text = "Skipped the following sensors because they're offline: "
            for item in self.skipped:
                text += "%s, " % item.hostname

            print(text[:-2])
            logging.info(text[:-2])

        if len(self.online_sensors) <= 0:
            print("No sensors to scan - exiting")
            logging.error("No sensors to scan - exiting")
            exit()

        if len(self.online_sensors) < 5:
            text = "Loaded the following sensors: "
            for item in self.online_sensors:
                text += "%s, " % item.hostname

            print(text[:-2])
            logging.info(text[:-2])
        else:
            print("Loaded %d online sensor(s)" % len(self.online_sensors))
            logging.info("Loaded %d online sensor(s)" % len(self.online_sensors))

        # Get active sessions
        # FIXME - Make this how the sessions are handled
        # Below is the start of something

        # Should do all session handling
        self.handle_sessions(fullfolderlocation, output_folder)
        return output_folder

    #def get_active_sessions(self):

    def handle_sessions(self, fullfolderlocation, output_folder):
        # Append everything to dictionary hostname: jobfinished
        self.curlist = []
        cnt = 0

        for sensor in self.all_sensors:
            self.curlist.append({"hostname": sensor.hostname})
            self.curlist[cnt]["analyzed"] = False
            self.curlist[cnt]["inprogress"] = False
            self.curlist[cnt]["sensor"] = sensor
            self.curlist[cnt]["timeout"] = self.timeout

            if sensor in self.online_sensors:
                self.curlist[cnt]["online"] = True
            else:
                self.curlist[cnt]["online"] = False

            cnt += 1

        #print("Finished generated hostlist")
        logging.info("Finished generated hostlist")

        finished = False
        cnt = 0
        currentsessions = 0
        iteration = 0
        self.currentsessions = 0

        logging.info("Started session handling loop")
        self.printProgressBar(prefix = 'Progress:', suffix = '(Online: %d/%d, Finished: 0, Sessions: %d)' % (len(self.online_sensors), len(self.curlist), self.currentsessions), length = 60)
        while(True):
            # Add jobs for items in currentsessions
            # Loop jobs with timeout
            # If finished = append to an array of data and return here:
            # finished = #self.get_all_results(job.get_zip_data, fullfolderlocation, output_folder)
            # for each finished: remove the finished one, and append a new one
            # Dictionary: {name: hostname, jobran: false/true}

            # Make currentsession counter every loop
            # This wont work hmm

            # FIXME
            #try:
            #    active_session_count = self.get_session_count()
            #except cbapi.errors.ApiError:
            #    active_session_count = 0
            active_session_count = 0

            #print("Active sessions: %d, iteration: %d" % (self.currentsessions+active_session_count, iteration))
            logging.info("Active sessions: %d, iteration: %d" % (self.currentsessions+active_session_count, iteration))
            if self.currentsessions == 0 and iteration != 0:
                # Verify, as there might be stuff leftover
                found = False
                for item in self.curlist:
                    if not item["analyzed"]:
                        print("Found something not analyzed ()")
                        logging.info("Found something not analyzed ()")
                        print("THIS PROBABLY MEANS SOMEONE ELSE ARE RUNNING THIS SCRIPT")
                        logging.warning("THIS PROBABLY MEANS SOMEONE ELSE ARE RUNNING THIS SCRIPT")
                        found = True
                        break

                if not found:
                    print("FINISHED FOR ALL TARGETS? :)")
                    logging.info("FINISHED FOR ALL TARGETS? :)")
                    break

            if active_session_count > self.currentsessions:
                self.currentsessions = active_session_count

            # Runs first iteration of commands
            # Basically not in use if the first round works out fine
            if self.currentsessions < self.max_sessions:
                newhosts = []
                for item in self.curlist:
                    if item["analyzed"] or not item["online"]:
                        continue

                    # Move on to check for results with timeout
                    if self.currentsessions == self.max_sessions:
                        break

                    if item["inprogress"]:
                        continue

                    item["inprogress"] = True
                    newhosts.append(item["hostname"])

                    self.new_run_command(self.job.put_local_file, item["sensor"])
                    self.new_run_command(self.job.unzip_remote, item["sensor"])
                    self.new_run_command(self.job.run_kansa, item["sensor"])
                    self.new_run_command(self.job.zip_remote, item["sensor"])

                    # Add to job
                    # Set item[analyzed] to false
                    self.currentsessions += 1

                if len(newhosts) > 0:
                    pass
                    #print("Putting files on %s" % ", ".join(newhosts))
                    logging.info("Putting files on %s" % ", ".join(newhosts))
                    #print("Unzipping %s" % ", ".join(newhosts))
                    logging.info("Unzipping %s" % ", ".join(newhosts))
                    #print("Running kansa on %s" % ", ".join(newhosts))
                    logging.info("Running kansa on %s" % ", ".join(newhosts))
                    #print("Zipping remote datafolder %s" % ", ".join(newhosts))
                    logging.info("Zipping remote datafolder %s" % ", ".join(newhosts))

            # Run analysis here before checking for online sensors again
            self.curlist = self.new_get_all_results(self.job.get_zip_data, fullfolderlocation, output_folder)

            # Check ALL sensors for every iteration again?
            for item in self.curlist:
                # Don't care about analyzed in list
                if item["analyzed"]:
                    continue

                newsensors = self.cb.select(Sensor).where("hostname:%s" % item["hostname"])
                try:
                    for cursensor in newsensors:
                        # Verifies if same (duplicates exist) with ID
                        if cursensor.id != item["sensor"].id:
                            continue

                        # Checks same host if status has changed
                        if cursensor.status != "Offline" and item["online"] != True:
                            #print("%s just turned online." % item["hostname"])
                            logging.info("%s just turned online." % item["hostname"])
                            item["online"] = True
                            self.online_sensors.append(cursensor)
                        elif cursensor.status == "Offline":
                            if item["online"]:
                                self.online_sensors.append(cursensor)

                            logging.info("%s is offline." % item["hostname"])
                            item["online"] = False
                            item["inprogress"] = False

                        # Break anyway, since you never reach here unelss the same sensor
                        break

                except TypeError as e:
                    #print("Error in sensor iteration 2: %s" % e)
                    logging.info("Error in sensor iteration 2: %s" % e)

            iteration += 1

    # Immediately starts a new session
    # Should return if sessions is 9 already
    def start_new_session(self):
        for item in self.curlist:
            if item["analyzed"] or not item["online"]:
                continue

            # Move on to check for results with timeout
            if self.currentsessions == self.max_sessions:
                break

            if item["inprogress"]:
                continue

            item["inprogress"] = True
            #print("Started new session on %s" % item["hostname"])
            logging.info("Started new session on %s" % item["hostname"])

            self.new_run_command(self.job.put_local_file, item["sensor"])
            self.new_run_command(self.job.unzip_remote, item["sensor"])
            self.new_run_command(self.job.run_kansa, item["sensor"])
            self.new_run_command(self.job.zip_remote, item["sensor"])

            # Set a time here
            #item["starttime"] =

            # Add to job
            # Set item[analyzed] to false
            self.currentsessions += 1

    def new_get_all_results(self, object, fullfolderlocation, output_folder):
        finishedsensornames = []

        # Loop self.curlist
        for i in range(len(self.curlist)):
            if not self.curlist[i]["inprogress"]:
                continue

            sensor = self.curlist[i]["sensor"]
            jobcheck = self.cb.live_response.submit_job(object, sensor)

            try:
                # Check if sensor is online at all?
                zipdata = jobcheck.result(timeout=self.curlist[i]["timeout"])

                # only gets here if it doesn't crash
                self.save_zip_data(sensor.computer_name, zipdata)
                self.curlist[i]["inprogress"] = False
                self.curlist[i]["analyzed"] = True
                #print("Finished saving zipdata for %s" % self.curlist[i]["hostname"])
                logging.info("Finished saving zipdata for %s" % self.curlist[i]["hostname"])
                self.new_run_command(self.job.cleanup_target, self.curlist[i]["sensor"])
                self.currentsessions -= 1

                self.finished += 1
                self.printProgressBar(prefix = 'Progress:', suffix = '(Online: %d/%d, Finished: %d, Sessions: %d)' % (len(self.online_sensors), len(self.curlist), self.finished, self.currentsessions), length = 60)

                #active_session_count = self.get_session_count()

                # FIXME
                active_session_count = 0
                #print("Active sessions: %d" % active_session_count)
                logging.info("Active sessions: %d" % active_session_count)

                # Immediately start new session
                #if active_session_count < self.max_sessions:
                if self.currentsessions < self.max_sessions:
                    self.start_new_session()

                self.printProgressBar(prefix = 'Progress:', suffix = '(Online: %d/%d, Finished: %d, Sessions: %d)' % (len(self.online_sensors), len(self.curlist), self.finished, self.currentsessions), length = 60)

            except (cbapi.errors.TimeoutError, concurrent.futures._base.TimeoutError) as e:
                # Increase time by 10 for each failure
                increase_amount = 10
                logging.info("Increased amount from %d to %d on host %s" % (self.curlist[i]["timeout"], self.curlist[i]["timeout"]+increase_amount, self.curlist[i]["hostname"]))
                self.printProgressBar(prefix = 'Progress:', suffix = '(Online: %d/%d, Finished: %d, Sessions: %d)' % (len(self.online_sensors), len(self.curlist), self.finished, self.currentsessions), length = 60)

                self.curlist[i]["timeout"] += increase_amount

            except cbapi.live_response_api.LiveResponseError as e:
                continue

        return self.curlist

    def new_run_command(self, object, sensor, critical=True):
        self.jobs.append(self.cb.live_response.submit_job(object, sensor))

    # Saves it in a temporary location (FIX - Not sure what to do here yet)
    def save_zip_data(self, targetname, zip_data):
        if not os.path.exists("data"):
            #print("Creating data folder")
            logging.info("Creating data folder")
            os.mkdir("data")
        if not os.path.exists("data/%s" % targetname):
            #print("Creating %s folder" % targetname)
            logging.info("Creating %s folder" % targetname)
            os.mkdir("data/%s" % targetname)

        with open("data/%s/data.zip" % targetname, "wb+") as tmp:
            tmp.write(zip_data)

    # Have to do some magic since extractall sucks, and cba doing more for now
    def unzip(self, folderpath):
        try:
            z = zipfile.ZipFile('%s/data.zip' % folderpath)
        except IOError:
            return

        z.extractall(folderpath)

    # Reorganizes the data for use with kansa analysis
    # This step is uneccesary, but I don't want to redo the mess above just yet :)
    def prepare_analysis(self, datafolder, analysisfolder):
        print("Moving files for analysis")
        logging.info("Moving files for analysis")

        if not os.path.exists(datafolder):
            os.mkdir(datafolder)

        if not os.path.exists(analysisfolder):
            os.mkdir(analysisfolder)

        filecount = 0
        computercount = 0
        foldercreate = True

        # Finds hosts
        for folder in os.listdir(datafolder):
            self.unzip("%s/%s" % (datafolder, folder))

            # Used while testing
            if folder.endswith(".zip"):
                continue

            # Finds hostfiles
            for filename in os.listdir("%s/%s" % (datafolder, folder)):
                if not filename.endswith(".csv"):
                    continue

                filenamesplit = filename.split("\\")

                # Creates the folder
                if foldercreate:
                    curfolder = filenamesplit[0]
                    try:
                        os.mkdir("%s/%s" % (analysisfolder, curfolder))
                    except OSError:
                        foldercreate = False

                # Moves the file to the associated location
                os.rename(
                    "%s/%s/%s" % (datafolder, folder, filename),
                    "%s/%s/%s" % (analysisfolder, filenamesplit[0], filenamesplit[1])
                )

                filecount += 1

            # Dont need to recreate folders, so done for efficiency
            foldercreate = False
            computercount += 1

            # Removes the original folder as its empty
            #rmtree(datafolder)

        print("%s is now ready for analysis with %d file(s) and %d system(s)" % (analysisfolder, filecount, computercount))
        logging.info("%s is now ready for analysis with %d file(s) and %d system(s)" % (analysisfolder, filecount, computercount))

# Cleans up the remote hosts files
    def cleanup_local(self):
        #rmtree(datafoldername)
        #os.remove("%s.zip" % datafoldername)

        foldernamecheck = self.args.targetfoldername
        if "_" in self.args.targetfoldername:
            foldernamecheck = self.args.targetfoldername.split("_")[0]

        for item in os.listdir():
            #if item.startswith("Output"):
            #    rmtree(item)
            if item.startswith(foldernamecheck):
                try:
                    rmtree(item)
                except NotADirectoryError:
                    os.remove(item)

        try:
            rmtree("__pycache__")
        except OSError:
            pass

        try:
            rmtree("targetdata.zip")
        except OSError:
            pass

    # Why isn't there any api call for this??
    # Based on live response 'session list'
    def get_session_count(self):
        sessions = self.cb.get_object("/api/v1/cblr/session")

        active = 0
        for item in sessions:
            if item["status"] == "active":
                active += 1

        return active

        #sensor = self.cb.info()
        #print(sensor)

    # Print iterations progress
    # Thanks https://stackoverflow.com/questions/3173320/text-progress-bar-in-the-console :)
    # Off by 1.8% (prolly per item thing)
    def printProgressBar (self, prefix = '', suffix = '', decimals = 1, length = 100, fill = '█'):
        """
            Call in a loop to create terminal progress bar
            @params:
            iteration   - Required  : current iteration (Int)
            total       - Required  : total iterations (Int)
            prefix      - Optional  : prefix string (Str)
            suffix      - Optional  : suffix string (Str)
            decimals    - Optional  : positive number of decimals in percent complete (Int)
            length      - Optional  : character length of bar (Int)
            fill        - Optional  : bar fill character (Str)
        """

        percent = ("{0:." + str(decimals) + "f}").format(100 * (self.finished / float(len(self.curlist))))
        filledLength = int(length * self.finished // len(self.curlist))
        bar = fill * filledLength + '-' * (length - filledLength)

        if filledLength > 0:
            timenow = datetime.now().timestamp()
            elapsed_time = timenow - self.starttime
            time_left = 100 * elapsed_time / filledLength - elapsed_time

            if self.firsttime == 0:
                self.firsttime = time_left
                self.time_left = time_left

            if time_left < self.time_left:
                if time_left >= 0:
                    self.time_left = time_left

            # Not sure why, but 2.2 is close
            #thistime = self.time_left-self.firsttime/2.1
            thistime = self.time_left-self.firsttime/2.1
            if thistime <= 2 or filledLength >= 98:
                suffix = "%s, time left: SOON^tm" % suffix
            else:
                suffix = "%s, time left: %f" % (suffix, thistime)

            print('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix), end = '\r')
        else:
            suffix = "%s time left: calculating" % (suffix)
            print('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix), end = '\r')

        if self.finished == len(self.curlist):
            print()

if __name__ == "__main__":
    # Prepares data for connection with CB sensor
    kansa = Kansa()

    kansa.handle_arguments()
    kansa.pack_target_data()
    zipname = kansa.compress_target_data()

    # Sends the actual data to the targets
    analysisfolder = kansa.loop_targets(zipname)

    # ANALYSIS
    datafolder = "data"
    kansa.prepare_analysis(datafolder, analysisfolder)

    kansa.cleanup_local()


import serial
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import numpy as np
import time

# Establish a serial connection
ser = serial.Serial('COM6', 115200)  # substitute 'COM3' with your port number and '9600' with your baud rate
time.sleep(1)
# Variables to store data
pitch_a_data = []
roll_a_data = []
pitch_g_data = []
roll_g_data = []
pitch_f_data = []
roll_f_data = []

fig, ax = plt.subplots()

lines = []
colors = ['r', 'b', 'g', 'y', 'm', 'c']
labels = ['Pitch Accelerometer', 'Roll Accelerometer', 'Pitch Gyroscope', 'Roll Gyroscope', 'Pitch Filtered', 'Roll Filtered']

for i in range(6):
    line, = ax.plot([], [], colors[i], label=labels[i])
    lines.append(line)

ax.legend(loc='upper left')

# Set y-axis limits
ax.set_ylim([-90, 90])
ax.yaxis.grid(True, linestyle='--', which='major', color='grey', alpha=.25)

def init():
    for line in lines:
        line.set_data([], [])
    return lines

def update(i):
    while (ser.inWaiting() == 0):  # wait for data
        pass
    arduinoString = ser.readline().decode('utf-8')  # read a line from the serial port
    data = arduinoString.split()

    if len(data) == 6:  # check if data array contains all data
        pitch_a = float(data[0])
        roll_a = float(data[1])
        pitch_g = float(data[2])
        roll_g = float(data[3])
        pitch_f = float(data[4])
        roll_f = float(data[5])

        pitch_a_data.append(pitch_a)
        roll_a_data.append(roll_a)
        pitch_g_data.append(pitch_g)
        roll_g_data.append(roll_g)
        pitch_f_data.append(pitch_f)
        roll_f_data.append(roll_f)

        # Limit the size of the data list
        if len(pitch_a_data) > 50:
            pitch_a_data.pop(0)
        if len(roll_a_data) > 50:
            roll_a_data.pop(0)
        if len(pitch_g_data) > 50:
            pitch_g_data.pop(0)
        if len(roll_g_data) > 50:
            roll_g_data.pop(0)
        if len(pitch_f_data) > 50:
            pitch_f_data.pop(0)
        if len(roll_f_data) > 50:
            roll_f_data.pop(0)

    lines[0].set_data(range(len(pitch_a_data)), pitch_a_data)
    lines[1].set_data(range(len(roll_a_data)), roll_a_data)
    lines[2].set_data(range(len(pitch_g_data)), pitch_g_data)
    lines[3].set_data(range(len(roll_g_data)), roll_g_data)
    lines[4].set_data(range(len(pitch_f_data)), pitch_f_data)
    lines[5].set_data(range(len(roll_f_data)), roll_f_data)

    ax.relim()
    ax.autoscale_view()

    return lines

ani = animation.FuncAnimation(fig, update, init_func=init, blit=False, interval=20)
plt.show()


# instance/config.py
# http://exploreflask.com/en/latest/configuration.html
# __file__ --> basic.py where is it in my computer - where do I start and it gives me the full directory path
# the os library saves you to grab that absolute path c://user/bart/something/scripts/program

import os
from dotenv import load_dotenv

load_dotenv()  # take environment variables
# environment variables are accessible with the load

# basedir = os.path.abspath(os.path.dirname(__file__))
PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__))

SECRET_KEY = os.getenv("API_KEY")
SQLALCHEMY_DATABASE_URI = "sqlite:///" + os.path.join(PROJECT_ROOT, "hornets.sqlite")
SQLALCHEMY_TRACK_MODIFICATIONS = False
JSON_SORT_KEYS = False

RAW_URI = True
REQUEST_URI = True

# MINOR ENVIRONMENT BASED CONFIG
DEBUG = True

SESSION_COOKIE_SECURE = True
SESSION_COOKIE_HTTPONLY = True
SESSION_COOKIE_SAMESITE = 'Lax'
SECURITY_CSRF_IGNORE_UNAUTH_ENDPOINTS = True


import pytest

from src.player.ai import AIPlayer
from src.player.human import HumanPlayer
from tests.stubs import ControllerStub

@pytest.fixture()
def setup() -> tuple[HumanPlayer, ControllerStub]:
    hp = HumanPlayer()
    stub = ControllerStub(hp)
    return hp, stub

def test_get_user_input(
    monkeypatch: pytest.MonkeyPatch,
    setup: tuple[HumanPlayer, ControllerStub],
) -> None:
    monkeypatch.setattr("builtins.input", lambda: "5")
    hp, controller = setup
    hp.get_input()

    assert controller.commands == {hp: "5"}

def test_get_ai_input():
    ai = AIPlayer(test_value=5)
    stub = ControllerStub(ai)

    ai.get_input()

    assert stub.commands == {ai: "5"}