from sympy import solve_triangulated
import torch
from torch import nn
from torch_geometric.nn import MessagePassing, TopKPooling
from torch_geometric.data import Data
from torch_geometric.utils import remove_self_loops, add_self_loops
from synthetic_data import datagen#, sourced
import numpy as np
import matplotlib.pyplot as plt
from Granola_GNN import NodeClassifier
from torch_geometric.data import DataLoader
import sys

"""

# ---------------------------------------   Data import ----------------------------------#

#Define problem variables
x_i = np.arange(0,10)
y_i = np.arange(0,10)
ev_mi = np.arange(0,10)
ev_ni = np.arange(0,10)
slist = np.arange(10)

#generate [100 x 2] node feature array with each index being a coordinate
x = []
for i in range(len(x_i)):
    for j in range(len(y_i)):
        x.append(np.array([i,j]))
nfeat = np.array(x)

#Create edge index array  (Defines graph connectivity using coordinate vector; code block below is specific to graphs with spatial coordinates)
source = []
target = []
for i in range(nfeat.shape[0]):
    ind = np.argwhere(np.all(np.abs(nfeat[i] - nfeat) <= 1,axis=1)).squeeze() # checking node distance on grid
    source.append(np.repeat(i,len(ind)))
    target.append(ind)

source = np.concatenate(source, axis=0)
target = np.concatenate(target, axis=0)

source = np.array(source).reshape((1,source.shape[0]))
target = np.array(target).reshape((1,target.shape[0]))
edge_index = torch.tensor(np.concatenate((source,target),axis=0))

#Defining Train/Test/Split Indeces

ntraingraphs = 700
nvadgraphs = 200
ntestgraphs = 100
ntotgraphs = ntraingraphs + ntestgraphs + nvadgraphs
graphs = np.arange(ntotgraphs)

trainind = np.random.choice(graphs,ntraingraphs,replace=False)
vadind = [x for x in graphs if x not in trainind][:nvadgraphs]
tempind =np.hstack((trainind,vadind))
testind = [x for x in graphs if x not in tempind]

#Getting 100 solution meshes

train_datalist = []
for i in range(len(trainind)):

    print(i)

    j= int(np.floor(trainind[i]/100)) # specifies the paramter function
    a = int(np.floor(trainind[i]/10)) # Specifies the eigenvalue coefficient

    vmesh = datagen(x_i,y_i,ev_mi,ev_ni,a,sourced(j))
    totfeat = np.concatenate((nfeat,vmesh),axis = 1)
    totparam  = np.array([sourced(j)(x,y) for x in range(10) for y in range(10)])

    train_datalist.append(Data(x=torch.tensor(totfeat,dtype=torch.float),y=torch.tensor(totparam,dtype=torch.float),edge_index=edge_index))

train_loader = DataLoader(train_datalist,batch_size=10,shuffle=True)
torch.save(train_loader,"train_data.pt")
print("Number of training samples",len(train_datalist))

vad_datalist = []
for i in range(len(vadind)):

    print(i)

    j=int(np.floor(vadind[i]/100)) # specifies the paramter function
    a=int(np.floor(vadind[i]/10)) # Specifies the eigenvalue coefficient

    vmesh = datagen(x_i,y_i,ev_mi,ev_ni,a,sourced(j))
    totfeat = np.concatenate((nfeat,vmesh),axis = 1)
    totparam  = np.array([sourced(j)(x,y) for x in range(10) for y in range(10)])

    vad_datalist.append(Data(x=torch.tensor(totfeat,dtype=torch.float),y=torch.tensor(totparam,dtype=torch.float),edge_index=edge_index))

vad_loader = DataLoader(vad_datalist,batch_size=10,shuffle=True)
torch.save(vad_loader,"vad_data.pt")
print("Number of Validation samples",len(vad_datalist))

test_datalist = []
for i in range(len(testind)):

    print(i)

    j= int(np.floor(testind[i]/100)) # specifies the paramter function
    a = int(np.floor(testind[i]/10)) # Specifies the eigenvalue coefficient

    vmesh = datagen(x_i,y_i,ev_mi,ev_ni,a, sourced(j))
    totfeat = np.concatenate((nfeat,vmesh),axis = 1)
    totparam  = np.array([sourced(j)(x,y) for x in range(10) for y in range(10)])

    test_datalist.append(Data(x=torch.tensor(totfeat,dtype=torch.float),y=torch.tensor(totparam,dtype=torch.float),edge_index=edge_index))

test_loader = DataLoader(test_datalist,batch_size=10,shuffle=True)
torch.save(test_loader,'test_data.pt')
print("Number of test samples",len(test_datalist))

#Check's number of btches
#for batch in train_loader:
#    print(batch)

"""

#Load train/test data

train_loader = torch.load('train_data.pt')
test_loader = torch.load('test_data.pt')
vad_loader = torch.load('vad_data.pt')

# Script for training GNN

model = NodeClassifier(num_node_features = 3, hidden_features = 3, num_classes = 1)
optimizer = torch.optim.Adam(model.parameters(),lr=0.001)
criterion = torch.nn.MSELoss()
model = model.float()

def train():
    loss_all = 0
    c = 0 # Counter for number of batches
    for data in train_loader:

        model.train()
        optimizer.zero_grad()
        out = model(data.x, data.edge_index)
        print("Model output:",out.size())
        loss = criterion( out, data.y )

        loss.backward()
        loss_all += loss.item() # adds loss for each batch
        optimizer.step()
        c = c + 1
        sys.exit()

    return loss_all/c # reporting average loss per batch

def validate():

    loss_all = 0
    c = 0 # counter for number of batches

    for data in vad_loader:
        model.eval()
        out = model(data.x, data.edge_index)
        loss = criterion(out, data.y)
        loss_all += loss.item()
        c = c + 1

    return loss_all/c #normalizing by number of batches

svad = []
svtrain = []

for epoch in range(1, 200):
    train_loss = train()
    validate_loss = validate()

    svtrain.append(train_loss)
    svad.append(validate_loss)

    if epoch % 10 == 0: #Print every 10 epochs
        print(f'Epoch: {epoch:03d}, Train Loss: {train_loss:.4f}, Validation Loss: {validate_loss:.4f}')

#Testing model with MSE
tloss = 0
for data in test_loader:

    model.eval()
    out = model(data.x,data.edge_index)
    tloss += criterion(out,data.y)
    #print(tloss)

print("Average Test Loss (MSE)",tloss/len(test_loader))

plt.figure()
plt.title("Experiment 8")
plt.plot(svtrain)
plt.plot(svad)
plt.yscale("log")
plt.xlabel("Number of Epochs")
plt.ylabel("Loss (Log-Scale)")
plt.legend(["Train Loss","Validation Loss"])
plt.savefig("/Users/oluwadamilolafasina/Inverse_GNN/exp/exp_8_train_vad.png")
plt.show()

plt.figure()
plt.title("Experiment 8 - Train Loss")
plt.plot(svtrain)
plt.yscale("log")
plt.xlabel("Number of Epochs")
plt.ylabel("Loss (Log-Scale)")
plt.legend(["Train Loss","Validation Loss"])
plt.show()
plt.savefig("/Users/oluwadamilolafasina/Inverse_GNN/exp/exp_8_train.png")

plt.figure()
plt.title("Experiment 8 - Validation Loss")
plt.plot(svad)
plt.yscale("log")
plt.xlabel("Number of Epochs")
plt.ylabel("Loss (Log-Scale)")
plt.show()
plt.savefig("/Users/oluwadamilolafasina/Inverse_GNN/exp/exp_8_vad.png")


from django.shortcuts import render
from mainattendance.models import StudentAttendance,CurrentAttendance,Batch,Student
# Create your views here.
def page(request):
	# batchs = CurrentAttendance.objects.values('batch','date')
	# if request.method == "GET":
	# 	xbatch = request.GET.get('batch')

	# params = {'batchs':batchs}
	# if xbatch is not None:
	# 	sbatch = xbatch.split(' on ')
	# 	batch = sbatch[0]
	# 	date = sbatch[1]
	# 	xstudents = CurrentAttendance.objects.get(batch=batch,date=date)

	# 	astudents = xstudents.students.all()
	# 	no_of_students =astudents.count()

	# 	att_list = []
	# 	for a in range(0,no_of_students):
	# 		at = StudentAttendance.objects.get(student_id = astudents[a].student_id,date=date)
	# 		att_list.append(at)

	# 	print(att_list)
	# 	params = {'xattendance':att_list,'date':date}

	batchs = Batch.objects.all()
	params={'batchs': batchs}
	if request.method == "GET":
		xbatch = request.GET.get('batch')
		xdates = CurrentAttendance.objects.filter(batch = xbatch)

		no_of_dates = xdates.count()
		# print(no_of_dates)
		att_list = []
		for a in range(0,no_of_dates):
			print(xdates[a].date)

			xstudents = CurrentAttendance.objects.get(batch =xbatch, date= xdates[a].date)
			astudents= xstudents.students.all()
			no_of_students = astudents.count()

			# cdate = xdates[a].date
			for i in range(0,no_of_students):
				print(astudents[i].student_id)

				at = StudentAttendance.objects.get(student_id = astudents[i].student_id,date = xdates[a].date)
				# print(at)
				att_list.append(at)

		print(att_list)

		params={'xattendance':att_list}
	return render(request, 'viewattendance/page.html',params)


from typing import Optional

from httpx import Client

from .responses import (
    InfoResponse,
    LoginResponse,
    SeasonObject,
    SeasonResponse,
    SeasonsObject,
    SeasonsResponse,
)

class Api:
    def __init__(self, client: Optional[Client] = None) -> None:
        """
        Creates an API instance.
        """

        self.client = client or Client(
            http2=True,
            headers={
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:96.0) Gecko/20100101 Firefox/96.0"
            },
        )
        self.conn_id: Optional[str] = None

    def login(self) -> LoginResponse:
        """
        Login to VVVVID.
        """

        r = self.client.get("https://www.vvvvid.it/user/login")
        login = LoginResponse(r)
        self.conn_id = login.conn_id
        return login

    def info(self, show_id: int) -> InfoResponse:
        """
        Get information about a show.
        """

        r = self.client.get(
            f"https://www.vvvvid.it/vvvvid/ondemand/{show_id}/info",
            params={"conn_id": self.conn_id},
        )
        return InfoResponse(r)

    def seasons(self, show_id: int) -> list[SeasonsObject]:
        """
        Get all the seasons the show has.
        """

        r = self.client.get(
            f"https://www.vvvvid.it/vvvvid/ondemand/{show_id}/seasons",
            params={"conn_id": self.conn_id},
        )
        s = SeasonsResponse(r)
        return s.get()

    def season(self, show_id: int, season_id: int) -> list[SeasonObject]:
        """
        Get information about a season.
        """

        r = self.client.get(
            f"https://www.vvvvid.it/vvvvid/ondemand/{show_id}/season/{season_id}",
            params={"conn_id": self.conn_id},
        )
        s = SeasonResponse(r)
        return s.get(self.client)


import json

from django.http import JsonResponse
from django.views import View

from movies.models import Actor, Movie

class ActorsView(View):

    def get(self, request):

        results = [
            {
                'first_name' : actor.first_name,
                'last_name' : actor.last_name,
                'movie_list' : [{
                    'title': movie.title
                } for movie in actor.movie.all()]
            } for actor in Actor.objects.all()
        ]
        return JsonResponse({"results" : results}, status=200)

class MoviesView(View):

    def get(self, request):

        results = [
            {
                'title' : movie.title,
                'running_time' : movie.running_time,
                'actor_list' : [{
                    'actor': actor.first_name
                } for actor in movie.actor_set.all()]
            } for movie in Movie.objects.all()
        ]
        return JsonResponse({"results" : results}, status=200)


from functools import cached_property

from typing import Dict, List, Sequence

from .column_names import COLUMN_NAME
from .events_factory import EventsFactory
from .lttng_info import LttngInfo
from .ros2_tracing.data_model import Ros2DataModel
from .value_objects import TimerCallbackValueLttng, TimerControl, TimerInit
from ...common import Columns, Util
from ...record import (merge, merge_sequential,
                       merge_sequential_for_addr_track,
                       RecordFactory,
                       RecordsFactory,
                       RecordsInterface)

class RecordsSource():

    def __init__(
        self,
        data: Ros2DataModel,
        info: LttngInfo
    ) -> None:
        self._data = data
        self._preprocess(self._data)
        self._info = info

    @staticmethod
    def _preprocess(data: Ros2DataModel):
        data.rclcpp_publish_instances.rename_columns(
            {COLUMN_NAME.RCLCPP_PUBLISH_TIMESTAMP: COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP}
        )

    @cached_property
    def _grouped_callback_start(self) -> Dict[int, RecordsInterface]:
        records = self._data.callback_start_instances.clone()
        group: Dict[int, RecordsInterface] = {}
        for k, v in records.groupby([COLUMN_NAME.IS_INTRA_PROCESS]).items():
            assert len(k) == 1
            group[k[0]] = v
        return group

    @cached_property
    def inter_proc_comm_records(self) -> RecordsInterface:
        """
        Compose inter process communication records.

        Used tracepoints
        - rclcpp_publish
        - dds_bind_addr_to_addr
        - dds_bind_addr_to_stamp
        - rcl_publish (Optional)
        - dds_write (Optional)
        - dispatch_subscription_callback
        - callback_start

        Returns
        -------
        RecordsInterface
            columns
            - callback_object
            - callback_start_timestamp
            - publisher_handle
            - rclcpp_publish_timestamp
            - rcl_publish_timestamp (Optional)
            - dds_write_timestamp (Optional)
            - message_timestamp
            - source_timestamp

        """
        publish = self._data.rclcpp_publish_instances

        publish = merge_sequential_for_addr_track(
            source_records=publish,
            copy_records=self._data.dds_bind_addr_to_addr,
            sink_records=self._data.dds_bind_addr_to_stamp,
            source_stamp_key=COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP,
            source_key=COLUMN_NAME.MESSAGE,
            copy_stamp_key=COLUMN_NAME.DDS_BIND_ADDR_TO_ADDR_TIMESTAMP,
            copy_from_key=COLUMN_NAME.ADDR_FROM,
            copy_to_key=COLUMN_NAME.ADDR_TO,
            sink_stamp_key=COLUMN_NAME.DDS_BIND_ADDR_TO_STAMP_TIMESTAMP,
            sink_from_key=COLUMN_NAME.ADDR,
            columns=[
                COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP,
                COLUMN_NAME.DDS_BIND_ADDR_TO_STAMP_TIMESTAMP,
                COLUMN_NAME.MESSAGE,
                COLUMN_NAME.SOURCE_TIMESTAMP,
            ],
            progress_label='binding: message_addr and rclcpp_publish',
        )

        rcl_publish_records = self._data.rcl_publish_instances
        rcl_publish_records.drop_columns([COLUMN_NAME.PUBLISHER_HANDLE])
        if len(rcl_publish_records) > 0:
            publish = merge_sequential(
                left_records=publish,
                right_records=rcl_publish_records,
                left_stamp_key=COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP,
                right_stamp_key=COLUMN_NAME.RCL_PUBLISH_TIMESTAMP,
                join_left_key=COLUMN_NAME.MESSAGE,
                join_right_key=COLUMN_NAME.MESSAGE,
                how='left',
                columns=[
                    COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP,
                    COLUMN_NAME.RCL_PUBLISH_TIMESTAMP,
                    COLUMN_NAME.PUBLISHER_HANDLE,
                    COLUMN_NAME.MESSAGE,
                    COLUMN_NAME.MESSAGE_TIMESTAMP,
                ],
                progress_label='binding: rclcpp_publish and rcl_publish',
            )

        dds_write = self._data.dds_write_instances
        if len(dds_write) > 0:
            publish = merge_sequential(
                left_records=publish,
                right_records=dds_write,
                left_stamp_key=COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP,
                right_stamp_key=COLUMN_NAME.DDS_WRITE_TIMESTAMP,
                join_left_key=COLUMN_NAME.MESSAGE,
                join_right_key=COLUMN_NAME.MESSAGE,
                columns=Columns(publish.columns + dds_write.columns).as_list(),
                how='left',
                progress_label='binding: rcl_publish and dds_write',
            )

        # When both intra_publish and inter_publish are used, value mismatch occurs when merging.
        # In order to merge at the latency time,
        # align the time to intra_publish if intra_process communication is used.
        intra_publish = self._data.rclcpp_intra_publish_instances.clone()
        intra_publish.drop_columns([
            COLUMN_NAME.MESSAGE, COLUMN_NAME.MESSAGE_TIMESTAMP
        ])
        publish = merge_sequential(
            left_records=intra_publish,
            right_records=publish,
            left_stamp_key=COLUMN_NAME.RCLCPP_INTRA_PUBLISH_TIMESTAMP,
            right_stamp_key=COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP,
            join_left_key=COLUMN_NAME.PUBLISHER_HANDLE,
            join_right_key=COLUMN_NAME.PUBLISHER_HANDLE,
            columns=Columns(intra_publish.columns + publish.columns).as_list(),
            how='right'
        )
        rclcpp_publish = [None] * len(publish.data)
        for i, record in enumerate(publish.data):
            if COLUMN_NAME.RCLCPP_INTRA_PUBLISH_TIMESTAMP in record.data:
                rclcpp_publish[i] = record.data[COLUMN_NAME.RCLCPP_INTRA_PUBLISH_TIMESTAMP]
            else:
                rclcpp_publish[i] = record.data[COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP]
        publish.append_column(
            COLUMN_NAME.RCLCPP_PUBLISH_TIMESTAMP, rclcpp_publish)
        publish.drop_columns([
            COLUMN_NAME.RCLCPP_INTRA_PUBLISH_TIMESTAMP,
            COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP
        ])

        callback_start_instances = self.inter_callback_records
        subscription = self._data.dispatch_subscription_callback_instances

        subscription = merge_sequential(
            left_records=subscription,
            right_records=callback_start_instances,
            left_stamp_key=COLUMN_NAME.DISPATCH_SUBSCRIPTION_CALLBACK_TIMESTAMP,
            right_stamp_key=COLUMN_NAME.CALLBACK_START_TIMESTAMP,
            join_left_key=COLUMN_NAME.CALLBACK_OBJECT,
            join_right_key=COLUMN_NAME.CALLBACK_OBJECT,
            columns=Columns(subscription.columns + callback_start_instances.columns).as_list(),
            how='left',
            progress_label='binding: dispatch_subscription_callback and callback_start',
        )

        # communication = merge(
        #     publish,
        #     self._data.on_data_available_instances,
        #     join_left_key=COLUMN_NAME.SOURCE_TIMESTAMP,
        #     join_right_key=COLUMN_NAME.SOURCE_TIMESTAMP,
        #     columns=publish.columns,
        #     how='left',
        #     progress_label='binding: source_timestamp and on_data_available',
        # )

        communication = merge(
            publish,
            subscription,
            join_left_key=COLUMN_NAME.SOURCE_TIMESTAMP,
            join_right_key=COLUMN_NAME.SOURCE_TIMESTAMP,
            columns=Columns(publish.columns + subscription.columns).as_list(),
            how='left',
            progress_label='binding: source_timestamp and callback_start',
        )

        communication.drop_columns(
            [
                COLUMN_NAME.IS_INTRA_PROCESS,
                COLUMN_NAME.ADDR,
                COLUMN_NAME.MESSAGE,
                COLUMN_NAME.DDS_BIND_ADDR_TO_STAMP_TIMESTAMP,
                COLUMN_NAME.DISPATCH_SUBSCRIPTION_CALLBACK_TIMESTAMP,
            ],
        )

        return communication

    @cached_property
    def publish_records(self) -> RecordsInterface:
        """
        Compose publish records.

        Returns
        -------
        RecordsInterface
            Columns
            - publisher_handle
            - rclcpp_publish_timestamp
            - rcl_publish_timestamp (Optional)
            - dds_write_timestamp (Optional)
            - message_timestamp
            - source_timestamp

        """
        inter_proc_publish = self._data.rclcpp_publish_instances

        rcl_publish_records = self._data.rcl_publish_instances
        rcl_publish_records.drop_columns([COLUMN_NAME.PUBLISHER_HANDLE])
        if len(rcl_publish_records) > 0:
            inter_proc_publish = merge_sequential(
                left_records=inter_proc_publish,
                right_records=rcl_publish_records,
                left_stamp_key=COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP,
                right_stamp_key=COLUMN_NAME.RCL_PUBLISH_TIMESTAMP,
                join_left_key='tid',
                join_right_key='tid',
                how='left',
                columns=[
                    COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP,
                    COLUMN_NAME.RCL_PUBLISH_TIMESTAMP,
                    COLUMN_NAME.PUBLISHER_HANDLE,
                    COLUMN_NAME.MESSAGE,
                    'tid',
                    COLUMN_NAME.MESSAGE_TIMESTAMP,
                ],
                progress_label='binding: rclcpp_publish and rcl_publish',
            )

        dds_write = self._data.dds_write_instances
        if len(dds_write) > 0:
            inter_proc_publish = merge_sequential(
                left_records=inter_proc_publish,
                right_records=dds_write,
                left_stamp_key=COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP,
                right_stamp_key=COLUMN_NAME.DDS_WRITE_TIMESTAMP,
                join_left_key='tid',
                join_right_key='tid',
                columns=Columns(inter_proc_publish.columns + dds_write.columns).as_list(),
                how='left',
                progress_label='binding: rclcppp_publish and dds_write',
            )

        inter_proc_publish = merge_sequential(
            left_records=inter_proc_publish,
            right_records=self._data.dds_bind_addr_to_stamp,
            left_stamp_key=COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP,
            right_stamp_key=COLUMN_NAME.DDS_BIND_ADDR_TO_STAMP_TIMESTAMP,
            join_left_key='tid',
            join_right_key='tid',
            columns=Columns(
                inter_proc_publish.columns + self._data.dds_bind_addr_to_stamp.columns).as_list(),
            how='left',
            progress_label='binding: rclcppp_publish and source_timestamp',
        )

        inter_proc_publish.drop_columns(
            [
                COLUMN_NAME.IS_INTRA_PROCESS,
                COLUMN_NAME.ADDR,
                COLUMN_NAME.MESSAGE,
                COLUMN_NAME.DDS_BIND_ADDR_TO_STAMP_TIMESTAMP,
                COLUMN_NAME.DISPATCH_SUBSCRIPTION_CALLBACK_TIMESTAMP,
            ],
        )

        intra_proc_publish = self._data.rclcpp_intra_publish_instances.clone()
        intra_proc_publish.drop_columns([COLUMN_NAME.MESSAGE])
        # intra_proc_publish.drop_columns([COLUMN_NAME.MESSAGE])

        # When publishing to both intra-process and inter-process,
        # intra-process communication is done first.
        # On the other hand,
        # there are cases where only one or the other is done, so we use outer join.
        # https://github.com/ros2/rclcpp/blob/galactic/rclcpp/include/rclcpp/publisher.hpp#L203
        publish = merge_sequential(
            left_records=intra_proc_publish,
            right_records=inter_proc_publish,
            left_stamp_key=COLUMN_NAME.RCLCPP_INTRA_PUBLISH_TIMESTAMP,
            right_stamp_key=COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP,
            join_left_key=COLUMN_NAME.PUBLISHER_HANDLE,
            join_right_key=COLUMN_NAME.PUBLISHER_HANDLE,
            columns=Columns(inter_proc_publish.columns + intra_proc_publish.columns).as_list(),
            how='outer',
            progress_label='binding intra_publish and inter_publish'
        )

        publish_stamps = []
        maxsize = 2**64-1
        for record in publish.data:
            rclcpp_publish, rclcpp_intra_publish = maxsize, maxsize
            if COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP in record.columns:
                rclcpp_publish = record.get(COLUMN_NAME.RCLCPP_INTER_PUBLISH_TIMESTAMP)
            if COLUMN_NAME.RCLCPP_INTRA_PUBLISH_TIMESTAMP in record.columns:
                rclcpp_intra_publish = record.get(COLUMN_NAME.RCLCPP_INTRA_PUBLISH_TIMESTAMP)
            inter_intra_publish = min(rclcpp_publish, rclcpp_intra_publish)
            publish_stamps.append(inter_intra_publish)

        publish.append_column(COLUMN_NAME.RCLCPP_PUBLISH_TIMESTAMP, publish_stamps)

        columns = []
        columns.append(COLUMN_NAME.PUBLISHER_HANDLE)
        columns.append(COLUMN_NAME.RCLCPP_PUBLISH_TIMESTAMP)
        if COLUMN_NAME.RCL_PUBLISH_TIMESTAMP in publish.columns:
            columns.append(COLUMN_NAME.RCL_PUBLISH_TIMESTAMP)
        if COLUMN_NAME.DDS_WRITE_TIMESTAMP in publish.columns:
            columns.append(COLUMN_NAME.DDS_WRITE_TIMESTAMP)
        columns.append(COLUMN_NAME.MESSAGE_TIMESTAMP)
        columns.append(COLUMN_NAME.SOURCE_TIMESTAMP)

        drop_columns = set(publish.columns) - set(columns)
        publish.drop_columns(list(drop_columns))
        publish.reindex(columns)

        return publish

    def create_timer_events_factory(
        self,
        timer_callback: TimerCallbackValueLttng
    ) -> EventsFactory:
        """
        Create tiemr events factory.

        Parameters
        ----------
        timer_callback : TimerCallbackValueLttng
            target callback to create timer events.

        Returns
        -------
        EventsFactory

        """
        class TimerEventsFactory(EventsFactory):

            def __init__(self, ctrls: Sequence[TimerControl]) -> None:
                self._ctrls = ctrls

            def create(self, until_ns: int) -> RecordsInterface:

                columns = [
                    COLUMN_NAME.TIMER_EVENT_TIMESTAMP,
                ]

                records = RecordsFactory.create_instance(None, columns)
                for ctrl in self._ctrls:

                    if isinstance(ctrl, TimerInit):
                        ctrl._timestamp
                        timer_timestamp = ctrl._timestamp
                        while timer_timestamp < until_ns:
                            record_dict = {
                                            COLUMN_NAME.TIMER_EVENT_TIMESTAMP: timer_timestamp,
                            }
                            record = RecordFactory.create_instance(record_dict)
                            records.append(record)
                            timer_timestamp = timer_timestamp+ctrl.period_ns

                return records

        timer_ctrls = self._info.get_timer_controls()

        filtered_timer_ctrls = Util.filter_items(
            lambda x: x.timer_handle == timer_callback.timer_handle, timer_ctrls)

        return TimerEventsFactory(filtered_timer_ctrls)

    @cached_property
    def tilde_publish_records(self) -> RecordsInterface:
        """
        Compose tilde publish records.

        Returns
        -------
        RecordsInterface
            columns:
            - tilde_publish_timestamp
            - tilde_publisher
            - tilde_message_id
            - tilde_subscription

        """
        records = self._data.tilde_publish
        records.rename_columns({'publisher': 'tilde_publisher'})

        subscription: List[int] = []
        for record in records:
            subscription_id = record.get('subscription_id')
            subscription.append(self._info.tilde_sub_id_map[subscription_id])

        records.append_column('tilde_subscription', subscription)
        records.drop_columns(['subscription_id'])
        return records

    @cached_property
    def tilde_subscribe_records(self) -> RecordsInterface:
        """
        Compose tilde subscribe records.

        Returns
        -------
        RecordsInterface
            columns:
            - tilde_subscribe_timestamp
            - tilde_subscription
            - tilde_message_id

        """
        records = self._data.tilde_subscribe
        records.rename_columns({'subscription': 'tilde_subscription'})
        return records

    @cached_property
    def intra_callback_records(self) -> RecordsInterface:
        intra_proc_subscribe = RecordsFactory.create_instance(
            None,
            ['callback_start_timestamp', 'callback_object', 'is_intra_process']
        )
        if 1 in self._grouped_callback_start:
            intra_callback_start = self._grouped_callback_start[1].clone()
            intra_proc_subscribe.concat(intra_callback_start)
        return intra_proc_subscribe

    @cached_property
    def inter_callback_records(self) -> RecordsInterface:
        intra_proc_subscribe = RecordsFactory.create_instance(
            None,
            ['callback_start_timestamp', 'callback_object', 'is_intra_process']
        )
        if 0 in self._grouped_callback_start:
            intra_callback_start = self._grouped_callback_start[0].clone()
            intra_proc_subscribe.concat(intra_callback_start)
        return intra_proc_subscribe

    @cached_property
    def subscribe_records(self) -> RecordsInterface:
        callback_start_instances = self.inter_callback_records
        inter_proc_subscrube = self._data.dispatch_subscription_callback_instances

        inter_proc_subscrube = merge_sequential(
            left_records=inter_proc_subscrube,
            right_records=callback_start_instances,
            left_stamp_key=COLUMN_NAME.DISPATCH_SUBSCRIPTION_CALLBACK_TIMESTAMP,
            right_stamp_key=COLUMN_NAME.CALLBACK_START_TIMESTAMP,
            join_left_key=COLUMN_NAME.CALLBACK_OBJECT,
            join_right_key=COLUMN_NAME.CALLBACK_OBJECT,
            columns=Columns(
                inter_proc_subscrube.columns + callback_start_instances.columns).as_list(),
            how='left',
            progress_label='binding: dispatch_subscription_callback and callback_start',
        )

        intra_proc_subscribe = self.intra_callback_records

        subscribe = merge_sequential(
            left_records=inter_proc_subscrube,
            right_records=intra_proc_subscribe,
            left_stamp_key=COLUMN_NAME.CALLBACK_START_TIMESTAMP,
            right_stamp_key=COLUMN_NAME.CALLBACK_START_TIMESTAMP,
            join_left_key=COLUMN_NAME.CALLBACK_OBJECT,
            join_right_key=COLUMN_NAME.CALLBACK_OBJECT,
            columns=Columns(inter_proc_subscrube.columns + intra_proc_subscribe.columns).as_list(),
            how='outer',
            progress_label='binding intra and inter subscribe'
        )

        return subscribe

    @cached_property
    def intra_proc_comm_records(self) -> RecordsInterface:
        """
        Compose intra process communication records.

        Used tracepoints
        - dispatch_intra_process_subscription_callback
        - rclcpp_publish
        - message_construct
        - callback_start

        Returns
        -------
        RecordsInterface
            columns:
            - callback_object
            - callback_start_timestamp
            - publisher_handle
            - rclcpp_publish_timestamp
            - message_timestamp

        """
        sink_records = self._data.dispatch_intra_process_subscription_callback_instances
        intra_publish_records = merge_sequential_for_addr_track(
            source_records=self._data.rclcpp_intra_publish_instances,
            copy_records=self._data.message_construct_instances,
            sink_records=sink_records,
            source_stamp_key=COLUMN_NAME.RCLCPP_INTRA_PUBLISH_TIMESTAMP,
            source_key=COLUMN_NAME.MESSAGE,
            copy_stamp_key=COLUMN_NAME.MESSAGE_CONSTRUCT_TIMESTAMP,
            copy_from_key=COLUMN_NAME.ORIGINAL_MESSAGE,
            copy_to_key=COLUMN_NAME.CONSTRUCTED_MESSAGE,
            sink_stamp_key=COLUMN_NAME.DISPATCH_INTRA_PROCESS_SUBSCRIPTION_CALLBACK_TIMESTAMP,
            sink_from_key=COLUMN_NAME.MESSAGE,
            columns=[
                'tid',
                COLUMN_NAME.DISPATCH_INTRA_PROCESS_SUBSCRIPTION_CALLBACK_TIMESTAMP,
                COLUMN_NAME.PUBLISHER_HANDLE,
                COLUMN_NAME.CALLBACK_OBJECT,
                COLUMN_NAME.RCLCPP_INTRA_PUBLISH_TIMESTAMP,
                COLUMN_NAME.MESSAGE_TIMESTAMP,
            ],
            progress_label='bindig: publish_timestamp and message_addr',
        )

        # note: Incorrect latency is calculated when intra_publish of ros-rclcpp is included.
        intra_publish_records = merge(
            left_records=intra_publish_records,
            right_records=sink_records,
            join_left_key=COLUMN_NAME.DISPATCH_INTRA_PROCESS_SUBSCRIPTION_CALLBACK_TIMESTAMP,
            join_right_key=COLUMN_NAME.DISPATCH_INTRA_PROCESS_SUBSCRIPTION_CALLBACK_TIMESTAMP,
            columns=intra_publish_records.columns + [COLUMN_NAME.MESSAGE],
            how='inner'
        )

        intra_publish_records = merge_sequential(
            left_records=intra_publish_records,
            right_records=sink_records,
            left_stamp_key=COLUMN_NAME.DISPATCH_INTRA_PROCESS_SUBSCRIPTION_CALLBACK_TIMESTAMP,
            right_stamp_key=COLUMN_NAME.DISPATCH_INTRA_PROCESS_SUBSCRIPTION_CALLBACK_TIMESTAMP,
            join_left_key=COLUMN_NAME.MESSAGE,
            join_right_key=COLUMN_NAME.MESSAGE,
            columns=intra_publish_records.columns,
            how='left_use_latest'
        )

        callback_start_instances = self.intra_callback_records

        intra_records = merge_sequential(
            left_records=intra_publish_records,
            right_records=callback_start_instances,
            left_stamp_key=COLUMN_NAME.DISPATCH_INTRA_PROCESS_SUBSCRIPTION_CALLBACK_TIMESTAMP,
            right_stamp_key=COLUMN_NAME.CALLBACK_START_TIMESTAMP,
            join_left_key=COLUMN_NAME.CALLBACK_OBJECT,
            join_right_key=COLUMN_NAME.CALLBACK_OBJECT,
            columns=intra_publish_records.columns + [
                COLUMN_NAME.CALLBACK_START_TIMESTAMP, COLUMN_NAME.IS_INTRA_PROCESS],
            how='left',
            progress_label='bindig: dispath_subsription and callback_start',
        )

        intra_records.drop_columns(
            [
                COLUMN_NAME.DISPATCH_INTRA_PROCESS_SUBSCRIPTION_CALLBACK_TIMESTAMP,
                COLUMN_NAME.MESSAGE,
                COLUMN_NAME.IS_INTRA_PROCESS
            ]
        )
        intra_records.rename_columns(
            {
                COLUMN_NAME.RCLCPP_INTRA_PUBLISH_TIMESTAMP: COLUMN_NAME.RCLCPP_PUBLISH_TIMESTAMP
            }
        )

        return intra_records

    @cached_property
    def callback_records(self) -> RecordsInterface:
        """
        Compose callback records.

        Used tracepoints
        - callback_start
        - callback_end

        Returns
        -------
        RecordsInterface
            columns:
            - callback_start_timestamp
            - callback_end_timestamp
            - callback_object

        """
        records: RecordsInterface

        records = merge_sequential(
            left_records=self._data.callback_start_instances,
            right_records=self._data.callback_end_instances,
            left_stamp_key=COLUMN_NAME.CALLBACK_START_TIMESTAMP,
            right_stamp_key=COLUMN_NAME.CALLBACK_END_TIMESTAMP,
            join_left_key=COLUMN_NAME.CALLBACK_OBJECT,
            join_right_key=COLUMN_NAME.CALLBACK_OBJECT,
            columns=[
                COLUMN_NAME.CALLBACK_START_TIMESTAMP,
                COLUMN_NAME.CALLBACK_END_TIMESTAMP,
                COLUMN_NAME.CALLBACK_OBJECT,
                COLUMN_NAME.IS_INTRA_PROCESS,
            ],
            how='inner',
            progress_label='binding: callback_start and callback_end'
        )

        records.drop_columns(
            [
                COLUMN_NAME.IS_INTRA_PROCESS
            ]
        )

        return records

    @cached_property
    def system_and_sim_times(self) -> RecordsInterface:
        return self._data.sim_time


import cv2
import torch
import sys
import os
sys.path.append('yolov5')
from models.experimental import attempt_load
from utils.general import non_max_suppression, xyxy2xywh
from utils.torch_utils import select_device

# Load YOLOv5 model
weights = 'yolov5s.pt'
device = select_device('')
model = attempt_load(weights, device)
model.eval()

# Set threshold for object detection
conf_thresh = 0.3

# Load class names for dataset
class_names_file = os.path.join('yolov5', 'data', 'coco.names')
with open(class_names_file, 'r') as f:
    class_names = f.read().splitlines()

# Open video stream or image file
cap = cv2.VideoCapture(0) # or replace 0 with the path to your image file

while True:
    # Read a frame from the video stream or image file
    ret, frame = cap.read()

    # Resize the image to match the expected input size of the model
    img = cv2.resize(frame, (640, 640))

    # Convert the image to a tensor
    img = torch.from_numpy(img).to(device).float() / 255.0
    img = img.permute(2, 0, 1).unsqueeze(0)

    # Run inference on the image
    with torch.no_grad():
        pred = model(img, augment=False)[0]

    # Apply non-maximum suppression to remove overlapping bounding boxes
    pred = non_max_suppression(pred, conf_thresh, 0.4)

    # Display the bounding boxes and class labels
    if pred is not None:
        for det in pred[0]:
            if det is not None and det[-1] > conf_thresh:
                x1, y1, x2, y2, conf, cls = det
                x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
                cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 255, 0), 2)
                cls_name = class_names[int(cls)]
                label = f'{cls_name}: {conf:.2f}'
                cv2.putText(frame, label, (x1, y1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)

    # Display the resulting image
    cv2.imshow('frame', frame)

    # Wait for a key press and check if 'q' was pressed to quit
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# Release the video stream or image file and close all windows
cap.release()
cv2.destroyAllWindows()


import data_util
import correlation

import pandas as pd
import numpy as np

import datetime
import pickle
import itertools
import time
import sys
import math
import warnings
import multiprocessing as mp
import os

warnings.simplefilter("error")

ctr = 0
tot = 0

NUM_CORES = 8

def analyse_corr(corr_pkl_file):
    with open(corr_pkl_file, "rb") as f:
        corr_list = pickle.load(f)
        corr_list = [list_item for list_item in corr_list if len(list_item[1]) != 0]
        for item in sorted(corr_list, key=lambda x: np.mean(list(map(lambda y: y[1], x[1])))):
            print(item)
        # print(max(map(lambda x: np.mean(x), corr_dict.values())))

def mp_get_corr(args):
    df1, df2 = args
    global ctr
    global tot
    ctr += 1
    if ctr % 100 == 0:
        print(ctr / tot * 100, os.getpid())
    return tuple(sorted((df1[data_util.KEY_SYMBOL].iloc[-1],
                         df2[data_util.KEY_SYMBOL].iloc[-1]))), correlation.get_correlation(df1, df2)

def main():
    global ctr
    global tot

    symbols_read_time = time.time()

    with open("symbols_df.pkl", "rb") as pkl_file:
        symbol_dfs = pickle.load(pkl_file)

    print("Time taken to read the symbols df pickle", time.time() - symbols_read_time, file=sys.stderr)
    symbol_dfs = [symbol_df for symbol_df in symbol_dfs if len(symbol_df.index) > 0]
    combs = itertools.combinations(symbol_dfs, 2)
    tot = len(symbol_dfs) * (len(symbol_dfs) - 1) // 2
    ctr = 0

    results = mp.Pool(processes=NUM_CORES).map(func=mp_get_corr, iterable=list(combs))
    # print(results)

    with open("avg_max_corrs.pkl", "wb") as pkl_file:
        pickle.dump(results, pkl_file)

if __name__ == '__main__':
    # main()

    analyse_corr("avg_max_corrs.pkl")


# %%
from os import path
import struct
import png

from lief import PE

from utils import align, compile, windres

src_dir = path.dirname(__file__)

# %%
# compile origin demo program
compile(path.join(src_dir, 'example.c'), '-m32 -O2 -o example.exe')
binary = PE.parse('example.exe')
print('[+] compile origin demo program success.')

# %%
IMG_PATH = 'packer5-packed.png'
ROW_LEN = 256
with open('example.exe', 'rb') as f:
    arr = []
    content = f.read()
    content = struct.pack('<I', len(content))+content

    for i in range(len(content)//ROW_LEN):
        t = content[i*ROW_LEN:i*ROW_LEN+ROW_LEN]
        arr.append(t)

    img = png.from_array(arr, 'L').save(IMG_PATH)

# %%
windres(path.join(src_dir, 'rsrc.rc'), path.join(src_dir, 'rsrc.o'))
print('[+] compile resource success.')

# %%
# compile shifted loader program
cflags = [
    '-m32',
    '-O2',
    '-Wall',
    '-I.',
    '-Wl,--entry=__start',
    '-nodefaultlibs',
    '-nostartfiles',
    '-lkernel32',
    '-luser32',
    '-lmsvcrt',
    '-lpng',
    '-o',
    'packed.exe'
]
compile([path.join(src_dir, src) for src in ['loader.c', 'png_decode.c', 'rsrc.o']], cflags)
print('[+] compile loader with resource success.')


# -*- coding: utf-8 -*-
"""
 Created on 2021/4/7 20:51
 Author: ff_wang
 E-mail: 1105936347@qq.com
 Zhihu : https://www.zhihu.com/people/1105936347
 Github: https://github.com/AFei19911012
-------------------------------------------------
"""
# Source: https://github.com/spmallick/learnopencv

from imutils import perspective
from skimage.filters import threshold_local
import cv2 as cv
import imutils
import matplotlib.pyplot as plt

# 读取图像
image = cv.imread(r'images/dog.jpg')
# 比例
orig = image.copy()
plt.figure('opencv contour')
plt.subplot(1, 3, 1)
plt.imshow(imutils.opencv2matplotlib(orig))
plt.axis('off')
plt.title('original')
# 灰度转化
gray_image = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
# 滤波
gray_image = cv.GaussianBlur(gray_image, (5, 5), 0)
# gray_image = cv.bilateralFilter(gray_image, 11, 17, 17)
# 二值化
# ret, gray_image = cv.threshold(gray_image, 100, 255, 0)
# 边缘检测
canny_image = cv.Canny(gray_image, 120, 200)
plt.subplot(1, 3, 2)
plt.imshow(imutils.opencv2matplotlib(canny_image))
plt.axis('off')
plt.title('Canny')
# 查找轮廓
contours = cv.findContours(canny_image.copy(), cv.RETR_LIST, cv.CHAIN_APPROX_SIMPLE)
# 保留最大轮廓
contours = imutils.grab_contours(contours)
contours = sorted(contours, key=cv.contourArea, reverse=True)[:5]
# 在原图上画轮廓
cv.drawContours(orig, contours, -1, (255, 0, 255), 2)
# 在原图上画矩形轮廓
for i in range(0, len(contours)):
    x, y, w, h = cv.boundingRect((contours[i]))
    cv.rectangle(orig, (x, y), (x+w, y+h), (255, 255, 0), 3)

plt.subplot(1, 3, 3)
plt.imshow(imutils.opencv2matplotlib(orig))
plt.axis('off')
plt.title('contour')
plt.show()


'''Trie.'''

class Trie:
    def __init__(self):
        self.root = _Node(None)

    def insert(self, word):
        current = self.root
        for char in word:
            if not current.has_child(char):
                current.add_child(char)
            current = current.get_child(char)
        current.is_end_of_word = True

    def contains(self, word):
        if word is None: return False

        current = self.root
        for char in word:
            if not current.has_child(char):
                return False
            current = current.get_child(char)
        return current.is_end_of_word

    def contains_recursive(self, word):
        def _contains(node, word):
            if len(word) == 1:
                return node.has_child(word[0])
            if not node.has_child(word[0]):
                return False
            return _contains(node.get_child(word[0]), word[1:])
        return _contains(self.root, word)

    def remove(self, word):
        def _remove(node, word):
            child = node.get_child(word[0])
            if child is None: return

            if len(word) == 1:
                child.is_end_of_word = False
            else:
                _remove(child, word[1:])
                if not child.is_end_of_word and not child.has_children():
                    node.remove_child(child.value)

        _remove(self.root, word)

    def traverse(self):
        def _traverse(node):
            if node.value is not None:
                print(node.value)
            for child in node.get_children():
                _traverse(child)
        _traverse(self.root)

    def count_words(self):
        def _count(node):
            total = 0
            if node.is_end_of_word:
                total += 1
            for child in node.get_children():
                total += _count(child)
            return total
        return _count(self.root)

    def longest_common_prefix(self, lst):
        maxlength = len(self.get_shortest(lst))
        trie = Trie()
        for word in lst:
            trie.insert(word)

        prefix = ''
        node = trie.root
        while len(prefix) <= maxlength:
            children = node.get_children()
            if len(children) != 1:
                return prefix
            node = children[0]
            prefix += node.value

    def get_shortest(self, lst):
        from functools import reduce
        f = lambda s1, s2: s1 if len(s1) < len(s2) else s2
        return reduce(f, lst)

    def autocomplete(self, word):
        lastnode = self.get_last_node(word)
        def _auto(node, word, lst):

            if node is None:
                return []

            if node.is_end_of_word:
                lst.append(word)

            for child in list(node.get_children()):
                _auto(child, word + child.value, lst)

            return lst

        return _auto(lastnode, word, lst=[])

    def get_last_node(self, word):
        if word is None:
            return
        node = self.root
        for char in word:
            node = node.get_child(char)
            if node is None:
                return
        return node

    def __repr__(self):
        pass

class _Node:

    def __init__(self, value=None):
        self.value = value  # char
        self.children = {}  # {char: Node}
        self.is_end_of_word = False

    def has_child(self, char):
        return char in self.children

    def add_child(self, char):
        self.children[char] = _Node(char)

    def remove_child(self, char):
        del self.children[char]

    def get_child(self, char):
        return self.children.get(char)

    def has_children(self):
        return len(self.children) > 0

    def get_children(self):
        return list(self.children.values())

    def __repr__(self):
        return f'{self.value}'

def main():
    print()
    words = ['car']
    words = ['canada', 'canadian']
    words = ['card', 'care', 'careful', 'carefree']
    words = ['dog', 'egg']

    trie = Trie()
    longest = trie.longest_common_prefix(words)
    print(f'\n\t longest common prefix in the trie: {longest} \n')
    quit()

    #####################################################################
    #words = ['a', 'ask', 'asked', 'asker', 'asking']
    print('\n\t the words inserted into the trie: ')
    print('\t', words)
    trie = Trie()
    for word in words:
        trie.insert(word)

    #####################################################################
    print(f'\n\t trie contains {trie.count_words()} words! \n')

    wd = 'eggs'
    print(f'\n\t trie contains {wd}: ', trie.contains_recursive(wd), '\n')

    lst = trie.autocomplete(wd)
    print(f' \n\t autocompleting {wd}: \t {lst}')
    print()

    #trie.traverse()
    #quit()
    #word1 = 'asked'
    #print(f'\n\t  removing ** {word1} ** ..\n')
    #trie.remove(word1)
    #for w in ['a', 'ask', 'aske', 'asked', 'asker', 'asking']:
    #    print(f'trie contains {w}: '.rjust(25), trie.contains(w))

    #print()

if __name__ == '__main__':
    main()


# # Linear Regression

# ## Importing the libraries

# In[61]:

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn import linear_model
import seaborn as sns
sns.set()
from sklearn.model_selection import train_test_split

# ## Importing the data and preprocessing

# In[62]:

# Importing the dataset
dataset = pd.read_csv('C:/Users/Harry/Desktop/Analytics/TSLA_stock.csv')
X = dataset.iloc[:, :1].values
y = dataset.iloc[:, 4:5].values

# Splitting the dataset into the Training set and Test set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 1/3, random_state = 0)

#View the data
dataset.head()

# ## Choosing the model

# In[63]:

from sklearn.linear_model import LinearRegression

regressor = linear_model.LinearRegression()

# ## Fitting the model and predicting the result

# In[79]:

# Fitting Simple Linear Regression to the Training set
regressor.fit(X_train, y_train)

# Predicting the Test set results
y_pred = regressor.predict(X_test)

# ## Checking the accuracy of the model

# In[80]:

acc = regressor.score(X_test, y_test)
print("Accuracy: " + str(acc))

# ## Plotting the graph

# In[83]:

plt.rcParams["figure.figsize"] = (20,8)
plt.scatter(X_train, y_train, color = 'red', label="Actual")
plt.plot(X_test,y_pred, color='blue', label="Linear Regression")

plt.title('Tesla Stock Prices 2015-Today')
plt.xlabel('Days')
plt.ylabel('Close Price')
plt.legend()
plt.show()


from django.shortcuts import render
from main.models import Message
from django.contrib.auth.decorators import login_required, user_passes_test
from main.forms import MessageForm
from django.http import HttpResponse

import user_auth
from user_auth import is_admin

@login_required
def front(request):

    # posts = Post.objects.all().order_by('-date_posted')

    context = {}

    if request.method == 'POST':

        # id = request.POST.get('id')

        # if id:

        #     message = Message.objects.get(id=id)
        #     form = PostForm(request.POST, request.FILES, instance=post)

        # else:

        form = MessageForm(request.POST)

        if is_admin(request.user):
            context['permission'] = 'is_admin'

        if form.is_valid():
            message = form.save()
            context['result'] = 'Your post has been saved'
        else:
            context['result'] = form.errors

    return render(request, 'index.html', context)

@login_required
def all_messages(request):
    context = {}
    messages = Message.objects.all().order_by('-date_posted')
    context['messages'] = messages

    if is_admin(request.user):
            context['permission'] = 'is_admin'

    return render(request, 'all_messages.html', context)

def edit_message(request, id):

    if request.method == 'DELETE':

        if request.user.is_superuser or is_admin(request.user):

            Message.objects.get(id=id).delete()

            return HttpResponse(status=204)
        else:
            return HttpResponse(status=401)

    elif request.method == 'GET':
        message = Message.objects.get(id=id)

        return render(request, 'message.html', {'message': message})


import csv
import datetime
import os.path
import json
from os import listdir

# directory of tsv files
inputdir = os.path.join(os.getcwd(), 'inputfiles')

# Input start date from the user
startDateInput = input("Enter start date in the format yyyy/mm/dd: ")
startDateInputArray = startDateInput.split("/")
startDate = datetime.datetime(int(startDateInputArray[0]), int(startDateInputArray[1]), int(startDateInputArray[2]))

# check if target directory exists, else create one
os.chdir('../mysite/dataengineering/static/dataengineering')
targetdir = os.getcwd()
if os.path.isdir(targetdir) is False:
    print("target directory does not exist.")
    exit(1)

last_day = 0
# create map of file names and the number of records
dict = {}
# read data from .tsv files
for filename in listdir(inputdir):
    print('Reading data from file', filename)
    tsv_file = open(os.path.join(inputdir, filename), 'r')
    read_tsv = csv.reader(tsv_file, delimiter="\t")

    # skip the headers
    next(read_tsv, None)
    for row in read_tsv:
        if row[2] is None:
            continue
        recordDate = datetime.datetime.strptime(row[2], '%d-%b-%y')
        diff = (recordDate - startDate).days + 1
        last_day = max(last_day, diff)
        count = dict.get(diff, 0)
        dict[diff] = count+1
    tsv_file.close()

print('Data Read Completed from tsv files. Comparing now with json files..')
for key in range(1, last_day+1):
    targetfile = os.path.join(targetdir, 'day' + str(key) + '.json')
    # read data from file in json format
    with open(targetfile, 'r') as jsonfile:
        j = json.load(jsonfile)
        records_in_json_file = len(j)
        records_in_tsv = dict.get(key, 0)
        if records_in_json_file != records_in_tsv:
            print("day {0} missing {1} records".format(key, abs(records_in_json_file - records_in_tsv)))

print("All Files checked successfully!")


# https://leetcode.com/problems/median-of-two-sorted-arrays/
# tags: #array, #binary_search, #divide_and_conquer
#
# Optimal solution (if you dare):
# https://leetcode.com/problems/median-of-two-sorted-arrays/discuss/2481/Share-my-O(log(min(mn)))-solution-with-explanation
#
# Solution: Find the median by traversing
# Time Complexity: O(m+n), Space complexity: O(1)
from typing import List

class Solution:
    def findMedianSortedArrays(self, nums1: List[int], nums2: List[int]) -> float:
        n, m = len(nums1), len(nums2)
        ia, ma, ib, mb = 0, 0, 0, 0

        for i in range((n + m) // 2 + 1):
            ma = mb

            if ia == n:
                mb = nums2[ib]
                ib += 1
            elif ib == m:
                mb = nums1[ia]
                ia += 1
            elif nums1[ia] < nums2[ib]:
                mb = nums1[ia]
                ia += 1
            else:  # nums1[ia] >= nums2[ib]
                mb = nums2[ib]
                ib += 1

        if (n + m) % 2 == 0:
            return (ma + mb) / 2

        return mb

if __name__ == "__main__":
    sol = Solution()
    print(sol.findMedianSortedArrays(nums1=[1, 2], nums2=[3, 4]))  # 2.5


import numpy as np
import pandas as pd
from subprocess import check_output
#print(check_output(["ls", "../input"]).decode("utf8"))
import datetime
import time
destinations = pd.read_csv("../input/destinations.csv")
train1 = pd.read_csv("../input/train.csv", nrows=100000)
test1 = pd.read_csv("../input/test.csv", nrows = 100)
train2 = pd.read_csv("../input/train.csv", nrows=100000)
train3 = pd.read_csv("../input/train.csv", nrows=200002)
train4.info()
train4 = train3.ix[100001:200002,:]
train1["date_time"] = pd.to_datetime(train1["date_time"])
train1["year"] = train1["date_time"].dt.year
train1["month"] = train1["date_time"].dt.month
train1 = train1[train1.is_booking == True]
train4 = train4[train4.is_booking == True]
train4["date_time"] = pd.to_datetime(train4["date_time"])
train4["year"] = train4["date_time"].dt.year
train4["month"] = train4["date_time"].dt.month
from sklearn.decomposition import PCA
X = destinations.ix[:,1:150]
y = destinations.ix[:,0:1]
pca = PCA(n_components=3)
X_r = pca.fit(X).transform(X)
X_r1 = pd.DataFrame(X_r)
X_r1["srch_destination_id"] = destinations["srch_destination_id"]
print('explained variance ratio (first 3 components): %s' % str(pca.explained_variance_ratio_))
def calc_fast_features(df):
    df["date_time"] = pd.to_datetime(df["date_time"])
    df["srch_ci"] = pd.to_datetime(df["srch_ci"], format='%Y-%m-%d', errors="coerce")
    df["srch_co"] = pd.to_datetime(df["srch_co"], format='%Y-%m-%d', errors="coerce")

    props = {}
    for prop in ["month", "day", "hour", "minute", "dayofweek", "quarter"]:
        props[prop] = getattr(df["date_time"].dt, prop)

    carryover = [p for p in df.columns if p not in ["date_time", "srch_ci", "srch_co"]]
    for prop in carryover:
        props[prop] = df[prop]

    date_props = ["month", "day", "dayofweek", "quarter"]
    for prop in date_props:
        props["ci_{0}".format(prop)] = getattr(df["srch_ci"].dt, prop)
        props["co_{0}".format(prop)] = getattr(df["srch_co"].dt, prop)
    props["stay_span"] = (df["srch_co"] - df["srch_ci"]).astype('timedelta64[h]')

    ret = pd.DataFrame(props)

    ret = ret.join(X_r1, on="srch_destination_id", how='left', rsuffix="dest")
    ret = ret.drop("srch_destination_iddest", axis=1)
    return ret
df = calc_fast_features(train2)
df.fillna(-1, inplace=True)
predictors = [c for c in df.columns if c not in ["hotel_cluster"]]
from sklearn import cross_validation
from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier(n_estimators=10, min_weight_fraction_leaf=0.1)
scores = cross_validation.cross_val_score(clf, df[predictors], df['hotel_cluster'], cv=3)
scores
from sklearn.ensemble import RandomForestClassifier
from sklearn.cross_validation import KFold
from itertools import chain
all_probs = []
unique_clusters = df["hotel_cluster"].unique()
prediction_frame = pd.DataFrame(all_probs).T
prediction_frame.columns = unique_clusters
import ml_metrics as metrics
def find_top_5(row):
    return list(row.nlargest(5).index)
preds = [ ]
for index, row in prediction_frame.iterrows():
    preds.append(find_top_5(row))
metrics.mapk([[l] for l in train4["hotel_cluster"]], preds, k=5)
test1["srch_ci"] = pd.to_datetime(test1["srch_ci"], format='%Y-%m-%d', errors="coerce")
test1["srch_co"] = pd.to_datetime(test1["srch_co"], format='%Y-%m-%d', errors="coerce")
test1["stay_span"] = (test1["srch_co"] - test1["srch_ci"]).astype('timedelta64[D]')
test2 = pd.to_datetime(test1["date_time"])
test2 = pd.DataFrame(test2)
test1['year'] = test2['date_time'].dt.year
test1['month'] = test2['date_time'].dt.month
test1['day_of_week'] = test2['date_time'].dt.dayofweek
test1['day'] = test2['date_time'].dt.day
test1['hour'] = test2['date_time'].dt.hour
test1["date_time"] = pd.to_datetime(test1["date_time"], format='%Y-%m-%d', errors="coerce")
test1["srch_ci"] = pd.to_datetime(test1["srch_ci"], format='%Y-%m-%d', errors="coerce")
test1["search_span"] = (test1["srch_ci"] - test1["date_time"]).astype('timedelta64[D]')
test1.ix[(test1['hour'] >= 10) & (test1['hour'] < 18), 'hour'] = 1
test1.ix[(test1['hour'] >= 18) & (test1['hour'] < 22), 'hour'] = 2
test1.ix[(test1['hour'] >= 22) & (test1['hour'] == 24), 'hour'] = 3
test1.ix[(test1['hour'] >= 1) & (test1['hour'] < 10), 'hour'] = 3
test1 = test1.drop('srch_ci', axis=1)
test1 = test1.drop('srch_co', axis=1)
test1 = test1.drop('date_time', axis=1)
test1.fillna(-1, inplace=True)
test1.info()
metrics.mapk([[l] for l in test1["stay_span"]], preds, k=5)


import pandas as pd
import numpy as np
from sklearn.linear_model import LinearRegression
import statsmodels.api as sm
from statsmodels.stats.outliers_influence import variance_inflation_factor
import scipy.stats as stats
import matplotlib.pyplot as plt
import seaborn as sns

#%% independent variables

path_csv = pd.read_csv('code/video_audio_paths.csv', index_col=0)
video_index = path_csv['orgPath_trun'].notna()
audio_index = path_csv['audio_path'].notna()

text_features = pd.read_csv('code/03-analysis_texts/analysis_texts_results.csv', index_col=0)
text_features['Text-Warmth'] = text_features['warmth'] / text_features['len_tokens']
text_features['Text-Ability'] = text_features['ability'] / text_features['len_tokens']
text_features['Text-Positive'] = text_features['positive'] / text_features['len_tokens']
text_features['Text-Negative'] = text_features['negative'] / text_features['len_tokens']

audio_features = pd.read_csv('code/02-analysis_audio/analysis_audio_results.csv', index_col=0)
audio_features.loc[audio_index, :] = audio_features.loc[audio_index, :].fillna(0)

video_features = pd.read_csv('code/01-analysis_video/analysis_video_results.csv', index_col=0)
video_features.loc[video_index, :] = video_features.loc[video_index, :].fillna(0)
video_features['Video-Positive'] = video_features['happy'] / 100
video_features['Video-Negative'] = video_features[['angry', 'sad', 'fear', 'disgust']].sum(axis=1) / 100

temp = pd.merge(text_features, audio_features, on='realpath')
ind_var = pd.merge(temp, video_features, on='realpath')
ind_var

#%% dependent variables

cover_info = pd.read_excel('kickstarter(total)/1cover&2advertising_page.xlsx')
update_info = pd.read_excel('kickstarter(total)/5update_page.xlsx')

# done_amount
amount = cover_info[['<realpath>', '已筹金额', '目标金额']].copy()
amount.columns = ['realpath', 'done_amount', 'goal_amount']
def remove_nonDigit(s):
    s_prime = ''
    for i in s:
        if i.isnumeric() is True:
            s_prime += i
    return s_prime
amount['done_amount'] = amount['done_amount'].apply(remove_nonDigit)
amount['goal_amount'] = amount['goal_amount'].apply(remove_nonDigit)
amount[['done_amount', 'goal_amount']] = amount[['done_amount', 'goal_amount']].astype(int)
amount['status'] = (amount['done_amount'] >= amount['goal_amount']).astype(int)

# investing population
population = cover_info[['<realpath>', '支持者人数']].copy()
population.columns = ['realpath', 'population']
population['population'] = population['population'].apply(remove_nonDigit)
population['population'] = population['population'].astype(int)

# posterior success
pos_url = cover_info[['<realpath>', '成功项目主页URL']].copy()
pos_url.columns = ['realpath', 'url']
pd.isna(pos_url.iloc[311, 1])
pos_url['pos_suc_url'] = (pos_url['url'].apply(pd.isna)).astype(int)

temp = pd.merge(amount, population, on='realpath')
dep_var = pd.merge(temp, pos_url, on='realpath')
dep_var.columns

#%% control variables

# founder info
founder_csv = pd.read_excel('kickstarter(total)/7founder_page.xlsx')
founder_hist = founder_csv[['<realpath>', '业绩']].copy()
def remove_split(s):
    s_list = s.split()
    nums = []
    for i in s_list:
        if i == 'First':
            nums.append(0)
            continue
        try:
            num = int(i)
            nums.append(num)
        except:
            continue
    return nums
founder_hist['nums'] = founder_hist.iloc[:,1].apply(remove_split)
founder_hist['created_num'] = founder_hist['nums'].apply(lambda x: x[0])
founder_hist['backed_num'] = founder_hist['nums'].apply(lambda x: x[1])
founder_hist = founder_hist.groupby('<realpath>').mean()

founder_coll = founder_csv[['<realpath>', '协作者']].copy()
founder_coll = founder_coll.groupby('<realpath>').count()
founder_coll.columns = ['coll_num']

founder_info = pd.merge(founder_hist, founder_coll, right_index=True, left_index=True)
founder_info = founder_info.reset_index()
founder_info.columns = ['realpath'] + list(founder_info.columns[1:])

# %% summary statistics
temp = pd.merge(dep_var, ind_var, on='realpath')
df_reg = pd.merge(temp, founder_info, on='realpath', how='left')
pitch_features = ['Text-Warmth', 'Text-Ability', 'Text-Positive', 'Text-Negative'
                 , 'Vocal-Arousal', 'Vocal-Valence', 'Vocal-Positive', 'Vocal-Negative'
                 , 'Video-Positive', 'Video-Negative'
                  # control
                 , 'backed_num', 'coll_num', 'len_tokens']
notna_index = ~ df_reg[pitch_features].isnull().T.any()
df_reg = df_reg.loc[notna_index]
with pd.option_context('display.max_rows', 10,
                       'display.max_columns', None,
                       'display.width', 1000,
                       'display.precision', 2,
                       'display.colheader_justify', 'left'):
    display(df_reg[list(dep_var.columns[1:])+pitch_features].describe())
print(df_reg[list(dep_var.columns[1:])+pitch_features].describe().T.to_latex(float_format="%.2f"))

# %% correaltion heatmap
temp = pd.merge(dep_var, ind_var, on='realpath')
df_reg = pd.merge(temp, founder_info, on='realpath', how='left')
pitch_features = ['Text-Warmth', 'Text-Ability', 'Text-Positive', 'Text-Negative'
                 , 'Vocal-Arousal', 'Vocal-Valence', 'Vocal-Positive', 'Vocal-Negative'
                 , 'Video-Positive', 'Video-Negative'
                  # control
                 , 'backed_num', 'coll_num', 'len_tokens']
notna_index = ~ df_reg[pitch_features].isnull().T.any()
df_reg = df_reg.loc[notna_index]

# max min standardization
for i in pitch_features+list(dep_var.columns[1:]):
    max_i = df_reg[i].max()
    min_i = df_reg[i].min()
    df_reg[i] = df_reg[i].apply(lambda x: (x - min_i) / (max_i - min_i))

# correlation
plt.subplots(figsize=(12,12),dpi=1080,facecolor='w')
fig=sns.heatmap(df_reg[pitch_features].corr(),annot=True, vmax=1, square=True, cmap="Blues", fmt='.2g')#annot为热力图上显示数据；fmt='.2g'为数据保留两位有效数字,square呈现正方形，vmax最大值为1
fig

#%% VIF
temp = pd.merge(dep_var, ind_var, on='realpath')
df_reg = pd.merge(temp, founder_info, on='realpath', how='left')
pitch_features = ['Text-Warmth', 'Text-Ability','Text-Positive', 'Text-Negative'
                 , 'Vocal-Valence', 'Vocal-Positive', 'Vocal-Negative'
                 , 'Video-Positive', 'Video-Negative'
                 # control
                 , 'backed_num', 'coll_num', 'len_tokens']
notna_index = ~ df_reg[pitch_features].isnull().T.any()
df_reg = df_reg.loc[notna_index]

# max min standardization
for i in pitch_features:
    max_i = df_reg[i].max()
    min_i = df_reg[i].min()
    df_reg[i] = df_reg[i].apply(lambda x: (x - min_i) / (max_i - min_i))

# multicollinearity
vif = [variance_inflation_factor(df_reg[pitch_features].values, df_reg[pitch_features].columns.get_loc(i)) for i in df_reg[pitch_features].columns]
vif

#%% ultimate dr_reg
temp = pd.merge(dep_var, ind_var, on='realpath')
df_reg = pd.merge(temp, founder_info, on='realpath', how='left')
pitch_features = ['Text-Ability','Text-Positive', 'Text-Negative'
                 , 'Vocal-Positive', 'Vocal-Negative'
                 , 'Video-Positive', 'Video-Negative'
                 # control
                 , 'backed_num', 'coll_num', 'len_tokens']
notna_index = ~ df_reg[pitch_features].isnull().T.any()
df_reg = df_reg.loc[notna_index]

# max min standardization
for i in pitch_features:
    max_i = df_reg[i].max()
    min_i = df_reg[i].min()
    df_reg[i] = df_reg[i].apply(lambda x: (x - min_i) / (max_i - min_i))

#%% OLS regression (status)
X = df_reg[pitch_features]
X_const = sm.add_constant(X)
Y = df_reg[['status']]
result = sm.OLS(Y, X_const).fit()
print(result.summary())
print(result.summary().as_latex())

# %% logistic regression (status)

model = sm.Logit(Y, X_const)
result = model.fit()

print(result.summary())
print(result.summary().as_latex())

#%% OLS regression (population)
X = df_reg[pitch_features]
X_const = sm.add_constant(X)
Y = df_reg[['population']]
result = sm.OLS(Y, X_const).fit()
print(result.summary())
print(result.summary().as_latex())

#%% OLS regression (posterior success)
X = df_reg[pitch_features]
X_const = sm.add_constant(X)
Y = df_reg[['pos_suc_url']]
result = sm.OLS(Y, X_const).fit()
print(result.summary())
print(result.summary().as_latex())

# %% logistic regression (posterior success)

model = sm.Logit(Y, X_const)
result = model.fit()

print(result.summary())
print(result.summary().as_latex())

#%% OLS regression (pledged amount)
X = df_reg[pitch_features]
X_const = sm.add_constant(X)
Y = df_reg[['done_amount']]
result = sm.OLS(Y, X_const).fit()
print(result.summary())
print(result.summary().as_latex())

#%% OLS regression (goal amount)
X = df_reg[pitch_features]
X_const = sm.add_constant(X)
Y = df_reg[['goal_amount']]
result = sm.OLS(Y, X_const).fit()
print(result.summary())
print(result.summary().as_latex())

# %%


'''
    I used Newspaper 3k for further scraping data from the URL's collected in the reddit post scraping.
    Newspaper 3k does a great job at scraping the relevant info from news articles.

    For more info on Newspaper 3k, check out this link:
    https://github.com/codelucas/newspaper

    This script primarily uses the URLs combined in data_merger.py to further scrape
    news websites for the content of the articles.

'''

import newspaper
from newspaper import Article
import pandas as pd
import time

def news_scraper(df):
    '''
        This function takes in a dataframe, takes the URL for each article, then scraps the news website for
        info on the article, including its text, post title, article title, date of publication and author.

        '''
    content = []
    for index, row in df.iterrows():
        url = row['URL']
        news=Article(url)
        try:
            # need to first download the article
            news.download()
            news.parse()
            content.append({'post_title': row['post_title'],'Article_Title':news.title, 'Text': news.text.replace('\n',' '),
                     'Date_of_pub':news.publish_date})
        except:
            print('could not download', news)
    return content

# importing dataset from merging.py
df = pd.read_csv('reddit_data.csv')
#scraping the newspapers
news_content = news_scraper(df)
#building newspaper dataframe to merge with reddit data
news_data = pd.DataFrame.from_dict(news_content)
# merging df with news content
all_reddit_data = pd.merge(df, news_data, how='inner', on=['post_title'])
#dropping duplicates
all_reddit_data = all_reddit_data.drop_duplicates('post_title')
#saving out CSV to use for classification
all_reddit_data.to_csv('clean_reddit_data.csv')


import github as gthb

inst = gthb.Github()

languages  = ['Python', 'Ruby', 'Javascript', 'Haskell']
extensions = ['.py', '.rb', '.js', '.hs']

with open('urls.txt','w') as f:

	for ind, lang in enumerate(languages):
		repos = inst.search_repositories(lang, language=lang, sort='stars', order='desc')

		size = 0
		i = 0
		while size < 1e6:

			url = repos[i].html_url
			size += repos[i].size
			i+=1

			f.write(lang+','+url+','+extensions[ind]+'\n')


import torch
from torch.autograd import Variable
batch_n = 100
hidden_layer = 100
input_data = 1000
output_data = 10
x = Variable(torch.randn(batch_n, input_data),requires_grad = False)
y = Variable(torch.randn(batch_n, output_data),requires_grad = False)

# 使用Sequential自动生成和初始化对应维度的权重参数
# 是一种序列容器， 嵌套各种实现神经网络的中具体功能相关的类
# 参数会按照定义好的序列自动传递
# 模块加入的两种方式：
# 1.代码中直接嵌套。
# 2.以orderdict有序字典传入

# 直接嵌套
models = torch.nn.Sequential(
    torch.nn.Linear(input_data, hidden_layer),
    torch.nn.ReLU(),
    torch.nn.Linear(hidden_layer, output_data)
)
print(models)

# 以orderdict有序字典传入
from collections import OrderedDict

models = torch.nn.Sequential(OrderedDict([
    ("linear1", torch.nn.Linear(input_data, hidden_layer)),
    ("relu1", torch.nn.ReLU()),
    ("linear2", torch.nn.Linear(hidden_layer, output_data))
]))
print(models)

# Sequential(
#   (0): Linear(in_features=1000, out_features=100, bias=True)
#   (1): ReLU()
#   (2): Linear(in_features=100, out_features=10, bias=True)
# )
# Sequential(
#   (linear1): Linear(in_features=1000, out_features=100, bias=True)
#   (relu1): ReLU()
#   (linear2): Linear(in_features=100, out_features=10, bias=True)
# )

epoch_n = 10000
learning_rate = 1e-4
loss_fn = torch.nn.MSELoss()

# loss_fn = torch.nn.L1Loss()
# loss_fn = torch.nn.CrossEntropyLoss()

for epoch in range(epoch_n):
    y_pred = models(x)
    loss = loss_fn(y_pred, y)
    if epoch%1000==0:
        print("Epoch:{}, Loss:{:.4f}".format(epoch, loss.data.item()))
    models.zero_grad()

    loss.backward()

    for param in models.parameters():
        param.data -= param.grad.data*learning_rate
# 通过models.parameters()进行参数更新梯度


from __future__ import annotations
from typing import Dict

class GameState:
    """
    This class represent game state.

    Attributes:
        current_scene (str): Id of the current scene in game
        variables (dict): Dictionary containing all the variables of the game. Default: {}
    """

    def __init__(
        self,
        current_scene: str,
        variables: Dict[str, bool] = {},
    ) -> None:
        """
        Initialize new GameState object.

        Params:
            current_scene (str): Id of the current scene in game
            variables (dict): Dictionary containing all the variables of the game. Default: {}
        """
        self.current_scene = current_scene
        self.variables = variables

    @classmethod
    def from_dict(cls, adict: dict) -> GameState:
        """
        Creates object from dictionary.

        Params:
            adict: A dictionary representing the object.

        Returns:
            Object of this class.
        """
        return cls(
            current_scene=adict["current_scene"],
            variables=adict["variables"],
        )

    def to_dict(self) -> dict:
        """
        Dumps object to dictionary.

        Returns:
           Dictionary representing this object.
        """
        return {
            "current_scene": self.current_scene,
            "variables": self.variables,
        }

    def __eq__(self, other: object) -> bool:
        """
        Compare equal method for object.

        Params:
            other: object to compare with

        Return:
            Compare result.
        """
        if not isinstance(other, GameState):
            raise NotImplementedError

        return self.to_dict() == other.to_dict()


'''
app 新建 urls.py 文件
'''

from django.conf.urls import url
import book.views as views

app_name = 'book' # 设置 app namespace
urlpatterns = [
    url(r'^index/$', views.index, name='index'), # url函数的name参数：设置 URL name

    # http://127.0.0.1:8000/book/分类id/书籍id/
    # http://127.0.0.1:8000/book/category_id/book_id/
    # 分组来获取正则中的数据
    # 1. 根据位置来获取 url中的参数
    url(r'^(\d+)/(\d+)/$', views.getid, name='getid'),
    # 2. 关键字参数--推荐大家使用关键字参数
    # url(r'^(?P<category_id>\d+)/(?P<book_id>\d+)/$', views.getid, name='getid'),
]


import pytest

from cygorithms.arrays import (
    OneDArray,
    DynamicOneDArray,
    selection_sort,
    merge_sort,
    quick_sort
)

@pytest.mark.parametrize(
    "array_type",
    [
        OneDArray,
        DynamicOneDArray
    ]
)
@pytest.mark.parametrize(
    "dtype, container, filler",
    [
        (
            int,
            [1, 2, 3, 4],
            5
        ),
        (
            str,
            ["a", "b", "c", "d"],
            "e"
        ),
        (
            tuple,
            [(1, 2, 3, 4), (1, 2, 3), (1, 2), (1,)],
            (0,)
        )
    ]
)
def test_arrays(array_type, dtype, container, filler):
    print(container)
    arr = array_type(dtype, 4, container)
    assert len(arr) == arr.size
    for el, check in zip(arr, container):
        assert el == check

    if "Dynamic" in array_type.__class__.__name__:
        arr.append(filler)
        assert arr[4] == filler
        arr.delete(2)
        assert arr[2] is None

@pytest.mark.parametrize(
    "array_type",
    [
        OneDArray,
        DynamicOneDArray
    ]
)
@pytest.mark.parametrize(
    "sort_alg",
    [
        selection_sort,
        merge_sort,
        quick_sort
    ]
)
@pytest.mark.parametrize(
    "dtype, container",
    [
        (
            int,
            [3, 1, 2, 5, 3, 2, 2, 1, 5, 6, 8, 1, 2, 4, 1]
        ),
        (
            str,
            ["c", "d", "a", "c", "ef", "f", "b", "ad", "dd", "e", "b"]
        )
    ]
)
@pytest.mark.parametrize(
    "comp",
    [
        None,
        lambda u, v: u >= v
    ]
)
def test_arrays_algs(array_type, sort_alg, dtype, container, comp):
    arr = array_type(dtype, container)
    if comp is None:
        sorted_container = sorted(container)
    else:
        sorted_container = sorted(container)[::-1]

    sort_alg(arr, comp=comp)

    for el, check in zip(sorted_container, arr):
        assert el == check


import os
import json
import utils
import interfaces.terraform.hetzner_provider_mapper

from abc import ABC
from commands.command import Command
from commands.validators.command_validators import KubesprayPatchesValidator
from exceptions.exceptions import UnexpectedTerraformFailureException, CommandConfirmationException
from interfaces.ansible.kubespray_manager import KubesprayManager
from interfaces.terraform import hetzner_provider_mapper
from interfaces.terraform.terraform_interface import Terraform

class TerraformBaseCommand(Command, ABC):

    def __init__(self, context):
        super().__init__(context, context_validators=[KubesprayPatchesValidator])
        self.terraform_content = None
        self.tf = Terraform()

    def prepare_arena(self):
        self.terraform_content = self.context.terraform_config.package_manager.get_content()
        if not os.listdir(self.context.cluster_space.tf_plugins_directory):
            if not self.tf.providers_mirror(self.terraform_content, self.context.cluster_space.tf_plugins_directory):
                raise UnexpectedTerraformFailureException('Cannot prepare terraform needed plugins')

        if not self.tf.init(self.terraform_content, self.context.cluster_space.tf_plugins_directory):
            raise UnexpectedTerraformFailureException('Terraform infrastructure initialization has failed')

    def get_dumped_infra_settings(self):
        tf_vars_file = self.context.temporal_fs.get_temporal_file(ext='.tfvars.json')
        with open(tf_vars_file, 'w') as file:
            json.dump(self.context.terraform_config.infra_config, file)
        return tf_vars_file

    def get_state_resources(self):
        res_ok, current_state = self.tf.show(self.terraform_content,
                                             input_file=self.context.cluster_space.tf_state_file)
        if not res_ok:
            raise UnexpectedTerraformFailureException('Cannot obtain current terraform state')
        return interfaces.terraform.hetzner_provider_mapper.parse_state(current_state)

class CreateClusterCommand(TerraformBaseCommand):

    def __init__(self, context):
        super().__init__(context)

    def run(self):
        try:
            self.prepare_arena()
            infra_settings_file = self.get_dumped_infra_settings()

            tf_vars = {'ssh_public_key': self.context.ssh_key_manager.get_public_rsa_key_opnessh()}

            plan_file = self.context.temporal_fs.get_temporal_file()
            res, output = self.tf.plan(self.terraform_content, [infra_settings_file], tf_vars, True,
                                       state_file=self.context.cluster_space.tf_state_file, plan_file=plan_file)
            if res:
                # TODO Make some validations
                plan_changes = hetzner_provider_mapper.parse_plan(output)
                if plan_changes.create or plan_changes.destroy or plan_changes.update:
                    res = self.tf.apply(self.terraform_content, [infra_settings_file], tf_vars,
                                        state_file=self.context.cluster_space.tf_state_file, plan_file=plan_file)
                if res:
                    self.logger.info('Infrastructure successfully created')
                    spray = KubesprayManager(self.context, self.get_state_resources())
                    spray.create_cluster()

                else:
                    self.logger.info('Failed to create infrastructure')

        except UnexpectedTerraformFailureException as err:
            self.logger.error(err)
            # TODO Raise to end with a proper exit code

class DestroyClusterCommand(TerraformBaseCommand):

    def __init__(self, context):
        super().__init__(context)

    def run(self):

        try:
            if not utils.get_optional_arg(self.context.run_options, 'confirm'):
                raise CommandConfirmationException('Destroy command needs destruction confirmation flag')

            self.prepare_arena()
            infra_settings_file = self.get_dumped_infra_settings()

            tf_vars = {'ssh_public_key': self.context.ssh_key_manager.get_public_rsa_key_opnessh()}
            res = self.tf.destroy(self.terraform_content, [infra_settings_file], tf_vars,
                                  state_file=self.context.cluster_space.tf_state_file)

            if res:
                self.context.cluster_space.destroy_cluster_space()
                self.logger.info('Infrastructure successfully destroyed')
            else:
                self.logger.info('Failed to destroy infrastructure')
        except UnexpectedTerraformFailureException as err:
            self.logger.error(err)
            return False