import numpy as np
from matplotlib import pyplot as plt
import sklearn.model_selection
import warnings
import seaborn as sns
import random
with warnings.catch_warnings():
    warnings.filterwarnings("ignore",category=FutureWarning)
    import tensorflow as tf
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)

def one_hot_encoding(seq) -> np.ndarray:
    """
    :param seq_array: np array of DNA sequences
    :return: np array of one-hot encodings of input DNA sequences
    """
    nuc2id = {'A' : 0, 'C' : 1, 'T' : 2, 'G' : 3}
    onehot_array = np.zeros((10000, 4))
    for seq_idx, nucleotide in enumerate(seq):
      if nucleotide != 'N':
        nuc_idx = nuc2id[nucleotide]
        onehot_array[seq_idx, nuc_idx] = 1

    return onehot_array

# Keys to npzfile of train & eval
train_cells = ['E065', 'E004', 'E066', 'E005', 'E012', 'E027', 'E053', 'E013', 'E028', 'E061', 'E109', 'E120', 'E062', 'E037', 'E038', 'E024', 'E105', 'E011', 'E106', 'E082', 'E097', 'E116', 'E098', 'E058',
'E117', 'E059', 'E070', 'E118', 'E085', 'E104', 'E119', 'E006', 'E127', 'E047', 'E094', 'E007', 'E054', 'E128', 'E095', 'E055', 'E114', 'E100', 'E056', 'E016', 'E122', 'E057', 'E123', 'E079', 'E003', 'E050']

eval_cells = ['E065', 'E004', 'E066', 'E005', 'E012', 'E027', 'E053', 'E013', 'E028', 'E061', 'E109', 'E120', 'E062', 'E037', 'E038', 'E024', 'E071', 'E105', 'E087', 'E011', 'E106', 'E096', 'E082', 'E097',
'E116', 'E098', 'E058', 'E117', 'E084', 'E059', 'E070', 'E118', 'E085', 'E104', 'E119', 'E006', 'E112', 'E127', 'E047', 'E094', 'E007', 'E054', 'E113', 'E128', 'E095', 'E055', 'E114', 'E100', 'E056', 'E016', 'E122', 'E057', 'E123', 'E079', 'E003', 'E050']

eval_only_cells = set(eval_cells)
eval_only_cells.difference_update(set(train_cells))
eval_only_cells = list(eval_only_cells)

# Load data
train_data = np.load('train.npz')
eval_data = np.load('eval.npz')

gene_sequences = {}
id_to_idx = {}
idx = 0

with open("seq_data.csv",'r+') as text:
  for line in text.readlines():
    (id, _) = line.strip().split(',')
    #gene_sequences[idx] = value
    id_to_idx[int(id)] = idx
    idx += 1

# Combine Train Data to use information from all cells
train_inputs = [] # Input histone mark data
train_dna_inputs = []
train_genes = []
train_cells_list = []
train_genes_set = set()
train_outputs = [] # Correct expression value
for cell in train_cells:
    cell_data = train_data[cell]
    hm_data = cell_data[:,:,0:6]
    for x in hm_data:
      train_genes_set.add(int(x[0,0]))
      train_inputs.append([x[:,1:6]])
      train_genes.append(id_to_idx[int(x[0,0])])
      train_cells_list.append(cell)
    exp_values = cell_data[:,0,6]
    #train_inputs.append(hm_data)
    train_outputs.append(exp_values)

train_inputs = np.concatenate(train_inputs, axis=0)
#train_inputs = np.concatenate(train_dna_inputs, axis=0)
train_outputs = np.concatenate(train_outputs, axis=0)

# Prepare Eval inputs in similar way
eval_inputs = []
eval_names = []
eval_only_genes = set()
eval_genes_list = []
eval_cells_list = []
for cell in eval_cells:
    cell_data = eval_data[cell]
    hm_data = cell_data[:,:,0:6]
    for x in hm_data:
      eval_inputs.append([x[:,1:6]])
    for gene in cell_data:
      eval_only_genes.add(int(gene[0,0]))
      eval_genes_list.append(int(gene[0,0]))
      eval_cells_list.append(cell)
      eval_names.append(cell+ "_" + str(int(gene[0,0])))
eval_inputs = np.concatenate(eval_inputs, axis=0)

eval_only_genes.difference_update(train_genes_set)
print(len(eval_only_genes))
print(eval_only_cells)

train_cell_vectors = np.zeros((50,8000000))
for c in range(50):
  curr_cell_data = []
  for i in range(len(train_cells_list)):
    if train_cells_list[i] == train_cells[c]:
        curr_cell_data.append(train_inputs[i].flatten())
  train_cell_vectors[c] = np.concatenate(curr_cell_data)

print(len(train_genes_set))

cell_vectors = np.zeros((6,8000000))
for c in range(6):
  curr_cell_data = []
  for i in range(len(eval_cells_list)):
    if eval_cells_list[i] == eval_only_cells[c]:
        if eval_genes_list[i] in train_genes_set:
            curr_cell_data.append(eval_inputs[i].flatten())
  cell_vectors[c] = np.concatenate(curr_cell_data)

print("Data done!")

cell_vectors = np.concatenate([cell_vectors, train_cell_vectors])

cell_to_idx = {}
idx = 0
for cell in train_cells:
  cell_to_idx[cell] = idx
  idx+=1
print(idx)
print(len(id_to_idx))

gene_vectors = np.zeros((17447,50*100*5))
for i in range(len(train_cells_list)):
  c_type = cell_to_idx[train_cells_list[i]]
  g_type = train_genes[i]
  gene_vectors[g_type,c_type*500:c_type*500+500] = train_inputs[i].flatten()

for i in range(len(eval_cells_list)):
  if eval_cells_list[i] in cell_to_idx:
    c_type = cell_to_idx[eval_cells_list[i]]
    g_type = id_to_idx[eval_genes_list[i]]
    gene_vectors[g_type,c_type*500:c_type*500+500] = eval_inputs[i].flatten()

print(gene_vectors)

from sklearn.manifold import MDS

mds = MDS(n_components=2, max_iter=30000, n_init=10, eps=1e-10)
pos = mds.fit_transform(gene_vectors[:500])
x = pos[:,0]
y = pos[:,1]
x = np.asarray(x)
y = np.asarray(y)
plt.scatter(x,y)
plt.savefig("genes.png")
plt.clf()

mds = MDS(n_components=2, max_iter=30000, n_init=10, eps=1e-10)
pos = mds.fit_transform(cell_vectors)
x = pos[:,0]
y = pos[:,1]
x = np.asarray(x)
y = np.asarray(y)
plt.scatter(x[:50],y[:50])
plt.scatter(x[50:],y[50:])
plt.savefig("cells.png")


import pandas as pd
import plotly.graph_objects as go
from plotly.subplots import make_subplots
from backend_code.constants import DATES, HOLIDAY_MAPPING, IDENTIFIER_MAP
from typing import Dict

def filter_and_rank_by_avg_sales(df: pd.DataFrame, mask, identifier:str="Dept", n: int=5, func=pd.DataFrame.head, select_type:str="All Types") -> pd.DataFrame:
    """
    identifier: can be Dept or Store
    select_type: can be All Types or A or B or C

    This function returns top 5/bottom 5 identifers based on select_type in terms of average weekly sales.

    For example, I want to get top 5 departments in Type A in terms of average weekly sales.
    """
    if select_type == "All Types":
        output_df = (df[mask]
                .groupby(identifier)["Weekly_Sales"]
                .mean()
                .sort_values(ascending=False))

    else:
        output_df = (df[(mask) & (df["Type"] == select_type)]
                .groupby(identifier)["Weekly_Sales"]
                .mean()
                .sort_values(ascending=False))

    output_df = func(output_df, n).reset_index()

    # Convert department identifiers to string (they are integers) to ensure they are treated as discrete categories when plotted,
    # preventing them from being interpreted as continuous data on a number line.

    output_df[identifier] = output_df[identifier].astype(str)

    return output_df

def aggregate_sales_by_holiday_and_identifier(df, identifier: str, func, select_type: str) -> dict:
    return {
        holiday: filter_and_rank_by_avg_sales(df=df, mask=df['Date'].isin(date_list), identifier=identifier, func=func, select_type=select_type)
        for holiday, date_list in DATES.items()
    }

def generate_holidays_sales_comparison(aggregated_data: Dict[str, pd.DataFrame], title: str) -> go.Figure:
    """
    This function return a figure which is subplots comparing between avg sales of the four holidays.
    """
    fig = make_subplots(rows=2, cols=2, subplot_titles=[info[0] for info in HOLIDAY_MAPPING.values()])

    for holiday_key, attrs in HOLIDAY_MAPPING.items():
        data = aggregated_data[holiday_key]
        holiday_name, color, row, col = attrs

        trace_kwargs = {"x": data.iloc[:, 0], "y": data["Weekly_Sales"], "marker_color": color, "name": holiday_name}

        fig.add_trace(go.Bar(**trace_kwargs), row=row, col=col)

    fig.update_layout(title=title, yaxis_title="Average Sales", autosize=True, template="plotly_dark")

    axes_kwargs = {"showgrid": True, "gridwidth": 1, "gridcolor": "Gray"}
    fig.update_xaxes(**axes_kwargs)
    fig.update_yaxes(**axes_kwargs)

    return fig

def assemble_holiday_vs_nonholiday_data(df, select_type, identifier, n=10, theme='dark'):
    assert identifier in ["Dept", "Store"], "Invalid identifier. Must be either 'Dept' or 'Store'."

    # Dark mode colors
    colors_dark = ['#FF6347', '#1E90FF']
    # Add more themes if needed
    themes = {'dark': colors_dark}

    if theme not in themes:
        raise ValueError("Invalid theme. Available themes are: " + ", ".join(themes.keys()))

    # print(df[holiday_mask])
    holiday_mask, non_holiday_mask = df["IsHoliday"], ~df["IsHoliday"]

    kwargs = {"df": df, "n": n, "identifier": identifier, "select_type": select_type}
    holiday_true_data = filter_and_rank_by_avg_sales(mask=holiday_mask, **kwargs)
    holiday_false_data = filter_and_rank_by_avg_sales(mask=non_holiday_mask, **kwargs)

    return holiday_true_data, holiday_false_data, themes[theme]

def generate_holiday_vs_nonholiday_subplot(holiday_data, non_holiday_data, identifier, theme) -> go.Figure:
    titles = ['Holiday Sales', 'Non-Holiday Sales']
    data = [holiday_data, non_holiday_data]

    fig = make_subplots(rows=1, cols=2, subplot_titles=titles)

    for i, (d, name) in enumerate(zip(data, titles)):
        trace_kwargs = {"x": d[identifier], "y": d["Weekly_Sales"], "marker_color": theme[i], "name": name}

        fig.add_trace(go.Bar(**trace_kwargs), row=1, col=i+1)

    layout_kwargs = {"yaxis_title": "Average Sales", "plot_bgcolor": "#333", "paper_bgcolor": "#333", "font": dict(color="white")}
    fig.update_layout(
        title=f"Sales Comparison based on {identifier}: Holiday vs Non-Holiday", **layout_kwargs
    )

    return fig

def select_and_prepare_dashboard_barcharts(df: pd.DataFrame, identifier: str, select_type: str) -> go.Figure:
    if identifier in ["Dept Sales: Holiday vs Non-Holiday", "Store Sales: Holiday vs Non-Holiday"]:
        key = IDENTIFIER_MAP[identifier]
        holiday_true_data, holiday_false_data, theme = assemble_holiday_vs_nonholiday_data(df=df, identifier=key, select_type=select_type)
        return generate_holiday_vs_nonholiday_subplot(holiday_true_data, holiday_false_data, key, theme)
    else:
        title, key, slice_func = IDENTIFIER_MAP[identifier]
        aggregated_data = aggregate_sales_by_holiday_and_identifier(df, key, slice_func, select_type)
        return generate_holidays_sales_comparison(aggregated_data, title)


import os
import sys
import numpy as np

from collections import deque
from itertools import cycle

import logging as log

from openvino.runtime import AsyncInferQueue

path = os.path.join(os.getcwd(), 'common\\python\\openvino\\model_zoo')
print(path)
sys.path.append(path)

from model_api.models.utils import resize_image

def crop(frame, roi):
    p1 = roi.position.astype(int)
    p1 = np.clip(p1, [0, 0], [frame.shape[1], frame.shape[0]])
    p2 = (roi.position + roi.size).astype(int)
    p2 = np.clip(p2, [0, 0], [frame.shape[1], frame.shape[0]])
    return frame[p1[1]:p2[1], p1[0]:p2[0]]

def cut_rois(frame, rois):
    return [crop(frame, roi) for roi in rois]

def resize_input(image, target_shape, nchw_layout):
    if nchw_layout:
        _, _, h, w = target_shape
    else:
        _, h, w, _ = target_shape
    resized_image = resize_image(image, (w, h))
    if nchw_layout:
        # HWC->CHW
        resized_image = resized_image.transpose((2, 0, 1))
    resized_image = resized_image.reshape(target_shape)
    return resized_image

def preprocess_frame(frame, size, chw_layout=True):
    frame = resize_input(frame, size, chw_layout)
    return frame

class AsyncWrapper:
    def __init__(self, ie_model, num_requests):
        self.model = ie_model
        self.num_requests = num_requests
        self._result_ready = False
        self._req_ids = cycle(range(num_requests))
        self._result_ids = cycle(range(num_requests))
        self._frames = deque(maxlen=num_requests)

    def infer(self, model_input, frame=None):
        next_req_id = next(self._req_ids)
        self.model.async_infer(model_input, next_req_id)

        last_frame = self._frames[0] if self._frames else frame

        self._frames.append(frame)
        if next_req_id == self.num_requests - 1:
            self._result_ready = True

        if self._result_ready:
            result_req_id = next(self._result_ids)
            result = self.model.wait_request(result_req_id)
            return result, last_frame
        else:
            return None, None

class SISO_IEModel:
    '''
    Single input/single output model.
    '''
    def __init__(self, model_path, core, target_device, num_requests, model_type):
        log.info('Reading {} model {}'.format(model_type, model_path))
        self.model = core.read_model(model_path)
        if len(self.model.inputs) != 1:
            log.error("SISO supports only models with 1 input")
            sys.exit(1)

        if len(self.model.outputs) != 1:
            log.error("SISO supports only models with 1 output")
            sys.exit(1)

        self.outputs = {}
        compiled_model = core.compile_model(self.model, target_device)
        self.output_tensor = compiled_model.outputs[0]
        self.input_name = self.model.inputs[0].get_any_name()
        self.input_shape = self.model.inputs[0].shape

        self.num_requests = num_requests
        self.infer_queue = AsyncInferQueue(compiled_model, num_requests)
        self.infer_queue.set_callback(self.completion_callback)
        log.info('The {} model {} is loaded to {}'.format(model_type, model_path, target_device))

    def completion_callback(self, infer_request, id):
        self.outputs[id] = infer_request.results[self.output_tensor]

    def async_infer(self, frame, req_id):
        input_data = {self.input_name: frame}
        self.infer_queue.start_async(input_data, req_id)

    def wait_request(self, req_id):
        self.infer_queue[req_id].wait()
        return self.outputs.pop(req_id, None)

    def cancel(self):
        for ireq in self.infer_queue:
            ireq.cancel()

class SIMO_IEModel:
    '''
    Single input/multiple output model.
    '''
    def __init__(self, model_path, core, target_device, num_requests, model_type):
        log.info('Reading {} model {}'.format(model_type, model_path))
        self.model = core.read_model(model_path)
        if len(self.model.inputs) != 1:
            log.error("SIMO supports only models with 1 input")
            sys.exit(1)

        self.outputs = {}
        compiled_model = core.compile_model(self.model, target_device)
        self.output_count = len(compiled_model.outputs)
        self.output_tensor = compiled_model.outputs[0]
        self.input_name = self.model.inputs[0].get_any_name()
        self.input_shape = self.model.inputs[0].shape

        self.num_requests = num_requests
        self.infer_queue = AsyncInferQueue(compiled_model, num_requests)
        self.infer_queue.set_callback(self.completion_callback)
        log.info('The {} model {} is loaded to {}'.format(model_type, model_path, target_device))

    def completion_callback(self, infer_request, id):
        results = infer_request.results
        aggregated_results = []
        for i in range(self.output_count):
            aggregated_results.append(results[self.output_tensor[i]])
        self.outputs[id] = aggregated_results

    def async_infer(self, frame, req_id):
        input_data = {self.input_name: frame}
        self.infer_queue.start_async(input_data, req_id)

    def wait_request(self, req_id):
        self.infer_queue[req_id].wait()
        return self.outputs.pop(req_id, None)

    def cancel(self):
        for ireq in self.infer_queue:
            ireq.cancel()

class MISO_IEModel:
    '''
    Multiple input/single output model.
    '''
    def __init__(self, model_path, core, target_device, num_requests, model_type):
        log.info('Reading {} model {}'.format(model_type, model_path))
        self.model = core.read_model(model_path)

        if len(self.model.outputs) != 1:
            log.error("MISO supports only models with 1 output")
            sys.exit(1)

        self.outputs = {}
        compiled_model = core.compile_model(self.model, target_device)
        self.output_tensor = compiled_model.outputs[0]
        self.input_count = len(self.model.inputs)
        self.input_names = [self.model.inputs[i].get_any_name() for i in range(self.input_count)]
        self.input_shapes = [self.model.inputs[i].shape for i in range(self.input_count)]

        self.num_requests = num_requests
        self.infer_queue = AsyncInferQueue(compiled_model, num_requests)
        self.infer_queue.set_callback(self.completion_callback)
        log.info('The {} model {} is loaded to {}'.format(model_type, model_path, target_device))

    def completion_callback(self, infer_request, id):
        self.outputs[id] = infer_request.results[self.output_tensor]

    def async_infer(self, inputs, req_id):
        assert self.input_count == len(inputs), 'Provided inputs do not match expected inputs'
        input_data = {}
        for i in range(self.input_count):
            input_data[self.input_names[i]] = inputs[i]
        self.infer_queue.start_async(input_data, req_id)

    def wait_request(self, req_id):
        self.infer_queue[req_id].wait()
        return self.outputs.pop(req_id, None)

    def cancel(self):
        for ireq in self.infer_queue:
            ireq.cancel()


import math
import time
from cscore import CameraServer

import cv2
import json
import numpy as np
import robotpy_apriltag
import ntcore
from ntcore import NetworkTableInstance as nt

inst = nt.getDefault()
inst.startClient4("wpilibpi")
inst.setServerTeam(9418)
inst.startDSClient()
vision_table = inst.getTable('Vision')
# visionTopic = inst.getIntegerArrayTopic("/Vision/Tags")
# visionTopic = vision_table.getIntegerArrayTopic("Tags")
# gtopic = inst.getIntegerArrayTopic("Tags")
# visionTopic = ntcore.IntegerArrayTopic(gtopic)
time.sleep(2.0)

# def __init__(self, visionTopic: ntcore.IntegerArrayTopic):
#     self.intPub = visionTopic.publish()

# __init__(visionTopic)
# nt.initialize(server = "10.94.18.2")

# Checks if a matrix is a valid rotation matrix.

def isRotationMatrix(R):
    Rt = np.transpose(R)
    shouldBeIdentity = np.dot(Rt, R)
    I = np.identity(3, dtype=R.dtype)
    n = np.linalg.norm(I - shouldBeIdentity)
    return n < 1e-6

# Calculates rotation matrix to euler angles
# of the euler angles ( x and z are swapped ).

def rotationMatrixToEulerAngles(R):

    # assert(isRotationMatrix(R))

    sy = math.sqrt(R[0, 0] * R[0, 0] + R[1, 0] * R[1, 0])

    singular = sy < 1e-6

    if not singular:
        x = math.atan2(R[2, 1], R[2, 2])
        y = math.atan2(-R[2, 0], sy)
        z = math.atan2(R[1, 0], R[0, 0])
    else:
        x = math.atan2(-R[1, 2], R[1, 1])
        y = math.atan2(-R[2, 0], sy)
        z = 0

    return np.array([x, y, z])

# def close(self):
    # self.intPub.close()

def main():
    width = 480
    height = 360

    CameraServer.startAutomaticCapture()
    input_stream = CameraServer.getVideo()
    output_stream = CameraServer.putVideo('Processed', width, height)
    img = np.zeros(shape=(height, width, 3), dtype=np.uint8)

    while True:
        self.intPub.setDefault(0)

        frame_time, input_img = input_stream.grabFrame(img)

        x_list = []
        y_list = []
        id_list = []
        x_euler_list = []
        y_euler_list = []
        z_euler_list = []

        if frame_time == 0:
            output_stream.notifyError(input_stream.getError())
            continue

        detector = robotpy_apriltag.AprilTagDetector()
        detector.addFamily("tag16h5")

        DETECTION_MARGIN_THRESHOLD = 100

        gray = cv2.cvtColor(input_img, cv2.COLOR_BGR2GRAY)
        tag_info = detector.detect(gray)

        filter_tags = [
            tag for tag in tag_info if tag.getDecisionMargin() > DETECTION_MARGIN_THRESHOLD]
        filter_tags = [tag for tag in filter_tags if (
            (tag.getId() > 0) & (tag.getId() < 9))]

        for tag in filter_tags:
            tag_id = tag.getId()
            self.intPub.set(tag_id, 0)
            center = tag.getCenter()
            homography = tag.getHomographyMatrix()
            euler_list = rotationMatrixToEulerAngles(homography)

           # x_list.append((center.x - width / 2) / (width / 2))
           # y_list.append((center.y - width / 2) / (width / 2))
            id_list.append(tag_id)
           # x_euler_list.append(euler_list[0])
           # y_euler_list.append(euler_list[1])
           # z_euler_list.append(euler_list[2])

        vision_table.putNumberArray("IDs", id_list)
        # vision_table.putNumberArray("X Coords", x_list)
        # vision_table.putNumberArray("Y Coords", y_list)
        # vision_table.putNumberArray("X Euler Angles", x_euler_list)
        # vision_table.putNumberArray("Y Euler Angles", y_euler_list)
        # vision_table.putNumberArray("Z Euler Angles", z_euler_list)

        print(x_euler_list)
        print(y_euler_list)
        print(z_euler_list)

        print(id_list)
        print("\n")

main()
# close()


try:
    import igraph
except ImportError:
    print("If you want to use igraph instead of networkx please install it.")

import types
from anomalous_vertices_detection.configs.config import *
from anomalous_vertices_detection.graphs import AbstractGraph
from anomalous_vertices_detection.utils.utils import *
from builtins import str

class IGraph(AbstractGraph):
    def __init__(self, is_directed=False, weight_field=None, graph_obj=None):
        super(IGraph, self).__init__(weight_field)
        self._is_directed = is_directed
        if graph_obj:
            self._graph = graph_obj.copy()
        else:
            self._graph = igraph.Graph(directed=is_directed)

    def save_graph(self):
        pass

    def load_saved_graph(self):
        pass

    def get_node_label(self, vertex):
        return self._labels_dict[self.get_vertex_name(vertex)]

    def add_edge(self, vertex1, vertex2, edge_atrr=None):
        if edge_atrr is None:
            edge_atrr = {}
        try:
            vertex1, vertex2 = vertex1.strip(), vertex2.strip()
        except:
            pass
        if not self.has_edge(vertex1, vertex2):
            vertex1 = self.add_vertex(vertex1)
            vertex2 = self.add_vertex(vertex2)
            self._graph.add_edge(vertex1, vertex2)
            edge_id = self._graph.get_eid(vertex1, vertex2)
            if edge_atrr:
                self._graph.es[edge_id][self._weight_field] = edge_atrr[self._weight_field]
                self._graph.es[edge_id]["edge_label"] = edge_atrr["edge_label"]

    def add_vertex(self, vertex):
        vertex = str(vertex)
        if not self.has_vertex(vertex):
            self._graph.add_vertex(vertex)
        return self._graph.vs.find(vertex).index

    def load_graph(self, graph_path, direction=1, start_line=1, limit=None, blacklist=set(),
                   delimiter=','):
        vetices_list = set()
        edges_list, label_list, weight_list = [], [], []
        if isinstance(graph_path, str):
            if graph_path.lower().endswith(".bz2"):
                f = read_bz2(graph_path)
            elif graph_path.lower().endswith(".gz"):
                f = read_gzip(graph_path)
            else:
                f = read_file(graph_path)
        else:
            f = graph_path
        if isinstance(f, types.GeneratorType):
            for i, edge in enumerate(f):

                if i >= start_line:
                    if limit and i == limit + start_line:
                        break
                    edge = extract_items_from_line(edge, delimiter)[0:3]
                    if direction is 0:
                        edge = reversed(edge)
                    if len(edge) >= 2 and edge[0] not in blacklist and edge[1] not in blacklist:
                        vetices_list.add(edge[0])
                        vetices_list.add(edge[1])
                        label_list.append(self.generate_edge_label(edge[0], edge[1]))
                        if len(edge) == 3:
                            weight_list.append(int(edge[2]))
                            edge = edge[:2]
                        edges_list.append(edge)
            self.load_igraph(list(vetices_list), edges_list, weight_list, label_list)

    def load_igraph(self, vertices_list, edges_list, weight_list, label_list):
        self._graph.add_vertices(len(vertices_list))
        self._graph.vs["name"] = vertices_list
        v_dict = {vertices_list[i]: i for i in range(len(vertices_list))}
        edges_list = [(v_dict[e[0]], v_dict[e[1]]) for e in edges_list]
        self._graph.add_edges(edges_list)
        if weight_list:
            self._graph.es[self._weight_field] = weight_list
        if edges_list:
            self._graph.es["edge_label"] = label_list
        self._graph = self._graph.simplify()

    def delete_graph(self):
        del self._graph
        self._graph = igraph.Graph(self._is_directed)

    def remove_edge(self, v, u, *args):
        self._graph.delete_edges(self._graph.get_eid(v, u))

    # return edges ids insted of edges
    def get_vertex_edges(self, vertex, mode="out"):
        edges_ind = []
        if mode == "out":
            edges_ind = self._graph.incident(vertex, mode="out")
        if mode == "in":
            edges_ind = self._graph.incident(vertex, mode="in")
        if mode == "all":
            edges_ind = self._graph.incident(vertex, mode="in") + self._graph.incident(vertex, mode="out")
        return [self._graph.es.find(x).tuple for x in edges_ind]

    def get_edge_label(self, vertex1, vertex2):
        if self.has_edge(vertex1, vertex2):
            # self._graph.get_eid(vertex1,vertex2)
            if "edge_label" in self._graph.es.attributes():
                return self._graph.es['edge_label'][self._graph.get_eid(vertex1, vertex2)]
        return self.negative_label

    def get_edge_weight(self, u, v):
        return self._graph.es[self._weight_field][self._graph.get_eid(u, v)]

    def edge(self, u, v):
        return self._graph.es[self._graph.get_eid(u, v)].attributes()

    @property
    def vertices(self):
        return self._graph.vs.indices

    @property
    def vertices_iter(self):
        return self.vertices()

    @property
    def edges(self):
        return [self._graph.es.find(x).tuple for x in self._graph.es.indices]

    @property
    def edges_iter(self):
        for x in self._graph.es.indices:
            yield self._graph.es.find(x).tuple

    @property
    def number_of_vertices(self):
        return self._graph.vcount()

    @property
    def is_directed(self):
        return self._is_directed

    # todo upgrade
    def pagerank(self):
        return self._graph.pagerank(weights=self._weight_field)

    def eigenvector(self):
        return self._graph.evcent(directed=self.is_directed, weights=self._weight_field)

    def closeness(self):
        return self._graph.closeness

    def get_shortest_path_length(self, vertex1, vertex2):
        try:
            # self._graph.bfs()
            # return self._graph.shortest_paths(source=vertex1, target=vertex2)
            sl = self._graph.shortest_paths_dijkstra(vertex1, vertex2)[0][0]
            if sl == float("inf"):
                sl = 0
            return sl
        except:
            return 0

    def get_vertex_name(self, vertex):
        if self.has_vertex(vertex):
            return self._graph.vs[vertex]["name"]
        else:
            return vertex

    def get_vertex_in_degree(self, vertex):
        return float(self._graph.indegree(vertex))

    def get_vertex_out_degree(self, vertex):
        return float(self._graph.outdegree(vertex))

    def has_edge(self, node1, node2):
        if not self.has_vertex(node1):
            return False
        if not self.has_vertex(node2):
            return False
        return self._graph.are_connected(node1, node2)

    def has_vertex(self, vertex):
        try:
            self._graph.vs.find(vertex)
            return True
        except ValueError:
            return False
        except IndexError:
            return False

    @memoize
    def get_neighbors(self, node):
        return self._graph.successors(node)

    def neighbors_iter(self, node):
        for node in self.get_neighbors(node):
            yield node

    def get_followers(self, node):
        if self.is_directed:
            return self._graph.predecessors(node)
        else:
            return self.get_neighbors(node)

    def get_subgraph(self, vertices):
        return IGraph(self.is_directed, self._weight_field, self._graph.subgraph(vertices))


import keras
from keras.datasets import mnist
from sklearn.metrics import accuracy_score

import numpy as np

from engines.model import Model
from layers import Dense, Conv2D, Flatten
from utils.train_utils import to_categorical

batch_size = 128
num_classes = 10
epochs = 10
num_kernels = 32
learning_rate = 0.01

# input image dimensions
img_rows, img_cols, num_channels = 28, 28, 1
input_shape = (img_rows, img_cols, num_channels)

def prepare_data():
    # the data, split between train and test sets
    (x_train, y_train), (x_test, y_test) = mnist.load_data()

    # reduce sample
    num_train_samples = 1000
    x_train = x_train[:num_train_samples]
    y_train = y_train[:num_train_samples]

    num_test_samples = int(0.3 * num_train_samples)
    x_test = x_test[:num_test_samples]
    y_test = y_test[:num_test_samples]

    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)

    x_train = x_train.astype('float32')
    x_test = x_test.astype('float32')
    x_train /= 255
    x_test /= 255
    print('x_train shape:', x_train.shape)
    print(x_train.shape[0], 'train samples')
    print(x_test.shape[0], 'test samples')
    return x_train, y_train, x_test, y_test

def train_with_keras(x_train, y_train, x_test, y_test):
    # convert class vectors to binary class matrices
    y_train = keras.utils.to_categorical(y_train, num_classes)
    y_test = keras.utils.to_categorical(y_test, num_classes)

    model = keras.models.Sequential()
    model.add(keras.layers.Conv2D(num_kernels, kernel_size=(3, 3),
                                  kernel_initializer='zeros',
                                  use_bias=False,
                                  activation='relu',
                                  input_shape=input_shape))
    model.add(keras.layers.Flatten())
    model.add(keras.layers.Dense(num_classes,
                                 kernel_initializer='zeros',
                                 bias_initializer='zeros',
                                 activation='sigmoid'))

    model.compile(loss=keras.losses.categorical_crossentropy,
                  optimizer=keras.optimizers.SGD(lr=learning_rate),
                  metrics=['accuracy'])

    model.fit(x_train, y_train,
              batch_size=len(x_train),
              epochs=epochs,
              verbose=1,
              validation_data=(x_test, y_test))
    score = model.evaluate(x_test, y_test, verbose=0)
    print('Test loss:', score[0])       # 0.36969225867455824
    print('Test accuracy:', score[1])   # 0.9074

def train_with_my_model(x_train, y_train, x_test, y_test):
    y_train = to_categorical(y_train, num_classes)
    # y_test = to_categorical(y_test, num_classes)

    model = Model()
    model.add(Conv2D(
        num_kernel=num_kernels,
        kernel_size=(3,3),
        kernel_initializer='zeros',
        use_bias=False,
        activation='relu',
        input_shape=input_shape))
    model.add(Flatten())
    model.add(Dense(
        units=num_classes,
        kernel_initializer='zeros',
        bias_initializer='zeros',
        activation='sigmoid'))

    model.compile(loss='categorical_crossentropy')
    model.fit(x_train, y_train, n_epochs=epochs, learning_rate=learning_rate)

    pred = model.predict(x_test)
    acc = accuracy_score(y_test, pred)
    print(acc)

def main():
    x_train, y_train, x_test, y_test = prepare_data()
    train_with_keras(x_train, y_train, x_test, y_test)
    # train_with_my_model(x_train, y_train, x_test, y_test)

if __name__ == '__main__':
    main()


import numpy as np
import pandas as pd
from .plotnine import *
from sklearn.decomposition import PCA, FactorAnalysis

def plot_pca_features(X, sensible_features) :

    data = X
    data[data == np.inf] = np.NaN
    data = data.dropna()

    sfindex = [data.columns.get_loc(col) for col in sensible_features]
    print(sfindex)

    # normalisation
    for col in data :
        data[col] -= data[col].min()
        data[col] /= data[col].max()

    pca = PCA(n_components=2)
    pca.fit(data.to_numpy().T)
    fa = FactorAnalysis(n_components=2)
    fa.fit(data.to_numpy().T)

    pca_df = pd.DataFrame(pca.transform(data.to_numpy().T))
    pca_df = pca_df.loc[[col in sensible_features for col in data.columns]]
    # pca_df = pca_df.loc[[data.columns.get_loc(col) for col in sensible_features]]
    pca_df.columns = ["PC1", "PC2"]
    pca_df["x_text"] = pca_df["PC1"] # + 0.1 * np.array([len(f) for f in data.columns])
    pca_df["y_text"] = pca_df["PC2"]
    pca_df["labels"] = data.columns
    exp1, exp2 = pca.explained_variance_ratio_.round(3)

    plt = ggplot() + \
          geom_hline(yintercept=0, color="white", linetype="dashed") + \
          geom_vline(xintercept=0, color="lightgrey", linetype="dashed") + \
          geom_point(data=pca_df, mapping=aes("PC1", "PC2")) + \
          geom_text(data=pca_df,
                    mapping=aes("x_text", "y_text", label="labels"),
                    size=10,
                    color="black") + \
          xlab(f"PCA1 (explained {exp1*100}% variance)") + \
          ylab(f"PCA2 (explained {exp2*100}% variance)") + \
          ggtitle("Features PCA plot")

    return plt


#!/usr/bin/python

"""
ZetCode PyQt5 tutorial

This program creates a menubar. The
menubar has one menu with an exit action.

Author: Jan Bodnar
Website: zetcode.com
"""

import sys
from PyQt5.QtWidgets import QMainWindow, QAction, qApp, QApplication
from PyQt5.QtGui import QIcon

class Example(QMainWindow):

    def __init__(self):
        super().__init__()

        self.initUI()

    def initUI(self):

        exitAct = QAction('&Exit', self)
        exitAct.setShortcut('Ctrl+Q')
        exitAct.setStatusTip('Exit application')
        exitAct.triggered.connect(self.exit)

        saveAct = QAction('&Save', self)
        saveAct.setShortcut('Ctrl+S')
        saveAct.setStatusTip('Save')
        saveAct.triggered.connect(self.save)

        preferencesAct = QAction('&Preferences', self)
        preferencesAct.setShortcut('Ctrl+,')
        preferencesAct.setStatusTip('Open the Preferences window')
        preferencesAct.triggered.connect(self.preferences)

        aboutAct = QAction('&About', self)
        aboutAct.setShortcut('Ctrl+A')
        aboutAct.setStatusTip('About the program')
        aboutAct.triggered.connect(self.about)

        self.statusBar()

        menubar = self.menuBar()
        fileMenu = menubar.addMenu('&File')
        aboutMenu = menubar.addMenu('&Help')
        aboutMenu.addAction(aboutAct)
        fileMenu.addAction(exitAct)
        fileMenu.addAction(saveAct)
        fileMenu.addAction(preferencesAct)

        self.setGeometry(300, 300, 300, 200)
        self.setWindowTitle('Simple menu')
        self.show()

    def exit(self):
        sys.exit()
    def save(self):
        print("save")
    def about(self):
        print("about")
    def preferences(self):
        print("settings")

def main():
    app = QApplication(sys.argv)
    ex = Example()
    sys.exit(app.exec_())

if __name__ == '__main__':
    main()


from selenium import webdriver
from selenium.webdriver.common.by import By
from time import sleep
from COVID.models.model import *
import matplotlib.pyplot as plt

# 各个国家的疫情数据
class Country(db.Model):
    __tablename__ = "country"
    __table_args__ = {'mysql_collate': 'utf8_general_ci'}
    country_id = db.Column(db.Integer, primary_key=True, autoincrement=True)
    country_name = db.Column(db.String(32))
    country_curConfirm = db.Column(db.Integer)
    country_confirmed = db.Column(db.Integer)
    country_cured = db.Column(db.Integer)
    country_died = db.Column(db.Integer)

def all_country_data():
    options = webdriver.ChromeOptions()
    options.binary_location = r"C:/application/Google/Chrome/Application/chrome.exe"
    options.add_experimental_option('excludeSwitches', ['enable-automation'])
    # options.add_argument("headless")
    prefs = {
        'profile.default_content_setting_values': {'images': 2}
    }
    options.add_experimental_option('prefs', prefs)
    bro = webdriver.Chrome("C:/code/python/COVID/static/chromedriver.exe", options=options)
    bro.get('https://voice.baidu.com/act/newpneumonia/newpneumonia/?from=osari_aladin_banner')
    bro.find_element(By.XPATH, '//div[@id="foreignTable"]/div').click()

    countrys = bro.find_elements(By.XPATH, '//div[@id="foreignTable"]//tbody//table//tr')
    # print(countrys)
    data = []
    for country in countrys:
        try:
            country_name = country.find_element(By.XPATH, './/a/div').text
        except:
            country_name = country.find_element(By.XPATH, './/div').text

        country_curConfirm = int(country.find_element(By.XPATH, './/td[2]').text)
        # print(country_curConfirm)
        country_confirmed = int(country.find_element(By.XPATH, './/td[3]').text)
        country_cured = int(country.find_element(By.XPATH, './/td[4]').text)
        country_died = int(country.find_element(By.XPATH, './/td[5]').text)
        data.append(Country(
            country_curConfirm=country_curConfirm, country_name=country_name,
            country_confirmed=country_confirmed, country_cured=country_cured, country_died=country_died)
        )

    bro.quit()

    db.session.add_all(data)
    db.session.commit()

def get_top5_data():
    countrys = Country.query.order_by(Country.country_curConfirm.desc()).limit(5)
    country_name = []
    country_curConfirm = []
    for country in countrys:
        country_name.append(country.country_name)
        country_curConfirm.append(country.country_curConfirm)

    plt.rcParams['font.family'] = ['Fangsong']
    plt.bar(country_name, country_curConfirm, width=0.5)
    plt.title('全球现存新冠病例TOP5')
    plt.grid(axis='y', linestyle=':')
    # plt.show()
    plt.savefig(r'./static/image/country.png', dpi=500, bbox_inches='tight', transparent=True)
    plt.clf()


from flask import Flask, render_template, request, session, redirect, url_for, flash, g
from functools import wraps
import data_manager
import os

UPLOAD_FOLDER = "./static/images/"

app = Flask(__name__)
app.config["UPLOAD_FOLDER"] = UPLOAD_FOLDER
app.config["MAX_CONTENT_LENGTH"] = 16 * 4000 * 4000
app.secret_key = os.environ.get("SECRET_KEY")

def login_required(f):
    @wraps(f)
    def decorated_function(*args, **kwargs):
        if not "user_name" in session:
            flash("You need to be logged in to user that feature!")
            return redirect(url_for('route_index'))
        return f(*args, **kwargs)
    return decorated_function

@app.route("/")
def route_index():
    questions = data_manager.get_latest_five_questions()
    tags = data_manager.get_tags_with_question_ids()
    order_by = request.args.get("order_by")
    order_direction = "desc" if request.args.get("order_direction") == "asc" else "asc"

    if order_by:
        questions = data_manager.order_list_of_dicts(questions, order_by, order_direction)
    return render_template("index.html",
                           questions=questions,
                           page_url="route_index",
                           order_direction=order_direction,
                           tags=tags)

@app.route("/list")
def route_all_questions():
    questions = data_manager.get_all_questions()
    tags = data_manager.get_tags_with_question_ids()
    order_by = request.args.get("order_by")
    order_direction = "desc" if request.args.get("order_direction") == "asc" else "asc"

    if order_by:
        questions = data_manager.order_list_of_dicts(questions, order_by, order_direction)
    return render_template("index.html",
                           questions=questions,
                           page_url="route_all_questions",
                           order_direction=order_direction,
                           tags=tags)

@app.route("/search")
def route_search():
    if request.args.get("search"):
        search_terms = request.args.get("search")
        question_titles = data_manager.search(search_terms, "question_titles")
        question_messages = data_manager.search(search_terms, "question_messages")
        answers = data_manager.search(search_terms, "answers")
        return render_template("search_results.html",
                               question_titles=question_titles,
                               question_messages=question_messages,
                               answers=answers,
                               search_terms=search_terms)

@app.route("/question/<question_id>/views")
def increase_view_counter(question_id):
    data_manager.view_count_increase(question_id)
    return redirect(url_for("display_a_question", question_id=question_id))

@app.route("/question/<question_id>")
def display_a_question(question_id):
    question = data_manager.get_question_details(question_id)
    unsorted_answers = data_manager.get_question_answers(question_id)
    answers = data_manager.order_list_of_dicts(unsorted_answers, "submission_time", "desc")
    question_comments = data_manager.get_question_comments(question_id)
    answer_comments = data_manager.get_answer_comments()
    tags = data_manager.get_question_tags(question_id)
    answers_with_comments = data_manager.get_answer_ids_with_comment("answer_id", answer_comments)
    users = data_manager.get_users()
    return render_template("display.html",
                           question=question,
                           answers=answers,
                           question_comments=question_comments,
                           answer_comments=answer_comments,
                           tags=tags,
                           answers_with_comments=answers_with_comments,
                           users=users)

@app.route("/add-question", methods=["GET", "POST"])
@login_required
def route_add_question():

    if request.method == "POST":
        file_url = data_manager.file_upload(request.files)
        title = request.form["title"]
        message = request.form["message"]
        data_manager.add_question(title, message, session["user_id"], file_url)
        return redirect(url_for("route_index"))

    return render_template("form.html",
                           form_url=url_for("route_add_question"),
                           add_question=True,
                           message_title="Details")

@app.route("/question/<question_id>/new-answer", methods=["GET", "POST"])
@login_required
def route_new_answer(question_id):

    if request.method == "POST":
        file_url = data_manager.file_upload(request.files)
        message = request.form["message"]
        data_manager.add_new_answer(message, question_id, session["user_id"], file_url)
        return redirect(url_for("display_a_question", question_id=question_id))

    return render_template("form.html",
                           form_url=url_for("route_new_answer", question_id=question_id),
                           message_title="Post answer",
                           add_answer=True)

@app.route("/question/<question_id>/new-comment", methods=["GET", "POST"])
@login_required
def route_new_question_comment(question_id):

    if request.method == "POST":
        message = request.form["message"]
        data_manager.add_comment("question_id", question_id, message, session["user_id"])
        return redirect(url_for("display_a_question", question_id=question_id))

    return render_template("form.html",
                           form_url=url_for("route_new_question_comment", question_id=question_id),
                           message_title="Post comment")

@app.route("/answer/<answer_id>/new-comment", methods=["GET", "POST"])
@login_required
def route_new_answer_comment(answer_id):

    if request.method == "POST":
        question_id = data_manager.get_question_id("answer", answer_id)
        message = request.form["message"]
        data_manager.add_comment("answer_id", answer_id, message, session["user_id"])
        return redirect(url_for("display_a_question", question_id=question_id))

    return render_template("form.html",
                           form_url=url_for("route_new_answer_comment", answer_id=answer_id),
                           message_title="Post comment")

@app.route("/question/<question_id>/edit", methods=["GET", "POST"])
@login_required
def route_edit_question(question_id):

    if request.method == "POST":
        file_url = data_manager.file_upload(request.files)
        title = request.form["title"]
        message = request.form["message"]
        data_manager.edit_question(title, message, question_id, file_url)
        return redirect(url_for("display_a_question", question_id=question_id))

    question = data_manager.get_question_details(question_id)
    question_title = question["title"]
    question_message = question["message"]
    return render_template("form.html",
                           form_url=url_for("route_edit_question", question_id=question_id),
                           add_question=True,
                           message_title="Details",
                           title=question_title,
                           message=question_message)

@app.route("/answer/<answer_id>/edit", methods=["GET", "POST"])
@login_required
def route_edit_answer(answer_id):

    if request.method == "POST":
        file_url = data_manager.file_upload(request.files)
        question_id = data_manager.get_question_id("answer", answer_id)
        message = request.form["message"]
        data_manager.edit_answer(message, answer_id, file_url)
        return redirect(url_for("display_a_question", question_id=question_id))

    answer = data_manager.get_answer(answer_id)
    answer_message = answer["message"]
    return render_template("form.html",
                           form_url=url_for("route_edit_answer", answer_id=answer_id),
                           message_title="Edit answer",
                           message=answer_message,
                           add_answer=True)

@app.route("/comments/<comment_id>/edit", methods=["GET", "POST"])
@login_required
def route_edit_comment(comment_id):

    if request.method == "POST":
        question_id = data_manager.get_question_id_from_comment(comment_id)
        message = request.form["message"]
        data_manager.edit_comment(message, comment_id)
        return redirect(url_for("display_a_question", question_id=question_id))

    comment = data_manager.get_comment(comment_id)
    comment_message = comment["message"]
    return render_template("form.html",
                           form_url=url_for("route_edit_comment", comment_id=comment_id),
                           message_title="Edit comment",
                           message=comment_message)

@app.route("/question/<question_id>/delete")
@login_required
def route_delete_question(question_id):
    data_manager.delete_question(question_id)
    return redirect(url_for("route_index"))

@app.route("/answer/<answer_id>/delete")
@login_required
def route_delete_answer(answer_id):
    question_id = data_manager.get_question_id("answer", answer_id)
    data_manager.delete_answer(answer_id)
    return redirect(url_for("display_a_question", question_id=question_id))

@app.route("/question/<question_id>/comments/<comment_id>/delete")
@login_required
def route_delete_comment(question_id, comment_id):
    data_manager.delete_comment(comment_id)
    return redirect(url_for("display_a_question", question_id=question_id))

@app.route("/question/<question_id>/vote-up")
@login_required
def route_question_vote_up(question_id):
    data_manager.vote("question", "up", question_id)
    return redirect(url_for("display_a_question", question_id=question_id))

@app.route("/question/<question_id>/vote-down")
@login_required
def route_question_vote_down(question_id):
    data_manager.vote("question", "down", question_id)
    return redirect(url_for("display_a_question", question_id=question_id))

@app.route("/answer/<answer_id>/vote-up")
@login_required
def route_answer_vote_up(answer_id):
    question_id = data_manager.get_question_id("answer", answer_id)
    data_manager.vote("answer", "up", answer_id)
    return redirect(url_for("display_a_question", question_id=question_id))

@app.route("/answer/<answer_id>/vote-down")
@login_required
def route_answer_vote_down(answer_id):
    question_id = data_manager.get_question_id("answer", answer_id)
    data_manager.vote("answer", "up", answer_id)
    return redirect(url_for("display_a_question", question_id=question_id))

@app.route("/question/<question_id>/new-tag", methods=["GET", "POST"])
@login_required
def tag_question(question_id):
    tags = data_manager.get_all_tags()

    if request.method == "POST":
        new_tag = request.form["new"]
        if new_tag:
            data_manager.add_new_tag(new_tag)
            tag_id = data_manager.get_tag_id(new_tag)
            data_manager.add_tag_to_question(question_id, tag_id)
        else:
            tag_id = request.form["existing"]
            data_manager.add_tag_to_question(question_id, tag_id)
        return redirect(url_for("display_a_question", question_id=question_id))

    return render_template("form.html",
                           form_url=url_for("tag_question", question_id=question_id),
                           add_tag=True,
                           tags=tags)

@app.route("/question/<question_id>/tag/<tag_id>/delete")
@login_required
def delete_tag(question_id, tag_id):
    data_manager.delete_question_tag(tag_id)
    return redirect(url_for("display_a_question", question_id=question_id))

@app.route("/registration", methods=["GET", "POST"])
def route_registration():
    if request.method == "POST":
        try:
            data_manager.add_new_user(request.form)
        except ValueError:
            flash("User name already exists")
            return redirect(url_for("route_registration"))
        except AssertionError:
            flash("Passwords don't match")
            return redirect(url_for("route_registration"))

        session["user_name"] = request.form["user_name"]
        session["user_id"] = data_manager.get_user_id(session["user_name"])
        return redirect(url_for("route_index"))

    return render_template("form.html",
                           registration=True)

@app.route("/login", methods=["POST"])
def route_login():
    if data_manager.login(request.form):
        session["user_name"] = request.form["user_name"]
        session["user_id"] = data_manager.get_user_id(session["user_name"])
        return redirect(url_for("route_index"))
    else:
        flash("You have entered an invalid user name or password. Please enter the correct details and try again.")
        return redirect(url_for("route_index"))

@app.route("/logout")
def route_logout():
    session.pop("user_name")
    session.pop("user_id")
    return redirect(url_for("route_index"))

@app.errorhandler(404)
def error_page_not_found(e):
    return render_template("404.html"), 404

if __name__ == "__main__":
    app.run(
        host='0.0.0.0',
        debug=True,
    )


# -*- coding: utf-8 -*-
from itertools import combinations, product,compress
import warnings
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats
from typing import Union
from IPython.display import display
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.compose import ColumnTransformer
from statsmodels.stats.outliers_influence import variance_inflation_factor
from sklearn.feature_selection import RFE, r_regression, SelectKBest
from sklearn.svm import SVR
from sklearn.metrics import (
    explained_variance_score,
    mean_squared_error,
    mean_absolute_error,
    max_error,
)

def chain_snap(data, fn=lambda x: x.shape, msg=None):
    r"""Print things in method chaining, leaving the dataframe untouched.
    Parameters
    ----------
    data: pandas.DataFrame
        the initial data frame for which the functions will be applied to in the pipe
    fn: lambda
        function that takes a pandas.DataFrame and that creates output to be printed
    msg: str or None
        optional message to be printed above output of the function
    Examples
    --------
    >>> from neuropy.utils import chain_snap
    >>> import pandas as pd
    >>> df = pd.DataFrame({'letter': ['a', 'b', 'c', 'c', 'd', 'c', 'a'],
    ...                    'number': [5,4,6,3,8,1,5]})
    >>> df = df.pipe(chain_snap, msg='Shape of the dataframe:')
    >>> df = df.pipe(chain_snap,
    ...              fn = lambda df: df['letter'].value_counts(),
    ...              msg="Frequency of letters:")
    >>> df = df.pipe(chain_snap,
    ...              fn = lambda df: df.loc[df['letter']=='c'],
    ...              msg="Dataframe where letter is c:")
    """
    if msg:
        print(msg + ": " + str(fn(data)))
    else:
        print(fn(data))
    return data

def standardise_data(
    X_train: pd.DataFrame, X_test: pd.DataFrame, quantitative_features: list
) -> pd.DataFrame:
    """Normalize numerical features using StandardScaler

    Args:
        X_train (pd.DataFrame): The train set coming from the train_test_split function
        X_test (pd.DataFrame): The test set coming from the train_test_split function
        quantitative_list (List): List of numerical features

    Returns:
        pd.DataFrame: The train and test set with the numerical features normalized
    """

    # initialize scaler
    scaler = StandardScaler()
    # fit scaler on train data
    _ = scaler.fit(X_train[quantitative_features])
    # transform train and test data
    X_train_scaled = pd.DataFrame(
        data=scaler.transform(X_train[quantitative_features]),
        columns=[x + "_scaled" for x in quantitative_features],
        index=X_train.index,
    )

    X_test_scaled = pd.DataFrame(
        data=scaler.transform(X_test[quantitative_features]),
        columns=[x + "_scaled" for x in quantitative_features],
        index=X_test.index,
    )

    return X_train_scaled, X_test_scaled

def encode_categorical_features(X_train, X_test, categorical_features):
    """Encode categorical features using sklearn OneHotEncoder
    Args:
        X_train (pd.DataFrame): The train set coming from the train_test_split function
        X_test (pd.DataFrame): The test set coming from the train_test_split function
        categorical_features (list): list of categorical features

    Returns:
        X_train_cat (pd.DataFrame): Train data with encoded categorical features
        X_test_cat (pd.DataFrame): Test data with encoded categorical features
    """

    # initialize encoder
    encoder = OneHotEncoder(drop="first", handle_unknown="ignore", sparse_output=False)
    # fit encoder on train data
    _ = encoder.fit(X_train[categorical_features])

    # transform train and test data
    X_train_cat = pd.DataFrame(
        data=encoder.transform(X_train[categorical_features]),
        columns=encoder.get_feature_names_out(categorical_features),
        index=X_train.index,
    )

    X_test_cat = pd.DataFrame(
        data=encoder.transform(X_test[categorical_features]),
        columns=encoder.get_feature_names_out(categorical_features),
        index=X_test.index,
    )

    return X_train_cat, X_test_cat


import argparse
import os
from colorama import Fore, Style, init

def rename_files(target_dir, filename=None, extension=None, start_name=None, new_name=None):
    # Set the target directory to the current directory if not provided
    if target_dir is None:
        target_dir = os.getcwd()

    files_to_rename = []

    # Recursively search for files in the target directory
    for root, _, files in os.walk(target_dir):
        for file in files:
            file_path = os.path.join(root, file)

            # Check if the file matches the specified criteria
            if (
                (filename is not None and file == filename) or
                (extension is not None and file.endswith(extension)) or
                (start_name is not None and file.startswith(start_name))
            ):
                files_to_rename.append(file_path)

    num_files = len(files_to_rename)
    print(Fore.YELLOW +
          f"Found {num_files} file(s) to rename." + Style.RESET_ALL)

    if num_files > 0:
        print("Files to rename:")
        for file in files_to_rename:
            print(file)

        confirm = input(
            Fore.YELLOW + "Do you want to continue? (y/n): " + Style.RESET_ALL).lower()
        if confirm != "y":
            print(Fore.YELLOW + "Aborted. No files were renamed." + Style.RESET_ALL)
            return

        files_renamed = 0

        for file_path in files_to_rename:
            try:
                dir_name = os.path.dirname(file_path)
                file_ext = os.path.splitext(file_path)[1]
                new_file_name = os.path.join(dir_name, new_name + file_ext)
                os.rename(file_path, new_file_name)
                print(
                    Fore.GREEN + f"Renamed file: {file_path} to {new_file_name}" + Style.RESET_ALL)
                files_renamed += 1
            except Exception as e:
                print(
                    Fore.RED + f"Error renaming file: {file_path} - {e}" + Style.RESET_ALL)

        print(Fore.YELLOW +
              f"Total files renamed: {files_renamed}" + Style.RESET_ALL)
    else:
        print(Fore.YELLOW +
              "No files found matching the specified criteria." + Style.RESET_ALL)

def main():

    # Initialize Colorama
    init()

    # Create the argument parser
    parser = argparse.ArgumentParser(
        description='Rename files based on criteria')

    # Add the command line arguments
    parser.add_argument('-d', '--directory', help='Target directory')
    parser.add_argument('-f', '--filename', help='Exact filename')
    parser.add_argument('-e', '--extension', help='File extension')
    parser.add_argument('-s', '--startname', help='Start of filename')
    parser.add_argument('-n', '--newname', required=True, help='New filename')

    # Parse the arguments
    args = parser.parse_args()

    # Call the rename_files function with the provided arguments
    rename_files(args.directory, args.filename,
                 args.extension, args.startname, args.newname)

if __name__ == '__main__':
    main()


from django.shortcuts import get_object_or_404, render
from django.http import HttpResponse
from .models import pc_parts
from collections import defaultdict

def view_part(request, part_name):
    print(type(part_name))
    print(part_name)
    quantity = defaultdict(int)
    data = pc_parts.objects.all()
    part_count = defaultdict(int)
    for d in data:
        total = 0
        model = d
        d = d.pub_date
        part_count[model.part_name]+=1
        if(model.part_name == part_name):
            if(d.find('2021',0,len(d)) != - 1):
                quantity[13] +=1
            else:
                total+=1
                if(d[2:5] == 'Dec'):
                    quantity[12]+=1
                if(d[2:5] == 'Nov'):
                    quantity[11]+=1
                if(d[2:5] == 'Oct'):
                    quantity[10]+=1
                if(d[2:6] == 'Sept'):
                    quantity[9]+=1
                if(d[2:5] == 'Aug'):
                    quantity[8]+=1
                if(d[2:6] == 'July'):
                    quantity[7]+=1
                if(d[2:6] == 'June'):
                    quantity[6]+=1
                if(d[2:5] == 'May'):
                    quantity[5]+=1
                if(d[2:7] == 'April'):
                    quantity[4]+=1
                if(d[2:7] == 'March'):
                    quantity[3]+=1
                if(d[2:5] == 'Feb'):
                    quantity[2]+=1
                if(d[2:5] == 'Jan'):
                    quantity[1]+=1

    # print(total)

    store = []
    if(part_count[part_name] > 2):
        for i in range(1,14):
            store.append(quantity[i])

        #find unique strings
        unique_cpu_names = set()
        for i in part_count.keys():
            if(part_count[i] > 2):
                unique_cpu_names.add(i)

        unique_cpu_names = list(unique_cpu_names)
        unique_cpu_names.sort()

        with_underscore = []

        for i in unique_cpu_names:
            with_underscore.append(i.replace("_", " "))

        tuple_list = []
        for i in range(len(unique_cpu_names)):
            tuple = (unique_cpu_names[i], with_underscore[i])
            tuple_list.append(tuple)

        ####

        return render(request, 'scrape/graph.html', {
            'data': store,
            'tuple_list': tuple_list,
            'title' : part_name
    })

    ####

def index(request):
    print("hello")
    return(view_part(request,"AMD Ryzen 5 2600 3.4 GHz 6-Core Processor"))


#!/usr/bin/env python

__author__ = "Christopher Hahne"
__email__ = "inbox@christopherhahne.de"
__license__ = """
Copyright (c) 2019 Christopher Hahne <inbox@christopherhahne.de>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""

try:
    import tkinter as tk
    from tkinter.filedialog import askopenfilename
except ImportError:
    import Tkinter as tk
    from tkFileDialog import askopenfilename

from tempfile import mkstemp
import sys
import os

# local python files
from plenoptisign import __version__
from plenoptisign.constants import ABBS, PF
from plenoptisign.mainclass import MainClass
from plenoptisign.gui.cfg import Config
from plenoptisign.gui.widget_men import MenWidget
from plenoptisign.gui.widget_cfg import CfgWidget, TwoStringVars
from plenoptisign.gui.widget_plt import PltWidget
from plenoptisign.gui.widget_cmd import CmdWidget
from plenoptisign.gui.widget_con import ConWidget

# generate blank icon on windows
ICON = (b'\x00\x00\x01\x00\x01\x00\x10\x10\x00\x00\x01\x00\x08\x00h\x05\x00\x00'
        b'\x16\x00\x00\x00(\x00\x00\x00\x10\x00\x00\x00 \x00\x00\x00\x01\x00'
        b'\x08\x00\x00\x00\x00\x00@\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
        b'\x00\x01\x00\x00\x00\x01') + b'\x00'*1282 + b'\xff'*64
_, ICON_PATH = mkstemp()
with open(ICON_PATH, 'wb') as icon_file:
    icon_file.write(ICON)

# object for application window
class PlenoptisignApp(tk.Tk):

    def __init__(self, parent):

        tk.Tk.__init__(self, parent)
        self.parent = parent

        # window title
        self.wm_title("PlenoptiSign-"+__version__)

        # icon handling
        if sys.platform == 'win32':
            self.wm_iconbitmap(default=ICON_PATH)
            cwd = sys._MEIPASS if hasattr(sys, '_MEIPASS') else os.getcwd()
            fp = os.path.join(cwd, 'misc', 'circlecompass_1055093.ico')
            fp = fp if os.path.exists(fp) else ICON_PATH
            self.iconbitmap(fp)

        # initialize parameters
        self.cfg = Config()

        # instantiate menu widget
        self.men_wid = MenWidget(self)
        self.men_wid.grid()

        # instantiate plot widget as object
        self.plt_wid = PltWidget(self)
        self.plt_wid.grid(row=0, column=0, padx=PF, pady=PF, sticky='NSWE')

        # instantiate config widget as object
        self.cfg_wid = CfgWidget(self)
        self.cfg_wid.grid(row=0, column=1, rowspan=3, padx=PF, pady=PF, sticky='NSWE')

        # instantiate command widget as object
        self.cmd_wid = CmdWidget(self)
        self.cmd_wid.grid(row=1, column=0, sticky='N')

        # instantiate console widget as object
        self.con_wid = ConWidget(self)
        self.con_wid.grid(row=2, column=0, padx=PF, pady=PF, sticky='NSWE')

        # about button in menu
        self.createcommand('tkAboutDialog', self.men_wid.open_about)

        # enable tkinter resizing
        self.resizable(True, False)

        # construct main class object
        self.obj = MainClass()

        # update results in GUI
        self.run()

    @staticmethod
    def tryfloat(val):
        try:
            return float(val)
        except:
            return val

    def run(self):

        # fetch parameter data from GUI
        self.obj.data = dict(zip(ABBS, [self.tryfloat(entry.get()) for entry in self.cfg_wid.entries]))

        # compute light field geometry
        self.obj.refo()
        self.obj.tria()

        # update widgets
        self.cfg_wid.refresh()
        self.plt_wid.refresh()

        # update console message
        self.con_wid.msg_box.config(text=self.obj.console_msg)

        return True

    @staticmethod
    def typefloat(var):
        if var:
            pass

    def save_cfg(self):
        ''' overwrite config file settings '''

        self.con_wid.msg_box.config(text='Save settings ...')

        # update results in GUI
        self.run()

        # transfer parameters from GUI to config object
        for i, key in enumerate(ABBS):
            self.cfg.params[key] = self.tryfloat(self.cfg_wid.entries[i].get())

        # write parameters from config object to hard drive
        self.cfg.write_json()

        self.con_wid.msg_box.config(text='Config saved')

        return True

    def load_cfg(self):
        ''' load config file settings '''

        # open window to select config file
        cfn_win = tk.Tk()
        cfn_win.withdraw()
        cfn = os.path.normpath(
            askopenfilename(parent=cfn_win, title="Select file", filetypes=[("Config files", "*.json")])
        )

        if cfn != '.':

            self.con_wid.msg_box.config(text='Load settings ...')

            # read parameters from hard drive to config object
            self.cfg.read_json(fp=cfn)

            # transfer parameters from config object to GUI
            for i, key in enumerate(ABBS):

                if isinstance(self.cfg.params[key], (list, tuple)):
                    tk_var = TwoStringVars(values=self.cfg.params[key])
                else:
                    tk_var = tk.StringVar()
                    tk_var.set(str(self.cfg.params[key]))

                self.cfg_wid.entries[i].config(textvariable=tk_var)

            # update results in GUI
            self.run()

            self.con_wid.msg_box.config(text='Config loaded')

        return True

    def quit_app(self):
        ''' quit app '''

        # destroy tkinter object
        self.destroy()
        sys.exit()

if __name__ == "__main__":

    MainWin = PlenoptisignApp(None)
    MainWin.mainloop()


# %%
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import tarfile
import urllib.request

from pandas.plotting import scatter_matrix
from pathlib import Path
from scipy import stats
from scipy.sparse.csr import csr_matrix
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.compose import ColumnTransformer
from sklearn.ensemble import RandomForestRegressor
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import cross_val_score, GridSearchCV, train_test_split, StratifiedShuffleSplit
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OrdinalEncoder, OneHotEncoder, StandardScaler
from sklearn.svm import SVR
from sklearn.tree import DecisionTreeRegressor
from zlib import crc32

# %%
DOWNLOAD_ROOT = "https://raw.githubusercontent.com/ageron/handson-ml2/master/"
HOUSING_URL = DOWNLOAD_ROOT + "datasets/housing/housing.tgz"
HOUSING_PATH = Path("data/housing")
HOUSING_CSV = HOUSING_PATH / "housing.csv"

# %%
def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):
    HOUSING_PATH.mkdir(exist_ok=True, parents=True)
    tgz_path = HOUSING_PATH / "housing.tgz"
    urllib.request.urlretrieve(housing_url, tgz_path)
    with tarfile.open(tgz_path) as housing_tgz:

        import os

        def is_within_directory(directory, target):

            abs_directory = os.path.abspath(directory)
            abs_target = os.path.abspath(target)

            prefix = os.path.commonprefix([abs_directory, abs_target])

            return prefix == abs_directory

        def safe_extract(tar, path=".", members=None, *, numeric_owner=False):

            for member in tar.getmembers():
                member_path = os.path.join(path, member.name)
                if not is_within_directory(path, member_path):
                    raise Exception("Attempted Path Traversal in Tar File")

            tar.extractall(path, members, numeric_owner=numeric_owner)

        safe_extract(housing_tgz, path=housing_path)

# %%
if not HOUSING_CSV.exists():
    fetch_housing_data()
assert HOUSING_CSV.exists()

# %%
def load_housing_data(housing_csv=HOUSING_CSV):
    return pd.read_csv(housing_csv)

# %%
housing = load_housing_data()

# %%
housing.info()

# %%
housing["ocean_proximity"].value_counts()

# %%
housing[["housing_median_age", "total_bedrooms"]].describe()

# %%
housing.hist(bins=50, figsize=(20, 15))
plt.show()

# %%
def split_train_test(data, test_ratio):
    shuffled_indices = np.random.permutation(len(data))
    test_set_size = int(len(data) * test_ratio)
    test_indices = shuffled_indices[:test_set_size]
    train_indices = shuffled_indices[test_set_size:]
    return data.iloc[train_indices], data.iloc[test_indices]

# %%
train_set, test_set = split_train_test(housing, 0.2)
len(train_set), len(test_set)

# %%
def test_set_check(identifier, test_ratio):
    return crc32(np.int64(identifier)) & 0x_ffff_ffff < test_ratio * 2 ** 32

# %%
def split_train_test_by_id(data, test_ratio, id_column):
    ids = data[id_column]
    in_test_set = ids.apply(lambda id_: test_set_check(id_, test_ratio))
    return data.loc[~in_test_set], data.loc[in_test_set]

# %%
housing_with_id = housing.reset_index()
train_set, test_set = split_train_test_by_id(housing_with_id, 0.2, "index")
len(train_set), len(test_set)

# %%
housing_with_id["id"] = housing["longitude"] * 1000 + housing["latitude"]
train_set, test_set = split_train_test_by_id(housing_with_id, 0.2, "id")
len(train_set), len(test_set)

# %%
train_set, test_set = train_test_split(housing, test_size=0.2, random_state=42)
len(train_set), len(test_set)

# %%
housing["income_cat"] = pd.cut(housing["median_income"], bins=[0., 1.5, 3.0, 4.5, 6.0, np.inf],
                               labels=[1, 2, 3, 4, 5])

# %%
housing["income_cat"].hist()
plt.title("Income (Whole Dataset)")
plt.show()

# %%
split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
strat_train_set, strat_test_set = None, None
for train_index, test_index in split.split(housing, housing["income_cat"]):
    strat_train_set = housing.loc[train_index]
    strat_test_set = housing.loc[test_index]
len(strat_train_set), len(strat_test_set)

# %%
strat_train_set, strat_test_set = train_test_split(housing, test_size=0.2, random_state=42,
                                                   stratify=housing["income_cat"])
len(strat_train_set), len(strat_test_set)

# %%
strat_train_set["income_cat"].hist()
plt.title("Income (strat_train_set)")
plt.show()
strat_test_set["income_cat"].hist()
plt.title("Income (strat_test_set)")
plt.show()

# %%
train_set, test_set = train_test_split(housing, test_size=0.2, random_state=42)
train_set["income_cat"].hist()
plt.title("Income (train_set)")
plt.show()
test_set["income_cat"].hist()
plt.title("Income (test_set)")
plt.show()

# %%
for set_ in (strat_train_set, strat_test_set):
    set_.drop("income_cat", axis=1, inplace=True)

# %%
housing = strat_train_set.copy()

# %%
housing.plot(kind="scatter", x="longitude", y="latitude")
plt.show()

# %%
housing.plot(kind="scatter", x="longitude", y="latitude", alpha=0.1)
plt.show()

# %%
housing.plot(kind="scatter", x="longitude", y="latitude", alpha=0.4, s=housing["population"] / 40,
             label="population", figsize=(15, 10), c="median_house_value", cmap="jet",
             colorbar=True)
plt.show()

# %%
corr_matrix = housing.corr()
corr_matrix["median_house_value"].sort_values(ascending=False)

# %%
attributes = ["median_house_value", "median_income", "total_rooms",
              "housing_median_age"]
scatter_matrix(housing[attributes], figsize=(15, 10))
plt.show()

# %%
housing.plot(kind="scatter", x="median_income", y="median_house_value", alpha=0.1)
plt.show()

# %%
housing["rooms_per_household"] = housing["total_rooms"] / housing["households"]
housing["bedrooms_per_room"] = housing["total_bedrooms"] / housing["total_rooms"]
housing["population_per_household"] = housing["population"] / housing["households"]
housing["bedrooms_per_household"] = housing["total_bedrooms"] / housing["households"]

# %%
corr_matrix = housing.corr()
corr_matrix["median_house_value"].sort_values(ascending=False)

# %%
housing = strat_train_set.drop("median_house_value", axis=1)
housing_labels = strat_train_set["median_house_value"].copy()

# %%
imputer: SimpleImputer = SimpleImputer(strategy="median")

# %%
housing_num: pd.DataFrame = housing.drop("ocean_proximity", axis=1)
imputer.fit(housing_num)

# %%
print(imputer.statistics_)

# %%
print(housing_num.median().values)

# %%
X = imputer.transform(housing_num)
housing_tr = pd.DataFrame(X, columns=housing_num.columns, index=housing_num.index)

# %%
housing_cat = housing[["ocean_proximity"]]

# %%
ordinal_encoder = OrdinalEncoder()
housing_cat_encoded = ordinal_encoder.fit_transform(housing_cat)

# %%
print(ordinal_encoder.feature_names_in_)
print(ordinal_encoder.categories_)

# %%
cat_encoder = OneHotEncoder()
housing_cat_1hot: csr_matrix = cat_encoder.fit_transform(housing_cat)
print(housing_cat_1hot[:5])

# %%
housing_cat_1hot.toarray()

# %%
print(cat_encoder.feature_names_in_)
print(cat_encoder.categories_)

# %%
rooms_ix, bedrooms_ix, population_ix, households_ix = 3, 4, 5, 6

# %%
class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
    def __init__(self, add_bedrooms_per_room=True):
        self.add_bedrooms_per_room = add_bedrooms_per_room

    # noinspection PyPep8Naming,PyUnusedLocal,PyShadowingNames
    def fit(self, X, y=None):
        return self

    # noinspection PyPep8Naming,PyShadowingNames
    def transform(self, X):
        rooms_per_household = X[:, rooms_ix] / X[:, households_ix]
        population_per_household = X[:, population_ix] / X[:, households_ix]
        if self.add_bedrooms_per_room:
            bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]
            return np.c_[X, rooms_per_household, population_per_household, bedrooms_per_room]
        else:
            return np.c_[X, rooms_per_household, population_per_household]

# %%
attr_adder = CombinedAttributesAdder(add_bedrooms_per_room=False)
housing_extra_attribs = attr_adder.transform(housing.values)

# %%
num_pipeline = Pipeline([
    ("imputer", SimpleImputer(strategy="median")),
    ("attribs_adder", CombinedAttributesAdder()),
    ("std_scaler", StandardScaler()),
])

# %%
housing_num_tr = num_pipeline.fit(housing_num)

# %%
num_attribs = list(housing_num)
cat_attribs = ["ocean_proximity"]

# %%
full_pipeline = ColumnTransformer([
    ("num", num_pipeline, num_attribs),
    ("cat", OneHotEncoder(), cat_attribs),
])

# %%
housing_prepared = full_pipeline.fit_transform(housing)

# %%
print(f"Number of input features: {len(full_pipeline.feature_names_in_)}")
print(full_pipeline.feature_names_in_)
# This does not work, unfortunately
# print(full_pipeline.get_feature_names())
# print(full_pipeline.get_feature_names_out())

# %%
lin_reg = LinearRegression()
lin_reg.fit(housing_prepared, housing_labels)

# %%
some_data = housing.iloc[:5]
some_labels = housing_labels[:5]
some_data_prepared = full_pipeline.transform(some_data)
some_data_predictions = lin_reg.predict(some_data_prepared)
print(f"Predictions: {[round(x, 1) for x in some_data_predictions]}")
print(f"Labels:      {list(some_labels)}")
print(f"Ratios:      {[round(pred / label, 2) for pred, label in zip(some_data_predictions, some_labels)]}")

# %%
housing_predictions = lin_reg.predict(housing_prepared)
lin_mse = mean_squared_error(housing_labels, housing_predictions)
lin_rmse = np.sqrt(lin_mse)
print(f"RMSE: {lin_rmse:.1f}")

# %%
tree_reg = DecisionTreeRegressor()
tree_reg.fit(housing_prepared, housing_labels)

# %%
# noinspection DuplicatedCode
housing_predictions = tree_reg.predict(housing_prepared)
tree_mse = mean_squared_error(housing_labels, housing_predictions)
tree_rmse = np.sqrt(tree_mse)
print(f"RMSE: {tree_rmse:.1f}")

# %%
scores = cross_val_score(tree_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)
tree_rmse_scores = np.sqrt(-scores)

# %%
# noinspection PyShadowingNames
def display_scores(scores):
    print(f"Scores:\n{scores}")
    print(f"Mean:               {scores.mean():.0f}")
    print(f"Standard deviation: {scores.std():.0f}")

# %%
display_scores(tree_rmse_scores)

# %%
forest_reg = RandomForestRegressor(n_jobs=32)
print("Fitting random forest...", flush=True, end="")
# noinspection DuplicatedCode
forest_reg.fit(housing_prepared, housing_labels)
print("done.", flush=True)

# %%
# noinspection DuplicatedCode
housing_predictions = forest_reg.predict(housing_prepared)
forest_mse = mean_squared_error(housing_labels, housing_predictions)
forest_rmse = np.sqrt(forest_mse)
print(f"RMSE: {forest_rmse:.1f}")

# %%
scores = cross_val_score(forest_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)
forest_rmse_scores = np.sqrt(-scores)

# %%
display_scores(forest_rmse_scores)

# %%
param_grid = [
    {"n_estimators": [3, 10, 30], "max_features": [2, 4, 6, 8]},
    {"bootstrap": [False], "n_estimators": [3, 10], "max_features": [2, 3, 4]}
]

# %%
forest_reg = RandomForestRegressor(n_estimators=32)

# %%
grid_search = GridSearchCV(forest_reg, param_grid, cv=5, scoring="neg_mean_squared_error", return_train_score=True)

# %%
grid_search.fit(housing_prepared, housing_labels)

# %%
print(grid_search.best_params_)

# %%
best_grid_search_estimator = grid_search.best_estimator_

# %%
cv_res = grid_search.cv_results_

# %%
for mean_score, params in zip(cv_res["mean_test_score"], cv_res["params"]):
    print(f"RMSE {np.sqrt(-mean_score):.1f} for {params}")

# %%
feature_importances = [round(x * 100, 1) for x in grid_search.best_estimator_.feature_importances_]
print(f"Number of features: {len(feature_importances)}.")
print(feature_importances)

# %%
extra_attribs = ["rooms_per_household", "pop_per_household", "bedrooms_per_room"]
cat_encoder = full_pipeline.named_transformers_["cat"]
cat_one_hot_attribs = list(cat_encoder.categories_[0])
attributes = num_attribs + extra_attribs + cat_one_hot_attribs
sorted(zip(feature_importances, attributes), reverse=True)

# %%
final_model = grid_search.best_estimator_

# %%
X_test = strat_test_set.drop("median_house_value", axis=1)
y_test = strat_test_set["median_house_value"].copy()

# %%
X_test_prepared = full_pipeline.transform(X_test)

# %%
final_predictions = final_model.predict(X_test_prepared)

# %%
final_mse = mean_squared_error(y_test, final_predictions)
final_rmse = np.sqrt(final_mse)
print(f"RMSE = {final_rmse:.1f}")

# %%
confidence = 0.95
squared_errors = (final_predictions - y_test) ** 2
confidence_interval = np.sqrt(stats.t.interval(confidence, len(squared_errors) - 1,
                                               loc=squared_errors.mean(), scale=stats.sem(squared_errors)))
print("Confidence interval:", confidence_interval)

# %%
svm_reg = RandomForestRegressor(n_jobs=32)
# noinspection DuplicatedCode
svm_reg.fit(housing_prepared, housing_labels)

# %%
# noinspection DuplicatedCode
housing_predictions = svm_reg.predict(housing_prepared)
svm_mse = mean_squared_error(housing_labels, housing_predictions)
svm_rmse = np.sqrt(svm_mse)
print(f"RMSE: {svm_rmse:.1f}")

# %%
scores = cross_val_score(svm_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)
svm_rmse_scores = np.sqrt(-scores)

# %%
display_scores(svm_rmse_scores)

# %%


import argparse
import logging
import json
from pathlib import Path
from urllib.parse import urljoin

import requests
import urllib3
from bs4 import BeautifulSoup

from download_books import download_books

logger = logging.getLogger('log')

def create_arg_parser():
    parser = argparse.ArgumentParser(
        prog='TULULU PARSER',
        description='Parser for online-library tululu.org',
    )
    parser.add_argument(
        '-from',
        '--start_page',
        default=1,
        help='Set start page of the category list',
        type=int,
    )
    parser.add_argument(
        '-to',
        '--end_page',
        help='Set end page of the category list',
        type=int,
    )
    parser.add_argument(
        '-f',
        '--dest_folder',
        default='parsing_result',
        help='Destination folder for downloaded parsing result',
    )
    parser.add_argument(
        '-i',
        '--skip_imgs',
        action='store_true',
        help='Bool argument to skip image downloading',
    )
    parser.add_argument(
        '-t',
        '--skip_txt',
        action='store_true',
        help='Bool argument to skip text downloading',
    )
    parser.add_argument(
        '-j',
        '--json_path',
        default='books',
        help='Set a path of the result json file',
    )
    return parser

def check_final_npage(category_url, end_page):
    response = requests.get(category_url)
    response.raise_for_status()
    soup = BeautifulSoup(response.text, 'lxml')
    final_page_number = int(soup.select('#content .npage')[-1].text)
    if not end_page:
        return final_page_number
    if end_page > final_page_number:
        raise Exception(
            f'There are only {final_page_number} pages in the category list. ' \
            f'Enter correct end page number.'
    )
    return end_page

def parse_tululu_category(category_url, start_page, end_page):
    book_urls = list()
    for page_number in range(start_page, end_page+1):
        logger.info(f'parsing page: {page_number}')
        list_page_url = f'{category_url}{page_number}/'
        response = requests.get(list_page_url)
        response.raise_for_status()
        soup = BeautifulSoup(response.text, 'lxml')
        card_tags = soup.select('.d_book')
        for book_url in [url.select_one('[href^="/b"]')['href'] for url in card_tags]:
            book_urls.extend([urljoin(category_url, book_url)])
    return book_urls

def main():
    logger.setLevel(logging.DEBUG)

    fh = logging.FileHandler('spam.log')
    fh.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    fmtstr = '%(asctime)s - %(name)s - %(levelname)s - %(funcName)s - %(message)s'
    fmtdate = '%H:%M:%S'
    formatter = logging.Formatter(fmtstr, fmtdate)
    fh.setFormatter(formatter)
    ch.setFormatter(formatter)

    logger.addHandler(fh)
    logger.addHandler(ch)

    logger.info('Starting program')

    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
    parser = create_arg_parser()
    args = parser.parse_args()
    if args.end_page and args.end_page < args.start_page:
        raise Exception('Start page number must be less than the end one')

    category_url = 'https://tululu.org/l55/'
    end_page = check_final_npage(category_url, args.end_page)

    Path(args.dest_folder).mkdir(parents=True, exist_ok=True)
    title_page_urls = parse_tululu_category(
        category_url,
        args.start_page,
        end_page,
    )
    books_summary = download_books(
        title_page_urls,
        args.dest_folder,
        args.skip_imgs,
        args.skip_txt,
    )

    with open(f'{args.dest_folder}/{args.json_path}.json', 'w', encoding='utf-8') as json_file:
        json.dump(books_summary, json_file, ensure_ascii=False)

if __name__ == '__main__':
    main()


import copy
import warnings
from collections import defaultdict
from typing import List, Mapping, Optional, Sequence, Tuple, Type, Union

import torch.nn as nn
from torch.nn.modules.container import Sequential

from qsparse.quantize import QuantizeLayer, quantize
from qsparse.sparse import PruneLayer, prune
from qsparse.util import auto_name_prune_quantize_layers, logging, nn_module

def copy_nn_module_on_demand(kwargs: Mapping) -> Mapping:
    return {
        k: copy.deepcopy(v) if isinstance(v, nn.Module) else v
        for k, v in kwargs.items()
    }

def convert(  # noqa: C901
    model: nn.Module,
    operator: Union[PruneLayer, QuantizeLayer],
    inplace: bool = True,
    weight_layers: Sequence[Type[nn.Module]] = [],
    activation_layers: Sequence[Type[nn.Module]] = [],
    input: bool = False,
    log: bool = True,
    excluded_weight_layer_indexes: Sequence[Tuple[Type[nn.Module], Sequence[int]]] = [],
    excluded_activation_layer_indexes: Sequence[
        Tuple[Type[nn.Module], Sequence[int]]
    ] = [],
    filter: Optional[Union[str, List[str]]] = None,
    exclude = None,
    order: str = "post",
    apoz={}
) -> nn.Module:
    """Automatically convert a model to a new model with its weights and
    activations transformed by the operator, e.g. [prune][qsparse.sparse.prune] or [quantize][qsparse.quantize.quantize].

    Args:
        model (nn.Module): input network module
        operator (Union[PruneLayer, QuantizeLayer]): operator used to transform the weights and activations.
        inplace (bool, optional): whether mutates the original module. Defaults to False.
        weight_layers (Sequence[Type[nn.Module]], optional): which layers to apply operator to transform weights. Defaults to [].
        activation_layers (Sequence[Type[nn.Module]], optional): which layers to apply operator to transform output activations. Defaults to [].
        input (bool, optional): whether apply operator to input. Defaults to False.
        log (bool, optional): whether print the conversion log. Defaults to True.
        excluded_weight_layer_indexes (Sequence[Tuple[Type[nn.Module], Sequential[int]]], optional): indexes of layers excluded in weight transformations from conversion. Defaults to [].
        excluded_activation_layer_indexes (Sequence[Tuple[Type[nn.Module], Sequential[int]]], optional): indexes of layers excluded in activation transformations from conversion. Defaults to [].
        filter (Union[str, List[str]], optional): Used to filter out a subnetwork to convert. For example, when filter="transition", then `convert` will only visit layers whose module paths contain "transition". When more than one filter is provided, the layer module path must contain all of them in order to be converted. Defaults to None, means to traverse the entire network.
        order (str, optional): whether insert the operator after ("post") or before ("pre") the activation layers, available choices: "post", "pre". Defaults to "post".

    Returns:
        nn.Module: converted module
    """
    assert isinstance(
        operator, (PruneLayer, QuantizeLayer)
    ), "`operator` does not belong to (PruneLayer, QuantizeLayer)"
    assert order in ["pre", "post"], "`order` must be either 'pre' or 'post'"

    closure = {'count': 0}

    filter = filter or []
    if isinstance(filter, str):
        filter = [filter]

    def met_filter_condition(module_path: str):
        return all([s in module_path for s in filter])

    if (len(weight_layers) + len(activation_layers)) == 0:
        warnings.warn(
            "No weight or activation layers specified, nothing will be converted."
        )

    def _print(msg):
        if log:
            logging.info(msg)

    def is_builtin_name(name):
        return "quantize" in name.lower() or "prune" in name.lower()

    def mstr(m: nn.Module) -> str:
        if isinstance(m, nn.Sequential):
            if len(m) > 0:
                assert len(m) == 2
                return [a for a in [mstr(m[0]), mstr(m[1])] if not is_builtin_name(a)][0]
            else:
                return None
        elif isinstance(m, nn.Module):
            return m.__class__.__name__
        else:
            return m.__name__

    def is_container(m: nn.Module) -> bool:
        if len(m._modules) == 0:
            return False
        else:
            for k in [
                "quantize",
                "prune",
                "quantize_bias",
            ]:  # a leaf module injected with qsparse layers
                if hasattr(m, k):
                    return False
            return True

    def apply_operator(layer: Optional[nn.Module] = None) -> nn.Module:
        if layer is not None:
            if isinstance(operator, QuantizeLayer):

                return quantize(layer, **copy_nn_module_on_demand(operator._kwargs))
            else:
                return prune(layer, **copy_nn_module_on_demand(operator._kwargs))
        else:
            r = copy.deepcopy(operator)
            # r.index = closure['count']
            # closure['count'] += 1
            return r

    if not inplace:
        model = copy.deepcopy(model)

    def count_occurrence(
        mod: nn.Module, layer_types: Sequence[Type[nn.Module]]
    ) -> Mapping[str, int]:
        def _count_occurrence(m: nn.Module, layer_type, scope):
            total = 0
            for name, layer in m.named_children():
                cur_scope = f"{scope}.{name}"
                if exclude is not None and exclude in cur_scope:
                    continue
                if not is_container(layer):
                    if mstr(layer) == mstr(layer_type) and met_filter_condition(
                        cur_scope
                    ):
                        total += 1
                else:
                    total += _count_occurrence(layer, layer_type, cur_scope)
            return total

        return {mstr(l): _count_occurrence(mod, l, "") for l in layer_types}

    def excluded_layer_indexes_to_dict(
        layer_indexes, layers_count
    ) -> Mapping[str, Sequence[int]]:
        return defaultdict(
            list,
            {
                mstr(cls): [
                    l if l >= 0 else (l + layers_count[mstr(cls)]) for l in indexes
                ]
                for cls, indexes in layer_indexes
            },
        )

    weight_counter = {mstr(c): 0 for c in weight_layers}
    weight_total_layers_counts = count_occurrence(model, weight_layers)
    excluded_weight_layer_indexes = excluded_layer_indexes_to_dict(
        excluded_weight_layer_indexes, weight_total_layers_counts
    )

    activation_counter = {mstr(c): 0 for c in activation_layers}
    activation_total_layers_counts = count_occurrence(model, activation_layers)
    excluded_activation_layer_indexes = excluded_layer_indexes_to_dict(
        excluded_activation_layer_indexes, activation_total_layers_counts
    )

    operator_name = str(operator).lower()
    for c in ["(", ")", "layer"]:
        operator_name = operator_name.replace(c, "")
    operator_name = f"`{operator_name}`"

    def _convert_weight(mod: nn.Module, scope: str = "") -> nn.Module:
        reassign = {}
        for name, m in mod.named_children():
            modified = False
            cur_scope = f"{scope}.{name}"
            if exclude is not None and exclude in cur_scope:
                continue

            if not is_container(m):
                if mstr(m) in weight_counter:
                    if (
                        weight_counter[mstr(m)]
                        not in excluded_weight_layer_indexes[mstr(m)]
                    ) and met_filter_condition(cur_scope):
                        _print(f"Apply {operator_name} on the {cur_scope} weight")
                        m = apply_operator(m)
                        modified = True
                    else:
                        _print(f"Exclude {cur_scope} weight")
                    weight_counter[mstr(m)] += 1
                if modified:
                    reassign[name] = m
            else:
                _convert_weight(m, cur_scope)

        for key, value in reassign.items():
            mod._modules[key] = value
        return mod

    def _convert_activation(mod: nn.Module, scope: str = "") -> nn.Module:
        reassign = {}
        for name, m in mod.named_children():
            origin_m = m
            cur_scope = f"{scope}.{name}"
            if exclude is not None and exclude in cur_scope:
                continue

            if (not is_container(m)) or hasattr(m, "_qsparse_conversion"):
                layer_type = mstr(m)
                if layer_type in activation_counter:
                    if (
                        activation_counter[layer_type]
                        not in excluded_activation_layer_indexes[layer_type]
                    ) and met_filter_condition(cur_scope):
                        _print(f"Apply {operator_name} on the {cur_scope} activation")
                        op = apply_operator()
                        if isinstance(op, PruneLayer):
                            if cur_scope in apoz:
                                op.sparsity = apoz[cur_scope]
                                apoz.pop(cur_scope)
                                logging.danger(f"assigning apoz sparsity {op.sparsity} to {cur_scope}")
                        # if spa:
                        #     last_prune_layer = None
                        if order == "post":
                            m = nn.Sequential(m, op)
                        else:
                            m = nn.Sequential(op, m)
                        # if spa:
                        #     print("connecting quantization and pruning layer")
                        #     op.callback.set_p(last_prune_layer)
                        setattr(m, "_qsparse_conversion", True)
                    else:
                        _print(f"Exclude {cur_scope} activation")
                    activation_counter[layer_type] += 1
                if origin_m is not m:
                    reassign[name] = m
            else:
                _convert_activation(m, cur_scope)

        for key, value in reassign.items():
            mod._modules[key] = value
        return mod

    def apply_to_input(mod):
        return nn.Sequential(apply_operator(), mod)

    if model == nn_module(model):
        model = _convert_weight(model)
        model = _convert_activation(model)
        if input:
            model = apply_to_input(model)
    else:
        model.module = _convert_weight(model.module)
        model.module = _convert_activation(model.module)
        if input:
            model.module = apply_to_input(model.module)
    auto_name_prune_quantize_layers(nn_module(model))
    return model