"""General purpose visualizer for semantic segmentation results on various datasets super-fueled by open3D.
"""
import os
import open3d
import torch
import numpy as np
from termcolor import colored

class ColorCompletionVisualizer:
    """Visualize meshes from various datasets with open3D. Key Events show RGB, ground truth, prediction or differences."""

    def __init__(self, dataset, save_dir: str = ""):
        """Initialize Semantic Segmentation Visualizer which shows meshes with optional prediction and ground truth

        Arguments:
            dataset  {BaseDataSet} -- Examples from which dataset we want to visualize
            save_dir {str}         -- Directory in which .ply files should be saved
        """

        # keep a pointer to the dataset to retrieve relevant information, such as color mapping
        self._dataset = dataset

        if not os.path.exists(os.path.dirname(save_dir)):
            os.makedirs(os.path.dirname(save_dir))

        assert os.path.isdir(save_dir) or save_dir == ""
        self._save_dir = save_dir

    def visualize_result(self, mesh_name, prediction=None, gt=None, mask=None):
        mesh = self._dataset.get_mesh(mesh_name)
        mesh.compute_vertex_normals()

        vis = open3d.visualization.VisualizerWithKeyCallback()
        vis.create_window(width=1600, height=1200)

        # PREPARE RGB COLOR SWITCH
        rgb_colors = open3d.utility.Vector3dVector(np.asarray(mesh.vertex_colors))

        def colorize_rgb(visu):
            mesh.vertex_colors = rgb_colors
            visu.update_geometry(mesh)
            visu.update_renderer()

        vis.register_key_callback(ord('H'), colorize_rgb)

        if type(prediction) == torch.Tensor:
            # PREPARE PREDICTION COLOR SWITCH
            pred_colors = open3d.utility.Vector3dVector(np.asarray(prediction))

            def colorize_pred(visu):
                mesh.vertex_colors = pred_colors
                visu.update_geometry(mesh)
                visu.update_renderer()

            vis.register_key_callback(ord('J'), colorize_pred)

        if type(gt) == torch.Tensor:
            # PREPARE GROUND TRUTH COLOR SWITCH
            gt_shape = np.asarray(gt).shape
            rgb_shape = np.asarray(rgb_colors).shape
            if gt_shape != rgb_shape:
                print('WARNING: ground truth shape', gt_shape, 'differs from loaded rgb shape', rgb_shape)
            gt_colors = open3d.utility.Vector3dVector(np.asarray(gt))

            def colorize_gt(visu):
                mesh.vertex_colors = gt_colors
                visu.update_geometry(mesh)
                visu.update_renderer()

            vis.register_key_callback(ord('K'), colorize_gt)

        if type(mask) == torch.Tensor:
            # PREPARE GROUND TRUTH COLOR SWITCH
            mask_colors = torch.where(mask.expand_as(gt), torch.zeros_like(gt), gt)
            mask_colors = open3d.utility.Vector3dVector(np.asarray(mask_colors))

            def colorize_mask(visu):
                mesh.vertex_colors = mask_colors
                visu.update_geometry(mesh)
                visu.update_renderer()
                print('Rendering mask')

            vis.register_key_callback(ord('M'), colorize_mask)

        if type(gt) == torch.Tensor and type(prediction) == torch.Tensor:
            # PREAPRE DIFFERENCE COLOR SWITCH
            def generate_heatvector(v, min=0.0, max=0.5):
                out = torch.zeros_like(v).unsqueeze(1).repeat(1, 3)
                ratio = 2 * (v - min) / (max - min)
                out[:, 0] = torch.maximum(torch.zeros_like(ratio), (ratio - 1))
                out[:, 2] = torch.maximum(torch.zeros_like(ratio), (1 - ratio))
                out[:, 1] = 1.0 - out[:, 0] - out[:, 2]
                return out

            d = torch.abs(gt - prediction).mean(dim=1)
            differences = generate_heatvector(d)
            differences = torch.where(mask.expand_as(differences), differences, torch.zeros_like(differences)+0.25)
            diff_colors = open3d.utility.Vector3dVector(np.asarray(differences))

            def colorize_diff(visu):
                mesh.vertex_colors = diff_colors
                visu.update_geometry(mesh)
                visu.update_renderer()

            vis.register_key_callback(ord('F'), colorize_diff)

        def take_screenshot(visu):
            path = f"{self._save_dir}/SemSegVisualizer_img.png"
            visu.capture_screen_image(path)
            print('Saving screenshot to', path)

        vis.register_key_callback(ord('P'), take_screenshot)

        def save_room(visu):
            mesh.vertex_colors = rgb_colors
            open3d.io.write_triangle_mesh(
                f"{self._save_dir}/SemSegVisualizer_{mesh_name}_rgb.ply", mesh)

            if type(prediction) == torch.Tensor:
                mesh.vertex_colors = pred_colors
                open3d.io.write_triangle_mesh(
                    f"{self._save_dir}/SemSegVisualizer_{mesh_name}_pred.ply", mesh)

            if type(gt) == torch.Tensor:
                mesh.vertex_colors = gt_colors
                open3d.io.write_triangle_mesh(
                    f"{self._save_dir}/SemSegVisualizer_{mesh_name}_gt.ply", mesh)

            if type(mask) == torch.Tensor:
                mesh.vertex_colors = mask_colors
                open3d.io.write_triangle_mesh(
                    f"{self._save_dir}/SemSegVisualizer_{mesh_name}_mask.ply", mesh)

            if type(gt) == torch.Tensor and type(prediction) == torch.Tensor:
                mesh.vertex_colors = diff_colors
                open3d.io.write_triangle_mesh(
                    f"{self._save_dir}/SemSegVisualizer_{mesh_name}_diff.ply", mesh)
            print(colored(
                f"PLY meshes successfully stored in {os.path.abspath(self._save_dir)}", 'green'))

        # Start with unlit prediction
        colorize_pred(vis)
        vis.get_render_option().light_on = False

        vis.register_key_callback(ord('D'), save_room)
        vis.add_geometry(mesh)
        vis.run()
        vis.destroy_window()


import pymongo
import json
import random
import calendar
import datetime

client = pymongo.MongoClient()
db = client.dummy_database
collection = db.sensorData

data = {}
now = calendar.timegm(datetime.datetime.now().timetuple())*1000

dates = [now + (i-5000)*100000 for i in range(10000)]
delta1 = [random.randint(-1,1)/10 for i in range(10000)]
delta2 = [random.randint(-1,1)/10 for i in range(10000)]

for i in range(10000):
    collection.insert({
        'date':dates[i],
        'oxygen':round(sum(delta1[:i])*10)/10,
        'temperature':round(sum(delta2[:i])*10)/10
        })
#print(data['oxygen'])


# -*- coding: utf-8 -*-
"""
Created on Fri Apr  3 19:09:37 2020

@author: Maldonado
"""

import sqlite3

conexion = sqlite3.connect("ejemplo.db")
cursor = conexion.cursor()
#cursor.execute("create table usuarios (nombre varchar(100), edad integer, email varchar(100))")
#cursor.execute("insert into usuarios values('Alberto', 29, 'soporte@email.com')")
# cursor.execute("select * from usuarios")
# print(cursor)

# usuario = cursor.fetchone()
# print(usuario)
# print(usuario[2])

# usuarios = [('Mauricio',29, 'soporte@tecno.com'),
#             ('Juan', 30, 'user@gmail.com'),
#             ('Maria', 56, 'ventas@comensal.com'),]
# cursor.executemany("insert into usuarios values(?,?,?)", usuarios)

cursor.execute("select * from usuarios")
usuarios = cursor.fetchall()
for usuario in usuarios:
    print(usuario)

conexion.commit()

conexion.close()


#!/usr/bin/env python

"""Module with Embedding and RankLoss classes. First one is linear embedding
with weights for features and so-called anchor point which roughly represents
position of clusters."""

__author__ = 'Anna Kukleva'
__date__ = 'August 2018'

import torch.nn as nn
import torch

from utils.arg_pars import opt

class Embedding(nn.Module):
    """Linear embedding"""
    def __init__(self, embed_dim, feature_dim, n_subact):
        super(Embedding, self).__init__()
        torch.manual_seed(opt.seed)

        self._embed_dim = embed_dim
        self._n_subact = n_subact
        self._W_a = nn.Linear(embed_dim, n_subact, bias=False)
        self._W_f = nn.Linear(feature_dim, embed_dim, bias=False)
        self._init_weights()

    def forward(self, x):
        x = self._W_f(x)
        x = self._W_a(x)
        return x

    def embedded(self, x):
        return self._W_f(x)

    def anchors(self):
        # mask = torch.ones((self._n_subact, self._embed_dim))
        return self._W_a.weight

    def _init_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Linear):
                nn.init.normal_(m.weight, 0, 0.01)

class RankLoss(nn.Module):
    def __init__(self, margin=0.3):
        super(RankLoss, self).__init__()
        self._margin = margin

    def forward(self, features, k):
        result = torch.sub(features.t(), features[k.byte()]).t()
        result = result + self._margin
        zeros = torch.zeros(result.size()).cuda()
        result = torch.max(zeros, result)
        result = torch.sum(result)
        result = result - self._margin * features.size()[0]
        result = result # / features.size()[0]
        return result


import datetime
import fcntl
import inspect
import json
import os
from types import FunctionType
from typing import Optional, Union

from apscheduler.executors.pool import ThreadPoolExecutor
from apscheduler.jobstores.memory import MemoryJobStore
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
from apscheduler.schedulers.background import BackgroundScheduler
from pytz import utc

from cnaas_nms.app_settings import app_settings
from cnaas_nms.db.job import Job
from cnaas_nms.db.session import sqla_session
from cnaas_nms.tools.log import get_logger

logger = get_logger()

class SingletonType(type):
    _instances = {}

    def __call__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super(SingletonType, cls).__call__(*args, **kwargs)
        return cls._instances[cls]

class Scheduler(object, metaclass=SingletonType):
    def __init__(self):
        threads = 10
        self.is_mule = False
        # If scheduler is already started, use uwsgi ipc to send job to mule process
        self.lock_f = open("/tmp/scheduler.lock", "w")
        try:
            fcntl.lockf(self.lock_f, fcntl.LOCK_EX | fcntl.LOCK_NB)
        except BlockingIOError:
            try:
                import uwsgi  # noqa: F401
            except Exception:
                self.use_mule = False
            else:
                self.use_mule = True
        else:
            self.use_mule = False
        caller = self.get_caller(caller=inspect.currentframe())
        if caller == "api":
            sqlalchemy_url = app_settings.POSTGRES_DSN
            self._scheduler = BackgroundScheduler(
                executors={"default": ThreadPoolExecutor(threads)},
                jobstores={"default": SQLAlchemyJobStore(url=sqlalchemy_url)},
                job_defaults={},
                timezone=utc,
            )
            logger.info("Scheduler started with persistent jobstore, {} threads".format(threads))
        elif caller == "mule":
            sqlalchemy_url = app_settings.POSTGRES_DSN
            self._scheduler = BackgroundScheduler(
                executors={"default": ThreadPoolExecutor(threads)},
                jobstores={"default": SQLAlchemyJobStore(url=sqlalchemy_url)},
                job_defaults={},
                timezone=utc,
            )
            logger.info("Scheduler started with persistent jobstore, {} threads".format(threads))
            self.is_mule = True
        elif self.use_mule:
            logger.info("Use uwsgi to send jobs to mule process")
            self._scheduler = None
        else:
            self._scheduler = BackgroundScheduler(
                executors={"default": ThreadPoolExecutor(threads)},
                jobstores={"default": MemoryJobStore()},
                job_defaults={},
                timezone=utc,
            )
            logger.info("Scheduler started with in-memory jobstore, {} threads".format(threads))

    def __del__(self):
        if self.lock_f:
            fcntl.lockf(self.lock_f, fcntl.LOCK_UN)
            self.lock_f.close()
            os.unlink("/tmp/scheduler.lock")

    def get_scheduler(self):
        return self._scheduler

    def get_caller(self, caller):
        """Check if API main run was the caller."""
        frameinfo = inspect.getframeinfo(caller.f_back.f_back)
        filename = "/".join(frameinfo.filename.split("/")[-2:])
        function = frameinfo.function
        if filename == "cnaas_nms/run.py" and function == "get_app":
            logger.info("Scheduler started from filename {} function {} (API mode)".format(filename, function))
            return "api"
        elif filename == "cnaas_nms/scheduler_mule.py":
            logger.info("Scheduler started from filename {} function {} (uwsgi mule mode)".format(filename, function))
            return "mule"
        else:
            logger.info("Scheduler started from filename {} function {} (Standalone mode)".format(filename, function))
            return "other"

    def start(self):
        if self._scheduler and not self.use_mule:
            return self._scheduler.start()

    def shutdown(self):
        if self._scheduler and not self.use_mule:
            return self._scheduler.shutdown()

    def add_local_job(self, func, **kwargs):
        """Add job to local scheduler."""
        return self._scheduler.add_job(func, **kwargs)

    def remove_local_job(self, job_id):
        """Remove job from local scheduler."""
        return self._scheduler.remove_job(str(job_id))

    def shutdown_mule(self):
        """Send a message to the mule worker to shut itself down."""
        if self.use_mule:
            try:
                import uwsgi
            except Exception as e:
                logger.exception("use_mule is set but not running in uwsgi")
                raise e
            args = {"scheduler_action": "shutdown_mule"}
            uwsgi.mule_msg(json.dumps(args))

    def remove_scheduled_job(self, job_id, abort_message="removed"):
        """Remove scheduled job from mule worker or local scheduler depending
        on setup."""
        if self.use_mule:
            try:
                import uwsgi
            except Exception as e:
                logger.exception("use_mule is set but not running in uwsgi")
                raise e
            args = {"scheduler_action": "remove", "id": str(job_id)}
            uwsgi.mule_msg(json.dumps(args))
        else:
            self.remove_local_job(job_id)

        with sqla_session() as session:
            job = session.query(Job).filter(Job.id == job_id).one_or_none()
            job.finish_abort(message=abort_message)

    def add_onetime_job(
        self, func: Union[str, FunctionType], when: Optional[int] = None, scheduled_by: Optional[str] = None, **kwargs
    ) -> int:
        """Schedule a job to run at a later time on the mule worker or
        local scheduler depending on setup.

        Some extra checks against kwargs are performed here. If kwarg
        with name 'dry_run' is included, (dry_run) is appended to function
        name. If kwarg job_comment or job_ticket_ref are included, those
        fields in the job will be populated.

        Args:
            func: The function to call
            when: Optional number of seconds to wait before starting job
            scheduled_by: Username that scheduled the job
            **kwargs: Arguments to pass through to called function
        Returns:
            int: job_id
        """
        if when and isinstance(when, int):
            trigger = "date"
            run_date = datetime.datetime.utcnow() + datetime.timedelta(seconds=when)
        else:
            trigger = None
            run_date = None

        if isinstance(func, FunctionType):
            func_qualname = str(func.__qualname__)
        else:
            func_qualname = str(func)
        func_name = func_qualname.split(":")[-1]

        try:
            json.dumps(kwargs)
        except TypeError as e:
            raise TypeError("Job args must be JSON serializable: {}".format(e))

        # Append (dry_run) to function name if set, so we can distinguish dry_run jobs
        try:
            if kwargs["kwargs"]["dry_run"]:
                func_name += " (dry_run)"
        except Exception:
            pass

        with sqla_session() as session:
            job = Job()
            if run_date:
                job.scheduled_time = run_date
            job.function_name = func_name
            if scheduled_by is None:
                scheduled_by = "unknown"
            job.scheduled_by = scheduled_by
            job_comment = kwargs["kwargs"].pop("job_comment", None)
            if job_comment and isinstance(job_comment, str):
                job.comment = job_comment[:255]
            job_ticket_ref = kwargs["kwargs"].pop("job_ticket_ref", None)
            if job_ticket_ref and isinstance(job_comment, str):
                job.ticket_ref = job_ticket_ref[:32]
            job.start_arguments = kwargs["kwargs"]
            session.add(job)
            session.flush()
            job_id = job.id

        kwargs["job_id"] = job_id
        kwargs["scheduled_by"] = scheduled_by
        kwargs["kwargs"]["scheduled_by"] = scheduled_by
        if self.use_mule:
            try:
                import uwsgi
            except Exception as e:
                logger.exception("use_mule is set but not running in uwsgi")
                raise e
            args = dict(kwargs)
            args["func"] = func_qualname
            args["trigger"] = trigger
            args["when"] = when
            args["id"] = str(job_id)
            uwsgi.mule_msg(json.dumps(args))
            return job_id
        else:
            self.add_local_job(
                func, trigger=trigger, kwargs=kwargs, id=str(job_id), run_date=run_date, name=func_qualname
            )
            return job_id


# Author: Salvatore Masecchia <salvatore.masecchia@disi.unige.it>
# License: New BSD

import numpy as np

## Prox and Proj operators ----------------------------------------------------
from utils import discrete_derivate, discrete_derivate_conj, \
                  interval_projection, prox_l1

# Unchanged functions ---------------------------------------------------------
from l12_B import c_phi, prox_phi

## Auxiliary functions --------------------------------------------------------
def c_psi(Y, Theta0, B0, lambda_, muw, zeta):
    L, J = B0.shape
    DB0 = np.empty((L-1, J))
    discrete_derivate(B0, DB0)
    gap0_psi = (
        (0.5 * np.sum((Y - np.dot(B0, Theta0))**2)) +
        (lambda_ * np.sum(np.abs(B0))) +
        (np.sum( (muw) * np.abs(DB0))) +
        (np.sum(B0*B0) / zeta)
    )
    C_psi = np.sqrt(gap0_psi * 2.*zeta)
    return C_psi

## CGH DL steps ---------------------------------------------------------------
def prox_psi(B, zeta, Theta, Y, muw, lambda_, eps, maxN=1e5, init=None):
    """ Fixed Theta """

    L, J = B.shape
    S = Theta.shape[1]

    # Initializations
    BNorm2 = np.sum(B*B)
    Zeta = np.empty_like(B)
    Zeta_aux = np.empty_like(B)
    Dconj_U3 = np.empty_like(B)
    Dconj_V3 = np.empty_like(B)
    DZeta = np.empty((L-1, J))

    mw = muw.ravel().reshape(-1, 1)

    if init is None:
        V1, V2, V3 = (np.zeros((L, S)), np.zeros((L, J)), np.zeros((L-1, J)))
    else:
        V1, V2, V3 = (V.copy() for V in init)
    U1, U2, U3 = V1.copy(), V2.copy(), V3.copy()
    V1_prev, V2_prev, V3_prev = (np.empty_like(V1), np.empty_like(V2),
                                 np.empty_like(V3))

    gamma = 1.0/(zeta * (np.linalg.norm(np.dot(Theta, Theta.T)) + 5.0))
    t = 1.

    # Duality gap value
    gap = None

    for n in xrange(int(maxN)):
        t_prev = t
        V1_prev, V2_prev, V3_prev = V1.copy(), V2.copy(), V3.copy()
        discrete_derivate_conj(U3, Dconj_U3)
        Zeta_aux = B - zeta*(np.dot(U1, Theta.T) + U2 + Dconj_U3)

        # Data fit:
        V1 = (1./(1. + gamma)) * (U1 + gamma*(np.dot(Zeta_aux, Theta) - Y))

        # L1 norm
        grad = U2 + gamma*Zeta_aux
        interval_projection(grad, lambda_, V2)

        # Weighted Total variation
        discrete_derivate(Zeta_aux, DZeta)
        interval_projection(U3 + gamma*DZeta, mw, V3)

        # Solution Update
        discrete_derivate_conj(V3, Dconj_V3)
        Zeta = B - zeta*(np.dot(V1, Theta.T) + V2 + Dconj_V3)

        if not (n % 10):
            discrete_derivate(Zeta, DZeta)
            primal = (
                (0.5 * np.sum((Y - np.dot(Zeta, Theta))**2)) +     # Data fit
                (lambda_ * np.sum(np.abs(Zeta)) ) +                # L1
                (np.sum( muw * np.abs(DZeta)) ) +                  # TV
                (np.sum((Zeta - B)**2) / (2.*zeta))                # Prox
            )
            dual = (
              ((np.sum(Zeta*Zeta) - BNorm2) / (2.*zeta)) +         # Prox*
              (0.5*np.sum(V1**2) + np.sum(V1*Y))                   # Fit*
              # Tv and L1 not included because their duals are 0
            )

            gap = primal+dual

        t = (1. + np.sqrt(1. + 4.*t_prev*t_prev)) * 0.5
        U1 = V1 + ((t_prev - 1.) / t) * (V1 - V1_prev)
        U2 = V2 + ((t_prev - 1.) / t) * (V2 - V2_prev)
        U3 = V3 + ((t_prev - 1.) / t) * (V3 - V3_prev)

        if gap <= (eps*eps)/(2.*zeta):
            break

    return Zeta, gap, (V1, V2, V3), n


import sys

from PySide6.QtWidgets import QMainWindow, QComboBox, QApplication, QLabel, QPushButton

class spinner(QMainWindow):
    def __init__(self):
        super().__init__()

        self.spinner=QComboBox(self)
        lista = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10"]
        self.spinner.addItems(lista)
        self.resultado=QLabel(self)

        self.boton=QPushButton("boton", self)
        self.boton.move(100, 0)
        self.boton.clicked.connect(self.mostrar)
        self.resultado.move(100, 100)

    def mostrar(self):
        indice_seleccionado = self.spinner.currentIndex()
        numero = self.spinner.itemText(indice_seleccionado)
        print("El numero elegido es:", numero)

if __name__ == "__main__":
    app = QApplication([])
    ventana1 = spinner()
    ventana1.show()
    sys.exit(app.exec())

02 Generacion de Interfaces Graficas/04 Layouts y Spinner/spinner.py


#!/usr/bin/env python3

from enum import Enum
import itertools

class ConnectionType(Enum):
    JumpTrue = 'JT'
    JumpFalse = 'JF'
    CallTrue = 'CT'
    CallFalse = 'CF'
    ReturnTrue = 'RT'
    ReturnFalse = 'RF'
    NextBlock = 'NB'

class Instruction:
    def __init__(self, name: str, is_branching: bool):
        self.name: str = name
        self.is_branching: bool = is_branching

class Language:
    def __init__(self,
                 jmp_instructions: list[Instruction],
                 call_instructions: list[Instruction],
                 ret_instructions: list[Instruction],
                 interrupt_instructions: list[Instruction],
                 interrupt_addresses: list[int]):
        self.jmp_instructions: list[Instruction] = jmp_instructions
        self.call_instructions: list[Instruction] = call_instructions
        self.ret_instructions: list[Instruction] = ret_instructions
        self.interrupt_instructions: list[Instruction] = interrupt_instructions
        self.interrupt_addresses: list[int] = interrupt_addresses

class DisassembledLine:
    def __init__(self, address: str, code: list[str]):
        self.is_leader: bool = False
        self.address: int = int(address, 16)
        self.code: list[str] = code
        self.instruction: str = code[0].strip()
        self.argument: str = code[1].strip() if len(code) > 1 else None

    def set_leader(self):
        self.is_leader = True

    def __str__(self):
        leader_str = 'L' if self.is_leader else ' '
        return f"{leader_str} {self.address:04x} {' '.join(self.code)}"

    @staticmethod
    def from_raw_string(raw_string: str):
        split = raw_string.split(' ')
        return DisassembledLine(split[0], split[1:])

class BasicBlock:
    _new_id = itertools.count()

    def __init__(self, lines: list[DisassembledLine]):
        self.id: int = next(self._new_id)
        self.lines: list[DisassembledLine] = lines
        self.successors: list[tuple[BasicBlock, ConnectionType]] = []
        self.is_entry_block: bool = False
        self.is_exit_block: bool = False

    def add_successor(self, connected_block: 'BasicBlock', connection_type: ConnectionType):
        self.successors.append((connected_block, connection_type))

    def line_from_address(self, address: int):
        for line in self.lines:
            if line.address == address:
                return line

        raise LookupError(f"Could not find {address:04x} in the basic block:\n{self}")

    def __str__(self):
        id_str = f'id: {self.id}'

        entry_exit = ''
        if self.is_entry_block and self.is_exit_block:
            entry_exit = ' (entry, exit)'
        elif self.is_entry_block:
            entry_exit = ' (entry)'
        elif self.is_exit_block:
            entry_exit = ' (exit)'

        connections_str = ', connected to: ' \
                          + ', '.join(map(lambda x: str(x[0].id) + " (" + str(x[1].value) + ")", self.successors))

        lines_str = '\n'.join(map(lambda x: str(x), self.lines))

        return id_str + entry_exit + connections_str + '\n' + lines_str


from TCR_Autoencoder.tcr_autoencoder import PaddingAutoencoder
import torch
import torch.autograd as autograd
import torch.nn as nn
import torch.nn.functional as F

class PathologyClassifier(nn.Module):
    def __init__(self, embedding_dim, device, max_len, input_dim, encoding_dim, output_dim,
                 batch_size, ae_file, train_ae=True):
        super(PathologyClassifier, self).__init__()
        # GPU
        self.device = device
        # Dimensions
        self.embedding_dim = embedding_dim
        self.max_len = max_len
        self.input_dim = input_dim
        self.batch_size = batch_size
        # TCR Autoencoder
        self.autoencoder = PaddingAutoencoder(max_len, input_dim, encoding_dim)
        checkpoint = torch.load(ae_file)
        self.autoencoder.load_state_dict(checkpoint['model_state_dict'])
        if train_ae is False:
            for param in self.autoencoder.parameters():
                param.requires_grad = False
            self.autoencoder.eval()
        self.mlp = nn.Sequential(nn.Linear(encoding_dim, 50),
                                 nn.Tanh(),
                                 nn.Linear(50, output_dim),
                                 nn.Softmax(dim=1))

    def forward(self, padded_tcrs):
        # TCR Encoder:
        # Embedding
        concat = padded_tcrs.view(self.batch_size, self.max_len * self.input_dim)
        encoded_tcrs = self.autoencoder.encoder(concat)
        # MLP Classifier
        mlp_output = self.mlp(encoded_tcrs)
        return mlp_output

    pass

class MLP(nn.Module):
    def __init__(self, input_dim, output_dim, device):
        super(MLP, self).__init__()
        # GPU
        self.device = device
        # Dimensions
        self.input_dim = input_dim
        self.output_dim = output_dim
        self.mlp = nn.Sequential(nn.Linear(input_dim, 50),
                                 nn.Tanh(),
                                 nn.Linear(50, output_dim),
                                 nn.Softmax())

    def forward(self, encoded_tcrs):
        # MLP Classifier
        output = self.mlp(encoded_tcrs)
        return output

    pass


import csv
import cv2
import numpy as np
import matplotlib.pyplot as plt

from keras.models import Sequential
from keras.layers import Flatten, Dense, Cropping2D, Convolution2D, Lambda
from keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint
from keras.utils.visualize_util import plot

from datetime import datetime

# Helper function to load data from a directory
def load_data(data_dir, X, y):

    with open(data_dir + '\\driving_log.csv') as csv_file:
        reader = csv.reader(csv_file)
        headers = next(reader)

        for line in reader:
            center_image_file_path = data_dir + '\\' + line[headers.index('center')]

            image = cv2.imread(center_image_file_path)
            steer = float(line[headers.index('steering')])

            X.append(image)
            y.append(steer)

            # add flipped image to dataset
            X.append(np.fliplr(image))
            y.append(-steer)

    return

def train(X, y):

    # define nvidia model
    model = Sequential()
    model.add(Lambda(lambda x : (x / 255.0) -0.5, input_shape = (160, 320, 3)))
    model.add(Cropping2D(cropping = ((70, 25), (0, 0)), input_shape = (160, 320, 3)))
    model.add(Convolution2D(24, 5, 5, subsample = (2, 2), activation = 'relu'))
    model.add(Convolution2D(36, 5, 5, subsample = (2, 2), activation = 'relu'))
    model.add(Convolution2D(48, 5, 5, subsample = (2, 2), activation = 'relu'))
    model.add(Convolution2D(64, 3, 3, activation = 'relu'))
    model.add(Convolution2D(64, 3, 3, activation = 'relu'))

    model.add(Flatten())
    model.add(Dense(100))
    model.add(Dense(50))
    model.add(Dense(10))
    model.add(Dense(1))

    model.compile(loss = 'mse', optimizer = 'adam')

    plot(model, to_file='model.png', show_shapes=True)

    # configure early_stopping_callback to prevent overfitting
    checkpoint_callback = ModelCheckpoint('./model_checkpoints/model_{epoch:02d}.h5',  verbose = 1)
    history = model.fit(X, y, validation_split = 0.2, shuffle = True, nb_epoch = 20, callbacks = [checkpoint_callback])

    plt.plot(history.history['loss'])
    plt.plot(history.history['val_loss'])
    plt.title('model loss')
    plt.ylabel('loss')
    plt.xlabel('epoch')
    plt.legend(['train', 'validation'], loc='upper left')
    plt.savefig('model_loss.png')

    return

if __name__ == '__main__':

    X = []
    y = []

    print(datetime.now())

    # load data from various directories in X, y arrays
    load_data('.\\data\\sample_data', X, y)
    load_data('.\\data\\lap', X, y)
    load_data('.\\data\\specific_training_1', X, y)
    load_data('.\\data\\specific_training_2', X, y)
    load_data('.\\data\\specific_training_3', X, y)

    print(datetime.now())

    train(np.array(X), np.array(y))

    print(datetime.now())


# -*- coding: utf-8 -*-
from metadata import tvshow_meta, season_episodes_meta, all_episodes_meta
from apis.trakt_api import trakt_fetch_collection_watchlist, trakt_get_hidden_items, trakt_get_my_calendar
from modules import kodi_utils, settings, watched_status as ws
from modules.utils import jsondate_to_datetime, adjust_premiered_date, make_day, get_datetime, title_key, date_difference, make_thread_list_enumerate
# logger = kodi_utils.logger

remove_meta_keys, set_view_mode, focus_index, external_browse = kodi_utils.remove_meta_keys, kodi_utils.set_view_mode, kodi_utils.focus_index, kodi_utils.external_browse
add_items, set_content, set_sort_method, end_directory = kodi_utils.add_items, kodi_utils.set_content, kodi_utils.set_sort_method, kodi_utils.end_directory
ls, make_listitem, build_url, dict_removals, sys = kodi_utils.local_string, kodi_utils.make_listitem, kodi_utils.build_url, kodi_utils.episode_dict_removals, kodi_utils.sys
get_art_provider, show_specials, calendar_sort_order, ignore_articles = settings.get_art_provider, settings.show_specials, settings.calendar_sort_order, settings.ignore_articles
nextep_content_settings, nextep_display_settings, calendar_focus_today = settings.nextep_content_settings, settings.nextep_display_settings, settings.calendar_focus_today
metadata_user_info, watched_indicators_info, show_unaired_info = settings.metadata_user_info, settings.watched_indicators, settings.show_unaired
date_offset_info, default_all_episodes = settings.date_offset, settings.default_all_episodes
single_ep_display_title, single_ep_format = settings.single_ep_display_title, settings.single_ep_format
tv_meta_function, season_meta_function, all_episodes_meta_function = tvshow_meta, season_episodes_meta, all_episodes_meta
adjust_premiered_date_function, jsondate_to_datetime_function = adjust_premiered_date, jsondate_to_datetime
date_difference_function, make_day_function, title_key_function, get_datetime_function = date_difference, make_day, title_key, get_datetime
get_progress_percent, get_watched_status, get_watched_info, get_bookmarks = ws.get_progress_percent, ws.get_watched_status_episode, ws.get_watched_info_tv, ws.get_bookmarks
get_in_progress_episodes, get_next_episodes, get_recently_watched = ws.get_in_progress_episodes, ws.get_next_episodes, ws.get_recently_watched
string, david_str, trakt_str, watched_str, unwatched_str, extras_str, options_str, clearprog_str = str, ls(32036), ls(32037), ls(32642), ls(32643), ls(32645), ls(32646), ls(32651)
upper = string.upper
poster_empty, fanart_empty = kodi_utils.empty_poster, kodi_utils.addon_fanart
run_plugin, unaired_label, tmdb_poster = 'RunPlugin(%s)', '[COLOR red][I]%s[/I][/COLOR]', 'https://image.tmdb.org/t/p/'

def build_episode_list(params):
	def _process():
		for item in episodes_data:
			try:
				cm = []
				listitem = make_listitem()
				set_properties = listitem.setProperties
				cm_append = cm.append
				item_get = item.get
				season, episode, ep_name = item_get('season'), item_get('episode'), item_get('title')
				episode_date, premiered = adjust_premiered_date_function(item_get('premiered'), adjust_hours)
				playcount, overlay = get_watched_status(watched_info, string(tmdb_id), season, episode)
				thumb = item_get('thumb', None) or fanart
				item.update({'trailer': trailer, 'tvshowtitle': title, 'premiered': premiered, 'genre': genre, 'duration': duration, 'mpaa': mpaa, 'studio': studio,
							'playcount': playcount, 'overlay': overlay})
				options_params = build_url({'mode': 'options_menu_choice', 'content': 'episode', 'tmdb_id': tmdb_id, 'season': season, 'episode': episode,
											'poster': show_poster, 'is_widget': is_widget})
				extras_params = build_url({'mode': 'extras_menu_choice', 'tmdb_id': tmdb_id, 'media_type': 'tvshow', 'is_widget': is_widget})
				url_params = build_url({'mode': 'play_media', 'media_type': 'episode', 'tmdb_id': tmdb_id, 'season': season, 'episode': episode})
				if not episode_date or current_date < episode_date:
					if not show_unaired: continue
					if season != 0:
						display, unaired = unaired_label % ep_name, True
						item['title'] = display
				else: display, unaired = ep_name, False
				cm_append((extras_str, run_plugin % extras_params))
				cm_append((options_str, run_plugin % options_params))
				clearprog_params, unwatched_params, watched_params = '', '', ''
				if not unaired:
					if playcount:
						if hide_watched: continue
						unwatched_params = build_url({'mode': 'mark_as_watched_unwatched_episode', 'action': 'mark_as_unwatched', 'tmdb_id': tmdb_id,
													'tvdb_id': tvdb_id, 'season': season, 'episode': episode,  'title': title, 'year': year})
						cm_append((unwatched_str % watched_title, run_plugin % unwatched_params))
					else:
						watched_params = build_url({'mode': 'mark_as_watched_unwatched_episode', 'action': 'mark_as_watched', 'tmdb_id': tmdb_id,
													'tvdb_id': tvdb_id, 'season': season, 'episode': episode,  'title': title, 'year': year})
						cm_append((watched_str % watched_title, run_plugin % watched_params))
					progress = get_progress_percent(bookmarks, tmdb_id, season, episode)
					if progress:
						clearprog_params = build_url({'mode': 'watched_unwatched_erase_bookmark', 'media_type': 'episode', 'tmdb_id': tmdb_id,
													'season': season, 'episode': episode, 'refresh': 'true'})
						cm_append((clearprog_str, run_plugin % clearprog_params))
						set_properties({'watchedprogress': progress, 'resumetime': progress, 'david_in_progress': 'true'})
				listitem.setLabel(display)
				listitem.addContextMenuItems(cm)
				listitem.setArt({'poster': show_poster, 'fanart': fanart, 'thumb': thumb, 'icon':thumb, 'banner': banner, 'clearart': clearart, 'clearlogo': clearlogo,
								'landscape': thumb, 'season.poster': season_poster, 'tvshow.poster': show_poster, 'tvshow.clearart': clearart, 'tvshow.clearlogo': clearlogo})
				listitem.setCast(cast + item_get('guest_stars', []))
				listitem.setUniqueIDs({'imdb': imdb_id, 'tmdb': string(tmdb_id), 'tvdb': string(tvdb_id)})
				listitem.setInfo('video', remove_meta_keys(item, dict_removals))
				if is_widget:
					set_properties({'david_widget': 'true', 'david_playcount': string(playcount), 'david_extras_params': extras_params, 'david_options_params': options_params,
									'david_unwatched_params': unwatched_params, 'david_watched_params': watched_params, 'david_clearprog_params': clearprog_params})
				else: set_properties({'david_widget': 'false'})
				yield (url_params, listitem, False)
			except: pass
	handle = int(sys.argv[1])
	try:
		item_list = []
		append = item_list.append
		meta_user_info, watched_indicators, watched_info, show_unaired, is_widget, fanart_enabled, hide_watched, current_date, adjust_hours, bookmarks = get_episode_info()
		poster_main, poster_backup, fanart_main, fanart_backup, clearlogo_main, clearlogo_backup = get_art_provider()
		all_episodes = True if params.get('season') == 'all' else False
		meta = tv_meta_function('tmdb_id', params.get('tmdb_id'), meta_user_info, current_date)
		meta_get = meta.get
		tmdb_id, tvdb_id, imdb_id = meta_get('tmdb_id'), meta_get('tvdb_id'), meta_get('imdb_id')
		title, year, rootname = meta_get('title'), meta_get('year'), meta_get('rootname')
		show_poster = meta_get(poster_main) or meta_get(poster_backup) or poster_empty
		fanart = meta_get(fanart_main) or meta_get(fanart_backup) or fanart_empty
		clearlogo = meta_get(clearlogo_main) or meta_get(clearlogo_backup) or ''
		if fanart_enabled: banner, clearart, landscape = meta_get('banner'), meta_get('clearart'), meta_get('landscape')
		else: banner, clearart, landscape = '', '', ''
		try:
			poster_path = [i['poster_path'] for i in meta_get('season_data') if i['season_number'] == int(params['season'])][0]
			season_poster =  ''.join([tmdb_poster, meta_user_info['image_resolution']['poster'], poster_path]) if poster_path is not None else show_poster
		except: season_poster = show_poster
		cast, mpaa, duration = meta_get('cast', []), meta_get('mpaa'), meta_get('duration')
		trailer, genre, studio = string(meta_get('trailer')), meta_get('genre'), meta_get('studio')
		tvshow_plot = meta_get('plot')
		watched_title = trakt_str if watched_indicators == 1 else david_str
		if all_episodes:
			episodes_data = all_episodes_meta_function(meta, meta_user_info)
			if not show_specials(): episodes_data = [i for i in episodes_data if not i['season'] == 0]
		else: episodes_data = season_meta_function(params['season'], meta, meta_user_info)
		add_items(handle, list(_process()))
	except: pass
	set_content(handle, 'episodes')
	set_sort_method(handle, 'episodes')
	end_directory(handle, False if is_widget else None)
	if not is_widget: set_view_mode('view.episodes', 'episodes')

def build_single_episode(list_type, data, handle):
	def _sort_results(items):
		if list_type_starts_with('next_episode'):
			def func(function):
				if sort_key == 'david_name': return title_key_function(function, ignore_articles_setting)
				elif sort_key == 'david_last_played': return jsondate_to_datetime_function(function, resformat)
				else: return function
			sort_key = nextep_settings['sort_key']
			sort_direction = nextep_settings['sort_direction']
			if nextep_settings['sort_airing_today_to_top']:
				airing_today = [i for i in items
								if date_difference_function(current_date, jsondate_to_datetime_function(i[1].getProperty('david_first_aired'), '%Y-%m-%d').date(), 0)]
				airing_today = sorted(airing_today, key=lambda i: i[1].getProperty('david_first_aired'))
				remainder = [i for i in items if not i in airing_today]
				remainder = sorted(remainder, key=lambda i: func(i[1].getProperty(sort_key)), reverse=sort_direction)
				unaired = [i for i in remainder if i[1].getProperty('david_unaired') == 'true']
				aired = [i for i in remainder if not i in unaired]
				remainder = aired + unaired
				items = airing_today + remainder
			else:
				items = sorted(items, key=lambda i: func(i[1].getProperty(sort_key)), reverse=sort_direction)
				unaired = [i for i in items if i[1].getProperty('david_unaired') == 'true']
				aired = [i for i in items if not i in unaired]
				items = aired + unaired
		else:
			items.sort(key=lambda k: int(k[1].getProperty('david_sort_order')))
			if list_type in ('trakt_calendar', 'trakt_recently_aired'):
				if list_type == 'trakt_calendar': reverse = calendar_sort_order() == 0
				else: reverse = True
				items.sort(key=lambda k: int(k[1].getProperty('david_sort_order')))
				items = sorted(items, key=lambda i: i[1].getProperty('david_first_aired'), reverse=reverse)
		return items
	def _process(item_position, ep_data):
		try:
			cm = []
			listitem = make_listitem()
			set_properties = listitem.setProperties
			cm_append = cm.append
			ep_data_get = ep_data.get
			meta = tv_meta_function('trakt_dict', ep_data_get('media_ids'), meta_user_info, current_date)
			if not meta: return
			meta_get = meta.get
			tmdb_id, tvdb_id, imdb_id = meta_get('tmdb_id'), meta_get('tvdb_id'), meta_get('imdb_id')
			title, year, rootname = meta_get('title'), meta_get('year'), meta_get('rootname')
			orig_season, orig_episode = ep_data_get('season'), ep_data_get('episode')
			show_poster = meta_get(poster_main) or meta_get(poster_backup) or poster_empty
			fanart = meta_get(fanart_main) or meta_get(fanart_backup) or fanart_empty
			clearlogo = meta_get(clearlogo_main) or meta_get(clearlogo_backup) or ''
			if fanart_enabled: banner, clearart, landscape = meta_get('banner'), meta_get('clearart'), meta_get('landscape')
			else: banner, clearart, landscape = '', '', ''
			try:
				poster_path = [i['poster_path'] for i in meta_get('season_data') if i['season_number'] == orig_season][0]
				season_poster =  ''.join([tmdb_poster, meta_user_info['image_resolution']['poster'], poster_path]) if poster_path is not None else show_poster
			except: season_poster = show_poster
			cast, mpaa, duration, tvshow_plot = meta_get('cast', []), meta_get('mpaa'), meta_get('duration'), meta_get('plot')
			trailer, genre, studio = string(meta_get('trailer')), meta_get('genre'), meta_get('studio')
			if list_type_starts_with('next_episode'):
				season_data = meta_get('season_data')
				curr_season_data = [i for i in season_data if i['season_number'] == orig_season][0]
				if orig_episode >= curr_season_data['episode_count']: orig_season, orig_episode, new_season = orig_season + 1, 1, True
				else: orig_episode, new_season = orig_episode + 1, False
			episodes_data = season_meta_function(orig_season, meta, meta_user_info)
			try: item = [i for i in episodes_data if i['episode'] == orig_episode][0]
			except: return
			item_get = item.get
			season, episode, ep_name = item_get('season'), item_get('episode'), item_get('title')
			str_season_zfill2, str_episode_zfill2 = string(season).zfill(2), string(episode).zfill(2)
			episode_date, premiered = adjust_premiered_date_function(item_get('premiered'), adjust_hours)
			if not episode_date or current_date < episode_date:
				if list_type_starts_with('next_episode'):
					if not nextep_include_unaired: return
					if episode_date and new_season and not date_difference_function(current_date, episode_date, 7): return
				elif not show_unaired: return
				unaired = True
				set_properties({'david_unaired': 'true'})
			else:
				unaired = False
				set_properties({'david_unaired': 'false'})
			playcount, overlay = get_watched_status(watched_info, string(tmdb_id), season, episode)
			if display_title == 0: title_string = ''.join([title, ': '])
			else: title_string = ''
			if display_title in (0,1): seas_ep = ''.join([str_season_zfill2, 'x', str_episode_zfill2, ' - '])
			else: seas_ep = ''
			if list_type_starts_with('next_episode'):
				unwatched = ep_data_get('unwatched', False)
				if episode_date: display_premiered = make_day_function(current_date, episode_date, date_format)
				else: display_premiered == 'UNKNOWN'
				airdate = ''.join(['[[COLOR magenta]', display_premiered, '[/COLOR]] ']) if nextep_include_airdate else ''
				highlight_color = nextep_unwatched_color if unwatched else nextep_unaired_color if unaired else ''
				italics_open, italics_close = ('[I]', '[/I]') if highlight_color else ('', '')
				if highlight_color: episode_info = ''.join([italics_open, '[COLOR', highlight_color, ']', seas_ep, ep_name, '[/COLOR]', italics_close])
				else: episode_info = ''.join([italics_open, seas_ep, ep_name, italics_close])
				display = ''.join([airdate, upper(title_string), episode_info])
			elif list_type == 'trakt_calendar':
				if episode_date: display_premiered = make_day_function(current_date, episode_date, date_format)
				else: display_premiered == 'UNKNOWN'
				display = ''.join(['[', display_premiered, '] ', upper(title_string), seas_ep, ep_name])
				if unaired:
					displays = display.split(']')
					display = ''.join(['[COLOR red]', displays[0], '][/COLOR]', displays[1]])
			else:
				color_tags = ('[COLOR red]', '[/COLOR]') if unaired else ('', '')
				display = ''.join([upper(title_string), color_tags[0], seas_ep, ep_name, color_tags[1]])
			thumb = item_get('thumb', None) or fanart
			item.update({'trailer': trailer, 'tvshowtitle': title, 'premiered': premiered, 'genre': genre, 'duration': duration,
						'mpaa': mpaa, 'studio': studio, 'playcount': playcount, 'overlay': overlay, 'title': display})
			options_params = build_url({'mode': 'options_menu_choice', 'content': 'episode_single', 'tmdb_id': tmdb_id, 'season': season, 'episode': episode,
										'poster': show_poster, 'is_widget': is_widget})
			extras_params = build_url({'mode': 'extras_menu_choice', 'tmdb_id': tmdb_id, 'media_type': 'tvshow', 'is_widget': is_widget})
			url_params = build_url({'mode': 'play_media', 'media_type': 'episode', 'tmdb_id': tmdb_id, 'season': season, 'episode': episode})
			cm_append((extras_str, run_plugin % extras_params))
			cm_append((options_str, run_plugin % options_params))
			clearprog_params, unwatched_params, watched_params = '', '', ''
			if not unaired:
				if playcount:
					if hide_watched: return
					unwatched_params = build_url({'mode': 'mark_as_watched_unwatched_episode', 'action': 'mark_as_unwatched', 'tmdb_id': tmdb_id,
												'tvdb_id': tvdb_id, 'season': season, 'episode': episode,  'title': title, 'year': year})
					cm_append((unwatched_str % watched_title, run_plugin % unwatched_params))
				else:
					watched_params = build_url({'mode': 'mark_as_watched_unwatched_episode', 'action': 'mark_as_watched', 'tmdb_id': tmdb_id,
												'tvdb_id': tvdb_id, 'season': season, 'episode': episode,  'title': title, 'year': year})
					cm_append((watched_str % watched_title, run_plugin % watched_params))
				progress = get_progress_percent(bookmarks, tmdb_id, season, episode)
				if progress:
					clearprog_params = build_url({'mode': 'watched_unwatched_erase_bookmark', 'media_type': 'episode', 'tmdb_id': tmdb_id,
												'season': season, 'episode': episode, 'refresh': 'true'})
					cm_append((clearprog_str, run_plugin % clearprog_params))
					set_properties({'WatchedProgress': progress, 'resumetime': progress, 'david_in_progress': 'true'})
			listitem.setLabel(display)
			listitem.addContextMenuItems(cm)
			listitem.setArt({'poster': show_poster, 'fanart': fanart, 'thumb': thumb, 'icon':thumb, 'banner': banner, 'clearart': clearart, 'clearlogo': clearlogo,
				'landscape': thumb, 'season.poster': season_poster, 'tvshow.clearart': clearart, 'tvshow.clearlogo': clearlogo, 'tvshow.landscape': thumb, 'tvshow.banner': banner})
			listitem.setCast(cast + item_get('guest_stars', []))
			listitem.setUniqueIDs({'imdb': imdb_id, 'tmdb': string(tmdb_id), 'tvdb': string(tvdb_id)})
			listitem.setInfo('video', remove_meta_keys(item, dict_removals))
			set_properties({'david_widget': 'false', 'david_first_aired': premiered, 'david_name': '%s - %sx%s' % (title, str_season_zfill2, str_episode_zfill2)})
			if list_type_starts_with('next_episode'):
				last_played = ep_data_get('last_played', resinsert)
				set_properties({'david_last_played': last_played})
			else: set_properties({'david_sort_order': string(item_position)})
			if is_widget:
				set_properties({'david_widget': 'true', 'david_playcount': string(playcount), 'david_extras_params': extras_params, 'david_options_params': options_params,
								'david_unwatched_params': unwatched_params, 'david_watched_params': watched_params, 'david_clearprog_params': clearprog_params})
			append((url_params, listitem, False))
		except: pass
	try:
		item_list = []
		append = item_list.append
		list_type_starts_with = list_type.startswith
		meta_user_info, watched_indicators, watched_info, show_unaired, is_widget, fanart_enabled, hide_watched, current_date, adjust_hours, bookmarks = get_episode_info()
		display_title, date_format, art_keys, all_episodes = get_single_episode_info()
		show_all_episodes = all_episodes in (1, 2)
		poster_main, poster_backup, fanart_main, fanart_backup, clearlogo_main, clearlogo_backup = art_keys
		ignore_articles_setting = ignore_articles()
		watched_title = trakt_str if watched_indicators == 1 else david_str
		if list_type_starts_with('next_episode'):
			nextep_settings, nextep_disp_settings = nextep_content_settings(), nextep_display_settings()
			nextep_unaired_color, nextep_unwatched_color = nextep_disp_settings['unaired_color'], nextep_disp_settings['unwatched_color']
			nextep_include_airdate, nextep_include_unaired = nextep_disp_settings['include_airdate'], nextep_settings['include_unaired']
			if watched_indicators == 1: resformat, resinsert = '%Y-%m-%dT%H:%M:%S.%fZ', '2000-01-01T00:00:00.000Z'
			else: resformat, resinsert = '%Y-%m-%d %H:%M:%S', '2000-01-01 00:00:00'
		threads = list(make_thread_list_enumerate(_process, data))
		[i.join() for i in threads]
		item_list = _sort_results(item_list)
		add_items(handle, item_list)
	except: pass
	set_content(handle, 'episodes')
	end_directory(handle, cacheToDisc=False)
	if not is_widget: set_view_mode('view.episodes', 'episodes')
	if list_type == 'trakt_calendar' and calendar_focus_today():
		try:
			today = upper('[%s]' % ls(32849))
			index = max([i for i, x in enumerate([i[1].getLabel() for i in item_list]) if today in x])
			focus_index(index)
		except: pass

def get_episode_info():
	meta_user_info = metadata_user_info()
	watched_indicators = watched_indicators_info()
	watched_info = get_watched_info(watched_indicators)
	show_unaired = show_unaired_info()
	is_widget = external_browse()
	fanart_enabled = meta_user_info['extra_fanart_enabled']
	hide_watched = is_widget and meta_user_info['widget_hide_watched']
	current_date = get_datetime_function()
	adjust_hours = date_offset_info()
	bookmarks = get_bookmarks(watched_indicators, 'episode')
	return meta_user_info, watched_indicators, watched_info, show_unaired, is_widget, fanart_enabled, hide_watched, current_date, adjust_hours, bookmarks

def get_single_episode_info():
	display_title = single_ep_display_title()
	date_format = single_ep_format()
	art_keys = get_art_provider()
	all_episodes = default_all_episodes()
	return display_title, date_format, art_keys, all_episodes

def build_in_progress_episode():
	data = get_in_progress_episodes()
	build_single_episode('progress', data, int(sys.argv[1]))

def build_recently_watched_episode():
	data = get_recently_watched('episode')[0]
	build_single_episode('recently_watched', data, int(sys.argv[1]))

def build_next_episode():
	nextep_settings = nextep_content_settings()
	include_unwatched = nextep_settings['include_unwatched']
	indicators = watched_indicators_info()
	watched_info = get_watched_info(indicators)
	data = get_next_episodes(watched_info)
	if indicators == 1:
		list_type = 'next_episode_trakt'
		try:
			hidden_data = trakt_get_hidden_items('progress_watched')
			data = [i for i in data if not i['media_ids']['tmdb'] in hidden_data]
		except: pass
	else: list_type = 'next_episode_david'
	if include_unwatched:
		try: unwatched = [{'media_ids': i['media_ids'], 'season': 1, 'episode': 0, 'unwatched': True} for i in trakt_fetch_collection_watchlist('watchlist', 'tvshow')]
		except: unwatched = []
		data += unwatched
	build_single_episode(list_type, data, int(sys.argv[1]))

def build_my_calendar(params):
	recently_aired = params.get('recently_aired', None)
	data = trakt_get_my_calendar(recently_aired, get_datetime())
	list_type = 'trakt_recently_aired' if recently_aired else 'trakt_calendar'
	data = sorted(data, key=lambda k: (k['sort_title'], k['first_aired']), reverse=True)
	build_single_episode(list_type, data, int(sys.argv[1]))


# Задание №6
# 📌 Создать страницу, на которой будет форма для ввода имени
# и возраста пользователя и кнопка "Отправить"
# 📌 При нажатии на кнопку будет произведена проверка
# возраста и переход на страницу с результатом или на
# страницу с ошибкой в случае некорректного возраста.
import logging
from pathlib import PurePath, Path

from flask import Flask, request, render_template, flash, abort
from werkzeug.utils import secure_filename

app = Flask(__name__)

logger = logging.getLogger(__name__)

@app.route('/')
@app.route('/index/')
def html_index():
    return render_template('index.html')

@app.route('/hello_world/')
def hello():
    return 'Hello world!'

@app.errorhandler(403)
def page_not_found(e):
    logger.warning(e)
    context = {
        'title': 'Доступ запрещен',
        'url': request.base_url
    }
    return render_template('403.html', **context), 403

@app.route('/send/', methods=['GET', 'POST'])
def send():
    if request.method == 'POST':
        age = int(request.form.get('age'))
        if age < 18:
            abort(403)
        return f'Привет, вы успешно вошли в систему'
    return render_template('send.html')

if __name__ == "__main__":
    app.run(debug=True)


import gsw
import numpy as np
import xarray as xr

from compass.step import Step

class Combine(Step):
    """
    A step for combining a January and an annual WOA 2023 climatology into
    a single file, where the January data is used where available and the
    annual in the deeper ocean where monthly data is not provided.
    """

    def __init__(self, test_case):
        """
        Create a new step

        Parameters
        ----------
        test_case : compass.ocean.tests.utility.extrap_woa.ExtraWoa
            The test case this step belongs to
        """
        super().__init__(test_case, name='combine', ntasks=1, min_tasks=1)
        self.add_output_file(filename='woa_combined.nc')

    def setup(self):
        """
        Set up the step in the work directory, including downloading any
        dependencies.
        """
        super().setup()

        base_url = \
            'https://www.ncei.noaa.gov/thredds-ocean/fileServer/woa23/DATA'

        dirs = dict(
            temp=dict(ann='temperature/netcdf/decav91C0/0.25',
                      jan='temperature/netcdf/decav91C0/0.25'),
            salin=dict(ann='salinity/netcdf/decav91C0/0.25',
                       jan='salinity/netcdf/decav91C0/0.25'))

        woa_files = dict(
            temp=dict(ann='woa23_decav91C0_t00_04.nc',
                      jan='woa23_decav91C0_t01_04.nc'),
            salin=dict(
                ann='woa23_decav91C0_s00_04.nc',
                jan='woa23_decav91C0_s01_04.nc'))

        for field in ['temp', 'salin']:
            for season in ['jan', 'ann']:
                woa_dir = dirs[field][season]
                woa_filename = woa_files[field][season]
                woa_url = f'{base_url}/{woa_dir}/{woa_filename}'

                self.add_input_file(
                    filename=f'woa_{field}_{season}.nc',
                    target=woa_filename,
                    database='initial_condition_database',
                    url=woa_url)

    def run(self):
        """
        Run this step of the test case
        """
        ds_ann = xr.open_dataset('woa_temp_ann.nc', decode_times=False)

        ds_out = xr.Dataset()

        for var in ['lon', 'lat', 'depth']:
            ds_out[var] = ds_ann[var]
            ds_out[f'{var}_bnds'] = ds_ann[f'{var}_bnds']

        var_map = dict(temp='t_an', salin='s_an')

        for field, var in var_map.items():
            slices = list()
            ds_ann = xr.open_dataset(
                f'woa_{field}_ann.nc',
                decode_times=False).isel(time=0).drop_vars('time')
            ds_jan = xr.open_dataset(
                f'woa_{field}_jan.nc',
                decode_times=False).isel(time=0).drop_vars('time')
            for index in range(ds_ann.sizes['depth']):
                if index < ds_jan.sizes['depth']:
                    ds = ds_jan
                else:
                    ds = ds_ann
                slices.append(ds[var].isel(depth=index))

            ds_out[var] = xr.concat(slices, dim='depth')
            ds_out[var].attrs = ds_ann[var].attrs

        ds_out = self._temp_to_pot_temp(ds_out)
        ds_out.to_netcdf('woa_combined.nc')

    @staticmethod
    def _temp_to_pot_temp(ds):
        dims = ds.t_an.dims

        slices = list()
        for depth_index in range(ds.sizes['depth']):
            temp_slice = ds.t_an.isel(depth=depth_index)
            in_situ_temp = temp_slice.values
            salin = ds.s_an.isel(depth=depth_index).values
            lat = ds.lat.broadcast_like(temp_slice).values
            lon = ds.lon.broadcast_like(temp_slice).values
            z = -ds.depth.isel(depth=depth_index).values
            pressure = gsw.p_from_z(z, lat)
            mask = np.isfinite(in_situ_temp)
            SA = gsw.SA_from_SP(salin[mask], pressure[mask], lon[mask],
                                lat[mask])
            pot_temp = np.nan * np.ones(in_situ_temp.shape)
            pot_temp[mask] = gsw.pt_from_t(SA, in_situ_temp[mask],
                                           pressure[mask], p_ref=0.)
            pot_temp_slice = xr.DataArray(data=pot_temp, dims=temp_slice.dims,
                                          attrs=temp_slice.attrs)

            slices.append(pot_temp_slice)

        ds['pt_an'] = xr.concat(slices, dim='depth').transpose(*dims)

        ds.pt_an.attrs['standard_name'] = \
            'sea_water_potential_temperature'
        ds.pt_an.attrs['long_name'] = \
            'Objectively analyzed mean fields for ' \
            'sea_water_potential_temperature at standard depth levels.'

        ds = ds.drop_vars('t_an')
        return ds


import matplotlib.pyplot as plt
import numpy as np

f = input("Enter Frequency of sine wave :  ")

Fs = 8000
time = 8000
x = np.arange(time)
y = np.sin(2 * np.pi * f * x / Fs)
# print(x)
plt.ylim(-1.5,1.5)
plt.plot(x, y)
plt.xlabel('t')
plt.ylabel('y(t) (sine wave)')
plt.grid(True)
plt.show()


import sys
import pickle
from pathlib import Path

from dotenv import find_dotenv, load_dotenv
# find .env automagically by walking up directories until it's found, then
# load up the .env entries as environment variables
load_dotenv(find_dotenv())

from tqdm import tqdm

import h5py
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import torch
import torch.nn as nn

from torchvision import transforms, models
from torch.utils.data import DataLoader
from nnunet.training.model_restore import restore_model
from nnunet.training.network_training.nnUNetTrainerV2 import nnUNetTrainerV2
from nnunet.training.network_training.nnUNet_variants.architectural_variants.nnUNetTrainerV2_ResencUNet import nnUNetTrainerV2_ResNetUNet

# local imports
from src.data import ADNIDatasetClassification
from src.net import BraTSnnUNet, load_from_wandb

PROJ_ROOT = Path('/home/jupyter/gama/bruno')
DATASET_FPATH = PROJ_ROOT/'data/interim/ADNI123_slices_fix_2mm_split_class.hdf5'

if __name__ == '__main__':
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

    # PARSE ARGS
    args = sys.argv
    model, run_id, split = args[-3:]

    model = model.lower()
    split = split.lower()

    assert model in ['unet', 'resnet']
    assert split in ['val', 'test']

    # LOAD DATA
    group_labels = ['CN', 'MCI', 'AD']
    dataset = ADNIDatasetClassification(
        DATASET_FPATH,
        get_age=True,
        dataset=split,
        labels=group_labels,
        transform=transforms.Compose([
            transforms.ToTensor(),
        ]),

    )
    dataloader = DataLoader(dataset, batch_size=40, shuffle=False)

    h = lambda x: x*25+75

    # LOAD MODEL
    if split == 'val':
        wandb_model_file = 'model_best'
    elif split == 'test':
        wandb_model_file = 'model_last'

    nnunet_trainer_kwargs = dict(
        dataset_directory='/home/jupyter/gama/nnUNet/data/processed/Task102_BraTS2020',
        batch_dice=True,
        stage=0,
        unpack_data=True,
        deterministic=False,
        fp16=True,
    )
    if model == 'unet':
        nnunet_trainer = nnUNetTrainerV2(
            '/home/jupyter/gama/nnUNet/data/processed/Task102_BraTS2020/nnUNetPlansv2.1_plans_2D.pkl',
            0,
            output_folder='/home/jupyter/gama/nnUNet/models/nnUNet/2d/Task102_BraTS2020/nnUNetTrainerV2__nnUNetPlansv2.1',
            **nnunet_trainer_kwargs,
        )
        nnunet_trainer.initialize(False)

        net = BraTSnnUNet(nnunet_trainer.network)
        net.pooling = nn.AvgPool2d(3)

        net = load_from_wandb(net, run_id, model_fname=wandb_model_file).to(device)
    elif model == 'resnet':
        nnunet_trainer = nnUNetTrainerV2_ResNetUNet(
            '/home/jupyter/gama/nnUNet/data/processed/Task102_BraTS2020/nnUNetPlans_ResNetUNet_v2.1_plans_2D.pkl',
            0,
            output_folder='/home/jupyter/gama/nnUNet/models/nnUNet/2d/Task102_BraTS2020/nnUNetTrainerV2_ResNetUNet__nnUNetPlans_ResNetUNet_v2.1',
            **nnunet_trainer_kwargs,
        )
        nnunet_trainer.initialize(False)

        resnet_backbone = nnunet_trainer.network.encoder
        resnet_backbone.default_return_skips = False
        net = nn.Sequential(
            resnet_backbone,
            nn.AdaptiveAvgPool2d(output_size=(1,1)),
            nn.Flatten(1),
            nn.Linear(2048, 1),
        )

    net = load_from_wandb(net, run_id, model_fname=wandb_model_file).to(device)
    net.eval()

    # MAKE PREDICTIONS
    age_hats = list()
    ages = list()
    groups = list()
    for X, a, y in tqdm(dataloader):
        group = group_labels[y.median().item()]

        X = X.repeat((1,4,1,1)).to(device)  # fix input channels

        with torch.no_grad():
            a_hat = h(net(X))

        age_hats.append(a_hat.mean().item())
        ages.append(a.median().item())
        groups.append(group)

    # SAVE PREDICTIONS
    with open(f'/home/jupyter/gama/bruno/data/preds/predictions_{model}_{run_id}_{split}.pkl', 'wb') as f:
        pickle.dump({
            'age_hats': age_hats,
            'ages': ages,
            'groups': groups,
        }, f)


from PIL import Image
import numpy as np
import os
import torch
from torch.utils.data.dataset import Dataset
import torchvision.transforms as T
import torchvision.transforms.functional as F
import random
import cv2
import albumentations as A
import json
import platform

import xml.etree.ElementTree as xmlET

CLASS_NAME_LIST = [#'__background__', # always index 0
                    'aeroplane',
                    'bicycle',
                    'bird',
                    'boat',
                    'bottle',
                    'bus',
                    'car',
                    'cat',
                    'chair',
                    'cow',
                    'diningtable',
                    'dog',
                    'horse',
                    'motorbike',
                    'person',
                    'pottedplant',
                    'sheep',
                    'sofa',
                    'train',
                    'tvmonitor']

def readXML(xml_path):
    # return ndarry: [[x0,y0,x1,y1,class],..]
    # value:0-1
    labels = []

    tree = xmlET.parse(xml_path)

    size = tree.findall('size')
    w = float(size[0].find('width').text)
    h = float(size[0].find('height').text)

    objs = tree.findall('object')
    for ix, obj in enumerate(objs):
        tmp = []
        bbox = obj.find('bndbox')
        xmin = float(bbox.find('xmin').text)/w
        ymin = float(bbox.find('ymin').text)/h
        xmax = float(bbox.find('xmax').text)/w
        ymax = float(bbox.find('ymax').text)/h

        cla = obj.find('name').text
        if cla not in CLASS_NAME_LIST:
            raise Exception("class name not in CLASS_NAME_LIST")

        labels.append(np.array([xmin, ymin, xmax, ymax, CLASS_NAME_LIST.index(cla)]))

    return labels

###### 1.Data aug
class TrainDataAug:
    def __init__(self, h, w):
        self.h = h
        self.w = w

    def __call__(self, img):
        # opencv img, BGR
        # new_width, new_height = self.size[0], self.size[1]
        img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)
        # raw_h, raw_w = img.shape[:2]
        # min_size = max(img.shape[:2])

        # input_h, input_w = img.shape[:2]
        # h_ratio = input_h/self.h
        # w_ratio = input_w/self.w
        # if (h_ratio>1 and w_ratio>1):
        #     if h_ratio>w_ratio:
        #         resize_h = self.h
        #         resize_w = int(input_w/h_ratio)
        #     else:
        #         resize_w = self.w
        #         resize_h = int(input_h/w_ratio)
        #     img = A.Resize(resize_h,resize_w,cv2.INTER_AREA)(image=img)['image']
        #     min_size = max(img.shape[:2])

        # img = A.OneOf([A.PadIfNeeded(min_height=min_size, min_width=min_size,
        #                 border_mode=3, value=0, mask_value=0,
        #                 always_apply=False, p=0.7),
        #             A.PadIfNeeded(min_height=min_size, min_width=min_size,
        #                 border_mode=0, value=0, mask_value=0,
        #                 always_apply=False, p=0.3)],
        #                 p=1.0)(image=img)['image']

        # img = A.ShiftScaleRotate(
        #                         shift_limit=0.1,
        #                         scale_limit=0.1,
        #                         rotate_limit=20,
        #                         interpolation=cv2.INTER_LINEAR,
        #                         border_mode=cv2.BORDER_CONSTANT,
        #                          value=0, mask_value=0,
        #                         p=0.6)(image=img)['image']

        # img = A.HorizontalFlip(p=0.5)(image=img)['image']

        # img = A.OneOf([A.RandomBrightness(limit=0.1, p=1),
        #             A.RandomContrast(limit=0.1, p=1),
        #             A.RandomGamma(gamma_limit=(50, 150),p=1),
        #             A.HueSaturationValue(hue_shift_limit=10,
        #                 sat_shift_limit=10, val_shift_limit=10,  p=1)],
        #             p=0.8)(image=img)['image']

        img = A.Resize(self.h,self.w,p=1)(image=img)['image']
        # img = A.OneOf([A.MotionBlur(blur_limit=3, p=0.2),
        #                 A.MedianBlur(blur_limit=3, p=0.2),
        #                 A.GaussianBlur(blur_limit=3, p=0.1),
        #                 A.GaussNoise(var_limit=(10.0, 50.0), mean=0, p=0.5)],
        #                 p=0.8)(image=img)['image']

        img = Image.fromarray(img)
        return img

class ValTestDataAug:
    def __init__(self, h, w):
        self.h = h
        self.w = w

    def __call__(self, img):
        # opencv img, BGR
        # new_width, new_height = self.size[0], self.size[1]

        img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)

        img = A.Resize(self.h,self.w,p=1)(image=img)['image']
        img = Image.fromarray(img)
        return img

######## 2.dataloader

class TensorDatasetTestBackbone(Dataset):

    def __init__(self, train_jpg, transform=None):
        self.train_jpg = train_jpg
        if transform is not None:
            self.transform = transform
        else:
            self.transform = None

    def __getitem__(self, index):
        # img = cv2.imread(self.train_jpg[index])
        # img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        #img = cv2.resize(img,(180, 180))

        #img = Image.open(self.train_jpg[index]).convert('RGB')
        img = cv2.imread(self.train_jpg[index])
        #img = imgPaddingWrap(img)
        #b
        if self.transform is not None:
            img = self.transform(img)

        return img, self.train_jpg[index]

    def __len__(self):
        return len(self.train_jpg)

class DatasetTrainVal(Dataset):

    def __init__(self, data_type, voc_dir, data_sets, transform=None):
        self.data_type = data_type
        self.voc_dir = voc_dir
        self.data_sets = data_sets
        self.transform = transform

        self.img_list = []
        self.xml_list = []
        self._getDataList()

    def _getDataList(self):

        if self.data_type == 'trainval':
            name_file = 'trainval.txt'
            for data_set in self.data_sets:
                base_dir = os.path.join(self.voc_dir, data_set)
                name_path = os.path.join(base_dir, "ImageSets","Main",name_file)
                with open(name_path, 'r') as f:
                    lines = f.readlines()
                for line in lines:
                    name = line.strip()
                    img_path = os.path.join(base_dir, 'JPEGImages', name+".jpg")
                    xml_path = os.path.join(base_dir, 'Annotations', name+'.xml')
                    self.img_list.append(img_path)
                    self.xml_list.append(xml_path)

        elif self.data_type == 'train':
            name_file = 'train.txt'
            for data_set in self.data_sets:
                base_dir = os.path.join(self.voc_dir, data_set)
                name_path = os.path.join(base_dir, "ImageSets","Main",name_file)
                with open(name_path, 'r') as f:
                    lines = f.readlines()
                for line in lines:
                    name = line.strip()
                    img_path = os.path.join(base_dir, 'JPEGImages', name+".jpg")
                    xml_path = os.path.join(base_dir, 'Annotations', name+'.xml')
                    self.img_list.append(img_path)
                    self.xml_list.append(xml_path)

        elif self.data_type == 'val':
            name_file = 'val.txt'
            for data_set in self.data_sets:
                base_dir = os.path.join(self.voc_dir, data_set)
                name_path = os.path.join(base_dir, "ImageSets","Main",name_file)
                with open(name_path, 'r') as f:
                    lines = f.readlines()
                for line in lines:
                    name = line.strip()
                    img_path = os.path.join(base_dir, 'JPEGImages', name+".jpg")
                    xml_path = os.path.join(base_dir, 'Annotations', name+'.xml')
                    self.img_list.append(img_path)
                    self.xml_list.append(xml_path)

        elif self.data_type == 'test':
            name_file = 'test.txt'
            for data_set in self.data_sets:
                base_dir = os.path.join(self.voc_dir, data_set)
                name_path = os.path.join(base_dir, "ImageSets","Main",name_file)
                with open(name_path, 'r') as f:
                    lines = f.readlines()
                for line in lines:
                    name = line.strip()
                    img_path = os.path.join(base_dir, 'JPEGImages', name+".jpg")
                    xml_path = os.path.join(base_dir, 'Annotations', name+'.xml')
                    self.img_list.append(img_path)
                    self.xml_list.append(xml_path)

    def __getitem__(self, index):

        img_path = self.img_list[index]
        # img_path = os.path.join(self.voc_dir, 'JPEGImages', name+".jpg")
        # print(img_path)
        img = cv2.imread(img_path)
        h,w = img.shape[:2]

        if self.transform:
            img = self.transform(img)

        # xml_path = os.path.join(self.voc_dir, 'Annotations', name+'.xml')
        xml_path = self.xml_list[index]
        # print(xml_path)
        # b
        labels = readXML(xml_path)

        #print(np.array(labels).shape)
        # print(img.shape, torch.from_numpy(np.array(labels)).shape)
        #bb
        return img, torch.from_numpy(np.array(labels)), len(labels), img_path, w, h

    def __len__(self):
        return len(self.img_list)

class DatasetTest(Dataset):

    def __init__(self, data_type, voc_dir, data_sets, transform=None):
        self.data_type = data_type
        self.voc_dir = voc_dir
        self.data_sets = data_sets
        self.transform = transform

        self._getDataList()

    def _getDataList(self):
        self.img_list = []
        self.xml_list = []
        if self.data_type == 'trainval':
            name_file = 'trainval.txt'
            for data_set in self.data_sets:
                base_dir = os.path.join(self.voc_dir, data_set)
                name_path = os.path.join(base_dir, "ImageSets","Main",name_file)
                with open(name_path, 'r') as f:
                    lines = f.readlines()
                for line in lines:
                    name = line.strip()
                    img_path = os.path.join(base_dir, 'JPEGImages', name+".jpg")
                    xml_path = os.path.join(base_dir, 'Annotations', name+'.xml')
                    self.img_list.append(img_path)
                    self.xml_list.append(xml_path)

        #

    def __getitem__(self, index):

        img_path = self.img_list[index]
        # img_path = os.path.join(self.voc_dir, 'JPEGImages', name+".jpg")
        # print(img_path)
        img = cv2.imread(img_path)
        h,w = img.shape[:2]

        if self.transform:
            img = self.transform(img)

        # xml_path = os.path.join(self.voc_dir, 'Annotations', name+'.xml')
        xml_path = self.xml_list[index]
        # print(xml_path)
        # b
        labels = readXML(xml_path)

        #print(np.array(labels).shape)
        # print(img.shape, torch.from_numpy(np.array(labels)).shape)
        #bb
        return img, torch.from_numpy(np.array(labels)), len(labels), img_path,w, h

    def __len__(self):
        return len(self.img_list)

class SubtractMeans(object):
    def __init__(self, mean):
        self.mean = np.array(mean, dtype=np.float32)

    def __call__(self, img):
        # image = image.astype(np.float32)
        img = np.array(img, dtype=np.float32) - self.mean
        img = torch.from_numpy(img)

        img = img.permute(2,0,1) #hwc->chw

        return img

###### 3. get data loader

def collate_fn(batch_data):
    """
    默认的batch合并函数是直接把不同item直接拼接
    但是目标检测中的label，不同图片的目标框数量不同
    直接使用就会报错 stack expects each tensor to be equal size,
    but got [2, 5] at entry 0 and [11, 5] at entry 1

    所以需要直接实现合并的函数
    """
    # print(len(batch_data)) #=batchsize
    # print(len(batch_data[0])) #= len(return thing)  here is 3 (img, torch.from_numpy(np.array(labels)), name)
    # print(batch_data[0][0].shape)  # [3,300,300]
    # print(batch_data[0][1].shape)  # [n_obj, 5]
    # print(batch_data[0][2])        #name

    # max_obj = max([x[1].shape[0] for x in batch_data])
    # print(batch_data[0][1].shape, batch_data[1][1].shape, max_obj)

    def _padBatchBox(box_tensor_list):
        max_obj = max([x.shape[0] for x in box_tensor_list])

        # print(box_tensor_list[0].shape, box_tensor_list[1].shape)
        box_tensor_list_with_same_shape = [torch.cat((x, torch.Tensor([[0]*x.shape[1]]*(max_obj-x.shape[0])))) for x in box_tensor_list]
        # print(box_tensor_list_with_same_shape[0].shape, box_tensor_list_with_same_shape[1].shape)
        # b
        return box_tensor_list_with_same_shape

    batch_img = torch.stack([x[0] for x in batch_data], 0)
    batch_box = torch.stack(_padBatchBox([x[1] for x in batch_data]), 0)
    batch_len_obj = [x[2] for x in batch_data]
    batch_name = [x[3] for x in batch_data]
    return batch_img, batch_box, batch_len_obj, batch_name

def getDataLoader(mode, voc_dir, data_sets, img_size, batch_size, kwargs):
    if platform.system() == "Windows":
        num_workers = 0
    else:
        num_workers = 4

    if mode=="train":

        train_loader = torch.utils.data.DataLoader(DatasetTrainVal("trainval",
                                                        voc_dir,
                                                        data_sets,
                                                        T.Compose([
                                                            TrainDataAug(img_size, img_size),
                                                            # SubtractMeans([123, 117, 104])#bgr
                                                            T.ToTensor(),
                                                            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
                                                        ])
                                                    ),
                                                    collate_fn = collate_fn,
                                                    batch_size=batch_size,
                                                    shuffle=True,
                                                    drop_last=True,
                                                    **kwargs
                                                    )

        return train_loader

    elif mode=="trainval":

        train_loader = torch.utils.data.DataLoader(DatasetTrainVal("train",
                                                        voc_dir,
                                                        data_sets,
                                                        T.Compose([
                                                            TrainDataAug(img_size, img_size),
                                                            T.ToTensor(),
                                                            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
                                                        ])
                                                    ),
                                                    collate_fn = collate_fn,
                                                    batch_size=batch_size,
                                                    shuffle=True,
                                                    drop_last=True,
                                                    **kwargs
                                                    )

        val_loader = torch.utils.data.DataLoader(DatasetTrainVal("val",
                                                        voc_dir,
                                                        data_sets,
                                                        T.Compose([
                                                            ValTestDataAug(img_size, img_size),
                                                            T.ToTensor(),
                                                            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
                                                        ])
                                                    ),
                                                    batch_size=batch_size,
                                                    shuffle=False,
                                                    num_workers=kwargs['num_workers'],
                                                    pin_memory=kwargs['pin_memory']
                                                    )

        return train_loader,val_loader

    elif mode=="val":
        val_loader = torch.utils.data.DataLoader(DatasetTrainVal("val",
                                                        voc_dir,
                                                        data_sets,
                                                        T.Compose([
                                                            ValTestDataAug(img_size, img_size),
                                                            T.ToTensor(),
                                                            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
                                                        ])
                                                    ),
                                                    batch_size=batch_size,
                                                    shuffle=False,
                                                    num_workers=kwargs['num_workers'],
                                                    pin_memory=kwargs['pin_memory']
                                                    )

        return val_loader

    elif mode=="eval":
        val_loader = torch.utils.data.DataLoader(DatasetTrainVal("test",
                                                        voc_dir,
                                                        data_sets,
                                                        T.Compose([
                                                            ValTestDataAug(img_size, img_size),
                                                            T.ToTensor(),
                                                            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
                                                        ])
                                                    ),
                                                    batch_size=batch_size,
                                                    shuffle=False,
                                                    num_workers=kwargs['num_workers'],
                                                    pin_memory=kwargs['pin_memory']
                                                    )

        return val_loader

    elif mode=="test":
        test_loader = torch.utils.data.DataLoader(DatasetTest("test",
                                                        voc_dir,
                                                        data_sets,
                                                        T.Compose([
                                                            ValTestDataAug(img_size, img_size),
                                                            T.ToTensor(),
                                                            T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
                                                        ])
                                                    ),
                                                    batch_size=1,
                                                    shuffle=False,
                                                    num_workers=kwargs['num_workers'],
                                                    pin_memory=kwargs['pin_memory']
                                                    )

        return test_loader

    elif mode=="testBackbone":
        test_loader = torch.utils.data.DataLoader(
                        TensorDatasetTestBackbone(data_list,
                        T.Compose([
                                    ValTestDataAug(img_size, img_size),
                                    T.ToTensor(),
                                    T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
                                    ])
                        ),
                        batch_size=batch_size,
                        shuffle=False,
                        num_workers=kwargs['num_workers'],
                        pin_memory=kwargs['pin_memory']
                        )

        return test_loader


import os
import shutil
from utils import copy_file, get_dir_length, read_file

class GithubScraper:
    """
    Scrape python files from github repo and save to buffer
    """

    def __init__(self, repo, owner, dest, search_root=''):
        self.repo = repo
        self.owner = owner
        self.dest = dest
        self.search_root = search_root

    def run(self):
        self.clone_repo()
        paths = self.search()
        idx = get_dir_length(self.dest)

        for path in paths:
            copy_file(path, os.path.join(self.dest, 'file{}.py'.format(idx)))
            idx += 1

        shutil.rmtree(self.repo)

    def clone_repo(self):
        repo_url = 'https://github.com/{}/{}.git'.format(self.owner, self.repo)
        os.system('git clone ' + repo_url + ' --depth 1')

    def search(self):
        file_paths = []
        for root, dirs, files in os.walk(os.path.join(self.repo, self.search_root)):
            file_paths += [os.path.join(root, f) for f in files if f.endswith('.py')]

        return file_paths

    @staticmethod
    def scrape_from_csv(filepath='../resources/repos.csv', dest='../data/buffer'):
        data = read_file(filepath)
        for line in data.split('\n'):
            if line == '':
                continue
            name, author = line.split(',')
            g = GithubScraper(name, author, dest)
            try:
                g.run()
            except Exception:
                print('Error scraping repo {} with name{}'.format(name, author))

if __name__ == '__main__':
    GithubScraper.scrape_from_csv()


# -*- coding:utf-8 -*-
import os
import time
import requests
from flask import Flask, render_template, request

app = Flask(__name__)
app.config['SECRET_KEY'] = '******'

@app.route('/', methods=['GET', 'POST'])
def index():
    weather = ''
    if request.method == 'POST':
        if 'query' in request.form.keys():
            city_name = request.form['city_name']
            weather = get_weather(city_name)
        elif 'history' in request.form.keys():
            history = get_history()
            return render_template('index.html', info=history)
        elif 'help' in request.form.keys():
            help_info = get_help()
            return render_template('index.html', info=help_info)
    return render_template('index.html', info=weather)

# 获取帮助信息
def get_help():
    return ['输入城市名，获取城市天气信息;', '单击帮助按钮，获取帮助文档;', '单击历史按钮，获取查询历史。']

# 历史文件的创建
def history_file():
    if os.path.isfile('history.txt') is False:
        return open('history.txt', 'w', encoding='utf-8')
    else:
        return open('history.txt', 'a+', encoding='utf-8')

# 查询历史
def get_history():
    if os.path.isfile('history.txt') is False:
        return ['您暂未查询任何城市天气信息']
    else:
        history_info = []
        with open('history.txt', 'r', encoding='utf-8') as history:
            for i in history:
                history_info.append(i)
            return history_info

# 查询信息并记录
def get_weather(keyword):

    history = history_file()

    r = requests.get('https://api.thinkpage.cn/v3/weather/now.json?key=r6fdjn8dz8hghngf&location=' + keyword + '&language=zh-Hans&unit=c')

    if r.status_code == 200:
        message_dict = r.json()['results'][0]
        city_name = message_dict['location']['name']
        temperature = message_dict['now']['temperature']
        text = message_dict['now']['text']

        history.write(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) + " " + city_name + " " + text + " " +
                      temperature + '度' + '\n')

        return [city_name + '现在的天气为' + text + ',温度是' + temperature + '度']
    else:
        return ['查询不到您输入的城市天气信息，请重新输入']

# 启动服务器
if __name__ == '__main__':
    app.run(debug=True)


import pandas as pd
import numpy as np
from math import sqrt
from numpy import concatenate
from matplotlib import pyplot
from pandas import read_csv
from pandas import DataFrame
from pandas import concat
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
import tensorflow as tf
from datetime import datetime

dataset = pd.read_csv('D:\ML Datasets\household_power_consumption.txt', parse_dates={'dt' : ['Date', 'Time']}, infer_datetime_format=True, index_col= 0, na_values=['nan','?'], sep=';')

dataset.fillna(0, inplace=True)
values = dataset.values
# ensure all data is float
values = values.astype('float32')

# normalizing input features
scaler = MinMaxScaler(feature_range=(0, 1))
scaled = scaler.fit_transform(values)
scaled = pd.DataFrame(scaled)

def create_ts_data(dataset, lookback=1, predicted_col=1):
    temp=dataset.copy()
    temp["id"]= range(1, len(temp)+1)
    temp = temp.iloc[:-lookback, :]
    temp.set_index('id', inplace =True)
    predicted_value=dataset.copy()
    predicted_value = predicted_value.iloc[lookback:,predicted_col]
    predicted_value.columns=["Predcited"]
    predicted_value= pd.DataFrame(predicted_value)

    predicted_value["id"]= range(1, len(predicted_value)+1)
    predicted_value.set_index('id', inplace =True)
    final_df= pd.concat([temp, predicted_value], axis=1)
    #final_df.columns = ['var1(t-1)', 'var2(t-1)', 'var3(t-1)', 'var4(t-1)', 'var5(t-1)', 'var6(t-1)', 'var7(t-1)', 'var8(t-1)','var1(t)']
    #final_df.set_index('Date', inplace=True)
    return final_df

reframed_df= create_ts_data(scaled, 1,0)
reframed_df.fillna(0, inplace=True)
reframed_df.columns = ['var1(t-1)', 'var2(t-1)', 'var3(t-1)', 'var4(t-1)', 'var5(t-1)', 'var6(t-1)', 'var7(t-1)','var1(t)']
# display(reframed_df.head(4))

y=reframed_df['var1(t)']
X=reframed_df.drop(['var1(t)'], axis=1)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)

print("X_train.shape: ", X_train.shape)
print("y_train.shape: ", y_train.shape)
print("X_test.shape: ", X_test.shape)
print("y_test.shape: ", y_test.shape)

# reshape input to be 3D [samples, time steps, features]
X_train = X_train.values.reshape((X_train.shape[0], 1, X_train.shape[1]))
X_test = X_test.values.reshape((X_test.shape[0], 1, X_test.shape[1]))
print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)

lstm_model = Sequential()
lstm_model.add(LSTM(75, return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])))
lstm_model.add(LSTM(units=30, return_sequences=True))
lstm_model.add(LSTM(units=30))
lstm_model.add(Dense(units=1))
lstm_model.compile(loss='mae', optimizer='adam')
lstm_model.summary()

history_lstm = lstm_model.fit(X_train, y_train, epochs=10, batch_size=64, validation_data=(X_test, y_test),  shuffle=False)

# Plotting Loss
pyplot.plot(history_lstm.history['loss'], label='LSTM train', color='red')
pyplot.plot(history_lstm.history['val_loss'], label='LSTM test', color= 'green')
pyplot.legend()
pyplot.show()

# make a prediction
yhat_test = lstm_model.predict(X_test)
X_test = X_test.reshape((X_test.shape[0], 7))

# invert scaling for forecast
print(yhat_test.shape, X_test.shape)
inv_yhat_test = np.concatenate((yhat_test, X_test[:, -6:]), axis=1)
print(inv_yhat_test.shape)
inv_yhat_test = scaler.inverse_transform(inv_yhat_test)
inv_yhat_test = inv_yhat_test[:,0]

# invert scaling for actual
y_test = y_test.values.reshape((len(y_test), 1))
inv_y = np.concatenate((y_test, X_test[:, -6:]), axis=1)
inv_y = scaler.inverse_transform(inv_y)
inv_y = inv_y[:,0]

# calculate RMSE
rmse = np.sqrt(mean_squared_error(inv_y, inv_yhat_test))
print('Test RMSE: %.3f' % rmse)

# Compare predictions with actual values
## time steps, every step is one hour (you can easily convert the time step to the actual time index)
## for a demonstration purpose, I only compare the predictions in 200 hours.
pyplot.figure(figsize=(12,8))
aa=[x for x in range(200)]
pyplot.plot(aa, inv_y[:200], marker='.', label="actual")
pyplot.plot(aa, inv_yhat_test[:200], 'r', label="prediction")
pyplot.ylabel('Global_active_power', size=15)
pyplot.xlabel('Time step', size=15)
pyplot.legend(fontsize=15)
pyplot.show()


import os
import torch
from rouge import Rouge
from datasets import load_dataset
from transformers import (AutoTokenizer,AutoModelForSeq2SeqLM)
import time
start = time.time()

def __init__():
    model_ckpt = 'model/Laysumm/PRIMERA-arxiv-TrainPLUStest-4096'
    tokenizer = AutoTokenizer.from_pretrained(model_ckpt)
    model = AutoModelForSeq2SeqLM.from_pretrained(model_ckpt)
    mode = 'LAYSUMM'        # 'LAYSUMM' or 'ABSTRACT'
    return tokenizer, model, mode

def load_Task2LaysumDataINmydisk():
    tokenizer, model, mode = __init__()
    dataset = load_dataset("scientific_lay_summarisation", "plos")
    #####? LAYSUMM #####
    if mode == 'LAYSUMM':
        dataset = dataset.map(lambda example: {'article': ' <doc-sep> '.join(example['article'].split('\n')[1:])})
    #####? ABSTRACT #####
    elif mode == 'ABSTRACT':
        dataset = dataset.map(lambda example: {'abstruct': ' '.join(example['article'].split('\n')[:1])}, remove_columns=['year'])
        dataset = dataset.map(lambda example: {'article': ' <doc-sep> '.join(example['article'].split('\n')[1:])})
    else:
        return 0
    return dataset

def create_file(path_save):
    with open(path_save, 'w') as f:
        f.write('predict')
        f.write('\n')

def main(model_save_path):
    tokenizer, model, mode = __init__()
    dataset = load_Task2LaysumDataINmydisk()
    create_file(model_save_path)
    # ***** predict *****
    rouge_1, rouge_2, rouge_l = 0, 0, 0
    k = len(dataset['validation'])
    for i in range(k):
        sample_text = dataset['validation'][i]["article"]
        #####? LAYSUMM #####
        if mode == 'LAYSUMM':
            reference = dataset['validation'][i]["summary"]
        #####? ABSTRACT #####
        elif mode == 'ABSTRACT':
            reference = dataset['validation'][i]["abstruct"]
        device = torch.device("cuda")
        model.to('cuda')
        input_ids = tokenizer(sample_text, max_length=4096, truncation=True, padding='max_length', return_tensors='pt').to(device)
        summaries = model.generate(input_ids=input_ids['input_ids'],
                            attention_mask=input_ids['attention_mask'],
                            max_length=256)
        decoded_summaries = [tokenizer.decode(s, skip_special_tokens=True, clean_up_tokenization_spaces=True) for s in summaries]

        # rouge = Rouge()
        # rouge_score = rouge.get_scores(decoded_summaries[0], reference)
        # rouge_1 += rouge_score[0]["rouge-1"]['f']
        # rouge_2 += rouge_score[0]["rouge-2"]['f']
        # rouge_l += rouge_score[0]["rouge-l"]['f']

        with open(model_save_path, 'a') as f:
            # f.write('number' + str(i))
            # f.write('\t')
            # f.write(str(reference))
            # f.write('\t')
            f.write(str(decoded_summaries[0]))
            # f.write('\t')
            # f.write(str(rouge_score[0]["rouge-1"]['f']))
            # f.write('\t')
            # f.write(str(rouge_score[0]["rouge-2"]['f']))
            # f.write('\t')
            # f.write(str(rouge_score[0]["rouge-l"]['f']))
            f.write('\n')
        print('predict number now = ', i)

if __name__ == '__main__':
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    # os.environ["CUDA_VISIBLE_DEVICES"] = "1"

    path_save = 'out/PRIMERA-arxiv-TrainPLUStest.txt'
    main(path_save)