from pynput import keyboard
import time

script_on = False

#Listener
def on_press(key):
    try: k = key.char
    except: k = key.name

    if key == keyboard.Key.esc:
        return False

    if key == 'F10':
        print("Script toggled")
        script_on = not script_on

#Main body
def main():
    #Setting listener
    listener = keyboard.Listener(on_press=on_press)
    listener.start()

    while True:
        time.sleep(10)


import requests
from bs4 import BeautifulSoup
import sqlite3

head = {"userAgent": "Mozilla/5.0 (X11; Linux i686 on x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2909.25 Safari/537.36"}

first_req = requests.get("https://www.myauto.ge/ka/s/0/0/25/ML-Klasse/2008/00/00/00/iyideba-mercedes-benz-ml-klasse-(%E1%83%A7%E1%83%95%E1%83%94%E1%83%9A%E1%83%90)-2008-geo?stype=0&for_rent=0&marka=25&model=ML-Klasse&year_from=2008&price_from=1000&currency_id=3&loc_id=1&det_search=0&ord=7&last_model=ML-Klasse&keyword=&category_id=m0&page=1")
req_cont = first_req.text

num_soup = BeautifulSoup(req_cont, "html.parser")

last_page = num_soup.find("li", {"class":"pagination-li last-page"})

a_number = last_page.find("a")["href"].split("=")[-1]

BASE_URL = "https://www.myauto.ge/ka/s/0/0/25/ML-Klasse/2008/00/00/00/iyideba-mercedes-benz-ml-klasse-(%E1%83%A7%E1%83%95%E1%83%94%E1%83%9A%E1%83%90)-2008-geo?stype=0&for_rent=0&marka=25&model=ML-Klasse&year_from=2008&price_from=1000&currency_id=3&loc_id=1&det_search=0&ord=7&last_model=ML-Klasse&keyword=&category_id=m0&page="

for page in range(1, int(a_number)):

	request = requests.get(BASE_URL+str(page), headers=head)
	content = request.text

	soup = BeautifulSoup(content, "html.parser")

	main_container = soup.find("div", {"class":"container-main"})

	cars_list = main_container.find_all("div", {"class":"car-short-info"})

	for item in cars_list:
		car_name = item.find("a").text.strip()

		car_id = item.find("div", {"class":"cr-vp-lbl"}).text

		year = item.find("p", {"class":"car-year"}).text.strip()

		try:
			levy = item.find("p", {"class":"car-levy passed"}).text.strip()
		except:
			levy = "განუბაჟებელი"

		engine = item.find("div", {"data-info":"ძრავი"}).text.strip()

		kilometrage = item.find("div", {"class":"cr-road"}).text.strip()

		transmission = item.find("div", {"class":"cr-wheel"}).text.strip()

		wheel = item.find("div", {"class":"cr-gas"}).text.strip()

		place = item.find("p", {"class":"car-list-row"}).text.strip().split()[0]

		prices = item.find("div", {"price-title-container"})
		price_gel = prices.find_all("label")[0].text.strip()
		price_usd = prices.find_all("label")[1].text.strip()

		print(f"{car_name}; {car_id}; \nმდებარეობა:->> {place}; წელი: {year};\
				\nძრავი:->> {engine}; განვლილი მანძილი(km):->> {kilometrage}\
				\nგადაცემათა კოლოფი:->> {transmission}; \nსაჭე:->> {wheel}; \nგანბაჟება: {levy};")

		print(f"USD: {price_usd}; GEL: {price_gel}\n")

	print("-------------------------------------------------------------------")


import contextlib
import io
import itertools
import json
import tempfile
import time
from loguru import logger
from tqdm import tqdm
import numpy as np

from chainercv.evaluations import eval_detection_voc, calc_detection_voc_prec_rec

import torch

from dass_det.utils import (
    postprocess,
    xyxy2xywh
)

class ComicEvaluator:
    """
    Manga109 AP Evaluation class.
    """

    def __init__(
        self, dataloader, img_size, confthre, nmsthre, num_classes,
        use_real_size=False, verbose=False
    ):
        """
        Args:
            dataloader (Dataloader): evaluate dataloader.
            img_size (int): image size after preprocess. images are resized
                to squares whose shape is (img_size, img_size).
            confthre (float): confidence threshold ranging from 0 to 1, which
                is defined in the config file.
            nmsthre (float): IoU threshold of non-max supression ranging from 0 to 1.
        """
        self.dataloader = dataloader
        self.verbose = verbose
        self.img_size = img_size
        self.confthre = confthre
        self.nmsthre = nmsthre
        self.num_classes = num_classes
        self.use_real_size = use_real_size # if set true, then self.img_size will not be used

    def evaluate(
        self,
        model,
        distributed=False,
        half=False,
        trt_file=None,
        decoder=None,
        test_size=None,
        use_07_metric=False,
        select_index :int=-1,
        mode=1,
        use_helper_pred=False,
    ):
        # TODO half to amp_test
        tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor
        model = model.eval()
        if half:
            model = model.half()
        ids = []
        data_list = []
        progress_bar = tqdm

        inference_time = 0
        nms_time = 0
        n_samples = max(len(self.dataloader) - 1, 1)

        if trt_file is not None:
            from torch2trt import TRTModule

            model_trt = TRTModule()
            model_trt.load_state_dict(torch.load(trt_file))

            x = torch.ones(1, 3, test_size[0], test_size[1]).cuda()
            model(x)
            model = model_trt

        pred_bboxes, pred_labels, pred_scores = [], [], []
        gt_bboxes, gt_labels = [], []

        dl_iter = iter(self.dataloader)

        for cur_iter in progress_bar(range(len(self.dataloader))):

            if self.use_real_size:
                file_ids = [[cur_iter]]
                imgs, (h, w) = self.dataloader.dataset.load_image(cur_iter)
                resize_size = ((h // 32) * 32, (w // 32) * 32)
                self.img_size = resize_size
                info_imgs = [[h], [w]]

                imgs, labels = self.dataloader.dataset.preproc(imgs, None, resize_size)
                imgs = torch.from_numpy(imgs).unsqueeze(0)

            else:
                imgs, _, info_imgs, file_ids = next(dl_iter)

            with torch.no_grad():

                imgs = imgs.type(tensor_type)

                if mode == 1:
                    outputs, _ = model(imgs, mode=mode)
                    if decoder is not None:
                        outputs = decoder(outputs, dtype=outputs.type())

                    outputs = postprocess(outputs, self.num_classes, self.confthre, self.nmsthre)

                elif mode == 2 and use_helper_pred:
                    foutputs, outputs = model(imgs, mode=0)
                    if decoder is not None:
                        foutputs = decoder(foutputs, dtype=outputs.type())
                        outputs = decoder(outputs, dtype=outputs.type())

                    foutputs = postprocess(foutputs, self.num_classes, 0.7, 0.4)
                    if foutputs is not None:
                        outputs = postprocess(outputs, self.num_classes, self.confthre,
                                              self.nmsthre, helper_prediction=foutputs)
                    else:
                        outputs = postprocess(outputs, self.num_classes, self.confthre, self.nmsthre)

                elif mode == 2:

                    _, outputs = model(imgs, mode=mode)

                    if decoder is not None:
                        outputs = decoder(outputs, dtype=outputs.type())

                    outputs = postprocess(outputs, self.num_classes, self.confthre, self.nmsthre)

            pred_box, pred_label, pred_score, gt_box, gt_label = self.extract_components(outputs,
                                                                                info_imgs,
                                                                                file_ids,
                                                                                select_index)

            pred_bboxes.extend(pred_box)
            pred_labels.extend(pred_label)
            pred_scores.extend(pred_score)
            gt_bboxes.extend(gt_box)
            gt_labels.extend(gt_label)

        result = eval_detection_voc(pred_bboxes,
                                    pred_labels,
                                    pred_scores,
                                    gt_bboxes,
                                    gt_labels,
                                    use_07_metric=use_07_metric)

        pre_rec_results = calc_detection_voc_prec_rec(pred_bboxes,
                                                      pred_labels,
                                                      pred_scores,
                                                      gt_bboxes,
                                                      gt_labels)

        aps = result['ap']
        aps = aps[~np.isnan(aps)]

        if self.verbose:
            print(' ---> mAP: {:f}'.format(100.0 * result['map']))

        def get_key(my_dict, val):
            for key, value in my_dict.items():
                 if val == value:
                        return key.capitalize()

            return "Not Found"

        print_text = "AP Values | "
        for i in range(len(aps)):
            print_text += f' {get_key(self.dataloader.dataset.class_dict, i)}: {100.0 * aps[i]}'

        if self.verbose:
            print(print_text)

        result["precision"], result["recall"] = [], []

        for ev_idx in range(len(pre_rec_results[0])):
            if pre_rec_results[0][ev_idx] is not None and len(pre_rec_results[0][ev_idx]) > 1:
                result["precision"].append(pre_rec_results[0][ev_idx].mean())
                result["recall"].append(pre_rec_results[1][ev_idx].mean())
            else:
                result["precision"].append(0.0)
                result["recall"].append(0.0)

        if self.verbose:
            print("Precision:", result["precision"])
            print("Recall:", result["recall"])

        return result

    def extract_components(self, outputs, info_imgs, ids, select_index :int):

        pred_bboxes, pred_labels, pred_scores = [], [], []
        gt_bboxes, gt_labels = [], []

        if outputs is None:
            outputs = [None]*len(ids)

        for (output, img_h, img_w, img_id) in zip(
            outputs, info_imgs[0], info_imgs[1], ids
        ):
            if output is not None:
                output = output.cpu()
                bbox = output[:, 0:4]

                # preprocessing: resize
                scale = min(self.img_size[0] / float(img_h), self.img_size[1] / float(img_w))
                bbox /= scale

                if output.shape[0] > 0:
                    cls = torch.zeros(output.shape[0]).fill_(max(0, select_index))
                else:
                    cls = torch.zeros(0)

                score = output[:, 4]

            else:
                bbox = torch.zeros(0, 4)
                score = torch.zeros(0)
                cls = torch.zeros(0)

            gt_annot = self.dataloader.dataset.load_anno(img_id[0])
            gt_box = gt_annot[:,:4]
            gt_label = gt_annot[:,-1]

            pred_bboxes.append(bbox.cpu().numpy().astype(int))
            pred_labels.append(cls.cpu().numpy())
            pred_scores.append(score.cpu().numpy())
            gt_bboxes.append(gt_box.astype(int))
            gt_labels.append(gt_label)

        return pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels


import time
import threading
import logging
import socket
import threading
import sys
import pickle

sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

try:
    import tkinter as tk # Python 3.x
    import tkinter.scrolledtext as ScrolledText
except ImportError:
    import Tkinter as tk # Python 2.x
    import ScrolledText

class TextHandler(logging.Handler):
    # This class allows you to log to a Tkinter Text or ScrolledText widget
    # Adapted from Moshe Kaplan: https://gist.github.com/moshekaplan/c425f861de7bbf28ef06

    def __init__(self, text):
        # run the regular Handler __init__
        logging.Handler.__init__(self)
        # Store a reference to the Text it will log to
        self.text = text

    def emit(self, record):
        msg = self.format(record)
        def append():
            self.text.configure(state='normal')
            self.text.insert(tk.END, msg + '\n')
            self.text.configure(state='disabled')
            # Autoscroll to the bottom
            self.text.yview(tk.END)
        # This is necessary because we can't modify the Text from other threads
        self.text.after(0, append)

class myGUI(tk.Frame):

    def __init__(self, parent, window, *args, **kwargs):
        tk.Frame.__init__(self, parent, *args, **kwargs)
        self.root = parent
        if window == 'main':
            self.build_gui()
        elif window == 'apps':
            self.build_app()
        elif window == 'file_manager':
            self.build_file_manager()

    def build_file_manager(self):
        self.root.title('FILE MANAGER')
        self.root.option_add('*tearOff', 'FALSE')
        self.grid(column=0, row=0, sticky='ew')

        entry = tk.Entry(self.root)
        entry.grid(row=1, column=0, sticky="nsew")

        button = tk.Button(self, text="Crear", command=lambda: send_message('create', 'FILE', entry.get()))
        button.configure(font='TkFixedFont')
        button.grid(row=2, column=1, sticky="nsew")

        button2 = tk.Button(self, text="Eliminar", command=lambda: send_message('delete', 'FILE', entry.get()))
        button2.configure(font='TkFixedFont')
        button2.grid(row=2, column=2, sticky="nsew")

    def build_app(self):
        self.root.title('APPS')
        self.root.option_add('*tearOff', 'FALSE')
        self.grid(column=0, row=0, sticky='ew')

        label = tk.Label(self, text="Modulo de aplicaciones")
        label.configure(font='TkFixedFont')
        label.grid(row=0, column=0, sticky="w")

        button = tk.Button(self, text="Iniciar", command=lambda: send_message('open', 'APP','APP'))
        button.configure(font='TkFixedFont')
        button.grid(row=1, column=0, sticky="nsew")

        button2 = tk.Button(self, text="Terminar", command=lambda: send_message('close', 'APP','APP'))
        button2.configure(font='TkFixedFont')
        button2.grid(row=1, column=1, sticky="nsew")

        label2 = tk.Label(self, text="App 1")
        label2.configure(font='TkFixedFont')
        label2.grid(row=2, column=0, sticky="w")

        button3 = tk.Button(self, text="Iniciar", command=lambda: send_message('open', 'APP','APP1'))
        button3.configure(font='TkFixedFont')
        button3.grid(row=3, column=0, sticky="nsew")

        button4 = tk.Button(self, text="Terminar", command=lambda: send_message('close', 'APP','APP1'))
        button4.configure(font='TkFixedFont')
        button4.grid(row=3, column=1, sticky="nsew")

        label3 = tk.Label(self, text="App  2")
        label3.configure(font='TkFixedFont')
        label3.grid(row=4, column=0, sticky="w")

        button5 = tk.Button(self, text="Iniciar", command=lambda: send_message('open', 'APP','APP2'))
        button5.configure(font='TkFixedFont')
        button5.grid(row=5, column=0, sticky="nsew")

        button6 = tk.Button(self, text="Terminar", command=lambda: send_message('close', 'APP','APP2'))
        button6.configure(font='TkFixedFont')
        button6.grid(row=5, column=1, sticky="nsew")

    def build_gui(self):
        # Build GUI
        self.root.title('GUI')
        self.root.option_add('*tearOff', 'FALSE')
        self.grid(column=0, row=0, sticky='ew')
        # self.grid_columnconfigure(0, weight=1, uniform='a')
        # self.grid_columnconfigure(1, weight=1, uniform='a')
        # self.grid_columnconfigure(2, weight=1, uniform='a')
        # self.grid_columnconfigure(3, weight=1, uniform='a')

        # Add text widget to display logging info
        label = tk.Label(self, text="Logger")
        label.configure(font='TkFixedFont')
        label.grid(row=1, column=1, sticky="w")

        st = ScrolledText.ScrolledText(self, state='disabled')
        st.configure(font='TkFixedFont')
        st.grid(column=0, row=2, sticky='w', columnspan=3)

        button = tk.Button(self, text="Aplicaciones", command=lambda: open_window('apps'))
        button.configure(font='TkFixedFont')
        button.grid(row=3, column=0, sticky="nsew")

        button2 = tk.Button(self, text="Archivos", command=lambda: open_window('file_manager'))
        button2.configure(font='TkFixedFont')
        button2.grid(row=3, column=1, sticky="nsew")

        # Create textLogger
        text_handler = TextHandler(st)

        # Logging configuration
        logging.basicConfig(filename='test.log',
            level=logging.INFO,
            format='%(asctime)s - %(levelname)s - %(message)s')

        # Add the handler to logger
        logger = logging.getLogger()
        logger.addHandler(text_handler)

def open_window(wtype):
    window = tk.Tk()
    myGUI(window, wtype)

def connect():
    while True:
        try:
            data = sock.recv(1024)
            if data:
                data = pickle.loads(data)
                if data['codeterm'] == 3:
                    for line in data['data'].split('\n'):
                        logging.info(line)
        except:
            pass

def main():
    global sock
    sock.connect(("localhost", 4000))
    root = tk.Tk()
    myGUI(root, 'main')
    process = threading.Thread(target=connect)
    process.daemon = True
    process.start()
    # send_message('info','FILE','data')
    root.mainloop()

def send_message(process, dst, message):
    global sock
    msg = {'cmd': process, 'src': 'GUI', 'dst': dst, 'msg': message}
    sock.send(pickle.dumps(msg))

if __name__ == '__main__':
    main()


import argparse
import asyncio
import os.path

from tornado.platform.asyncio import AsyncIOMainLoop
from tornado.web import Application
from tornado.web import StaticFileHandler
from tornado.httpserver import HTTPServer
from tornado.options import define

from libs import setup

from handlers import (
    IndividualView,
    TwilioView,
    WebSocketHandler
)
from handlers.websocket.view import infinite_update_loop

# CONSTANTS
parser = argparse.ArgumentParser(description='commodus')
parser.add_argument('dbuser', help='database username string')
parser.add_argument('dbpass', help='database password string')
parser.add_argument('dbhost', help='database host string (ex: 120.0.0.1)')
parser.add_argument('dbname', help='database name string')
parser.add_argument('--dbport', help='database port integer (ex: 5432)', type=int, default=5432)
parser.add_argument('--port', help='API server port integer (ex: 8080)', type=int, default=8080)
config = parser.parse_args()
define('username', default=config.dbuser, type=str, group='db')
define('password', default=config.dbpass, type=str, group='db')
define('host', default=config.dbhost, type=str, group='db')
define('name', default=config.dbname, type=str, group='db')
define('port', default=config.dbport, type=int, group='db')

logger = setup.log(name='commodus', version='v1')

def main():
    AsyncIOMainLoop().install()
    server = HTTPServer(
        Application(
            [
                (r'/v1/twilio/', TwilioView),
                (r"/v1/individual/([0-9A-F]{32})", IndividualView),
                (r'/static/(.*)', StaticFileHandler, {'path': 'static'}),
                (r'/ws', WebSocketHandler),
                (r'/(.*)', StaticFileHandler, {'path': 'static', 'default_filename': 'index.html'}),
            ],
            # default_handler_class=NotFoundView,
            template_path=os.path.join(os.path.dirname(__file__), "static")
        )
    )
    server.bind(8080)
    server.start()  # Forks multiple sub-processes
    logger.info("installing background services")
    asyncio.ensure_future(infinite_update_loop())
    logger.info("server is waiting for a connection")
    asyncio.get_event_loop().run_forever().start()

if __name__ == "__main__":
    main()


# apis/serializers.py
from rest_framework import serializers
from todos import models

class TodoSerializer(serializers.ModelSerializer):
    title123 = serializers.CharField(source='title')
    class Meta:
        model = models.Todo
        fields = (
            'id',
            'title123',
            'description',
        )


import torch
import random
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.utils.rnn import pack_padded_sequence

if torch.cuda.is_available():
    device = torch.device('cuda')
else:
    device = torch.device('cpu')
class DecoderNet(nn.Module):
    def __init__(
        self, word_vectors, output_size, embed_size, hidden_size=1024,
        dropout_keep_prob=1, max_length=110, num_layers=4
    ):
        super(DecoderNet, self).__init__()
        self.embed = nn.Embedding(output_size, embed_size)
        self.gru = nn.GRU(
            (embed_size + 3*32*32), hidden_size, num_layers, batch_first=True,
        )
        self.linear = nn.Linear(hidden_size, output_size)
        self.max_length = max_length
        self.word_vectors = word_vectors
        self.output_size = output_size
        # self._init()

    def _init(self):
        self.embed.weight.data = self.word_vectors

    # Foward and sample stolen from https://github.com/yunjey/pytorch-tutorial
    def forward(self, features, captions, lengths, teacher_forcing_ratio=0.75):
        """Decode image feature vectors and generate captions."""
        # use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False
        loss = 0.0;
        embeddings = self.embed(captions); features = features.unsqueeze(1)
        # print(embeddings.shape, features.shape)
        features = features.expand(embeddings.shape[0], embeddings.shape[1], features.shape[-1])
        embeddings = torch.cat((features, embeddings), -1)

        states = None; weights = torch.ones(self.output_size).to(device); weights[622] = 0
        for t in range(embeddings.shape[1]):
            use_teacher_forcing = True
            if t == 0: inputs = embeddings[:, t, :].unsqueeze(1)
            hiddens, states = self.gru(inputs, states)
            cur_output = self.linear(hiddens.squeeze(1))
            _, predicted = cur_output.max(1)

            loss_function = nn.CrossEntropyLoss(weight=weights, reduce=False)
            if not use_teacher_forcing:
                predicted = predicted.detach()
                inputs = self.embed(predicted).unsqueeze(1)
                inputs = torch.cat((features[:, t, :].unsqueeze(1), inputs), -1)
                loss += loss_function(cur_output, captions[:, t])
            else:
                inputs = embeddings[:, t, :].unsqueeze(1)
                loss += loss_function(cur_output, captions[:, t])
        return loss

    def sample(self, features, states=None):
        """Generate captions for given image features using greedy search."""
        sampled_ids = []
        START = self.word_vectors[621].expand((features.shape[0], 1, 100))
        embeddings = START.to(device); features = features.unsqueeze(1)
        features = features.expand(embeddings.shape[0], embeddings.shape[1], features.shape[-1])
        embeddings = torch.cat((features, embeddings), -1)

        for i in range(self.max_length):
            hiddens, states = self.gru(embeddings, states)       # hiddens: (batch_size, 1, hidden_size)
            outputs = self.linear(hiddens.squeeze(1))            # outputs:  (batch_size, vocab_size)
            _, predicted = outputs.max(1)                        # predicted: (batch_size)
            sampled_ids.append(predicted)
            embeddings = self.embed(predicted).unsqueeze(1)      # inputs: (batch_size, embed_size)
            embeddings = torch.cat((features, embeddings), -1)   # inputs: (batch_size, 1, embed_size)
        sampled_ids = torch.stack(sampled_ids, 1)                # sampled_ids: (batch_size, max_length)
        return sampled_ids


from PIL import Image

import torch
from torch.optim.lr_scheduler import LambdaLR
import torchvision.transforms.functional as TF
import pytorch_lightning as pl

from .losses import focal_loss, water_obstacle_separation_loss, object_projection_loss, object_focal_loss
from .pairwise_affinity import pairwise_affinity_loss, object_pairwise_affinity_loss
from .metrics import PixelAccuracy, ClassIoU
from .utils import bool_arg

NUM_EPOCHS = {'warmup':25, 'finetune':50}
LEARNING_RATE = 1e-6
MOMENTUM = 0.9
WEIGHT_DECAY = 1e-6
LR_DECAY_POW = 0.9
FOCAL_LOSS_SCALE = 'labels'
SEPARATION_LOSS = {'warmup':False, 'finetune':True}
SL_LAMBDA = 0.01
PA_LOSS = {'warmup':True, 'finetune':True}
PA_LOSS_LAMBDA = 1
PA_LOSS_TAU = 0.1
OBJ_LOSS = {'warmup':True, 'finetune':False}
OBJ_LOSS_LAMBDA = 1

class LitModel(pl.LightningModule):
    """ Pytorch Lightning wrapper for a model, ready for distributed training. """

    @staticmethod
    def add_argparse_args(parser, phase):
        """Adds phase-specific model parameters to parser."""

        parser.add_argument("--learning-rate", type=float, default=LEARNING_RATE,
                            help="Base learning rate for training with polynomial decay.")
        parser.add_argument("--momentum", type=float, default=MOMENTUM,
                            help="Momentum component of the optimiser.")
        parser.add_argument("--epochs", type=int, default=NUM_EPOCHS[phase],
                            help="Number of training epochs.")
        parser.add_argument("--lr-decay-pow", type=float, default=LR_DECAY_POW,
                            help="Decay parameter to compute the learning rate decay.")
        parser.add_argument("--weight-decay", type=float, default=WEIGHT_DECAY,
                            help="Regularisation parameter for L2-loss.")
        parser.add_argument("--focal-loss-scale", type=str, default=FOCAL_LOSS_SCALE, choices=['logits', 'labels'],
                            help="Which scale to use for focal loss computation (logits or labels).")
        parser.add_argument("--separation-loss", default=SEPARATION_LOSS[phase], type=bool_arg,
                            help="Use separation loss.")
        parser.add_argument("--separation-loss-lambda", default=SL_LAMBDA, type=float,
                            help="The separation loss lambda (weight).")
        parser.add_argument("--separation-loss-sky", action='store_true',
                            help="Include sky in the separation loss computation.")
        parser.add_argument("--pairwise-affinity-loss", type=bool_arg, default=PA_LOSS[phase],
                            help="Add the pairwise affinity loss term.")
        parser.add_argument("--pairwise-affinity-loss-lambda", default=PA_LOSS_LAMBDA, type=float,
                            help="The pairwise affinity loss lambda (weight).")
        parser.add_argument("--pairwise-affinity-loss-tau", default=PA_LOSS_TAU, type=float,
                            help="The pairwise affinity loss tau (similarity threshold).")
        parser.add_argument("--object-loss", type=bool_arg, default=OBJ_LOSS[phase],
                            help="Add the object loss term.")
        parser.add_argument("--object-loss-pa", type=bool_arg, default=True,
                            help="Enable object loss PA term.")
        parser.add_argument("--object-loss-proj", type=bool_arg, default=True,
                            help="Enable object loss projection loss term.")
        parser.add_argument("--object-loss-aux", type=bool_arg, default=True,
                            help="Enable object loss auxiliary segmentation term.")
        parser.add_argument("--object-loss-lambda", default=OBJ_LOSS_LAMBDA, type=float,
                            help="The object loss lambda (weight).")

        return parser

    def __init__(self, model, num_classes, args):
        super().__init__()

        self.model = model
        self.num_classes = num_classes

        self.epochs = args.epochs
        self.learning_rate = args.learning_rate
        self.momentum = args.momentum
        self.weight_decay = args.weight_decay
        self.lr_decay_pow = args.lr_decay_pow
        self.focal_loss_scale = args.focal_loss_scale
        self.separation_loss = args.separation_loss
        self.separation_loss_lambda = args.separation_loss_lambda
        self.separation_loss_sky = args.separation_loss_sky
        self.pairwise_affinity_loss = args.pairwise_affinity_loss
        self.pairwise_affinity_loss_lambda = args.pairwise_affinity_loss_lambda
        self.pairwise_affinity_loss_tau = args.pairwise_affinity_loss_tau
        self.object_loss = args.object_loss
        self.object_loss_lambda = args.object_loss_lambda
        self.object_loss_pa = args.object_loss_pa
        self.object_loss_proj = args.object_loss_proj
        self.object_loss_aux = args.object_loss_aux

        self.val_accuracy = PixelAccuracy(num_classes)
        self.val_iou_0 = ClassIoU(0, num_classes)
        self.val_iou_1 = ClassIoU(1, num_classes)
        self.val_iou_2 = ClassIoU(2, num_classes)

    def forward(self, x):
        output = self.model(x)
        return output['out']

    def training_step(self, batch, batch_idx):
        features, labels = batch

        out = self.model(features)

        fl = focal_loss(out['out'], labels['segmentation'], target_scale=self.focal_loss_scale)

        separation_loss = torch.tensor(0.0)
        if self.separation_loss:
            separation_loss = water_obstacle_separation_loss(
                out['aux'], labels['segmentation'], include_sky=self.separation_loss_sky)

        pa_loss = torch.tensor(0.0)
        if self.pairwise_affinity_loss:
            ignore_mask = None
            if self.object_loss is not None:
                ignore_mask = labels['objects'].max(1).values
            pa_loss = pairwise_affinity_loss(out['out'], labels['segmentation'], labels['pa_similarity'],
                                             tau=self.pairwise_affinity_loss_tau, target_scale=self.focal_loss_scale, ignore_mask=ignore_mask)

        separation_loss = self.separation_loss_lambda * separation_loss
        pa_loss = self.pairwise_affinity_loss_lambda * pa_loss
        bg_loss = fl + separation_loss + pa_loss

        obj_loss_o = torch.tensor(0.0)
        obj_loss_pa = torch.tensor(0.0)
        obj_loss_aux = torch.tensor(0.0)
        obj_loss = torch.tensor(0.0)
        if self.object_loss:
            if self.object_loss_proj:
                obj_loss_o = object_projection_loss(out['out'], labels['objects'], labels['n_objects'], target_scale=self.focal_loss_scale,
                                                    normalize=True, reduce='sum')

            if self.object_loss_pa:
                obj_loss_pa = object_pairwise_affinity_loss(out['out'], labels['objects'], labels['n_objects'], labels['pa_similarity'],
                                                            tau=self.pairwise_affinity_loss_tau, normalize=True, reduce='sum')

            if self.object_loss_aux:
                obj_loss_aux = object_focal_loss(out['out'], labels['objects'], labels['n_objects'], labels['instance_seg'],
                                                normalize=True, reduce='sum')

            obj_loss = obj_loss_o + obj_loss_pa + obj_loss_aux

        loss = bg_loss + obj_loss

        # log losses
        self.log('train/loss', loss.item())
        self.log('train/bg/focal_loss', fl.item())
        self.log('train/bg/separation_loss', separation_loss.item())
        self.log('train/bg/pa_loss', pa_loss.item())
        self.log('train/bg_loss', bg_loss.item())
        self.log('train/obj/proj_loss', obj_loss_o.item())
        self.log('train/obj/pa_loss', obj_loss_pa.item())
        self.log('train/obj/aux_loss', obj_loss_aux.item())
        self.log('train/obj_loss', obj_loss.item())

        return loss

    def training_epoch_end(self, outputs):
        # Bugfix
        pass

    def validation_step(self, batch, batch_idx):
        features, labels = batch

        out = self.model(features)

        loss = focal_loss(out['out'], labels['segmentation'], target_scale=self.focal_loss_scale)

        # Log loss
        self.log('val/loss', loss.item())

        # Metrics
        labels_size = (labels['segmentation'].size(2), labels['segmentation'].size(3))
        logits = TF.resize(out['out'], labels_size, interpolation=Image.BILINEAR)
        preds = logits.argmax(1)

        # Create hard labels from soft
        labels_hard = labels['segmentation'].argmax(1)
        ignore_mask = labels['segmentation'].sum(1) < 0.9
        labels_hard = labels_hard * ~ignore_mask + 4 * ignore_mask

        self.val_accuracy(preds, labels_hard)
        self.val_iou_0(preds, labels_hard)
        self.val_iou_1(preds, labels_hard)
        self.val_iou_2(preds, labels_hard)

        self.log('val/accuracy', self.val_accuracy)
        self.log('val/iou/obstacle', self.val_iou_0)
        self.log('val/iou/water', self.val_iou_1)
        self.log('val/iou/sky', self.val_iou_2)

        return {'loss': loss, 'preds': preds}

    def configure_optimizers(self):
        # Separate parameters for different LRs
        encoder_parameters = []
        decoder_w_parameters = []
        decoder_b_parameters = []
        for name, parameter in self.model.named_parameters():
            if name.startswith('backbone'):
                encoder_parameters.append(parameter)
            elif 'weight' in name:
                decoder_w_parameters.append(parameter)
            else:
                decoder_b_parameters.append(parameter)

        optimizer = torch.optim.RMSprop([
            {'params': encoder_parameters, 'lr': self.learning_rate},
            {'params': decoder_w_parameters, 'lr': self.learning_rate * 10},
            {'params': decoder_b_parameters, 'lr': self.learning_rate * 20},
        ], momentum=self.momentum, alpha=0.9, weight_decay=self.weight_decay)

        # Decaying LR function
        lr_fn = lambda epoch: (1 - epoch/self.epochs) ** self.lr_decay_pow

        # Decaying learning rate (updated each epoch)
        scheduler = LambdaLR(optimizer, lr_fn)

        return [optimizer], [scheduler]

    def on_save_checkpoint(self, checkpoint):
        # Export the model weights
        checkpoint['model'] = self.model.state_dict()


from yapsy.IPlugin import IPlugin
import wolframalpha
import yaml

keys_file = open("friday/plugins/KEYS")
keys = yaml.load(keys_file)
keys_file.close()

class Wolfram(IPlugin):
    def can_perform(self, friday, request):
        return 'result' in request and 'resolvedQuery' in request['result']\
                and 'action' in request['result'] and request['result']['action'] == 'wisdom.unknown'
        # result = request['result']  # Assumes we're using gTTS
        # # Get the text that is supposed to be spoken aloud
        # reply = result['fulfillment']['speech']
        # # Get what the service thought you said
        # question = result['resolvedQuery']

    def perform(self, friday, request):
        question = request['result']['resolvedQuery']
        client = wolframalpha.Client(keys['WOLFRAM'])
        res = client.query(question)
        answer = str(list(res))
        """if len(res):
            results = list(res.results)
            if len(results):
                answer = results[0].text[0]
            else:
                answer = ' '.join([each_answer.subpods[0].text for each_answer in res.pods
                                   if each_answer.subpods[0].text])
        else:
            # answer = "Sorry, Wolfram doesn't know the answer."
            answer = ""
        """
        """# Replace some of its notation so it's more easily read.
        answer = answer.replace('\n', '. ').replace('~~', ' or about ')
        # Get the result to a computation and don't bother reading the original question.
        if '=' in answer:
            answer = answer[answer.index('=') + 1:].strip()
        """
        return answer

#
# def wolfram_query(question):
#     # Every service should have a general set of requirements under which
#     # it is activated, this would be one of the ones that Wolfram Alpha
#     # uses, it does have others as well. Consider having a single method
#     # in the plugin system that returns a boolean determining whether
#     # a plugin should be activated.
#     if question:
#
#
# def wolfram_query_old(question):
#     import wolframalpha
#     # Every service should have a general set of requirements under which
#     # it is activated, this would be one of the ones that Wolfram Alpha
#     # uses, it does have others as well. Consider having a single method
#     # in the plugin system that returns a boolean determining whether
#     # a plugin should be activated.
#     if question.lower().startswith('wolfram'):
#         question = question[8:]
#     client = wolframalpha.Client(user_info.WOLFRAM_KEY)
#     res = client.query(question)
#     try:
#         return next(res.results).text  # This really needs to be changed.
#         # I shouldn't have to rely upon error catching for my flow control.
#     except StopIteration:
#         pass
#     try:
#         answer = '   '.join([each_answer.text for each_answer in res.pods if each_answer])
#     except TypeError:
#         answer = None
#     if not answer:
#         answer = "Sorry, Wolfram doesn't know the answer."
#
#     # Replace some of its notation so it's more easily read.
#     answer = answer.replace('\n', '; ').replace('~~', ' or about ')
#     # Get the result to a computation and don't bother reading the original question.
#     if '=' in answer:
#         answer = answer[answer.index('=')+1:]
#     return [answer, None]  # Follows answer format of [text, action]
#


import socketio
import eventlet

sio = socketio.Server()
app = socketio.WSGIApp(sio)

frames = {}

class ConnectionNameSpace(socketio.Namespace):

    def __init__(self, namespace=None):
        super().__init__(namespace)
        print (self.namespace)

    def on_connect(self, sid, environ):
        print('connected')
        print (self.namespace)
        self.enter_room(sid, self.namespace)

    def on_disconnect(self, sid):
        self.leave_room(sid, self.namespace)

    def on_frame(self, sid, data):
        print ('frame')
        self.emit('new_frame', data, room = self.namespace, skip_sid = sid)

device1 = ConnectionNameSpace('/device1')
device2 = ConnectionNameSpace('/device2')
sio.register_namespace(device1)
sio.register_namespace(device2)

eventlet.wsgi.server(eventlet.listen(('', 8000)), app)


from telebot import TeleBot

api = '1722995619:AAFD3SqWOCp2ZCjlUhkkYCYp6MvZfJeVPu8'
bot = TeleBot(api, parse_mode='html')

@bot.message_handler(commands=['start'])
def send_welcome(message):
    bot.send_message(message.chat.id, 'Привет, ты попал в бота для получения данных'
                                      ' с сервера в виде exele файлов\n\n'
                                      'Напиши боту команду /info он пришлет тебе все файлы с статистикой')

@bot.message_handler(commands=['info'])
def send_exele(message):
    chat_id = message.chat.id

    rekshop = open('rekshop.xls', 'rb')
    bot.send_document(chat_id, rekshop)

    voltiq = open('voltiq.xls', 'rb')
    bot.send_document(chat_id, voltiq)

    roboshop = open('roboshop.xls', 'rb')
    bot.send_document(chat_id, roboshop)

    arduinopro = open('arduinopro.xls', 'rb')
    bot.send_document(chat_id, arduinopro)

    bot.send_message(chat_id, text='Для завершения анализа обратитесь к программисту')

bot.polling(none_stop=True)


import configparser
import os

def has_config(path):
    """Returns true if gritter config file exists."""
    return True if os.path.isfile(path) else False

def create_config(path):
    """Create the config file iff it doesn't exist."""
    if not os.path.exists(os.path.dirname(path)):
        os.makedirs(os.path.dirname(path))
    if not has_config(path):
        open(path, 'a').close()

def set_auth(path, args):
    """Sets auth args in config."""
    config = configparser.ConfigParser()
    config.read(path)
    # clobber old auth values
    config["auth"] = {}
    for k, v in args.items():
        config["auth"][k] = v

    with open(path, 'w') as configfile:
        config.write(configfile)

def get_auth(path):
    """Gets dict of auth args from config."""
    config = configparser.ConfigParser()
    config.read(path)
    return config["auth"]


import os
import pandas as pd
import matplotlib.pyplot as plt

from transformers import GPT2TokenizerFast

from langchain.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter

from langchain.vectorstores import FAISS
from langchain.chains.question_answering import load_qa_chain
from langchain.llms import HuggingFaceHub
from langchain.chains import ConversationalRetrievalChain
from langchain.chains import RetrievalQA

from langchain.embeddings.huggingface import HuggingFaceEmbeddings
from llama_index import LangchainEmbedding

import textract

from dotenv import load_dotenv
import gradio as gr

class PDFQA():

    def __init__(self) -> None:
        super().__init__()

        os.environ['HUGGINGFACEHUB_API_TOKEN'] = os.getenv('HUGGINGFACEHUB_API_TOKEN')

        self.source_folder = './data/'
        self.output_folder = './output/'

    def loading_pdf():
        return 'Loading...'

    def pdf_changes(self, pdf_doc, repo_id):
        # # Simple method - Split by pages
        # loader = PyPDFLoader(pdf_doc.name)
        # pages = loader.load_and_split()

        # # SKIP TO STEP 2 IF YOU'RE USING THIS METHOD
        # chunks = pages

        if pdf_doc is None:
            return

        doc = textract.process(pdf_doc.name)

        # Step 2: Save to .txt and reopen (helps prevent issues)
        with open(self.output_folder+'test.txt', 'w') as f:
            f.write(doc.decode('utf-8'))

        with open(self.output_folder+'test.txt', 'r') as f:
            text = f.read()

        # Step 3: Create function to count tokens
        tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')

        def count_tokens(text: str) -> int:
            return len(tokenizer.encode(text))

        # Step 4: Split text into chunks
        text_splitter = RecursiveCharacterTextSplitter(
            # Set a really small chunk size, just to show.
            chunk_size = 512,
            chunk_overlap  = 24,
            length_function = count_tokens,
        )

        chunks = text_splitter.create_documents([text])

        embeddings = HuggingFaceEmbeddings()
        # embed_model = LangchainEmbedding(embeddings)

        # Create vector database
        db = FAISS.from_documents(chunks, embeddings)
        retriever = db.as_retriever()

        flan_ul2 = HuggingFaceHub(repo_id=repo_id, model_kwargs={'temperature': 0.1})
        # chain = load_qa_chain(flan_ul2, chain_type='stuff')
        self.chain = RetrievalQA.from_chain_type(llm=flan_ul2, chain_type='stuff', retriever=retriever, return_source_documents=True)
        return 'Ready'

    def add_text(self, history, text):
        history = history + [(text, None)]
        return history, ''

    def bot(self, history):
        response = self.infer(history[-1][0])
        history[-1][1] = response['result']
        return history

    def infer(self, question):

        query = question
        result = self.chain({'query': query})

        return result

if __name__ == '__main__':
    load_dotenv()

    pdf_qa = PDFQA()

    # query = 'What is the copay for Diagnostic test?'
    # query = 'what is my out of pocket limit?'
    # query = 'what is the radiology cost for having a baby'
    # query = how much will be my radiology cost for new born baby?

    css='''
    #col-container {max-width: 700px; margin-left: auto; margin-right: auto;}
    '''

    title = '''

Upload a PDF document and start asking questions from the document

Upload a pdf document from your computer, click the 'Process PDF' button,

        once ready (check for the Ready status), you can start asking questions about the uploaded pdf document

    '''

    with gr.Blocks(css=css) as demo:
        with gr.Column(elem_id='col-container'):
            gr.HTML(title)

            with gr.Column():
                pdf_doc = gr.File(label='Load a pdf', file_types=['.pdf'], type='file')
                repo_id = gr.Dropdown(label='LLM', choices=['google/flan-t5-large', 'bigscience/mt0-large', 'microsoft/prophetnet-large-uncased', 'bigscience/T0_3B'], value='google/flan-t5-large')
                with gr.Row():
                    langchain_status = gr.Textbox(label='Status', placeholder='', interactive=False)
                    load_pdf = gr.Button('Process PDF')

            chatbot = gr.Chatbot([], elem_id='chatbot').style(height=350)
            question = gr.Textbox(label='Question', placeholder='Ask your question and hit enter')
            submit_btn = gr.Button('Send message')

        repo_id.change(pdf_qa.pdf_changes, inputs=[pdf_doc, repo_id], outputs=[langchain_status], queue=False)
        load_pdf.click(pdf_qa.pdf_changes, inputs=[pdf_doc, repo_id], outputs=[langchain_status], queue=False)
        question.submit(pdf_qa.add_text, [chatbot, question], [chatbot, question]).then(
            pdf_qa.bot, chatbot, chatbot
        )
        submit_btn.click(pdf_qa.add_text, [chatbot, question], [chatbot, question]).then(
            pdf_qa.bot, chatbot, chatbot
        )

    demo.launch()


import copy
from collections import Sequence
from math import floor

import matplotlib.pyplot as plt
import mmcv
import numpy as np
import torch
from mmcv.runner import obj_from_dict

from .. import datasets

def to_tensor(data):
    """Convert objects of various python types to :obj:`torch.Tensor`.

    Supported types are: :class:`numpy.ndarray`, :class:`torch.Tensor`,
    :class:`Sequence`, :class:`int` and :class:`float`.
    """
    if isinstance(data, torch.Tensor):
        return data
    elif isinstance(data, np.ndarray):
        return torch.from_numpy(data)
    elif isinstance(data, Sequence) and not mmcv.is_str(data):
        return torch.tensor(data)
    elif isinstance(data, int):
        return torch.LongTensor([data])
    elif isinstance(data, float):
        return torch.FloatTensor([data])
    else:
        raise TypeError('type {} cannot be converted to tensor.'.format(
            type(data)))

def random_scale(img_scales, mode='range'):
    """Randomly select a scale from a list of scales or scale ranges.

    Args:
        img_scales (list[tuple]): Image scale or scale range.
        mode (str): "range" or "value".

    Returns:
        tuple: Sampled image scale.
    """
    num_scales = len(img_scales)
    if num_scales == 1:  # fixed scale is specified
        img_scale = img_scales[0]
    elif num_scales == 2:  # randomly sample a scale
        if mode == 'range':
            img_scale_long = [max(s) for s in img_scales]
            img_scale_short = [min(s) for s in img_scales]
            long_edge = np.random.randint(
                min(img_scale_long),
                max(img_scale_long) + 1)
            short_edge = np.random.randint(
                min(img_scale_short),
                max(img_scale_short) + 1)
            img_scale = (long_edge, short_edge)
        elif mode == 'value':
            img_scale = img_scales[np.random.randint(num_scales)]
    else:
        if mode != 'value':
            raise ValueError(
                'Only "value" mode supports more than 2 image scales')
        img_scale = img_scales[np.random.randint(num_scales)]
    return img_scale

def show_ann(coco, img, ann_info):
    plt.imshow(mmcv.bgr2rgb(img))
    plt.axis('off')
    coco.showAnns(ann_info)
    plt.show()

def get_dataset(data_cfg):

    data_info = copy.deepcopy(data_cfg)
    data_info['ann_file'] = data_cfg['ann_file']
    data_info['proposal_file'] = None
    data_info['img_prefix'] = data_cfg['img_prefix']
    dset = obj_from_dict(data_info, datasets)
    return dset

def make_gaussian(size, sigma=10, center=None, d_type=np.float64):
    """ Make a square gaussian kernel.
    size: is the dimensions of the output gaussian
    sigma: is full-width-half-maximum, which
    can be thought of as an effective radius.
    """

    x = np.arange(0, size[1], 1, float)
    y = np.arange(0, size[0], 1, float)
    y = y[:, np.newaxis]

    if center is None:
        x0 = y0 = size[0] // 2
    else:
        x0 = center[0]
        y0 = center[1]

    return np.exp(-4 * np.log(2) * ((x - x0) ** 2 + (y - y0) ** 2) / sigma ** 2).astype(
        d_type
    )

def generate_heatmap(
    bbox_shape, kx_bbox, ky_bbox, heatmap_size=(64, 48), gaussian_size=5, gaussian_std=2
):
    """
    Args:
        bbox_shape (list): h, w of bbox
        kx_bbox, ky_bbox (int): keypoint coordination in bbox
        heatmap_size (tuple): default value is (64, 48)
        gaussian_size (int): gaussian size
        gaussian_std (int): gaussian std

    Returns:
        heatmap (np.darray): heatmap w/wo gaussian
        target_weight (np.darray): if heatmap has gaussian, 1 else 0
    """
    assert gaussian_size % 2 != 0

    h, w = bbox_shape
    assert 0 <= kx_bbox < w and 0 <= ky_bbox < h

    heatmap = np.zeros(heatmap_size)
    ratio_w = heatmap_size[1] / w
    ratio_h = heatmap_size[0] / h

    resized_kx_bbox = floor(kx_bbox * ratio_w)
    resized_ky_bbox = floor(ky_bbox * ratio_h)

    # Add temp pad for gaussian addition
    pad_size = gaussian_size // 2
    heatmap_pad = np.pad(heatmap, (pad_size, pad_size), mode="constant")

    gaussian = make_gaussian([gaussian_size, gaussian_size], gaussian_std)

    # heatmap coords adjusting gaussian
    gx1 = resized_kx_bbox
    gy1 = resized_ky_bbox

    gx2 = resized_kx_bbox + gaussian_size
    gy2 = resized_ky_bbox + gaussian_size

    heatmap_pad[gy1:gy2, gx1:gx2] = gaussian
    heatmap = heatmap_pad[pad_size:-pad_size, pad_size:-pad_size]
    target_weight = np.ones([1, 1])
    return heatmap, target_weight

def add_boundary_noise(img_shape, x1, y1, x2, y2, kx_bbox=None, ky_bbox=None):
    """Add noise when cropping bbox

    Args:
        img_shape (list): [h, w]
        x1, y1, x2, y2 (int): bbox coordination
        kx_bbox, ky_bbox (int): keypoint coordination, default is None

    Returns:
        new_x1, new_x2, new_y1, new_y2 (int): new bbox coordination
    """
    img_h, img_w = img_shape

    randint_w = sorted((0, (x2 - x1) // 3 + 1))
    randint_h = sorted((0, (y2 - y1) // 3 + 1))

    dx1 = min(x1, np.random.randint(*randint_w))
    dx2 = min(img_w - x2, np.random.randint(*randint_w))
    dy1 = min(y1, np.random.randint(*randint_h))
    dy2 = min(img_h - y2, np.random.randint(*randint_h))

    new_x1 = x1 - dx1
    new_x2 = x2 + dx2
    new_y1 = y1 - dy1
    new_y2 = y2 + dy2

    if kx_bbox is not None and ky_bbox is not None:
        new_kx_bbox = kx_bbox + dx1
        new_ky_bbox = ky_bbox + dy1
        return new_x1, new_x2, new_y1, new_y2, new_kx_bbox, new_ky_bbox
    return new_x1, new_x2, new_y1, new_y2

def bb_intersection_over_union(boxA, boxB):
    # determine the (x, y)-coordinates of the intersection rectangle
    xA = max(boxA[0], boxB[0])
    yA = max(boxA[1], boxB[1])
    xB = min(boxA[2], boxB[2])
    yB = min(boxA[3], boxB[3])

    # compute the area of intersection rectangle
    interArea = max(0, xB - xA + 1) * max(0, yB - yA + 1)

    # compute the area of both the prediction and ground-truth
    # rectangles
    boxAArea = (boxA[2] - boxA[0] + 1) * (boxA[3] - boxA[1] + 1)
    boxBArea = (boxB[2] - boxB[0] + 1) * (boxB[3] - boxB[1] + 1)

    # compute the intersection over union by taking the intersection
    # area and dividing it by the sum of prediction + ground-truth
    # areas - the interesection area
    iou = interArea / float(boxAArea + boxBArea - interArea)

    # return the intersection over union value
    return iou


from typing import Optional

from Token import Token
from TokenType import TokenType

class Scanner():
    KEYWORDS = {
        "and": TokenType.AND,
        "class": TokenType.CLASS,
        "else": TokenType.ELSE,
        "false": TokenType.FALSE,
        "fun": TokenType.FUN,
        "for": TokenType.FOR,
        "if": TokenType.IF,
        "nil": TokenType.NIL,
        "or": TokenType.OR,
        "print": TokenType.PRINT,
        "return": TokenType.RETURN,
        "super": TokenType.SUPER,
        "this": TokenType.THIS,
        "true": TokenType.TRUE,
        "var": TokenType.VAR,
        "while": TokenType.WHILE,
    }

    def __init__(self, source: str, lox):
        self.lox = lox
        self.source = source
        self.tokens = []
        self.start = 0
        self.current = 0
        self.line = 1

    def scanTokens(self) -> list[Token]:
        while not self.isAtEnd():
            # We are at the beginning of the next lexeme
            self.start = self.current
            self.scanToken()

        self.tokens.append(Token(TokenType.EOF, "", None, self.line))
        return self.tokens

    def scanToken(self) -> None:
        c = self.advance()
        if c == "(":
            self.addToken(TokenType.LEFT_PAREN)
        elif c == ")":
            self.addToken(TokenType.RIGHT_PAREN)
        elif c == "{":
            self.addToken(TokenType.LEFT_BRACE)
        elif c == "}":
            self.addToken(TokenType.RIGHT_BRACE)
        elif c == ",":
            self.addToken(TokenType.COMMA)
        elif c == ".":
            self.addToken(TokenType.DOT)
        elif c == "-":
            self.addToken(TokenType.MINUS)
        elif c == "+":
            self.addToken(TokenType.PLUS)
        elif c == ";":
            self.addToken(TokenType.SEMICOLON)
        elif c == "*":
            self.addToken(TokenType.STAR)

        # operators
        elif c == "!":
            self.addToken(TokenType.BANG_EQUAL if self.match("=") else TokenType.BANG)
        elif c == "=":
            self.addToken(TokenType.EQUAL_EQUAL if self.match("=") else TokenType.EQUAL)
        elif c == "<":
            self.addToken(TokenType.LESS_EQUAL if self.match("=") else TokenType.LESS)
        elif c == ">":
            self.addToken(TokenType.GREATER_EQUAL if self.match("=") else TokenType.GREATER)

        # comments
        elif c == "/":
            if self.match("/"):
                # A comment goes until the end of the line.
                while self.peek() != "/n" and not self.isAtEnd():
                    self.advance()
            else:
                self.addToken(TokenType.SLASH)

        # whitespaces
        elif c == " ":
            pass
        elif c == "\r":
            pass
        elif c == "\t":
            # Ignore whitespace
            pass
        elif c == "\n":
            self.line += 1

        # string literals
        elif c == '"':
            self.string()

        else:
            # number literals
            if self.isDigit(c):
                self.number()
            # reserved number and identifier
            elif self.isAlpha(c):
                self.identifier()
            else:
                self.lox.error(self.line, "Unexpected character.")

    def identifier(self) -> None:
        while self.isAlphaNumeric(self.peek()):
            self.advance()

        text = self.source[self.start:self.current]
        type = self.KEYWORDS.get(text)
        if type is None:
            type = TokenType.IDENTIFIER

        self.addToken(type)

    def string(self) -> None:
        while self.peek() != '"' and not self.isAtEnd():
            if self.peek() == "\n":
                self.line += 1
            self.advance()

        if self.isAtEnd():
            self.lox.error(self.line, "Unterminated string.")

        # The closing ".
        self.advance()

        # Trim the surrounding quotes.
        value = self.source[self.start + 1: self.current - 1]
        self.addTokenLiteral(TokenType.STRING, value)

    def number(self) -> None:
        while self.isDigit(self.peek()):
            self.advance()

        # Look for fractional part
        if self.peek() == "." and self.isDigit(self.peekNext()):
            # Consume the "."
            self.advance()

            while self.isDigit(self.peek()):
                self.advance()

        value = float(self.source[self.start:self.current])
        self.addTokenLiteral(TokenType.NUMBER, value)

    def match(self, expected: str) -> bool:
        if self.isAtEnd():
            return False
        if self.source[self.current] != expected:
            return False

        self.current += 1
        return True

    def peek(self) -> str:
        if self.isAtEnd():
            return ""
        current_char = self.source[self.current]
        return current_char

    def peekNext(self) -> str:
        if self.current + 1 >= len(self.source):
            return ""
        return self.source[self.current + 1]

    def isAlpha(self, c:str) -> bool:
        return (c >= "a" and c <= "z") or (c >= "A" and c <= "Z") or c == "_"

    def isAlphaNumeric(self, c: str) -> bool:
        return self.isAlpha(c) or self.isDigit(c)

    def isDigit(self, c: str) -> bool:
        return c >= "0" and c <= "9"

    def isAtEnd(self) -> bool:
        return self.current >= len(self.source)

    def advance(self) -> str:
        current_char = self.source[self.current]
        self.current += 1
        return current_char

    def addToken(self, type: str) -> None:
        self.addTokenLiteral(type, None)

    def addTokenLiteral(self, type: str, literal: Optional[str]) -> None:
        text = self.source[self.start:self.current]
        self.tokens.append(Token(type, text, literal, self.line))


"""Ledger configuration."""

import asyncio
from collections import OrderedDict
import logging
import re
import sys

from aiohttp import ClientSession, ClientError, ClientResponse
import markdown
import prompt_toolkit
from prompt_toolkit.eventloop.defaults import use_asyncio_event_loop
from prompt_toolkit.formatted_text import HTML

from ..ledger.base import BaseLedger

from .base import ConfigError
from .injection_context import InjectionContext

LOGGER = logging.getLogger(__name__)

async def fetch_genesis_transactions(genesis_url: str):
    """Get genesis transactions."""
    headers = {}
    headers["Content-Type"] = "application/json"
    retry = 5
    LOGGER.info("Fetching genesis transactions from: %s", genesis_url)
    while True:
        try:
            async with ClientSession() as client_session:
                response: ClientResponse = await client_session.get(
                    genesis_url, headers=headers
                )
                if response.status < 200 or response.status >= 300:
                    raise ClientError("Bad response from server")
                genesis_txns = await response.text()
                return genesis_txns
        except ClientError as e:
            if retry:
                retry -= 1
                await asyncio.sleep(5.0)
            else:
                raise ConfigError("Error retrieving ledger genesis transactions") from e

async def ledger_config(
    context: InjectionContext, public_did: str, provision: bool = False
) -> bool:
    """Perform Indy ledger configuration."""

    # Fetch genesis transactions if necessary
    if not context.settings.get("ledger.genesis_transactions"):
        if context.settings.get("ledger.genesis_url"):
            context.settings[
                "ledger.genesis_transactions"
            ] = await fetch_genesis_transactions(context.settings["ledger.genesis_url"])
        elif context.settings.get("ledger.genesis_file"):
            try:
                genesis_path = context.settings["ledger.genesis_file"]
                LOGGER.info("Reading genesis transactions from: %s", genesis_path)
                with open(genesis_path, "r") as genesis_file:
                    context.settings["ledger.genesis_transactions"] = genesis_file.read(
                        -1
                    )
            except IOError as e:
                raise ConfigError("Error reading genesis transactions") from e

    ledger: BaseLedger = await context.inject(BaseLedger, required=False)
    if not ledger:
        LOGGER.info("Ledger instance not provided")
        return False
    elif ledger.LEDGER_TYPE != "indy":
        LOGGER.info("Non-indy ledger provided")
        return False

    async with ledger:
        # Check transaction author agreement acceptance
        taa_info = await ledger.get_txn_author_agreement()
        if taa_info["taa_required"] and public_did:
            taa_accepted = await ledger.get_latest_txn_author_acceptance()
            if (
                not taa_accepted
                or taa_info["taa_record"]["digest"] != taa_accepted["digest"]
            ):
                if not await accept_taa(ledger, taa_info, provision):
                    return False

        # Publish endpoint if necessary - skipped if TAA is required but not accepted
        endpoint = context.settings.get("default_endpoint")
        if public_did:
            await ledger.update_endpoint_for_did(public_did, endpoint)

    return True

async def accept_taa(ledger: BaseLedger, taa_info, provision: bool = False) -> bool:
    """Perform TAA acceptance."""

    if not sys.stdout.isatty():
        LOGGER.warning("Cannot accept TAA without interactive terminal")
        return False

    mechanisms = taa_info["aml_record"]["aml"]
    allow_opts = OrderedDict(
        [
            (
                "wallet_agreement",
                (
                    "Accept the transaction author agreement and store the "
                    + "acceptance in the wallet"
                ),
            ),
            (
                "on_file",
                (
                    "Acceptance of the transaction author agreement is on file "
                    + "in my organization"
                ),
            ),
        ]
    )
    if not provision:
        allow_opts["for_session"] = (
            "Accept the transaction author agreement for the duration of "
            + "the current session"
        )

    found = []
    for opt in allow_opts:
        if opt in mechanisms:
            found.append(opt)

    md = markdown.Markdown()
    taa_html = md.convert(taa_info["taa_record"]["text"])
    taa_html = re.sub(
        r"<h[1-6]>(.*?)</h[1-6]>", r"
\1
\n", taa_html
    )
    taa_html = re.sub(r"
(.*?)
", r" - \1", taa_html)

    taa_html = (
        "\nTransaction Author Agreement version "
        + taa_info["taa_record"]["version"]
        + "\n\n"
        + taa_html
    )

    # setup for prompt_toolkit
    use_asyncio_event_loop()

    prompt_toolkit.print_formatted_text(HTML(taa_html))

    opts = []
    num_mechanisms = {}
    for idx, opt in enumerate(found):
        num_mechanisms[str(idx + 1)] = opt
        opts.append(f" {idx+1}. {allow_opts[opt]}")
    opts.append(" X. Skip the transaction author agreement")
    opts_text = "\nPlease select an option:\n" + "\n".join(opts) + "\n[1]> "

    while True:
        try:
            opt = await prompt_toolkit.prompt(opts_text, async_=True)
        except EOFError:
            return False
        if not opt:
            opt = "1"
        opt = opt.strip()
        if opt in ("x", "X"):
            return False
        if opt in num_mechanisms:
            mechanism = num_mechanisms[opt]
            break

    await ledger.accept_txn_author_agreement(taa_info["taa_record"], mechanism)

    return True


import datetime
import itertools
import os

import numpy as np
from path import Path
from tensorboardX import SummaryWriter
from torch.backends import cudnn
from torch.utils.data import DataLoader

from dvmvs.dataset_loader import MVSDataset
from dvmvs.losses import LossMeter, update_losses
from dvmvs.pairnet.model import *
from dvmvs.train import train
from dvmvs.utils import zip_code, print_number_of_trainable_parameters, calculate_cost_volume_by_warping

class TrainingHyperparameters:
    Config.train_subsequence_length = 2
    Config.train_predict_two_way = True
    batch_size = 14
    learning_rate = 1e-4
    momentum = 0.9
    beta = 0.999
    weight_decay = 0
    # loss_type = "Huber"
    # loss_type = "L1"
    loss_type = "L1-inv"
    # loss_type = "L1-rel"

    finetune_epochs = 2

    use_augmentation = True
    use_checkpoint = False

scaling = 0.5
x = np.linspace(0, Config.train_image_width * scaling - 1, num=int(Config.train_image_width * scaling))
y = np.linspace(0, Config.train_image_height * scaling - 1, num=int(Config.train_image_height * scaling))
ones = np.ones(shape=(int(Config.train_image_height * scaling), int(Config.train_image_width * scaling)))
x_grid, y_grid = np.meshgrid(x, y)
warp_grid = np.stack((x_grid, y_grid, ones), axis=-1)
warp_grid = torch.from_numpy(warp_grid).float()
warp_grid = warp_grid.view(-1, 3).t().cuda()

def main():
    # set the manual seed for reproducibility
    torch.manual_seed(Config.train_seed)

    # create the directory for this run of the training
    run_directory = os.path.join(Config.train_run_directory, datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
    os.mkdir(run_directory)

    # zip every code file
    zip_code(run_directory)

    summary_writer = SummaryWriter(run_directory)

    print("=> fetching scenes in '{}'".format(Config.dataset))
    train_set = MVSDataset(
        root=Config.dataset,
        seed=Config.train_seed,
        split="TRAINING",
        subsequence_length=Config.train_subsequence_length,
        scale_rgb=255.0,
        mean_rgb=[0.485, 0.456, 0.406],
        std_rgb=[0.229, 0.224, 0.225],
        geometric_scale_augmentation=True
    )

    val_set = MVSDataset(
        root=Config.dataset,
        seed=Config.train_seed,
        split="VALIDATION",
        subsequence_length=Config.train_subsequence_length,
        scale_rgb=255.0,
        mean_rgb=[0.485, 0.456, 0.406],
        std_rgb=[0.229, 0.224, 0.225]
    )

    print('{} samples found in {} train scenes'.format(len(train_set), len(train_set.scenes)))
    print('{} samples found in {} valid scenes'.format(len(val_set), len(val_set.scenes)))
    train_loader = DataLoader(dataset=train_set,
                              batch_size=TrainingHyperparameters.batch_size,
                              shuffle=True,
                              num_workers=Config.train_data_pipeline_workers,
                              pin_memory=True,
                              drop_last=True)

    val_loader = DataLoader(dataset=val_set,
                            batch_size=TrainingHyperparameters.batch_size,
                            shuffle=False,
                            num_workers=Config.train_data_pipeline_workers,
                            pin_memory=True,
                            drop_last=True)

    feature_extractor = FeatureExtractor()
    feature_shrinker = FeatureShrinker()
    cost_volume_encoder = CostVolumeEncoder()
    cost_volume_decoder = CostVolumeDecoder()
    feature_extractor = feature_extractor.cuda()
    feature_shrinker = feature_shrinker.cuda()
    cost_volume_encoder = cost_volume_encoder.cuda()
    cost_volume_decoder = cost_volume_decoder.cuda()
    model = [feature_extractor, feature_shrinker, cost_volume_encoder, cost_volume_decoder]

    if TrainingHyperparameters.use_checkpoint:
        for i in range(len(model)):
            try:
                checkpoint = Path(".").files("module_" + str(i) + "*")[0]
                weights = torch.load(checkpoint)
                model[i].load_state_dict(weights)
                print("Loaded weights for", checkpoint)
            except Exception as e:
                print(e)
                print("Skipping...")

    cudnn.benchmark = True

    best_loss = [np.inf, np.inf, np.inf, np.inf]

    # TRAIN MY PARTS
    parameters = itertools.chain(feature_shrinker.parameters(),
                                 cost_volume_encoder.parameters(),
                                 cost_volume_decoder.parameters())
    optimizer = torch.optim.Adam(parameters,
                                 lr=TrainingHyperparameters.learning_rate,
                                 betas=(TrainingHyperparameters.momentum, TrainingHyperparameters.beta),
                                 weight_decay=TrainingHyperparameters.weight_decay)
    print_number_of_trainable_parameters(optimizer)
    for epoch in range(TrainingHyperparameters.finetune_epochs):
        print("\n\nEPOCH:", epoch)
        train(train_loader=train_loader,
              val_loader=val_loader,
              model=model,
              optimizer=optimizer,
              summary_writer=summary_writer,
              epoch=epoch,
              best_loss=best_loss,
              run_directory=run_directory,
              forward_pass_function=forward_pass)

    # TRAIN EVERYTHING
    parameters = itertools.chain(feature_extractor.parameters(),
                                 feature_shrinker.parameters(),
                                 cost_volume_encoder.parameters(),
                                 cost_volume_decoder.parameters())
    optimizer = torch.optim.Adam(parameters,
                                 lr=TrainingHyperparameters.learning_rate,
                                 betas=(TrainingHyperparameters.momentum, TrainingHyperparameters.beta),
                                 weight_decay=TrainingHyperparameters.weight_decay)
    print_number_of_trainable_parameters(optimizer)
    for epoch in range(TrainingHyperparameters.finetune_epochs, Config.train_epochs):
        print("\n\nEPOCH:", epoch)
        train(train_loader=train_loader,
              val_loader=val_loader,
              model=model,
              optimizer=optimizer,
              summary_writer=summary_writer,
              epoch=epoch,
              best_loss=best_loss,
              run_directory=run_directory,
              forward_pass_function=forward_pass)

def forward_pass(images, depths, poses, K, model, is_training):
    feature_extractor = model[0]
    feature_shrinker = model[1]
    cost_volume_encoder = model[2]
    cost_volume_decoder = model[3]

    K[:, 0:2, :] = K[:, 0:2, :] * scaling
    K = K.cuda()

    images_cuda = []
    depths_cuda = []
    poses_cuda = []
    feature_halfs = []
    feature_quarters = []
    feature_one_eights = []
    feature_one_sixteens = []
    # Extract image features
    for i in range(0, len(images)):
        images_cuda.append(images[i].cuda())
        poses_cuda.append(poses[i].cuda())
        depths_cuda.append(depths[i].cuda())
        feature_half, feature_quarter, feature_one_eight, feature_one_sixteen = feature_shrinker(*feature_extractor(images_cuda[i]))
        feature_halfs.append(feature_half)
        feature_quarters.append(feature_quarter)
        feature_one_eights.append(feature_one_eight)
        feature_one_sixteens.append(feature_one_sixteen)

    optimizer_loss = 0
    predictions = None
    l1_meter = LossMeter()
    huber_meter = LossMeter()
    l1_inv_meter = LossMeter()
    l1_rel_meter = LossMeter()
    for i in range(1, len(images)):
        reference_index = i
        measurement_index = i - 1

        if Config.train_predict_two_way:
            iterations = [[measurement_index, reference_index],
                          [reference_index, measurement_index]]
        else:
            iterations = [[reference_index, measurement_index]]

        for [index1, index2] in iterations:
            initial_cost_volume = calculate_cost_volume_by_warping(image1=feature_halfs[index1],
                                                                   image2=feature_halfs[index2],
                                                                   pose1=poses_cuda[index1],
                                                                   pose2=poses_cuda[index2],
                                                                   K=K,
                                                                   warp_grid=warp_grid,
                                                                   min_depth=Config.train_min_depth,
                                                                   max_depth=Config.train_max_depth,
                                                                   n_depth_levels=Config.train_n_depth_levels,
                                                                   device=torch.device('cuda'),
                                                                   dot_product=True)
            flipped = False
            to_be_used_feature_one_sixteen = feature_one_sixteens[index1]
            to_be_used_feature_one_eight = feature_one_eights[index1]
            to_be_used_feature_quarter = feature_quarters[index1]
            to_be_used_feature_half = feature_halfs[index1]
            to_be_used_image = images_cuda[index1]
            to_be_used_depth = depths_cuda[index1]
            to_be_used_cost_volume = initial_cost_volume
            if is_training and TrainingHyperparameters.use_augmentation and np.random.random() > 0.5:
                to_be_used_feature_one_sixteen = torch.flip(feature_one_sixteens[index1], dims=[-1])
                to_be_used_feature_one_eight = torch.flip(feature_one_eights[index1], dims=[-1])
                to_be_used_feature_quarter = torch.flip(feature_quarters[index1], dims=[-1])
                to_be_used_feature_half = torch.flip(feature_halfs[index1], dims=[-1])
                to_be_used_image = torch.flip(images_cuda[index1], dims=[-1])
                to_be_used_depth = torch.flip(depths_cuda[index1], dims=[-1])
                to_be_used_cost_volume = torch.flip(initial_cost_volume, dims=[-1])
                flipped = True

            skip0, skip1, skip2, skip3, bottom = cost_volume_encoder(features_half=to_be_used_feature_half,
                                                                     features_quarter=to_be_used_feature_quarter,
                                                                     features_one_eight=to_be_used_feature_one_eight,
                                                                     features_one_sixteen=to_be_used_feature_one_sixteen,
                                                                     cost_volume=to_be_used_cost_volume)

            depth_full, depth_half, depth_quarter, depth_one_eight, depth_one_sixteen = cost_volume_decoder(to_be_used_image,
                                                                                skip0,
                                                                                skip1,
                                                                                skip2,
                                                                                skip3,
                                                                                bottom)

            predictions = [depth_one_sixteen, depth_one_eight, depth_quarter, depth_half, depth_full]

            weights = [1, 1, 1, 1, 1]
            optimizer_loss = optimizer_loss + update_losses(predictions=predictions,
                                                            weights=weights,
                                                            groundtruth=to_be_used_depth,
                                                            is_training=is_training,
                                                            l1_meter=l1_meter,
                                                            huber_meter=huber_meter,
                                                            l1_inv_meter=l1_inv_meter,
                                                            l1_rel_meter=l1_rel_meter,
                                                            loss_type=TrainingHyperparameters.loss_type)

            if flipped and index1 == len(images) - 1:
                depth_quarter = torch.flip(depth_quarter, dims=[-1])
                depth_half = torch.flip(depth_half, dims=[-1])
                depth_full = torch.flip(depth_full, dims=[-1])

            predictions = [depth_quarter, depth_half, depth_full]

    predictions_names = ["prediction_quarter", "prediction_half", "prediction_full"]

    return l1_meter, huber_meter, l1_inv_meter, l1_rel_meter, optimizer_loss, predictions, predictions_names

if __name__ == '__main__':
    main()


import sys

import click
import subprocess

from . import log
from .utils import dd, dump

# Exit if not running as root - NO, can be use backups too
# if os.geteuid() != 0: exit('This script must be run as root')

CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])

@click.group(context_settings=CONTEXT_SETTINGS)
def handle():
    """
    \b
    mreschke.serverbackups
    Copyright (c) 2018 Matthew Reschke
    License http://mreschke.com/license/mit
    """
    pass

@handle.command()
# @click.option('-a', '--all', default='backups.py', show_default=True)
@click.option('--all', is_flag=True, help="Run all backups (both config.yml and custom code)")
@click.option('--custom', is_flag=True, help="Run only custom backup code")
@click.option('--servers', is_flag=True, help="Run entire config.yml only, no custom code")
@click.option('--server', help="Run just one server from config.yml")
@click.option('--cluster', help="Run all servers in a cluster from config.yml")
def run(all, custom, servers, server, cluster):
    """Run backups"""

def start(backups):
    # Only run with proper cli flags
    if 'run' not in sys.argv and '--help' not in sys.argv:
        log.error('Missing run option, see --help')
        exit(1)

    if not any(x in ['--all', '--servers', '--server', '--cluster'] for x in sys.argv):
        if 'run' in sys.argv and '--custom' not in sys.argv and '--help' not in sys.argv and '-h' not in sys.argv:
            log.error('Missing run option, see --help')
            exit(1)
        return

    log.blank()
    log.separator()
    log.header3("Running server backups")
    log.separator()

    # Run a single server
    if '--server' in sys.argv:
        server = sys.argv[sys.argv.index("--server") + 1]
        log.bullet("Running single server '{}'".format(server))
        backups.run(server=server)

    # Run all servers in a single cluster
    elif '--cluster' in sys.argv:
        cluster = sys.argv[sys.argv.index("--cluster") + 1]
        log.bullet("Running servers in cluster '{}'".format(cluster))
        backups.run(cluster=cluster)

    else:
        log.bullet("Running all servers defined in config")
        backups.run()

# def backupserversOLD(config_path='/etc/mreschke/serverbackups', servers=None, defaults=None):
#     """Run the main BackupServer class from the server config.yml and config.d files
#     """
#     # Only run with proper cli flags
#     if not any(x in ['--all', '--servers', '--server', '--cluster'] for x in sys.argv): return

#     log.blank()
#     log.separator()
#     log.header3("Running server backups")
#     log.separator()

#     if servers:
#         # Passing in a dictionary of servers
#         log.bullet("Backups defined as inline dictionary")
#         if not defaults:
#             log.bullet("No DEFAULTS dictionary passed inline, using builtin DEFAULTS")
#             defaults_file = os.path.dirname(os.path.realpath(__file__)) + '/templates/' + 'defaults.yml'
#             with io.open(defaults_file, 'r') as stream:
#                 defaults = yaml.load(stream, Loader=yaml.SafeLoader)
#         else:
#             log.bullet("DEFAULTS defined as inline dictionary string")
#             defaults = yaml.load(defaults)
#     else:
#         # Using config directory
#         log.bullet("Using config dir {}".format(config_path))
#         configd_path = config_path + '/config.d'
#         defaults_file = config_path + '/defaults.yml'

#         # Read defaults.yml file
#         with io.open(defaults_file, 'r') as stream:
#             defaults = yaml.load(stream)

#         # Read each server config in config.d folder
#         servers = {}
#         for file in os.listdir(configd_path):
#             with io.open(configd_path + "/" + file, 'r') as stream:
#                 servers = {**servers, **yaml.load(stream)}

#     # Run a single server
#     if '--server' in sys.argv:
#         server = sys.argv[sys.argv.index("--server") + 1]
#         log.bullet("Running single server '{}'".format(server))
#         BackupServer(log, server, servers[server], defaults).run()

#     # Run all servers in a single cluster
#     elif '--cluster' in sys.argv:
#         cluster = sys.argv[sys.argv.index("--cluster") + 1]
#         log.bullet("Running servers in cluster '" + cluster + "'")
#         for server, options in servers.items():
#             if options['cluster'] == cluster:
#                 BackupServer(log, server, options, defaults).run()

#     else:
#         log.bullet("Running all servers defined in config")
#         for server, options in servers.items():
#             BackupServer(log, server, options, defaults).run()

def allowcustom():
    """Helper for backups.py to determine if custom backup code should be run
    """
    return any(x in ['--custom', '--all'] for x in sys.argv)

def kill_if_running(script):
    # Kill myself if already running
    procs = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
    found = 0
    for line in procs.stdout:
        if script in line: found += 1
    if found > 1:
        log.error('Backups already running.  Exiting')
        exit()

# Explicit exports
__all__ = ['handle', 'allowcustom', 'start', 'log', 'dump', 'dd']


from redis import StrictRedis
if __name__ == "__main__":
    # 创建一个StrictRedis对象,链接redis数据库
    try:
        sr = StrictRedis(host='192.168.21.100')
        # 添加一对键值
        #res = sr.set('name','guido')
        # print(res)

        # 获取name的值
        # res = sr.get('name')
        # print(res)

        # 删除name及对应的值
        #res = sr.delete('name')
        #print(res)
        #删除多个键值
        #res = sr.delete('a1','a2')
        #print(res)

        # 获取数据库中所有的键
        res = sr.keys()
        print(res)
    except Exception as e:
        print(e)


import pandas
import os
import random
import argparse
import pysam
from FastqStreamer.FastqReader import FastqReader
from BamStreamer.BamUtils import read_alignments
##from FastqIterator import FastqIterator
##from link_bams import BamLinker

class IPeteSpikeIn(FastqReader):
    """
    Filter spike-in reads from fastq file

    Attributes
    ----------
    spikes : pandas data frame
    Pandas data frame containing spike-in alignments

    spike_stats : dict
    Dictionary for counting spike-in alignments
    """

    def __init__(self, spikein, reference_annot_file, basename):
        FastqReader.__init__(self)
        self.spikes = self.load_spike_aln(spikein)
        self.spike_stats = {}
        self.basename = basename

    def load_spike_aln(self, spikeBam):
        """
        Read spike in alignment names into memory, store read and spikeID together

        Parameters
        ----------
        spikeBam : str
        path to spikein alignment bam file

        Returns
        -------
        spikes : dict
        dictionary of read name keys and spikeId values

        """
        spikes = {}
        for read in read_alignments(spikeBam):
            read_name = read["query_name"]
            spikes[read_name] = read['reference_name']
        return spikes

    def split_bams(self, Vbam, Jbam):
        """
        use spikein read alignments to split V and J alignments into spike-in and native read files.

        Sbam : str
        spikein alignments bam file
        Vbam : str
        Vgene alignments bam file
        Jbam : str
        Jgene alignments bam file
        """

        Vsam = pysam.AlignmentFile(Vbam, "rb", check_sq=False)
        nativeV = pysam.AlignmentFile("{}_Vgene_native.bam".format(self.basename), "wb", template=Vsam)
        spikeV = pysam.AlignmentFile("{}_Vgene_spikein.bam".format(self.basename), "wb", template=Vsam)
        Vsam.close()
        ##split V gene alignments reads based on spikein alignments
        for read in read_alignments(Vbam):
            if read["query_name"] in self.spikes:
                spikeV.write(read['pyread'])
            else:
                nativeV.write(read['pyread'])
        nativeV.close()
        spikeV.close()
        Jsam = pysam.AlignmentFile(Jbam, "rb", check_sq=False)
        nativeJ = pysam.AlignmentFile("{}_Jgene_native.bam".format(self.basename), "wb", template=Jsam)
        spikeJ = pysam.AlignmentFile("{}_Jgene_spikein.bam".format(self.basename), "wb", template=Jsam)
        Jsam.close()
        ##split J gene alignments reads based on spikein alignments
        for read in read_alignments(Jbam):
            if read["query_name"] in self.spikes:
                spikeJ.write(read['pyread'])
            else:
                nativeJ.write(read['pyread'])
        nativeJ.close()
        spikeJ.close()

        ##can also process with `read_two_bams` via BamParser, but that is slower with larger files.

    def label_spikeins(self, spikeDF, spikeUMI_df):
        UMI_groups = spikeDF.UMI_group.unique()
        spikeGroups = {}
        for group in UMI_groups:
            groupDF = spikeDF.loc[spikeDF["UMI_group"] == group]
            family_size = groupDF.shape[0]
            Spikes = groupDF["spikein"].value_counts(dropna=False)
            spikeGroups[group] = Spikes.idxmax()
        spikeUMI_df["spikein"] = spikeUMI_df["UMI_group"].map(lambda x : spikeGroups[x])
        return spikeUMI_df

    def cummulative_stats(self, df):
        df["cummulative_reads"] = df["UMI_family_size"].cumsum()
        df["cummulative_fraction"] = df["cummulative_reads"] / sum(df["UMI_family_size"])
        return df

    def split_umi_reports(self, VDJfile, UMIfams):
        VDJreport = pandas.read_csv(VDJfile, sep="\t")
        UMIreport = pandas.read_csv(UMIfams, sep="\t")
        ## spikein reads by name
        VDJspikein1 = VDJreport[VDJreport["name"].isin(self.spikes)]
        VDJspikein1["spikein"] = VDJspikein1["name"].map(lambda x : self.spikes[x])
        ## native reads by name
        VDJnative = VDJreport[~VDJreport["name"].isin(self.spikes)]
        ## spike in reads by UMI group
        VDJspikein2 = VDJnative[VDJnative["UMI_group"].isin(VDJspikein1["UMI_group"])]
        VDJspikein = pandas.concat([VDJspikein1, VDJspikein2])
        NativeReport = UMIreport[~UMIreport["UMI_group"].isin(VDJspikein["UMI_group"])]
        SpikeinReport = UMIreport[UMIreport["UMI_group"].isin(VDJspikein["UMI_group"])]
        SpikeinReport = self.cummulative_stats(SpikeinReport)
        NativeReport = self.cummulative_stats(NativeReport)
        ## count spikeins
        SpikeinReport = self.label_spikeins(VDJspikein, SpikeinReport)
        ##label UMI report with consensus spikein label
        ##output files
        nativeTabOut = "{}_on_target_umi_groups_native.tsv".format(self.basename)
        spikeTabOut = "{}_on_target_umi_groups_spikein.tsv".format(self.basename)
        nativeUMIOut = "{}_cdr3_dedup_report_native.tsv".format(self.basename)
        spikeUMIOut = "{}_cdr3_dedup_report_spikein.tsv".format(self.basename)
        ##write output
        VDJnative.to_csv(nativeTabOut, sep="\t", index=False)
        VDJspikein.to_csv(spikeTabOut, sep="\t", index=False)
        NativeReport.to_csv(nativeUMIOut, sep="\t", index=False)
        SpikeinReport.to_csv(spikeUMIOut, sep="\t", index=False)

    def report_spikein_counts(self, VJaln, outFile):
        """
        write a report file of spike-in sequences

        Parameters
        ----------
        outFile : str
        Path to output file

        Returns
        -------
        None
        """
        # summarize VJ alignments of spike-in reads
        VJspike = {}
        try:
            aln = pandas.read_csv(VJaln, sep="\t")
            if len(aln) > 0:
                for name, Vgene, Jgene, uid_group in zip(aln["name"], aln["v_gene"], aln["j_gene"], aln["UMI_group"]):
                    VJpair = "{}__{}".format(Vgene, Jgene)
                    if name in self.spikes:
                        spikeId = self.spikes[name]
                        if not spikeId in VJspike:
                            VJspike[spikeId] = {"match": 0, "diff": 0, "uids": []}
                        if VJpair == spikeId:
                            VJspike[spikeId]["match"] += 1
                        else:
                            VJspike[spikeId]["diff"] += 1
                        uid = "{}_{}".format(VJpair, uid_group)
                        VJspike[spikeId]["uids"].append(uid)
                # create summary for spike-in reads
                for name in self.spikes:
                    spike = self.spikes[name]
                    if not spike in self.spike_stats:
                        self.spike_stats[spike] = {"reads": 0,
                                                   "on_target": 0,
                                                   "VJ_match": 0,
                                                   "dedup_count": 0}
                    self.spike_stats[spike]["reads"] += 1
                for spike in self.spike_stats:
                    if spike in VJspike:
                        self.spike_stats[spike]["on_target"] = (
                            VJspike[spike]["match"] + VJspike[spike]["diff"])
                        self.spike_stats[spike]["VJ_match"] = VJspike[spike]["match"]
                        self.spike_stats[spike]["dedup_count"] = len(set(VJspike[spike]["uids"]))
        except Exception as e:
            print("Error: ", e)
            pass
        # spike-in report
        with open(outFile, 'w') as fhOut:
            fhOut.write("synthetic,reads,on_target,VJ_match,dedup_count\n")
            for spike in self.spike_stats:
                vals = [spike,
                        str(self.spike_stats[spike]["reads"]),
                        str(self.spike_stats[spike]["on_target"]),
                        str(self.spike_stats[spike]["VJ_match"]),
                        str(self.spike_stats[spike]["dedup_count"])]
                vals = ",".join(vals)+"\n"
                fhOut.write(vals)

def main():
    parser = argparse.ArgumentParser(
        description="gather amplicon stats from bam file")
    parser.add_argument("-s", "--spike_aln",
                        required=True,
                        help="Spike in alignment bam")
    parser.add_argument("-v", "--v_aln",
                        required=True,
                        help="V gene alignment bam")
    parser.add_argument("-j", "--j_aln",
                        required=True,
                        help="J gene alignment bam")
    parser.add_argument("-VJ", "--VJ_umi",
                        required=True,
                        help="V & J gene alignment summary file")
    parser.add_argument("-d", "--dedup_report",
                        required=True,
                        help="V & J gene alignment summary file")
    parser.add_argument("-r", "--reference_annot",
                        required=True,
                        help="reference annotation file")
    parser.add_argument("-b", "--basename",
                        required=True,
                        help="basename argument for report file name")
    args = parser.parse_args()
    parser = IPeteSpikeIn(args.spike_aln, args.reference_annot, args.basename)
    parser.split_bams(args.v_aln, args.j_aln)
    parser.split_umi_reports(args.VJ_umi, args.dedup_report)
    ##parser.report_spikein_counts(args.VJaln, "{}_{}".format(
    ##args.basename, "spikein_stats.csv"))

if __name__ == '__main__':
    main()


from os import access
import dropbox

class TransferData:
    def __init__(self, access_token):
        self.access_token= access_token

    def upload_files(self, file_from, file_to):
        dbx=dropbox.Dropbox(self.access_token)
        f=open(file_from, "rb")
        dbx.files_upload(f.read(), file_to)

def main():
    access_token="wslHcaboW6sAAAAAAAAAAcAMNwLbCUkjfVa8ygrgZb1XHl0V7PSH2qVLbSzxYLWw"
    transfer_data=TransferData(access_token)
    file_from=input("Enter the file path to upload: ")
    file_to=input("Enter the file path for dropbox: ")

    transfer_data.upload_files(file_from, file_to)
    print("File successfully uploaded!")

main()


import torch
import torch.nn.functional as F
import torch.nn as nn
from torchvision.models import resnet18

class Network(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.fc1 = torch.nn.Linear(input_size, 256)
        self.fc2 = torch.nn.Linear(256, 128)
        self.fc3 = torch.nn.Linear(128, 64)
        self.fc4 = torch.nn.Linear(64, classes)

        # Dropout module with 0.2 drop probability
        self.dropout = torch.nn.Dropout(p=0.2)

    def forward(self, x):
        # make sure input tensor is flattened
        x = x.view(x.shape[0], -1)

        # Now with dropout
        x = self.dropout(F.relu(self.fc1(x)))
        x = self.dropout(F.relu(self.fc2(x)))
        x = self.dropout(F.relu(self.fc3(x)))

        # output so no dropout here
        x = F.log_softmax(self.fc4(x), dim=1)

        return x

class Lenet5(torch.nn.Module):
    def __init__(self, classes):
        super().__init__()
        self.conv = torch.nn.Sequential(

            torch.nn.Conv2d(3, 6, 5), # out = (in - kernal + 2*padding) / stride + 1  (1, 28, 28) -> (6, 24, 24)
            torch.nn.ReLU(),
            torch.nn.AvgPool2d(2, stride=2), # (N, 6, 24, 24) - > (N, 6, 12, 12)

            torch.nn.Conv2d(6, 16, 5),  #  (N, 6, 12, 12) -> (N. 6, 8, 8)
            torch.nn.ReLU(),
            torch.nn.AvgPool2d(2, stride=2)  # (N, 16, 8, 8) -> (N, 16, 4 4)
        )
        self.fc = torch.nn.Sequential(
            # torch.nn.Linear(256, 120), #(N, 256) - > (N, 120)
            # torch.nn.ReLU(),
            # torch.nn.Linear(120, 84), #(N, 120) - > (N, 84)
            # torch.nn.ReLU(),
            # torch.nn.Linear(84, classes)

            # torch.nn.Linear(576, 256),
            # torch.nn.ReLU(),

            torch.nn.Linear(256, classes),
            torch.nn.ReLU(),
            torch.nn.Linear(classes, classes),

            )
    def forward(self, x):
        x = self.conv(x)
        # print(x.size()) # torch.Size([64, 16, 4, 4])
        x = x.view(x.size(0), -1)  # batch_size is x.size(0)

        x = self.fc(x)
        return x

def my_resnt18(classes):
    resnet = resnet18()
    resnet.fc = torch.nn.Linear(in_features=512, out_features=classes, bias=True)
    resnet.add_module("softmax", torch.nn.Softmax(dim=-1))

    return resnet

# classes =145
# myresnt18 = resnet18()
# myresnt18.fc = torch.nn.Linear(in_features=512, out_features=classes, bias=True)
# myresnt18.add_module("softmax", torch.nn.Softmax(dim=-1))

class FashionCNN(nn.Module):
    def __init__(self, classes):
        super().__init__()
        self.layer1 = nn.Sequential(
            nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding=1),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2)
        )

        self.layer2 = nn.Sequential(
            nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(2)
        )

        self.fc1 = nn.Linear(in_features=64*6*6, out_features=600)
        self.drop = nn.Dropout2d(0.25)
        self.fc2 = nn.Linear(in_features=600, out_features=300)  #   classes: 300 to 120 to 10
        self.fc3 = nn.Linear(in_features=300, out_features=classes)

    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        out = out.view(out.size(0), -1)
        out = self.fc1(out)
        out = self.drop(out)
        out = self.fc2(out)
        out = self.fc3(out)

        return out