import os
import argparse
from model import ConvBlock, Wav2Letter
import torch
import time

import numpy as np
import torch
import torchaudio
import torch.nn as nn
from torch.nn.utils.rnn import *
from torch.utils.data import DataLoader, Dataset
from utils.transcript_utils import *

import os
import sys
import numpy as np
import pandas as pd
import torch
import timeit
import torch.nn as nn
import onnxruntime as rt
from torch.autograd import Variable
from ctcdecode import CTCBeamDecoder

from data.dataloader import *
from hyperparameters import get_hyperparameters
from utils.transcript_utils import *
from utils.train_utils import *
from utils.metric_utils import *
from model import Wav2Letter

parser = argparse.ArgumentParser(description='Script to evaluate a pytorch model on HW4 data')
parser.add_argument('--model', type=str, help='path to the model file')
parser.add_argument('--beamwidth', type=int, default=20, help='beamwidth for the CTCBeamDecoder')
parser.add_argument('--outvocab', type=int, default=35, help='Out vocab for the model')
parser.add_argument('--batchsize', type=int, default=1, help='Batch size to evaluate the model')
parser.add_argument('--datadir', type=str, default='data',
                    help='data directory where train.npy, dev.npy, train_transcripts.npy, and dev_transcripts.npy are present')
parser.add_argument('--device', type=str, default='cuda',
                    help='cuda or cpu are valid options')
parser.add_argument('--quantized', action='store_true', default=False)

# Evaluate the model
@torch.no_grad()
def evaluate_model(val_loader, model, criterion, batchsize=32, beamwidth=20, device=None, quantized=False):
    predictions = []
    actuals = []
    val_loss = 0

    # distances = []
    input_name = ""
    # Set model in validation mode
    if not quantized:
        model.eval()
    else:
        input_name = model.get_inputs()[0].name

    stime = time.time()

    for i, (inputs, targets, input_lengths, target_lengths, input_len_ratio) in enumerate(val_loader):
        inputs = inputs.transpose(0, 1).transpose(1, 2).reshape((batchsize, 40, -1)).to(device)
        input_lengths = input_lengths.to(device)
        targets = targets.to(device)
        target_lengths = target_lengths.to(device)

        # evaluate the model on the validation set
        if not quantized:
            out = model(inputs.float())
        else:
            out = torch.tensor(model.run(None, {input_name: inputs.numpy().astype(np.float32)})[0])

        out = out.permute(2, 1, 0)

        seq_length = out.size(0)
        out_lengths = Variable(input_len_ratio.mul_(int(seq_length)).int(), requires_grad=False)

        # Calculate validation loss
        loss = criterion(out, targets, out_lengths, target_lengths)

        actual = targets.cpu().numpy()
        actual_lengths = target_lengths.cpu().numpy()

        decoder = CTCBeamDecoder(LETTER_LIST + ["&"], beam_width=beamwidth, log_probs_input=True, blank_id=len(LETTER_LIST))
        out, _, _, out_lengths = decoder.decode(out.transpose(0, 1), out_lengths)

        # reshape for stacking
        actual = unpad_beam1(actual, actual_lengths)
        out = unpad(torch.squeeze(out, 1).cpu().numpy(), torch.squeeze(out_lengths, 1).cpu().numpy())

        # store
        predictions.append(out)
        actuals.append(actual)

        if i%50==0:
            print(f"\tBatch: {i}/{len(val_loader)}\t Loss: {loss.item():3.3f}\ttime-elapsed: {time.time()-stime:3.3f} Secs")

        del out
        del out_lengths
        del targets
        del target_lengths
        del inputs
        del input_lengths
        del input_len_ratio

        torch.cuda.empty_cache()
        val_loss += loss.item()

    p = generate_all_outputs(predictions)
    a = generate_all_outputs(actuals)

    # Calculate Validation Levenshtein distance
    if len(p) != len(a):
        print("ERROR")

    dist = calc_levenshtein_dist(a, p)
    val_loss /= len(val_loader)
    print(f"Evaluation done in {time.time()-stime: 3.3f} Secs")
    print(f"\rAverage Loss: {val_loss:3.3f} and Average Edit Distance: {dist:3.3f}")
    return dist, val_loss

if __name__=="__main__":
    LETTER_LIST = ['<', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q',
                   'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '-', "'", '.', '_', '+', ' ', '>']

    letter2index, index2letter = create_dictionaries(LETTER_LIST)
    num_workers = get_num_workers()
    hparams = get_hyperparameters()
    hparams["num_workers"] = num_workers

    args = parser.parse_args()
    print(args)
    print(f"Loading model: {args.model}")

    DATA_PATH = args.datadir
    DEVICE = torch.device(args.device)

    model = Wav2Letter(args.outvocab)

    quantized = args.quantized
    if quantized and args.device=='cuda':
      print("Please note quantized models are not supported on GPU yet, so changing the backend to CPU")
      DEVICE = torch.device('cpu')

    if not quantized:
        mdl = torch.load(args.model, map_location=DEVICE)
        model.load_state_dict(mdl.state_dict())
        model.to(DEVICE)
    else:
        model = rt.InferenceSession(args.model)

    # Filepaths
    dev_filename = os.path.join(DATA_PATH, "dev.npy")
    dev_transcripts_filename = os.path.join(DATA_PATH, "dev_transcripts.npy")

    # criterion
    criterion = nn.CTCLoss().to(DEVICE)

    # # Validation dataloader
    val_data = RnnDataset(dev_filename, dev_transcripts_filename, letter2index, is_train=False)
    val_args = dict(shuffle=False, batch_size=args.batchsize, num_workers=hparams["num_workers"], drop_last=True,
                    collate_fn=collate_train_val)
    val_loader = DataLoader(val_data, **val_args)
    evaluate_model(val_loader, model, criterion, args.batchsize, args.beamwidth, DEVICE, quantized)


from PIL import Image, ImageFont, ImageDraw
import requests
from io import BytesIO

def currentHeight(text, font):
    return font.getsize(text)[1]
def currentWidth(text, font):
    return font.getsize(text)[0]
fontFamily = "impact.ttf"

class Meme:
    def __init__(self):
        self.requestFileName()
        self.requestLink()
        self.getImage()
        self.requestText()
        self.save()
        self.show()
    def requestFileName(self):
        print("gimme file name")
        self.name = input("name: ")
    def requestLink(self):
        print("gimme link")
        self.url = input("link: ")
    def getImage(self):
        response = requests.get(self.url)
        self.img = Image.open(BytesIO(response.content))
        self.draw = ImageDraw.Draw(self.img)
        self.img_fraction = 0.15
    def requestText(self):
        print("gimme top text")
        topText = input("top text: ").upper()
        topPos = 0
        self.addText(topText, topPos)
        print("gimme bottom text")
        bottomText = input("bottom text: ").upper()
        tempFontSize = 1
        tempFont = ImageFont.truetype(fontFamily, tempFontSize)
        while currentHeight(bottomText, tempFont) < self.desiredHeight and currentWidth(bottomText, tempFont) < self.desiredWidth:
                tempFontSize += 1
                tempFont = ImageFont.truetype(fontFamily, tempFontSize)
        bottomPos = self.img.size[1]-tempFont.getsize(bottomText)[1]
        self.addText(bottomText, bottomPos)
    def save(self):
        self.img.save(self.name+'.jpg')
    def show(self):
        self.img.show()
    def addText(self, text, yPos):
        self.desiredHeight = self.img_fraction*self.img.size[1]
        self.desiredWidth = self.img.size[0]
        fontSize = 1
        font = ImageFont.truetype(fontFamily, fontSize)
        while currentHeight(text, font) < self.desiredHeight and currentWidth(text, font) < self.desiredWidth:
            fontSize += 1
            font = ImageFont.truetype(fontFamily, fontSize)
        halfWidth = self.img.size[0]/2-font.getsize(text)[0]/2
        x = halfWidth
        y = yPos
        black = (0,0,0)
        white = (255,255,255)
        Left = (x-2, y-2)
        Right = (x+2, y-2)
        bottomRight = (x+2, y+2)
        bottomLeft = (x-2, y+2)
        self.draw.text(Left, text, black, font=font)
        self.draw.text(Right, text, black, font=font)
        self.draw.text(bottomRight, text, black, font=font)
        self.draw.text(bottomLeft, text, black, font=font)
        self.draw.text((x,y),text, white, font=font)


#!/usr/bin/env python

import sys

import click
import numpy as np
import pandas as pd
import tifffile as tiff
from rich import print, traceback

@click.command()
@click.option("-m", "--meta", required=True, type=str, help="Path to metadata file")
@click.option("-r", "--ratios", required=True, type=str, help="Path to ratios")
@click.option("-s", "--segs", required=True, type=str, help="Path to segmentations")
@click.option("-o", "--output", default="./", type=str, help="Output path")
def main(meta: str, ratios: str, segs: str, output: str):
    """Command-line interface for rtastat"""

    print(
        r"""[bold blue]
        rtastat
        """
    )

    print("[bold blue]Run [green]rtastat --help [blue]for an overview of all commands\n")
    df = pd.read_csv(meta, header=0)
    df["Ratio"], df["Zone"] = zip(*[(calc_ratio(ratios, segs, x)) for x in df["Filename"]])
    df.to_csv("ratios.tsv", sep="\t", index=False)

def calc_ratio(ratios, segs, x):
    ratio_img = tiff.imread(ratios + x + "_ratio.tif")
    ratio_img = np.nan_to_num(ratio_img)
    tif_img = np.load(segs + x + ".npy")
    ratio_ee = extract_ph(ratio_img, tif_img, 2)
    ratio_m = extract_ph(ratio_img, tif_img, 4)
    # Early elongation zone is generally a bit smaller in size.
    if ratio_m > 1.25 * ratio_ee:
        ratio = ratio_m
        zone = "Meristematic Zone"
    else:
        ratio = ratio_ee
        zone = "Early Elongation Zone"
    if ratio == 0:
        return np.nan, "None"
    else:
        return ratio, zone

def extract_ph(ratio_img, tif_img, class_index):
    late_array = ratio_img[tif_img == class_index]
    # Exclude extremely small predictions, as they are likely to be FP.
    if late_array.shape[0] < 1000:
        return 0
    return np.true_divide(late_array.sum(0), (late_array != 0).sum(0))

if __name__ == "__main__":
    traceback.install()
    sys.exit(main())  # pragma: no cover


from collections import Counter
class Solution:
    def findAnagrams(self, s, p):
        targetHM = Counter(p)
        sw = len(p)
        result = []
        i = 0
        j = i+sw-1
        tempHM = Counter(s[i:j])
        while i<(len(s)-sw+1):
            tempHM[s[j]]+=1
            match = True
            for k,v in targetHM.items():
                if k not in tempHM or tempHM[k]!=v:
                    match=False
                    break
            if match:
                result.append(i)
            tempHM[s[i]]-=1
            i+=1
            j+=1

        return result


from django.http import HttpResponse
from django.views.decorators.csrf import csrf_exempt

from . import api

# Create your views here.
@csrf_exempt
def index(request):
    wx = api.Base()
    data = request.GET
    try:
        echostr = data['echostr']
        result = wx.check_sign(data)
        if result:
            return HttpResponse(echostr)
        else:
            return HttpResponse('error')
    except KeyError:
        wx_res = api.Response(request)
        result = wx_res.get_response()
        return HttpResponse(result)


import dynet as dy
import argparse
import time
import random
import numpy as np
from collections import defaultdict
import math
import pickle
import json
from tqdm import tqdm
from model import LM

parser = argparse.ArgumentParser()
parser.add_argument("--dynet-mem", help="allocate memory for dynet")
parser.add_argument("--dynet-gpu", help="use GPU")
parser.add_argument("--dynet-gpus", help="use GPU")
parser.add_argument("--dynet-seed", help="set random seed for dynet")
parser.add_argument("--dynet-gpu-ids", default=3, help="choose which GPU to use")
parser.add_argument("--dynet-autobatch", default=0, help="choose which GPU to use")
parser.add_argument("--dynet-devices", help="set random seed for dynet")
parser.add_argument("--dynet-weight-decay", help="choose weight decay")

parser.add_argument("--train", default="alternate_sents_seame_train_filtered.json", help="location of training file")
parser.add_argument("--dev", default="alternate_sents_seame_val_filtered.json", help="new dev file")
parser.add_argument("--test", default="alternate_sents_seame_test_filtered.json", help="new test file")
parser.add_argument("--train_finetune", default="alternate_sents_mono_all_filtered.json", help="location of training file for finetune")
parser.add_argument("--epochs", default=25, type=int, help="number of epochs")
parser.add_argument("--batch_size", default=20, type=int, help="size of batches")
parser.add_argument("--trainer", default="sgd", help="choose trainer for the optimization")
parser.add_argument("--num_layers", default=2, type=int, help="number of layers in RNN")
parser.add_argument("--input_dim", default=300, type=int, help="dimension of the input to the RNN")
parser.add_argument("--hidden_dim", default=650, type=int, help="dimension of the hidden layer of the RNN")
parser.add_argument("--x_dropout", default=0.35, type=float, help="value for rnn dropout")
parser.add_argument("--h_dropout", default=0.35, type=float, help="value for rnn dropout")
parser.add_argument("--w_dropout_rate", default=0.2, type=float, help="value for rnn dropout")
parser.add_argument("--logfile", default="log.txt", help="location of log file for debugging")
parser.add_argument("--learning_rate", default=1, type = float, help="set initial learning rate")
parser.add_argument("--lr_decay_factor", default=2.5, type = float, help="set clipping threshold")
parser.add_argument("--clip_thr", default=1, type=float, help="set clipping threshold")
parser.add_argument("--init_scale_rnn", default=0.05, type=float, help="scale to init rnn")
parser.add_argument("--init_scale_params", default=None, type=float, help="scale to init params")
parser.add_argument("--check_freq", default=None, help="frequency of checking perp on dev and updating lr")
parser.add_argument("--finetune_p1", action='store_true',help="train a model from monolingual data")
parser.add_argument("--finetune_p2", help="name of model learned from momnolingual data")
parser.add_argument("--resume_train", default=False, action="store_true", help="resume training from model")
parser.add_argument("--lamb", default=1, type=float, help="value of lambda")
parser.add_argument("--margin", default="wer", help="value of margin for loss")
parser.add_argument("--evaluate_model", help="model to load and evaluate, no training")

class Set:
    orig = None
    orig_type = None
    cs = None
    en = None
    zh = None
    distances = None
    raw = None

def remove_tag(sentence):
    no_tag = ""
    for token in sentence.split():
        no_tag += token.split("__")[0] + " "

    return no_tag

def detect_lang(sentence):
    tokens = sentence.split()
    en = False
    zh = False
    for token in tokens:
        if token.endswith("__en"):
            en = True
        elif token.endswith("__zh"):
            zh = True

    if en and zh:
        # code switched
        return "cs"
    if en:
        # english
        return "en"
    if zh:
        # chinese
        return "zh"

    # something wrong
    return None

def read_corpus(data_file, train = False):
    all_sets = []
    with open(data_file) as f:
        data = json.load(f)
    for s in tqdm(data):
        new_s = Set()
        new_s.cs = []
        new_s.en = []
        new_s.zh = []
        new_s.distances = []
        new_s.raw = []
        new_s.orig_type = detect_lang(s["orig"])

        new_s.orig = [w2i[w.split("__")[0].lower().lower()] for w in [START_TOKEN] + s["orig"].split() + [END_TOKEN]]
        new_s.raw.append(s["orig"])
        for idx, lang_type in enumerate(["cs", "en", "zh"]):
            for v in s[lang_type + "_alternatives"]:
                #assert(detect_lang(v) == idx)
                sent = []
                for w in [START_TOKEN] + v.split() + [END_TOKEN]:
                    sent.append(w2i[w.split("__")[0].lower()])
                getattr(new_s, lang_type).append(sent)
                new_s.raw.append(v)
        all_sets.append(new_s)
    assert(len(all_sets) == len(data))
    if train:
        print("length of train:", len(all_sets))
    return all_sets, len(all_sets)

def evaluate(test):
    all_scores = []
    all_norms = []
    ranks = []
    for item in tqdm(test):
        batch = [item.orig] + item.cs + item.en + item.zh
        output = [s.npvalue().flatten().tolist() for s in get_batch_scores(batch, True)]
        indices = list(range(len(scores)))
        indices.sort(key=lambda x: scores[x], reverse=True)
        ranks.append(indices.index(0)+1)
        all_norms.append(norms)
        all_scores.append(scores)

    cs = []
    en = []
    zh = []
    for rank, item in zip(ranks, test):
        if item.orig_type == "cs":
            cs.append(rank)
        elif item.orig_type == "en":
            en.append(rank)
        elif item.orig_type == "zh":
            zh.append(rank)

    tot_acc = sum([1 for r in ranks if r == 1])*100/float(len(ranks))
    cs_acc = sum([1 for r in cs if r == 1])*100/float(len(cs))
    en_acc = sum([1 for r in en if r == 1])*100/float(len(en))
    zh_acc = sum([1 for r in zh if r == 1])*100/float(len(zh))

    return tot_acc, cs_acc, en_acc, zh_acc, all_norms, all_scores

def logstr(f, s):
    print(f'[{str(time.time())}]  {s}')
    f.write(f'[{str(time.time())}]  {s}')

if __name__ == '__main__':

    args = parser.parse_args()
    random.seed(10)
    np.random.seed(10)

    print(str(args) + "\n\n")

    START_TOKEN = "<s>"
    END_TOKEN = "</s>"
    w2i = defaultdict(lambda: len(w2i))
    UNK = w2i["<unk>"]

    if args.finetune_p1 or args.finetune_p2:
        # need vocabulary in w2i for both p1 and p2
        train_set_finetune, train_nbatches_finetune = read_corpus(args.train_finetune, True)
        print("Loaded monolingual train set")

    train_set, train_nbatches = read_corpus(args.train, True)
    print("Loaded code switched train set")
    dev_set, dev_nbatches = read_corpus("data/" + args.dev)
    print("Loaded validation set")
    test_set, test_nbatches = read_corpus("data/" + args.test)
    print("Loaded test set")

    # create model
    # NOTE: need to read corpus before creating model since read_corpus() will fill up w2i
    word_num = len(w2i)
    model = LM(args.num_layers, args.input_dim, args.hidden_dim, word_num, args.init_scale_rnn, args.init_scale_params,
               args.x_dropout, args.h_dropout, args.w_dropout_rate, args.learning_rate, args.clip_thr)

    model.load("models/" + args.evaluate_model + "_model")
    print("loaded model", args.evaluate_model)
    dev_acc, dev_cs_acc, dev_en_acc, dev_zh_acc, dev_scores = evaluate(dev_set)
    test_acc, test_cs_acc, test_en_acc, test_zh_acc, test_scores = evaluate(test_set)
    print("dev accs", dev_acc, dev_cs_acc, dev_en_acc, dev_zh_acc)
    print("test accs", test_acc, test_cs_acc, test_en_acc, test_zh_acc)

    json.dump(dev_scores, open("dev_scores.json", 'w'), indent=2, ensure_ascii=False)
    logging.info("Saved dev scores to dev_scores.txt")
    json.dump(test_scores, open("test_scores.json", 'w'), indent=2, ensure_ascii=False)
    logging.info("Saved test scores to test_scores.txt")


import sys
import math

from io import open
from conllu import parse

def load_conll_file(filename):
    text = open(filename, "r", encoding="utf-8").read()
    return parse(text)

def save_tb(tb):
    for sent in tb:
        if sent.metadata:
            for key, value in sent.metadata.items():
                if value is None:
                    sent.metadata[key] = 'None'

        print(sent.serialize(), end='')

def split_sentence_impl(sent):
    ret = []

    last_idx = -1
    last_word_id = sent[last_idx]['id']
    if not isinstance(last_word_id, int):
        last_idx -= 1
        last_word_id = sent[last_idx]['id']

    possible_split_points = {}

    for token in sent:
        if not isinstance(token['id'], int):
            continue

        pos = token['id']
        # consider splitting sentence after token pos

        if pos < 3 or pos > last_word_id - 3:
            continue

        links_to_update_before = 0
        links_to_update_after = 0

        for tok in sent:
            if not isinstance(tok['id'], int):
                continue

            if tok['id'] <= pos:
                if tok['head'] > pos:
                    links_to_update_before += 1
            else:
                if tok['head'] <= pos:
                    links_to_update_after += 1

        possible_split_points[pos] = links_to_update_before + links_to_update_after

    good_split_points = sorted(possible_split_points.keys(), key=lambda x: possible_split_points[x])
    min_value = possible_split_points[good_split_points[0]]
    min_values_count = sum(v == min_value for v in possible_split_points.values())
    good_split_points = good_split_points[:min_values_count]

    if len(good_split_points) == 0:
        raise

    good_split_points_centred = sorted(good_split_points, key=lambda x: abs(int(math.floor(len(sent) / 2)) - x))

    if len(good_split_points_centred) == 0:
        raise

    split_pos = good_split_points_centred[0]
    split_idx = None

    if split_pos is None:
        raise

    for i in range(len(sent)):
        if isinstance(sent[i]['id'], int) and sent[i]['id'] == split_pos:
            split_idx = i + 1
            break

    first = sent.copy()
    first.metadata = first.metadata.copy()
    first.tokens = first[:split_idx]
    if 'sent_id' in first.metadata:
        first.metadata['sent_id'] = first.metadata['sent_id'] + " / left"

    update_heads_in_left_part(first, sent)

    for tok in first:
        if tok['head'] is None:
            continue
        if tok['head'] != 0 and tok['head'] > split_pos:
            sys.stderr.write('F prev head of token %s is %d\n' % (str(tok['id']), tok['head']))

    second = sent.copy()
    second.metadata = second.metadata.copy()
    second.tokens = second[split_idx:]
    if 'sent_id' in second.metadata:
        second.metadata['sent_id'] = second.metadata['sent_id'] + " / right"

    update_heads_in_right_part(second, sent)

    i = len(second.tokens) - 1
    last_token_id = second[i]['id']
    while not isinstance(last_token_id, int):
        i -= 1
        last_token_id = second[i]['id']

    for tok in second:
        if 'head' not in tok or tok['head'] is None or tok['head'] == '_':
            continue
        #if tok['head'] != 0 and tok['head'] > last_token_id:
        #    sys.stderr.write('S prev head of token %s is %d\n' % (str(tok['id']), tok['head']))

    ret = [first, second]

    return ret

def get_token_with_id(sent, id):
    for tok in sent:
        if str(tok['id']) == id:
            return tok
    raise

def update_heads_in_left_part(part, whole_sent):
    i = len(part) - 1
    last_token_id = part[i]['id']
    while not isinstance(last_token_id, int):
        i -= 1
        last_token_id = part[i]['id']

    root_pos = None
    orphans = []
    for i in range(len(part)):
        if 'head' not in part[i] or part[i]['head'] is None or part[i]['head'] == '_':
            continue
        if part[i]['head'] == 0:
            root_pos = part[i]['id']
        if part[i]['head'] > last_token_id:
            orphans.append(part[i]['id'])

    if root_pos is None:
        branch_len = [ len(part) + 1 ] * len(part)
        for i in range(len(part)):
            #pos = orphans[i]
            if not isinstance(part[i]['id'], int):
                continue
            if 'upostag' not in part[i]:
                continue
            if part[i]['upostag'] == 'PUNCT':
                continue
            token = get_token_with_id(whole_sent, str(part[i]['id']))
            parent = token['head']
            branch_len[i] = 0
            while parent != 0:
                if branch_len[i] > len(whole_sent) + 1:
                    raise
                branch_len[i] += 1
                parent = get_token_with_id(whole_sent, str(parent))['head']

        root_pos = part[branch_len.index(min(branch_len))]['id']
        get_token_with_id(part, str(root_pos))['head'] = 0

    for tok in part:
        if 'head' not in tok or tok['head'] is None or tok['head'] == '_':
            continue
        if tok['head'] != 0 and tok['head'] > last_token_id:
            tok['head'] = root_pos

    update_indices(part)

def update_heads_in_right_part(part, whole_sent):
    i = 0
    first_token_id = part[i]['id']
    while not isinstance(first_token_id, int):
        i += 1
        first_token_id = part[i]['id']

    root_pos = None
    orphans = []
    for i in range(len(part)):
        if 'head' not in part[i] or part[i]['head'] is None or part[i]['head'] == '_':
            continue
        if part[i]['head'] == 0:
            root_pos = part[i]['id']
        if part[i]['head'] < first_token_id:
            orphans.append(part[i]['id'])

    if root_pos is None:
        branch_len = [ len(part) + 1 ] * len(part)
        for i in range(len(part)):
            #pos = orphans[i]
            if not isinstance(part[i]['id'], int):
                continue
            if 'upostag' not in part[i]:
                continue
            if part[i]['upostag'] == 'PUNCT':
                continue
            token = get_token_with_id(whole_sent, str(part[i]['id']))
            parent = token['head']
            branch_len[i] = 0
            while parent != 0:
                if branch_len[i] > len(whole_sent) + 1:
                    raise
                branch_len[i] += 1
                parent = get_token_with_id(whole_sent, str(parent))['head']

        root_pos = part[branch_len.index(min(branch_len))]['id']
        get_token_with_id(part, str(root_pos))['head'] = 0

    for tok in part:
        if 'head' not in tok or tok['head'] is None or tok['head'] == '_':
            continue
        if tok['head'] != 0 and tok['head'] < first_token_id:
            tok['head'] = root_pos

    update_indices(part)

def update_indices(sent):
    pos = 1
    old2new = {}

    for i in range(len(sent)):
        id = sent[i]['id']
        if id not in old2new:
            if isinstance(id, int):
                new_id = pos
                old2new[id] = pos
                pos += 1
            else:
                if '-' in str(id):
                    if len(id) == 3:
                        begin, end = id[0], id[2]
                        begin = int(begin)
                        end = int(end)
                        n = 0
                        for x in range(begin, end+1):
                            old2new[x] = pos + n
                            n += 1
                        new_id = (old2new[begin], '-', old2new[end])
                        old2new[id] = new_id
                        pos += end - begin + 1
                    else:
                        raise
                elif '.' in str(id):
                    if len(id) == 3:
                        begin, idx = id[0], id[2]
                        begin = int(begin)
                        idx = int(idx)
                        if begin not in old2new:
                            old2new[begin] = pos
                            pos += 1
                            #raise
                        #old2new[begin] = pos
                        #pos += 1
                        new_id = (old2new[begin], '.', idx)
                        old2new[id] = new_id
                    else:
                        raise

        sent[i]['id'] = old2new[id]

    for tok in sent:
        if 'head' not in tok or tok['head'] is None or tok['head'] == '_':
            continue
        if tok['head'] != 0 and tok['head'] not in old2new:
            raise
        if tok['head'] != 0:
            tok['head'] = old2new[tok['head']]

    pass

def split_sentence(sent, max):
    rv = []
    parts = split_sentence_impl(sent)
    for s in parts:
        if len(s) > max:
            rv.extend(split_sentence(s, max))
        else:
            rv.append(s)
    return rv

def guess_max_sentence_len(tb, out_ratio):
    lengths = []
    for sent in tb:
        lengths.append(len(sent))

    lengths.sort(reverse=True)

    num_outliers = int(len(lengths) * out_ratio)
    if num_outliers < 1:
        num_outliers = 2

    outliers = lengths[:num_outliers]
    max_len = outliers[-1]

    return max_len

def is_sentence_supported(sent):
    for t in sent:
        if isinstance(t['id'], tuple):
            return False
        if isinstance(t['head'], tuple):
            return False
        if t['head'] is None:
            print(str(sent.serialize()))
        i = int(t['head'])
    return True

def augment(tb, max_len):
    new = []

    for sent in tb:
        if not is_sentence_supported(sent):
            print('Not supported:\n%s\n' % str(sent))
            continue
        num = len(sent)
        if num > max_len:
            parts = split_sentence(sent, max_len)
            for s in parts:
                new.append(s)
        else:
            new.append(sent)

    return new

def main():
    if len(sys.argv) <= 1:
        return

    tb = load_conll_file(sys.argv[1])

    new_tb = augment(tb, guess_max_sentence_len(tb, 0.02))

    save_tb(new_tb)

if __name__ == '__main__':
    main()


# encoding: utf-8
# (C) 2001 - 2020 Rajkumar Palani <rajkumarpalani07@gmail.com>

import pytest

from tamil import utf8

def test_get_tamil_utf_count():
    expected = 364
    assert(utf8.get_tamil_utf_count() == expected)

def test_get_tamil_utf():
    expected = 364
    assert(len(utf8.get_tamil_utf()) == expected)

    temp1 = list(filter(lambda x: (x == 'அ'), utf8.get_tamil_utf()))
    assert(('அ' in temp1[0]) == True)

def test_validate_char():
    input1 = "ட"
    assert(utf8.validate_char(input1) == True)
    input2 = 'அ'
    assert(utf8.validate_char(input2) == True)

def test_validate_char_failing():
    input1 = 1000
    with pytest.raises(TypeError, match="Input value must be unicode string"):
        utf8.validate_char(input1)

    input2 = ''
    with pytest.raises(ValueError, match="Input value is empty"):
        utf8.validate_char(input2)

    input3 = 'Ж'
    with pytest.raises(ValueError, match="Input value is not a valid utf8 tamil character"):
        utf8.validate_char(input3)


import IAFlow as flw
from scipy import ndimage
from matplotlib.pyplot import imshow, show
import random
import numpy as np

def load_data():
    data = np.loadtxt("Data.txt", dtype='float32')
    return data

def num(y):
    resultRaw = []
    for i in range(len(y)):
        resultRaw.append(np.where(y[i] == y[i].max())[0]+1)
    return resultRaw

def comp(y, result):
    count = 0
    for i in range(len(result)):
        if np.argmax(y[i]) == np.argmax(result[i]):
            count += 1
    percentage = count / len(result)
    return percentage

def plots(vector):
    matrix = np.reshape(vector, (20, 20))
    matrix = ndimage.rotate(matrix, 90)
    imshow(matrix, cmap='gray', origin='lower')
    show()

def toRaw(data):
    yProcessed = []
    for i in data:
        temp = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
        temp[int(i) - 1] = 1
        yProcessed.append(temp)
    yProcessed = np.array(yProcessed, dtype='float64')
    return yProcessed

if __name__ == "__main__":
    pNN = flw.snn([400, 25, 10], flw.relu, flw.softmax, 'regression', 0)

    preResources = load_data()
    randomize = [x for x in range(5000)]
    random.shuffle(randomize)
    resources = []
    for i in range(len(randomize)):
        resources.append(preResources[randomize[i]])
    resources = np.array(resources)
    xTrain = np.resize(resources[0:4500, 0:len(resources[0])-1], (4500,400))
    rawY = np.resize(resources[0:4500, 400], (4500, 1))
    yTrain = toRaw(rawY)

    pNN.insertData(xTrain, yTrain)
    pNN.feedforward()
    pNN.backpropagation()
    pNN.gradientDescent(0.1, 200)

    train_result = pNN.z[2]

    efficiency = comp(train_result, yTrain)
    efficiency *= 100
    print("The efficiency in the train set was of %.2f %%" % (efficiency))

    xTest = np.resize(resources[4500:, 0:len(resources[0]) - 1], (500, 400))
    rawYTest = np.resize(resources[4500:, 400], (500, 1))
    yTest = toRaw(rawYTest)

    pNN.feedforward(xTest)
    test_result = pNN.z[2]

    efficiency = comp(test_result, yTest)
    efficiency *= 100
    print("The efficiency in the validation set was of %.2f %%" % (efficiency))
    rep = 1
    #while rep != 0:
        #pos = random.randint(0,500)
        #print("El número predicho por la red neuronal es %i" % (rawXval[pos]))
        #plots(rawResult[pos])
        #rep = input("Desea predecir otro número? si = 1 | no = 0")


#!/usr/bin/env python
"""
Global Average Plot
===================

Using cartopy_ we can easily visualize gridded model output on maps with
different cartographic projections, and then configure them with any
aesthetics or features such as continents, geopolitical borders, gridlines,
and more.

.. _cartopy: http://scitools.org.uk/cartopy/docs/latest/
"""
# Author: Daniel Rothenberg
# Version: June 2, 2017

import matplotlib.pyplot as plt
plt.style.use(['seaborn-talk', 'seaborn-ticks'])

import xbpch

# First we read in a sample dataset containing GEOS-Chem output
ds = xbpch.open_bpchdataset(
    "/Users/daniel/workspace/bpch/test_data/ND49_20060101_ref_e2006_m2010.bpch",
    diaginfo_file="/Users/daniel/Desktop/sample_nd49/diaginfo.dat",
    tracerinfo_file="/Users/daniel/Desktop/sample_nd49/tracerinfo.dat",
    dask=False, memmap=True
)

# This dataset contains multiple timesteps of multiple tracers, but we can
# extract just ozone, and average over the timesteps to get a slice to plot.
avg_o3 = ds['IJ_AVG_S_O3'].mean('time')

################################################
# To create a cartopy plot, we have to pass a **projection** to our matplotlib
# axis. This contains the utility/logic to transform data from the coordinate
# system we give to matplotlib to the actual cartographic/geographic system.
#
# Now, when we plot on this axis, we just need to tell matplotlib how to transform
# our data to its coordinate system. We do this by passing a **transform** object.
# No matter what projection you use, if you're plotting rectangular lat/lon data
# from GEOS-Chem or another model, you should use the ``PlateCarree()`` projection
# for this transform.
#
# For convenience, we'll use the wrapper that each ``DataArray`` has for
# matplotlib functions. This is just for convenience; we could always get the
# vector of lon/lat values and plot the data in **avg_o3** directly.

import cartopy.crs as ccrs
fig = plt.figure(figsize=(8, 4))
ax = fig.add_subplot(111, projection=ccrs.PlateCarree())

avg_o3.plot.imshow(x='lon', y='lat', vmin=0, vmax=60., cmap='gist_stern',
                   ax=ax, transform=ccrs.PlateCarree())
ax.coastlines()
plt.show()


from django.contrib import admin

from .models import Category, Question

class QuestionAdmin(admin.ModelAdmin):
    fieldsets = [
        (None,               {'fields': ['question_text']}),
    ]

admin.site.register(Question, QuestionAdmin)
admin.site.register(Category)

# Register your models here.


import matplotlib.pyplot as plt
import itertools
from sklearn.cluster import KMeans
from sklearn import metrics
from time import time
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification
import numpy as np
import matplotlib.pyplot as plt
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from sklearn.datasets import load_digits
from sklearn.model_selection import learning_curve
from sklearn.model_selection import ShuffleSplit
from collections import defaultdict
import numpy as np

import modulation as modulate
import channel as chan

def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
                        n_jobs=1, train_sizes=np.linspace(.1, 1.0, 5), scoreFunction="accuracy"):
    """
    Generate a simple plot of the test and training learning curve.

    Parameters
    ----------
    estimator : object type that implements the "fit" and "predict" methods
        An object of that type which is cloned for each validation.

    title : string
        Title for the chart.

    X : array-like, shape (n_samples, n_features)
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    y : array-like, shape (n_samples) or (n_samples, n_features), optional
        Target relative to X for classification or regression;
        None for unsupervised learning.

    ylim : tuple, shape (ymin, ymax), optional
        Defines minimum and maximum yvalues plotted.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:
          - None, to use the default 3-fold cross-validation,
          - integer, to specify the number of folds.
          - An object to be used as a cross-validation generator.
          - An iterable yielding train/test splits.

        For integer/None inputs, if ``y`` is binary or multiclass,
        :class:`StratifiedKFold` used. If the estimator is not a classifier
        or if ``y`` is neither binary nor multiclass, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validators that can be used here.

    n_jobs : integer, optional
        Number of jobs to run in parallel (default 1).
    """
    plt.figure()
    plt.title(title)
    if ylim is not None:
        plt.ylim(*ylim)
    plt.xlabel("Training examples")
    plt.ylabel("Score")
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes, scoring=scoreFunction)
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    plt.grid()

    print(train_scores_mean)

    print(test_scores_mean)

    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                     train_scores_mean + train_scores_std, alpha=0.1,
                     color="r")
    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                     test_scores_mean + test_scores_std, alpha=0.1, color="g")
    plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
             label="Training score")
    plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
             label="Cross-validation score")

    plt.legend(loc="best")
    return plt

def genKSignals(K, mode, size, snr):
    X = np.zeros((K, size),dtype=np.complex)
    for i in range(0, K):
        V = modulate.genmodsig(mode, size)
        X[i] = chan.awgn(V, snr)
    return X

def genYTargets(sampleSize, nSignals):
    Y = np.concatenate([np.zeros(sampleSize, dtype=np.int)+k for k in range(0,nSignals)])
    return Y

def createDataset(modulationMethodList, burstSize, sampleSize, snrRange):
    snrToDatasetList = defaultdict(list)
    snrToMatrixDataset = {}

    for snr in snrRange:
        for mode in modulationMethodList:
            snrToDatasetList[snr].append(genKSignals(sampleSize, mode, burstSize, snr))
        snrToMatrixDataset[snr] = np.concatenate(snrToDatasetList[snr])

    del snrToDatasetList
    return snrToMatrixDataset


import string

from bs4 import BeautifulSoup
from classObj import *
import requests

class SubjectObj:

    #Holds to top level URL
    mainURL = 'https://catalog.unc.edu/courses/'

    def __init__(self, name, abb, href):
        self.name = name
        self.abbreviation = abb
        self.href = href
        self.classes = []

    def __str__(self):
        return f'{self.name}({self.abbreviation}) {self.href}'

    def getClasses(self,csv_writer):

        #Opens the subject page and gets the soup from it
        newUrl = self.mainURL + self.href
        page = requests.get(newUrl).text
        subjectHTML = BeautifulSoup(page,'lxml')

        #List holding the course blocks
        courseblocks = []
        for cb in subjectHTML.find_all('div', class_='courseblock'):
            courseblocks.append(cb)

        #Loop that creates the classObj's
        for cb in courseblocks:
            courseBlockTitle = cb.find('p',class_='courseblocktitle').strong.text.strip()
            courseBlockTitleParts = courseBlockTitle.split('.')

            self.prepTitleBlock(courseBlockTitleParts)

            #Holds the abbreviation for the class
            abb = self.getGetAbb(courseBlockTitleParts)

            #Holds the number for the class
            num = self.getNum(courseBlockTitleParts)

            #Holds the full title for the class
            fullTitle = self.getFullTitle(courseBlockTitleParts)

            #Holds the raw credits for the class
            rawCreds = self.getRawCreds(courseBlockTitleParts)

            #Gets all of the course descriptions
            courseBlockDesc = cb.find('p',class_='courseblockdesc')

            #Holds the description of the class
            rawDesc = self.getRawClassDesc(courseBlockDesc)

            #Creates the classObj for each class
            a = ClassObj()
            a.setRawCredits(rawCreds)
            a.setAbbreviation(abb)
            a.setNumber(num)
            a.setFullTitle(fullTitle)
            a.setDesc(rawDesc)
            self.csvWrite(a,csv_writer)
            #print(a)

    #Preps the courseblockTitleParts array and handles the various formats in the titles of the classes
    def prepTitleBlock(self,courseBlockTitleParts):

        # Handling the M.F.A Classes
        if (len(courseBlockTitleParts) == 7):
            courseBlockTitleParts[
                1] = f'{courseBlockTitleParts[1]}.{courseBlockTitleParts[2]}.{courseBlockTitleParts[3]}.{courseBlockTitleParts[4]}'
            courseBlockTitleParts.remove(courseBlockTitleParts[2])
            courseBlockTitleParts.remove(courseBlockTitleParts[2])
            courseBlockTitleParts.remove(courseBlockTitleParts[2])

        # Handling of the classes with M.A. and Ph.D in their titles
        if (len(courseBlockTitleParts) == 6 and (courseBlockTitleParts[1] == '  M' or courseBlockTitleParts[2] == 'D')):
            courseBlockTitleParts[
                1] = f'{courseBlockTitleParts[1]}.{courseBlockTitleParts[2]}. {courseBlockTitleParts[3]}'
            courseBlockTitleParts.remove(courseBlockTitleParts[2])
            courseBlockTitleParts.remove(courseBlockTitleParts[2])

        subjectsWithPeriodInTitle = ['ENGL', 'ARTH', 'GERM', 'HPM', 'HIST', 'INLS']

        #Handling of the classes with a period in the class title
        if (len(courseBlockTitleParts) == 5 and (courseBlockTitleParts[0][:4] in subjectsWithPeriodInTitle or courseBlockTitleParts[0][:3] in subjectsWithPeriodInTitle) and not('  0' in courseBlockTitleParts or '  1' in courseBlockTitleParts)):
            courseBlockTitleParts[1] = f'{courseBlockTitleParts[1]}.{courseBlockTitleParts[2]}'
            courseBlockTitleParts.remove(courseBlockTitleParts[2])

        # Handling of the classes with half credit-hour classes
        if (len(courseBlockTitleParts) == 5 and (courseBlockTitleParts[0][:4] != 'ARCH')):
            courseBlockTitleParts[-2] = f'{courseBlockTitleParts[-3]}.{courseBlockTitleParts[-2]}'
            courseBlockTitleParts.remove(courseBlockTitleParts[-3])

        # Handling of the first weird ARCH class
        if (len(courseBlockTitleParts) == 5):
            courseBlockTitleParts[0] = courseBlockTitleParts[0] + courseBlockTitleParts[1]
            courseBlockTitleParts.remove(courseBlockTitleParts[1])

    #Returns the abbreviation of the subject
    def getGetAbb(self,courseBlockTitleParts):
        return courseBlockTitleParts[0].split('\xa0')[0]

    #Returns the number of the class
    def getNum(self,courseBlockTitleParts):
        num = courseBlockTitleParts[0].split('\xa0')[1]

        # try:
        #     num = num.replace("H", "")
        #     num = num.replace("L", "")
        #     num = num.replace("B", "")
        #     num = num.replace("A", "")
        #     num = num.replace("C", "")
        #     num = num.replace("I", "")
        #     num = num.replace("P", "")
        #
        # except:
        #     print('Missing H/L Argument')
        #
        # try:
        #     num = int(num)
        # except:
        #     num = 0
        #     print('Problem with num assignment')

        return num

    #Returns the full title of the class
    def getFullTitle(self,courseBlockTitleParts):
        return courseBlockTitleParts[1].strip()

    #Returns the raw credits string of the class
    def getRawCreds(self,courseBlockTitleParts):
        return float(courseBlockTitleParts[-2].strip(string.ascii_letters).replace('-',''))

    #Returns the class description
    def getRawClassDesc(self,courseBlockDesc):
        return courseBlockDesc.text.strip()

    def csvWrite(self,a,csv_writer):
        csv_writer.writerow([a.abb,a.number,a.fullTitle,a.rawCredits,a.minCredits,a.maxCredits,a.hasCreditRange,a.genEds,a.desc])


#!/usr/bin/env python3
import argparse
from mlagents_envs.environment import UnityEnvironment
from mlagents_envs.side_channel.engine_configuration_channel import EngineConfigurationChannel
from gym_unity.envs import UnityToGymWrapper

from lib import model
from PIL import Image

import numpy as np
import torch

ENV_ID = "../../Robotic-RL-Env/Build/Robotic-RL-Env"

if __name__ == "__main__":
    """
    Dies ist das Programm mit dem das trainierte Modell gezeigt werden kann. Dem Skript muss über das Argument das Modell
    übergeben werden.
    """
    parser = argparse.ArgumentParser()
    parser.add_argument("-m", "--model", required=True, help="Model file to load")
    parser.add_argument("-e", "--env", default=ENV_ID, help="Environment name to use, default=" + ENV_ID)
    args = parser.parse_args()

    # Wrappen der Unity-Umgebung in eine Gym-Umgebung
    channel = EngineConfigurationChannel()
    unity_env = UnityEnvironment(ENV_ID, seed=1, side_channels=[channel])
    channel.set_configuration_parameters(time_scale=1.0)
    env = UnityToGymWrapper(unity_env)

    # Erstellen des Policy-Netzes mit den optimierten Gewichtungen aus dem Training
    net = model.ModelActor(env.observation_space.shape[0], env.action_space.shape[0])
    net.load_state_dict(torch.load(args.model))

    total_reward = 0.0
    total_steps = 0
    while True:
        # Für jeden Durchgang wird die Umgebung in einen definierten Startzustand gefahren.
        obs = env.reset()
        obs_v = torch.FloatTensor(obs)
        mu_v = net(obs_v)
        action = mu_v.squeeze(dim=0).data.numpy()
        action = np.clip(action, -1, 1)
        # Die berechnete Aktion wird folgend mit der .Step() Methode ausgeführt
        obs, reward, done, _ = env.step(action)
        total_reward += reward
        total_steps += 1

        # Zur Besseren Darstellung werden die Aktionen in Geschwindigkeit und Winkel umgerechnet
        action.data[0] = action.data[0] / 2 + 0.5
        action.data[1] = action.data[1] / 2 + 0.5
        action.data[0] = 10 + action.data[0] * 360
        action.data[1] = (action.data[1]*-150)
        print("The Target ist %.3f away and with the Actions %.3f and %.3f, we get %.3f Points" % (obs_v, action.data[0], action.data[1], reward))


import logging

import pytorch_lightning as pl
import torch
import torchmetrics
from torch.nn import functional as F

from nevermore.data import NYUv2Dataset
from nevermore.metric import Abs_CosineSimilarity
from nevermore.model.network import SingleSegNet

__all__ = ['SingleSegnetNYUv2Model']

logger = logging.getLogger(__name__)

class SingleSegnetNYUv2Model(pl.LightningModule):

    def __init__(self, learning_rate, task):
        super().__init__()
        self.save_hyperparameters()

        allowed_task = ("segmentation", "depth", "normal")
        if task not in allowed_task:
            raise ValueError(
                f"Expected argument `tsak` to be one of "
                f"{allowed_task} but got {task}"
            )
        task_output_channels = dict(
            segmentation=len(NYUv2Dataset.CLASSES),
            depth=1,
            normal=3,
        )

        self.task = task
        self.segnet = SingleSegNet(
            input_channels=3,
            output_channels=task_output_channels[task],
        )

        self.miou = torchmetrics.IoU(
            num_classes=len(NYUv2Dataset.CLASSES), ignore_index=0
        )
        self.rmse = torchmetrics.MeanSquaredError(squared=False)
        self.cos = Abs_CosineSimilarity(reduction='abs')

    def forward(self, x):
        return self.segnet.forward(x)

    def training_step(self, batch, batch_idx):
        x = batch['image']
        y_hat, y_hat_softmax = self(x)

        if self.task == 'segmentation':
            y_seg = batch['mask']
            loss_seg = F.cross_entropy(y_hat_softmax, y_seg)
            loss = loss_seg
            self.log('train_loss_seg', loss, prog_bar=True)

        if self.task == 'depth':
            y_dep = batch['depth']
            y_hat = y_hat.squeeze()
            loss_dep = F.mse_loss(y_hat, y_dep)
            loss = loss_dep
            self.log('train_loss_dep', loss, prog_bar=True)

        if self.task == 'normal':
            y_nor = batch['normal'].flatten(start_dim=1)
            y_hat = y_hat.flatten(start_dim=1)
            loss_nor = torch.mean(F.cosine_similarity(y_hat, y_nor))
            loss = loss_nor
            self.log('train_loss_nor', loss, prog_bar=True)

        return loss

    def training_epoch_end(self, training_step_outputs):
        print(self.trainer.lr_schedulers[0]['scheduler'].get_lr())
        for out in training_step_outputs:
            pass

    def validation_step(self, batch, batch_idx):
        x = batch['image']
        y_hat, y_hat_softmax = self(x)

        if self.task == 'segmentation':
            y_seg = batch['mask']
            loss_seg = F.cross_entropy(y_hat_softmax, y_seg)
            self.log('val_seg_loss', loss_seg)
            self.log('val_seg_iou_step', self.miou(y_hat_softmax, y_seg))

        if self.task == 'depth':
            y_dep = batch['depth']
            y_hat = y_hat.squeeze()
            loss_dep = F.mse_loss(y_hat, y_dep)
            self.log('val_dep_loss', loss_dep)
            self.log('val_dep_rmse_step', self.rmse(y_hat, y_dep))

        if self.task == 'normal':
            y_nor = batch['normal'].flatten(start_dim=1)
            y_hat = y_hat.flatten(start_dim=1)
            loss_nor = torch.mean(F.cosine_similarity(y_hat, y_nor))
            self.log('val_nor_loss', loss_nor)
            self.log('val_dep_cos_step', self.cos(y_hat, y_nor))

    def validation_epoch_end(self, validation_step_outputs):

        if self.task == 'segmentation':
            val_miou = self.miou.compute()
            self.log('val_seg_iou', val_miou)
            logger.info("val_seg_iou:" + str(val_miou.item()))
            self.miou.reset()

        if self.task == 'depth':
            val_rmse = self.rmse.compute()
            self.log('val_dep_mse', val_rmse)
            logger.info("val_dep_mse:" + str(val_rmse.item()))
            self.rmse.reset()

        if self.task == 'normal':
            val_cos = self.cos.compute()
            self.log('val_nor_cos', val_cos)
            logger.info("val_nor_cos:" + str(val_cos.item()))
            self.cos.reset()

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(
            self.segnet.parameters(), lr=self.hparams.learning_rate
        )
        lr_schedule = torch.optim.lr_scheduler.StepLR(
            optimizer, step_size=462, gamma=0.2
        )
        optim_dict = {'optimizer': optimizer, 'lr_scheduler': lr_schedule}
        return optim_dict


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import itertools
import os
import shutil

import tensorflow as tf

from tensorflow_recommenders_addons import dynamic_embedding as de

from tensorflow.core.protobuf import config_pb2
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import test_util
from tensorflow.python.framework.errors_impl import NotFoundError
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import variables
from tensorflow.python.platform import test
from tensorflow.python.training import adam
from tensorflow.python.training import monitored_session
from tensorflow.python.training.optimizer import Optimizer as tf1_opt
from tensorflow.python.training import training_util
try:
  from tensorflow.keras.optimizers.legacy import Adam
except:
  from tensorflow.keras.optimizers import Adam
from tensorflow_recommenders_addons.utils.check_platform import is_macos, is_arm64

default_config = config_pb2.ConfigProto(
    allow_soft_placement=True,
    gpu_options=config_pb2.GPUOptions(allow_growth=True))

def get_emb_sequential_model(emb_t, opt, *args, **kwargs):
  l0 = tf.keras.layers.InputLayer(input_shape=(None,), dtype=dtypes.int64)
  l1 = emb_t(*args, **kwargs)
  l2 = tf.keras.layers.Dense(8, 'relu', kernel_initializer='zeros')
  l3 = tf.keras.layers.Dense(1, 'sigmoid', kernel_initializer='zeros')
  if emb_t == tf.keras.layers.Embedding:
    model = tf.keras.Sequential([l0, l1, l2, l3])
  elif emb_t == de.keras.layers.HvdAllToAllEmbedding:
    model = tf.keras.Sequential([l0, l1, l2, l3])
  else:
    raise TypeError('Unsupported embedding layer {}'.format(emb_t))
  model.compile(optimizer=opt, loss='mean_absolute_error')
  return model

class HorovodTest(test.TestCase):

  @test_util.deprecated_graph_mode_only
  def test_adam_minimize_trainable(self):
    if (is_macos() and is_arm64()):
      self.skipTest(
          "Apple silicon devices don't support synchronous training based on Horovod."
      )

    base_opt = adam.AdamOptimizer(1.0)
    test_opt = adam.AdamOptimizer(1.0)
    self.common_minimize_trainable_v1(base_opt, test_opt, name="adam")
    tf.keras.backend.clear_session()
    keras_base_opt = Adam(1.0)
    keras_test_opt = Adam(1.0)
    self.common_minimize_trainable_v2(keras_base_opt,
                                      keras_test_opt,
                                      name="keras_adam")

  @test_util.run_all_in_graph_and_eager_modes
  def test_all_to_all_embedding_trainable(self):
    # TODO: Resolve the conflict arising from the 'save' function incompatibility with TensorFlow 2.11.
    if (tf.__version__ == "2.11.0" or tf.__version__ == "2.11.1"):
      self.skipTest(
          "The save function doesn't work with TF 2.11, skip the test.")
    if (is_macos() and is_arm64()):
      self.skipTest(
          "Apple silicon devices don't support synchronous training based on Horovod."
      )
    keras_base_opt = Adam(1.0)
    keras_test_opt = Adam(1.0)
    self.common_all_to_all_embedding_trainable_v2(keras_base_opt,
                                                  keras_test_opt,
                                                  name="keras_adam")

  def common_minimize_trainable_v1(self, base_opt, test_opt, name):
    # TODO(rhdong): Recover the testing, if the horovod import error is fixed on macOS+TF2.7+.
    try:
      import horovod.tensorflow as hvd
    except (NotFoundError):
      self.skipTest(
          "Skip the test for horovod import error with Tensorflow-2.7.0 on MacOS-12."
      )

    tf.config.set_soft_device_placement(True)

    hvd.init()

    # These cases need 2 GPUs at least if available.
    logical_devices = tf.config.list_logical_devices('GPU')
    _device = "GPU" if len(logical_devices) >= hvd.size() else "CPU"
    _device_id = hvd.local_rank(
    ) if _device == "GPU" and len(logical_devices) >= 2 else 0

    if _device == "GPU":
      os.environ["CUDA_VISIBLE_DEVICES"] = str(_device_id)

    base_opt = de.DynamicEmbeddingOptimizer(base_opt, synchronous=True)
    for dtype, run_step, dim in itertools.product([dtypes.float32], [1], [10]):
      print("device=", "/{}:{}".format(_device, _device_id))
      with tf.device("/{}:{}".format(_device, _device_id)):
        x = tf.random.uniform(shape=[32, dim])
        y = tf.zeros([32, 1])

        global_step = training_util.create_global_step()

        base_weight = tf.compat.v1.get_variable(name="base_weights",
                                                initializer=tf.ones([10, 1]))

        base_logits = tf.nn.relu(math_ops.matmul(x, base_weight))
        base_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y,
                                                            logits=base_logits)

        base_opt_op = base_opt.minimize(base_loss,
                                        global_step,
                                        var_list=[base_weight])

        test_weight = tf.compat.v1.get_variable(name="test_weights",
                                                initializer=tf.ones([10, 1]))

        test_logits = tf.nn.relu(math_ops.matmul(x, test_weight))
        test_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y,
                                                            logits=test_logits)

        grads_and_vars = test_opt.compute_gradients(test_loss,
                                                    var_list=[test_weight])
        var_list = []
        aggregated_grad = []
        for grad, var in grads_and_vars:
          var_list.append(var)
          aggregated_grad.append(hvd.allreduce(grad, op=hvd.Sum))
        aggregated_grads_and_vars = zip(aggregated_grad, var_list)
        test_opt_op = test_opt.apply_gradients(aggregated_grads_and_vars,
                                               global_step)

      with monitored_session.MonitoredTrainingSession(
          is_chief=True, config=default_config) as sess:
        for _ in range(run_step):
          sess.run(base_opt_op)
          sess.run(test_opt_op)

        self.assertAllCloseAccordingToType(
            sess.run(base_weight),
            sess.run(test_weight),
            msg="Cond:{},{},{}".format(dtype, run_step, dim),
        )

  def common_minimize_trainable_v2(self, base_opt, test_opt, name):
    # TODO(rhdong): Recover the testing, if the horovod import error is fixed on macOS+TF2.7+.
    try:
      import horovod.tensorflow as hvd
    except (NotFoundError):
      self.skipTest(
          "Skip the test for horovod import error with Tensorflow-2.7.0 on MacOS-12."
      )

    tf.config.set_soft_device_placement(True)

    hvd.init()

    # These cases need 2 GPUs at least if available.
    logical_devices = tf.config.list_logical_devices('GPU')
    _device = "GPU" if len(logical_devices) >= hvd.size() else "CPU"
    _device_id = hvd.local_rank(
    ) if _device == "GPU" and len(logical_devices) >= 2 else 0

    if _device == "GPU":
      os.environ["CUDA_VISIBLE_DEVICES"] = str(_device_id)

    base_opt = de.DynamicEmbeddingOptimizer(base_opt, synchronous=True)
    for dtype, run_step, dim in itertools.product([dtypes.float32], [1], [10]):
      print("device=", "/{}:{}".format(_device, _device_id))
      with tf.device("/{}:{}".format(_device, _device_id)):
        x = tf.random.uniform(shape=[32, dim])
        y = tf.zeros([32, 1])

        with tf.GradientTape() as tape_base:
          base_weight = tf.compat.v1.get_variable(name="base_weights",
                                                  initializer=tf.ones([10, 1]))

          base_logits = tf.nn.relu(math_ops.matmul(x, base_weight))
          base_loss = tf.nn.sigmoid_cross_entropy_with_logits(
              labels=y, logits=base_logits)

          # grad_base = tape_base.gradient(base_loss, base_weight)
          # base_opt.
        base_opt_op = base_opt.minimize(base_loss,
                                        var_list=[base_weight],
                                        tape=tape_base)

        with tf.GradientTape() as tape_test:
          test_weight = tf.compat.v1.get_variable(name="test_weights",
                                                  initializer=tf.ones([10, 1]))

          test_logits = tf.nn.relu(math_ops.matmul(x, test_weight))
          test_loss = tf.nn.sigmoid_cross_entropy_with_logits(
              labels=y, logits=test_logits)

        grads_and_vars = test_opt._compute_gradients(test_loss,
                                                     var_list=[test_weight],
                                                     tape=tape_test)
        var_list = []
        aggregated_grad = []
        for grad, var in grads_and_vars:
          var_list.append(var)
          aggregated_grad.append(hvd.allreduce(grad, op=hvd.Sum))
        aggregated_grads_and_vars = zip(aggregated_grad, var_list)
        test_opt_op = test_opt.apply_gradients(aggregated_grads_and_vars)

      with monitored_session.MonitoredTrainingSession(
          is_chief=True, config=default_config) as sess:
        for _ in range(run_step):
          sess.run(base_opt_op)
          sess.run(test_opt_op)

        self.assertAllCloseAccordingToType(
            sess.run(base_weight),
            sess.run(test_weight),
            msg="Cond:{},{},{}".format(dtype, run_step, dim),
        )

  def common_all_to_all_embedding_trainable_v2(self, base_opt, test_opt, name):
    # TODO(rhdong): Recover the testing, if the horovod import error is fixed on macOS+TF2.7+.
    try:
      import horovod.tensorflow as hvd
    except (NotFoundError):
      self.skipTest(
          "Skip the test for horovod import error with Tensorflow-2.7.0 on MacOS-12."
      )

    tf.config.set_soft_device_placement(True)

    hvd.init()

    # These cases need 2 GPUs at least if available.
    logical_devices = tf.config.list_logical_devices('GPU')
    _device = "GPU" if len(logical_devices) >= hvd.size() else "CPU"
    _device_id = hvd.local_rank(
    ) if _device == "GPU" and len(logical_devices) >= 2 else 0

    if _device == "GPU":
      os.environ["CUDA_VISIBLE_DEVICES"] = str(_device_id)

    base_opt = de.DynamicEmbeddingOptimizer(base_opt, synchronous=True)
    test_opt = hvd.DistributedOptimizer(test_opt)
    init = tf.keras.initializers.Zeros()
    kv_creator = de.CuckooHashTableCreator(
        saver=de.FileSystemSaver(proc_size=hvd.size(), proc_rank=hvd.rank()))
    batch_size = 8
    start = 0
    for dtype, run_step, dim in itertools.product([dtypes.float32], [10], [10]):
      print("device=", "/{}:{}".format(_device, _device_id))
      with tf.device("/{}:{}".format(_device, _device_id)):
        for i in range(1, run_step):
          x = math_ops.range(start, start + batch_size * i, dtype=dtypes.int64)
          x = tf.reshape(x, (batch_size, -1))
          start += batch_size * i
          y = tf.zeros((batch_size, 1), dtype=dtypes.float32)

          base_model = get_emb_sequential_model(
              de.keras.layers.HvdAllToAllEmbedding,
              base_opt,
              embedding_size=dim,
              initializer=init,
              bp_v2=False,
              kv_creator=kv_creator,
              name='all2all_emb')
          test_model = get_emb_sequential_model(tf.keras.layers.Embedding,
                                                test_opt,
                                                input_dim=start +
                                                batch_size * i,
                                                output_dim=dim,
                                                embeddings_initializer=init,
                                                name='tf_emb')

          base_model.fit(x, y, verbose=0)
          test_model.fit(x, y, verbose=0)

        self.assertAllCloseAccordingToType(
            base_model.layers[1].weights[0],
            test_model.layers[1].weights[0],
            msg="Cond:{},{},{}".format(dtype, run_step, dim),
        )

        self.assertAllCloseAccordingToType(
            base_model.layers[2].weights[0],
            test_model.layers[2].weights[0],
            msg="Cond:{},{},{}".format(dtype, run_step, dim),
        )

        a2aemb_size = base_model.layers[0].params.size()
        save_dir = "/tmp/hvd_save_restore" + str(
            hvd.size()) + str(run_step) + str(
                dim)  # All ranks should share same save directory
        save_options = tf.saved_model.SaveOptions(namespace_whitelist=['TFRA'])
        if hvd.rank() == 0:
          if os.path.exists(save_dir):
            shutil.rmtree(save_dir)
        hvd.join()  # Sync for avoiding files conflict
        # base_model.save(save_dir, options=save_options)
        de.keras.models.de_hvd_save_model(base_model,
                                          save_dir,
                                          options=save_options)
        ckpt = de.train.DEHvdCheckpoint(base_model)
        ckpt.save(save_dir + '/ckpt/test')
        tf.keras.backend.clear_session()
        del base_model
        new_base_model = get_emb_sequential_model(
            de.keras.layers.HvdAllToAllEmbedding,
            base_opt,
            dense_init='ones',
            embedding_size=dim,
            initializer=init,
            bp_v2=False,
            kv_creator=kv_creator,
            name='all2all_emb')
        ckpt = de.train.DEHvdCheckpoint(new_base_model)
        hvd.join()  # Sync for avoiding files conflict
        ckpt.restore(tf.train.latest_checkpoint(save_dir + '/ckpt/'))
        new_a2aemb_size = new_base_model.layers[0].params.size()
        self.assertEqual(a2aemb_size, new_a2aemb_size)
        hvd.join()  # Sync for avoiding files conflict
        tf.keras.backend.clear_session()
        new_base_model.load_weights(save_dir + '/variables/variables')
        new_a2aemb_size = new_base_model.layers[0].params.size()
        self.assertEqual(a2aemb_size, new_a2aemb_size)
        hvd.join()  # Sync for avoiding files conflict

if __name__ == "__main__":
  test.main()


import logging
import os
import shutil
import subprocess
import tempfile

import multiprocessing as mp
from autometa.common.exceptions import ExternalToolError

logger = logging.getLogger(__name__)

def sort(sam, bam, cpus=mp.cpu_count()):
    """
    Views then sorts sam file by leftmost coordinates and outputs to bam.

    Parameters
    ----------
    sam : str
        </path/to/alignment.sam>
    bam : str
        </path/to/output/alignment.bam>
    cpus : int, optional
        Number of processors to be used. By default uses all the processors of the system

    Raises
    ------
    TypeError
        cpus must be an integer greater than zero
    FileNotFoundError
        Specified path is incorrect or the file is empty
    ExternalToolError
        Samtools did not run successfully, returns subprocess traceback and command run
    """

    if not isinstance(cpus, int) or cpus <= 0:
        raise TypeError(f"cpus must be an integer greater than zero! Given: {cpus}")
    if not os.path.exists(sam) or not os.path.getsize(sam):
        raise FileNotFoundError(
            f"The specified path: {sam} is either incorrect or the file is empty"
        )
    samtools_out_dir = os.path.dirname(os.path.abspath(bam))
    err = os.path.join(samtools_out_dir, "samtools.err")
    out = os.path.join(samtools_out_dir, "samtools.out")
    with tempfile.TemporaryDirectory() as tempdir:  # this will delete the temporary files even if program stops in between
        temp_bam = os.path.join(tempdir, os.path.basename(bam))
        cmd = f"samtools view -@{cpus} -bS {sam} | samtools sort -@{cpus} -o {temp_bam}"
        logger.debug(f"cmd: {cmd}")
        with open(out, "w") as stdout, open(err, "w") as stderr:
            try:
                subprocess.run(
                    cmd, stdout=stdout, stderr=stderr, shell=True, check=True
                )
            except subprocess.CalledProcessError as err:
                raise ExternalToolError(cmd, err)
            shutil.move(temp_bam, bam)
        os.remove(err)
        os.remove(out)
    return bam

def main():
    import argparse
    import logging as logger

    logger.basicConfig(
        format="%(asctime)s : %(name)s : %(levelname)s : %(message)s",
        datefmt="%m/%d/%Y %I:%M:%S %p",
    )
    parser = argparse.ArgumentParser(
        description="Takes a sam file, sorts it and returns the output to a bam file"
    )
    parser.add_argument("sam", help="</path/to/alignment.sam>", type=str)
    parser.add_argument("bam", help="</path/to/output/alignment.bam>", type=str)
    parser.add_argument(
        "--cpus", help="Number of processors to use", default=mp.cpu_count(), type=int
    )
    args = parser.parse_args()
    sort(sam=args.sam, bam=args.bam, cpus=args.cpus)

if __name__ == "__main__":

    main()


from django.shortcuts import render
from django.conf import settings
from django.core.mail import send_mail
# Create your views here.

def index(request):
    return render(request,'email/index.html',context=None)

def send_email(request):
    subject='Django mail'
    message='trying to send mail'
    from_email=settings.EMAIL_HOST_USER
    to_list=['subham.mishra14@gmail.com']

    send_mail(subject,message,from_email,to_list,fail_silently=False)

    return render(request,'email/send_email.html',context=None)


import os
import sys
import pandas as pd
import pytest
import shutil
from datetime import datetime
sys.path.append(os.path.abspath(os.path.dirname(__file__)+'/../..'))
from src import todos

@pytest.fixture(scope="function")
def tmp_dir(tmpdir_factory):
    my_tmpdir = tmpdir_factory.mktemp("pytestdata")
    todos.PATH_TO_DATA = my_tmpdir
    yield my_tmpdir
    shutil.rmtree(str(my_tmpdir))

@pytest.fixture(scope="function")
def new_row():
    return {
        "created": datetime.now().strftime("%Y-%m-%d %H-%M-%S"),
        "task": "cocinar",
        "summary": "Cocinar algo rico",
        "status": "todo",
        "owner": "Andre",
    }

@pytest.fixture(scope="function")
def df_full(new_row):
    return pd.DataFrame(
        [new_row], columns=["created", "task", "summary", "status", "owner"]
    )

n1 = {
        "created": datetime.now().strftime("%Y-%m-%d %H-%M-%S"),
        "task": "cocinar",
        "summary": "Cocinar algo rico",
        "status": "todo",
        "owner": "Andre",
    }

n2 = {
        "created": datetime.now().strftime("%Y-%m-%d %H-%M-%S"),
        "task": "Programar",
        "summary": "Tarea python",
        "status": "todo",
        "owner": "Rodrigo",
    }

def obtener_datos_test_integration():
    return [(False, n1,
            'my_todo_list_1', pd.DataFrame(columns=["created", "task",
"summary", "status", "owner"])),
            (False, n2,
            'my_todo_list_2', pd.DataFrame(columns=["create", "task",
"summary", "status", "owner"]))]

@pytest.mark.parametrize('Bool, nr, tl, dataframe',
                         obtener_datos_test_integration())
def test_add_to_list(Bool, nr, tl, dataframe, df_full, tmp_dir):

    todos.create_list(tl)
    df0 = todos.load_list(tl)
    d0 = (pd.concat([df0, dataframe], axis=1).columns.unique().shape[0] == 5)

    todos.add_to_list(tl, nr)
    df1 = todos.load_list(tl)
    d1 = (df_full.append(df1).drop_duplicates().shape[0] > 1)

    assert (d0 and d1) == Bool


#!/usr/bin/env python
"""Copy random files form source to the destination"""
import logging
import os

def parseArgs():
    """Parse arguments from command line"""
    from argparse import ArgumentParser
    from sys import exit

    parser = ArgumentParser(
        description='Copy random files from source to destination')

    parser.add_argument('source',
                        help='source directory')
    parser.add_argument('destination',
                        help='destination directory')
    parser.add_argument('-d', '--debug', action='store_true',
                        help='enable debug output')
    parser.add_argument('-e', '--extension', default='.mp3',
                        help='file extension filter (defaults to ".mp3")')
    parser.add_argument('-n', '--number', type=int,
                        help='limit the number of files to copy')
    parser.add_argument('-r', '--random', action='store_true',
                        help='insert random id on the destination file name')
    parser.add_argument('-s', '--size', type=int,
                        help='limit the total size (in MB) to copy')

    args = parser.parse_args()

    if not os.path.exists(args.source):
        exit('Invalid source: {}'.format(args.source))

    if not os.path.exists(args.destination):
        exit('Invalid destination: {}'.format(args.destination))

    if args.debug:
        logging.basicConfig(level=logging.DEBUG, format='%(message)s')
        logging.debug('Debugging enabled.')
    else:
        logging.basicConfig(level=logging.INFO, format='%(message)s')

    return args

def scan(directory, extension):
    """Scan directory and return a list of files with the desired extension"""
    files_list = []

    logging.info('Scanning source...')
    for (dir_path, dir_names, file_names) in os.walk(directory):
        if file_names:
            logging.debug('Scanning directory: {}'.format(dir_path))

        for file_name in file_names:
            if file_name.lower().endswith(extension):
                logging.debug('  Match, adding to list: {}'.format(file_name))
                files_list.append((dir_path, file_name))

    return files_list

def copy(files_list, destination, insert_rid=False, max_files=None, max_size=None):
    """Copy files from the list to the destination"""
    from random import SystemRandom
    from shutil import copy

    random = SystemRandom()
    total_files = 0
    total_size = 0

    logging.info('Copying files...')
    while True:
        source, file_name = random.choice(files_list)
        src_path = os.path.join(source, file_name)
        file_size = os.stat(src_path).st_size / 1024.0 / 1024.0

        if insert_rid:
            dst_path = os.path.join(destination,
                                    '{:03}_{}'.format(total_files, file_name))
        else:
            dst_path = os.path.join(destination, file_name)

        logging.debug('  {} ({:.2f} MB)'
                      .format(file_name, file_size))

        if max_size:
            if total_size + file_size > max_size:
                logging.debug('    Maximum size reached ({} MB).'
                              .format(max_size))
                break

        if max_files:
            if total_files == max_files:
                logging.debug('    Number of files reached ({} files).'
                              .format(max_files))
                break

        try:
            copy(src_path, dst_path)

        except IOError as e:
            # No space left
            logging.debug('    {}'.format(e.strerror))
            logging.debug('  Removing file: {}'.format(dst_path))
            os.unlink(dst_path)
            break

        except KeyboardInterrupt:
            # User interruption
            logging.debug('    Keyboard interruption.')
            logging.debug('  Removing file: {}'.format(dst_path))
            os.unlink(dst_path)
            break

        else:
            # Increment the number of copied files
            total_files += 1

            # Increment the total data size copied
            total_size += file_size

    logging.info('Copied {} files ({:.2f} MB)'.format(total_files, total_size))

def run():
    args = parseArgs()
    files_list = scan(args.source, args.extension)

    opts = {}
    if args.random:
        opts['insert_rid'] = True
    if args.number:
        opts['max_files'] = args.number
    if args.size:
        opts['max_size'] = args.size

    copy(files_list, args.destination, **opts)

if __name__ == '__main__':
    try:
        run()
    except (KeyboardInterrupt, SystemExit):
        pass


import requests
import json_extract
from bs4 import BeautifulSoup

# Create session
s = requests.session()

# Extract data from JSON files
site = json_extract.extract('site.json')[0]
urls = site.urls
widgets = site.widgets
ext = site.app_window_ext
login_data = json_extract.extract('login_data.json')[0]
keywords = json_extract.extract('keywords.json')[0]
resume_config = json_extract.extract('resume-config.json')[0]

# Log-in to website
s.post(urls.get('sign-in'), login_data)

# Scrape data from home page
# home_data = s.get(site.get('home'))

# Filter jobs by keywords
job_list_data = s.post(urls.get('jobs_list'), keywords)
job_list_soup = BeautifulSoup(job_list_data.text, 'html.parser')

# Fetch the titles of all jobs on the page
jobs_list = job_list_soup.findAll('div', {'class': 'list-item-title'})

for job in jobs_list:
    # Get job posting url and html
    job_link = job.find('a')
    job_url = urls.get('default') + job_link['href'] + ext
    job_data = s.get(job_url)
    job_soup = BeautifulSoup(job_data.text, 'html.parser')

    # Set job type
    mode = 'default_options'

    # Selecting documents
    resume_option = job_soup.find('div', {'id': widgets.resume})
    resume_option.get('option')['selected'] = 'selected>' + resume_config.get(mode).get('resume_name')

    cover_option = job_soup.find('div', {'id': widgets.cover_letter})
    resume_option.get('option')['selected'] = 'selected>' + resume_config.get(mode).get('cover_letter')

    transcript_option = job_soup.find('div', {'id': widgets.transcript})
    transcript_option.get('option')['selected'] = 'selected>' + resume_config.get(mode).get('transcript')

# Close session
s.close()


import websocket
import json
import requests
from config import username, password, rooms, header, admin, avatar
from battle import Battle

current_battle = None
current_information = None
opp_pokemon = {
    "name": None,
    "hp": None
}

def parser(wsapp, message):
    #websocket.enableTrace(True) # for debugging purposes
    message = message.split("|")
    if message[1] == "challstr": # creates necessary string to log in
        if len(message[3]) != 0:
            challstr = message[2] + "|" + message[3]
            log_in(challstr)
    if message[1] == "pm":
        pm_parser(message[2], message[4])
    if message[-2] == "gametype":
        global current_battle
        current_battle = Battle()
    elif message[1] == "request":
        if current_battle.getTeam() == None:
            current_battle.organize_team(message[2])
        global current_information
        current_information = json.loads(message[2])
        if "forceSwitch" in current_information:
            global opp_pokemon
            battle(message[0], current_information, current_battle, opp_pokemon)
    elif message[-2] == "turn":
        for i in range(0, len(message)):
            if message[i][0:4] == "p1a:":
                if message[i][-2:] == r'\n':
                    opp_pokemon["name"] = message[i][5:-2]
                else:
                    opp_pokemon["name"] = message[i][5:]
                if message[i-1] == "switch":
                    opp_pokemon["hp"] = message[i+2][:-1]
                elif message[i-1] == "-damage":
                    opp_pokemon["hp"] = message[i+1]
        battle(message[0], current_information, current_battle, opp_pokemon)

def log_in(challstr):
    body = {
        "act": "login",
        "name": username,
        "pass": password,
        "challstr": challstr
    }
    # sends a post request to the server
    r = requests.post("https://play.pokemonshowdown.com/~~showdown/action.php", headers = header, data = body)
    assertion = r.text.partition('"assertion":"')[2]
    assertion = assertion[:-2]
    # finishes log-in process to the server
    wsapp.send("|/trn " + username + "," + avatar + "," + assertion)
    # joins rooms that are wanted
    if len(rooms) > 0:
        for i in rooms:
            wsapp.send("|/join " + i)
    wsapp.send("|/avatar " + avatar)

def pm_parser(sender, message):
    if message[0:10] == "/challenge":
        if message[11:] == "gen8randombattle":
            wsapp.send("|/utm null")
            wsapp.send("|/accept " + sender)
        else:
            wsapp.send("|/reject " + sender)
            wsapp.send("|/pm " + sender + ", I do not support battling in this format. Send .help for more info.")
    elif message == ".kill" and sender[1:] == admin:
        wsapp.close()
    elif message == ".help":
            wsapp.send("|/pm " + sender + ", Hi! I am a bot programmed by " + admin + ". If you want to battle me, I can battle in [Gen 8] Random Battle, though I am still in development and could stop working during a battle.")

def battle(battle_link, information, current_battle, opponent = None):
    currentMon = current_battle.currentPokemon(information)
    if "active" in information:
        wsapp.send("|/msgroom " + battle_link[1:-1] + ", " + current_battle.move_selection_priority(information, opponent))
    elif "forceSwitch" in information:
        current_battle.faint(currentMon)
        wsapp.send("|/msgroom " + battle_link[1:-1] + ", " + "/choose switch " + current_battle.switch(opponent, information))

# connects to websocket
wsapp = websocket.WebSocketApp("wss://sim3.psim.us/showdown/websocket", on_message=parser)
# makes it an endless connection
wsapp.run_forever()


from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
import time
import requests

def geturl(url):
    res = requests.head(url)
    url = res.headers.get('location')
    return url

a = 1
with open('id.txt', 'r') as x:
	for line in x:
		id = line.replace('\n', '')
		url = 'http://clients2.google.com/service/update2/crx?response=redirect&prodversion=49.0&acceptformat=crx3&x='+id+'%26installsource%3Dondemand%26uc'
		new = geturl(url)
		print(new)
		r= requests.get(new)
		name = str(a)+'.crx'
		with open(name, "wb") as code:
			code.write(r.content)
		a=a+1


# coding=utf-8
# auther：Liul5
# date：2019/4/26 13:08
# tools：PyCharm
# Python：2.7.15
# https://py.checkio.org/en/mission/sendgrid-daily-spam/
# https://github.com/sendgrid/sendgrid-python/blob/master/USAGE.md#api-keys

import sendgrid
import json
from datetime import datetime
from datetime import timedelta

# need to register and get the key https://app.sendgrid.com
API_KEY = 'your api key'

sg = sendgrid.SendGridAPIClient(apikey=API_KEY)

def how_spammed(str_date):
    end_time = datetime.strptime(str_date, '%Y-%m-%d')
    start_time = end_time + timedelta(days=1)
    response = sg.client.suppression.spam_reports.get(query_params={
    'end_time':int(end_time.timestamp()),
    'start_time': int(start_time.timestamp())
    })
    data = json.loads(response.body)
    return len(data)


import torch
import torch.nn as nn
import os

class Model1(torch.nn.Module):
    def __init__(self):
        super().__init__()

        self.conv = nn.Conv2d(1, 1, 3, stride=1, padding=1,bias=True)
        self.conv.weight.data.fill_(1)
        self.conv.bias.data.fill_(0)

    def forward(self, x):
        x = self.conv(x)
        x = x.view(x.size(0), -1)
        return x

class Model2(torch.nn.Module):
    def __init__(self):
        super().__init__()

        self.conv = nn.Conv2d(1, 1, 3, stride=1, padding=1,bias=True)
        self.conv.weight.data.fill_(1)
        self.conv.bias.data.fill_(0)

    def forward(self, x):
        x = self.conv(x)
        # 用int函数求值
        batch = int(x.size(0))
        x = x.view(batch, -1)
        return x

# 这个包对应opset11的导出代码，如果想修改导出的细节，可以在这里修改代码
# import torch.onnx.symbolic_opset11
print("对应opset文件夹代码在这里：", os.path.dirname(torch.onnx.__file__))

model1 = Model1()
model2 = Model2()
dummy = torch.zeros(1, 1, 3, 3)
torch.onnx.export(model1,  (dummy,), "D:/0/demo1.onnx")
torch.onnx.export(model2,  (dummy,), "D:/0/demo2.onnx")

print("Done.!")


"""OT-3 Module and it's attributes."""
from typing import Optional

from pydantic import Field
from typing_extensions import Literal

from emulation_system.compose_file_creator.config_file_settings import (
    EmulationLevels,
    Hardware,
    OpentronsRepository,
    SourceRepositories,
)
from emulation_system.compose_file_creator.pipette_utils import get_valid_ot3_pipettes
from emulation_system.compose_file_creator.types.intermediate_types import (
    IntermediateEnvironmentVariables,
    IntermediatePorts,
)
from emulation_system.consts import OT3_STATE_MANAGER_BOUND_PORT

# cannot import from . because of circular import issue
from .robot_model import RobotAttributes, RobotInputModel

class OT3Attributes(RobotAttributes):
    """Attributes specific to Robots."""

    left_pipette: get_valid_ot3_pipettes() | None = None  # type: ignore[valid-type]
    right_pipette: get_valid_ot3_pipettes() | None = None  # type: ignore[valid-type]

class OT3SourceRepositories(SourceRepositories):
    """Source repositories for OT3."""

    firmware_repo_name: Literal[None] = None
    # This is the monorepo because we are specifying where the robot server
    # should get it's source code. Not the hardware emulators, those are hardcoded
    # to ot3-firmware
    hardware_repo_name: OpentronsRepository = OpentronsRepository.OT3_FIRMWARE

class OT3InputModel(RobotInputModel):
    """Model for OT3."""

    hardware: Literal[Hardware.OT3]
    source_repos: OT3SourceRepositories = Field(
        default=OT3SourceRepositories(), const=True, exclude=True
    )
    hardware_specific_attributes: OT3Attributes = Field(default=OT3Attributes())
    emulation_level: Literal[EmulationLevels.HARDWARE]
    can_server_exposed_port: int = Field(default=9898)
    can_server_bound_port: int = Field(default=9898)
    ot3_state_manager_exposed_port: int = Field(default=OT3_STATE_MANAGER_BOUND_PORT)

    can_server_env_vars: IntermediateEnvironmentVariables | None
    gripper_env_vars: IntermediateEnvironmentVariables | None
    gantry_x_env_vars: IntermediateEnvironmentVariables | None
    gantry_y_env_vars: IntermediateEnvironmentVariables | None
    pipettes_env_vars: IntermediateEnvironmentVariables | None
    head_env_vars: IntermediateEnvironmentVariables | None
    bootloader_env_vars: IntermediateEnvironmentVariables | None
    state_manager_env_vars: IntermediateEnvironmentVariables | None

    def get_can_server_bound_port(self) -> Optional[IntermediatePorts]:
        """Get can server port string."""
        return (
            [f"{self.can_server_exposed_port}:{self.can_server_bound_port}"]
            if self.can_server_exposed_port is not None
            else None
        )

    def get_ot3_state_manager_bound_port(self) -> IntermediatePorts:
        """Get OT-3 State Manager bound port.

        Note that it is UDP protocol.
        """
        return [
            f"{self.ot3_state_manager_exposed_port}:{OT3_STATE_MANAGER_BOUND_PORT}/udp"
        ]