#mount drive for both input and output
#from google.colab import drive
#drive.mount('/content/drive')

# code
from caption import *
from models	 import *

# miscellaneous
import numpy as np
from collections import Counter
import h5py
import json
import time
from tqdm import tqdm
from nltk.translate.bleu_score import corpus_bleu, sentence_bleu

# PyTorch
import torch
import torch.optim
from torch.optim import Adam
import torch.utils.data as data
import torch.nn.functional as F
import torch.backends.cudnn as cudnn
import torchvision.transforms as transforms
from torch.nn.utils.rnn import pack_padded_sequence
import pdb
from pathlib import Path
# computing environment
cuda = torch.cuda.is_available()
#args.cuda = False

if cuda:
    print('Running on GPU')
    device = torch.device ('cuda:1')
else:
    print('Running on CPU')
    device = torch.device ('cpu')
#device= 'cpu'
#cudnn.benchmark = True

# PARAMETERS

# dir
# input_dir = 'drive/My Drive/img_cap/processed_input/' # input directory (processed data)
input_dir = '/run/user/1435515045/gvfs/smb-share:server=terracotta.psychology.pitt.edu,share=projects/Sanchayan/flickr_localized_narrative/all_info/'
output_dir = "/run/user/1435515045/gvfs/smb-share:server=terracotta.psychology.pitt.edu,share=projects/Sanchayan/flickr_localized_narrative/all_info/model/" # save model directory
trained_model = None   # path to pre-trained model, if there is any
Path(output_dir).mkdir(parents=True, exist_ok=True)
# model
emb_dim = 512       # word embedding dimension
att_dim = 512       # attention dimension/size
dec_dim = 512       # decoder RNN/LSTM dimension

# training
enc_lr = 1e-4       # encoder learning rate
dec_lr = 4e-4       # decoder learning rate
batch_size = 16     # batch size, the destroyer of all RAM
grad_threshold = 5. # clip gradient to prevent exploding
alpha_c = 1.        # regularization parameter from the paper
best_bleu = 0.      # highest bleu-4 score
log_freq = 50       # print train/val stats every number of batches
fine_tune = False   # determines whether to fine tune the encoder (resnet 101)

begin_epoch = 0     # beginning epoch (resume training)
num_epoch = 500     # training epochs unless Colab disconnects or early stop
epoch_from_best = 0 # keep track of how many epochs with no improvement

with open(os.path.join(input_dir, "wordmap.json"), 'r') as f:
    word_map = json.load(f)
word_map['<gap>'] = 2055
class StatMeter:
    """
    Keep track of newest val, sum, count, avg.
    """
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, repeat=1):
        self.val = val
        self.sum += val * repeat
        self.count += repeat
        self.avg = self.sum / self.count

def lr_decay(opt, factor):
    """
    Perform controlled learning rate decay.
    """
    for pg in opt.param_groups:
        pg['lr'] *= factor

def accuracy(scores, targets, k):
    """
    Compute top-k acc.
    """

    batch_size = targets.size(0)
    _, idx = scores.topk(k, dim=1, largest=True, sorted=True)

    targets = targets.view(-1, 1).expand_as(idx)
    right = idx.eq(targets)

    # sum the right list all up
    right_sum = right.view(-1).float().sum()
    acc = right_sum.item() * (100. / batch_size)
    return acc

def clip_gradient(optimizer, threshold):
	"""
	Clips gradients of every parameter.
	"""

	for pg in optimizer.param_groups:
		for p in pg['params']:

			if p.requires_grad:
				p.grad.data.clamp_(-threshold, threshold)

def train(data_loader, encoder, decoder, encoder_opt, decoder_opt, criterion, epoch):
    """
    performs an epoch of training.
    """

    decoder.train()
    encoder.train()

    batch_time = StatMeter()   # forward + backward time by batch
    data_time = StatMeter()    # data loading time by batch
    losses = StatMeter()       # loss per word
    top_accs = StatMeter()     # top 5 accuracy
    tick = time.time()         # initialize time

    # each batch
    for i, (imgs, caps, len_caps) in enumerate(data_loader):
        data_time.update(time.time() - tick)

        # send them to GPU
        imgs = imgs.to(device)
        caps = caps.to(device)
        len_caps = len_caps.to(device)

        # forward: encode images and decode
        imgs = encoder(imgs)

        sorted_caps, decode_lengths, scores, alphas, sorted_idxs = decoder(imgs, caps, len_caps)

       	#print (i, sorted_caps.shape)

        # get targets (excluding <start>)
        targets = sorted_caps[:, 1:]

        # remove undecoded or simply pads with the PackedSequence object
        scores, _ ,  _, _ = pack_padded_sequence(scores, decode_lengths, batch_first=True)
        targets, _ , _, _= pack_padded_sequence(targets, decode_lengths, batch_first=True)

        # compute loss
        loss = criterion(scores, targets)

        # add regularization for soft attention (according to paper)
        loss += alpha_c * ((1. - alphas.sum(dim=1)) ** 2).mean()

        # backward (autograd)
        if encoder_opt: # training encoder is very expensive
            encoder_opt.zero_grad()
        decoder_opt.zero_grad()
        loss.backward()

        # clip gradients if needed
        if grad_threshold:
            clip_gradient(decoder_opt, grad_threshold)
            if encoder_opt:
                clip_gradient(encoder_opt, grad_threshold)

        # update weights with optimizers
        decoder_opt.step()
        if encoder_opt:
            encoder_opt.step()

        # update metrics
        losses.update(loss.item(), sum(decode_lengths))

        top_accs.update(accuracy(scores, targets, 5), sum(decode_lengths))
        batch_time.update(time.time() - tick)
        tick = time.time()

        '''
        if i % log_freq == 0:
            print(f'E [{epoch}][{i}/{len(data_loader)}]\t'
                  f'D {data_time.val:.3f}->{data_time.avg:.3f}\t'
                  f'B {batch_time.val:.3f}->{batch_time.avg:.3f}\t'
                  f'L {loss.val:.4f}->{loss.avg:.4f}\t'
                  f'A {top_accs.val:.3f}->{top_accs.avg:.3f}')
        '''
        print (loss)

def validate_old(data_loader, encoder, decoder, criterion):
    """
    Performs an epoch of validation.
    """
    decoder.eval()
    if encoder:
        encoder.eval()

    # same as train() but without data loading timer
    batch_time = StatMeter()
    losses = StatMeter()
    top_accs = StatMeter()
    tick = time.time()

    ground_truths = []
    predictions = []

    # always disable gradient when evaluating
    with torch.no_grad():
        # all captions also passed in from caption.py
        for i, (imgs, caps, len_caps, all_caps) in enumerate(data_loader):
            # the uncommented operations are similar to train(), please refer to that
            imgs = imgs.to(device)
            caps = caps.to(device)
            len_caps = len_caps.to(device)

            if encoder:
                imgs = encoder(imgs)
            sorted_caps, decode_lengths, scores, alphas, sorted_idxs = decoder(imgs, caps, len_caps)
            targets = sorted_caps[:, 1:]

            scores_cp = scores.clone() # save a copy for bleu score
            scores, _ = pack_padded_sequence(scores, decode_lengths, batch_first=True)
            targets, _ = pack_padded_sequence(targets, decode_lengths, batch_first=True)

            loss = criterion(scores, targets)
            loss += alpha_c * ((1. -  alpha.sum(dim=1)) ** 2).mean()

            losses.update(loss.item(), sum(decode_lengths))
            losses.update(loss.item(), sum(decode_lengths))
            top_accs.update(accuracy(scores, targets, 5), sum(decode_lengths))
            batch_time.update(time.time() - tick)
            tick = time.time()

            if i % log_freq == 0:
                print(f'V [{i}/{len(data_loader)}]\t'
                      f'B {batch_time.val:.3f}->{batch_time.avg:.3f}\t'
                      f'L {loss.val:.4f}->{loss.avg:.4f}\t'
                      f'A {top_accs.val:.3f}->{top_accs.avg:.3f}')

            # get ground truths (sort captions and get rid of start and end tokens)
            all_caps = all_caps[sorted_idxs]
            for j in range(all_caps.shape[0]):
                img_caps = all_caps[j].tolist()
                # get rid of <start> and <end> because they increase the bleu score
                img_caps = list(map(lambda cap: [w for w in cap if (w != word_map['<start>'] and w != word_map['<pad>'])],
                                img_caps))
                ground_truths.append(img_caps)

            # get predictions
            _, preds = torch.max(scores_cp, dim=2)
            preds = preds.tolist()
            temp = []
            for j, p in enumerate(preds):
                # not including pads
                temp.append(preds[j][:decode_lengths[j]])
            preds = temp
            predictions.extend(preds)

            assert len(ground_truths) == len(predictions)

        # use corpus_bleu library functions to calculate bleu score
        bleu_score = corpus_bleu(ground_truths, predictions)
        print(f'\nL {loss.avg:.3f} A {top_5.avg:.3f}, B {bleu_score}\n')

    return bleu_score

def validate(data_loader, encoder, decoder, criterion):
    """
    Performs an epoch of validation.
    """
    decoder.eval()
    if encoder:
        encoder.eval()

    # same as train() but without data loading timer
    batch_time = StatMeter()
    losses = StatMeter()
    top_accs = StatMeter()
    tick = time.time()

    ground_truths = []
    predictions = []

    # always disable gradient when evaluating
    with torch.no_grad():
        # all captions also passed in from caption.py
        for i, (imgs, caps, len_caps) in enumerate(data_loader):
            # the uncommented operations are similar to train(), please refer to that
            imgs = imgs.to(device)
            caps = caps.to(device)
            len_caps = len_caps.to(device)

            if encoder:
                imgs = encoder(imgs)
            sorted_caps, decode_lengths, scores, alphas, sorted_idxs = decoder(imgs, caps, len_caps)
            targets = sorted_caps[:, 1:]

            scores_cp = scores.clone() # save a copy for bleu score
            scores, _, _, _ = pack_padded_sequence(scores, decode_lengths, batch_first=True)
            targets, _, _,_ = pack_padded_sequence(targets, decode_lengths, batch_first=True)

            loss = criterion(scores, targets)
            loss += alpha_c * ((1. -  alphas.sum(dim=1)) ** 2).mean()

            losses.update(loss.item(), sum(decode_lengths))
            losses.update(loss.item(), sum(decode_lengths))
            top_accs.update(accuracy(scores, targets, 5), sum(decode_lengths))
            batch_time.update(time.time() - tick)
            tick = time.time()

            '''
            if i % log_freq == 0:
                print(f'V [{i}/{len(data_loader)}]\t'
                      f'B {batch_time.val:.3f}->{batch_time.avg:.3f}\t'
                      f'L {loss.val:.4f}->{loss.avg:.4f}\t'
                      f'A {top_accs.val:.3f}->{top_accs.avg:.3f}')
            '''

            # get ground truths (sort captions and get rid of start and end tokens)
            #all_caps = all_caps[sorted_idxs]
            for j in range(sorted_caps.shape[0]):
                img_caps = sorted_caps[j].tolist()

                # get rid of <start> and <end> because they increase the bleu score

                selected_caps= [x for x in img_caps if x!=word_map['<start>'] and x!=word_map['<pad>'] and x!=word_map['<end>']]
                '''
                img_caps = list(map(lambda cap: [w for w in cap if (w != word_map['<start>'] and w != word_map['<pad>'])],
                                img_caps))
                '''

                ground_truths.append([selected_caps])

            # get predictions
            _, preds = torch.max(scores_cp, dim=2)
            preds = preds.tolist()
            temp = []
            for j, p in enumerate(preds):
                # not including pads
                temp.append(preds[j][:decode_lengths[j]])
            preds = temp
            predictions.extend(preds)

            assert len(ground_truths) == len(predictions)

            print (loss)

        # use corpus_bleu library functions to calculate bleu score
        bleu_score = corpus_bleu(ground_truths, predictions)

        #print(f'\nL {loss.avg:.3f} A {top_5.avg:.3f}, B {bleu_score}\n')

    return bleu_score, predictions, ground_truths

def save_model(epoch, encoder, decoder, encoder_opt, decoder_opt, new_bleu, predictions, ground_truth,save_path):

	torch.save ({'epoch' : epoch,
                'encoder' : encoder,
                 'decoder' : decoder,
				'encoder_state_dict': encoder.state_dict(),
                'decoder_state_dict': decoder.state_dict(),
                'encoder_opt': encoder_opt,
                'decoder_opt': decoder_opt,
                'bleu': new_bleu,
                'predictions':predictions,
                'ground_truth': ground_truth}, save_path+str(epoch))

	return
def train_and_validate(trained_model=trained_model, best_bleu=best_bleu, begin_epoch=begin_epoch,
                       epoch_from_best=epoch_from_best, fine_tune=fine_tune, word_map=word_map):

    decoder = DecoderWithAttention(len(word_map), emb_dim, dec_dim, att_dim, device=device )
    decoder_opt = Adam(params=filter(lambda p: p.requires_grad, decoder.parameters()), lr = dec_lr)
    encoder = Encoder()

    encoder.fine_tune(fine_tune)

    if fine_tune:
        encoder_opt = Adam(params=filter(lambda p: p.requires_grad, decoder.parameters()), lr = enc_lr)
    else:
        encoder_opt = None

    # move everything to device
    decoder = decoder.to(device)
    encoder = encoder.to(device)
    criterion = nn.CrossEntropyLoss().to(device)

    # transform while loading data
    normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                     std=[0.229, 0.224, 0.225])

    trainset = CapData(input_dir, 'tr', transform=transforms.Compose([normalize]))
    validset = CapData(input_dir, 'vl', transform=transforms.Compose([normalize]))
    testset = CapData(input_dir, 'ts', transform=transforms.Compose([normalize]))
    # create data loaders, one worker for moving data, pin_memory for fast data transfer
    train_data_loader = data.DataLoader(
        trainset,
        batch_size=batch_size, collate_fn= trainset.collate_fn,shuffle=True, num_workers=0, pin_memory=False)
    val_data_loader = data.DataLoader(
        validset,
        batch_size=batch_size, collate_fn= validset.collate_fn, shuffle=False, num_workers=0, pin_memory=False)
    test_data_loader = data.DataLoader(
        testset,
        batch_size=batch_size, collate_fn= testset.collate_fn, shuffle=False, num_workers=0, pin_memory=False)

    for epoch in range(begin_epoch, num_epoch):
        if epoch_from_best >= 25:
            break
        # learning rate decay every 10 epochs with no improvement
        if epoch_from_best % 10 == 0 and epoch_from_best != 0:
            if fine_tune:
                lr_decay(encoder_opt, 0.8)
            lr_decay(decoder_opt, 0.8)

        # train and validate
        train(train_data_loader, encoder, decoder, encoder_opt, decoder_opt, criterion, epoch)
        val_bleu, val_predictions, val_ground_truth = validate(val_data_loader, encoder, decoder, criterion)
        test_bleu, test_predictions, test_ground_truth = validate(test_data_loader, encoder, decoder, criterion)

        update = val_bleu > best_bleu

        if update:
            epoch_from_best = 0
            best_bleu = val_bleu
            print("New improvement!")
            save_model(epoch,encoder, decoder, encoder_opt, decoder_opt, best_bleu, test_predictions, test_ground_truth, output_dir)

        else:
            epoch_from_best += 1
            print(f"No improvement for {epoch_from_best} epochs")

train_and_validate()


import re
import requests
from bs4 import BeautifulSoup
from redis import StrictRedis, ConnectionPool

def get_urls(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.content
        else:
            print('request error code:', response.status_code)
    except Exception as e:
        print(e)
        return None

def parse(input, cutLetter):
    re_tag = re.compile('</?\w+[^>]*>')
    content = re_tag.sub('', str(input))
    if cutLetter:
        re_useless = re.compile('[a-zA-Z,]')
        return re_useless.sub('', content)
    return content

def getRedisConnectionPool():
    url = 'redis://@localhost:6379/0'
    pool = ConnectionPool.from_url(url)
    return pool

def main():
    headers = {
        'user-agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36',
    }

    url_co_prefix = 'https://www.youtube.com/watch?v='

    url_ids = {
        'hitomi': [
            'MwEU_09TaLs','nXjKqkAJ-R8','4lZx2f-TbY0','rftFz2tUbKE','VFgpwfEIrnI',
            'u-GRWRs_d6I','yQFzeBzO_ZE','zfaGTwmxG3A','kMYEntl-_js','mVH_dzY3wRI',
            'IH7cIEByWWU','t_LcpM-QG-4','0hr-DyTWLU4','9eIfStw-e_c','1r6fPvCt1UE',
            'XCZEnJ5L81Q','KScSFIrl8HQ','txH9eX8hD_Y','bxWD4TpxKWI','fwrQE9pGizM',
            'j25j272Z9hQ','jFPz8wsQurw','7IFWkA0FM9o',
        ],
        'nako': [
            'vIcFnNdHc2w', '5YHLH9CVff0', 'o6VpbfOT9R0', 'Rry_viMzp_M',
            'rfvYiRUeK0I', 'GSVurT6v6gs', 'Lq7rgg_Rq7E', 'ucg-KHmPrXY',
            'UQD62p2TDqQ', 'PR6ZdZZiQNE', 'A-3YKK1NNJ4', 'VtxvZq6dB3Q',
            'atI1ZzBArrs', '6R5ORUUiLRQ', '-l7etfJDAso', 'MfdCxncxbQw',
            'vqHLAVx40t0', 'c-IP3n-VUm0', '-xPD5CovM70', 'DAEph5aJGlQ'
        ],
        'sakura': [
            'm2K26HTAi_E', 'eWdxzvTy88k', 'YeJ9s9Xtzug', 'qp7dHeGPi08',
            'nKcXsTOIesE', 'ywEkHQyxevQ', 'sDBPKy5jHRY', 'tnDHnbk1Pds',
            'T2Bvxr2OTUU', 'Y3RkiW0eWwQ', 'idgk0b0lioo', 'XIbrYlvCvE8',
            'DOF0M6KKw8E', 'KDfebdR8XyM', 'U_vNFe1XrlM', 'ks6fbGC2SP8',
            'Upi90X8DlbU', 'GuU_ob0diSE', '3fzXLGMFWRo', 'hdIf6j4O5TQ',
            's5oc82ZPyEs', 'nSMt0IPwAhQ', '3EkshpX5_AQ', 'f-ehRuVT19k',
            'V0WWskPWmLQ', 'W0hntlJHAXY', 'evhL6ZE4Iq0', 'tXojxAwO2R0',
            'gGpNfVCIkxU', 'zAvvJ8OxVlk', '5HjjumzgNtY', 'MXqG0e6wuY4',
            'bDAoIX1nyjY', 'xPhSvAJvbVA', 'M17XCxxAWGs', 'hcSBFdMbWVk',
            '25bEVyeTg9c', 'Hpp1oMorYNI', 'waWpOPLZPMU'
        ],
        'nayeon': [
            '6fbxCnIyS4c', 'O4XEJhLh124', 'KQiqzFmKti0', '2I8hTLs9B8c',
            'TqhQWwV9hSc', 'fhyLpq-7OL0', 'Rwyr4Q6tD4Q', 'FL6TpPBqTZk',
            'Vq2q6cLQIvc', '3iu0Y8UbRw0', 'rc35kimy7VU', 'IpvZrVPH_NE',
            'xquaIy535_c', 'ZdWfXyX-dxQ', '-RcUOYxo3tk', 'H1ky7IKp914',
            '838Bj2TTxOA'
        ]
    }

    pool = getRedisConnectionPool()
    redis = StrictRedis(connection_pool=pool)

    for key in url_ids.keys():
        ids = url_ids[key]
        print(key + ":")
        for id in ids:
            html = get_urls(url_co_prefix + id)
            soup = BeautifulSoup(html, 'html.parser')
            title = parse(soup.head.title, False)
            content = soup.find('div', {'class': 'watch-view-count'})
            viewCount = re.findall(r'\d+', parse(content, True))[0]
            delta = int(viewCount)
            if redis.exists(key, title):
                preViewCount = int(redis.hget(key, title).decode('utf-8'))
                delta = delta - preViewCount
            redis.hset(key, title, str(viewCount))
            print(delta, viewCount, title)

if __name__ == "__main__":
    main()


from graphframes import *
from pyspark.sql import SQLContext
from pyspark import SparkContext

# Create DataFrames from HIVE query
v=SQLContext.sql("select  id, name, total_seconds from my_schema.nodes")
e=SQLContext.sql("select src, dst, relationship from my_schema.edges")

# Create DataFrames manually for testing purposes
v = SQLContext.createDataFrame([
    ("A", "ARON"  ,350 ),
    ("B", "BILL"  ,360 ),
    ("C", "CLAIR" ,195 ),
    ("D", "DANIEL",90),
    ("E", "ERIC"  ,90),
    ("F", "FRANK" ,215 ),
    ("G", "GRAHAM",30 ),
    ("H", "HENRY" ,25 ),
    ("I", "INNA"  ,25 ),
    ("J", "JEN"   ,20 )
], ["id", "name", "total_seconds"])

e=SQLContext.createDataFrame([
    ("A", "B", 60),
    ("B", "A", 50),
    ("A", "C", 50),
    ("C", "A", 100),
    ("A", "D", 90),
    ("C", "I", 25),
    ("C", "J", 20),
    ("B", "F", 50),
    ("F", "B", 110),
    ("F", "G", 30),
    ("F", "H", 25),
    ("B", "E", 90)
],["src","dst","relationship"])

# Now lets construct the graph
g = GraphFrame(v,e)


#!/usr/bin/env python
# coding: utf-8

# In[ ]:

import os
import json

class Inventory:
    def _init_(self):
        #AT LAUNCH GROUPS AND LOADING FUNCTION
        self.items = {}
        self.load()

    def remove(self, ID):
        #REMOVING ITEMS FOR LISTS AND OUTPUT DOCUMENT
        del self.items[str(ID)]
        self.save()

    def add(self, ID, name, qty):
        #ADDING ITEMS FOR LISTS AND OUTPUT DOCUMENT
        self.items[ID] = {"name": name, "qty": qty}
        self.save()

    def update(self, ID, update):
        #UPDATING ITEMS FOR LISTS AND OUTPUT DOCUMENT
        self.items[ID]["qty"] += update
        self.save()

    def search(self, ID):
        #SEARCHING ITEMS FOR LISTS
        item = self.items.get(ID, None)
        if item:
            return ID, item['name'], item['qty']
        else:
            return None

    def _str_(self):
        #FORMATTING
        out = ""
        for id, d in self.items.items():
            out += f"ID Number : {id} \nItem Name : {d['name']}\nQuantity : {d['qty']}\n"
            out += "----------\n"
        return out

    def save(self):
        #WHERE TO SAVE TO
        with open('data.txt','w') as outfile:
           json.dump(self.items, outfile)

    def load(self):
        #WHERE TO PUT DATA FROM WHEN RELAUNCHING PROGRAM
        try:
            with open('data.txt','r') as json_file:
               self.items = json.load(json_file)
        except:
            print("Can't load old inventory, starting fresh")
            self.items = {}

def menuDisplay():
    #MENU FOR PROGRAM
    """Display the menu"""
    print('=============================')
    print('= Inventory Management Menu =')
    print('=============================')
    print('(1) Add New Item to Inventory')
    print('(2) Remove Item from Inventory')
    print('(3) Update Inventory')
    print('(4) Search Item in Inventory')
    print('(5) Print Inventory Report')
    print('(99) Quit')

def add_one_item(inventory):
    #ADDING PROMPT AND ERROR CHECKING
    print('Adding Inventory')
    print('================')
    while True:
        try:
            new_ID = int(input("Enter an ID number for the item: "))
            if inventory.search(new_ID):
                print("ID number is taken, please enter a different ID number")
                continue
            new_name = input('Enter the name of the item: ').lower()
            assert new_name.isalpha(), "Only letters are allowed!"
            new_qty = int(input("Enter the quantity of the item: "))
            inventory.add(new_ID, new_name, new_qty)
            break
        except Exception as e:
            print("Invalid choice! try again! " + str(e))
            print()

def remove_one_item(inventory):
    #REMOVING PROMPT AND ERROR CHECKING
    print('Removing Inventory')
    print('==================')
    removing = int(input("Enter the item's ID number to remove from inventory: "))
    inventory.remove(removing)

def ask_exit_or_continue():
    #OPTION TO CONTINUE OR QUITE PROGRAM
    return int(input('Enter 98 to continue or 99 to exit: '))

def update_inventory(inventory):
    #UPDATING PROMPT AND ERROR CHECKING
    print('Updating Inventory')
    print('==================')
    ID = int(input("Enter the item's ID number to update: "))
    update = int(input("Enter the updated quantity. Enter 5 for additional or -5 for less: "))
    inventory.update(ID, update)

def search_inventory(inventory):
    #SEARCHING PROMPT AND ERROR CHECKING
    print('Searching Inventory')
    print('===================')
    search = int(input("Enter the ID number of the item: "))
    result = inventory.search(search)
    if result is None:
        print("Item not in inventory")
    else:
        ID, name, qty = result
        print('ID Number: ', ID)
        print('Item:     ', name)
        print('Quantity: ', qty)
        print('----------')

def print_inventory(inventory):
    #PRINT CURRENT LIST OF ITEMS IN INVENTORY
    print('Current Inventory')
    print('=================')
    print(inventory)

def main():
    #PROGRAM RUNNING COMMAND AND ERROR CHECKING
    inventory = Inventory()
    while True:
        try:
            menuDisplay()
            CHOICE = int(input("Enter choice: "))
            if CHOICE in [1, 2, 3, 4, 5]:
                if CHOICE == 1:
                    add_one_item(inventory)
                elif CHOICE == 2:
                    remove_one_item(inventory)
                elif CHOICE == 3:
                    update_inventory(inventory)
                elif CHOICE == 4:
                    search_inventory(inventory)
                elif CHOICE == 5:
                    print_inventory(inventory)
                exit_choice = ask_exit_or_continue()
                if exit_choice == 99:
                    exit()
            elif CHOICE == 99:
                exit()
        except Exception as e:
            print("Invalid choice! try again!"+str(e))
            print()

        # If the user pick an invalid choice,
        # the program will come to here and
        # then loop back.

main()


import torch

def iou(pred, target, num_classes):
    ious = []
    for cls in range(num_classes):
        pred_inds = pred == cls
        target_inds = target == cls
        intersection = (pred_inds[target_inds]).long().sum().data.cpu()  # Cast to long to prevent overflows
        union = pred_inds.long().sum().data.cpu() + target_inds.long().sum().data.cpu() - intersection
        if union == 0:
            ious.append(float('nan'))  # If there is no ground truth, do not include in evaluation
        else:
            ious.append(float(intersection) / max(union, 1))
    return ious

def dice(pred, target, num_classes):
    dice = []
    for cls in range(num_classes):
        pred_inds = pred == cls
        target_inds = target == cls
        intersection = (pred_inds[target_inds]).long().sum().data.cpu()  # Cast to long to prevent overflows
        dice.append(float(2 * intersection) / (pred_inds.long().sum().data.cpu() + target_inds.long().sum().data.cpu()))
    return dice

class MetricMaker:

    def __init__(self, classes) -> None:
        self.dices = {cls: [] for cls in classes}
        self.ious = {cls: [] for cls in classes}
        self.classes = classes

    def update(self, pred, target):
        dices = dice(pred, target, len(self.classes))
        ious = iou(pred, target, len(self.classes))
        for idx, cls in enumerate(self.classes):
            self.dices[cls].append(dices[idx])
            self.ious[cls].append(ious[idx])

    @property
    def last_dice(self):
        return {cls: self.dices[cls][-1] for cls in self.classes}

    @property
    def last_iou(self):
        return {cls: self.ious[cls][-1] for cls in self.classes}

    @property
    def mean_dice(self):
        return {cls: torch.tensor(self.dices[cls]).mean().item() for cls in self.classes}

    @property
    def mean_iou(self):
        return {cls: torch.tensor(self.ious[cls]).mean().item() for cls in self.classes}

    def reset(self):
        self.dices = {cls: [] for cls in self.classes}
        self.ious = {cls: [] for cls in self.classes}


import typing
from .character import Character
from ...definition import AbstractGameLogic, AbstractGameState, CharacterStatus, InternalQuestion
from ...server import CharacterType

class CharacterGrey(Character):

    def __init__(self, state: CharacterType) -> None:
        super().__init__(state)
        self._post_movement_power = CharacterStatus.MANDATORY

    def get_power_possibilities(self, game_logic: AbstractGameLogic, game_state: AbstractGameState) -> typing.Tuple[InternalQuestion, tuple]:
        available_rooms = [room for room in range(10) if room is not game_state.room_shadow]
        game_state._end_of_turn = True
        return (InternalQuestion.ROOM_SHADOW, available_rooms)

    def reset(self) -> None:
        self._movement = CharacterStatus.AVAILABLE
        self._post_movement_power = CharacterStatus.MANDATORY


# from django.http import JsonResponse #to return python data as JSON data # removed after adding api decarotor
from rest_framework.decorators import api_view, permission_classes
from rest_framework.permissions import IsAuthenticated, IsAdminUser
from rest_framework.response import Response
from .serializers import ProjectSerializer
from projects.models import Project, Review

# list specifies what methods function can take (only GET in this case)
@api_view(['GET'])
# gives all url paths or routes inside of api
def getRoutes(request):
    # list dictionaries that will be turned into javascript objects
    # specifying method that a specific route takes
    routes = [
        # returns a list of project objects
        {'GET': 'api/projects'},
        # returns a single project object
        {'GET': 'api/projects/id'},
        # passes in vote
        {'POST': 'api/projects/id/vote'},

        # built-in class to generate a token for user login
        {'POST': 'api/users/token'},
        # built-in class to generate a new json web token after it expires
        {'POST': 'api/users/token/refresh'},
    ]
    # return python data as json data
    # safe parameter allows us to return back something other than
    # a python dictionary, since we're sending a list, set safe to False
    # this is saying it's ok to turn any data into JSON data
    # return JsonResponse(routes, safe=False)
    # can use this instead of above b/c function is now wrapped in an api decarator
    return Response(routes)

# list specifies what methods function can take (only GET in this case)
@api_view(['GET'])
# requires authentication for user to get projects
# @permission_classes([IsAuthenticated])
def getProjects(request):
    projects = Project.objects.all()
    # takes queryset and turns it into json data
    # many = True b/c serializing multiple objects
    serializer = ProjectSerializer(projects, many=True)
    # .data used to get data out of serializer which is a class
    return Response(serializer.data)

# list specifies what methods function can take (only GET in this case)
@api_view(['GET'])
# requires authentication for user to get projects
# @permission_classes([IsAuthenticated])
def getProject(request, pk):
    project = Project.objects.get(id=pk)
    # takes queryset and turns it into json data
    # many = False b/c not serializing multiple objects
    serializer = ProjectSerializer(project, many=False)
    # .data used to get data out of serializer which is a class
    return Response(serializer.data)

# list specifies what methods function can take (only POST in this case)
@api_view(['POST'])
# requires authentication for user to get projects
@permission_classes([IsAuthenticated])
# takes in post request and modifies a project vote
def projectVote(request, pk):
    project = Project.objects.get(id=pk)
    user = request.user.profile
    # property available b/c of api decorator
    data = request.data
    # checks if object exists already and gets that instance
    # if it does and creates one if not
    # created will be true or false
    review, created = Review.objects.get_or_create(
        owner=user,
        project=project,
        )
    # getting vote value and setting it
    review.value = data['value']
    # update project vote count
    review.save()
    # because used @property decorator in models do not have to call as
    # traditional function
    project.getVoteCount

    # takes queryset and turns it into json data
    # many = False b/c not serializing multiple objects
    serializer = ProjectSerializer(project, many=False)
    # .data used to get data out of serializer which is a class
    return Response(serializer.data)


# -*- coding: utf-8 -*
# 载入模块
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import numpy as np

# 创建 1 张画布
fig = plt.figure()

#===============

# 向画布添加子图 1
ax1 = fig.add_subplot(1, 2, 1, projection='3d')

# 生成子图 1 数据
x = np.linspace(-6 * np.pi, 6 * np.pi, 1000)
y = np.sin(x)
z = np.cos(x)

# 绘制第 1 张图
ax1.plot(x, y, z)

#===============

# 向画布添加子图 2
ax2 = fig.add_subplot(1, 2, 2, projection='3d')

# 生成子图 2 数据
X = np.arange(-2, 2, 0.1)
Y = np.arange(-2, 2, 0.1)
X, Y = np.meshgrid(X, Y)
Z = np.sqrt(X ** 2 + Y ** 2)

# 绘制第 2 张图
ax2.plot_surface(X, Y, Z, cmap=plt.cm.winter)

# 显示图
plt.show()


import torch
from torch.nn import Module, ModuleList, Conv1d, Sequential, ReLU, Dropout
from torch_geometric.nn import Linear, EdgeConv, GATv2Conv, SAGEConv, BatchNorm

class DilatedResidualLayer(Module):
    def __init__(self, dilation, in_channels, out_channels):
        super(DilatedResidualLayer, self).__init__()
        self.conv_dilated = Conv1d(in_channels, out_channels, kernel_size=3, padding=dilation, dilation=dilation)
        self.conv_1x1 = Conv1d(out_channels, out_channels, kernel_size=1)
        self.relu = ReLU()
        self.dropout = Dropout()

    def forward(self, x):
        out = self.relu(self.conv_dilated(x))
        out = self.conv_1x1(out)
        out = self.dropout(out)
        return x + out

# This is for the iterative refinement (we refer to MSTCN++: https://github.com/sj-li/MS-TCN2)
class Refinement(Module):
    def __init__(self, final_dim, num_layers=10, interm_dim=64):
        super(Refinement, self).__init__()
        self.conv_1x1 = Conv1d(final_dim, interm_dim, kernel_size=1)
        self.layers = ModuleList([DilatedResidualLayer(2**i, interm_dim, interm_dim) for i in range(num_layers)])
        self.conv_out = Conv1d(interm_dim, final_dim, kernel_size=1)

    def forward(self, x):
        f = self.conv_1x1(x)
        for layer in self.layers:
            f = layer(f)
        out = self.conv_out(f)
        return out

class SPELL(Module):
    def __init__(self, cfg):
        super(SPELL, self).__init__()
        self.use_spf = cfg['use_spf'] # whether to use the spatial features
        self.use_ref = cfg['use_ref']
        self.num_modality = cfg['num_modality']
        channels = [cfg['channel1'], cfg['channel2']]
        final_dim = cfg['final_dim']
        num_att_heads = cfg['num_att_heads']
        dropout = cfg['dropout']

        if self.use_spf:
            self.layer_spf = Linear(-1, cfg['proj_dim']) # projection layer for spatial features

        self.layer011 = Linear(-1, channels[0])
        if self.num_modality == 2:
            self.layer012 = Linear(-1, channels[0])

        self.batch01 = BatchNorm(channels[0])
        self.relu = ReLU()
        self.dropout = Dropout(dropout)

        self.layer11 = EdgeConv(Sequential(Linear(2*channels[0], channels[0]), ReLU(), Linear(channels[0], channels[0])))
        self.batch11 = BatchNorm(channels[0])
        self.layer12 = EdgeConv(Sequential(Linear(2*channels[0], channels[0]), ReLU(), Linear(channels[0], channels[0])))
        self.batch12 = BatchNorm(channels[0])
        self.layer13 = EdgeConv(Sequential(Linear(2*channels[0], channels[0]), ReLU(), Linear(channels[0], channels[0])))
        self.batch13 = BatchNorm(channels[0])

        if num_att_heads > 0:
            self.layer21 = GATv2Conv(channels[0], channels[1], heads=num_att_heads)
        else:
            self.layer21 = SAGEConv(channels[0], channels[1])
            num_att_heads = 1
        self.batch21 = BatchNorm(channels[1]*num_att_heads)

        self.layer31 = SAGEConv(channels[1]*num_att_heads, final_dim)
        self.layer32 = SAGEConv(channels[1]*num_att_heads, final_dim)
        self.layer33 = SAGEConv(channels[1]*num_att_heads, final_dim)

        if self.use_ref:
            self.layer_ref1 = Refinement(final_dim)
            self.layer_ref2 = Refinement(final_dim)
            self.layer_ref3 = Refinement(final_dim)

    def forward(self, x, edge_index, edge_attr, c=None):
        feature_dim = x.shape[1]

        if self.use_spf:
            x_visual = self.layer011(torch.cat((x[:, :feature_dim//self.num_modality], self.layer_spf(c)), dim=1))
        else:
            x_visual = self.layer011(x[:, :feature_dim//self.num_modality])

        if self.num_modality == 1:
            x = x_visual
        elif self.num_modality == 2:
            x_audio = self.layer012(x[:, feature_dim//self.num_modality:])
            x = x_visual + x_audio

        x = self.batch01(x)
        x = self.relu(x)

        edge_index_f = edge_index[:, edge_attr<=0]
        edge_index_b = edge_index[:, edge_attr>=0]

        # Forward-graph stream
        x1 = self.layer11(x, edge_index_f)
        x1 = self.batch11(x1)
        x1 = self.relu(x1)
        x1 = self.dropout(x1)
        x1 = self.layer21(x1, edge_index_f)
        x1 = self.batch21(x1)
        x1 = self.relu(x1)
        x1 = self.dropout(x1)

        # Backward-graph stream
        x2 = self.layer12(x, edge_index_b)
        x2 = self.batch12(x2)
        x2 = self.relu(x2)
        x2 = self.dropout(x2)
        x2 = self.layer21(x2, edge_index_b)
        x2 = self.batch21(x2)
        x2 = self.relu(x2)
        x2 = self.dropout(x2)

        # Undirected-graph stream
        x3 = self.layer13(x, edge_index)
        x3 = self.batch13(x3)
        x3 = self.relu(x3)
        x3 = self.dropout(x3)
        x3 = self.layer21(x3, edge_index)
        x3 = self.batch21(x3)
        x3 = self.relu(x3)
        x3 = self.dropout(x3)

        x1 = self.layer31(x1, edge_index_f)
        x2 = self.layer32(x2, edge_index_b)
        x3 = self.layer33(x3, edge_index)

        out = x1+x2+x3

        if self.use_ref:
            xr0 = torch.permute(out, (1, 0)).unsqueeze(0)
            xr1 = self.layer_ref1(torch.softmax(xr0, dim=1))
            xr2 = self.layer_ref2(torch.softmax(xr1, dim=1))
            xr3 = self.layer_ref3(torch.softmax(xr2, dim=1))
            out = torch.stack((xr0, xr1, xr2, xr2), dim=0).squeeze(1).transpose(2, 1).contiguous()

        return out


import aiohttp
import crescent
import hikari
from pydub import AudioSegment
import speech_recognition

plugin = crescent.Plugin()
recognizer = speech_recognition.Recognizer()

@plugin.include
@crescent.message_command(name="Process voice")
async def voice(ctx: crescent.Context, message: hikari.Message):
	if not hikari.MessageFlag.IS_VOICE_MESSAGE in message.flags:
		return await ctx.respond(content="not voice", ephemeral=True)
	await ctx.defer(ephemeral=True)
	with open("f.mp3", mode="wb") as f:
		async with aiohttp.ClientSession() as session:
			async with session.get(message.attachments[0].url) as response:
				thing = await response.read()
				f.write(thing)

	sound = AudioSegment.from_mp3("f.mp3")
	sound.export("f.wav", format="wav")

	with speech_recognition.AudioFile("f.wav") as source:
		audio_data = recognizer.record(source)
		text = recognizer.recognize_google(audio_data)

	await ctx.respond(text, ephemeral=True)


import random
import torch
import torch.nn as nn
import torch.utils
import torch.nn.functional as F
import torchvision.datasets as dset
import torch.utils.data as data
import torchvision.transforms as transforms
import torch.backends.cudnn as cudnn
import numpy as np
import threading
import logging
import os
import gc
import shutil
from io import BytesIO
from PIL import Image, ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True

class Cutout(object):
    def __init__(self, length):
        self.length = length

    def __call__(self, img):
        h, w = img.size(1), img.size(2)
        mask = np.ones((h, w), np.float32)
        y = np.random.randint(h)
        x = np.random.randint(w)

        y1 = np.clip(y - self.length // 2, 0, h)
        y2 = np.clip(y + self.length // 2, 0, h)
        x1 = np.clip(x - self.length // 2, 0, w)
        x2 = np.clip(x + self.length // 2, 0, w)

        mask[y1: y2, x1: x2] = 0.
        mask = torch.from_numpy(mask)
        mask = mask.expand_as(img)
        img *= mask
        return img

IMG_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif']

def has_file_allowed_extension(filename, extensions):
    filename_lower = filename.lower()
    return any(filename_lower.endswith(ext) for ext in extensions)

def convert_to_pil(bytes_obj):
    img = Image.open(BytesIO(bytes_obj))
    #img = Image.open(bytes_obj)
    #img = bytes_obj
    return img.convert('RGB')

def _data_transforms_mnist(cutout_size):
    CIFAR_MEAN = [0.49139968]
    CIFAR_STD = [0.24703233]

    train_transform = transforms.Compose([
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize(CIFAR_MEAN, CIFAR_STD),
    ])
    if cutout_size is not None:
        train_transform.transforms.append(Cutout(cutout_size))

    valid_transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize(CIFAR_MEAN, CIFAR_STD),
    ])
    return train_transform, valid_transform

def _data_transforms_cifar10(cutout_size):
    CIFAR_MEAN = [0.49139968, 0.48215827, 0.44653124]
    CIFAR_STD = [0.24703233, 0.24348505, 0.26158768]

    train_transform = transforms.Compose([
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize(CIFAR_MEAN, CIFAR_STD),
    ])
    if cutout_size is not None:
        train_transform.transforms.append(Cutout(cutout_size))

    valid_transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize(CIFAR_MEAN, CIFAR_STD),
    ])
    return train_transform, valid_transform

class ReadImageThread(threading.Thread):
    def __init__(self, root, fnames, class_id, target_list):
        threading.Thread.__init__(self)
        self.root = root
        self.fnames = fnames
        self.class_id = class_id
        self.target_list = target_list

    def run(self):
        for fname in self.fnames:
            if has_file_allowed_extension(fname, IMG_EXTENSIONS):
                path = os.path.join(self.root, fname)
                try:
                    image = Image.open(path)
                except(OSError, NameError):
                    os.remove(path)
                    continue
                image.close()
                if(random.random()<0.1):
                    continue
                with open(path, 'rb') as f:
                    image = f.read()
                item = (image, self.class_id)
                self.target_list.append(item)

class InMemoryDataset(data.Dataset):
    def __init__(self, path, transform=None, num_workers=1):
        super(InMemoryDataset, self).__init__()
        self.path = path
        self.transform = transform
        self.samples = []
        classes, class_to_idx = self.find_classes(self.path)
        dir = os.path.expanduser(self.path)
        for target in sorted(os.listdir(dir)):
            d = os.path.join(dir, target)
            if not os.path.isdir(d):
                continue
            for root, _, fnames in sorted(os.walk(d)):
                if num_workers == 1:
                    for fname in sorted(fnames):
                        if has_file_allowed_extension(fname, IMG_EXTENSIONS):
                            path = os.path.join(root, fname)
                            try:
                               image = Image.open(path)
                            except(OSError, NameError):
                               os.remove(path)
                               continue
                            image.close()
                            with open(path, 'rb') as f:
                               image = f.read()
                            item = (image, class_to_idx[target])
                            self.samples.append(item)
                else:
                    fnames = sorted(fnames)
                    num_files = len(fnames)
                    threads = []
                    res = [[] for i in range(num_workers)]
                    num_per_worker = num_files // num_workers
                    for i in range(num_workers):
                        start_index = num_per_worker * i
                        end_index = num_files if i == num_workers - \
                            1 else num_per_worker * (i+1)
                        thread = ReadImageThread(
                            root, fnames[start_index:end_index], class_to_idx[target], res[i])
                        threads.append(thread)
                    for thread in threads:
                        thread.start()
                    for thread in threads:
                        thread.join()
                    for item in res:
                        self.samples += item
                    del res, threads
                    gc.collect()
    def __len__(self):
        return len(self.samples)

    def __getitem__(self, index):
        sample, target = self.samples[index]
        sample = convert_to_pil(sample)
        if self.transform is not None:
            sample = self.transform(sample)
        return sample, target

    def __repr__(self):
        fmt_str = 'Dataset ' + self.__class__.__name__ + '\n'
        fmt_str += '    Number of datapoints: {}\n'.format(self.__len__())
        fmt_str += '    Root Location: {}\n'.format(self.path)
        tmp = '    Transforms (if any): '
        fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        tmp = '    Target Transforms (if any): '
        fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        return fmt_str

    @staticmethod
    def find_classes(root):
        classes = [d for d in os.listdir(root) if os.path.isdir(os.path.join(root, d))]
        classes.sort()
        class_to_idx = {classes[i]: i for i in range(len(classes))}
        return classes, class_to_idx

class ZipDataset(data.Dataset):
    def __init__(self, path, transform=None):
        super(ZipDataset, self).__init__()
        self.path = os.path.expanduser(path)
        self.transform = transform
        self.samples = []
        with zipfile.ZipFile(self.path, 'r') as reader:
            classes, class_to_idx = self.find_classes(reader)
            fnames = sorted(reader.namelist())
        for fname in fnames:
            if self.is_directory(fname):
                continue
            target = self.get_target(fname)
            item = (fname, class_to_idx[target])
            self.samples.append(item)

    def __len__(self):
        return len(self.samples)

    def __getitem__(self, index):
        sample, target = self.samples[index]
        with zipfile.ZipFile(self.path, 'r') as reader:
            sample = reader.read(sample)
        sample = convert_to_pil(sample)
        if self.transform is not None:
            sample = self.transform(sample)
        return sample, target

    def __repr__(self):
        fmt_str = 'Dataset ' + self.__class__.__name__ + '\n'
        fmt_str += '    Number of datapoints: {}\n'.format(self.__len__())
        fmt_str += '    Root Location: {}\n'.format(self.path)
        tmp = '    Transforms (if any): '
        fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        tmp = '    Target Transforms (if any): '
        fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
        return fmt_str

    @staticmethod
    def is_directory(fname):
        if fname.startswith('n') and fname.endswith('/'):
            return True
        return False

    @staticmethod
    def get_target(fname):
        assert fname.startswith('n')
        return fname.split('/')[0]

    @staticmethod
    def find_classes(reader):
        classes = [ZipDataset.get_target(name) for name in reader.namelist() if ZipDataset.is_directory(name)]
        classes.sort()
        class_to_idx = {classes[i]: i for i in range(len(classes))}
        return classes, class_to_idx

def build_mnist(data_path,
                  cutout_size=16,
                  num_worker=10,
                  train_batch_size=32,
                  eval_batch_size=32,
                  split_train_for_valid: float = None,
                  small_set=1, **kwargs):

    train_transform, valid_transform = _data_transforms_mnist(cutout_size)
    if split_train_for_valid is None:
        train_data = dset.CIFAR10(
            root=data_path, train=True, download=True, transform=train_transform)
        valid_data = dset.CIFAR10(
            root=data_path, train=False, download=True, transform=valid_transform)

        train_queue = torch.utils.data.DataLoader(
            train_data, batch_size=train_batch_size, shuffle=True, pin_memory=True, num_workers=num_worker)
        valid_queue = torch.utils.data.DataLoader(
            valid_data, batch_size=eval_batch_size, shuffle=False, pin_memory=True, num_workers=num_worker)
    else:
        train_data = dset.CIFAR10(
            root=data_path, train=True, download=True, transform=train_transform)
        valid_data = dset.CIFAR10(
            root=data_path, train=True, download=True, transform=valid_transform)
        n = int(len(train_data)*small_set)
        indices = list(range(n))
        split = int(np.floor(split_train_for_valid * n))
        np.random.shuffle(indices)
        train_queue = torch.utils.data.DataLoader(
            train_data, batch_size=train_batch_size,
            sampler=torch.utils.data.sampler.SubsetRandomSampler(
                indices[:split]),
            pin_memory=True, num_workers=num_worker)
        valid_queue = torch.utils.data.DataLoader(
            valid_data, batch_size=eval_batch_size,
            sampler=torch.utils.data.sampler.SubsetRandomSampler(
                indices[split:n]),
            pin_memory=True, num_workers=num_worker)

    return train_queue, valid_queue

def build_cifar10(data_path,
                  cutout_size=16,
                  num_worker=10,
                  train_batch_size=32,
                  eval_batch_size=32,
                  split_train_for_valid: float = None,
                  small_set=1, **kwargs):

    train_transform, valid_transform = _data_transforms_cifar10(cutout_size)
    if split_train_for_valid is None:
        train_data = dset.CIFAR10(
            root=data_path, train=True, download=True, transform=train_transform)
        valid_data = dset.CIFAR10(
            root=data_path, train=False, download=True, transform=valid_transform)

        train_queue = torch.utils.data.DataLoader(
            train_data, batch_size=train_batch_size, shuffle=True, pin_memory=True, num_workers=num_worker)
        valid_queue = torch.utils.data.DataLoader(
            valid_data, batch_size=eval_batch_size, shuffle=False, pin_memory=True, num_workers=num_worker)
    else:
        train_data = dset.CIFAR10(
            root=data_path, train=True, download=True, transform=train_transform)
        valid_data = dset.CIFAR10(
            root=data_path, train=True, download=True, transform=valid_transform)
        n = int(len(train_data)*small_set)
        indices = list(range(n))
        split = int(np.floor(split_train_for_valid * n))
        np.random.shuffle(indices)
        train_queue = torch.utils.data.DataLoader(
            train_data, batch_size=train_batch_size,
            sampler=torch.utils.data.sampler.SubsetRandomSampler(
                indices[:split]),
            pin_memory=True, num_workers=num_worker)
        valid_queue = torch.utils.data.DataLoader(
            valid_data, batch_size=eval_batch_size,
            sampler=torch.utils.data.sampler.SubsetRandomSampler(
                indices[split:n]),
            pin_memory=True, num_workers=num_worker)

    return train_queue, valid_queue

def build_cifar100(data_path,
                   cutout_size=16,
                   num_worker=10,
                   train_batch_size=32,
                   eval_batch_size=32,
                   split_train_for_valid: float = None, **kwargs):

    train_transform, valid_transform = _data_transforms_cifar10(cutout_size)
    if split_train_for_valid is None:
        train_data = dset.CIFAR100(
            root=data_path, train=True, download=True, transform=train_transform)
        valid_data = dset.CIFAR100(
            root=data_path, train=False, download=True, transform=valid_transform)

        train_queue = torch.utils.data.DataLoader(
            train_data, batch_size=train_batch_size, shuffle=True, pin_memory=True, num_workers=num_worker)
        valid_queue = torch.utils.data.DataLoader(
            valid_data, batch_size=eval_batch_size, shuffle=False, pin_memory=True, num_workers=num_worker)
    else:
        train_data = dset.CIFAR100(
            root=data_path, train=True, download=True, transform=train_transform)
        valid_data = dset.CIFAR100(
            root=data_path, train=True, download=True, transform=valid_transform)
        n = len(train_data)
        indices = list(range(n))
        split = int(np.floor(split_train_for_valid * n))
        np.random.shuffle(indices)
        train_queue = torch.utils.data.DataLoader(
            train_data, batch_size=train_batch_size,
            sampler=torch.utils.data.sampler.SubsetRandomSampler(
                indices[:split]),
            pin_memory=True, num_workers=num_worker)
        valid_queue = torch.utils.data.DataLoader(
            valid_data, batch_size=eval_batch_size,
            sampler=torch.utils.data.sampler.SubsetRandomSampler(
                indices[split:n]),
            pin_memory=True, num_workers=num_worker)

    return train_queue, valid_queue

def build_imagenet(data_path,
                   load_num_work=32,
                   feed_num_work=16,
                   zip_file: bool = False,
                   lazy_load: bool = False,
                   train_batch_size: int = 128,
                   valid_batch_size: int = 128):
    normalize = transforms.Normalize(
        mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
    train_transform = transforms.Compose([
        transforms.RandomResizedCrop(224),
        transforms.RandomHorizontalFlip(),
        transforms.ColorJitter(
            brightness=0.4,
            contrast=0.4,
            saturation=0.4,
            hue=0.2),
        transforms.ToTensor(),
        normalize])

    valid_transform = transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        normalize])
    if zip_file:
        logging.debug('Loading data from zip file')
        traindir = os.path.join(data_path, 'train.zip')
        validdir = os.path.join(data_path, 'valid.zip')
        if lazy_load:
            train_data = ZipDataset(traindir, train_transform)
            valid_data = ZipDataset(validdir, valid_transform)
        else:
            logging.debug('Loading data into memory')
            train_data = InMemoryZipDataset(
                traindir, train_transform, num_workers=load_num_work)
            valid_data = InMemoryZipDataset(
                validdir, valid_transform, num_workers=load_num_work)
    else:
        logging.debug('Loading data from directory')
        traindir = os.path.join(data_path, 'ILSVRC2012_img_train/')
        validdir = os.path.join(data_path, 'ILSVRC2012_img_val/')
        if lazy_load:
            train_data = dset.ImageFolder(traindir, train_transform)
            valid_data = dset.ImageFolder(validdir, valid_transform)
        else:
            logging.debug('Loading data into memory')
            train_data = InMemoryDataset(
                traindir, train_transform, num_workers=load_num_work)
            valid_data = InMemoryDataset(
                validdir, valid_transform, num_workers=load_num_work)

    logging.info('[ImageNet] Found %d in training data', len(train_data))
    logging.info('[ImageNet] Found %d in validation data', len(valid_data))

    train_queue = torch.utils.data.DataLoader(
        train_data, batch_size=train_batch_size, shuffle=True, pin_memory=True, num_workers=feed_num_work)
    valid_queue = torch.utils.data.DataLoader(
        valid_data, batch_size=valid_batch_size, shuffle=True, pin_memory=True, num_workers=feed_num_work)
    return train_queue, valid_queue

class auto_data(object):
    @staticmethod
    def generate_data(value_range=(0,15), length_range=(10,20), num_samples=500000, data_parser=lambda x: ' '.join(['.'.join([str(j) for j in i]) for i in x.reshape(-1, 3)])):
        dataset =  list()
        data1 = list()
        data2 = list()
        for i in range(length_range[0], length_range[1], 1):
            data1.extend([i for i in np.random.randint(value_range[0], value_range[1], size=(np.ceil(num_samples/(length_range[1]-length_range[0])).astype('int'), i*3))])
            data2.extend([i for i in np.random.randint(value_range[0], value_range[1], size=(np.ceil(num_samples/(length_range[1]-length_range[0])).astype('int'), i*3))])
        np.random.shuffle(data1)
        np.random.shuffle(data2)
        for d0, d1 in zip(data1, data2):
            dataset.append('{0} <---> {1}\t{0} <---> {1}'.format(data_parser(d0), data_parser(d1)))
        return dataset

    @staticmethod
    def save_data(data, save_path='./Res/PretrainModel/', file_name='trainset.txt'):
        file = open(os.path.join(save_path, file_name), 'a')
        file.write("\n".join(data))
        file.flush()
        file.close()


import sqlite3

#function to create a new table in database
def crt_table():
    connect = sqlite3.connect("students.db")
    cursor = connect.cursor()
    c = ""

    while(c != 'n'):
        n = input("Name for new table: ")
        num = int(input("How many columns do you want in your table: "))
        l1 = []

        for i in range(num):
            l1.append( input(f"Enter column {i+1} name:"))

        s1 = f'CREATE TABLE {n} (\n'

        s2 = ""

        for item in range(len(l1)):
            if item < (len(l1) - 1):
                s2 = s2 +  "\t" + f"{l1[item]}" + ",\n"
            else:
                s2 = s2 + "\t" + f"{l1[item]}" + "\n"

        s3 = ');'

        query = s1 + s2 + s3

        cursor.execute(query)

        connect.commit()
        c = input("Do you want to add another table: ")

    connect.close()

#function to add new records to table in database
def add_data(tb_nm):
    l = get_nm(tb_nm)

    connect = sqlite3.connect("students.db")
    cursor = connect.cursor()

    c = ""

    while(c != 'n'):
        inp = []
        inp2 = []
        for i in range(len(l)):
            inp2.append(input(f"Enter the value for {l[i][0]} column:"))

        tu = tuple(inp2)
        inp.append(tu)

        #creating a query string
        s1 = f"INSERT INTO {tb_nm} VALUES ("
        s2 = ""

        for i in range(len(l)):
            if i < (len(l) - 1):
                s2 = s2 + "?" + ", "
            else:
                s2 = s2 + "?);"

        query = s1 + s2

        #executing query
        cursor.executemany(query,inp)
        connect.commit()
        print("\n")
        c = input("Do you want to add another record? : ")

    connect.close()

#function to get column names of specified table
def get_nm(tb_nm):
    connect = sqlite3.connect("students.db")
    cursor = connect.cursor()

    query = f"SELECT name FROM pragma_table_info('{tb_nm}')"

    cursor.execute(query)

    l = cursor.fetchall()

    connect.commit()
    connect.close()
    return l

#this function displays the whole table i.e. specified
def show_tb():
    connect = sqlite3.connect("students.db")
    cursor = connect.cursor()
    c = ""

    while(c != "n"):
        print("\n\n")
        tb_nm = input("Enter the name of the table you want to see: ")
        l = get_nm(tb_nm)

        cursor.execute(f"SELECT * FROM {tb_nm}")
        l1 = cursor.fetchall()

        print(f"\nHere is your {tb_nm} table:")

        s1 = ""

        for i in range(len(l)):
            s1 = s1 + l[i][0] + "\t\t"

        s2 = ""

        for i in l1:
            for j in range(len(i)):
                s2 = s2 + i[j] + "\t\t"
            s2 = s2 + "\n"

        print(s1)
        print(s2)
        connect.commit()

        c = input("Do you want to see another table: ")

    connect.close()

#function to query a database
def query_tb():
    connect = sqlite3.connect("students.db")
    cursor = connect.cursor()
    c = ""

    while(c != "n"):
        q = input("Enter the query you want to execute: ")

        cursor.execute(q)

        items = cursor.fetchall()

        if items:
            print(items)

        connect.commit()
        c = input("Do you want to execute another query: ")

    connect.close()

#function to delete a whole table
def del_tb():
    connect = sqlite3.connect("students.db")
    cursor = connect.cursor()
    c = ""

    while(c !='n'):
        tb_nm = input("Name of the table you want to delete: ")

        cursor.execute(f"DROP TABLE {tb_nm}")

        connect.commit()
        c = input("Do you want to delete another table?")

    connect.close()

if __name__ == "__main__":
    crt_table("nothing", [1,2,3,4,5])


from typing import List

class Solution:
    def canVisitAllRooms(self, rooms: List[List[int]]) -> bool:
        n = len(rooms)

        # Guarda quais salas já visitei
        visited = [False] * n
        visited[0] = True  # Começa pela primeira sala (0)

        queue = [0]  # Fila pra guardar quais salas ainda precisa visitar

        while queue:
            # Remove a sala da fila
            room = queue.pop(0)

            for key in rooms[room]:
                # Se ainda não tiver visitado a sala, atribui true para o valor de visited
                if not visited[key]:
                    visited[key] = True
                    queue.append(key)

        return all(visited)


#!/usr/bin/env python

import datetime
import json
import logging
import os
import pdb
from sqlalchemy import and_
import time
import uuid

from vib.vwf.VWorker import VWorker

import vib.config.AppConfig
config = vib.config.AppConfig.AppConfig( 'viblio' ).config()

import vib.utils.s3 as s3
import vib.db.orm
from vib.db.models import *

import vib.vwf.Transcode.transcode_utils as tutils

log = logging.getLogger( __name__ )

class Transcode( VWorker ):
    # This line controls how we interact with SWF, and changes here
    # must be made in coordination with VPWorkflow.py
    task_name = 'Transcode'

    def run_task( self, options ):
        '''Transcode a video.  Input options are:
        { media_uuid, user_uuid, input_file : { s3_bucket, s3_key },
          metadata_uri,
          original_uuid,
          outputs : [ { output_file : { s3_bucket, s3_key},
                      format : "mp4",
                      max_video_bitrate: 1500,
                      audio_bitrate : 160,
                      scale: "640:-1", # Passed to ffmpeg -vf scale argument, overrides size
                      size: "640x360", # Currently ignored
                      asset_type: "main",
                      thumbnails : [ {
                        times : [0.5], size: "320x240", label: "poster",
                        format : "png",
                        output_file: { s3_bucket, s3_key } } ]
                     } ] }
                     # If multiple output_files are present, only one need have thumbnails.
        '''
        try:
            media_uuid = options['media_uuid']
            user_uuid = options['user_uuid']
            input_file = options['input_file']
            outputs = options['outputs']

            log.info( json.dumps( {
                        'media_uuid' : media_uuid,
                        'user_uuid' : user_uuid,
                        'message' : "Starting transcode for user %s media %s" % ( user_uuid, media_uuid )
                    } ) )

            # Download to our local disk.
            original_file = config.transcode_dir + "/"  + media_uuid
            log.debug( json.dumps( {
                        'media_uuid' : media_uuid,
                        'user_uuid' : user_uuid,
                        'message' : "Downloading original file from s3 for user %s media %s" % ( user_uuid, media_uuid )
                    } ) )

            s3.download_file( original_file, input_file['s3_bucket'], input_file['s3_key'] )

            # There seems to be a race condition where somehow the OS
            # doesn't present the file to the next command sometimes -
            # give things a few seconds to catch up.
            time.sleep( 5 )

            log.debug( json.dumps( {
                        'media_uuid' : media_uuid,
                        'user_uuid' : user_uuid,
                        'message' : "Generating exif for original file for user %s media %s" % ( user_uuid, media_uuid )
                    } ) )
            exif = tutils.get_exif( media_uuid, original_file, user_uuid )

            try:
                log.debug( json.dumps( {
                            'media_uuid' : media_uuid,
                            'user_uuid' : user_uuid,
                            'message' : "Running qt-faststart for user %s media %s" % ( user_uuid, media_uuid )
                    } ) )
                tutils.move_atom( media_uuid, original_file, user_uuid )
            except Exception as e:
                log.warning( json.dumps( {
                            'media_uuid' : media_uuid,
                            'user_uuid' : user_uuid,
                            'message' : "Exception thrown while running qt-faststart for user %s media %s : %s" % ( user_uuid, media_uuid, e ) } ) )
                # Explicitly do nothing upon error in this case.

            # Process files into S3
                log.debug( json.dumps( { 'media_uuid' : media_uuid,
                                         'user_uuid' : user_uuid,
                                         'message' : "Transcoding and storing the result in S3 for user %s media %s : " % ( user_uuid, media_uuid ) } ) )

            outputs = tutils.transcode_and_store( media_uuid, original_file, outputs, exif, options.get( 'try_photos', False ), user_uuid )

            orm = vib.db.orm.get_session()
            orm.commit()

            # Get the media object to which all our subordinate files relate.
            media = orm.query( Media ).filter( Media.uuid == media_uuid ).first()
            if media is None:
                log.error( json.dumps( { 'media_uuid' : media_uuid,
                                         'user_uuid' : user_uuid,
                                         'message' : 'Failed to locate database record for media %s, perhaps it has been deleted?' % ( media_uuid ) } ) )
                # Terminate execution permanently, if there is no
                # database record for this we can't possibly proceed.
                return { 'ACTIVITY_ERROR' : True, 'retry' : False }

            if options.get( 'viblio_added_content_type', '' ) != '':
                # This is a viblio generated video, set it's location to Palo Alto
                media.lat = 37.442174
                media.lng = -122.143199
            else:
                media.lat = exif['lat']
                media.lng = exif['lng']

            media.status = 'visible'

            mwfs = MediaWorkflowStages( workflow_stage = self.task_name + 'Complete' )
            media.media_workflow_stages.append( mwfs )

            # Calculate the recording date for video versions.
            if options.get( 'viblio_added_content_type', '' ) != '':
                # This is a viblio generated video, the recording date should be now.
                recording_date = datetime.datetime.now()
            else:
                recording_date = datetime.datetime.utcfromtimestamp( 0 )

            if exif['create_date'] and exif['create_date'] != '' and exif['create_date'] != '0000:00:00 00:00:00':
                try:
                    recording_date = datetime.datetime.strptime( exif['create_date'], '%Y:%m:%d %H:%M:%S' )
                except Exception as e:
                    recording_date = exif['create_date']
            log.debug( 'Setting recording date to ' + str( recording_date ) )
            log.debug( 'Exif data for create was ' + exif['create_date'] )
            media.recording_date = recording_date

            original = orm.query( MediaAssets ).filter( MediaAssets.uuid == options['original_uuid'] ).first()
            if original is None:
                log.error( json.dumps( {
                            'media_uuid' : media_uuid,
                            'user_uuid' : user_uuid,
                            'message' : 'Failed to locate database record for original media asset %s, perhaps it has been deleted?' % ( media_uuid ) } ) )
                # Terminate execution permanently, if there is no
                # database record for this we can't possibly proceed.
                return { 'ACTIVITY_ERROR' : True, 'retry' : False }

            original.mimetype = 'video/%s' % exif.get( 'format', 'mp4' )
            original.width = exif.get( 'width', None )
            original.height = exif.get( 'height', None )

            # We will return to the next stage the key/bucket of the
            # "main" asset_type.
            return_bucket = None
            return_key = None

            for output in outputs:
                output_uuid = str( uuid.uuid4() )

                output_exif = tutils.get_exif( media_uuid, output['output_file_fs'], user_uuid )

                log.info( json.dumps( {
                            'media_uuid' : media_uuid,
                            'user_uuid' : user_uuid,
                            'asset_type' : output['asset_type'],
                            'output_uuid' : output_uuid,
                            'message' : "Creating video database row of uuid %s for user %s, media %s of asset_type %s and uri %s" % ( output_uuid, user_uuid, media_uuid, output['asset_type'], output['output_file']['s3_key'] )
                    } ) )

                if output['asset_type'] == 'main':
                    return_bucket = output['output_file']['s3_bucket']
                    return_key = output['output_file']['s3_key']

                video_asset = MediaAssets(
                    uuid         = output_uuid,
                    asset_type   = output['asset_type'],
                    mimetype     = 'video/%s' % exif.get( 'format', 'mp4' ),
                    metadata_uri = options['metadata_uri'],
                    bytes        = os.path.getsize( output['output_file_fs'] ),
                    uri          = output['output_file']['s3_key'],
                    location     = 'us',
                    view_count   = 0,
                    width        = output_exif.get( 'width', None ),
                    height       = output_exif.get( 'height', None ),
                    duration     = output_exif['duration'] )
                media.assets.append( video_asset )

                if output['asset_type'] == 'main' and recording_date != datetime.datetime.utcfromtimestamp( 0 ):
                    try:
                        # Add a Month Year tag to the main media asset.
                        date_string = recording_date.strftime( "%B %Y" )
                        date_tag = MediaAssetFeatures(
                            feature_type = 'activity',
                            coordinates = date_string
                        )
                        video_asset.media_asset_features.append( date_tag )
                    except Exception as e:
                        log.warning( json.dumps( { 'media_uuid' : media_uuid,
                                                   'user_uuid' : user_uuid,
                                                   'message' : 'Failed to add date tag for video, error was: %s' % ( e ) } ) )

                video_poster = None

                for thumbnail in output.get( 'thumbnails', [] ):
                    thumbnail_uuid = str( uuid.uuid4() )

                    log.info( json.dumps( {
                                'media_uuid' : media_uuid,
                                'user_uuid' : user_uuid,
                                'asset_type' : thumbnail['label'],
                                'output_uuid' : thumbnail_uuid,
                                'message' : "Creating image database row of uuid %s for user %s, media %s of asset_type %s and uri %s" % ( thumbnail_uuid, user_uuid, media_uuid, thumbnail['label'], thumbnail['output_file']['s3_key'] )
                                } ) )

                    thumbnail_size = thumbnail.get( 'size', "320x240" )
                    thumbnail_x, thumbnail_y = thumbnail_size.split( 'x' )
                    thumbnail_asset = MediaAssets( uuid       = thumbnail_uuid,
                                                   asset_type = thumbnail['label'],
                                                   mimetype   = 'image/%s' % thumbnail.get( 'format', 'png' ),
                                                   bytes      = os.path.getsize( thumbnail['output_file_fs'] ),
                                                   width      = int( thumbnail_x ),
                                                   height     = int( thumbnail_y ),
                                                   uri        = thumbnail['output_file']['s3_key'],
                                                   location   = 'us',
                                                   view_count = 0 )
                    media.assets.append( thumbnail_asset )

                    if thumbnail['label'] == 'poster':
                        video_poster = thumbnail_asset

            # Determine if we need to create and or add to the special Video summary album.
            if options.get( 'viblio_added_content_type', '' ) == config.viblio_summary_video_type:
                viblio_summary_album = orm.query( Media ).filter( and_( Media.user_id == media.user_id, Media.is_viblio_created == True, Media.title == config.viblio_summary_album_name ) )[:]

                if len( viblio_summary_album ) == 0:
                    viblio_summary_album = Media( user_id = media.user_id,
                                                  uuid = str( uuid.uuid4() ),
                                                  media_type = 'original',
                                                  is_album = True,
                                                  title = config.viblio_summary_album_name,
                                                  is_viblio_created = True )
                    orm.add( viblio_summary_album )

                    media_album_row = MediaAlbums()
                    orm.add( media_album_row )
                    media.media_albums_media.append( media_album_row )
                    viblio_summary_album.media_albums.append( media_album_row )

                    album_poster = MediaAssets( user_id = media.user_id,
                                                uuid = str( uuid.uuid4() ),
                                                asset_type = 'poster',
                                                mimetype = video_poster.mimetype,
                                                location = video_poster.location,
                                                uri = video_poster.uri,
                                                width = video_poster.width,
                                                height = video_poster.height )
                    viblio_summary_album.assets.append( album_poster )

                elif len( viblio_summary_album ) == 1:
                    media_album_row = MediaAlbums()
                    orm.add( media_album_row )
                    media.media_albums_media.append( media_album_row )
                    viblio_summary_album[0].media_albums.append( media_album_row )
                else:
                    raise Exception( "ERROR: Found multiple %s albums for user: %s " % ( config.viblio_summary_album_name, media.user_id ) )

            # Determine if we need to create and or add to the special Try Photo Finder album.
            if options.get( 'try_photos', False ):
                try_photos_album = orm.query( Media ).filter( and_( Media.user_id == media.user_id, Media.is_viblio_created == True, Media.title == config.try_photos_album_name ) )[:]

                if len( try_photos_album ) == 0:
                    try_photos_album = Media( user_id = media.user_id,
                                              uuid = str( uuid.uuid4() ),
                                              media_type = 'original',
                                              is_album = True,
                                              title = config.try_photos_album_name,
                                              is_viblio_created = True )
                    orm.add( try_photos_album )

                    media_album_row = MediaAlbums()
                    orm.add( media_album_row )
                    media.media_albums_media.append( media_album_row )
                    try_photos_album.media_albums.append( media_album_row )

                    album_poster = MediaAssets( user_id = media.user_id,
                                                uuid = str( uuid.uuid4() ),
                                                asset_type = 'poster',
                                                mimetype = video_poster.mimetype,
                                                location = video_poster.location,
                                                uri = video_poster.uri,
                                                width = video_poster.width,
                                                height = video_poster.height )
                    try_photos_album.assets.append( album_poster )

                elif len( try_photos_album ) == 1:
                    media_album_row = MediaAlbums()
                    orm.add( media_album_row )
                    media.media_albums_media.append( media_album_row )
                    try_photos_album[0].media_albums.append( media_album_row )
                else:
                    raise Exception( "ERROR: Found multiple %s albums for user: %s " % ( config.try_photos_album_name, media.user_id ) )

            log.info( json.dumps( { 'media_uuid' : media_uuid,
                                    'user_uuid' : user_uuid,
                                    'message' : "Committing rows to database for user %s, media %s" % ( user_uuid, media_uuid ) } ) )
            orm.commit()

            self.cleanup_files( media_uuid, user_uuid, original_file, outputs )

            return_value = {
                'media_uuid' : media_uuid,
                'user_uuid' : user_uuid,
                'output_file' : {
                    's3_bucket' : return_bucket,
                    's3_key' : return_key
                    }
                }

            return return_value

        except Exception as e:
            log.error( json.dumps( {
                        'media_uuid' : media_uuid,
                        'user_uuid' : user_uuid,
                        'message' : "Exception while transcoding video for user %s, media %s, error was: %s" % ( user_uuid, media_uuid, e )
                        } ) )

            self.cleanup_files( media_uuid, user_uuid, original_file, outputs )
            raise

    def cleanup_files( self, media_uuid, user_uuid, original_file, outputs ):
        try:
            # Delete the original file
            if os.path.exists( original_file ):
                log.debug( json.dumps( {
                            'media_uuid' : media_uuid,
                            'user_uuid' : user_uuid,
                            'message' : "Deleting temporary file %s for user %s, media %s" % ( original_file, user_uuid, media_uuid )
                            } ) )
                os.remove( original_file )

                exif_file = os.path.splitext( original_file )[0] + '_exif.json'

                if os.path.exists( exif_file ):
                    log.debug( json.dumps( {
                                'media_uuid' : media_uuid,
                                'user_uuid' : user_uuid,
                                'message' : "Deleting temporary file %s for user %s, media %s" % ( exif_file, user_uuid, media_uuid )
                                } ) )
                    os.remove( exif_file )

            # Delete file files created and stored in the outputs data
            # structure.
            for output in outputs:
                if 'output_file_fs' in output:
                    if os.path.exists( output['output_file_fs'] ):
                        log.debug( json.dumps( {
                                    'media_uuid' : media_uuid,
                                    'user_uuid' : user_uuid,
                                    'message' : "Deleting temporary file %s for user %s, media %s" % ( output['output_file_fs'], user_uuid, media_uuid )
                                } ) )
                        os.remove( output['output_file_fs'] )

                        exif_file = os.path.splitext( output['output_file_fs'] )[0] + '_exif.json'

                        if os.path.exists( exif_file ):
                            log.debug( json.dumps( {
                                        'media_uuid' : media_uuid,
                                        'user_uuid' : user_uuid,
                                        'message' : "Deleting temporary file %s for user %s, media %s" % ( output['output_file_fs'], user_uuid, media_uuid )
                                        } ) )
                            os.remove( exif_file )

                for thumbnail in output.get( 'thumbnails', [] ):
                    if 'output_file_fs' in thumbnail:
                        if os.path.exists( thumbnail['output_file_fs'] ):
                            log.debug( json.dumps( {
                                        'media_uuid' : media_uuid,
                                        'user_uuid' : user_uuid,
                                        'message' : "Deleting temporary file %s for user %s, media %s" % ( thumbnail['output_file_fs'], user_uuid, media_uuid )
                                        } ) )
                            os.remove( thumbnail['output_file_fs'] )

        except Exception as e:
            log.error( json.dumps( {
                        'media_uuid' : media_uuid,
                        'user_uuid' : user_uuid,
                        'message' : "Exception while cleaning up temporary files for user %s, media %s, error was: %s" % ( user_uuid, media_uuid, e )
                        } ) )
            raise


import json
from pymongo import MongoClient

filename = "mongo-seed/employees.json"
# filename = "mongo-seed/employees_short.json"

with open(filename, "r") as f:
    data = json.load(f)

client = MongoClient("localhost", 27017)
db = client.employees
employees_collection = db.employees_collection
employees_id = employees_collection.insert_many(data).inserted_ids
print("Вставлено {} записей в коллекцию".format(len(employees_id)))