import constants
from seq2seq import Seq2Seq
from preprocessing import Vocabulary
from logger import log, save_dataframe, read_dataframe

import os

import numpy as np
import pandas as pd

import torch

class Environment:
    def __init__(self):
        self.train_methods = None
        self.valid_methods = None
        self.test_methods = None
        self.input_vocab = Vocabulary()
        self.output_vocab = Vocabulary()
        self.model = None
        self.history = None
        self.iters_completed = None
        self.total_training_time = None

    def save(self):
        log('Saving the state of the environment')
        self.__save_methods()
        self.__save_vocabs()
        self.__save_model()
        self.__save_history()
        self.__save_iters_completed()
        self.__save_total_training_time()

    def save_data(self):
        log('Saving dataset splits and vocabularies')
        self.__save_methods()
        self.__save_vocabs()

    def save_train(self):
        log('Saving the state of training')
        self.__save_model()
        self.__save_history()
        self.__save_iters_completed()
        self.__save_total_training_time()

    def load(self):
        log('Loading the last saved environment')
        self.__load_methods()
        self.__load_vocabs()
        self.__load_model()
        self.__load_history()
        self.__load_iters_completed()
        self.__load_total_training_time()

        log("Starting from iteration {}\n{} more iterations to go ({:.1f}%)".format(
            self.iters_completed + 1,
            constants.NUM_ITER - self.iters_completed,
            (constants.NUM_ITER - self.iters_completed) / constants.NUM_ITER * 100))

    def initialize_new(self):
        log('Initializing a new environment')
        self.__init_methods()
        self.__init_vocabs()
        self.__init_model()
        self.__init_history()
        self.__init_iters_completed()
        self.__init_total_training_time()

    def __save_methods(self):
        log('Saving train, validation, and test methods')
        join = lambda tokens: ' '.join(tokens)

        save_dataframe(self.train_methods.applymap(join), constants.TRAIN_METHODS_FILE)
        save_dataframe(self.valid_methods.applymap(join), constants.VALID_METHODS_FILE)
        save_dataframe(self.test_methods.applymap(join), constants.TEST_METHODS_FILE)

    def __save_vocabs(self):
        log('Saving vocabularies')
        self.input_vocab.save(constants.INPUT_VOCAB_FILE)
        self.output_vocab.save(constants.OUTPUT_VOCAB_FILE)

    def __save_model(self):
        log('Saving the model')
        torch.save(self.model.state_dict(), constants.TRAINED_MODEL_FILE)

    def __save_history(self):
        log('Saving score history')
        save_dataframe(self.history, constants.HISTORIES_FILE)

    def __save_iters_completed(self):
        log('Saving the number of completed iterations')
        with open(constants.ITERS_COMPLETED_FILE, 'w') as f:
            f.write(str(self.iters_completed))

    def __save_total_training_time(self):
        log('Saving the total training time')
        with open(constants.TRAINING_TIME_FILE, 'w') as f:
            f.write(str(self.total_training_time))

    def __load_methods(self):
        log('Loading train, validation, and test methods')
        self.train_methods = read_dataframe(constants.TRAIN_METHODS_FILE)
        self.valid_methods = read_dataframe(constants.VALID_METHODS_FILE)
        self.test_methods = read_dataframe(constants.TEST_METHODS_FILE)

        self.__log_split()
        self.__split_tokens()

    def __load_vocabs(self):
        log('Loading vocabularies')
        self.input_vocab.load(constants.INPUT_VOCAB_FILE)
        self.output_vocab.load(constants.OUTPUT_VOCAB_FILE)
        self.__log_vocabs()

    def __load_model(self):
        self.__init_model()

        log('Loading the last trained model')
        self.model.load_state_dict(torch.load(constants.TRAINED_MODEL_FILE))

    def __load_history(self):
        log('Loading score history')
        self.history = read_dataframe(constants.HISTORIES_FILE)

    def __load_iters_completed(self):
        log('Loading the number of completed iterations')
        with open(constants.ITERS_COMPLETED_FILE, 'r') as f:
            self.iters_completed = int(f.read())

    def __load_total_training_time(self):
        log('Loading the total training time')
        with open(constants.TRAINING_TIME_FILE, 'r') as f:
            self.total_training_time = float(f.read())

    def __init_methods(self):
        self.__load_dataset()
        self.__split_tokens()
        self.__filter_dataset()
        self.__split_dataset()

        self.__log_split()

    def __init_vocabs(self):
        log('Collecting the tokens from train set into the vocabularies')
        self.input_vocab.collectWordsFrom(self.train_methods['source'])
        self.output_vocab.collectWordsFrom(self.train_methods['name'])
        self.__log_vocabs()

    def __init_model(self):
        log('Initializing an empty model')
        self.model = Seq2Seq(
            input_size=len(self.input_vocab),
            output_size=len(self.output_vocab),
            hidden_size=constants.HIDDEN_SIZE,
            learning_rate=constants.LEARNING_RATE,
            teacher_forcing_ratio=constants.TEACHER_FORCING_RATIO,
            device=constants.DEVICE)

        log(str(self.model))

    def __init_history(self):
        log('Initializing an empty DataFrame for score history')
        self.history = pd.DataFrame(
            columns=['Loss', 'BLEU', 'ROUGE', 'F1', 'num_names'])

    def __init_iters_completed(self):
        log('Setting the number of completed iterations equal to 0')
        self.iters_completed = 0

    def __init_total_training_time(self):
        log('Setting the total training time equal to 0')
        self.total_training_time = 0

    def __load_dataset(self):
        log('Loading the dataset')
        self.__methods = read_dataframe(constants.DATASET_FILE)
        self.__methods = self.__methods[['name', 'source']]

    def __split_dataset(self):
        log('Splitting the dataset into train, validation, and test sets')

        if constants.TRAIN_PROP + constants.VALID_PROP + constants.TEST_PROP != 1.0:
            raise ValueError("Train, validation, and test proportions don't sum up to 1.")

        test_size = int(constants.TEST_PROP * len(self.__methods))
        valid_size = int(constants.VALID_PROP * len(self.__methods))
        train_size = int(constants.TRAIN_PROP * len(self.__methods))

        indices = np.random.permutation(len(self.__methods))

        train_idx = indices[:train_size]
        valid_idx = indices[train_size:(train_size + valid_size)]
        test_idx = indices[-test_size:]

        self.train_methods = self.__methods.iloc[train_idx]
        self.valid_methods = self.__methods.iloc[valid_idx]
        self.test_methods = self.__methods.iloc[test_idx]

    def __filter_dataset(self):
        # TODO: Move it elsewhere
        log('Filtering data from dataset')
        initial_size = len(self.__methods)

        log('Removing empty methods')
        self.__methods = self.__methods[self.__methods['source'].apply(
            lambda code: len(code) > 0)]

        log('{} methods left ({:.2f}%)'.format(
            len(self.__methods), len(self.__methods) / initial_size * 100))

        log('Removing methods with empty names')
        self.__methods = self.__methods[self.__methods['name'].apply(
            lambda name: len(name) > 0)]

        log('{} methods left ({:.2f}%)'.format(
            len(self.__methods), len(self.__methods) / initial_size * 100))

        log('Removing getters')
        self.__methods = self.__methods[self.__methods['name'] != self.__methods['source']]

        log('{} methods left ({:.2f}%)'.format(
            len(self.__methods), len(self.__methods) / initial_size * 100))

        log('Removing test methods')
        self.__methods = self.__methods[self.__methods['name'].apply(lambda name: name[0] != 'test')]

        log('{} methods left ({:.2f}%)'.format(
            len(self.__methods), len(self.__methods) / initial_size * 100))

        log('Removing abstract methods')
        self.__methods = self.__methods[self.__methods['source'].apply(lambda source: source != ['self', 'subclass', 'responsibility'])]

        log('{} methods left ({:.2f}%)'.format(
            len(self.__methods), len(self.__methods) / initial_size * 100))

        log('Removing shouldNotImplement methods')
        self.__methods = self.__methods[self.__methods['source'].apply(lambda source: source != ['self', 'should', 'not', 'implement'])]

        log('{} methods left ({:.2f}%)'.format(
            len(self.__methods), len(self.__methods) / initial_size * 100))

        log('Removing keywords')
        self.__methods = self.__methods[self.__methods['source'].apply(lambda source: not set(source).issubset(['self', 'should', 'not', 'implement']))]

        log('{} methods left ({:.2f}%)'.format(
            len(self.__methods), len(self.__methods) / initial_size * 100))

        if constants.DROP_DUPLICATES:
            log('Removing duplicate methods')
            self.__methods = self.__methods.loc[self.__methods.astype(str).drop_duplicates().index]

            log('{} methods left ({:.2f}%)'.format(
                len(self.__methods), len(self.__methods) / initial_size * 100))

        log('Removing methods that exceed max size')
        self.__methods = self.__methods[self.__methods['source'].apply(
            lambda code: len(code) <= constants.MAX_LENGTH)]

        log('{} methods left ({:.2f}%)'.format(
            len(self.__methods), len(self.__methods) / initial_size * 100))

    def __join_tokens(self):
        log('Joining tokens into strings')
        join = lambda tokens: ' '.join(tokens)

        self.train_methods = self.train_methods.applymap(join)
        self.valid_methods = self.valid_methods.applymap(join)
        self.test_methods = self.test_methods.applymap(join)

    def __split_tokens(self):
        log('Splitting strings into tokens')
        split = lambda sentence: str(sentence).split()

        # self.train_methods = self.train_methods.applymap(split)
        # self.valid_methods = self.valid_methods.applymap(split)
        # self.test_methods = self.test_methods.applymap(split)

        self.__methods = self.__methods.applymap(split)

    def __log_split(self):
        log('Train size: {}\n'
            'Validation size: {}\n'
            'Test size: {}'.format(
                len(self.train_methods),
                len(self.valid_methods),
                len(self.test_methods)))

    def __log_vocabs(self):
        log('Number of unique input tokens: {}\n'
            'Number of unique output tokens: {}'.format(
                len(self.input_vocab),
                len(self.output_vocab)))


# -*- coding: utf-8 -*-
"""
 Created on Mon Nov 19 20:38:00 2018
Modified on Fri Dec 7   1:41:06 2018

@author: Jason
@modifier: Jeff
"""
import torch
import torch.utils.data
import numpy as np
import glob
import cv2
import pandas as pd

class GANDataset(torch.utils.data.Dataset):
    """
    __len__ will return the # of data
    """
    def __init__(self, data_x, data_y, from_file = False):
        if from_file:
            self.data = np.load(data_x)
            self.label = np.load(data_y)
        else:
            self.data = data_x
            self.label = data_y
    def __getitem__(self, index):
        x = torch.tensor(self.data[index])
        feature1 = torch.tensor(self.label[index][0])
        feature2 = torch.tensor(self.label[index][1])
        return [x,feature1,feature2]
    def __len__(self):
        return len(self.data)

def GetDataset():
    # put the img path into glob to get all the img name
    ### *** Jeff modified *** ###
    imgname = glob.glob(r'../hw3-1/data/extra_data/images/*.jpg')

    img_list = []
    for item in imgname:
        im = cv2.imread(item)
        #print(type(im))
        #### changed ####
        #im = cv2.cvtColor(im, cv2.COLOR_BGR2Lab)
        if type(im)!=type(None):
            # im is of shape (64, 64, 3)
            # np.flip(...) added by Sun
            im = np.flip(im, axis=2) # BGR to RGB
            img_list.append(np.transpose(im, (2,0,1))/255.0) # to shape (3, 64, 64)

    ### *** Jeff modified *** ###
    tag = np.array(pd.read_csv('../hw3-1/data/extra_data/tags.csv'))
    tag_head = np.array(['aqua hair aqua eyes'])
    tags = np.insert(tag[:,1], 0, values=tag_head, axis=0)

    # get two feature of img: eye color and hair color
    hair_set = set()
    eye_set = set()
    for it in tags:
        tmp = it.split(' ')
        hair_set.add(tmp[0])
        eye_set.add(tmp[2])
    hair_dict = {}
    eye_dict = {}
    i = 0
    for it in hair_set:
        hair_dict[it] = i
        i = i + 1
    i = 0
    for it in eye_set:
        eye_dict[it] = i
        i = i + 1

    # img_list:
    # shape:36740*(64, 64, 3), type:np.ndarray
    # feature:
    # shape: (36740,2), type:np.ndarray
    feature = []
    for it in tags:
        tmp = it.split(' ')
        hair = np.zeros(len(hair_dict))
        hair[hair_dict[tmp[0]]] = 1
        eye = np.zeros(len(eye_dict))
        eye[eye_dict[tmp[2]]] = 1
        feature.append([hair,eye])
    '''
    extra_imgname1 = glob.glob(r'./data/faces_data1/*.jpg')
    hair = np.zeros(len(hair_dict))
    eye = np.zeros(len(eye_dict))
    for item in extra_imgname1:
        im = cv2.imread(item)
        im = cv2.resize(im, (64,64), interpolation=cv2.INTER_CUBIC)
        img_list.append(np.transpose(im , (2,0,1))/255.0)
        feature.append([hair,eye])
    '''
    """
    hair = np.zeros(len(hair_dict))
    eye = np.zeros(len(eye_dict))
    extra_imgname2 = glob.glob(r'./data/faces/*.jpg')
    for item in extra_imgname2:
        im = cv2.imread(item)
        #### changed ####
        #im = cv2.cvtColor(im, cv2.COLOR_BGR2Lab)
        im = cv2.resize(im, (64,64), interpolation=cv2.INTER_CUBIC)
        img_list.append(np.transpose(im , (2,0,1))/255.0)
        feature.append([hair,eye])
    """
    feature = np.array(feature)
    # print(feature[0],len(feature),type(feature),feature.shape)
    img_list = np.array(img_list)
    # print(len(img_list),type(img_list),img_list.shape)

    dataset = GANDataset(img_list, feature)
    return dataset
    '''
    dataloader = torch.utils.data.DataLoader(dataset, batch_size = 4)

    for i, (x,f1,f2) in enumerate(dataloader):
        print(i)
        print(type(x),x.shape)
        print(type(f1),f1.shape)
        print(type(f2),f2.shape)
    #     if(i>2):
    #         break
    '''

    # in each iteration, dataloader will return:
    # x: batch of images
    # <class 'torch.Tensor'> torch.Size([batch size, 3, 64, 64])
    # f1: hair color
    # <class 'torch.Tensor'> torch.Size([batch size, 12])
    # f2: eye colar
    # <class 'torch.Tensor'> torch.Size([batch size, 10])


# -*- coding: utf-8 -*-

import time
from lxml import etree
from odoo import api, fields, models, _
from odoo.tools import DEFAULT_SERVER_DATETIME_FORMAT
import logging
_logger = logging.getLogger(__name__)

class StockMove(models.Model):
	_inherit = 'stock.move'

	@api.multi
	def action_set_draft(self):
		""" Cancels the moves and if all moves are cancelled it cancels the picking. """
		# TDE DUMB: why is cancel_procuremetn in ctx we do quite nothing ?? like not updating the move ??
		_logger.warn('none> StockMove.action_set_draft()')
		if any(move.state == 'done' for move in self):
			raise UserError(_('You cannot set to Draft a stock move that has been set to \'Done\'.'))

		procurements = self.env['procurement.order']
		_logger.warn('info> self: %s;',self)
		for move in self:
			_logger.warn('info> move_id: %s; state: %s; reserved_quant_ids: %s; move_dest_id: %s; procurement_id: %s; propagate: %s;',move.id, move.state, move.reserved_quant_ids, move.move_dest_id, move.procurement_id, move.propagate)
			if move.reserved_quant_ids:
				move.quants_unreserve()
			if self.env.context.get('cancel_procurement'):
				if move.propagate:
					pass
					# procurements.search([('move_dest_id', '=', move.id)]).cancel()
			else:
				if move.move_dest_id:
					if move.propagate:
						move.move_dest_id.action_set_draft()
					elif move.move_dest_id.state == 'waiting':
						# If waiting, the chain will be broken and we are not sure if we can still wait for it (=> could take from stock instead)
						move.move_dest_id.write({'state': 'confirmed'})
				if move.procurement_id:
					procurements |= move.procurement_id

		self.write({'state': 'draft', 'move_dest_id': False})
		if procurements:
			procurements.check()
		return True

class StockBackorderConfirmation(models.TransientModel):
	_inherit = 'stock.backorder.confirmation'

	@api.multi
	def process(self):
		_logger.warn('none> StockBackorderConfirmation.process()')
		super(StockBackorderConfirmation, self).process()
		backorder_pick = self.env['stock.picking'].search([('backorder_id', '=', self.pick_id.id)])
		backorder_pick.action_set_draft()

class StockPicking(models.Model):
	_inherit = 'stock.picking'

	is_return_picking = fields.Boolean('Is a Return Picking', default=False, readonly=True)

	@api.multi
	def action_set_draft(self):
		_logger.warn('none> StockPicking.action_set_draft()')
		self.mapped('move_lines').action_set_draft()
		return True

	@api.multi
	def action_confirm(self):
		_logger.warn('none> StockPicking.action_confirm()')
		for picking in self:
			_logger.warn('info> picking: %s; name: %s',picking.id, picking.name)
			if picking.backorder_id and picking.name[0]=='#':
				name = picking.picking_type_id.sequence_id.next_by_id()
				_logger.warn('info> new name: %s;',name)
				picking.write({'name':name})

		return super(StockPicking, self).action_confirm()

	@api.multi
	def do_new_transfer(self):
		_logger.warn('none> StockPicking.do_new_transfer()')
		for picking in self:
			_logger.warn('info> picking: %s; name: %s',picking.id, picking.name)
			if picking.backorder_id and picking.name[0]=='#':
				name = picking.picking_type_id.sequence_id.next_by_id()
				_logger.warn('info> new name: %s;',name)
				picking.write({'name':name})

		return super(StockPicking, self).do_new_transfer()

	@api.multi
	def _create_backorder(self, backorder_moves=[]):
		""" Move all non-done lines into a new backorder picking. If the key 'do_only_split' is given in the context, then move all lines not in context.get('split', []) instead of all non-done lines.
		"""
		# TDE note: o2o conversion, todo multi
		_logger.warn('none> StockPicking._create_backorder()')
		backorders = self.env['stock.picking']
		for picking in self:
			backorder_moves = backorder_moves or picking.move_lines
			if self._context.get('do_only_split'):
				not_done_bo_moves = backorder_moves.filtered(lambda move: move.id not in self._context.get('split', []))
			else:
				not_done_bo_moves = backorder_moves.filtered(lambda move: move.state not in ('done', 'cancel'))
			if not not_done_bo_moves:
				continue
			backorder_picking = picking.copy({
				'name': '<'+picking.name,
				'move_lines': [],
				'pack_operation_ids': [],
				'backorder_id': picking.id
			})
			backorder_picking.write({'name':'#'+str(backorder_picking.id)})

			picking.message_post(body=_("Back order %s created.") % (backorder_picking.name))
			not_done_bo_moves.write({'picking_id': backorder_picking.id})
			if not picking.date_done:
				picking.write({'date_done': time.strftime(DEFAULT_SERVER_DATETIME_FORMAT)})
			#backorder_picking.action_confirm()
			#backorder_picking.action_assign()
			backorders |= backorder_picking
		return backorders

class ReturnPicking(models.TransientModel):
	_inherit = 'stock.return.picking'

	@api.multi
	def create_returns(self):
		_logger.warn('none> ReturnPicking.create_returns()')
		ret = super(ReturnPicking, self).create_returns()
		_logger.warn('\ninfo> ret: %s', ret)
		self.env['stock.picking'].browse(ret['res_id']).write({'is_return_picking':True})
		return ret

class PackOperation(models.Model):
	_inherit = "stock.pack.operation"

	def _compute_lpm_lot_domain(self):
		_logger.warn('none> PackOperation._compute_lpm_lot_domain()')
		_logger.warn('\ninfo> picking_id.picking_type_code: %s; picking_id.is_return_picking: %s', self.picking_id.picking_type_code, self.picking_id.is_return_picking)
		lot_domain_ids = []
		if self.picking_id.is_return_picking:
			if self.picking_id.picking_type_code == 'outgoing':
				orm_obj = self.env['purchase.order']
				procurement_group_id = 'group_id'
			elif self.picking_id.picking_type_code == 'incoming':
				orm_obj = self.env['sale.order']
				procurement_group_id = 'procurement_group_id'
			so_po_search = orm_obj.search([(procurement_group_id,'=',self.picking_id.group_id.id)])
			_logger.warn('\ninfo> orm_obj: %s; so_po_search:%s', orm_obj, so_po_search)
			if so_po_search:
				for sp in so_po_search.picking_ids:
					for spo in sp.pack_operation_ids:
						for pl in spo.pack_lot_ids:
							if pl.lot_id:
								lot_domain_ids.append(pl.lot_id.id)
		else:
			datenow = fields.Date.context_today(self)
			search_not_expired = self.env['stock.production.lot'].search([('product_id', '=', self.product_id.id),('expiry_date','>',datenow)])
			_logger.warn('\ninfo> datenow: %s; search_not_expired: %s', datenow, search_not_expired)
			lot_domain_ids = search_not_expired.ids

		_logger.warn('\ninfo> lot_domain_ids: %s', lot_domain_ids)
		return lot_domain_ids

	@api.model
	def fields_view_get(self, view_id=None, view_type='form', toolbar=False, submenu=False):
		_logger.warn('none> PackOperation.fields_view_get()')
		res = super(PackOperation, self).fields_view_get(
			view_id=view_id, view_type=view_type, toolbar=toolbar, submenu=submenu)
		_logger.warn("\ninfo> yo man !")
		if self._context.get('lot_domain_ids'):
			#_logger.warn("\ninfo> res['fields']['pack_lot_ids']:\n%s", res['fields']['pack_lot_ids'])
			res_lot_id = res['fields']['pack_lot_ids']['views']['tree']['fields']['lot_id']
			res_lot_id['domain'] = [('id', 'in', self._context['lot_domain_ids'])]
			_logger.warn("\nINFO> res_lot_id:\n%s", res_lot_id)
		return res

	@api.multi
	def action_split_lots(self):
		_logger.warn('none> PackOperation.action_split_lots()')
		_logger.warn('\ninfo> state: %s; context: %s', self.state, self.env.context)
		ret = super(PackOperation, self).action_split_lots()
		if ret['context']['only_create'] == False:
			lot_domain = self._compute_lpm_lot_domain()
			ret['context'].update({'lot_domain_ids': lot_domain})
		_logger.warn('\ninfo> super: %s', ret)
		return ret
	split_lot2 = action_split_lots

class PackOperationLot(models.Model):
	_inherit = "stock.pack.operation.lot"

	expiry_date = fields.Date(string='Expiry Date')
	expiry_date_rel = fields.Date(related='lot_id.expiry_date',string='Lot Expiry Date')

	@api.multi
	def write(self, values):
		_logger.warn('none> PackOperationLot.write()')
		ret = super(PackOperationLot, self).write(values)
		if values.has_key('lot_id'):
			_logger.warn('info> self.expiry_date: %s; self.lot_id.expiry_date: %s;',self.expiry_date, self.lot_id.expiry_date)
			if self.expiry_date:
				self.lot_id.write({'expiry_date':self.expiry_date})
		return ret


import pytest
import System
import Student

def test_viewAssignments(student_system):
    course = 'fakeCourse'
    assert student_system.view_assignments(course)

@pytest.fixture
def student_system():
    gradingSystem = System.System()
    gradingSystem.load_data()
    name = 'akend3'
    studentSystem = Student.Student(name, gradingSystem.users, gradingSystem.courses)
    return studentSystem


# -*- coding: utf-8 -*-
"""
Created on Wed Jan 13 11:32:13 2021

@author: Hrishi_rich
"""
# import computer vision library
import cv2

# Face & Smile Classifier
face_detector = cv2.CascadeClassifier('frontalface_default.xml')
smile_detector = cv2.CascadeClassifier('haarcascade_smile.xml')

# Grab Webcam
webcam = cv2.VideoCapture(0) #'Why_so_serious.mp4'

# Show the current frame
while True:

    # Read current frame from webcam
    successful_frame_read, frame = webcam.read()

    # If there's an error abort
    if not successful_frame_read:
        break

    # Change to grayscale
    frame_grayscale = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # Detect faces
    faces = face_detector.detectMultiScale(frame_grayscale)

    # Run face detection within each of those faces
    for (x, y, w, h) in faces:

        # Draw a rectangle around the face
        cv2.rectangle(frame, (x, y), (x+w, y+h), (100, 200, 50), 2)

        # Get the sub frame (using numpy N-dimensional array slicing)
        the_face = frame[y:y+h, x:x+w]

        # Change to grayscale
        face_grayscale = cv2.cvtColor(the_face, cv2.COLOR_BGR2GRAY)

        smile = smile_detector.detectMultiScale(face_grayscale, scaleFactor=1.7, minNeighbors=20)

        # Lable this faces as smiling
        if len(smile) > 0:
            cv2.putText(frame, 'smiling', (x, y+h+40), fontScale=3, fontFace=cv2.FONT_HERSHEY_PLAIN, color=(255, 255, 255))

# Show the current frame
cv2.imshow('Smile Detector', frame)

# Display
cv2.waitkey(1)

#cleanup
webcam.release()
cv2.destroyAllWindows()

print("code complete")


from __future__ import print_function, division
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import organize_data2 as od
import pandas as pd

num_epochs = 100
total_series_length = 1000000
truncated_backprop_length = 30
state_size = 64
num_classes = 2
echo_step = 3
batch_size = 21
num_batches = total_series_length//batch_size//truncated_backprop_length
num_layers = 3

def generateData():
    #x = np.array(nddp.random.choice(2, total_series_length, p=[0.5, 0.5]))
    mnist = od.read_data()

    #mnist.dropna(axis = 0, how='any')
    #mnist['date'] = pd.to_datetime(mnist['date'],unit='d')
    mnist[['price_max','return_max','volume_max']]= mnist.groupby('gvkey')['price','return','volume'].transform('max')
    mnist['price_norm'] = mnist['price']/(mnist['price_max'] * 1.25)
    mnist['return_norm'] = mnist['return']/(mnist['return_max'] * 1.25)
    mnist['volume_norm'] = mnist['volume']/(mnist['volume_max'] * 1.25)
    mnist['gvkeyCount'] = mnist.groupby(['gvkey'])['gvkey'].transform('count')
    mnist = mnist[(mnist.gvkeyCount == 756)]
    dataY = mnist.pivot(index='gvkey',columns='date',values='price_norm')
    dataY = dataY.dropna()
    dataX = np.empty((2205,756,3))
    dataX[:,:,0] = pd.pivot_table(mnist,values='price_norm',index='gvkey',columns='date')
    dataX[:,:,1] = pd.pivot_table(mnist,values='return_norm',index='gvkey',columns='date')
    dataX[:,:,2] = pd.pivot_table(mnist,values='volume_norm',index='gvkey',columns='date')

    #y = np.roll(x, echo_step)
    #y[0:echo_step] = 0

    x = dataX.reshape((batch_size, -1,3))  # The first index changing slowest, subseries as rows
    y = dataY.values
    y = y.reshape((batch_size, -1))

    return (x, y)

batchX_placeholder = tf.placeholder(tf.float32, [batch_size, truncated_backprop_length,3])
batchY_placeholder = tf.placeholder(tf.int32, [batch_size, truncated_backprop_length])

init_state = tf.placeholder(tf.float32, [num_layers, 2, batch_size, state_size])

state_per_layer_list = tf.unstack(init_state, axis=0)
rnn_tuple_state = tuple(
    [tf.contrib.rnn.LSTMStateTuple(state_per_layer_list[idx][0,:,:], state_per_layer_list[idx][1,:,:])
     for idx in range(num_layers)]
)

W2 = tf.Variable(np.random.rand(state_size, num_classes),dtype=tf.float32)
b2 = tf.Variable(np.zeros((1,num_classes)), dtype=tf.float32)

# Unpack columns
inputs_series = tf.unstack(batchX_placeholder, axis=1)
#inputs_series = tf.split(axis=1, num_or_size_splits=truncated_backprop_length, value=batchX_placeholder)
labels_series = tf.unstack(batchY_placeholder, axis=1)

# Forward passes
#cell = tf.contrib.rnn.LSTMCell(state_size, state_is_tuple=True)
#cell = tf.contrib.rnn.MultiRNNCell([cell] * num_layers, state_is_tuple=True)
stacked_rnn = []
for _ in range(num_layers):
    stacked_rnn.append(tf.contrib.rnn.LSTMCell(state_size, state_is_tuple=True))

cell = tf.contrib.rnn.MultiRNNCell(stacked_rnn, state_is_tuple=True)

states_series, current_state = tf.contrib.rnn.static_rnn(cell, inputs_series, initial_state=rnn_tuple_state)

logits_series = [tf.matmul(state, W2) + b2 for state in states_series] #Broadcasted addition
predictions_series = [tf.nn.softmax(logits) for logits in logits_series]

losses = [tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels) for logits, labels in zip(logits_series,labels_series)]
total_loss = tf.reduce_mean(losses)

train_step = tf.train.AdagradOptimizer(0.3).minimize(total_loss)

def plot(loss_list, predictions_series, batchX, batchY):
    plt.subplot(2, 3, 1)
    plt.cla()
    plt.plot(loss_list)

    for batch_series_idx in range(5):
        one_hot_output_series = np.array(predictions_series)[:, batch_series_idx, :]
        single_output_series = np.array([(1 if out[0] < 0.5 else 0) for out in one_hot_output_series])

        plt.subplot(2, 3, batch_series_idx + 2)
        plt.cla()
        plt.axis([0, truncated_backprop_length, 0, 2])
        left_offset = range(truncated_backprop_length)
        plt.bar(left_offset, batchX[batch_series_idx, :,:], width=1, color="blue")
        plt.bar(left_offset, batchY[batch_series_idx, :] * 0.5, width=1, color="red")
        plt.bar(left_offset, single_output_series * 0.3, width=1, color="green")

    plt.draw()
    plt.pause(0.0001)

with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    plt.ion()
    #plt.figure()
    plt.show()
    loss_list = []

    for epoch_idx in range(num_epochs):
        x,y = generateData()

        _current_state = np.zeros((num_layers, 2, batch_size, state_size))

        print("New data, epoch", epoch_idx)

        for batch_idx in range(num_batches):
            start_idx = batch_idx * truncated_backprop_length
            end_idx = start_idx + truncated_backprop_length

            batchX = x[:,start_idx:end_idx,:]
            batchY = y[:,start_idx:end_idx]

            _total_loss, _train_step, _current_state, _predictions_series = sess.run(
                [total_loss, train_step, current_state, predictions_series],
                feed_dict={
                    batchX_placeholder: batchX,
                    batchY_placeholder: batchY,
                    init_state: _current_state
                })

            loss_list.append(_total_loss)

            if batch_idx%100 == 0:
                print("Step",batch_idx, "Batch loss", _total_loss)
                #plot(loss_list, _predictions_series, batchX, batchY)

plt.ioff()
#plt.show()


# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the License);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an AS IS BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and#
# limitations under the License.

import copy
import logging

from oslo_config import cfg

from cloudferry.lib.base.action import converter
from cloudferry.lib.utils import utils as utl

CONF = cfg.CONF
LOG = logging.getLogger(__name__)
CEPH = 'ceph'
ACTIVE = 'active'
BARE = "bare"

def require_methods(methods, obj):
    for method in methods:
        if method not in dir(obj):
            return False
    return True

class ConvertVolumeToImage(converter.Converter):
    def run(self, storage_info=None, **kwargs):
        disk_format = self.cfg.migrate.disk_format
        container_format = self.cfg.migrate.container_format
        volumes_info = storage_info and copy.deepcopy(storage_info) or {}
        resource_storage = self.cloud.resources[utl.STORAGE_RESOURCE]
        resource_image = self.cloud.resources[utl.IMAGE_RESOURCE]
        images_info = {}
        if not require_methods(['upload_volume_to_image'], resource_storage):
            raise RuntimeError("No require methods")
        images_from_volumes = {}
        for volume in volumes_info[utl.VOLUMES_TYPE].values():
            vol = volume['volume']
            LOG.debug(
                "| | uploading volume %s [%s] to image service bootable=%s",
                vol['display_name'], vol['id'],
                vol['bootable'] if hasattr(vol, 'bootable') else False)
            _, image_id = resource_storage.upload_volume_to_image(
                vol['id'], force=True, image_name=vol['id'],
                container_format=container_format,
                disk_format=disk_format)
            timeout = CONF.migrate.storage_backend_timeout
            resource_image.wait_for_status(image_id,
                                           resource_image.get_status,
                                           ACTIVE, timeout=timeout)
            resource_image.patch_image(resource_image.get_backend(), image_id)
            image_vol = resource_image.get_image_by_id_converted(image_id)
            img_new = {
                utl.IMAGE_BODY: (
                    image_vol[utl.IMAGES_TYPE][image_id][utl.IMAGE_BODY]),
                utl.META_INFO: volume[utl.META_INFO]
            }
            img_new[utl.META_INFO][utl.VOLUME_BODY] = vol
            images_from_volumes[image_id] = img_new
        images_info[utl.IMAGES_TYPE] = images_from_volumes
        return {
            'images_info': images_info
        }


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2021 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Script for BLEU metric."""

import re
import six
import sys
import unicodedata
from typing import List, Sequence

from .bleu_util import compute_bleu
from .metric import metric_registry

class UnicodeRegex(object):
    """Ad-hoc hack to recognize all punctuation and symbols.

    Attributes:
        nondigit_punct_re: The compiled regular expressions to recognize
          punctuation preceded with a digit.
        punct_nondigit_re: The compiled regular expressions to recognize
          punctuation followed by a digit.
        symbol_re: The compiled regular expressions to recognize symbols.
    """

    def __init__(self) -> None:
        """Initialize the regular expressions."""
        punctuation = self.property_chars("P")
        self.nondigit_punct_re = re.compile(r"([^\d])([" + punctuation + r"])")
        self.punct_nondigit_re = re.compile(r"([" + punctuation + r"])([^\d])")
        self.symbol_re = re.compile("([" + self.property_chars("S") + "])")

    def property_chars(self, prefix: str) -> str:
        """Collect all Unicode strings starting with a specific prefix.

        Args:
            prefix: The specific prefix.

        Returns:
            punctuation: The join result of all Unicode strings starting
              with a specific prefix.
        """
        punctuation = "".join(six.unichr(x) for x in range(sys.maxunicode) \
                              if unicodedata.category(six.unichr(x)).startswith(prefix))
        return punctuation

uregex = UnicodeRegex()

def bleu_tokenize(string: str) -> List[str]:
    """Tokenize a string following the official BLEU implementation.

    See https://github.com/moses-smt/mosesdecoder/"
            "blob/master/scripts/generic/mteval-v14.pl#L954-L983

    Args:
        string: The string to be tokenized.

    Returns:
        tokens: A list of tokens.
    """
    string = uregex.nondigit_punct_re.sub(r"\1 \2 ", string)
    string = uregex.punct_nondigit_re.sub(r" \1 \2", string)
    string = uregex.symbol_re.sub(r" \1 ", string)
    tokens = string.split()
    return tokens

@metric_registry('BLEU', 'tensorflow, tensorflow_itex')
class BLEU(object):
    """Computes the BLEU (Bilingual Evaluation Understudy) score.

    BLEU is an algorithm for evaluating the quality of text which has
    been machine-translated from one natural language to another.
    This implementent approximate the BLEU score since we do not
    glue word pieces or decode the ids and tokenize the output.
    By default, we use ngram order of 4 and use brevity penalty.
    Also, this does not have beam search.

    Attributes:
        predictions: List of translations to score.
        labels: List of the reference corresponding to the prediction result.
    """

    def __init__(self) -> None:
        """Initialize predictions and labels."""
        self.predictions = []
        self.labels = []

    def reset(self) -> None:
        """Clear the predictions and labels in the cache."""
        self.predictions = []
        self.labels = []

    def update(self, prediction: Sequence[str], label: Sequence[str]) -> None:
        """Add the prediction and label.

        Args:
            prediction: The prediction result.
            label: The reference corresponding to the prediction result.

        Raises:
            ValueError: An error occurred when the length of the prediction
            and label are different.
        """
        if len(label) != len(prediction):
            raise ValueError("Reference and prediction files have different number "
                             "of lines. If training only a few steps (100-200), the "
                             "translation may be empty.")
        label = [x.lower() for x in label]
        prediction = [x.lower() for x in prediction]
        label = [bleu_tokenize(x) for x in label]
        prediction = [bleu_tokenize(x) for x in prediction]
        self.labels.extend(label)
        self.predictions.extend(prediction)

    def result(self) -> float:
        """Compute the BLEU score.

        Returns:
            bleu_score: The approximate BLEU score.
        """
        bleu_score = compute_bleu(self.labels, self.predictions) * 100
        return bleu_score


import pandas as pd

body_url = 'https://raw.githubusercontent.com/wmgriffi3/barbell-beasts/master/FitNotes_BodyTracker_Export.csv'
body_data = pd.read_csv(body_url, error_bad_lines=False)

standards_url = 'https://raw.githubusercontent.com/wmgriffi3/barbell-beasts/master/weight_lifting_standards.csv'
standards_data = pd.read_csv(standards_url, error_bad_lines=False)

exercise_url = 'https://raw.githubusercontent.com/wmgriffi3/barbell-beasts/master/FitNotes_Export.csv'
exercise_data = pd.read_csv(exercise_url, error_bad_lines=False)

body_data.head()
standards_data.head()
exercise_data.head()

from io import BytesIO

import requests
r = requests.get('https://docs.google.com/spreadsheets/d/e/2PACX-1vQmHR9YOt6BhTolpVXR9m8eVSHM-2AnPmhVhYypo2liHRpZ1REQeEtE5rEoBQQfQfCt-5MGk6bU1ipQ/pub?output=csv')
data = r.content

dataf = pd.read_csv(BytesIO(data), index_col=0)

dataf.head()


import os
import matplotlib.pyplot as plt
import csv
import numpy as np
import math
import unicodedata

problemy = ["MST", "SP", "MF"]
algorytmy = {
    "MF": ["Ford-Fulkerson-BFS", "Ford-Fulkerson-DFS"],
    "MST": ["Kruskal", "Prim"],
    "SP": ["Dijkstra", "Bellman-Ford"]
}

"""
Wczytywanie plików csv z danymi
"""
sciezka = os.getcwd()
listaPlikow = []
for root, dirs, files in os.walk(sciezka+'\\Dane\\'):
    for file in files:
        if file.endswith(".csv"):
             listaPlikow.append(os.path.join(root, file))

#print(listaPlikow)

print("Autor: Mikołaj Chmielecki")
print("Generowanie wykresów na podstawie plików csv z aktualnego katalogu...")

listaListDanych = []
listaTytulow = []
"""
Tworzenie listy list plików do każdego wykresu
"""
nrProblemu = 0
for problem in problemy:
    for i in range(6):
        listaListDanych.append([])
    listaTytulow.append(problem + " - macierz sąsiedztwa")
    listaTytulow.append(problem + " - lista sąsiedztwa")
    listaTytulow.append(problem + " - gęstość 25%")
    listaTytulow.append(problem + " - gęstość 50%")
    listaTytulow.append(problem + " - gęstość 75%")
    listaTytulow.append(problem + " - gęstość 99%")
    for plik in listaPlikow:
        head, tail = os.path.split(plik)
        plikName = tail
        if (algorytmy[problem][0] in plikName or algorytmy[problem][1] in plikName):
            if "macierz" in plik:
                listaListDanych[nrProblemu*6].append(plik)
            if "lista" in plik:
                listaListDanych[nrProblemu*6+1].append(plik)
            if "25" in plik:
                listaListDanych[nrProblemu*6+2].append(plik)
            if "50" in plik:
                listaListDanych[nrProblemu*6+3].append(plik)
            if "75" in plik:
                listaListDanych[nrProblemu*6+4].append(plik)
            if "99" in plik:
                listaListDanych[nrProblemu*6+5].append(plik)

    nrProblemu += 1

"""
Generowanie wykresu dla każdego pliku z danymi
"""
licznikWykresu = 0
for lista in listaListDanych:
    plt.close()
    plt.title(listaTytulow[licznikWykresu])

    for plik in lista:

        liczbaElemetnow = []
        czasOperacji = []
        nazwa = ""
        """
        Otwieranie pliku z danymi
        """
        with open(plik, 'r') as csvfile:
            plots = csv.reader(csvfile, delimiter=';')
            i = 0
            """
            Czytanie kolejnych wierszy
            """
            for row in plots:
                if i == 0:
                    nazwa = row[0]
                else:
                    liczbaElemetnow.append(int(row[0]))
                    czasOperacji.append(float(row[1].replace(',', '.')))
                i += 1

        nowaLista = [x / 1000 for x in czasOperacji]
        czasOperacji = nowaLista[:]
        jednostka = "ms"
        """
        jednostka = "µs"
        if  max(czasOperacji) > 999:
            nowaLista = [x/1000 for x in czasOperacji]
            czasOperacji = nowaLista[:]
            jednostka = "ms"
        if max(czasOperacji) > 999:
            nowaLista = [x/1000 for x in czasOperacji]
            czasOperacji = nowaLista[:]
            jednostka = "s"
        """

        plt.plot(liczbaElemetnow, czasOperacji, marker='o', label=nazwa)
        plt.xticks(liczbaElemetnow)
        plt.xlabel('Liczba elementów')
        plt.ylabel(f'Czas operacji [{jednostka}]')

    #plt.ticklabel_format(useOffset=False, style='plain')
    plt.legend()
    plt.tight_layout()
    normalize = unicodedata.normalize('NFD', listaTytulow[licznikWykresu]).encode('ascii', 'ignore')
    plt.savefig("Dane/" + str(normalize).replace("b\'","").replace("\'","").replace("%","") + ".pdf" )
    plt.close()
    licznikWykresu += 1

print("Generowanie zakończone")


from sklearn.model_selection import train_test_split
import xgboost as xgb
import numpy as np
import pandas as pd

def read_data(TRAIN_DIR, TEST_DIR, nrows=None):
    data_train = pd.read_csv(TRAIN_DIR,
                             parse_dates=["pickup_datetime",
                                          "dropoff_datetime"],
                             nrows=nrows)
    data_test = pd.read_csv(TEST_DIR,
                            parse_dates=["pickup_datetime"],
                            nrows=nrows)
    data_train = data_train.drop(['dropoff_datetime'], axis=1)
    data_train.loc[:, 'store_and_fwd_flag'] = data_train['store_and_fwd_flag'].map({'Y': True,
                                                                                'N': False})
    data_test.loc[:, 'store_and_fwd_flag'] = data_test['store_and_fwd_flag'].map({'Y': True,
                                                                                'N': False})
    data_train = data_train[data_train.trip_duration < data_train.trip_duration.quantile(0.99)]

    xlim = [-74.03, -73.77]
    ylim = [40.63, 40.85]
    data_train = data_train[(data_train.pickup_longitude> xlim[0]) & (data_train.pickup_longitude < xlim[1])]
    data_train = data_train[(data_train.dropoff_longitude> xlim[0]) & (data_train.dropoff_longitude < xlim[1])]
    data_train = data_train[(data_train.pickup_latitude> ylim[0]) & (data_train.pickup_latitude < ylim[1])]
    data_train = data_train[(data_train.dropoff_latitude> ylim[0]) & (data_train.dropoff_latitude < ylim[1])]

    return (data_train, data_test)

def train_xgb(X_train, labels):
    Xtr, Xv, ytr, yv = train_test_split(X_train.values,
                                        labels,
                                        test_size=0.2,
                                        random_state=0)

    dtrain = xgb.DMatrix(Xtr, label=ytr)
    dvalid = xgb.DMatrix(Xv, label=yv)

    evals = [(dtrain, 'train'), (dvalid, 'valid')]

    params = {
        'min_child_weight': 1, 'eta': 0.166,
        'colsample_bytree': 0.4, 'max_depth': 9,
        'subsample': 1.0, 'lambda': 57.93,
        'booster': 'gbtree', 'gamma': 0.5,
        'silent': 1, 'eval_metric': 'rmse',
        'objective': 'reg:linear',
    }

    model = xgb.train(params=params, dtrain=dtrain, num_boost_round=60, #227,
                      evals=evals, early_stopping_rounds=60, maximize=False,
                      verbose_eval=10)

    print('Modeling RMSE %.5f' % model.best_score)
    return model

def predict_xgb(model, X_test):
    dtest = xgb.DMatrix(X_test.values)
    ytest = model.predict(dtest)
    X_test['trip_duration'] = np.exp(ytest) - 1
    return X_test[['trip_duration']]

def feature_importances(model, feature_names):
    feature_importance_dict = model.get_fscore()
    fs = ['f%i' % i for i in range(len(feature_names))]
    f1 = pd.DataFrame({'f': list(feature_importance_dict.keys()),
                       'importance': list(feature_importance_dict.values())})
    f2 = pd.DataFrame({'f': fs, 'feature_name': feature_names})
    feature_importance = pd.merge(f1, f2, how='right', on='f')
    feature_importance = feature_importance.fillna(0)
    return feature_importance[['feature_name', 'importance']].sort_values(by='importance',
                                                                          ascending=False)

def get_train_test_fm(feature_matrix):
    X_train = feature_matrix[feature_matrix['test_data'] == False]
    X_train = X_train.drop(['test_data'], axis=1)
    labels = X_train['trip_duration']
    X_train = X_train.drop(['trip_duration'], axis=1)
    X_test = feature_matrix[feature_matrix['test_data'] == True]
    X_test = X_test.drop(['test_data', 'trip_duration'], axis=1)
    return (X_train, labels, X_test)

def duplicate_columns(frame):
    groups = frame.columns.to_series().groupby(frame.dtypes).groups
    dups = []
    for t, v in groups.items():
        dcols = frame[v].to_dict(orient="list")

        vs = dcols.values()
        ks = dcols.keys()
        lvs = len(vs)

        for i in range(lvs):
            for j in range(i+1,lvs):
                if vs[i] == vs[j]:
                    dups.append(ks[i])
                    break
    return dups


import logging
from fractions import Fraction

from django.db import models
from django.core.exceptions import ValidationError
from django.utils.translation import gettext_lazy as _
from django.contrib.auth.models import Group

from taggit.managers import TaggableManager

from .utils import get_exif

# Get an instance of a logger
logger = logging.getLogger(__name__)

class Exif(models.Model):
    """
    Represents the exif data.
    """
    taken_date = models.DateField(editable=False, verbose_name=_('Taken Date'))
    focal_length = models.CharField(
        max_length=8, editable=False, verbose_name=_('Focal Length'))
    aperture = models.CharField(
        max_length=16, editable=False, verbose_name=_('Aperture'))
    iso = models.IntegerField(editable=False, verbose_name=_('ISO'))
    shutter_speed = models.CharField(
        max_length=16, editable=False, verbose_name=_('Shutter Speed'))
    camera_make = models.CharField(
        max_length=32, editable=False, verbose_name=_('Camera Make'))
    camera_model = models.CharField(
        max_length=32, editable=False, verbose_name=_('Camera Model'))
    lens_make = models.CharField(
        max_length=32,
        editable=False,
        blank=True,
        null=True,
        verbose_name=_('Lens Make'))
    lens_model = models.CharField(
        max_length=32, editable=False, verbose_name=_('Lens Model'))

    @classmethod
    def create(cls, exif_data: dict):
        """
        Constructor helper
        See https://docs.djangoproject.com/en/1.9/ref/models/instances/#django.db.models.Model.clean
        """
        exif = cls(
            iso=exif_data['ISOSpeedRatings'],
            taken_date=exif_data['DateTimeOriginal'].replace(':', '-')[0:10],
            camera_make=exif_data['Make'],
            camera_model=exif_data['Model'],
            lens_make=exif_data.get('LensMake'),
            lens_model=exif_data['LensModel'],
            focal_length=exif_data['FocalLength'][0] /
            exif_data['FocalLength'][1],
            shutter_speed=Fraction(exif_data['ExposureTime'][0],
                                   exif_data['ExposureTime'][1]),
            aperture=exif_data['FNumber'][0] / exif_data['FNumber'][1],
        )
        return exif

    def __str__(self):
        return '{} {}, ƒ/{}, {}s, {}mm, ISO {}'.format(
            self.camera_make.capitalize(), self.camera_model, self.aperture,
            self.shutter_speed, self.focal_length, self.iso)

class Picture(models.Model):
    """
    Represents a Picture. Could be a photograph or a screenshot.
    """
    title = models.CharField(max_length=255, verbose_name=_('Title'))
    description = models.TextField(blank=True, verbose_name=_('Description'))
    published_date = models.DateTimeField(
        auto_now_add=True, verbose_name=_('Published Date'))
    exif = models.OneToOneField(
        Exif,
        on_delete=models.CASCADE,
        editable=False,
        null=True,
        blank=True,
        verbose_name=_('EXIF'))

    def __str__(self):
        return self.title

    def validate_image(fieldfile_obj):
        """Check whether the exif data can be retrieve without issue."""
        exif = get_exif(fieldfile_obj)
        try:
            # Test whether the exif data is valid by attempting
            # to create an Exif instance.
            Exif.create(exif)
        except KeyError as ex:
            logger.exception('Failed to retrieve exif data.')
            raise ValidationError('Failed to retrieve exif data.')

    image = models.ImageField(upload_to='photos/', validators=[validate_image])
    tags = TaggableManager()

    CC0 = 'CC0'
    BY = 'BY'
    BY_SA = 'BY-SA'
    BY_NC = 'BY-NC'
    BY_NC_SA = 'BY-NC-SA'
    NO_LICENSE = 'NONE'

    AVAILABLE_LICENSES = (
        (CC0, _('CC0')),
        (BY, _('Attribution')),
        (BY_SA, _('Attribution-ShareAlike')),
        (BY_NC, _('Attribution-NonCommercial')),
        (BY_NC_SA, _('Attribution-NonCommercial-ShareAlike')),
        (NO_LICENSE, _('All rights reserved')),
    )

    license = models.CharField(
        max_length=10,
        choices=AVAILABLE_LICENSES,
        default=BY,
        verbose_name=_('License'),
        help_text=
        "See https://en.wikipedia.org/wiki/Creative_Commons_license for more information"
    )

    view_count = models.PositiveIntegerField(
        verbose_name=_('View count'), default=0, editable=False)

    download_count = models.PositiveIntegerField(
        verbose_name=_('Download count'), default=0, editable=False)

    is_private = models.BooleanField(verbose_name=_("Private"), default=True)

    public_for_user_groups = models.ManyToManyField(
        Group, verbose_name=_('Public for user groups'), blank=True)

    def increment_view_count(self):
        self.view_count += 1
        models.Model.save(self)

    def increment_download_count(self):
        self.download_count += 1
        models.Model.save(self)


import json
import math
import numpy as np
import random
from collections import defaultdict, namedtuple

NUM_NODES = 10
NUM_CHANNELS = 30

def popup_node_with_channel(
        visible_nodes,
        hidden_nodes,
        node_to_unichannels
):
    # First attempt following channels of visible nodes at random.
    i = 0
    while i < 10:
        i += 1
        rand = random.randint(0, len(visible_nodes) - 1)
        rand_visible_node = visible_nodes[rand]
        for unichannel in node_to_unichannels[rand_visible_node]:
            if unichannel.partner in hidden_nodes:
                return unichannel.partner, unichannel.channel

    # Alternatively, attempt finding a channel from a hidden node to a visible one.
    i = 0
    while i < 10:
        i += 1
        rand = random.randint(0, len(hidden_nodes) - 1)
        rand_hidden_node = hidden_nodes[rand]
        for unichannel in node_to_unichannels[rand_hidden_node]:
            if unichannel.partner in visible_nodes:
                return rand_hidden_node, unichannel.channel

    print('No hidden node with connection to visible node found. Giving up.')

    return None, None

def popup_channel(visible_nodes, hidden_channels, channels):
    i = 0
    while i < 10:
        i += 1
        rand = random.randint(0, len(hidden_channels) - 1)
        rand_hidden_channel = hidden_channels[rand]
        channel = channels[rand_hidden_channel]
        if channel[0] in visible_nodes and channel[1] in visible_nodes:
            return rand_hidden_channel

    print('No hidden channel with connection between visible nodes found. Giving up.')

    return None

def flash_channel(visible_channels):
    return random.sample(visible_channels, 1)[0]

def generate_nodes():
    """
    Generate random coordinates on the surface of a sphere.
    """
    nodes = np.random.randn(3, NUM_NODES)
    nodes /= np.linalg.norm(nodes, axis=0)
    return list(nodes.transpose().tolist())

def generate_channels(num_nodes):
    """
    Generate random index pairs to connect nodes via channels.
    """
    channels = set()
    for i in range(NUM_CHANNELS):
        channels.add(frozenset(random.sample(range(num_nodes), 2)))

    return [tuple(channel) for channel in channels]

def generate_animations(num_nodes, channels):
    # Map nodes to their channels.
    Unichannel = namedtuple('UniChannel', ['channel', 'partner'])
    node_to_unichannels = defaultdict(list)
    for i, channel in enumerate(channels):
        node_to_unichannels[channel[0]].append(Unichannel(channel=i, partner=channel[1]))
        node_to_unichannels[channel[1]].append(Unichannel(channel=i, partner=channel[0]))

    Animation = namedtuple('Animation', [
        'start_frame',
        'animation_type',
        'element_type',
        'element_id',
        'transfer_id'
    ])

    # Show nodes through random connections.
    animations = []
    visible_nodes = []
    hidden_nodes = list(range(num_nodes))
    visible_channels = []
    hidden_channels = list(range(len(channels)))
    popup_delay = 10

    first_node = random.randint(0, num_nodes - 1)
    first_animation = Animation(
        start_frame=0,
        animation_type='show',
        element_type='node',
        element_id=first_node,
        transfer_id=-1
    )
    animations.append(first_animation)
    hidden_nodes.remove(first_node)
    visible_nodes.append(first_node)

    last_frame = 0
    transfer_id = 0
    while hidden_nodes or hidden_channels:
        # Find a node with a channel to a visible node.
        node = None
        channel = None
        type = None

        rand = random.randint(0, 2)
        if hidden_nodes and rand == 0:
            node, channel = popup_node_with_channel(
                visible_nodes,
                hidden_nodes,
                node_to_unichannels
            )
            type = 'show'
        elif rand == 1:
            channel = popup_channel(visible_nodes, hidden_channels, channels)
            type = 'show'
        elif visible_channels:
            channel = flash_channel(visible_channels)
            type = 'flash'

        # Create animation.
        if node is not None or channel is not None:
            popup_delay = int(math.ceil(0.95 * popup_delay))
            last_frame = last_frame + popup_delay
        if node is not None:
            node_animation = Animation(
                start_frame=last_frame,
                animation_type=type,
                element_type='node',
                element_id=node,
                transfer_id=transfer_id if type == 'flash' else -1
            )
            animations.append(node_animation)
            if type == 'show':
                hidden_nodes.remove(node)
                visible_nodes.append(node)
        if channel is not None:
            channel_animation = Animation(
                start_frame=last_frame,
                animation_type=type,
                element_type='channel',
                element_id=channel,
                transfer_id=transfer_id if type == 'flash' else -1
            )
            animations.append(channel_animation)
            if type == 'show':
                hidden_channels.remove(channel)
                visible_channels.append(channel)

        if type == 'flash':
            transfer_id += 1

    return animations

def run():
    nodes = generate_nodes()
    channels = generate_channels(len(nodes))

    # Write network topography to network file.
    with open('network.json', 'w') as network_file:
        json.dump({'nodes': nodes, 'channels': channels}, network_file, indent=2)

    animations = generate_animations(len(nodes), channels)

    with open('animation.json', 'w') as animation_file:
        json.dump(animations, animation_file, indent=2)

if __name__ == "__main__":
    run()


import pygame
pygame.init()

SIZE = WIDTH, HEIGHT = 720, 480
FPS = 60
BACKGROUND_COLOR = pygame.Color('white')

screen = pygame.display.set_mode(SIZE)
clock = pygame.time.Clock()

class Hunter(pygame.sprite.Sprite):

    def __init__(self, position):
        super(Hunter, self).__init__()

        self.image = pygame.Surface((64, 64))
        self.image.fill(pygame.Color('red'))
        self.rect = self.image.get_rect(topleft=position)
        self.position = pygame.math.Vector2(position)
        self.speed = 2

    def hunt_player(self, player):
        player_position = player.rect.topleft
        direction = player_position - self.position
        velocity = direction.normalize() * self.speed

        self.position += velocity
        self.rect.topleft = self.position

    def update(self, player):
        self.hunt_player(player)

class Player(pygame.sprite.Sprite):

    def __init__(self, position):
        super(Player, self).__init__()

        self.image = pygame.Surface((64, 64))
        self.image.fill(pygame.Color('green'))
        self.rect = self.image.get_rect(topleft=position)

        self.position = pygame.math.Vector2(position)
        self.velocity = pygame.math.Vector2(0, 0)
        self.speed = 3

    def update(self):
        keys = pygame.key.get_pressed()
        if keys[pygame.K_LEFT]:
            self.velocity.x = -self.speed
        elif keys[pygame.K_RIGHT]:
            self.velocity.x = self.speed
        else:
            self.velocity.x = 0

        if keys[pygame.K_UP]:
            self.velocity.y = -self.speed
        elif keys[pygame.K_DOWN]:
            self.velocity.y = self.speed
        else:
            self.velocity.y = 0

        self.position += self.velocity
        self.rect.topleft = self.position

player = Player(position=(350, 220))
monster = Hunter(position=(680, 400))
running = True
while running:

    clock.tick(FPS)

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False

    player.update()
    monster.update(player)

    screen.fill(BACKGROUND_COLOR)
    screen.blit(player.image, player.rect)
    screen.blit(monster.image, monster.rect)

    pygame.display.update()


import os
import os.path
import numpy
import scipy.sparse
import torch
from download_torch_ready_data import Reddit, SmallerReddit, OneQuarterReddit, TinyReddit
from torch_geometric.data import Data

from torch_geometric.utils.convert import to_networkx
import networkx
import metis

def metis_reorder(edge_index, x, y, split, P=8):
    # a reorder with gap example:
    # parts        = 0,1,0,1,0,1
    # node_ids     = 1,2,5,6,7,8
    # will sort to = 1,5,7,2,6,8
    # e.g. the old id=5 should be loc to idx=2(which is idx of old id=2), so change id=5 to 2.
    # e.g. id=7 should be loc to idx=3(which is idx of old id=5), so change id=7 to 5.
    # node_mapping = 1:1, 5:2, 7:5, ...

    #print('edge index', edge_index)
    nxg = to_networkx(Data(x=x, edge_index=edge_index))
    #print('partition begin')
    # print('nxg', nxg.nodes)
    # print('nxg', nxg.edges)
    (edgecuts, parts) = metis.part_graph(nxg, P)
    #print('parted', edgecuts, parts)
    node_ids = list(nxg) # only works when no gap in ids
    parted_ids = [(p,old_node_id) for p, old_node_id in zip(parts, node_ids)]
    sorted_node_ids = [node_id for p, node_id in sorted(parted_ids)]
    #print('sorted node ids', sorted_node_ids)

    id_mapping = dict((old_id, new_id) for old_id, new_id in zip(sorted_node_ids, node_ids))
    #print('id mapping', id_mapping)
    old_edge_index = torch.clone(edge_index)

    edge_index[0] = torch.tensor([id_mapping[old_id.item()] for old_id in old_edge_index[0]])
    edge_index[1] = torch.tensor([id_mapping[old_id.item()] for old_id in old_edge_index[1]])
    #print('node swapped', edge_index)

    def swap_tensor(t):
        # st = torch.empty_like(t)
        st = t[sorted_node_ids].clone()
        return st

    x = swap_tensor(x)
    #print(y.size(), y)
    y = swap_tensor(y)
    #print(y.size(), y)
    #print(split.size(), split)
    split = swap_tensor(split)
    #print(split.size(), split)

    return edge_index, x, y, split

def create_parted_dataset(original_dataset):
    original_name = original_dataset.__class__.__name__
    edge_index, x, y, split = metis_reorder(*list(map(original_dataset.data_dict.get, ['edge_index', 'x', 'y', 'split'])))
    parted_data_path = os.path.join('..', 'torch_ready_data', 'Parted'+original_name, 'processed', 'data.pt')
    os.makedirs(os.path.dirname(parted_data_path), exist_ok=True)
    torch.save({"x": x, "y": y, "edge_index": edge_index, "split": split}, parted_data_path)

def main():
    #torch.set_printoptions(threshold=5000)
    # list(map(create_parted_dataset, [SmallerReddit(), OneQuarterReddit(), Reddit()]))
    create_parted_dataset(Reddit())
    # create_parted_dataset(TinyReddit())

    # Cora()
    # Amazon()
    pass

if __name__ == '__main__':
    main()


from flask import Flask, render_template,request
from flask_bootstrap import Bootstrap
import json
from main import get_stories,get_random
app = Flask(__name__)

store_id=0
@app.route("/",methods=['GET', 'POST'])
def main():
    if request.method == 'POST':
        album_id = int(request.form['album_id'])
        global store_id
        store_id=album_id
        try:
            input_image,input_text,output_image,output_text=get_stories(album_id,shuffle=True)
        except:
            return render_template('error.html')
        return render_template('index.html',sorted_image=output_image,sorted_text=output_text,shuffled_image=input_image,shuffled_text=input_text)
    else:
        input_image,input_text,output_image,output_text=get_stories(Random=True)
        return render_template('index.html',sorted_image=output_image,sorted_text=output_text,shuffled_image=input_image,shuffled_text=input_text)

@app.route("/shuffle",methods=['POST'])

def shuffle():
    album_id = store_id
    try:
        input_image,input_text,output_image,output_text=get_stories(album_id,shuffle=True)
    except:
        return render_template('error.html')
    return render_template('index.html',sorted_image=output_image,sorted_text=output_text,shuffled_image=input_image,shuffled_text=input_text)

@app.route("/arrange",methods=['POST'])

def arrange():
    album_id = store_id
    try:
        input_image,input_text,output_image,output_text=get_stories(album_id,shuffle=False)
    except:
        return render_template('error.html')
    return render_template('index.html',sorted_image=output_image,sorted_text=output_text,shuffled_image=input_image,shuffled_text=input_text)

@app.route("/example")
def example():
    input_image_list=[]
    input_text_list=[]
    output_image_list=[]
    output_text_list=[]
    index=[243,654,342,676,23]
    for i in index:
        input_image,input_text,output_image,output_text=get_random(i)
        input_image_list.append(input_image)
        input_text_list.append(input_text)
        output_image_list.append(output_image)
        output_text_list.append(output_text)
    return render_template('example.html',im1=input_image_list,text1=input_text_list,im2=output_image_list,text2=output_text_list)

@app.route("/about")
def about():
    return render_template('about.html')

if __name__ == "__main__":
    app.run(debug=True)


import sys
import pickle
with open(sys.argv[1], 'rb') as f:
	x = pickle.load(f)
	preds = x['test']['preds']

from bs4 import BeautifulSoup
import glob
import collections
import shutil
import os

dev_files = sorted(glob.glob('/home/devanshg27/semtabfact/xml/test_a/*.xml'))

SUBMISSION_DIR = './submit'

if os.path.exists(SUBMISSION_DIR) and os.path.isdir(SUBMISSION_DIR):
    shutil.rmtree(SUBMISSION_DIR)

os.makedirs(SUBMISSION_DIR)

CONST_DICT = {
	'refuted': 0,
	'entailed': 1,
	'unknown': 2,
}

REV_CONST_DICT = {
	0: 'refuted',
	1: 'entailed',
	2: 'unknown',
}

idx = 0
for name in dev_files:
	with open(name) as f:
		soup = BeautifulSoup(f, 'xml')
		statements = soup.find_all("statement")

		for statement in statements:
			statement['type'] = REV_CONST_DICT[int(preds[idx])]
			idx += 1

		f2 = open(SUBMISSION_DIR + '/' + os.path.basename(name), "w")
		f2.write(soup.prettify())
		f2.close()

assert(idx == len(preds))

shutil.make_archive('submission', 'zip', SUBMISSION_DIR)


import sys
import sqlite3
sys.path.append('/afs/cern.ch/cms/PPD/PdmV/tools/McM/')
from rest import McM

# McM instance
mcm = McM(dev=False)

def pick_chained_requests(chained_requests):
    """
    Select chained requests with newest NanoAOD version
    """
    tree = {}
    selected_chained_requests = []
    for chained_request in chained_requests:
        steps = chained_request_to_steps(chained_request)
        mini_step = steps.get('miniaod')
        nano_step = steps.get('nanoaod')
        if mini_step is None or nano_step is None:
            selected_chained_requests.append(chained_request)
            continue

        mini_step = mini_step.split('-')[1]
        nano_step = nano_step.split('-')[1]
        if mini_step not in tree:
            tree[mini_step] = {}

        if nano_step not in tree[mini_step]:
            tree[mini_step][nano_step] = []

        tree[mini_step][nano_step].append(chained_request)

    for mini_campaign in tree:
        print('  %s' % (mini_campaign))
        nano_campaigns = sorted(tree[mini_campaign].keys())
        for n in nano_campaigns:
            print('    %s' % (n))

        selected_chained_requests.extend(tree[mini_campaign][nano_campaigns[-1]])

    if len(chained_requests) > 1:
        prepids = [x['prepid'] for x in chained_requests]
        selected_prepids = [x['prepid'] for x in selected_chained_requests]
        for p in prepids:
            print('%s %s' % ('*' if p in selected_prepids else ' ', p))

    return selected_chained_requests

def chained_request_to_steps(chained_request):
    """
    Split chained request into dictionary of step prepids
    """
    steps = {}
    for req_prepid in chained_request['chain']:
        if 'pLHE' in req_prepid:
            steps['plhe'] = req_prepid
        elif 'GS' in req_prepid:
            steps['gs'] = req_prepid
        elif 'DR' in req_prepid:
            steps['dr'] = req_prepid
        elif 'MiniAOD' in req_prepid:
            steps['miniaod'] = req_prepid
        elif 'NanoAOD' in req_prepid:
            steps['nanoaod'] = req_prepid

    return steps

def get_sample_if_exists(sql_args, cursor, table):
    campaign = sql_args[0]
    chained_request_prepid = sql_args[2]
    dataset_name = sql_args[3]
    root_request_prepid = sql_args[4]
    campaign_equals = '=' if campaign is not None else 'IS'
    chained_request_equals = '=' if chained_request_prepid is not None else 'IS'
    dataset_equals = '=' if dataset_name is not None else 'IS'
    root_request_equals = '=' if root_request_prepid is not None else 'IS'
    rows = [r for r in cursor.execute('''SELECT uid, interested_pwgs, original_interested_pwgs, root_request, miniaod, notes
                                         FROM %s
                                         WHERE campaign %s ? AND
                                               chained_request %s ? AND
                                               dataset %s ? AND
                                               root_request %s ?''' % (table,
                                                                       campaign_equals,
                                                                       chained_request_equals,
                                                                       dataset_equals,
                                                                       root_request_equals),
                                      [campaign,
                                       chained_request_prepid,
                                       dataset_name,
                                       root_request_prepid])]
    return rows

def insert_or_update(sql_args, cursor, table):
    """
    0. campaign,
    1. campaign_group
    2. chained_request_prepid
    3. dataset_name
    4. root_request_prepid
    5. root_request_priority
    6. root_request_total_events
    7. root_request_done_events
    8. root_request_status
    9. root_request_output
    10. miniaod_prepid
    11. miniaod_priority
    12. miniaod_total_events
    13. miniaod_done_events
    14. miniaod_status
    15. miniaod_output
    16. interested_pwgs
    17. original_interested_pwgs
    18. notes
    """
    existing_sample = get_sample_if_exists(sql_args, cursor, table)
    nice_description = '%s %s %s %s' % (sql_args[0], sql_args[2], sql_args[3], sql_args[4])
    if len(existing_sample) > 1:
        print('Something bad, there are multiple %s in %s' % (nice_description, table))
        print(existing_sample)
        return

    if len(existing_sample) == 1:
        existing_sample = existing_sample[0]
        if table == 'samples':
            current_interested_pwgs = set([x.strip().upper() for x in existing_sample[1].split(',') if x.strip()])
            original_interested_pwgs = set([x.strip().upper() for x in existing_sample[2].split(',') if x.strip()])
            current_notes = existing_sample[5]
            # Get root request or MiniAOD request from McM that was used before
            mcm_request = mcm.get('requests', existing_sample[4] if existing_sample[4] else existing_sample[3])
            print('Will check %s' % (mcm_request['prepid']))
            mcm_interested_pwgs = set([x.strip().upper() for x in mcm_request.get('interested_pwg', [])])
            samples_added = current_interested_pwgs - original_interested_pwgs
            samples_removed = original_interested_pwgs - current_interested_pwgs
            mcm_added = mcm_interested_pwgs - original_interested_pwgs
            mcm_removed = original_interested_pwgs - mcm_interested_pwgs
            all_added = samples_added.union(mcm_added)
            all_removed = samples_removed.union(mcm_removed)
            new_pwgs = (original_interested_pwgs - all_removed).union(all_added)
            original_interested_pwgs_string = ','.join(sorted(original_interested_pwgs))
            new_interested_pwgs_string = ','.join(sorted(new_pwgs))
            mcm_notes = mcm_request.get('notes')
            if original_interested_pwgs_string != new_interested_pwgs_string or mcm_notes != current_notes:
                print('Interested PWGs:')
                print('  Reference: %s' % (','.join(original_interested_pwgs)))
                print('  Samples added: %s' % (','.join(samples_added)))
                print('  Samples removed: %s' % (','.join(samples_removed)))
                print('  McM added: %s' % (','.join(mcm_added)))
                print('  McM removed: %s' % (','.join(mcm_removed)))
                print('  All added: %s' % (','.join(all_added)))
                print('  All removed: %s' % (','.join(all_removed)))
                print('  New PWGs: %s' % (','.join(new_pwgs)))
                print('%s must be updated in McM. Set interested PWGs to %s' % (mcm_request['prepid'], new_interested_pwgs_string))
                print('%s must be updated in McM. Set notes to %s' % (mcm_request['prepid'], current_notes))
                mcm_request['interested_pwg'] = list(new_pwgs)
                mcm_request['notes'] = current_notes
                print(mcm.update('requests', mcm_request))
                new_request = sql_args[10] if sql_args[10] else sql_args[4]
                if new_request == mcm_request['prepid']:
                    sql_args[16] = sql_args[17] = new_interested_pwgs_string

        # interested_pwgs = ','.join(sorted(x.strip().upper() for x in sql_args[14].split(',') if x.strip()))
        # if table == 'samples' and samples_interested_pwgs != interested_pwgs:
        #     miniaod_prepid = sql_args[9]
        #     print('This must be updated in McM. %s != %s' % (samples_interested_pwgs, interested_pwgs))
        #     if miniaod_prepid:
        #         req = mcm.get('requests', miniaod_prepid)
        #     else:
        #         req = mcm.get('requests', root_request_prepid)

        #     req['interested_pwg'] = [x for x in samples_interested_pwgs.split(',') if x.strip()]
        #     print(mcm.update('requests', req))
        #     sql_args[14] = samples_interested_pwgs

        print('Updating %s' % (nice_description))
        sql_args.append(existing_sample[0])
        cursor.execute('''UPDATE %s
                          SET campaign = ?,
                              campaign_group = ?,
                              chained_request = ?,
                              dataset = ?,
                              root_request = ?,
                              root_request_priority = ?,
                              root_request_total_events = ?,
                              root_request_done_events = ?,
                              root_request_status = ?,
                              root_request_output = ?,
                              miniaod = ?,
                              miniaod_priority = ?,
                              miniaod_total_events = ?,
                              miniaod_done_events = ?,
                              miniaod_status = ?,
                              miniaod_output = ?,
                              interested_pwgs = ?,
                              original_interested_pwgs = ?,
                              notes = ? WHERE uid = ?''' % (table), sql_args)
    else:
        print('Inserting %s' % (nice_description))
        cursor.execute('''INSERT INTO %s
                          VALUES (NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0)''' % (table), sql_args)

def process_request(request, campaign, cursor, table):
    campaign_group = campaign.replace('wmLHEGEN', '').replace('wmLHEGS', '').replace('pLHE', '').replace('GEN', '').replace('GS', '')
    if len(request['member_of_chain']) == 0:
        # Create a fake chain request
        chained_requests = [{'prepid': None,
                             'chain': [request['prepid']]}]
    else:
        chained_requests = []
        for chained_request_prepid in request['member_of_chain']:
            chained_request = mcm.get('chained_requests', chained_request_prepid)
            if chained_request:
                chained_requests.append(chained_request)
            else:
                print('No %s?' % (chained_request_prepid))

        chained_requests = pick_chained_requests(chained_requests)

    for chained_request in chained_requests:
        steps = chained_request_to_steps(chained_request)
        chained_request_prepid = chained_request['prepid']
        root_request_prepid = chained_request['chain'][0]
        root_request = mcm.get('requests', root_request_prepid)
        dataset_name = root_request['dataset_name']
        root_request_priority = root_request['priority']
        root_request_total_events = root_request['total_events']
        root_request_done_events = root_request['completed_events']
        root_request_status = root_request['status']
        root_request_output = None
        if root_request['output_dataset']:
            root_request_output = root_request['output_dataset'][-1]

        miniaod_prepid = steps.get('miniaod')
        if miniaod_prepid:
            miniaod_request = mcm.get('requests', miniaod_prepid)
        else:
            miniaod_request = {}

        miniaod_priority = miniaod_request.get('priority')
        miniaod_total_events = miniaod_request.get('total_events')
        miniaod_done_events = miniaod_request.get('completed_events')
        miniaod_status = miniaod_request.get('status')
        miniaod_output = None
        if miniaod_request:
            # If MiniAOD exists, use MiniAOD interested PWGs
            interested_pwgs = ','.join(miniaod_request.get('interested_pwg', []))
            notes = miniaod_request.get('notes')

            if miniaod_request['output_dataset']:
                miniaod_output = miniaod_request['output_dataset'][-1]
        else:
            # If MiniAOD does not exist, use root request PWGs
            interested_pwgs = ','.join(root_request.get('interested_pwg', []))
            notes = root_request.get('notes')

        sql_args = [campaign,
                    campaign_group,
                    chained_request_prepid,
                    dataset_name,
                    root_request_prepid,
                    root_request_priority,
                    root_request_total_events,
                    root_request_done_events,
                    root_request_status,
                    root_request_output,
                    miniaod_prepid,
                    miniaod_priority,
                    miniaod_total_events,
                    miniaod_done_events,
                    miniaod_status,
                    miniaod_output,
                    interested_pwgs,
                    interested_pwgs,
                    notes]

        insert_or_update(sql_args, cursor, table)

campaigns = ['RunIISummer19UL16GEN', 'RunIISummer19UL16wmLHEGEN', 'RunIISummer19UL16pLHEGEN',
             'RunIISummer19UL17GEN', 'RunIISummer19UL17wmLHEGEN', 'RunIISummer19UL17pLHEGEN',
             'RunIISummer19UL18GEN', 'RunIISummer19UL18wmLHEGEN', 'RunIISummer19UL18pLHEGEN',
             #'RunIIFall18GS', 'RunIIFall18wmLHEGS', 'RunIIFall18pLHE'
             #'RunIIFall17GS', 'RunIIFall17wmLHEGS', 'RunIIFall17pLHE'
             #'RunIISummer15GS', 'RunIISummer15wmLHEGS', 'RunIIWinter15pLHE'
         ]

conn = sqlite3.connect('data.db')
c = conn.cursor()
# Create table
c.execute('''CREATE TABLE IF NOT EXISTS samples
             (uid integer PRIMARY KEY AUTOINCREMENT,
              campaign text,
              campaign_group text,
              chained_request text,
              dataset text,
              root_request text,
              root_request_priority integer,
              root_request_total_events integer,
              root_request_done_events integer,
              root_request_status text,
              root_request_output text,
              miniaod text,
              miniaod_priority integer,
              miniaod_total_events integer,
              miniaod_done_events integer,
              miniaod_status text,
              miniaod_output text,
              interested_pwgs text,
              original_interested_pwgs text,
              updated integer,
              notes text)''')

c.execute('''CREATE TABLE IF NOT EXISTS twiki_samples
             (uid integer PRIMARY KEY AUTOINCREMENT,
              campaign text,
              campaign_group text,
              chained_request text,
              dataset text,
              root_request text,
              root_request_priority integer,
              root_request_total_events integer,
              root_request_done_events integer,
              root_request_status text,
              root_request_output text,
              miniaod text,
              miniaod_priority integer,
              miniaod_total_events integer,
              miniaod_done_events integer,
              miniaod_status text,
              miniaod_output text,
              interested_pwgs text,
              original_interested_pwgs text,
              updated integer,
              notes text)''')

c.execute('''CREATE TABLE IF NOT EXISTS action_history
             (campaign text NOT NULL,
              dataset text NOT NULL,
              root_request text NOT NULL,
              chained_request text,
              username text NOT NULL,
              role text,
              action text NOT NULL,
              pwg text NOT NULL,
              updated integer)
             ''')

conn.commit()

# TWiki samples - old mode
with open('MainSamplesFall18.txt') as f:
    twiki_samples_fall_18 = [line.strip().split('\t') for line in f if line.strip()]

# Remove this!
# twiki_samples_fall_18 = twiki_samples_fall_18[:5]
# campaigns = [campaigns[3], campaigns[5]]
# Remove this!

twiki_added_requests = set()
for index, sample in enumerate(twiki_samples_fall_18):
    print('%s/%s %s' % (index + 1, len(twiki_samples_fall_18), sample[0]))
    requests_for_sample = mcm.get('requests', query='prepid=%s-%s*&dataset_name=%s' % (sample[4], sample[3], sample[0]))
    if not requests_for_sample:
        continue

    for req in requests_for_sample:
        prepid = req['prepid']
        if prepid in twiki_added_requests:
            # print('%s is already there' % (prepid))
            continue

        if int(req['extension']) != int(sample[1]):
            # print('Extension is wrong %s != %s' % (req['extension'], sample[1]))
            continue

        if int(req['total_events']) != int(sample[2]):
            # print('Total events is wrong %s != %s' % (req['total_events'], sample[2]))
            continue

        twiki_added_requests.add(prepid)
        print('Found %s' % (prepid))
        process_request(req, req['member_of_campaign'], c, 'twiki_samples')
        conn.commit()
        break
    else:
        print('Could not find anything for %s' % (' '.join(sample)))

conn.commit()

# McM samples
for campaign in campaigns:
    # This is a very stupid thing
    requests_in_campaign = mcm.get('requests', query='member_of_campaign=%s' % (campaign))
    for index, request in enumerate(requests_in_campaign):
        print('%s/%s %s' % (index + 1, len(requests_in_campaign), request['prepid']))
        process_request(request, campaign, c, 'samples')
        conn.commit()

    conn.commit()

conn.commit()
conn.close()


# -*- coding: utf-8 -*-

import os
import pytest

from giraffez.__main__ import main
from giraffez.core import MainCommand
from giraffez.errors import *
from giraffez.encrypt import *

@pytest.mark.usefixtures('tmpfiles')
class TestCommandLine(object):
    def test_print_help(self, mocker):
        # Should raise a SystemExit and cause help to print
        with pytest.raises(SystemExit):
            main()

    @pytest.mark.usefixtures('config')
    def test_cmd_error(self, mocker, tmpfiles):
        os.remove(tmpfiles.key)
        create_key_file(tmpfiles.key)
        with pytest.raises(TeradataError):
            MainCommand().run(test_args=["cmd", "select * from dbc.dbcinfo", "--conf", tmpfiles.conf, "--key", tmpfiles.key])

    def test_config_not_found(self, mocker, tmpfiles):
        mock_prompt = mocker.patch('giraffez.core.prompt_bool')
        mock_prompt.return_value = False
        mock_prompt.side_effect = ConfigNotFound("Did it!")
        with pytest.raises(ConfigNotFound):
            MainCommand().run(test_args=["cmd", "select * from dbc.dbcinfo", "--conf", tmpfiles.noconf])