# This function should take in RGB image input
# Analyze that image using your feature creation code and output a one-hot encoded label

#boolean flag if hot encoder is double encoded
def double_encoded(one_hot_encoded):
    if (np.sum(one_hot_encoded) > 1):
        return True
    else:
        return False

# boolean flag if hot encoder has a label
def has_label(one_hot_encoded):
    if (np.sum(one_hot_encoded) > 0):
        return True
    else:
        return False

# hot encode the image based on where the mean of the feature is in relation to the height of the
# traffic light.
def hot_encode_height(mean):
    # bin ranges
    # red = 0, 15
    # yellow = 10, 23
    # green = 20, 32
    one_hot_encoded = [0, 0, 0]

    if mean >= 0 and mean < 15:
        one_hot_encoded[0] = 1
    if mean >= 10 and mean < 23:
        one_hot_encoded[1] = 1
    if mean >= 20 and mean <= 32:
        one_hot_encoded[2] = 1

    return one_hot_encoded

def estimate_value(rgb_image):

    feature = feature_value(rgb_image)

    # get mean and bimodal boolean
    max_list2 = max_idx_rank(feature)
    mean = max_list2[0]
    bimodal = is_bimodal(max_list2, feature)

    one_hot_encoded = hot_encode_height(mean)
    return one_hot_encoded, bimodal

def estimate_hueXvalue(rgb_image):

    feature = feature_valueXHue(rgb_image)

    # get mean and bimodal boolean
    max_list2 = max_idx_rank(feature)
    mean = max_list2[0]
    bimodal = is_bimodal(max_list2, feature)

    one_hot_encoded = hot_encode_height(mean)
    return one_hot_encoded, bimodal

def estimate_color(rgb_image):
    feature = feature_rgb(rgb_image)

    one_hot_encoded = [0, 0, 0]
    # sum channels representing each light color
    red_sum = np.sum(feature[:,:,0])
    green_sum = np.sum(feature[:,:,1])
    yellow_sum = np.sum(feature[:,:,2])

    # one hot encode the color who has the greatest sum
    if red_sum > (yellow_sum + green_sum):
        one_hot_encoded[0] = 1
    if yellow_sum > (green_sum + red_sum):
        one_hot_encoded[1] = 1
    if green_sum > (yellow_sum + red_sum):
        one_hot_encoded[2] = 1

    return one_hot_encoded

# rules
    # * if all hues detected, or white is predominant. color is most liekly yellow.
    # * if a bimodal distribution of value vs traffic light length is found.
    #     - compute hue * value.
    #     - use hue * value mean retrieved from this plot.

def estimate_label(rgb_image, print_stats = False):

    ## TODO: Extract feature(s) from the RGB image and use those features to
    ## classify the image and output a one-hot encoded label
    predicted_label = [0,0,0]
    predicted_label_height = []
    predicted_label_color = []
    bimodal = False
    # copy picture
    image = np.copy(rgb_image)

    # get hot encode for value vs height
    predicted_label_Value, bimodal_value = estimate_value(image)

    if print_stats:
        print('height: {}, bimodal: {}'.format(predicted_label_Value, bimodal_value))

    # if bimodal compute hue * value and get the hot encode vs height
    if bimodal == True:
        predicted_label_height, bimodal = estimate_hueXvalue(image)
    else:
        predicted_label_height = predicted_label_Value
        bimodal = bimodal_value

    # get hot encode for color
    predicted_label_color = estimate_color(image)

    if print_stats:
        print('height: {}, bimodal: {}'.format(predicted_label_height, bimodal))
        print('color: {}'.format(predicted_label_color))

    #union of two hot_encoded labels
    union = matrix_multiplication(predicted_label_height, predicted_label_color)

    #if height label and color label match return encoder
    if predicted_label_height == predicted_label_color:
        predicted_label = predicted_label_height
        # check to make sure two colors arn't labeled
        if not double_encoded(predicted_label_height):
            return predicted_label

    # if height label and color label do not match
    #if double encoded
    if double_encoded(predicted_label_height):
        #use color if not double encoded
        if not double_encoded(predicted_label_color) and has_label(predicted_label_color):
            # check to make sure estimates do not conflict, and contain a union
            if np.sum(union) > 0:
                predicted_label = predicted_label_color
            else:
                #if double encoded on green or red choose red
                if predicted_label_height[0]*predicted_label_height[1] > 0:
                    predicted_label = [1, 0, 0]
            return predicted_label
        #if color is double encoded
        else:
            # if they are encoded the same choose red if not use the union
            if np.sum(union) == 1:
                predicted_label = union
            else:
                predicted_label = [1, 0, 0]
            return predicted_label
    # if height label is not double labelled and conflicts with color
    # choose color if available
    else:
        if has_label(predicted_label_color) and predicted_label_color != [0, 1, 0]:
            predicted_label = predicted_label_color
        # if no color got off of value feature
        # if bimodal
        else:
            if bimodal_value:
                predicted_label = predicted_label_height
            else:
                predicted_label = predicted_label_Value

    return predicted_label

image, label, image_num = getImage(color='red', Random=True, showImg=False)
#image_num = random.randint(0, total_num_of_images-1)
#image_num = 757
test_im = STANDARDIZED_LIST[image_num][0]
test_label = STANDARDIZED_LIST[image_num][1]

encoder = estimate_label(test_im)
print('image: ', image_num)
print('guess: ', encoder)
print('actual: ', test_label)
plt.imshow(test_im)
plt.show()


import pandas as pd
import os

def excel():
    folders = os.listdir("classifiedD")
    for folder in folders:
        files = os.listdir("classifiedD/"+folder)
        print(folder)
        for file in files:
            if ".h5" in file:
                print(file)
                hdf = pd.HDFStore("classifiedD/"+folder+"/"+file)
                excel = pd.ExcelWriter("classifiedD/"+folder+"/"+file[:-3]+".xlsx")
                hdf['df'].to_excel(excel, 'Sheet1')
                hdf['dp'].to_excel(excel,'Sheet2')
                excel.save()
        os.system("clear")


INPUT_SIZE = 30
EMBEDDING_OUTPUT = 32
K = 8
CH_NUM = 8
CLASS_NUM = 6
NUM_LAYERS = 2
L2_REG = 5e-4
LR = 0.1
DROPOUT_X = 0.84
DROPOUT_A = 0.001
BATCH_SIZE = 2500


'''
attribs: value, next, prev; methods: get_next, set_next, get_prev, set_prev, get_value
'''
class DLLNode(object):
	def __init__(self, value):
		self.value = value
		self.next = None
		self.prev = None

	def get_value(self):
		return self.value

	def get_next(self):
		return self.next

	def set_next(self, node):
		self.next = node

	def get_prev(self):
		return self.prev

	def set_prev(self, node):
		self.prev = node

'''
attribs: head; methods: size, add_top, add_bot, search, delete
'''
class DLL(object):
	def __init__(self):
		self.head = None

	def size(self):
		count = 0
		current = self.head
		while current:
			count += 1
			current = current.get_next()
		return count

	def add_top(self, value):
		node = DLLNode(value)
		current = self.head
		self.head = node
		node.set_next(current)
		if current:
			current.set_prev(node)

	def add_bot(self, value):
		node = DLLNode(value)
		prev = None
		current = self.head
		while current:
			prev = current
			current = current.get_next()
		if prev:
			prev.set_next(node)
			node.set_prev(prev)
		else:
			self.head = node

	def search(self, value): # returns first node from top which contains value else None
		found = False
		current = self.head
		while not found and current:
			found = current.get_value() == value
			if not found:
				current = current.get_next()
		return current

	def delete(self, node):
		prev = node.get_prev()
		next = node.get_next()
		if not prev:
			self.head = next
		elif not next:
			prev.set_next(None)
		else:
			prev.set_next(next)
			next.set_prev(prev)


import logging
import random
import re
from types import SimpleNamespace

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

import stanza.models.classifiers.classifier_args as classifier_args
import stanza.models.classifiers.data as data
from stanza.models.common.vocab import PAD_ID, UNK_ID
from stanza.models.common.data import get_long_tensor, sort_all
# TODO: move CharVocab to common
from stanza.models.pos.vocab import CharVocab

"""
The CNN classifier is based on Yoon Kim's work:

https://arxiv.org/abs/1408.5882

The architecture is simple:

- Embedding at the bottom layer
  - separate learnable entry for UNK, since many of the embeddings we have use 0 for UNK
- Some number of conv2d layers over the embedding
- Maxpool layers over small windows, window size being a parameter
- FC layer to the classification layer

One experiment which was run and found to be a bit of a negative was
putting a layer on top of the pretrain.  You would think that might
help, but dev performance went down for each variation of
  - trans(emb)
  - relu(trans(emb))
  - dropout(trans(emb))
  - dropout(relu(trans(emb)))
"""

logger = logging.getLogger('stanza')

class CNNClassifier(nn.Module):
    def __init__(self, pretrain, extra_vocab, labels,
                 charmodel_forward, charmodel_backward,
                 args):
        """
        pretrain is a pretrained word embedding.  should have .emb and .vocab

        extra_vocab is a collection of words in the training data to
        be used for the delta word embedding, if used.  can be set to
        None if delta word embedding is not used.

        labels is the list of labels we expect in the training data.
        Used to derive the number of classes.  Saving it in the model
        will let us check that test data has the same labels

        args is either the complete arguments when training, or the
        subset of arguments stored in the model save file
        """
        super(CNNClassifier, self).__init__()
        self.labels = labels
        # we build a separate config out of the args so that we can easily save it in torch
        self.config = SimpleNamespace(filter_channels = args.filter_channels,
                                      filter_sizes = args.filter_sizes,
                                      fc_shapes = args.fc_shapes,
                                      dropout = args.dropout,
                                      num_classes = len(labels),
                                      wordvec_type = args.wordvec_type,
                                      extra_wordvec_method = args.extra_wordvec_method,
                                      extra_wordvec_dim = args.extra_wordvec_dim,
                                      extra_wordvec_max_norm = args.extra_wordvec_max_norm,
                                      char_lowercase = args.char_lowercase,
                                      charlm_projection = args.charlm_projection,
                                      model_type = 'CNNClassifier')

        if args.char_lowercase:
            self.char_case = lambda x: x.lower()
        else:
            self.char_case = lambda x: x

        self.unsaved_modules = []

        emb_matrix = pretrain.emb
        self.add_unsaved_module('embedding', nn.Embedding.from_pretrained(torch.from_numpy(emb_matrix), freeze=True))
        self.vocab_size = emb_matrix.shape[0]
        self.embedding_dim = emb_matrix.shape[1]

        self.add_unsaved_module('charmodel_forward', charmodel_forward)
        if charmodel_forward is not None:
            self.add_unsaved_module('charmodel_forward_vocab', charmodel_forward.char_vocab())
            logger.debug("Got forward char model of dimension {}".format(charmodel_forward.hidden_dim()))
        self.add_unsaved_module('charmodel_backward', charmodel_backward)
        if charmodel_backward is not None:
            self.add_unsaved_module('charmodel_backward_vocab', charmodel_backward.char_vocab())
            logger.debug("Got backward char model of dimension {}".format(charmodel_backward.hidden_dim()))

        # The Pretrain has PAD and UNK already (indices 0 and 1), but we
        # possibly want to train UNK while freezing the rest of the embedding
        # note that the /10.0 operation has to be inside nn.Parameter unless
        # you want to spend a long time debugging this
        self.unk = nn.Parameter(torch.randn(self.embedding_dim) / np.sqrt(self.embedding_dim) / 10.0)

        self.vocab_map = { word: i for i, word in enumerate(pretrain.vocab) }

        if self.config.extra_wordvec_method is not classifier_args.ExtraVectors.NONE:
            if not extra_vocab:
                raise ValueError("Should have had extra_vocab set for extra_wordvec_method {}".format(self.config.extra_wordvec_method))
            if not args.extra_wordvec_dim:
                self.config.extra_wordvec_dim = self.embedding_dim
            if self.config.extra_wordvec_method is classifier_args.ExtraVectors.SUM:
                if self.config.extra_wordvec_dim != self.embedding_dim:
                    raise ValueError("extra_wordvec_dim must equal embedding_dim for {}".format(self.config.extra_wordvec_method))

            self.extra_vocab = list(extra_vocab)
            self.extra_vocab_map = { word: i for i, word in enumerate(self.extra_vocab) }
            # TODO: possibly add regularization specifically on the extra embedding?
            self.extra_embedding = nn.Embedding(num_embeddings = len(extra_vocab),
                                                embedding_dim = self.config.extra_wordvec_dim,
                                                max_norm = self.config.extra_wordvec_max_norm,
                                                padding_idx = 0)
            logger.debug("Extra embedding size: {}".format(self.extra_embedding.weight.shape))
        else:
            self.extra_vocab = None
            self.extra_vocab_map = None
            self.config.extra_wordvec_dim = 0
            self.extra_embedding = None

        # Pytorch is "aware" of the existence of the nn.Modules inside
        # an nn.ModuleList in terms of parameters() etc
        if self.config.extra_wordvec_method is classifier_args.ExtraVectors.NONE:
            total_embedding_dim = self.embedding_dim
        elif self.config.extra_wordvec_method is classifier_args.ExtraVectors.SUM:
            total_embedding_dim = self.embedding_dim
        elif self.config.extra_wordvec_method is classifier_args.ExtraVectors.CONCAT:
            total_embedding_dim = self.embedding_dim + self.config.extra_wordvec_dim
        else:
            raise ValueError("unable to handle {}".format(self.config.extra_wordvec_method))

        if charmodel_forward is not None:
            if args.charlm_projection:
                self.charmodel_forward_projection = nn.Linear(charmodel_forward.hidden_dim(), args.charlm_projection)
                total_embedding_dim += args.charlm_projection
            else:
                self.charmodel_forward_projection = None
                total_embedding_dim += charmodel_forward.hidden_dim()

        if charmodel_backward is not None:
            if args.charlm_projection:
                self.charmodel_backward_projection = nn.Linear(charmodel_backward.hidden_dim(), args.charlm_projection)
                total_embedding_dim += args.charlm_projection
            else:
                self.charmodel_backward_projection = None
                total_embedding_dim += charmodel_backward.hidden_dim()

        self.conv_layers = nn.ModuleList([nn.Conv2d(in_channels=1,
                                                    out_channels=self.config.filter_channels,
                                                    kernel_size=(filter_size, total_embedding_dim))
                                          for filter_size in self.config.filter_sizes])

        previous_layer_size = len(self.config.filter_sizes) * self.config.filter_channels
        fc_layers = []
        for shape in self.config.fc_shapes:
            fc_layers.append(nn.Linear(previous_layer_size, shape))
            previous_layer_size = shape
        fc_layers.append(nn.Linear(previous_layer_size, self.config.num_classes))
        self.fc_layers = nn.ModuleList(fc_layers)

        self.max_window = max(self.config.filter_sizes)

        self.dropout = nn.Dropout(self.config.dropout)

    def add_unsaved_module(self, name, module):
        self.unsaved_modules += [name]
        setattr(self, name, module)

    def build_char_reps(self, batch_chars, batch_offsets, device, forward=True):
        if forward:
            model = self.charmodel_forward
            vocab = self.charmodel_forward_vocab
            projection = self.charmodel_forward_projection
        else:
            model = self.charmodel_backward
            vocab = self.charmodel_backward_vocab
            projection = self.charmodel_backward_projection

        batch_charlens = [len(x) for x in batch_chars]
        chars_sorted, char_orig_idx = sort_all([batch_chars, batch_offsets], batch_charlens)
        batch_chars, batch_offsets = chars_sorted
        batch_charlens = [len(x) for x in batch_chars]
        chars = get_long_tensor(batch_chars, len(batch_chars),
                                pad_id=vocab.unit2id(' ')).to(device=device)
        char_reps = model.get_representation(chars, batch_offsets,
                                             batch_charlens, char_orig_idx)
        char_reps = char_reps.data
        if projection is not None:
            char_reps = projection(char_reps)
        char_reps = torch.reshape(char_reps, [max(len(x) for x in batch_offsets), len(batch_chars), char_reps.shape[-1]])
        char_reps = torch.transpose(char_reps, 0, 1)

        return char_reps

    def forward(self, inputs, device=None):
        if not device:
            # assume all pieces are on the same device
            device = next(self.parameters()).device

        # we will pad each phrase so either it matches the longest
        # conv or the longest phrase in the input, whichever is longer
        max_phrase_len = max(len(x) for x in inputs)
        if self.max_window > max_phrase_len:
            max_phrase_len = self.max_window

        batch_indices = []
        batch_unknowns = []
        extra_batch_indices = []
        batch_forward_chars = []
        batch_forward_offsets = []
        batch_backward_chars = []
        batch_backward_offsets = []
        for phrase in inputs:
            # we use random at training time to try to learn different
            # positions of padding.  at test time, though, we want to
            # have consistent results, so we set that to 0 begin_pad
            if self.training:
                begin_pad_width = random.randint(0, max_phrase_len - len(phrase))
            else:
                begin_pad_width = 0
            end_pad_width = max_phrase_len - begin_pad_width - len(phrase)

            # the initial lists are the length of the begin padding
            sentence_indices = [PAD_ID] * begin_pad_width
            # the "unknowns" will be the locations of the unknown words.
            # these locations will get the specially trained unknown vector
            sentence_unknowns = []

            for word in phrase:
                if word in self.vocab_map:
                    sentence_indices.append(self.vocab_map[word])
                    continue
                new_word = word.replace("-", "")
                # google vectors have words which are all dashes
                if len(new_word) == 0:
                    new_word = word
                if new_word in self.vocab_map:
                    sentence_indices.append(self.vocab_map[new_word])
                    continue

                if new_word[-1] == "'":
                    new_word = new_word[:-1]
                    if new_word in self.vocab_map:
                        sentence_indices.append(self.vocab_map[new_word])
                        continue

                # TODO: split UNK based on part of speech?  might be an interesting experiment
                sentence_unknowns.append(len(sentence_indices))
                sentence_indices.append(PAD_ID)

            sentence_indices.extend([PAD_ID] * end_pad_width)
            batch_indices.append(sentence_indices)
            batch_unknowns.append(sentence_unknowns)

            if self.extra_vocab:
                extra_sentence_indices = [PAD_ID] * begin_pad_width
                for word in phrase:
                    if word in self.extra_vocab_map:
                        # the extra vocab is initialized from the
                        # words in the training set, which means there
                        # would be no unknown words.  to occasionally
                        # train the extra vocab's unknown words, we
                        # replace 1% of the words with UNK
                        # we don't do that for the original embedding
                        # on the assumption that there may be some
                        # unknown words in the training set anyway
                        # TODO: maybe train unk for the original embedding?
                        if self.training and random.random() < 0.01:
                            extra_sentence_indices.append(UNK_ID)
                        else:
                            extra_sentence_indices.append(self.extra_vocab_map[word])
                    else:
                        extra_sentence_indices.append(UNK_ID)
                extra_sentence_indices.extend([PAD_ID] * end_pad_width)
                extra_batch_indices.append(extra_sentence_indices)

            if self.charmodel_forward is not None:
                start_id, end_id = self.charmodel_forward_vocab.unit2id('\n'), self.charmodel_forward_vocab.unit2id(' ') # special token
                processed_phrase = ([[end_id]] * begin_pad_width +
                                    [self.charmodel_forward_vocab.map([self.char_case(x) for x in w]) + [end_id] for w in phrase] +
                                    [[end_id]] * end_pad_width)
                chars = [start_id]
                offsets = []
                for word in processed_phrase:
                    # -1 because we want the position of the last character
                    chars.extend(word)
                    offsets.append(len(chars)-1)
                batch_forward_chars.append(chars)
                batch_forward_offsets.append(offsets)

            if self.charmodel_backward is not None:
                start_id, end_id = self.charmodel_backward_vocab.unit2id('\n'), self.charmodel_backward_vocab.unit2id(' ') # special token
                processed_phrase = ([[end_id]] * begin_pad_width +
                                    [self.charmodel_backward_vocab.map([self.char_case(x) for x in w[::-1]]) + [end_id] for w in phrase[::-1]] +
                                    [[end_id]] * end_pad_width)
                chars = [start_id]
                offsets = []
                for word in processed_phrase:
                    # -1 because we want the position of the last character
                    chars.extend(word)
                    # backwards because that seems to be what get_representation wants
                    offsets = [len(chars)-1] + offsets
                batch_backward_chars.append(chars)
                batch_backward_offsets.append(offsets)

        # creating a single large list with all the indices lets us
        # create a single tensor, which is much faster than creating
        # many tiny tensors
        # we can convert this to the input to the CNN
        # it is padded at one or both ends so that it is now num_phrases x max_len x emb_size
        # there are two ways in which this padding is suboptimal
        # the first is that for short sentences, smaller windows will
        #   be padded to the point that some windows are entirely pad
        # the second is that a sentence S will have more or less padding
        #   depending on what other sentences are in its batch
        # we assume these effects are pretty minimal
        batch_indices = torch.tensor(batch_indices, requires_grad=False, device=device)
        input_vectors = self.embedding(batch_indices)
        # we use the random unk so that we are not necessarily
        # learning to match 0s for unk
        for phrase_num, sentence_unknowns in enumerate(batch_unknowns):
            for unknown in sentence_unknowns:
                input_vectors[phrase_num, unknown, :] = self.unk

        if self.extra_vocab:
            extra_batch_indices = torch.tensor(extra_batch_indices, requires_grad=False, device=device)
            extra_input_vectors = self.extra_embedding(extra_batch_indices)
            if self.config.extra_wordvec_method is classifier_args.ExtraVectors.CONCAT:
                all_inputs = [input_vectors, extra_input_vectors]
            elif self.config.extra_wordvec_method is classifier_args.ExtraVectors.SUM:
                all_inputs = [input_vectors + extra_input_vectors]
            else:
                raise ValueError("unable to handle {}".format(self.config.extra_wordvec_method))
        else:
            all_inputs = [input_vectors]

        if self.charmodel_forward is not None:
            char_reps_forward = self.build_char_reps(batch_forward_chars, batch_forward_offsets, device, forward=True)
            all_inputs.append(char_reps_forward)

        if self.charmodel_backward is not None:
            char_reps_backward = self.build_char_reps(batch_backward_chars, batch_backward_offsets, device, forward=False)
            all_inputs.append(char_reps_backward)

        if len(all_inputs) > 1:
            input_vectors = torch.cat(all_inputs, dim=2)
        else:
            input_vectors = all_inputs[0]

        # reshape to fit the input tensors
        x = input_vectors.unsqueeze(1)

        conv_outs = [self.dropout(F.relu(conv(x).squeeze(3)))
                     for conv in self.conv_layers]
        pool_outs = [F.max_pool1d(out, out.shape[2]).squeeze(2) for out in conv_outs]
        pooled = torch.cat(pool_outs, dim=1)

        previous_layer = pooled
        for fc in self.fc_layers[:-1]:
            previous_layer = self.dropout(F.relu(fc(previous_layer)))
        out = self.fc_layers[-1](previous_layer)
        return out

# TODO: make some of the following methods part of the class

# TODO: all this code is basically the same as for POS and NER.  Should refactor
def save(filename, model, skip_modules=True):
    model_state = model.state_dict()
    # skip saving modules like pretrained embeddings, because they are large and will be saved in a separate file
    if skip_modules:
        skipped = [k for k in model_state.keys() if k.split('.')[0] in model.unsaved_modules]
        for k in skipped:
            del model_state[k]
    params = {
        'model': model_state,
        'config': model.config,
        'labels': model.labels,
        'extra_vocab': model.extra_vocab,
    }
    try:
        torch.save(params, filename, _use_new_zipfile_serialization=False)
        logger.info("Model saved to {}".format(filename))
    except (KeyboardInterrupt, SystemExit):
        raise
    except BaseException as e:
        logger.warning("Saving failed to {}... continuing anyway.  Error: {}".format(filename, e))

def load(filename, pretrain, charmodel_forward, charmodel_backward):
    try:
        checkpoint = torch.load(filename, lambda storage, loc: storage)
    except BaseException:
        logger.exception("Cannot load model from {}".format(filename))
        raise
    logger.debug("Loaded model {}".format(filename))

    # TODO: should not be needed when all models have this value set
    setattr(checkpoint['config'], 'char_lowercase', getattr(checkpoint['config'], 'char_lowercase', False))
    setattr(checkpoint['config'], 'charlm_projection', getattr(checkpoint['config'], 'charlm_projection', None))

    # TODO: the getattr is not needed when all models have this baked into the config
    model_type = getattr(checkpoint['config'], 'model_type', 'CNNClassifier')
    if model_type == 'CNNClassifier':
        extra_vocab = checkpoint.get('extra_vocab', None)
        model = CNNClassifier(pretrain=pretrain,
                              extra_vocab=extra_vocab,
                              labels=checkpoint['labels'],
                              charmodel_forward=charmodel_forward,
                              charmodel_backward=charmodel_backward,
                              args=checkpoint['config'])
    else:
        raise ValueError("Unknown model type {}".format(model_type))
    model.load_state_dict(checkpoint['model'], strict=False)

    logger.debug("-- MODEL CONFIG --")
    for k in model.config.__dict__:
        logger.debug("  --{}: {}".format(k, model.config.__dict__[k]))

    logger.debug("-- MODEL LABELS --")
    logger.debug("  {}".format(" ".join(model.labels)))

    return model

def label_text(model, text, batch_size=None, reverse_label_map=None, device=None):
    """
    Given a list of sentences, return the model's results on that text.
    """
    model.eval()
    if reverse_label_map is None:
        reverse_label_map = {x: y for (x, y) in enumerate(model.labels)}
    if device is None:
        device = next(model.parameters()).device

    text = [data.update_text(s, model.config.wordvec_type) for s in text]

    if batch_size is None:
        intervals = [(0, len(text))]
    else:
        # TODO: results would be better if we sort by length and then unsort
        intervals = [(i, min(i+batch_size, len(text))) for i in range(0, len(text), batch_size)]
    labels = []
    for interval in intervals:
        if interval[1] - interval[0] == 0:
            # this can happen for empty text
            continue
        output = model(text[interval[0]:interval[1]], device)
        predicted = torch.argmax(output, dim=1)
        labels.extend(predicted.tolist())

    logger.debug("Found labels")
    for (label, sentence) in zip(labels, text):
        logger.debug((label, sentence))

    return labels


import logging

import numpy as np
from pandas import Categorical

from core import prepare
from core import draw

# === helpers === #
BENCH_NAME = 'apache'
EXP_NAME = '%s_perf' % BENCH_NAME

# NOTE 1: since all compilers perform roughly the same, we leave gcc only for simplicity
def main(t="perf"):
    logging.info("Processing data")
    df = prepare.process_results(t)
    if t == "tput":
        prepare.reorder_compilers(df, t)
        plot = draw.LinePlotTput()
        plot.get_data(df, [])
        plot.build_plot(
            xlim=(0, 54),
            xticks=range(0, 60, 10),
            ylim=(0.42, 0.92),
            yticks=np.arange(0.1, 1, 0.1),
        )
        plot.save_plot("apache_%s.pdf" % t)

    else:
        logging.error("Unknown plot type")
        exit(-1)


import json

from django.http import HttpResponse

from .RedisUtil import RedisUtil

from .base_response import BaseResponse

# 获取redis key列表
def keys(request):
    key_list = RedisUtil.get_conn().keys()
    print(len(key_list))
    result_list = []
    for key in key_list:
        value = {"key": key, "type": RedisUtil.get_conn().type(key)}
        result_list.append(value)
    return HttpResponse(json.dumps(result_list), content_type="application/json")

def db_size(request):
    return HttpResponse(RedisUtil.get_conn().dbsize(), content_type="application/json")

def get_value(request, key):
    return HttpResponse(RedisUtil.get_conn().get(key), content_type="application/json")

def set_value(request, key, value):
    print(key, value)
    RedisUtil.get_conn().type(key)
    r = BaseResponse.success(RedisUtil.get_conn().set(key, value))
    return HttpResponse(json.dumps(r.__dict__), content_type="application/json")

def value_clear(request, key):
    is_exist = RedisUtil.get_conn().exists(key)
    if is_exist:
        value_type = RedisUtil.get_conn().type(key)
        if value_type == "string":
            RedisUtil.get_conn().set(key, "")
    return None


import re

_UTF8_TYPES = (bytes, type(None))
unicode_type = str
_TO_UNICODE_TYPES = (unicode_type, type(None))

def force_text(value):
    """Converts a string argument to a unicode string.

    If the argument is already a unicode string or None, it is returned
    unchanged.  Otherwise it must be a byte string and is decoded as utf8.
    """
    if isinstance(value, _TO_UNICODE_TYPES):
        return value
    if not isinstance(value, bytes):
        raise TypeError(
            "Expected bytes, unicode, or None; got %r" % type(value)
        )
    return value.decode("utf-8")

def force_bytes(value):
    """Converts a string argument to a byte string.

    If the argument is already a byte string or None, it is returned unchanged.
    Otherwise it must be a unicode string and is encoded as utf8.
    """
    if isinstance(value, _UTF8_TYPES):
        return value
    if not isinstance(value, unicode_type):
        raise TypeError(
            "Expected bytes, unicode, or None; got %r" % type(value)
        )
    return value.encode("utf-8")

def get_valid_filename(s):
    """
    Returns the given string converted to a string that can be used for a clean
    filename. Specifically, leading and trailing spaces are removed; other
    spaces are converted to underscores; and anything that is not a unicode
    alphanumeric, dash, underscore, or dot, is removed.
    >>> get_valid_filename("john's portrait in 2004.jpg")
    'johns_portrait_in_2004.jpg'
    """
    s = force_text(s).strip().replace(' ', '_')
    return re.sub(r'(?u)[^-\w.]', '', s)


# coding=utf8

from bs4 import BeautifulSoup
from operator import itemgetter
import base64, Commands, Config, importlib, Logger, praw, re, requests, Soaker, socket, ssl, sys, time, types, URLInfo

class IRC:
    def __init__(self):
        Config.config(self)
        self.activeDict = {}
        for channel in self.info['CHAN']:
            self.activeDict[channel] = {}
        Commands.redditAPI(self)
        Soaker.Soaker(self)
        self.Connect()
        self.fLog = {}
        self.Main()

    def Connect(self):
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((self.info['HOST'], int(self.info['PORT'])))
        self.irc = ssl.wrap_socket(sock)
        connectMSG = []
        if self.info['SASL']:
            self.ircSend('CAP LS')
        self.ircSend('NICK %s' % self.info['NICK'])
        self.ircSend('USER %s %s %s :%s' % (self.info['NICK'], self.info['NICK'], self.info['NICK'], self.info['NAME']))
        self.ircSend('JOIN %s' % ','.join(self.info['CHAN']))

    def Main(self):
        while True:
            rawdata = self.irc.recv(4096)
            try:
                data = rawdata.decode('utf-8')
            except UnicodeDecodeError:
                try:
                    data = rawdata.decode('cp1252')
                except UnicodeDecodeError:
                    try:
                        data = rawdata.decode('iso8859-1')
                    except:
                        continue
            lines = str(data).split('\n')
            for line in lines:
                if len(line) < 1:
                    continue

                Log = Logger.interpret(line)
                self.fLog[Log['time']] = Log

                # reply to pings
                if Log['command'] == 'PING':
                    self.ircSend('PONG :%s' % Log['trail'][0])
                    continue

                # SASL
                if self.info['SASL']:
                    if Log['command'] == 'CAP':
                        if Log['parameters'] [0] == '*' and Log['parameters'][1] == 'LS':
                            self.ircSend('CAP REQ :%s' % ' '.join(Log['trail']))
                            continue
                        if Log['parameters'] [1] == 'ACK':
                            self.ircSend('AUTHENTICATE PLAIN')
                            continue
                    if Log['command'] == 'AUTHENTICATE' and Log['parameters'][0] == '+':
                        sasl_token = '\0'.join([self.info['NICK'], self.info['NICK'], self.info['PASS']])
                        self.ircSend('AUTHENTICATE %s' % base64.b64encode(sasl_token.encode('utf-8')).decode('utf-8'))
                        continue
                    if Log['command'] == '903':
                        self.ircSend('CAP END')
                        self.ircSend('JOIN %s' % ','.join(self.info['CHAN']))
                        continue

                # checks when identified with nickserv
                if Log['command'] == 'NOTICE' and Log['nick'] == 'NickServ':
                    if len(Log['trail']) > 3:
                        if 'registered' in Log['trail'][3]:
                            self.ircSend('PRIVMSG NickServ :identify %s' % self.info['PASS'])
                            continue
                        if Log['trail'][3] == 'identified':
                            self.ircSend('JOIN %s' % ','.join(self.info['CHAN']))
                            continue

                # checks for INVITE received
                if Log['command'] == 'INVITE' and Log['parameters'][0] == self.info['NICK']:
                    if Log['trail'][0] not in self.info['CHAN']:
                        self.info['CHAN'].append(Log['trail'][0])
                        self.updateFile()
                        self.ircSend('JOIN %s' % Log['trail'][0])

                # checks channel join
                if Log['command'] == 'JOIN':
                    if Log['nick'] == self.info['NICK']:
                        self.activeDict[Log['parameters'][0]] = {}
                        self.ircSend('WHO %s %%na' % Log['parameters'][0])
                    elif Log['nick'] not in self.activeDict[Log['parameters'][0]]:
                        self.activeDict[Log['parameters'][0]][Log['nick']] = 0
                        if Log['nick'] not in list(self.userDict.values()):
                            self.ircSend('WHOIS %s' % Log['nick'])
                    continue

                # checks nick change
                if Log['command'] == 'NICK':
                    if Log['nick'] == self.info['NICK']:
                        self.info['NICK'] = Log['trail'][0]
                    else:
                        for channels in self.info['CHAN']:
                            if Log['nick'] in self.activeDict[channels]:
                                self.activeDict[channels][Log['trail'][0]] = self.activeDict[channels][Log['nick']]
                                del self.activeDict[channels][Log['nick']]
                        self.ircSend('WHOIS %s' % Log['trail'][0])
                    continue

                # parses NAMES result
                if str(Log['command']) == '353' and len(Log['parameters']) > 2:
                    if Log['parameters'][-1] not in self.activeDict:
                        self.activeDict[Log['parameters'][-1]] = {}
                    for names in Log['trail']:
                        names = names.lstrip('@+')
                        if names != self.info['NICK']:
                            self.activeDict[Log['parameters'][-1]][names] = 0
                    continue

                # parses WHOIS result
                if (str(Log['command']) == '330' or str(Log['command']) == '354') and len(Log['parameters']) > 2:
                    if Log['parameters'][2] != str(0):
                        if Log['parameters'][2] not in self.userDict:
                            self.userDict[Log['parameters'][2]] = []
                        if Log['parameters'][1] not in self.userDict[Log['parameters'][2]] and Log['parameters'][1] != self.info['NICK']:
                            self.userDict[Log['parameters'][2]].append(Log['parameters'][1])
                        self.updateFile()
                    continue

                # updates active list if user leaves
                if Log['command'] == 'PART':
                    if Log['nick'] in self.activeDict[Log['parameters'][0]]:
                        del self.activeDict[Log['parameters'][0]][Log['nick']]
                    continue
                if Log['command'] == 'QUIT':
                    for channels in self.info['CHAN']:
                        if Log['nick'] in self.activeDict[channels]:
                            del self.activeDict[channels][Log['nick']]
                    continue

                # checks when PRIVMSG received
                if Log['command'] == 'PRIVMSG':
                    if Log['context'] not in self.info['UNLOAD'].keys():
                        self.info['UNLOAD'][Log['context']] = []

                    # builds last spoke list
                    if Log['context'] not in self.activeDict:
                        self.activeDict[Log['context']] = {}
                    self.activeDict[Log['context']][Log['nick']] = Log['time']
                    validList = []
                    for group in self.userDict.values():
                        validList.extend(group)
                    if Log['nick'] not in validList and Log['cap'] == '+':
                        self.ircSend('WHOIS %s' % Log['nick'])

                    if Log['host'] in self.info['OWNER'] + self.info['SUDOER']:
                        if len(Log['trail']) > 1:
                            def imports():
                                for name, val in globals().items():
                                    if isinstance(val, types.ModuleType):
                                        yield val.__name__
                            try:
                                if Log['trail'][0].lower() == '!load':
                                    if Log['trail'][1] in self.info['UNLOAD'][Log['context']]:
                                        self.info['UNLOAD'][Log['context']].remove(Log['trail'][1])
                                        self.PRIVMSG(Log['context'],'Module \'%s\' loaded' % Log['trail'][1])

                                if Log['trail'][0].lower() == '!unload':
                                    if Log['trail'][1] in list(imports()) and Log['trail'][1] not in self.info['UNLOAD'][Log['context']]:
                                        self.info['UNLOAD'][Log['context']].append(Log['trail'][1])
                                        self.PRIVMSG(Log['context'],'Module \'%s\' unloaded' % Log['trail'][1])
                                        self.updateFile()

                                if Log['trail'][0].lower() == '!reload':
                                    module = __import__(Log['trail'][1])
                                    if Log['trail'][1] == 'IRCbot':
                                        self.ircSend('QUIT')
                                        self.irc.close()
                                    if Log['trail'][1] in list(imports()):
                                        importlib.reload(module)
                                        self.PRIVMSG(Log['context'],'Module \'%s\' reloaded' % Log['trail'][1])
                            except Exception as e:
                                print(e)

                        if Log['trail'][0] == '!quit':
                            self.ircSend('QUIT')
                            self.irc.close()
                            sys.exit('User exited')

                    if 'Soaker' not in self.info['UNLOAD'][Log['context']]:
                        Soaker.Handler(self, Log)

                    if 'Commands' not in self.info['UNLOAD'][Log['context']]:
                        Commands.Handler(self, Log)

                    if 'URLInfo' not in self.info['UNLOAD'][Log['context']]:
                        URLInfo.Handler(self, Log)

    def updateFile(self):
        with open('nwobot.conf', 'r') as file:
            oldConf = eval(file.read())
        with open('users', 'r') as file:
            oldUser = eval(file.read())
        if self.info != oldConf:
            with open('nwobot.conf', 'w+') as file:
                file.write(str(self.info))
        if self.userDict != oldUser:
            with open('users', 'w+') as file:
                file.write(str(self.userDict))

    def listActive(self, chan, minutes = 10, caller = None, full = False, exclude = []):
        return Logger.listActive(self, chan, minutes, caller, full, exclude)

    current_milli_time = lambda: int(round(time.time() * 1000))

    def PRIVMSG(self, context, message):
        self.ircSend('PRIVMSG %s :%s' % (context, message))

    def ircSend(self, message):
        print('%s %s' % (time.strftime('%H:%M:%S', time.gmtime(time.time())), message))
        self.irc.send(bytes('%s\r\n' % message, 'UTF-8'))

IRC()


#import statements go here
from flask import Flask, request, render_template, redirect, url_for, flash
from flask_wtf import FlaskForm
from wtforms import StringField, IntegerField, StringField, SubmitField
from wtforms.validators import Required, Email ##add separate validators

import requests
import json

app = Flask(__name__)
app.config['SECRET_KEY'] = 'hard to guess string'
app.debug = True

@app.route('/')
def home():
    return "Hello, world!"

class iTunesForm(FlaskForm): #inherits a parameter from flask form
    artist = StringField('enter an artist', validators=[Required() ]) #input types, labels, required validators
    numresult = IntegerField('enter a number of results', validators=[Required() ])
    email = StringField('enter an email address', validators=[Required(), Email()])
    submit = SubmitField('Submit') #creates submit button

@app.route('/itunes-form')
def itunes_form():
    #what code goes here?
    itunes_form = iTunesForm() #create itunes-form.html
    return render_template('itunes-form.html', form=itunes_form) # HINT : create itunes-form.html to represent the form defined in your class

@app.route('/itunes-result', methods = ['GET', 'POST'])
def itunes_result():
    #what code goes here?
    form = iTunesForm(request.form)
    params = {}
    if request.method == 'POST' and form.validate_on_submit():
        params['term'] = form.artist.data
        params['limit'] = form.apires.data
        response = requests.get('https://itunes.apple.com/search?', params = params)
        response_text = json.loads['results']
        result_py = response_text['results']

        return render_template('itunes-results.html', result_html = result_py)

    flash('All fields are required!')
    return redirect(url_for('itunes_form')) #this redirects you to itunes_form if there are errors

if __name__ == '__main__':
    app.run()


#!/usr/bin/env python
"""
@author: dongting@asu.edu
"""
import rospy
from robotiq_ft_sensor.msg import ft_sensor
import numpy as np
from sensor_listener.msg import force_torque_sensor

def callback(msg):
    global Fx,Fy,Fz,Mx,My,Mz
    Fx = msg.Fx
    Fy = msg.Fy
    Fz = msg.Fz
    Mx = msg.Mx
    My = msg.My
    Mz = msg.Mz

def Calibrate_FT_Sensor():
    rospy.loginfo("Start sensor calibiration")
    rospy.Subscriber("robotiq_ft_sensor",ft_sensor,callback)
    global ft_cal_data
    ft_cal_data = np.array([Fx,Fy,Fz,Mx,My,Mz])
    for num in range(1,10000):
        rospy.Subscriber("robotiq_ft_sensor",ft_sensor,callback)
        ft_cal_data = ft_cal_data + [Fx,Fy,Fz,Mx,My,Mz]
    ft_cal_data = ft_cal_data/(num)
    rospy.loginfo("Finish calibration")
    rospy.loginfo("Sensor calibration result: "+np.array2string(ft_cal_data))

def ft_sensor_talker():
    pub = rospy.Publisher('rq_calibrated_sensor',force_torque_sensor,queue_size= 10)
    global pub_rate
    pub_rate = 30
    rate = rospy.Rate(pub_rate)
    rospy.Subscriber("robotiq_ft_sensor",ft_sensor,callback)
    ft_raw_data = np.array([Fx,Fy,Fz,Mx,My,Mz])
    ft_data = ft_raw_data - ft_cal_data
    force_torque_sensor_data = force_torque_sensor()
    force_torque_sensor_data.Fx = ft_data[0]
    force_torque_sensor_data.Fy = ft_data[1]
    force_torque_sensor_data.Fz = ft_data[2]
    force_torque_sensor_data.Mx = ft_data[3]
    force_torque_sensor_data.My = ft_data[4]
    force_torque_sensor_data.Mz = ft_data[5]
    pub.publish(force_torque_sensor_data)
    rate.sleep()

#rospy.init_node('ft_calibrated_sensor')
rospy.init_node('ft_sensor_data')
try:
    Calibrate_FT_Sensor()
    rospy.loginfo("Publish rate = "+str(pub_rate)+"Hz")
    while not rospy.is_shutdown():
        talker = ft_sensor_talker()
except KeyboardInterrupt:
    rospy.loginfo("Collection ended by user")
except:
    rospy.loginfo("check sensor connection or internet configuration")
    rospy.loginfo("Collection ended by user")
except:
    rospy.loginfo("check sensor connection")
#    print("check sensor connection")


# Copyright (c) 2016 Cloudbase Solutions Srl
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Handling of block device information and mapping

Module contains helper methods for dealing with block device information
"""

import os

from nova import block_device
from nova import exception
from nova import objects
from nova.virt import block_device as driver_block_device
from nova.virt import configdrive
from nova.virt import driver
from os_win import constants as os_win_const
from os_win import exceptions as os_win_exc
from os_win import utilsfactory
from oslo_log import log as logging
from oslo_serialization import jsonutils

from compute_hyperv.i18n import _
from compute_hyperv.nova import constants
from compute_hyperv.nova import pathutils
from compute_hyperv.nova import volumeops

LOG = logging.getLogger(__name__)

class BlockDeviceInfoManager(object):

    _VALID_BUS = {constants.VM_GEN_1: (constants.CTRL_TYPE_IDE,
                                       constants.CTRL_TYPE_SCSI),
                  constants.VM_GEN_2: (constants.CTRL_TYPE_SCSI,)}

    _DEFAULT_BUS = constants.CTRL_TYPE_SCSI

    _TYPE_FOR_DISK_FORMAT = {'vhd': constants.DISK,
                             'vhdx': constants.DISK,
                             'iso': constants.DVD}

    _DEFAULT_ROOT_DEVICE = '/dev/sda'

    def __init__(self):
        self._volops = volumeops.VolumeOps()
        self._pathutils = pathutils.PathUtils()

        self._vmutils = utilsfactory.get_vmutils()

    @staticmethod
    def _get_device_bus(ctrl_type, ctrl_addr, ctrl_slot):
        """Determines the device bus and it's hypervisor assigned address.
        """
        if ctrl_type == constants.CTRL_TYPE_SCSI:
            address = ':'.join(map(str, [0, 0, ctrl_addr, ctrl_slot]))
            return objects.SCSIDeviceBus(address=address)
        elif ctrl_type == constants.CTRL_TYPE_IDE:
            address = ':'.join(map(str, [ctrl_addr, ctrl_slot]))
            return objects.IDEDeviceBus(address=address)

    def _get_vol_bdm_attachment_info(self, bdm):
        drv_vol_bdm = driver_block_device.convert_volume(bdm)
        if not drv_vol_bdm:
            return

        connection_info = drv_vol_bdm['connection_info']
        if not connection_info:
            LOG.warning("Missing connection info for volume %s.",
                        bdm.volume_id)
            return

        attachment_info = self._volops.get_disk_attachment_info(
            connection_info)
        attachment_info['serial'] = connection_info['serial']
        return attachment_info

    def _get_eph_bdm_attachment_info(self, instance, bdm):
        # When attaching ephemeral disks, we're setting this field so that
        # we can map them with bdm objects.
        connection_info = self.get_bdm_connection_info(bdm)
        eph_filename = connection_info.get("eph_filename")
        if not eph_filename:
            LOG.warning("Missing ephemeral disk filename in "
                        "BDM connection info. BDM: %s", bdm)
            return

        eph_path = os.path.join(
            self._pathutils.get_instance_dir(instance.name), eph_filename)
        if not os.path.exists(eph_path):
            LOG.warning("Could not find ephemeral disk %s.", eph_path)
            return

        return self._vmutils.get_disk_attachment_info(eph_path,
                                                      is_physical=False)

    def _get_disk_metadata(self, instance, bdm):
        attachment_info = None
        if bdm.is_volume:
            attachment_info = self._get_vol_bdm_attachment_info(bdm)
        elif block_device.new_format_is_ephemeral(bdm):
            attachment_info = self._get_eph_bdm_attachment_info(
                instance, bdm)

        if not attachment_info:
            LOG.debug("No attachment info retrieved for bdm %s.", bdm)
            return

        tags = [bdm.tag] if bdm.tag else []
        bus = self._get_device_bus(
            attachment_info['controller_type'],
            attachment_info['controller_addr'],
            attachment_info['controller_slot'])
        serial = attachment_info.get('serial')

        return objects.DiskMetadata(bus=bus,
                                    tags=tags,
                                    serial=serial)

    def get_bdm_metadata(self, context, instance):
        """Builds a metadata object for instance devices, that maps the user
           provided tag to the hypervisor assigned device address.
        """
        bdms = objects.BlockDeviceMappingList.get_by_instance_uuid(
            context, instance.uuid)

        bdm_metadata = []
        for bdm in bdms:
            try:
                device_metadata = self._get_disk_metadata(instance, bdm)
                if device_metadata:
                    bdm_metadata.append(device_metadata)
            except (exception.DiskNotFound, os_win_exc.DiskNotFound):
                LOG.debug("Could not find disk attachment while "
                          "updating device metadata. It may have been "
                          "detached. BDM: %s", bdm)

        return bdm_metadata

    def set_volume_bdm_connection_info(self, context, instance,
                                       connection_info):
        # When attaching volumes to already existing instances, the connection
        # info passed to the driver is not saved yet within the BDM table.
        #
        # Nova sets the volume id within the connection info using the
        # 'serial' key.
        volume_id = connection_info['serial']
        bdm = objects.BlockDeviceMapping.get_by_volume_and_instance(
            context, volume_id, instance.uuid)
        bdm.connection_info = jsonutils.dumps(connection_info)
        bdm.save()

    @staticmethod
    def get_bdm_connection_info(bdm):
        # We're using the BDM 'connection_info' field to store ephemeral
        # image information so that we can map them. In order to do so,
        # we're using this helper.
        # The ephemeral bdm object wrapper does not currently expose this
        # field.
        try:
            conn_info = jsonutils.loads(bdm.connection_info)
        except TypeError:
            conn_info = {}

        return conn_info

    @staticmethod
    def update_bdm_connection_info(bdm, **kwargs):
        conn_info = BlockDeviceInfoManager.get_bdm_connection_info(bdm)
        conn_info.update(**kwargs)
        bdm.connection_info = jsonutils.dumps(conn_info)
        bdm.save()

    def _initialize_controller_slot_counter(self, instance, vm_gen):
        # we have 2 IDE controllers, for a total of 4 slots
        free_slots_by_device_type = {
            constants.CTRL_TYPE_IDE: [
                os_win_const.IDE_CONTROLLER_SLOTS_NUMBER] * 2,
            constants.CTRL_TYPE_SCSI: [
                os_win_const.SCSI_CONTROLLER_SLOTS_NUMBER]
            }
        if configdrive.required_by(instance):
            if vm_gen == constants.VM_GEN_1:
                # reserve one slot for the config drive on the second
                # controller in case of generation 1 virtual machines
                free_slots_by_device_type[constants.CTRL_TYPE_IDE][1] -= 1
        return free_slots_by_device_type

    def validate_and_update_bdi(self, instance, image_meta, vm_gen,
                                block_device_info):
        slot_map = self._initialize_controller_slot_counter(instance, vm_gen)
        self._check_and_update_root_device(vm_gen, image_meta,
                                           block_device_info, slot_map)
        self._check_and_update_ephemerals(vm_gen, block_device_info, slot_map)
        self._check_and_update_volumes(vm_gen, block_device_info, slot_map)

        if vm_gen == constants.VM_GEN_2 and configdrive.required_by(instance):
            # for Generation 2 VMs, the configdrive is attached to the SCSI
            # controller. Check that there is still a slot available for it.
            if slot_map[constants.CTRL_TYPE_SCSI][0] == 0:
                msg = _("There are no more free slots on controller %s for "
                        "configdrive.") % constants.CTRL_TYPE_SCSI
                raise exception.InvalidBDMFormat(details=msg)

    def _check_and_update_root_device(self, vm_gen, image_meta,
                                      block_device_info, slot_map):
        # either booting from volume, or booting from image/iso
        root_disk = {}

        root_device = driver.block_device_info_get_root_device(
                block_device_info)
        root_device = root_device or self._DEFAULT_ROOT_DEVICE

        if self.is_boot_from_volume(block_device_info):
            root_volume = self._get_root_device_bdm(
                block_device_info, root_device)
            root_disk['type'] = constants.VOLUME
            root_disk['path'] = None
            root_disk['connection_info'] = root_volume['connection_info']
        else:
            root_disk['type'] = self._TYPE_FOR_DISK_FORMAT.get(
                image_meta['disk_format'])
            if root_disk['type'] is None:
                raise exception.InvalidImageFormat(
                    format=image_meta['disk_format'])
            root_disk['path'] = None
            root_disk['connection_info'] = None

        root_disk['disk_bus'] = (constants.CTRL_TYPE_IDE if
            vm_gen == constants.VM_GEN_1 else constants.CTRL_TYPE_SCSI)
        (root_disk['drive_addr'],
         root_disk['ctrl_disk_addr']) = self._get_available_controller_slot(
            root_disk['disk_bus'], slot_map)
        root_disk['boot_index'] = 0
        root_disk['mount_device'] = root_device

        block_device_info['root_disk'] = root_disk

    def _get_available_controller_slot(self, controller_type, slot_map):
        max_slots = (os_win_const.IDE_CONTROLLER_SLOTS_NUMBER if
                     controller_type == constants.CTRL_TYPE_IDE else
                     os_win_const.SCSI_CONTROLLER_SLOTS_NUMBER)
        for idx, ctrl in enumerate(slot_map[controller_type]):
            if slot_map[controller_type][idx] >= 1:
                drive_addr = idx
                ctrl_disk_addr = max_slots - slot_map[controller_type][idx]
                slot_map[controller_type][idx] -= 1
                return (drive_addr, ctrl_disk_addr)

        msg = _("There are no more free slots on controller %s"
                ) % controller_type
        raise exception.InvalidBDMFormat(details=msg)

    def is_boot_from_volume(self, block_device_info):
        if block_device_info:
            root_device = block_device_info.get('root_device_name')
            if not root_device:
                root_device = self._DEFAULT_ROOT_DEVICE

            return block_device.volume_in_mapping(root_device,
                                                  block_device_info)

    def _get_root_device_bdm(self, block_device_info, mount_device=None):
        for mapping in driver.block_device_info_get_mapping(block_device_info):
            if mapping['mount_device'] == mount_device:
                return mapping

    def _check_and_update_ephemerals(self, vm_gen, block_device_info,
                                     slot_map):
        ephemerals = driver.block_device_info_get_ephemerals(block_device_info)
        for eph in ephemerals:
            self._check_and_update_bdm(slot_map, vm_gen, eph)

    def _check_and_update_volumes(self, vm_gen, block_device_info, slot_map):
        volumes = driver.block_device_info_get_mapping(block_device_info)
        root_device_name = block_device_info['root_disk']['mount_device']
        root_bdm = self._get_root_device_bdm(block_device_info,
                                             root_device_name)
        if root_bdm:
            volumes.remove(root_bdm)
        for vol in volumes:
            self._check_and_update_bdm(slot_map, vm_gen, vol)

    def _check_and_update_bdm(self, slot_map, vm_gen, bdm):
        disk_bus = bdm.get('disk_bus')
        if not disk_bus:
            bdm['disk_bus'] = self._DEFAULT_BUS
        elif disk_bus not in self._VALID_BUS[vm_gen]:
            msg = _("Hyper-V does not support bus type %(disk_bus)s "
                    "for generation %(vm_gen)s instances."
                    ) % {'disk_bus': disk_bus,
                         'vm_gen': vm_gen}
            raise exception.InvalidDiskInfo(reason=msg)

        device_type = bdm.get('device_type')
        if not device_type:
            bdm['device_type'] = 'disk'
        elif device_type != 'disk':
            msg = _("Hyper-V does not support disk type %s for ephemerals "
                    "or volumes.") % device_type
            raise exception.InvalidDiskInfo(reason=msg)

        (bdm['drive_addr'],
         bdm['ctrl_disk_addr']) = self._get_available_controller_slot(
            bdm['disk_bus'], slot_map)

        # make sure that boot_index is set.
        bdm['boot_index'] = bdm.get('boot_index')

    def _sort_by_boot_order(self, bd_list):
        # we sort the block devices by boot_index leaving the ones that don't
        # have a specified boot_index at the end
        bd_list.sort(key=lambda x: (x['boot_index'] is None, x['boot_index']))

    def get_boot_order(self, vm_gen, block_device_info):
        if vm_gen == constants.VM_GEN_1:
            return self._get_boot_order_gen1(block_device_info)
        else:
            return self._get_boot_order_gen2(block_device_info)

    def _get_boot_order_gen1(self, block_device_info):
        if block_device_info['root_disk']['type'] == 'iso':
            return [os_win_const.BOOT_DEVICE_CDROM,
                    os_win_const.BOOT_DEVICE_HARDDISK,
                    os_win_const.BOOT_DEVICE_NETWORK,
                    os_win_const.BOOT_DEVICE_FLOPPY]
        else:
            return [os_win_const.BOOT_DEVICE_HARDDISK,
                    os_win_const.BOOT_DEVICE_CDROM,
                    os_win_const.BOOT_DEVICE_NETWORK,
                    os_win_const.BOOT_DEVICE_FLOPPY]

    def _get_boot_order_gen2(self, block_device_info):
        devices = [block_device_info['root_disk']]
        devices += driver.block_device_info_get_ephemerals(
            block_device_info)
        devices += driver.block_device_info_get_mapping(block_device_info)

        self._sort_by_boot_order(devices)

        boot_order = []
        for dev in devices:
            if dev.get('connection_info'):
                dev_path = self._volops.get_disk_resource_path(
                    dev['connection_info'])
                boot_order.append(dev_path)
            else:
                boot_order.append(dev['path'])

        return boot_order


from __future__ import unicode_literals

import re

from .common import InfoExtractor

class NuvidIE(InfoExtractor):
    _VALID_URL = r'^https?://(?:www|m)\.nuvid\.com/video/(?P<id>[0-9]+)'
    _TEST = {
        'url': 'http://m.nuvid.com/video/1310741/',
        'md5': 'eab207b7ac4fccfb4e23c86201f11277',
        'info_dict': {
            'id': '1310741',
            'ext': 'mp4',
            "title": "Horny babes show their awesome bodeis and",
            "age_limit": 18,
        }
    }

    def _real_extract(self, url):
        mobj = re.match(self._VALID_URL, url)
        video_id = mobj.group('id')

        murl = url.replace('://www.', '://m.')
        webpage = self._download_webpage(murl, video_id)

        title = self._html_search_regex(
            r'
\s+
]*>([^<]+)
',
            webpage, 'title').strip()

        url_end = self._html_search_regex(
            r'href="(/mp4/[^"]+)"[^>]*data-link_type="mp4"',
            webpage, 'video_url')
        video_url = 'http://m.nuvid.com' + url_end

        thumbnail = self._html_search_regex(
            r'href="(/thumbs/[^"]+)"[^>]*data-link_type="thumbs"',
            webpage, 'thumbnail URL', fatal=False)

        return {
            'id': video_id,
            'url': video_url,
            'ext': 'mp4',
            'title': title,
            'thumbnail': thumbnail,
            'age_limit': 18,
        }


import sys,os,re
from multiprocessing import Process, Queue, Pool

usage = """
<folder of alignments>
(N)ucleotide or (P)rotein?>
<Num bootstraps (0 for no bootstrapping)>
<output folder>
<cores to use>
<cores per tree>\n\n"""
argv = sys.argv
junk = argv.pop(0)

if len(sys.argv) == 6:
	in_folder = argv.pop(0)
	type_flag = argv.pop(0)
	s_flag = int(argv.pop(0))
	out_folder = argv.pop(0)
	cores=int(argv.pop(0))
	coresPerTree = int(argv.pop(0))
else:
	sys.exit(usage)

def multi_run(cmd):
	os.system(cmd)

rxml = '~/tools_and_software/RAxML-8.1.24/raxmlHPC-PTHREADS-AVX2'

os.system('mkdir -p {0}'.format(out_folder))
aligns = os.listdir(in_folder)

cmd = []
NAMES = {}

for f in aligns:
	f_name = f
	f_name = f_name.replace('.fna','')
	f_name = f_name.replace('.faa','')
	f_name = f_name.replace('.mafft','')
	NAMES[f_name] = ''
	if type_flag == 'N':
		if s_flag > 0:
			data = (rxml,in_folder,f,f_name,coresPerTree,s_flag)
			cmd.append('{0} -s {1}/{2} -n {3} -x 3524627 -p 3524627 -f a -T {4} -N {5} -m GTRCAT -c 6'.format(*data))
		else:
			data = (rxml,in_folder,f,f_name,coresPerTree,s_flag)
			cmd.append('{0} -s {1}/{2} -n {3} -T {4} -m GTRCAT -c 6'.format(*data))
	elif type_flag == 'P':
		if s_flag > 0:
			data = (rxml,in_folder,f,f_name,coresPerTree,s_flag)
			cmd.append('{0} -s {1}/{2} -n {3} -x 3524627 -f a -T {4} -N {5} -m PROTGAMMABLOSUM62'.format(*data))
		else:
			data = (rxml,in_folder,f,f_name,coresPerTree,s_flag)
			cmd.append('{0} -s {1}/{2} -n {3} -T {4} -m PROTGAMMABLOSUM62'.format(*data))

numRun = cores / coresPerTree
p=Pool(numRun)
for c in cmd:
	p.apply_async(multi_run, args=(c,))
	pass
p.close()
p.join()

outputs = os.listdir('./')
for f in outputs:
	front = f.split('.')[0]
	tail = '.'.join(f.split('.')[1:])
	if s_flag == 0:
		if front == 'RAxML_bestTree' and NAMES.has_key(tail):
			os.system('mv {0} {1}/{2}.nwk'.format(f,out_folder,tail))
	else:
		if front == 'RAxML_bipartitionsBranchLabels' and NAMES.has_key(tail):
			os.system('mv {0} {1}/{2}.nwk'.format(f,out_folder,tail))


# -*- coding: utf-8 -*-
import scrapy
import re
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.http import Request
from urlparse import urlparse
from factcheck.items import FactcheckItem
from bs4 import BeautifulSoup

class FactSpider(CrawlSpider):
    name = 'factcheck'

    allowed_domains = ['news.naver.com']
    '''
    start_urls = ['http://news.naver.com/main/list.nhn?oid=023&mid=sec&mode=LPOD&date=20160304'] #sbstv

    rules = (
        Rule(LinkExtractor(allow=r'023&mid=sec&mode=LPOD&date=20160304'), callback='parse_list', follow=True),
    )
    '''
    #snac url http:// 없이 들어옴

    def __init__(self, *a, **kw):

        self.start_urls = ['http://news.naver.com/main/election/president2017/factcheck/index.nhn?type=NEW&page=1']
        FactSpider.rules = (Rule(LinkExtractor(allow=r'type=NEW&page='), callback='parse_list', follow=True),)

        super(FactSpider, self).__init__(*a, **kw)

    def parse_list(self, response):
#//*[@id="factcheck_page"]/div[3]/div[1]/div[1]/div[2]/p
        factItem = FactcheckItem()
        item_list = response.xpath("//div[@class='fact_lst _tab_NEW']/div[@class='section']")
        #self.log("is it here?  ")
        #self.logger.debug(len(item_list))

        for item in item_list:

            name = item.xpath(".//span[@class='name']/text()").extract_first()
            action = item.xpath(".//h4[@class='tit']/text()").extract_first()
            title = item.xpath(".//div[@class='info_area _rcount']/p/text()").extract_first()
            result = item.xpath(".//dl[@class='chk_lst']/dd/text()")

            self.logger.info(name)
            self.logger.info(action)
            self.logger.info(action.split()[-1])
            self.logger.info(title)
            self.logger.info(result.extract())

            factItem['name'] = name
            factItem['action'] = action.split()[-1]
            factItem['title'] = title
            factItem['result'] = result.extract()

            yield factItem
'''

        item_list = response.xpath("//ul[contains(@class,'type06_headline') or contains(@class,'type06')]/li/dl/dt[not(@class='photo')]")

        for item in item_list:
            detail_url = item.xpath("a/@href")[0].extract().strip()
            detail_urls = ''.join(detail_url)
            request = Request(detail_urls, callback=self.parse_detail)
            yield request

    def parse_detail(self, response):

        if response.status == 200:
            if(response.url.find("entertain") == -1 and response.url.find("sports") == -1 ):
                naver_url = response.xpath("//meta[@property='og:url']/@content")[0].extract()
                naver_section = response.xpath("//meta[@property='me2:category2']/@content").extract_first()
                detail_content = response.xpath("//div[@class='content']")
                datetime = detail_content.xpath("//span[@class='t11']/text()").extract_first().strip()
                #comment_cnt = detail_content.xpath("//span[@class='lo_txt']/text()").extract_first().strip()
                comment_cnt = 0
                #origin_url = detail_content.xpath("//div[@class='article_btns']/a/@href").extract_first().strip()
                origin_url = detail_content.xpath("//a[@class='btn_artialoriginal']/@href").extract_first().strip()

                vod = detail_content.xpath("//div[@class='vod_area']").extract()
                origin_content = detail_content.xpath("//div[@id='articleBodyContents']").extract_first().strip()

                article = BeautifulSoup(origin_content, 'lxml')
                # kill all script and style elements
                for script in article(["script", "style"]):
                    script.extract()    # rip it out

                for br in article(["br"]):
                    br.replace_with("\n")
                # get text
                #text = article.get_text()
                article_text = article.getText()

                image_count = len(article.find_all("img"))
                vod_count = len(vod)

                if (origin_url.startswith('http')):
                    yield Request(origin_url, callback=self.parse_origin, meta={'naver_section':naver_section, 'naver_url':naver_url,'datetime':datetime, 'img_cnt':image_count, 'vod_cnt':vod_count, 'cmt_cnt': comment_cnt, 'article':article_text})

    def parse_origin(self, response):
        newsItem = NewscrawlItem()
        parsed_url = urlparse(response.url)
        domain_name = '{uri.netloc}'.format(uri=parsed_url)

        if response.status == 200:
            newsItem['naver_url'] = response.meta['naver_url']
            newsItem['naver_section'] = response.meta['naver_section']
            newsItem['date'] = response.meta['datetime'].split()[0]
            newsItem['time'] = response.meta['datetime'].split()[1]
            newsItem['image'] = response.meta['img_cnt']
            newsItem['vod'] = response.meta['vod_cnt']
            newsItem['comments'] = response.meta['cmt_cnt']
            newsItem['article'] = response.meta['article']
            title = response.xpath("//meta[@property='og:title']/@content")
            #title = response.xpath("//div[contains(@class,'article_title02') or contains(@class, 'articleTit') or contains(@class, 'article_title') or contains(@class, 'artiTitle')]/h1/text()")

            if(len(title)>0):
                newsItem['title'] = title.extract_first()
            else :
                newsItem['title'] = response.xpath("//title/text()").extract_first()

            response_url = response.xpath("//meta[@property='og:url']/@content")
            if(len(response_url) > 0 ):
                newsItem['org_url'] = response_url[0].extract()
            else :
                newsItem['org_url'] = response.url

            #//*[@id="container"]/div[2]/p[1]
            author_items = []
            if(domain_name == "news.chosun.com"):
                author_items = response.xpath("//li[@id='j1']/a/text()").extract()
            elif(domain_name == "biz.chosun.com"):
                author_items = response.xpath("//li[@id='j1']/text()").extract()

            authors = []
            if(len(author_items) > 0):
                for author_item in author_items:
                    authors.append(author_item.strip())

                if(len(authors)>1):
                    newsItem['author'] = ' '.join(authors)
                else:
                    newsItem['author'] = authors[0]

            section = ""
            if(domain_name == "news.chosun.com"):
                title_text = response.xpath('//title/text()').extract_first()
                title_list = title_text.split('-')
                section = title_list[-1]

                section = section.split('>')[0].strip()

                #javascript_codes = response.xpath('//body/script/text()').extract()

                #for code in javascript_codes:
                #    section = re.findall(r"C1_rp.*?=\s*(.*?);", code, re.DOTALL | re.MULTILINE)
            elif(domain_name == "biz.chosun.com"):
                section = domain_name.split('.')[0]
                #response.xpath('//div[@class="art_title_2011"]/dl/dt/a/text()').extract_first()
            else:
                section = domain_name.split('.')[0]
                #self.logger.info("non section url   " + response.url)

            #self.logger.info(section)

            if(len(section) > 0):
                newsItem['section'] = section

            yield newsItem
    '''


def findAt(email_addr):
    """ Returns the index of the @ character in an email address.

        findAt('user@cs.yonsei.ac.kr') -> 4
        findAt('@.foo.bar') -> -1 (error, user name empty)
        findAt('foo@.ba@r') -> -1 (error, multiple @ signs not allowed)
        findAt('foo@')      -> -1 (error, domain name empty)
        findAt('foo.bar')   -> -1 (error, no @ sign)
        No other errors need to be checked!
    """
    # no @ sign or multiple @ sign
    if email_addr.count('@') != 1:
        return -1

    # domain or user name empty
    elif email_addr.index('@') == 0 or \
         email_addr.index('@') == len(email_addr) - 1:
        return -1

    # return index of the @ character
    else:
        return email_addr.index('@')


from .MarieLogger import MarieIOLog

def getHighestIntentScore(result):
    first_intent = result['intent']
    name = first_intent['name']
    print('intent name:', name)
    score = first_intent['confidence']
    return score

if __name__ == "__main__":
    rst = {'intent': {'name': 'query_quantum_chemistry', 'confidence': 0.7796140909194946}, 'entities': [
        {'entity': 'attribute', 'start': 0, 'end': 16, 'confidence_entity': 0.5236995705636405,
         'value': 'geometry c=c=c=c', 'extractor': 'CRFEntityExtractor'}],
           'intent_ranking': [{'name': 'query_quantum_chemistry', 'confidence': 0.7796140909194946},
                              {'name': 'geometry_type', 'confidence': 0.12144190818071365},
                              {'name': 'electronic_energy', 'confidence': 0.02252243459224701},
                              {'name': 'spin_multiplicity', 'confidence': 0.022378213703632355},
                              {'name': 'symmetry_number', 'confidence': 0.017157595604658127},
                              {'name': 'rotational_relaxation_collision', 'confidence': 0.009460683912038803},
                              {'name': 'vibration_frequency', 'confidence': 0.007247160654515028},
                              {'name': 'rotational_constants', 'confidence': 0.007164621260017157},
                              {'name': 'guassian_file', 'confidence': 0.006535339634865522},
                              {'name': 'query_thermodynamic', 'confidence': 0.006477926392108202}],
           'text': 'geometry c=c=c=c\n'}


# import the required modules for game
import random
import pygame
import tkinter as tk
from tkinter import messagebox

# initialise our game
pygame.init()
# rename our pygame window
pygame.display.set_caption("Snake Game")
# add background music to play throughout our game
music = pygame.mixer.music.load('music.mp3')
pygame.mixer.music.play(-1)  # the -1 ensures our music plays indefinitely

# START PYGAME

# CLASSES
class cube(object):
    rows = 20
    w = 500

    def __init__(self, start, color=(255, 100, 180)):
        self.pos = start
        self.dirnx = 0  # starts the position of the snake on the screen but not moving in either direction
        self.dirny = 0
        self.color = color

    def move(self, dirnx, dirny):  # defines position
        self.dirnx = dirnx
        self.dirny = dirny
        self.pos = (self.pos[0] + self.dirnx, self.pos[1] + self.dirny)

    def draw(self, surface):
        dis = self.w // self.rows  # determines the width/height of each cube
        i = self.pos[0]  # the current row
        j = self.pos[1]  # the current column
        pygame.draw.rect(surface, self.color, (i * dis + 1, j * dis + 1, dis - 2, dis - 2))  # determines where to draw
        # our cube by multiplying the row and column value of our cube

class snake(object):  # everything required for our snake to work
    body = []  # blank list of our snake coordinates
    turns = {}

    def __init__(self, color, pos):
        self.color = color
        self.head = cube(pos)  # Determines the head of the snake will be the front position
        self.body.append(self.head)  # We will add the head to our body list
        self.dirnx = 0  # Represents the direction our snake is moving
        self.dirny = 0

    def move(self):
        for event in pygame.event.get():
            if event.type == pygame.QUIT:  # Checks if the user has hit the red x to cancel game
                pygame.quit()

            keys = pygame.key.get_pressed()  # Determines which keys are being pressed by user

            for _ in keys:  # Loops through all of the arrow keys
                if keys[pygame.K_LEFT]:
                    self.dirnx = -1  # -1 means the object is moving left
                    self.dirny = 0  # does not move in 2 directions at once (diagonal)
                    self.turns[self.head.pos[:]] = [self.dirnx, self.dirny]  # moves our head position to the direction
                    # of whatever key was pressed (in this case left)
                # elif is used as we don't want to move in more than one direction
                elif keys[pygame.K_RIGHT]:
                    self.dirnx = 1  # 1 means the object is moving right
                    self.dirny = 0
                    self.turns[self.head.pos[:]] = [self.dirnx, self.dirny]

                elif keys[pygame.K_UP]:
                    self.dirnx = 0
                    self.dirny = -1  # -1 for y means the object is moving up
                    self.turns[self.head.pos[:]] = [self.dirnx, self.dirny]

                elif keys[pygame.K_DOWN]:
                    self.dirnx = 0
                    self.dirny = 1  # 1 for y means the object is moving down
                    self.turns[self.head.pos[:]] = [self.dirnx, self.dirny]

# Followed Tech with Tim guide (referenced in report) for the following function:
        for i, c in enumerate(self.body):
            p = c.pos[:]
            if p in self.turns:  # if the copied position above is in the turns, then the object will move/turn
                turn = self.turns[p]
                c.move(turn[0], turn[1])
                if i == len(self.body) - 1:  # if we are on the last cube, we're going to remove the last turn
                    self.turns.pop(p)
            else:  # if position is not in the turn, we still need to move
                # this checks whether or not we have reached the end of the screen
                if c.dirnx == -1 and c.pos[0] <= 0:  # if we're moving left, and the position is less then or equal to 0
                    # then we will change position to go to the right side of the screen
                    c.pos = (c.rows - 1, c.pos[1])
                elif c.dirnx == 1 and c.pos[0] >= c.rows - 1:
                    c.pos = (0, c.pos[1])
                elif c.dirny == 1 and c.pos[1] >= c.rows - 1:
                    c.pos = (c.pos[0], 0)
                elif c.dirny == -1 and c.pos[1] <= 0:
                    c.pos = (c.pos[0], c.rows - 1)
                else:
                    c.move(c.dirnx, c.dirny)

    def reset(self, pos):  # resets our game once you lose
        self.head = cube(pos)  # new position of snake head
        self.body = []  # clears list of snake body
        self.body.append(self.head)
        self.turns = {}
        self.dirnx = 0
        self.dirny = 0

    def addCube(self):  # this function will add a cube onto the snake tail for each snack collected
        tail = self.body[-1]  # this will collect the last cube of the snake (tail)
        dx, dy = tail.dirnx, tail.dirny  # collects the position of tail cube

        # we need to know which side of the snake to add the new cube to (tail)
        # so we need to check the direction we are currently moving in to determine whether we add the tail to the
        # left, right, above or below
        if dx == 1 and dy == 0:
            self.body.append(cube((tail.pos[0] - 1, tail.pos[1])))
        elif dx == -1 and dy == 0:  # e.g. if moving in a left direction, adds cube to the furthest right of the snake
            self.body.append(cube((tail.pos[0] + 1, tail.pos[1])))
        elif dx == 0 and dy == 1:
            self.body.append(cube((tail.pos[0], tail.pos[1] - 1)))
        elif dx == 0 and dy == -1:
            self.body.append(cube((tail.pos[0], tail.pos[1] + 1)))

        self.body[-1].dirnx = dx  # sets the direction of the tail, ensures it moves with the rest of the snake
        self.body[-1].dirny = dy

    def draw(self, surface):
        for i, c in enumerate(self.body):
            c.draw(surface)

# Drawing our grid
def drawGrid(w, rows, surface):
    sizeBtwn = w // rows  # Gives us the distance between the lines (width divided by rows)

    x = 0  # Keeps track of current x
    y = 0  # Keeps track of current y
    for l in range(rows):  # Draws one vertical and one horizontal line for each loop
        x = x + sizeBtwn
        y = y + sizeBtwn

        pygame.draw.line(surface, (255, 255, 255), (x, 0), (x, w))  # vertical line so y value doesn't change
        pygame.draw.line(surface, (255, 255, 255), (0, y), (w, y))  # horizontal line so x value doesn't change

def redrawWindow(surface):
    global rows, width, s, snack
    surface.fill((0, 0, 0))  # Fills the screen with colour black
    s.draw(surface)  # draws our snake object
    snack.draw(surface)  # draws our snack object on the screen
    drawGrid(width, rows, surface)  # Draws our grid lines
    pygame.display.update()  # Updates our screen

def randomSnack(rows, snake):
    positions = snake.body  # gets all of the cube positions of our snake

    while True:  # generates random positions on grid until a valid one is received
        x = random.randrange(rows)
        y = random.randrange(rows)
        if len(list(filter(lambda z: z.pos == (x, y), positions))) > 0:  # checks if the position generated is already
            # occupied by the snake , ensures snack is not put on top of snake
            continue
        else:
            break

    return x, y

def message_box(subject, content):
    root = tk.Tk()  # opens a new window for message box
    root.attributes("-topmost", True)  # creates the window to pop up over any other windows you may have open
    root.withdraw()
    messagebox.showinfo(subject, content)
    try:
        root.destroy()
    except:
        pass

def main():
    # GLOBAL VARIABLES
    global width, rows, s, snack
    width = 500  # Width of our screen
    height = 500  # Height of our screen
    rows = 20  # Amount of rows on our screen
    win = pygame.display.set_mode((width, height))  # This creates our pygame window screen
    s = snake((255, 100, 180), (10, 10))  # Creates our snake object and the figures 10 give it a position on screen
    snack = cube(randomSnack(rows, s), color=(255, 255, 255))  # generates snack object
    flag = True

    clock = pygame.time.Clock()  # Creates our clock object

    while flag:
        # starts our main loop
        pygame.time.delay(50)  # Delays our game to prevent it from running too quickly
        clock.tick(10)  # Ensures our game runs at no more than 10 frames per second
        s.move()
        if s.body[0].pos == snack.pos:  # checks if or snake has hit the snack object
            s.addCube()  # if it has, it adds another cube onto the snake tail
            snack = cube(randomSnack(rows, s), color=(255, 255, 255))  # generates a new snack

        for x in range(len(s.body)):
            if s.body[x].pos in list(map(lambda z: z.pos, s.body[x + 1:])):  # checks for collision of snake's body
                print('Score: ', len(s.body))  # prints score
                message_box('GAME OVER!', 'Try again?')
                s.reset((10, 10))  # resets game to start over
                break

        redrawWindow(win)  # This will refresh our screen

main()


# coding:iso-8859-9 Türkçe
# p_41001.py: Metin bileşeninin sade, kaydıraçlı ve resimli örneği

from tkinter import *
from p_315 import Renk

kök = Tk()
kök.title ("Metin Bileşeni-1")

metin = Text (kök, height=2, width=30)
metin.pack()

metin.insert (END, "Bu, sadece iki satırlık küçük\nbir metin bileşenidir.\n")

kök.mainloop()
#---------------------------------------------------------------------------------------------------------

kök = Tk()
kök.title ("Metin Bileşeni-2")

metin = Text (kök, height=2, width=30)
metin.pack()

alıntı = """HAMLET: Olmak yada olmamak--Esas mesele budur:
Hangisi daha soyludur, acaba çirkin bahtın
Ok ve mızraklarının anılar eziyeti mi,
Yoksa bir acun sıkıntıya göğüs germek,
Ve sonunda hepsiyle zıtlaşmak mı? Ölmek, uyumak--
Fazlası değil--ve uyuyunca deriz ki sonlandırdık
Kalp sızısını, ve binlerce doğal şokları
Bedenin miraslandığı. Bu'dur kemale erme
Sofucasına arzulanan."""

metin.insert (END, alıntı)

kök.mainloop()
#---------------------------------------------------------------------------------------------------------

kök = Tk()
kök.title ("Kaydıraçlı Metin Bileşeni")

kaydıraç = Scrollbar (kök)
kaydıraç.pack (side=RIGHT, fill=Y)

metin = Text (kök, height=4, width=50)
metin.pack(side=LEFT)

kaydıraç.config (command=metin.yview)
metin.config (yscrollcommand=kaydıraç.set)

metin.insert (END, alıntı)

kök.mainloop()
#---------------------------------------------------------------------------------------------------------

kök = Tk()
kök.title ("Resimli ve Kaydıraçlı Metin")

metin1 = Text (kök, height=15, width=25, background=Renk.renk() )
metin1.pack (side=LEFT)
# kök renklendirmesi bg ve fg ile değil, background ve foreground iledir...
resim = PhotoImage (file='resim/shakespeare.png')
metin1.image_create (END, image=resim)

metin2 = Text (kök, height=15, width=45, background=Renk.renk() )
metin2.pack (side=LEFT)

kaydıraç = Scrollbar (kök, command=metin2.yview)
kaydıraç.pack (side=RIGHT, fill=Y)

metin2.configure (yscrollcommand=kaydıraç.set)

metin2.tag_configure ('koyu-yatık', foreground="blue", font=('Arial', 10, 'bold', 'italic'))
metin2.tag_configure ('büyük', foreground="Brown", font=('Verdana', 20, 'bold'))
metin2.tag_configure ('koyu-renkli', foreground='#476042', background="Tan", font=('Tempus Sans ITC', 12, 'bold'))
metin2.tag_bind ('devam', "<1>", lambda olay, t=metin2: t.insert (END, "Şimdi değil, belki daha sonra!\n", "koyu-yatık") )
# tag_configure ve tag_bind'lar sonradan metin.insert(END, metin, tag) olarak kullanılacaktır...

metin2.insert (END,'William Shakespeare\n', 'büyük')

alıntı = """
Olmak yada olmamak--Esas mesele budur:
Hangisi daha soyludur, acaba çirkin bahtın
Ok ve mızraklarının anılar eziyeti mi,
Yoksa bir acun sıkıntıya göğüs germek,
"""

metin2.insert (END, alıntı, 'koyu-renkli')
metin2.insert (END, 'devamı var...\n', 'devam')

kök.mainloop()


import pickle
import numpy as np
import io
import time
import zlib

a = np.array(range(100000))
b = np.array(range(200000, 300000))
file_obj = io.BytesIO()

start = time.time()
file_obj.write(zlib.compress(pickle.dumps({'foo': a, 'bar': b})))
print('pickle dump: %.04f s' % (time.time() - start))

start = time.time()
p = pickle.loads(zlib.decompress(file_obj.getvalue()))
p['foo']
p['bar']
print('pickle load: %.04f s' % (time.time() - start))

print('pickled size: %s' % len(file_obj.getvalue()))

file_obj = io.BytesIO()
start = time.time()
file_obj.seek(0)
np.savez_compressed(file_obj, foo=a, bar=b)
print('np dump: %.04f s' % (time.time() - start))

start = time.time()
file_obj.seek(0)
npz = np.load(file_obj)
npz['foo']
npz['bar']

print('np load: %.04f s' % (time.time() - start))
print('np size: %s' % len(file_obj.getvalue()))


# -*- coding: utf-8 -*-\
'''
    unit.cloud_config_test
    ~~~~~~~~~~~~~~~~~~~~~~

    Configuration related unit testing

    :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)`
    :copyright: © 2013 by the SaltStack Team, see AUTHORS for more details.
    :license: Apache 2.0, see LICENSE for more details.
'''

# Import python libs
import os
import shutil
import tempfile

# Import 3rd-party libs
import yaml

# Import salt testing libs
from salttesting import TestCase
from salttesting.helpers import ensure_in_syspath
ensure_in_syspath('../')

# Import salt libs
import salt.utils
import integration
from salt import config as cloudconfig

class CloudConfigTestCase(TestCase):

    def test_load_cloud_config_from_environ_var(self):
        original_environ = os.environ.copy()

        tempdir = tempfile.mkdtemp()
        try:
            env_root_dir = os.path.join(tempdir, 'foo', 'env')
            os.makedirs(env_root_dir)
            env_fpath = os.path.join(env_root_dir, 'config-env')

            salt.utils.fopen(env_fpath, 'w').write(
                'root_dir: {0}\n'
                'log_file: {1}\n'.format(env_root_dir, env_fpath)
            )

            os.environ['SALT_CLOUD_CONFIG'] = env_fpath
            # Should load from env variable, not the default configuration file
            config = cloudconfig.cloud_config('/etc/salt/cloud')
            self.assertEqual(config['log_file'], env_fpath)
            os.environ.clear()
            os.environ.update(original_environ)

            root_dir = os.path.join(tempdir, 'foo', 'bar')
            os.makedirs(root_dir)
            fpath = os.path.join(root_dir, 'config')
            salt.utils.fopen(fpath, 'w').write(
                'root_dir: {0}\n'
                'log_file: {1}\n'.format(root_dir, fpath)
            )
            # Let's set the environment variable, yet, since the configuration
            # file path is not the default one, ie, the user has passed an
            # alternative configuration file form the CLI parser, the
            # environment variable will be ignored.
            os.environ['SALT_CLOUD_CONFIG'] = env_fpath
            config = cloudconfig.cloud_config(fpath)
            self.assertEqual(config['log_file'], fpath)
        finally:
            # Reset the environ
            os.environ.clear()
            os.environ.update(original_environ)

            if os.path.isdir(tempdir):
                shutil.rmtree(tempdir)

    def test_deploy_search_path_as_string(self):
        temp_conf_dir = os.path.join(integration.TMP, 'issue-8863')
        config_file_path = os.path.join(temp_conf_dir, 'cloud')
        deploy_dir_path = os.path.join(temp_conf_dir, 'test-deploy.d')
        try:
            for directory in (temp_conf_dir, deploy_dir_path):
                if not os.path.isdir(directory):
                    os.makedirs(directory)

            default_config = cloudconfig.cloud_config(config_file_path)
            default_config['deploy_scripts_search_path'] = deploy_dir_path
            with salt.utils.fopen(config_file_path, 'w') as cfd:
                cfd.write(yaml.dump(default_config))

            default_config = cloudconfig.cloud_config(config_file_path)

            # Our custom deploy scripts path was correctly added to the list
            self.assertIn(
                deploy_dir_path,
                default_config['deploy_scripts_search_path']
            )

            # And it's even the first occurrence as it should
            self.assertEqual(
                deploy_dir_path,
                default_config['deploy_scripts_search_path'][0]
            )
        finally:
            if os.path.isdir(temp_conf_dir):
                shutil.rmtree(temp_conf_dir)

    def test_includes_load(self):
        '''cloud.{providers,profiles}.d directories are loaded even if not directly passed'''
        config_path = os.path.join(integration.FILES, 'conf', 'cloud')
        config = cloudconfig.cloud_config(config_path)
        self.assertIn('ec2-config', config['providers'])
        self.assertIn('Ubuntu-13.04-AMD64', config['profiles'])

if __name__ == '__main__':
    from salttesting.parser import run_testcase
    run_testcase(CloudConfigTestCase)


'''
Знайти добуток всіх елементів масиву цілих чисел, менших 0. Розмірність
масиву - 10. Заповнення масиву здійснити за допомогою клавіатури.
Постолович А.С. 122-В
'''
from functools import reduce
import numpy as np

A = np.zeros(10) #створюємо пустий масив
z = [] #створюємо пустий список
for i in range(10):
    A[i] = int(input('Введіть значення:')) #заповнюємо масив значенями
    if A[i] < 0:
        z.append(A[i]) #
z_mult = reduce(lambda x, y: x*y, z) #знаходимо добуток всіх чисел у списку
print(f"Добуток всіх від'ємних чисел - {z_mult}") #виводимо добуток всіх чисел у списку


import nltk
from nltk.tree import ParentedTree
from nltk.tokenize import RegexpTokenizer
from nltk.sentiment.vader import SentimentIntensityAnalyzer

class ChunkParser(nltk.ChunkParserI):
    def __init__(self, train_sents):
        train_data = [[(t, c) for w, t, c in nltk.chunk.tree2conlltags(sent)] for sent in train_sents]
        self.tagger = nltk.TrigramTagger(train_data)

    def parse(self, sentence):
        pos_tags = [pos for (word, pos) in sentence]
        tagged_pos_tags = self.tagger.tag(pos_tags)
        chunktags = [chunktag for (pos, chunktag) in tagged_pos_tags]
        conlltags = [(word, pos, chunktag) for ((word, pos), chunktag) in zip(sentence, chunktags)]
        return nltk.chunk.conlltags2tree(conlltags)

with open('./drug_abst.txt') as fp:
    text = fp.readlines()
    string = ''.join(text)
    sentences = nltk.sent_tokenize(string)
    sid = SentimentIntensityAnalyzer()

    neg_count = 0
    pos_count = 0
    neu_count = 0

    for sentence in sentences:
        print(sentence)
        ss = sid.polarity_scores(sentence)
        if ss['neg'] > ss['pos']:
            neg_count += 1

        else:
            pos_count += 1

    print('POS: {}, NEG: {}'.format(pos_count, neg_count))

    """
    sentences = nltk.sent_tokenize(string)
    tokens = []
    for sent in sentences:
        pos = nltk.pos_tag(nltk.word_tokenize(sent))
        tokens.append(pos)
    train_sents = nltk.corpus.conll2000.chunked_sents('train.txt', chunk_types=['NP'])
    # print(train_sents)
    NPChunker = ChunkParser(train_sents)
    foo = NPChunker.parse(tokens[0])
    foo = ParentedTree.convert(foo)
    for t in foo.subtrees():
        if t.label() == 'NP':
            print(t)
    """