from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import itertools
import random
import shutil
import os
import sys
import time
import gc

import torch
import torch.nn as nn
from torch.autograd import Variable

import numpy as np

from hparams import Iwslt16EnDeBpe32SharedParams

class DataLoader(object):
  def __init__(self, hparams, decode=False):
    """Encloses both train and valid data.

    Args:
      hparams: must be ['tiny' 'bpe16' 'bpe32']
    """

    self.hparams = hparams
    self.decode = decode

    print("-" * 80)
    print("Building data for '{0}' from '{1}'".format(
      self.hparams.dataset, self.hparams.data_path))

    # vocab
    (self.source_word_to_index,
     self.source_index_to_word) = self._build_vocab(self.hparams.source_vocab)
    self.source_vocab_size = len(self.source_word_to_index)

    (self.target_word_to_index,
     self.target_index_to_word) = self._build_vocab(self.hparams.target_vocab)
    self.target_vocab_size = len(self.target_word_to_index)

    assert (self.source_word_to_index[self.hparams.pad] ==
            self.target_word_to_index[self.hparams.pad])
    assert (self.source_word_to_index[self.hparams.unk] ==
            self.target_word_to_index[self.hparams.unk])
    assert (self.source_word_to_index[self.hparams.bos] ==
            self.target_word_to_index[self.hparams.bos])
    assert (self.source_word_to_index[self.hparams.eos] ==
            self.target_word_to_index[self.hparams.eos])

    if (self.hparams.raml_source or (hasattr(self.hparams, "raml_target")
                                     and self.hparams.raml_target)):
      self.softmax = torch.nn.Softmax(dim=-1)

    if self.hparams.dist_corrupt:
      assert self.hparams.glove_emb_file is not None
      print("Reading GloVe from '{0}'".format(self.hparams.glove_emb_file))
      num_lines = 0
      vocab_size = 0
      #glove = {UNK: np.zeros(300, dtype=np.float32)}
      self.glove = torch.zeros(self.source_vocab_size, self.hparams.glove_emb_dim)
      with open(self.hparams.glove_emb_file, "r") as finp:
        for line in finp:
          # print(line)
          line = line.strip()
          if not line:
            continue

          tokens = line.split(" ")
          word = tokens[0]
          num_lines += 1
          if word not in self.source_word_to_index:
            continue
          vec = [float(v.strip()) for v in tokens[1:] if v.strip()]
          wid = self.source_word_to_index[word]
          self.glove[wid] = torch.FloatTensor(vec)
          #print(wid, word)
          #print(self.glove[wid])
          vocab_size += 1

          if num_lines % 1000 == 0:
            print("Read {0:>8d} lines, vocab_size={1}".format(num_lines,
                                                              vocab_size))
            gc.collect()
          if self.hparams.max_glove_vocab_size is not None and num_lines >= self.hparams.max_glove_vocab_size:
            break
        print("Read {0:>8d} lines, vocab_size={1}".format(num_lines,
                                                          vocab_size))
      if self.hparams.cuda:
        self.glove = self.glove.cuda()
    if self.decode:
      self.x_test, self.y_test = self._build_parallel(
        self.hparams.source_test, self.hparams.target_test, is_training=False)
      self.test_size = len(self.x_test)
      self.reset_test()
      return
    else:
      # train data
      self.x_train, self.y_train = self._build_parallel(
        self.hparams.source_train, self.hparams.target_train, is_training=True,
        sort=True)

      # signifies that x_train, y_train are not ready for batching
      self.train_size = len(self.x_train)
      self.n_train_batches = None
      self.reset_train()

      # valid data
      self.x_valid, self.y_valid = self._build_parallel(
        self.hparams.source_valid, self.hparams.target_valid, is_training=False)
      self.valid_size = len(self.x_valid)
      self.reset_valid()

  def reset_train(self):
    """Shuffle training data. Prepare the batching scheme if necessary."""

    if self.hparams.batcher == "word":
      if self.n_train_batches is None:
        start_indices, end_indices = [], []
        start_index = 0
        while start_index < self.train_size:
          end_index = start_index
          word_count = 0
          while (end_index + 1 < self.train_size and
                 (word_count +
                  len(self.x_train[end_index + 1]) +
                  len(self.y_train[end_index + 1])) <= self.hparams.batch_size):
            end_index += 1
            word_count += (len(self.x_train[end_index]) +
                           len(self.y_train[end_index]))
          start_indices.append(start_index)
          end_indices.append(end_index + 1)
          start_index = end_index + 1
        assert len(start_indices) == len(end_indices)
        self.n_train_batches = len(start_indices)
        self.start_indices = start_indices
        self.end_indices = end_indices
    elif self.hparams.batcher == "sent":
      if self.n_train_batches is None:
        self.n_train_batches = ((self.train_size + self.hparams.batch_size - 1)
                                // self.hparams.batch_size)
    else:
      raise ValueError("Unknown batcher scheme '{0}'".format(self.batcher))
    self.train_queue = np.random.permutation(self.n_train_batches)
    self.train_index = 0

  def reset_valid(self):
    self.valid_index = 0

  def reset_test(self):
    self.test_index = 0

  def next_test(self, test_batch_size=1, raml=False):
    end_of_epoch = False
    start_index = self.test_index
    end_index = min(start_index + test_batch_size, self.test_size)
    batch_size = end_index - start_index

    # pad data
    x_test = self.x_test[start_index: end_index]
    y_test = self.y_test[start_index: end_index]
    if raml:
      x_test = [sent for sent in x_test for _ in range(self.hparams.n_corrupts)]
      no_corrupt_mask = [0] * (self.hparams.n_corrupts-self.hparams.n_cleans) + [1] * self.hparams.n_cleans
      no_corrupt_mask = no_corrupt_mask * batch_size
      no_corrupt_mask = torch.ByteTensor(no_corrupt_mask).unsqueeze(1).byte()
      if self.hparams.cuda:
        no_corrupt_mask = no_corrupt_mask.cuda()
      x_test_raml, x_test, x_mask, x_pos_emb_indices, x_count = self._pad(
        sentences=x_test, pad_id=self.pad_id, raml=True,
        vocab_size=self.hparams.source_vocab_size, volatile=True,
        pad_corrupt=self.hparams.src_pad_corrupt,
        dist_corrupt=self.hparams.dist_corrupt,
        no_corrupt_mask=no_corrupt_mask, raml_tau=self.hparams.raml_src_tau)
    else:
      x_test, x_mask, x_pos_emb_indices, x_count = self._pad(
        sentences=x_test, pad_id=self.pad_id, volatile=True)

    y_test, y_mask, y_pos_emb_indices, y_count = self._pad(
      sentences=y_test, pad_id=self.pad_id, volatile=True)

    if end_index >= self.test_size:
      end_of_epoch = True
      self.test_index = 0
    else:
      self.test_index += batch_size

    if raml:
      return ((x_test_raml, x_test, x_mask, x_pos_emb_indices, x_count),
              (y_test, y_mask, y_pos_emb_indices, y_count),
              batch_size, end_of_epoch)
    else:
      return ((x_test, x_mask, x_pos_emb_indices, x_count),
              (y_test, y_mask, y_pos_emb_indices, y_count),
              batch_size, end_of_epoch)

  def next_valid(self, valid_batch_size=20):
    """Retrieves a sentence of testing examples.

    Returns:
      (x_valid, x_len): a pair of torch Tensors of size [batch, source_length]
        and [batch_size].
      (y_valid, y_len): a pair of torch Tensors of size [batch, target_length]
        and [batch_size].
      end_of_epoch: whether we reach the end of training examples.
    """

    end_of_epoch = False
    start_index = self.valid_index
    end_index = min(start_index + valid_batch_size, self.valid_size)
    batch_size = end_index - start_index

    # pad data
    x_valid = self.x_valid[start_index : end_index]
    y_valid = self.y_valid[start_index : end_index]
    x_valid, x_mask, x_pos_emb_indices, x_count = self._pad(
      sentences=x_valid, pad_id=self.pad_id, volatile=True)
    y_valid, y_mask, y_pos_emb_indices, y_count = self._pad(
      sentences=y_valid, pad_id=self.pad_id, volatile=True)

    # shuffle if reaches the end of data
    if end_index >= self.valid_size:
      end_of_epoch = True
      self.valid_index = 0
    else:
      self.valid_index += batch_size

    return ((x_valid, x_mask, x_pos_emb_indices, x_count),
            (y_valid, y_mask, y_pos_emb_indices, y_count),
            batch_size, end_of_epoch)

  def next_train(self):
    """Retrieves a batch of training examples.

    Returns:
      (x_train, x_len): a pair of torch Tensors of size [batch, source_length]
        and [batch_size].
      (y_train, y_len): a pair of torch Tensors of size [batch, target_length]
        and [batch_size].
      end_of_epoch: whether we reach the end of training examples.
    """
    if self.hparams.batcher == "word":
      start_index = self.start_indices[self.train_queue[self.train_index]]
      end_index = self.end_indices[self.train_queue[self.train_index]]
    elif self.hparams.batcher == "sent":
      start_index = (self.train_queue[self.train_index] *
                     self.hparams.batch_size)
      end_index = min(start_index + self.hparams.batch_size, self.train_size)
    else:
      raise ValueError("Unknown batcher '{0}'".format(self.hparams.batcher))

    x_train = self.x_train[start_index : end_index]
    y_train = self.y_train[start_index : end_index]
    self.train_index += 1
    batch_size = len(x_train)

    # pad data
    if self.hparams.raml_source:
      x_train_raml, x_train, x_mask, x_pos_emb_indices, x_count = self._pad(
        sentences=x_train, pad_id=self.pad_id, raml=True,
        raml_tau=self.hparams.raml_src_tau,
        vocab_size=self.hparams.source_vocab_size,
        pad_corrupt=self.hparams.src_pad_corrupt,
        dist_corrupt=self.hparams.dist_corrupt)
    else:
      x_train, x_mask, x_pos_emb_indices, x_count = self._pad(
        sentences=x_train, pad_id=self.pad_id)

    if self.hparams.raml_target:
      y_train_raml, y_train, y_mask, y_pos_emb_indices, y_count = self._pad(
        sentences=y_train, pad_id=self.pad_id, raml=True,
        raml_tau=self.hparams.raml_tau,
        vocab_size=self.hparams.target_vocab_size,
        pad_corrupt=self.hparams.trg_pad_corrupt)
    else:
      y_train, y_mask, y_pos_emb_indices, y_count = self._pad(
        sentences=y_train, pad_id=self.pad_id)

    # shuffle if reaches the end of data
    if self.train_index > self.n_train_batches - 1:
      self.reset_train()

    if self.hparams.raml_source and self.hparams.raml_target:
      return ((x_train_raml, x_train, x_mask, x_pos_emb_indices, x_count),
              (y_train_raml, y_train, y_mask, y_pos_emb_indices, y_count),
              batch_size)

    if self.hparams.raml_source:
      return ((x_train_raml, x_train, x_mask, x_pos_emb_indices, x_count),
              (y_train, y_mask, y_pos_emb_indices, y_count),
              batch_size)

    if self.hparams.raml_target:
      return ((x_train, x_mask, x_pos_emb_indices, x_count),
              (y_train_raml, y_train, y_mask, y_pos_emb_indices, y_count),
              batch_size)

    return ((x_train, x_mask, x_pos_emb_indices, x_count),
            (y_train, y_mask, y_pos_emb_indices, y_count),
            batch_size)

  def _pad(self, sentences, pad_id, volatile=False,
           raml=False, raml_tau=1., vocab_size=None, pad_corrupt=False,
           dist_corrupt=False, no_corrupt_mask=None):
    """Pad all instances in [data] to the longest length.

    Args:
      sentences: list of [batch_size] lists.

    Returns:
      padded_sentences: Variable of size [batch_size, max_len], the sentences.
      mask: Variable of size [batch_size, max_len]. 1 means to ignore.
      pos_emb_indices: Variable of size [batch_size, max_len]. indices to use
        when computing positional embedding.
      sum_len: total number of words.
      raml: if True, the sentences will be replaced by their samples according
        to the exp payoff distribution, ie. exp(-HammingDist(s, sents)).
      vocab_size: if raml is True, vocab_size has to be specified for negative
        sampling.
      dist_corrupt: if is set to True, sample corrupt words based on word
        embedding distance.
      no_corrupt_mask: [batch_size, 1] if is set, then 1 means not to corrupt
        the sentence.
    """

    batch_size = len(sentences)
    lengths = [len(sentence) for sentence in sentences]
    sum_len = sum(lengths)
    max_len = max(lengths)

    padded_sentences = [
      sentence + ([pad_id] * (max_len - len(sentence)))
      for sentence in sentences]
    mask = [
      ([0] * len(sentence)) + ([1] * (max_len - len(sentence)))
      for sentence in sentences]
    pos_emb_indices = [
      [i+1 for i in range(len(sentence))] + ([0] * (max_len - len(sentence)))
      for sentence in sentences
    ]
    if raml:
      raml_mask = [
        [1] + ([0] * (len(sentence)-2)) + ([1] * (max_len-len(sentence)+1))
        for sentence in sentences]
      raml_mask = torch.ByteTensor(raml_mask)
      if self.hparams.cuda:
        raml_mask = raml_mask.cuda()
    padded_sentences = Variable(torch.LongTensor(padded_sentences))
    mask = torch.ByteTensor(mask)
    pos_emb_indices = Variable(torch.FloatTensor(pos_emb_indices))

    if self.hparams.cuda:
      padded_sentences = padded_sentences.cuda()
      mask = mask.cuda()
      pos_emb_indices = pos_emb_indices.cuda()

    if not raml:
      return padded_sentences, mask, pos_emb_indices, sum_len

    assert vocab_size is not None
    # first, sample the number of words to corrupt for each sentence
    logits = torch.arange(max_len)
    if self.hparams.cuda:
      logits = logits.cuda()
    logits = logits.mul_(-1).unsqueeze(0).expand_as(
      padded_sentences).contiguous().masked_fill_(mask, -self.hparams.inf)
    logits = Variable(logits, volatile=True)
    if self.hparams.cuda:
      logits = logits.cuda()
    probs = self.softmax(logits.mul_(raml_tau))
    num_words = torch.distributions.Categorical(probs).sample()

    # sample the indices
    lengths = torch.FloatTensor(lengths)
    if self.hparams.cuda:
      lengths = lengths.cuda()

    # mask out bos, eos
    corrupt_pos = num_words.data.float().div_(lengths-2).unsqueeze(
      1).expand_as(padded_sentences).contiguous().masked_fill_(raml_mask, 0)
    if no_corrupt_mask is not None:
      corrupt_pos.masked_fill_(no_corrupt_mask, 0)
    corrupt_pos = torch.bernoulli(corrupt_pos, out=corrupt_pos).byte()
    total_words = int(corrupt_pos.sum())
    if total_words == 0:
      return padded_sentences, padded_sentences, mask, pos_emb_indices, sum_len
    if dist_corrupt:
      # sample words according to distance
      words_to_corrupt = padded_sentences.data.masked_select(corrupt_pos)
      # (num_words, dim)
      words_emb_to_corrupt = torch.index_select(self.glove, dim=0, index=words_to_corrupt)
      # (num_words, num_vocab)
      w12 = torch.mm(words_emb_to_corrupt, self.glove.permute(1, 0))
      # (num_words, 1)
      w1 = torch.norm(words_emb_to_corrupt, 2, dim=1, keepdim=True)
      # (num_vocab, 1)
      w2 = torch.norm(self.glove, 2, dim=1, keepdim=True)
      # (num_words, num_vocab)
      distance = w12 / ((torch.mm(w1, w2.permute(1, 0))).clamp(min=1e-8))
      distance = Variable(distance)
      if self.hparams.cuda:
        distance = distance.cuda()
      # mask out the original words
      add = torch.arange(total_words).long()*vocab_size
      if self.hparams.cuda:
        add = add.cuda()
      corrupt_mask_index = (words_to_corrupt + add)
      distance = distance.view(-1)
      distance.data.index_fill_(0, corrupt_mask_index, -self.hparams.inf)
      distance = distance.view(total_words, -1)
      probs = torch.nn.functional.softmax(distance.mul_(self.hparams.dist_corrupt_tau), dim=1)
      corrupt_val = torch.distributions.Categorical(probs).sample().view(-1)
      #_, corrupt_val = torch.topk(probs, 1)

      if self.hparams.cuda:
        corrupt_val = corrupt_val.long().cuda()
        corrupt_pos = corrupt_pos.cuda()
      sample_sentences = padded_sentences.masked_scatter(Variable(corrupt_pos), corrupt_val)
    else:
      # sample the corrupts, which will be added to padded_sentences
      corrupt_val = torch.LongTensor(total_words)
      if pad_corrupt:
        corrupt_val.fill_(self.hparams.pad_id + vocab_size)
      else:
        corrupt_val = corrupt_val.random_(1, vocab_size)
      corrupts = torch.zeros(batch_size, max_len).long()
      if self.hparams.cuda:
        corrupt_val = corrupt_val.long().cuda()
        corrupts = corrupts.cuda()
        corrupt_pos = corrupt_pos.cuda()
      corrupts = corrupts.masked_scatter_(corrupt_pos, corrupt_val)

      sample_sentences = padded_sentences.add(Variable(corrupts)).remainder_(
        vocab_size).masked_fill_(Variable(mask), pad_id)
    #if dist_corrupt:
    #  print(sample_sentences)
    #  print(corrupt_val)
    #  print(words_to_corrupt)
    #  print(padded_sentences)
    #  print(corrupt_pos)
    #  corrupt_val = corrupt_val.data.view(-1).cpu().numpy()
    #  words_to_corrupt = words_to_corrupt.cpu().numpy()

    #  for c, w in zip(corrupt_val, words_to_corrupt):
    #    print("corrupt", self.source_index_to_word[c])
    #    print("orig", self.source_index_to_word[w])
    #  exit(0)
    return sample_sentences, padded_sentences, mask, pos_emb_indices, sum_len

  def _build_parallel(self, source_file, target_file, is_training, sort=False):
    """Build pair of data."""

    print("-" * 80)
    print("Loading parallel data from '{0}' and '{1}'".format(
      source_file, target_file))

    source_file = os.path.join(self.hparams.data_path, source_file)
    with open(source_file, encoding='utf-8') as finp:
      source_lines = finp.read().split("\n")

    target_file = os.path.join(self.hparams.data_path, target_file)
    with open(target_file, encoding='utf-8') as finp:
      target_lines = finp.read().split("\n")

    source_data, target_data = [], []
    source_lens = []
    total_sents = 0
    source_unk_count, target_unk_count = 0, 0
    for i, (source_line, target_line) in enumerate(
        zip(source_lines, target_lines)):
      source_line = source_line.strip()
      target_line = target_line.strip()
      if not source_line or not target_line:
        continue

      source_indices, target_indices = [self.bos_id], [self.bos_id]
      source_tokens = source_line.split(" ")
      target_tokens = target_line.split(" ")
      if is_training and len(target_line) > self.hparams.max_len:
        continue

      total_sents += 1

      for source_token in source_tokens:
        source_token = source_token.strip()
        if source_token not in self.source_word_to_index:
          source_token = self.hparams.unk
          source_unk_count += 1
        source_index = self.source_word_to_index[source_token]
        source_indices.append(source_index)

      for target_token in target_tokens:
        target_token = target_token.strip()
        if target_token not in self.target_word_to_index:
          target_token = self.hparams.unk
          target_unk_count += 1
        target_index = self.target_word_to_index[target_token]
        target_indices.append(target_index)

      source_indices.append(self.eos_id)
      target_indices.append(self.eos_id)

      source_lens.append(len(source_indices))
      source_data.append(source_indices)
      target_data.append(target_indices)

      if (self.hparams.n_train_sents is not None and
          self.hparams.n_train_sents <= total_sents):
        break

      if total_sents % 10000 == 0:
        print("{0:>6d} pairs. src_unk={1}. tgt_unk={2}".format(
          total_sents, source_unk_count, target_unk_count))

    assert len(source_data) == len(target_data)
    print("{0:>6d} pairs. src_unk={1}. tgt_unk={2}".format(
      total_sents, source_unk_count, target_unk_count))

    if sort:
      print("Heuristic sort based on source lens")
      indices = np.argsort(source_lens)
      source_data = [source_data[index] for index in indices]
      target_data = [target_data[index] for index in indices]

    return np.array(source_data), np.array(target_data)

  def _build_vocab(self, file_name):
    """Build word_to_index and index_to word dicts."""

    print("-" * 80)
    print("Loading vocab from '{0}'".format(file_name))
    file_name = os.path.join(self.hparams.data_path, file_name)
    with open(file_name, encoding='utf-8') as finp:
      lines = finp.read().split("\n")

    word_to_index, index_to_word = {}, {}
    for line in lines:
      line = line.strip()
      if not line:
        continue
      word_index = line.split("▁")
      if len(word_index) != 2:
        print("Weird line: '{0}'. split_len={1}".format(line, len(word_index)))
        continue
      word, index = word_index
      index = int(index)
      word_to_index[word] = index
      index_to_word[index] = word
      if word == self.hparams.unk:
        self.unk_id = index
      elif word == self.hparams.bos:
        self.bos_id = index
      elif word == self.hparams.eos:
        self.eos_id = index
      elif word == self.hparams.pad:
        self.pad_id = index

    assert len(word_to_index) == len(index_to_word), (
      "|word_to_index|={0} != |index_to_word|={1}".format(len(word_to_index),
                                                        len(index_to_word)))
    print("Done. vocab_size = {0}".format(len(word_to_index)))

    return word_to_index, index_to_word


        # "How often do you travel with an overnight stay for the purpose of cycling?": "1-3 times a year",
        # "How many nights is your average bike-centric trip?": "3-5 nights",
        # "What is your preferred mode of travel with bike(s)?": "Drive",
        # "How long are you willing to drive to get to a trail?": "I'll travel any length for a good ride!",
        # "What is your preferred accommodations on a bike trip? ": "Camping",
        # "What is your preferred maximum nightly cost for accommodations while on a bike trip? ": "$40 per night",

travel = {
    "1-3 times a year": 0,
    "4-8 times a year": 0,
    "9-15 times a year": 0,
    "16-30 times a year": 0,
    "30+ times a year": 0,
}
travelQuestion = "How often do you travel with an overnight stay for the purpose of cycling?"
travelFileName = "howOftenDoYouTravel.json"

nights = {
    "1-2 nights": 0,
    "3-5 nights": 0,
    "6-10 nights": 0,
    "10-14 nights": 0,
    "14+ nights": 0,
}

nightsQuestion = "How many nights is your average bike-centric trip?"
nightsFileName = "howManyNights.json"

mode = {
    "Backpacking": 0,
    "Drive": 0,
    "Fly": 0,
    "Train/Rail": 0,
}

modeQuestion = "What is your preferred mode of travel with bike(s)?"
modeFileName = "preferredModeOfTravel.json"

drive = {
    "Less than 30 minutes": 0,
    "Up to 1 hour": 0,
    "Up to 90 minutes": 0,
    "I'll travel any length for a good ride!": 0,
    "Other": 0,
}
driveQuestion = "How long are you willing to drive to get to a trail?"
driveFileName = "howLongAreYouWillingToDrive.json"

accommodations = {
    "Camping": 0,
    "Hotel": 0,
    "AirBnB/VRBO": 0,
    "Other": 0,
}
accommodationsQuestion = "What is your preferred accommodations on a bike trip? "
accommodationsFileName = "preferredAccommodations.json"

cost = {
    "$40 per night": 0,
    "$75 per night": 0,
    "$150 per night": 0,
    "No preference": 0,
}
costQuestion = "What is your preferred maximum nightly cost for accommodations while on a bike trip? "
costFileName = "preferredMaximumNightlyCost.json"

import json
def makeJson(answers, question, fileName):
     # What is your favorite MORC trail?
    with open("morc_data.json", "r") as json_file:
        data = json.load(json_file)
        for entry in data:
            for key, value in entry.items():
                if key == question:
                    for v in answers:
                        if v in value:
                            answers[v] += 1
        print(answers)
        question_data = []
        for q in answers:
            temp_data = {"answer": q, "value": answers[q]}
            question_data.append(temp_data)

        filePath = "../src/components/data/" + fileName
        with open(filePath, "w") as outfile:
            json.dump(question_data, outfile, indent=4)

    return question_data

print("Travel Question: ", makeJson(travel, travelQuestion, travelFileName))
print("Nights Question: ", makeJson(nights, nightsQuestion, nightsFileName))
print("Mode Question: ", makeJson(mode, modeQuestion, modeFileName))
print("Drive Question: ", makeJson(drive, driveQuestion, driveFileName))
print("Accommodations Question: ", makeJson(accommodations, accommodationsQuestion, accommodationsFileName))
print("Cost Question: ", makeJson(cost, costQuestion, costFileName))


import os
from werkzeug.contrib.fixers import ProxyFix
from flask import Flask, redirect, url_for
from flask_dance.contrib.slack import make_slack_blueprint, slack
from flask_sslify import SSLify
from raven.contrib.flask import Sentry

app = Flask(__name__)
app.wsgi_app = ProxyFix(app.wsgi_app)
sentry = Sentry(app)
sslify = SSLify(app)
app.secret_key = os.environ.get("FLASK_SECRET_KEY", "supersekrit")
app.config["SLACK_OAUTH_CLIENT_ID"] = os.environ.get("SLACK_OAUTH_CLIENT_ID")
app.config["SLACK_OAUTH_CLIENT_SECRET"] = os.environ.get("SLACK_OAUTH_CLIENT_SECRET")
slack_bp = make_slack_blueprint(scope=["identify", "chat:write:bot"])
app.register_blueprint(slack_bp, url_prefix="/login")

@app.route("/")
def index():
    if not slack.authorized:
        return redirect(url_for("slack.login"))
    resp = slack.post("chat.postMessage", data={
        "channel": "#general",
        "text": "ping",
        "icon_emoji": ":robot_face:",
    })
    assert resp.ok, resp.text
    return resp.text

if __name__ == "__main__":
    app.run()


"""
Testeo de la calculadora.
"""

from pywinauto import Desktop, Application
from time import sleep

def runCalc():
    app = Application(backend="uia").start('calc.exe')
    dlg = Desktop(backend="uia").Calculator

    log = open('log.txt', 'w', encoding='utf-8')

    # Comprobación de la suma
    suma = "Sumando 23 + 39 => Resultado esperado: 62 | Resultado obtenido: "
    dlg.type_keys('23{+}39=')
    resSuma = int(dlg.Static3.texts()[0].split(' ')[3])
    suma += str(resSuma) + " => CORRECTO\n" if 62 == resSuma else " => INCORRECTO\n"
    print(suma)
    sleep(1)

    # Comprobación de la resta
    resta = "Restando 23 - 39 => Resultado esperado: -16 | Resultado obtenido: "
    dlg.type_keys('23{-}39=')
    resResta = int(dlg.Static3.texts()[0].split(' ')[3])
    resta += str(resResta) + " => CORRECTO\n" if -16 == resResta else " => INCORRECTO\n"
    print(resta)
    sleep(1)

    # Comprobación de la multiplicación
    mult = "Multiplicando 23 * 39 => Resultado esperado: 897 | Resultado obtenido: "
    dlg.type_keys('23{*}39=')
    resMult = int(dlg.Static3.texts()[0].split(' ')[3])
    mult += str(resMult) + " => CORRECTO\n" if 897 == resMult else " => INCORRECTO\n"
    print(mult)
    sleep(1)

    # Comprobación de la división
    div = "Dividiendo 23 / 39 => Resultado esperado: 0,589743 | Resultado obtenido: "
    dlg.type_keys('23{/}39=')
    resDiv = float(dlg.Static3.texts()[0].split(' ')[3].replace(',', '.')[0:8])
    div += str(resDiv) + " => CORRECTO\n" if 0.589743 == resDiv else " => INCORRECTO\n"
    print(div)
    sleep(1)

    # Comprobación de la división por 0
    div0 = "Dividiendo 23 / 0 => Resultado esperado: No se puede dividir entre cero | Resultado obtenido: "
    dlg.type_keys('23{/}0=')
    resDiv0 = ' '.join(dlg.Static3.texts()[0].split(' ')[3:])
    div0 += str(resDiv0) +\
            " => CORRECTO\n" if "No se puede dividir entre cero" == resDiv0 else " => INCORRECTO\n"
    print(div0)
    sleep(1)

    log.writelines([suma, resta, mult, div, div0])

    log.close()

runCalc()


from flask import Flask, render_template, url_for, redirect, request
import cv2
import numpy as np
from keras.models import load_model
import statistics as st

app = Flask(__name__)

@app.route("/")
def home():
    return render_template("index.html")

@app.route("/<string:name_page>")
def page(name_page):
    return render_template(name_page)

@app.route("/<string:path>")
def path_fd(path):
    return path

@app.route('/camera', methods=['GET', 'POST'])
def camera():
        GR_dict={0:(0,255,0),1:(0,0,255)}
        model = load_model('./deep/model.hdf5')
        i = 0
        output = []
        # prevents openCL usage and unnecessary logging messages
        cv2.ocl.setUseOpenCL(False)

        # dictionary which assigns each label an emotion (alphabetical order)
        emotion_dict = {0: "Angry", 1: "Disgusted", 2: "Fearful", 3: "Happy", 4: "Neutral", 5: "Sad", 6: "Surprised"}

        # start the webcam feed
        # cap = cv2.VideoCapture(0)
        cap = cv2.VideoCapture(0)
        while (i <= 50):
            # Find haar cascade to draw bounding box around face
            ret, frame = cap.read()
            facecasc = cv2.CascadeClassifier('./deep/haarcascade_frontalface_default.xml')
            gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
            faces = facecasc.detectMultiScale(gray, scaleFactor=1.3, minNeighbors=5)

            for (x, y, w, h) in faces:
                # cv2.rectangle(frame, (x, y - 50), (x + w, y + h + 10), (255, 0, 0), 2)
                roi_gray = gray[y:y + h, x:x + w]
                cropped_img = np.expand_dims(np.expand_dims(cv2.resize(roi_gray, (48, 48)), -1), 0)
                prediction = model.predict(cropped_img)
                maxindex = int(np.argmax(prediction))
                predicted_emotion = emotion_dict[maxindex]
                output.append(predicted_emotion)
                cv2.rectangle(frame,(x,y),(x+w,y+h),GR_dict[1],2)
                cv2.rectangle(frame,(x,y-40),(x+w,y),GR_dict[1],-1)
                cv2.putText(frame, predicted_emotion, (x, y-10),cv2.FONT_HERSHEY_SIMPLEX,0.8,(255,255,255),2)

            i = i + 1

            # show the output frame
            cv2.imshow("Frame", frame)
            key = cv2.waitKey(1) & 0xFF
            if key == 27:
                cap.release()
                cv2.destroyAllWindows()
                break

            # if the `q` key was pressed, break from the loop
            if key == ord("q"):
                break
        print(output)
        cap.release()
        cv2.destroyAllWindows()
        final_output1 = st.mode(output)
        print(final_output1)
        return render_template("index.html", final_output=final_output1)
        # return redirect(url_for('home', final_output=final_output1))

    # return "hello"

if __name__ == "__main__":
    app.run(debug=True)


import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVR

# Function to calculate inflation_adjust price
def adjust_for_inflation(price, inflation_rate):
    return round(price * (1 + inflation_rate/100), 2)

# Read the CSV file
data = pd.read_csv('rice_beef_coffee_price_changes.csv')

# Filling values for 2006,2022
data.loc[data['Year'] == 2006, 'Inflation_rate'] = 31.96
data.loc[data['Year'] == 2022, 'Inflation_rate'] = 7.41

# Filling values for 2006,2022
data.loc[data['Year'].isin([2006, 2022]), 'Price_rice_infl'] = \
    data.loc[data['Year'].isin([2006, 2022])].apply(lambda row: adjust_for_inflation(row['Price_rice_kilo'], row['Inflation_rate']),axis=1)
data.loc[data['Year'].isin([2006, 2022]), 'Price_coffee_infl'] = \
    data.loc[data['Year'].isin([2006, 2022])].apply(lambda row: adjust_for_inflation(row['Price_coffee_kilo'], row['Inflation_rate']),axis=1)
data.loc[data['Year'].isin([2006, 2022]), 'Price_beef_infl'] = \
    data.loc[data['Year'].isin([2006, 2022])].apply(lambda row: adjust_for_inflation(row['Price_beef_kilo'], row['Inflation_rate']),axis=1)

# Transform to dateTime
data['Date'] = pd.to_datetime(data[['Year', 'Month']].assign(day=1))

# Set the datetime column as the index
data.set_index('Date', inplace=True)

# Drop the original nominal price columns and the inflation rate column
data.drop(columns=['Year', 'Month', 'Price_rice_kilo', 'Price_beef_kilo', 'Price_coffee_kilo', 'Inflation_rate'], inplace=True)

# Split the dataset into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(data.index, data, test_size=0.2, shuffle=False)

print(data.info())

# Train a SVR model for each target variable on the training set
models = {}
for col in data.columns:
    models[col] = make_pipeline(StandardScaler(), SVR(kernel='linear', C=10, epsilon=0.1, gamma='scale'))
    models[col].fit(X_train.to_numpy().reshape(-1, 1), y_train[col])

# Generate a date range for the next 50 years
start_date = pd.to_datetime('2023-03-26')
end_date = start_date + pd.DateOffset(years=50)
date_range = pd.date_range(start_date, end_date, freq='MS')

# Predict the prices for each product for each month in the date range
price_beef_infl_pred = models['Price_beef_infl'].predict(date_range.to_numpy().reshape(-1, 1))
price_rice_infl_pred = models['Price_rice_infl'].predict(date_range.to_numpy().reshape(-1, 1))
price_coffee_infl_pred = models['Price_coffee_infl'].predict(date_range.to_numpy().reshape(-1, 1))

# Store the predicted prices in a data frame to write into a csv
pred_df = pd.DataFrame({
    'Date': date_range,
    'Price_beef_infl': price_beef_infl_pred,
    'Price_rice_infl': price_rice_infl_pred,
    'Price_coffee_infl': price_coffee_infl_pred
})

# Set the date column as the index
pred_df.set_index('Date', inplace=True)

# Store the predicted prices to a CSV file
#pred_df.to_csv('predicted_prices.csv', float_format='%.2f')
print(pred_df)


import os
from lxml import etree
import numpy as np

class Config_XML():
    def __init__(self):
        self.config = {}
        self.project_dir = '.'

    def init_config(self):
        self.config['mesh_filename'] = os.path.realpath(os.path.join(self.project_dir, 'data', 'phantom.stl'))
        self.config['phantom_filename'] = os.path.realpath(os.path.join(self.project_dir, 'data', 'phantom.npz'))
        self.config['mesh_rot'] = np.zeros(3)
        self.config['mesh_trans'] = np.zeros(3)
        self.config['mesh_scale'] = np.ones(3)
        self.config['mesh_color'] = np.array([0.69411765, 0.47843137, 0.39607843])
        self.config['loop_slideshow'] = False

    def read(self, filename):
        tree = etree.parse(filename)
        root = tree.getroot()
        self.config['phantom_filename'] = os.path.realpath(os.path.join(self.project_dir, root.find('PhantomFilename').attrib['value']))
        self.config['mesh_filename'] = os.path.realpath(os.path.join(self.project_dir, root.find('MeshFilename').attrib['value']))
        self.config['mesh_rot'] = np.fromstring(root.find('MeshRotation').attrib['value'], sep=' ')
        self.config['mesh_trans'] = np.fromstring(root.find('MeshTranslation').attrib['value'], sep=' ')
        self.config['mesh_scale'] = np.fromstring(root.find('MeshScale').attrib['value'], sep=' ')
        self.config['mesh_color'] = np.fromstring(root.find('MeshColor').attrib['value'], sep=' ')
        self.config['loop_slideshow'] = root.find('LoopSlideshow').attrib['value'].lower() == 'true'

    def write(self, filename):
        root = etree.Element('InteractiveConeBeamReconstruction', version='0.1')
        etree.SubElement(root, 'PhantomFilename', value=self.config['phantom_filename'].replace(os.path.realpath(self.project_dir), '')[1:])
        etree.SubElement(root, 'MeshFilename', value=self.config['mesh_filename'].replace(os.path.realpath(self.project_dir), '')[1:])
        etree.SubElement(root, 'MeshRotation', value=np.array2string(self.config['mesh_rot'])[1:-1])
        etree.SubElement(root, 'MeshTranslation', value=np.array2string(self.config['mesh_trans'])[1:-1])
        etree.SubElement(root, 'MeshScale', value=np.array2string(self.config['mesh_scale'])[1:-1])
        etree.SubElement(root, 'MeshColor', value=np.array2string(self.config['mesh_color'])[1:-1])
        etree.SubElement(root, 'LoopSlideshow', value="True" if self.config['loop_slideshow'] else "False")
        tree = etree.ElementTree(root)
        tree.write(filename, encoding='utf8', method='xml', pretty_print=True)

if __name__ == '__main__':
    c = Config_XML()
    c.init_config()
    c.write('test.xml')
    c.read('test.xml')
    print(c.config)


from django.shortcuts import render
from enroll import forms

def first(request):
	#if request.method == 'POST':
	if request.POST.get('logpage'): #multiple html pages unnappudu ela if condition first line ela rayali...
		fm = forms.Student(request.POST)
		if fm.is_valid():
			print("Form is Valid!!")
			name = fm.cleaned_data['name']

			return render(request,'enroll/page2.html',{'nm':name})

	if request.POST.get('signpage'):
		fm = forms.Student(request.POST)
		if fm.is_valid():
			print("Form is Valid!!")
			name = fm.cleaned_data['name']

			return render(request,'enroll/page3.html',{'nm':name})

	else:
		fm =forms.Student()
	return render(request,'enroll/page1.html',context={'form':fm})


# coding=utf-8
import sys
from rdflib import Graph, Namespace, RDF, Literal, BNode, URIRef, XSD
import json
import random

class WSQueries:

	def __init__(self, ttl_file):
		self.g = Graph()
		self.g.parse(ttl_file, format="n3")

	def getDistinctGenres(self):
		query = "SELECT DISTINCT ?genre WHERE { ?a :hasGenre ?genre }"

		qres = self.g.query(query)

		l = []

		for row in qres:
			l.append(str(row[0]).split("#")[1])

		return l

	def getFilmsByGenre(self, genre):
		namespace="<http://www.semanticmovies.pt/#"+genre+">"

		l=[]
		query = "SELECT  ?film ?filmID ?poster  WHERE { "+namespace+" :isGenreOf ?a. ?a :hasTitle ?film. ?a :hasFilmID ?filmID. ?a :hasPoster ?poster}"

		qres=self.g.query(query)

		for row in qres:
			l.append({"hasTitle":str(row[0]),"hasFilmID":str(row[1]),"hasPoster":str(row[2])})

		return l

	def getRelatedFilms(self, id):

		listFilmGenre = []
		listDirectors = []
		listRelated = []

		filmGenre = self.getFilmGenre(id)
		for i in filmGenre:
			listFilmGenre.append( i["hasGenre"])

		directors = self.getDirectorsByFilm(id)

		listDirectors.append( directors[0]["hasPersonID"])

		for i in listFilmGenre:
			listRelated += random.sample(self.getFilmsByGenre(i), 5)

		for i in listDirectors:
			listRelated += self.getFilmsByDirector(i)

		aux = 0
		index = -1
		for i in listRelated:
			if i["hasFilmID"] == id:
				index = aux
				break
			aux+=1

		if index >= 0:
			#del listRelated[index]
			listRelated.pop(index)

		return random.sample(listRelated,6)

	def getFilmGenre(self, id):

		l=[]
		query = "SELECT ?filmID ?title ?genre WHERE {?a :hasGenre ?genre. ?a :hasFilmID ?filmID. ?a :hasTitle ?title. FILTER (regex(?filmID,'"+id+"')).}"
		qres=self.g.query(query)

		for row in qres:
			l.append({"hasFilmID":str(row[0]),"hasTitle":str(row[1]),"hasGenre":str(row[2]).split("#")[1]})

		return l

	def getFilmsByActor(self, personID):
		l=[]
		query = "SELECT ?actorID ?actor ?film ?filmID ?poster WHERE {?a :isActorOf ?b. ?a :hasPersonID ?actorID. ?a :hasName ?actor. ?b :hasTitle ?film. ?b :hasFilmID ?filmID. ?b :hasPoster ?poster. FILTER (regex(?actorID,'"+personID+"')).}"
		qres=self.g.query(query)

		for row in qres:
			l.append({"hasPersonID":str(row[0]),"hasName":str(row[1]),"hasTitle":str(row[2]), "hasFilmID":str(row[3]), "hasPoster":str(row[4])})

		return l

	def getFilmsByDirector(self, personID):
		l=[]
		query = "SELECT ?directorID ?director ?film ?filmID ?poster WHERE {?a :isDirectorOf ?b. ?a :hasPersonID ?directorID. ?a :hasName ?director. ?b :hasTitle ?film. ?b :hasFilmID ?filmID. ?b :hasPoster ?poster. FILTER (regex(?directorID,'"+personID+"')).}"
		qres=self.g.query(query)

		for row in qres:
			l.append({"hasPersonID":str(row[0]),"hasName":str(row[1]),"hasTitle":str(row[2]), "hasFilmID":str(row[3]), "hasPoster":str(row[4])})

		return l

	def getFilmsByWriter(self, personID):
		l=[]
		query = "SELECT ?writerID ?writer ?film ?filmID ?poster WHERE {?a :isWriterOf ?b. ?a :hasPersonID ?writerID. ?a :hasName ?writer. ?b :hasTitle ?film. ?b :hasFilmID ?filmID. ?b :hasPoster ?poster. FILTER (regex(?writerID,'"+personID+"')).}"
		qres=self.g.query(query)

		for row in qres:
			l.append({"hasPersonID":str(row[0]),"hasName":str(row[1]),"hasTitle":str(row[2]), "hasFilmID":str(row[3]), "hasPoster":str(row[4])})

		return l

	def getAllPersons(self):
		l=[]
		query = "SELECT ?personID ?person  WHERE { ?a :hasName ?person. ?a :hasPersonID ?personID.}"
		qres=self.g.query(query)

		for row in qres:
			l.append({"hasPersonID":str(row[0]),"hasName":str(row[1])})

		return l

	def getAllFilms(self):
		l=[]
		query = "SELECT ?filmID ?film ?poster WHERE {?a :hasTitle ?film. ?a :hasFilmID ?filmID. ?a :hasPoster ?poster }"
		qres=self.g.query(query)

		for row in qres:
			l.append({"hasFilmID":str(row[0]),"hasTitle":str(row[1]),"hasPoster":str(row[2])})

		return l

	def getPersonInfo(self, personID):
		l=[]
		query = "SELECT ?name ?id ?job WHERE { ?a :hasName ?name. ?a :hasPersonID ?id. ?a :hasJob ?job FILTER (regex(?id,'"+personID+"')).}"
		qres=self.g.query(query)

		for row in qres:
			l.append({"hasName":str(row[0]),"hasPersonID":personID,"hasJob":str(row[2])})

		return l[0]

	def getActorsByFilm(self, id):
		l=[]
		query = "SELECT ?filmID ?actor ?actorID WHERE {?a :hasActor ?b. ?a :hasFilmID ?filmID. ?b :hasName ?actor. ?b :hasPersonID ?actorID FILTER (regex(?filmID,'"+id+"')).}"
		qres=self.g.query(query)
		for row in qres:
			l.append({"hasFilmID":str(row[0]),"hasName":str(row[1]),"hasPersonID":str(row[2])})

		return l

	def getWritersByFilm(self, id):
		l=[]
		query = "SELECT ?filmID ?writer ?writerID WHERE {?a :hasWriter ?b. ?a :hasFilmID ?filmID. ?b :hasName ?writer. ?b :hasPersonID ?writerID. FILTER (regex(?filmID,'"+id+"')).}"
		qres=self.g.query(query)
		for row in qres:
			l.append({"hasFilmID":str(row[0]),"hasName":str(row[1]),"hasPersonID":str(row[2])})

		return l

	def getDirectorsByFilm(self, id):
		l=[]
		query = "SELECT ?filmID ?director ?directorID WHERE {?a :hasDirector ?b. ?a :hasFilmID ?filmID. ?b :hasName ?director. ?b:hasPersonID ?directorID. FILTER (regex(?filmID,'"+id+"')).}"
		qres=self.g.query(query)
		for row in qres:
			l.append({"hasFilmID":str(row[0]),"hasName":str(row[1]),"hasPersonID":str(row[2])})

		return l

	def getProducersByFilm(self, id):
		l=[]
		query = "SELECT ?filmID ?producer ?producerID WHERE {?a :hasProducer ?b. ?a :hasFilmID ?filmID. ?b :hasName ?producer. ?b :hasPersonID ?producerID. FILTER (regex(?filmID,'"+id+"')).}"
		qres=self.g.query(query)
		for row in qres:
			l.append({"hasFilmID":str(row[0]),"hasName":str(row[1]),"hasPersonID":str(row[2])})

		return l

	def getFilmInfo(self, id):
		l=[]
		query = "SELECT ?filmID ?title ?runtime ?release_date ?rating ?votes ?tagline ?plot ?poster ?actor WHERE {?a a :Film. ?a :hasFilmID ?filmID.  ?a :hasTitle ?title. ?a :hasRuntime ?runtime. ?a :hasReleaseDate ?release_date. ?a :hasRating ?rating. ?a :hasVotes ?votes. ?a :hasTagline ?tagline. ?a :hasPlot ?plot. ?a :hasPoster ?poster.  FILTER (regex(?filmID,'"+id+"')).}"

		qres=self.g.query(query)
		for row in qres:
			l.append({"hasFilmID":str(row[0]),"hasTitle":str(row[1]),"hasRuntime":int(row[2]),"hasReleaseDate":str(row[3]),"hasRating":str(row[4]),"hasVotes":str(row[5]),"hasTagline":str(row[6]),"hasPlot":str(row[7]),"hasPoster":str(row[8]),"hasActor":str(row[9])})

		return l[0]

	def searchFilm(self, keyword,dates,genre):
		l=[]

		namespace="<http://www.semanticmovies.pt/#"+genre+">"
		if dates[0] != "" and dates[1] != "":
			dates[0] = dates[0]+"-01-01"
			dates[1] = dates[1]+"-12-31"
			query = "SELECT ?id ?title ?runtime ?release_date ?genre ?rating ?votes ?tagline ?plot ?poster WHERE {?a a :Film. ?a :hasFilmID ?id.  ?a :hasTitle ?title. ?a :hasRuntime ?runtime. ?a :hasReleaseDate ?release_date. ?a :hasRating ?rating. ?a :hasVotes ?votes. ?a :hasTagline ?tagline. ?a :hasPlot ?plot. ?a :hasPoster ?poster. FILTER (regex(?title,'.*"+keyword+".*','i')). FIlTER (?release_date >= '"+dates[0]+"').FIlTER (?release_date <= '"+dates[1]+"')."

		elif dates[0] != "":
			dates[0] = dates[0]+"-01-01"
			query = "SELECT ?id ?title ?runtime ?release_date ?rating ?votes ?tagline ?plot ?poster WHERE {?a a :Film. ?a :hasFilmID ?id.  ?a :hasTitle ?title. ?a :hasRuntime ?runtime. ?a :hasReleaseDate ?release_date. ?a :hasRating ?rating. ?a :hasVotes ?votes. ?a :hasTagline ?tagline. ?a :hasPlot ?plot. ?a :hasPoster ?poster. FILTER (regex(?title,'.*"+keyword+".*','i')). FIlTER (?release_date >= '"+dates[0]+"')."

		elif dates[1] != "":
			dates[1] = dates[1]+"-12-31"
			query = "SELECT ?id ?title ?runtime ?release_date ?rating ?votes ?tagline ?plot ?poster WHERE {?a a :Film. ?a :hasFilmID ?id.  ?a :hasTitle ?title. ?a :hasRuntime ?runtime. ?a :hasReleaseDate ?release_date. ?a :hasRating ?rating. ?a :hasVotes ?votes. ?a :hasTagline ?tagline. ?a :hasPlot ?plot. ?a :hasPoster ?poster. FILTER (regex(?title,'.*"+keyword+".*','i')). FIlTER (?release_date <= '"+dates[1]+"')."

		else:
			query = "SELECT ?id ?title ?runtime ?release_date ?rating ?votes ?tagline ?plot ?poster WHERE {?a a :Film. ?a :hasFilmID ?id.  ?a :hasTitle ?title. ?a :hasRuntime ?runtime. ?a :hasReleaseDate ?release_date. ?a :hasRating ?rating. ?a :hasVotes ?votes. ?a :hasTagline ?tagline. ?a :hasPlot ?plot. ?a :hasPoster ?poster. FILTER (regex(?title,'.*"+keyword+".*','i')). "

		if genre != "":
			query += "?a :hasGenre "+namespace+"."

		query +="}"

		qres=self.g.query(query)
		for row in qres:
			l.append({"hasFilmID":str(row[0]),"hasTitle":str(row[1]),"hasRuntime":str(row[2]),"hasReleaseDate":str(row[3]),"hasRating":str(row[4]),"hasVotes":str(row[5]),"hasTagline":str(row[6]),"hasPlot":str(row[7]),"hasPoster":str(row[8])})

		return l

	def searchPerson(self, keyword,job):
		l = []
		namespace="<http://www.semanticmovies.pt/#"+job+">"

		if job != "":
			 query = "SELECT ?id ?person ?job  WHERE {?a a :Person. ?a :hasName ?person. ?a :hasPersonID ?id. ?a :hasJob "+namespace+". FILTER (regex(?person,'.*"+keyword+".*','i')).}"
		else:
			query = "SELECT ?id ?person ?job  WHERE {?a a :Person. ?a :hasName ?person. ?a :hasPersonID ?id. ?a :hasJob ?job. FILTER (regex(?person,'.*"+keyword+".*','i')).}"

		qres=self.g.query(query)

		for row in qres:
			l.append({"hasPersonID":str(row[0]),"hasName":str(row[1]),"hasJob":str(row[2])})

		for i in range(len(l)-1):

			for j in range(i+1,len(l)):
				if type(l[i])!=int and type(l[j])!=int:
					if (l[i]["hasName"]==l[j]["hasName"]):
						l[i]["hasJob"]=[l[i]["hasJob"],l[j]["hasJob"]]
						l[j] = -1

		if -1 in l:
			l.remove(-1)

	def getInfo(self, lista):
		l=[]

		if type(lista) == dict:
			lista=[lista]

		for i in lista:
			v = list(i.values())[0]
			if "Film" in v:
				query = "SELECT ?filmID ?title ?runtime ?release_date ?rating ?votes ?tagline ?plot ?poster ?actor WHERE {<"+v+"> :hasFilmID ?filmID. <"+v+"> :hasTitle ?title. <"+v+"> :hasRuntime ?runtime. <"+v+"> :hasReleaseDate ?release_date. <"+v+"> :hasRating ?rating. <"+v+"> :hasVotes ?votes. <"+v+"> :hasTagline ?tagline. <"+v+"> :hasPlot ?plot. <"+v+"> :hasPoster ?poster. }"

				qres=self.g.query(query)
				for row in qres:
					l.append({"hasFilmID":str(row[0]),"hasTitle":str(row[1]),"hasRuntime":str(row[2]),"hasReleaseDate":str(row[3]),"hasRating":str(row[4]),"hasVotes":str(row[5]),"hasTagline":str(row[6]),"hasPlot":str(row[7]),"hasPoster":str(row[8]),"hasActor":str(row[9])})
			else:
				query = "SELECT ?name ?id ?job WHERE { <"+v+"> :hasName ?name. <"+v+"> :hasPersonID ?id. <"+v+"> :hasJob ?job.}"
				qres=self.g.query(query)

				for row in qres:
					l.append({"hasName":str(row[0]),"hasPersonID":str(row[1]),"hasJob":str(row[2])})

		return l

	def generateQuery(self, dict,length):
		l = []

		if length==1:
			query="SELECT "+dict["selectWord"]+" WHERE {"+dict["query1"]+" "+dict["query2"]+"}"

		elif (":hasGenre" in dict["query1"] or ":hasGenre" in dict["query2"] ):
			namespace="<http://www.semanticmovies.pt/#"+dict["filterWord"].capitalize()+">"

			if ":hasGenre" in dict["query1"]:
				dict["query1"]=dict["query1"].replace(dict["filter"],namespace)
			else:
				dict["query2"]=dict["query2"].replace(dict["filter"],namespace)
			query="SELECT "+dict["selectWord"]+" WHERE {"+dict["query1"]+" "+dict["query2"]+"}"
		elif (":hasReleaseDate" in dict["query1"] or ":hasReleaseDate" in dict["query2"] ):
			 query="SELECT "+dict["selectWord"]+" WHERE {"+dict["query1"]+" "+dict["query2"]+" FILTER ("+dict["filter"]+" >= '"+dict["filterWord"]+"-01-01').  FILTER ("+dict["filter"]+" <= '"+dict["filterWord"]+"-12-31').}"
		elif (":hasRating" in dict["query1"] or ":hasRating" in dict["query2"] ):
			query="SELECT "+dict["selectWord"]+" WHERE {"+dict["query1"]+" "+dict["query2"]+" FILTER ("+dict["filter"]+" >= "+dict["filterWord"]+"). FILTER ("+dict["filter"]+" <= "+dict["filterWord"].split(".")[0]+".9).}"
		else:
			query="SELECT "+dict["selectWord"]+" WHERE {"+dict["query1"]+" "+dict["query2"]+" FILTER (regex("+dict["filter"]+",'.*"+dict["filterWord"]+".*','i')).}"

		qres=self.g.query(query)
		for row in qres:
			l.append({"hasFilmID":str(row[0])})

		if len(l) > 1 or len(l) == 0:
			return l
		else:
			return l[0]

	def searchSemantic(self, frase):
		reservedWords=["todos","tem","cujo","de","da","do","com","em","que","os","o","as","a","onde","são","com","foi","é","no","participam","participa","participou","por","pelo"]

		ontology = {"filmes":"","filme":"","classificação":":hasRating","classificacao":":hasRating","genero":":hasGenre","ano":":hasReleaseDate","titulo":":hasTitle","nome":":hasName","atrizes":":hasActor","atriz":":hasActor","ator":":hasActor","atores":":hasActor","realizado":":hasDirector","realizados":":hasDirector","realizador":":hasDirector","dirigido":":hasDirector","dirigidos":":hasDirector","realizadores":":hasDirector","diretor":":hasDirector","diretores":":hasDirector","produzidos":":hasProducer","produzido":":hasProducer","produtor":":hasProducer","produtores":":hasProducer","escrito":":hasWriter","escritor":":hasWriter","escritores":":hasWriter"}
		string = frase.split()
		query1=""
		query2=""
		filterWord=""
		listaQuery=[]
		dict={}

		for i in string:
			if i.lower() in reservedWords:
				continue;
			elif i.lower() in ontology.keys():
				listaQuery.append(i)
			else:
				filterWord+=i+" "

		filterWord=filterWord.rstrip()

		if len(listaQuery)==1:
			dict["filterWord"] = ""
			dict["filter"] = "?a"
			dict["selectWord"] ="?a"
			dict["query2"] = ""

			if listaQuery[0] == "filme" or listaQuery[0] == "filmes":
				dict["query1"] = "?a a :Film."
			else:
				job = ontology[listaQuery[0]]
				job = job.replace(":has","")
				namespace = "<http://www.semanticmovies.pt/#"+job+">"
				dict["query1"] = "?a :hasJob "+namespace+"."

		elif len(listaQuery)==2:
			if ontology[listaQuery[1]]!="":
				query1="?a "+ontology[listaQuery[1]]+" ?b"
			else:
				query1="?a "+ontology[listaQuery[0]]+" ?b"

			if listaQuery[1] == "filme" or listaQuery[1] == "filmes":
				dict["filter"]="?a"
				dict["selectWord"]="?b"
			else:
				dict["filter"]="?b"
				dict["selectWord"]="?a"
			dict["filterWord"]=filterWord
			dict["query1"]=query1+"."
			if(":hasName" not in dict["query1"] and ":hasTitle" not in dict["query1"] and ":hasGenre" not in dict["query1"] and ":hasReleaseDate" not in dict["query1"] and ":hasRating" not in dict["query1"]):
				if listaQuery[1] == "filme" or listaQuery[1] == "filmes":
					 dict["query1"]+=dict["filter"]+" :hasTitle ?c."
				else:
					dict["query1"]+=dict["filter"]+" :hasName ?c."
				dict["filter"]="?c"
			dict["query2"]=""

		elif len(listaQuery)==3:
			query1+="?a "+ontology[listaQuery[0]]+ontology[listaQuery[1]]+" ?b"

			if listaQuery[1] == "filme" or listaQuery[1] == "filmes":
				compose="?a"
				dict["selectWord"]="?b"
			else:
				compose="?b"
				dict["selectWord"]="?a"

			query2+=compose+" "+ontology[listaQuery[2]]+" ?c"
			dict["filter"]="?c"
			dict["filterWord"]=filterWord
			dict["query1"]=query1+"."
			dict["query2"]=query2+". "
			if(":hasName" not in dict["query2"] and ":hasTitle" not in dict["query2"] and ":hasGenre" not in dict["query2"] and ":hasReleaseDate" not in dict["query2"] and ":hasRating" not in dict["query2"]):
				if listaQuery[1] == "filme" or listaQuery[1] == "filmes":
					 dict["query2"]+=dict["filter"]+" :hasTitle ?d."
				else:
					dict["query2"]+=dict["filter"]+" :hasName ?d."
				dict["filter"]="?d"

		l = self.generateQuery(dict,len(listaQuery))
		l = self.getInfo(l)
		return l


# -*- coding = utf-8 -*-
# @Time : 2023/2/27 0:34
# @Author : 彭睿
# @File : plot_draw.py
# @Software : PyCharm

#因为买了才能评论，而且几天后会自动评论，所以按照评论量来定义购买量
import pandas as pd
import matplotlib.pyplot as plt
import codecs
import jieba
from wordcloud import WordCloud

plt.rcParams['font.sans-serif'] = 'SimHei'  # 设置中文显示
plt.rcParams['axes.unicode_minus'] = False

def ReadFile(filePath,encoding="utf-8"):
    with codecs.open(filePath,"r",encoding) as f:
        return f.read()

pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)

frigedf = pd.read_csv('冰箱.csv')
washdf = pd.read_csv('洗衣机.csv')
waterdf = pd.read_csv('热水器.csv')
tvdf = pd.read_csv('电视.csv')
breakdf = pd.read_csv('破壁机.csv')
frieddf = pd.read_csv('空气炸锅.csv')
airdf = pd.read_csv('空调.csv')
wetdf = pd.read_csv('除湿器.csv')

def draw_market_buy_times(x):
    x = x.drop('Unnamed: 0', axis=1, level=None, inplace=False)
    df = x.groupby(['item']).agg(item=('item','first'),comments=('comments','sum'))
    df = df.sort_values('comments', ascending=False)
    print(df)
    values = df['comments']
    labels = []
    for i in df['item']:
        labels.append(i)
    labels = labels[0:10]
    plt.figure(figsize=(8,8))
    plt.pie(values[0:10], labels=labels, autopct='%1.1f%%')
    plt.title('市场购买份额占比前十名')
    plt.savefig('D:/数据爬取/test/市场份额前十占比图/frige.jpg')
    plt.show()

def top_good_comment(x):
    x = x.drop('Unnamed: 0', axis=1, level=None, inplace=False)
    df = x.groupby(['item']).agg(item=('item', 'first'), comments=('good', 'sum'))
    df = df.sort_values('comments', ascending=False)
    print(df)
    values = df['comments']
    labels = []
    for i in df['item']:
        labels.append(i)
    labels = labels[0:10]
    plt.figure(figsize=(8, 8))
    plt.pie(values[0:10], labels=labels, autopct='%1.1f%%')
    plt.title('市场购买好评数前十名')
    plt.savefig('D:/数据爬取/test/市场购买好评前十占比图/frige.jpg')
    plt.show()

def top_bad_comment(x):
    x = x.drop('Unnamed: 0', axis=1, level=None, inplace=False)
    df = x.groupby(['item']).agg(item=('item', 'first'), comments=('bad', 'sum'))
    df = df.sort_values('comments', ascending=False)
    print(df)
    values = df['comments']
    labels = []
    for i in df['item']:
        labels.append(i)
    labels = labels[0:10]
    plt.figure(figsize=(8, 8))
    plt.pie(values[0:10], labels=labels, autopct='%1.1f%%')
    plt.title('市场购买差评数前十名')
    plt.savefig('D:/数据爬取/test/市场购买差评前十占比图/frige.jpg')
    plt.show()

def hot_selling_goods_mean_price_draw(x):
    x = x.drop('Unnamed: 0', axis=1, level=None, inplace=False)
    x =x[x['comments'] >= 100000]
    # print(x)
    df = x.groupby(['item']).agg(item=('item', 'first'), mean_price=('price', 'mean'))
    df = df.sort_values('mean_price', ascending=False)
    print(df)
    columns = []
    values = []
    for i in df['item']:
        columns.append(i)
    for i in df['mean_price']:
        values.append(int(i))
    plt.figure(figsize=(8, 6))
    plt.title('产品销售量超10万的产品售价均值')
    plt.ylabel('元')
    plt.bar(columns[0:], values[0:])
    my_height = values[0:]
    for i in range(len(my_height)):
        plt.text(i, my_height[i] + 100, my_height[i], va='bottom', ha='center')
    plt.savefig('D:/数据爬取/test/市场热卖商品价格均值/frige.jpg')
    plt.show()

if __name__ == '__main__':
    x = frigedf
    draw_market_buy_times(x)
    top_good_comment(x)
    top_bad_comment(x)
    hot_selling_goods_mean_price_draw(x)


# -*- coding: utf-8 -*-
import torch  # pytorch 0.4.0! fft
import torch.nn as nn
import numpy as np

def complex_mul(x, z):
    out_real = x[..., 0] * z[..., 0] - x[..., 1] * z[..., 1]
    out_imag = x[..., 0] * z[..., 1] + x[..., 1] * z[..., 0]
    return torch.stack((out_real, out_imag), -1)

def complex_mulconj(x, z):
    out_real = x[..., 0] * z[..., 0] + x[..., 1] * z[..., 1]
    out_imag = x[..., 1] * z[..., 0] - x[..., 0] * z[..., 1]
    return torch.stack((out_real, out_imag), -1)

class DCFNetFeature(nn.Module):
    def __init__(self):
        super(DCFNetFeature, self).__init__()
        self.feature = nn.Sequential(
            nn.Conv2d(3, 32, 3, padding=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(32, 32, 3, padding=1),
            nn.LocalResponseNorm(size=5, alpha=0.0001, beta=0.75, k=1),
        )

    def forward(self, x):
        return self.feature(x)

class CACFNet(nn.Module):

    def __init__(self, config=None):
        super(CACFNet, self).__init__()
        self.feature = DCFNetFeature()
        self.yf = config.yf.clone()
        self.lambda0 = config.lambda0
        self.config = config
        self.flag = False  # True for train, Flase for interference

    def forward(self, x):
        if self.flag:
            # self.feature.feature[0].padding = (0, 0)
            # self.feature.feature[2].padding = (0, 0)
            x = self.feature(x)
        else:
            # self.feature.feature[0].padding = (1, 1)
            # self.feature.feature[2].padding = (1, 1)
            x = self.feature(x) * self.config.cos_window
        xf = torch.rfft(x, signal_ndim=2)
        kxzf = torch.sum(complex_mulconj(xf, self.model_zf), dim=1, keepdim=True)
        response = torch.irfft(complex_mul(kxzf, self.model_alphaf), signal_ndim=2)
        return response

    def CACF_update(self, inputs, lr=1.):

        inputs[0] = self.feature(inputs[0]) * self.config.cos_window
        inputs[1] = self.feature(inputs[1]) * self.config.cos_window
        inputs[2] = self.feature(inputs[2]) * self.config.cos_window
        inputs[3] = self.feature(inputs[3]) * self.config.cos_window
        inputs[4] = self.feature(inputs[4]) * self.config.cos_window
        zf = torch.rfft(inputs[0], signal_ndim=2)  # target region
        cf1 = torch.rfft(inputs[1], signal_ndim=2)  # contex 1 region
        cf2 = torch.rfft(inputs[2], signal_ndim=2)  # contex 2 region
        cf3 = torch.rfft(inputs[3], signal_ndim=2)  # contex 3 region
        cf4 = torch.rfft(inputs[4], signal_ndim=2)  # contex 4 region
        kzzf = torch.sum(torch.sum(zf ** 2, dim=4, keepdim=True), dim=1, keepdim=True)
        kccf1 = torch.sum(torch.sum(cf1 ** 2, dim=4, keepdim=True), dim=1, keepdim=True)
        kccf2 = torch.sum(torch.sum(cf2 ** 2, dim=4, keepdim=True), dim=1, keepdim=True)
        kccf3 = torch.sum(torch.sum(cf3 ** 2, dim=4, keepdim=True), dim=1, keepdim=True)
        kccf4 = torch.sum(torch.sum(cf4 ** 2, dim=4, keepdim=True), dim=1, keepdim=True)

        if lr > 0.99:
            alphaf = self.config.yf / (kzzf + self.config.lambda0)
        else:
            alphaf = self.config.yf / (kzzf + self.config.lambda0 + self.config.lambda1 * (kccf1 + kccf2 + kccf3 + kccf4))

        if lr > 0.99:
            self.model_alphaf = alphaf
            self.model_zf = zf
        else:
            self.model_alphaf = (1 - lr) * self.model_alphaf.data + lr * alphaf.data
            self.model_zf = (1 - lr) * self.model_zf.data + lr * zf.data

    def load_param(self, net_path):
        checkpoint = torch.load(net_path)
        # for param
        self.feature.feature[0].weight.data = checkpoint['feature.0.weight'].data
        self.feature.feature[0].bias.data = checkpoint['feature.0.bias'].data
        self.feature.feature[2].weight.data = checkpoint['feature.2.weight'].data
        self.feature.feature[2].bias.data = checkpoint['feature.2.bias'].data
        # for checkpoint
        # self.load_state_dict(checkpoint['state_dict'])

if __name__ == '__main__':

    # network test
    pass


import csv
import json

from datetime import datetime

def structure_date_by_iso_format(date: str | None) -> str | None:
    if date is None:
        return None
    date = [int(elem) for elem in date.split('.')]
    if len(date) == 2:
        return None
    date.reverse()
    iso_date_format = datetime(year=date[0], month=date[1], day=date[2]).date().isoformat()
    return iso_date_format

def structure_data_by_fields(data: list) -> list:
    structure_data = {}
    report = []
    items = data
    for item in items:
        structure_data['first_name'] = item['first_name'] if item.get('first_name') else None
        structure_data['last_name'] = item['last_name'] if item.get('last_name') else None
        structure_data['country'] = item['country']['title'] if item.get('country') else None
        structure_data['city'] = item['city']['title'] if item.get('city') else None
        structure_data['birthdate'] = structure_date_by_iso_format(item.get('bdate'))
        structure_data['sex'] = "male" if item['sex'] == 2 else "female"
        report.append(structure_data.copy())
    return report

def write_data_to_file(friends: list, file_path_and_name: str, file_format: str) -> None:
    with open(f"{file_path_and_name}.{file_format}", 'w', encoding="utf-8") as file:
        if file_format == 'json':
            json.dump(friends, file, indent=4, ensure_ascii=False)
        elif file_format == 'csv':
            writer = csv.writer(file)
            writer.writerow(friends[0].keys())
            writer.writerows(map(dict.values, friends))
        elif file_format == 'tsv':
            for friend in friends:
                line = [elem if elem else '' for elem in friend.values()]
                line = '\t'.join(line)
                file.write(f"{line}\n")

def sort_friends_data_by_name(unsorted_items: list) -> list:
    sorted_items = sorted(unsorted_items, key=lambda k: k['first_name'])
    return sorted_items

def create_report(friends_data: list, file_path_and_name: str, file_format: str) -> None:
    structure_friends_data = structure_data_by_fields(friends_data)
    sorted_friends_by_name = sort_friends_data_by_name(structure_friends_data)
    write_data_to_file(sorted_friends_by_name, file_path_and_name, file_format)


#! /usr/bin/env python3

from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
import json
import secrets

# CAUTION: `random` isn't cryptographically secure. In general, prefer to use
# `secrets`. But `secrets` currently lacks a shuffle() function, so we just
# import that specific function from `random`.
from random import shuffle

animals = [
    "aardvark",
    "butterfly",
    "cat",
    "dog",
    "elephant",
    "fox",
    "giraffe",
    "hippopotamus",
    "iguana",
    "jaguar",
    "kangaroo",
    "llama",
    "manatee",
    "narwhal",
    "octopus",
    "pig",
    "quail",
    "rhinoceros",
    "sheep",
    "turkey",
    "unicorn",
    "vulture",
    "wombat",
    "xenoceratops",
    "yak",
    "zebra",
]

def random_animals(n):
    return " ".join(secrets.choice(animals) for _ in range(n))

def random_string(length):
    return random_animals(length)[:length]

def xor_bytes(a, b):
    assert len(a) == len(b)
    return bytes(x ^ y for x, y in zip(a, b))

def AES_encrypt_block(key, block):
    assert len(key) == 16
    assert len(block) == 16
    return Cipher(algorithms.AES(key), modes.ECB()).encryptor().update(block)

inputs = {}

# Problem 1
inputs["problem1"] = random_string(16)

# Problem 2
key = b"A" * 16
inputs["problem2"] = AES_encrypt_block(key, random_string(16).encode()).hex()

# Problem 3
plaintext_len = (secrets.randbelow(4) + 2) * 16
plaintext = random_string(plaintext_len)
assert len(plaintext) % 16 == 0
inputs["problem3"] = plaintext

# Problem 4
plaintext_len = (secrets.randbelow(3) + 2) * 16
plaintext = random_string(plaintext_len)
assert len(plaintext) % 16 == 0
ciphertext = (
    Cipher(algorithms.AES(key), modes.ECB()).encryptor().update(plaintext.encode())
)
assert len(plaintext) == len(ciphertext)
inputs["problem4"] = ciphertext.hex()

# Problem 5
inputs["problem5"] = [
    s.encode().hex()
    for s in [
        "z" * 15,
        random_animals(3),
        random_animals(4),
        "z" * 32,
    ]
]

# Problem 6
strings = [random_animals(3).encode().hex() for _ in range(3)]
inputs["problem6"] = []
for i in range(len(strings)):
    s = bytes.fromhex(strings[i])
    padding = 16 - (len(s) % 16)
    s += bytes([padding] * padding)
    inputs["problem6"].append(s.hex())

# Problem 7
lyrics = "John Jacob Jingleheimer Schmidt! His name is my name too. Whenever we go out the people always shout there goes John Jacob Jingleheimer Schmidt! Nanananananana..."
inputs["problem7"] = {
    "lyrics": lyrics,
    "key": secrets.token_bytes(16).hex(),
}

# Problem 8
inputs["problem8"] = {
    "key": secrets.token_bytes(16).hex(),
    "nonce": secrets.token_bytes(12).hex(),
    "plaintext": random_animals(5),
}

# Problem 9
inputs["problem9"] = secrets.token_bytes(16).hex()

print(json.dumps(inputs, indent="  "))


import requests
from bs4 import BeautifulSoup

def get_bs_obj(company_code):
    url     = "https://finance.naver.com/item/main.nhn?code="+company_code
    result  = requests.get(url)
    bsObj   = BeautifulSoup(result.content, "html.parser")
    return bsObj

def get_candle_chart_data(company_code):
    bsObj    = get_bs_obj(company_code)

    td_first = bsObj.find("td", {"class":"first"})
    close = td_first.find("span", {"class":"blind"}).text #종가

    tbl_no_info = bsObj.find("table", {"class":"no_info"})

    tr = tbl_no_info.findAll("tr")[0] #첫번째 tr
    td = tr.findAll("td")[1] #두번째 td
    high = td.findAll("span", {"class":"blind"})[0].text #고가

    tr      = tbl_no_info.findAll("tr")[1]  # 두번째 tr
    td_open = tr.find("td", {"class":"first"})  # td.first
    open    = td_open.find("span", {"class":"blind"}).text #시가
    td_low  = tr.findAll("td")[1]
    low     = td_low.find("span", {"class":"blind"}).text #저가
    print(low)

    return {"close":close, "high":high, "open":open, "low":low}

close = get_candle_chart_data("035420") #종가
print(close)


import torch
import torch.nn as nn
from thop import profile

# https://github.com/d-li14/octconv.pytorch
class OctConv(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, alpha_in=0.5, alpha_out=0.5, stride=1, padding=0, dilation=1,
                 groups=1, bias=True):
        super(OctConv, self).__init__()
        self.downsample = nn.AvgPool2d(kernel_size=(2, 2), stride=2)
        self.upsample = nn.Upsample(scale_factor=2, mode='nearest')
        self.alpha_in, self.alpha_out = alpha_in, alpha_out
        self.low_ch_in, self.high_ch_in = int(self.alpha_in * in_channels), int((1 - self.alpha_in) * in_channels)
        self.low_ch_out, self.high_ch_out = int(self.alpha_out * out_channels), int((1 - self.alpha_out) * out_channels)

        self.conv_l2l = None if alpha_in == 0 or alpha_out == 0 else \
            nn.Conv2d(self.low_ch_in, self.low_ch_out, kernel_size, stride, padding, dilation, groups, bias)
        self.conv_l2h = None if alpha_in == 0 or alpha_out == 1 else \
            nn.Conv2d(self.low_ch_in, self.high_ch_out, kernel_size, stride, padding, dilation, groups, bias)
        self.conv_h2l = None if alpha_in == 1 or alpha_out == 0 else \
            nn.Conv2d(self.high_ch_in, self.low_ch_out, kernel_size, stride, padding, dilation, groups, bias)
        self.conv_h2h = None if alpha_in == 1 or alpha_out == 1 else \
            nn.Conv2d(self.high_ch_in, self.high_ch_out, kernel_size, stride, padding, dilation, groups, bias)

    def forward(self, x):
        x_h, x_l = x if type(x) is tuple else (x, None)

        if x_h is not None:
            # High -> High conv
            x_h2h = self.conv_h2h(x_h)

            # High -> Low conv
            x_h2l = self.downsample(x_h)
            x_h2l = self.conv_h2l(x_h2l) if self.alpha_out > 0 else None

        if x_l is not None:
            # Low -> High conv
            x_l2h = self.conv_l2h(x_l)
            x_l2h = self.upsample(x_l2h)

            # Low -> Low conv
            x_l2l = self.conv_l2l(x_l) if self.alpha_out > 0 else None

            # Cross Add
            x_h = x_h2h + x_l2h
            x_l = x_l2l + x_h2l if x_h2l is not None and x_l2l is not None else None
            return x_h, x_l
        else:
            return x_h2h, x_h2l

class OctBlock(nn.Module):
    def __init__(self, ch_in=64, ch_out=64, kernel_size=3, alpha=0.25, mode='middle'):
        super(OctBlock, self).__init__()
        padding = kernel_size // 2
        self.mode = mode
        if self.mode == 'first':        # First
            alpha_in, alpha_out = 0, alpha
        elif self.mode == 'last':       # Last
            alpha_in, alpha_out = alpha, 0
        else:                           # Middle
            alpha_in, alpha_out = alpha, alpha
        self.conv_oct = OctConv(ch_in, ch_out, kernel_size=kernel_size, padding=padding, alpha_in=alpha_in, alpha_out=alpha_out)
        self.relu = nn.ReLU()

    def forward(self, x):
        x_h, x_l = self.conv_oct(x)
        if type(x) is not tuple:        # First
            x_h, x_l = self.relu(x_h), self.relu(x_l)
        else:
            if self.mode == 'last':     # Last
                x_h = self.relu(x_h)
                return x_h
            else:                       # Middle
                x_h += x[0]             # High
                x_l += x[1]             # Low
                x_h, x_l = self.relu(x_h), self.relu(x_l)
        return x_h, x_l

class CNN_OCT(nn.Module):
    def __init__(self, channels=64, kernel_size=3, alpha=0.25):
        super(CNN_OCT, self).__init__()
        padding = kernel_size // 2

        self.conv_first = nn.Conv2d(1, channels, kernel_size=kernel_size, padding=padding)
        blocks = [OctBlock(ch_in=64, ch_out=64, kernel_size=3, alpha=alpha, mode='first')]
        for i in range(1, 9):
            blocks.append(OctBlock(ch_in=64, ch_out=64, kernel_size=3, alpha=alpha, mode='middle'))
        blocks.append(OctBlock(ch_in=64, ch_out=64, kernel_size=3, alpha=alpha, mode='last'))
        self.blocks = nn.ModuleList(blocks)
        self.conv_recon = nn.Conv2d(channels, 1, kernel_size=kernel_size, padding=padding)
        self.relu = nn.ReLU()

    def forward(self, x):
        x_in = x
        x = self.conv_first(x)
        x = self.relu(x)

        for block in self.blocks:
            x = block(x)

        x = self.conv_recon(x)
        x += x_in
        x = self.relu(x)
        return x

if __name__ == "__main__":
    x = torch.ones((16, 1, 64, 64))
    model = CNN_OCT(channels=64, kernel_size=3, alpha=0.75)
    z = model(x)

    device = "cpu"
    if torch.cuda.is_available():
        device = "cuda"

    model = model.to(device)
    x = x.to(device)
    total_ops, total_params = profile(model, (x,), verbose=False)

    print()
    print("Params(M): {:.4f} ".format(total_params / (1000 ** 2)))
    print(" FLOPs(G): {:.2f}".format(total_ops / (1000 ** 3)))
    print("Output shape:", z.shape)
    print()


from flask import Blueprint
main = Blueprint('main', __name__)
import os
import json
from recommendationFramework import Engine

import logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

from flask import Flask, request

def create_app():
    global recommendation_engine

    cur_dir = os.getcwd()

    books_path = os.path.join(cur_dir, 'datasets', 'books6ml.csv')
    ratings_path = os.path.join(cur_dir, 'datasets', 'ratings6ml.csv')
    model_path = os.path.join(cur_dir, 'models', 'ALS')

    recommendation_engine =  Engine.RecommendationEngine(ratings_path, books_path, model_path, 'ALS')

    app = Flask(__name__)
    app.register_blueprint(main)
    return app

@main.route("/<int:user_id>/ratings/top/<int:count>", methods=["GET"])
def top_ratings(user_id, count):
    logger.debug("User %s TOP ratings requested", user_id)
    top_ratings = recommendation_engine.alsTopNForUser(user_id,count)
    return json.dumps(top_ratings)

#  add this when adding recommendation for a book
# @main.route("/<int:user_id>/ratings/<int:book_id>", methods=["GET"])
# def book_ratings(user_id, book_id):
#     logger.debug("User %s rating requested for book %s", user_id, book_id)
#     ratings =  recommendation_engine.get_ratings_for_book_ids(user_id, [book_id])
#     return json.dumps(ratings)

@main.route("/<int:user_id>/ratings", methods = ["POST"])
def add_ratingsALS(user_id):
    # get the ratings from the Flask POST request object
    ratings_list = request.form.keys()[0].strip().split("\n")
    ratings_list = map(lambda x: x.split(","), ratings_list)
    # create a list with the format required by the engine (user_id, book_id, rating)
    ratings = map(lambda x: (user_id, int(x[0]), float(x[1])), ratings_list)
    # add them to the model using then engine API
    recommendation_engine.add_ratingsALS(ratings)

    return json.dumps(ratings)


from distutils.command.build import build
import pandas as pd
import numpy as np
import tensorflow as tf
from matplotlib import pyplot as plt

pd.options.display.max_rows = 10
pd.options.display.float_format = "{:.1f}".format

if tf.config.list_physical_devices('GPU'):
  print("TensorFlow **IS** using the GPU")
else:
  print("TensorFlow **IS NOT** using the GPU")

def build_model(my_learning_rate):
    model = tf.keras.models.Sequential()

    model.add(tf.keras.layers.Dense(units=10, input_shape=(1,)))
    model.add(tf.keras.layers.Dense(units=10, input_shape=(1,)))
    model.add(tf.keras.layers.Dense(units=10, input_shape=(1,)))

    model.compile(optimizer=tf.keras.optimizers.RMSprop(learning_rate=my_learning_rate), loss="mean_squared_error", metrics=[tf.keras.metrics.RootMeanSquaredError()])

    return model

# def build_model(my_learning_rate):
#     model = tf.keras.models.Sequential()

#     model.add(tf.keras.layers.Dense(units=10))
#     model.add(tf.keras.layers.Dense(units=10))

#     model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=my_learning_rate), loss=tf.keras.losses.BinaryCrossentropy(), metrics=[tf.keras.metrics.BinaryAccuracy(),tf.keras.metrics.FalseNegatives()])

#     return model

def train_model(model, df, feature, label, epochs, batch_size):
    history = model.fit(x=df[feature], y=df[label], batch_size=batch_size, epochs=epochs)

    trained_weight = model.get_weights()[0]
    trained_bias = model.get_weights()[1]

    epochs = history.epoch

    hist = pd.DataFrame(history.history)

    rmse = hist["root_mean_squared_error"]

    return trained_weight, trained_bias, epochs, rmse

print("Defined build_model and train_model")

def plot_the_model(trained_weight, trained_bias, feature, label):
    plt.xlabel("feature")
    plt.ylabel("label")

    random_examples = training_df.sample(n=200)
    plt.scatter(random_examples[feature], random_examples[label])

    x0 = 0
    y0 = trained_bias
    x1 = 10000
    y1 = trained_bias + (trained_weight * x1)
    plt.plot([x0, x1], [y0, y1], c='r')

    plt.show()

def plot_the_loss_curve(epochs, rmse):
    plt.figure()
    plt.xlabel("Epoch")
    plt.ylabel("Root Mean Squared Error")

    plt.plot(epochs, rmse, label="Loss")
    plt.legend()
    plt.ylim([rmse.min()*0.97, rmse.max()])
    plt.show()

print("Defined the plot_the_model and plot_the_loss_curve functions.")

def predict_house_values(n, feature, label):
  """Predict house values based on a feature."""

  batch = training_df[feature][10000:10000 + n]
  predicted_values = my_model.predict_on_batch(x=batch)

  print("feature   label          predicted")
  print("  value   value          value")
  print("          in thousand$   in thousand$")
  print("--------------------------------------")
  for i in range(n):
    print ("%5.0f %6.0f %15.0f" % (training_df[feature][10000 + i], training_df[label][10000 + i], predicted_values[i][0] ))

training_df = pd.read_csv(r"C:\Users\Tom\Projects\Data Stash\california_housing_train.csv")

training_df["median_house_value"] /= 1000.0
training_df["rooms_per_person"] = training_df["total_rooms"] / training_df["population"]

learning_rate=0.001
epochs=30
my_batch_size=60

my_feature = "median_income"
my_label = "median_house_value"

my_model = None

my_model = build_model(learning_rate)
trained_weight, trained_bias, epochs, rmse = train_model(my_model, training_df, my_feature, my_label, epochs, my_batch_size)

# my_model.summary()
# predict_house_values(10, my_feature, my_label)
# corr_matrix = training_df.corr()
# print(corr_matrix)
# plot_the_model(trained_weight, trained_bias, my_feature, my_label)
# plot_the_loss_curve(epochs, rmse)

# imageFile = r"C:\Users\Tom\Projects\Data Stash\train-images.idx3-ubyte"
# trainingFile = r"C:\Users\Tom\Projects\Data Stash\train-labels.idx1-ubyte"
# trainingImage = idx2numpy.convert_from_file(imageFile)
# trainingLabel = idx2numpy.convert_from_file(trainingFile)

# check = random.randrange(0,60000)

# image = trainingImage[check]
# label = trainingLabel[check]
# label = label.astype(np.uint8)

# inputs = []
# for rows_of_pixels in image:
# 	for pixel in rows_of_pixels:
# 		inputs.append(pixel)

# new = [inputs, label]


from typing import List
class Solution:
    def change(self, amount: int, coins: List[int]) -> int:
        dp = [0]*(amount+1)
        dp [0] = 1
        for i in coins:
            for j in range(1,amount+1):
                if j-i>=0:
                    dp[j] +=dp[j-i]
        return dp[-1]
A = Solution()
a = 5
b = [1,2,5]
print(A.change(a,b))


from django.shortcuts import render, redirect, get_object_or_404
from django.contrib.auth.decorators import login_required
from ..models import Crop, Transaction
from django.http import HttpResponse, JsonResponse
from django.db.models import Sum
import pandas as pd
import numpy as np

class RecommendedSystem:
    def __init__(self):
        self.crops = []
        self.kgs_sold = []
        self.kgs_received = []
        self.selling_price_per_kg = []
        self.production_cost_per_kg = []

    def dataframe(self):
        data = {
            "Crops": self.crops,
            "KG received": self.kgs_received,
            "KG Sold": self.kgs_sold,
            "Production Price": self.production_cost_per_kg,
            "Selling Price": self.selling_price_per_kg,
        }

        self.df = pd.DataFrame(data)
        return self.df

    def computations(self):
        self.df["Total_Produced_Price"] = (
            self.df["KG Sold"] * self.df["Production Price"]
        )
        self.df["Total_Selling_Price"] = self.df["KG Sold"] * self.df["Selling Price"]

        self.df["Unsold KG"] = self.df["KG received"] - self.df["KG Sold"]
        self.df["Profit made"] = (
            self.df["Total_Selling_Price"] - self.df["Total_Produced_Price"]
        )

    def recommended_crop(self):
        trial = self.df
        row_index = trial.loc[
            trial["Unsold KG"] == (trial.groupby("Crops")["Unsold KG"].min()).min()
        ]["Profit made"].idxmax()

        return self.crops[row_index]

@login_required(login_url="signin")
def analytics(request):
    if request.method == "GET":
        return render(request, "analytics.html")

    elif request.method == "POST":
        end = request.POST["end"]
        start = request.POST["start"]
        analysis = RecommendedSystem()

        for crop in Crop.objects.all():
            analysis.crops.append(crop.name)
            analysis.selling_price_per_kg.append(crop.selling_price)
            analysis.production_cost_per_kg.append(crop.production_cost)

            analysis.kgs_sold.append(
                Transaction.objects.filter(
                    transaction_type="sold", crop_id=crop.id
                ).aggregate(Sum("quantity"))["quantity__sum"]
            )
            analysis.kgs_received.append(
                Transaction.objects.filter(
                    transaction_type="received", crop_id=crop.id
                ).aggregate(Sum("quantity"))["quantity__sum"]
            )

        dataframe = analysis.dataframe()
        analysis.computations()
        recommended_crop = analysis.recommended_crop()

        return render(
            request,
            "analytics.html",
            {
                "dataframe": dataframe,
                "recommended_crop": recommended_crop,
            },
        )


from django.conf.urls import include, url
from django.contrib import admin
from django.contrib.auth import views as auth_views

from mastersheet.views import *

urlpatterns = [
    url(r'^report_table_cm_activity_count/$', report_table_cm_activity_count, name="report_table_cm_activity_count"),
    url(r'^report_table_cm/$', report_table_cm, name="report_table_cm"),
	url(r'^report_table/$', give_report_table_numbers, name="give_report_table_numbers"),
    url(r'^report_table_accounts/$', give_report_table_numbers_accounts, name="give_report_table_numbers_accounts"),
	url(r'^details/$', give_details, name="give_details"),
	url(r'^report/$', render_report, name="render_report"),
	url(r'^show/report/$', create_report, name="create_report"),
    url(r'^delete_selected/$', delete_selected, name="delete_selected"),
    url(r'^files/$', file_ops, name="file_ops"),
    url(r'^columns/$', define_columns, name="define_columns"),
    url(r'^show/mastersheet/$', renderMastersheet, name="renderMastersheet"),
    url(r'^list/show/$', masterSheet, name="masterSheet"),
    url(r'^sync/slum/$', sync_kobo_data, name='sync_kobo_data'),
    url(r'^account/excel/$', accounts_excel_generation, name='accounts_excel_generation'),
    url(r'^show/summery/$', renderSummery, name="renderSummery"),
    url(r'^show/showSummery/$', ProcessShortView, name='ProcessShortView'),
    url(r'^gisDataDownload/$', gisDataDownload, name='gisDataDownload')
]


from django.contrib import messages
from django.contrib.auth.decorators import login_required, permission_required
from django.shortcuts import render, get_object_or_404, redirect

from utils.forms import DeleteConfirmForm
from .forms import CoffeeForm
from .models import Coffee

# Create your views here.
@login_required
def index(request):
    coffees = Coffee.objects.all()
    return render(request, 'coffees/index.html', {'coffees': coffees})

@login_required
def show(request, pk):
    coffee = get_object_or_404(Coffee, pk=pk)
    return render(request, 'coffees/show.html', {'coffee': coffee})

@permission_required('coffees.add_coffee', raise_exception=True)
def add(request):
    form = CoffeeForm(request.POST or None)
    if form.is_valid():
        form.save()
        messages.success(request, '新增成功')
        return redirect('coffees:index')
    return render(request, 'coffees/add.html', {'form': form})

@permission_required('coffees.change_coffee', raise_exception=True)
def edit(request, pk):
    coffee = get_object_or_404(Coffee, pk=pk)
    form = CoffeeForm(request.POST or None, instance=coffee)
    if form.is_valid():
        form.save()
        messages.success(request, '更新成功')
        return redirect('coffees:index')
    return render(request, 'coffees/edit.html', {'form': form})

@permission_required('coffees.delete_coffee', raise_exception=True)
def delete(request, pk):
    coffee = get_object_or_404(Coffee, pk=pk)
    form = DeleteConfirmForm(request.POST or None)
    if form.is_valid() and form.cleaned_data['check']:
        coffee.delete()
        messages.success(request, '刪除成功')
        return redirect('coffees:index')
    return render(request, 'coffees/delete.html', {'form': form})


'''
Um ano é bissexto se:

O ano for divisivel por 4, mas não divisível por 100, exceto se ele for também
divisivel por 400.

Exemplo anos bissexto: 1600, 1732, 1944, 2008

Exemplo anos não bissexto: 1742, 1889, 1951, 2011
'''

import pytest

# Função para calcular ano bissexto
def eh_bissexto(ano):
    resto4 = ano % 4
    resto100 = ano % 100
    resto400 = ano % 400

    if ((not resto4) and (resto100)) or (not resto400):
        return True
    return False

### Não deve ser bissexto
@pytest.mark.parametrize('ano', [500, 600, 1500, 1742, 1889, 1951, 2011])  # Setup
def test_nao_deve_ser_bissexto(ano):
    resp = eh_bissexto(ano)  # Execução do teste

    assert resp is False  # Verificação do resultado do teste

### Deve ser bissexto
@pytest.mark.parametrize('ano', [1600, 1732, 1944, 2008])  # Setup
def test_deve_ser_bissexto(ano):
    resp = eh_bissexto(ano)  # Execução do teste

    assert resp is True  # Verificação do resultado do teste


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import copy
import time
import threading
from pysc2.agents import base_agent
from pysc2.env import sc2_env
from pysc2.lib import actions, features, units
from absl import app
import random

#import scripted_actr_agent as scripted

import logging
import sys

from absl import flags

from pysc2.agents import base_agent
from pysc2.lib import actions

#game setup
FLAGS = flags.FLAGS #for passing arguments
steps = 400 # = 4 Episodes
step_mul = 1 #no frame skipping = realtime

import numpy
import ccm
log = ccm.log()

from pysc2.agents import base_agent
from pysc2.lib import actions
from pysc2.lib import features

from pysc2.env import run_loop
from pysc2.env import sc2_env

_PLAYER_SELF = features.PlayerRelative.SELF
_PLAYER_NEUTRAL = features.PlayerRelative.NEUTRAL  # beacon/minerals
_PLAYER_ENEMY = features.PlayerRelative.ENEMY

FUNCTIONS = actions.FUNCTIONS

from ccm import model
from ccm.lib.actr import *

############### Globals ######################

def _xy_locs(mask):
  """Mask should be a set of bools from comparison with a feature layer."""
  y, x = mask.nonzero()
  return list(zip(x, y))

#********************* ACT-R_CMU Stuff **********************
class GameEnvModel(ccm.Model):
    next_action=ccm.Model(isa='action',press='no_op')
    screen=ccm.Model(isa='screen',event='beacon')

class MotorModule(ccm.Model):
    beacons = 0
    next_action = ''

    def do_action(self,action):
        print(f"do the action: {action}")
        self.parent.parent.next_action = action
        self.parent.parent.screen.event='none'
        time.sleep(0.1)

class ActrAgent(ACTR):
    focus = Buffer()
    motor = MotorModule()
    visual = Buffer()
    vision_module=SOSVision(visual,delay=0)

    retrieve = Buffer()
    memory = Memory(retrieve)

    def init(self):
        print('actr init')
        self.focus.set('find_beacon')

    #def noOp(screen="event:none"):
    def seeBeacon(screen='event:?beacon'):
        print('found the beacon!')
        motor.do_action(beacon)
        self.focus.clear()

    def moveToBeacon(screen='event:?event'):
        motor.do_action(event)
        #print(f'production:{moving}')
        self.focus.clear()

#--------------------------- StarCraft Stuff ---------------------
class BeaconAgent(base_agent.BaseAgent):
  """An agent specifically for solving the MoveToBeacon map."""
  #actr_env =
  def __init__(self):
      super(BeaconAgent, self).__init__()

      self.status = 'none'
      self.target = numpy.zeros(2)
  #def nextAction(actions.FUNCTIONS func):

  def step(self, obs):

    super(BeaconAgent, self).step(obs)

    #self.response = [_NO_OP, []] #default response is to do nothing

    if FUNCTIONS.Move_screen.id in obs.observation.available_actions:

      player_relative = obs.observation.feature_screen.player_relative

      beacon = _xy_locs(player_relative == _PLAYER_NEUTRAL)

      if not beacon:
        actrGameEnv.screen.event = 'beacon'
        return FUNCTIONS.no_op()

      beacon_center = numpy.mean(beacon, axis=0).round()

      if numpy.array_equal(beacon_center,self.target):
          return FUNCTIONS.no_op()
      else:
        self.target = beacon_center
        actrGameEnv.screen.event = 'beacon'
        self.status = 'moving'
        return FUNCTIONS.Move_screen("now", beacon_center)

    else:
      actrGameEnv.screen.event = 'moving'
      return FUNCTIONS.select_army("select")

def sc2_thread(*args):

    FLAGS(args)

    sc2env = sc2_env.SC2Env(
        map_name="MoveToBeacon",
        # players=[sc2_env.Agent(sc2_env.Race.terran),
        #         sc2_env.Bot(sc2_env.Race.random,
        #                     sc2_env.Difficulty.very_easy)],
        players=[sc2_env.Agent(sc2_env.Race.terran)],

        agent_interface_format=features.AgentInterfaceFormat(
            feature_dimensions=features.Dimensions(screen=84, minimap=64),
            use_feature_units=True),
        step_mul=1,
        game_steps_per_episode=0,
        #screen_size_px=(128, 128),
        visualize=True)
        #visualize=False)

    sc2agent.setup(sc2env.observation_spec(), sc2env.action_spec())

    timesteps = sc2env.reset()

    sc2agent.reset()

    #    actrAgent.run()

    while True:
        step_actions = [sc2agent.step(timesteps[0])]
        if timesteps[0].last():
            break
        timesteps = sc2env.step(step_actions)

def actr_thread2(*args): #not currently used

    FLAGS(args)

    env = MatchEnvironment()
    env.model = actr_model
    env.model.goal.set('attend')

    ccm.display(env)

    #actr_agent = ActrAgent()
    #actrEnv = StarCraftEnvironment()
    #actrEnv.agent = actr_agent

    #ccm.log_everything(env)

    env.run()

def actr_thread(*args): #not currently used

    FLAGS(args)

    env = StarCraftEnvironment()
    env.model = actr_agent
    env.model.focus.set('no_op')

    ccm.log(env)

    env.run()

#common_model = SimpleModel()
actrGameEnv = GameEnvModel()

if __name__ == "__main__":
    #app.run(main)

    sc2agent = BeaconAgent()

    actrGameEnv.agent = ActrAgent()

    #actr_agent.focus.set('no_op')
    #sc2agent.actr_setup(deepcopy(actr_agent))

    ccm.log_everything(actrGameEnv)
    #ccm.log(actrGameEnv)

    game_thread = threading.Thread(target=sc2_thread, args=sys.argv)
    game_thread.start()

    #model_thread = Process(target=actr_thread, args=sys.argv)
    model_thread = threading.Thread(target=actrGameEnv.run(), args=sys.argv)
    model_thread.start()

    #time.sleep(25)

    # game_thread = Process(target=sc2_thread, args=sys.argv)


from flask import Blueprint, Response, request, jsonify
import json

import logging
import requests
import re
import threading

from file_worker import FileWorker

logger = logging.getLogger(__name__)

def construct_blueprint(app_config):
    file_info_index_blueprint = Blueprint('file_info_index', __name__)
    app_config = app_config

    """
    Index file info documents for the files in a dataset
    Returns
    -------
    json
        A json containing details of the dataset indexing operation
    """
    @file_info_index_blueprint.route('/datasets/<dataset_id>/reindex', methods=['PUT'])
    def create_update_dataset_file_infos(dataset_id):
        try:
            fworker = FileWorker(app_config=app_config, request_headers=request.headers)

            # Specify argument to confirm dataset_id is for a Dataset entity.
            theDataset = fworker.get_entity(entity_id=dataset_id, entity_type_check=['DATASET','PUBLICATION'])

            if not fworker.verify_user_in_write_group(aDataset=theDataset) and not fworker.verify_user_is_data_admin():
                raise Exception(f"Permission denied for modifying ES index entries for '{theDataset['uuid']}'.")

            dataset_uuid = theDataset['uuid']

            # Confirm the Elastic Search index exists? Or leave that to search-api?
            # https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-exists.html
            # Need to add "HEAD id" to search-api, if not calling ES directly.

            asynchronous = request.args.get('async')

            index_response_dict = {}

            if asynchronous:
                index_response_dict[dataset_uuid] = fworker.index_dataset(aDataset=theDataset)

                # Consolidate the successful responses together into a subdictionary in the JSON response.
                dataset_files_info_response_dict = {'dataset_uuid': dataset_uuid,
                                                    'file_count': len(index_response_dict[dataset_uuid]),
                                                    'index_responses': {}}
                for file_uuid in index_response_dict[dataset_uuid].keys():
                    if not file_uuid in dataset_files_info_response_dict['index_responses']:
                        dataset_files_info_response_dict['index_responses'][file_uuid] = []
                    for es_index_name in index_response_dict[dataset_uuid][file_uuid].keys():
                        dataset_files_info_response_dict['index_responses'][file_uuid].append(index_response_dict[dataset_uuid][file_uuid][es_index_name])

                return Response(json.dumps(dataset_files_info_response_dict)), 200
            else:
                try:
                    threading.Thread(target=fworker.index_dataset, args=[theDataset]).start()
                    # discard the index_response_dict, as it will be logged at the INFO level by the FileWorker

                    logger.info(f"Started to clear documents for uuid '{dataset_uuid}' from the Elasticsearch index.")
                except Exception as e:
                    logger.exception(e)
                    raise Exception(e)

                return f"Request to load file info documents for uuid '{dataset_uuid}' into the Elasticsearch index accepted", 202

        except requests.exceptions.HTTPError as he:
            eMsg = he.response.text if he.response.text is not None else 'Undescribed HTTPError'
            eCode = he.response.status_code if he.response.status_code is not None else 500
            logger.error(str(eCode) + ':' + eMsg, exc_info=True)
            return Response(eMsg, eCode)
        except Exception as e:
            logger.error(str(e), exc_info=True)
            return Response('Unexpected error: ' + str(e), 500)
        return Response(json.dumps(dataset_files_info_response_dict))

    """
    Index all Datasets.
    Returns
    -------
    json
        A json containing details of the dataset indexing operation
    """
    @file_info_index_blueprint.route('/datasets/reindex-all', methods=['PUT'])
    def reindex_dataset_files_docs():
        fworker = FileWorker(app_config=app_config, request_headers=request.headers)

        # Verify the user is a Data Admin, who can index all datasets
        if not fworker.verify_user_is_data_admin():
            raise Exception("Permission denied for requested operation.")

        asynchronous = request.args.get('async')

        if asynchronous:
            return fworker.index_all_datasets()
        else:
            try:
                threading.Thread(target=fworker.index_all_datasets, args=[]).start()
                # discard the index_response_dict, as it will be logged at the INFO level by the FileWorker

                logger.info("Started to index file info documents for all Datasets to the public and private Elasticsearch indices.")
            except Exception as e:
                logger.exception(e)
                raise Exception(e)

        return "Request to index file info documents for all Datasets into the public and private Elasticsearch indices accepted", 202

    @file_info_index_blueprint.route('/datasets/refresh-indices', methods=['PUT'])
    def refresh_indices():
        fworker = FileWorker(app_config=app_config, request_headers=request.headers)

        # Verify the user is a Data Admin, who can index all datasets
        if not fworker.verify_user_is_data_admin():
            raise Exception("Permission denied for requested operation.")

        asynchronous = request.args.get('async')

        if asynchronous:
            return fworker.refresh_indices()
        else:
            try:
                threading.Thread(target=fworker.refresh_indices, args=[]).start()
                # discard the index_response_dict, as it will be logged at the INFO level by the FileWorker
                logger.info(f"Started to refresh the public and private Elasticsearch indices with"
                            f" file info documents for added, changed, or deleted Datasets.")
            except requests.HTTPError as he:
                # OpenSearch errors come back in the JSON, so return them that way.
                return jsonify(he.response.json()), he.response.status_code
            except Exception as e:
                logger.exception(e)
                raise Exception(e)

        return  (   f"Request to refresh the public and private OpenSearch indices with file info documents for"
                    f" added, changed, or deleted Datasets accepted"), 202

    # end construct_blueprint()
    return file_info_index_blueprint