import os
import os.path as osp
import random
import re
import pickle
import argparse
import numpy as np
from random import randint
from collections import defaultdict

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
from torch.nn.utils.rnn import pad_sequence

from gensim.models import KeyedVectors

class Vocab:
    def build_vocabs(self, sentence_list):
        self.stoi_dict = defaultdict(lambda: 1)
        self.stoi_dict["<PAD>"] = 0
        self.stoi_dict["<UNK>"] = 1
        _index = 2
        for sentence in sentence_list:
            tokens_list = sentence
            for word in tokens_list:
                if word in self.stoi_dict:
                    pass
                else:
                    self.stoi_dict[word] = _index
                    _index += 1
        self.itos_dict = {v: k for k, v in self.stoi_dict.items()}

    def stoi(self, token_list):
        return [self.stoi_dict[word] for word in token_list]

    def itos(self, indices):
        return " ".join(
            [
                self.itos_dict[index]
                for index in indices
                if self.itos_dict[index] != "<PAD>"
            ]
        )

class CNNDataset:
    def __init__(self, path, w2v_path):
        data = self.load_data(path)
        zipped_data = list(zip(*data))

        self.text = zipped_data[0]
        self.text = [self.clean_str(sen) for sen in self.text]
        self.text = [[word for word in self.tokenizer(sen)] for sen in self.text]
        self.label = zipped_data[1]

        self.vocab = Vocab()
        self.vocab.build_vocabs(self.text)
        self.pad_index = self.vocab.stoi_dict["<PAD>"]
        self.w2v = self.load_word2vec(w2v_path)
        self.embedding_dim = self.w2v.vector_size
        self.pretrained_embedding = self.get_pretrained_embeddings()

    def __len__(self):
        return len(self.label)

    def __getitem__(self, idx):
        sample_label = self.label[idx]
        sample_text = self.text[idx]
        sample_text = self.vocab.stoi(sample_text)
        return torch.Tensor(sample_text).long(), sample_label

    def load_data(self, path):
        with open(
            f"{path}/CNN_sentence/rt-polarity.pos", "r", encoding="ISO-8859-1"
        ) as f:
            pos = f.readlines()
        with open(
            f"{path}/CNN_sentence/rt-polarity.neg", "r", encoding="ISO-8859-1"
        ) as f:
            neg = f.readlines()
        pos = [(p, 1) for p in pos]
        neg = [(n, 0) for n in neg]
        return pos + neg

    def tokenizer(self, sentence):
        return sentence.split()

    def load_word2vec(self, w2v_path):
        return KeyedVectors.load_word2vec_format(w2v_path, binary=True)

    def get_pretrained_embeddings(self):
        pretrained_embedding = []
        for word in self.vocab.stoi_dict:
            if word in self.w2v:
                pretrained_embedding.append(self.w2v[word])
            else:
                pretrained_embedding.append(
                    np.random.uniform(-0.25, 0.25, self.embedding_dim)
                )
        return torch.from_numpy(np.array(pretrained_embedding))

    def pad_collate(self, batch):
        (xx, yy) = zip(*batch)
        xx_pad = pad_sequence(xx, batch_first=True, padding_value=0)
        return xx_pad, yy

    def clean_str(self, string, TREC=False):
        """
        Tokenization/string cleaning for all datasets except for SST.
        Every dataset is lower cased except for TREC
        """
        string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
        string = re.sub(r"\'s", " 's", string)
        string = re.sub(r"\'ve", " 've", string)
        string = re.sub(r"n\'t", " n't", string)
        string = re.sub(r"\'re", " 're", string)
        string = re.sub(r"\'d", " 'd", string)
        string = re.sub(r"\'ll", " 'll", string)
        string = re.sub(r",", " , ", string)
        string = re.sub(r"!", " ! ", string)
        string = re.sub(r"\(", " \( ", string)
        string = re.sub(r"\)", " \) ", string)
        string = re.sub(r"\?", " \? ", string)
        string = re.sub(r"\s{2,}", " ", string)
        return string.strip() if TREC else string.strip().lower()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Dataset Builder")
    parser.add_argument("-b", "--batch_size", type=int, default=10)
    parser.add_argument("-p", "--path", type=str, default="/home/long8v")
    parser.add_argument(
        "-w",
        "--w2v_path",
        type=str,
        default="/home/long8v/Downloads/GoogleNews-vectors-negative300.bin.gz",
    )
    args = parser.parse_args()

    dataset = CNNDataset(args.path, args.w2v_path)
    data_loader = DataLoader(
        dataset=dataset, batch_size=args.batch_size, collate_fn=dataset.pad_collate
    )

    for i in data_loader:
        print(i)
        break


"""Wren - An interface for examining data and its relationships in a grid."""

from io import BytesIO
import logging
import os
import sys

from PyQt5.QtGui import QImage, QPixmap

# -- Globals ------------------------------------------------------------------

log = logging.getLogger('wren')
log.setLevel(logging.DEBUG)
if not log.handlers:
    log.addHandler(logging.StreamHandler())
    log.info(__doc__)

SELECTION_COLOR = {
    1: 'red',  # Cursor
    2: 'blue',  # Selection 1
    3: 'green'  # Selection 2
}

# GRID_WIDTH and GRID_HEIGHT set to None will be calculated.
GRID_WIDTH = None#4#12  # Number of clips
GRID_HEIGHT = None# 4#7  # Number of clips
CLIP_WIDTH = 192  # Pixels
CLIP_HEIGHT = 150  # Pixels
GRID_BACKGROUND = '#999'
GRID_BACKGROUND_HOMEROW = '#777'
CLIP_BACKGROUND = '#ff0'
CLIP_BACKGROUND_HOMEROW = '#cc0'

# -- Miscellaneous ------------------------------------------------------------

def render_equation_to_pixmap(text):
    import matplotlib.pyplot as plt
    plt.text(0.2, 0.6, r"$%s$" % text, fontsize=100)

    # hide axes
    fig = plt.gca()
    fig.axes.get_xaxis().set_visible(False)
    fig.axes.get_yaxis().set_visible(False)

    buffer = BytesIO()
    plt.savefig(buffer, bbox_inches='tight', frameon=False, transparent=True)
    plt.close()
    pixmap = QPixmap()
    pixmap.loadFromData(buffer.getvalue())
    buffer.close()
    return pixmap

def render_equation_to_label(label, text):
    pixmap = render_equation_to_pixmap(text)
    # Not full size so cursor can still appear
    pixmap = pixmap.scaled(CLIP_WIDTH * 0.85, CLIP_HEIGHT * 0.85)
    label.setPixmap(pixmap)
    label.adjustSize()

def render_equation_to_image(text):
    return QImage(render_equation_to_pixmap(text))

def spiral_coords(start_x, start_y):
    """A spiraling iteration of coordinates"""
    yield(start_x, start_y)

    level = 1
    x = start_x - level
    y = start_y - level
    area = 'top'

    while True:
        yield(x, y)
        if area == 'top':
            if x == start_x + level:
                y += 1
                area = 'right'
            else:
                x += 1
        elif area == 'right':
            if y == start_y + level:
                x -= 1
                area = 'bottom'
            else:
                y += 1
        elif area == 'bottom':
            if x == start_x - level:
                y -= 1
                area = 'left'
            else:
                x -= 1
        elif area == 'left':
            if y == (start_y - level) + 1:
                level += 1
                x = start_x - level
                y = start_y - level
                area = 'top'
            else:
                y -= 1

class StyleSheet:
    def __init__(self, selector):
        self.selector = selector
        self.sheet = {}

    def render(self):
        result = ["{0}{{".format(self.selector)]
        for element, value in self.sheet.items():
            result.append('{0}: {1};'.format(element, value))
        result.append('}')
        return ''.join(result)

    def set(self, element, value):
        self.sheet[element] = value

    def remove(self, element):
        del self.sheet[element]

class IDMap:
    """Map of datum key to its Controller"""
    def __init__(self):
        # Map of key to object.
        self.id_map = {}

    def get(self, key):
        return self.id_map.get(key)

    def set(self, key, obj):
        self.id_map[key] = obj

    def _reset(self):
        """Reset map for testing purposes only"""
        self.id_map = {}

# -- Main ---------------------------------------------------------------------

if __name__ == '__main__':
    # Confirm data storage is initialized (need: user to name the session.)
    from app import get_application
    wren = get_application()  # Creates application if does not exist.
    wren.init_ui()
    sys.exit(wren.run())


#!/usr/bin/env python3.9

import re
import json
import os
import sys
from typing import Any
from io import IOBase
from scapy.all import PcapReader, IP
from datetime import datetime
from datetime import timedelta
import argparse

#=============================
#   Some General Definitions
#=============================
PROGRAM_PATH = os.path.dirname(os.path.abspath(__file__))
PAJE_HEADER_FILE = f'{PROGRAM_PATH}/header.trace'
CPU_MEM_TIME_MASK = "%a %Y-%m-%d %H:%M:%S.%f"
CPU_MEM_START = " ********* Start Monitoring **********\n"
CPU_MEM_END = " ********* Stopping Monitoring **********\n"
NETWORK_START = " ******** IP traffic monitor started ********\n"
NETWORK_END = " ******** IP traffic monitor stopped ********\n"
UNKNOWN_HOST = "Unknown"

EXPERIMENT_ID = datetime.now().strftime('%d-%m-%Y_%H:%M')

PAJE_CODES = {
  'PajeDefineContainerType': '0',
  'PajeDefineVariableType': '1',
  'PajeDefineLinkType': '4',
  'PajeCreateContainer': '6',
  'PajeDestroyContainer': '7',
  'PajeSetVariable': '8',
  'PajeStartLink': '14',
  'PajeEndLink': '15',
}

#=============================
#   Including Project Libs
#=============================
sys.path.append(f'{PROGRAM_PATH}/../libs')

import helper

#=============================
#   Local Functions
#=============================

# Inserting into "env" entries input and output data paths for each element.
def agregatePaths(envs: dict, input_folder: str, output_folder: str):
  envs['output_root_trace'] = f'{output_folder}/traces/{EXPERIMENT_ID}/root.trace'
  envs['unknown_trace'] = f'{output_folder}/traces/{EXPERIMENT_ID}/unknown_host.trace'
  for host in envs['hosts']:
    host['trace_folder'] = f'{output_folder}/traces/{EXPERIMENT_ID}/{host["hostname"]}'
    host['cpu_mem_source'] = f'{input_folder}/{host["hostname"]}/cpu_mem_output.txt'
    host['pcap_source'] = f'{input_folder}/{host["hostname"]}/network_output.pcap'
    for vm in host['virtual_machines']:
      vm['trace_path'] = f'{host["trace_folder"]}/{vm["name"]}.trace'

# Recovering environment data from json to dictionary.
def recoverEnvironments(path: str):
  if(not os.path.isfile(path)):
    sys.exit(f'File {path} does not exist.')
  with open(path, 'r') as envs_file:
    envs = json.load(envs_file)

  return envs

# Checking if folder exists.
def checkFoldersExistence(f_paths: list):
  for path in f_paths:
    if(not os.path.isdir(path)):
      sys.exit(f"Folder '{path}' does not exists.")

# Recovering vm's from 'envs' into a list.
def generateVmList(envs: dict):
  vm_list = []

  for host in envs['hosts']:
    vm_list.extend(host["virtual_machines"])

  return vm_list

# Recovering vm from 'vm list' if there is a 'key' 'value' match.
def searchVM(vm_list: list, key: str, value: Any):
  vm = []
  for vm_entry in vm_list:
    if(vm_entry[key] == value):
      vm = vm_entry
      break
  return vm

# Formatting and outputting 'PajeSetVariable' trace line.
def outputPAJEVariable(time: float, vm_name: str, var_name: str, var_value: Any, file: IOBase):
  print(PAJE_CODES['PajeSetVariable'], time, vm_name, var_name, var_value, file=file)

# Formating and outputing 'PajeStartLink' trace line.
def outputPAJEStartLink(time: float, vm_name: str, package_size: int, link_key: str, file: IOBase):
  print(PAJE_CODES["PajeStartLink"], time, "LINK", "root", vm_name, package_size, link_key, file=file)

# Formating and outputing 'PajeEndLink' trace line.
def outputPAJEEndLink(time: float, vm_name: str, package_size: int, link_key: str, file: IOBase):
  print(PAJE_CODES["PajeEndLink"], time, "LINK", "root", vm_name, package_size, link_key, file=file)

# Formating and outputing 'PajeCreateContainer' trace line.
def outputPAJECreateContainer(time: float, container_name: str, container_type: str, container_parent: str, file: IOBase):
  print(PAJE_CODES["PajeCreateContainer"], time, container_name, container_type, container_parent, container_name, file=file)

# Generating CPU, MEM and Network trace consumption from 'fcm_input' and 'fpcap_input' into 'f_output'.
# For each CPU/MEM trace timestamp interval, trace packets that has been sent by 'vm' in this interval.
def tracing(target_vm: dict, fcm_input: IOBase, fpcap_input: IOBase, vm_list: list, f_output: IOBase, unk_output: IOBase):

  # Checking 'fcm_input' file first line.
  line = fcm_input.readline()
  line_cols = line.split(';')

  if(line_cols[1] != CPU_MEM_START):
    sys.exit(f"Wrong file format:'{fcm_input.name}'.")

  # Iniitiating time variables
  start_time = datetime.strptime(line_cols[0][0:30], CPU_MEM_TIME_MASK)
  total_time = timedelta()

  # Maps link comunication "vm_origin:vm_dest" to its packet order.
  vm_link_map = {}

  # Initiating packets list 'all_pkts' (from pcap file 'fpcap_input')
  all_pkts = PcapReader(fpcap_input)
  # Recover first packet into 'next_pkt', setting as None there in no packet (except StopIteration).
  try:
    next_pkt = all_pkts.next()
  except StopIteration:
    next_pkt = None

  for line in fcm_input.readlines():
    line_cols = line.split(';')

    # Checking if it's 'fcm_input' last line.
    if(line_cols[1] == CPU_MEM_END):
      break

    curr_time = datetime.strptime(line_cols[0][0:30], CPU_MEM_TIME_MASK)
    total_time = (curr_time - start_time)
    tt_seconds = total_time.total_seconds()

    # Loop through each vm entry in "line"
    for vm_entry in line_cols[1:]:
      vm_cols = vm_entry.split()

      # If entry corresponds to the current vm, register values.
      if(vm_cols[0] == target_vm["name"]):
        # Add CPU trace for 'tt_seconds' timestamp.
        outputPAJEVariable(tt_seconds, target_vm["name"], 'CPU', vm_cols[3], f_output)

        # Gattering and writing Network traces that occurs until 'curr_time'.
        next_pkt = tracingPcap(target_vm, next_pkt, all_pkts, vm_list, vm_link_map, f_output, unk_output, start_time, curr_time)

        # Add MEM trace for 'tt_seconds' timestamp.
        outputPAJEVariable(tt_seconds, target_vm["name"], 'MEM', vm_cols[1], f_output)

# Generating NETWORK trace traffict into 'f_output' while 'next_pkt' time and consecutives packets of 'all_pkts' are under 'end_time'.
# If 'target_vm' commuticates with unknow host, output is redirected to 'unk_output'.
def tracingPcap(target_vm: list, next_pkt, all_pkts, vm_list: list, vm_link_map, f_output: IOBase, unk_output: IOBase, start_time, end_time):

  # If there are no more packtes (next_pkt == None) or 'next_pkt.time' is grater than 'end_time',
  # exits from function returning 'next_pkt' value.
  if((next_pkt == None) or (datetime.fromtimestamp(float(next_pkt.time)) > end_time)):
    return next_pkt

  # Gettering and checking packets.
  while next_pkt:

    # Check if the next_pkt has an IP protocol header.
    if(IP in next_pkt):

      # Recovering data from 'next_pkt'
      curr_time = datetime.fromtimestamp(float(next_pkt.time))
      total_time = (curr_time - start_time)
      tt_seconds = total_time.total_seconds()

      sender = next_pkt[IP].src
      receiver = next_pkt[IP].dst

      size = next_pkt.wirelen

      # If 'next_pkt' time is grater than 'end_time', exits from function returning 'next_pkt' value.
      if(curr_time > end_time):
        return next_pkt

      # Check if vm is the sender of the message.
      if(sender == target_vm["ip"]):
        vm_counterpart = searchVM(vm_list, "ip", receiver)
        # Check if the counterpar is another vm from experiment.
        if(vm_counterpart):
          link = f'{target_vm["name"]}:{vm_counterpart["name"]}'
          order = vm_link_map.setdefault(link, 0)
          outputPAJEStartLink(tt_seconds,target_vm["name"], size, f'{link}|{order}', f_output)
          vm_link_map[link] += 1
        # If the conterpart is unknown.
        else:
          link = f'{target_vm["name"]}:{UNKNOWN_HOST}'
          order = vm_link_map.setdefault(link, 0)
          outputPAJEStartLink(tt_seconds,target_vm["name"], size, f'{link}|{order}', unk_output)
          vm_link_map[link] += 1

      # Check if vm is the receiver of the message.
      if(receiver == target_vm["ip"]):
        vm_counterpart = searchVM(vm_list, "ip", sender)
        # Check if the counterpar isnt't another vm from experiment (UNKNOWN_HOST).
        if(not vm_counterpart):
          link = f'{UNKNOWN_HOST}:{target_vm["name"]}'
          order = vm_link_map.setdefault(link, 0)
          outputPAJEStartLink((tt_seconds), UNKNOWN_HOST, size, f'{link}|{order}', unk_output)
          vm_link_map[link] += 1

    try:
      # Recover next packet from 'all_pkts' into 'next_pkt', retuning None if there in no next packet (except StopIteration).
      next_pkt = all_pkts.next()
    except (StopIteration) as e:
      return None

# Calling trace generators for each vm into 'host'.
def generateTraceFiles(host: dict, vm_list: list, unk_output):

  c_m_in = open(host['cpu_mem_source'], 'r')

  # PARALELIZATION POINT.
  for vm in host['virtual_machines']:
    print(f'\tGenerating Traces for {vm["name"]}...')
    with open(vm['trace_path'], 'w') as f_output:
      c_m_in.seek(0)

      tracing(vm, c_m_in, host['pcap_source'], vm_list, f_output, unk_output)

    print('\tDone!')

  c_m_in.close()

# Agrupates the traces into 'root.trace' file (readable by ViTE).
def agrupateTraces(envs: dict):
  print(f'Agrupating traces into "root.trace" file (readable by ViTE)...')

  with open(envs['output_root_trace'], 'w') as r_trace:
    with open(PAJE_HEADER_FILE, 'r') as h_trace:
      for line in h_trace:
        r_trace.write(line)

    r_trace.write('\n# ----------------------------------------\n# -- Initializing Container & Variables --\n# ----------------------------------------\n')
    outputPAJECreateContainer(0.0, "root", 'ROOT', '0', r_trace)
    outputPAJECreateContainer(0.0, UNKNOWN_HOST, 'NODE', 'root', r_trace)
    for host in envs['hosts']:
      outputPAJECreateContainer(0.0, host['hostname'], 'NODE', 'root', r_trace)
      for vm in host['virtual_machines']:
        outputPAJECreateContainer(0.0, vm['name'], 'VM', host['hostname'], r_trace)
        outputPAJEVariable(0.0, vm["name"], 'MEM', 0, r_trace)
        outputPAJEVariable(0.0, vm["name"], 'CPU', 0, r_trace)

    r_trace.write('\n# -----------------------------------------\n# --- Aggregating Virtual Machines Data ---\n# -----------------------------------------\n')
    for host in envs['hosts']:
      for vm in host['virtual_machines']:
        r_trace.write(f'# --- {vm["name"]} ---\n')
        with open(vm['trace_path'], 'r') as vm_trace:
          for line in vm_trace:

            r_trace.write(line)

            line_cols = re.split(' |:|\|', line)
            # If 'line' represents a packet send (PajeStartLink), write the complement (PajeEndLink) in 'root.trace' for proper visualization with ViTE.
            if(line_cols[0] == PAJE_CODES['PajeStartLink']):
              outputPAJEEndLink(float(line_cols[1]), line_cols[7], int(line_cols[5]), f'{line_cols[6]}:{line_cols[7]}|{line_cols[8][:-1]}', r_trace)

        r_trace.write('\n\n')

    r_trace.write('\n# -----------------------------------------------\n# --- Aggregating Unknown Host Communications ---\n# -----------------------------------------------\n')
    with open(envs['unknown_trace'], 'r') as unknown_trace:
      for line in unknown_trace:
        r_trace.write(line)

        line_cols = re.split(' |:|\|', line)
        # If 'line' represents a packet send (PajeStartLink), write the complement (PajeEndLink) in 'root.trace' for proper visualization with ViTE.
        if(line_cols[0] == PAJE_CODES['PajeStartLink']):
          outputPAJEEndLink(float(line_cols[1]), line_cols[7], int(line_cols[5]), f'{line_cols[6]}:{line_cols[7]}|{line_cols[8][:-1]}', r_trace)

    print('Done!')

# Parsing program initialization arguments.
def parsingArguments():
    parser = argparse.ArgumentParser()
    parser.add_argument("input_folder", help="Path to source folder.")
    parser.add_argument("output_folder", help="Path to output folder.")
    parser.add_argument("-g", action='store_const', const=True, help="Indicates to agrupate the generated traces into a single trace 'root.trace' that will be stored in 'output_folder'.")
    args = parser.parse_args()

    return os.path.normpath(args.input_folder), os.path.normpath(args.output_folder), args.g

#=============================
#       Main code
#=============================

if __name__ == '__main__':

  input_folder, output_folder, root_trace = parsingArguments()
  checkFoldersExistence([input_folder, output_folder])

  # Generating output folders.
  helper.createFolder(f'{output_folder}/traces')
  helper.createFolder(f'{output_folder}/traces/{EXPERIMENT_ID}')

  # Recovering environments.
  envs = recoverEnvironments(f'{input_folder}/environments.json')

  # Registering input an ouput file paths.
  agregatePaths(envs, input_folder, output_folder)

  # Recoverting list of vms.
  vm_list = generateVmList(envs)

  # Defining file output for unknown host communications.
  with open(envs['unknown_trace'], 'w') as unknown_host_dsc:

    # PARALELIZATION POINT.
    for host in envs['hosts']:
      helper.createFolder(host['trace_folder'])
      print(f'Generating Traces for {host["hostname"]}...')
      generateTraceFiles(host, vm_list, unknown_host_dsc)
      print(f'Done!')

  # If vizualition trace was selected ('-g' argument.)
  if(root_trace):
    agrupateTraces(envs)


from builtins import range
import numpy as np
import scipy.sparse as sp
import scipy.sparse.linalg
import time
import logging

def lm_optimize(eq_func, x0, jac_func, min_chisqr = 0.1,
                stopping_criteria = 0.001, max_iteration = 50, boost = 2,
                drop = 3, lambda_initial =0.1):
    '''This is a simple implementation of Levenberg Marquardt for minimizing
    the residuals of a set of equations. scipy already has an optimizer
    called scipy.optimize.leastsq. You should generally use this scipy
    function in preference to this class, it is implemented in Fortran and
    is a more sophisticated algorithm. But unlike that code, this algorithm
    takes advantage of sparse matrixes.

    We use as a stopping criteria one of the following conditions:
    1. We exceed max_iteration. In that case, we quit and raise an exception
    2. The chisqr of the residuals is < min_chisqr,
    3. The change in chisqr from one iteration to the next is
       < stopping_criteria
   '''
    start_time = time.process_time()
    x = x0
    t1 = time.process_time()
    res = eq_func(x)
    log = logging.getLogger("geocal-python.lm_optimize")
    log.info("Done with residual.")
    log.info("  Total time: %f " % (time.process_time() - start_time))
    log.info("  Delta time: %f" % (time.process_time() - t1))
    chisq = np.inner(res, res) / (len(res) - len(x0))
    lam = lambda_initial
    for i in range(max_iteration):
        t1 = time.process_time()
        j = jac_func(x).tocsr()
        log.info("Done with jacobian.")
        log.info("  Total time: %f " % (time.process_time() - start_time))
        log.info("  Delta time: %f" % (time.process_time() - t1))
        jtj = j.transpose() * j
        chisqold = chisq
        for k in range(max_iteration):
            c = jtj + lam * sp.eye(jtj.shape[0], jtj.shape[1], format="csr")
            jtres = j.transpose() * res
            t1 = time.process_time()
            # Note permc_spec has *no* effect on umfpack library, but we put
            # this in place in case umf isn't available and we are using
            # superlu. For superlu, this permutation significantly speeds
            # this up.
            xnew = x - sp.linalg.spsolve(c, jtres, use_umfpack=True,
                                         permc_spec="MMD_ATA")
            log.info("Done with spsolve.")
            log.info("  Total time: %f " % (time.process_time() - start_time))
            log.info("  Delta time: %f" % (time.process_time() - t1))
            t1 = time.process_time()
            resnew = eq_func(xnew)
            log.info("Done with residual.")
            log.info("  Total time: %f " % (time.process_time() - start_time))
            log.info("  Delta time: %f" % (time.process_time() - t1))
            chisq = np.inner(resnew, resnew) / (len(resnew) - len(x0))
            if(chisq < chisqold):
                x = xnew
                lam /= drop
                res = resnew
                break
            else:
                lam *= boost
                log.info("Rejected Chisq: %f" % chisq)
                log.info("Redoing iteration with lambda boosted to %f" % lam)
        else:
            raise RuntimeError("Exceeded maximum number of iterators")
        if(chisq < min_chisqr or
           (chisqold - chisq) / chisq < stopping_criteria):
            break
        log.info("Done with iteration %d, chisq %f" % (i, chisq))
    else:
        raise RuntimeError("Exceeded maximum number of iterators")
    log.info("Done with optimize.")
    log.info("  Total time: %f" %(time.process_time() - start_time))
    return x

__all__ = ["lm_optimize"]


import sys
import requests
import json

# init weights
weight_title = sys.argv[2]
weight_date = sys.argv[3]

# preparing query data sys.argv[1] contains the query text
query = {
    "query": {
        "multi_match": {
            "query": sys.argv[1],
            "fields": ["title" + "^" + weight_title, "data" + "^" + weight_date]
         }
     }
}

# our index name is cran-data which we had formed on importing cran field dataset
def search(query):
    url = 'http://localhost:9200/cran-data/_search'
    httpResp = requests.get(url, data=json.dumps(query), headers={"content-type":"application/json"})
    searchHits = json.loads(httpResp.text)["hits"]
    print(sys.argv[4])

    for idx, hit in enumerate(searchHits['hits']):
        print("%s" %(hit['_source']['id']))
search(query)


import numpy
import mnist as mnist
from keras.models import Sequential
from keras.layers import Dense
from keras.utils import np_utils

# make seed for cycle results
numpy.random.seed(42)

# load data
(X_train, y_train), (X_test, y_test) = mnist.load_data()

# resize data
X_train = X_train.reshape(60000, 784)

# normalize data
X_train = X_train.astype('float32')
X_train /= 255

# transform tags to categories
y_train = np_utils.to_categorical(y_train, 10)

# 0 -> [1, 0, 0, 0, 0, 0, 0, 0, 0, 0]
# 2 -> [0, 0, 1, 0, 0, 0, 0, 0, 0, 0] # only the right element == 1

# create sequential model
model = Sequential()

# add network levels
model.add(Dense(800, input_dim=784, init="normal", activation="relu"))
model.add(Dense(10, init="normal", activation="softmax"))

# compile model
model.compile(loss="categorical_crossentropy", optimizer="SGD", metrics=["accuracy"])
print(model.summary())

#_________training_______________________
# batch_size - size of data portion
# nb_epoch - amount times of teaching (100 times)
model.fit(X_train, y_train, batch_size=200, nb_epoch=100, verbose=1)

#___________working_____________
# work on input data
predictions = model.predict(X_train)

# transform output data
# from categories to tags of classes (num from 0 to 9)
predictions = np_utils.categorical_probas_to_classes(predictions)

# now we can compare


"""Library for multiagent concerns."""
import collections
import numpy as np
import random

from pysc2.lib import actions, features, units
from utils import get_entity_obs, get_upgrade_obs, get_gameloop_obs, get_race_onehot, get_agent_statistics
from network import EntityEncoder, ScalarEncoder, SpatialEncoder, Core

_SELECT_ARMY = actions.FUNCTIONS.select_army.id
_SELECT_ALL = [0]

_MOVE_SCREEN = actions.FUNCTIONS.Move_screen.id
_NOT_QUEUED = [0]
_QUEUED = [1]
_SELECT_ALL = [2]

_SELECT_POINT = actions.FUNCTIONS.select_point.id
_ATTACK_SCREEN = actions.FUNCTIONS.Attack_screen.id
_ATTACK_MINIMAP = actions.FUNCTIONS.Attack_minimap.id
_SELECT_CONTROL_GROUP = actions.FUNCTIONS.select_control_group.id

_BUILD_SUPPLY_DEPOT = actions.FUNCTIONS.Build_SupplyDepot_screen.id
_BUILD_BARRACKS = actions.FUNCTIONS.Build_Barracks_screen.id
_BUILD_REFINERY = actions.FUNCTIONS.Build_Refinery_screen.id
_BUILD_TECHLAB = actions.FUNCTIONS.Build_TechLab_screen.id

_TRAIN_MARINE = actions.FUNCTIONS.Train_Marine_quick.id
_TRAIN_MARAUDER = actions.FUNCTIONS.Train_Marauder_quick.id
_TRAIN_SCV = actions.FUNCTIONS.Train_SCV_quick.id
_SELECT_ARMY = actions.FUNCTIONS.select_army.id
_SELECT_IDLE_WORKER = actions.FUNCTIONS.select_idle_worker.id

_RETURN_SCV = actions.FUNCTIONS.Harvest_Return_SCV_quick.id
_HARVEST_GATHER = actions.FUNCTIONS.Harvest_Gather_screen.id
_HARVEST_GATHER_SCV = actions.FUNCTIONS.Harvest_Gather_SCV_screen.id

class Agent(object):
  """Demonstrates agent interface.

  In practice, this needs to be instantiated with the right neural network
  architecture.
  """
  def __init__(self, race, initial_weights):
    self.race = race
    self.steps = 0
    self.weights = initial_weights
    self.weights = None

    self.spatial_encoder = SpatialEncoder(img_height=128, img_width=128, channel=27)
    self.scalar_encoder = ScalarEncoder(128)
    self.entity_encoder = EntityEncoder(464, 8)
    self.core = Core(12)

    self.home_upgrade_array = np.zeros(89)
    self.away_upgrade_array = np.zeros(89)

  def initial_state(self):
    """Returns the hidden state of the agent for the start of an episode."""
    # Network details elided.
    initial_state = None

    return initial_state

  def set_weights(self, weights):
    self.weights = weights

  def get_weights(self):
    return self.weights

  def get_steps(self):
    """How many agent steps the agent has been trained for."""
    return self.steps

  def step(self, observation, last_state):
    """Performs inference on the observation, given hidden state last_state."""
    # We are omitting the details of network inference here.
    # ...
    feature_screen = observation[3]['feature_screen']
    feature_minimap = observation[3]['feature_minimap']
    feature_units = observation[3]['feature_units']
    feature_player = observation[3]['player']
    available_actions = observation[3]['available_actions']
    score_by_category = observation[3]['score_by_category']
    game_loop = observation[3]['game_loop']

    unit_type = feature_screen.unit_type
    empty_space = np.where(unit_type == 0)
    empty_space = np.vstack((empty_space[0], empty_space[1])).T
    random_point = random.choice(empty_space)
    #target = [random_point[0], random_point[1]]
    #action = [actions.FunctionCall(_BUILD_SUPPLY_DEPOT, [_NOT_QUEUED, target])]
    policy_logits = None
    new_state = None

    spatial_encoder_output = self.spatial_encoder(np.reshape(feature_screen, [1,128,128,27]))

    agent_statistics = get_agent_statistics(score_by_category)

    home_race = 'Terran'
    away_race = 'Terran'
    race = get_race_onehot(home_race, away_race)

    time = get_gameloop_obs(game_loop)

    upgrade_value = get_upgrade_obs(feature_units)
    if upgrade_value != -1:
      self.home_upgrade_array[np.where(upgrade_value[0] == 1)] = 1
      self.away_upgrade_array[np.where(upgrade_value[1] == 1)] = 1

    embedded_scalar = np.concatenate((agent_statistics, race, time, self.home_upgrade_array, self.away_upgrade_array), axis=0)
    scalar_encoder_output = self.scalar_encoder(np.reshape(embedded_scalar, [1,307]))
    embedded_feature_units = get_entity_obs(feature_units)
    entity_encoder_output = self.entity_encoder(np.reshape(embedded_feature_units, [1,512,464]))
    encoder_input = np.concatenate((spatial_encoder_output, scalar_encoder_output, entity_encoder_output), axis=1)

    core_input = np.reshape(encoder_input, [16, 8, 131])
    whole_seq_output, final_memory_state, final_carry_state = self.core(core_input)
    print(whole_seq_output.shape)
    print(final_memory_state.shape)
    print(final_carry_state.shape)

    action = [actions.FUNCTIONS.no_op()]

    return action, policy_logits, new_state

  def unroll(self, trajectory):
    """Unrolls the network over the trajectory.

    The actions taken by the agent and the initial state of the unroll are
    dictated by trajectory.
    """
    # We omit the details of network inference here.
    return policy_logits, baselines

def remove_monotonic_suffix(win_rates, players):
  if not win_rates:
    return win_rates, players

  for i in range(len(win_rates) - 1, 0, -1):
    if win_rates[i - 1] < win_rates[i]:
      return win_rates[:i + 1], players[:i + 1]

  return np.array([]), []

def pfsp(win_rates, weighting="linear"):
  weightings = {
      "variance": lambda x: x * (1 - x),
      "linear": lambda x: 1 - x,
      "linear_capped": lambda x: np.minimum(0.5, 1 - x),
      "squared": lambda x: (1 - x)**2,
  }
  fn = weightings[weighting]
  probs = fn(np.asarray(win_rates))
  norm = probs.sum()
  if norm < 1e-10:
    return np.ones_like(win_rates) / len(win_rates)

  return probs / norm

class Payoff:
  def __init__(self):
    self._players = []
    self._wins = collections.defaultdict(lambda: 0)
    self._draws = collections.defaultdict(lambda: 0)
    self._losses = collections.defaultdict(lambda: 0)
    self._games = collections.defaultdict(lambda: 0)
    self._decay = 0.99

  def _win_rate(self, _home, _away):
    if self._games[_home, _away] == 0:
      return 0.5

    return (self._wins[_home, _away] +
            0.5 * self._draws[_home, _away]) / self._games[_home, _away]

  def __getitem__(self, match):
    home, away = match

    if isinstance(home, Player):
      home = [home]
    if isinstance(away, Player):
      away = [away]

    win_rates = np.array([[self._win_rate(h, a) for a in away] for h in home])
    if win_rates.shape[0] == 1 or win_rates.shape[1] == 1:
      win_rates = win_rates.reshape(-1)

    return win_rates

  def update(self, home, away, result):
    for stats in (self._games, self._wins, self._draws, self._losses):
      stats[home, away] *= self._decay
      stats[away, home] *= self._decay

    self._games[home, away] += 1
    self._games[away, home] += 1
    if result == "win":
      self._wins[home, away] += 1
      self._losses[away, home] += 1
    elif result == "draw":
      self._draws[home, away] += 1
      self._draws[away, home] += 1
    else:
      self._wins[away, home] += 1
      self._losses[home, away] += 1

  def add_player(self, player):
    self._players.append(player)

  @property
  def players(self):
    return self._players

class Player(object):
  def get_match(self):
    pass

  def ready_to_checkpoint(self):
    return False

  def _create_checkpoint(self):
    return Historical(self, self.payoff)

  @property
  def payoff(self):
    return self._payoff

  @property
  def race(self):
    return self._race

  def checkpoint(self):
    raise NotImplementedError

class MainPlayer(Player):
  def __init__(self, race, agent, payoff):
    self.agent = Agent(race, agent.get_weights())
    self._payoff = payoff
    self._race = agent.race
    self._checkpoint_step = 0

  def _pfsp_branch(self):
    historical = [
        player for player in self._payoff.players
        if isinstance(player, Historical)
    ]
    win_rates = self._payoff[self, historical]
    return np.random.choice(
        historical, p=pfsp(win_rates, weighting="squared")), True

  def _selfplay_branch(self, opponent):
    # Play self-play match
    if self._payoff[self, opponent] > 0.3:
      return opponent, False

    # If opponent is too strong, look for a checkpoint
    # as curriculum
    historical = [
        player for player in self._payoff.players
        if isinstance(player, Historical) and player.parent == opponent
    ]
    win_rates = self._payoff[self, historical]
    return np.random.choice(
        historical, p=pfsp(win_rates, weighting="variance")), True

  def _verification_branch(self, opponent):
    # Check exploitation
    exploiters = set([
        player for player in self._payoff.players
        if isinstance(player, MainExploiter)
    ])
    exp_historical = [
        player for player in self._payoff.players
        if isinstance(player, Historical) and player.parent in exploiters
    ]
    win_rates = self._payoff[self, exp_historical]
    if len(win_rates) and win_rates.min() < 0.3:
      return np.random.choice(
          exp_historical, p=pfsp(win_rates, weighting="squared")), True

    # Check forgetting
    historical = [
        player for player in self._payoff.players
        if isinstance(player, Historical) and player.parent == opponent
    ]
    win_rates = self._payoff[self, historical]
    win_rates, historical = remove_monotonic_suffix(win_rates, historical)
    if len(win_rates) and win_rates.min() < 0.7:
      return np.random.choice(
          historical, p=pfsp(win_rates, weighting="squared")), True

    return None

  def step(self, observation, state):
    action, policy_logits, new_state = self.agent.step(observation, state)
    return action, policy_logits, new_state

  def initial_state(self):
    """Returns the hidden state of the agent for the start of an episode."""
    # Network details elided.
    return self.agent.initial_state()

  def get_weights(self):
    return self.agent.get_weights()

  def get_race(self):
    return self._race

  def get_match(self):
    coin_toss = np.random.random()

    # Make sure you can beat the League
    if coin_toss < 0.5:
      return self._pfsp_branch()

    main_agents = [
        player for player in self._payoff.players
        if isinstance(player, MainPlayer)
    ]
    opponent = np.random.choice(main_agents)

    # Verify if there are some rare players we omitted
    if coin_toss < 0.5 + 0.15:
      request = self._verification_branch(opponent)
      if request is not None:
        return request

    return self._selfplay_branch(opponent)

  def ready_to_checkpoint(self):
    steps_passed = self.agent.get_steps() - self._checkpoint_step
    if steps_passed < 2e9:
      return False

    historical = [
        player for player in self._payoff.players
        if isinstance(player, Historical)
    ]
    win_rates = self._payoff[self, historical]
    return win_rates.min() > 0.7 or steps_passed > 4e9

  def checkpoint(self):
    self._checkpoint_step = self.agent.get_steps()
    return self._create_checkpoint()

class MainExploiter(Player):
  def __init__(self, race, agent, payoff):
    self.agent = Agent(race, agent.get_weights())
    self._initial_weights = agent.get_weights()
    self._payoff = payoff
    self._race = agent.race
    self._checkpoint_step = 0

  def step(self, observation, state):
    action, policy_logits, new_state = self.agent.step(observation, state)
    return action, policy_logits, new_state

  def initial_state(self):
    """Returns the hidden state of the agent for the start of an episode."""
    # Network details elided.
    return self.agent.initial_state()

  def get_match(self):
    main_agents = [
        player for player in self._payoff.players
        if isinstance(player, MainPlayer)
    ]
    opponent = np.random.choice(main_agents)

    if self._payoff[self, opponent] > 0.1:
      return opponent, True

    historical = [
        player for player in self._payoff.players
        if isinstance(player, Historical) and player.parent == opponent
    ]
    win_rates = self._payoff[self, historical]

    return np.random.choice(
        historical, p=pfsp(win_rates, weighting="variance")), True

  def get_race(self):
    return self._race

  def checkpoint(self):
    self.agent.set_weights(self._initial_weights)
    self._checkpoint_step = self.agent.get_steps()
    return self._create_checkpoint()

  def ready_to_checkpoint(self):
    steps_passed = self.agent.get_steps() - self._checkpoint_step
    if steps_passed < 2e9:
      return False

    main_agents = [
        player for player in self._payoff.players
        if isinstance(player, MainPlayer)
    ]
    win_rates = self._payoff[self, main_agents]
    return win_rates.min() > 0.7 or steps_passed > 4e9

class LeagueExploiter(Player):
  def __init__(self, race, agent, payoff):
    self.agent = Agent(race, agent.get_weights())
    self._initial_weights = agent.get_weights()
    self._payoff = payoff
    self._race = agent.race
    self._checkpoint_step = 0

  def step(self, observation, state):
    action, policy_logits, new_state = self.agent.step(observation, state)
    return action, policy_logits, new_state

  def initial_state(self):
    """Returns the hidden state of the agent for the start of an episode."""
    # Network details elided.
    return self.agent.initial_state()

  def get_match(self):
    historical = [
        player for player in self._payoff.players
        if isinstance(player, Historical)
    ]
    win_rates = self._payoff[self, historical]
    return np.random.choice(
        historical, p=pfsp(win_rates, weighting="linear_capped")), True

  def get_race(self):
    return self._race

  def checkpoint(self):
    if np.random.random() < 0.25:
      self.agent.set_weights(self._initial_weights)
    self._checkpoint_step = self.agent.get_steps()
    return self._create_checkpoint()

  def ready_to_checkpoint(self):
    steps_passed = self._agent.get_steps() - self._checkpoint_step
    if steps_passed < 2e9:
      return False
    historical = [
        player for player in self._payoff.players
        if isinstance(player, Historical)
    ]
    win_rates = self._payoff[self, historical]
    return win_rates.min() > 0.7 or steps_passed > 4e9

class Historical(Player):
  def __init__(self, agent, payoff):
    self._agent = Agent(agent.race, agent.get_weights())
    self._payoff = payoff
    self._race = agent.race
    self._parent = agent

  def step(self, observation, state):
    action, policy_logits, new_state = self._agent.step(observation, state)
    return action, policy_logits, new_state

  @property
  def parent(self):
    return self._parent

  def initial_state(self):
    """Returns the hidden state of the agent for the start of an episode."""
    # Network details elided.
    return self._agent.initial_state()

  def get_match(self):
    raise ValueError("Historical players should not request matches")

  def ready_to_checkpoint(self):
    return False

class League(object):
  def __init__(self,
               initial_agents,
               main_agents=1,
               main_exploiters=1,
               league_exploiters=1):
    self._payoff = Payoff()
    self._learning_agents = []

    for race in initial_agents:

      for _ in range(main_agents):
        main_agent = MainPlayer(race, initial_agents[race], self._payoff)

        self._learning_agents.append(main_agent)
        self._payoff.add_player(main_agent.checkpoint())

      for _ in range(main_exploiters):
        self._learning_agents.append(
            MainExploiter(race, initial_agents[race], self._payoff))

      for _ in range(league_exploiters):
        self._learning_agents.append(
            LeagueExploiter(race, initial_agents[race], self._payoff))

    for player in self._learning_agents:
      self._payoff.add_player(player)

  def update(self, home, away, result):
    return self._payoff.update(home, away, result)

  def get_player(self, idx):
    return self._learning_agents[idx]

  def add_player(self, player):
    self._payoff.add_player(player)


# module plotter
'''
Plots the residual vector vs. the number of iterations.
'''

import matplotlib.pyplot as plt
import scienceplots # pylint: disable=unused-import
import pandas as pd

res_data = pd.read_csv('../data/res.csv')
frc_data = pd.read_csv('../data/frc.csv')

plt.style.use(['science', 'grid'])
plt.figure(figsize=(1920/96, 1080/96), dpi=96)

plt.plot(res_data['iter'], res_data['rho'],   label='$\\rho$')
plt.plot(res_data['iter'], res_data['rho_u'], label='$\\rho{}u$')
plt.plot(res_data['iter'], res_data['rho_v'], label='$\\rho{}v$')
plt.plot(res_data['iter'], res_data['rho_e'], label='$\\rho{}E$')

plt.plot(res_data['iter'], res_data['err_rho'],   label='$\\rho$ Error',    linestyle='dashdot')
plt.plot(res_data['iter'], res_data['err_rho_u'], label='$\\rho{}u$ Error', linestyle='dashdot')
plt.plot(res_data['iter'], res_data['err_rho_v'], label='$\\rho{}v$ Error', linestyle='dashdot')
plt.plot(res_data['iter'], res_data['err_rho_e'], label='$\\rho{}E$ Error', linestyle='dashdot')

plt.yscale('log')
plt.title('$R_{ij} - D_{ij}$ \\& Respective Error vs. Iteration Number')
plt.xlabel('Iteration Number')
plt.ylabel('Residual')
plt.legend()

# plt.savefig('../img/res.webp', dpi=96 * 2)
plt.show()

plt.plot(frc_data['dist'], frc_data['fp_x'], label='$F_{px}$')
plt.plot(frc_data['dist'], frc_data['fp_y'], label='$F_{py}$')
plt.title('Pressure Force on the Bottom Wall')
plt.xlabel('$x$-distance')
plt.ylabel('Pressure Force')
plt.legend()

# plt.savefig('../img/frc.webp', dpi=96 * 2)
plt.show()


# this problem is same as the max_sub_arr in same directory where it has been explain using kadan algorithm only differce is here the length of sub arr has been given.
# https://www.youtube.com/watch?v=__guhvzO540
# asymtotic function = len of arr * len of win (m*n)

def brute_force(arr, win=4):
	max_sum = 0
	for i in range(len(arr) - win + 1):
		win_sum = 0
		for j in range(win):
			win_sum += arr[i:][j]
			if win_sum > max_sum:
				max_sum = win_sum
	return max_sum

def sliding_window(nums: list, k: int) -> float:
	max_sum = 0
	_sum, start = 0, 0
	for end in range(len(nums)):
		_sum += nums[end]
		if end >= k - 1: # making equal to the sliding window size
			max_sum = max(_sum, max_sum)
			_sum -= nums[start]
			start += 1  # slide the window

	return max_sum

from itertools import combinations

def choose_best_dist(num, k):
	temp = 0
	for i in combinations(num, k):
		temp = max(sum(i), temp)
	return temp

def main():
	arr = [1, 24, 6, -3, 10, -2, 5, 12 ,13]
	#arr = [2,2,2,2,2,2,2,2]
	#arr = [4,3,2,1,6]
	print(sliding_window(arr, 4))
	#print(brute_force(arr))
	print(choose_best_dist(arr, 4))

if __name__ == '__main__':
	main()