from collections import deque

with open('i19.txt') as f:
    scn = [[(*map(int, line.split(',')),) for line in sc.split()[4:]]
           for sc in f.read().strip().split('\n\n')]

N = len(scn)
MIN_OVERLAP = 12

def rot90z(coords):
    return [(-y, x, z) for x, y, z in coords]

def rot90x(coords):
    return [(x, -z, y) for x, y, z in coords]

def rot90y(coords):
    return [(z, y, -x) for x, y, z in coords]

def getAllOrient(coords):
    for _ in range(4):
        for _ in range(4):
            yield coords
            coords = rot90z(coords)
        coords = rot90x(coords)
    coords = rot90y(coords)
    for _ in range(4):
        yield coords
        coords = rot90z(coords)
    coords = rot90y(rot90y(coords))
    for _ in range(4):
        yield coords
        coords = rot90z(coords)

def mapToPairs(coords):
    for x1, y1, z1 in coords:
        for x2, y2, z2 in coords:
            if x1 == x2 and y1 == y2 and z1 == z2:
                continue
            yield x2-x1, y2-y1, z2-z1

def mapDelta(arr, dx, dy, dz):
    return [(x+dx, y+dy, z+dz) for x, y, z in arr]

def findMapedDeltas(a, b, ovl=None):
    if ovl is None:
        ovl = MIN_OVERLAP
    for ax, ay, az in a:
        for bx, by, bz in b:
            maped = mapDelta(b, ax-bx, ay-by, az-bz)
            if overlap(a, maped) >= ovl:
                return maped, ax-bx, ay-by, az-bz

def overlap(a, b):
    a = {*a}
    c = 0
    for e in b:
        if e in a:
            a.remove(e)
            c += 1
    return c

scnOr = [list(getAllOrient(sc)) for sc in scn]

scwor = [[list(mapToPairs(o)) for o in s] for s in scnOr]

prs = []
for i in range(N):
    print('Precheck:', i, '/', N, end='\r')
    for j in range(i + 1, N):
        for bo in scwor[j]:
            if overlap(scwor[i][0], bo) >= MIN_OVERLAP * (MIN_OVERLAP - 1):
                prs += [(i, j)]
                break
print(end='Precheck: Done     \r')

curr = {*scn[0]}
q = deque(x[1] for x in prs if x[0] == 0)
done = set([0]+[*q])
scnpos = [(0, 0, 0)]

while q:
    print('Done:', len(done)-len(q), '/', N, end='      \r')
    i = q.popleft()
    for o in getAllOrient(scn[i]):
        r = findMapedDeltas(curr, o)
        if r:
            maped, *dp = r
            curr.update(maped)
            scnpos += [dp]
            for a, b in prs:
                if b == i:
                    a, b = b, a
                if a == i and b not in done:
                    q.append(b)
                    done.add(b)
            break

print('Part 1:', len(curr), '   ')

mx = 0
for x1, y1, z1 in scnpos:
    for x2, y2, z2 in scnpos:
        mx = max(mx, abs(x1-x2)+abs(y1-y2)+abs(z1-z2))

print('Part 2:', mx)


import contextlib
import enum
import functools
import os
import tempfile
from typing import Any, Dict, List, Optional, TypeVar, Union

from sklearn import base
from sklearn import dummy
from sklearn import ensemble
from sklearn import tree
import tensorflow as tf
import tensorflow_decision_forests as tfdf

class TaskType(enum.Enum):
  """The type of task that a scikit-learn model performs."""
  UNKNOWN = 1
  SCALAR_REGRESSION = 2
  SINGLE_LABEL_CLASSIFICATION = 3

ScikitLearnModel = TypeVar("ScikitLearnModel", bound=base.BaseEstimator)
ScikitLearnTree = TypeVar("ScikitLearnTree", bound=tree.BaseDecisionTree)

def convert(
    sklearn_model: ScikitLearnModel,
    intermediate_write_path: Optional[os.PathLike] = None,
) -> tf.keras.Model:
  """Converts a tree-based scikit-learn model to a tensorflow model.

  Currently supported models are:
  *   sklearn.tree.DecisionTreeClassifier
  *   sklearn.tree.DecisionTreeRegressor
  *   sklearn.tree.ExtraTreeClassifier
  *   sklearn.tree.ExtraTreeRegressor
  *   sklearn.ensemble.RandomForestClassifier
  *   sklearn.ensemble.RandomForestRegressor
  *   sklearn.ensemble.ExtraTreesClassifier
  *   sklearn.ensemble.ExtraTreesRegressor
  *   sklearn.ensemble.GradientBoostingRegressor

  Additionally, only single-label classification and scalar regression are
  supported (e.g. multivariate regression models will not convert).

  Args:
    sklearn_model: the scikit-learn tree based model to be converted.
    intermediate_write_path: path to a directory. As part of the conversion
      process, a TFDF model is written to disk. If intermediate_write_path is
      specified, the TFDF model is written to this directory. Otherwise, a
      temporary directory is created that is immediately removed after this
      function executes. Note that in order to save the converted model and
      load it again later, this argument must be provided.

  Returns:
    a keras Model that emulates the provided scikit-learn model.
  """
  if not intermediate_write_path:
    # No intermediate directory was provided, so this creates one using the
    # TemporaryDirectory context mananger, which handles teardown.
    intermediate_write_directory = tempfile.TemporaryDirectory()
    path = intermediate_write_directory.name
  else:
    # Uses the provided write path, and creates a null context manager as a
    # stand-in for TemporaryDirectory.
    intermediate_write_directory = contextlib.nullcontext()
    path = intermediate_write_path
  with intermediate_write_directory:
    tfdf_model = _build_tfdf_model(sklearn_model, path)
  # The resultant tfdf model only receives the features that are used
  # to split samples in nodes in the trees as input. But we want to pass the
  # full design matrix as an input to match the scikit-learn API, thus we
  # create another tf.keras.Model with the desired call signature.
  template_input = tf.keras.Input(shape=(sklearn_model.n_features_in_,))
  # Extracts the indices of the features that are used by the TFDF model.
  # The features have names with the format "feature_<index-of-feature>".
  feature_names = tfdf_model.signatures[
      "serving_default"].structured_input_signature[1].keys()
  template_output = tfdf_model(
      {i: template_input[:, int(i.split("_")[1])] for i in feature_names})
  return tf.keras.Model(inputs=template_input, outputs=template_output)

@functools.singledispatch
def _build_tfdf_model(
    sklearn_model: ScikitLearnModel,
    path: os.PathLike,
) -> tf.keras.Model:
  """Builds a TFDF model from the given scikit-learn model."""
  raise NotImplementedError(
      f"Can't build a TFDF model for {type(sklearn_model)}")

@_build_tfdf_model.register(tree.DecisionTreeRegressor)
@_build_tfdf_model.register(tree.ExtraTreeRegressor)
def _(sklearn_model: ScikitLearnTree, path: os.PathLike) -> tf.keras.Model:
  """Converts a single scikit-learn regression tree to a TFDF model."""
  # The label argument is unused when the model is loaded, so we pass a
  # placeholder.
  objective = tfdf.py_tree.objective.RegressionObjective(label="label")
  pytree = convert_sklearn_tree_to_tfdf_pytree(sklearn_model)
  cart_builder = tfdf.builder.CARTBuilder(path=path, objective=objective)
  cart_builder.add_tree(pytree)
  cart_builder.close()
  return tf.keras.models.load_model(path)

@_build_tfdf_model.register(tree.DecisionTreeClassifier)
@_build_tfdf_model.register(tree.ExtraTreeClassifier)
def _(sklearn_model: ScikitLearnTree, path: os.PathLike) -> tf.keras.Model:
  """Converts a single scikit-learn classification tree to a TFDF model."""
  objective = tfdf.py_tree.objective.ClassificationObjective(
      label="label",
      # TF doesnt accept classes that aren't bytes or unicode,
      # so we convert the classes into strings in case they are not.
      classes=[str(c) for c in sklearn_model.classes_],
  )
  pytree = convert_sklearn_tree_to_tfdf_pytree(sklearn_model)
  cart_builder = tfdf.builder.CARTBuilder(path=path, objective=objective)
  cart_builder.add_tree(pytree)
  cart_builder.close()
  return tf.keras.models.load_model(path)

@_build_tfdf_model.register(ensemble.ExtraTreesRegressor)
@_build_tfdf_model.register(ensemble.RandomForestRegressor)
def _(
    sklearn_model: Union[ensemble.ExtraTreesRegressor, ensemble.RandomForestRegressor],
    path: os.PathLike,
) -> tf.keras.Model:
  """Converts a forest regression model into a TFDF model."""
  objective = tfdf.py_tree.objective.RegressionObjective(label="label")
  rf_builder = tfdf.builder.RandomForestBuilder(path=path, objective=objective)
  for single_tree in sklearn_model.estimators_:
    rf_builder.add_tree(convert_sklearn_tree_to_tfdf_pytree(single_tree))
  rf_builder.close()
  return tf.keras.models.load_model(path)

@_build_tfdf_model.register(ensemble.ExtraTreesClassifier)
@_build_tfdf_model.register(ensemble.RandomForestClassifier)
def _(
    sklearn_model: Union[ensemble.ExtraTreesClassifier, ensemble.RandomForestClassifier],
    path: os.PathLike,
) -> tf.keras.Model:
  """Converts a forest classification model into a TFDF model."""
  objective = tfdf.py_tree.objective.ClassificationObjective(
      label="label",
      classes=[str(c) for c in sklearn_model.classes_],
  )
  rf_builder = tfdf.builder.RandomForestBuilder(path=path, objective=objective)
  for single_tree in sklearn_model.estimators_:
    rf_builder.add_tree(convert_sklearn_tree_to_tfdf_pytree(single_tree))
  rf_builder.close()
  return tf.keras.models.load_model(path)

@_build_tfdf_model.register(ensemble.GradientBoostingRegressor)
def _(
    sklearn_model: ensemble.GradientBoostingRegressor,
    path: os.PathLike,
) -> tf.keras.Model:
  """Converts a gradient boosting regression model into a TFDF model."""
  if isinstance(sklearn_model.init_, dummy.DummyRegressor):
    # If the initial estimator is a DummyRegressor, then it predicts a constant
    # which can be passed to GradientBoostedTreeBuilder as a bias.
    init_pytree = None
    bias = sklearn_model.init_.constant_[0][0]
  elif isinstance(sklearn_model.init_, tree.DecisionTreeRegressor):
    # If the initial estimator is a DecisionTreeRegressor, we add it as the
    # first tree in the ensemble and set the bias to zero. We could also support
    # other tree-based initial estimators (e.g. RandomForest), but this seems
    # like a niche enough use case that we don't for the moment.
    init_pytree = convert_sklearn_tree_to_tfdf_pytree(sklearn_model.init_)
    bias = 0.0
  elif sklearn_model.init_ == "zero":
    init_pytree = None
    bias = 0.0
  else:
    raise ValueError("The initial estimator must be either a DummyRegressor"
                     "or a DecisionTreeRegressor, but got"
                     f"{type(sklearn_model.init_)}.")

  gbt_builder = tfdf.builder.GradientBoostedTreeBuilder(
      path=path,
      objective=tfdf.py_tree.objective.RegressionObjective(label="label"),
      bias=bias,
  )
  if init_pytree:
    gbt_builder.add_tree(init_pytree)

  for weak_learner in sklearn_model.estimators_.ravel():
    gbt_builder.add_tree(convert_sklearn_tree_to_tfdf_pytree(
        weak_learner,
        weight=sklearn_model.learning_rate,
    ))
  gbt_builder.close()
  return tf.keras.models.load_model(path)

def convert_sklearn_tree_to_tfdf_pytree(
    sklearn_tree: ScikitLearnTree,
    weight: Optional[float] = None,
) -> tfdf.py_tree.tree.Tree:
  """Converts a scikit-learn decision tree into a TFDF pytree.

  Args:
    sklearn_tree: a scikit-learn decision tree.
    weight: an optional weight to apply to the values of the leaves in the tree.
      This is intended for use when converting gradient boosted tree models.

  Returns:
    a TFDF pytree that has the same structure as the scikit-learn tree.
  """
  try:
    sklearn_tree_data = sklearn_tree.tree_.__getstate__()
  except AttributeError as e:
    raise ValueError(
        "Scikit-Learn model must be fit to data before converting.") from e

  field_names = sklearn_tree_data["nodes"].dtype.names
  task_type = _get_sklearn_tree_task_type(sklearn_tree)
  if weight and task_type is TaskType.SINGLE_LABEL_CLASSIFICATION:
    raise ValueError("weight should not be passed for classification trees.")

  nodes = []
  for node_properties, target_value in zip(
      sklearn_tree_data["nodes"],
      sklearn_tree_data["values"],
  ):
    node = {
        field_name: field_value
        for field_name, field_value in zip(field_names, node_properties)
    }
    if task_type is TaskType.SCALAR_REGRESSION:
      scaling_factor = weight if weight else 1.0
      node["value"] = tfdf.py_tree.value.RegressionValue(target_value[0][0] *
                                                         scaling_factor)
    elif task_type is TaskType.SINGLE_LABEL_CLASSIFICATION:
      # Normalise to probabilities if we have a classification tree.
      probabilities = list(target_value[0] / target_value[0].sum())
      node["value"] = tfdf.py_tree.value.ProbabilityValue(probabilities)
    else:
      raise ValueError(
          "Only scalar regression and single-label classification are "
          "supported.")
    nodes.append(node)

  root_node = _convert_sklearn_node_to_tfdf_node(
      # The root node has index zero.
      node_index=0,
      nodes=nodes,
  )
  return tfdf.py_tree.tree.Tree(root_node)

def _get_sklearn_tree_task_type(sklearn_tree: ScikitLearnTree) -> TaskType:
  """Finds the task type of a scikit learn tree."""
  if hasattr(sklearn_tree, "n_classes_") and sklearn_tree.n_outputs_ == 1:
    return TaskType.SINGLE_LABEL_CLASSIFICATION
  elif sklearn_tree.n_outputs_ == 1:
    return TaskType.SCALAR_REGRESSION
  else:
    return TaskType.UNKNOWN

def _convert_sklearn_node_to_tfdf_node(
    node_index: int,
    nodes: List[Dict[str, Any]],
) -> tfdf.py_tree.node.AbstractNode:
  """Converts a node within a scikit-learn tree into a TFDF node."""
  if node_index == -1:
    return None

  node = nodes[node_index]
  neg_child = _convert_sklearn_node_to_tfdf_node(
      node_index=node["left_child"],
      nodes=nodes,
  )
  pos_child = _convert_sklearn_node_to_tfdf_node(
      node_index=node["right_child"],
      nodes=nodes,
  )
  if pos_child:
    feature = tfdf.py_tree.dataspec.SimpleColumnSpec(
        name=f"feature_{node['feature']}",
        # In sklearn, all fields must be numerical.
        type=tfdf.py_tree.dataspec.ColumnType.NUMERICAL,
        col_idx=node["feature"],
    )
    return tfdf.py_tree.node.NonLeafNode(
        condition=tfdf.py_tree.condition.NumericalHigherThanCondition(
            feature=feature,
            threshold=node["threshold"],
            missing_evaluation=False,
        ),
        pos_child=pos_child,
        neg_child=neg_child,
    )
  else:
    return tfdf.py_tree.node.LeafNode(value=node["value"])


import time

import gym
import gym_rover

from gym import spaces

import numpy as np

from gym_rover.learning.ccea import CCEA

RENDER = True
GENS = 10000
NUM_POIS = 1
NUM_AGENTS = 1
TIME_LIMIT = 10
LENGTH = 10

class FitnessEval(object):
    def __init__(self, env):
        self.env = env
        self.input_shape = (1, 8)

    def fitness(self, team, debug=False):
        agents = team

        states = self.env.reset()
        done = False

        while not done:
            if RENDER:
                self.env.render()

            actions = []
            for i in range(self.env.num_agents):
                state = states[i]
                agent = agents[i]
                state = np.reshape(state, self.input_shape)
                action = agent.predict(state)
                actions.append(action)

            next_states, reward, done, info = self.env.step(np.array(actions))

        return [reward]

def init_ccea(env):
    fitness = FitnessEval(env)
    ccea = CCEA(env.num_agents, 12, [8,12,2], ['relu', 'relu', 'relu'], 0.05, fitness)
    return ccea

def init_env():
    env = gym.make('rover-cont-feature-v0')
    env._max_epsidoes = GENS * NUM_AGENTS * 12

    env.world_height = LENGTH
    env.world_width = LENGTH
    env.num_agents = NUM_AGENTS
    env.num_pois = NUM_POIS
    env.time_limit = TIME_LIMIT

    env.observation_mode = 'feature'

    env.set_observation_space()
    env.set_action_space()
    env.reset()
    return env

def run_ccea():
    env = init_env()
    ccea = init_ccea(env)

    for _ in range(10000):
        ccea.generation()
        print(ccea.best_team()[0])

if __name__ == "__main__":
    run_ccea()


import logging
from os.path import join, exists, splitext, basename, split
from os import makedirs, remove, listdir
from franklin.backbone.specifications import (BACKBONE_DIRECTORIES,
                                              BACKBONE_BASENAMES)
from franklin.utils.cmd_utils import call
from franklin.utils.misc_utils import get_num_threads, rel_symlink
from franklin.seq.readers import guess_seq_file_format
from franklin.utils.seqio_utils import seqio
from tempfile import NamedTemporaryFile
from franklin.backbone.analysis import scrape_info_from_fname

LOGGER_NAME = 'franklin'

def blast_runner_plus(seq_fpath, blast_db, blast_type, result_fpath,
                       threads=False):
    'It runs a blast giving a file and a database path'
    cmd = [blast_type, '-db', blast_db, '-num_alignments', '25',
           '-num_descriptions', '25', '-evalue', '0.0001', '-outfmt', '5',
           '-query', seq_fpath, '-out', result_fpath]
    if threads:
        cmd.extend(['-num_threads', str(threads)])
    call(cmd, raise_on_error=True, log=True)

def makeblastdb_plus(seq_fpath, dbtype, outputdb=None):
    'It creates the blast db database'
    cmd = ['makeblastdb', '-in', seq_fpath, '-dbtype', dbtype]
    if outputdb is not None:
        cmd.extend(['-out', outputdb])
    call(cmd, raise_on_error=True)

def _get_basename(fpath):
    'It returns the base name without path and extension'
    return splitext(basename(fpath))[0]

def _create_temp_fasta_file(fpath):
    'It creates a fasta file format temfile'
    fasta_fhand = NamedTemporaryFile(suffix='.fasta', mode='a')
    seqio(in_seq_fhand=open(fpath),
          out_seq_fhand=fasta_fhand,
          out_format='fasta')
    return fasta_fhand

def guess_blastdb_kind(blastdb):
    'it infers the kind of the blastdb'
    blastdir, basename_ = split(blastdb)
    for file_ in listdir(blastdir):
        if file_.startswith(basename_ + '.') and file_ != basename_ :
            if splitext(file_)[1][1] == 'n':
                return 'nucl'
            elif splitext(file_)[1][1] == 'p':
                return 'prot'

def guess_blast_program(seq_type, db_type, prefer_tblastx=False):
    'It guess the blast program to use for the given seq and db type'
    if seq_type == 'nucl' and db_type == 'nucl':
        if prefer_tblastx:
            blast_program = 'tblastx'
        else:
            blast_program = 'blastn'
    elif seq_type == 'nucl' and db_type == 'prot':
        blast_program = 'blastx'
    elif seq_type == 'prot' and db_type == 'nucl':
        blast_program = 'tblastn'
    elif seq_type == 'prot' and db_type == 'prot':
        blast_program = 'blastp'

    return blast_program

def backbone_blast_runner(query_fpath, project_dir, blast_program,
                          blast_db=None, blast_db_seq=None, dbtype='nucl',
                          threads=False):
    '''It returns the blast if the results doesn't exist'''
    if blast_db is None and blast_db_seq is None:
        raise RuntimeError('It needs a blast database or seqfile')

    #create a logger
    logger = logging.getLogger(LOGGER_NAME)
    query_basename = _get_basename(query_fpath)
    blast_dir = join(project_dir, BACKBONE_DIRECTORIES['blast_dir'])

    if blast_db:
        result_dir = join(blast_dir, query_basename, _get_basename(blast_db))
    else:
        result_dir = join(blast_dir, query_basename,
                          _get_basename(blast_db_seq))
    if not exists(result_dir):
        makedirs(result_dir)
    result_fpath = join(result_dir,
                        '%s.%s.xml' % (BACKBONE_BASENAMES['blast_basename'],
                                       blast_program))
    if exists(result_fpath):
        logger.info('Using the stored blast result %s' % result_fpath)
        return result_fpath

    #the input file should be fasta
    fasta_query_fhand = None
    fasta_db_fhand = None
    if guess_seq_file_format(open(query_fpath)) != 'fasta':
        fasta_query_fhand = _create_temp_fasta_file(query_fpath)
        query_fpath = fasta_query_fhand.name

    #we have to create a database in BACKBONE_DIRECTORIES['blast_databases']
    if blast_db_seq:
        blast_db = make_backbone_blast_db(project_dir, blast_db_seq, dbtype)

    logger.info('Running the blast %s' % result_fpath)
    try:
        blast_runner_plus(query_fpath, blast_db, blast_program,
                                 result_fpath, threads=threads)
    except RuntimeError as error:
        if exists(result_fpath):
            remove(result_fpath)
        msg = '%s \n database: %s\n database type: %s' % (str(error),
                                                             blast_db, dbtype)
        raise RuntimeError(msg)

    if fasta_query_fhand:
        fasta_query_fhand.close()
    if fasta_db_fhand:
        fasta_db_fhand.close()

    return result_fpath

def make_backbone_blast_db(project_dir, blast_db_seq, dbtype):
    'It formats a blastdb when need it'
    logger = logging.getLogger(LOGGER_NAME)
    #the name should be the basename of the blast_db_seq
    db_dir = join(project_dir, BACKBONE_DIRECTORIES['blast_databases'])
    if not exists(db_dir):
        makedirs(db_dir)
    db_seq_fpath = join(db_dir, _get_basename(blast_db_seq))
    if not exists(db_seq_fpath):
        #which is the name of the new databae?
        blast_db_seq_format = guess_seq_file_format(open(blast_db_seq))
        if blast_db_seq_format == 'fasta':
            rel_symlink(blast_db_seq, db_seq_fpath)
        else:
            seqio(in_seq_fhand=open(blast_db_seq),
                  out_seq_fhand=open(db_seq_fpath, 'w'),
                  out_format='fasta')
        logger.info('Formatting the database %s' % db_seq_fpath)
        try:
            makeblastdb_plus(db_seq_fpath, dbtype=dbtype)
        except RuntimeError:
            msg = 'Error making blastdb. db:%s\n dbtype:%s\n' % \
                                               (db_seq_fpath, dbtype)
            remove(db_seq_fpath)
            raise RuntimeError(msg)
    return db_seq_fpath


# -*- coding: utf-8 -*-
"""
Created on Thu Sep  6 06:26:39 2018

@author: vishal
"""

url = 'https://www.imdb.com/india/top-rated-indian-movies/?sort=rk,asc&mode=simple&page=1'
from selenium import webdriver
driver =  webdriver.Chrome(r"C:\Users\visha\chromedriver")
driver.get(url)

#for storing all titles in a list
titles = driver.find_elements_by_class_name('titleColumn')
newtitles = []
for i in titles:
    j = i.text
    newtitles.append(j)

#looping for ratings
string_title = '//*[@id="main"]/div/span/div/div/div[3]/table/tbody/tr[1]/td[3]/strong'
ratings = []
for er in range(1,len(titles)+1):
    up_rate = string_title[0:-15] +str(er)+ string_title[-14:]
    k = driver.find_element_by_xpath(up_rate)
    ratings.append(k)

newratings = []
for k in ratings:
    q = k.text
    newratings.append(q)

'''we came to conclusion later that to access all films in rating wise
it would be great if we stick with list as dictionaries are inherently orderless '''

list1 = []
for i in range(0,249):
    k = newtitles[i] + ' : ' + newratings[i]
    list1.append(k)

a = 1
while a != 0:
    test1 = int(input('Enter a number between 1-250 to check rating of a movie: '))
    if test1>=1 and test1<=250:
        print(list1[test1-1])
    else:
        print('wrong input')
    a = input("Enter 'done' to stop or press enter to continue: ")
    if a == 'done':
        break


import warnings

warnings.filterwarnings("ignore")
import argparse
import torch
import os
import numpy as np
import pdb

from torch import nn, optim
from torch.utils import data
from torch.nn import functional as F
from data_utils import *
from vis_tools import *

# from model_utils import *
from torchvision.utils import save_image
from torchvision import transforms
from time import time
from tqdm import tqdm

from torch.optim.lr_scheduler import StepLR
from utils.trajectory_loader import PushDataset
from models.image_autoencoder import Decoder, Encoder

# Configurations and Hyperparameters
port_num = 8082
gpu_id = 1
lr_rate = 2e-4
num_epochs = 50
num_sample = 6
noise_dim = 2
report_feq = 10

display = visualizer(port=port_num)

# Random Initialization
torch.manual_seed(1)
np.random.seed(1)

def denorm(tensor):
    return ((tensor + 1.0) / 2.0) * 255.0

def norm(image):
    return (image / 255.0 - 0.5) * 2.0

# Dataloader
dataset = PushDataset("128_128_data")
loader = data.DataLoader(dataset, batch_size=16, shuffle=True)

# Models
encoder = Encoder().to(gpu_id)
decoder = Decoder().to(gpu_id)
decoder.weight_init(mean=0.0, std=0.02)
encoder.weight_init(mean=0.0, std=0.02)

# Initialize Loss
l1, mse, bce = nn.L1Loss(), nn.MSELoss(), nn.BCELoss()

# Initialize Optimizer
optimizer = optim.Adam(
    [{"params": decoder.parameters()}, {"params": encoder.parameters()}],
    lr=lr_rate,
    betas=(0.5, 0.999),
)

step = 0
min_pred_error = np.inf
for epoch in range(num_epochs):
    for i, inputs in enumerate(loader):
        ########## Inputs ########
        images, _, _, _ = inputs
        images = images.to(gpu_id)

        # Flatten image trajectories to image batch
        state_cur = images.view(-1, *(images.size()[2:]))
        # state_cur = norm(state_cur)

        z = encoder(state_cur)
        state_cur_hat = decoder(z)

        recon_loss = mse(state_cur_hat, state_cur)

        optimizer.zero_grad()
        recon_loss.backward()
        optimizer.step()

        step += 1

        recon_loss_np = recon_loss.cpu().data.numpy()
        print(epoch, step, "recon_loss_np: ", recon_loss_np)

        if step % report_feq == 0:
            state_cur_vis = denorm(state_cur[0]).detach().cpu().numpy().astype(np.uint8)
            state_cur_hat_vis = (
                denorm(state_cur_hat[0]).detach().cpu().numpy().astype(np.uint8)
            )

            display.img_result(state_cur_vis, win=1, caption="state_cur_vis")
            display.img_result(state_cur_hat_vis, win=2, caption="state_cur_hat_vis")

    if epoch % 10 == 1:
        if not os.path.exists("models"):
            os.makedirs("models")

        torch.save(encoder, "models/encoder_" + str(epoch) + ".pt")
        torch.save(decoder, "models/decoder_" + str(epoch) + ".pt")


"""
You are given a sorted unique integer array nums.

Return the smallest sorted list of ranges that cover all the numbers
in the array exactly. That is, each element of nums is covered by
exactly one of the ranges, and there is no integer x such that x is
in one of the ranges but not in nums.

Each range [a,b] in the list should be output as:

"a->b" if a != b
"a" if a == b

Example 1:
Input: nums = [0,1,2,4,5,7]
Output: ["0->2","4->5","7"]
Explanation: The ranges are:
[0,2] --> "0->2"
[4,5] --> "4->5"
[7,7] --> "7"

Example 2:
Input: nums = [0,2,3,4,6,8,9]
Output: ["0","2->4","6","8->9"]
Explanation: The ranges are:
[0,0] --> "0"
[2,4] --> "2->4"
[6,6] --> "6"
[8,9] --> "8->9"

Example 3:
Input: nums = []
Output: []

Example 4:
Input: nums = [-1]
Output: ["-1"]

Example 5:
Input: nums = [0]
Output: ["0"]

Constraints:
0 <= nums.length <= 20
-231 <= nums[i] <= 231 - 1
All the values of nums are unique.
"""
from typing import List

class Solution:
    def summaryRanges(self, nums: List[int]) -> List[str]:
        summary: List[str] = []
        start_index: int = 0
        for index in range(len(nums)):
            if index + 1 < len(nums) and nums[index + 1] == nums[index] + 1:
                continue
            if start_index == index:
                summary.append(str(nums[start_index]) + "")
            else:
                summary.append(str(nums[start_index]) + "->" + str(nums[index]))

            start_index = index + 1

        return summary


from __future__ import annotations

from typing import Generator

from ansible_collections.arista.avd.plugins.filter.convert_dicts import convert_dicts
from ansible_collections.arista.avd.plugins.plugin_utils.errors import AvdConversionWarning, AvdDeprecationWarning
from ansible_collections.arista.avd.plugins.plugin_utils.utils import get_all

SCHEMA_TO_PY_TYPE_MAP = {
    "str": str,
    "int": int,
    "bool": bool,
    "float": float,
    "dict": dict,
    "list": list,
}
SIMPLE_CONVERTERS = {
    "str": str,
    "int": int,
    "bool": bool,
}

class AvdDataConverter:
    """
    AvdDataConverter is used to convert AVD Data Types based on schema options.

    avdschema argument is an instance of AvdSchema. Type hinting is not working because of circular import
    TODO: Refactor to take a fully resolved schema as dict
    """

    def __init__(self, avdschema):
        self._avdschema = avdschema

        # We run through all the regular keys first, to ensure that all data has been converted
        # in case some of it is referenced in "dynamic_keys" below
        self.converters = {
            "items": self.convert_items,
            "keys": self.convert_keys,
            "dynamic_keys": self.convert_dynamic_keys,
            "deprecation": self.deprecation,
        }

    def convert_data(self, data, schema: dict = None, path: list[str] = None) -> Generator:
        """
        Perform in-place conversion of data according to the provided schema.
        Main entry function which is recursively called from the child functions performing the actual conversion of keys/items.
        """
        if schema is None:
            # Get fully resolved schema (where all $ref has been expanded recursively)
            schema = self._avdschema.resolved_schema

        if path is None:
            path = []

        for key, converter in self.converters.items():
            if key not in schema:
                # Ignore keys not in schema
                continue

            # Converters will do inplace update of data. Any returns will be yielded conversion messages.
            yield from converter(schema[key], data, schema, path)

    def convert_keys(self, keys: dict, data: dict, schema: dict, path: list[str]):
        """
        This function performs conversion on each key with the relevant subschema
        """
        if not isinstance(data, dict):
            return

        for key, childschema in keys.items():
            if key not in data:
                # Skip key since there is nothing to convert if the key is not set in data
                continue

            # Perform type conversion of the data for the child key if required based on "convert_types"
            if "convert_types" in childschema:
                yield from self.convert_types(childschema["convert_types"], data, key, childschema, path + [key])

            # Convert to lower case if set in schema and value is a string
            if childschema.get("convert_to_lower_case") and isinstance(data[key], str):
                data[key] = data[key].lower()

            yield from self.convert_data(data[key], childschema, path + [key])

    def convert_dynamic_keys(self, dynamic_keys: dict, data: dict, schema: dict, path: list[str]):
        """
        This function resolves "dynamic_keys" by looking in the actual data.
        Then calls convert_keys to performs conversion on each resolved key with the relevant subschema
        """
        if not isinstance(data, dict):
            return

        # Resolve "keys" from schema "dynamic_keys" by looking for the dynamic key in data.
        keys = {}
        for dynamic_key, childschema in dynamic_keys.items():
            resolved_keys = get_all(data, dynamic_key)
            for resolved_key in resolved_keys:
                keys.setdefault(resolved_key, childschema)

        # Reuse convert_keys to perform the actual conversion on the resolved dynamic keys
        yield from self.convert_keys(keys, data, schema, path)

    def convert_items(self, items: dict, data: list, schema: dict, path: list[str]):
        """
        This function performs conversion on each item with the items subschema
        """
        if not isinstance(data, list):
            return

        for index, item in enumerate(data):
            # Perform type conversion of the items data if required based on "convert_types"
            if "convert_types" in items:
                yield from self.convert_types(items["convert_types"], data, index, items, path + [index])

            # Convert to lower case if set in schema and item is a string
            if items.get("convert_to_lower_case") and isinstance(item, str):
                data[index] = item.lower()

            # Dive in to child items/schema
            yield from self.convert_data(item, items, path + [index])

    def convert_types(self, convert_types: list, data: dict | list, index: str | int, schema: dict, path: list[str]):
        """
        This function performs type conversion if necessary on a single data instance.
        It is invoked for child keys during "keys" conversion and for child items during
        "items" conversion

        "data" is either the parent dict or the parent list.
        "index" is either the key of the parent dict or the index of the parent list.

        Conversion is performed in-place using the provided "data" and "index"

        Yields AvdConversionWarning and/or AvdDeprecationWarning except for simple str/int/bool conversions

        Any conversion errors are ignored and the original value is returned
        """
        schema_type = schema.get("type")

        # Get value from input data
        value = data[index]

        # For simple conversions, skip conversion if the value is of the correct type
        if schema_type in SIMPLE_CONVERTERS and isinstance(value, SCHEMA_TO_PY_TYPE_MAP.get(schema_type)):
            # Avoid corner case where we want to convert bool to int. Bool is a subclass of Int so it passes the check above.
            if not (schema_type == "int" and isinstance(value, bool)):
                return

        for convert_type in convert_types:
            if isinstance(value, SCHEMA_TO_PY_TYPE_MAP.get(convert_type)):
                if schema_type in SIMPLE_CONVERTERS:
                    try:
                        data[index] = SIMPLE_CONVERTERS[schema_type](value)
                    except Exception:
                        # Ignore errors
                        # TODO: Log message
                        return

                    # Here we do not yield an AvdConversionWarning, since these will be accepted going forward.

                elif convert_type in ["dict", "list"] and schema_type == "list" and "primary_key" in schema:
                    try:
                        data[index] = convert_dicts(value, schema["primary_key"], secondary_key=schema.get("secondary_key"))
                    except Exception:
                        # Ignore errors
                        # TODO: Log message
                        return

                    if data[index] == value:
                        # No change - when input is list and have the correct format in all items
                        return

                    yield AvdConversionWarning(key=path, oldtype=convert_type, newtype=schema_type)

                elif convert_type == "dict" and schema_type == "list":
                    try:
                        data[index] = list(value)
                    except Exception:
                        # Ignore errors
                        # TODO: Log message
                        return

                    yield AvdConversionWarning(key=path, oldtype=convert_type, newtype=schema_type)

                elif convert_type == "str" and schema_type == "list":
                    try:
                        data[index] = list(map(str.strip, value.split(",")))
                    except Exception:
                        # Ignore errors
                        # TODO: Log message
                        return

                    yield AvdConversionWarning(key=path, oldtype=convert_type, newtype=schema_type)

    def deprecation(self, deprecation: dict, data, schema: dict, path: list):
        """
        deprecation:
          warning: bool, default = True
          new_key: str
          removed: bool
          remove_in_version: str
          remove_after_date: str
          url: str
        """

        if not deprecation.get("warning", True):
            return

        yield AvdDeprecationWarning(
            key=path,
            new_key=deprecation.get("new_key"),
            remove_in_version=deprecation.get("remove_in_version"),
            remove_after_date=deprecation.get("remove_after_date"),
            url=deprecation.get("url"),
            removed=deprecation.get("removed", False),
        )


"""Align multiple sequence FASTA files with MAFFT.

This script allows the user to concatenate multiple sequence files in FASTA format using the
concatenate_contigs() function and to align the multifasta file with the mafft_align() function.

"""

import os
import logging
import subprocess
from Bio import SeqIO
from Bio.Seq import Seq

def concatenate_contigs(contigs_list, out_file="all_mitogenomes.rotated.fa"):
    # create a list of BioSeqs that contains all contigs
    contigs = []
    for contig in contigs_list:
        with open(contig) as handle:
            for record in SeqIO.parse(handle, "fasta"):
                contigs.append(record)
    # write concatenated FASTA file
    with open(out_file, "w") as output_handle:
        SeqIO.write(contigs, output_handle, "fasta")

    return out_file

def mafft_align(multifasta_file, threads='1', out_file="all_mitogenomes.rotated.aligned.aln", clustal_format=False):
    if clustal_format:
        #logging.info("MAFFT output will be in clustal format")
        mafft = "mafft --quiet --clustalout --thread {} {} > {}".format(threads, multifasta_file, out_file)
    else:
        mafft = "mafft --quiet --thread {} {} > {}".format(threads, multifasta_file, out_file)
    logging.info("MAFFT alignment will be called with:\n" + mafft + "\n")
    subprocess.run(mafft, shell=True)

if __name__ == '__main__':
    contigs_files = []
    for curr_file in os.listdir('.'):
        if curr_file.endswith('mitogenome.rotated.fa'):
            contigs_files.append(curr_file)
    concatenate_contigs(contigs_files)


import os
from PyPDF2 import PdfReader, PdfWriter

path = r"D:\python\real_python_materials\book1-exercises\chp11\practice_files"

input_file_name = os.path.join(path, 'Pride and Prejudice.pdf')
input_file = PdfReader(open(input_file_name, 'rb'))

output_pdf = PdfWriter()

for page_num in range(1, 4):
    output_pdf.add_page(input_file.pages[page_num])

with open('lecture_output.pdf', 'wb') as output_file:
    output_pdf.write(output_file)

from PyPDF2 import PdfReader, PdfWriter
import copy
import os

path = r"D:\python\real_python_materials\book1-exercises\chp11\practice_files"
input_file = PdfReader(open(path + '\\half and half.pdf', 'rb'))
output_pdf = PdfWriter()

for page_num in range(len(input_file.pages)):
    page_left = input_file.pages[page_num]
    page_right = copy.copy(page_left)
    upper_right = page_left.mediabox.upper_right

    page_left.mediabox.upper_right = (upper_right[0] / 2, upper_right[1])
    output_pdf.add_page(page_left)

    page_right.mediabox.upper_left = (upper_right[0] / 2, upper_right[1])
    output_pdf.add_page(page_right)

output_file_name = 'The little Mermaid.pdf'
output_file = open(output_file_name, 'wb')
output_pdf.write(output_file)
output_file.close()

import os
from PyPDF2 import PdfReader, PdfWriter

path = r"D:\python\real_python_materials\book1-exercises\chp11\practice_files"
input_file_name = os.path.join(path, "The Emperor.pdf")
input_file = PdfReader(open(input_file_name, "rb"))
output_PDF = PdfWriter()

watermark_file_name = os.path.join(path, "top secret.pdf")
watermark_file = PdfReader(open(watermark_file_name, "rb"))

for page_num in range(0, len(input_file.pages)):
    page = input_file.pages[page_num]
    page.merge_page(watermark_file.pages[0]) # add watermark image
    output_PDF.add_page(page)

output_PDF.encrypt("good2Bking") # add a password to the PDF file
output_file_name = "New Suit.pdf"
output_file = open(output_file_name, "wb")
output_PDF.write(output_file)
output_file.close()


import reflex as rx

from app import style
from app.state import State

def chat() -> rx.Component:
    return rx.box(
        rx.text(State.chat_history, style=style.question_style)
        #rx.foreach(
        #    State.chat_history,
        #    lambda messages: qa(messages[0], messages[1]),
        #)
    )

def action_bar() -> rx.Component:
    return rx.hstack(
        rx.input(
            value=State.question,
            placeholder="Put in a review...",
            on_change=State.set_question,
            style=style.input_style,
        ),
        rx.button(
            "Ask",
            on_click=State.answer,
            style=style.button_style,
        ),
    )
color = "rgb(107,99,246)"

#def upload():
#    """The main view."""
#    return rx.vstack(
#        rx.upload(
#            rx.vstack(
#                rx.button(
#                    "Select File",
#                    color=color,
#                    bg="white",
#                    border=f"1px solid {color}",
#                ),
#                rx.text(
#                    "Drag and drop files here or click to select files"
#                ),
#            ),
#            border=f"1px dotted {color}",
#            padding="5em",
#        ),
#        rx.hstack(rx.foreach(rx.selected_files, rx.text)),
#        rx.button(
#            "Upload",
#            on_click=lambda: State.handle_upload(
#               rx.upload_files()
#            ),
#        ),
#        rx.button(
#            "Clear",
#            on_click=rx.clear_selected_files,
#        ),
#        rx.foreach(
#            State.img, lambda img: rx.image(src=img)
#        ),
#        padding="5em",
#    )

def index() -> rx.Component:
    return rx.container(
        action_bar(),
        chat(),
    )

app = rx.App()
app.add_page(index)
app.compile()


# The Nature of Code
# Daniel Shiffman
# http://natureofcode.com

# A basic implementation of John Conway's Game of Life CA

import pygame
import random as rnd
import os.path

width = 640
height = 360
window = pygame.display.set_mode((width,height))
window.fill((0,0,0))  # Fills the screen with black

w = 8

class GOL:
    def __init__(self, r=None):
        global w
        self.w = w
        self.columns = int(width/self.w)
        self.rows = int(height/self.w)
        self.board = [[rnd.randint(0,2) for x in range(1, self.rows - 1)] for y in range(1, self.columns - 1)]

    def generate(self):
        nextGen = [[0 for x in range(1, self.rows - 1)] for y in range(1, self.columns - 1)]
        for x in range(1, self.columns - 3):
            for y in range(1, self.rows - 3):
                # Add up all the states in a 3x3 surrounding grid
                neighbors = 0
                for i in range(-1, 2):
                    for j in range(-1, 2):
                        neighbors += self.board[x + i][y + j]

                # A little trick to subtract the current cell's state since
                # we added it in the above loop
                neighbors -= self.board[x][y]

                # Rules of Life
                if ((self.board[x][y] == 1) and (neighbors < 2)): # Loneliness
                    nextGen[x][y] = 0
                elif ((self.board[x][y] == 1) and (neighbors > 3)): # Overpopulation
                    nextGen[x][y] = 0
                elif ((self.board[x][y] == 0) and (neighbors == 3)): # Reproduction
                    nextGen[x][y] = 1
                else:                                           # Stasis
                    nextGen[x][y] = self.board[x][y]

        self.board = nextGen # Next is now our board

    def display(self):
        window.fill((0,0,0))  # Fills the screen with black
        for i in range(0, self.columns - 3):
            for j in range(0, self.rows - 3):
                if(self.board[i][j] == 1):
                    pygame.draw.rect(window, pygame.Color('white'), (i * self.w, j * self.w, self.w, self.w))
                    #pygame.draw.rect(window, (255,255,255), (i * self.w, j * self.w, self.w, self.w))

        pygame.display.update()

gol = GOL()

def draw():
    global window
    global gol

    gol.generate()
    gol.display()

    # reset board when mouse is pressed
    for event in pygame.event.get():
        if event.type == pygame.MOUSEBUTTONDOWN:
            window.fill((0,0,0))  # Fills the screen with black
            gol = GOL()

while (True):
    draw()


import numpy as np
import pickle
import sys
import os
import RBFN as RBFN
from sklearn.ensemble import RandomForestRegressor
from sklearn import linear_model
from sklearn.neighbors import KNeighborsRegressor
from sklearn import svm
# import logging
# logging.basicConfig(filename='/home/naamah/Documents/CatES/result_All/GECCO/svm_25/logger/logger.log', filemode='a',level=logging.INFO)
import smac_KNN, smac_svm, smac_RF
import time
#import pyKriging
#from pyKriging.krige import kriging

# from pyKriging_master import pyKriging
# from pyKriging_master.pyKriging.krige import kriging

def surrogateRF(Xarchive, Farchive, X,file_loc,file_loc_general,toUpdate, first_iter=False,problem='LABS',index=1):
    Xnew=Xarchive.T
    X_pred=X.T
    SMAC = False

    if SMAC:
        with open("/home/naamah/Documents/CatES/result_All/smac/RF/X1.p", "wb") as fp:
            pickle.dump(Xnew, fp)
        with open("/home/naamah/Documents/CatES/result_All/smac/RF/F1.p", "wb") as fp:
            pickle.dump(Farchive, fp)

        anf=smac_RF.main_loop(problem)

        print("SMAC {}".format(anf))
        sys.exit("Error message")

    clf = RandomForestRegressor(criterion="mse",n_estimators=49, min_samples_leaf=1, min_samples_split=2,
                                    min_weight_fraction_leaf=0.0001060554, max_leaf_nodes=1000,
                                    min_impurity_decrease=0.0,min_impurity_split=None, warm_start=False, max_depth=None,
                                    max_features="auto",random_state=7
                                    ) #RF_9111

    # if problem=="LABS":
    #
    #     clf = RandomForestRegressor(criterion="mse",n_estimators=49, min_samples_leaf=1, min_samples_split=2,
    #                                 min_weight_fraction_leaf=0.0001060554, max_leaf_nodes=1000,
    #                                 min_impurity_decrease=0.0,min_impurity_split=None, warm_start=False, max_depth=None,
    #                                 max_features="auto",random_state=7
    #                                 ) #RF_9111
    #
    #
    # elif problem=="NKL":
    #     clf = RandomForestRegressor(criterion="mse",n_estimators=43, min_samples_leaf=1, min_samples_split=4,
    #                                 min_weight_fraction_leaf=0.0005238657425634613, max_leaf_nodes=673, min_impurity_decrease=0.0,
    #                                 warm_start=True, max_depth=None, max_features="auto", random_state=8
    #                                 ) #RF_1_sig
    #
    #
    #
    # else: #problem=="QAP"
    #     clf = RandomForestRegressor(criterion="mse",n_estimators=38, min_samples_leaf=1, min_samples_split=2,
    #                                 min_weight_fraction_leaf=0.0002313685, max_leaf_nodes=551, min_impurity_decrease=2.86E-08,
    #                                 warm_start=False, max_depth=None, max_features="auto", random_state=None
    #                                 )#RF_9333

    if not os.path.exists(file_loc_general + "/surrogate_configuration"):
        with open(file_loc_general + "/surrogate_configuration", 'a') as file:
            file.write("clf:\n{}\n\nTuning Algorithem: {} ".format(clf.get_params(), "smac"))
        file.close()

    clf.fit(Xnew, Farchive)
    F_pred = clf.predict(X_pred)

    return F_pred

def surrogateKNN(Xarchive, Farchive, X,file_loc,file_loc_general,toUpdate, first_iter=False,problem='LABS'):
    Xnew=Xarchive.T
    X_pred=X.T
    SMAC = False

    if SMAC:
        with open("/home/naamah/Documents/CatES/result_All/X1.p", "wb") as fp:
            pickle.dump(Xnew, fp)
        with open("/home/naamah/Documents/CatES/result_All/F1.p", "wb") as fp:
            pickle.dump(Farchive, fp)

        anf=smac_KNN.main_loop(problem)

        print("SMAC {}".format(anf))
        sys.exit("Error message")

    neigh = KNeighborsRegressor(n_neighbors=9,  algorithm="ball_tree", p=1, weights="distance",leaf_size=60) # KNN_LABS_444

    # if problem=="LABS":
    #     neigh = KNeighborsRegressor(n_neighbors=9,  algorithm="ball_tree", p=1, weights="distance",leaf_size=60) # KNN_LABS_444
    #
    #
    #
    # elif problem=="NKL":
    #     neigh = KNeighborsRegressor(n_neighbors=10, algorithm="brute", p=1, weights="distance",leaf_size=76) # KNN_NKL_4442
    #
    #
    #
    # else: #problem=="QAP"
    #     neigh = KNeighborsRegressor(n_neighbors=8,  algorithm="auto", p=1, weights="distance",leaf_size=19) # KNN_QAP_4446

    if not os.path.exists(file_loc_general + "/surrogate_configuration"):
        with open(file_loc_general + "/surrogate_configuration", 'a') as file:
            file.write("clf:\n{}\n\nTuning Algorithem: {} ".format(neigh.get_params(), "smac"))
        file.close()

    neigh.fit(Xnew, Farchive)
    F_pred = neigh.predict(X_pred)

    return F_pred

def surrogateSVM(Xarchive, Farchive, X,file_loc,file_loc_general,toUpdate, first_iter=True,problem='LABS'):
    Xnew=Xarchive.T
    X_pred=X.T
    SMAC = False

    if SMAC:
        # smac - no conda
        with open("/home/naamah/Documents/CatES/result_All/smac/svm/X1.p", "wb") as fp:
            pickle.dump(Xnew, fp)
        with open("/home/naamah/Documents/CatES/result_All/smac/svm/F1.p", "wb") as fp:
            pickle.dump(Farchive, fp)

        ans=smac_svm.main_loop(problem)

        print("SMAC {}".format(ans))
        sys.exit("Error message")

        ans_clf=ans.get_dictionary()
        C=ans_clf['C']
        epsilon=ans_clf['epsilon']
        kernel=ans_clf['kernel']
        shrinking =True
        if ans_clf['shrinking']=='false':
            shrinking=False
        if ans_clf['gamma']=='value':
            gamma_value = ans_clf['gamma']
        else:
            gamma_value='auto'

    # time1=time.time()
    # clf = svm.SVR(kernel='rbf', C=307)
    if problem == "MIS":
        # clf = svm.SVR(kernel='rbf', C=307, epsilon=0.0023, gamma=0.1223 ,shrinking=False,degree=3) #IRACE_SURROGATE SVM_NKL_3331
        clf = svm.SVR(kernel='rbf', C=307, epsilon=0.5, gamma=0.1223 ,shrinking=False,degree=3) #IRACE_SURROGATE SVM_NKL_3331

    else:
        # clf = svm.SVR(kernel='rbf', C=237.689013261001, epsilon=0.00125697429061751, gamma=0.0445112387400382,shrinking=False)  # BYSMAC_9111
        clf = svm.SVR(kernel='rbf', C=237.689013261001, epsilon=0.5, gamma=0.0445112387400382,shrinking=False)  # BYSMAC_9111

    # if problem=="LABS":
    #     # clf = svm.SVR(kernel='rbf', C=307, epsilon=0.0023, gamma=0.1223 ,shrinking=False,degree=3) #R
    #     clf = svm.SVR(kernel='rbf', C=237.689013261001, epsilon=0.00125697429061751, gamma=0.0445112387400382 ,shrinking=False) #BYSMAC_9111
    #
    #
    # elif problem=="NKL":
    #     clf = svm.SVR(kernel='poly', C=999.7180904447123, epsilon=0.0014989615126237686, gamma=1,shrinking=True) #BYSMAC_9111
    #
    # else: #"QAP"
    #     clf = svm.SVR(kernel='rbf', C=307, epsilon=0.0023, gamma=0.1223 ,shrinking=False,degree=3) #IRACE_SURROGATE SVM_NKL_3331

    # time2=time.time()

    if not os.path.exists(file_loc_general + "/surrogate_configuration"):
        with open(file_loc_general + "/surrogate_configuration", 'a') as file:
            file.write("clf:\n{}\n\nTuning Algorithem: {} ".format(clf.get_params(), "smac"))
        file.close()
    clf.fit(Xnew, Farchive)
    # time3=time.time()
    F_pred = clf.predict(X_pred)
    # time4=time.time()
    # logging.warning("\nModel creation: {}  Training: {}  Prediction:  {} \n\n".format(time2-time1, time3-time2, time4-time3))
    return F_pred

def surrogateLM(Xarchive, Farchive, X, toUpdate, test=False, problem='LABS'):
    Fnew = np.asarray(Farchive).reshape((len(Farchive), 1))
    Xnew = Xarchive.T
    X_pred = X.T

    reg = linear_model.LinearRegression()
    reg.fit(Xnew, Fnew)

    F_pred = reg.predict(X_pred)
    F_pred = np.squeeze(F_pred, axis=1)

    return F_pred

def surrogateRBFN (Xarchive, Farchive, X,file_loc,file_loc_general,toUpdate, first_iter=False,problem='LABS',isKmeans=True):
    # http://www.rueckstiess.net/research/snippets/show/72d2363e
    Xnew=Xarchive.T
    X_pred=X.T
    Xnew_unique, idx_unique = np.unique(Xnew, axis=0, return_index=True)
    F_unique =Farchive[idx_unique]
    numCenter= int(len(F_unique)*0.75)

    if (toUpdate):
        Rbfn=RBFN.RBF(len(Xnew[0]),numCenter,len(Xnew[0]))
        Rbfn.train(Xnew_unique,F_unique,isKmeans)
        with open(file_loc+"/Rbfn.p", "wb") as fp:
            pickle.dump(Rbfn, fp)
    else:
        with open(file_loc+"/Rbfn.p", "rb") as fp:
            Rbfn = pickle.load(fp)

    F_pred=Rbfn.test(X_pred)
    return F_pred

def surrogateKriging (Xarchive, Farchive, X, file_loc,file_loc_general,toUpdate, first_iter=False,problem='LABS'):
    X_pred=X.T
    F_pred=[]

    # update or build the model
    if (toUpdate):
        Xnew = Xarchive.T
        Xnew_unique, idx_unique = np.unique(Xnew, axis=0, return_index=True)
        F_unique = Farchive[idx_unique]

        if first_iter: #build the model

            testfun = pyKriging.testfunctions().stybtang

            # Now that we have our initial data, we can create an instance of a Kriging model
            k = kriging(Xnew_unique, F_unique, testfunction=testfun, name='simple')
            k.train(optimizer='ga')

        else: #update the model
            with open(file_loc+"kriging.p", "rb") as fp:
                k = pickle.load(fp)
            for point,y_point in zip(Xnew_unique,F_unique):
                k.addPoint(point,y_point)

        #save the new/updates model
        with open(file_loc+"kriging.p", "wb") as fp:
            pickle.dump(k, fp)

    # no new points to update the model
    # only predicts new points.
    else:
        with open(file_loc+"kriging.p","rb") as fp:
            k = pickle.load(fp)

    for x in X_pred:
        F_pred.append(k.predict(x))

    F_pred=np.asarray(F_pred)
    return F_pred

#
# X= np.asarray([[1,0,1,0,1,0],[4,5,6],[7,8,9],[10,11,12],[13,14,15],[13,14,15]], dtype=float)
# y= np.asarray([6,15,24,36,42], dtype=float)
# x_test=np.asarray([[1,2,3],[13,14,16],[16,11,8],[16,12,8],[2,3,4],[13,14,15]],dtype=float)
# X= np.asarray([[1,0,1,0,1,0],[1,1,0,0,1,1],[0,0,1,1,1,1],[0,1,1,0,0,0],[0,0,1,0,0,0],[1,1,1,1,1,1],[1,1,0,0,1,1]
#                ,[1,1,1,1,0,1],[0,1,1,1,1,1],[0,0,0,1,0,0]], dtype=float)
# y= np.asarray([3,4,4,2,1,6,4,5,5,1], dtype=float)
# x_test=np.asarray([[1,1,1,0,0,0],[1,1,1,1,1,1],[0,0,0,0,0,0],[0,1,0,1,0,1],[1,0,1,0,1,0]],dtype=float)
# X=X.T
# x_test=x_test.T
# #
# X=np.loadtxt('/home/naamah/Documents/CatES/data/X')
# y=np.loadtxt('/home/naamah/Documents/CatES/data/F')
# x_test=np.loadtxt('/home/naamah/Documents/CatES/data/X_test')
# f_test=np.loadtxt('/home/naamah/Documents/CatES/data/F_test')
# X=X[0:12,:]
# x_test=x_test[0:12,:]
# # x_test=X
# # f_test=y
#
# ans=surrogateKriging(X,y,x_test,file_loc="/home/naamah/Documents/CatES/result_All/result_test/exp_num_1", toUpdate=True,first_iter=True)
# ans2=np.linalg.norm(f_test-ans)
#
# print("the ans for the surrogate is:\n {}\n".format(ans))
# print("the reall fittnes is: \n{}\n".format(f_test))
# print("the diff between the fittnes and the surrogate is:\n {}\n".format(f_test-ans))
# print("the euclidean diff between the fittnes and the surrogate is: {}\n".format(ans2))


from django.conf.urls import url, include
from . import views
from rest_framework.routers import DefaultRouter

router = DefaultRouter()
router.register(r'users', views.UserViewSet)
router.register(r'tasks', views.NewTaskViewSet)
router.register(r'projects', views.ProjectViewSet)

urlpatterns = [
    url(r'^', include(router.urls))
]


import pygame
import os
from pygame.math import Vector2

class Player:
    def __init__(self):
        self.animations = {}
        self.initialize_animation_frames("./assets/dwarf/")

        self.frame = 0
        self.direction = "right"
        self.animation = "wait" + "_" + self.direction
        # animations_forced are here to loop entirely through these ones before moving to a next one
        self.animations_forced = ["hattack"]
        self.can_change_next_animation = True
        self.sprite = self.animations[self.animation][self.frame]
        self.sprite_rect = self.sprite.get_rect()
        self.speed = 4

    def initialize_animation_frames(self, main_path):
        # this method initialize all animations from the main_path by itself
        # given that main_path only contains one folder per animation
        # MAIN PATH /
        #       FOLDER_1 /
        #           FRAME_1
        #           FRAME_2
        #           FRAME_3
        #       FOLDER_2 /
        #           FRAME_1
        #           FRAME_2
        animation_subdirectories = os.walk(main_path)
        for name, sub_directories, files in filter(
            lambda sub_dir: sub_dir[0] != main_path, animation_subdirectories
        ):
            # we have to take name of the directory, excluding all path from it
            # ./assets/dwarf/death, we only want death
            # split all the path
            path_names = name.split("/")
            # and get the last element
            animation_type = path_names[len(path_names) - 1]

            for index, animation_file in enumerate(sorted(files)):
                if index == 0:
                    self.animations.update(
                        {
                            animation_type: [
                                pygame.image.load("{}/{}".format(name, animation_file))
                            ]
                        }
                    )
                    continue
                self.animations[animation_type].append(
                    pygame.image.load("{}/{}".format(name, animation_file))
                )

    def animate(self, animation):
        if self.can_change_next_animation:
            # will animate player with given animation
            match_previous_animation = (
                self.animation == animation + "_" + self.direction
            )
            # if the animation changes from previous frame, we will reset frame
            self.frame = self.frame if match_previous_animation else 0

            self.animation = animation + "_" + self.direction

        if animation in self.animations_forced:
            self.can_change_next_animation = False

        animation_sprites = self.animations[self.animation]
        # modulo, to always have frame index of animation_sprites, even if self.frame is way higher than the len of animation_sprites
        self.sprite = animation_sprites[int(self.frame) % len(animation_sprites)]
        # animation is rendered in sprite, we will now update frame to next animation
        # by updating of 0.5, we will slow down animation
        # 0.5 means animation will need 2 frames rendered by pygame to pass to the next frame
        self.frame += 0.5
        # if we have finished the loop for the forced_animation, we can change the next animation
        if (
            int(self.frame) == len(self.animations[self.animation])
            and not self.can_change_next_animation
        ):
            self.frame = 0
            self.can_change_next_animation = True

    def move(self, direction, camera):
        self.direction = "left" if direction.x < 0 else "right"
        # camera follow player
        # so we move player as we moved the camera
        camera_movement = camera.update(direction)
        self.sprite_rect = self.sprite_rect.move(camera_movement)

    def input(self, camera=None):
        # get state of all keys (pressed or not)
        keys = pygame.key.get_pressed()

        vector_direction = Vector2(0, 0)

        up = keys[pygame.K_UP]
        down = keys[pygame.K_DOWN]
        left = keys[pygame.K_LEFT]
        right = keys[pygame.K_RIGHT]
        space = keys[pygame.K_SPACE]

        if up:
            vector_direction.y -= self.speed
            self.move(vector_direction, camera)
        elif down:
            vector_direction.y += self.speed
            self.move(vector_direction, camera)
        elif left:
            vector_direction.x -= self.speed
            self.move(vector_direction, camera)
        elif right:
            vector_direction.x += self.speed
            self.move(vector_direction, camera)

        # can_change_next_animation is here to avoid register too many animation when spamming escape
        if space and self.can_change_next_animation:
            self.animate("hattack")
        elif up or down or left or right:
            self.animate("run")
        else:
            self.animate("wait")


#!/usr/bin/env python3.10
import subprocess
import os
from PIL import Image
import re
import numpy as np
from datetime import datetime
from astropy.io import fits
import sys
import argparse

fitcropX=1.0
fitcropY=1.0
speedPara=1.0
downSample=2 ##default downSample=2
depthStar='11-20'
dt=datetime.now()

def solvCoord(myImage, mydepth, mydownsample):
    depthStar=mydepth
    dt=datetime.now()
    ##run solve-field again
    cmd = f'solve-field --no-plot --scale-low 0.1 --downsample {mydownsample} --depth {mydepth} --overwrite  {myImage}' #resized_img_filename
    output = subprocess.check_output(cmd, shell=True, universal_newlines=True)
    print(output)
    dt2=datetime.now()-dt
    print ("Platesolve time:", dt2)
    # Parse the output to get the RA and Dec coordinates
    ra = None
    dec = None
    for line in output.split('\n'):

        if line.startswith('simplexy:'):
            starno=line.split('simplexy: found ')[1].split(' sources.')
            print('simplyxy:',starno,' stars')
        if line.startswith('Field center: (RA,Dec) = ('):
           # ra, dec = re.findall(r"[-+]?(?:\d*\.*\d+)",(line.split(') = (')[1].split(',')))
            ra,dec=line.split(' (RA,Dec) = (')[1].split(',')
            #radd,decdd=line.split(' D:M:S) = (')[1].split(',')
            #radecnp =np.array( re.findall(r"[-+]?(?:\d*\.*\d+)",radec))
            print(ra,dec)
            ra = float(ra)
            dec = float(dec[:-6])
            #print (radd,decdd)
            #radd=radd
            #decdd=decdd[:-2]
        if line.startswith('Field size: '):
           fieldInfo = line.split('Field size: ')[1].split()
           print(fieldInfo)
           #fieldX=float(fieldInfo[0])
           #fieldY=float(fieldInfo[2])

           arcUnit=fieldInfo[3]
           #translate unit to degrees
           if arcUnit=='degrees':
              fieldX=float(fieldInfo[0])
              fieldY=float(fieldInfo[2])
           elif arcUnit[:7]=='arcminu':
              fieldX=float(fieldInfo[0])/60
              fieldY=float(fieldInfo[2])/60
           else:
              fieldX=float(fieldInfo[0])/3600
              fieldY=float(fieldInfo[2])/3600

        if line.startswith('Field rotation angle: '):
           rotateA = line.split(' up is ')[1].split()
           #rotateA=float(rotateA)
           print('rot', rotateA)
           break
    return (ra, dec, rotateA, fieldX, fieldY, starno)

def rawTotiff_resize(myImage, resizeRateX, resizeRateY):
    # Convert the raw image file to FITS format using dcraw
    dt=datetime.now()
    outfitfile=myImage[:-4]+'.fit'
    #os.system(f"rawtran -c R -o {outfitfile} {myImage}")
    os.system(f"dcraw -4 -T -h  {myImage}")
    print("dcraw img conv to tiff:",datetime.now()-dt)
    # Get the name of the FITS file that was created by dcraw
    #fits_image_file = os.path.splitext(myImage)[0] + '.fit'
    fits_image_file = os.path.splitext(myImage)[0] + '.tiff'
    print("t img:",datetime.now()-dt)
    image = Image.open(fits_image_file)
    width,height = image.size
    if (resizeRateX is not None or  resizeRateY is not None):
      if float(resizeRateX) > 1.0 or float(resizeRateX)< 0.10 or  float(resizeRateY) > 1.0 or float(resizeRateY) < 0.10  :
         resizeRateX=1.0
         resizeRateY=1.0
    else:
      resizeRateX=0.25 ##default resizeset to 0.25
      resizeRateY=0.25
    resized_img=image.resize((int(width*resizeRateX),int(height*resizeRateY)))
    resized_img_filename=os.path.splitext(myImage)[0] + '_q.jpg'
    resized_img.save(resized_img_filename)
    print("q img:",datetime.now()-dt,resized_img_filename)
    #resized_img3=img.resize((int(width/3),int(height/3)))
    #resized_img3_filename=os.path.splitext(myImage)[0] + '_s.jpg'
    #resized_img3.save(resized_img3_filename)
    outImage=resized_img_filename
    #image = fits.getdata(fits_image_file)
    return (outImage)

##for fits crop to reduce image size and solve-field speed.
def fits_crop(myImage,cropX,cropY):
    ##directly fits translate and crop into 800x800 pixels
    image = fits.getdata(myImage)

    height, width = image.shape
    ##crop fits if crop <1.0
    if cropX is not None or cropY is not None:
     if float(cropX) >1.0 or float(cropY)<0.1:
        cropX=1.0
        cropY=1.0
    #default crop=0.5
    else:
    	cropX=0.5
    	cropY=0.5
    # Calculate the coordinates of the center of the image
    center_x = width // 2
    center_y = height // 2
    fitcropXw=int(cropX*width)
    fitcropYh=int(cropY*height)
    # Calculate the coordinates of the top-left corner of the crop
    crop_x = center_x - fitcropXw // 2
    crop_y = center_y - fitcropYh // 2
    # Crop the image
    cropped_image = image[crop_y:crop_y+fitcropXw, crop_x:crop_x+fitcropYh]
    # Save the cropped image to a new file
    fitcrop2_name = myImage[:-4]+'_crop'  +myImage[-4:]  ##'.tiff'
    print ('\n',datetime.now()-dt,'\t',fitcrop2_name ,'\t',fitcropXw,'x',fitcropYh, ' pixels fits image cropped')
    fits.writeto(fitcrop2_name, cropped_image,overwrite=True)
    outImage=fitcrop2_name
    return (outImage)

#main arg parser for input parameters
def solvefield_arg_parse():
    ##input args
    #if len(sys.argv)<2:
    #    print("Usage python3 solveLocal2.py <input image (fit/fits/cr2/nef/arw)> for plate-    solve")
    #    sys.exit(1)
    parser = argparse.ArgumentParser(description='input the image, crop ratio and speed parameter')
    #group = parser.add_mutually_exclusive_group()
    parser.add_argument('-c', '--crop', type=float, help='Resize(.raw)/Crop(.fits) factor 0.02(0.1)-1.0 to make sure pixels ~ 800 pixels for shorter solve-field time.(default=1.0)')
    parser.add_argument('-z', '--downsample', type=int, help='downsample parameter in solve-field (defualt=2)')
    parser.add_argument('-d', '--depth', type=str, help='depth sequence of stars in solve-field (default=11-20)')
    #group.add_argument('-s', '--speed', type=int, help='speed for solve-field **test only**')
    parser.add_argument('imagefile', type=str, help='input image filename cr2/nef/fit')
    args = parser.parse_args()
    if args.imagefile =='':
        print("Usage python3 solveLocal6.py -c 1.0 -z 2 <input image (fit/fits/cr2/nef/arw)> for plate-solve.")
        sys.exit(1)
    else:
       raw_image_file=args.imagefile
    if args.crop is not None:
      if float(args.crop) <= 1.0 and float(args.crop)>=0.02:
        fitcropX=float(args.crop)  ##set the crop ration
        fitcropY=float(args.crop)
    else:  ##default crop=1.0
      fitcropX=1.0
      fitcropY=1.0
    #elif args.downsample is not None :
    if args.downsample is not None:
      if int(args.downsample)>1:
        downSample=int(args.downsample)
        print('downSample',downSample)
      else:
        downSample=1
    else:
      downSample=2
      print(args.crop,args.downsample)
   #elif args.speed > 1:
   #    speedPara=args.speed
    if args.depth is not None:
      depthStar=args.depth
    else:
      depthStar='11-20'
      ##default nu,ber is 11th to 20th bright stars
    return(raw_image_file, downSample, fitcropX,fitcropY, depthStar)

def main():
    dt=datetime.now()
    #input arguments truncation loop
    raw_image_file, downSample, fitcropX, fitcropY, depthStar = solvefield_arg_parse()

    ##

    # Path to the raw image file
    #raw_image_file = args.imagefile
    verifyRaw=raw_image_file.lower()[-4:]

    print(verifyRaw, downSample, fitcropX, fitcropY, depthStar)
    if verifyRaw == '.cr2' or verifyRaw == '.nef' or verifyRaw=='.arw':
      ##dslr image resize routine it will slower 7-8sec than fits file for convertion.
      outImage=rawTotiff_resize(raw_image_file,fitcropX,fitcropY)

    else:
      ##fits image croped routine
      outImage=fits_crop(raw_image_file,fitcropX,fitcropY)

    dt1=datetime.now()-dt
    print ("image translate", dt1,outImage)
    #depthStar='11-20'
    ra, dec, rotateA, fieldX, fieldY, starno = solvCoord(outImage, depthStar, downSample)

    if ra is not None and dec is not None:
        print(f'RA: {ra}, Dec: {dec}')
        print('Field:', fieldX, 'x', fieldY,'degs')
        print('Rotation angle:', rotateA[0])
        #print(f'RA: {radd}, Dec: {decdd}')
    else:
        print('Plate-solving failed.')
    print('star in croped img:',starno[0])

    if starno in (10, 20) : ##smaller 20 star
        depthStar='2-11'
        ra, dec, rotateA, fieldX, fieldY, starno = solvCoord(outImage, depthStar, downSample)

    if ra is not None and dec is not None:
        print(f'RA: {ra}, Dec: {dec}')
        print('Field:', fieldX, 'x', fieldY,'degs')
        print('Rotation angle:', rotateA[0])
        #print(f'RA: {radd}, Dec: {decdd}')
    else:
        print('Plate-solving failed.')
    print('star a croped img:',starno[0])
    ##printfinal proess time
    dt2=datetime.now()-dt
    print ("Platesolve time:", dt2)

if __name__ == "__main__":
    main()


from _csv import register_dialect, QUOTE_ALL
import csv
import codecs
import cStringIO
from HTMLParser import HTMLParser
from itertools import groupby
from django.conf import settings

def currency_converter(amount, year):
    return amount * settings.OPENAID_CURRENCY_CONVERSIONS[year]

def eur_to_usd_converter(amount, year):
    return amount / settings.OPENAID_CURRENCY_CONVERSIONS[year]

def sanitize_get_param(param_type, param, default, top=None, length=None):
    # simple sanitize of GET parameter and converting to param_type
    # if fails: return default value
    # if top is defined and the value is > than top: return default
    if not param:
        return default
    if length:
        if len(param) > length:
            param = param[:length]
    try:
        value = param_type(param)
    except ValueError:
        return default
    else:
        if top:
            if value > top:
                return default
            else:
                return value
        return value

class UTF8Recoder:
    """
    Iterator that reads an encoded stream and reencodes the input to UTF-8
    """

    def __init__(self, f, encoding):
        self.reader = codecs.getreader(encoding)(f)

    def __iter__(self):
        return self

    def next(self):
        return self.reader.next().encode("utf-8")

class UnicodeDictReader:
    """
    A CSV reader which will iterate over lines in the CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        f = UTF8Recoder(f, encoding)
        self.reader = csv.DictReader(f, dialect=dialect, **kwds)

    def next(self):
        row = self.reader.next()
        return dict((k, unicode(s, "utf-8")) for k, s in row.iteritems() if s is not None)

    def __iter__(self):
        return self

class UnicodeDictWriter(object):
    def __init__(self, f, fieldnames, dialect=csv.excel, encoding="utf-8", **kwds):
        # Redirect output to a queue
        self.queue = cStringIO.StringIO()
        self.writer = csv.DictWriter(self.queue, fieldnames, dialect=dialect, **kwds)
        self.stream = f
        self.encoder = codecs.getincrementalencoder(encoding)()

    def writerow(self, D):
        try:
            self.writer.writerow(dict([(k,v.encode("utf-8")) for k,v in D.items()]))
        except UnicodeEncodeError:
            import pprint
            pprint.pprint(D['geography'])
            return
        # Fetch UTF-8 output from the queue ...
        data = self.queue.getvalue()
        data = data.decode("utf-8")
        # ... and reencode it into the target encoding
        data = self.encoder.encode(data)
        # write to the target stream
        self.stream.write(data)
        # empty queue
        self.queue.truncate(0)

    def writerows(self, rows):
        for D in rows:
            self.writerow(D)

class ExcelSemicolon(csv.excel):
    """Extends excel Dialect in order to set semicolon as delimiter."""
    delimiter = ';'
    quoting = QUOTE_ALL

register_dialect("excel_semicolon", ExcelSemicolon)

class UnicodeWriter:
    """
    A CSV writer which will write rows to CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        # Redirect output to a queue
        self.queue = cStringIO.StringIO()
        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
        self.stream = f
        self.encoder = codecs.getincrementalencoder(encoding)()

    def writerow(self, row):
        self.writer.writerow([s.encode("utf-8") for s in row])
        # Fetch UTF-8 output from the queue ...
        data = self.queue.getvalue()
        data = data.decode("utf-8")
        # ... and reencode it into the target encoding
        data = self.encoder.encode(data)
        # write to the target stream
        self.stream.write(data)
        # empty queue
        self.queue.truncate(0)

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)

class MLStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self.fed = []

    def handle_data(self, d):
        self.fed.append(d)

    def get_data(self):
        return ' '.join(self.fed)

def strip_tags(html):
    s = MLStripper()
    s.feed(html)
    return s.get_data()

# converts a qset into a dict using the key field
def convert2dict(queryset, key_field):
    if type(key_field) == tuple or type(key_field) == list:
        if len(key_field) == 0:
            raise Exception
        else:
            indice_keygen = lambda x: tuple(x[key] for key in key_field)

    else:
        indice_keygen = lambda x: x[key_field]

    return dict((k, list(v)) for k, v in groupby(queryset, key=indice_keygen))


import networkx as nx
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np
import json
import os
from io import BytesIO
from typing import List
import openai
import PyPDF2
from langchain.embeddings import OpenAIEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
import chainlit as cl

openai.api_key = ""
openai_api_key = ""

text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)

system_template = """Use the following pieces of context to answer the users question.
If you don't know the answer, just say that you don't know, don't try to make up an answer.
ALWAYS return a "SOURCES" part in your answer.
The "SOURCES" part should be a reference to the source of the document from which you got your answer.

And if the user greets with greetings like Hi, hello, How are you, etc reply accordingly as well.

Example of your response should be:

The answer is foo
SOURCES: xyz

Begin!
----------------
{summaries}"""

@cl.on_chat_start
async def on_chat_start():
    files = None
    # Wait for the user to upload a file
    while files == None:
        files = await cl.AskFileMessage(
            content="Please upload a text file to begin!",
            accept=["application/pdf", "text/plain"],
            max_size_mb=100,
            timeout=180,
            max_files=5
        ).send()

    chunks = []
    for file in files:
        _, ext = os.path.splitext(file.name)
        print(f"File type: {ext}")

        msg = cl.Message(
                content=f"Processing `{file.name}`...", disable_human_feedback=True
            )
        await msg.send()

        if ext == ".txt":
            tex = file.content.decode("utf-8")
            chunks.extend(text_splitter.split_text(text=tex))
        elif ext == ".pdf":
            print("entered pdf elif")
            pdf_file = PyPDF2.PdfReader(BytesIO(file.content))

            # Get the total number of pages in the PDF file.
            num_pages = len(pdf_file.pages)

            # Create a list to store the decoded text from each page.
            decoded_text = []

            # Decode each page of the PDF file and add the decoded text to the list.
            for i in range(num_pages):
                page = pdf_file.pages[i]
                decoded_text.append(page.extract_text())

            for text in decoded_text:
                chunks.extend(text_splitter.split_text(text))

    metadatas = [{"source": f"{i}-pl"} for i in range(len(chunks))]

    embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key, openai_model="text-embedding-ada-002")

    documents_embedded = embeddings.embed_documents(chunks)

    G = nx.Graph()
    if os.path.exists('graph.json'):
        with open("graph.json", "r") as f:
            data = json.load(f)
            G = data

    for idx, (text, embedding, source) in enumerate(zip(chunks, documents_embedded, metadatas)):
        G.add_node(idx, text=text, embedding=embedding, source=source)
    for idx1, data1 in G.nodes(data=True):
        for idx2, data2 in G.nodes(data=True):
            if idx1 != idx2:
                similarity = cosine_similarity([data1['embedding']], [data2['embedding']])[0][0]
                if similarity > 0.75:
                    G.add_edge(idx1, idx2, weight=similarity)
    data = nx.node_link_data(G)
    with open("graph.json", "w") as f:
        json.dump(data, f)

    msg.content = f"Processing `{file.name}` done. You can now ask questions!"
    await msg.update()

    cl.user_session.set("Graph", G)

@cl.on_message
async def main(message: cl.Message):
    graph = cl.user_session.get("Graph")  # type: ConversationalRetrievalChain

    embeddings = OpenAIEmbeddings(openai_api_key=openai_api_key, openai_model="text-embedding-ada-002")
    userembed = embeddings.embed_documents(message.content)
    resentry, resneighbors = query_graph(userembed, graph)
    neighbor_texts = [neighbor['text'] for neighbor in resneighbors]

    # Joining neighbor texts into a single string with newline separator
    neighbor_texts_str = '\n'.join(neighbor_texts)

    # Updating the prompt to include all neighbor texts
    prompt = f"{system_template}\n\nContext: {resentry['text']}\n{neighbor_texts_str}\n\nUser: {message.content}\n"
    print(prompt)
    messages = [
        {"role": "system", "content": "You will use the context in the user input to generate factual output about new information"}
        ]
    finalprompt = {"role": "user", "content": prompt}
    messages.append(finalprompt)
    completion = openai.ChatCompletion.create(
        model = "gpt-4",
        messages = messages,
        max_tokens=1024,
        n=1,
        stop=None,
        temperature=0.5,
    )
    result = completion["choices"][0]["message"]
    answer = result['content']
    source_documents =[]    # type: List[Document]
    source_documents.append(resentry['source'])
    for neighbor in resneighbors:
        source_documents.append(neighbor['source'])
    text_elements = []  # type: List[cl.Text]

    if source_documents:
        for source_idx, source_doc in enumerate(source_documents):
            source_name = f"source_{source_idx}"
            text_elements.append(
                cl.Text(content=source_doc['source'], name=source_name)
            )
        source_names = [text_el.name for text_el in text_elements]
        if source_names:
            answer += f"\nSources: {', '.join(source_names)}"
        else:
            answer += "\nNo sources found"
    await cl.Message(content=answer, elements=text_elements).send()

def query_graph(query_embedding, G):
    # Ensure query_embedding is a numpy array
    query_embedding = np.array(query_embedding).flatten()

    # Determine the smallest dimension among all embeddings
    min_dimension = min(
        [len(data['embedding']) for idx, data in G.nodes(data=True)],
        default=len(query_embedding)
    )

    # Slice each embedding and the query_embedding to have the same size
    similarities = [
        (
            idx,
            cosine_similarity(
                [np.array(data['embedding']).flatten()[:min_dimension]],
                [query_embedding[:min_dimension]]
            )[0][0]
        )
        for idx, data in G.nodes(data=True)
    ]
    entry_point_node_idx, _ = max(similarities, key=lambda x: x[1])

    # Gather nodes that are one relation away
    neighbors = list(G.neighbors(entry_point_node_idx))

    # Return the entry point node and its neighbors
    return G.nodes[entry_point_node_idx], [G.nodes[neighbor_idx] for neighbor_idx in neighbors]


import collections
import copy
from typing import Any, Mapping, Optional
from ..components.structures import ComponentSpec, ComponentReference
from ..components._components import _default_component_name, _resolve_command_line_and_paths
from ..components._naming import _sanitize_python_function_name, generate_unique_name_conversion_table
from .. import dsl

def _create_container_op_from_component_and_arguments(
    component_spec: ComponentSpec,
    arguments: Mapping[str, Any],
    component_ref: Optional[ComponentReference] = None,
) -> 'dsl.ContainerOp':
    # Check types of the reference arguments and serialize PipelineParams
    arguments = arguments.copy()
    for input_name, argument_value in arguments.items():
        if isinstance(argument_value, dsl.PipelineParam):
            input_type = component_spec._inputs_dict[input_name].type
            reference_type = argument_value.param_type
            dsl.types.verify_type_compatibility(reference_type, input_type, 'Incompatible argument passed to the input "{}" of component "{}": '.format(input_name, component_spec.name))

            arguments[input_name] = str(argument_value)
        if isinstance(argument_value, dsl.ContainerOp):
            raise TypeError('ContainerOp object was passed to component as an input argument. Pass a single output instead.')

    resolved_cmd = _resolve_command_line_and_paths(
        component_spec=component_spec,
        arguments=arguments,
    )

    container_spec = component_spec.implementation.container

    old_warn_value = dsl.ContainerOp._DISABLE_REUSABLE_COMPONENT_WARNING
    dsl.ContainerOp._DISABLE_REUSABLE_COMPONENT_WARNING = True

    # Merge output_paths and output_uris to get the file_outputs.
    file_outputs = collections.OrderedDict(resolved_cmd.output_paths or {})
    for name, output_uri in resolved_cmd.output_uris.items():
        file_outputs[name] = output_uri

    artifact_argument_paths = [
        dsl.InputArgumentPath(
            argument=arguments[input_name],
            input=input_name,
            path=path,
        )
        for input_name, path in resolved_cmd.input_paths.items()
    ]

    for input_name, input_uri in resolved_cmd.input_uris.items():
        artifact_argument_paths.append(
            dsl.InputArgumentPath(
                argument=arguments[input_name],
                input=input_name,
                path=input_uri
            ))

    task = dsl.ContainerOp(
        name=component_spec.name or _default_component_name,
        image=container_spec.image,
        command=resolved_cmd.command,
        arguments=resolved_cmd.args,
        file_outputs=file_outputs,
        artifact_argument_paths=artifact_argument_paths,
    )
    dsl.ContainerOp._DISABLE_REUSABLE_COMPONENT_WARNING = old_warn_value

    component_meta = copy.copy(component_spec)
    task._set_metadata(component_meta)
    component_ref_without_spec = copy.copy(component_ref)
    component_ref_without_spec.spec = None
    task._component_ref = component_ref_without_spec

    task._parameter_arguments = resolved_cmd.inputs_consumed_by_value

    # Previously, ContainerOp had strict requirements for the output names, so we had to
    # convert all the names before passing them to the ContainerOp constructor.
    # Outputs with non-pythonic names could not be accessed using their original names.
    # Now ContainerOp supports any output names, so we're now using the original output names.
    # However to support legacy pipelines, we're also adding output references with pythonic names.
    # TODO: Add warning when people use the legacy output names.
    output_names = [output_spec.name for output_spec in component_spec.outputs or []] # Stabilizing the ordering
    output_name_to_python = generate_unique_name_conversion_table(output_names, _sanitize_python_function_name)
    for output_name in output_names:
        pythonic_output_name = output_name_to_python[output_name]
        # Note: Some component outputs are currently missing from task.outputs (e.g. MLPipeline UI Metadata)
        if pythonic_output_name not in task.outputs and output_name in task.outputs:
            task.outputs[pythonic_output_name] = task.outputs[output_name]

    if container_spec.env:
        from kubernetes import client as k8s_client
        for name, value in container_spec.env.items():
            task.container.add_env_variable(k8s_client.V1EnvVar(name=name, value=value))

    if component_spec.metadata:
        annotations = component_spec.metadata.annotations or {}
        for key, value in annotations.items():
            task.add_pod_annotation(key, value)
        for key, value in (component_spec.metadata.labels or {}).items():
            task.add_pod_label(key, value)
        # Disabling the caching for the volatile components by default
        if annotations.get('volatile_component', 'false') == 'true':
            task.execution_options.caching_strategy.max_cache_staleness = 'P0D'

    return task


#!/usr/bin/env python
import asyncio
import websockets

async def hello():
    async with websockets.connect("ws://localhost:8001") as websocket:
        speed = input("Enter the speed : ")
        height = input("Enter the height : ")
        while speed != str(-1) and height != str(-1):
            await websocket.send((speed,height))
            msg = await websocket.recv()
            print(msg)
            speed = input("Enter the speed : ")
            height = input("Enter the height : ")
        await websocket.send("close")
        print("client CLOSED")

asyncio.run(hello())


import scipy.cluster.hierarchy as hcluster
from sklearn.cluster import DBSCAN
import numpy as np
import pandas as pd

def get_cluster_id(clusters,route_id):
    """
        Returns the cluster_id of a given route_id
    """
    for i in range(len(clusters)):
        try:
            clusters[i].loc[route_id]
            return i
        except:
            continue

def calculate_route_similarity(routeA,routeB):
    """
        calculates the similarity between two routes
    """
    return len(list(set(routeA).intersection(routeB))) / len(list(dict.fromkeys(routeA + routeB)))

def clustering_by_similarity(routes):
    """
        NOT IMPLEMENTED YET
        clustering of routes by the factor of similarity between them
    """
    clusters = np.zeros(shape=(len(routes.items()),len(routes.items())))
    for key,route in routes.items():
        for subkey,subroute in routes.items():
            if(key>subkey):
                clusters[key][subkey]=SUMO.calculate_route_similarity(route,subroute)
            elif(key==subkey):
                clusters[key][subkey]=1
    print("implemented only to generate similarity matrix")

def clustering_by_location(coords):
    """
        Clustering points by location
    """
    kms_per_radian = 6371.0088
    # by setting epsilon, we set max distance that points can be from each other in a cluster
    # 0.2 km in this example
    epsilon = 0.2 / kms_per_radian
    db = DBSCAN(eps=epsilon, min_samples=1, algorithm='ball_tree', metric='haversine').fit(np.radians(coords))
    cluster_labels = db.labels_
    num_clusters = len(set(cluster_labels))
    clusters = pd.Series([coords[cluster_labels == n] for n in range(num_clusters)])
    return clusters


from __future__ import annotations

from typing import Callable, List, Optional, Union
from copy import deepcopy

import torch

class LossWeightTree:
    def __init__(
        self, name: str,
        weight: Union[float, Callable[[int], float]],
        children: Optional[List[LossWeightTree]],
    ) -> None:
        self.name = name
        self.weight = weight
        self.children = children

    def __repr__(self):
        # if self.children is None:
        #     return f'{self.weight}_{self.name}'
        # else:
        #     return f'''{self.weight}x({"+".join([
        #         repr(x) for x in self.children
        #     ])})'''
        return f'<LossWeightTree {self.name} weight={repr(self.weight)}>'

    def __getitem__(self, key):
        for child in self.children:
            if child.name == key:
                return child
        raise KeyError(f'{key} not found.')

    def __setitem__(self, key, value):
        if isinstance(value, __class__):
            assert value.name == key
            for child in self.children:
                if child.name == key:
                    raise ValueError(f'{key} already exists.')
            self.children.append(value)
        else:
            raise TypeError(f'{__class__.__name__} can only setitem {__class__.__name__}.')

    def __contains__(self, key):
        return key in [x.name for x in self.children]

    def getWeight(self, epoch: int):
        try:
            return self.weight(epoch)
        except TypeError:
            return self.weight

    def to(self, device):
        if self.children is None:
            children = None
        else:
            children = [x.to(device) for x in self.children]
        return __class__(
            self.name,
            torch.tensor(self.weight, device=device),
            children,
        )

    def print(self, depth=0):
        print(' ' * depth, self.name, ': ', self.weight, sep='')
        for child in self.children or []:
            child.print(depth + 1)

    def __deepcopy__(self, memo):
        if self.children is None:
            children = None
        else:
            children = [deepcopy(x, memo) for x in self.children]
        return __class__(self.name, self.weight, children)

    def errorAboutWeight(*_):
        raise TypeError('You are probably forgetting `.weight`')
    def __init_class__(cls):
        for operator in (
            '__bool__', '__eq__', '__ne__',
            '__lt__', '__le__',
            '__gt__', '__ge__',
        ):
            setattr(cls, operator, cls.errorAboutWeight)

LossWeightTree.__init_class__(LossWeightTree)


# numeric imports
import numpy as np
import torch

# typing imports
from typing import List, Tuple, Dict

def get_theta_and_V(
    daily_excess_return_series : np.ndarray,
    daily_risk_free_series : np.ndarray,
    signal : np.ndarray,
    initial_cash : float = 200_000.,
    leverage : float = 5.) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
    """
    Evaluates a mean reversion strategy given a signal and returns the theta, V_asset, V_cap and V_total

    Arguments:
    ----------
    daily_excess_return_series  : {np.ndarray}
                                    > The daily excess return series
    daily_risk_free_series      : {np.ndarray}
                                    > The daily risk free rate series
    signal                      : {np.ndarray}
                                    > The signal series
    initial_cash                : {float}
                                    > The initial cash
    leverage                    : {float}
                                    > The leverage

    Returns:
    ----------
    theta                       : {np.ndarray}
                                    > The dollar values (either long or short)
    V_asset                     : {np.ndarray}
                                    > The unlevereaged value of the assets.
    V_cap                       : {np.ndarray}
                                    > The value of the unused capital.
    V_total                     : {np.ndarray}
                                    > The total value of the holdings.
    """
    # ge let n_days
    n_days = len(signal)

    # set up
    theta = np.zeros((n_days,))

    # assets
    V_t = np.zeros((n_days,))
    delta_V_t = np.zeros((n_days,))

    # money market
    V_cap = np.zeros((n_days,))
    delta_V_cap = np.zeros((n_days,))

    # total
    V_total = np.zeros((n_days,))
    delta_V_total = np.zeros((n_days,))

    # # initial values
    V_t[0] = initial_cash
    delta_V_t[0] = 0
    V_cap[0] = 0
    delta_V_cap[0] = 0
    V_total[0] = V_t[0] + delta_V_cap[0]
    delta_V_total[0] = 0

    # backtest strategy
    for i, s in enumerate(signal[:-1]):

        if i == 0:
            continue

       # get position size
        # theta[i] = V_t[i-1] * leverage * s
        theta[i] = V_total[i-1] * leverage * s

        # get V_t
        delta_V_t[i] = (daily_excess_return_series[i+1] * theta[i])
        V_t[i] = delta_V_t[i] + V_t[i-1]

        # get V_cap_t
        M = np.abs(theta[i]) / leverage
        delta_V_cap[i] = (V_total[i-1] - M) * (daily_risk_free_series[i+1])
        V_cap[i] = delta_V_cap[i] + V_cap[i-1]

        # Get V_total_t
        delta_V_total[i] = delta_V_t[i] + delta_V_cap[i]
        V_total[i] = delta_V_total[i] + V_total[i-1]

    strategy_data = {
        "theta" : theta,
        "V_t" : V_t,
        "delta_V_t" : delta_V_t,
        "V_cap" : V_cap,
        "delta_V_cap" : delta_V_cap,
        "V_total" : V_total,
        "delta_V_total" : delta_V_total}

    return strategy_data

def get_turnover_dollars(
    theta : np.ndarray,
    ) -> float:
    """
    Calculates the turnover in dollars traded over time

    Arguments:
    ----------
        theta       : {np.ndarray}
                        > The dollar values invested

    Returns:
    ----------
        turnover    : {float}
                       > The turnover in dollars traded over time
    """
    theta_delta = np.diff(theta)
    turnover = np.sum(np.abs(theta_delta))
    return turnover

def get_turnover_units(
    theta : np.ndarray,
    price_series : np.ndarray
        ) -> float:
    """
    Calculates the turnover in units traded over time

    Arguments:
    ----------
        theta           : {np.ndarray}
                            > The dollar values invested
        price_series    : {np.ndarray}
                            > The price series

    Returns:
    ----------
        turnover        : {float}
                            > The turnover in units traded over time
    """
    theta_over_price = theta / price_series
    turnover = np.sum(np.abs(np.diff(theta_over_price)))
    return turnover

def get_daily_trading_pnl(
    unleveraged_asset_holdings : np.ndarray,
    ) -> np.ndarray:
    """
    Calculates the daily trading PnL for the strategy.

    Arguments:
    ----------
    unleveraged_asset_holdings  : {np.ndarray}
                                    > The daily holdings of the asset.

    Returns:
    --------
    daily_trading_pnl           : {np.ndarray}
                                    > The daily trading PnL.
    """
    # calculate the daily trading pnl
    daily_trading_pnl = np.diff(unleveraged_asset_holdings)
    return daily_trading_pnl

def get_daily_money_market_capital_account(
    money_market_capital_account : np.ndarray,
    ) -> np.ndarray:
    """
    Calculates the daily money market capital account for unused cash for the the strategy.

    Arguments:
    ----------
    money_market_capital_account        : {np.ndarray}
                                            > The daily money market capital account for unused cash.

    Returns:
    ----------
    daily_money_market_capital_account  : {np.ndarray}
                                            > The daily money market capital account for unused cash.
    """
    # calculate the daily money market capital account
    daily_money_market_capital_account = np.diff(money_market_capital_account)
    return daily_money_market_capital_account

def get_daily_total_holdings(
    total_holdings : np.ndarray,
    ) -> np.ndarray:
    """
    Calculates the daily total holdings for the strategy.

    Arguments:
    ----------
    total_holdings      : {np.ndarray}
                            > The daily total holdings for the strategy.

    Returns:
    ----------
    daily_total_holdings : {np.ndarray}
                            > The daily total holdings for the strategy.
    """
    # calculate the daily total holdings
    daily_total_holdings = np.diff(total_holdings)
    return daily_total_holdings