from enum import Enum
import math
import random
import torch
import torch.nn as nn
import xxx  # you own distributed module

class SlimBlockType(Enum):
    FAT = 1
    JOINT = 2
    MUSCLE = 3
    @classmethod
    def type2bn(cls, type):
        map_dict = {
            cls.FAT: "no_bn",
            cls.JOINT: "run_bn",
            cls.MUSCLE: "post_bn"
        }
        return map_dict[type]

    @classmethod
    def value2name(cls, value):
        if value not in cls.__members__.values():
            raise ValueError
        else:
            return cls.FAT

class SlimBNType(Enum):
    NORMAL_BN = 1
    RUN_NORM_BN = 2
    NO_BN = 3
    @classmethod
    def value2name(cls, value):
        if value not in cls.__members__.values():
            raise ValueError
        else:
            for k, v in cls.__members__.items():
                if v == value:
                    return k

class SlimBlock(object):
    def __init__(self, op, block_type):
        self.op = op
        self.block_type = block_type

    def __repr__(self):
        return '{}'.format(self.op)

class SlimBN2d(nn.Module):
    def __init__(self, *args, bn_type=SlimBNType.NORMAL_BN, **kwargs):
        super(SlimBN2d, self).__init__()
        self.bn = nn.BatchNorm2d(*args, **kwargs)
        self.bn_type = bn_type
        self.bn_func_dict = {
            SlimBNType.NORMAL_BN: self._normal_bn,
            SlimBNType.NO_BN: self._no_bn,
            SlimBNType.RUN_NORM_BN: self._run_norm_bn
        }
        self.bn_func = self.bn_func_dict[bn_type]

    def _normal_bn(self, x):
        return self.bn(x)

    def _no_bn(self, x):
        return x

    def _run_norm_bn(self, x):
        epsilon = 1e-8
        mean = x.mean(dim=[0, 2, 3], keepdim=True)
        std = x.std(dim=[0, 2, 3], keepdim=True)
        return (x - mean) / (std + epsilon)

    def forward(self, x):
        return self.bn_func(x)

    def __repr__(self):
        return 'SlimBN2d({}, bn_type={})'.format(self.bn, SlimBNType.value2name(self.bn_type))

class IDepthSlimTrainProbGenerator(object):
    def _valid_check(self, x):
        return x >= 0 and x <= 1

    def _step(self):
        raise NotImplementedError

    def step(self):
        x = self._step()
        if self._valid_check(x):
            return x
        else:
            raise ValueError('invalid prob value: {}'.format(x))

class ConstantProbGenerator(IDepthSlimTrainProbGenerator):
    def __init__(self, init_val):
        self.init_val = init_val

    # override
    def _step(self):
        return self.init_val

class EGreedyProbGenerator(IDepthSlimTrainProbGenerator):
    def __init__(self, min_val=0, max_val=1, decay_coeff=50):
        assert(max_val > min_val)
        self.min_val = min_val
        self.max_val = max_val
        self.decay_coeff = decay_coeff
        self.count = 0

    def _scale(self, x):
        return (self.max_val - self.min_val) * x + self.min_val

    def _step(self):
        self.count += 1
        x = 1 * math.exp(-1. * self.count / self.decay_coeff)
        return self._scale(x)

class DepthSlimTrainStrategy(object):
    def __init__(self, necessary, option_num, option_block_num, option_prob_func=ConstantProbGenerator(1.0)):
        N = int(math.pow(2, option_block_num))
        if necessary == 'sandwich':
            self.necessary = {0, N-1}
        elif necessary == 'max':
            self.necessary = {N-1}
        elif necessary == 'min':
            self.necessary = {0}
        else:
            raise ValueError
        self.option_block_num = option_block_num
        self.option_prob_func = option_prob_func

        assert(option_num <= N - len(self.necessary))
        self.option_set = {x for x in range(N)} - self.necessary
        self.option_num = option_num

    def _get_binary_code_tuple(self, code):
        binary_code = bin(code)[2:]
        binary_code = binary_code.zfill(self.option_block_num)
        code = [int(x) for x in list(binary_code)]
        code = tuple(code)
        return code

    def get_necessary(self):
        result = []
        for item in self.necessary:
            result.append(self._get_binary_code_tuple(item))
        return result

    def _option_step(self):
        self.option = random.sample(self.option_set, self.option_num)

    def get_option(self):
        prob = random.uniform(0, 1)
        prob = torch.Tensor([prob])
        xxx.broadcast(prob, 0)
        self.option_prob = self.option_prob_func.step()
        if prob.item() > self.option_prob:
            return []
        self._option_step()
        result = []
        for item in self.option:
            result.append(self._get_binary_code_tuple(item))
        return result

class DepthSlimStage(nn.Module):
    def __init__(self, blocks, train_strategy):
        super(DepthSlimStage, self).__init__()

        self.joints = []
        self.fats = nn.ModuleList()
        self.muscles = []
        for item in blocks:
            if item.block_type == SlimBlockType.FAT:
                self.fats.append(item.op)
            elif item.block_type == SlimBlockType.JOINT:
                self.joints.append(item.op)
            elif item.block_type == SlimBlockType.MUSCLE:
                self.muscles.append(item.op)
        self.train_strategy = train_strategy

        assert(len(self.joints) == 2)
        self.begin = self.joints[0]
        self.end = nn.Sequential(*self.muscles, self.joints[1])

    def forward(self, x):
        x = self.begin(x)
        x = self.end(x)
        return x

    def multi_transform(self, x):
        def binary_code_exe(func, code, x):
            assert(len(func) == len(code))
            for i in range(len(code)):
                if i == 0:
                    x = x
                else:
                    x = func[i](x)
            return x

        transform_x = {}
        necessary = self.train_strategy.get_necessary()
        option = self.train_strategy.get_option()
        execute = necessary + option
        for item in execute:
            transform_x['-'.join([str(t) for t in item])] = binary_code_exe(self.fats, item, x)
        return transform_x

    def posterior_bn(self):
        for m in self.modules():
            if isinstance(m, SlimBN2d):
                if m.bn_type == SlimBNType.RUN_NORM_BN:
                    m.bn_type = SlimBNType.NORMAL_BN
                    continue
            m.requires_grad = False
            m.eval()

    def train_forward(self, x):
        x = self.begin(x)
        transform_x = self.multi_transform(x)
        result_x = {}
        for k, x in transform_x.items():
            result_x[k] = self.end(x)

        return result_x

    def __repr__(self):
        return ('DepthSlimStage' + '(\n' +
               'joint_begin: {}\n'.format(str(self.joints[0])) +
               'fat: (\n{}\n)\n'.format('\n'.join([str(x) for x in self.fats])) +
               'muscle: (\n{}\n)\n'.format('\n'.join([str(x) for x in self.muscles])) +
               'joint_end: {}\n'.format(str(self.joints[1])) +
               ')')

class IDepthSlimModel(object):
    def train_mode(self):
        raise NotImplementedError

    def posterior_bn_mode(self):
        raise NotImplementedError

    def eval_mode(self):
        raise NotImplementedError


import string
import random
from Levenshtein import distance
import numpy as np

# Optimization Algorithm Genetic
# Set a Final Value : "vipulnath"
# Create initial Samples
# Number of elements in Population: 10
# Fitness levenstien distance
# CRossover + Mutation
def create_in_element(len_trgt):
    return "".join([random.choice(string.ascii_letters) for i in range(len_trgt)])
def select_fit(target, population, pop_cnt=10):
    fitness_dict = {indv:distance(target,indv) for indv in population}
    # sorted_dict = dict(sorted(fitness_dict.items()))
    sorted_dict = sorted(fitness_dict.items(), key=lambda x: x[1])
    return [i[0] for i in sorted_dict[0:pop_cnt]], sorted_dict[0]

def gene_mutate(gene_pool,mr=0.25):
    tl = len(gene_pool)
    ml = int(tl*mr)
    random.shuffle(gene_pool)
    mutd = gene_pool[0:ml].copy()
    for i in mutd:
        mu = random.choice(string.ascii_letters)
        ind = random.choice([n for n in range(len(i))])
        mi = i.replace(i[ind], mu)
        gene_pool.append(mi)
    return gene_pool

def crossover(population, mix_num):
    pop_dict = {i:[v[0:3],v[3:6],v[6:10]] for i,v in enumerate(population)}
    new_pop = []
    gene_pool =[]
    for i in pop_dict.values():
        for k in i:
            gene_pool.append(k)
    gene_pool = gene_mutate(gene_pool)
    for num_indv in range(0,mix_num):
        ndv_ls =""
        fst = np.random.choice(gene_pool)
        ndv_ls =ndv_ls+ fst
        fst = np.random.choice(gene_pool)
        ndv_ls = ndv_ls + fst
        fst = np.random.choice(gene_pool)
        ndv_ls = ndv_ls + fst

        new_pop.append(ndv_ls)

    return new_pop

generations= 400

target = "vipulnath"
intial_population = {create_in_element(len(target)) for i in range(0,15)}
print("Initial Population", intial_population)
selected, gen_best_fitness = select_fit(target, intial_population)
print("Generation : {}, Best Individual :{}, Best Fitness Score {}"
          .format(0, gen_best_fitness[0], gen_best_fitness[1]))
for gen in range(generations):
    # do crossover

    breeded = crossover(selected,500)
    selected, gen_best_fitness = select_fit(target, breeded)
    print("Generation : {}, Best Individual :{}, Best Fitness Score {}"
          .format(gen + 1, gen_best_fitness[0], gen_best_fitness[1]))
    if gen_best_fitness[1]==0:
        break


import logging
import pathlib
import typing

from ._exceptions import ConfigurationPathsError

log = logging.getLogger(__name__)

PIPELINE_CONFIG_FILES = {
    "clients.yml",
    "release_states.yml",
    "deployments.yml",
    "frames.yml",
    "puff_map.yml",
    "service_principals.vault",
    "subscriptions.yml",
    "systems.yml",
    "tenants.yml",
}

T = typing.TypeVar("T", bound="PipelineConfigurationPaths")

class PipelineConfigurationPaths:
    """Paths to pipeline configuration files."""

    clients: pathlib.Path
    context: typing.Set[pathlib.Path]
    deployments: pathlib.Path
    frames: pathlib.Path
    puff_map: pathlib.Path
    release_states: pathlib.Path
    service_principals: pathlib.Path
    static_secrets: typing.Set[pathlib.Path]
    subscriptions: pathlib.Path
    systems: pathlib.Path
    tenants: pathlib.Path

    CONFIG_SUBDIRS: typing.Set[str] = {"static_secrets", "context"}

    def __init__(self: T) -> None:
        """Construct ``PipelineConfigurationPaths`` object."""
        for this_file in PIPELINE_CONFIG_FILES:
            path = pathlib.Path(this_file)
            setattr(self, path.stem, path)

    @classmethod
    def from_dict(cls: typing.Type[T], data: dict) -> T:
        """
        Construct ``PipelineConfigurationPaths`` object.

        NOTE: Delivering valid paths is the users responsibility.

        Args:
            data: Dictionary of data to populate in object.
        """
        this_object = cls()
        for x, y in data.items():
            setattr(this_object, x, y)

        return this_object

    @classmethod
    def from_paths(
        cls: typing.Type[T],
        client_config: pathlib.Path,
        system_config: pathlib.Path,
    ) -> T:
        """
        Construct ``PipelineConfigurationPaths`` object.

        Args:
            client_config: Path to client configuration directory.
            system_config: Path to system configuration directory.

        Raises:
            ConfigurationPathsError: If any paths are duplicated between
                client and system.
        """
        this_object = cls()
        client_files = this_object.__acquire_client_files(client_config)
        system_files = this_object.__acquire_system_files(system_config)

        if len(client_files + system_files) > len(PIPELINE_CONFIG_FILES):
            # must be duplicate files between the directories
            log.debug("client files, {0}".format(str(client_files)))
            log.debug("system files, {0}".format(str(system_files)))
            raise ConfigurationPathsError(
                "Duplicate files between "
                "directories, {0}, {1}".format(client_config, system_config)
            )

        this_object.static_secrets = cls.__acquire_static_secrets(client_config)
        this_object.context = cls.__acquire_template_context(
            client_config, system_config
        )

        return this_object

    @staticmethod
    def __acquire_static_secrets(
        client_config: pathlib.Path,
    ) -> typing.Set[pathlib.Path]:
        secrets_path = client_config / "static_secrets"
        result = PipelineConfigurationPaths.__acquire_subdir_files(
            secrets_path, "static secrets"
        )
        return result

    @staticmethod
    def __acquire_template_context(
        client_config: pathlib.Path, system_config: pathlib.Path
    ) -> typing.Set[pathlib.Path]:
        # template context could be located in either client or system config.
        context_client_path = client_config / "context"
        client_context_files = (
            PipelineConfigurationPaths.__acquire_subdir_files(
                context_client_path, "client template context"
            )
        )
        context_system_path = system_config / "context"
        system_context_files = (
            PipelineConfigurationPaths.__acquire_subdir_files(
                context_system_path, "system template context"
            )
        )
        result = client_context_files.union(system_context_files)
        return result

    def __acquire_client_files(
        self: T, client_config: pathlib.Path
    ) -> typing.List[pathlib.Path]:
        client_files = list()
        for x in client_config.iterdir():
            if (
                x.is_file()
                and (x.name in PIPELINE_CONFIG_FILES)
                and (x.stem not in self.CONFIG_SUBDIRS)
            ):
                log.info("adding client configuration file, {0}".format(x))
                setattr(self, x.stem, x)
                client_files.append(x)

        return client_files

    def __acquire_system_files(
        self: T, system_config: pathlib.Path
    ) -> typing.List[pathlib.Path]:
        system_files = list()
        for x in system_config.iterdir():
            if x.is_file() and (x.name in PIPELINE_CONFIG_FILES):
                log.info("adding system configuration file, {0}".format(x))
                setattr(self, x.stem, x)
                system_files.append(x)

        return system_files

    @staticmethod
    def __acquire_subdir_files(
        this_path: pathlib.Path, category: str
    ) -> typing.Set[pathlib.Path]:
        result = set()
        if this_path.is_dir():
            log.debug(
                "{2} directory, {0}, {1}".format(
                    this_path, str(list(this_path.iterdir())), category
                )
            )
            for x in this_path.iterdir():
                if x.is_file():
                    log.info(
                        "adding file to configuration, {1}, {0}".format(
                            x, category
                        )
                    )
                    result.add(x)
        elif this_path.exists():
            log.debug(f"{category} not a directory, {this_path}")
        else:
            log.debug(f"{category} not present, {this_path}")

        return result


from django.conf.urls import url
from . import views

app_name='customer'

urlpatterns=[
        url(r'^$',views.index,name='index'),
        url(r'^transaction/init/$',views.maketransactions,name='maketransactions'),
        url(r'^viewtransactions/pending/$',views.viewpendingtransactions,name='viewpending'),
        url(r'^transaction/statements/$',views.transactionstatements,name='transactionstatements')
]


import logging
import random

from allennlp.data.iterators import BucketIterator
from allennlp.data.vocabulary import Vocabulary
from allennlp.training.trainer import Trainer
import torch

from data_readers.brown import BrownDatasetReader
from data_readers.linzen import LinzenLMDatasetReader
from data_readers.wsj import WSJDatasetReader
from stack_rnn_LM import StackRNNLanguageModel
from simple_rnn_LM import SimpleRNNLanguageModel

def main():
    # reader = BrownDatasetReader()
    # train_dataset = reader.read("data/brown.txt")
    # vocab = Vocabulary.from_instances(train_dataset)
    # dataset_name = "brown"

    reader = WSJDatasetReader(max_num=5000)
    train_dataset = reader.read("data/treebank_3/raw/wsj")
    vocab = Vocabulary.from_instances(train_dataset)
    dataset_name = "wsj"

    # reader = LinzenLMDatasetReader()
    # train_dataset = reader.read("StackNN/data/linzen/rnn_agr_simple/numpred.train")
    # validation_dataset = reader.read("StackNN/data/linzen/rnn_agr_simple/numpred.val")
    # vocab = Vocabulary.from_instances(train_dataset + validation_dataset)
    # dataset_name = "linzenlm"

    swap_push_pop = True

    # if torch.cuda.is_available():
    #     logging.info("Found and using CUDA.")
    #     device = 0
    # else:
    #     logging.info("No CUDA found.")
    #     device = -1

    # For some reason, this code runs MUCH slower on the GPU.
    device = -1

    model = StackRNNLanguageModel(vocab,
                                  rnn_dim=100,
                                  stack_dim=16,
                                  swap_push_pop=swap_push_pop,
                                  device=None if device == -1 else device)

    optimizer = torch.optim.Adam(model.parameters())
    iterator = BucketIterator(batch_size=16,
                              sorting_keys=[("sentence", "num_tokens")])
    iterator.index_with(vocab)

    if device == 0:
        # Best practice to call this after constructing optimizer.
        model.cuda(device)

    trainer = Trainer(model=model,
                      optimizer=optimizer,
                      iterator=iterator,
                      train_dataset=train_dataset,
                      num_epochs=5,
                      # validation_dataset=validation_dataset,
                      # patience=5
                      cuda_device=device)
    trainer.train()

    filename = dataset_name
    if isinstance(model, StackRNNLanguageModel):
        filename = "stack-" + filename
    if swap_push_pop:
        filename += "-swap"

    with open("saved_models/%s.th" % filename, "wb") as fh:
        torch.save(model.state_dict(), fh)
    vocab.save_to_files("saved_models/vocabulary-%s" % dataset_name)

if __name__ == "__main__":
    main()


import logging

from finance.financedb import FinanceDB

logger = logging.getLogger(__name__)

class DBTable:
    def __init__(self, db, table_name, keys, columns, order_by, **kwargs):
        self._db = db
        self._table_name = table_name
        self._keys = keys
        self._columns = columns
        self._order_by = order_by
        # These are columns that are automatically generated by the database
        self._auto_generate_columns = kwargs.get('auto_generate_columns')
        self._where = kwargs.get('where')
        self._key_values = None
        self._update_count = 0
        self._insert_count = 0

    def __iter__(self):
        """Fetch rows from the table."""
        cursor = self.select()

        for row in cursor:
            yield row

    @property
    def update_count(self):
        """The number of records updated in this session."""
        return self._update_count

    @property
    def insert_count(self):
        """The number of records inserted in this session."""
        return self._insert_count

    def select(self, **kwargs):
        """Fetch data from the table and return a cursor.

        Keyword arguments:
        columns -- subset of columns to fetch (default all columns)
        where -- where clause to filter the result
        order_by -- list of column names with optional direction to sort by
        """
        # keys = kwargs.get('keys', self._keys)
        columns = kwargs.get('columns', self._keys + self._columns)
        order_by = kwargs.get('order_by', self._order_by)
        where = kwargs.get('where', self._where)
        t = kwargs.get('t')

        sql = """
            SELECT {}
            FROM {}
            {}
            {}""".format(', '.join(columns),
                         self._table_name,
                         '' if where is None else 'WHERE {}'.format(where),
                         '' if order_by is None else 'ORDER BY {}'.format(', '.join(order_by)))

        c = self._db.conn.cursor()
        if t is None:
            cursor = c.execute(sql)
        else:
            cursor = c.execute(sql, t)

        return cursor

    def check_exists(self, row_dict):
        """Check the supplied row exists in the table based on the primary key."""
        if self._keys is not None and self._key_values is None:
            # build dictionary of unique column values for lookup to determine update or insert
            cursor = self.select(columns=self._keys)
            self._key_values = set([':'.join([str(c) for c in r]) for r in cursor])
            logger.debug('{} unique keys loaded.'.format(len(self._key_values)))
            for i, key in enumerate(self._key_values):
                if i > 3:
                    break
                logger.debug('Key: {}'.format(key))
        key = ':'.join([str(row_dict[c]) for c in self._keys])
        logger.debug('New key: {}'.format(key))

        return False if self._keys is None else key in self._key_values

    def save(self, row_dict):
        """Save the supplied row to the table."""
        c = self._db.conn.cursor()
        if self._keys is not None and self.check_exists(row_dict):
            update_columns = frozenset(self._columns)
            update_keys = frozenset(self._keys)
            if self._auto_generate_columns is not None:
                update_columns -= frozenset(self._auto_generate_columns)
                update_keys -= frozenset(self._auto_generate_columns)

            sql = """\
                UPDATE {}
                SET {}
                WHERE {}""".format(self._table_name,
                                   ', '.join(['{} = ?'.format(c) for c in update_columns if c in row_dict]),
                                   ' AND '.join(['{} = ?'.format(c) for c in update_keys]))
            t = tuple([row_dict.get(c) for c in list([c for c in update_columns if c in row_dict]) + list(update_keys)])
            self._update_count += 1
        else:
            insert_columns = frozenset(self._keys + self._columns)
            if self._auto_generate_columns is not None:
                insert_columns -= frozenset(self._auto_generate_columns)
            sql = """\
                INSERT INTO {} ({})
                VALUES ({})""".format(self._table_name,
                                      ', '.join(insert_columns),
                                      ', '.join(['?'] * len(insert_columns)))
            t = tuple([row_dict.get(c) for c in insert_columns])
            self._insert_count += 1
        # print(sql)
        # print(t)
        c.execute(sql, t)

class Tran(DBTable):
    def __init__(self, db):
        super().__init__(db, 'tran', ('bank', 'account', 'tran_date', 'description', 'amount'),
                         ('sequence', 'balance',
                          'file_name', 'file_sequence', 'calc_balance', 'head', 'source', 'notes'),
                         ('tran_date', 'sequence'),
                         auto_generate_columns=('sequence',))

        self._trans_accounts = None

    def get_accounts_with_transaction(self, row_dict):
        tran_key_columns = ('tran_date', 'description', 'amount')
        bank_account_columns = ('bank', 'account')
        if self._trans_accounts is None:
            # build dictionary of unique column values for lookup to determine update or insert
            cursor = self.select(columns=('bank', 'account', 'tran_date', 'description', 'amount'))
            self._trans_accounts = dict()
            for r in cursor:
                key = ':'.join([str(r[c]) for c in tran_key_columns])
                bank_account = ':'.join([r[c] for c in bank_account_columns])
                if key not in self._trans_accounts:
                    self._trans_accounts[key] = list()
                self._trans_accounts[key].append(bank_account)
            logger.debug('{} unique keys loaded.'.format(len(self._trans_accounts)))
            for i, key in enumerate(self._trans_accounts.keys()):
                if i > 3:
                    break
                logger.debug('Key: {}'.format(key))
        key = ':'.join([str(row_dict[c]) for c in tran_key_columns])
        logger.debug('New key: {}'.format(key))

        return self._trans_accounts[key] if key in self._trans_accounts else list()

def main():
    from finance import logutil
    #    logging.lastResort = None
    _ = logutil.setup_logging()

    db = FinanceDB()
    _ = db.connect('finance.db')

    tran = Tran(db)
    for i, row in enumerate(tran):
        print(dict(row))
        print([x for x in row])
        print(row[0])

        j = dict(row)

        del j['head']
        tran.save(j)
        j = dict(row)
        j['amount'] = 7.77
        tran.save(j)

        if i > 2:
            break
    tran.save(dict(tran_date='2018-02-23',
                   description='Test transaction',
                   bank='Test',
                   account='Test Account',
                   amount=500.25,
                   file_name='test',
                   file_sequence=7))
    db.disconnect()

if __name__ == '__main__':
    main()


from typing import List
import torch
from torch import nn
import numpy as np

def make_pad_mask(lengths, xs=None, length_dim=-1, maxlen=None):
    """Make mask tensor containing indices of padded part.

    Args:
        lengths (LongTensor or List): Batch of lengths (B,).
        xs (Tensor, optional): The reference tensor.
            If set, masks will be the same shape as this tensor.
        length_dim (int, optional): Dimension indicator of the above tensor.
            See the example.

    Returns:
        Tensor: Mask tensor containing indices of padded part.
                dtype=torch.uint8 in PyTorch 1.2-
                dtype=torch.bool in PyTorch 1.2+ (including 1.2)

    Examples:
        With only lengths.

        >>> lengths = [5, 3, 2]
        >>> make_pad_mask(lengths)
        masks = [[0, 0, 0, 0 ,0],
                 [0, 0, 0, 1, 1],
                 [0, 0, 1, 1, 1]]

        With the reference tensor.

        >>> xs = torch.zeros((3, 2, 4))
        >>> make_pad_mask(lengths, xs)
        tensor([[[0, 0, 0, 0],
                 [0, 0, 0, 0]],
                [[0, 0, 0, 1],
                 [0, 0, 0, 1]],
                [[0, 0, 1, 1],
                 [0, 0, 1, 1]]], dtype=torch.uint8)
        >>> xs = torch.zeros((3, 2, 6))
        >>> make_pad_mask(lengths, xs)
        tensor([[[0, 0, 0, 0, 0, 1],
                 [0, 0, 0, 0, 0, 1]],
                [[0, 0, 0, 1, 1, 1],
                 [0, 0, 0, 1, 1, 1]],
                [[0, 0, 1, 1, 1, 1],
                 [0, 0, 1, 1, 1, 1]]], dtype=torch.uint8)

        With the reference tensor and dimension indicator.

        >>> xs = torch.zeros((3, 6, 6))
        >>> make_pad_mask(lengths, xs, 1)
        tensor([[[0, 0, 0, 0, 0, 0],
                 [0, 0, 0, 0, 0, 0],
                 [0, 0, 0, 0, 0, 0],
                 [0, 0, 0, 0, 0, 0],
                 [0, 0, 0, 0, 0, 0],
                 [1, 1, 1, 1, 1, 1]],
                [[0, 0, 0, 0, 0, 0],
                 [0, 0, 0, 0, 0, 0],
                 [0, 0, 0, 0, 0, 0],
                 [1, 1, 1, 1, 1, 1],
                 [1, 1, 1, 1, 1, 1],
                 [1, 1, 1, 1, 1, 1]],
                [[0, 0, 0, 0, 0, 0],
                 [0, 0, 0, 0, 0, 0],
                 [1, 1, 1, 1, 1, 1],
                 [1, 1, 1, 1, 1, 1],
                 [1, 1, 1, 1, 1, 1],
                 [1, 1, 1, 1, 1, 1]]], dtype=torch.uint8)
        >>> make_pad_mask(lengths, xs, 2)
        tensor([[[0, 0, 0, 0, 0, 1],
                 [0, 0, 0, 0, 0, 1],
                 [0, 0, 0, 0, 0, 1],
                 [0, 0, 0, 0, 0, 1],
                 [0, 0, 0, 0, 0, 1],
                 [0, 0, 0, 0, 0, 1]],
                [[0, 0, 0, 1, 1, 1],
                 [0, 0, 0, 1, 1, 1],
                 [0, 0, 0, 1, 1, 1],
                 [0, 0, 0, 1, 1, 1],
                 [0, 0, 0, 1, 1, 1],
                 [0, 0, 0, 1, 1, 1]],
                [[0, 0, 1, 1, 1, 1],
                 [0, 0, 1, 1, 1, 1],
                 [0, 0, 1, 1, 1, 1],
                 [0, 0, 1, 1, 1, 1],
                 [0, 0, 1, 1, 1, 1],
                 [0, 0, 1, 1, 1, 1]]], dtype=torch.uint8)

    """
    if length_dim == 0:
        raise ValueError(f"length_dim cannot be 0: {length_dim}")

    if not isinstance(lengths, list):
        lengths = lengths.long().tolist()

    bs = int(len(lengths))
    if maxlen is None:
        if xs is None:
            maxlen = int(max(lengths))
        else:
            maxlen = xs.size(length_dim)
    else:
        assert xs is None
        assert maxlen >= int(max(lengths))

    seq_range = torch.arange(0, maxlen, dtype=torch.int64)
    seq_range_expand = seq_range.unsqueeze(0).expand(bs, maxlen)
    seq_length_expand = seq_range_expand.new(lengths).unsqueeze(-1)
    mask = seq_range_expand >= seq_length_expand

    if xs is not None:
        assert xs.size(0) == bs, (xs.size(0), bs)

        if length_dim < 0:
            length_dim = xs.dim() + length_dim
        # ind = (:, None, ..., None, :, , None, ..., None)
        ind = tuple(
            slice(None) if i in (0, length_dim) else None for i in range(xs.dim())
        )
        mask = mask[ind].expand_as(xs).to(xs.device)
    return mask

class BLIP_VQA(nn.Module):
    def __init__(self,
                 med_config='configs/med_config.json',
                 image_size=480,
                 cased=True,
                 vit='base',
                 vit_grad_ckpt=False,
                 vit_ckpt_layer=0,
                 multitask_qcate=True,
                 ):
        """
        Args:
            med_config (str): path for the mixture of encoder-decoder model's configuration file
            image_size (int): input image size
            vit (str): model size of vision transformer
        """
        from models.med import BertConfig, BertModel, BertLMHeadModel
        from models.blip import create_vit, init_tokenizer

        super().__init__()

        self.visual_encoder, vision_width = create_vit(vit, image_size, vit_grad_ckpt, vit_ckpt_layer,
                                                       drop_path_rate=0.1)
        self.tokenizer = init_tokenizer(cased)

        encoder_config = BertConfig.from_json_file(med_config)
        encoder_config.encoder_width = vision_width
        self.text_encoder = BertModel(config=encoder_config, add_pooling_layer=True)

        decoder_config = BertConfig.from_json_file(med_config)
        self.text_decoder = BertLMHeadModel(config=decoder_config)

        self.num_heads = encoder_config.num_attention_heads
        self.num_patches = self.visual_encoder.patch_embed.num_patches + 1
        self.multitask_qcate = multitask_qcate
        if multitask_qcate:
            # self.retr_ffn = nn.Linear(self.num_patches, 1)
            self.multitask_ffn = nn.Linear(encoder_config.hidden_size, 6)

    def encode_images(self, images: torch.Tensor, n_facts: List[int]):
        """
        :param images: (batch, n_facts, channel, H, W)
        :param n_facts: Number of image facts in each sample in the batch
        :return:
            - image_embeds: (batch, n_facts * seq_len, embed_size)
            - lengths: Valid lengths (dim 1) of image_embeds
        """
        batch_size, max_n_facts, C, H, W = images.shape
        images = images.view(-1, C, H, W)
        image_embeds = self.visual_encoder(images)  # (batch * n_facts, seq_len, embed_size)

        n_facts = [nf * image_embeds.size(1) for nf in n_facts]
        image_embeds = image_embeds.view(batch_size, max_n_facts * image_embeds.size(1), image_embeds.size(2))
        return image_embeds, n_facts

    def enable_med(self):
        tie_encoder_decoder_weights(self.text_encoder, self.text_decoder.bert, '', '/attention')

    def forward(
            self,
            image: torch.Tensor,
            captions: List[List[str]],
            question: List[str],
            answer: List[str],
            n_img_facts: List[int],
            train=True,
    ):
        """
        :param image: (batch, n_img_facts, channel, H, W)
        :param captions: Batch of list of captions
        :param question: Batch of questions
        :param answer: Batch of answers
        :param n_img_facts: Batch of number of image facts
        :param train: train or inference
        """

        image_embeds, lengths = self.encode_images(image, n_img_facts)
        image_atts = ~make_pad_mask(lengths, image_embeds[:, :, 0], 1).to(image.device)

        question = self.tokenizer(question, padding='longest', return_tensors="pt").to(image.device)
        question.input_ids[:, 0] = self.tokenizer.enc_token_id

        # concatenate captions and tokenize them
        captions = [f' {self.tokenizer.sep_token} '.join(cap) for cap in captions]
        captions = self.tokenizer(captions, padding='longest', return_tensors="pt").to(image.device)
        # mask the first token since we already have a sep_token_id set to the last token of question
        captions.input_ids[:, 0] = self.tokenizer.pad_token_id
        captions.attention_mask[:, 0] = 0

        # image-grounded text encoder
        input_ids = torch.cat([question.input_ids, captions.input_ids], dim=-1)
        attention_mask = torch.cat([question.attention_mask, captions.attention_mask], dim=-1)
        input_ids = input_ids[:, :512]
        attention_mask = attention_mask[:, :512]

        cross_attention_weight = torch.ones_like(input_ids, dtype=torch.float)
        cross_attention_weight[torch.as_tensor(n_img_facts, dtype=torch.long) == 0] = 0.0
        question_output = self.text_encoder(
            input_ids,
            attention_mask=attention_mask,
            encoder_hidden_states=image_embeds,
            encoder_attention_mask=image_atts,
            cross_attention_weight=cross_attention_weight,
            # output_attentions=True,
            return_dict=True,
        )

        # (batch, num_heads, question_len, image_embeds_len)
        multimodal_cross_atts = None
        if train:
            if self.multitask_qcate:
                # multimodal_cross_atts = question_output.cross_attentions[-1]  # last layer's cross attention
                # atts = torch.sum(multimodal_cross_atts, dim=2)  # (batch, num_heads, image_embeds_len)
                # atts = torch.sum(atts, dim=1)  # (batch, image_embeds_len)

                # # (batch, n_facts, num_patches)
                # atts = atts.view(atts.shape[0], -1, self.num_patches)
                # retr = self.retr_ffn(atts).squeeze(dim=-1)  # (batch, n_facts)

                mt_res = self.multitask_ffn(question_output.pooler_output)
            else:
                mt_res = None

            '''
            n: number of answers for each question
            weights: weight for each answer
            '''
            answer = self.tokenizer(answer, padding='longest', return_tensors="pt").to(image.device)
            answer.input_ids[:, 0] = self.tokenizer.bos_token_id
            answer_targets = answer.input_ids.masked_fill(answer.input_ids == self.tokenizer.pad_token_id, -100)

            answer_output = self.text_decoder(answer.input_ids,
                                              attention_mask=answer.attention_mask,
                                              encoder_hidden_states=question_output.last_hidden_state,
                                              encoder_attention_mask=attention_mask,
                                              labels=answer_targets,
                                              return_dict=True,
                                              reduction='none',
                                              )

            loss = answer_output.loss
            loss = loss.sum() / image.size(0)
            return loss, mt_res, multimodal_cross_atts
        else:
            num_beams = 10
            question_states = question_output.last_hidden_state.repeat_interleave(num_beams, dim=0)
            question_atts = attention_mask.repeat_interleave(num_beams, dim=0).to(question_states.device)

            bos_ids = torch.full((image.size(0), 1), fill_value=self.tokenizer.bos_token_id, device=image.device)

            outputs = self.text_decoder.generate(
                input_ids=bos_ids,
                max_length=50,
                min_length=1,
                num_beams=num_beams,
                eos_token_id=self.tokenizer.sep_token_id,
                pad_token_id=self.tokenizer.pad_token_id,
                encoder_hidden_states=question_states,
                encoder_attention_mask=question_atts,
            )

            answers = []
            for output in outputs:
                answer = self.tokenizer.decode(output, skip_special_tokens=True)
                answers.append(answer)
            return answers

def blip_vqa(pretrained='', **kwargs):
    from models.blip import load_checkpoint

    model = BLIP_VQA(**kwargs)
    if pretrained:
        model, msg = load_checkpoint(model, pretrained)
    return model

def tile(x, dim, n_tile):
    init_dim = x.size(dim)
    repeat_idx = [1] * x.dim()
    repeat_idx[dim] = n_tile
    x = x.repeat(*(repeat_idx))
    order_index = torch.LongTensor(np.concatenate([init_dim * np.arange(n_tile) + i for i in range(init_dim)]))
    return torch.index_select(x, dim, order_index.to(x.device))

def tie_encoder_decoder_weights(encoder: nn.Module, decoder: nn.Module, base_model_prefix: str, skip_key: str):
    """
    https://github.com/salesforce/BLIP/blob/main/models/blip_pretrain.py
    """
    uninitialized_encoder_weights: List[str] = []
    if decoder.__class__ != encoder.__class__:
        print(
            f"{decoder.__class__} and {encoder.__class__} are not equal. "
            f"In this case make sure that all encoder weights are correctly initialized."
        )

    def tie_encoder_to_decoder_recursively(
            decoder_pointer: nn.Module,
            encoder_pointer: nn.Module,
            module_name: str,
            uninitialized_encoder_weights: List[str],
            skip_key: str,
            depth=0,
    ):
        assert isinstance(decoder_pointer, nn.Module) and isinstance(
            encoder_pointer, nn.Module
        ), f"{decoder_pointer} and {encoder_pointer} have to be of type torch.nn.Module"
        if hasattr(decoder_pointer, "weight") and skip_key not in module_name:
            assert hasattr(encoder_pointer, "weight")
            encoder_pointer.weight = decoder_pointer.weight
            if hasattr(decoder_pointer, "bias"):
                assert hasattr(encoder_pointer, "bias")
                encoder_pointer.bias = decoder_pointer.bias
            print(module_name + ' is tied')
            return

        encoder_modules = encoder_pointer._modules
        decoder_modules = decoder_pointer._modules
        if len(decoder_modules) > 0:
            assert (
                    len(encoder_modules) > 0
            ), f"Encoder module {encoder_pointer} does not match decoder module {decoder_pointer}"

            all_encoder_weights = set([module_name + "/" + sub_name for sub_name in encoder_modules.keys()])
            encoder_layer_pos = 0
            for name, module in decoder_modules.items():
                if name.isdigit():
                    encoder_name = str(int(name) + encoder_layer_pos)
                    decoder_name = name
                    if not isinstance(decoder_modules[decoder_name], type(encoder_modules[encoder_name])) and len(
                            encoder_modules
                    ) != len(decoder_modules):
                        # this can happen if the name corresponds to the position in a list module list of layers
                        # in this case the decoder has added a cross-attention that the encoder does not have
                        # thus skip this step and subtract one layer pos from encoder
                        encoder_layer_pos -= 1
                        continue
                elif name not in encoder_modules:
                    continue
                elif depth > 500:
                    raise ValueError(
                        "Max depth of recursive function `tie_encoder_to_decoder` reached. "
                        "It seems that there is a circular dependency between two or more `nn.Modules` of your model."
                    )
                else:
                    decoder_name = encoder_name = name
                tie_encoder_to_decoder_recursively(
                    decoder_modules[decoder_name],
                    encoder_modules[encoder_name],
                    module_name + "/" + name,
                    uninitialized_encoder_weights,
                    skip_key,
                    depth=depth + 1,
                )
                all_encoder_weights.remove(module_name + "/" + encoder_name)

            uninitialized_encoder_weights += list(all_encoder_weights)

    # tie weights recursively
    tie_encoder_to_decoder_recursively(decoder, encoder, base_model_prefix, uninitialized_encoder_weights, skip_key)

{'BertConfig': 'models.med.BertConfig', 'BertModel': 'models.med.BertModel', 'BertLMHeadModel': 'models.med.BertLMHeadModel', 'create_vit': 'models.blip.create_vit', 'init_tokenizer': 'models.blip.init_tokenizer'}


from pandas import read_csv, DataFrame
from tqdm import tqdm
import numpy as np
import json

# Load our Data From the CSV File in this Directory
raw_df = read_csv('final/orignal-data/bank-additional-full.csv', delimiter=';')
# raw_df = read_csv('final/bank-additional-full.csv', delimiter=';')

# translation dataframe to be populated
translated_df = DataFrame()

# create an enumerated value for each new field we get
translation_field_bank = dict()

# custom months dictionary
translation_field_bank['month'] = {
    'jan': 1,
    'feb': 2,
    'mar': 3,
    'apr': 4,
    'may': 5,
    'jun': 6,
    'jul': 7,
    'aug': 8,
    'sep': 9,
    'oct': 10,
    'nov': 11,
    'dec': 12
}

translation_field_bank['day_of_week'] = {
    'sun': 1,
    'mon': 2,
    'tue': 3,
    'wed': 4,
    'thu': 5,
    'fri': 6,
    'sat': 7
}

# columns where we need to perform "translation"
translation_columns = [
    "job",
    "marital",
    "education",
    "default",
    "housing",
    "loan",
    "contact",
    "month",
    "day_of_week",
    "poutcome",
    "y"
]

binary_dict = {
    "no": 0,
    "yes": 1,
    "unknown": 2
}

print("Translating Data...")

# iterate through each column in the dataframe
for column_name in tqdm(raw_df.columns):

    new_column_values = np.zeros(len(raw_df), dtype=np.float64)

    if column_name == "emp.var.rate":
        print()

    if column_name not in translation_columns:

        # iterate through each row in the column and copy the row value. These are the default values.
        for index, row_value in enumerate(raw_df[column_name]):
            new_column_values[index] = row_value

        translated_df[column_name] = new_column_values
        continue

    elif column_name == "month":

        # iterate through each row and place the enumerated month instead of the string
        for index, row_value in enumerate(raw_df['month']):
            new_column_values[index] = translation_field_bank['month'][row_value]

        translated_df[column_name] = new_column_values
        continue

    elif column_name == "day_of_week":

        # iterate through each row and place the enumerated day instead of the string
        for index, row_value in enumerate(raw_df['day_of_week']):
            new_column_values[index] = translation_field_bank['day_of_week'][row_value]

        translated_df[column_name] = new_column_values
        continue

    # binary assignment
    if raw_df[column_name][0] == "yes" or raw_df[column_name][0] == "no":
        translation_field_bank[column_name] = binary_dict.copy()

    # create a new, unique dictionary that will populate this enumeration object.
    else:
        translation_field_bank[column_name] = dict()

    # enumeration field, starts at zero and increments by one as a uniuqe field is added to our bank.
    enumeration_idx: int = 0

    # iterate through each row in the column
    for index, row_value in enumerate(raw_df[column_name]):

        # if the row value doesn't exist, add it
        if row_value not in translation_field_bank[column_name].keys():
            translation_field_bank[column_name][row_value] = enumeration_idx
            enumeration_idx += 1

        new_column_values[index] = translation_field_bank[column_name][row_value]

    translated_df[column_name] = new_column_values

# rename target data
translated_df = translated_df.rename(columns={'y': 'target'})

# if you want to look at the dataframe
translated_df.to_csv("translated.df.csv")

with open("translation_field_bank.json", "w") as f:
    json.dump(translation_field_bank, f)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""Compute j tot as a function of theta for dm_halos project."""

from scipy import integrate
import numpy as np
import matplotlib.pyplot as plt
import h_funcs as h

g = {}
gsfunc, gpfunc, gdfunc, gsomfunc = [0, 0, 0, 0]

def jval():
    """H func computation."""
    h.g_import()
    # normalizations and f_values
    limit = 1000
    yval = np.logspace(-1, np.log10(50), num=20)
    jst = [integrate.quad(lambda x: x * h.hts(x), 0, y, limit=limit)[0] for y in yval]
    js = [integrate.quad(lambda x: x * h.hs(x), 0, y, limit=limit)[0] for y in yval]
    jp = [integrate.quad(lambda x: x * h.hp(x), 0, y, limit=limit)[0] for y in yval]
    jd = [integrate.quad(lambda x: x * h.hd(x), 0, y, limit=limit)[0] for y in yval]
    jsom = [integrate.quad(lambda x: x * h.hsom(x), 0, y, limit=limit)[0] for y in yval]

    with open("j_func_thetas.txt", 'wb') as outfile:
        np.savez(outfile, jst=np.array(jst), js=np.array(js), jp=np.array(
            jp), jd=np.array(jd), jsom=np.array(jsom), theta=np.array(yval))

if __name__ == '__main__':
    # jval()
    with np.load('j_func_thetas.txt', 'rb') as infile:
        theta = infile['theta']
        js = infile['js']
        jp = infile['jp']
        jd = infile['jd']
        jsom = infile['jsom']
        # plt.plot(theta, js)
        # plt.plot(theta, jp)
        # plt.plot(theta, jd)
    h.g_import()
    # yval = np.logspace(-1, 1, num=20)
    # plt.plot(theta, [h.hsom(y) for y in yval])
    # plt.show()
    print([(y, h.hsom(y)) for y in np.logspace(-4, 1, num=50)])
    print(integrate.quad(lambda x: x * h.hsom(x), 0, 0.1, limit=50))
    print(integrate.quad(lambda x: x * h.hsom(x), 0, 1, limit=50))


# ---------------------------------------------------------------------------------------------------------- #
# Author: Victor Cardel                                                                                #
# ---------------------------------------------------------------------------------------------------------- #
import tensorflow as tf
import sys
import numpy as np
import cv2
import os

# ---------------------------------------------------------------------------------------------------------- #
# Linha de comando:                                                                                #
#         python3 train_and_test.py /path/to/train_folder /path/to/test_folder /path/to/output.txt           #
# Descrição:                                                                                #
#         * carrega todas as imagens nos subdiretórios de /path/to/train_folder e cria modelo de             #
#           classificação                                                                                #
#         * carrega todas as imagens a serem classificadas no diretório /path/to/test_folder                 #
#         * para cada imagem de teste carregada, imprime uma linha no arquivo /path/to/output.txt com dois   #
#           valores separados por um espaço, sendo eles o nome do arquivo da imagem seguido do nome da sua   #
#           respectiva classe ('futebol', 'golfe', 'sinuca' ou 'tenis')                                      #
# ---------------------------------------------------------------------------------------------------------- #

def calcula_tamanho_dos_diretorios():
	global num_treino,num_teste

	for dir_treino in os.listdir(path):
		num_treino += len(os.listdir(path + '/'+ dir_treino))

	num_teste = len(os.listdir(sys.argv[2]))

def preenche_vetores_de_treino_e_teste():
	global treino,teste,labels_treino,tam_classes,num_classes

	k = 0
	diretorios = os.listdir(path)
	diretorios.sort()
	for classe,dir_treino in enumerate(diretorios):
		lista_imagens_classe = os.listdir(path + '/' + dir_treino)
		tam_classes.append(len(lista_imagens_classe))
		for img in lista_imagens_classe:
			imagem = cv2.imread(path + '/' + dir_treino + '/' + img,cv2.IMREAD_COLOR)
			treino[k] = imagem
			one_hot_encoding = np.zeros([num_classes])
			one_hot_encoding[classe] = 1
			labels_treino[k] = one_hot_encoding
			k = k + 1
	k = 0
	for img in os.listdir(sys.argv[2]):
		imagem = cv2.imread(str(sys.argv[2]) + '/' +  img,cv2.IMREAD_COLOR)
		teste[k] = imagem
		k = k + 1

def constroi_validacao():
	global labels_treino,tam_classes,validacao,treino,labels_validacao,labels_treino,treino_oficial,labels_treino_oficial
	#construindo validacao
	classe_atual = 0
	tam = 0
	classe_de_insercao = 0
	cont = 0
	v = 0
	j = 0
	for k in range(len(treino)):
		indice = np.where(labels_treino[k] == 1)
		if indice[0][0] != classe_atual:
			classe_atual = classe_atual + 1
		if cont < len(tam_classes) and tam > tam_classes[cont] - 1:
			classe_de_insercao = classe_de_insercao + 1
			cont = cont + 1
			tam = 0
		if classe_atual == classe_de_insercao:
			validacao[v] = treino[k]
			labels_validacao[v] = labels_treino[k]
			v = v + 1
			tam = tam + 1
		else:
			treino_oficial[j] = treino[k]
			labels_treino_oficial[j] = labels_treino[k]
			j = j + 1

def processa_base():
	global tam_classes
	preenche_vetores_de_treino_e_teste()
	#aplicando os pesos para cada classe
	tam_classes = map(lambda i: int(i*size_of_validation),tam_classes)
	tam_classes = list(tam_classes)
	constroi_validacao()

###############GLOBALS#################
path = sys.argv[1]
heigth = 64
width = 64
channels = 3
num_filters = 64
size_of_validation = 0.2
num_treino,num_teste = 0,0
num_classes = 4
num_features = heigth*width*channels
batch_size = 32
num_iteracoes_treino = 10000
lr = 0.001
#######################################
calcula_tamanho_dos_diretorios()
treino = np.empty([num_treino,heigth,width,channels], dtype=np.uint8)
teste = np.empty([num_teste,heigth,width,channels], dtype=np.uint8)
labels_treino = np.empty([num_treino,num_classes], dtype = np.uint8)
tam_validacao = int(size_of_validation*num_treino)
tam_treino_oficial = num_treino - tam_validacao
tam_classes = []
validacao = np.empty([tam_validacao,heigth,width,channels], dtype=np.uint8)
labels_validacao = np.empty([tam_validacao,num_classes],dtype=np.uint8)
treino_oficial = np.empty([tam_treino_oficial,heigth,width,channels], dtype=np.uint8)
labels_treino_oficial = np.empty([tam_treino_oficial,num_classes], dtype = np.uint8)
#######################################

processa_base()

#creating vector of labels
labels_treino_escalar = np.empty([tam_treino_oficial], dtype = np.uint8)
labels_val = np.empty([tam_validacao], dtype = np.uint8)

for index,one_hot in enumerate(labels_treino_oficial):
	for classe,value in enumerate(one_hot):
		if value == 1:
			labels_treino_escalar[index] = classe
			break

for index,one_hot in enumerate(labels_validacao):
	for classe,value in enumerate(one_hot):
		if value == 1:
			labels_val[index] = classe
			break

#inicializa seu grafo de operacoes
graph = tf.Graph()
with graph.as_default():
	#imagens, labels e learning rate
	img = tf.compat.v1.placeholder(tf.float32,shape=(None,heigth,width,channels))
	labels = tf.compat.v1.placeholder(tf.int64, shape=(None,))
	learning_rate = tf.compat.v1.placeholder(tf.float32)
	training = tf.compat.v1.placeholder(tf.bool)
	#camadas de convolução
	cv1 = tf.compat.v1.layers.conv2d(img,
				     	num_filters,
				     	(3,3),
				     	(1,1),
				     	padding = 'valid',
				     	activation = tf.nn.relu)

	cv2 = tf.compat.v1.layers.max_pooling2d(cv1,
					       (2,2),
					       (2,2),
					       padding = 'valid')

	cv2 = tf.layers.dropout(cv2,0.15, training = training)

	cv3 = tf.compat.v1.layers.conv2d(cv2,
				     	 num_filters,
				     	 (3,3),
				     	 (1,1),
				     	 padding = 'valid',
				     	 activation = tf.nn.relu)

	cv4 = tf.compat.v1.layers.max_pooling2d(cv3,
					       (2,2),
					       (2,2),
					       padding = 'valid')

	cv4 = tf.layers.dropout(cv4,0.15, training = training)

	cv5 = tf.compat.v1.layers.conv2d(cv4,
				         num_filters,
				         (3,3),
				         (1,1),
				         padding = 'valid',
				         activation = tf.nn.relu)

	cv6 = tf.compat.v1.layers.max_pooling2d(cv5,
					       (2,2),
					       (2,2),
					       padding = 'valid')

	cv6 = tf.layers.dropout(cv6,0.15, training = training)

	cv7 = tf.compat.v1.layers.conv2d(cv6,
				     	 num_filters,
				        (3,3),
				        (1,1),
				        padding = 'valid',
				        activation = tf.nn.relu)

	cv8 = tf.compat.v1.layers.max_pooling2d(cv7,
					       (2,2),
					       (2,2),
					       padding = 'valid')

	cv8 = tf.layers.dropout(cv8,0.15, training = training)

	#Need to reshape output of cnn for classifier
	shape = cv8.shape
	cv8 = tf.reshape(cv8,[-1,shape[1]*shape[2]*shape[3]])
	#classifier
	d1 = tf.compat.v1.layers.dense(cv8, 1024, activation=tf.nn.relu, name='d1')
	d1 = tf.layers.dropout(d1,0.5, training = training)
	d2 = tf.compat.v1.layers.dense(d1, 64, activation=tf.nn.relu, name='d2')
	d2 = tf.layers.dropout(d2,0.2, training = training)

	output = tf.compat.v1.layers.dense(d2, 4, name='output')
	#loss
	loss = tf.compat.v1.nn.sparse_softmax_cross_entropy_with_logits(labels=labels, logits=output)

	train_op = tf.compat.v1.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)

	correct = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(output, axis=1), labels), dtype=tf.float32))

	print(correct)

def accuracy(session, Xi, yi):
	batch_size = 32
	cont = 0
	for i in range(0, len(Xi), batch_size):
		X_batch = Xi[i:i+batch_size]
		y_batch = yi[i:i+batch_size]
		ret = session.run([correct], feed_dict = {img : X_batch, labels : y_batch, training : False})
		cont += ret[0]
	return 100.0*cont/len(Xi)

with tf.compat.v1.Session(graph = graph) as session:
	#random initialization of the weigths
	session.run(tf.compat.v1.global_variables_initializer())

	for i in range(num_iteracoes_treino):

		indexes = np.random.permutation(len(treino_oficial))[:batch_size]
		imgs_batch = np.take(treino_oficial, indexes, axis=0)
		labels_batch = np.take(labels_treino_escalar, indexes, axis=0)

		ret = session.run([train_op], feed_dict = {img : imgs_batch, labels : labels_batch, learning_rate : lr, training: True})

		if i%100 == 99:
			print("Iteration #%d" % (i))
			print("TRAIN: ACC=%.5f" % (accuracy(session, treino_oficial, labels_treino_escalar)))
			print("VAL: ACC=%.5f\n" % (accuracy(session, validacao, labels_val)))


# -*- coding: utf-8 -*-

"""

@author: bartulem

Assign real times (based on elapsed IPIs) to spikes.

If there is sync misalignment between tracking and imec clocks,
such that sampling frequencies cannot be trusted, it is possible
to convert every event into time/frame measured by the IPI generator.

That works in the following way:
(1) spikes that happen before or after tracking are eliminated
(2) each spike time is calculated as the sum between how much time
    elapsed until the LED just prior to the spike, and the sampling
    rate between two bounding LEDs multiplied by the number of samples
    until the spike

!NB: Spike times are now zeroed to tracking start!

"""

import numpy as np
from numba import njit

@njit(parallel=False)
def times_to_events(sync_data,
                    event_data,
                    imec_data_col,
                    time_data_col,
                    frame_data_col):

    """
    Inputs
    ----------
    sync_data : np.ndarray
        The sync data; necessary input.
    event_data : np.ndarray
        The spike train; necessary input.
    imec_data_col : int
        IMEC column in the sync DataFrame; necessary input.
    time_data_col : int
        IPI time column in the sync DataFrame; necessary input.
    frame_data_col : int
        Tracking column in the sync DataFrame; necessary input.
    ----------

    Outputs
    ----------
    new_event_times : np.ndarray
        An array with spike times converted to the synce pulse clock.
    new_event_frames : np.ndarray
        An array with spike frames adjusted to the synce pulse clock.
    ----------
    """

    # get first and last imec LEDs
    first_led_samples = sync_data[1, imec_data_col]
    last_led_samples = sync_data[-2, imec_data_col]

    # eliminate events before 1st and after last LED onset
    # this means there are no more spikes before or after tracking
    event_data = event_data[(event_data > first_led_samples) & (event_data < last_led_samples)]

    # create new array for event times
    new_event_times = np.zeros(len(event_data))
    new_event_frames = np.zeros(len(event_data), dtype=np.int64)

    for idx, spike in enumerate(event_data):

        for xx in range(1, sync_data.shape[0] - 2):

            # find bounding LED events in samples
            lower_bound_samples = sync_data[xx, imec_data_col]
            upper_bound_samples = sync_data[xx + 1, imec_data_col]

            if lower_bound_samples <= spike < upper_bound_samples:

                # find bounding LED events in sync time and frames
                lower_bound_time = sync_data[xx, time_data_col]
                upper_bound_time = sync_data[xx + 1, time_data_col]

                lower_bound_frames = sync_data[xx, frame_data_col]
                upper_bound_frames = sync_data[xx + 1, frame_data_col]

                # get total number of samples between two bounding LED events
                total_samples_between = upper_bound_samples - lower_bound_samples

                # samples between spike and lower bound led
                samples_to_spike = spike - lower_bound_samples + 1

                # get total time/frames between two bounding LED events
                total_time_between = upper_bound_time - lower_bound_time
                total_frames_between = upper_bound_frames - lower_bound_frames

                # calculate spike time/frame
                spike_time = lower_bound_time + (total_time_between / total_samples_between) * samples_to_spike
                spike_frame = (lower_bound_frames - sync_data[1, frame_data_col]) + \
                              int(round((total_frames_between / total_samples_between) * samples_to_spike))

                # save results converted to seconds/frames to array
                new_event_times[idx] = spike_time / 1e3
                new_event_frames[idx] = spike_frame

                break

    return new_event_times, new_event_frames


#test2 - query the 2 chromadbs!
#https://python.langchain.com/en/latest/modules/agents/agents/custom_multi_action_agent.html
import logging
import os                  #access the chroma folder/files
import re                  #regex to replace multiple spaces
import chromadb            #store data locally for querying

from langchain                  import PromptTemplate
from langchain.embeddings       import HuggingFaceEmbeddings
from langchain.vectorstores     import Chroma
from langchain.chains           import RetrievalQA
from langchain.memory           import ConversationBufferMemory

from langchain.agents           import Tool, AgentExecutor, initialize_agent, AgentType
from langchain.agents           import BaseSingleActionAgent
from langchain.agents           import BaseMultiActionAgent
from langchain.prompts          import StringPromptTemplate
from langchain                  import LLMChain
from typing                     import List, Tuple, Any, Union
from langchain.schema           import AgentAction, AgentFinish

from langchain.tools            import BaseTool

#logging.basicConfig(level=logging.DEBUG)

ABS_PATH    = os.path.dirname(os.path.abspath(__file__))
RUFF_DB_DIR = os.path.join(ABS_PATH, "ruff_db")
IKEA_DB_DIR = os.path.join(ABS_PATH, "ikea_db")

embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")

#llm_chain = LLMChain(prompt=prompt, llm=llm)
#tool_names = [tool.name for tool in tools]
#
#class CustomOutputParser(AgentOutputParser):
#    def parse(self, llm_output: str) -> Union[AgentAction, AgentFinish]:
#        if \"Final Answer:\" in llm_output:
#            print(\"we reached a Final Answer\\n\")
#            return AgentFinish(
#                return_values={\"output\": llm_output.split(\"Final Answer:\")[-1].strip()},
#                log=llm_output,
#            )
#        print(\"We are still looping through potential answers\\n\")
#        # Parse out the action and action input
##        regex = r\"Action\\s*\\d*\\s*:(.*?)\\nAction\\s*\\d*\\s*Input\\s*\\d*\\s*:[\\s]*(.*)\"
##        match = re.search(regex, llm_output, re.DOTALL)
##        print(match)
##        if not match:
##            raise ValueError(f\"Could not parse LLM output: `{llm_output}`\")
##        action = match.group(1).strip()
##        action_input = match.group(2)
#        # Return the action and action input
##        return AgentAction(tool=action, tool_input=action_input.strip(\" \").strip('\"'), log=llm_output)
#        return AgentAction(tool=llm_output.strip(\" \").strip('\"'), #tool_input=llm_output.strip(\" \").strip('\"'), log=llm_output)
#
#    output_parser = CustomOutputParser()

#CHAT_CONVERSATIONAL_REACT_DESCRIPTION = 'chat-conversational-react-description'
#CHAT_ZERO_SHOT_REACT_DESCRIPTION = 'chat-zero-shot-react-description'
#CONVERSATIONAL_REACT_DESCRIPTION = 'conversational-react-description'
#REACT_DOCSTORE = 'react-docstore'
#SELF_ASK_WITH_SEARCH = 'self-ask-with-search'
#ZERO_SHOT_REACT_DESCRIPTION = 'zero-shot-react-description'
#agent = initialize_agent(tools, llm, agent="zero-shot-react-description", verbose=True)
#agent.run("How much did the economy grow by?")
#agent = LLMSingleActionAgent(llm_chain=llm_chain,output_parser=output_parser,stop=["Observation:"])
#agent = LLMSingleActionAgent(llm_chain=llm_chain,
#                             output_parser=output_parser,
#                             stop=["Observation:"],
#                             allowed_tools=tool_names
#                            )
#agent_executor = AgentExecutor.from_agent_and_tools(agent=agent, tools=tools, verbose=True)
#agent_executor.run("How much did the economy grow by?")
#

def search_ruff(search_input):
    print("\nwithin search_ruff we are searching for:", search_input)

    client_settings = chromadb.config.Settings(
        chroma_db_impl     ="duckdb+parquet",
        persist_directory  =RUFF_DB_DIR,
        anonymized_telemetry=False        #do not allow tracking of data - just incase
    )

    vectorstore = Chroma(
        collection_name    ="ruff_store",
        embedding_function = embeddings,
        client_settings    = client_settings,
        persist_directory  = RUFF_DB_DIR
    )

    docs = vectorstore.similarity_search_with_score(search_input, k=1) #k=1 as we want 1 answer!
    content = docs[0][0]
    my_str = '"'+str(content)+'"'
    tt = my_str.split("'")
    score = docs[0][1]
    return tt[1]

def search_ikea(search_input):
    print("\nwithin search_ikea, we are searching for:", search_input)

    client_settings = chromadb.config.Settings(
        chroma_db_impl     ="duckdb+parquet",
        persist_directory  =IKEA_DB_DIR,
        anonymized_telemetry=False        #do not allow tracking of data - just incase
    )

    vectorstore = Chroma(
        collection_name    ="ikea_store",
        embedding_function = embeddings,
        client_settings    = client_settings,
        persist_directory  = IKEA_DB_DIR
    )

    docs = vectorstore.similarity_search_with_score(search_input, k=1) #k=1 as we want 1 answer!
    content = docs[0][0]
    my_str = '"'+str(content)+'"'
    tt = my_str.split("'")
    score = docs[0][1]
    return tt[1]

def search_janes(query: str) -> str:
    print("\n...and now I am querying JANES data source....\nI'll be right back.\n")
    return "This is the result from the JANES response that we created"

def query_chromadb():
    #if the folder does not exist then stop as we cannot do anything without data
    if not os.path.exists(RUFF_DB_DIR):
        raise Exception(f"{RUFF_DB_DIR} does not exist, nothing can be queried")
    if not os.path.exists(IKEA_DB_DIR):
        raise Exception(f"{IKEA_DB_DIR} does not exist, nothing can be queried")

    query1 = "Why would I use ruff over flake8 and can I purchase it at the IKEA store?"
#    query1 = "What is the weight of a challenger 2 tank?"

    tools = [
        Tool(
            name = "SearchIKEA",
            func=search_ikea,
            description="use for query information about the IKEA FAQ"
        ),
        Tool(
            name = "SearchRuff",
            func=search_ruff,
            description="use for query information about the Python Ruff linter"
        ),
        Tool(
            name = "SearchJanes",
            func=search_janes,
            description="call this to query the JANES database"
        ),
    ]

    print("Initializing EL.VI.S....")
#    memory = ConversationBufferMemory(memory_key="chat_history")
#keep this here incase we need to do something with history later on
    llm = embeddings

    class FakeAgent(BaseMultiActionAgent):
        """Fake Custom Agent."""

        @property
        def input_keys(self):
            return ["input"]

        def plan(
            self, intermediate_steps: List[Tuple[AgentAction, str]], **kwargs: Any
        ) -> Union[AgentAction, AgentFinish]:
            """Given input, decided what to do. If you cannot provide a correct answer do not guess.

            Args:
                intermediate_steps: Steps the LLM has taken to date,
                    along with observations
                **kwargs: User inputs.

            Returns:
                Action specifying what tool to use.
            """
            if len(intermediate_steps) == 0:
                return [
                    AgentAction(tool="SearchIKEA", tool_input=kwargs["input"], log=""),
                    AgentAction(tool="SearchRuff", tool_input=kwargs["input"], log=""),
                    AgentAction(tool="SearchJanes", tool_input=kwargs["input"], log=""),
                ]
            else:
                return AgentFinish(return_values={"output": "\nplan - I have absolutely no idea"}, log="")

        async def aplan(
            self, intermediate_steps: List[Tuple[AgentAction, str]], **kwargs: Any
        ) -> Union[List[AgentAction], AgentFinish]:
            """Given input, decided what to do. If you cannot provide a correct answer do not guess.

            Args:
                intermediate_steps: Steps the LLM has taken to date,
                    along with observations
                **kwargs: User inputs.

            Returns:
                Action specifying what tool to use.
            """
            if len(intermediate_steps) == 0:
                return [
                    AgentAction(tool="SearchIKEA", tool_input=kwargs["input"], log=""),
                    AgentAction(tool="SearchRuff", tool_input=kwargs["input"], log=""),
                    AgentAction(tool="SearchJanes", tool_input=kwargs["input"], log=""),
                ]
            else:
                return AgentFinish(return_values={"output": "\naplan - I have no idea"}, log="")

    agent = FakeAgent()
    agent_executor = AgentExecutor.from_agent_and_tools(agent=agent, tools=tools, verbose=True, max_iterations=1)

#we can call this directly and the output will be shown in the command-line:
#    agent_executor.run(query1)
#or we can contain the responses into a single string and do something with the result
    doc = agent_executor.run(query1)
#    print(type(doc)) #str
    print(doc)
#ideally, we would store each of the individual responses from each tool so that we can have that individual data available, maybe store it as we are gathering it?

def main():
    query_chromadb()

if __name__ == '__main__':
    main()


import numpy as np
import re
import pickle
import feedparser

'''

'''

def loadDataSet():
    postList = [['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
                ['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
                ['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
                ['stop', 'posting', 'stupid', 'worthless', 'garbage'],
                ['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
                ['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]

    classVec = [0, 1, 0, 1, 0, 1]   # 1 for abusive, 0 fro not
    return postList, classVec

def createVovabularyList(dataset):
    vocabSet = set([])
    for document in dataset:
        vocabSet = vocabSet | set(document) # | is a union operation

    return list(vocabSet)

def setOfWords2Vec(vocabList, inputSet):
    returnVec = [0]*len(vocabList)
    for word in inputSet:
        if word in vocabList:
            returnVec[vocabList.index(word)] = 1
        else:
            print('the word %s is not in the Vocabulary'%word)

    return returnVec

def bagOfWords2Vec(vocabList, inputSet):
    returnVec = [0]*len(vocabList)
    for word in inputSet:
        if word in vocabList:
            returnVec[vocabList.index(word)] += 1
        # else:
        #     print('the word %s is not in the Vocabulary'%word)

    return returnVec

def trainNB0(trainMatrix, labels):
    number_of_samples = len(trainMatrix)
    number_of_words = len(trainMatrix[0])
    p_Abusive = sum(labels)/number_of_samples

    #防止因为没出现而造成0的现象，连乘之后导致结果为0
    p0_numerator = np.ones(number_of_words)
    p1_numerator = np.ones(number_of_words)
    '''
    为整数，假设有5个training document， 其中一个词每次都出现，则p0_numerator=6为了让分母与分子不同且分母永远比分子大，就让分母从
    2开始加
    '''
    p0_denominator = 2.0
    p1_denominator = 2.0

    for i in range(number_of_samples):
        if labels[i] == 1:
            #为所有1情况下，各个单词occurance各自之和

            p1_numerator += trainMatrix[i]
            #为所有w的occurance之和
            p1_denominator += sum(trainMatrix[i])

        else:
            p0_numerator += trainMatrix[i]

            p0_denominator += sum(trainMatrix[i])
    # 因为log单调递增切此处概率大于零，可以让log(A*B) = logA+logB，变为和的形式p(w1|c)*p(w2|c) = log(p(w1|c))+log(p(w2|c))

    p0_Vec = np.log(p0_numerator/p0_denominator)
    p1_Vec = np.log(p1_numerator/p1_denominator)

    return p0_Vec, p1_Vec, p_Abusive

def classifyNB(testVec, p0, p1, pA):
    p_0 = sum(testVec * p0) + np.log(pA)
    p_1 = sum(testVec * p1) + np.log(pA)

    if p_0>p_1:
        return 0
    else:
        return 1

def testingNB():
    # vocabulary = list()
    # temp = list()
    # for i in range(len(labels)):
    #     if labels[i] == 1:
    #         temp.append(postList[i])
    postList, labels = loadDataSet()
    vocabulary = createVovabularyList(postList)
    trainMatrix = list()

    for i in range(len(postList)):
        trainMatrix.append(setOfWords2Vec(vocabulary, postList[i]))

    p0, p1, pA = trainNB0(trainMatrix, labels)

    testDoc_1 = ['garbage', 'stupid', 'dog']
    testDoc_2 = ['popo', 'tsy', 'cute']

    testVec_1 = setOfWords2Vec(vocabulary, testDoc_1)
    testVec_2 = setOfWords2Vec(vocabulary, testDoc_2)

    print(classifyNB(testVec_1, p0, p1, pA))
    print(classifyNB(testVec_2, p0, p1, pA))

def getTokens(textString):

    textString = textString.strip()
    words = re.findall(r'\w*', textString)
    tokenList = [word.lower() for word in words if len(word)>2]
    # tokenList = list()
    # for word in words:
    #     if len(word)>3:
    #         tokenList.append(word)
    return tokenList

# with open('ham/1.txt', 'r') as file:
#     line = str(file.read())
#     tokens = getTokens(line)
#     print(tokens)

def testingNB_email():
    testSet = []
    trainingLabels = []
    testLabels = []
    trainingMatrix = []
    fullText = []

    for i in range(1, 26):

        with open('ham/%s.txt'%i, 'r') as file:
            line = str(file.read())
        tokens = getTokens(line)
        trainingMatrix.append(tokens)
        fullText.extend(tokens)
        trainingLabels.append(0)

        with open('spam/%s.txt' % i, 'r') as file:
            line = str(file.read())
        tokens = getTokens(line)
        trainingMatrix.append(tokens)
        trainingLabels.append(1)
        fullText.extend(tokens)
    vocabulary = createVovabularyList(trainingMatrix)
    for _ in range(10):
        index = int(np.random.uniform(len(trainingMatrix)+1)-1)

        testSet.append(trainingMatrix[index])
        testLabels.append(trainingLabels[index])
        del(trainingMatrix[index])
        del(trainingLabels[index])

    #now we get the training set and the testing set

    trainingSet = []
    for item in trainingMatrix:
        trainingSet.append(setOfWords2Vec(vocabulary, item))

    p0, p1, pA = trainNB0(np.array(trainingSet), np.array(trainingLabels))
    error = 0
    for i in range(len(testSet)):
        if classifyNB(np.array(setOfWords2Vec(vocabulary, testSet[i])), p0, p1, pA) != testLabels[i]:
            error += 1
            print('error document is: ', testSet[i])
    error_rate = error/len(testSet)
    return error_rate

# error_rate = testingNB_email()
# print('error rate is :', error_rate)

def calcMostFreq(vocabList, fullText):
    import operator
    freqDict = {}
    for token in vocabList:
        freqDict[token] = fullText.count(token)
    sortedFreq = sorted(freqDict.items(), key=operator.itemgetter(1), reverse=True)
    return sortedFreq[:30]

def localWords(feed1, feed0):
    import feedparser

    docList = []
    classList = []
    fullText = []

    minLen = min(len(feed1['entries']), len(feed0['entries']))
    for i in range(minLen):
        wordList = getTokens(feed1['entries'][i]['summary'])
        docList.append(wordList)
        fullText.extend(wordList)
        classList.append(1)
        wordList = getTokens(feed0['entries'][i]['summary'])
        docList.append(wordList)
        fullText.extend(wordList)
        classList.append(0)

    vocabulary = createVovabularyList(docList)
    top30words = calcMostFreq(vocabulary, fullText)
    for word in top30words:
        if word[0] in vocabulary:
            vocabulary.remove(word[0])

    training_index_Set = list(range(2*minLen))   #因为从两个源各去了minLen长度的内容, 此处需要用list把range转化，否则无法删除操作
    test_index_Set = []
    testClasses = []

    for i in range(20):
        index = int(np.random.uniform(0, len(training_index_Set)))
        test_index_Set.append(training_index_Set[index])
        del(training_index_Set[index])

    trainingMatix = []
    trainingClasses = []

    for index in training_index_Set:
        trainingMatix.append(bagOfWords2Vec(vocabulary, docList[index]))
        trainingClasses.append(classList[index])

    p0, p1, pA = trainNB0(trainingMatix, trainingClasses)
    error = 0
    for index in test_index_Set:
        if classList[index] != classifyNB(bagOfWords2Vec(vocabulary, docList[index]), p0, p1, pA):
            error += 1

    error_rate = float(error)/len(test_index_Set)

    print(error_rate)

    return p0, p1, vocabulary

def getTopWord(ny, sf):
    pSF, pNY, vocabulary = localWords(ny, sf)#ny -->feed1, sf -->feed0
    topSF = []
    topNY = []
    for i in range(len(pSF)):
        if pSF[i]>-6:
            topSF.append((vocabulary[i], pSF[i]))
        if pNY[i]>-6:
            topNY.append((vocabulary[i], pNY[i]))

    sortedSF = sorted(topSF, key=lambda item: item[1], reverse=True)
    print('SF*'*10)
    for item in sortedSF[:10]:
        print(item[0])

    print('NY*'*10)
    sortedNY = sorted(topNY, key=lambda item: item[1], reverse=True)
    for item in sortedNY[:10]:
        print(item[0])

ny = feedparser.parse('http://newyork.craigslist.org/stp/index.rss')
sf = feedparser.parse('http://sfbay.craigslist.org/stp/index.rss')
#if we get rid of some stop words, then we can make the error rate lower

getTopWord(ny, sf)
'''
0.45
SF*SF*SF*SF*SF*SF*SF*SF*SF*SF*
butt
married
all
great
from
hard
workout
man
could
face
NY*NY*NY*NY*NY*NY*NY*NY*NY*NY*
home
did
any
there
breasts
wear
boy
love
well
see

'''

#
# ny = feedparser.parse('http://newyork.craigslist.org/stp/index.rss')
# print(ny['entries'][0]['summary'])


import socket
from multiprocessing import Process
import re
import sys

HTML_ROOT_DIR = "./html"

class HTTPServer(object):
    def __init__(self, app):
        self.serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.app = app

    def bind(self, port):
        self.serverSocket.bind(("", port))

    def start(self):
        self.serverSocket.listen(1000)
        while 1:
            clientSocket, clientAddress = self.serverSocket.accept()
            process = Process(target=self.handleClient, args=(clientSocket, ))
            process.start()
            clientSocket.close()

    def startResponse(self, status, headers):
        responseHeaders = "HTTP/1.1" + status + "\r\n"
        for header in headers:
            responseHeaders += "%s: %s\r\n" % header
        self.responseHeaders = responseHeaders

    def handleClient(self, clientSocket):
        requestData = clientSocket.recv(1024)
        requestLines = requestData.splitlines()
        requestStartLine = requestLines[0]
        print(requestStartLine.decode("utf-8"))
        fileName = re.match(r"\w+ +(/[^ ]*) ", requestStartLine.decode("utf-8")).group(1)
        method = re.match(r"(\w+) +/[^ ]* ", requestStartLine.decode("utf-8")).group(1)

        env = {
            "PATH_INFO": fileName,
            "METHOD": method
        }
        responseBody = self.app(env, self.startResponse)
        response = self.responseHeaders + "\r\n" + responseBody

        clientSocket.send(bytes(response, "utf-8"))
        clientSocket.close()

def main():
    sys.path.insert(1, HTML_ROOT_DIR)
    if len(sys.argv) < 2:
        sys.exit("framework!!!")

    moduleName, appName = sys.argv[1].split(":")
    m = __import__(moduleName)
    app = getattr(m, appName)
    httpServer = HTTPServer(app)
    httpServer.bind(4567)
    httpServer.start()

if __name__ == '__main__':
    main()


from typing import Any

import structlog

from confluent_kafka_helpers.message import Message

from eventsourcing_helpers import metrics
from eventsourcing_helpers.handler import Handler
from eventsourcing_helpers.utils import get_callable_representation

logger = structlog.get_logger(__name__)

class EventHandler(Handler):
    """
    Application service that calls the correct domain handler for an event.
    """

    def _can_handle_command(self, message: Message) -> bool:
        """
        Checks if the event is something we can handle.

        Args:
            message: Consumed message from the bus.

        Returns:
            bool: Flag to indicate if we can handle the event.
        """
        event_class = message.value['class']
        if event_class not in self.handlers:
            logger.debug("Unhandled event", event_class=event_class)
            return False

        return True

    def _handle_event(self, event: Any) -> None:
        """
        Get and call the correct event handler.

        Args:
            event: Event to be handled.
        """
        handler = self.handlers[event._class]

        handler_name = get_callable_representation(handler)
        handler_wrapper = metrics.timed(
            'eventsourcing_helpers.handler.handle',
            tags=['message_type:event', f'message_class:{event._class}', f'handler:{handler_name}']
        )(handler)

        handler_wrapper(event)

    def handle(self, message: dict) -> None:
        """
        Apply correct handler for received event.

        Args:
            message: Consumed message from the bus.
        """
        if not self._can_handle_command(message):
            return

        event = self.message_deserializer(message)
        logger.info("Handling event", event_class=event._class)

        self._handle_event(event)


import torch
from dataset import test_loader
from generator_model import Generator
from tqdm import tqdm
import config
from torchvision.utils import save_image
import numpy as np
import cv2 as cv
from skimage.measure._label import label

def cv_imread(filePath):

    cv_img = cv.imdecode(np.fromfile(filePath, dtype=np.uint8), -1)
    cv_img = cv.cvtColor(cv_img, cv.COLOR_RGB2BGR)

    return cv_img

def threshold_image(image):

    gray = cv.cvtColor(image, cv.COLOR_BGR2GRAY)
    ret, binary = cv.threshold(gray, 255, 255, cv.THRESH_BINARY|cv.THRESH_OTSU)

    return binary

def save_some_examples(gen, x, y, epoch, folder):

    x, y = x.to(config.DEVICE), y.to(config.DEVICE)
    gen.eval()
    with torch.no_grad():
        y_fake = gen(x)

        y_fake += torch.flip(y_fake, (2,))   # row
        y_fake += torch.flip(y_fake, (3,))   # column
        y_fake += y_fake.transpose(-1, -2)   # diagonal

        save_image(y_fake, folder + f"/y_gen_{epoch}.png")
        save_image(y, folder + f"/input_{epoch}.png")

        y_fake_name = folder + f"/y_gen_{epoch}.png"
        y_fake_image = cv_imread(y_fake_name)
        y_fake_image = cv.medianBlur(y_fake_image, ksize=3)
        y_binary = threshold_image(image=y_fake_image)

        labels, num = label(y_binary, return_num=True, background=255)

        max_area = 0
        for i in range(1, num + 1):
            value_area = (np.ones_like(labels) * i == labels).sum()

            if value_area > max_area:
                max_area = value_area
                value = i

        y_binary = 1 - np.array(np.ones_like(labels) * value == labels).astype('uint8')

        y_fake = torch.from_numpy(y_binary)[None, None, ...]
        y_fake = torch.clamp(y_fake.float(), min=0., max=1.)

        save_image(y_fake, folder + f"/y_gen_{epoch}_binary.png")

        return y, y_fake

gen = Generator(in_channels=1, features=32).to(config.DEVICE)

gen.load_state_dict(torch.load('checkpoints/gen_500.pth.tar')['state_dict'], strict=False)

loop = tqdm(test_loader, leave=True)

output_50_50 = []
label_50_50 = []
label_10_31 = []
for idx, (x, y) in enumerate(loop):

    if idx>14120:

        y, y_fake = save_some_examples(gen, x, y, idx+1, folder="inference")

        label_50_50.append(y.squeeze(1).cpu().numpy())
        output_50_50.append(y_fake.squeeze(1).cpu().numpy())
        label_10_31.append(x.reshape(-1, 10, 31).cpu().numpy())

label_10_31 = np.concatenate(label_10_31, axis=0)
label_50_50 = np.concatenate(label_50_50, axis=0)
output_50_50 = np.concatenate(output_50_50, axis=0)

import scipy.io as sio

sio.savemat('inference.mat', {'label_10_31': label_10_31,
                              'label_50_50': label_50_50,
                              'output_50_50': output_50_50})


# -*- coding: utf-8 -*-
"""
Created on Tue Nov 26 17:03:44 2019

@author: Morimoto Lab
"""
import openmdao.api as om
import numpy as np
from model import CTR
import h5py
import time
import scipy.io
from math import cos,sin,radians
from stiffness import Stiffness

t0=time.time()
# Load the path point
f = h5py.File('path.mat','r')
data_pa = f.get('Pa')
data_pb = f.get('Pb')
data_pa = np.array(data_pa[:,49:])
data_pb = np.array(data_pb[:,49:])
# Rotate the path by 180 degrees
Rot = [[-1,0,0],[0,-1,0],[0,0,1]]
Rot = np.array(Rot)
data_pa = np.dot(Rot,data_pa)
# Entry Angle
theta = 45
Rot2 = np.array([[1,0,0],[0,cos(radians(theta)),-sin(radians(theta))],[0,sin(radians(theta)),cos(radians(theta))]])
data_pa = np.dot(Rot2,data_pa)
data_pb = np.dot(Rot2,data_pb)
# Adjust the base position of the concentric tube robot
"""data_pa[0] = data_pa[0] + 50
data_pa[1] = data_pa[1] - 20
data_pa[2] = data_pa[2] - 120"""

# We'll use the component that was defined in the last tutorial

dim = np.shape(data_pa)
N = dim[1]
E=80
# build the model
prob = om.Problem()
indeps = prob.model.add_subsystem('indeps', om.IndepVarComp())

indeps.add_output('length1', np.ones(N)*(-30))
indeps.add_output('length2', np.ones(N)*(-30))
indeps.add_output('length4', np.ones(N)*(-30))
indeps.add_output('kappa2', .05)
indeps.add_output('d1', 1.370)
indeps.add_output('d2', 1.850)
indeps.add_output('d3', 2.007)
indeps.add_output('d4', 2.400)
indeps.add_output('d5', 2.590)
indeps.add_output('d6', 3.048)
indeps.add_output('l22', np.ones(N)*(-60))
indeps.add_output('psi2', np.ones(N))

"""indeps.add_output('length1', np.ones(N)*(-30))
indeps.add_output('length2', np.ones(N)*(-30))
indeps.add_output('length4', np.ones(N)*(-30))
indeps.add_output('kappa2', .04)
indeps.add_output('d1', 1.2)
indeps.add_output('d2', 1.5)
indeps.add_output('d3', 2.1)
indeps.add_output('d4', 2.5)
indeps.add_output('d5', 2.9)
indeps.add_output('d6', 3.1)
indeps.add_output('l22', np.ones(N)*(-60))
indeps.add_output('psi2', np.ones(N))"""

prob.model.add_subsystem('ctr', CTR())
prob.model.add_subsystem('Stiffness',Stiffness())
# define the component whose output will be constrained
prob.model.add_subsystem('const1', om.ExecComp('x = length1*(cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*cos(psi2)*sin((kappa2*kb2*(length2))/(kb1 + kb2)) + sin((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*cos((kappa2*kb2*(length2))/(kb1 + kb2))*cos(psi2)) - (cos(psi2)*(cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3)) - 1)*(kb1 + kb2 + kb3))/(kappa2*kb2) - (cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*cos(psi2)*(cos((kappa2*kb2*(length2))/(kb1 + kb2)) - 1)*(kb1 + kb2))/(kappa2*kb2) + (sin((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*cos(psi2)*sin((kappa2*kb2*(length2))/(kb1 + kb2))*(kb1 + kb2))/(kappa2*kb2)-3',x=np.zeros(N),l22=np.zeros(N),length2=np.zeros(N),length1=np.zeros(N),psi2=np.zeros(N)))
prob.model.add_subsystem('const2', om.ExecComp('y = length1*(cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*sin((kappa2*kb2*(length2))/(kb1 + kb2))*sin(psi2) + sin((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*cos((kappa2*kb2*(length2))/(kb1 + kb2))*sin(psi2)) - (sin(psi2)*(cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3)) - 1)*(kb1 + kb2 + kb3))/(kappa2*kb2) - (cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*sin(psi2)*(cos((kappa2*kb2*(length2))/(kb1 + kb2)) - 1)*(kb1 + kb2))/(kappa2*kb2) + (sin((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*sin((kappa2*kb2*(length2))/(kb1 + kb2))*sin(psi2)*(kb1 + kb2))/(kappa2*kb2)-20',y=np.zeros(N),l22=np.zeros(N),length2=np.zeros(N),length1=np.zeros(N),psi2=np.zeros(N)))
prob.model.add_subsystem('const3', om.ExecComp('z = length4 + length1*(cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*cos((kappa2*kb2*(length2))/(kb1 + kb2)) - sin((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*sin((kappa2*kb2*(length2))/(kb1 + kb2))) + (sin((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*(kb1 + kb2 + kb3))/(kappa2*kb2) + (cos((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*sin((kappa2*kb2*(length2))/(kb1 + kb2))*(kb1 + kb2))/(kappa2*kb2) + (sin((kappa2*kb2*(l22 - length2))/(kb1 + kb2 + kb3))*(cos((kappa2*kb2*(length2))/(kb1 + kb2)) - 1)*(kb1 + kb2))/(kappa2*kb2)+120',z=np.zeros(N),l22=np.zeros(N),length4=np.zeros(N),length1=np.zeros(N),length2=np.zeros(N)))
prob.model.add_subsystem('const4', om.ExecComp('g=l22-length2', g=np.zeros(N),l22=np.zeros(N),length2=np.zeros(N)))
prob.model.add_subsystem('const5', om.ExecComp('k1=kb1'))
prob.model.add_subsystem('const6', om.ExecComp('k2=kb2'))
prob.model.add_subsystem('const12', om.ExecComp('k3=kb3'))
"""prob.model.add_subsystem('const5', om.ExecComp('k1=kb2-kb1'))
prob.model.add_subsystem('const6', om.ExecComp('k2=kb3-kb2'))"""
prob.model.add_subsystem('const7', om.ExecComp('t1=d2-d1'))
prob.model.add_subsystem('const8', om.ExecComp('t2=d4-d3'))
prob.model.add_subsystem('const9', om.ExecComp('t3=d6-d5'))
prob.model.add_subsystem('const10', om.ExecComp('t12=d3-d2'))
prob.model.add_subsystem('const11', om.ExecComp('t23=d5-d4'))

# Connect the variables
prob.model.connect('indeps.d1', ['Stiffness.d1','const7.d1'])
prob.model.connect('indeps.d2', ['Stiffness.d2','const10.d2','const7.d2'])
prob.model.connect('indeps.d3', ['Stiffness.d3','const10.d3','const8.d3'])
prob.model.connect('indeps.d4', ['Stiffness.d4','const11.d4','const8.d4'])
prob.model.connect('indeps.d5', ['Stiffness.d5','const11.d5','const9.d5'])
prob.model.connect('indeps.d6', ['Stiffness.d6','const9.d6'])
prob.model.connect('indeps.l22', ['ctr.l22','const1.l22','const2.l22','const3.l22','const4.l22'])
prob.model.connect('indeps.length1', ['ctr.length1','const1.length1','const2.length1','const3.length1'])
prob.model.connect('indeps.length2', ['ctr.length2','const1.length2','const2.length2','const3.length2','const4.length2'])
prob.model.connect('indeps.length4', ['ctr.length4','const3.length4'])
prob.model.connect('indeps.kappa2', ['ctr.kappa2','const1.kappa2','const2.kappa2','const3.kappa2'])
prob.model.connect('Stiffness.kb1', ['ctr.kb1','const1.kb1','const2.kb1','const3.kb1','const5.kb1'])
prob.model.connect('Stiffness.kb2', ['ctr.kb2','const1.kb2','const2.kb2','const3.kb2','const6.kb2'])
prob.model.connect('Stiffness.kb3', ['ctr.kb3','const1.kb3','const2.kb3','const3.kb3','const12.kb3'])
prob.model.connect('indeps.psi2', ['ctr.psi2','const1.psi2','const2.psi2'])

# setup the optimization
prob.driver = om.ScipyOptimizeDriver()
prob.driver.options['optimizer'] = 'SLSQP'
prob.driver.options['maxiter'] = 1500

# Add design variables
prob.model.add_design_var('indeps.length1', lower=-500, upper=0)
prob.model.add_design_var('indeps.length2', lower=-500, upper=0)
prob.model.add_design_var('indeps.length4', lower=-500, upper=0)
prob.model.add_design_var('indeps.kappa2', lower=0.04, upper=.1)
"""prob.model.add_design_var('indeps.kb1', lower=0, upper=60)
prob.model.add_design_var('indeps.kb2', lower=60, upper=100)
prob.model.add_design_var('indeps.kb3', lower=100, upper=200)"""
prob.model.add_design_var('indeps.l22', lower=-500, upper=-15)
prob.model.add_design_var('indeps.psi2', lower=-3.14, upper=3.14)
"""prob.model.add_design_var('indeps.d1', lower=1, upper=3.5)
prob.model.add_design_var('indeps.d2', lower=1,upper=3.5)
prob.model.add_design_var('indeps.d3', lower=1,upper=3.5)
prob.model.add_design_var('indeps.d4', lower=1,upper=3.5)
prob.model.add_design_var('indeps.d5', lower=1,upper=3.5)
prob.model.add_design_var('indeps.d6', lower=1,upper=3.5)"""
prob.model.add_objective('ctr.f')

# Add constraints
prob.model.add_constraint('const1.x', equals=data_pa[0])
prob.model.add_constraint('const2.y', equals=data_pa[1])
prob.model.add_constraint('const3.z', equals=data_pa[2])
prob.model.add_constraint('const4.g', upper=0)
"""prob.model.add_constraint('const5.k1', lower=0,upper=60)
prob.model.add_constraint('const6.k2', lower=60,upper=100)
prob.model.add_constraint('const12.k3', lower=100,upper=250)"""
"""prob.model.add_constraint('const5.k1', lower=0)
prob.model.add_constraint('const6.k2', lower=0)"""
"""prob.model.add_constraint('const7.t1', lower=0.1,upper=1)
prob.model.add_constraint('const8.t2', lower=0.1,upper=1)
prob.model.add_constraint('const9.t3', lower=0.1,upper=1)
prob.model.add_constraint('const10.t12', lower=0.12, upper=0.15)
prob.model.add_constraint('const11.t23', lower=0.12, upper=0.15)"""
#prob.model.add_constraint('const7.l23', lower=0)
prob.setup()
prob.run_driver()

t1=time.time()
# print the optimal results
print('time:', t1-t0)
print('Length1 =',prob['indeps.length1'])
print('Length3 =',prob['indeps.l22']-prob['indeps.length2'])
print('Length2 =',prob['indeps.length2'])
print('Length4 =',prob['indeps.length4'])
print('kappa2 = ',prob['indeps.kappa2'])
print('kb1 = ',prob['Stiffness.kb1'])
print('kb2 = ',prob['Stiffness.kb2'])
print('kb3 = ',prob['Stiffness.kb3'])
print('d1 = ',prob['indeps.d1'])
print('d2 = ',prob['indeps.d2'])
print('d3 = ',prob['indeps.d3'])
print('d4 = ',prob['indeps.d4'])
print('d5 = ',prob['indeps.d5'])
print('d6 = ',prob['indeps.d6'])
print('l22 = ',prob['indeps.l22'])
print('psi2 = ',prob['indeps.psi2'])

# Save joint values and tube parameters to .mat file
mdict = {'Length1':prob['indeps.length1'],'Length2':prob['indeps.length2'],'Length3':prob['indeps.l22']-prob['indeps.length2'],
        'Length4':prob['indeps.length4'],'l22':prob['indeps.l22'],'kb1':prob['Stiffness.kb1'],'kb2':prob['Stiffness.kb2'],
        'kb3':prob['Stiffness.kb3'],'kappa2':prob['indeps.kappa2'],'d1':prob['indeps.d1'],'d2':prob['indeps.d2'],'d3':prob['indeps.d3'],
        'd4':prob['indeps.d4'],'d5':prob['indeps.d5'],'d6':prob['indeps.d6'],'psi2':prob['indeps.psi2']}
# scipy.io.savemat('D:/Desktop/Fred/CTR/CTR optimization/CTR/Inverse_kinematics/jointvalues/jointvalue_distance_001.mat',mdict)
scipy.io.savemat('D:/Desktop/Fred/CTR_tubeselection/CTR_tubeselection/jointvalues/jonitvalue_ob3_test001.mat',mdict)


"""
自定义返回处理
"""

from rest_framework.renderers import JSONRenderer

class CustomerRenderer(JSONRenderer):
    def render(self, data, accepted_media_type=None, renderer_context=None):
        print('CustomerRenderer data:', data)
        print('accepted_media_type:', accepted_media_type)
        print('renderer_context:', renderer_context)
        if renderer_context:
            detail = None
            # 如果返回的data为字典
            if isinstance(data, dict):
                message = data.pop('errorCode', 'success')
                detail = None
                if 'errorDetail' in data:
                    detail = data.pop('errorDetail')
                code = data.pop('code', renderer_context['response'].status_code)
            # 如果不是字典则将msg内容改为请求成功 code改为请求的状态码
            else:
                message = 'success'
                detail = None
                code = renderer_context['response'].status_code

            ret = {
                'errorCode': message,
                'errorDetail': detail,
                'code': code,
                'data': data
            }
            return super().render(ret, accepted_media_type, renderer_context)

        else:
            return super().render(data, accepted_media_type, renderer_context)


from selenium import webdriver
import time
import os
import sys
class ebabot():
    def giris(self):
        self.browserprofile=webdriver.ChromeOptions()
        self.browserprofile.add_experimental_option('prefs',{'intl.accpet_languages':'en,en_US'})
        self.browser=webdriver.Chrome('chromedriver.exe',chrome_options=self.browserprofile)
        self.girisurl="https://giris.eba.gov.tr/EBA_GIRIS/giris.jsp"
        self.canli_dersurl="https://ders.eba.gov.tr/ders/proxy/VCollabPlayer_v0.0.772/index.html#/main/livesessionview?tab=liveLessons&pageNumber=1&pageSize=25"
        self.tc="tc"
        self.sifre="sifre"
        self.browser.get(self.girisurl)
        self.tcarea=self.browser.find_element_by_xpath("//*[@id='tckn']").send_keys(self.tc)
        self.passarea=self.browser.find_element_by_xpath("/html/body/div[2]/div/div[1]/div/div[2]/form/div[3]/div/input").send_keys(self.sifre)
        time.sleep(1)
        self.submit=self.browser.find_element_by_xpath("/html/body/div[2]/div/div[1]/div/div[2]/form/div[5]/button").click()
        time.sleep(3)
        self.browser.get(self.canli_dersurl)
        time.sleep(3)
    def tikla(self):
        self.tikla=self.browser.find_element_by_xpath("/html/body/div[1]/div[1]/div/div/div[1]/div/div[3]/div/div/div[2]/div/div[3]/div/div[3]/div[2]/div[2]/div/table/tbody/tr/td/div[2]/div/div[3]/table/tbody/tr/td[3]/div/div[2]/div[1]").click()
        time.sleep(3)
    def bot(self):

        self.katil=self.browser.find_element_by_xpath("/html/body/div[1]/div[1]/div/div/div[1]/div/div[3]/div/div/div[2]/div[1]/div/div/div/div[3]/div").get_attribute("class")
        if self.katil=="btn btn-md ng-binding btn-success disabled":
            print("katıl kapalı")
            time.sleep(10)
            self.browser.refresh()
            time.sleep(10)
            return False
        else:
            print("katıl açık")
            time.sleep(5)
            return True
    def derse_gir(self):
        print("derse girildi")
    def kappa(self):
        self.browser.close()

bot=ebabot()
bot.giris()
bot.tikla()
while True:
    bot.bot()
    if bot.bot==True:
        print("katıl aççık")
        break
    else:
        print("katıl kapalı")
        time.sleep(10)
bot.derse_gir()
bot.kappa()
os.execl(sys.executable, os.path.abspath(__file__), *sys.argv)


"""
views related to application user
"""

from django.contrib import messages
from django.http import HttpResponseRedirect
from django.shortcuts import render
from django.urls import reverse

from .forms import ProfileForm

def user_profile(request):
    """ return user profile """
    return render(
        request, 'User/profile.html', {})

def user_profile_edit(request):
    """ return page to edit user profile """

    user = request.user

    if request.method == 'POST':
        form = ProfileForm(request.POST)

        if 'save' in request.POST and form.is_valid():
            data = form.cleaned_data
            user.first_name = data['first_name']
            user.last_name = data['last_name']
            user.climber.middle_name = data['middle_name']
            user.climber.save()
            user.save()

            messages.success(request, 'Profile details updated.')

        return HttpResponseRedirect(reverse('user-profile'))

    data = {
        'username': user.username,
        'first_name': user.first_name,
        'last_name': user.last_name,
        'middle_name': user.climber.middle_name,
    }
    form = ProfileForm(initial=data)

    return render(
        request, 'User/profile_edit.html', {'form': form})