"""
Module to deal with BlueFS. Locates the superblock, reads the transaction log
and extracts files from BlueFS.
"""

import os
import shutil
from vampyr.datatypes import CephDataType, CephBlockHeader, CephUUID,\
    CephInteger, CephUnknown, CephVarInteger, CephLBA, CephList, CephUTime,\
    CephString, CephVarIntegerLowz
from vampyr.exceptions import VampyrMagicException
import logging
import hashlib
import time

class BlueFS(object):
    """
    Contains the information about the BlueFS.
    """
    def __init__(self, osd):
        """
        osd: The file handle where to read the data from.
        """
        self.initialized = False
        self.allocated_areas = {}
        self.deallocated_areas = {}
        self.allocated_ino = []
        self.deallocated_ino = []
        self.dirlist = []
        self.removed_dirlist = []
        self.allocated_extents = []
        self.deallocated_extents = []
        self.ino_to_file_map = {}

        self.osd = osd

        o = osd
        self.superblock = BlueFSSuperblock(o)
        assert(self.superblock.data['osd_uuid'] == osd.bluestorelabel['uuid'])
        self.ino_to_file_map[self.superblock.data['log_fnode'].ino] =\
            BlueFSFile(self.superblock.data['log_fnode'].ino,
                       self.superblock.data['log_fnode'].size,
                       self.superblock.data['log_fnode'].mtime,
                       self.superblock.data['log_fnode'].extents)
        self.transactions = {}
        self.skipped_transactions = {}

        self.next_offset = 0
        self.jump_seq = False

        self.blocksize = self.superblock.data['block_size'].value

        extents = self.superblock.data['log_fnode'].extents
        oldextents = []
        assert(len(extents) > 0)
        logical_offset = 0

        start = time.time()
        while len(extents) > 0:
            oldextents += extents
            logical_offset = self.read_bluefs_extents(extents, logical_offset)
            newextents = self.get_file(1).extents
            extents = [x for x in newextents if x not in oldextents]
        stop = time.time()
        logging.info("Parsed %d txs from transaction log in %d seconds." %
                     (len(self.transactions), stop - start))
        assert(self._validate_extent_location())

    def _validate_extent_location(self):
        """
        An exent should never be outside of the allocated BlueFS areas.
        """
        for e in self.allocated_extents:
            match = False
            for a in self.allocated_areas.values():
                if e.offset >= a[0] and e.offset + e.length <= a[0] + a[1]:
                    match = True
            if not match:
                logging.error("Extent %e is not within BlueFS." % e)
                return False
        return True

    def read_bluefs_extents(self, extents, logical_offset):
        # Tries to decode transactions in the given transaction log extents
        # logical_offset is the current offset in the transaction log.
        for e in extents:
            block = 0
            fnodesize = e.length
            offset = e.offset
            # self.next_offset = 0
            while block * self.blocksize < fnodesize:
                try:
                    thisoff = offset + block * self.blocksize
                    if self.jump_seq:
                        self.jump_seq = False
                        self.read_bluefs_transaction(thisoff, skip=True)
                    elif self.next_offset > logical_offset:
                        self.read_bluefs_transaction(thisoff, skip=True)
                    else:
                        self.next_offset = 0
                        self.read_bluefs_transaction(thisoff)
                except (VampyrMagicException, AssertionError) as e:
                    # print(traceback.format_exc())
                    pass
                block += 1
                logical_offset += self.blocksize
        return logical_offset

    def read_bluefs_transaction(self, seek, skip=False):
        # Decodes a transaction and the contained operations of the
        # transaction log.
        # seek is the offset of the transaction on the OSD.
        # skip indicates whether the transaction is to be ignored.
        h = {}
        handle = self.osd
        handle.seek(seek)
        h['header'] = CephBlockHeader(handle)
        assert(h['header'].blength <= self.blocksize)
        h['uuid'] = CephUUID(handle)
        assert(h['uuid'] == self.superblock.data['uuid'])
        h['seq'] = CephInteger(handle, 8).value
        if skip:
            # We want to read the transaction, but don't want to apply the
            # transaction to our own BlueFS state that we're recovering.
            h['transaction'] = BlueFSTransaction(handle, None)
        else:
            # We want to read the transaction and apply it.
            h['transaction'] = BlueFSTransaction(handle, self)
        h['crc'] = CephInteger(handle, 4)
        assert(handle.tell() == h['header'].end_offset)
        h['unknown'] = CephUnknown(handle, 0x10)

        if not skip:
            self.transactions[seek] = h
        else:
            self.skipped_transactions[seek] = h

    def op_init(self):
        self.initialized = True

    def op_alloc_add(self, id, offset, length):
        assert(self.initialized)
        self.allocated_areas[id] = (offset, length)

    def op_alloc_rm(self, id, offset, length):
        assert(self.initialized)
        assert(id in self.allocated_areas)
        self.allocated_areas.remove(id)
        self.deallocated_areas[id] = (offset, length)

    def op_dir_create(self, dirname):
        assert(self.initialized)
        if dirname not in [x.dirname for x in self.dirlist]:
            self.dirlist.append(BlueFSDir(dirname))

    def op_dir_remove(self, dirname):
        assert(self.initialized)
        rmdir = None
        for d in self.dirlist:
            if d.dirname == dirname:
                rmdir = d
        if rmdir is None:
            raise IndexError("directory %d cannot be removed" % rmdir)
        self.dirlist.remove(rmdir)
        self.removed_dirlist.append(rmdir)

    def op_dir_link(self, dirname, filename, ino):
        assert(self.initialized)
        for bfsdir in self.dirlist:
            if bfsdir.dirname == dirname:
                bfsdir.link(filename, ino)

    def op_dir_unlink(self, dirname, filename):
        assert(self.initialized)
        for bfsdir in self.dirlist:
            if bfsdir.dirname == dirname:
                bfsdir.unlink(filename)

    def op_file_update(self, ino, size, mtime, extents):
        assert(self.initialized)
        if ino not in self.allocated_ino:
            self.allocated_ino.append(ino)
        if ino in self.deallocated_ino:
            self.deallocated_ino.remove(ino)
        for e in extents:
            if e not in self.allocated_extents:
                self.allocated_extents.append(e)
            if e in self.deallocated_extents:
                self.deallocated_extents.remove(e)
        if ino not in self.ino_to_file_map:
            self.ino_to_file_map[ino] = BlueFSFile(ino, size, mtime, extents)
        else:
            self.ino_to_file_map[ino].update(size, mtime, extents)

    def op_file_remove(self, ino):
        assert(self.initialized)
        if ino in self.allocated_ino:
            self.allocated_ino.remove(ino)
        if ino not in self.deallocated_ino:
            self.deallocated_ino.append(ino)
        assert(ino in self.ino_to_file_map)
        f = self.ino_to_file_map[ino]
        del self.ino_to_file_map[ino]
        for e in f.extents:
            if e in self.allocated_extents:
                self.allocated_extents.remove(e)
            if e not in self.deallocated_extents:
                self.deallocated_extents.append(e)

    def op_jump(self, next_seq, offset):
        assert(self.initialized)
        if self.next_offset == 0:
            self.next_offset = offset
        else:
            logging.debug("There still is an offset. Not jumping.")

    def op_jump_seq(self):
        assert(self.initialized)
        self.jump_seq = True
        logging.debug("Jump next transaction")

    def get_file(self, ino):
        return self.ino_to_file_map[ino]

    def dump_state(self, verbose=False):
        print("----------------")
        print("State of BlueFS:")
        print("----------------")
        print("Allocated areas:")
        for mid, alloc in self.allocated_areas.items():
            print("Alloc %d: 0x%x+0x%x" % (mid, alloc[0], alloc[1]))
        print("----------------")
        print("Transaction log:")
        print("Transaction log --> %s" % self.ino_to_file_map[1])
        print("----------------")
        print("Files:")
        for d in self.dirlist:
            for ino, filename in sorted(d.ino_to_file_map.items(),
                                        key=lambda x: x[0]):
                o = "file %s/%s --> ino %d" % (d.dirname, filename, ino)
                if ino in self.ino_to_file_map:
                    file = self.ino_to_file_map[ino]
                    print("%s --> %s" % (o, file))
                else:
                    print(o)
        print("----------------")
        if verbose:
            print("Allocated extents:")
            for e in sorted(self.allocated_extents, key=lambda x: x.offset):
                print(e)
            print("----------------")
            print("Deallocated extents:")
            for e in sorted(self.deallocated_extents, key=lambda x: x.offset):
                print(e)
            print("----------------")
        print("")

    def print_transactions(self, skipped=False):
        transactions = self.transactions
        header = "BlueFS Transaction List:"
        if skipped:
            transactions = self.skipped_transactions
            header = "BlueFS Skipped Transaction List:"
        print("-" * len(header))
        print(header)
        print("-" * len(header))
        for t in sorted(transactions.items(), key=lambda x: x[1]['seq']):
            print("0x%016x -> seq: 0x%08x: %s" %
                  (t[0], t[1]['seq'], t[1]['transaction']))
        print("")

    def extract_state(self, destination):
        for d in self.dirlist:
            d.mkdir(destination)
        self.superblock.extract_slack(destination)
        for ino, file in self.ino_to_file_map.items():
            filename = None
            for d in self.dirlist:
                filename = d.get_file(ino)
                if filename:
                    file.mkfile(filename, d.dirname, destination, self.osd)

class BlueFSDir(object):
    # Represents a directory in BlueFS.

    def __init__(self, dirname):
        self.dirname = dirname
        self.ino_to_file_map = {}

    def link(self, filename, ino):
        # Add a file to the directory.
        self.ino_to_file_map[ino] = filename

    def unlink(self, filename):
        # Remove a file from the directory.
        rm = None
        for ino in self.ino_to_file_map.keys():
            if self.ino_to_file_map[ino] == filename:
                rm = ino
                break
        if rm:
            del self.ino_to_file_map[rm]

        assert(filename not in self.ino_to_file_map.values())

    def list_files(self):
        # Print all files in the directory with the inode.
        for ino, filename in self.ino_to_file_map.items():
            print("file %s/%s --> ino %d" %
                  (self.dirname, filename, ino))

    def mkdir(self, destination):
        # Create this directory on a certain destination.
        assert(os.path.isdir(destination))
        destinationfull = os.path.join(destination, self.dirname)
        if os.path.isdir(destinationfull):
            shutil.rmtree(destinationfull)
        assert(not os.path.isdir(destinationfull))
        os.makedirs(destinationfull)
        assert(os.path.isdir(destinationfull))

    def get_file(self, ino):
        # Get a file from the directory.
        # Returns None, if not found.
        rc = None
        if ino in self.ino_to_file_map:
            rc = self.ino_to_file_map[ino]
        return rc

    def __eq__(self, other):
        return self.dirname == other.dirname

class BlueFSFile(object):
    # Represents a file in BlueFS.

    def __init__(self, ino, size, mtime, extents):
        self.ino = ino
        self.size = size
        self.mtime = mtime
        self.extents = extents

    def update(self, size, mtime, extents):
        # Update the file metadata.
        self.size = size
        self.mtime = mtime
        self.extents = extents

    def __str__(self):
        extents = " | ".join(str(e) for e in self.extents)
        return "ino: %d, size: %d, mtime: %s, extents: %s" % \
               (self.ino, self.size, self.mtime, extents)

    def mkfile(self, filename, dirname, destination, osd):
        # Extract the file content.
        # Extracts also the slack and will compute the MD5 sum of the file.
        logging.info("Make file %s" % filename)
        destinationfull = os.path.join(destination, dirname)
        assert(os.path.isdir(destinationfull))
        filenamefull = os.path.join(destinationfull, filename)
        filenameslack = os.path.join(destinationfull, "slack_%s" % filename)
        filenamemd5 = os.path.join(destinationfull, "md5_%s" % filename)
        assert(not os.path.exists(filenamefull))
        logging.debug("create %s" % filenamefull)
        logging.debug("- size:  %d (0x%x)" % (self.size, self.size))
        logging.debug("- mtime: %s" % str(self.mtime))

        bsize = 0
        if self.size:
            bsize = self.size

        md5sum = hashlib.md5()

        o = osd
        with open(filenamefull, 'wb') as f, open(filenameslack, 'wb') as s:
            for e in self.extents:
                o.seek(e.offset)
                length = e.length
                logging.info("read extent %s, bsize: 0x%x" % (str(e), bsize))
                if bsize == 0:
                    break
                elif bsize <= length:
                    chunk = o.read(bsize)
                    md5sum.update(chunk)
                    f.write(chunk)
                    slack = length - bsize
                    s.write(o.read(slack))
                    bsize = 0
                else:
                    chunk = o.read(length)
                    md5sum.update(chunk)
                    f.write(chunk)
                    bsize = bsize - length
        with open(filenamemd5, 'w') as f:
            f.write(md5sum.hexdigest())
            f.write('\n')
        assert(bsize == 0)

        ts = self.mtime.timestamp
        os.utime(filenamefull, (ts, ts))

class BlueFSSuperblock(object):
    # Reads the BlueFS superblock at the fixed address 0x1000 of the OSD.

    def __init__(self, handle):
        h = {}
        handle.seek(0x1000)  # Superblock starts at 0x1000
        self.start = handle.tell()
        h['header'] = CephBlockHeader(handle)
        h['uuid'] = CephUUID(handle)
        h['osd_uuid'] = CephUUID(handle)
        h['version'] = CephInteger(handle, 8)
        h['block_size'] = CephInteger(handle, 4)
        h['log_fnode'] = BlueFSFNode(handle)
        assert(handle.tell() == h['header'].end_offset)
        self.end = handle.tell()
        h['crc'] = CephInteger(handle, 4).value
        self.data = h

        slack_length = 0x2000 - handle.tell()  # Reserved block to 8k offset
        assert(slack_length >= 0)
        self.superblock_slack = handle.read(slack_length)

    def pretty_print(self):
        # Print nicely formatted information about the superblock.
        print("------------------------------")
        print("BlueFS Superblock Information:")
        print("------------------------------")
        print("Start at: %s" % hex(self.start))
        print("End at:   %s" % hex(self.end))
        print("------------------------------")

        print("BlueFS UUID: %s" % str(self.data['uuid']))
        print("OSD UUID:    %s" % str(self.data['osd_uuid']))
        print("Version: %d" % self.data['version'].value)
        print("Block size: %s" % hex(self.data['block_size'].value))
        print("Log fnode Information:")
        f = self.data['log_fnode']
        print("- ino: %d" % f.ino)
        print("- size: %d, %s" % (f.size, hex(f.size)))
        print("- mtime: %s" % str(f.mtime))
        print("- prefer block device: %d" % f.prefer_bdev)
        print("- extents:")
        for e in f.extents:
            print("  - %s" % (str(e)))
            print("    (at ~ %d GiB offset)" % (e.offset / 2014**3))
        print("CRC32 checksum: 0x%08x" % self.data['crc'])

        print("------------------------------")
        print("")

    def extract_slack(self, edir):
        # Write the superblock slack to the file "slack_bfssuperblock".
        slackfile = os.path.join(edir, "slack_bfssuperblock")
        with open(slackfile, 'wb') as s:
            s.write(self.superblock_slack)

class BlueFSFNode(CephDataType):
    # fnode data structure of the transaction log and superblock.
    def __init__(self, handle):
        start = handle.tell()
        self.header = CephBlockHeader(handle)
        self.ino = CephVarInteger(handle).value
        self.size = CephVarInteger(handle).value
        self.mtime = CephUTime(handle)
        self.prefer_bdev = CephInteger(handle, 1).value
        self.extents = CephList(handle, BlueFSExtent, None).elements
        end = handle.tell()
        assert(end == self.header.end_offset)
        super().__init__(start, end)

    def __str__(self):
        offsets = ", ".join("0x%x+0x%x" %
                            (x.offset, x.length)
                            for x in self.extents)
        return "ino: %d, size: %d, mtime: %s, extents: %s" % \
               (self.ino, self.size, self.mtime, offsets)

class BlueFSExtent(CephDataType):
    # Extent data structure of the transaction log.
    def __init__(self, handle):
        start = handle.tell()
        self.header = CephBlockHeader(handle)
        self.offset = CephLBA(handle).value
        self.length = CephVarIntegerLowz(handle).value
        self.bdev = CephInteger(handle, 1).value
        end = handle.tell()
        assert(end == self.header.end_offset)
        super().__init__(start, end)

    def __str__(self):
        return "0x%x+0x%x (bdev %d)" % \
               (self.offset, self.length, self.bdev)

    def __eq__(self, other):
        return self.offset == other.offset

class BlueFSTransaction(CephDataType):
    # Decodes a transaction.
    def __init__(self, handle, bluefs):
        start = handle.tell()
        self.operations = []
        self.structlen = CephInteger(handle, 4).value
        end_offset = handle.tell() + self.structlen
        while handle.tell() < end_offset:
            t = BlueFSOperation(handle, bluefs)
            self.operations.append(t)
        end = handle.tell()
        assert(end == end_offset)
        super().__init__(start, end)

    def __str__(self):
        return " | ".join(str(x) for x in self.operations)

class BlueFSOperation(CephDataType):
    # Decodes an operation in an transaction.

    def __init__(self, handle, bluefs):
        start = handle.tell()
        self.op = BlueFSOperationCode(handle)
        op = self.op.value

        if op == 0:
            self.hint = "none"
        elif op == 1:
            self.hint = "init"
            if bluefs:
                bluefs.op_init()
        elif op == 2 or op == 3:
            # ALLOC_ADD
            # ALLOC_RM
            self.id = CephInteger(handle, 1).value
            self.offset = CephInteger(handle, 8).value
            self.length = CephInteger(handle, 8).value
            self.hint = "id: 0x%x, offset: 0x%x, length: 0x%x" % \
                (self.id, self.offset, self.length)
            if bluefs:
                if op == 2:
                    bluefs.op_alloc_add(self.id, self.offset,
                                        self.length)
                else:
                    bluefs.op_alloc_rm(self.id, self.offset,
                                       self.length)
        elif op == 4:
            # DIR_LINK
            self.dir = CephString(handle).value
            self.file = CephString(handle).value
            self.ino = CephInteger(handle, 8).value
            self.hint = "dir: %s, file: %s, ino: %d" % \
                (self.dir, self.file, self.ino)
            if bluefs:
                bluefs.op_dir_link(self.dir, self.file, self.ino)
        elif op == 5:
            # DIR_UNLINK
            self.dir = CephString(handle).value
            self.file = CephString(handle).value
            self.hint = "dir: %s, file: %s" % \
                (self.dir, self.file)
            if bluefs:
                bluefs.op_dir_unlink(self.dir, self.file)
        elif op == 6 or op == 7:
            # DIR_CREATE
            # DIR_REMOVE
            self.dir = CephString(handle).value
            self.hint = "dir: %s" % self.dir
            if bluefs:
                if op == 6:
                    bluefs.op_dir_create(self.dir)
                else:
                    bluefs.op_dir_remove(self.dir)
        elif op == 8:
            # FILE_UPDATE
            self.file = BlueFSFNode(handle)
            self.hint = str(self.file)
            if bluefs:
                bluefs.op_file_update(self.file.ino, self.file.size,
                                      self.file.mtime, self.file.extents)
        elif op == 9:
            # FILE_REMOVE
            self.ino = CephInteger(handle, 8).value
            self.hint = "ino: %d" % self.ino
            if bluefs:
                bluefs.op_file_remove(self.ino)
        elif op == 10:
            # JUMP
            self.next_seq = CephInteger(handle, 8).value
            self.offset = CephInteger(handle, 8).value
            self.hint = "next_seq: %d, offset: 0x%x" % \
                (self.next_seq, self.offset)
            if bluefs:
                bluefs.op_jump(self.next_seq, self.offset)
        elif op == 11:
            # JUMP_SEQ
            self.next_seq = CephInteger(handle, 8).value
            self.hint = "next_seq: %d" % self.next_seq
            if bluefs:
                bluefs.op_jump_seq()

        end = handle.tell()
        super().__init__(start, end)

    def __str__(self):
        return "OP: %s, %s" % (str(self.op), self.hint)

class BlueFSOperationCode(CephInteger):
    translation_map = ["NONE", "INIT", "ALLOC_ADD", "ALLOC_RM",
                       "DIR_LINK", "DIR_UNLINK", "DIR_CREATE",
                       "DIR_REMOVE", "FILE_UPDATE", "FILE_REMOVE",
                       "JUMP", "JUMP_SEQ"]

    def __init__(self, handle):
        super().__init__(handle, 1)

    def __str__(self):
        return BlueFSOperationCode.translation_map[self.value]


#bases
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
import graphviz

#scikit learn
from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import export_graphviz
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, precision_score, confusion_matrix, classification_report
from sklearn.utils import shuffle

#trabalhando com 4 classes alvo
def seta_classe(df, coluna_ordem, coluna_classe):
    quartil_1 = df[coluna_ordem].quantile(q=0.25, interpolation='linear')
    quartil_2 = df[coluna_ordem].quantile(q=0.50, interpolation='linear')
    quartil_3 = df[coluna_ordem].quantile(q=0.75, interpolation='linear')
    quartil_4 = df[coluna_ordem].quantile(q=0.85, interpolation='linear')

    for _, row in df.iterrows():
        if row[coluna_ordem] >= 1 and row[coluna_ordem] <= quartil_1:
            row[coluna_classe] = 1
        elif row[coluna_ordem] > quartil_1 and row[coluna_ordem] <= quartil_2:
            row[coluna_classe] = 2
        elif row[coluna_ordem] > quartil_2 and row[coluna_ordem] <= quartil_3:
            row[coluna_classe] = 3
        elif row[coluna_ordem] > quartil_3 and row[coluna_ordem] <= quartil_4:
            row[coluna_classe] = 4

#trabalhando com 10 faixas para atributos contínuos
def seta_faixa(df, coluna_ordem, coluna_faixa):
    quartil_1 = df[coluna_ordem].quantile(q=0.1, interpolation='linear')
    quartil_2 = df[coluna_ordem].quantile(q=0.2, interpolation='linear')
    quartil_3 = df[coluna_ordem].quantile(q=0.3, interpolation='linear')
    quartil_4 = df[coluna_ordem].quantile(q=0.4, interpolation='linear')
    quartil_5 = df[coluna_ordem].quantile(q=0.5, interpolation='linear')
    quartil_6 = df[coluna_ordem].quantile(q=0.6, interpolation='linear')
    quartil_7 = df[coluna_ordem].quantile(q=0.7, interpolation='linear')
    quartil_8 = df[coluna_ordem].quantile(q=0.8, interpolation='linear')
    quartil_9 = df[coluna_ordem].quantile(q=0.9, interpolation='linear')

    for _, row in df.iterrows():
        if row[coluna_ordem] <= quartil_1:
            row[coluna_faixa] = 1
        elif row[coluna_ordem] > quartil_1 and row[coluna_ordem] <= quartil_2:
            row[coluna_faixa] = 2
        elif row[coluna_ordem] > quartil_2 and row[coluna_ordem] <= quartil_3:
            row[coluna_faixa] = 3
        elif row[coluna_ordem] > quartil_3 and row[coluna_ordem] <= quartil_4:
            row[coluna_faixa] = 4
        elif row[coluna_ordem] > quartil_4 and row[coluna_ordem] <= quartil_5:
            row[coluna_faixa] = 5
        elif row[coluna_ordem] > quartil_5 and row[coluna_ordem] <= quartil_6:
            row[coluna_faixa] = 6
        elif row[coluna_ordem] > quartil_6 and row[coluna_ordem] <= quartil_7:
            row[coluna_faixa] = 7
        elif row[coluna_ordem] > quartil_7 and row[coluna_ordem] <= quartil_8:
            row[coluna_faixa] = 8
        elif row[coluna_ordem] > quartil_8 and row[coluna_ordem] <= quartil_9:
            row[coluna_faixa] = 9
        elif row[coluna_ordem] > quartil_9:
            row[coluna_faixa] = 10

def analise_geral(df, rotulo, titulo):
    #alguns agrupamentos para visualização
    df_top_ano = df.groupby('cd_ano').agg({rotulo: [np.mean]})
    df_top_mes = df.groupby('cd_mes').agg({rotulo: [np.mean]})
    df_pes_ano = df.groupby('cd_ano').agg({'pesocargabruta': [np.sum]})
    df_pes_mes = df.groupby('cd_mes').agg({'pesocargabruta': [np.sum]})
    df_teu_ano = df.groupby('cd_ano').agg({'teu': [np.sum]})
    df_teu_mes = df.groupby('cd_mes').agg({'teu': [np.sum]})

    #graficos gerais
    fig, axes = plt.subplots(nrows=3, ncols=2)
    fig.suptitle(titulo + ' - Agrupamentos', fontsize=16)

    axes[0,0].set_title('Tempo vs Ano')
    axes[0,1].set_title('Tempo vs Mes')
    axes[1,0].set_title('Peso vs Ano')
    axes[1,1].set_title('Peso vs Mes')
    axes[2,0].set_title('TEU vs Ano')
    axes[2,1].set_title('TEU vs Mes')

    df_top_ano.plot(kind='bar', ax=axes[0,0], legend=False, color='cyan')
    df_top_mes.plot(kind='line', ax=axes[0,1], legend=False, color='blue')
    df_pes_ano.plot(kind='bar', ax=axes[1,0], legend=False, color='red')
    df_pes_mes.plot(kind='line', ax=axes[1,1], legend=False, color='green')
    df_teu_ano.plot(kind='bar', ax=axes[2,0], legend=False, color='gray')
    df_teu_mes.plot(kind='line', ax=axes[2,1], legend=False, color='black')

    plt.show()

    #estatistica descritiva
    print('>>> antes')
    print(df[['toperacao', 'tesperaatracacao', 'testadia']].describe())

    #olhando os histogramas e boxplot
    fig, axes = plt.subplots(nrows=3, ncols=2)
    fig.suptitle(titulo + ' - Histogramas e BoxPlot >> antes', fontsize=16)

    df.hist(column='toperacao', ax=axes[0, 0])
    df.boxplot(column='toperacao', ax=axes[0, 1])

    df.hist(column='tesperaatracacao', ax=axes[1, 0])
    df.boxplot(column='tesperaatracacao', ax=axes[1, 1])

    df.hist(column='testadia', ax=axes[2, 0])
    df.boxplot(column='testadia', ax=axes[2, 1])
    plt.show()

    #eliminando outliers e criando 4 classes para trabalho com árvores de decisão - label: toperacao
    df['classe_toperacao'] = 5
    seta_classe(df, 'toperacao', 'classe_toperacao')
    df = df.loc[df['classe_toperacao'] != 5]

    #eliminando outliers e criando 4 classes para trabalho com árvores de decisão - label: tesperaatracacao
    df['classe_tesperaatracacao'] = 5
    seta_classe(df, 'tesperaatracacao', 'classe_tesperaatracacao')
    df = df.loc[df['classe_tesperaatracacao'] != 5]

    #eliminando outliers e criando 4 classes para trabalho com árvores de decisão - label: tesperaatracacao
    df['classe_testadia'] = 5
    seta_classe(df, 'testadia', 'classe_testadia')
    df = df.loc[df['classe_testadia'] != 5]

    print('>>> depois')
    print(df[['toperacao', 'tesperaatracacao', 'testadia']].describe())

    #olhando os histogramas e boxplot
    fig, axes = plt.subplots(nrows=3, ncols=2)
    fig.suptitle(titulo + ' - Histogramas e BoxPlot >> depois', fontsize=16)

    df.hist(column='toperacao', ax=axes[0, 0])
    df.boxplot(column='toperacao', ax=axes[0, 1])

    df.hist(column='tesperaatracacao', ax=axes[1, 0])
    df.boxplot(column='tesperaatracacao', ax=axes[1, 1])

    df.hist(column='testadia', ax=axes[2, 0])
    df.boxplot(column='testadia', ax=axes[2, 1])
    plt.show()

    return df

def classifica_arvore(df, rotulo):
    #arvore de decisão com scikit learn
    classes = ['1', '2', '3', '4']
    atributos = ['cd_berco', 'cd_ano', 'cd_mes','cd_terminal','nacionalidadearmador', 'teu', 'pesocargabruta']

    #df_shuffle = shuffle(df, random_state=1)
    df_shuffle = df.reindex(np.random.permutation(df.index))
    X = df_shuffle[atributos]
    y = df_shuffle[rotulo]                                                                                # classe resultado
    X_treino, X_teste, y_treino, y_teste = train_test_split(X, y, test_size=0.2, random_state=1)                   # 80% para treina e 20% para teste

    #parâmetros básicos
    clf_arvore = DecisionTreeClassifier(criterion='entropy', max_depth=8, random_state=1)                         # entropia como métrica

    #fase de treinamento
    clf_arvore = clf_arvore.fit(X_treino, y_treino)
    y_predicao_treino = clf_arvore.predict(X_treino)
    print("Acuracidade no treino com DT:", accuracy_score(y_treino, y_predicao_treino))
    print("Precisão no treino com DT:", precision_score(y_treino, y_predicao_treino, average='weighted'))

    #matriz de confusão - para treino
    print(confusion_matrix(y_treino, y_predicao_treino))
    print(classification_report(y_treino, y_predicao_treino))

    #fase de testes
    y_predicao_teste = clf_arvore.predict(X_teste)

    #resultado nos testes
    print("Acuracidade no teste com DT:", accuracy_score(y_teste, y_predicao_teste))
    print("Precisão no teste com DT:", precision_score(y_teste, y_predicao_teste, average='weighted'))

    #matriz de confusão - para treino
    print(confusion_matrix(y_teste, y_predicao_teste))
    print(classification_report(y_teste, y_predicao_teste))

    clf_rf = RandomForestClassifier(n_estimators=20, random_state=1)
    clf_rf.fit(X_treino, y_treino)
    y_predicao_treino = clf_rf.predict(X_treino)
    print("Acuracidade no treino com RF:", accuracy_score(y_treino, y_predicao_treino))

    #fase de testes
    y_predicao = clf_rf.predict(X_teste)

    #resultado nos testes
    print("Acuracidade no teste com RF:", accuracy_score(y_teste, y_predicao))

    #visualizando a arvore
    dot_data = export_graphviz(clf_arvore, out_file=None, filled=True, rounded = True, feature_names=atributos, class_names=classes)
    graph = graphviz.Source(dot_data)
    graph.render('arvore', view=True)

#carregando dataset completo - gerado pelas views do sqlite - vw_dataset_cg + vw_dataset_gs + vw_dataset_gl
df_atraca = pd.read_csv('../csv/atraca.csv', low_memory=False)
df_atraca.set_index('idatracacao')

#filtrando Santos / Movimentação de Carga / Longo Curso
df_atraca = df_atraca.loc[df_atraca['cd_portoatracacao'] == 11]      #Santos
df_atraca = df_atraca.loc[df_atraca['cd_tipooperacao'] == 2]         #Movimentação da Carga
df_atraca = df_atraca.loc[df_atraca['cd_tiponavegacaoatracao'] == 3] #Longo Curso

#removendo tempos "inválidos"
df_atraca = df_atraca.loc[df_atraca['toperacao'] > 0]                #Remover tempos zerados
df_atraca = df_atraca.loc[df_atraca['testadia'] > 0]                 #Remover tempos zerados

#separando datasets por tipo de carga devido aos comportamentos diferentes nas atracações
df_cg = df_atraca.loc[df_atraca['cd_tipocarga'] == 1] #carga geral
df_gs = df_atraca.loc[df_atraca['cd_tipocarga'] == 2] #granel sólido
df_gl = df_atraca.loc[df_atraca['cd_tipocarga'] == 3] #granel líquido e gasoso
df_cg_cs = df_cg.loc[df_cg['teu'] == 0]               #somente carga solta
df_cg_cntr = df_cg.loc[df_cg['teu'] > 0]              #somente container

#
#CONTAINERS
#
print('**********************************')
print('*** TRABALHANDO COM CONTAINERS ***')
print('**********************************')

#análise geral
df_cg_cntr = analise_geral(df_cg_cntr, 'toperacao', 'TRABALHANDO COM CONTAINERS')

#classificação com árvores
classifica_arvore(df_cg_cntr, 'classe_toperacao')

#
#CARGA SOLTA
#
print('***********************************')
print('*** TRABALHANDO COM CARGA SOLTA ***')
print('***********************************')

#análise geral
df_cg_cs = analise_geral(df_cg_cs, 'toperacao', 'TRABALHANDO COM CARGA SOLTA')

#classificação com árvores
classifica_arvore(df_cg_cs, 'classe_toperacao')

#
#GRANEL SÓLIDO
#
print('*************************************')
print('*** TRABALHANDO COM GRANEL SÓLIDO ***')
print('*************************************')

#análise geral
df_gs = analise_geral(df_gs, 'toperacao', 'TRABALHANDO COM GRANEL SÓLIDO')

#classificação com árvores
classifica_arvore(df_gs, 'classe_toperacao')

#
#GRANEL LÍQUIDO E GASOSO
#
print('***********************************************')
print('*** TRABALHANDO COM GRANEL LÍQUIDO E GASOSO ***')
print('***********************************************')

#análise geral
df_gl = analise_geral(df_gl, 'toperacao')

#classificação com árvores
classifica_arvore(df_gl, 'classe_toperacao')


from cStringIO import StringIO
import os, re, cgi
from twisted.python import text
from twisted.web import domhelpers, microdom
import latex, tree

class DocbookSpitter(latex.BaseLatexSpitter):

    currentLevel = 1

    def writeNodeData(self, node):
        self.writer(node.data)

    def visitNode_body(self, node):
        self.visitNodeDefault(node)
        self.writer('
'*self.currentLevel)

    def visitNodeHeader(self, node):
        level = int(node.tagName[1])
        difference, self.currentLevel = level-self.currentLevel, level
        self.writer('
'*difference+'
'*-difference)
        if difference<=0:
            self.writer('
\n
')
        self.writer('<title>')
        self.visitNodeDefault(node)

    def visitNode_a_listing(self, node):
        fileName = os.path.join(self.currDir, node.getAttribute('href'))
        self.writer('<programlisting>\n')
        self.writer(cgi.escape(open(fileName).read()))
        self.writer('</programlisting>\n')

    def visitNode_a_href(self, node):
        self.visitNodeDefault(node)

    def visitNode_a_name(self, node):
        self.visitNodeDefault(node)

    def visitNode_li(self, node):
        for child in node.childNodes:
            if getattr(child, 'tagName', None) != 'p':
                new = microdom.Element('p')
                new.childNodes = [child]
                node.replaceChild(new, child)
        self.visitNodeDefault(node)

    visitNode_h2 = visitNode_h3 = visitNode_h4 = visitNodeHeader
    end_h2 = end_h3 = end_h4 = '</title><para />'
    start_title, end_title = '
<title>', '</title><para />'
    start_p, end_p = '<para>', '</para>'
    start_strong, end_strong = start_em, end_em = '<emphasis>', '</emphasis>'
    start_span_footnote, end_span_footnote = '<footnote><para>', '</para></footnote>'
    start_q = end_q = '"'
    start_pre, end_pre = '<programlisting>', '</programlisting>'
    start_div_note, end_div_note = '<note>', '</note>'
    start_li, end_li = '<listitem>', '</listitem>'
    start_ul, end_ul = '<itemizedlist>', '</itemizedlist>'
    start_ol, end_ol = '<orderedlist>', '</orderedlist>'
    start_dl, end_dl = '<variablelist>', '</variablelist>'
    start_dt, end_dt = '<varlistentry><term>', '</term>'
    start_dd, end_dd = '<listitem><para>', '</para></listitem></varlistentry>'


import numpy as np
import matplotlib.pyplot as plt

from src.Shear_flow import shear_flow
import src.AppliedForcesMoments as AFM
from src.StructuralCalculations.Bending import *
from src.AppliedForcesMoments import moment_functions

def shear_stress(shear_flow):

    ''':param t: thickness wing
       :param shear_flow: shear flow in a point

       :return shear stress'''

    t=0.0011

    stress=shear_flow*t

    return stress

def von_mises(sigma_xx,shear_xy):

    ''':param sigma_xx: normal stress caused by My and Mz
       :param shear_xy: shear stress

       :return von mises stress'''

    v_mises_stress=np.sqrt((sigma_xx)**2+3*shear_xy)

    return v_mises_stress

#calculating the shear stress at two important locations rib A and rib C (These were chosen since th FEM predicts highest stresses here)

shear_stresses_C=shear_stress(shear_flow(AFM.distance_dict['x2']+AFM.distance_dict['xa']/2)[0])
shear_stresses_A=shear_stress(shear_flow(AFM.distance_dict['x1'])[0])

#set the aileron specification
ba = 2.661
ha = 0.205
Ca = 0.605
h_stringer = 1.6 * 10 ** (-2)
N=100

#get the coordinates and the full model
coordinates = get_crossectional_coordinates(Ca, ha, h_stringer)
normal_model = FullModel(coordinates, (10e-10, ba), N, transform=False)

#find the moment Mx and My along the x axis
Mz=moment_functions(AFM.distance_dict,AFM.force_dict)[2]
My=moment_functions(AFM.distance_dict,AFM.force_dict)[1]

#determine the distance of rib A and C from the origin
distance_A=AFM.distance_dict['x2']+AFM.distance_dict['xa']/2
distance_C=AFM.distance_dict['x1']

#determine the corresponding index of rib A and C
index_A=round(distance_A/AFM.distance_dict['ba']*N)
index_C=round(distance_C/AFM.distance_dict['ba']*N)

#calculate the von mises stresses at rib location A and C per each boom
von_mises_stressC=von_mises(NumericalBending(normal_model[index_C]).calculate_bending_stress(Mz,My),shear_stresses_C)[0]
von_mises_stressA=von_mises(NumericalBending(normal_model[index_A]).calculate_bending_stress(Mz,My),shear_stresses_A)[0]
# i don't know why but it saves the von mises as many times as we have booms, so i take the first element of the array

#plot the stresses in order to make a comparison with the validation model
fig=plt.figure(figsize=(15,7))
# added conversion to kN/mm^2
ax=fig.add_subplot(211)
ax.set_title('Aileron stresses at rib C ULC1')
ribC=ax.scatter(normal_model[index_C].get_all_coordinates()[:,2], normal_model[index_C].get_all_coordinates()[:,1], c=von_mises_stressC*10**(-9))
cbar = plt.colorbar(ribC)
cbar.ax.set_ylabel('avg Von Mises Stress at rib C (kN/$mm^2$)', rotation=90)

ax=fig.add_subplot(212)
ax.set_title('Aileron stresses at rib A ULC1')
ribA=ax.scatter(normal_model[index_A].get_all_coordinates()[:,2], normal_model[index_A].get_all_coordinates()[:,1], c=von_mises_stressA*10**(-9))
cbar = plt.colorbar(ribA)
cbar.ax.set_ylabel('avg Von Mises Stress at rib A (kN/$mm^2$)', rotation=90)

plt.show()
distances=[0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45,0.5,0.55,0.6]

twist=0
twists=[]
for distance in distances:
    twist=twist+shear_flow(distance)[1]*distance
    twists.append(twist)

plt.plot(twists)
plt.show()


"""Default settings for tests."""

import os
from pathlib import Path

import numpy as np

###################################################################################################
###################################################################################################

# Define general settings for test simulations
FS = 100
N_SECONDS = 10.0

FS_ODD = 123
N_SECONDS_ODD = 1/7

N_SECONDS_CYCLE = 1.0

# Define parameter options for test simulations
FREQ1 = 10
FREQ2 = 25
FREQ_SINE = 1
FREQS_LST = [8, 12, 1]
FREQS_ARR = np.array([5, 10, 15])
EXP1 = -1
EXP2 = -2
KNEE = 100

# Define settings for testing analyses
F_RANGE = (FREQ1-2, FREQ1+2)

# Define error tolerance levels for test comparisons
EPS = 10**(-10)
EPS_FILT = 10**(-3)

# Set paths for test files
TESTS_PATH = Path(os.path.abspath(os.path.dirname(__file__)))
BASE_TEST_FILE_PATH = TESTS_PATH / 'test_files'
TEST_PLOTS_PATH = os.path.join(BASE_TEST_FILE_PATH, 'plots')


from flask import url_for
import pytest

endpoints = ('user.register', 'user.logout', 'user.login',
             'equipment.equipment', 'vizit.start_page')

@pytest.mark.parametrize('endpoint', endpoints)
def test_pages_healthcheck(client, init_database, endpoint):
    """Проверка эндпойнтов на доступность."""
    response = client.get(url_for(endpoint), follow_redirects=True)
    assert response.status_code == 200


#!/usr/bin/python

__author__ = 'ilia'
from color.colorize import Color
import rpm
import argparse

def main():
    parser = argparse.ArgumentParser("rpm like behavior")

    subparsers = parser.add_subparsers(help="file-like option")

    parser_search = subparsers.add_parser('search', help="search")
    parser_search.add_argument('pattern', help='enter file pattern to find', nargs=1)

    args = parser.parse_args()

    pattern = args.pattern[0]

    ts = rpm.TransactionSet()
    mi = ts.dbMatch()
    for h in mi:
        package_name = h[rpm.RPMTAG_NAME]
        files = h[rpm.RPMTAG_FILENAMES]
        matches = []
        for fln in files:
            if pattern in fln:
                matches.append(fln.replace(pattern, Color(pattern).as_red()))

        if matches:
            print ('%s:' % Color(package_name).as_green())
            for match in matches:
                print ('\t%s' % match)

if __name__ == "__main__":
    main()


bif = "ball.png"
bgif = "bg.jpg"

import pygame, sys
from pygame.locals import *
pygame.init()

randomColor = [100, 240, 160]
color = randomColor

size = [600, 400]

move = [0, 0]
pos = [0, 0]
speed = 0.5

screen = pygame.display.set_mode(size)

bg = pygame.image.load(bgif).convert()
ball = pygame.image.load(bif).convert_alpha()

while True:
    for event in pygame.event.get():
        if event.type == QUIT:
            pygame.quit()
            sys.exit()
        if event.type == KEYDOWN:
            if event.key == K_w:
                move[1] -= speed
            elif event.key == K_a:
                move[0] -= speed
            elif event.key == K_s:
                move[1] += speed
            elif event.key == K_d:
                move[0] += speed
        elif event.type == KEYUP:
            if event.key == K_w:
                move[1] = 0
            elif event.key == K_a:
                move[0] = 0
            elif event.key == K_s:
                move[1] = 0
            elif event.key == K_d:
                move[0] = 0

    pos[0] += move[0]
    pos[1] += move[1]

    screen.blit(bg, (0, 0))
    screen.blit(ball, pos)

    pygame.display.update()


import cv2
import numpy as np

# Importing the Libraries
image_file = r"F:\Fall 2021 NTNU\Computer Vision NTU\Chapter-7\HomeWork-10\lena.bmp"
gray_image = cv2.imread(image_file, 0)  # Read Image in Grayscale

def convolve(img, kernel):
    row_img, col_img = img.shape
    row_k, col_k = kernel.shape
    res = 0
    for h_row in range(row_img):
        for w_col in range(col_img):
            if 0 <= row_img - h_row - 1 < row_k and 0 <= col_img - w_col - 1 < col_k:
                res += img[h_row, w_col] * kernel[row_img - h_row - 1, col_img - w_col - 1]
    return res

def convolution_image(gray_image, kernel, threshold):
    # To avoid index out of bounds Error
    mask_img = np.zeros((gray_image.shape[0] - kernel.shape[0] + 1, gray_image.shape[1] - kernel.shape[1] + 1))
    for h_row in range(mask_img.shape[0]):
        for w_col in range(mask_img.shape[1]):
            value = convolve(gray_image[h_row:h_row + kernel.shape[0], w_col:w_col + kernel.shape[1]], kernel)
            if value >= threshold:
                mask_img[h_row, w_col] = 1
            elif value <= -threshold:
                mask_img[h_row, w_col] = -1
            else:
                mask_img[h_row, w_col] = 255

    return mask_img

def zero_crossing(masked_image):
    # 8 neighborhood zero crossing
    zero_cross_image = np.copy(masked_image)
    for h_row in range(0, masked_image.shape[0] - 1):
        for w_col in range(0, masked_image.shape[1] - 1):
            if masked_image[h_row, w_col] <= 0:
                zero_cross_image[h_row, w_col] = 255
            elif masked_image[h_row, w_col] == 1:
                if masked_image[h_row - 1, w_col - 1] == -1 or masked_image[h_row, w_col - 1] == -1 or masked_image[
                    h_row + 1, w_col - 1] == -1 \
                        or masked_image[h_row - 1, w_col] == -1 or masked_image[h_row + 1, w_col] == -1 or masked_image[
                    h_row - 1, w_col + 1] == -1 \
                        or masked_image[h_row, w_col + 1] == -1 or masked_image[h_row + 1, w_col + 1] == -1:
                    zero_cross_image[h_row, w_col] = 0
                else:
                    zero_cross_image[h_row,w_col] = 255
    return zero_cross_image

if __name__ == "__main__":
    # Laplace mask 1
    laplace_mask1_kernel = np.array([
        [0, 1, 0],
        [1, -4, 1],
        [0, 1, 0]
    ])
    laplace_mask1_image = convolution_image(gray_image, laplace_mask1_kernel, 15)
    zero_crossing_laplace_mask_image = zero_crossing(laplace_mask1_image)
    cv2.imshow('Zero Crossing Image', zero_crossing_laplace_mask_image)
    cv2.waitKey(0)
    # Laplace Mask 2
    laplace_mask2_kernel = np.array([
        [1, 1, 1],
        [1, -8, 1],
        [1, 1, 1]
    ]) / 3
    laplace_mask2_image = convolution_image(gray_image, laplace_mask2_kernel, 15)
    zero_crossing_laplace_mask2_image = zero_crossing(laplace_mask2_image)
    cv2.imshow('Zero Crossing laplace mask 2 Image', zero_crossing_laplace_mask2_image)
    cv2.waitKey(0)
    # Minimum Variance Laplacian
    minimum_variance_laplacian_kernel = np.array([
        [2., -1, 2],
        [-1, -4, -1],
        [2, -1, 2]
    ]) / 3
    minimum_variance_laplacian_image = convolution_image(gray_image, minimum_variance_laplacian_kernel, 15)
    zero_crossing_mvl_image = zero_crossing(minimum_variance_laplacian_image)
    cv2.imshow('Zero Crossing Minimum Variance Laplacian Image', zero_crossing_mvl_image)
    cv2.waitKey(0)
    # Laplace of Gaussian
    laplace_of_gaussian_kernel = np.array([
        [0, 0, 0, -1, -1, -2, -1, -1, 0, 0, 0],
        [0, 0, -2, -4, -8, -9, -8, -4, -2, 0, 0],
        [0, -2, -7, -15, -22, -23, -22, -15, -7, -2, 0],
        [-1, -4, -15, -24, -14, -1, -14, -24, -15, -4, -1],
        [-1, -8, -22, -14, 52, 103, 52, -14, -22, -8, -1],
        [-2, -9, -23, -1, 103, 178, 103, -1, -23, -9, -2],
        [-1, -8, -22, -14, 52, 103, 52, -14, -22, -8, -1],
        [-1, -4, -15, -24, -14, -1, -14, -24, -15, -4, -1],
        [0, -2, -7, -15, -22, -23, -22, -15, -7, -2, 0],
        [0, 0, -2, -4, -8, -9, -8, -4, -2, 0, 0],
        [0, 0, 0, -1, -1, -2, -1, -1, 0, 0, 0]
    ])
    laplace_of_gaussian_image = convolution_image(gray_image, laplace_of_gaussian_kernel, 3000)
    zero_crossing_log_image = zero_crossing(laplace_of_gaussian_image)
    cv2.imshow('Zero Crossing Laplace of Gaussian Image', zero_crossing_log_image)
    cv2.waitKey(0)
    # Difference of Gaussian
    difference_of_gaussian_kernel = np.array([
        [-1, -3, -4, -6, -7, -8, -7, -6, -4, -3, -1],
        [-3, -5, -8, -11, -13, -13, -13, -11, -8, -5, -3],
        [-4, -8, -12, -16, -17, -17, -17, -16, -12, -8, -4],
        [-6, -11, -16, -16, 0, 15, 0, -16, -16, -11, -6],
        [-7, -13, -17, 0, 85, 160, 85, 0, -17, -13, -7],
        [-8, -13, -17, 15, 160, 283, 160, 15, -17, -13, -8],
        [-7, -13, -17, 0, 85, 160, 85, 0, -17, -13, -7],
        [-6, -11, -16, -16, 0, 15, 0, -16, -16, -11, -6],
        [-4, -8, -12, -16, -17, -17, -17, -16, -12, -8, -4],
        [-3, -5, -8, -11, -13, -13, -13, -11, -8, -5, -3],
        [-1, -3, -4, -6, -7, -8, -7, -6, -4, -3, -1],
    ])
    difference_of_gaussian_image = convolution_image(gray_image, difference_of_gaussian_kernel, 1)
    zero_crossing_dog_image = zero_crossing(difference_of_gaussian_image)
    cv2.imshow('Zero Crossing Difference of Gaussian Image', zero_crossing_dog_image)
    cv2.waitKey(0)


#! /usr/bin/python

import numpy as np
import sympy as sym
from opty.direct_collocation import Problem
import matplotlib.pyplot as plt

duration = 10.0
num_nodes = 500

interval_value = duration / (num_nodes - 1)

# Symbolic equations of motion
t = sym.symbols('t')
x, y, theta, vpx, vpy, L2, L3, L4, L5 = sym.symbols('x, y, theta, vpx, vpy, L2, L3, L4, L5', cls=sym.Function)

state_symbols = (x(t), y(t), theta(t))
specified_symbols = (vpx(t), vpy(t))

def get_pushing_eom(rox1_0, roy1_0):
    c = 0.38259785823210635
    r1sq = rox1_0**2 + roy1_0**2
    rox1 = (sym.cos(theta(t))*rox1_0 - sym.sin(theta(t))*roy1_0)
    roy1 = (sym.sin(theta(t))*rox1_0 + sym.cos(theta(t))*roy1_0)
    rox1roy1 = (rox1_0*roy1_0*sym.cos(2*theta(t)) + 0.5*(rox1_0*rox1_0 - roy1_0*roy1_0)*sym.sin(2*theta(t)))
    rox1sq = (0.5*(r1sq + (rox1_0*rox1_0 - roy1_0*roy1_0)*sym.cos(2*theta(t)) - 2*rox1_0*roy1_0*sym.sin(2*theta(t))))
    roy1sq = (0.5*(r1sq + (roy1_0*roy1_0 - rox1_0*rox1_0)*sym.cos(2*theta(t)) + 2*rox1_0*roy1_0*sym.sin(2*theta(t))))
    eom1 = sym.Matrix([x(t).diff() - ((c*c + rox1sq)*vpx(t) + rox1roy1*vpy(t))/(c*c + r1sq),
                      y(t).diff() - (rox1roy1*vpx(t) + (c*c + roy1sq)*vpy(t))/(c*c + r1sq),
                      theta(t).diff() - (rox1*vpy(t) - roy1*vpx(t))/(c*c + r1sq)])
    return eom1

eom2 = get_pushing_eom(0.0, -0.5)
eom3 = get_pushing_eom(0.5, 0.0)
eom4 = get_pushing_eom(0.0, 0.5)
eom5 = get_pushing_eom(-0.5, 0.0)

eom = L2(t)*eom2 + L3(t)*eom3 + L4(t)*eom4 + L5(t)*eom5

def obj(free):
    """Minimize the sum of the squares of the control input."""
    u = free[2 * num_nodes:]
    return interval_value * np.sum(u**2)

def obj_grad(free):
    grad = np.zeros_like(free)
    grad[2 * num_nodes:] = 2.0 * interval_value * free[2 * num_nodes:]
    return grad

# Specify the symbolic instance constraints, i.e. initial and end
# conditions.
instance_constraints = (x(0.0),
                        x(duration) - 3.0,
                        y(0.0),
                        y(duration) - 1.0,
                        theta(0.0),
                        theta(duration))

bounds = {
    vpx(t): (-1, 1),
    vpy(t): (-1, 1),

    ((vpx(t)*1.0 + vpy(t)*0.0)**2 - 0.030153689607045803*(vpx(t)**2 + vpy(t)**2)*1.0)*L2(t): (-float("inf"), 0),
    (-vpx(t)*0.0 + vpy(t)*1.0)*L2(t): (0, float("inf")),

    ((vpx(t)*0.0 + vpy(t)*1.0)**2 - 0.030153689607045803*(vpx(t)**2 + vpy(t)**2)*1.0)*L3(t): (-float("inf"), 0),
    (-vpx(t)*1.0 + vpy(t)*0.0)*L3(t): (0, float("inf")),

    ((vpx(t)*-1.0 + vpy(t)*0.0)**2 - 0.030153689607045803*(vpx(t)**2 + vpy(t)**2)*1.0)*L4(t): (-float("inf"), 0),
    (-vpx(t)*0.0 + vpy(t)*-1.0)*L4(t): (0, float("inf")),

    ((vpx(t)*0.0 + vpy(t)*-1.0)**2 - 0.030153689607045803*(vpx(t)**2 + vpy(t)**2)*1.0)*L5(t): (-float("inf"), 0),
    (-vpx(t)*-1.0 + vpy(t)*0.0)*L5(t): (0, float("inf")),
}

known_trajectory_map = {
    L2(t): np.concatenate([np.ones(num_nodes/2),np.zeros(num_nodes/2)]),
    L3(t): np.concatenate([np.zeros(num_nodes/2),np.zeros(num_nodes/2)]),
    L4(t): np.concatenate([np.zeros(num_nodes/2),np.zeros(num_nodes/2)]),
    L5(t): np.concatenate([np.zeros(num_nodes/2),np.ones(num_nodes/2)]),
}

# Create an optimization problem.
prob = Problem(obj, obj_grad, eom, state_symbols, num_nodes, interval_value,
               instance_constraints=instance_constraints,
               bounds=bounds,
               known_trajectory_map=known_trajectory_map)

# Use a random positive initial guess.
initial_guess = np.random.randn(prob.num_free)

# Find the optimal solution.
solution, info = prob.solve(initial_guess)

# Make some plots
prob.plot_trajectories(solution)

plt.show()


#!/usr/bin/env python3

import sympy

def complex_symplify(A):
	while True:
		last_A = A
		A = A.subs(i*j, k)
		A = A.subs(j*k, i)
		A = A.subs(k*i, j)
		A = A.subs(j*i, -k)
		A = A.subs(k*j, -i)
		A = A.subs(i*k, -j)
		if A == last_A:
			break
	return A

sympy.var("x y z w a b")

i=sympy.Symbol('i', commutative=False)
j=sympy.Symbol('j', commutative=False)
k=sympy.Symbol('k', commutative=False)

A = (w+z*k)*(a*i+b*j)*(w-z*k)
A = sympy.expand(A)
print(A)
A = complex_symplify(A)
print(A)
A = sympy.simplify(A)
print(A)


#!/usr/bin/env python

"""The setup script."""

from setuptools import setup, find_packages

file_requirements = open('requirements.txt', 'r')
requirements = file_requirements.readlines()
requirements = [line.replace("\n", "") for line in requirements]

setup(
    author="Camilo Caceres F",
    python_requires='>=3.8',
    classifiers=[
        'Development Status :: 2 - Pre-Alpha',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
        'Natural Language :: English',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.8',
    ],
    description="Image API converter from blackboard to B/W.",
    install_requires=requirements,
    license="GNU General Public License v3",
    include_package_data=True,
    keywords='blackboard_photo2cleanimage',
    name='blackboard_photo2cleanimage',
    packages=find_packages(include=['blackboard_photo2cleanimage', 'blackboard_photo2cleanimage.*']),
    test_suite='tests',
    url='https://github.com/camilo-cf/blackboard_photo2cleanimage',
    version='0.1.0',
    zip_safe=False,
)


import os
import openai
from datetime import datetime
import pyttsx3
import speech_recognition as sr
from config import apikey

def say(text):
    engine = pyttsx3.init(driverName='espeak')  # Linux voice engine
    engine.setProperty('voice', 'm1')
    engine.setProperty('rate', 150)
    engine.say(text)
    engine.runAndWait()

def save_conversation(convo_id, chat_dict):
    if not os.path.exists("Openai"):
        os.mkdir("Openai")

    timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
    filepath = f"Openai/convo_{convo_id}_{timestamp}.txt"

    with open(filepath, "w") as f:
        f.write(f"User: {chat_dict[convo_id]['user_query']}\nAI: {chat_dict[convo_id]['ai_response']}")

    print(f"Conversation saved to: {filepath}")

def load_conversations():
    chat_dict = {}
    folder_path = 'Openai'

    try:
        # Iterate through files in the Openai folder
        for file_name in os.listdir(folder_path):
            if file_name.endswith(".txt"):
                # Extract conversation ID and timestamp from the file name
                parts = file_name.split("_")
                convo_id = int(parts[1])
                timestamp = parts[2].split(".")[0]

                # Read the content of the text file
                file_path = os.path.join(folder_path, file_name)
                with open(file_path, 'r') as file:
                    content = file.read()

                # Update chat_dict with the conversation details
                chat_dict[convo_id] = {
                    'user_query': content.split("User: ")[1].split("\n")[0].strip(),
                    'ai_response': content.split("AI: ")[1].strip(),
                    'timestamp': timestamp
                }

        return chat_dict

    except FileNotFoundError:
        # Return an empty dictionary if no saved conversations are found
        return {}

def recall_conversation(convo_id, chat_dict):
    if convo_id in chat_dict:
        print(f"Recalling conversation with ID {convo_id}:")
        print(f"User: {chat_dict[convo_id]['user_query']}")
        print(f"AI Response: {chat_dict[convo_id]['ai_response']}")
        say(f"Recalling conversation with ID {convo_id}. You said: {chat_dict[convo_id]['user_query']}. The AI responded: {chat_dict[convo_id]['ai_response']}")
    else:
        print(f"No conversation found with ID {convo_id}")
        say(f"No conversation found with ID {convo_id}")

def chat(query, chat_dict):
    try:
        print(f"API Key: {apikey}")
        openai.api_key = apikey
        # Load existing conversations at the beginning
        chat_dict = load_conversations()

        # Generate a unique ID for the conversation
        convo_id = len(chat_dict) + 1

        # Update chat_dict with the current conversation
        #print("ChatDict After Adding Conversation:", chat_dict)

        chat_dict[convo_id] = {
            'user_query': query,
            'ai_response': '',
            'timestamp': datetime.now().strftime("%Y%m%d%H%M%S")
        }

        # Use a separate variable for saving to a file
        save_convo_id = convo_id

        # Construct the prompt for OpenAI
        prompt = ""
        for c_id, convo in chat_dict.items():
            prompt += f"User: {convo['user_query']}\nAI: {convo['ai_response']}\n"

        # Add the current user query to the prompt
        prompt += f"User: {query}\n"

        #print("Prompt to OpenAI:")
        print(prompt)

        print(f"ID of the current conversation: {convo_id}")
        # print("ChatDict After Adding Conversation:", chat_dict)

        response = openai.Completion.create(
            model="gpt-3.5-turbo-instruct-0914",
            prompt=prompt,
            temperature=0.7,
            max_tokens=256,
            top_p=1,
            frequency_penalty=0,
            presence_penalty=0
        )

        # Update the chat_dict with the AI's response
        chat_dict[convo_id]['ai_response'] = response['choices'][0]['text']

        # Print and say the response
        response_text = chat_dict[convo_id]['ai_response']
        print(f"AI: {response_text}")
        say(response_text)

        # Save the conversation to a file using the separate variable
        save_conversation(save_convo_id, chat_dict)

    except Exception as e:
        print(f"An error occurred while interacting with OpenAI: {e}")

# Example usage
# Initialize chat dictionary
chat_dict = load_conversations()

# Print existing conversation IDs
#print("Existing Conversation IDs:", list(chat_dict.keys()))

# Initialize recognizer
recognizer = sr.Recognizer()

# Example usage

# Example usage
while True:
    with sr.Microphone() as source:
        print("Please start speaking...")

        # Adjust for ambient noise
        recognizer.adjust_for_ambient_noise(source)

        try:
            # Listen to the speech and store it in audio_text variable
            audio_text = recognizer.listen(source)
            print("Audio recording complete.")

            # Use Google Speech Recognition to convert audio to text
            recognized_text = recognizer.recognize_google(audio_text)

            if recognized_text:
                print("User: " + recognized_text)

                # Check if the user wants to stop the conversation
                if recognized_text.lower() == "stop":
                    print("Conversation stopped.")
                    break  # Exit the loop

                # Check if the user wants to recall a past conversation
                elif recognized_text.lower().startswith("recall id"):
                    parts = recognized_text.lower().split("recall id ")
                    if len(parts) == 2:
                        convo_id = int(parts[1].strip())

                        recall_conversation(convo_id, chat_dict)
                        continue  # Skip the rest of the loop to avoid processing as a new query

                # Update chat_dict using the chat function
                chat(recognized_text, chat_dict)

            else:
                print("Sorry, I did not understand what you said.")

        except sr.UnknownValueError:
            print("Google Speech Recognition could not understand audio.")

        except sr.RequestError as e:
            print(f"Could not request results from Google Speech Recognition service; {e}")


import folium
from Grid import Region
import pandas as pd
from grid_visualize import get_index_high_ratio_24h

GRID_CONGESTED_THREASHOLD = 0.3

base_map = folium.Map(location=(42.30, -71.05), zoom_start=13)
congested_indexes = get_index_high_ratio_24h(GRID_CONGESTED_THREASHOLD)
reg = Region(42.70, -71.50, 42.10, -70.75)
reg.add_ratio()

for label in congested_indexes:
	grid = reg.find_grid(label)

	grid_center = ((grid.x + grid.m) / 2, (grid.y + grid.n) / 2)
	folium.Marker(grid_center, icon=folium.Icon(color='blue')).add_to(base_map)

grids = [reg.find_grid(label) for label in congested_indexes]
maxi_g = max(grids, key=lambda x: x.congestion_ratio)
maxi_center = ((maxi_g.x + maxi_g.m) / 2, (maxi_g.y + maxi_g.n) / 2)
folium.Marker(maxi_center, popup='most congested', icon=folium.Icon(color='red')).add_to(base_map)

base_map.save('q1.html')


from django.urls import path, include
from rest_framework.routers import SimpleRouter
from rest_framework.urlpatterns import format_suffix_patterns
from product.views.viewsdrf import (
    ProductViewSet,
    CategorysViewSet,
    SubcategorysViewSet,
    AddStarRatingViewSet,
    ReviewCreateViewSet,
)

router_product = SimpleRouter()
router_product.register('product', ProductViewSet, basename='product')
router_product.register('category', CategorysViewSet, basename='category')
router_product.register('subcategory', SubcategorysViewSet, basename='subcategory')

urlpatterns = format_suffix_patterns ([
    path('', include(router_product.urls)),
    path('review/', ReviewCreateViewSet.as_view({'post': 'create'})),
    path('rating/', AddStarRatingViewSet.as_view({'post': 'create'})),
])


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
        ('research', '0001_initial'),
    ]

    operations = [
        migrations.AddField(
            model_name='studyparticipation',
            name='created',
            field=models.DateTimeField(null=True, blank=True),
        ),
        migrations.AddField(
            model_name='studyparticipation',
            name='created_by',
            field=models.ForeignKey(related_name='created_research_studyparticipation_subrecords', blank=True, to=settings.AUTH_USER_MODEL, null=True),
        ),
        migrations.AddField(
            model_name='studyparticipation',
            name='updated',
            field=models.DateTimeField(null=True, blank=True),
        ),
        migrations.AddField(
            model_name='studyparticipation',
            name='updated_by',
            field=models.ForeignKey(related_name='updated_research_studyparticipation_subrecords', blank=True, to=settings.AUTH_USER_MODEL, null=True),
        ),
    ]


from torch.utils.data import DataLoader
from torch.optim import lr_scheduler
from torchvision import transforms
from data import ImageFilelist, ImageFolder
import torch
import os
import math
import torchvision.utils as vutils
import yaml
import torch.nn.init as init

def get_all_data_loaders(conf):
    batch_size = conf['batch_size']
    num_workers = conf['num_workers']
    if 'new_size' in conf:
        new_size_a = new_size_b = conf['new_size']
    else:
        new_size_a = conf['new_size_a']
        new_size_b = conf['new_size_b']
    height = conf['crop_image_height']
    width = conf['crop_image_width']

    if 'data_root' in conf:
        train_loader_a = get_data_loader_folder(
            # os.path.join(
            #     conf['data_root'],
            #     'trainA'),
            conf['trainA'],
            batch_size,
            True,
            new_size_a,
            height,
            width,
            num_workers,
            True)
        test_loader_a = get_data_loader_folder(
            # os.path.join(
            #     conf['data_root'],
            #     'testA'),
            conf['testA'],
            batch_size,
            False,
            new_size_a,
            new_size_a,
            new_size_a,
            num_workers,
            True)
        train_loader_b = get_data_loader_folder(
            # os.path.join(
            #     conf['data_root'],
            #     'trainB'),
            conf['trainB'],
            batch_size,
            True,
            new_size_b,
            height,
            width,
            num_workers,
            True)
        test_loader_b = get_data_loader_folder(
            # os.path.join(
            #     conf['data_root'],
            #     'testB'),
            conf['testB'],
            batch_size,
            False,
            new_size_b,
            new_size_b,
            new_size_b,
            num_workers,
            True)
    else:
        train_loader_a = get_data_loader_list(
            conf['data_folder_train_a'],
            conf['data_list_train_a'],
            batch_size,
            True,
            new_size_a,
            height,
            width,
            num_workers,
            True)
        test_loader_a = get_data_loader_list(
            conf['data_folder_test_a'],
            conf['data_list_test_a'],
            batch_size,
            False,
            new_size_a,
            new_size_a,
            new_size_a,
            num_workers,
            True)
        train_loader_b = get_data_loader_list(
            conf['data_folder_train_b'],
            conf['data_list_train_b'],
            batch_size,
            True,
            new_size_b,
            height,
            width,
            num_workers,
            True)
        test_loader_b = get_data_loader_list(
            conf['data_folder_test_b'],
            conf['data_list_test_b'],
            batch_size,
            False,
            new_size_b,
            new_size_b,
            new_size_b,
            num_workers,
            True)
    return train_loader_a, train_loader_b, test_loader_a, test_loader_b

def get_data_loader_list(root, file_list, batch_size, train, new_size=None,
                         height=256, width=256, num_workers=4, crop=True):
    transform_list = [transforms.ToTensor(),
                      transforms.Normalize((0.5, 0.5, 0.5),
                                           (0.5, 0.5, 0.5))]
    transform_list = [
        transforms.RandomCrop(
            (height, width))] + transform_list if crop else transform_list
    transform_list = [transforms.Resize(
        new_size)] + transform_list if new_size is not None else transform_list
    transform_list = [transforms.RandomHorizontalFlip()] + \
        transform_list if train else transform_list
    transform = transforms.Compose(transform_list)
    dataset = ImageFilelist(root, file_list, transform=transform)
    loader = DataLoader(
        dataset=dataset,
        batch_size=batch_size,
        shuffle=train,
        drop_last=True,
        num_workers=num_workers)
    return loader

def get_data_loader_folder(
        input_folder,
        batch_size,
        train,
        new_size=None,
        height=256,
        width=256,
        num_workers=4,
        crop=True,
        return_path=False):
    transform_list = [transforms.ToTensor(),
                      transforms.Normalize((0.5, 0.5, 0.5),
                                           (0.5, 0.5, 0.5))]
    transform_list = [
        transforms.RandomCrop(
            (height, width))] + transform_list if crop else transform_list
    transform_list = [transforms.Resize(
        new_size)] + transform_list if new_size is not None else transform_list
    transform_list = [transforms.RandomHorizontalFlip()] + \
        transform_list if train else transform_list
    # 加入ColorJitter用于测试，记得改回来
    # transform_list = [transforms.ColorJitter(hue=0.2)] + transform_list
    transform = transforms.Compose(transform_list)
    dataset = ImageFolder(
        input_folder,
        transform=transform,
        return_paths=return_path)
    print("load ", len(dataset), " images")
    loader = DataLoader(
        dataset=dataset,
        batch_size=batch_size,
        shuffle=train,
        drop_last=True,
        num_workers=num_workers)
    return loader

def get_config(config):
    with open(config, 'r') as stream:
        return yaml.load(stream)

def __write_images(image_outputs, display_image_num, file_name):
    # expand gray-scale images to 3 channels
    image_outputs = [images.expand(-1, 3, -1, -1) for images in image_outputs]
    image_tensor = torch.cat([images[:display_image_num]
                              for images in image_outputs], 0)
    image_grid = vutils.make_grid(
        image_tensor.data,
        nrow=display_image_num,
        padding=0,
        normalize=True)
    vutils.save_image(image_grid, file_name, nrow=1)

def write_2images(image_outputs, display_image_num, image_directory, postfix):
    # n = len(image_outputs)
    __write_images(image_outputs, display_image_num, '%s/gen_a2b_%s.jpg' %
                   (image_directory, postfix))
    # __write_images(image_outputs[n //
    #                              2:n], display_image_num, '%s/gen_b2a_%s.jpg' %
    #                (image_directory, postfix))

def prepare_sub_folder(output_directory):
    image_directory = os.path.join(output_directory, 'images')
    if not os.path.exists(image_directory):
        print("Creating directory: {}".format(image_directory))
        os.makedirs(image_directory)
    checkpoint_directory = os.path.join(output_directory, 'checkpoints')
    if not os.path.exists(checkpoint_directory):
        print("Creating directory: {}".format(checkpoint_directory))
        os.makedirs(checkpoint_directory)
    return checkpoint_directory, image_directory

def write_loss(iterations, trainer, train_writer):
    members = [
        attr for attr in dir(trainer) if not callable(getattr(trainer, attr))
                                         and not attr.startswith("__")
                                         and ('loss' in attr or 'grad' in attr or 'nwd' in attr)]
    loss_gen_dir = {}
    loss_dis_dir = {}
    for m in members:
        if 'gen' in m:
            loss_gen_dir[m] = getattr(trainer, m)
        elif 'dis' in m:
            loss_dis_dir[m] = getattr(trainer, m)
    train_writer.add_scalars('Generator', loss_gen_dir, iterations + 1)
    train_writer.add_scalars('Discriminator', loss_dis_dir, iterations + 1)

# Get model list for resume
def get_model_list(dirname, key):
    if os.path.exists(dirname) is False:
        return None
    gen_models = [
        os.path.join(
            dirname,
            f) for f in os.listdir(dirname) if os.path.isfile(
            os.path.join(
                dirname,
                f)) and key in f and ".pt" in f]
    if gen_models is None:
        return None
    gen_models.sort()
    last_model_name = gen_models[-1]
    return last_model_name

def get_scheduler(optimizer, hyperparameters, iterations=-1):
    if 'lr_policy' not in hyperparameters or hyperparameters['lr_policy'] == 'constant':
        scheduler = None  # constant scheduler
    elif hyperparameters['lr_policy'] == 'step':
        scheduler = lr_scheduler.StepLR(
            optimizer,
            step_size=hyperparameters['step_size'],
            gamma=hyperparameters['gamma'],
            last_epoch=iterations)
    else:
        return NotImplementedError(
            'learning rate policy [%s] is not implemented',
            hyperparameters['lr_policy'])
    return scheduler

def weights_init(init_type='gaussian'):
    def init_fun(m):
        classname = m.__class__.__name__
        if (classname.find('Conv') == 0 or classname.find(
                'Linear') == 0) and hasattr(m, 'weight'):
            # print m.__class__.__name__
            if init_type == 'gaussian':
                init.normal_(m.weight.data, 0.0, 0.02)
            elif init_type == 'xavier':
                init.xavier_normal_(m.weight.data, gain=math.sqrt(2))
            elif init_type == 'kaiming':
                init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
            elif init_type == 'orthogonal':
                init.orthogonal_(m.weight.data, gain=math.sqrt(2))
            elif init_type == 'default':
                pass
            else:
                assert 0, "Unsupported initialization: {}".format(init_type)
            if hasattr(m, 'bias') and m.bias is not None:
                init.constant_(m.bias.data, 0.0)

    return init_fun


#!/usr/local/bin/python
# encoding: utf-8
import extensions
from extensions import *
from xml.dom import minidom, Node # USE cElementTree for FAST stream or BeautifulSoup for html !!™

# https://en.wiktionary.org/w/api.php?action=query&titles=money&prop=revisions&rvprop=content&format=xml
base_url="https://en.wiktionary.org/w/api.php?action=query&prop=revisions&rvprop=content&format=xml&titles="
base_url_xml="https://en.wiktionary.org/w/api.php?action=query&prop=revisions&rvprop=content&format=xml&rvexpandtemplates=true&titles="
http_base="https://en.wiktionary.org/w/index.php?title="

title="Category:Ugaritic_nouns"

html=wget(http_base+title)
link_ex=regex("a href=\"/wiki/(.*)\"")
# str.matches=regex_matches
# links=html.matches(link_ex)
# print(links)
links=match(html,link_ex)
# links=matches(html,link_ex)
print(links)

quit()

def getChild(self,name):
	if name=="__str__": return self.__repr__
	if name=="text": return self.firstChild.nodeValue
	# if name=="__str__": return lambda:self.localName# self.toxml
	if name=="xml": return self.toxml()
	xs=self.getElementsByTagName(name)
	return self.getAttribute(name) or len(xs)==1 and xs[0] or xs

minidom.Element.__getitem__=getChild
minidom.Element.__getattr__=getChild
import xml.dom.minidom
# xmldoc = minidom.parseString(wget(base_url+title))
xmldoc = minidom.parseString(wget(base_url_xml+title))

root = xmldoc.documentElement
# print(root)
# print(root.xml)
page=root['query']['pages']['page']
# minidom.Element.__getattr__=minidom.Element.getAttribute
# minidom.Element.__getattr__=getChild
# print(root.query)
# print(root.toxml())
# print(page.title)
print(page['title'])
text=page['revisions']['rev']
print(text)
print(text.text)


import os
from pathlib import Path

import yaml
from flask import request, Flask, render_template
from flask_cors import CORS
from flask_sqlalchemy import SQLAlchemy

from server import MAIN_FOLDER
from server.db_structures import UserLogs

with (MAIN_FOLDER / Path("app_config.yaml")).open("r") as stream:
    try:
        config = yaml.safe_load(stream)
    except yaml.YAMLError as exc:
        exit()

basedir = os.path.abspath(os.path.dirname(__file__))
db_name = config['database_rel_path']
host = config['host']
port = config['port']

app = Flask(__name__)
CORS(app)

app.config['SQLALCHEMY_DATABASE_URI'] = f"sqlite:///{os.path.join(basedir, db_name)}"
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True
db = SQLAlchemy(app)

@app.route('/db')
def render_logs_database():
    data = db.session.query(UserLogs).all()
    rows = [d.as_dict() for d in data]
    columns = (
        "ip_address", "time", "session_id", "kernel_id",
        "notebook_name", "cell_index", "cell_num",
        "event", "cell_source", "cell_output"
    )

    return render_template('db_template.html', rows=rows, columns=columns)

@app.route('/', methods=['GET', 'POST'])
def add_message():
    content = request.json
    content['cell_source'] = str(content['cell_source']) if content['cell_source'] else None
    content['cell_output'] = str(content['cell_output']) if content['cell_output'] else None

    user_log = UserLogs(**content)
    db.session.add(user_log)
    db.session.commit()
    return content

def run_server() -> None:
    app.run(host=host, port=port)

if __name__ == "__main__":
    run_server()


from typing import List

class Solution1:
    def shortestWordDistance(self, words: List[str], word1: str, word2: str) -> int:
        result = float('inf')
        wp, wd = -1, None

        for i, w in enumerate(words):
            if w == word1 or w == word2:
                if wp == -1:
                    wd = w
                    wp = i
                elif w == wd:
                    if word1 == word2:
                        result = min(result, i - wp)
                    wp = i
                else:
                    result = min(result, i - wp)
                    wd = w
                    wp = i

        return result

# Only check if word1 and word2 is same once and save it in a boolean and use it wisely.
# This way we don't lose the speed and the code remain clean.

class Solution:
    def shortestWordDistance(self, words: List[str], word1: str, word2: str) -> int:
        idx1, idx2 = -1, -1
        min_diff = float('inf')

        for i, w in enumerate(words):
            if idx1 < idx2:                     # always update the smaller loc first
                if w == word1:      idx1 = i
                elif w == word2:    idx2 = i
            else:
                if w == word2:      idx2 = i
                elif w == word1:    idx1 = i

            if idx1 != -1 and idx2 != -1:
                min_diff = min(min_diff, abs(idx2 - idx1))

        return min_diff

class Solution:
    def shortestWordDistance(self, words: List[str], word1: str, word2: str) -> int:
        n = len(words)
        min_diff = n
        idx1, idx2 = -1, -1

        for i, w in enumerate(words):
            if w == word1:
                idx1 = i
                if idx2 != -1: min_diff = min(min_diff, idx1 - idx2)

            if w == word2:
                idx2 = i
                if idx1 != -1 and idx1 != idx2: min_diff = min(min_diff, idx2 - idx1)

        return min_diff

# O(n) time O(1) space:

# 245. Shortest Word Distance III
# https://leetcode.com/problems/shortest-word-distance-iii/description/


from os import path
import pandas
import sys
from sklearn.preprocessing import LabelBinarizer

class AudioDataPreprocessor:
    def __init__(self):
        self.completeDataset = None
        self.training = None
        self.validation = None
        self.test = None

    def quitIfFileNotExist(self, file):
        if not path.exists(file):
            print(file + ' does not exists')
            sys.exit(1)

    # Return one-hot encoded labels (y) and raw data (x)
    def dataframeToXandY(self, dataframe):
        encoder = LabelBinarizer()
        y = dataframe['voyelle']
        transformedY = encoder.fit_transform(y)
        x = dataframe.drop(columns=['voyelle'])

        return (x, transformedY)

    def load(self, filename):
        self.quitIfFileNotExist(filename)

        completeDataset = pandas.read_csv(
            filename,
            sep='\t',
            usecols=['voyelle', 'F1', 'F2', 'F3', 'F4', 'Z1', 'Z2', 'f0']
        )

        completeDataset.dropna() # remove NaN rows
        completeDataset = completeDataset[completeDataset['F4'] != '--undefined--'] # remove rows with F4 equal to --undefined--

        # Types casting
        completeDataset[['F1', 'F2', 'F3', 'F4', 'f0']] = completeDataset[['F1', 'F2', 'F3', 'F4', 'f0']].astype(float)

        splittingSourceDataset = completeDataset.copy()

        # Use 80% as training
        training_sample = splittingSourceDataset.sample(frac=0.8, random_state=0)
        x_train, y_train = self.dataframeToXandY(training_sample)

        # Update splitting source dataset
        splittingSourceDataset = splittingSourceDataset.drop(x_train.index)

        # Split what's left in 2
        # 100% - 80% = 20% / 2 = 10% of total dataset
        validation_sample = splittingSourceDataset.sample(frac=0.5, random_state=0)
        x_validation, y_validation = self.dataframeToXandY(validation_sample)

        test_sample = splittingSourceDataset.drop(validation_sample.index)
        x_test, y_test = self.dataframeToXandY(test_sample)

        self.completeDataset = completeDataset
        self.training = {'x': x_train, 'y': y_train}
        self.validation = {'x': x_validation, 'y': y_validation}
        self.test = {'x': x_test, 'y': y_test}


import socket
import ctypes as ct
import struct
import sys
import time
import threading
import cv2
import numpy as np
from PIL import Image
from djitellopy import Tello
from messagerie import *

def init():
    myDrone = Tello()   # create drone object
    myDrone.connect()   # connect drone
    myDrone.streamon()
    return myDrone

def moveto(x,y,Drone): # pos est la position actuelle du Drone (avant déplacement)
    if(x-drone.pos.x>0):
        Drone.move_forward(2*(x-drone.pos.x))
    if(x-drone.pos.x<0):
        Drone.move_back(-2*(x-drone.pos.x))
    if(y-drone.pos.y>0):
        Drone.move_right(2*(y-drone.pos.y))
    if(y-drone.pos.y<0):
        Drone.move_left(-2*(y-drone.pos.y))
    return [x,y]

def photo(Drone,nom): # prend une photo à l'aide du Drone et l'enregristre sous le nom "nom"
    frame_read = Drone.get_frame_read()
    cv2.imwrite(nom+".png", frame_read.frame)

    # Load image and convert to '1' mode
    im = Image.open(nom+".png").convert('1')

    # Make Numpy array from image
    na = np.array(im)

    # Save array
    with open('image.pbm','w') as f:
        f.write(f'P2\n{im.width} {im.height}\n')
        np.savetxt(f, na, fmt='%d')

def asservissement (drone, pos):
    #moveto(pos.x,pos.y,myDrone)
    myDrone.move_forward(25)
    drone.pos.x = pos.x
    drone.pos.y = pos.y
    return

myDrone=init()

systemON = 0

print('Enter drone id:')
droneID = str.encode(input())

drone = Tdrone()
drone.droneID = droneID
drone.battery = 80
drone.pos.x = 0
drone.pos.y = 0
drone.pos.z = 0
drone.isON = 0

req = Tmessage()

req.codereq = 0
req.drone.droneID = droneID
req.drone.presenceImage = 0

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('127.0.0.1', 3000))

#IDENTIFY
print("Drone identification request")
req.drone = drone
req.codereq = DRONE_IDENTIFIER

s.send(req)
data = s.recv(65536)
ack = Tmessage.from_buffer_copy(data)
if (ack.codereq == ACK_DRONE_IDENTIFIER):
    print("Drone identified\n")
elif (ack.codereq == ERROR_DRONE_IDENTIFIER):
    print("Error to identify drone\n")

while(ack.codereq != ACK_DRONE_DISCONNECT):

    drone.battery = myDrone.get_battery()
    drone.vitesse.x = myDrone.get_speed_x()
    drone.vitesse.y = myDrone.get_speed_y()
    drone.vitesse.z = myDrone.get_speed_z()
    drone.pos.z = myDrone.get_height()

    #DRONE-STATUS
    req.drone = drone
    req.codereq = DRONE_STATUS

    print("Sending drone status")
    afficherDrone(drone)

    s.send(req)
    data = s.recv(65536)
    ack = Tmessage.from_buffer_copy(data)

    if (ack.codereq == ACK_DRONE_STATUS):
        print("Drone status sent\n")
    elif (ack.codereq == ERROR_DRONE_IDENTIFIER):
        print("Error to send drone status\n")

    #DRONE-DEMANDE-ACTION
    req.drone = drone
    req.codereq = DRONE_DEMANDE_ACTION
    print("Waiting for action...")

    s.send(req)
    data = s.recv(65536)
    ack = Tmessage.from_buffer_copy(data)

    if (ack.codereq == ACK_DRONE_DEMANDE_ACTION_NONE):
        print("Drone ne fait rien\n")
    elif (ack.codereq == ACK_DRONE_DEMANDE_ACTION_START):
        print("Drone demarre\n")
        drone.isON = 1
        myDrone.takeoff()
        myDrone.move_down(50)
        photo(myDrone,droneID)
    elif (ack.codereq == ACK_DRONE_DEMANDE_ACTION_POS):
        print("Drone va à la position ("+str(ack.pos.x)+", "+str(ack.pos.y)+", "+str(ack.pos.z)+")\n")
        if __name__ == "__main__":
            threadpos = threading.Thread(target=asservissement, args=(drone,ack.pos, ))
            threadpos.start()

    elif (ack.codereq == ACK_DRONE_DEMANDE_ACTION_FIN):
        print("Drone arrete\n")
        drone.isON = 0
        myDrone.land()
    time.sleep(0.5)

#DRONE-DISCONNECT
print("Drone disconnect request")

req.drone = drone
req.codereq = DRONE_DISCONNECT

s.send(req)
data = s.recv(65536)
ack = Tmessage.from_buffer_copy(data)

if (ack.codereq == ACK_DRONE_DISCONNECT):
    print("Drone disconnected\n")
elif (ack.codereq == ERROR_DRONE_DISCONNECT):
    print("Error to disconnect drone\n")

req.drone = drone
req.codereq = 555

s.send(req)

s.close()


import zookeeper

from twisted.internet.defer import inlineCallbacks, fail, succeed

from txzookeeper import ZookeeperClient
from txzookeeper.utils import retry_change
from txzookeeper.tests.mocker import MATCH
from txzookeeper.tests import ZookeeperTestCase, utils

MATCH_STAT = MATCH(lambda x: isinstance(x, dict))

class RetryChangeTest(ZookeeperTestCase):

    def update_function_increment(self, content, stat):
        if not content:
            return str(0)
        return str(int(content) + 1)

    def setUp(self):
        super(RetryChangeTest, self).setUp()
        self.client = ZookeeperClient("127.0.0.1:2181")
        return self.client.connect()

    def tearDown(self):
        utils.deleteTree("/", self.client.handle)
        self.client.close()

    @inlineCallbacks
    def test_node_create(self):
        """
        retry_change will create a node if one does not exist.
        """
        #use a mock to ensure the change function is only invoked once
        func = self.mocker.mock()
        func(None, None)
        self.mocker.result("hello")
        self.mocker.replay()

        yield retry_change(
            self.client, "/magic-beans", func)

        content, stat = yield self.client.get("/magic-beans")
        self.assertEqual(content, "hello")
        self.assertEqual(stat["version"], 0)

    @inlineCallbacks
    def test_node_update(self):
        """
        retry_change will update an existing node.
        """
        #use a mock to ensure the change function is only invoked once
        func = self.mocker.mock()
        func("", MATCH_STAT)
        self.mocker.result("hello")
        self.mocker.replay()

        yield self.client.create("/magic-beans")
        yield retry_change(
            self.client, "/magic-beans", func)

        content, stat = yield self.client.get("/magic-beans")
        self.assertEqual(content, "hello")
        self.assertEqual(stat["version"], 1)

    def test_error_in_change_function_propogates(self):
        """
        an error in the change function propogates to the caller.
        """

        def error_function(content, stat):
            raise SyntaxError()

        d = retry_change(self.client, "/magic-beans", error_function)
        self.failUnlessFailure(d, SyntaxError)
        return d

    @inlineCallbacks
    def test_concurrent_update_bad_version(self):
        """
        If the node is updated after the retry function has read
        the node but before the content is set, the retry function
        will perform another read/change_func/set cycle.
        """
        yield self.client.create("/animals")
        content, stat = yield self.client.get("/animals")
        yield self.client.set("/animals", "5")

        real_get = self.client.get
        p_client = self.mocker.proxy(self.client)
        p_client.get("/animals")
        self.mocker.result(succeed((content, stat)))

        p_client.get("/animals")
        self.mocker.call(real_get)

        self.mocker.replay()

        yield retry_change(
            p_client, "/animals", self.update_function_increment)

        content, stat = yield real_get("/animals")
        self.assertEqual(content, "6")
        self.assertEqual(stat["version"], 2)

    @inlineCallbacks
    def test_create_node_exists(self):
        """
        If the node is created after the retry function has determined
        the node doesn't exist but before the node is created by the
        retry function. the retry function will perform another
        read/change_func/set cycle.
        """
        yield self.client.create("/animals", "5")

        real_get = self.client.get
        p_client = self.mocker.patch(self.client)
        p_client.get("/animals")
        self.mocker.result(fail(zookeeper.NoNodeException()))

        p_client.get("/animals")
        self.mocker.call(real_get)
        self.mocker.replay()

        yield retry_change(
            p_client, "/animals", self.update_function_increment)

        content, stat = yield real_get("/animals")
        self.assertEqual(content, "6")
        self.assertEqual(stat["version"], 1)

    def test_set_node_does_not_exist(self):
        """
        if the retry function goes to update a node which has been
        deleted since it was read, it will cycle through to another
        read/change_func set cycle.
        """
        real_get = self.client.get
        p_client = self.mocker.patch(self.client)
        p_client.get("/animals")
        self.mocker.result("5", {"version": 1})

        p_client.get("/animals")
        self.mocker.call(real_get)

        yield retry_change(
            p_client, "/animals", self.update_function_increment)

        content, stat = yield real_get("/animals")
        self.assertEqual(content, "0")
        self.assertEqual(stat["version"], 0)

    def test_identical_content_noop(self):
        """
        If the change function generates identical content to
        the existing node, the retry change function exits without
        modifying the node.
        """
        self.client.create("/animals", "hello")

        def update(content, stat):
            return content

        yield retry_change(self.client, "/animals", update)
        content, stat = self.client.get("/animals")
        self.assertEqual(content, "hello")
        self.assertEqual(stat["version"], 0)


# 导包
import time
from appium import webdriver

# desired_caps为字典格式 - 常用参数：
desired_caps = {}
# 必填-且正确
desired_caps['platformName'] = 'Android'
# 必填-且正确
desired_caps['platformVersion'] = '5.1'
# 必填，设备名称，通过adb命令（adb devices）获取
desired_caps['deviceName'] = 'emulator-5554'
# APP包名
desired_caps['appPackage'] = 'com.android.settings'
# 启动名
desired_caps['appActivity'] = '.Settings'

# 地址通过Ctrl+p可以获取
driver=webdriver.Remote('http://127.0.0.1:4723/wd/hub', desired_caps)

time.sleep(3)

#获取爱客App的包名和启动名com.vcooline.aike/.umanager.LoginActivity
driver.start_activity("com.vcooline.aike",".umanager.LoginActivity")

driver.quit()


from sklearn import metrics
from core.findNN import *

def ComputeGap(ushapeletCandidate, lenSubsequence, timeseries, deltas, labels, similarity_measure):
    numTimeseries = len(timeseries)

    ts_index = int(ushapeletCandidate[0])
    loc = int(ushapeletCandidate[1])
    shapelet = timeseries[ts_index, loc : loc + lenSubsequence]
    delta_shapelet = deltas[ts_index, loc : loc + lenSubsequence]

    # Compute the distances from the shapelet to all timeseries
    distances = np.zeros(numTimeseries, dtype=float)
    delta_distances = np.zeros(numTimeseries, dtype=float)
    locations = np.zeros(numTimeseries, dtype=int)
    for i in range(numTimeseries):
        # if i % 100 == 0:
        #     print('findNN for i:', i)
        distances[i], delta_distances[i], locations[i] = findNN(shapelet, delta_shapelet, timeseries[i, :], deltas[i, :], similarity_measure)

    # Sorting the distances to get the orderline
    if similarity_measure == 'UED':
        # Simple ordering
        indices = np.lexsort((delta_distances, distances))
        orderline = distances[indices]
    else:
        orderline = np.sort(distances)

    # Finding the biggest gap in the orderline
    lb = int(np.ceil(numTimeseries * 0.167))
    ub = int(np.floor(numTimeseries * 0.833))

    maxGap = 0
    threshold = 0
    for i in range(lb, ub):
        curThreshold = orderline[i]
        curGap = gap(orderline, curThreshold)
        if curGap > maxGap:
            maxGap = curGap
            threshold = curThreshold

    # Evaluation of the threshold
    label = labels[ts_index]
    evaluation_labels = np.zeros(numTimeseries, dtype=int)
    for i in range(numTimeseries):
        if labels[i] == label:
            evaluation_labels[i] = 1

    IDX = np.zeros(numTimeseries, dtype=int)
    IDX[distances <= threshold] = 1

    RI = metrics.rand_score(evaluation_labels, IDX)

    return maxGap, RI, IDX, distances, delta_distances, locations

def gap(orderline, d):
    Da = np.argwhere(orderline <= d)
    Db = np.argwhere(d < orderline)

    # Both groups have to contain at least two objects
    if Da.shape[0] < 2 or Db.shape[0] < 2:
       return 0

    gap = 0
    r = len(np.argwhere(Da)) / len(np.argwhere(Db))

    # Ensure a high separation power
    if 0.2 < r and r < 5:
        ma = np.mean(orderline[Da])
        mb = np.mean(orderline[Db])
        sa = np.std(orderline[Da])
        sb = np.std(orderline[Db])
        gap = mb - sb - (ma + sa)

    return gap


from token_vk import token
import bot
import requests
import sql_uploader

class VK:

    def __init__(self):
        self.vk_token = token

# Информация для поиска
    def get_info_for_search(self, user_id):
        user_info = self.get_info_about_user(user_id)
        bot.Bot().write_msg(user_id, f"Введите возраст:")
        user_info['age'] = bot.Bot().get_age()

        if user_info['city'] == None:
            bot.Bot().write_msg(user_id, f"Введите город:")
            user_info['city'] = bot.Bot().get_city()
        if user_info['sex'] == None:
            bot.Bot().write_msg(user_id, f"""
                                        Укажите цифру соответствующую полу:
                                        1 - женский
                                        2 - мужской""")
            user_info['sex'] = bot.Bot().get_sex()
        if user_info['relation'] == None:
            bot.Bot().write_msg(user_id, f"""
                                        Укажите цифру соответствующую статусу:
                                        1 — не женат (не замужем);
                                        2 — встречается;
                                        3 — помолвлен(-а);
                                        4 — женат (замужем);
                                        5 — всё сложно;
                                        7 — влюблен(-а);
                                        8 — в гражданском браке.""")
            user_info['relation'] = bot.Bot().get_relation()
        return user_info

    # Информация по пользователю
    def get_info_about_user(self, id):
        response = requests.get("https://api.vk.com/method/users.get",
                                params={"access_token": self.vk_token, "v": "5.131", "user_ids": id,
                                        'fields': 'bdate, sex, relation, city'})
        if 'error' not in response.json():
            user_info = {}
            user_info['id'] = response.json()['response'][0]['id']
            user_info['name'] = response.json()['response'][0]['first_name']
            if 'city' in response.json()['response'][0]:
                user_info['city'] = response.json()['response'][0]['city']['id']
            else:
                user_info['city'] = None
            if 'sex' in response.json()['response'][0]:
                user_info['sex'] = response.json()['response'][0]['sex']
            else:
                user_info['sex'] = None
            if 'relation' in response.json()['response'][0]:
                user_info['relation'] = response.json()['response'][0]['relation']
            else:
                user_info['relation'] = None
            return user_info
        else:
            return False

    # Проверка города
    def check_city(self, city):
        response = requests.get("https://api.vk.com/method/database.getCities",
                                params={"access_token": self.vk_token, "v": "5.131", 'country_id': 1, 'q': city})
        for item in response.json()['response']['items']:
            if city.lower() == item['title'].lower():
                return item['id']
        return False

    # Результат поиска
    def show_users(self, user_id, user_info):
        users = self.search_users(user_info)
        sql_uploader.SqlUploader().create_table(user_id)
        for find_user_id, first_name, last_name in users:
            if not sql_uploader.SqlUploader().check_id_in_base(user_id, find_user_id):
                photos_list = []
                response = requests.get("https://api.vk.com/method/photos.get", params={"access_token": self.vk_token,
                                                                                "owner_id": find_user_id,
                                                                                "v": "5.131",
                                                                                "album_id": "profile",
                                                                                "extended": "1"})
                photos = response.json()['response']['items']
                for photo in photos:
                    photo_id = photo['id']
                    likes = photo['likes']['count']
                    comments = photo['comments']['count']
                    url = photo['sizes'][-1]['url']
                    new_tuple = (likes, comments, photo_id, url)
                    photos_list.append(new_tuple)
                max_3_photos = sorted(photos_list, key=lambda x: (x[0], x[1]), reverse=True)[:3]
                for item in max_3_photos:
                    bot.Bot().vk.method('messages.send',{'user_id': user_id, "attachment": f'photo{find_user_id}_{item[2]}',
                                                    'random_id': 0})
                user_url = f'https://vk.com/id{find_user_id}'
                bot.Bot().write_msg(user_id, f"{first_name} {last_name} {user_url}")
                bot.Bot().write_msg(user_id, f'Отправьте в чат "+", чтобы добавить в избраное и продолжить\n'
                                              f'или любой символ для продолжения без сохранения\n'
                                              f'Для выхода отправьте "exit"')
                if bot.Bot().choose_from_find(find_user_id, max_3_photos) == False:
                    break

    # Поиск
    def search_users(self, user_info):
        if user_info['sex'] == 2:
            gender = user_info['sex'] - 1
        else:
            gender = user_info['sex'] + 1
        response = requests.get("https://api.vk.com/method/users.search",
                                params={"access_token": self.vk_token, "v": "5.131", "count": "1000",
                                        'has_photo': '1', 'sex': gender, 'status': user_info['relation'],
                                        'city': user_info['city'], 'age_from': user_info['age'] - 5,
                                        'age_to': user_info['age'] + 5})
        response.raise_for_status()
        users_list = response.json()['response']['items']
        find_list = [(user['id'], user['first_name'],
                  user['last_name']) for user in users_list if user['is_closed'] == False]

        return find_list