import itertools
import logging
import operator
import pathlib
import re

from retrying import retry

import medusa.index

from medusa.storage.cluster_backup import ClusterBackup
from medusa.storage.node_backup import NodeBackup
from medusa.storage.google_storage import GoogleStorage
from medusa.storage.local_storage import LocalStorage
from medusa.storage.s3_storage import S3Storage
from medusa.storage.s3_rgw import S3RGWStorage
from medusa.storage.azure_storage import AzureStorage
from medusa.storage.s3_base_storage import S3BaseStorage
from medusa.utils import evaluate_boolean

# pattern meant to match just the blob name, not the entire path
# the path is covered by the initial .*
# also retains extension if the name has any
INDEX_BLOB_NAME_PATTERN = re.compile('.*(tokenmap|schema|manifest|differential|incremental)_(.*)$')
INDEX_BLOB_WITH_TIMESTAMP_PATTERN = re.compile('.*(started|finished)_(.*)_([0-9]+).timestamp$')

def divide_chunks(values, step):
    """
    Yield successive step-sized chunks from values.
    :param values: A list of items to split into sub-lists
    :param step: The size of sub-lists
    :return: A list of lists of maximum 'step' size.
    """
    for i in range(0, len(values), step):
        yield values[i:i + step]

def format_bytes_str(value):
    for unit_shift, unit in enumerate(['B', 'KB', 'MB', 'GB', 'TB']):
        if value >> (unit_shift * 10) < 1024:
            break
    return '{:.2f} {}'.format(value / (1 << (unit_shift * 10)), unit)

class Storage(object):
    def __init__(self, *, config):
        self._config = config
        # Used to bypass dependency checks when running in Kubernetes
        self._k8s_mode = evaluate_boolean(config.k8s_mode) if config.k8s_mode else False
        self._prefix = pathlib.Path(config.prefix or '.')
        self.prefix_path = str(self._prefix) + '/' if len(str(self._prefix)) > 1 else ''
        self.storage_driver = self._load_storage()
        self.storage_provider = self._config.storage_provider

    def __enter__(self):
        self.storage_driver.connect()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.storage_driver.disconnect()

    def _load_storage(self):
        logging.debug('Loading storage_provider: {}'.format(self._config.storage_provider))
        if self._config.storage_provider.lower() == 'google_storage':
            google_storage = GoogleStorage(self._config)
            return google_storage
        elif self._config.storage_provider.lower() == 'azure_blobs':
            azure_storage = AzureStorage(self._config)
            return azure_storage
        elif self._config.storage_provider.lower() == 's3_rgw':
            return S3RGWStorage(self._config)
        elif self._config.storage_provider.lower() == "s3_compatible":
            s3_storage = S3BaseStorage(self._config)
            return s3_storage
        elif self._config.storage_provider.lower().startswith('s3'):
            s3_storage = S3Storage(self._config)
            return s3_storage
        elif self._config.storage_provider.lower() == 'local':
            return LocalStorage(self._config)
        elif self._config.storage_provider.lower() == "ibm_storage":
            s3_storage = S3BaseStorage(self._config)
            return s3_storage

        raise NotImplementedError("Unsupported storage provider")

    @property
    def config(self):
        return self._config

    @retry(stop_max_attempt_number=7, wait_exponential_multiplier=10000, wait_exponential_max=120000)
    def get_node_backup(self, *, fqdn, name, differential_mode=False):
        return NodeBackup(
            storage=self,
            name=name,
            fqdn=fqdn,
            differential_mode=differential_mode
        )

    def discover_node_backups(self, *, fqdn=None):
        """
        Discovers nodes backups by traversing data folders.
        This operation is very taxing for cloud backends and should be avoided.
        We keep it in the codebase for the sole reason of allowing the compute-backup-indices to work.
        """

        def get_backup_name_from_blob(blob):
            blob_path = pathlib.Path(blob.name)
            if self.prefix_path == '':
                fqdn, name, *_ = blob_path.parts
            else:
                _, fqdn, name, *_ = blob_path.parts
            return fqdn, name

        def is_schema_blob(blob):
            return blob.name.endswith('/schema.cql')

        def includes_schema_blob(blobs):
            return any(map(is_schema_blob, blobs))

        prefix_path = fqdn if fqdn else ''

        logging.debug("Listing blobs with prefix '{}'".format(prefix_path))

        storage_objects = filter(
            lambda blob: "meta" in blob.name,
            self.storage_driver.list_objects(path=prefix_path)
        )

        all_blobs = sorted(storage_objects, key=operator.attrgetter('name'))

        logging.debug("Finished listing blobs")

        for (fqdn, backup_name), blobs in itertools.groupby(all_blobs, key=get_backup_name_from_blob):
            # consume the _blobs_ iterator into a list because we need to traverse it twice
            backup_blobs = list(blobs)
            if includes_schema_blob(backup_blobs):
                logging.debug("Found backup {}.{}".format(fqdn, backup_name))
                yield NodeBackup(storage=self, fqdn=fqdn, name=backup_name, preloaded_blobs=backup_blobs)

    def list_node_backups(self, *, fqdn=None, backup_index_blobs=None):
        """
        Lists node backups using the index.
        If there is no backup index, no backups will be found.
        Use discover_node_backups to discover backups from the data folders.
        """

        def is_tokenmap_file(blob):
            return "tokenmap" in blob.name

        def get_blob_name(blob):
            return blob.name

        def get_all_backup_blob_names(blobs):
            # if the tokenmap file exists, we assume the whole backup exists too
            all_backup_blobs = filter(is_tokenmap_file, blobs)
            return list(map(get_blob_name, all_backup_blobs))

        def get_blobs_for_fqdn(blobs, fqdn):
            return list(filter(lambda b: f'_{fqdn}.' in b, blobs))

        if backup_index_blobs is None:
            backup_index_blobs = self.list_backup_index_blobs()

        blobs_by_backup = self.group_backup_index_by_backup_and_node(backup_index_blobs)

        all_backup_blob_names = get_all_backup_blob_names(backup_index_blobs)

        if len(all_backup_blob_names) == 0:
            logging.info('No backups found in index. Consider running "medusa build-index" if you have some backups')

        # possibly filter out backups only for given fqdn
        if fqdn is not None:
            relevant_backup_names = get_blobs_for_fqdn(all_backup_blob_names, fqdn)
        else:
            relevant_backup_names = all_backup_blob_names

        # use the backup names and fqdns from index entries to construct NodeBackup objects
        node_backups = list()
        for backup_index_entry in relevant_backup_names:
            if self.prefix_path == '':
                # no prefix in the buckets
                _, _, backup_name, tokenmap_file = backup_index_entry.split('/')
            else:
                # prefix is being used for multi tenancy in the cluster
                _, _, _, backup_name, tokenmap_file = backup_index_entry.split('/')

            # tokenmap file is in format 'tokenmap_fqdn.json'
            tokenmap_fqdn = self.get_fqdn_from_any_index_blob(tokenmap_file)
            manifest_blob, schema_blob, tokenmap_blob = None, None, None
            started_blob, finished_blob = None, None
            started_timestamp, finished_timestamp = None, None
            if tokenmap_fqdn in blobs_by_backup[backup_name]:
                manifest_blob = self.lookup_blob(blobs_by_backup, backup_name, tokenmap_fqdn, 'manifest')
                schema_blob = self.lookup_blob(blobs_by_backup, backup_name, tokenmap_fqdn, 'schema')
                tokenmap_blob = self.lookup_blob(blobs_by_backup, backup_name, tokenmap_fqdn, 'tokenmap')
                started_blob = self.lookup_blob(blobs_by_backup, backup_name, tokenmap_fqdn, 'started')
                finished_blob = self.lookup_blob(blobs_by_backup, backup_name, tokenmap_fqdn, 'finished')
                differential_blob = self.lookup_blob(blobs_by_backup, backup_name, tokenmap_fqdn, 'differential')
                # Should be removed after while. Here for backwards compatibility.
                incremental_blob = self.lookup_blob(blobs_by_backup, backup_name, tokenmap_fqdn, 'incremental')
                if started_blob is not None:
                    started_timestamp = self.get_timestamp_from_blob_name(started_blob.name)
                else:
                    started_timestamp = None
                if finished_blob is not None:
                    finished_timestamp = self.get_timestamp_from_blob_name(finished_blob.name)
                else:
                    finished_timestamp = None

            nb = NodeBackup(storage=self, fqdn=tokenmap_fqdn, name=backup_name,
                            manifest_blob=manifest_blob, schema_blob=schema_blob, tokenmap_blob=tokenmap_blob,
                            started_timestamp=started_timestamp, started_blob=started_blob,
                            finished_timestamp=finished_timestamp, finished_blob=finished_blob,
                            differential_blob=differential_blob if differential_blob is not None else incremental_blob)
            node_backups.append(nb)

        # once we have all the backups, we sort them by their start time. we get oldest ones first
        sorted_node_backups = sorted(
            # before sorting the backups, ensure we can work out at least their start time
            filter(lambda nb: nb.started is not None, node_backups),
            key=lambda nb: nb.started
        )

        # then, before returning the backups, we pick only the existing ones
        previous_existed = False
        for node_backup in sorted_node_backups:
            # we try to be smart here - once we have seen an existing one, we assume all later ones exist too
            if previous_existed:
                yield node_backup
                continue

            # the idea is to save .exist() calls as they actually go to the storage backend and cost something
            # this is mostly meant to handle the transition period when backups expire before the index does,
            # which is a consequence of the transition period and running the build-index command

            if node_backup.exists():
                previous_existed = True
                yield node_backup
            else:
                logging.debug('Backup {} for fqdn {} present only in index'.format(node_backup.name, node_backup.fqdn))
                # if a backup doesn't exist, we should remove its entry from the index too
                try:
                    self.remove_backup_from_index(node_backup)
                except Exception:
                    logging.debug(
                        'This account cannot perform the cleanup_storage'
                        '{} for fqdn {} present only in index.'
                        'Ignoring and continuing...'
                        .format(node_backup.name, node_backup.fqdn))

    def list_backup_index_blobs(self):
        path = '{}index/backup_index'.format(self.prefix_path)
        return self.storage_driver.list_objects(path)

    def list_root_blobs(self):
        return self.storage_driver.list_objects(self.prefix_path)

    def group_backup_index_by_backup_and_node(self, backup_index_blobs):

        def get_backup_name(blob):
            blob_name_chunks = blob.name.split('/')
            return blob_name_chunks[2] if len(str(self.prefix_path)) <= 1 else blob_name_chunks[3]

        def name_and_fqdn(blob):
            return get_backup_name(blob), Storage.get_fqdn_from_any_index_blob(blob)

        def group_by_backup_name(blobs):
            return itertools.groupby(blobs, get_backup_name)

        def group_by_fqdn(blobs):
            return itertools.groupby(blobs, Storage.get_fqdn_from_any_index_blob)

        def has_proper_name(blob):
            blob_name_chunks = blob.name.split('/')
            is_proper = len(blob_name_chunks) == 4 if len(str(self.prefix_path)) <= 1 else len(blob_name_chunks) == 5
            if not is_proper:
                logging.warning('File {} in backup index has improper name'.format(blob.name))
            return is_proper

        blobs_by_backup = {}
        properly_named_index_blobs = filter(
            has_proper_name,
            backup_index_blobs
        )
        sorted_backup_index_blobs = sorted(
            properly_named_index_blobs,
            key=name_and_fqdn
        )

        for backup_name, blobs in group_by_backup_name(sorted_backup_index_blobs):
            blobs_by_node = {}
            for fqdn, node_blobs in group_by_fqdn(blobs):
                blobs_by_node[fqdn] = list(node_blobs)
            blobs_by_backup[backup_name] = blobs_by_node

        return blobs_by_backup

    @staticmethod
    def get_fqdn_from_any_index_blob(blob):
        if not isinstance(blob, str):
            blob_name = blob.name
        else:
            blob_name = blob
        # it's important to check in this order, because the 2nd pattern is more generic
        match = INDEX_BLOB_WITH_TIMESTAMP_PATTERN.match(blob_name)
        if match is None:
            match = INDEX_BLOB_NAME_PATTERN.match(blob_name)
        assert match is not None, 'Encountered malformed index blob name {}'.format(blob_name)
        return Storage.remove_extension(match.group(2))

    @staticmethod
    def remove_extension(fqdn_with_extension):
        replaces = {
            '.json': '',
            '.cql': '',
            '.txt': '',
            '.timestamp': ''
        }
        r = fqdn_with_extension
        for old, new in replaces.items():
            r = r.replace(old, new)
        return r

    @staticmethod
    def get_timestamp_from_blob_name(blob_name):
        match = INDEX_BLOB_WITH_TIMESTAMP_PATTERN.match(blob_name)
        assert match is not None, 'Encountered malformed index blob name with timestamp {}'.format(blob_name)
        return int(match.group(3))

    def lookup_blob(self, blobs_by_backup, backup_name, fqdn, blob_name_chunk):
        """
        This function looks up blobs in blobs_by_backup, which is a double dict (k->k->v).
        The blob_name_chunk tells which blob for given backup and fqdn we want.
        It can be 'schema', 'manifest', 'started', 'finished'
        """
        blob_list = list(filter(lambda blob: blob_name_chunk in blob.name,
                                blobs_by_backup[backup_name][fqdn]))
        return blob_list[0] if len(blob_list) > 0 else None

    def list_cluster_backups(self, backup_index=None):
        node_backups = sorted(
            self.list_node_backups(backup_index_blobs=backup_index),
            key=lambda b: (b.name, b.started)
        )

        for name, node_backups in itertools.groupby(node_backups, key=operator.attrgetter('name')):
            yield ClusterBackup(name, node_backups)

    def latest_node_backup(self, *, fqdn):
        index_path = '{}index/latest_backup/{}/backup_name.txt'.format(self.prefix_path, fqdn)
        try:
            latest_backup_name = self.storage_driver.get_blob_content_as_string(index_path)
            differential_blob = self.storage_driver.get_blob(
                '{}{}/{}/meta/differential'.format(self.prefix_path, fqdn, latest_backup_name))
            # Should be removed after while. Here for backwards compatibility.
            incremental_blob = self.storage_driver.get_blob(
                '{}{}/{}/meta/incremental'.format(self.prefix_path, fqdn, latest_backup_name))

            node_backup = NodeBackup(
                storage=self,
                fqdn=fqdn,
                name=latest_backup_name,
                differential_blob=differential_blob if differential_blob is not None else incremental_blob
            )

            if not node_backup.exists():
                logging.warning('Latest backup points to non-existent backup. Deleting the marker')
                self.remove_latest_backup_marker(fqdn)
                raise Exception

            return node_backup

        except Exception:
            logging.info('Node {} does not have latest backup'.format(fqdn))
            return None

    def latest_cluster_backup(self, backup_index=None):
        """
        Get the latest backup attempted (successful or not)
        """
        last_started = max(
            self.list_cluster_backups(backup_index=backup_index),
            key=operator.attrgetter('started'),
            default=None
        )

        logging.debug("Last cluster backup : {}".format(last_started))
        return last_started

    def latest_complete_cluster_backup(self, backup_index=None):
        """
        Get the latest *complete* backup (ie successful on all nodes)
        """
        finished_backups = filter(
            operator.attrgetter('finished'),
            self.list_cluster_backups(backup_index=backup_index)
        )

        last_finished = max(finished_backups, key=operator.attrgetter('finished'), default=None)
        return last_finished

    def get_cluster_backup(self, backup_name):
        for cluster_backup in self.list_cluster_backups():
            if cluster_backup.name == backup_name:
                return cluster_backup
        raise KeyError('The backup {} does not exist'.format(backup_name))

    def remove_backup_from_index(self, node_backup):
        """
        Takes a node backup and tries to remove corresponding items from the index.
        This usually happens when the node_backup.exists() returns false, which means it's schema in the
        meta folder does not exist.
        We are checking and deleting each blob separately because there is no easy way to list and get the objects.
        """
        medusa.index.clean_backup_from_index(self, node_backup)

    def remove_latest_backup_marker(self, fqdn):
        """
        Removes the markers of the latest backup for a fqdn.
        Unlike remove_backup_from_index, here we do call list because the path is not ambiguous, and because we can't
        get the blobs from anywhere.
        Then we can call the delete object on the results.
        """
        markers = self.storage_driver.list_objects('{}index/latest_backup/{}/'.format(self.prefix_path, fqdn))
        for marker in markers:
            self.storage_driver.delete_object(marker)

    def delete_objects(self, objects, concurrent_transfers=None):
        self.storage_driver.delete_objects(objects, concurrent_transfers)


from fastapi import APIRouter
from sql_app import seeder, models
from sql_app.database import get_db_sesh

# class that allows for separate routers (like Blueprints in Flask)
user_routes = APIRouter(
    prefix="/users",
    tags=["users"],
    responses={404: {"description": "Not found"}}
)

# Users route
# Seeds users and returns all records in users table
@user_routes.get("/")
def get_all_users():
    # We need next() to get return value from Python functions that `yield`
    # rather than `return`. This gets the actual value from the yielded
    # generator object.
    db = next(get_db_sesh())

    if db:
        seeder.seed(db)
        return db.query(models.User).all()
    else:
        return {"error": "There was a problem retrieving users"}


import functools
import torch

import numpy as np

from torch.utils.data import Dataset, Subset

from gantk2.data.transforms import transform_factory

datasets = ['simple', 'mnist', 'celeba']

class IndicesDataset(Dataset):
    def __init__(self, dataset):
        super(IndicesDataset).__init__()
        self.dataset = dataset

    def __len__(self):
        return len(self.dataset)

    def __getitem__(self, index):
        return self.dataset[index], index

def source_collate_fn(collate_fn, batch):
    indices = torch.tensor([batch[i][1] for i in range(len(batch))])
    return collate_fn([batch[i][0] for i in range(len(batch))]), indices

def dataset_dimensionality(dataset):
    assert dataset in datasets
    if dataset == 'mnist':
        return [32, 32, 1]
    elif dataset == 'celeba':
        return [32, 32, 3]
    else:
        raise ValueError(f'No defined dimensionality for `{dataset}`')

def dataset_factory(config, source):
    if source:
        dataset_name = config.in_dataset
        opposite_dataset = config.out_dataset
        transform = transform_factory(config.in_transform)
    else:
        dataset_name = config.out_dataset
        opposite_dataset = config.in_dataset
        transform = transform_factory(config.out_transform)

    assert dataset_name in datasets
    if dataset_name == 'simple':
        if opposite_dataset != 'simple' and not config.generator:
            config.data_dim = dataset_dimensionality(opposite_dataset)
        from gantk2.data.simple import get_dataset
        dataset, collate_fn = get_dataset(config, transform, source)
    else:
        assert not source or not config.generator
        config.data_dim = dataset_dimensionality(dataset_name)
        if dataset_name == 'mnist':
            from gantk2.data.mnist import get_dataset
            dataset, collate_fn = get_dataset(config.data_path, transform, True)
        elif dataset_name == 'celeba':
            from gantk2.data.celeba import get_dataset
            dataset, collate_fn = get_dataset(config.data_path, transform, True)
        else:
            raise ValueError(f'No dataset named `{dataset_name}`')
        if source:
            nb_samples = config.in_nb_samples
        else:
            nb_samples = config.out_nb_samples
        dataset_size = len(dataset)
        assert nb_samples <= dataset_size
        if nb_samples > 0:
            chosen_indices = np.arange(dataset_size)
            np.random.shuffle(chosen_indices)
            dataset = Subset(dataset, chosen_indices[:nb_samples])

    if source and not config.generator:
        if collate_fn is not None:
            collate_fn = functools.partial(source_collate_fn, collate_fn)
        return IndicesDataset(dataset), collate_fn
    else:
        return dataset, collate_fn


import sqlite3

conn = sqlite3.connect('zadacha3.db')
cursor = conn.cursor()
cursor.execute('''CREATE TABLE IF NOT EXISTS zadacha_3 (id INTEGER PRIMARY KEY, col_1 TEXT)''') #ключ primary key содержит в себе констр. unique под капотом

#i = int(input('Введите число:')) #заполнила таблицу, чтобы было с чем работать
#for el in range(1,i+1):
#    cursor.execute('''INSERT INTO zadacha_3(id) VALUES (?)''', (el,))
#    conn.commit()

class UpdateData:

    def __init__(self, id, name):
        self.id = id
        self.name = name

    def update_data(self):
            cursor.execute(f'''UPDATE zadacha_3 SET col_1 = '{self.name}' WHERE id = {self.id}''')
            conn.commit()

a = UpdateData(1,'Karina')
a.update_data()

cursor.execute('''SELECT * FROM zadacha_3''')
k = cursor.fetchall()
print(k)


import json

from models.chess.chess_prism import ChessPrism
from models.hedron import Hedron

class ChessHedron(Hedron):
    def json(self):
        parent_json = super().json()
        parent_json["chess"] = {}
        return json.dumps(parent_json)

    def __init__(self, hid=None, prisms=None):
        super().__init__(hid=hid, prisms=prisms)

    def get_prism(self, pid=None):
        prism = super().get_prism(pid)
        return ChessPrism(
            pid=prism.pid,
            hid=prism.hid,
            first_name=prism.first_name,
            family_name=prism.family_name,
            birth_sign=prism.birth_sign,
            gender=prism.gender,
            body=prism.body,
            hedron_network=prism.hedron_network,
            skills=prism.skills
        )


# package import
from parsivar import Tokenizer
import re
import pandas as pd

farsi_total = pd.read_csv("farsi_total.csv")

def CleanText(readData):

    # Remove Retweets
    #text = re.sub('RT @[\w_]+: ', '', readData)

    #text = readData

	# Remove Mentions
    #text = re.sub('@[\w_]+', '', text)

    # Remove or Replace URL
    # text = url_re.sub('URL', text)
    #text = re.sub(r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+", ' ', text) # start with http
    #text = re.sub(r"[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{2,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)", ' ', text) # Don't start with http

    # Remove Hashtag
    # text = re.sub('[#]+[0-9a-zA-Z_]+', ' ', text)

    # Remove Zero-width non-joiner
    text = re.sub('\u200c' , ' ', readData)

    # Remove Garbage Words (ex. &lt, &gt, etc)
    text = re.sub(r'[^\w\s]',' ',text)
    text = re.sub('_',' ',text)

    # Remove Special Characters
    # text = re.sub('[^0-9a-zA-Z]', ' ', text)

    # Remove Numbers (If you want, activate the code)
    text = re.sub(r'\d+',' ',text)

    # Remove English (If you want, activate the code)
    text = re.sub('[a-zA-Z]' , ' ', text)

    # Remove newline
    text = text.replace('\n',' ')

    # Remove multi spacing & Reform sentence
    # text = ' '.join(text.split())

	# If you want to normalize Korean text, activate code below:
	# from konlpy.tag import Okt # Must use Konlpy ver 0.5.2 above
	# okt = Okt()
	# if len(text) != 0:
		# text = okt.normalize(text)

    return text

# Persian Stopwords Load
persian_stopwords = pd.read_csv("./persian_stopwords.txt", delimiter='\t', names=["ستون"])

# Add Custom Persian Stopwords
my_data = []
my_persian_stopwords = pd.DataFrame(my_data, columns = ["ستون"])

# Main

my_tokenizer = Tokenizer()

def preprocessing_parsivar(readData):

    #### Clean text
    sentence = CleanText(readData)

    #### Tokenize
    morphs = my_tokenizer.tokenize_words(sentence)

    # Remove Stopwords
    morphs[:] = (morph for morph in morphs if morph not in persian_stopwords["ستون"].tolist())
    morphs[:] = (morph for morph in morphs if morph not in my_persian_stopwords["ستون"].tolist())

    # Remove Numbers
    #morphs[:] = (morph for morph in morphs if morph[1] not in NUMBER)

    return morphs

#print(f"Before preprocessing : {farsi_total[tweet]}")
#print(f"After preprocessing : {preprocessing_parsivar(farsi_total)}")

text_file = open("Results.txt", "w")
text_file.write(preprocessing_parsivar(SAMPLE_TEXT))
text_file.close()


import math
import sys
import time
import torch
import torchvision
import pandas as pd
import networkx
from networkx.algorithms.components.connected import connected_components

import utils
from coco_utils import get_coco_api_from_dataset
from coco_eval import CocoEvaluator

def train_one_epoch(model, optimizer, data_loader, device, epoch, print_freq):
    model.train()
    metric_logger = utils.MetricLogger(delimiter="  ")
    metric_logger.add_meter('lr', utils.SmoothedValue(window_size=1, fmt='{value:.6f}'))
    header = 'Epoch: [{}]'.format(epoch)

    lr_scheduler = None
    if epoch == 0:
        warmup_factor = 1. / 1000
        warmup_iters = min(1000, len(data_loader) - 1)
        lr_scheduler = utils.warmup_lr_scheduler(optimizer, warmup_iters, warmup_factor)

    for images, targets, _ in metric_logger.log_every(data_loader, print_freq, header):
        images = list(image.to(device) for image in images)
        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]
        loss_dict = model(images, targets)
        losses = sum(loss for loss in loss_dict.values())
        # reduce losses over all GPUs for logging purposes
        loss_dict_reduced = utils.reduce_dict(loss_dict)
        losses_reduced = sum(loss for loss in loss_dict_reduced.values())
        loss_value = losses_reduced.item()
        if not math.isfinite(loss_value):
            print("Loss is {}, stopping training".format(loss_value))
            sys.exit(1)

        optimizer.zero_grad()
        losses.backward()
        optimizer.step()

        if lr_scheduler is not None:
            lr_scheduler.step()
        metric_logger.update(loss=losses_reduced, **loss_dict_reduced)
        metric_logger.update(lr=optimizer.param_groups[0]["lr"])
    return loss_dict_reduced, losses_reduced

def _get_iou_types(model):
    model_without_ddp = model
    if isinstance(model, torch.nn.parallel.DistributedDataParallel):
        model_without_ddp = model.module
    iou_types = ["bbox"]
    if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN):
        iou_types.append("segm")
    if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN):
        iou_types.append("keypoints")
    return iou_types

@torch.no_grad()
def evaluate(model, data_loader, device):
    n_threads = torch.get_num_threads()
    # FIXME remove this and make paste_masks_in_image run on the GPU
    torch.set_num_threads(n_threads)
    model.eval()
    metric_logger = utils.MetricLogger(delimiter="  ")
    header = 'Test:'

    coco = get_coco_api_from_dataset(data_loader.dataset)
    iou_types = _get_iou_types(model)
    coco_evaluator = CocoEvaluator(coco, iou_types)

    for images, targets, _ in metric_logger.log_every(data_loader, 100, header):
        images = list(img.to(device) for img in images)
        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]
        if device.type == 'cuda':
            torch.cuda.synchronize()
        model_time = time.time()
        outputs = model(images)

        outputs = [{k: v.to(device) for k, v in t.items()} for t in outputs]
        model_time = time.time() - model_time

        res = {target["image_id"].item(): output for target, output in zip(targets, outputs)}
        evaluator_time = time.time()
        coco_evaluator.update(res)
        evaluator_time = time.time() - evaluator_time
        metric_logger.update(model_time=model_time, evaluator_time=evaluator_time)

    # gather the stats from all processes
    metric_logger.synchronize_between_processes()
    print("Averaged stats:", metric_logger)
    coco_evaluator.synchronize_between_processes()

    # accumulate predictions from all images
    coco_evaluator.accumulate()
    coco_evaluator.summarize()
    torch.set_num_threads(n_threads)
    return coco_evaluator

def _area(a, b):  # returns None if rectangles don't intersect
    '''
    return intersect area of two boxes if overlaping otherwise return None
    '''
    dx = min(a.x1, b.x1) - max(a.x0, b.x0)
    dy = min(a.y1, b.y1) - max(a.y0, b.y0)
    if (dx>=0) and (dy>=0):
        return dx*dy

def _to_graph(l):
    G = networkx.Graph()
    for part in l:
        # each sublist is a bunch of nodes
        G.add_nodes_from(part)
        # it also imlies a number of edges:
        G.add_edges_from(_to_edges(part))
    return G

def _to_edges(l):
    """
        treat `l` as a Graph and returns it's edges
        to_edges(['a','b','c','d']) -> [(a,b), (b,c),(c,d)]
    """
    it = iter(l)
    last = next(it)
    for current in it:
        yield last, current
        last = current

def idx_cleanboxes(boxes, scores, second_cutoff=0.83):
    '''
    boxes: 2d npy containing all boxes in the image
    socres: 1d npy containing score for each box in boxes
    '''
    df = pd.DataFrame(boxes)
    df.columns = ['x0', 'y0', 'x1', 'y1']
    df['score'] = scores

    clusters = []
    for box1 in df.itertuples():
        cluster = {box1.Index}
        df2 = df.iloc[box1.Index+1:, :]
        for box2 in df2.itertuples():
            if _area(box1, box2):
                cluster.add(box2.Index)
        clusters.append(cluster)
    G = _to_graph(clusters)

    final_index = []
    for c in connected_components(G):
        c = list(c)
        scores = df.loc[c, 'score']
        cdas = scores[scores>=second_cutoff]
        cdas_idx = cdas.index.to_list()
        if len(cdas_idx)>0:
            final_index.extend(cdas_idx)
        else:
            final_index.append(scores.idxmax())
    return final_index


from datetime import datetime
from glob import glob
import time
import asyncio
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.cron import CronTrigger
from discord.ext.commands import Bot as BotBase
from discord.ext.commands import CommandNotFound
from discord.ext.commands import MissingPermissions
from discord import Intents, Embed, File
from discord.ext import commands
from apscheduler.schedulers.background import BackgroundScheduler
from lib.cogs.timetable import Timetable
import schedule

from ..db import db

PREFIX = "!"
OWNER_IDS = [317315671224483840]
COGS = [path.split("\\")[-1][:-3] for path in glob("./lib/cogs/*.py")]

class Bot(BotBase):
    def __init__(self):
        self.OWNER_IDS = OWNER_IDS
        self.PREFIX = PREFIX
        self.ready = False
        self.guild = None
        self.scheduler = AsyncIOScheduler()
        db.autosave(self.scheduler)

        intents = Intents.default()
        intents.members = True
        intents.presences = True
        intents.messages = True
        intents.reactions = True
        intents.voice_states = True
        intents.emojis = True
        intents.voice_states = True

        super().__init__(
            command_prefix=self.PREFIX,
            owner_ids=OWNER_IDS,
            intents=Intents.all()
        )

    def setup(self):
        for cog in COGS:
            self.load_extension(f"lib.cogs.{cog}")
            print(f"[log]{cog} cog loaded")
        print("[log]setup complete")

    async def on_error(self, err, *args, **kwargs):
        if err == "on_command_error":
            await args[0].send('Что-то пошло не так...')
        channel = self.get_channel(809020872895496232)
        await channel.send('Произошла ошибка...')
        raise

    async def on_command_error(self, ctx, exc):
        channel = self.get_channel(809020872895496232)
        await channel.send('Произошла ошибка...')
        if isinstance(exc, CommandNotFound):
            await ctx.send('Не существует такой команды...')
        elif isinstance(exc, MissingPermissions):
            await ctx.send(f'{ctx.author.mention}, у тебя недостаточно прав...')
        elif hasattr(exc, "original"):
            raise exc.original
        else:
            raise exc

    def run(self, version):
        self.VERSION = version
        self.setup()
        with open("./lib/bot/token.0", "r", encoding="UTF-8") as tf:
            self.TOKEN = tf.read()

        print("[log]running bot...")
        super().run(self.TOKEN, reconnect=True)

    async def on_connect(self):
        print('[log]bot connected...')

    async def on_disconnect(self):
        print('[log]bot disconnected')

    async def on_ready(self):
        if not self.ready:
            self.READY = True
            self.guild = self.get_guild(759167756317884436)
            self.scheduler.start()
            print("[log]bot ready")

        else:
            print("[log]bot reconnected")

    async def on_message(self, message):
        if not message.author.bot:
            await self.process_commands(message)

bot = Bot()


import requests
from bs4 import BeautifulSoup

# URL = "https://web.archive.org/web/20200518073855/https://www.empireonline.com/movies/features/best-movies-2/"
URL = "http://web.archive.org/web/20200518055830/https://www.empireonline.com/movies/features/best-movies-2/"

response=requests.get(URL)
response.encoding="utf-8"
data=response.text

soup=BeautifulSoup(data,"html.parser")

title=[]
heading=soup.find_all(name="h3")
for i in heading:
    title.append(i.text)

title=title[::-1]
print(title)

with open("movies","w") as file:
    for i in title:
        file.write(f"{i}\n")

# Write your code below this line ðŸ‘‡

Day 45 web scraping/Starting+Code+-+100+movies+to+watch+start/Starting Code - 100 movies to watch start/main.py


# -*- coding: utf-8 -*-
from gensim.models import word2vec
from sklearn.manifold import TSNE
# import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib import rc, font_manager
import gensim.models as g
import pandas as pd
import PresenTALKUtil
from operator import itemgetter
from collections import OrderedDict

class W2VRecommender:

    def __init__(self, model_path):
        if model_path != '':
            self.model = g.Word2Vec.load(model_path)

    def train_word2vec(self, prefix, file_path, params):
        data = word2vec.LineSentence(file_path)
        self.model = word2vec.Word2Vec(data, size=params['size'], window=params['window'], min_count=params['min_count']
                                       , workers=4, iter=30, sg=1)
        self.model.save('rec_model_{0}_{1}_{2}_{3}_30'.format(prefix, params['size'], params['window']
                                                              , params['min_count']))

    def analyze_data(self, file_path):
        present_list = PresenTALKUtil.get_all_present_list()
        present_target = PresenTALKUtil.get_feature_target()
        present_purpose = PresenTALKUtil.get_feature_purpose()

        data_file = open(file_path, 'r', encoding='utf-8')
        lines = data_file.readlines()

        word_count_dict = dict()

        for line in lines:
            word_list = line.split(' ')

            for word in word_list:
                word = word.replace('\n', '')
                if word in word_count_dict:
                    word_count_dict[word] += 1
                else:
                    word_count_dict[word] = 1

        # word Count
        sorted_list = sorted(word_count_dict.items(), key=itemgetter(1), reverse=True)
        # for key, value in sorted_list:
        #     print(key, value)

        # present word count
        for key, value in sorted_list:
            if key in present_list:
                print(key, value)

        # target word count
        for key, value in sorted_list:
            if key in present_target:
                print(key, value)

        # purpose word count
        for key, value in sorted_list:
            if key in present_purpose:
                print(key, value)

    def visualization_data(self, word_count):
        font_path = 'C:/Windows/Fonts/NanumGothic.ttf'
        font_manager.FontProperties(fname=font_path)
        rc('font', family="NanumGothic")

        vocab = list(self.model.wv.vocab)
        X = self.model[vocab]

        tsne = TSNE(n_components=2)
        X_tsne = tsne.fit_transform(X[:word_count, :])

        df = pd.DataFrame(X_tsne, index=vocab[:word_count], columns=['x', 'y'])

        fig = plt.figure()
        fig.set_size_inches(40, 20)
        ax = fig.add_subplot(1, 1, 1)

        ax.scatter(df['x'], df['y'])

        for word, pos in df.iterrows():
            ax.annotate(word, pos, fontsize=10)
        plt.show()


# Illustration
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit

x = np.linspace(0, 10, num = 100)
y = np.log(x)

plt.plot(x, x, color = "black")
plt.plot(x, y, color = "black")
plt.show()

# Toy model
rng = np.random.default_rng(seed = 313)
x = np.linspace(0, 1, num = 100)
u = rng.uniform(-1, 1, 100)
y = np.exp(1 + 2 * x + u)

func = lambda x, a, b: np.exp(a + b * x)
param = curve_fit(func, x, y)[0]
param
res_ls = smf.ols("np.log(y) ~ x", data = pd.DataFrame([y, x]).T).fit()
res_ls.params

plt.scatter(x, y)
plt.plot(x, func(x, 1, 2), color = "black")
plt.plot(x, func(x, *param), color = "red")
plt.show()

lower = np.sum((y - np.mean(y)) ** 2)
upper = np.sum((y - func(x, param[0], param[1])) ** 2)
1 - upper / lower
np.corrcoef(x, np.log(y))[1, 0] ** 2

# Illustration
import numpy as np
import matplotlib.pyplot as plt

x = np.linspace(0, 10, num = 100)
u = rng.uniform(-1, 1, 100)
y = np.exp(1 + 2 * x + u)

plt.scatter(x, np.log(y))
plt.plot(x, 1 + 2 * x)
plt.show()

# Interest
x = np.linspace(0, 50, num = 50)
u = rng.uniform(-0.1, 0.1, 50)
plt.scatter(x, 100 * 1.03 ** x * np.exp(u))
plt.plot(x, 100 * 1.03 ** x)
plt.show()

extras/2021 Fall EBA3500 Data Analysis and Programming/notes/lecture3/lecture3-logarithm.py


import sqlite3
import time
import validators
import json
import threading
import schedule
import telebot
from telebot import types
from telebot.storage import StateMemoryStorage
from database.db import User_database, Banned_users
from datetime import datetime
from pathlib import Path
from tools.markups import Start_markup
from tools.ban_message_parser import ban_message_parser
from tools.my_states import MyStates

with open('config.json', 'r', encoding='utf-8') as f:
    config = json.load(f)
    BOT_TOKEN = config['BOT_TOKEN']
    ADMINS_ID = config['ADMINS_ID']
    SUPPORT_CHAT = config['SUPPORT_CHAT']
    LIST_ID = config['LIST_ID']

state_storage = StateMemoryStorage()
bot = telebot.TeleBot(BOT_TOKEN, state_storage=state_storage)
bot.delete_webhook()
db = User_database(str(Path.cwd() / 'database' / 'database.db'))
bans_database = Banned_users(str(Path.cwd() / 'database' / 'database.db'))

# Тут работаем с командой start
@bot.message_handler(commands=['start'])
def welcome_start(message):
    try:
        text = message.from_user.first_name + ', приветствую вас!'
        bot.send_message(message.chat.id, text, parse_mode='Markdown', reply_markup=Start_markup)
    except Exception as ex:
        print('Ops... Hello error')
        print(ex)

@bot.message_handler(commands=['ban'], func=lambda message: str(
    message.chat.id) == SUPPORT_CHAT and SUPPORT_CHAT != 'None' and message.reply_to_message is not None)
def ban(message):
    try:
        res = ban_message_parser(message.text)
        if bans_database.check_user(message.reply_to_message.forward_from.id):
            bans_database.upd_user_time(message.reply_to_message.forward_from.id, res['time'])
            with open(Path.cwd() / 'messages' / 'ban_write_update.txt', 'r', encoding='utf-8') as file:
                text = file.read()
                text = text.replace('[[ days ]]', res['days'])
            bot.send_message(message.reply_to_message.forward_from.id, text, parse_mode='Markdown')
        else:
            bans_database.add_user(message.reply_to_message.forward_from.id,
                                   message.reply_to_message.forward_from.username, res['time'], res['reason'])
            with open(Path.cwd() / 'messages' / 'ban_write.txt', 'r', encoding='utf-8') as file:
                text = file.read()
                text = text.replace('[[ days ]]', res['days'])
                text = text.replace('[[ reason ]]', str(res['reason']))
            bot.send_message(message.reply_to_message.forward_from.id, text, parse_mode='Markdown')
        text = 'Ограничения произведены успешно.'
        bot.reply_to(message, text)
    except Exception as ex:
        with open(Path.cwd() / 'messages' / 'ban_tags_error.txt', 'r', encoding='utf-8') as file:
            text = 'Произошла ошибка!\n' + str(ex) + '\n' + file.read()
        bot.reply_to(message, text)
        print(str(ex))

@bot.message_handler(commands=['cancel_ban'], func=lambda message: str(
    message.chat.id) == SUPPORT_CHAT and SUPPORT_CHAT != 'None' and message.reply_to_message is not None)
def cancel_ban(message):
    try:
        if bans_database.check_user(message.reply_to_message.forward_from.id):
            bans_database.delete_user(message.reply_to_message.forward_from.id)
            text = 'С пользователя успешно сняты ограничения.'
            bot.reply_to(message, text)
            text = "С вас сняты ограничения."
            bot.send_message(message.reply_to_message.forward_from.id, text, parse_mode='Markdown')
        else:
            text = 'Данный пользователь не находится под ограничениями на данный момент.'
            bot.reply_to(message, text)
    except Exception as ex:
        text = 'Произошла ошибка!\n' + str(ex)
        bot.reply_to(message, text)
        print(str(ex))

@bot.message_handler(commands=['cancel_ban'], func=lambda message: str(
    message.chat.id) == SUPPORT_CHAT and SUPPORT_CHAT != 'None' and message.reply_to_message is None)
def cancel_ban_id(message):
    try:
        user_id = int(message.text.split()[-1])
    except ValueError:
        text = 'Id указано некорректно.'
        bot.reply_to(message, text)
        return
    except Exception as ex:
        text = 'Произошла ошибка!\n' + str(ex)
        bot.reply_to(message, text)
        print(str(ex))
        return
    try:
        if bans_database.check_user(user_id):
            bans_database.delete_user(user_id)
            text = 'С пользователя успешно сняты ограничения.'
            bot.reply_to(message, text)
            text = "С вас сняты ограничения."
            bot.send_message(user_id, text, parse_mode='Markdown')
        else:
            text = 'Данный пользователь не находится под ограничениями на данный момент.'
            bot.reply_to(message, text)
    except Exception as ex:
        text = 'Произошла ошибка!\n' + str(ex)
        bot.reply_to(message, text)
        print(str(ex))

@bot.message_handler(commands=['ban_info'],
                     func=lambda message: str(message.chat.id) == SUPPORT_CHAT and SUPPORT_CHAT != 'None')
def user_info(message):
    try:
        user_id = int(message.text.split()[-1])
    except ValueError:
        text = 'Id указано некорректно.'
        bot.reply_to(message, text)
        return
    except Exception as ex:
        text = 'Произошла ошибка!\n' + str(ex)
        bot.reply_to(message, text)
        print(str(ex))
        return
    try:
        if bans_database.check_user(user_id):
            info = bans_database.get_user_info(user_id)
            text = 'До ' + datetime.fromtimestamp(info[0]).date().isoformat() + '\nПричина: ' + info[1]
            bot.reply_to(message, text)
        else:
            text = 'Данный пользователь не находится под ограничениями на данный момент.'
            bot.reply_to(message, text)
    except Exception as ex:
        text = 'Произошла ошибка!\n' + str(ex)
        bot.reply_to(message, text)
        print(str(ex))

@bot.message_handler(commands=['ban_list'],
                     func=lambda message: str(message.chat.id) == SUPPORT_CHAT and SUPPORT_CHAT != 'None')
def ban_list(message):
    lst = bans_database.get_users()
    text = 'Список забаненных:\n'
    for i in lst:
        text += '@' + i[0] + ': ' + str(i[1]) + '\n'
    bot.reply_to(message, text)

@bot.message_handler(content_types=['text'], func=lambda message: str(
    message.chat.id) == SUPPORT_CHAT and SUPPORT_CHAT != 'None' and message.reply_to_message is not None)
def answer(message):
    print(message.chat.id)
    text = '> ' + message.reply_to_message.text + '\n\n' + message.text
    bot.send_message(message.reply_to_message.forward_from.id, text, parse_mode='Markdown')

@bot.message_handler(content_types=['sticker'], func=lambda message: str(
    message.chat.id) == SUPPORT_CHAT and SUPPORT_CHAT != 'None' and message.reply_to_message is not None)
def answer(message):
    text = '> ' + message.reply_to_message.text
    bot.send_message(message.reply_to_message.forward_from.id, text, parse_mode='Markdown')
    sticker = message.sticker.file_id
    bot.send_sticker(message.reply_to_message.forward_from.id, sticker)

@bot.callback_query_handler(func=lambda call: True)
def callback_inline(call):
    try:
        if call.data == 'start_markup_support_chat':
            buttons = types.ReplyKeyboardMarkup(resize_keyboard=True, one_time_keyboard=True)
            btn1 = types.KeyboardButton('Да')
            btn2 = types.KeyboardButton('Нет')
            buttons.add(btn1, btn2)
            text = 'Хотите изменить чат поддержки?'
            message = bot.send_message(call.from_user.id, text, parse_mode='Markdown', reply_markup=buttons)
            bot.register_next_step_handler(message, approve_change_support_chat_id)
        elif call.data == 'unseen':
            bot.delete_message(call.message.chat.id, call.message.id)
    except Exception as ex:
        print('Ops... callback error')
        print(ex)

def approve_change_support_chat_id(message):
    if message.text.lower() == 'да':
        text = 'Хорошо, отправьте id чата, который хотите использовать.'
        bot.send_message(message.from_user.id, text, parse_mode='Markdown')
        bot.register_next_step_handler(message, change_support_chat_id)
    elif message.text.lower() == 'нет':
        text = 'Хорошо, отмена.'
        bot.send_message(message.from_user.id, text, parse_mode='Markdown')
    else:
        text = 'Этот ответ не был ожидаем, так что отмена.'
        bot.send_message(message.from_user.id, text, parse_mode='Markdown')

def change_support_chat_id(message):
    try:
        text = "Теперь это чат поддержки"
        bot.send_message(message.text, text, parse_mode='Markdown', disable_web_page_preview=True)
        with open('config.json', 'r', encoding='utf-8') as file:
            config = json.load(file)
        config['SUPPORT_CHAT'] = message.text
        global SUPPORT_CHAT
        SUPPORT_CHAT = message.text
        with open('config.json', 'w') as file:
            file.write(json.dumps(config, indent=4))
        text = 'Id чата поддержки изменено на ' + message.text
        bot.send_message(message.from_user.id, text, parse_mode='Markdown', disable_web_page_preview=True)
    except Exception as ex:
        text = 'Ошибка. Возможно вы не дали боту возможность писать в чате, который хотите сделать чатом поддержки.'
        text += '\n\n' + str(ex)
        bot.send_message(message.from_user.id, text, parse_mode='Markdown', disable_web_page_preview=True)

@bot.chat_join_request_handler()
def invite(message):
    try:
        flag = False
        try:
            if bot.get_chat_member(message.chat.id, message.from_user.id).status == 'left':
                flag = True
        except:
            flag = True
        if flag:
            if not db.check_user(message.from_user.id):
                db.add_user(message.from_user.id, datetime.now().timestamp(), 0)
            if db.check_user_status(message.from_user.id):
                bot.approve_chat_join_request(message.chat.id, message.from_user.id)
                with open(Path.cwd() / 'messages' / 'welcome.txt', 'r', encoding='utf-8') as file:
                    bot.send_message(message.from_user.id, file.read(), parse_mode='Markdown',
                                     disable_web_page_preview=True)
            else:
                bot.decline_chat_join_request(message.chat.id, message.from_user.id)
                with open(Path.cwd() / 'messages' / 'no_welcome.txt', 'r', encoding='utf-8') as file:
                    bot.send_message(message.from_user.id, file.read(), parse_mode='Markdown',
                                     disable_web_page_preview=True)
        else:
            with open(Path.cwd() / 'messages' / 'already_member.txt', 'r', encoding='utf-8') as file:
                bot.send_message(message.from_user.id, file.read(), parse_mode='Markdown',
                                 disable_web_page_preview=True)
    except Exception as ex:
        print('Ops...\n', ex)

# рассылка
@bot.message_handler(commands=['startmailing'], func=lambda message: str(message.chat.id) != SUPPORT_CHAT)
def start_mailing(message):
    if message.from_user.id in ADMINS_ID:
        text = bot.send_message(message.chat.id, 'Пришлите текст рассылки')
        bot.register_next_step_handler(text, approve)
        bot.set_state(message.from_user.id, MyStates.mailing_text, message.chat.id)

@bot.message_handler(state=MyStates.mailing_text)
def approve(message):
    with bot.retrieve_data(message.from_user.id, message.chat.id) as data:
        data['mailing_text'] = message.text
    bot.send_message(message.chat.id, f'Текст вашей рассылки-{message.text}')
    btns = types.ReplyKeyboardMarkup(resize_keyboard=True)
    btn1 = types.KeyboardButton('Да')
    btn2 = types.KeyboardButton('Нет')
    btn3 = types.KeyboardButton('Отмена')
    btns.add(btn1, btn2, btn3)
    bot.send_message(message.chat.id, 'Добавить кнопку с ссылкой?', reply_markup=btns)
    bot.set_state(message.from_user.id, MyStates.adding_button, message.chat.id)

@bot.message_handler(state=MyStates.adding_button)
def answer(message):
    removing_buttons = types.ReplyKeyboardRemove()
    if message.text == 'Да':
        bot.send_message(message.chat.id, 'Отлично, теперь пришлите текст для кнопки', reply_markup=removing_buttons)
        bot.set_state(message.from_user.id, MyStates.button_text, message.chat.id)
    elif message.text == 'Нет':
        bot.send_message(message.chat.id, 'Отлично, переходим к финальной проверке', reply_markup=removing_buttons)
        with bot.retrieve_data(message.from_user.id, message.chat.id) as data:
            data['button_text'] = ''
            data['button_link'] = ''
            final_approve(data, message.chat.id)
        bot.set_state(message.from_user.id, MyStates.final_approve, message.chat.id)
    elif message.text == 'Отмена':
        bot.send_message(message.chat.id, 'Вы отменили рассылку', reply_markup=removing_buttons)
        bot.delete_state(message.from_user.id, message.chat.id)
    else:
        bot.send_message(message.chat.id, 'Пожалуйста, воспользуйтесь кнопками')

@bot.message_handler(state=MyStates.button_text)
def add_button_text(message):
    with bot.retrieve_data(message.from_user.id, message.chat.id) as data:
        data['button_text'] = message.text
    bot.send_message(message.chat.id, 'Теперь пришлите ссылку для кнопки')
    bot.set_state(message.from_user.id, MyStates.button_link, message.chat.id)

@bot.message_handler(state=MyStates.button_link)
def add_button_link(message):
    if not validators.url(message.text):
        bot.send_message(message.chat.id, 'Это не ссылка')
    else:
        bot.set_state(message.from_user.id, MyStates.final_approve, message.chat.id)
        with bot.retrieve_data(message.from_user.id, message.chat.id) as data:
            data['button_link'] = message.text
        final_approve(data, message.chat.id, button=True)

def final_approve(data, chat_id, button=False):
    buttons = None
    if button:
        buttons = types.InlineKeyboardMarkup()
        b = types.InlineKeyboardButton(text=data['button_text'], url=data['button_link'])
        buttons.add(b)
    bot.send_message(chat_id, 'Вот как выглядит ваша рассылка:')
    bot.send_message(chat_id, text=data['mailing_text'], reply_markup=buttons)
    final_confirm_buttons = types.ReplyKeyboardMarkup(resize_keyboard=True)
    confirm_btn = types.KeyboardButton('Да')
    cancel_btn = types.KeyboardButton('Отмена')
    final_confirm_buttons.add(confirm_btn, cancel_btn)
    bot.send_message(chat_id, 'Начать рассылку? (Это может занять некоторое время)', reply_markup=final_confirm_buttons)

@bot.message_handler(state=MyStates.final_approve)
def answer(message):
    removing_btns = types.ReplyKeyboardRemove()
    if message.text == 'Да':
        with bot.retrieve_data(message.from_user.id, message.chat.id) as data:
            mail_text = data['mailing_text']
            btn_text = data['button_text']
            btn_link = data['button_link']
            sendall(message, mail_text, btn_text, btn_link)
    elif message.text == 'Отмена':
        bot.send_message(message.chat.id, 'Вы отменили рассылку', reply_markup=removing_btns)
        bot.delete_state(message.from_user.id, message.chat.id)
    else:
        bot.send_message(message.chat.id, 'Пожалуйста, воспользуйтесь кнопками')

def sendall(message, mail_text, btn_text, btn_link):
    removing_buttons = types.ReplyKeyboardRemove()
    mail_btns = None
    if btn_text and btn_link:
        mail_btns = types.InlineKeyboardMarkup(row_width=1)
        mail_btn = types.InlineKeyboardButton(text=btn_text, url=btn_link)
        mail_btns.add(mail_btn)
    try:
        list_of_users = db.get_all_users()
        for user in list_of_users:
            bot.send_message(user[0], text=mail_text, reply_markup=mail_btns)
            # у телеграмма вроде есть ограничение 30 сообщ/сек, так что на всякий случай добавила задержку
            time.sleep(0.1)
        bot.send_message(message.chat.id, text='Успех!', reply_markup=removing_buttons)
        bot.set_state(message.from_user.id, MyStates.neutral, message.chat.id)
    except sqlite3.Error:
        bot.send_message(message.chat.id, 'Упс! Проблемы с соединением с базой данных',
                         reply_markup=removing_buttons)
    except Exception as ex:
        bot.send_message(message.chat.id, 'Кажется, что-то пошло не так...', reply_markup=removing_buttons)
        print('Ops...\n', ex)

@bot.message_handler(commands=['payforsub'])
def pay(message):
    with open('purchase_info.json', encoding='utf-8') as file:
        info = json.load(file)
        if 'test' in message.text:
            payment_token = info["TEST_PAYMENT_TOKEN"]
        else:
            payment_token = info["PAYMENT_TOKEN"]
        sub_info = info["subscription_info"]
        title = sub_info["title"]
        description = sub_info["description"]
        purchase_label = sub_info["purchase_label"]
        start_p = sub_info["start_p"]
        invoice_p = sub_info["invoice_p"]
        sub_price = sub_info["price"]

    price = types.LabeledPrice(label=purchase_label, amount=sub_price * 100)
    try:
        bot.send_invoice(message.chat.id,
                         title=title,
                         description=description,
                         provider_token=payment_token,
                         currency="rub",
                         prices=[price],
                         start_parameter=start_p,
                         invoice_payload=invoice_p)
    except Exception as ex:
        print('Проблема с платёжкой')
        print(ex)

@bot.pre_checkout_query_handler(func=lambda query: True)
def checkout(pre_checkout_query):
    bot.answer_pre_checkout_query(pre_checkout_query.id, ok=True,
                                  error_message="Возникла непредвиденная ошибка. Попробуйте позже")

@bot.message_handler(content_types=['successful_payment'])
def successful_payment(message):
    if not db.check_user(message.from_user.id):
        db.add_user(message.from_user.id, datetime.now().timestamp(), 0)
    db.upd_user(message.from_user.id, db.check_user_datetime(message.from_user.id) + int(30 * 24 * 60 * 60), 1)
    bot.send_message(message.chat.id, f"Платёж прошел успешно! "
                                      f"Сумма оплаты составила {message.successful_payment.total_amount / 100}"
                                      f" {message.successful_payment.currency}")

bot.add_custom_filter(telebot.custom_filters.StateFilter(bot))

@bot.message_handler(content_types=['text'],
                     func=lambda message: str(message.chat.id) != SUPPORT_CHAT and SUPPORT_CHAT != 'None')
def forwarding_to_support_chat(message):
    if not bans_database.check_user(message.from_user.id):
        button = types.InlineKeyboardMarkup()
        btn = types.InlineKeyboardButton('Скрыть', callback_data='unseen')
        button.add(btn)
        bot.forward_message(SUPPORT_CHAT, message.chat.id, message.message_id)
        text = 'Ваш вопрос отправлен в чат поддержки, вскоре вы получите на него ответ.'
        bot.send_message(message.chat.id, text, parse_mode='Markdown', reply_markup=button)
    else:
        button = types.InlineKeyboardMarkup()
        btn = types.InlineKeyboardButton('Скрыть', callback_data='unseen')
        button.add(btn)
        info = bans_database.get_user_info(message.from_user.id)
        with open(Path.cwd() / 'messages' / 'impossible_write.txt', 'r', encoding='utf-8') as file:
            text = file.read()
            text = text.replace('[[ date ]]', datetime.fromtimestamp(info[0]).date().isoformat())
        bot.send_message(message.chat.id, text, parse_mode='Markdown', reply_markup=button)

def checking_subscription():
    current_date_time = datetime.now().timestamp()
    all_users = db.get_all()
    for i in all_users:
        if i[3]:
            if current_date_time > i[2]:  # конец подписки
                try:
                    normal_kick(i[1])
                    db.upd_user_status(i[1], 0)
                except Exception as ex:
                    print("Выгнать человека не удалось\n" + str(ex))
                with open(Path.cwd() / 'messages' / 'end_subscription.txt', 'r', encoding='utf-8') as file:
                    bot.send_message(LIST_ID, file.read(), parse_mode='Markdown',
                                     disable_web_page_preview=True)
            elif int(-1 * (i[2] / 86400) // 1 * -1) == 7:  # предупреждение за неделю
                with open(Path.cwd() / 'messages' / 'week_until_end_subscription.txt', 'r',
                          encoding='utf-8') as file:
                    bot.send_message(LIST_ID, file.read(), parse_mode='Markdown',
                                     disable_web_page_preview=True)
            elif int(-1 * (i[2] / 86400) // 1 * -1) == 3:  # предупреждение за три дня
                with open(Path.cwd() / 'messages' / 'three_days_until_end_subscription.txt', 'r',
                          encoding='utf-8') as file:
                    bot.send_message(LIST_ID, file.read(), parse_mode='Markdown',
                                     disable_web_page_preview=True)
            elif int(-1 * (i[2] / 86400) // 1 * -1) == 1:  # предупреждение за один день
                with open(Path.cwd() / 'messages' / 'day_until_end_subscription.txt', 'r', encoding='utf-8') as file:
                    bot.send_message(LIST_ID, file.read(), parse_mode='Markdown',
                                     disable_web_page_preview=True)

def check_bans():
    current_date_time = datetime.now().timestamp()
    all_bans = bans_database.get_all_users()
    for i in all_bans:
        if current_date_time > i[3] - 300:
            try:
                bans_database.delete_user(i[1])
            except Exception as ex:
                print("Не удалось удалить строку из таблицы bans")
                print(ex)
                return
            try:
                text = 'С вас сняты ограничения'
                bot.send_message(i[1], text, parse_mode='Markdown')
            except Exception as ex:
                print(ex)

def normal_kick(user_id):
    try:
        bot.kick_chat_member(LIST_ID, user_id)
        bot.unban_chat_member(LIST_ID, user_id)
    except:
        pass

def run_threaded(job_func):
    job_thread = threading.Thread(target=job_func)
    job_thread.start()

def time_checker():
    trigger_time = "20:07"
    schedule.every().day.at(trigger_time).do(run_threaded, checking_subscription)
    schedule.every().day.at(trigger_time).do(run_threaded, check_bans)
    while RUNNING:
        schedule.run_pending()
        time.sleep(1)

if __name__ == "__main__":
    thread = threading.Thread(target=time_checker)
    RUNNING = True
    thread.start()
    print("Бот запущен")
    bot.polling()
    RUNNING = False
print('Бот остановлен')


import inject
from flask import Flask, g, request
from flask_compress import Compress
from daily_jira.resource import (efficient, jira)

def create_app():
    app = Flask(__name__)
    Compress(app)

    # 注册路由表
    blueprints = (
        efficient.get_resources(),
        jira.get_resources()
    )
    for bp in blueprints:
        app.register_blueprint(bp)
    app.url_map.strict_slashes = False

    return app


import pandas as pd
import numpy as np

df = pd.read_csv("outputfile2.csv",header=None).to_numpy()
#print(df.head())
print(df.shape)
for i in range(0,df.shape[0]):
    if df[i,:][0]==df[i,:][1]:
        print("******************")
'''
for row in df.iterrows():
    print("row:",row)
    #print(row[0],"**",row[1])
'''


from webapp import mysql

class Students(object):

    def __init__(self, idnumber=None, fname=None, mname=None, lname=None, gender=None, yearlvl=None, course=None):
        self.idnumber = idnumber
        self.fname = fname
        self.mname = mname
        self.lname = lname
        self.gender = gender
        self.yearlvl = yearlvl
        self.course = course

    def add(self):
        curs = mysql.connection.cursor()

        sql = f"INSERT INTO data(idnumber,fname,mname,lname,gender,yearlvl,course) \
                VALUES('{self.idnumber}','{self.fname}','{self.mname}','{self.lname}','{self.gender}','{self.yearlvl}','{self.course}')"

        curs.execute(sql)
        mysql.connection.commit()

    @classmethod
    def all(cls):
        curs = mysql.connection.cursor()
        sql = curs.execute("SELECT * from data")
        if sql > 0:
            result = curs.fetchall()
        return result

    @classmethod
    def edit(cls, idnumber):
        curs = mysql.connection.cursor()
        sql = f"SELECT * from data where STUDENTID = {idnumber}"
        curs.execute(sql)
        idnumber = curs.fetchall()
        return idnumber

    @classmethod
    def update(cls, idnumber):
        try:
            curs = mysql.connection.cursor()
            sql = f"UPDATE from data where STUDENTID= {idnumber}"
            curs.execute(sql)
            mysql.connection.commit()
            return True
        except:
            return False

    @classmethod
    def delete(cls,id):
        try:
            curs = mysql.connection.cursor()
            sql = f"DELETE from users where id= {id}"
            curs.execute(sql)
            mysql.connection.commit()
            return True
        except:
            return False

class Courses(object):

    def __init__(self, course=None, coursedesc=None):
        self.course = course
        self.coursedesc = coursedesc

    def add(self):
        cursor = mysql.connection.cursor()

        sql = f"INSERT INTO courses(courses,coursdesc) \
                VALUES('{self.course}','{self.coursedesc}')"

        cursor.execute(sql)
        mysql.connection.commit()
    @classmethod
    def all(cls):
        cursor = mysql.connection.cursor()

        sql = f"SELECT * from courses"
        cursor.execute(sql)
        result = cursor.fetchall()
        return result

    @classmethod
    def populate(cls):
        con = mysql.connection
        curs = con.cursor()
        sql = "SELECT COURSEID FROM courses"
        curs.execute(sql)
        result = [item[0] for item in curs.fetchall()]
        return result


from db.db import session
from flask import Flask, jsonify, request
from flask_restful import Resource, fields, marshal_with, abort, reqparse
import os
import werkzeug
from flask import Flask, request, redirect, url_for
from werkzeug.utils import secure_filename
from db_models.models import Projects,Documents,Chats
import uuid
import re
import subprocess
from pathlib import Path
from settings import ENGINE_PATH, UPLOAD_FOLDER, ALLOWED_EXTENSIONS
import unicodedata

#UPLOAD_FOLDER = 'd:\\uploads'
#ALLOWED_EXTENSIONS = set(['xls','xlsx'])
def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

def generate_file_name():
    pass
class UploadFile(Resource):
    def post(self):
        try:
            f = request.form
            userId = f.get('userId')
            product_id = f.get('productId')
            project = Projects(userId,product_id)
            session.add(project)
            session.commit()
            system_chat = Chats('Обсуждение проекта', userId, project.id, [], True)
            session.add(system_chat)
            session.commit()
            if (project.id>-1):
                f_list =[]
                files =[]
                for t in request.files:
                    f_list = request.files.getlist(str(t))
                    for j_file in f_list:
                        files.append(j_file)
                #files= request.files.getlist("file[]")
                dir_id = str(uuid.uuid4().hex)
                project_folder = os.path.join(UPLOAD_FOLDER, dir_id)
                for file in files:

                    if not os.path.exists(project_folder):
                        os.makedirs(project_folder)
                    if file and allowed_file(file.filename):
                        # From flask uploading tutorial
                        filename =file.filename #str(secure_filename(file.filename)).lower()
                        filename = unicodedata.normalize('NFC', filename).encode('utf-8').decode('utf-8')
                        v = Path(filename)
                        short_name = Path(filename).stem

                        short_name = re.sub(r'[\\/*?:"<>|]', "", short_name)

                        extension =  Path(filename).suffix
                        file_id =str(uuid.uuid4().hex)
                        result_file_name = short_name+"_"+file_id+extension

                        l_res_file_name = len(result_file_name)
                        l_project_folder =len(project_folder)

                        if (l_res_file_name+l_project_folder >250):
                            result_file_name = file_id+extension

                        print("Save to "+result_file_name)
                        file_path = os.path.join(project_folder,result_file_name)
                        file.save(file_path)
                        file_size = os.path.getsize(file_path)
                        document = Documents(project.id,userId,file.filename,file_path,file_size)
                        session.add(document)
                        session.commit()

                        #return {}
                    else:
                        # return error
                        return {}

                #"python d:\Projects\Github\Landau.Pyzzle.Engine\__init__.py "
                if (product_id==None):
                    product_id=-1
                tt = ENGINE_PATH+str(project.id)+" "+str(userId)+" 0 -1"+' '+str(product_id)

                subprocess.Popen(tt, shell=True)
                return {"State":"OK"}
        except Exception as e:
            session.rollback()
            return {"State":"Error"}


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2021/12/26 15:10
# @Author  : strawsyz
# @File    : rakuten-intern.py
# @desc:

def solution(A, K):
    n = len(A)
    best = 0
    count = 0
    for i in range(n - K - 1):
        if (A[i] == A[i + 1]):
            count = count + 1
        else:
            count = 0
        best = max(best, count)
    result = best + 1 + K

    return min(result,n)

if __name__ == '__main__':
    A = [1, 1, 3,3,3,3,3,3,3,3]
    A = [1,  5,5]
    K = 0

    res = solution(A, K)
    print(res)


class Solution:
    def minOperations(self, nums: List[int], x: int) -> int:
        total = sum(nums)
        left = 0
        current = 0
        maxlen = -1

        for right in range(len(nums)):
            current += nums[right]

            while current > total - x and left <= right:
                current -= nums[left]
                left += 1

            if current == total - x:
                maxlen = max(maxlen, right - left + 1)

        return len(nums) - maxlen if maxlen != -1 else -1


"""
In this module lives the type tree.
"""
from collections import defaultdict
from typing import Any, DefaultDict, Dict, List, Optional, Tuple, Type, Union

import dask.dataframe as dd
import numpy as np
import pandas as pd
from ..clean import validate_country, validate_lat_long
from ..errors import UnreachableError

CATEGORICAL_NUMPY_DTYPES = [bool, object]
CATEGORICAL_PANDAS_DTYPES = [pd.CategoricalDtype, pd.PeriodDtype]
CATEGORICAL_DTYPES = CATEGORICAL_NUMPY_DTYPES + CATEGORICAL_PANDAS_DTYPES

STRING_PANDAS_DTYPES = [pd.StringDtype]
STRING_DTYPES = STRING_PANDAS_DTYPES

NUMERICAL_NUMPY_DTYPES = [np.number]
NUMERICAL_DTYPES = NUMERICAL_NUMPY_DTYPES

DATETIME_NUMPY_DTYPES = [np.datetime64]
DATETIME_PANDAS_DTYPES = [pd.DatetimeTZDtype]
DATETIME_DTYPES = DATETIME_NUMPY_DTYPES + DATETIME_PANDAS_DTYPES

NULL_VALUES = {
    float("NaN"),
    "#N/A",
    "#N/A N/A",
    "#NA",
    "-1.#IND",
    "-1.#QNAN",
    "-NaN",
    "-nan",
    "1.#IND",
    "1.#QNAN",
    "<NA>",
    "N/A",
    "NA",
    "NULL",
    "NaN",
    "n/a",
    "nan",
    "null",
    "",
}

class DType:
    """
    Root of Type Tree
    """

############## Syntactic DTypes ##############
class Categorical(DType):
    """
    Type Categorical
    """

class Nominal(Categorical):
    """
    Type Nominal, Subtype of Categorical
    """

class Ordinal(Categorical):
    """
    Type Ordinal, Subtype of Categorical
    """

class Numerical(DType):
    """
    Type Numerical
    """

class Continuous(Numerical):
    """
    Type Continuous, Subtype of Numerical
    """

class Discrete(Numerical):
    """
    Type Discrete, Subtype of Numerical
    """

############## Semantic DTypes ##############

class DateTime(Numerical):
    """
    Type DateTime, Subtype of Numerical
    """

class Text(Nominal):
    """
    Type Text, Subtype of Nominal
    """

class GeoGraphy(Categorical):
    """
    Type GeoGraphy, Subtype of Categorical
    """

class GeoPoint(DType):
    """
    Type GeoPoint
    """

class LatLong(GeoPoint):
    """
    Type LatLong, Tuple
    """

    def __init__(self, lat_col: str, long_col: str) -> None:
        self.lat = lat_col
        self.long = long_col

############## End of the Type Tree ##############

DTypeOrStr = Union[DType, Type[DType], str, None]
DTypeDict = Union[Dict[str, Union[DType, Type[DType], str]], None]
DTypeDef = Union[Dict[str, Union[DType, Type[DType], str]], DType, Type[DType], None]

def detect_dtype(
    col: dd.Series,
    known_dtype: Optional[DTypeDef] = None,
    detect_small_distinct: bool = True,
) -> DType:
    """
    Given a column, detect its type or transform its type according to users' specification

    Parameters
    ----------
    col: dask.datafram.Series
        A dataframe column
    known_dtype: Optional[Union[Dict[str, Union[DType, str]], DType]], default None
        A dictionary or single DType given by users to specify the types for designated columns or
        all columns. E.g.  known_dtype = {"a": Continuous, "b": "Nominal"} or
        known_dtype = {"a": Continuous(), "b": "nominal"} or
        known_dtype = Continuous() or known_dtype = "Continuous" or known_dtype = Continuous()
    detect_small_distinct: bool, default True
        Whether to detect numerical columns with small distinct values as categorical column.
    """
    if not known_dtype:
        return detect_without_known(col, detect_small_distinct=detect_small_distinct)

    if isinstance(known_dtype, dict):
        if col.name in known_dtype:
            dtype = normalize_dtype(known_dtype[col.name])
            return map_dtype(dtype)

    elif isinstance(normalize_dtype(known_dtype), DType):
        return map_dtype(normalize_dtype(known_dtype))

    return detect_without_known(col, detect_small_distinct=detect_small_distinct)

def map_dtype(dtype: DType) -> DType:
    """
    Currently, we want to keep our Type System flattened.
    We will map Categorical() to Nominal() and Numerical() to Continuous()
    """
    if (
        isinstance(dtype, Categorical) is True
        and isinstance(dtype, Ordinal) is False
        and isinstance(dtype, Nominal) is False
    ):
        return Nominal()
    elif (
        isinstance(dtype, Numerical) is True
        and isinstance(dtype, Continuous) is False
        and isinstance(dtype, Discrete) is False
    ):
        return Continuous()
    else:
        return dtype

def detect_without_known(col: dd.Series, detect_small_distinct: bool) -> DType:
    # pylint: disable=too-many-return-statements
    """
    This function detects dtypes of column when users didn't specify.
    """
    if is_nominal(col.dtype):
        if is_geography(col):
            return GeoGraphy()
        if is_geopoint(col):
            return GeoPoint()
        else:
            return Nominal()

    elif is_continuous(col.dtype):
        if detect_small_distinct:
            # detect as categorical if distinct value is small
            nuniques = col.nunique_approx().compute()
            if nuniques < 10:
                return Nominal()
            else:
                return Continuous()
        else:
            return Continuous()

    elif is_datetime(col.dtype):
        return DateTime()
    else:
        raise UnreachableError

def is_dtype(dtype1: Any, dtype2: DType) -> bool:
    """
    This function detects if dtype2 is dtype1.
    """
    return isinstance(dtype1, dtype2.__class__)

def normalize_dtype(dtype_repr: Any) -> DType:
    """
    This function normalizes a dtype repr.
    """
    normalized: DType
    str_dic = {
        "Categorical": Categorical,
        "Ordinal": Ordinal,
        "Nominal": Nominal,
        "Numerical": Numerical,
        "Continuous": Continuous,
        "Discrete": Discrete,
        "DateTime": DateTime,
        "Text": Text,
    }
    for str_dtype, dtype in str_dic.items():
        if isinstance(dtype_repr, str):
            if dtype_repr.lower() == str_dtype.lower():
                normalized = dtype()
                break

        elif isinstance(dtype_repr, dtype):
            normalized = dtype_repr
            break

        elif dtype_repr == dtype:
            normalized = dtype()
            break

    return normalized

def is_nominal(dtype: Any) -> bool:
    """
    Given a type, return if that type is a nominal type
    """

    if is_continuous(dtype) or is_datetime(dtype):
        return False

    if isinstance(dtype, np.dtype):
        dtype = dtype.type

        return any(issubclass(dtype, c) for c in CATEGORICAL_NUMPY_DTYPES)
    else:
        return any(isinstance(dtype, c) for c in CATEGORICAL_PANDAS_DTYPES)

def is_geography(col: dd.Series) -> bool:
    """
    Given a column, return if its type is a geography type
    """
    geo = col.compute()[:100]
    geo_ratio: float = np.sum(validate_country(geo)) / geo.shape[0]
    return geo_ratio > 0.8

def is_geopoint(col: dd.Series) -> bool:
    """
    Given a column, return if its type is a geopoint type
    """
    lat_long = pd.Series(col.compute()[:100], dtype="string")
    lat_long_ratio: float = np.sum(validate_lat_long(lat_long)) / lat_long.shape[0]
    return lat_long_ratio > 0.8

def is_continuous(dtype: Any) -> bool:
    """
    Given a type, return if that type is a continuous type
    """
    dtype = dtype.type
    return any(issubclass(dtype, c) for c in NUMERICAL_NUMPY_DTYPES)

def is_datetime(dtype: Any) -> bool:
    """
    Given a type, return if that type is a datetime type
    """
    if isinstance(dtype, np.dtype):
        dtype = dtype.type
        return any(issubclass(dtype, c) for c in DATETIME_NUMPY_DTYPES)
    else:
        return any(isinstance(dtype, c) for c in DATETIME_PANDAS_DTYPES)

def is_pandas_categorical(dtype: Any) -> bool:
    """
    Detect if a dtype is categorical and from pandas.
    """
    return any(isinstance(dtype, c) for c in CATEGORICAL_PANDAS_DTYPES)

def drop_null(
    var: Union[dd.Series, pd.DataFrame, dd.DataFrame]
) -> Union[pd.Series, dd.Series, pd.DataFrame, dd.DataFrame]:
    """
    Drop the null values (specified in NULL_VALUES) from a series or DataFrame
    """

    if isinstance(var, (pd.Series, dd.Series)):
        if is_datetime(var.dtype):
            return var.dropna()
        return var[~var.isin(NULL_VALUES)]

    elif isinstance(var, (pd.DataFrame, dd.DataFrame)):
        df = var
        for values in df.columns:
            if is_datetime(df[values].dtype):
                df = df.dropna(subset=[values])
            else:
                df = df[~df[values].isin(NULL_VALUES)]
        return df

    raise ValueError("Input should be a Pandas/Dask Dataframe or Series")

def get_dtype_cnts_and_num_cols(
    df: dd.DataFrame,
    dtype: Union[Dict[str, Union[DType, Type[DType], str]], DType, Type[DType], None],
) -> Tuple[Dict[str, int], List[str]]:
    """
    Get the count of each dtype in a dataframe
    """
    dtype_cnts: DefaultDict[str, int] = defaultdict(int)
    num_cols: List[str] = []
    for col in df.columns:
        col_dtype = detect_dtype(df[col], dtype)
        if is_dtype(col_dtype, Nominal()):
            dtype_cnts["Categorical"] += 1
        elif is_dtype(col_dtype, Continuous()):
            dtype_cnts["Numerical"] += 1
            num_cols.append(col)
        elif is_dtype(col_dtype, DateTime()):
            dtype_cnts["DateTime"] += 1
        elif is_dtype(col_dtype, GeoGraphy()):
            dtype_cnts["GeoGraphy"] += 1
        elif is_dtype(col_dtype, GeoPoint()):
            dtype_cnts["GeoPoint"] += 1
        else:
            raise NotImplementedError
    return dtype_cnts, num_cols


#!/usr/bin/env python
# -*- coding: utf-8 -*-

#from sys import argv
import os
import re
import glob
import xlwt

#file_path = input('Insert path where config file are ( for example: /var/logs/conf.cfg ): ')
file_path = 'D:\TEST_CFG\policy_config.txt'

book = xlwt.Workbook(encoding="utf-8")
sheet1 = book.add_sheet("SRX_POLICY_List_#1")

FILTER_ZONES = r'from-zone (.+) to-zone (.+) .+'
FILTER_POLIC = r'.+ policy (.+) .+'
FILTER_SOURC = r'.+ source-address (.+)'
FILTER_DESTN = r'.+ destination-address (.+)'
FILTER_APPLI = r'.+ application (.+)'

with open(file_path, 'r') as f:
    n = 0
    for line in f:
        match = re.match(FILTER_ZONES, line)
        if match:
            n = n + 1
            sheet1.write(n, 1, match.group(1))
            sheet1.write(n, 2, '---->')
            sheet1.write(n, 3, match.group(2))
        match = re.match(FILTER_POLIC, line)
        if match:
            n = n + 1
            sheet1.write(n, 4, match.group(1))
        match = re.match(FILTER_SOURC, line)
        if match:
            sheet1.write(n, 5, match.group(1))
        match = re.match(FILTER_DESTN, line)
        if match:
            sheet1.write(n, 6, match.group(1))
        match = re.match(FILTER_APPLI, line)
        if match:
            sheet1.write(n, 7, match.group(1))
        if "permit" in line:
            sheet1.write(n, 8, "permit")
        elif "reject" in  line:
            sheet1.write(n, 8, "reject")
        elif "deny" in line:
            sheet1.write(n, 8, "deny")
        elif "log {" in line:
            sheet1.write(n, 9, "log")
        elif "session-init;" in line:
            sheet1.write(n, 10, "session-init")
        elif "session-close;" in line:
            sheet1.write(n, 11, "session-close")

book.save("SRX_POLICY_table1.xls")


import pygame
from gameobject import GameObject

class Bullet(GameObject):
    def __init__(self, screen, speedx, speedy, start_x, start_y, animation_speed = 60 ):
        GameObject.__init__(self, screen, speedx, speedy, start_x, start_y, animation_speed)
        self.image = pygame.image.load("assets/objects/shoots/laserGreen.png").convert_alpha()
        self.rect = self.image.get_rect()
        self.rect.bottom = start_y
        self.rect.centerx = start_x
        self.speedy = speedy

    def update(self):
        self.rect.y += self.speedy
        # kill if it moves off the top of the screen
        if self.rect.top < 0:
            self.kill()