import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

df = pd.read_csv("./data/Bullying_2018.csv",sep=';')
#print(df.head())
#df.info()

df.drop(['record', 'Were_underweight', 'Were_overweight', 'Were_obese'], axis=1, inplace=True)

# Replace empty strings with NaN
df.replace(r'^\s*$', np.nan, regex=True, inplace=True)
#print(df.isnull().sum())

# Convert "Yes" and "No" to 1 and 0
df.replace({'Yes': 1, 'No': 0}, inplace=True)

df['Sex'].fillna("Prefers not to answer", inplace=True)
df.dropna(inplace=True)

columns_to_check = ['Bullied_on_school_property_in_past_12_months', 'Bullied_not_on_school_property_in_past_12_months', 'Cyber_bullied_in_past_12_months']

# Find rows with null values in all specified columns
rows_with_all_null = df[df[columns_to_check].isnull().all(axis=1)]

# Drop the rows from the DataFrame
df.drop(rows_with_all_null.index, inplace=True)

# Join 3 columns into 1
df['Bullied_in_past_12_months'] = df[columns_to_check].apply(lambda row: 1 if row.any() == 1 else 0, axis=1)

"""# Fill null values
df.fillna("Prefers not to answer", inplace=True)"""

age_mapping = {
    '11 years old or younger': 11,
    '12 years old': 12,
    '13 years old': 13,
    '14 years old': 14,
    '15 years old': 15,
    '16 years old': 16,
    '17 years old': 17,
    '18 years old or older': 18,
    'Prefers not to answer': 15
}
df['Custom_Age'] = df['Custom_Age'].map(age_mapping)

physically_attacked_mapping = {
    '0 times': 0.0,
    '1 time': 1.0,
    '2 or 3 times': 2.5,
    '4 or 5 times': 4.5,
    '6 or 7 times': 6.5,
    '8 or 9 times': 8.5,
    '10 or 11 times': 10.5,
    '12 or more times': 12.0,
    'Prefers not to answer': 5.0
}
df['Physically_attacked'] = df['Physically_attacked'].map(physically_attacked_mapping)

physical_fighting_mapping = {
    '0 times': 0.0,
    '1 time': 1.0,
    '2 or 3 times': 2.5,
    '4 or 5 times': 4.5,
    '6 or 7 times': 6.5,
    '8 or 9 times': 8.5,
    '10 or 11 times': 10.5,
    '12 or more times': 12.0,
    'Prefers not to answer': 5.0
}
df['Physical_fighting'] = df['Physical_fighting'].map(physical_fighting_mapping)

close_friends_mapping = {
    '0': 0,
    '1': 1,
    '2': 2,
    '3 or more': 3,
    'Prefers not to answer': 0
}
df['Close_friends'] = df['Close_friends'].map(close_friends_mapping)

miss_school_mapping = {
    '0 days': 0.0,
    '1 or 2 days': 1.5,
    '3 to 5 days': 4.0,
    '6 to 9 days': 7.5,
    '10 or more days': 10.0,
    'Prefers not to answer': 5.0
}
df['Miss_school_no_permission'] = df['Miss_school_no_permission'].map(miss_school_mapping)

felt_lonely_mapping = {
    'Never': 0,
    'Sometimes': 1,
    'Rarely': 2,
    'Most of the time': 3,
    'Always': 4
}
df['Felt_lonely'] = df['Felt_lonely'].map(felt_lonely_mapping)

other_students_mapping = {
    'Most of the time': 4,
    'Sometimes': 3,
    'Rarely': 2,
    'Always': 1,
    'Never': 0
}
df['Other_students_kind_and_helpful'] = df['Other_students_kind_and_helpful'].map(other_students_mapping)

parents_mapping = {
    'Always': 4,
    'Never': 0,
    'Rarely': 1,
    'Most of the time': 3,
    'Sometimes': 2
}
df['Parents_understand_problems'] = df['Parents_understand_problems'].map(parents_mapping)

sex_mapping = {
    'Female': 0,
    'Male': 1,
    'Prefers not to answer': 2
}
df['Sex'] = df['Sex'].map(sex_mapping)

"""# Create a new feature 'Has_close_friends' based on 'Close_Friends'
df['Has_close_friends'] = df['Close_friends'].apply(lambda x: 1 if x != '0' else 0)

# Create a new feature 'Physically_attacked_num' based on 'Physically_attacked'
mapping = {
    '0 times': 0,
    '1 time': 1,
    '2 or 3 times': 2.5,
    '4 or 5 times': 4.5,
    '6 or 7 times': 6.5,
    'Prefers not to answer': 0,
    '8 or 9 times': 8.5,
    '10 or 11 times': 10.5
}

df['Physically_attacked_num'] = df['Physically_attacked'].map(mapping)"""

# Write in txt all the features and their count
result = ""
for col in df.columns:
    col_count = str(df[col].value_counts())
    lines = col_count.split('\n')
    lines.pop()
    fixed_col_count = '\n'.join(lines)
    result += fixed_col_count + "\n" + "\n"

with open("./results/fixed-feature-count.txt", "w") as f:
    f.write(result)

"""for col in df.columns:
    col_count = str(df[col].value_counts())
    lines = col_count.split('\n')
    lines.pop()
    fixed_col_count = '\n'.join(lines)
    result += fixed_col_count + "\n" + "Null " + str(df[col].isnull().sum()) + "\n" + "\n"
print(result)"""

# Save in csv file
df.to_csv("./results/fixed-Bullying_2018.csv", index=False, sep=';')

""" # Plot the distribution of 'Bullied_in_past_12_months'
plt.figure(figsize=(8, 6))
sns.countplot(x='Bullied_in_past_12_months', data=df, palette=['#87CEEB', '#FFA500'])
plt.title('Distribution of Bullied_in_past_12_months')
plt.xlabel('Bullied_in_past_12_months')
plt.ylabel('Count')
plt.savefig('./results/plots/Bullied_in_past_12_months.png') """

"""# Contar la cantidad de 0s y 1s para cada característica
counts = df[['Bullied_in_past_12_months', 'Bullied_on_school_property_in_past_12_months',
              'Bullied_not_on_school_property_in_past_12_months', 'Cyber_bullied_in_past_12_months']].apply(pd.Series.value_counts)

# Crear un gráfico de barras apiladas
ax = counts.T.plot(kind='bar', stacked=True, color=['#87CEEB', '#FFA500'])

# Configurar el gráfico
ax.set_ylabel('Cantidad')
ax.set_xlabel('Tipo de Bullying')
ax.set_title('Distribución de los tipos de Bullying')
ax.legend(labels=['No', 'Sí'], loc='lower center')

# Abreviar y mostrar los nombres horizontalmente
abbreviated_labels = ['De cualquier tipo', 'En la escuela', 'Fuera de la escuela', 'Ciberbullying']
ax.set_xticklabels(abbreviated_labels, rotation=0)

plt.savefig('./results/plots/Bullied_Distribution.png')"""

fig, ax = plt.subplots(figsize=(10, 6))

distribution = pd.crosstab(df['Sex'], df['Bullied_in_past_12_months'], margins=True, margins_name='Total')

# Crear un gráfico de barras apiladas con la cantidad total en lugar del porcentaje
ax = distribution[[0, 1]].plot(kind='bar', stacked=True, color=['#87CEEB', '#FFA500'], ax=ax)

# Configurar el gráfico
ax.set_ylabel('Cantidad')
ax.set_xlabel('Sexo')
ax.set_title('Cantidad de estudiantes que sufrieron bullying, de acuerdo a su sexo')
plt.legend(title='Sufre bullying', labels=['No', 'Sí'], loc='lower center')

abbreviated_labels = ['Femenino', 'Masculino', 'No responde', 'Total']
ax.set_xticklabels(abbreviated_labels, rotation=0)

plt.tight_layout()
plt.savefig('./results/plots/Bullied_Sex_Distribution.png')

fig, ax = plt.subplots(figsize=(10, 6))

distribution = pd.crosstab(df['Felt_lonely'], df['Bullied_in_past_12_months'], margins=True, margins_name='Total')

# Crear un gráfico de barras apiladas con la cantidad total en lugar del porcentaje
ax = distribution[[0, 1]].plot(kind='bar', stacked=True, color=['#87CEEB', '#FFA500'], ax=ax)

# Configurar el gráfico
ax.set_ylabel('Cantidad')
ax.set_xlabel('Se siente solo/a')
ax.set_title('Cantidad de estudiantes que sufrieron bullying, de acuerdo a si se sentían solos')
plt.legend(title='Sufre bullying', labels=['No', 'Sí'], loc='lower center')

abbreviated_labels = ['Siempre', 'Casi siempre', 'Nunca', 'Rara vez', 'A veces', 'Total']
ax.set_xticklabels(abbreviated_labels, rotation=0)

plt.tight_layout()
plt.savefig('./results/plots/Bullied_Felt_lonely_Distribution.png')


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os

from setuptools import setup, find_packages

def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

setup(
    name='dood',
    version='0.1dev',
    description='A simplified API for creating polls on doodle.com.',
    long_description=read('README.rst'),
    author='Marc Brinkmann',
    author_email='git@marcbrinkmann.de',
    url='http://github.com/mbr/dood',
    license='MIT',
    packages=find_packages(exclude=['test']),
    install_requires=['lxml', 'rauth', 'requests', 'xmltodict'],
)


import re
import logging
from config_generator.connections import connect, get_hostname
from config_generator.fstools import get_script_parameters
from config_generator.parameters import get_evpn_groups, get_ncs_ports_ranges

############################
# CONSTANTS
############################
SCRIPT_PARAMETERS = get_script_parameters()
EVPN_GROUPS = get_evpn_groups()
MIN_AGGR_PORT, MAX_AGGR_PORT = get_ncs_ports_ranges("min_aggr_port", "max_aggr_port")
MIN_INTERLINK_PORT, MAX_INTERLINK_PORT = get_ncs_ports_ranges("min_interlink_port",
                                                              "max_interlink_port")
MIN_UPLINK_PORT, MAX_UPLINK_PORT = get_ncs_ports_ranges("min_uplink_port",
                                                        "max_uplink_port")
############################
# CLASSES
############################

class AccessList:
    def __init__(self, name):
        self.name = name
        self.rules = []
        self.is_on_se = False
        self.is_on_pe = False

    def set_on_se(self):
        self.is_on_se = True

    def set_on_pe(self):
        self.is_on_pe = True

    def add_rules(self, rules):
        self.rules.append(rules)

class RemoteNeighbor:
    def __init__(self, ip, device_type):
        self.ip = ip
        self.pwid_list = []
        self.ssh = None
        self.old_loopback = None
        self.ncs_loopback = None
        self.ncs_loopback_backup = None
        self.hostname = None
        self.device_type = device_type
        self.ac_interfaces = []
        self._command_template = None

    def connect(self, username, password):
        self.ssh = connect(username, password, self.ip,
                           show_status=False,
                           device_type=self.device_type)
        self.hostname = get_hostname(self.ssh)

    def disconnect(self):
        self.ssh.disconnect()

    def add_pwid(self, pwid):
        self.pwid_list.append(pwid)

    def add_pwid_list(self, pwid_list):
        self.pwid_list.extend(pwid_list)

    def _get_ac_interface(self, pwid, *old_loopbacks):
        command = 'show xc all | i '
        for loopback in old_loopbacks:
            command += f'{loopback}:{pwid}'
            if old_loopbacks.index(loopback) != len(old_loopbacks) - 1:
                command += '|'
        output = self.ssh.send_command(command)
        if self.device_type == 'cisco_ios':
            search = re.search(r'((?:Gi|Te)(?:\d+\/?)*\.\d+)', output)
        elif self.device_type == 'cisco_xe':
            search = re.search(r'((?:Gi|Te)(?:\d+\/?)*:\d+)', output)
        if search:
            interface = search.group(1)
            logging.info(f'Remote {self.ip}: pwid {pwid} is on {interface}')
            return interface
        else:
            logging.info(f'Remote {self.ip}: pwid {pwid} hasn\'t been found')

    def get_ac_interfaces(self, *old_loopbacks, pwid_list=None):
        """Search for AC-interfaces for all pwids in pwid_list"""
        # Если pwid_list не передан, как атрибут, берем список из self.pwid_list
        if not pwid_list:
            if self.pwid_list:
                pwid_list = self.pwid_list
            else:
                logging.warning('pwid_list is empty. '
                                'No interface_list has been generated')
                raise ValueError
        for pwid in pwid_list:
            self.ac_interfaces.append(self._get_ac_interface(pwid, *old_loopbacks))
        logging.info('Interface_list is {}'.format(self.ac_interfaces))

    def _create_ios_config(self, se_loopback, se_loopback_backup, *old_loopbacks):
        """Create config for one interface"""
        self.get_ac_interfaces(*old_loopbacks)
        config = []
        for interface in self.ac_interfaces:
            config.append(f'no interface {interface}\ninterface {interface}')
            output = self.ssh.send_command(f'show running-config interface {interface}')
            for line in output.split('\n'):
                if re.search(r'(^\s)', line):
                    if re.search(r'^ xconnect', line):
                        pwid = re.search(r'xconnect \d+\.\d+\.\d+\.\d+ (\d+)', line).group(1)
                        config.append(re.sub(r'\d+\.\d+\.\d+\.\d+', se_loopback, line))
                        config.append(f'  backup peer {se_loopback_backup} {pwid}')
                    elif re.search(r'^  backup', line):
                        continue
                    else:
                        config.append(line)
            config.append('exit\n')
        return '\n'.join(config)

    def _create_xr_command_template(self, *old_loopbacks):
        command = 'show run formal l2vpn | i "'
        for loopback in old_loopbacks:
            command += f'neighbor.*{loopback} '
            command += 'pw-id {0}'
            if old_loopbacks.index(loopback) != len(old_loopbacks) - 1:
                command += '|'
        command += '"'
        return command

    def _create_xr_config(self, se_loopback, se_loopback_backup, *old_loopbacks):
        self._command_template = self._create_xr_command_template(*old_loopbacks)
        config = []
        for pwid in self.pwid_list:
            output = self.ssh.send_command(self._command_template.format(pwid))
            for line in output.split('\n'):
                if is_bd_config(line) and 'backup' not in line:
                    if 'split-horizon' not in line:
                        config.append('no ' + line)
                    config_line = re.sub(r'neighbor .*? ', f'neighbor {se_loopback} ',
                                         line, count=1)
                    config.append(config_line)
                    if 'split-horizon' not in line:
                        config.append(config_line + f'backup neighbor '
                                                    f'{se_loopback_backup} pw-id {pwid}')
                elif is_xc_config(line) and 'backup' not in line:
                    config.append('no ' + line)
                    config_line = re.sub(r'neighbor ipv4 .*? ',
                                         f'neighbor ipv4 {se_loopback} ', line, count=1)
                    config.append(config_line)
                    config.append(config_line + f'backup neighbor {se_loopback_backup} '
                                                f'pw-id {pwid}')
        return '\n'.join(config)

    def create_config(self, se_loopback, se_loopback_backup, *old_loopbacks):
        """Create config for all neighbor pwids"""
        config = '{}\n{}\n\n'.format(self.hostname, '*' * len(self.hostname))
        if self.device_type == 'cisco_ios':
            config += self._create_ios_config(se_loopback,
                                              se_loopback_backup,
                                              *old_loopbacks)
        elif self.device_type == 'cisco_xr':
            config += self._create_xr_config(se_loopback,
                                             se_loopback_backup,
                                             *old_loopbacks)
        # elif self.device_type == 'cisco_xe':
            # config += self._create_xr_config(se_loopback,
            #                                  se_loopback_backup,
            #                                  *old_loopbacks)

        # elif self.device_type == 'juniper_junos':
            # config += self._create_xr_config(se_loopback,
            #                                  se_loopback_backup,
            #                                  *old_loopbacks)
        config += '\n\n'
        return config

############################
# FUNCTIONS
############################

def is_shaping_policy(policy_name):
    return True if re.search(r'(\d+)(m|k)', policy_name) else False

def get_shaping_policies(vlans):
    policies = set()
    for vlan in vlans:
        policy_in = vlan.get('svi_policy_in')
        policy_out = vlan.get('svi_policy_out')
        if policy_in and is_shaping_policy(policy_in):
            policies.add(policy_in)
        if policy_out and is_shaping_policy(policy_out):
            policies.add(policy_out)

    return policies

def create_policy(name):
    policy_parameters = re.search(r'(\d+)(m|k)', name)
    if policy_parameters:
        shape_speed = round(int(policy_parameters.group(1)) * 1.1)
        prefix = policy_parameters.group(2)
        policy = f'policy-map {name}\n' \
                 f' class class-default\n' \
                 f'  shape average {shape_speed} {prefix}bps' \
                 f'\n !' \
                 f'\n  end-policy-map\n' \
                 f'!\n'
    else:
        return None
    return policy

def get_acl_list(connection):
    """

    :param connection:
    :return:
    """
    logger = logging.getLogger('get_acl_list')
    logger.info('Starting function')

    acl_names = set()
    acl_list = []
    acl_output = connection.send_command('sh run partition access-list')
    for line in acl_output.split(sep='\n'):

        # Стандартный и расширенный ACL:
        if line.startswith('ip access-list'):
            acl_name = re.search(r'ip access-list (?:standard|extended) (\S+)',
                                 line).group(1)
            if acl_names:  # Если это не первый ACL, сохраняем предыдущий
                acl_list.append(acl)
            acl = AccessList(acl_name)
            acl_names.add(acl_name)
        elif line.startswith(' permit') or line.startswith(' deny'):
            acl.add_rules(re.sub('^ ', '', line))

        # Нумерованный ACL:
        elif line.startswith('access-list'):
            num_acl_output = re.search(r'access-list (?P<name>\d+) (?P<rule>.*)', line)
            if num_acl_output:
                acl_name = num_acl_output.group('name')
                # Проверяем был ли расширенный ACL уже добавлен:
                if acl_name not in acl_names:
                    if acl_names:  # Если это не первый ACL, сохраняем предыдущий
                        acl_list.append(acl)
                    acl = AccessList(acl_name)
                    acl_names.add(acl_name)
                rule = num_acl_output.group('rule')
                acl.add_rules(rule)
        else:
            continue
    acl_list.append(acl)
    return acl_list

def is_vlan_fake(vlan):
    return True if vlan['fake'] else False

def get_acl_inuse(vlans):
    """Return unique set of ACL, which are used on BVIs"""
    acl_inuse = set()
    for vlan in vlans:
        if vlan.get('svi_acl_in'):
            acl_inuse.add(vlan.get('svi_acl_in'))
        if vlan.get('svi_acl_out'):
            acl_inuse.add(vlan.get('svi_acl_out'))
    return acl_inuse

def get_acl_location(vlans):
    """Return two sets, acl_on_pe and acl_on_se"""
    acl_on_pe = set()
    acl_on_se = set()
    for vlan in vlans:
        if vlan.get('svi_acl_in') and is_vlan_fake(vlan):
            acl_on_se.add(vlan.get('svi_acl_in'))
        elif vlan.get('svi_acl_in') and not is_vlan_fake(vlan):
            acl_on_pe.add(vlan.get('svi_acl_in'))
        if vlan.get('svi_acl_out') and is_vlan_fake(vlan):
            acl_on_se.add(vlan.get('svi_acl_out'))
        elif vlan.get('svi_acl_out') and not is_vlan_fake(vlan):
            acl_on_pe.add(vlan.get('svi_acl_out'))
    return acl_on_pe, acl_on_se

def get_unique_acl_list(acl_list, vlans):
    """

    :param vlans:
    :param acl_list:
    :return:
    """

    logger = logging.getLogger('get_unique_acl_list')
    logger.info('Starting function')

    acl_names = set()  # Множество всех имен ACL, проверка на повторы
    unique_acl_list = []  # Лист, содержащий уникальные используемые ACL
    acl_inuse = get_acl_inuse(vlans)
    acl_on_pe, acl_on_se = get_acl_location(vlans)

    for acl in acl_list:
        if acl.name in acl_inuse and acl.name not in acl_names:
            if acl.name in acl_on_se:
                acl.set_on_se()
            if acl.name in acl_on_pe:
                acl.set_on_pe()
            unique_acl_list.append(acl)
            acl_names.add(acl.name)
    return unique_acl_list

def get_neighbor_role(neighbor):
    return neighbor['role']

def split_neighbors_to_group(neighbor_list):
    """

    :param neighbor_list:
    :return:
    """

    logger = logging.getLogger('split_neighbors_to_group')
    logger.info('Starting function')

    aggregation_list = []  # Список соседей-агрегаций
    uplink_list = []  # Список соседей-аплинков
    interlink_list = []  # Список межкомплектов

    for neighbor in neighbor_list:
        if neighbor['alarm']:  # Обрабатываем только соседей без флага ALARM
            logger.info(f'Neighbor {neighbor["neighbor"]} has ALARM flag. Skipping')
            continue
        elif get_neighbor_role(neighbor) == 'aggregation':
            logger.info(f'Neighbor {neighbor["neighbor"]} is an aggregation')
            aggregation_list.append(neighbor)
        elif get_neighbor_role(neighbor) == 'uplink':
            logger.info(f'Neighbor {neighbor["neighbor"]} is an uplink')
            uplink_list.append(neighbor)
        elif get_neighbor_role(neighbor) == 'interlink':
            logger.info(f'Neighbor {neighbor["neighbor"]} is an interlink')
            interlink_list.append(neighbor)

    return aggregation_list, uplink_list, interlink_list

def sort_aggregations(aggregations):
    """Move MTU switches to first positions"""
    mtu1 = mtu2 = None
    for aggr in aggregations:
        if re.search(r'-MTU\d+-01', aggr['neighbor']):
            mtu1 = aggr
        elif re.search(r'-MTU\d+-02', aggr['neighbor']):
            mtu2 = aggr
    sorted_aggregations = []
    if mtu1:
        sorted_aggregations.append(mtu1)
    if mtu2:
        sorted_aggregations.append(mtu2)
    for aggr in aggregations:
        if aggr != mtu1 and aggr != mtu2:
            sorted_aggregations.append(aggr)

    return sorted_aggregations

def is_gist_vlan(vlan):
    return re.search(r'gist', vlan['vlan_name'].lower())

def set_evpn_groups_for_vlan(vlans):
    """

    :param vlans:
    :return:
    """

    logger = logging.getLogger('set_evpn_groups_for_vlan')
    logger.info('Starting function')

    for vlan in vlans:
        if vlan['is_l2/l3'] == 'L3':
            try:
                vlan['evpn_group'] = EVPN_GROUPS[vlan['svi_vrf']]
                logger.info(f"Vlan {vlan['vlan_id']} has marked as "
                            f"{vlan['evpn_group']} group")
            except KeyError:  # Если ключа нет, то это отдельный клиент
                vlan['evpn_group'] = 'EVPN_CLIENTS'
                logger.info(f"Vlan {vlan['vlan_id']} has marked as "
                            f"{vlan['evpn_group']} group")
        # L2 это EVPN_CLIENTS, либо EVPN_GIST:
        elif is_gist_vlan(vlan):
            vlan['evpn_group'] = 'EVPN_GIST'
            logger.info(f"Vlan {vlan['vlan_id']} has marked as "
                        f"{vlan['evpn_group']} group")
        else:
            vlan['evpn_group'] = 'EVPN_CLIENTS'
            logger.info(f"Vlan {vlan['vlan_id']} has marked as "
                        f"{vlan['evpn_group']} group")

def is_ucn_xc(xc):
    return re.search('UCN', str(xc['source_int_description']))

def is_szo_xc(xc):
    return re.search('SZO', str(xc['source_int_description']))

def is_gist_xc(xc):
    return re.search('gist', str(xc['source_int_description']).lower())

def is_pppoe_xc(xc):
    return re.search('pppoe', str(xc['source_int_description']).lower())

def add_is_pppoe_flags(xc_list):
    for xc in xc_list:
        if xc.get('source_int_description') and is_pppoe_xc(xc):
            xc['is_pppoe'] = True
        else:
            xc['is_pppoe'] = False

def set_evpn_groups_for_xc(xc_list):
    """

    :param xc_list:
    :return:
    """

    logger = logging.getLogger('set_evpn_groups_for_xc')
    logger.info('Starting function')

    for xc in xc_list:
        if xc.get('source_int_description') and is_ucn_xc(xc):
            xc['evpn_group'] = 'EVPN_UCN'
        elif xc.get('source_int_description') and is_szo_xc(xc):
            xc['evpn_group'] = 'EVPN_SZO'
        elif xc.get('source_int_description') and is_gist_xc(xc):
            xc['evpn_group'] = 'EVPN_GIST'
        elif xc.get('source_int_description') and is_pppoe_xc(xc):
            xc['evpn_group'] = 'EVPN_PPPoE'
        else:
            xc['evpn_group'] = 'EVPN_CLIENTS'

        logger.info(f"XC from {xc['source_port']} to {xc['remote_ip']} "
                    f"has marked as {xc['evpn_group']}")

def get_evi_id_for_vlan(vlan, ncs_evi_list):
    """

    :param ncs_evi_list:
    :param vlan:
    :return:
    """

    logger = logging.getLogger('get_evi_id_for_vlan')
    logger.info('Starting function')

    if vlan['vlan_id'] not in ncs_evi_list:
        evi_id = vlan['vlan_id']
        ncs_evi_list.append(evi_id)
    # В случае дублирования добавляем слева к номеру EVI ID 5, 6 и т.д:
    else:
        vlan['is_duplicated'] = True
        for evi_id_addiction in range(5, 30):
            if len(vlan['vlan_id']) == 1:
                evi_id = str(evi_id_addiction) + '00' + vlan['vlan_id']
            elif len(vlan['vlan_id']) == 2:
                evi_id = str(evi_id_addiction) + '0' + vlan['vlan_id']
            else:
                evi_id = str(evi_id_addiction) + vlan['vlan_id']
            if evi_id not in ncs_evi_list:
                ncs_evi_list.append(evi_id)
                break

    return evi_id

def get_evi_id_for_xc(xc, ncs_evi_list):
    """

    :param ncs_evi_list:
    :param xc:
    :return:
    """

    logger = logging.getLogger('get_evi_id_for_xc')
    logger.info('Starting function')

    if xc['dot1q'] not in ncs_evi_list:
        evi_id = xc['dot1q']
        ncs_evi_list.append(evi_id)
    else:
        # PPPoE XC и XC между PE могут повторяться, это не дубликаты
        if xc['evpn_group'] != 'EVPN_PPPoE' and not xc.get('not_xc'):
            xc['is_duplicated'] = True
            for evi_id_addiction in range(5, 30):
                if len(xc['dot1q']) == 1:
                    evi_id = str(evi_id_addiction) + '00' + xc['dot1q']
                elif len(xc['dot1q']) == 2:
                    evi_id = str(evi_id_addiction) + '0' + xc['dot1q']
                else:
                    evi_id = str(evi_id_addiction) + xc['dot1q']
                if evi_id not in ncs_evi_list and xc['evpn_group'] != 'EVPN_PPPoE':
                    ncs_evi_list.append(evi_id)
                    break
        else:
            evi_id = xc['dot1q']

    return evi_id

def get_pppoe_xc_diff(xc_list, hostname):
    """

    :param hostname:
    :param xc_list:
    :return: pppoe_xc_diff_list
    """

    logger = logging.getLogger('get_pppoe_xc_diff')
    logger.info('{} Starting function'.format(hostname))

    pe1_pppoe_xc = []  # Список XC (dot1q) с текущей PE
    pe2_pppoe_xc = []  # Список XC (dot1q) с другой PE
    pppoe_xc_diff = []  # Список XC (dot1q), отсутствующих на текущей PE

    for xc in xc_list:
        if xc['evpn_group'] == 'EVPN_PPPoE':
            if xc['hostname'] == hostname:
                pe1_pppoe_xc.append(xc['dot1q'])
            else:
                pe2_pppoe_xc.append(xc['dot1q'])
    for pppoe_xc in pe2_pppoe_xc:
        if pppoe_xc not in pe1_pppoe_xc:
            pppoe_xc_diff.append(pppoe_xc)
    logger.info(f"{hostname} PPPoE XC diff list: {pppoe_xc_diff}")

    return pppoe_xc_diff

def get_pppoe_neighbors(xc_list, hostname1, hostname2):
    """

    :param xc_list:
    :param hostname1:
    :param hostname2:
    :return:
    """

    logger = logging.getLogger('get_pppoe_neighbors')
    logger.info('Starting function')

    neighbors = {}  # Словарь, содержащий значения pe_hostname:remote_pe_ip
    for xc in xc_list:
        if xc['evpn_group'] == 'EVPN_PPPoE' and xc['hostname'] == hostname1:
            neighbors[hostname1] = xc['remote_ip']
        elif xc['evpn_group'] == 'EVPN_PPPoE' and xc['hostname'] == hostname2:
            neighbors[hostname2] = xc['remote_ip']
    logger.info(f'PPPoE neighbors dictionary contains: {neighbors}')

    return neighbors

def get_old_neighbor_lo30(connection):
    return connection['old_neighbor_lo30']

def need_convertation_to_vlan(xc_list, hostname, old_neighbor_lo30):
    logger = logging.getLogger('need_convertation_to_vlan')
    logger.info(f'{hostname} Starting function')
    for xc in xc_list:
        if xc['hostname'] == hostname and xc['remote_ip'] == old_neighbor_lo30:
            xc['not_xc'] = True

def convert_xc_to_vlan(xc_list, vlans, hostname):
    logger = logging.getLogger('convert_xc_to_vlan')
    logger.info('Starting function')

    pwid_list = []
    xc_to_convert = []
    converted_xc = []

    for xc in xc_list:
        if xc.get('not_xc'):
            if xc['hostname'] == hostname:
                pwid_list.append(xc['pwid'])
                xc_to_convert.append(xc)
            elif xc['pwid'] in pwid_list:
                for second_xc in xc_to_convert:
                    if xc['pwid'] == second_xc['pwid']:
                        vlans.append({'vlan_id': xc['dot1q'],
                                      'is_l2/l3': 'L2',
                                      'is_merged': True,
                                      'neighbors': xc['source_neighbor'] + ', ' +
                                                   second_xc['source_neighbor'],
                                      'vlan_name': xc['source_int_description'],
                                      'alarm_neighbors': None,
                                      'alarm_hsrp': None,
                                      'alarm_subnets': None,
                                      'alarm_intersection': None,
                                      'is_in_eigrp': None
                                      })
                        converted_xc.append(xc)
                        converted_xc.append(second_xc)
                        break

    for xc in converted_xc:
        xc_list.remove(xc)

def get_fake_scheme(vlan_list):
    """

    :param vlan_list:
    :return:
    """

    logger = logging.getLogger('get_fake_scheme')

    for vlan in vlan_list:
        logger.info(f"Starting func for vlan {vlan['vlan_id']}")
        # Проверка MGMT вланов:
        if vlan.get('svi_vrf') == 'MGMT' \
                and vlan.get('vlan_id') not in ('1', '2', '888', '901'):
            vlan['fake'] = 2
            logger.info(f"Vlan {vlan['vlan_id']} is FAKE2 (MGMT)")
        # Проверка банков:
        elif vlan.get('svi_vrf') and 'bank' in vlan['svi_vrf'].lower():
            vlan['fake'] = 1
            logger.info(f"Vlan {vlan['vlan_id']} is FAKE1 ('BANK' IN VRF)")
        elif vlan.get('svi_descr') and 'bank' in vlan['svi_descr'].lower():
            vlan['fake'] = 1
            logger.info(f"Vlan {vlan['vlan_id']} is FAKE1 ('BANK' IN DESCR)")
        # Проверка остальных исключений:
        elif vlan.get('svi_vrf') in ['TATTRANSGAZ', 'kes', 'n_Zelax', 'MONITOR_TLS']:
            vlan['fake'] = 1
            logger.info(f"Vlan {vlan['vlan_id']} is FAKE1 "
                        f"(VRF {vlan['svi_vrf']} is an exception)")
        elif vlan.get('vlan_id') in ['3004']:
            vlan['fake'] = 1
            logger.info(f"Vlan {vlan['vlan_id']} is FAKE1 (Vlan is an exception)")
        else:
            vlan['fake'] = 0
            logger.info(f"Vlan {vlan['vlan_id']} is not fake")

def format_acl_rule(rule):
    return rule.replace(' ip ', ' ipv4 ')

def create_acl_config(acl):
    logger = logging.getLogger('create_acl_config')
    logger.info('Starting function')

    config = ''
    rule_index = 10
    for rule in acl.rules:
        config += f"ipv4 access-list {acl.name} {rule_index} {format_acl_rule(rule)}\n"
        rule_index += 10

    return config

def format_acl_name_for_se(acl_name, hostname):
    logger = logging.getLogger('change_acl_name_for_se')
    logger.info('Starting function')

    return re.search(r'(\S+)+PE\d', hostname).group(1) + acl_name

def create_remotes_config(**kwargs):

    logger = logging.getLogger('create_remotes_config')
    logger.info('Starting function')

    # Промежуточный словарь ip:[pwid_list]
    d = {}
    for xc in kwargs["xconnects"]:
        d.setdefault(xc['remote_lo2'], []).append(xc['pwid'])

    # Собираем список всех удаленных соседей (объектов RemoteNeighbor):

    remote_neighbors = []
    for remote_ip in d.keys():
        neighbor = RemoteNeighbor(remote_ip, kwargs['device_type'])
        neighbor.add_pwid_list(d[remote_ip])
        remote_neighbors.append(neighbor)

    # Проходим по всем соседям и генерируем конфиг
    config = ''
    for neighbor in remote_neighbors:
        neighbor.connect(kwargs["ssh_username"], kwargs["ssh_password"])
        config += neighbor.create_config(kwargs["se_loopback"],
                                         kwargs["se_loopback_backup"],
                                         kwargs["pe1_old_lo30"],
                                         kwargs["pe2_old_lo30"])
        neighbor.disconnect()

    return config

def format_data_from_table(sheet_title, data):
    if sheet_title == 'Общий список VLAN':
        return format_vlan_list(data)
    elif sheet_title == 'Общий список соседей':
        return format_neighbor_list(data)
    elif sheet_title == 'Список XC':
        return format_xc_list(data)
    else:
        return data

def format_vlan_list(vlan_list):
    logger = logging.getLogger('format_vlan_list')
    logger.info('Start function')

    formatted_vlan_list = []
    for vlan in vlan_list:
        if not vlan['alarm_neighbors'] and not vlan['alarm_hsrp'] \
                and not vlan['alarm_subnets'] \
                and not vlan['alarm_intersection'] \
                and not vlan['is_in_eigrp']:
            if vlan.get('svi_ip_secondary'):
                vlan['svi_ip_secondary'] = vlan['svi_ip_secondary'].split(', ')
            formatted_vlan_list.append(vlan)
        else:
            logger.info(f'Vlan {vlan["vlan_id"]} has ALARM flag. Skipping')
    return formatted_vlan_list

def format_neighbor_list(neighbor_list):
    for neighbor in neighbor_list:
        if re.search(r'(^|\s)gist', neighbor['neighbor'].lower()):
            neighbor['is_gist_router'] = True
            neighbor['neighbor'] = 'GIST_ROUTER'
    return neighbor_list

def format_xc_list(xc_list):
    logger = logging.getLogger('format_xc_list')
    logger.info('Start function')
    formatted_xc_list = []
    for xc in xc_list:
        if not xc.get('alarm'):
            formatted_xc_list.append(xc)
        else:
            logger.info(f'{xc["hostname"]} XC from {xc["source_port"]} '
                        f'to {xc["remote_ip"]} has ALARM flag. Skipping')
    return formatted_xc_list

def get_esi_template(loopback30):
    logger = logging.getLogger('get_esi_template')
    logger.info(f'Creating Ethernet Segment ID for {loopback30}')

    lo30_octets = []
    esi_template = ''
    for group in range(1, 5):
        lo30_octet = re.search(r'(\d+)\.(\d+)\.(\d+)\.(\d+)', loopback30).group(group)
        if len(lo30_octet) == 1:
            lo30_octet = '0' + lo30_octet
        elif len(lo30_octet) == 3:
            lo30_octets.append(lo30_octet[0:2])
            lo30_octets.append(lo30_octet[2] + '0')
            continue
        lo30_octets.append(lo30_octet)

    for esi_octet in range(0, 8):
        try:
            esi_template += lo30_octets[esi_octet] + '.'
        except IndexError:
            esi_template += '00.'

    esi_template += '{}'
    logger.info(f'Ethernet Segment ID template is: {esi_template}')

    return esi_template

def get_7600s_lo30(connection):
    output = connection.send_command('sh run int lo30 | i ip address')
    return re.search(r'ip address (\d+.\d+.\d+.\d+)', output).group(1)

def create_ncs_hostname(old_hostname):
    hostname = re.search(r'(\S+)+PE\d', old_hostname).group(1) + 'NCS-' + \
               re.search(r'(PE.*)', old_hostname).group(1)
    return hostname

def get_bgp_vpnv4_neighbors(connection):
    output = connection.send_command('show run partition router bgp 65000 | '
                                     'sec address-family vpnv4')
    return re.findall(r'neighbor (\S+) activate', output)

def is_bd_config(line):
    return True if line.startswith('l2vpn bridge') else False

def is_xc_config(line):
    return True if line.startswith('l2vpn xconnect') else False

def write_init_config(hostname, loopback2, loopback30, ncs_config_file):
    init_config = [
        "service unsupported-transceiver\n",
        f"hostname {hostname}\n",
        "group auto-bw\n",
        " interface 'tunnel-te.*'\n",
        " interface 'tunnel-te.*'  ipv4 unnumbered Loopback30\n",
        " interface 'tunnel-te.*'  load-interval 30\n",
        " interface 'tunnel-te.*'  auto-bw \n",
        " interface 'tunnel-te.*'  auto-bw  bw-limit min 50000 max 300000000\n",
        " interface 'tunnel-te.*'  auto-bw  overflow threshold 10 min 5000 "
        "limit 3\n",
        " interface 'tunnel-te.*'  auto-bw  adjustment-threshold 10 min 10000\n",
        " interface 'tunnel-te.*'  auto-bw  underflow threshold 10 "
        "min 5000 limit 3\n",
        " interface 'tunnel-te.*'  auto-bw  application 600\n",
        " interface 'tunnel-te.*'  logger events lsp-status state\n",
        " interface 'tunnel-te.*'  logger events pcalc-failure\n",
        " interface 'tunnel-te.*'  logger events lsp-status "
        "insufficient-bandwidth\n",
        " interface 'tunnel-te.*'  path-option 10 dynamic\n",
        " interface 'tunnel-te.*'  logger events link-status\n",
        "end-group\n",
        "clock timezone MSK Europe/Moscow\n",
        # LOGGING:
        "logger events threshold 90\n",
        "logger events buffer-size 1000000\n",
        "logger events level notifications\n",
        "logger buffered 10000000\n",
        "logger buffered notifications\n",
        "logger 172.16.31.31 vrf MGMT severity info port default\n",
        "logger suppress rule license alarm LICENSE SMART_LIC "
        "EVAL_EXPIRED_WARNING\n",
        "logger suppress apply rule license\n",
        "logger suppress apply rule license all-of-router\n",
        "logger suppress rule tr69block alarm L2 L2RIB DUP_HOST\n",
        "logger suppress apply rule tr69block\n",
        "logger suppress apply rule tr69block all-of-router\n",
        "logger suppress rule ldpduplicate alarm ROUTING LDP PEER_DUP_ADDRS\n",
        "logger suppress apply rule ldpduplicate\n",
        "logger suppress apply rule ldpduplicate all-of-router\n",
        f"logger hostnameprefix {hostname}\n",
        # ACCESS:
        "telnet vrf MGMT ipv4 server max-servers 20 access-list VTY_ACCESS\n",
        "telnet vrf default ipv4 server max-servers 10\n",
        "tacacs source-interface Loopback2 vrf MGMT\n",
        "aaa accounting exec default start-stop group TAT\n",
        "aaa accounting network default start-stop group TAT\n",
        "aaa accounting commands default start-stop group TAT\n",
        "aaa group server tacacs+ TAT\n",
        "aaa group server tacacs+ TAT vrf MGMT\n",
        "aaa group server tacacs+ TAT server-private 172.16.31.251 port 4949\n",
        "aaa group server tacacs+ TAT server-private 172.16.31.251 port 4949 "
        "key 7 075B781817\n",
        "aaa authentication login default local group TAT\n",
        "line default exec-timeout 0 0\n",
        "line default session-limit 20\n",
        "vty-pool default 0 15\n",
        # CDP:
        "cdp\n",
        # LACP:
        "lacp system mac 0001.0001.0001\n",
        # SNMP:
        "snmp-server ifindex persist\n",
        "snmp-server vrf MGMT\n",
        "snmp-server host 10.100.1.10 traps version 2c ttmon4vz4zd322\n",
        "snmp-server host 172.16.31.41 traps version 2c ttmon4vz4zd322\n",
        "snmp-server user Billing-V3-Snmp IRBIS-GROUP v3 auth md5 encrypted "
        "105D2D1F4846015F41002C7D027164787A4A201C5152 priv aes 128 encrypted "
        "03160C535F2832486A44110F105F0F040E67200C1F6321\n",
        "snmp-server view IRBIS-VIEW system included\n",
        "snmp-server view IRBIS-VIEW interfaces included\n",
        "snmp-server community ttmon4vz4zd322 RW IPv4 SNMP_MON\n",
        "snmp-server group IRBIS-GROUP v3 priv write IRBIS-VIEW IPv4 SNMP_MON\n",
        "snmp-server User Billing-V3-Snmp IRBIS-GROUP v3 auth md5 "
        "sDf-1s4-df6F97-89Gj23 priv aes 128 r789GsdD-hjg-dhj-kHW0t IPv4 11\n",
        "snmp-server traps snmp linkup\n",
        "snmp-server traps snmp linkdown\n",
        "snmp-server contact NULL\n",
        "snmp-server trap-source Loopback2\n",
        # DHCP:
        "dhcp ipv4 vrf WAP relay profile RELAY\n",
        "dhcp ipv4 vrf WUS relay profile RELAY\n",
        "dhcp ipv4 vrf FTTH relay profile RELAY_FTTH\n",
        "dhcp ipv4 vrf IPTV relay profile RELAY\n",
        "dhcp ipv4 vrf TR69 relay profile RELAY\n",
        "dhcp ipv4 vrf SIP_ADSL relay profile RELAY\n",
        "dhcp ipv4 profile RELAY relay\n",
        "dhcp ipv4 profile RELAY relay helper-address vrf MGMT 10.16.252.20 "
        f"giaddr {loopback2}\n",
        "dhcp ipv4 profile RELAY relay helper-address vrf MGMT 10.16.252.21 "
        f"giaddr {loopback2}\n",
        "dhcp ipv4 profile RELAY relay relay information option vpn\n",
        "dhcp ipv4 profile RELAY relay relay information option\n",
        "dhcp ipv4 profile RELAY relay relay information policy encapsulate\n",
        "dhcp ipv4 profile RELAY relay relay information option vpn-mode cisco\n",
        "dhcp ipv4 profile RELAY relay relay information option allow-untrusted\n",
        "dhcp ipv4 profile trust snoop\n",
        "dhcp ipv4 profile trust snoop trusted\n",
        "dhcp ipv4 profile untrust snoop\n",
        "dhcp ipv4 profile untrust snoop relay information option "
        "allow-untrusted\n",
        # NTP:
        "ntp server vrf MGMT 10.1.1.1 version 3 source Loopback2\n",
        # CALL-HOME:
        "call-home service active\n",
        "call-home contact smart-licensing\n",
        "call-home profile CiscoTAC-1\n",
        "call-home profile CiscoTAC-1 active\n",
        "call-home profile CiscoTAC-1 destination transport-method http\n",
        # HW-MODULE:
        "hw-module profile acl egress layer3 interface-based\n",
        "hw-module profile qos hqos-enable\n",
        "hw-module profile bundle-scale 1024\n",
        "hw-module vrrpscale enable\n",
        "!\n",
        "hw-module quad 0 location 0/0/CPU0 mode 10g\n",
        "hw-module quad 1 location 0/0/CPU0 mode 10g\n",
        # INTERFACES:
        "interface Loopback2\n",
        "interface Loopback2 vrf MGMT\n",
        f"interface Loopback2 ipv4 address {loopback2} "
        "255.255.255.255\n"
        "interface Loopback30\n",
        f"interface Loopback30 ipv4 address {loopback30} "
        "255.255.255.255\n",
        "interface MgmtEth0/RP0/CPU0/0 shutdown\n",
        "interface HundredGigE0/0/1/0 shutdown\n",
        "interface HundredGigE0/0/1/1 shutdown\n",
        "!\n",
        # PREFIX-SETS:
        "prefix-set FTTH_STB_NET\n",
        "  10.232.0.0/14 le 24,\n",
        "  10.236.0.0/14 le 24,\n",
        "  10.244.0.0/14 le 24,\n",
        "  10.216.0.0/13 le 24,\n",
        "  10.200.0.0/13 le 27,\n",
        "  10.240.0.0/14 le 27,\n",
        "  10.192.0.0/13 le 27\n",
        "end-set\n",
        "!\n",
        "prefix-set stream\n",
        "  10.20.5.152/30\n",
        "end-set\n",
        "!\n",
        # ROUTE-POLICIES:
        "route-policy FTTH_STB_TO_IPTV\n"
        "   if destination in FTTH_STB_NET then\n"
        "     set extcommunity rt(65000: 90) additive"
        "endif\n"
        "end-policy\n"
        "!\n"
        "route-policy stream\n",
        "  if destination in stream then\n",
        "    drop\n",
        "  else\n",
        "    pass\n",
        "  endif\n",
        "end-policy\n",
        "!\n",
        # ROUTER OSPF:
        f"router ospf 1 router-id {loopback30}\n",
        "router ospf 1 bfd minimum-interval 100\n",
        "router ospf 1 bfd fast-detect\n",
        "router ospf 1 bfd multiplier 3\n",
        "router ospf 1 mpls ldp sync\n",
        "router ospf 1 mpls ldp auto-config\n",
        "router ospf 1 timers throttle lsa all 30 100 1000\n",
        "router ospf 1 timers throttle spf 30 100 1000\n",
        "router ospf 1 auto-cost reference-bandwidth 80000\n",
        "router ospf 1 redistribute connected\n",
        "router ospf 1 area 0\n",
        "router ospf 1 area 0 mpls traffic-eng\n",
        "router ospf 1 area 0 interface Loopback30\n",
        "router ospf 1 area 0 interface Loopback30 passive enable\n"
        # ROUTER BGP:
        "router bgp 65000\n",
        "router bgp 65000 timers bgp 1 20\n",
        f"router bgp 65000 bgp router-id {loopback30}\n",
        "router bgp 65000 bgp update-delay 1\n",
        "router bgp 65000 bgp log neighbor changes detail\n",
        "router bgp 65000 address-family vpnv4 unicast\n",
        "router bgp 65000 address-family l2vpn evpn\n",
        "router bgp 65000 neighbor-group iBGP\n",
        "router bgp 65000 neighbor-group iBGP remote-as 65000\n",
        "router bgp 65000 neighbor-group iBGP update-source Loopback30\n",
        # EVPN:
        "evpn\n",
        "evpn bgp\n",
        # L2VPN:
        "l2vpn\n",
        "l2vpn load-balancing flow src-dst-mac\n",
        # MPLS:
        "mpls traffic-eng\n",
        "mpls ldp\n",
        "mpls ldp log hello-adjacency\n",
        "mpls ldp log neighbor\n",
        "mpls ldp log session-protection\n",
        "mpls ldp discovery hello interval 1\n",
        f"mpls ldp router-id {loopback30}\n",
        "mpls ldp session protection\n",
        "mpls ldp address-family ipv4\n",
        # MULTICAST:
        "multicast-routing address-family ipv4\n",
        "multicast-routing address-family ipv4 rate-per-route\n",
        "multicast-routing address-family ipv4 interface all enable\n",
        "multicast-routing address-family ipv4 accounting per-prefix\n",
        "multicast-routing\n",
        # IGMP:
        "router igmp version 2\n",
        # PIM:
        "router pim\n",
        "router pim address-family ipv4 rp-address 10.30.0.14 AUTORP_NETWORKS\n",
        "router pim address-family ipv4 old-register-checksum\n",
        "router pim address-family ipv4 log neighbor changes\n",
        # IGMP:
        "igmp snooping profile multicast\n",
        "igmp snooping profile multicast router-alert-check disable\n",
        "igmp snooping profile multicast-guard\n",
        "igmp snooping profile multicast-guard router-guard\n",
        "igmp snooping profile multicast-mrouter\n",
        "igmp snooping profile multicast-mrouter mrouter\n",
        # CLASS-MAP, POLICY-MAP:
        "class-map match-any BULK\n",
        " match dscp ipv4 af11\n",
        " match mpls experimental topmost 1\n",
        " end-class-map\n",
        "!\n",
        "class-map match-any IPTV\n",
        " description -=Match_Streaming_Video_Traffic=-\n",
        " match dscp ipv4 cs4\n",
        " match mpls experimental topmost 4\n",
        " end-class-map\n",
        "!\n",
        "class-map match-any MGMT\n",
        " description -=Match_Routing_and_Management_Traffic=-\n",
        " match dscp ipv4 cs6 cs2\n",
        " match mpls experimental topmost 6\n",
        " end-class-map\n",
        "!\n",
        "class-map match-any VOICE\n",
        " description -=Match_VOICE_and_Videoconference_Traffic=-\n",
        " match mpls experimental topmost 5\n",
        " match dscp ipv4 ef cs5\n",
        " end-class-map\n",
        "!\n",
        "class-map match-any BULK-TC\n",
        " description 'Match BULK priority traffic-class 4'\n",
        " match traffic-class 4\n",
        " end-class-map\n",
        "!\n",
        "class-map match-any IPTV-TC\n",
        " description 'Match video traffic-class 6'\n",
        " match traffic-class 6\n",
        " end-class-map\n",
        "!\n",
        "class-map match-any MGMT-TC\n",
        " description 'Match highest priority traffic-class 1'\n",
        " match traffic-class 1\n",
        " end-class-map\n",
        "!\n",
        "class-map match-any BUSINESS\n",
        " description -=Match_Business_Critical_Traffic=-\n",
        " match dscp ipv4 af31 cs3\n",
        " match mpls experimental topmost 3\n",
        " end-class-map\n",
        "!\n",
        "class-map match-any VOICE-TC\n",
        " description 'Match high priority traffic-class 7'\n",
        " match traffic-class 7\n",
        " end-class-map\n",
        "!\n",
        "class-map match-any BUSINESS-TC\n",
        " description 'Match medium traffic-class 3'\n",
        " match traffic-class 3\n",
        " end-class-map\n",
        "!\n",
        "policy-map QOS-IN\n",
        " class MGMT\n",
        "  set traffic-class 7\n",
        " !\n",
        " class VOICE\n",
        "  set traffic-class 7\n",
        " !\n",
        " class BUSINESS\n",
        "  set traffic-class 4\n",
        " !\n",
        " class BULK\n",
        "  set traffic-class 3\n",
        " !\n",
        " class IPTV\n",
        "  set traffic-class 6\n",
        " !\n",
        " class class-default\n",
        "  set mpls experimental topmost 0\n",
        "  set traffic-class 6\n",
        "  set dscp 0\n",
        " !\n",
        " end-policy-map\n",
        "!\n",
        "policy-map QOS-OUT\n",
        " class MGMT-TC\n",
        "  priority level 1\n",
        " !\n",
        " class BUSINESS-TC\n",
        "  priority level 3\n",
        " !\n",
        " class BULK-TC\n",
        "  priority level 4\n",
        " !\n",
        " class VOICE-TC\n",
        "  priority level 1\n",
        " !\n",
        " class IPTV-TC\n",
        "  priority level 2\n",
        " !\n",
        " class class-default\n",
        " !\n",
        " end-policy-map\n",
        "!\n"
    ]
    ncs_config_file.writelines(init_config)

def write_bgp_vpnv4_config(bgp_vpnv4_neighbors, ncs_config_file):
    logger = logging.getLogger('write_bgp_vpnv4_config')
    for neighbor in bgp_vpnv4_neighbors:
        logger.info(f'Adding config for iBGP-neighbor {neighbor} (family vpnv4)')
        ncs_config_file.writelines([
            f"router bgp 65000 neighbor {neighbor}\n",
            f"router bgp 65000 neighbor {neighbor} use neighbor-group iBGP\n",
            f"router bgp 65000 neighbor {neighbor} address-family vpnv4 unicast\n"
        ])

def write_vrf_config(vrf_list, ncs_config_file):
    for vrf in vrf_list:
        ncs_config_file.writelines([
            f"vrf {vrf['vrf_name']} \n",
            f"vrf {vrf['vrf_name']} address-family ipv4 unicast\n",
        ])
        for rt in vrf['vrf_rt_import'].split(sep=','):
            ncs_config_file.write(f"vrf {vrf['vrf_name']} address-family ipv4 unicast "
                                  f"import route-target {rt}\n")
        for rt in vrf['vrf_rt_export'].split(sep=','):
            ncs_config_file.write(f"vrf {vrf['vrf_name']} address-family ipv4 unicast "
                                  f"export route-target {rt}\n")
        ncs_config_file.writelines([
            f"router bgp 65000 vrf {vrf['vrf_name']}\n",
            f"router bgp 65000 vrf {vrf['vrf_name']} rd {vrf['vrf_rd']}\n",
            f"router bgp 65000 vrf {vrf['vrf_name']} address-family ipv4 unicast\n",
            f"router bgp 65000 vrf {vrf['vrf_name']} address-family ipv4 unicast "
            f"label mode per-vrf\n",
            f"router bgp 65000 vrf {vrf['vrf_name']} address-family ipv4 unicast "
            f"redistribute connected\n",
            f"router bgp 65000 vrf {vrf['vrf_name']} address-family ipv4 unicast "
            f"redistribute static\n",
        ])

def write_acl_config(acl_list, hostname, ncs_config_file, se_config_file):
    for acl in acl_list:
        if acl.is_on_pe:
            ncs_config_file.write(create_acl_config(acl))
        if acl.is_on_se:
            acl.name = format_acl_name_for_se(acl.name, hostname)
            se_config_file.write(create_acl_config(acl))

def write_aggregations_config(aggregations, esi_template, ncs_config_file):
    logger = logging.getLogger('write_aggregations_config')

    bundle_id = MIN_AGGR_PORT
    for neighbor in sort_aggregations(aggregations):
        if bundle_id > MAX_AGGR_PORT:
            print('ВНИМАНИЕ! Кончились порты для выделения агрегациям.')
            logger.error(f'No more ports for aggregations')
            break
        logger.info(f'GigabitEthernet0/0/0/{bundle_id} is allocated '
                    f'for neighbor {neighbor["neighbor"]}')

        # Физика:
        neighbor['ncs_interface'] = []
        ncs_config_file.writelines([
            f"interface preconfigure GigabitEthernet0/0/0/{bundle_id} "
            f"description -=={neighbor['description']}==-\n",
            f"interface preconfigure GigabitEthernet0/0/0/{bundle_id}\n",
            f"interface preconfigure GigabitEthernet0/0/0/{bundle_id} bundle id "
            f"{bundle_id} mode on\n"])

        # Бандл:
        ncs_config_file.writelines([
            f"interface Bundle-Ether{bundle_id}\n",
            f"interface Bundle-Ether{bundle_id} description "
            f"-=={neighbor['description']}==-\n",
            f"interface Bundle-Ether{bundle_id} mtu {neighbor['mtu']}\n",
            f"interface Bundle-Ether{bundle_id} service-policy output QOS-OUT\n",
            f"interface Bundle-Ether{bundle_id} bundle load-balancing hash dst-ip\n",
            f"interface Bundle-Ether{bundle_id} load-interval 30\n"
        ])
        logger.info('Bundle-Ether{} is created'.format(bundle_id))

        # ESI:
        ncs_config_file.writelines([
            f"evpn interface Bundle-Ether{bundle_id}\n",
            f"evpn interface Bundle-Ether{bundle_id} ethernet-segment\n",
            f"evpn interface Bundle-Ether{bundle_id} ethernet-segment identifier "
            f"type 0 {esi_template.format(bundle_id)}\n"
        ])
        logger.info(f'ESI for Bundle-Ether{bundle_id}: {esi_template.format(bundle_id)}')

        # В словарь с параметрами соседа записываем назначенный порт:
        neighbor['ncs_interface'].append(bundle_id)

        # Если порт-ченнел:
        if neighbor.get('port_channel_interface'):
            bundle_id += 1
            logger.info(f'Adding GigabitEthernet0/0/0/{bundle_id} to '
                        f'Bundle-Ether{bundle_id - 1}')
            ncs_config_file.writelines([
                f"interface preconfigure GigabitEthernet0/0/0/{bundle_id} description "
                f"-=={neighbor['description']}==-\n",
                f"interface preconfigure GigabitEthernet0/0/0/{bundle_id}\n",
                f"interface preconfigure GigabitEthernet0/0/0/{bundle_id} "
                f"bundle id {bundle_id - 1} mode on\n"
            ])
            neighbor['ncs_interface'].append(bundle_id)

        bundle_id += 1

def write_interlinks_config(interlinks, hostname, ncs_config_file):
    logger = logging.getLogger('write_interlinks_config')

    bundle_id = MIN_INTERLINK_PORT
    for neighbor in interlinks:
        # Не обрабатываем межкомплект, если он уходит c другой PE:
        if neighbor['hostname'] != hostname:
            logger.info(f'{hostname} Neighbor {neighbor["neighbor"]} '
                        f'belongs to another PE. Skipping')
            continue
        if bundle_id > MAX_INTERLINK_PORT:
            print('ВНИМАНИЕ! Кончились порты для выделения межкомплектам.')
            logger.error(f'{hostname} No more ports for interlinks')
            break
        logger.info(f'{hostname} TenGigE0/0/0/{bundle_id} is allocated for interlink')
        ncs_config_file.writelines([
            f"interface preconfigure TenGigE0/0/0/{bundle_id} description "
            f"-=={neighbor['description']}==-\n",
            f"interface preconfigure TenGigE0/0/0/{bundle_id} cdp\n",
            f"interface preconfigure TenGigE0/0/0/{bundle_id} mtu {neighbor['mtu']}\n",
            f"interface preconfigure TenGigE0/0/0/{bundle_id} service-policy "
            f"input QOS-IN\n",
            f"interface preconfigure TenGigE0/0/0/{bundle_id} ipv4 address "
            f"{neighbor['ip']} 255.255.255.252\n"
        ])
        neighbor['ncs_interface'] = [bundle_id]

        bundle_id += 1

def write_uplink_config(uplinks, hostname, ncs_config_file):
    logger = logging.getLogger('write_uplink_config')

    bundle_id = MIN_UPLINK_PORT
    for neighbor in uplinks:
        if neighbor['hostname'] != hostname:  # Eсли аплинк с другой PE
            logger.info(f'{hostname} Neighbor {neighbor["neighbor"]} '
                        f'belongs to another PE. Skipping')
            continue
        if bundle_id > MAX_UPLINK_PORT:
            print('ВНИМАНИЕ! Кончились порты для выделения аплинкам.')
            logger.error(f'{hostname} No more ports for uplinks')
            exit()
        logger.info(f'{hostname} TenGigE0/0/0/{bundle_id} is allocated for '
                    f'neighbor {neighbor["neighbor"]}')
        ncs_config_file.writelines([
            f"interface preconfigure TenGigE0/0/0/{bundle_id} description "
            f"-=={neighbor['description']}==-\n",
            "interface preconfigure TenGigE0/0/0/{bundle_id} cdp\n",
            f"interface preconfigure TenGigE0/0/0/{bundle_id} mtu {neighbor['mtu']}\n",
            f"interface preconfigure TenGigE0/0/0/{bundle_id} service-policy "
            f"input QOS-IN\n",
            f"interface preconfigure TenGigE0/0/0/{bundle_id} service-policy "
            f"output QOS-OUT\n",
            f"interface preconfigure TenGigE0/0/0/{bundle_id} ipv4 address "
            f"{neighbor['ip']} 255.255.255.252\n"
        ])
        neighbor['ncs_interface'] = [bundle_id]

        bundle_id += 1

def write_policies_config(policies, ncs_config_file):
    for policy in policies:
        ncs_config_file.writelines(create_policy(policy))


import os
import random
import yaml
from yaml import FullLoader
import hashlib

from packedbits import PackedBitsWriter, PackedBitsReader
from logic.logic import logic
from loot_tables import loot_tables
from pack_build import build
from paths import RANDO_ROOT_PATH

class randomize:
    def __randomize__(self, options):
        self.rng = random.Random()
        self.rng.seed(options['seed'])

        options['hash'] = randomize.rando_hash(self, options)
        options['permalink'] = randomize.get_permalink(self, options)

        self.mob_loot_table_list, self.unrandomized_mob_loot_table_list, self.chest_loot_table_list, self.unrandomized_chest_loot_table_list = loot_tables.get_loot_tables(self, options)

        self.rng.shuffle(self.mob_loot_table_list)
        self.rng.shuffle(self.chest_loot_table_list)
        self.mob_loot_table_list.extend(self.unrandomized_mob_loot_table_list)
        self.chest_loot_table_list.extend(self.unrandomized_chest_loot_table_list)

        self.mob_loot_tables, self.s_mob_loot_tables, self.chest_loot_tables, self.s_chest_loot_tables, self.mob_loot_table_list = loot_tables.read_loot_tables(self, self.mob_loot_table_list, self.chest_loot_table_list)

        #self.v_loot_tables = logic.logic_expression(self, self.v_loot_tables)
            # Logic is a WIP

        self.mob_patched_loot_tables, self.spoiler_mob_loot_tables, self.chest_patched_loot_tables, self.spoiler_chest_loot_tables = loot_tables.write_loot_tables(self, self.mob_loot_tables, self.s_mob_loot_tables, self.chest_loot_tables, self.s_chest_loot_tables)

        pack_build = build.build_pack(self, options, self.mob_patched_loot_tables, self.chest_patched_loot_tables)

        build.write_spoiler_log(self, options, self.spoiler_mob_loot_tables, self.spoiler_chest_loot_tables, self.mob_loot_table_list)
            #functional, but off until a UI option is created

        return (pack_build, options)

    def rando_hash(self, options):  # From https://github.com/lepelog/sslib
        rando_hash = hashlib.md5()
        rando_hash.update(str(options['seed']).encode('ASCII'))
        rando_hash.update(randomize.get_permalink(self, options).encode('ASCII'))
        with open(RANDO_ROOT_PATH / 'version.txt') as VERSION:
            rando_hash.update(VERSION.read().strip().encode('ASCII'))
        with open(RANDO_ROOT_PATH / 'hashnames.txt') as f:
            names = [s.strip() for s in f.readlines()]
        hash_random = random.Random()
        hash_random.seed(rando_hash.digest())
        return ' '.join(hash_random.choice(names) for _ in range(3))

    def get_permalink(self, options):  # From https://github.com/lepelog/sslib
        with (RANDO_ROOT_PATH / 'gui/options.yaml').open('r') as optf:
            OPTIONS = yaml.load(optf, Loader=FullLoader)

        writer = PackedBitsWriter()
        for option in OPTIONS:
            if not option.get('permalink', True):
                continue
            value = options[option['option']]
            if option['type'] == 'boolean':
                # one bit
                writer.write(int(value), 1)
            elif option['type'] == 'int':
                # needs information, how many bits this number is
                writer.write(value, option['bits'])
            elif option['type'] == 'multichoice':
                # as many bits as choices
                for choice in option['choices']:
                    writer.write(int(choice in value), 1)
            elif option['type'] == 'singlechoice':
                # needs information, how many bits this number is, then it's just the index to the choices
                if option['option'] == 'version':
                    writer.write([str(c) for c in option['choices']].index(value), option['bits'])
                else:
                    writer.write(option['choices'].index(value), option['bits'])
            else:
                raise Exception(f'unknown type: {option["type"]}')
        writer.flush()
        permalink = writer.to_base64()
        #if self['seed'] != -1:
        #    permalink += "#" + str(self['seed'])
        return permalink


"""remove_feature_flipping_for_bookings_prefilters

Revision ID: 1ee2283828e7
Revises: f151ee109bd4
Create Date: 2021-07-12 15:35:34.789724

"""
import enum

from pcapi.models import feature

# revision identifiers, used by Alembic.
revision = "1ee2283828e7"
down_revision = "f151ee109bd4"
branch_labels = None
depends_on = None

class FeatureToggle(enum.Enum):
    ENABLE_BOOKINGS_PAGE_FILTERS_FIRST = "Active le pré-filtrage de la page des réservations"

FLAG = FeatureToggle.ENABLE_BOOKINGS_PAGE_FILTERS_FIRST

def upgrade():
    feature.remove_feature_from_database(FLAG)

def downgrade():
    feature.add_feature_to_database(FLAG)


from .source.utils import daily_pattern
import pandas as pd
import numpy as np
import os
import matplotlib.pyplot as plt
import seaborn as sns
from .source.utils import compute_barycenter, load_data
from .source.plot_scripts import plot_cluster_analysis, plot_silhouette_values, plot_doughnut
from .source.utils import normalize
from matplotlib.dates import DateFormatter
sns.set_style('darkgrid')
import itertools

lines = ['-', '--', '-.', ':']
linecycler = itertools.cycle(lines)

def plot_barycenter(data=None, labels=None, meta=None, clusters=2, method='softdtw', show=True, colors=None, norm=True,
                    alpha=0.5, legend=True):

    if colors is None:
        colors = sns.color_palette('viridis', 5)

    data.index = pd.to_datetime(data.index)

    cl = labels.loc[clusters]
    figs = []
    for c in sorted(cl.unique()):

        fig, ax = plt.subplots()

        idx = cl[cl == c].index

        s = data[idx]
        user = meta.loc[idx]['inhabitants']

        u1_plot = plt.plot([-100, -99], [-100, -99], c=colors[1], linestyle='-', alpha=0.8, linewidth=2.0)
        u2_plot = plt.plot([-100, -99], [-100, -99], c=colors[2], linestyle='-', alpha=0.8, linewidth=2.0)
        u3_plot = plt.plot([-100, -99], [-100, -99], c=colors[3], linestyle='-', alpha=0.8, linewidth=2.0)
        u4_plot = plt.plot([-100, -99], [-100, -99], c=colors[4], linestyle='-', alpha=0.8, linewidth=2.0)

        for num_u in sorted(user.unique()):
            u = s[user[user == num_u].index]
            ax.plot(u.index, u.values, c=colors[num_u], alpha=alpha)

        if method == 'euclidean':
            x = s.mean(axis=1)

        elif method == 'softdtw':
            if norm:
                # Scale barycenter if it is produced on the normalised data
                x = compute_barycenter(normalize(s), method='softdtw')
                sm = s.mean(axis=1).mean()
                xm = x.mean()
                x = x * sm / xm
            else:
                x = compute_barycenter(s, method='softdtw')

        bary_plot = ax.plot(x.index, x.values, color='k', linewidth=2, linestyle='-', label=r'$\mathbf{x}^\ast$')

        e = compute_barycenter(s, method='euclidean')
        mu_plot = ax.plot(e.index, e.values, color='k', linewidth=1, linestyle='--', label=r'$\mathbf{\mu}$')

        if legend:
            plt.legend([u1_plot[0], u2_plot[0], u3_plot[0],  u4_plot[0], bary_plot[0], mu_plot[0]],
                       [f'1 resident', f'2 residents', f'3 residents', f'4 residents', r'$\mathbf{x}^\ast$', r'$\mathbf{\mu}$'],
                       loc='upper right',
                       frameon=False,
                       fontsize=12)
            legend = False
        plt.xlim((data.index[0], data.index[-1]))
        plt.ylim((0, 0.05))

        hfmt = DateFormatter('%H:%M')
        ax.xaxis.set_major_formatter(hfmt)
        plt.xticks(rotation=45)
        plt.ylabel(r'$Q \quad [L/s]$', fontsize=18)
        ax.tick_params(axis='both', which='major', labelsize=14)
        if show:
            plt.show()
        figs.append(fig)

    return figs

def plot_patterns(data=None, meta=None, show=True):

    data.index = pd.to_datetime(data.index)

    users = meta['inhabitants']
    uniques = sorted(users.unique())

    colors = sns.color_palette('viridis', len(uniques))

    fig, ax = plt.subplots()

    u1_plot = ax.plot([-100, -99], [-100, -99], c=colors[0], linestyle='-', alpha=0.8, linewidth=2.0)
    u2_plot = ax.plot([-100, -99], [-100, -99], c=colors[1], linestyle='-', alpha=0.8, linewidth=2.0)
    u3_plot = ax.plot([-100, -99], [-100, -99], c=colors[2], linestyle='-', alpha=0.8, linewidth=2.0)
    u4_plot = ax.plot([-100, -99], [-100, -99], c=colors[3], linestyle='-', alpha=0.8, linewidth=2.0)
    # u5_plot = ax.plot([-100, -99], [-100, -99], c=colors[4], linestyle='-', alpha=0.8, linewidth=2.0)

    for user in sorted(users.unique(), reverse=True):
        idx = meta[meta['inhabitants'] == user].index
        x = data[idx]
        ax.plot(x.index, x.values, color=colors[user - 1], alpha=0.4)

    mu_plot = ax.plot(x.index, x.mean(axis=1).values, color='k', linestyle='--')

    plt.xlim([x.index[0], x.index[-1]])
    plt.ylim((0, 0.04))
    hfmt = DateFormatter('%H:%M')
    ax.xaxis.set_major_formatter(hfmt)
    plt.xticks(rotation=45)
    plt.ylabel(r'$Q \quad [L/s]$', fontsize=18)
    ax.tick_params(axis='both', which='major', labelsize=14)

    plt.legend([u1_plot[0], u2_plot[0], u3_plot[0], u4_plot[0], mu_plot[0]],
               [f'1 resident', f'2 residents', f'3 residents', f'4 residents',
                r'$\mathbf{\mu}$'],
               loc='upper right',
               frameon=False,
               fontsize=12)

    if show:
        plt.show()

    return fig

def plot_simple_patterns(data=None, show=True):

    data.index = pd.to_datetime(data.index)
    colors = sns.color_palette('viridis', len(data.columns)+1)

    fig, ax = plt.subplots()

    for ind, (name, x) in enumerate(data.iteritems()):
        name = name.replace('sm_milford_', 'Home ')
        name = name.replace('_', ' (')
        name += ')'

        ax.plot(x.index, x.values, color=colors[ind], alpha=0.8, label=name, linestyle=next(linecycler), linewidth=2)

    plt.xlim([x.index[0], x.index[-1]])
    plt.ylim((0, 0.04))
    hfmt = DateFormatter('%H:%M')
    ax.xaxis.set_major_formatter(hfmt)
    plt.xticks(rotation=45)
    plt.ylabel(r'$Q \quad [L/s]$', fontsize=18)
    ax.tick_params(axis='both', which='major', labelsize=14)

    plt.legend(loc='upper right',
               frameon=False,
               fontsize=12)

    if show:
        plt.show()

    return fig

def numuser_barplot(meta=None, labels=None, colors=None, show=True, clusters=2, silhouette=None):

    l = labels.loc[clusters]

    if colors is None:
        colors = sns.color_palette('viridis', clusters)

    clist = []
    for c in sorted(l.unique()):
        idx = l[l == c].index
        clist.append(meta['inhabitants'].loc[idx].values)
    fig, ax = plt.subplots()
    ax.hist(clist, color=colors, bins=np.arange(5) + 0.5, align='mid', rwidth=0.7,  alpha=0.7,
            edgecolor='k', label=[f'$C_{x}$' for x in range(1, clusters+1)])
    ax.hist(meta['inhabitants'], bins=np.arange(5) + 0.5, edgecolor='k', facecolor='None', linewidth=1, rwidth=0.7,
            alpha=0.8, label='All')

    if silhouette is not None:
        s = silhouette.loc[clusters]
        ls = l.copy()
        s = s[s < 0.0]

        ls = ls[s.index]

        slist = []
        for c in sorted(l.unique()):
            idx = ls[ls == c].index
            slist.append(meta['inhabitants'].loc[idx].values)
        ax.hist(slist, bins=np.arange(5) + 0.5, align='mid', rwidth=0.7, alpha=0.7,
                edgecolor='k', fill=False, hatch='//', label=r'$S(\mathbf{y_l}) < 0$')

    plt.legend(fontsize=16, frameon=False)
    plt.xlabel(r'Number of residents', fontsize=16)
    plt.ylabel(r'Occurences', fontsize=16)

    if show:
        plt.show()

    return fig

def type_day_barplot(meta=None, labels=None, colors=None, show=True, clusters=2, silhouette=None):

    l = labels.loc[clusters]

    if colors is None:
        colors = sns.color_palette('viridis', clusters)

    clist = []
    for c in sorted(l.unique()):
        idx = l[l == c].index
        clist.append(meta['type_day'].loc[idx].values)
    fig, ax = plt.subplots()
    ax.hist(clist, color=colors, bins=np.arange(3) - 0.5, align='mid', rwidth=0.7,  alpha=0.7,
            edgecolor='k', label=[f'$C_{x}$' for x in range(1, clusters+1)])
    ax.hist(meta['type_day'], bins=np.arange(3) - 0.5, edgecolor='k', facecolor='None', linewidth=1, rwidth=0.7,
            alpha=0.8, label='All')

    if silhouette is not None:
        s = silhouette.loc[clusters]
        ls = l.copy()
        s = s[s < 0.0]

        ls = ls[s.index]

        slist = []
        for c in sorted(l.unique()):
            idx = ls[ls == c].index
            slist.append(meta['type_day'].loc[idx].values)
        ax.hist(slist, bins=np.arange(3) - 0.5, align='mid', rwidth=0.7, alpha=0.7,
                edgecolor='k', fill=False, hatch='//', label=r'$S(\mathbf{y_l}) < 0$')

    plt.legend(fontsize=16, frameon=False)
    plt.xlabel(r'weekday / weekend', fontsize=16)
    plt.ylabel(r'Occurences', fontsize=16)

    if show:
        plt.show()

    return fig

def combine_barplot(meta=None, labels=None, colors=None, show=True, clusters=2, silhouette=None):

    # x = list(np.arange(4) + 1)
    # y = ['weekday', 'weekend']
    # order = [f'{b} {a}' for a, b in itertools.product(y, x)]

    l = labels.loc[clusters]

    if colors is None:
        colors = sns.color_palette('viridis', clusters)

    clist = []
    for c in sorted(l.unique()):
        idx = l[l == c].index
        clist.append(meta['both'].loc[idx].values)
    fig, ax = plt.subplots()
    ax.hist(clist, color=colors, bins=np.arange(9) - 0.5, align='mid', rwidth=0.7,  alpha=0.7,
            edgecolor='k', label=[f'$C_{x}$' for x in range(1, clusters+1)])
    ax.hist(meta['both'], bins=np.arange(9) - 0.5, edgecolor='k', facecolor='None', linewidth=1, rwidth=0.7,
            alpha=0.8, label='All')

    if silhouette is not None:
        s = silhouette.loc[clusters]
        ls = l.copy()
        s = s[s < 0.0]

        ls = ls[s.index]

        slist = []
        for c in sorted(l.unique()):
            idx = ls[ls == c].index
            slist.append(meta['both'].loc[idx].values)
        ax.hist(slist, bins=np.arange(9) - 0.5, align='mid', rwidth=0.7, alpha=0.7,
                edgecolor='k', fill=False, hatch='//', label=r'$S(\mathbf{y_l}) < 0$')

    plt.legend(fontsize=16, frameon=False)
    plt.xlabel(r'weekday / weekend', fontsize=16)
    plt.ylabel(r'Occurences', fontsize=16)

    if show:
        plt.show()

    return fig

def type_barplot(meta=None, labels=None, colors=None, show=True, clusters=2, silhouette=None):

    l = labels.loc[clusters]

    if colors is None:
        colors = sns.color_palette('viridis', clusters)

    clist = []
    for c in sorted(l.unique()):
        idx = l[l == c].index
        clist.append(meta['type'].loc[idx].values)
    fig, ax = plt.subplots()
    ax.hist(clist, color=colors, bins=np.arange(4) - 0.5, align='mid', rwidth=0.7,  alpha=0.7,
            edgecolor='k', label=[f'$C_{x}$' for x in range(1, clusters+1)])
    ax.hist(meta['type'], bins=np.arange(4) - 0.5, edgecolor='k', facecolor='None', linewidth=1, rwidth=0.7,
            alpha=0.8, label='All')

    if silhouette is not None:
        s = silhouette.loc[clusters]
        ls = l.copy()
        s = s[s < 0.0]

        ls = ls[s.index]

        slist = []
        for c in sorted(l.unique()):
            idx = ls[ls == c].index
            slist.append(meta['type'].loc[idx].values)
        ax.hist(slist, bins=np.arange(4) - 0.5, align='mid', rwidth=0.7, alpha=0.7,
                edgecolor='k', fill=False, hatch='//', label=r'$S(\mathbf{y_l}) < 0$')

    plt.legend(fontsize=16, frameon=False)
    plt.xlabel(r'Pattern type', fontsize=16)
    plt.ylabel(r'Occurences', fontsize=16)

    if show:
        plt.show()

    return fig

def plot_consumption(data=None, meta=None, show=True, labels=None):

    user = meta['inhabitants']
    # user[user > 3] = 3
    cons = data.mean() * 3600 * 24

    user = user[cons.index]
    user.name = 'user'
    cons.name = 'cons'
    df = pd.concat([cons, user], axis=1)

    if labels is None:
        violinplot = sns.violinplot(x='user', y='cons', data=df, palette='viridis', inner='stick')

    else:
        labels = labels[cons.index]
        labels = labels + 1
        labels.name = 'Cluster'
        df = pd.concat([df, labels], axis=1)
        violinplot = sns.violinplot(x='user', y='cons', hue='Cluster', data=df, palette='viridis', split=True,
                                    inner="stick")

    plt.xlabel('Number of residents', fontsize=16)
    plt.ylabel('Consumption   (L/day)', fontsize=16)
    plt.xticks(fontsize=14)
    plt.yticks(fontsize=14)

    if show:
        plt.show()

    fig = violinplot.get_figure()

    return fig

def checker(x):

    if (x['type'] == 'work') and (x['labels'] == 0):
        return True
    elif (x['type'] == 'home') and (x['labels'] == 1):
        return True
    else:
        return False

def get_positives(labels=None, meta=None):
    labels.name = 'labels'

    house_type = meta['type'].copy(deep=True)

    house_type = house_type[labels.index]

    df = pd.concat([labels, house_type], axis=1)

    positives = df.apply(checker, axis=1)

    success_rate = positives.sum() / len(positives) * 100
    error_rate = 100 - success_rate
    print(f'Success Rate={success_rate}; Error Rate={error_rate}')

    return positives, success_rate, error_rate

if __name__ == '__main__':

    # Parameters
    dataset = '3'
    clusters = 2
    norm = True
    methods = ['euclidean', 'softdtw']

    # Program Start:
    collection = load_data(dataset=dataset, norm=norm)
    file_meta = os.path.join('data', f'Milford_metadata.txt')

    meta = pd.read_csv(file_meta, index_col=0, header=0, skiprows=[1])
    meta.index = list(map(lambda x: f'sm_milford_{x}', meta.index))

    idx = collection['euclidean']['data'].columns

    type_day = [x.split('_')[-1] for x in idx]
    num_users = [meta.loc[x[:-8]]['inhabitants'] for x in idx]
    type_household = [meta.loc[x[:-8]]['type'] for x in idx]

    type_day = pd.Series(index=idx, data=type_day, name='type_day')
    num_users = pd.Series(index=idx, data=num_users, name='inhabitants')
    type_household = pd.Series(index=idx, data=type_household, name='type')
    type_household[list(map(lambda x: x.endswith('weekend'), type_household.index))] = 'home'

    new_meta = pd.concat([type_day, num_users, type_household], axis=1)
    new_meta['both'] = [' '.join(i) for i in zip(new_meta["inhabitants"].map(str), new_meta["type_day"])]

    data = collection['softdtw']['data']

    # pattern plots
    fig = plot_patterns(data=data, meta=new_meta)

    # Conumption plot
    fig = plot_consumption(data=data, meta=new_meta)

    # Cluster analysis plot
    fig = plot_cluster_analysis(dataset=dataset, norm=norm, legend_location='lower right', ylim=(0, None))

    for method in methods:

        d = collection[method]

        if (method == 'euclidean') and (clusters == 2):
            d['labels'] = (d['labels'] + 1) % 2

        positives, success_rate, error_rate = get_positives(labels=d['labels'].loc[clusters], meta=new_meta)

        # Doughnut plots
        fig = plot_doughnut(success_rate, legend_loc='best')

        fig = plot_consumption(data=d['data'], meta=new_meta, labels=positives)

        legend = True
        figs = plot_barycenter(data=d['data'], labels=d['labels'], meta=new_meta, norm=True, method=method,
                               clusters=clusters, alpha=0.5, legend=legend)

        # Silhouette plots:
        fig = plot_silhouette_values(silhouette=d['silhouette'], labels=d['labels'], clusters=clusters)

        # Link clusters to number of users:
        fig = numuser_barplot(meta=new_meta, labels=d['labels'], clusters=clusters, silhouette=d['silhouette'])

        # Link clusters to weekday and weekend:
        fig = type_day_barplot(meta=new_meta, labels=d['labels'], clusters=clusters, silhouette=d['silhouette'])

        # Link clusters to both:
        fig = combine_barplot(meta=new_meta, labels=d['labels'], clusters=clusters, silhouette=d['silhouette'])

        fig = type_barplot(meta=new_meta, labels=d['labels'], clusters=clusters, silhouette=d['silhouette'])

        # Conumption plot
        fig = plot_consumption(data=d['data'], meta=new_meta,  labels=d['labels'].loc[clusters])

        keys = ['sm_milford_3_weekend',
                'sm_milford_6_weekend',
                'sm_milford_11_weekday',
                'sm_milford_15_weekday',
                'sm_milford_15_weekend',]

        subset = d['data'][keys]

        # Special pattern plot
        fig = plot_simple_patterns(data=subset)


import glob
import os
import pandas as pd
import numpy as np
import pandas_ta as ta
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import matplotlib.ticker as mticker
from mplfinance.original_flavor import candlestick_ohlc
import datetime

# Python for finance #7
def computeRSI(df, time_window):
    diff = df.diff(1).dropna()  # diff in one field(one day)

    up_chg = 0 * diff
    down_chg = 0 * diff

    up_chg[diff > 0] = diff[diff > 0]
    down_chg[diff < 0] = diff[diff < 0]

    up_chg_avg = up_chg.ewm(com=time_window - 1, min_periods=time_window).mean()
    down_chg_avg = down_chg.ewm(com=time_window - 1, min_periods=time_window).mean()

    rs = abs(up_chg_avg / down_chg_avg)
    rsi = 100 - 100 / (1 + rs)
    return rsi

def computeBollingerBands(df, n, m):
    df['TP'] = (df["high"] + df["low"] + df["close"]) / 3

    df['std'] = df['TP'].rolling(n).std(ddof=0)
    df['MA-TP']  = df['TP'].rolling(n).mean()
    df['BOLU'] = df['MA-TP'] + m * df['std']
    df['BOLD'] = df['MA-TP'] - m * df['std']

    return df

def computeProfit(i, Profit, ax):
    if i == 0:
        with open('temp.txt', 'w') as f:
            f.write('0')
    else:
        with open('temp.txt', 'r') as f:
            Current_Profit = float(f.readlines()[0])
            Current_Profit = Current_Profit + Profit
        with open('temp.txt', 'w') as f:
            f.write(str(Current_Profit))

        ax.text(0.83, 1.15, 'Total Profit: $' + str(round(Current_Profit, 3)) + ' per lot',
                bbox=dict(facecolor='#FF7A01', alpha=0.5),
                transform=ax.transAxes, color='white', fontsize=16, fontweight='bold',
                horizontalalignment='left', verticalalignment='center')

def backtest_day(i, data_full):
    time_stamp = date_list[i]
    #Changes made
    year = int(time_stamp[0:4])
    month = int(time_stamp[5:7])
    day = int(time_stamp[8:10])

    data_day = data_full[   (data_full['year'] == year) &
                            (data_full['month'] == month) &
                            (data_full['day'] == day)]

    data_day.reset_index(inplace=True)
    current_date = str(year) + '-' + str(month) + '-' + str(day)
    return data_day, current_date

def MA_Strategy(data):
    data['MA5'] = data["close"].rolling(5).mean()
    data['MA10'] = data["close"].rolling(10).mean()
    data['MA20'] = data["close"].rolling(20).mean()

    Buy=[] #show buy in the graph
    Sell=[] #show sell in the graph
    Record=[] #record buy and sell
    position = False # no short selling

    for i in range(len(data['close'])):
        if pd.notna(data['MA20'][i]):
            if (data['MA5'][i] > data['MA10'][i]) & (data['MA5'][i] > data['MA20'][i]):
                if position == False:
                    Buy.append(data['close'][i])
                    Sell.append(np.nan)
                    position = True
                    Record.append([i, data['close'][i], 'Buy'])
                else:
                    Buy.append(np.nan)
                    Sell.append(np.nan)
            elif (data['MA5'][i] < data['MA10'][i]) & (data['MA5'][i] < data['MA20'][i]):
                if position == True:
                    Buy.append(np.nan)
                    Sell.append(data['close'][i])
                    position = False
                    Record.append([i, data['close'][i], 'Sell'])
                else:
                    Buy.append(np.nan)
                    Sell.append(np.nan)
            else:
                Buy.append(np.nan)
                Sell.append(np.nan)
        else:
            Buy.append(np.nan)
            Sell.append(np.nan)

    data['Buy'] = Buy
    data['Sell'] = Sell

    return data, Record

def MACD_Strategy(data):
    macd = ta.macd(data["close"])*100
    macd.rename(columns={'MACD_12_26_9': 'MACD', 'MACDh_12_26_9': 'Histogram', 'MACDs_12_26_9': 'Signal'}, inplace=True)
    data = pd.concat([data, macd], axis=1).reindex(data.index)

    Buy=[] #show buy in the graph
    Sell=[] #show sell in the graph
    Record=[] #record buy and sell
    Buy_position = False
    Sell_position = False

    for i in range(len(data['close'])):
        if i == 0:
            Buy.append(np.nan)
            Sell.append(np.nan)
        elif pd.notna(data['Histogram'][i - 1]):
            if ((data['Histogram'][i - 1]<0) & (data['Histogram'][i]>0)) & \
                ((data['MACD'][i]<0) & (data['Signal'][i]<0)):
                # Check buying signal
                if Sell_position == True:
                    Buy.append(data['close'][i])
                    Sell.append(np.nan)
                    Buy_position = False
                    Sell_position = False
                    Record.append([i, data['close'][i], 'Buy'])
                elif Buy_position == False:
                    Buy.append(data['close'][i])
                    Sell.append(np.nan)
                    Buy_position = True
                    Sell_position = False
                    Record.append([i, data['close'][i], 'Buy'])
                else:
                    Buy.append(np.nan)
                    Sell.append(np.nan)

            elif ((data['Histogram'][i - 1]>0) & (data['Histogram'][i]<0)) & \
                ((data['MACD'][i]>0) & (data['Signal'][i]>0)):
                # Check selling signal
                if Buy_position == True:
                    Buy.append(np.nan)
                    Sell.append(data['close'][i])
                    Buy_position = False
                    Sell_position = False
                    Record.append([i, data['close'][i], 'Sell'])
                elif Sell_position == False:
                    Buy.append(np.nan)
                    Sell.append(data['close'][i])
                    Buy_position = False
                    Sell_position = True
                    Record.append([i, data['close'][i], 'Sell'])
                else:
                    Buy.append(np.nan)
                    Sell.append(np.nan)
            else:
                Buy.append(np.nan)
                Sell.append(np.nan)
        else:
            Buy.append(np.nan)
            Sell.append(np.nan)

    if len(Record) % 2:
        if Record[-1][2] == 'Buy':
            Buy[Record[-1][0]] = np.nan
        if Record[-1][2] == 'Sell':
            Sell[Record[-1][0]] = np.nan
        Record.pop()

    data['Buy'] = Buy
    data['Sell'] = Sell

    return data, Record

def RSI_Strategy(data):
    data['RSI'] = ta.rsi(data['close'], timeperiod=14)

    Buy=[] #show buy in the graph
    Sell=[] #show sell in the graph
    Record=[] #record buy and sell
    Buy_position = False
    Sell_position = False

    for i in range(len(data['close'])):
        if i == 0:
            Buy.append(np.nan)
            Sell.append(np.nan)
        elif pd.notna(data['RSI'][i - 1]):
            if ((data['RSI'][i - 1]<30) & (data['RSI'][i]>30)):
                # Check buying signal
                if Sell_position == True:
                    Buy.append(data['close'][i])
                    Sell.append(np.nan)
                    Buy_position = False
                    Sell_position = False
                    Record.append([i, data['close'][i], 'Buy'])
                elif Buy_position == False:
                    Buy.append(data['close'][i])
                    Sell.append(np.nan)
                    Buy_position = True
                    Sell_position = False
                    Record.append([i, data['close'][i], 'Buy'])
                else:
                    Buy.append(np.nan)
                    Sell.append(np.nan)

            elif ((data['RSI'][i - 1]>70) & (data['RSI'][i]<70)):
                # Check selling signal
                if Buy_position == True:
                    Buy.append(np.nan)
                    Sell.append(data['close'][i])
                    Buy_position = False
                    Sell_position = False
                    Record.append([i, data['close'][i], 'Sell'])
                elif Sell_position == False:
                    Buy.append(np.nan)
                    Sell.append(data['close'][i])
                    Buy_position = False
                    Sell_position = True
                    Record.append([i, data['close'][i], 'Sell'])
                else:
                    Buy.append(np.nan)
                    Sell.append(np.nan)
            else:
                Buy.append(np.nan)
                Sell.append(np.nan)
        else:
            Buy.append(np.nan)
            Sell.append(np.nan)

    if len(Record) % 2:
        if Record[-1][2] == 'Buy':
            Buy[Record[-1][0]] = np.nan
        if Record[-1][2] == 'Sell':
            Sell[Record[-1][0]] = np.nan
        Record.pop()

    data['Buy'] = Buy
    data['Sell'] = Sell

    return data, Record

def BB_Strategy(data):
    bb = ta.bbands(data['close'], length=20, std=2)
    bb.rename(columns={'BBU_20_2.0': 'BBU', 'BBL_20_2.0': 'BBL'}, inplace = True)
    data = pd.concat([data, bb], axis=1).reindex(data.index)

    Buy=[] #show buy in the graph
    Sell=[] #show sell in the graph
    Record=[] #record buy and sell
    Buy_position = False
    Sell_position = False

    for i in range(len(data['close'])):
        if i == 0:
            Buy.append(np.nan)
            Sell.append(np.nan)
        elif pd.notna(data['BBU'][i]):
            if (data['BBL'][i] > (data['close'][i]+data['open'][i])/2):
                # Check buying signal
                if Sell_position == True:
                    Buy.append(data['close'][i])
                    Sell.append(np.nan)
                    Buy_position = False
                    Sell_position = False
                    Record.append([i, data['close'][i], 'Buy'])
                elif Buy_position == False:
                    Buy.append(data['close'][i])
                    Sell.append(np.nan)
                    Buy_position = True
                    Sell_position = False
                    Record.append([i, data['close'][i], 'Buy'])
                else:
                    Buy.append(np.nan)
                    Sell.append(np.nan)

            elif (data['BBU'][i] < (data['close'][i]+data['open'][i])/2):
                # Check selling signal
                if Buy_position == True:
                    Buy.append(np.nan)
                    Sell.append(data['close'][i])
                    Buy_position = False
                    Sell_position = False
                    Record.append([i, data['close'][i], 'Sell'])
                elif Sell_position == False:
                    Buy.append(np.nan)
                    Sell.append(data['close'][i])
                    Buy_position = False
                    Sell_position = True
                    Record.append([i, data['close'][i], 'Sell'])
                else:
                    Buy.append(np.nan)
                    Sell.append(np.nan)
            else:
                Buy.append(np.nan)
                Sell.append(np.nan)
        else:
            Buy.append(np.nan)
            Sell.append(np.nan)

    if len(Record) % 2:
        if Record[-1][2] == 'Buy':
            Buy[Record[-1][0]] = np.nan
        if Record[-1][2] == 'Sell':
            Sell[Record[-1][0]] = np.nan
        Record.pop()

    data['Buy'] = Buy
    data['Sell'] = Sell

    return data, Record

# Python for finance #3
def figure_design(ax):
    ax.set_facecolor('#091217')
    ax.tick_params(axis='both', labelsize=14, colors='white')
    ax.ticklabel_format(useOffset=False)
    ax.spines['bottom'].set_color('#808080')
    ax.spines['top'].set_color('#808080')
    ax.spines['left'].set_color('#808080')
    ax.spines['right'].set_color('#808080')

# Python for finance #4
def main_plot(data, ax, current_date, showMA=True, showBB=True, showEMA=True, Strategy=True):

    ax.clear()

    if showMA==True:
        data['MA5'] = data["close"].rolling(5).mean()
        data['MA10'] = data["close"].rolling(10).mean()
        data['MA20'] = data["close"].rolling(20).mean()

        ax1.plot(data['MA5'], color='pink', linestyle="-", linewidth=1, label='5 periods SMA')
        ax1.plot(data['MA10'], color='orange', linestyle="-", linewidth=1, label='10 periods SMA')
        ax1.plot(data['MA20'], color='#08a0e9', linestyle="-", linewidth=1, label='20 periods SMA')

    if showEMA == True:
        data['EMA20'] = data["close"].ewm(span=20, adjust=False).mean()
        ax1.plot(data['EMA20'], color='#08a0e9', linestyle="-", linewidth=1, label='20 periods EMA')

    if showBB == True:
        #data = computeBollingerBands(data, 20, 2)
        bb = ta.bbands(data['close'], length=20, std=2)
        data = pd.concat([data, bb], axis=1).reindex(data.index)

        ax1.fill_between(data.index, data['BBU_20_2.0'], data['BBL_20_2.0'],
                         facecolor='#666699', alpha=0.2,
                         label='Bollinger Bands')
        ax1.plot(data['BBU_20_2.0'], color='#666699', linestyle="-", linewidth=0.2)
        ax1.plot(data['BBL_20_2.0'], color='#666699', linestyle="-", linewidth=0.2)

    if (showMA == True) | (showBB == True) | (showEMA == True):
        leg = ax1.legend(loc='upper left', facecolor='#121416', fontsize=10)
        #for text in leg.get_texts():
        #    text.set_color('w')
        plt.setp(leg.get_texts(), color='w')

    if Strategy == True:
        #data, Record = MA_Strategy(data)
        #data, Record = MACD_Strategy(data)
        #data, Record = RSI_Strategy(data)
        data, Record = BB_Strategy(data)

        ax1.scatter(data.index, data['Buy'], label='Buy', marker='^', color='#00FFBD', alpha=1, s=150)
        ax1.scatter(data.index, data['Sell'], label='Sell', marker='v', color='#FF6FFF', alpha=1, s=150)

        Profit=0
        margin = 0.95
        i = 1
        for item in Record:
            message = str(i) + ' ' + str(item[2]) + '@' + str(item[1])
            if item[2] == 'Buy':
                ax1.text(1.01, margin, message,
                         bbox=dict(facecolor='green', alpha=0.5),
                         transform=ax1.transAxes, color='white', fontsize=9, fontweight='bold',
                         horizontalalignment='left', verticalalignment='center')

                Profit = Profit - float(item[1])
            else:
                ax1.text(1.01, margin, message,
                         bbox=dict(facecolor='red', alpha=0.5),
                         transform=ax1.transAxes, color='white', fontsize=9, fontweight='bold',
                         horizontalalignment='left', verticalalignment='center')

                Profit = Profit + float(item[1])

            margin = margin - 0.055
            i = i + 1

        #if Record[-1][2] == 'Buy':
        #    Profit = Profit + float(Record[-1][1])

        ax1.text(0.83, 1.05, 'Daily Profit: $' + str(round(Profit, 3)) + ' per lot',
                 bbox=dict(facecolor='white', alpha=0.5),
                 transform=ax1.transAxes, color='black', fontsize=16, fontweight='bold',
                 horizontalalignment='left', verticalalignment='center')

    else:
        Profit=0

    ####################################################################
    candle_counter = range(len(data["open"]) - 1)
    ohlc = []
    for candle in candle_counter:
        append_me = candle_counter[candle], data["open"][candle], \
                    data["high"][candle], data["low"][candle], \
                    data["close"][candle]
        ohlc.append(append_me)
    ####################################################################
    ####################################################################
    if Strategy == True:
        candlestick_ohlc(ax, ohlc, width=0.4, colorup='#006400', colordown='#8B0000')
    else:
        candlestick_ohlc(ax, ohlc, width=0.4, colorup='#18b800', colordown='#ff3503')
    ####################################################################

    figure_design(ax)

#Given date - us

    ax1.text(0.5, 1.05, 'Date - 05-05-2023' ,
             transform=ax1.transAxes, color='white', fontsize=16, fontweight='bold',
             horizontalalignment='center', verticalalignment='center')

#    ax1.text(0.5, 1.05, 'Date ' + current_date,
#             transform=ax1.transAxes, color='white', fontsize=16, fontweight='bold',
#             horizontalalignment='center', verticalalignment='center')

    ax.grid(True, color='grey', linestyle='-', which='major', axis='both', linewidth=0.3)
    ax.set_xticklabels([])

    return Profit

def subplot_macd(data, ax):
    ax.clear()
    figure_design(ax)

    # 1st change
    macd = ta.macd(data["close"]).fillna(0)  * 100
    data = pd.concat([data, macd], axis=1).reindex(data.index)

    # 2nd change
    ax2.plot(np.where(data['MACD_12_26_9'] == 0, data['MACD_12_26_9'], None), label='MACD', linewidth=1, alpha = 0)
    ax2.plot(np.where(data['MACD_12_26_9'] != 0, data['MACD_12_26_9'], None), label='MACD', linewidth=1, color='white')

    ax2.plot(np.where(data['MACDs_12_26_9'] == 0, data['MACDs_12_26_9'], None), label='signal', linewidth=1,  alpha = 0)
    ax2.plot(np.where(data['MACDs_12_26_9'] != 0, data['MACDs_12_26_9'], None) , label='signal', linewidth=1, color='orange')

    pos = data['MACDh_12_26_9'] > 0
    neg = data['MACDh_12_26_9'] < 0

    ax2.bar(data.index[pos], data['MACDh_12_26_9'][pos], color='#006400', width=0.8, align='center')
    ax2.bar(data.index[neg], data['MACDh_12_26_9'][neg], color='#8B0000', width=0.8, align='center')

    if len(data['MACD_12_26_9']) != 0:
        ax.text(0.01, 0.95, 'MACD(12, 26, 9)', transform=ax.transAxes, color='white',
                fontsize=10, fontweight='bold',
                horizontalalignment='left', verticalalignment='top')

    ax.grid(True, color='grey', linestyle='-', which='major', axis='both', linewidth=0.3)
    ax.set_xticklabels([])

def subplot_rsi(data, ax):
    ax.clear()
    figure_design(ax)

    ax.axes.yaxis.set_ticks([30, 70])
    ax.set_ylim([-2, 102])

    #data['RSI'] = computeRSI(data["close"], 14)

    # 3rd change
    data['RSI'] = ta.rsi(data['close'], timeperiod=14).fillna(0)
    #data['x_axis'] = list(range(1, len(data['close']) + 1))

    # 4th change
    ax.plot(np.where(data['RSI'] == 0, data['RSI'], None), color='white', alpha = 0)
    ax.plot(np.where(data['RSI'] != 0, data['RSI'], None), color='white', linewidth=1)

    if len(data['RSI']) != 0:
        ax.text(0.01, 0.95, 'RSI(14)', transform=ax.transAxes, color='white',
                fontsize=10, fontweight='bold',
                horizontalalignment='left', verticalalignment='top')

    data['datetime'] = pd.to_datetime(data['datetime'], format="%Y-%m-%d %H:%M:%S")
    xdate = [i for i in data['datetime']]

    def mydate(x, pos=None):
        try:
            t = xdate[int(x)].strftime('%H:%M')
            return xdate[int(x)].strftime('%H:%M')
        except IndexError:
            return ''

    ax.xaxis.set_major_formatter(mticker.FuncFormatter(mydate))

    ax.axhline(30, linestyle='-', color='green', linewidth=0.5)
    ax.axhline(50, linestyle='-', alpha=0.5, color='white', linewidth=0.5)
    ax.axhline(70, linestyle='-', color='red', linewidth=0.5)

    ax.grid(True, color='grey', linestyle='-', which='major', axis='both', linewidth=0.3)
    ax.tick_params(axis='x', which='major', labelsize=10)

# Python for finance #3
fig = plt.figure()
#fig = plt.figure(figsize=(16.0, 10.0))
fig = plt.figure(figsize=(14.0, 10.0))
fig.patch.set_facecolor('#121416')
gs = fig.add_gridspec(6, 6)
ax1 = fig.add_subplot(gs[0:4, 0:6])
ax2 = fig.add_subplot(gs[4, 0:6])
ax3 = fig.add_subplot(gs[5, 0:6])

#list_of_files = glob.glob('C:\\Users\\daryle\\Desktop\\Acquire AI\\Python Finance - Algo Trade\\AAPL\\check point\\*.csv')
list_of_files = glob.glob('C:\\Users\\ashut\\Final_year-proj_Ashu\\Data_Collect\\VNO_final_c.csv')
data_full = pd.read_csv(list_of_files[-1], header=0)

date_list = [x[0:10] for x in data_full['datetime']]
date_list = sorted(set(date_list))

def animate(i):
    data_day, current_date = backtest_day(i, data_full)

    if not data_day.empty:
        Profit = main_plot(data_day, ax1, current_date, showMA=False, showBB=True, showEMA=True, Strategy=True)
        subplot_macd(data_day, ax2)
        subplot_rsi(data_day, ax3)
        computeProfit(i, Profit, ax1)
    plt.savefig('C:\\Users\\ashut\\Final_year-proj_Ashu\\Data_Collect\\'
                + str(current_date) +'.png', dpi = 300)

ani = animation.FuncAnimation(fig, animate, interval=1)
plt.show()