import os.path
from collections import defaultdict
import operator
from datetime import datetime
import matplotlib.pyplot as plt
import numpy as np
import math
import sys

sys.path.append('../')
from util import read_auxiliary_file, create_dir

ORG_TYPES = {
    'mass_media': ['media', 'radio', 'newspaper', 'jyrnal', 'gazet', 'press', 'television', 'GUP RB "Bashkortostan"',
                   'Rossiya Segodnya', 'RIK "Iskra"', 'Inforos Co., Ltd'],
    'publishing': ['publishing', 'izdatelstvo', 'izdatelskiy', 'editorial'],
    'education': ['state university'],
    'gov': ['ministry of', 'ministry for', 'administration of', 'ministerstvo', 'administratsiya',
            'Gosudarstvennogo Sobraniya', 'head of the republic', 'mincult', 'municipalnoe',
            'regionalnaya', 'Spetssvyaz FSO RF'],
    'rostelecom': ['rostelecom', 'rostelekom'],
    'hosters': ['LLC "Ucoz Media"', 'Compubyte Limited', 'Automattic, Inc.', '1&1 Internet Inc', 'Host Gate',
                'JSC Bashinformsvyaz'],
    'blogs': ['LiveJournal, Inc.', 'Google Inc.'],
    'art': ['library', 'teatr', '"TGAT im.G.Kamala"', 'kulturniy center']
}

LANG_COLORS = {
    'Башкирский': 'orange',
    'Бурятский': 'red',
    'Чеченский': '#DFD91B',
    'Чувашский': 'cyan',
    'Кабардино-черкесский': 'green',
    'Лезгинский': '#7516BD',
    'Лугово-восточный марийский': '#DF1BDF',
    'Эрзя-мордовский': '#1E90FF',
    'Якутский': 'blue',
    'Татарский': 'grey',
    'Тувинский': '#457019',
    'Удмуртский': 'black',
    'Калмыцкий': '#298A08'
}

STYLES_REG = {
    'Общее': 'solid',
    'Частные лица': 'dashed',
    'Организации': 'dotted',
    'Неизвестно': '-.'
}

CODE_LABEL = {
    'overall': 'Общее',
    'overall_private': 'Частные лица',
    'overall_org': 'Организации',
    'overall_none': 'Неизвестно'
}

# Stavropol, Saratov
LANG_CAPITAL = {
    'Абазинский': 'cherkessk',
    'Аварский': 'makhachkala',
    'Адыгейский': 'maykop',
    'Башкирский': 'ufa',
    'Бурятский': 'ulaan-ude',
    'Ингушский': 'magas',
    'Кабардино-черкесский': 'nalchik',  # кабардино-балкария
    'Калмыцкий': 'elista',
    'Карачаево-балкарский': 'cherkessk',
    'Коми-зырянский': 'syktyvkar',
    'Коми-пермяцкий': 'perm',
    'Кумыкский': 'makhachkala',
    'Лакский': 'makhachkala',
    'Лезгинский': 'makhachkala',
    'Литературный даргинский': 'makhachkala',
    'Лугово-восточный марийский': 'yoshkar-ola',
    'Рутульский': 'makhachkala',
    'Татарский': 'kazan',
    'Татский': 'makhachkala',
    'Тувинский': 'kyzyl',
    'Тундровый ненецкий': 'salekhard',
    'Удмуртский': 'izhevsk',
    'Хакасский': 'abakan',
    'Чеченский': 'groznyi',
    'Чувашский': 'cheboksary',
    'Чукотский': 'anadyr',
    'Эвенкийский': '',
    'Эрзя-мордовский': 'saransk',
    'Якутский': 'yakutsk',
}

def main():
    reg_file = read_auxiliary_file('data/domains_registration_info_with_ip.tsv')

    date_stats = get_date_stats(reg_file)
    draw_date_stats(date_stats)

    city_stats = get_city_stats(reg_file)
    draw_city_per_lang(city_stats, True, True)
    draw_city_per_lang(city_stats, True)
    draw_city_per_lang(city_stats)
    draw_cities_overall(city_stats, True)
    draw_cities_overall(city_stats)
    # for city, freq in sorted(city_stats['overall_private'].items(), key=operator.itemgetter(1), reverse=True):
    #     print(city, freq, sep='\t')

    reg_stats = get_registers_stats(reg_file)
    # print_registers_stats(reg_stats)
    draw_registers_stats(reg_stats)
    draw_registers_stats_separately(reg_stats)

    # for key, val in sorted(count_major_domains(reg_file).items(), key=operator.itemgetter(1), reverse=True):
    #     if val == 1:
    #         continue
    #     print(key, val, sep='\t')

# general function
def get_code_lang_mapping(lang_info_data):
    lang_code = dict()
    for row in lang_info_data:
        lang = row['lang_ru'].decode('utf-8')
        code = row['iso_code'].decode('utf-8')
        lang_code[code] = lang
    return lang_code

# auxiliary function to look at frequencies of first and second level domains
def count_major_domains(reg_data):
    dom_stats = dict()
    for row in reg_data:
        domain = row['domain'].decode('utf-8')
        major_dom = '.'.join(domain.split('.')[-2:])
        dom_stats[major_dom] = dom_stats.get(major_dom, 0) + 1
    return dom_stats

#########################
# who registered        #
#########################
def get_registers_stats(reg_data):
    lang_stats = defaultdict(dict)
    for row in reg_data:
        lang = row['lang'].decode('utf-8')
        person = row['name'].decode('utf-8')
        org = row['org'].decode('utf-8')
        if person != 'None' and org == 'None':
            lang_stats[lang]['private_person'] = lang_stats[lang].get('private_person', 0) + 1
        elif person == 'None' and org == 'None':
            lang_stats[lang]['none'] = lang_stats[lang].get('none', 0) + 1
        elif org != 'None':
            is_org_type_set = False
            org_type = None
            for key, val in sorted(ORG_TYPES.items()):
                for el in val:
                    if el.lower() in org.lower():
                        lang_stats[lang][key] = lang_stats[lang].get(key, 0) + 1
                        is_org_type_set = True
                        org_type = key
                        break
                if is_org_type_set:
                    break
            if not is_org_type_set:
                lang_stats[lang]['other'] = lang_stats[lang].get('other', 0) + 1
            if org_type in ['hosters', 'rostelecom']:
                lang_stats[lang]['none'] = lang_stats[lang].get('none', 0) + 1
            else:
                lang_stats[lang]['org'] = lang_stats[lang].get('org', 0) + 1
        lang_stats[lang]['overall'] = lang_stats[lang].get('overall', 0) + 1
    return lang_stats

def print_registers_stats(lang_stats):
    header = ['lang', 'private_person', 'org', 'mass_media', 'publishing', 'gov', 'blogs',
              'education', 'art', 'rostelecom', 'hosters', 'other', 'none', 'overall']
    print('\t'.join(header))

    for lang, stat in sorted(lang_stats.items()):
        for org_type in ORG_TYPES.keys():
            if org_type not in stat:
                stat[org_type] = 0
        vals_to_print = [lang] + [stat['private_person'] if 'private_person' in stat else 0]
        vals_to_print.append(stat['org'] if 'org' in stat else 0)
        for h in header[3:11]:
            vals_to_print.append(stat[h])
        vals_to_print.append(stat['other'] if 'other' in stat else 0)
        vals_to_print.append(stat['none'] if 'none' in stat else 0)
        vals_to_print.append(stat['overall'])
        print("\t".join(str(val) for val in vals_to_print))
        # print(lang, stat['private_person'] if 'private_person' else 0, stat['mass_media'], stat[''])
        # print('\t', stat)

# general graph with private and organizations registrations
def draw_registers_stats(lang_stats):
    langs = list()
    private = list()
    org = list()
    unknown = list()  # nones
    langs_info = read_auxiliary_file('../aux_files/langs_all_info.csv')
    code_to_lang = get_code_lang_mapping(langs_info)
    lang_stats_ru = dict([(code_to_lang[lang], val) for lang, val in lang_stats.items() if '_' not in lang])
    for lang, stat in sorted(lang_stats_ru.items()):
        langs.append(lang)
        private.append(stat['private_person'] if 'private_person' in stat else 0)
        org.append(stat['org'] if 'org' in stat else 0)
        unknown.append(stat['none'] if 'none' in stat else 0)
    n = len(langs)
    ind = np.arange(n)
    width = 0.25
    fig, ax = plt.subplots()
    rects1 = ax.bar(ind - width, private, width, color='#DF7401')
    rects2 = ax.bar(ind, org, width, color='#0B614B')
    rects3 = ax.bar(ind + width, unknown, width, color='#848484')

    # add labels
    ax.set_ylabel('Количество сайтов', fontsize=22)
    ttl = ax.set_title('Распределение по регистрировавшим домены', fontsize=30)
    ax.set_xticks(ind + width/2)
    ax.set_xticklabels(langs, rotation=90, fontsize=16)

    # arrange labels
    ttl.set_position([.5, 1.02])
    plt.subplots_adjust(bottom=0.35)
    plt.axis([-1, len(langs), 0, max(private + org + unknown) + 5])

    ax.legend((rects1[0], rects2[0], rects3[0]), ('Частные лица', 'Организации', 'Неизвестно'))
    plt.show()

# two graphs: one for major private regs, another for major org regs
def draw_registers_stats_separately(lang_stats):
    langs = defaultdict(list)
    private = defaultdict(list)
    org = defaultdict(list)
    unknown = defaultdict(list)  # nones
    langs_info = read_auxiliary_file('../aux_files/langs_all_info.csv')
    code_to_lang = get_code_lang_mapping(langs_info)
    lang_stats_ru = dict([(code_to_lang[lang], val) for lang, val in lang_stats.items() if '_' not in lang])
    for lang, stat in sorted(lang_stats_ru.items()):
        if ('private_person' in stat and 'org' not in stat) \
                or ('private_person' in stat and 'org' in stat
                    and stat['private_person'] > stat['org']):
            langs['private'].append(lang)
            private['private'].append(stat['private_person'] if 'private_person' in stat else 0)
            org['private'].append(stat['org'] if 'org' in stat else 0)
            unknown['private'].append(stat['none'] if 'none' in stat else 0)
        else:
            langs['org'].append(lang)
            private['org'].append(stat['private_person'] if 'private_person' in stat else 0)
            org['org'].append(stat['org'] if 'org' in stat else 0)
            unknown['org'].append(stat['none'] if 'none' in stat else 0)
    for reg_type, stat in langs.items():
        n = len(langs[reg_type])
        ind = np.arange(n)
        width = 0.25
        fig, ax = plt.subplots()
        rects1 = ax.bar(ind - width, private[reg_type], width, color='#DF7401')
        rects2 = ax.bar(ind, org[reg_type], width, color='#0B614B')
        rects3 = ax.bar(ind + width, unknown[reg_type], width, color='#848484')

        # add labels
        ax.set_ylabel('Количество сайтов', fontsize=22)
        ttl = ax.set_title('Распределение по регистрировавшим домены', fontsize=30)
        ax.set_xticks(ind + width/2)
        ax.set_xticklabels(langs[reg_type], rotation=90, fontsize=16)

        # arrange labels
        ttl.set_position([.5, 1.02])
        plt.subplots_adjust(bottom=0.35)
        plt.axis([-1, len(langs[reg_type]), 0, max(private[reg_type] + org[reg_type] + unknown[reg_type]) + 5])

        ax.legend((rects1[0], rects2[0], rects3[0]), ('Частные лица', 'Организации', 'Неизвестно'))
        plt.show()

MONTH_TO_TEXT = {
    1: 'Январь',
    2: 'Февраль',
    3: 'Март',
    4: 'Апрель',
    5: 'Май',
    6: 'Июнь',
    7: 'Июль',
    8: 'Август',
    9: 'Сентябрь',
    10: 'Октябрь',
    11: 'Ноябрь',
    12: 'Декабрь'
}

#########################
# when registered       #
#########################

def get_date_stats(reg_data, by_month=False):
    date_stats = defaultdict(dict)
    for row in reg_data:
        lang = row['lang'].decode('utf-8')
        date_str = row['creation_date'].decode('utf-8')
        if date_str in ['None', 'unknown']:
            continue
        date = datetime.strptime(date_str.split(' ')[0], "%Y-%m-%d")
        key_str = date.year
        if by_month:
            if date.year == 2012:
                key_str = date.month
            else:
                continue
        date_stats[lang][key_str] = date_stats[lang].get(key_str, 0) + 1
        date_stats['overall'][key_str] = date_stats['overall'].get(key_str, 0) + 1
        add_info_into_right_reg_type(row, key_str, date_stats)

    return date_stats

def draw_date_stats(date_stats):
    langs_info = read_auxiliary_file('../aux_files/langs_all_info.csv')
    code_to_lang = get_code_lang_mapping(langs_info)
    code_to_lang['overall'] = 'Общее распределение по датам'

    graphs_dir = create_dir('date_graphs')
    # draw_each_lang_separately(date_stats, code_to_lang, graphs_dir)
    # draw_all_langs_same_plot(date_stats, code_to_lang, graphs_dir)
    draw_overall_dates_with_registrars(date_stats, graphs_dir)

def draw_each_lang_separately(date_stats, code_to_lang, graphs_dir):
    for lang, date_val in sorted(date_stats.items()):
        if len(date_val) in [1, 2] or (set(date_val.values()) == {1} and len(date_val) < 5) \
                or 'overall_' in lang:
            # print(date_val)
            continue
        sorted_date = sorted(date_val.items())
        plt.figure(figsize=(16.0, 9.0), dpi=80)
        x = [s_date[0] for s_date in sorted_date]
        y = [s_date[1] for s_date in sorted_date]
        plt.plot(x, y)
        plt.plot(x, y, 'or')
        max_y = max(y) + 1
        plt.ylim([0, max_y])
        plt.xlim([min(x) - 1, max(x) + 1])
        plt.xticks(np.arange(min(x) - 1, max(x) + 1, 1.0), fontsize=12)
        step = 2.0 if lang == 'overall' else 1.0
        plt.yticks(np.arange(min(y) - 1, max(y) + 1, step), fontsize=12)
        plt.xlabel('Даты', fontsize=16, labelpad=20)
        plt.ylabel('Количество сайтов', fontsize=16, labelpad=20)
        if lang not in code_to_lang:
            langs = lang.split('_')
            new_lang = ", ".join([code_to_lang[l] for l in langs])
        else:
            new_lang = code_to_lang[lang]
        ttl = plt.title(new_lang, fontsize=22)
        ttl.set_position([.5, 1.02])  # x and y of position
        # plt.show()
        plt.savefig(os.path.join(graphs_dir, lang + '.png'), dpi=100)
        # print(lang, sorted(date_val.items()))

def draw_all_langs_same_plot(date_stats, code_to_lang):
    plt.figure(figsize=(16.0, 9.0), dpi=80)
    sorted_dates = sorted(date_stats['overall'].keys())
    plt.xlim([min(sorted_dates) - 1, max(sorted_dates) + 2])
    plt.ylim([0, 15])  # magic number
    plt.xticks(np.arange(min(sorted_dates), max(sorted_dates) + 1, 1.0), fontsize=12)
    plt.yticks(np.arange(0, 15, 1.0), fontsize=12)

    for lang, date_val in sorted(date_stats.items()):
        if len(date_val) in [1, 2] or (set(date_val.values()) == {1} and len(date_val) < 5):
            # if len(date_val) in [1, 2] or set(date_val.values()) == {1}\
            #         or set(date_val.values()) == {1, 2}:
            # print(date_val)
            continue
        if lang == 'overall' or '_' in lang:
            continue
        sorted_date = sorted(date_val.items())
        x = [s_date[0] for s_date in sorted_date]
        y = [s_date[1] for s_date in sorted_date]
        new_lang = code_to_lang[lang]
        plt.plot(x, y, color=LANG_COLORS[new_lang], label=new_lang)

    plt.xlabel('Даты', fontsize=16, labelpad=20)
    plt.ylabel('Количество сайтов', fontsize=16, labelpad=20)
    ttl = plt.title('Все языки', fontsize=22)
    ttl.set_position([.5, 1.02])  # x and y of position
    leg = plt.legend(loc='upper right', fontsize='large')
    for legobj in leg.legendHandles:
        legobj.set_linewidth(2.5)
    plt.show()
    # plt.savefig(os.path.join(graphs_dir, 'all_on_one' + '.png'), dpi=100)

def draw_overall_dates_with_registrars(date_stats, graphs_dir):
    plt.figure(figsize=(16.0, 9.0), dpi=80)
    sorted_dates = sorted(date_stats['overall'].keys())
    y_range = date_stats['overall'].values()
    plt.xlim([min(sorted_dates) - 1, max(sorted_dates) + 2])
    plt.ylim([0, max(y_range) + 1])
    # plt.xticks(np.arange(min(sorted_dates), max(sorted_dates) + 1, 1.0), fontsize=12)
    labels = list(MONTH_TO_TEXT.values())
    plt.xticks(np.arange(min(sorted_dates), max(sorted_dates) + 1, 1.0), labels, fontsize=12)
    plt.yticks(np.arange(0, max(y_range), 2.0), fontsize=12)

    for lang in sorted(CODE_LABEL.keys()):
        date_val = date_stats[lang]
        sorted_date = sorted(date_val.items())
        x = [s_date[0] for s_date in sorted_date]
        y = [s_date[1] for s_date in sorted_date]
        plt.plot(x, y, linestyle=STYLES_REG[CODE_LABEL[lang]], label=CODE_LABEL[lang], color='#2C6B3A')
    plt.xlabel('Даты', fontsize=16, labelpad=20)
    plt.ylabel('Количество сайтов', fontsize=16, labelpad=20)
    ttl = plt.title('Общее распределение по датам', fontsize=22)
    ttl.set_position([.5, 1.02])  # x and y of position
    leg = plt.legend(loc='upper right', fontsize='large')
    for legobj in leg.legendHandles:
        legobj.set_linewidth(2.5)
    # plt.show()
    plt.savefig(os.path.join(graphs_dir, 'overall_reg_mono' + '.png'), dpi=100)

def add_info_into_right_reg_type(row, data, res_dict):
    person = row['name'].decode('utf-8')
    org = row['org'].decode('utf-8')
    if person != 'None' and org == 'None':
        res_dict['overall_private'][data] = res_dict['overall_private'].get(data, 0) + 1
    elif person == 'None' and org == 'None':
        res_dict['overall_none'][data] = res_dict['overall_none'].get(data, 0) + 1
    elif org != 'None':
        trash_org_set = False
        for el in ORG_TYPES['hosters'] + ORG_TYPES['rostelecom']:
            if el.lower() in org.lower():
                res_dict['overall_none'][data] = res_dict['overall_none'].get(data, 0) + 1
                trash_org_set = True
                break
        if not trash_org_set:
            res_dict['overall_org'][data] = res_dict['overall_org'].get(data, 0) + 1

#########################
# where registered      #
#########################
def get_city_stats(reg_data):
    city_stats = defaultdict(dict)
    for row in reg_data:
        lang = row['lang'].decode('utf-8')
        # city_str = row['city'].decode('utf-8').lower()
        city_str = row['city_by_ip'].decode('utf-8').lower().split('(')[0].strip().strip('')
        # if city_str in ['none', 'unknown']:
        #     continue
        if row['name'].decode('utf-8') == 'None' and row['org'].decode('utf-8') == 'None':
            city_str = 'none'
        city_stats[lang][city_str] = city_stats[lang].get(city_str, 0) + 1
        city_stats['overall'][city_str] = city_stats['overall'].get(city_str, 0) + 1
        add_info_into_right_reg_type(row, city_str, city_stats)
    return city_stats

# http://matplotlib.org/examples/pylab_examples/bar_stacked.html
def draw_city_per_lang(city_stats, with_none=False, log_scale=False):
    langs = list()
    region_capitals = list()
    ru_capital = list()
    spb = list()
    other_cities = list()
    nones = list()
    langs_info = read_auxiliary_file('data/langs_all_info_merged.tsv')
    code_to_lang = get_code_lang_mapping(langs_info)
    lang_stats_ru = dict([(code_to_lang[lang], val) for lang, val in city_stats.items()
                          if '_' not in lang and lang != 'overall'])
    for lang, stat in sorted(lang_stats_ru.items()):
        # don't draw langs with none bar
        if 'none' in stat and len(stat) == 1:
            continue
        else:
            langs.append(lang)
            other_cities_num = 0
            for city, freq in stat.items():
                if city not in (LANG_CAPITAL[lang], 'moscow', 'none', 'saint petersburg'):
                    other_cities_num += freq
            region_capitals.append(stat[LANG_CAPITAL[lang]] if LANG_CAPITAL[lang] in stat else 0)
            ru_capital.append(stat['moscow'] if 'moscow' in stat else 0)
            spb.append(stat['saint petersburg'] if 'saint petersburg' in stat else 0)
            other_cities.append(other_cities_num)
            nones.append(stat['none'] if 'none' in stat else 0)

    n = len(langs)
    ind = np.arange(n)
    width = 0.25

    if log_scale:
        other_cities = [math.log(city) if city != 0 else 0 for city in other_cities]
        ru_capital = [math.log(city) if city != 0 else 0 for city in ru_capital]
        region_capitals = [math.log(city) if city != 0 else 0 for city in region_capitals]
        nones = [math.log(city) if city != 0 else 0 for city in nones]

    p1 = plt.bar(ind, other_cities, width, color='#D8D8D8')
    p2 = plt.bar(ind, ru_capital, width, color='#848484', bottom=other_cities)
    p5 = plt.bar(ind, spb, width, color='#58ACFA',
                 bottom=[i + j for i, j in zip(other_cities, ru_capital)])
    p3 = plt.bar(ind, region_capitals, width, color='#A9E2F3',
                 bottom=[i + j + k for i, j, k in zip(other_cities, ru_capital, spb)])
    if with_none:
        p4 = plt.bar(ind, nones, width, color='white',
                     bottom=[i + j + k + z for i, j, k, z in zip(other_cities, ru_capital, region_capitals, spb)])

    # add labels
    plt.ylabel('Количество сайтов', fontsize=22)
    ttl = plt.title('Распределение сайтов по городам', fontsize=30)
    plt.xticks(ind + width / 2., langs, rotation=90, fontsize=16)

    # arrange labels
    ttl.set_position([.5, 1.02])
    plt.subplots_adjust(bottom=0.35)
    if with_none:
        max_y = max(region_capitals) + max(ru_capital) + max(nones) + max(other_cities) + max(spb)
    else:
        max_y = max(region_capitals) + max(ru_capital) + max(other_cities)
    plt.axis([-1, len(langs), 0, max_y + 5])

    if with_none:
        plt.legend((p3[0], p5[0], p2[0], p1[0], p4[0]), ('Столицы регионов', 'Санкт-Петербург', 'Москва', 'Другие города', 'Неизвестно'))
    else:
        plt.legend((p3[0], p2[0], p1[0]), ('Столицы регионов', 'Москва', 'Другие города'))
    plt.show()

# http://matplotlib.org/examples/api/barchart_demo.html
def draw_cities_overall(city_stats, with_none=False):
    labels = ['Частные лица', 'Организации']
    region_capitals = list()
    ru_capital = list()
    spb = list()
    other_cities = list()
    nones = list()
    keys = ['overall_private', 'overall_org']
    for key in keys:
        region_capitals_num = 0
        other_cities_num = 0
        for city, freq in city_stats[key].items():
            if city not in list(LANG_CAPITAL.values()) + ['moscow', 'none', 'saint petersburg']:
                other_cities_num += freq
            elif city in list(LANG_CAPITAL.values()):
                region_capitals_num += freq
        region_capitals.append(region_capitals_num)
        other_cities.append(other_cities_num)
        ru_capital.append(city_stats[key]['moscow'] if 'moscow' in city_stats[key] else 0)
        spb.append(city_stats[key]['saint petersburg'] if 'saint petersburg' in city_stats[key] else 0)
        nones.append(city_stats[key]['none'] if 'none' in city_stats[key] else 0)
    n = 2
    ind = np.arange(n)
    width = 0.10
    fig, ax = plt.subplots()
    rects1 = ax.bar(ind - 2*width, region_capitals, width, color='#A9E2F3')
    rects2 = ax.bar(ind - width, ru_capital, width, color='#848484')
    rects5 = ax.bar(ind, spb, width, color='#58ACFA')
    rects3 = ax.bar(ind + width, other_cities, width, color='#D8D8D8')
    if with_none:
        rects4 = ax.bar(ind + 2 * width, nones, width, color='white')

    # add labels
    ax.set_ylabel('Количество сайтов', fontsize=22)
    ttl = ax.set_title('Распределение сайтов по городам', fontsize=30)
    if with_none:
        ax.set_xticks(ind + width)
    else:
        ax.set_xticks(ind + width / 2)
        # ax.set_xticks(ind + width)
    ax.set_xticklabels(labels, fontsize=16)

    # arrange labels
    ttl.set_position([.5, 1.02])
    # plt.subplots_adjust(bottom=0.35)
    if with_none:
        plt.axis([-0.5, len(region_capitals), 0, max(region_capitals + ru_capital + other_cities + nones) + 25])
    else:
        plt.axis([-0.3, len(region_capitals) - 0.5, 0, max(region_capitals + ru_capital + other_cities + spb) + 5])

    if with_none:
        ax.legend((rects1[0], rects2[0], rects5[0], rects3[0], rects4[0]),
                  ('Столицы регионов', 'Москва', 'Санкт-Петербург', 'Другие города', 'Неизвестно'))
    else:
        ax.legend((rects1[0], rects2[0], rects5[0], rects3[0]), ('Столицы регионов', 'Москва', 'Санкт-Петербург', 'Другие города'))

    autolabel(rects1, ax)
    autolabel(rects2, ax)
    autolabel(rects3, ax)
    autolabel(rects5, ax)
    if with_none:
        autolabel(rects4, ax)

    plt.show()

def autolabel(rects, ax):
    # attach some text labels
    for rect in rects:
        height = rect.get_height()
        ax.text(rect.get_x() + rect.get_width() / 2., height,
                '%d' % int(height),
                ha='center', va='bottom')
    return

if __name__ == "__main__":
    main()


from torchvision import transforms
from PIL import Image
import numpy as np

import os

from .PreTrainingDataset import PreTrainingDataset

class TestPreTrainingDataset(PreTrainingDataset):
    def __init__(self, dataset_root, train_size):
        # super().__init__(dataset_root, train_size)

        # self.joint_transform = JointResize(train_size)
        # here in Python, not C++, is-a relationship is up to ourselves

        self.data_list = self._make_list(dataset_root, [ 'RGB' ])
        self.mean = np.array([0.447, 0.407, 0.386])
        self.std = np.array([0.244, 0.250, 0.253])

        self.image_transform = transforms.Compose([
                transforms.Resize(train_size),
                transforms.ToTensor(),
                transforms.Normalize(self.mean, self.std)
            ])

    def __getitem__(self, index):
        image_path = self.data_list[index][0]

        image = Image.open(image_path).convert('RGB')

        image = self.image_transform(image)

        image_main_file_name = os.path.splitext(os.path.basename(image_path))[0]

        return image, image_path, image_main_file_name


#!/usr/bin/env python3
import sys, subprocess, gzip, os, shutil, zipfile
from os.path import join, relpath, abspath

inAppPath = sys.argv[1] 		# "air/myapp.air"
inTargetFile = sys.argv[2]		# "localserver/app/testharness.tivoipkg"
tempDir = "tivo-package_temp"

if sys.platform == "win32":
	# Windows-specific code
	cmd_cpioBomPack = "cd {0} & type {1} | cpio --force-local --warning=truncate --quiet -o -Hcrc --file {2}"
else:
	# Non windows code
	cmd_cpioBomPack = "cd {0}; cat {1} | cpio --force-local --warning=truncate --quiet -o -Hcrc -O {2}"

def main():
	# set up temp dir, clean out if present for previous failure
	shutil.rmtree( tempDir, True )
	os.mkdir( tempDir )

	# extract AIR package
	airDir = join( tempDir, "air-extract" )
	os.mkdir( airDir )
	extractAir( inAppPath, airDir )

	# create TiVo package
	tivoDir = join( tempDir, "tivo-build" )
	os.mkdir( tivoDir )
	tmpPkg = makeFlashPackage( airDir, tivoDir )

	# move TiVo package to desired location and clean up
	os.replace( tmpPkg, inTargetFile )
	shutil.rmtree( tempDir, True )

def extractAir( airPath, tempAirDir ):
	"""Unzip AIR package into specified dir"""

	with zipfile.ZipFile( airPath, 'r' ) as airZip:
		# file "hash" seems to always get a bad CRC, hopefully ignoring it won't be a problem
		airZip.extractall( tempAirDir, filter(
			lambda f: f != "META-INF/AIR/hash" and not f.endswith('/'), airZip.namelist() ) )
		airZip.close()

def makeFlashPackage( airDir, tempMakeDir ):
	"""Create a .tivopkg from a decompressed AIR package"""

	files = []
	pkgType = "flash2"

	# create manifest
	manifestFileName = "manifest.txt"
	out_file = open( join( tempMakeDir, manifestFileName ), "wt" )
	out_file.writelines( [
		"Type: " + pkgType,
		"\nX-Format: cpio-gz",
		"\nX-File: data.cpio.gz"] )
	out_file.close();

	files.append( manifestFileName )

	packWithBOM( tempMakeDir, airDir, makeFileList( airDir ), "data.cpio" )
	compressWithGZip( join( tempMakeDir, "data.cpio" ), join( tempMakeDir, "data.cpio.gz" ) )
	files.append( "data.cpio.gz" )

	packWithBOM( tempMakeDir, tempMakeDir, files, "tmp-pkg.cpio" )
	finalGzPath = join( tempMakeDir, "tmp-pkg.cpio.gz" );
	compressWithGZip( join( tempMakeDir, "tmp-pkg.cpio" ), finalGzPath )
	return finalGzPath

def makeFileList( dirPath ):
	"""Returns a list of files in the specified directory, with paths relative to it"""

	fileList = []
	for root, dirs, files in os.walk( dirPath ):
		if '.svn' in dirs:
			dirs.remove( '.svn' )
		fileList.extend( [ relpath( join( root, name ), dirPath ) for name in files ] )
	return fileList

def packWithBOM( workDir, sourceDir, fileList, cpioFileName ):
	"""Packs files using the external CPIO tool and a BOM (Bill Of Materials? Maybe; that's what TiVo's Perl script calls it"""

	bomPath = abspath( join( workDir, "bom" ) );
	out_file = open( bomPath, "wt" )
	out_file.writelines( [ name + "\n" for name in fileList ] )
	out_file.close()
	cmdStr = cmd_cpioBomPack.format( sourceDir, bomPath, abspath( join( workDir, cpioFileName ) ) )
	subprocess.call( cmdStr, shell=True )
	os.remove( bomPath );

def compressWithGZip( sourcePath, targetPath ):
	with open( sourcePath, 'rb' ) as f_in:
		with gzip.open( targetPath, 'wb' ) as f_out:
			f_out.writelines( f_in )

if __name__ ==	'__main__':
	main()


from flask import Flask
from firebase_admin import credentials, firestore, initialize_app
from dotenv import load_dotenv
import os
from mockfirestore import MockFirestore
from flask_cors import CORS
import json

load_dotenv()
initialize_app(credentials.Certificate(json.loads(os.environ.get('KEY'))))
db = firestore.client()
mock_db = MockFirestore()

# Initialize Flask app
def create_app(test_config=None):
    app = Flask(__name__)

    if test_config:
        global db
        app.config['ENV'] = 'development'
        app.config['DEBUG'] = True
        app.config['TESTING'] = True
        db = mock_db
    else:
        app.config['databaseURL'] = os.environ.get('DB')

    from .routes import user_bp, game_bp
    app.register_blueprint(user_bp)
    app.register_blueprint(game_bp)

    CORS(app)

    if __name__ == '__main__':
        app.run(port=int(os.environ.get('PORT')))

    return app


#%%
# imports
import time

from sklearn.datasets import load_sample_image
import faimg as fg
import numpy as np
import random
import matplotlib.pyplot as plt

#%%
# Tests for Image class:
china = load_sample_image("china.jpg")
imgProc = fg.ImageProcessor(china)

# %%
# Gini index

y = [1, 1, 1, 1, 2, 32, 3, 12, 312, 31, 231, 23]
print(len(np.bincount(y)))
gini = fg.giniIndex(y)
print(gini)

#%%
# Extracts patches speed test
start = time.time()
all_patches = imgProc.extract_all_patches(10, 3, 2)
end = time.time()
print("Time consumed in working: ", end - start)
print(np.array(all_patches).shape)

start_np = time.time()
all_patches_np = imgProc.extract_all_patches_np(10, 3, 2)
end_np = time.time()
print("Time consumed in working: ", end_np - start_np)
print((all_patches_np == all_patches).all())
print(all_patches_np.shape)

#%%
# Extract regions
region_size = 3
patch = fg.Patch(china[0:20, 0:20], region_size)

# region_positions = np.array([[0,0], [9,7]])

regions = patch.retrieve_random_regions()

print(regions[0])

#%% Extract regions and operators
patch_x = 150
patch_y = 150
patch_size = 20
region_size = 3
region_positions = np.array([[0, 0], [9, 7]])

patch = fg.Patch(
    china[patch_x : patch_x + patch_size, patch_y : patch_y + patch_size], region_size
)

regions = patch.retrieve_regions(region_positions)

print("Region : ")
print(regions[0])

print("Span : ")
span = patch.get_span(regions[0])
print(span)

for op in fg.RegionOperator:
    print(op)
    print(patch.apply_operator(regions[0], op))

#%% Referential value (used when 1 regions for distance) :

patch = fg.Patch(np.array([[[1, 1, 1]]]), 0)

reference_value = patch.get_random_reference_value()

print(reference_value)
print([patch])

#%% Random regions
patchSize = 20
regionSize = 5

regions = fg.getRandomRegionsPosition(patchSize, regionSize)
bar = random.choice(list(fg.RegionOperator))
print(bar)
print("regions : ", regions)

#%% Predictions playground

prediction = np.array([[1, 1, 1], [2, 2, 1], [2, 2, 2]])

print(prediction)
print(prediction.T)

def pred(array):
    counts = np.bincount(array)
    return np.argmax(counts)

print(pred(prediction.T[0]))

#%% Compute distance
patch_x = 150
patch_y = 150
patch_size = 20
region_size = 3
operator = fg.RegionOperator.MAX_SPAN
patch = fg.Patch(
    china[patch_x : patch_x + patch_size, patch_y : patch_y + patch_size], region_size
)

# 1 region
region_positions = np.array([[0, 0]])

res = patch.compute_feature(region_positions, operator)
print("Result type : ", res.dtype)
print("For 1 region : ", res)
# 2 region
region_positions = np.array([[0, 0], [4, 15]])

res = patch.compute_feature(region_positions, operator)
print("For 2 regions : ", res)

# 4 region
region_positions = np.array([[1, 1], [4, 15], [16, 2], [7, 9]])

res = patch.compute_feature(region_positions, operator)
print("For 4 regions : ", res)

#%% Loading utils
from PIL import Image

def loadDataAndLabel(imgDataPath, labelDataPath):
    imData = Image.open(imgDataPath)
    imLabel = Image.open(labelDataPath)

    X = np.array(imData)
    yRaw = np.array(imLabel)

    class_mapping = {
        (0x3C, 0x10, 0x98): 1,  # Building
        (0x84, 0x29, 0xF6): 2,  # Land (unpaved area)
        (0x6E, 0xC1, 0xE4): 3,  # Road
        (0xFE, 0xDD, 0x3A): 4,  # Vegetation
        (0xE2, 0xA9, 0x29): 5,  # Water
        (0x9B, 0x9B, 0x9B): 6,  # Unlabeled
    }

    def rgb_to_labels(image, class_mapping):
        # Initialize an empty array to store the labels
        labels = np.empty(image.shape[:2], dtype=np.uint8)

        # Iterate over each pixel in the image
        for i in range(image.shape[0]):
            for j in range(image.shape[1]):
                # Get the RGB value of the current pixel
                rgb = tuple(image[i, j])

                # Map the RGB value to a class label using the provided mapping
                label = class_mapping.get(
                    rgb, 6
                )  # Default to class 6 - Unlabeled if no mapping exists

                # Assign the label to the corresponding pixel in the labels array
                labels[i, j] = label

        return labels

    y = rgb_to_labels(yRaw, class_mapping)

    return X, y

#%% Loading the images
# - Tile 1 - part 005
patchSize = 20
regionSize = 5

print("Loading img...")
start = time.time()
# Images from https://www.kaggle.com/datasets/humansintheloop/semantic-segmentation-of-aerial-imagery
X, y = loadDataAndLabel("data/image_part_005_data.jpg", "data/image_part_005_label.png")
end = time.time()
print("Image loaded in ", end - start, "secs !")
print(X.shape)
print(y.shape)

X = X[0:125, 0:125]
y = y[0:125, 0:125]

print("Processing patches ...")
imProc = fg.ImageProcessor(X)
patches = imProc.extract_all_patches_np(patchSize, regionSize)

yWithPatch = y[
    patchSize // 2 : y.shape[0] - patchSize // 2 + 1,
    patchSize // 2 : y.shape[1] - patchSize // 2 + 1,
]

y = yWithPatch.flatten()

print("Patches processed ! shape : ", patches.shape, y.shape)
assert patches.shape == y.shape

#%%
# Testing Hansch tree fit on loaded images
print("Creating Hansch Tree ...")
start = time.time()
tree = fg.HanschTree(patchSize, regionSize, maxDepth=2, minSampleSize=10)
tree.fit(patches, y)
end = time.time()
print("Tree created in : ", end - start, "secs !")

#%%
# Print tree
print(tree)

#%%
# Testing Hansch tree predict on images
from sklearn.dummy import DummyClassifier

dummy = DummyClassifier()
dummy.fit(patches, y)
dummy_predict = dummy.predict(patches)

prediction = tree.predict(patches)
print(dummy_predict.shape)
print(prediction.shape)
print("dummy Accuracy : ", np.count_nonzero(y == dummy_predict) / y.shape[0])
print("tree Accuracy : ", np.count_nonzero(y == prediction) / y.shape[0])

#%%
# Testing Hansch forest fit on loaded images
print("Creating Hansch Forest ...")
start = time.time()
forest = fg.HanschForest(patchSize=patchSize, regionSize=regionSize, nbTrees=5, maxDepth=3, minSampleSize=10)
forest.fit(patches, y)
end = time.time()
print("forest created in : ", end - start, "secs !")

#%%
# Testing Hansch forest predict on images
from sklearn.dummy import DummyClassifier

dummy = DummyClassifier()
dummy.fit(patches, y)
dummy_predict = dummy.predict(patches)

prediction = forest.predict(patches)
print(dummy_predict.shape)
print(prediction.shape)
print("dummy Accuracy : ", np.count_nonzero(y == dummy_predict) / y.shape[0])
print("forest Accuracy : ", np.count_nonzero(y == prediction) / y.shape[0])

#%%
# Extracts patches
patch_size = 50
patch = imgProc.retrieve_random_image_patch(patch_size)
assert patch.shape[:2] == (patch_size, patch_size)

regions = imgProc.retrieve_regions(patch)
assert len(regions) >= 1 and len(regions) <= 4
for region in regions:
    assert region.shape[0] >= 10 and region.shape[1] >= 10
plt.imshow(china)
plt.figure()
plt.imshow(patch)
plt.axis("off")

for region in regions:
    plt.figure()
    plt.imshow(region)
    plt.axis("off")


import mediapipe as mp
import numpy as np
import cv2
from draw_landmarks import draw_landmarks_on_image

model_path = 'pose_landmarker_lite.task'

BaseOptions = mp.tasks.BaseOptions
PoseLandmarker = mp.tasks.vision.PoseLandmarker
PoseLandmarkerOptions = mp.tasks.vision.PoseLandmarkerOptions
PoseLandmarkerResult = mp.tasks.vision.PoseLandmarkerResult
VisionRunningMode = mp.tasks.vision.RunningMode

# Create a pose landmarker instance with the live stream mode:
def print_result(result: PoseLandmarkerResult, output_image: mp.Image, timestamp_ms: int):
  annotated_image = draw_landmarks_on_image(output_image, result)
  annotated_image = cv2.cvtColor(annotated_image, cv2.COLOR_RGB2BGR)
  cv2.imshow("OUTPUT", annotated_image)
  if cv2.waitKey(1) & 0xFF == ord('q'):
    cap.release()
    cv2.destroyAllWindows()

def Live_detector(video_path, people, callback_function):

  options = PoseLandmarkerOptions(
      base_options=BaseOptions(model_asset_path=model_path),
      running_mode=VisionRunningMode.LIVE_STREAM,
      num_poses= people,
      result_callback=callback_function)

  landmarker = mp.tasks.vision.PoseLandmarker.create_from_options(options)

  # Use OpenCV’s VideoCapture to start capturing from the webcam.
  cap = cv2.VideoCapture(video_path)
  frame_timestamp_ms = 0

  # Create a loop to read the latest frame from the camera using VideoCapture#read()
  if not cap.isOpened():
    print("Cannot open camera")
    exit()

  while True:
    # Capture frame-by-frame
    ret, frame = cap.read()

    # if frame is read correctly ret is True
    if not ret:
        print("Can't receive frame")
        break

    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

    # Convert the frame received from OpenCV to a MediaPipe’s Image object.
    mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame_rgb)

    # The landmarker is initialized. Use it here.
    annotated_image = landmarker.detect_async(mp_image, frame_timestamp_ms)
    frame_timestamp_ms += 1

  cap.release()
  cv2.destroyAllWindows()


import time
import sys

INITIAL = {
    'out1': 'G',
    'out2': 'R',
    'clock': 0,
    'walk': False,
}

G1 = lambda s: (s['out1'] == 'G' and
                s['out2'] == 'R' and
                (
                    (s['clock'] < 30 and dict(s, clock=s['clock'] + 1, walk=False)) or
                    (s['clock'] == 30 and dict(s, clock=0, out1='Y', walk=False)))
               )

Y1 = lambda s: (s['out1'] == 'Y' and
                s['out2'] == 'R' and
                (
                    (s['clock'] < 5 and dict(s, clock=s['clock'] + 1, walk=False)) or
                    (s['clock'] == 5 and dict(s, clock=0, out1='R', out2='G', walk=False)))
               )

G2 = lambda s: (s['out1'] == 'R' and
                s['out2'] == 'G' and
                (
                    (s['clock'] >= 30 and s['walk'] and dict(s, clock=0, out2='Y', walk=False)) or
                    (s['clock'] < 60 and dict(s, clock=s['clock'] + 1)) or
                    (s['clock'] == 60 and dict(s, clock=0, out2='Y', walk=False)))
               )

Y2 = lambda s: (s['out1'] == 'R' and
                s['out2'] == 'Y' and
                (
                    (s['clock'] < 5 and dict(s, clock=s['clock'] + 1, walk=False)) or
                    (s['clock'] == 5 and dict(s, clock=0, out1='G', out2='R', walk=False)))
               )

TICK = lambda evt, s: evt == 'tick' and (G1(s) or Y1(s) or G2(s) or Y2(s))
BUTTON = lambda evt, s: evt == 'button' and dict(s, walk=True)

NEXT = lambda evt, s: TICK(evt, s) or BUTTON(evt, s)

def run():
    import queue
    import threading

    event_queue = queue.Queue()

    def run_timer():
        while True:
            time.sleep(0.2)
            event_queue.put('tick')

    def run_button():
        while True:
            sys.stdin.readline()
            event_queue.put('button')

    threading.Thread(target=run_timer, daemon=True).start()
    threading.Thread(target=run_button, daemon=True).start()

    state = INITIAL
    while state:
        print(state)
        evt = event_queue.get()
        state = NEXT(evt, state)

def freeze(d):
    return tuple(sorted(d.items()))

def simulate():
    from collections import deque
    seen_states = set()
    must_check = deque([INITIAL])   # Queue of pending states

    while must_check:
        state = must_check.popleft()    # Get a state
        if freeze(state) in seen_states:
            continue
        seen_states.add(freeze(state))
        num_pending = len(must_check)
        for evt in ['tick', 'button']:   # ALL possible events
            nextstate = NEXT(evt, state)
            if nextstate:
                must_check.append(nextstate)
        if num_pending == len(must_check):
            print("DEADLOCK in", state)

    print("Checked", len(seen_states), "states")
simulate()

# try:
#     run()
# except KeyboardInterrupt:
#     pass


#!/usr/bin/env python3
#
# Project homepage: https://github.com/mwoolweaver
# Licence: <http://unlicense.org/>
# Created by Michael Woolweaver <m.woolweaver@icloud.com>
# ================================================================================

from os import path
from sqlite3 import connect
from sqlite3 import Error as sqlError

from lib.debug import debuginfo, debuginfoDBV, sqlError, args

def findGravity():

    # IF custom directory
    if args.dir:
        pihole_location = args.dir # set custom directory
    else:
        pihole_location = r'/etc/pihole' # set default pihole directory

    # Check for pihole path exsists
    if path.isdir(pihole_location):

        debuginfo('[i] Pi-hole path exists. - {}\n'.format(pihole_location))
        gravity_db_location = path.join(pihole_location, r'gravity.db')

        if path.isfile(gravity_db_location) and path.getsize(gravity_db_location) > 100:

            debuginfo('[i] Pi-Hole Gravity database found. - {}\n'.format(gravity_db_location))

            # see ---> https://stackoverflow.com/a/15355790
            # validate SQLite header
            with open(gravity_db_location, 'rb') as db:
                header = db.read(100)
            debuginfo('[i] Checking Gravity database header. - {}\n'.format(header[:16]))
            if header[:16] == b'SQLite format 3\x00':

                try:
                    gravityConnection = connect(gravity_db_location)
                    gravity = gravityConnection.cursor()

                    debuginfo('[i] Connected to Gravity. - {}\n'.format(gravity_db_location))
                     #      0           1              2
                    return (True, gravityConnection, gravity)

                except sqlError as error:
                    debuginfo("Failed to connect to Gravity. Error: {}".format(error))
                    exit(1)

            elif header[:16] != b'SQLite format 3\x00':
                debuginfo('Something is wrong @ reading {}'.format(gravity_db_location))
                return(exit(1))

        elif path.isfile(gravity_db_location) == False:
            debuginfo ("Can not find {}.".format(gravity_db_location))
            return (exit(1))

        elif path.getsize(gravity_db_location) == 0:
            debuginfo ("{} is empty.".format(gravity_db_location))
            return (exit(1))

        else:
            debuginfo("Something is wrong @ check for file.")
            return (exit(1))

    elif path.isdir(pihole_location) == False:
        debuginfo ("Can not find {}.".format(pihole_location))
        return (exit(1))

    else:
        debuginfo("Something is wrong @ check for file.")
        return (exit(1))


import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proTwo.settings')

import django
django.setup()

import random
from appTwo.models import AccessRecord, Topic, Webpage
from faker import Faker
fakegen = Faker()

topics = ['Games', 'Books', 'News', 'Entertainment', 'Food', 'Health']

def add_topic():
    t = Topic.objects.get_or_create(name = random.choice(topics))[0]
    return t

def populate(N = 5):
    for entry in range(N):
        top = add_topic()

        fake_company = fakegen.company()
        fake_url = fakegen.url()
        fake_date = fakegen.date()

        webpg = Webpage.objects.get_or_create(topic = top, company = fake_company, url = fake_url)[0]

        AccessRecord.objects.get_or_create(webpage = webpg, date = fake_date)

if __name__ == '__main__':
    print('Populating access records!')
    populate(20)
    print('population completed!!')


import pickle
import time

from absl import app, flags

from utils.graphwave.graphwave import *

# flags
FLAGS = flags.FLAGS
flags.DEFINE_integer('emb_dim', 64, 'Embedding dimension.')
flags.DEFINE_integer('max_seq', 100, 'Max length of cascade sequence.')
flags.DEFINE_integer('num_s', 2, 'Number of s for spectral graph wavelets.')
flags.DEFINE_integer('t_o', 3600, 'Observation time.')

# paths
flags.DEFINE_string( 'input', '../datasets/weibo/', 'Pre-training data path.')

def main(argv):
    # hyper-params
    emb_dim = FLAGS.emb_dim
    max_seq = FLAGS.max_seq
    NUM_S = FLAGS.num_s
    observation_time = FLAGS.t_o
    # hyper-params

    def sequence2list(filename):
        graphs = dict()
        with open(filename, 'r') as f:
            for line in f:
                paths = line.strip().split('\t')[:-1][:max_seq + 1]
                graphs[paths[0]] = list()
                for i in range(1, len(paths)):
                    nodes = paths[i].split(":")[0]
                    time = paths[i].split(":")[1]
                    graphs[paths[0]] \
                        .append([[int(x) for x in nodes.split(",")],
                                 int(time)])

        return graphs

    def read_labels(filename):
        labels = dict()
        with open(filename, 'r') as f:
            for line in f:
                id = line.strip().split('\t')[0]
                labels[id] = line.strip().split('\t')[-1]
        return labels

    def write_cascade(graphs, labels, filename, print_note='Write cascade graphs', weight=True):
        """
        Input: cascade graphs, global embeddings
        Output: cascade embeddings, with global embeddings appended
        """
        y_data = list()
        new_input = list()
        cascade_i = 0
        cascade_size = len(graphs)
        total_time = 0

        # for each cascade graph, generate its embeddings via wavelets
        for key, graph in graphs.items():
            start_time = time.time()
            y = int(labels[key])

            # list for saving embeddings
            new_temp = list()

            # build graph
            g = nx.Graph()
            nodes_index = list()
            list_edge = list()
            cascade_embedding = list()
            times = list()
            t_o = observation_time

            # add edges into graph
            for path in graph:
                t = path[1]
                if t >= t_o:
                    continue
                nodes = path[0]
                if len(nodes) == 1:
                    nodes_index.extend(nodes)
                    times.append(1)
                    continue
                else:
                    nodes_index.extend([nodes[-1]])
                if weight:
                    edge = (nodes[-1], nodes[-2], (1 - t / t_o))
                    times.append(1 - t / t_o)
                else:
                    edge = (nodes[-1], nodes[-2])
                list_edge.append(edge)
            if weight:
                g.add_weighted_edges_from(list_edge)
            else:
                g.add_edges_from(list_edge)

            # this list is used to make sure the node order of `chi` is same to node order of `cascade`
            nodes_index_unique = list(set(nodes_index))
            nodes_index_unique.sort(key=nodes_index.index)

            if g.number_of_nodes() <= 1:
                continue

            # embedding dim check
            d = emb_dim / (2 * NUM_S)
            if emb_dim % (2 * NUM_S) != 0:
                raise ValueError

            # generate embeddings
            chi, _, _ = graphwave_alg(g, np.linspace(0, 100, int(d)),
                                      taus='auto', verbose=False,
                                      nodes_index=nodes_index_unique,
                                      nb_filters=NUM_S)
            # save embeddings into list
            for node in nodes_index:
                cascade_embedding.append(chi[nodes_index_unique.index(node)])
            # concat node features to node embedding
            if weight:
                cascade_embedding = np.concatenate([np.reshape(times, (-1, 1)), np.array(cascade_embedding)[:, 1:]],
                                                   axis=1)

            # save embeddings
            new_temp.extend(cascade_embedding)
            new_input.append(new_temp)

            # save label
            y_data.append(y)

            # log
            total_time += time.time() - start_time
            cascade_i += 1
            if cascade_i % 100 == 0:
                speed = total_time / cascade_i
                eta = (cascade_size - cascade_i) * speed
                print("{}, {}/{}, eta: {:.2f} minutes".format(
                    print_note, cascade_i, cascade_size, eta / 60))

        # save embeddings and labels into file
        with open(filename, 'wb') as fin:
            pickle.dump((new_input, y_data), fin)

    def write_aug_cascade(graphs, filename, print_note='augmentation 1', weight=True):
        """
        Input: cascade graphs, global embeddings
        Output: cascade embeddings, with global embeddings appended
        """
        new_input = list()
        embedding_size = emb_dim
        cascade_i = 0
        cascade_size = len(graphs)
        total_time = 0

        # for each cascade graph, generate its embeddings via wavelets
        for key, graph in graphs.items():
            start_time = time.time()
            new_temp = list()

            # build graph
            g = nx.Graph()
            nodes_index = list()
            list_edge = list()
            cascade_embedding = list()
            times = list()
            t_o = observation_time

            # add edges into graph
            for path in graph:
                t = path[1]
                if t >= t_o:
                    continue
                nodes = path[0]
                if len(nodes) == 1:
                    nodes_index.extend(nodes)
                    times.append(1)
                    g.add_node(nodes[-1])
                    continue
                else:
                    nodes_index.extend([nodes[-1]])
                if weight:
                    edge = (nodes[-1], nodes[-2], (1 - t / t_o))
                    times.append(1 - t / t_o)
                else:
                    edge = (nodes[-1], nodes[-2])
                list_edge.append(edge)
            if weight:
                g.add_weighted_edges_from(list_edge)
            else:
                g.add_edges_from(list_edge)

            # this list is used to make sure the node order of `chi` is same to node order of `cascade`
            nodes_index_unique = list(set(nodes_index))
            nodes_index_unique.sort(key=nodes_index.index)

            if g.number_of_nodes() <= 1:
                continue

            # embedding dim check
            d = embedding_size / (2 * NUM_S)
            if embedding_size % (2 * NUM_S) != 0:
                raise ValueError

            # generate embeddings
            chi, _, _ = graphwave_alg(g, np.linspace(0, 100, int(d)),
                                      taus='auto', verbose=False,
                                      nodes_index=nodes_index_unique,
                                      nb_filters=NUM_S)
            # save embeddings into list
            for node in nodes_index:
                cascade_embedding.append(chi[nodes_index_unique.index(node)])
            # concat node features to node embedding
            if weight:
                cascade_embedding = np.concatenate([np.reshape(times, (-1, 1)), np.array(cascade_embedding)[:, 1:]],
                                                   axis=1)

            # save embeddings
            new_temp.extend(cascade_embedding)
            new_input.append(new_temp)

            # log
            total_time += time.time() - start_time
            cascade_i += 1
            if cascade_i % 100 == 0:
                speed = total_time / cascade_i
                eta = (cascade_size - cascade_i) * speed
                print("{}, {}/{}, eta: {:.2f} minutes".format(
                    print_note, cascade_i, cascade_size, eta / 60))

        # save embeddings into file
        with open(filename, 'wb') as fin:
            pickle.dump(new_input, fin)

    time_start = time.time()

    # get the information of nodes/users of cascades
    graphs_train = sequence2list(FLAGS.input + 'train.txt')
    graphs_val = sequence2list(FLAGS.input + 'val.txt')
    graphs_test = sequence2list(FLAGS.input + 'test.txt')
    graphs_aug_1 = sequence2list(FLAGS.input + 'aug_1.txt')
    graphs_aug_2 = sequence2list(FLAGS.input + 'aug_2.txt')
    graphs_unlabel_aug_1 = sequence2list(FLAGS.input + 'unlabel_aug_1.txt')
    graphs_unlabel_aug_2 = sequence2list(FLAGS.input + 'unlabel_aug_2.txt')

    # get the information of labels and sizes of cascades
    label_train = read_labels(FLAGS.input + 'train.txt')
    label_val = read_labels(FLAGS.input + 'val.txt')
    label_test = read_labels(FLAGS.input + 'test.txt')

    print("Start writing train set into file.")
    write_cascade(graphs_train, label_train, FLAGS.input + 'train.pkl', print_note='(1/7) Write train set')
    print("Start writing validation set into file.")
    write_cascade(graphs_val, label_val, FLAGS.input + 'val.pkl', print_note='(2/7) Write val set')
    print("Start writing test set into file.")
    write_cascade(graphs_test, label_test, FLAGS.input + 'test.pkl', print_note='(3/7) Write test set')
    print("Start writing aug set 1 into file.")
    write_aug_cascade(graphs_aug_1, FLAGS.input + 'data_aug_1.pkl', print_note='(4/7) Write augmented cascade graphs')
    print("Start writing aug set 2 into file.")
    write_aug_cascade(graphs_aug_2, FLAGS.input + 'data_aug_2.pkl', print_note='(5/7) Write augmented cascade graphs')
    print("Start writing unlabel aug set 1 into file.")
    write_aug_cascade(graphs_unlabel_aug_1, FLAGS.input + 'data_unlabel_aug_1.pkl',
                      print_note='(6/7) Write unlabeled augmented cascade graphs')
    print("Start writing unlabel aug set 2 into file.")
    write_aug_cascade(graphs_unlabel_aug_2, FLAGS.input + 'data_unlabel_aug_2.pkl',
                      print_note='(7/7) Write unlabeled augmented cascade graphs')

    time_end = time.time()
    print("Processing time: {0:.2f}s".format(time_end - time_start))

if __name__ == "__main__":
    app.run(main)


import numpy as np
from matplotlib import pyplot as plt
import cv2
import glob
import math
from pathlib import Path
import argparse

def search_point(args):
    # images_path = sorted(glob.glob(input_path+folder+"/*/l.jpg", recursive=True))

    distance_table = []
    with open(args.input_directory+'/'+args.folder+"/distance.txt",'r') as file:
        for line in file:
            angle,distance = line.split('-')
            distance_table.append([int(angle),float(distance)+4])
    distance_table = np.array(distance_table)
    distance_table = distance_table[distance_table[:,0].argsort()]

    panorama_i = cv2.imread(args.output_directory+'/panorama/'+args.folder+".png")
    panorama = cv2.cvtColor(panorama_i,cv2.COLOR_BGR2GRAY)

    Path(args.output_directory+'/'+args.folder).mkdir(parents=True,exist_ok=True)

    point_distance_file = open(args.output_directory+'/'+args.folder+'/distance.csv','w')
    point_distance_file.write("Angle,x,y,xp,yp,Distance\n")

    output_directory = Path(args.output_directory+'/'+args.folder+'/point')
    output_directory.mkdir(parents=True,exist_ok=True)

    for index,(angle,distance) in enumerate(distance_table):
        angle = int(angle)
        # if angle not in {937}:
        #     continue
        image_i = cv2.imread(args.input_directory+'/'+args.folder+"/"+str(angle)+"/l.jpg")
        image_i = cv2.resize(image_i,(511,384),interpolation = cv2.INTER_AREA)
        image = cv2.cvtColor(image_i,cv2.COLOR_BGR2GRAY)

        # SIFT algorithm
        sift = cv2.xfeatures2d.SIFT_create()
        kp1, des1 = sift.detectAndCompute(panorama, None)
        kp2, des2 = sift.detectAndCompute(image, None)

        bf = cv2.BFMatcher(cv2.NORM_L1,crossCheck=True)
        matches = bf.match(des1,des2)

        matches = sorted(matches, key = lambda x:x.distance)

        # Coordinate of point
        xk = round(3.04/2.76*384*3.75/distance+image.shape[1]/2)
        yk = round(3.04/2.76*384*0.9/distance+image.shape[0]/2)

        # Average delta
        sum_x=0
        sum_y=0
        denominator = 0
        for i in range(0,20):
            x1,y1 = np.round(kp1[matches[i].queryIdx].pt)
            x2, y2 = np.round(kp2[matches[i].trainIdx].pt)
            dx = x1-x2 if x1-x2>0 else panorama.shape[1]+x1-x2
            dy = y1-y2
            if i<5 or (abs(dx-delta_x)<30 and abs(y1-y2-delta_y)<15):
                sum_x+=dx/matches[i].distance
                sum_y+=(y1-y2)/matches[i].distance
                denominator+=abs(1/matches[i].distance)
                delta_x = sum_x/denominator
                delta_y = sum_y/denominator
                # print(i,y1,y2,x1,x2,dx,y1-y2,delta_x,delta_y)
        delta_x = round(sum_x/denominator)
        delta_y = round(sum_y/denominator)
        # print(delta_x,delta_y)

        dmin = 1e+10
        index=0
        for i in range(0,int(len(matches))):
            x1,y1 = np.round(kp1[matches[i].queryIdx].pt)
            x2, y2 = np.round(kp2[matches[i].trainIdx].pt)
            dx = x1-x2 if x1-x2>0 else panorama.shape[1]+x1-x2
            d = math.sqrt(math.pow(x2-xk,2)+math.pow(y2-yk,2))
            if d<dmin and abs(dx-delta_x)<50 and abs(y1-y2-delta_y)<20:
                dmin = d
                index = i
        x1,y1 = np.round(kp1[matches[index].queryIdx].pt)
        x2, y2 = np.round(kp2[matches[index].trainIdx].pt)
        dx_min = int(x1-x2 if x1-x2>0 else panorama.shape[1]+x1-x2)
        dy_min = int(y1-y2)
        # print(index,y1,y2,x1,y2,dx_min,dy_min)

        xk_panorama = xk+dx_min if xk+dx_min<panorama.shape[1] else xk+dx_min-panorama.shape[1]
        yk_panorama = yk+dy_min

        print(angle,distance)
        point_distance_file.write("{},{},{},{},{},{}\n".format(angle,xk,yk,xk_panorama,yk_panorama,distance))

        if dx_min<panorama.shape[1]-image.shape[1]:
            img_new = panorama_i[:,dx_min:dx_min+image.shape[1]].copy()
        else:
            img_new = np.zeros((panorama.shape[0],image.shape[1],3)).astype(np.uint8)
            img_new[:,:panorama.shape[1]-dx_min] = panorama_i[:,dx_min:]
            img_new[:,panorama.shape[1]-dx_min:] = panorama_i[:,:image.shape[1]-panorama.shape[1]+dx_min]

        # draw circle
        cv2.circle(img_new, (xk,yk+dy_min), 5, (0, 0, 255), thickness=2)
        cv2.circle(image_i, (xk,yk), 5, (0, 0, 255), thickness=2)

        fig = plt.figure(figsize=(12, 6))
        ax = fig.add_subplot(1, 2, 1)
        ax.imshow(cv2.cvtColor(image_i, cv2.COLOR_BGR2RGB))
        ax.set_title(str(angle)+' original',fontsize=10)

        ax = fig.add_subplot(1, 2, 2)
        ax.imshow(cv2.cvtColor(img_new, cv2.COLOR_BGR2RGB))
        ax.set_title(str(angle)+' panorama',fontsize=10)

        fig.tight_layout()
        fig.savefig(f'{args.output_directory}/{args.folder}/point/{angle}.png')
        plt.show()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-i','--input_directory', help="directory to input images", default="input")
    parser.add_argument('-f','--folder', help="folder of input images", default="27072023-1628")
    parser.add_argument('-o','--output_directory', help="directory to input images", default="output")
    args = parser.parse_args()
    search_point(args)


import execjs

def get_js():
    f = open("./qd/index.js", 'r', encoding='UTF-8')
    line = f.readline()
    htmlstr = ''
    while line:
        htmlstr = htmlstr + line
        line = f.readline()
    return htmlstr

jsster = get_js()
ctx = execjs.compile(jsster)
print(ctx.call('enString','123456'))


#!/usr/bin/env python

import sys
import os
import logging

import torch.optim as optim

# Assure that python can find the deeprank files:
deeprank_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, deeprank_root)

from deeprank.learn.NeuralNet import NeuralNet
from deeprank.learn.DataSet import DataSet
from deeprank.learn.model3d import cnn_class
from deeprank.models.metrics import OutputExporter
from deeprank.models.metrics import TensorboardBinaryClassificationExporter

logging.basicConfig(filename="learn-%d.log" % os.getpid(), filemode="w", level=logging.INFO)

def interpret_args(args, usage):
    """ Convert a list of commandline arguments into a set of positional and keyword arguments.

        Args (list of str): the commandline arguments

        Returns: (tuple(list of str, dict of str)): the positional and keyword arguments
    """

    if len(args) == 0:
        print(usage)
        sys.exit(1)

    if "--help" in args or "-h" in args:
        print(usage)
        sys.exit(0)

    positional_args = []
    kwargs = {}
    i = 0
    while i < len(args):

        if args[i].startswith("--"):
            key = args[i][2:]

            i += 1
            kwargs[key] = args[i]

        elif args[i].startswith("-"):
            key = args[i][1:2]

            if len(args[i]) > 2:
                kwargs[key] = args[i][2:]
            else:
                i += 1
                kwargs[key] = args[i]
        else:
            positional_args.append(args[i])

        i += 1

    return (positional_args, kwargs)

if __name__ == "__main__":

    usage = "Usage: %s [-e EPOCH_COUNT] *preprocessed_hdf5_files" % sys.argv[0]

    args, kwargs = interpret_args(sys.argv[1:], usage)

    dataset = DataSet(args, normalize_features=True)

    if len(args) == 0:
        raise RuntimeError("No preprocessed HDF5 files given")

    epoch_count = int(kwargs.get('e', 50))

    run_directory = "run-{}".format(os.getpid())

    neural_net = NeuralNet(dataset, cnn_class, model_type='3d',task='class', cuda=False,
                           metrics_exporters=[OutputExporter(run_directory),
                                              TensorboardBinaryClassificationExporter(run_directory)])
    neural_net.optimizer = optim.AdamW(neural_net.net.parameters(), lr=0.001, weight_decay=0.005)
    neural_net.train(nepoch = epoch_count, divide_trainset=None, train_batch_size = 128, num_workers=18, prefetch_factor=20)


import requests
import time
import json
from pprint import pprint
import spotipy

SPOTIFY_GET_CURRENT_TRACK_URL = 'https://api.spotify.com/v1/me/player/currently-playing'
ACCESS_TOKEN = 'BQBvhtfdCngkgmTBhWj4b7XHlJ3tu1I47EB8f-ofjTWVuWyWt_zAVHKzAFmkUrSFyF_0P2n09Wsh3UvnNfmuXmI8_233CqaKkdTmuTqI8OtWWnk3gPhrFMqgMDdonZcOkKT_gKKRKDtWxYxaBQVPp9iu_T5AcDCm5Z_9vcmijHsIpWMZnmr-aenKTdM5ABfnA30dxm8'

def get_current_track(access_token):
    response = requests.get(
        SPOTIFY_GET_CURRENT_TRACK_URL,
        headers={
            "Authorization": f"Bearer {access_token}"
        }
    )
    json_resp = response.json()

    track_id = json_resp['item']['id']
    track_name = json_resp['item']['name']
    artists = [artist for artist in json_resp['item']['artists']]

    link = json_resp['item']['external_urls']['spotify']

    artist_names = ', '.join([artist['name'] for artist in artists])

    current_track_info = {
    	"id": track_id,
    	"track_name": track_name,
    	"artists": artist_names,
    	"link": link
    }

    return current_track_info

def main():
	current_track_id = None
	while True:
	    current_track_info = get_current_track(ACCESS_TOKEN)

	    if current_track_info['id'] != current_track_id:
		    pprint(
		    	current_track_info,
		    	indent=4,
		    )
		    current_track_id = current_track_info['id']

	    time.sleep(1)

if __name__ == '__main__':
    main()


#!/usr/bin/python3
"""
script that lists all State objects that contain the
letter a from the database hbtn_0e_6_usa
"""

from model_state import State, Base
from sqlalchemy import (create_engine)
from sqlalchemy.orm import sessionmaker
import sys

def state_a():
    engine = create_engine('mysql+mysqldb://{}:{}@localhost:3306/{}'
                           .format(sys.argv[1], sys.argv[2], sys.argv[3]))

    Base.metadata.create_all(engine)
    Session = sessionmaker(bind=engine)
    session = Session()

    for all_state in session.query(State).filter(State.name.like('%a%')):
        print(all_state.id, all_state.name, sep=": ")

if __name__ == '__main__':
    state_a()


from BaseRequest.BaseApi import BaseApi
from tools.get_time_schedule import get_schedule

class UpdateCampaign(BaseApi):

    @staticmethod
    async def post(session, form):
        params = {
            'title': form.title.data,
            'budget': form.budget.data,
            'promoteTime': get_schedule(form.promoteTime.data),
            'promoteArea': form.promoteArea.data,
            'cositeFlag': form.cositeFlag.data,
            'id': form.id.data,
            '_page_csrf_token': form.csrf_token.data
        }

        async with session.post('https://p4p.1688.com/campaign/updateCampaign.html',
                                timeout=5,
                                cookies={'_csrf_token': form.csrf_token.data, 'cookie2': form.cookie2.data},
                                data=params) as resp:
            text = await resp.text()

        return text


import cooler
import os.path as op
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import h5py
from toolz.curried import interleave, reduce, concat, concatv
from toolz.curried import unique
from toolz.curried import compose, compose_left, comp, complement
from toolz.curried import pipe, thread_first, thread_last
from toolz.curried import first, second, nth
from toolz.curried import partial

# https://github.com/open2c/cooler-binder/blob/master/cooler_api.ipynb
from numpy import flip
from numpy import log
from scipy.signal import convolve2d

def addOffsetColumn(df, csize):
    """param df: a bed file with genomic ranges. first 3 coluns are chr, start,
    stop.
    param: csize: a list of the chromosomes' sizes.
    returns a column 'offset' which is the size of all chromosomes preceding
    chr.
    """
    #x = np.zeros_like(df['chr']).astype('int64')
    x = df['chr']
    y = [np.sum(csize[:c]) for c in x]
    return y

def dummy():
    """
    dummy function
    """
    print("hi")

def plotHicMat(arr, transform=np.log10):
    fig = plt.figure(figsize=(10, 10))
    ax = fig.add_subplot(111)
    im = ax.matshow(transform(arr), cmap='YlOrRd')
    fig.colorbar(im)

def getOffsets(chroms, cj):
    """
    chroms: a sorted list of chromnames, in oreder of chromosomes
    cj: a cooler object.
    returns a dictionary with the offset for each chromosome
    in a submatrix that contains only the listed chromosomes.
    """
    n = len(chroms)
    offset = {}
    offset[chroms[0]] = 0
    for i in range(1,n):
        offset[chroms[i]] = offset[chroms[i-1]] + cj.chromsizes[chroms[i-1]] // cj.binsize + 1
    return offset

def addOffsets(bedDF, offset):
    """
    bedDF: pandas data frame with genomic ranges.
    its first columns are: 'chr', 'start', 'end'.
    offset contains the offset of 'chr'.
    Returns a list which contains the correct offset for every row
    of the bedDF (can be added as an extra column 'offset').
    """
    n = len(bedDF)
    x = np.zeros(n)
    for i in range(n):
        x[i] = offset[bedDF['chr'][i]]
    return x.astype('int64')

def getIntervals(bedDF, c):
    """
    bedDF is a data frame with the information about identified contiguous
    intervals in the hic data -- start, end and chromosome.
    c is a cooler object for the same data.
    Will return a sorted list of the implied segments, including the chromosomes
    start and ends within the matrix.
    Example for 2 chromosomes and given sections a,b in chr1 and c,d in chr2,
    the implied segmentation is:
    chr1_start=0 <= a < b <= chr1_end < chr2_start <= c < d <= chr2_end
    """
    #incy = lambda x : (first(x), second(x) + 1)
    #foo = compose(incy, c.extent)
    #l = map(incy, map(c.extent, c.chromnames))
    l = unique(concat(map(c.extent, c.chromnames[:-1])))
    l = sorted(list(l))
    for  i in range(len(bedDF)):
        #x = bedDF.iloc[i]['start'] // c.binsize + bedDF.iloc[i]['offset']
        #y = bedDF.iloc[i]['end'] // c.binsize  + bedDF.iloc[i]['offset']
        x = bedDF.iloc[i]['start'] // c.binsize + c.offset(bedDF.iloc[i]['chr'])
        y = bedDF.iloc[i]['end'] // c.binsize  + c.offset(bedDF.iloc[i]['chr'])
        if abs(y - c.extent(bedDF.iloc[i]['chr'])[1]) <=2:
            y = c.extent(bedDF.iloc[i]['chr'])[1]
        l.append(x)
        l.append(y)
    # we need to correct the boundries a bit
    #l = sorted(l, reverse=True)
    #return l
    return sorted(list(unique(l)))

#l = getIntervals(bedDF, c)
#l


from __future__ import absolute_import, unicode_literals, print_function

from ply import lex

from ..errors import ThriftParserError

__all__ = ['Lexer']

THRIFT_KEYWORDS = (
    'namespace',
    'include',
    'void',
    'bool',
    'byte',
    'i8',
    'i16',
    'i32',
    'i64',
    'double',
    'string',
    'binary',
    'map',
    'list',
    'set',
    'oneway',
    'typedef',
    'struct',
    'union',
    'exception',
    'extends',
    'throws',
    'service',
    'enum',
    'const',
    'required',
    'optional',
    'true',
    'false',
)

class LexerSpec(object):
    """Lexer specification for Thrift IDL files.

    Adapted from thriftpy.parser.lexer."""

    literals = ':;,=*{}()<>[]'

    tokens = (
        'INTCONSTANT',
        'DUBCONSTANT',
        'LITERAL',
        'IDENTIFIER',
    ) + tuple(map(str.upper, THRIFT_KEYWORDS))

    t_ignore = ' \t\r'  # whitespace

    def t_error(self, t):
        raise ThriftParserError(
            'Illegal characher %r at line %d' % (t.value[0], t.lexer.lineno)
        )

    def t_newline(self, t):
        r'\n+'
        t.lexer.lineno += len(t.value)

    def t_ignore_SILLYCOMM(self, t):
        r'\/\*\**\*\/'
        t.lexer.lineno += t.value.count('\n')

    def t_ignore_MULTICOMM(self, t):
        r'\/\*[^*]\/*([^*/]|[^*]\/|\*[^/])*\**\*\/'
        t.lexer.lineno += t.value.count('\n')

    def t_ignore_DOCTEXT(self, t):
        r'\/\*\*([^*/]|[^*]\/|\*[^/])*\**\*\/'
        t.lexer.lineno += t.value.count('\n')

    def t_ignore_UNIXCOMMENT(self, t):
        r'\#[^\n]*'

    def t_ignore_COMMENT(self, t):
        r'\/\/[^\n]*'

    def t_DUBCONSTANT(self, t):
        r'-?\d+\.\d*(e-?\d+)?'
        t.value = float(t.value)
        return t

    def t_HEXCONSTANT(self, t):
        r'0x[0-9A-Fa-f]+'
        t.value = int(t.value, 16)
        t.type = 'INTCONSTANT'
        return t

    def t_INTCONSTANT(self, t):
        r'[+-]?[0-9]+'
        t.value = int(t.value)
        return t

    def t_LITERAL(self, t):
        r'(\"([^\\\n]|(\\.))*?\")|\'([^\\\n]|(\\.))*?\''
        s = t.value[1:-1]
        maps = {
            't': '\t',
            'r': '\r',
            'n': '\n',
            '\\': '\\',
            '\'': '\'',
            '"': '\"'
        }
        i = 0
        length = len(s)
        val = ''
        while i < length:
            if s[i] == '\\':
                i += 1
                if s[i] in maps:
                    val += maps[s[i]]
                else:
                    msg = 'Cannot escape character: %s' % s[i]
                    raise ThriftParserError(msg)
            else:
                val += s[i]
            i += 1

        t.value = val
        return t

    def t_IDENTIFIER(self, t):
        r'[a-zA-Z_](\.[a-zA-Z_0-9]|[a-zA-Z_0-9])*'

        if t.value in THRIFT_KEYWORDS:
            # Not an identifier after all.
            t.type = t.value.upper()

        return t

class Lexer(LexerSpec):
    """Lexer for Thrift IDL files."""

    def __init__(self, **kwargs):
        self._lexer = lex.lex(module=self, **kwargs)

    def input(self, data):
        """Reset the lexer and feed in new input.

        :param data:
            String of input data.
        """
        # input(..) doesn't reset the lineno. We have to do that manually.
        self._lexer.lineno = 1
        return self._lexer.input(data)

    def token(self):
        """Return the next token.

        Returns None when the end of the input is reached.
        """
        return self._lexer.token()


from http import HTTPStatus

"""
Code in this module is based on https://auth0.com/docs/quickstart/backend/python#validate-access-tokens
and course material
"""

class AuthError(Exception):
    """
    AuthError Exception
    A standardized way to communicate auth failure modes
    """
    def __init__(self, status_code: HTTPStatus, error: dict = None):
        self.error = error
        self.status_code = status_code

    @staticmethod
    def auth_error(status_code: HTTPStatus, code: str, description: str):
        return AuthError(status_code, {
            'code': code,
            'description': description
        })


"""
The assistant for brain of Sara.

Created on 19.01.2017

@author: Ruslan Dolovanyuk

"""

import logging
import os
import time

import configs

from extensions import birthday
from extensions import calendar
from extensions import events
from extensions import notes
from extensions import presser
from extensions import rss
from extensions import weather

class Pointer:
    """Class pointer on resources for extensions."""

    def __init__(self, speech, phrases, bd, func_notice):
        """Initialize class for save pointers."""
        self.speech = speech
        self.phrases = phrases
        self.bd = bd
        self.notice = func_notice

        self.rss_start = 1
        self.rss_next = False
        self.rss_source = None

class Assistant:
    """The assistant class for brain of Sara."""

    def __init__(self, speech, config, phrases, bd):
        """Initialize assistant class for brain Sara."""
        self.log = logging.getLogger()
        self.log.info('initialize assistant...')

        self.speech = speech
        self.config = config
        self.phrases = phrases
        self.bd = bd

        self.ptr = Pointer(self.speech, self.phrases, self.bd, self.notice)
        self.reset()
        if "true" == self.config.general.setup:
            configs.open_settings(self.ptr, self.config)

    def reset(self):
        """First run and reset assistant."""
        self.hello()

        if birthday.if_exists(self.ptr, self.config.birthday):
            birthday.notice(self.ptr)
        else:
            birthday.setup(self.ptr, self.config.birthday)

        if events.if_exists(self.ptr, self.config.events):
            events.notice(self.ptr)
        else:
            events.setup(self.ptr, self.config.events)

        if not notes.if_exists(self.ptr, self.config.notes):
            notes.setup(self.ptr, self.config.notes)

        if not rss.if_exists(self.ptr, self.config.rss):
            rss.setup(self.ptr, self.config.rss)

        if not weather.if_exists(self.ptr, self.config.weather):
            weather.setup(self.ptr, self.config.weather)

    def check(self, text, arg):
        """Check need actions for use."""
        if 'hello' == text:
            self.hello()
        elif 'settings' == text:
            configs.open_settings(self.ptr, self.config)
        elif 'version' == text:
            self.speech.speak(
              self.phrases.general.version + str(self.config.general.version))

        calendar.check(self.ptr, self.config.calendar, text)
        presser.check(self.ptr, self.config.presser, text)
        birthday.check(self.ptr, self.config.birthday, text, arg)
        events.check(self.ptr, self.config.events, text, arg)
        notes.check(self.ptr, self.config.notes, text, arg)
        rss.check(self.ptr, self.config.rss, text, arg)
        weather.check(self.ptr, self.config.weather, text,
                      arg, self.config.language.code)

    def hello(self):
        """Say hello method."""
        self.log.info('say hello...')

        hour = int(time.strftime('%H', time.localtime()))
        if 6 <= hour < 12:
            self.speech.speak(
              self.phrases.hello.morning % self.config.general.user)
        elif 12 <= hour < 18:
            self.speech.speak(
              self.phrases.hello.day % self.config.general.user)
        elif 18 <= hour < 24:
            self.speech.speak(
              self.phrases.hello.evening % self.config.general.user)
        else:
            self.speech.speak(
              self.phrases.hello.night % self.config.general.user)

    def notice(self):
        """Play sound and say notes."""
        self.log.info('out notice...')

        sound = os.path.join(
          self.config.general.sounds_dir, self.config.general.notice)
        self.speech.play(sound)
        self.speech.speak(self.phrases.general.notice)

    def setup(self):
        """Run setup methods in extensions."""
        birthday.setup(self.ptr, self.config.birthday)
        events.setup(self.ptr, self.config.events)
        notes.setup(self.ptr, self.config.notes)
        rss.setup(self.ptr, self.config.rss)
        weather.setup(self.ptr, self.config.weather)


# -*- coding: utf-8 -*-
'''
Configuration of network interfaces
===================================

The network module is used to create and manage network settings,
interfaces can be set as either managed or ignored. By default
all interfaces are ignored unless specified.

.. note::

    Prior to version 2014.1.0, only RedHat-based systems (RHEL,
    CentOS, Scientific Linux, etc.) are supported. Support for Debian/Ubuntu is
    new in 2014.1.0 and should be considered experimental.

    Other platforms are not yet supported.

.. code-block:: yaml

    system:
        network.system:
          - enabled: True
          - hostname: server1.example.com
          - gateway: 192.168.0.1
          - gatewaydev: eth0
          - nozeroconf: True
          - nisdomain: example.com
          - require_reboot: True

    eth0:
      network.managed:
        - enabled: True
        - type: eth
        - proto: none
        - ipaddr: 10.1.0.1
        - netmask: 255.255.255.0
        - dns:
          - 8.8.8.8
          - 8.8.4.4

    routes:
      network.routes:
        - name: eth0
        - routes:
          - name: secure_network
            ipaddr: 10.2.0.0
            netmask: 255.255.255.0
            gateway: 10.1.0.3
          - name: HQ_network
            ipaddr: 10.100.0.0
            netmask: 255.255.0.0
            gateway: 10.1.0.10

    eth2:
      network.managed:
        - enabled: True
        - type: slave
        - master: bond0

    eth3:
      network.managed:
        - enabled: True
        - type: slave
        - master: bond0

    eth4:
      network.managed:
        - enabled: True
        - type: eth
        - proto: dhcp
        - bridge: br0

    bond0:
      network.managed:
        - type: bond
        - ipaddr: 10.1.0.1
        - netmask: 255.255.255.0
        - mode: active-backup
        - proto: static
        - dns:
          - 8.8.8.8
          - 8.8.4.4
        - ipv6:
        - enabled: False
        - slaves: eth2 eth3
        - require:
          - network: eth2
          - network: eth3
        - miimon: 100
        - arp_interval: 250
        - downdelay: 200
        - lacp_rate: fast
        - max_bonds: 1
        - updelay: 0
        - use_carrier: on
        - xmit_hash_policy: layer2
        - mtu: 9000
        - autoneg: on
        - speed: 1000
        - duplex: full
        - rx: on
        - tx: off
        - sg: on
        - tso: off
        - ufo: off
        - gso: off
        - gro: off
        - lro: off

    bond0.2:
      network.managed:
        - type: vlan
        - ipaddr: 10.1.0.2
        - use:
          - network: bond0
        - require:
          - network: bond0

    bond0.3:
      network.managed:
        - type: vlan
        - ipaddr: 10.1.0.3
        - use:
          - network: bond0
        - require:
          - network: bond0

    bond0.10:
      network.managed:
        - type: vlan
        - ipaddr: 10.1.0.4
        - use:
          - network: bond0
        - require:
          - network: bond0

    bond0.12:
      network.managed:
        - type: vlan
        - ipaddr: 10.1.0.5
        - use:
          - network: bond0
        - require:
          - network: bond0
    br0:
      network.managed:
        - enabled: True
        - type: bridge
        - proto: dhcp
        - bridge: br0
        - delay: 0
        - ports: eth4
        - bypassfirewall: True
        - use:
          - network: eth4
        - require:
          - network: eth4

    system:
        network.system:
          - enabled: True
          - hostname: server1.example.com
          - gateway: 192.168.0.1
          - gatewaydev: eth0
          - nozeroconf: True
          - nisdomain: example.com
          - require_reboot: True
          - apply_hostname: True

    .. note::
        Apply changes to hostname immediately.

    .. versionadded:: 2015.5.0

.. note::

    When managing bridged interfaces on a Debian or Ubuntu based system, the
    ports argument is required.  Red Hat systems will ignore the argument.
'''
from __future__ import absolute_import

# Import python libs
import difflib
import salt.utils
import salt.utils.network
import salt.loader

# Set up logging
import logging
log = logging.getLogger(__name__)

def __virtual__():
    '''
    Confine this module to non-Windows systems with the required execution
    module available.
    '''
    if not salt.utils.is_windows() and 'ip.get_interface' in __salt__:
        return True
    return False

def managed(name, type, enabled=True, **kwargs):
    '''
    Ensure that the named interface is configured properly.

    name
        The name of the interface to manage

    type
        Type of interface and configuration.

    enabled
        Designates the state of this interface.

    kwargs
        The IP parameters for this interface.

    '''
    # For this function we are purposefully overwriting a bif
    # to enhance the user experience. This does not look like
    # it will cause a problem. Just giving a heads up in case
    # it does create a problem.
    ret = {
        'name': name,
        'changes': {},
        'result': True,
        'comment': 'Interface {0} is up to date.'.format(name),
    }
    if 'test' not in kwargs:
        kwargs['test'] = __opts__.get('test', False)

    # Build interface
    try:
        old = __salt__['ip.get_interface'](name)
        new = __salt__['ip.build_interface'](name, type, enabled, **kwargs)
        if kwargs['test']:
            if old == new:
                pass
            if not old and new:
                ret['result'] = None
                ret['comment'] = 'Interface {0} is set to be ' \
                                 'added.'.format(name)
            elif old != new:
                diff = difflib.unified_diff(old, new, lineterm='')
                ret['result'] = None
                ret['comment'] = 'Interface {0} is set to be ' \
                                 'updated:\n{1}'.format(name, '\n'.join(diff))
        else:
            if not old and new:
                ret['comment'] = 'Interface {0} ' \
                                 'added.'.format(name)
                ret['changes']['interface'] = 'Added network interface.'
            elif old != new:
                diff = difflib.unified_diff(old, new, lineterm='')
                ret['comment'] = 'Interface {0} ' \
                                 'updated.'.format(name)
                ret['changes']['interface'] = '\n'.join(diff)
    except AttributeError as error:
        ret['result'] = False
        ret['comment'] = str(error)
        return ret

    # Setup up bond modprobe script if required
    if type == 'bond':
        try:
            old = __salt__['ip.get_bond'](name)
            new = __salt__['ip.build_bond'](name, **kwargs)
            if kwargs['test']:
                if old == new:
                    pass
                if not old and new:
                    ret['result'] = None
                    ret['comment'] = 'Bond interface {0} is set to be ' \
                                     'added.'.format(name)
                elif old != new:
                    diff = difflib.unified_diff(old, new, lineterm='')
                    ret['result'] = None
                    ret['comment'] = 'Bond interface {0} is set to be ' \
                                     'updated:\n{1}'.format(name, '\n'.join(diff))
            else:
                if not old and new:
                    ret['comment'] = 'Bond interface {0} ' \
                                     'added.'.format(name)
                    ret['changes']['bond'] = 'Added bond {0}.'.format(name)
                elif old != new:
                    diff = difflib.unified_diff(old, new, lineterm='')
                    ret['comment'] = 'Bond interface {0} ' \
                                     'updated.'.format(name)
                    ret['changes']['bond'] = '\n'.join(diff)
        except AttributeError as error:
            #TODO Add a way of reversing the interface changes.
            ret['result'] = False
            ret['comment'] = str(error)
            return ret

    if kwargs['test']:
        return ret

    # Bring up/shutdown interface
    try:
        # Get Interface current status
        interfaces = salt.utils.network.interfaces()
        interface_status = False
        if name in interfaces:
            interface_status = interfaces[name].get('up')
        else:
            for iface in interfaces:
                if 'secondary' in interfaces[iface]:
                    for second in interfaces[iface]['secondary']:
                        if second.get('label', '') == 'name':
                            interface_status = True
        if enabled:
            if interface_status:
                if ret['changes']:
                    # Interface should restart to validate if it's up
                    __salt__['ip.down'](name, type)
                    __salt__['ip.up'](name, type)
                    ret['changes']['status'] = 'Interface {0} restart to validate'.format(name)
                    return ret
            else:
                __salt__['ip.up'](name, type)
                ret['changes']['status'] = 'Interface {0} is up'.format(name)
        else:
            if interface_status:
                __salt__['ip.down'](name, type)
                ret['changes']['status'] = 'Interface {0} down'.format(name)
    except Exception as error:
        ret['result'] = False
        ret['comment'] = str(error)
        return ret

    # TODO: create saltutil.refresh_grains that fires events to the minion daemon
    grains_info = salt.loader.grains(__opts__, True)
    __grains__.update(grains_info)
    __salt__['saltutil.refresh_modules']()
    return ret

def routes(name, **kwargs):
    '''
    Manage network interface static routes.

    name
        Interface name to apply the route to.

    kwargs
        Named routes
    '''
    ret = {
        'name': name,
        'changes': {},
        'result': True,
        'comment': 'Interface {0} routes are up to date.'.format(name),
    }
    apply_routes = False
    if 'test' not in kwargs:
        kwargs['test'] = __opts__.get('test', False)

    # Build interface routes
    try:
        old = __salt__['ip.get_routes'](name)
        new = __salt__['ip.build_routes'](name, **kwargs)
        if kwargs['test']:
            if old == new:
                return ret
            if not old and new:
                ret['result'] = None
                ret['comment'] = 'Interface {0} routes are set to be added.'.format(name)
                return ret
            elif old != new:
                diff = difflib.unified_diff(old, new, lineterm='')
                ret['result'] = None
                ret['comment'] = 'Interface {0} routes are set to be ' \
                                 'updated:\n{1}'.format(name, '\n'.join(diff))
                return ret
        if not old and new:
            apply_routes = True
            ret['comment'] = 'Interface {0} routes added.'.format(name)
            ret['changes']['network_routes'] = 'Added interface {0} routes.'.format(name)
        elif old != new:
            diff = difflib.unified_diff(old, new, lineterm='')
            apply_routes = True
            ret['comment'] = 'Interface {0} routes updated.'.format(name)
            ret['changes']['network_routes'] = '\n'.join(diff)
    except AttributeError as error:
        ret['result'] = False
        ret['comment'] = str(error)
        return ret

    # Apply interface routes
    if apply_routes:
        try:
            __salt__['ip.apply_network_settings'](**kwargs)
        except AttributeError as error:
            ret['result'] = False
            ret['comment'] = str(error)
            return ret

    return ret

def system(name, **kwargs):
    '''
    Ensure that global network settings are configured properly.

    name
        Custom name to represent this configuration change.

    kwargs
        The global parameters for the system.

    '''
    ret = {
        'name': name,
        'changes': {},
        'result': True,
        'comment': 'Global network settings are up to date.',
    }
    apply_net_settings = False
    kwargs['test'] = __opts__['test']
    # Build global network settings
    try:
        old = __salt__['ip.get_network_settings']()
        new = __salt__['ip.build_network_settings'](**kwargs)
        if __opts__['test']:
            if old == new:
                return ret
            if not old and new:
                ret['result'] = None
                ret['comment'] = 'Global network settings are set to be added.'
                return ret
            elif old != new:
                diff = difflib.unified_diff(old, new, lineterm='')
                ret['result'] = None
                ret['comment'] = 'Global network settings are set to be ' \
                                 'updated:\n{0}'.format('\n'.join(diff))
                return ret
        if not old and new:
            apply_net_settings = True
            ret['changes']['network_settings'] = 'Added global network settings.'
        elif old != new:
            diff = difflib.unified_diff(old, new, lineterm='')
            apply_net_settings = True
            ret['changes']['network_settings'] = '\n'.join(diff)
    except AttributeError as error:
        ret['result'] = False
        ret['comment'] = str(error)
        return ret

    # Apply global network settings
    if apply_net_settings:
        try:
            __salt__['ip.apply_network_settings'](**kwargs)
        except AttributeError as error:
            ret['result'] = False
            ret['comment'] = str(error)
            return ret

    return ret


import json
import mage
import tempfile
import time
import unittest

class TestAssessment(unittest.TestCase):

    TEST_ASSET = "unittest.example.com"

    @classmethod
    def setUpClass(cls):
        mage.connect()

    def setUp(self):
        self.a = mage.Assessment.create('EXTERNAL', name='UNITTEST')
        self.ar_id = None
        self.used_asset = False
        self.used_credential = False
        self.ag = None

    def tearDown(self):
        if self.ar_id:
            mage.AssessmentRun.stop(assessment_run_id=self.ar_id)
        if self.a:
            self.a.delete()
        if self.used_asset:
            for asset in mage.Asset.by_identifier(self.TEST_ASSET).auto_paging_iter():
                asset.delete()
        if self.ag:
            self.ag.delete()

    def test_create_assessment(self):
        self.assertIsInstance(self.a,mage.Assessment)

    def test_find_assessment_by_id(self):
        #BUG: created items are not always immediately findable
        time.sleep(2)
        result = mage.Assessment.eq(id=self.a.id).limit(1).search()
        self.assertEqual(len(result),1)

    def test_find_assessment_by_name(self):
        #BUG: created items are not always immediately findable
        time.sleep(2)
        result = mage.Assessment.eq(name=self.a.name).limit(1).search()
        self.assertEqual(len(result),1)

    def test_list_assessment(self):
        #BUG: created items are not always immediately findable
        time.sleep(2)
        result = mage.Assessment.list()
        self.assertTrue(len(result) > 0)

    def test_update_assessment_by_assignment(self):
        self.a.name = "KILROY"
        self.assertEqual(self.a.name, 'KILROY')

    def test_update_assessment_by_call(self):
        self.a.update(name="KILROY")
        self.assertEqual(self.a.name, 'KILROY')

    def test_start_assessment(self):
        self.ar_id = self.a.start()
        self.assertIsNotNone(self.ar_id)

    def test_load_assets(self):
        self.used_asset = True
        data = {'domains': [self.TEST_ASSET]}
        self.a.load_assets(data)
        self.assertEqual(len(self.a.assets), 1)

    def test_load_assets_bad_key(self):
        self.used_asset = True
        data = {'invalid':127}
        self.assertRaises(KeyError, self.a.load_assets, data)

    def test_load_assets_bad_type(self):
        self.used_asset = True
        data = {'ips':127}
        self.assertRaises(TypeError, self.a.load_assets, data)

    def test_load_assets_credential(self):
        self.used_credential = True
        data = {'cloud_credentials': [{'cloud_platform': 'AWS', 'access_key': '123'}]}
        self.a.load_assets(data)
        self.assertEqual(len(list(self.a.cloud_credentials.auto_paging_iter())), 1)

    def test_load_assets_from_filename(self):
        self.used_asset = True
        data = {'domains': self.TEST_ASSET}
        with tempfile.NamedTemporaryFile(mode='w+') as temp:
            temp.write(json.dumps(data))
            temp.flush()
            self.a.load_assets(temp.name)
        self.assertEqual(len(self.a.assets), 1)

    def test_load_assets_from_fileobj(self):
        self.used_asset = True
        data = {'domains': self.TEST_ASSET}
        with tempfile.TemporaryFile(mode='w+') as temp:
            temp.write(json.dumps(data))
            temp.flush()
            temp.seek(0)
            self.a.load_assets(temp)
        self.assertEqual(len(self.a.assets), 1)

    def test_load_assets_multiple(self):
        self.used_asset = True
        self.test_load_assets()
        self.test_load_assets()

    def test_load_assets_str(self):
        self.used_asset = True
        data = {'domains': self.TEST_ASSET}
        self.a.load_assets(data)
        self.assertEqual(len(self.a.assets), 1)

    def test_delete_assessment(self):
        self.assertIsNotNone(self.a.delete())
        self.a = None

    def test_list_runs(self):
        self.a.runs

    def test_create_schedule(self):
        self.assertTrue(self.a.create_schedule('DAILY', {'hour':'23', 'minute':'55'}))
        self.assertTrue(self.a.schedule)
        self.assertEqual(self.a.schedule.frequency, 'DAILY')
        self.assertEqual(self.a.schedule.time_expression.hour, '23')
        self.assertEqual(self.a.schedule.time_expression.minute, '55')

    def test_delete_schedule(self):
        self.test_create_schedule()
        self.assertTrue(self.a.delete_schedule())
        self.assertFalse(self.a.schedule)

    def test_filters(self):
        self.assertIsInstance(self.a.runs_filter, mage.Filter)
        self.assertEqual(self.a.runs_filter._cls, mage.AssessmentRun)
        self.assertIsInstance(self.a.asset_groups_filter, mage.Filter)
        self.assertEqual(self.a.asset_groups_filter._cls, mage.AssetGroup)
        self.assertIsInstance(self.a.cloud_credentials_filter, mage.Filter)
        self.assertEqual(self.a.cloud_credentials_filter._cls, mage.CloudCredential)

    def test_connect_asset(self):
        self.used_asset = True
        self.asset = mage.Asset.create('DOMAIN', self.TEST_ASSET)
        connection = self.a.connect(self.asset)
        self.assertIsInstance(connection, mage.AssessmentAssetConnection)
        self.assertEqual(len(list(self.a.assets.auto_paging_iter())), 1)

    def test_connect_asset_by_id(self):
        self.used_asset = True
        self.asset = mage.Asset.create('DOMAIN', self.TEST_ASSET)
        connection = self.a.connect(asset_id=self.asset.id)
        self.assertIsInstance(connection, mage.AssessmentAssetConnection)
        self.assertEqual(len(list(self.a.assets.auto_paging_iter())), 1)

    def test_connect_asset_group(self):
        self.ag = mage.AssetGroup.create('UNITTESTGROUP')
        connection = self.a.connect(self.ag)
        self.assertIsInstance(connection, mage.AssessmentAssetGroupConnection)
        self.assertEqual(len(list(self.a.asset_groups.auto_paging_iter())), 1)

    def test_connect_asset_group_by_id(self):
        self.ag = mage.AssetGroup.create('UNITTESTGROUP')
        connection = self.a.connect(asset_group_id=self.ag.id)
        self.assertIsInstance(connection, mage.AssessmentAssetGroupConnection)
        self.assertEqual(len(list(self.a.asset_groups.auto_paging_iter())), 1)

    def test_connect_bad_type(self):
        self.assertRaises(TypeError, self.a.connect, self.a)

if __name__ == '__main__':
    unittest.main()