import functools
import re
from copy import deepcopy
from itertools import chain, permutations
from math import sqrt, ceil, log10, exp
from random import choice, sample, randrange, random
from string import ascii_lowercase as lc, ascii_uppercase as uc

# with open(('\\'.join(__file__.split("\\")[:-1])+r"\words_longer.txt").lstrip('\\')) as f:
#     lines = f.readlines()
#     english_words_longer = [x.strip("\n") for x in lines]

with open(('\\'.join(__file__.split("\\")[:-1])+r"\words_list.txt").lstrip('\\')) as f:
    lines = f.readlines()
    english_words = [x.strip("\n") for x in lines]
    english_words_longer = list(filter(lambda x: len(x) > 2, english_words))

with open(('\\'.join(__file__.split("\\")[:-1]) + r"\words_list_plurals.txt").lstrip('\\')) as f:
    lines = f.readlines()
    english_words_plural = [x.strip("\n .,'").lower() for x in lines]

with open(('\\'.join(__file__.split("\\")[:-1])+r"\trigrams.txt").lstrip('\\')) as f:
    lines = f.readlines()
    english_trigrams_count = {}
    for line in lines:
        l = line.strip("\n").split(" ")
        english_trigrams_count[l[0]] = int(l[1])
    n = sum(english_trigrams_count.values())
    english_trigrams = {k.lower(): log10(float(v)/n) for k, v in english_trigrams_count.items()}
    del english_trigrams_count
    floor = log10(0.01 / n)

class Utils:
    @staticmethod
    def get_stats(cipher: str):
        print("Stats of", cipher)
        print("Length: %d Factors:" % len(cipher), Utils.get_factors(len(cipher)))
        print("Letter Frequency:", Utils.sorted_dict_str(Utils.frequency(cipher)))
        unformatted = Utils.unformat(cipher)
        print("\nUnformatted:", unformatted)
        print("Length: %d Factors:" % len(unformatted), sorted(Utils.get_factors(len(unformatted))))

        print("Letter Frequency:", Utils.sorted_dict_str(Utils.frequency_count_letters(unformatted)))
        print("IoC: %.7f Chi²: %.2f" % (Utils.index_of_coincidence(unformatted), Utils.chi_squared(unformatted)))
        print()
        print("Ngram Frequency:", Utils.sorted_dict_str(Utils.frequency_count_ngrams(cipher)))
        print("Trigram Similarity: %.2f" % Utils.ngram_score(unformatted))
        print()

    @staticmethod
    def unformat(formatted: str, spaces=False) -> str:
        return ''.join(x for x in formatted.lower() if (x in lc) or (spaces and x == " "))

    @staticmethod
    def reformat(unformatted: str, format_pattern: str) -> str:
        result = ''
        queue = list(unformatted)
        for f in format_pattern:
            if f in lc: result += queue.pop(0).lower()
            elif f in uc: result += queue.pop(0).upper()
            else: result += f
        return result

    @staticmethod
    def check_english_percent(plain: str, in_words=True) -> float:
        included, excluded = 0, 0
        if in_words:
            for word in plain.split(' '):
                if Utils.unformat(word) in english_words_longer: included += 1
            return float(included)/len(plain.split(' '))
        else:
            return Utils.chi_squared(plain)

    @staticmethod
    def is_english(plain: str, in_words=True) -> bool:
        if in_words: return Utils.check_english_percent(plain, in_words) > 0.2
        else: return Utils.check_english_percent(plain, in_words) > 0.7 * (11 * pow(len(plain), 2))

    @staticmethod
    def chi_squared(plain_f) -> float:
        actual = {"a": 0.08167, "b": 0.01492, "c": 0.02782, "d": 0.04253, "e": 0.12702, "f": 0.02228, "g": 0.02015, "h": 0.06094, "i": 0.06966, "j": 0.00153, "k": 0.00772, "l": 0.04025, "m": 0.02406, "n": 0.06749, "o": 0.07507, "p": 0.01929, "q": 0.00095, "r": 0.05987, "s": 0.06327, "t": 0.09056, "u": 0.02758, "v": 0.00978, "w": 0.0236, "x": 0.0015, "y": 0.01974, "z": 0.00074}
        plain = Utils.unformat(plain_f)
        freq = Utils.frequency_count_letters(plain)
        return sum(((freq[c] - actual[c]*len(plain)) ** 2) / actual[c]*len(plain) for c in lc)

    @staticmethod
    def index_of_coincidence(text: str) -> float:
        num = den = 0.0
        for val in Utils.frequency_count_letters(text).values():
            i = val
            num += i * (i - 1)
            den += i
        if den == 0.0: return 0.0
        else: return num / (den * (den - 1))

    @staticmethod
    def ngram_score(plain_f, l=3):
        plain = Utils.unformat(plain_f)
        score = 0
        for i in range(len(plain)-l+1):
            if plain[i:i+l] in english_trigrams: score += english_trigrams[plain[i:i+l]]
            else: score += floor
        return score

    @staticmethod
    def frequency(text) -> dict:
        freq = {}
        for c in text:
            if freq.get(c): freq[c] += 1
            else: freq[c] = 1
        return freq

    @staticmethod
    def frequency_count_letters(cipher: str) -> dict:
        cipher = Utils.unformat(cipher)
        freq = dict((c, 0) for c in lc)
        for c in cipher: freq[c] += 1
        return freq

    @staticmethod
    def frequency_count_words(cipher: str) -> dict:
        cipher = Utils.unformat(cipher, spaces=True).split(" ")
        return Utils.frequency(cipher)

    @staticmethod
    def frequency_count_ngrams(cipher_f: str, min_length: int = 3, max_length: int = 5) -> dict:
        cipher = Utils.unformat(cipher_f)
        ngrams = {}
        for i in range(min_length, max_length + 1):
            igrams = [cipher[j:j + i] for j in range(len(cipher) - i + 1)]
            igrams_count = {igram: igrams.count(igram) for igram in set(igrams)}
            del igrams
            ngrams.update({k: v for k, v in igrams_count.items() if v > 2})
            del igrams_count
        return ngrams

    @staticmethod
    @functools.lru_cache(maxsize=None)
    def get_factors(numb: int) -> list:
        return list(chain(*[(i, int(numb // i)) for i in range(1, int(ceil(sqrt(numb)))) if not numb % i]))

    @staticmethod
    @functools.lru_cache(maxsize=None)
    def get_useful_factors(numb: int, min_key_len=3, max_key_len=16) -> list:
        return list(chain(*[(i, int(numb // i)) for i in range(max(2, min_key_len),  min(int(ceil(sqrt(numb))), max_key_len)) if not numb % i]))

    @staticmethod
    def sorted_dict_str(dictionary: dict, sorting=lambda x: -x[1]) -> str:
        return "Length: " + str(len(dictionary.keys())) + "    " + ', '.join('%s: %s' % (k, v) for k, v in sorted(dictionary.items(), key=sorting))

    @staticmethod
    def insert_spaces(cipher: str) -> str:
        longest_words = sorted(english_words_plural, key=lambda x: -len(x))
        # print(longest_words)

        cipher = list(cipher)
        result = ""
        while len(cipher) > 0:
            for word in longest_words:
                w = ''.join(cipher[:len(word)])
                if w.lower() == word:
                    result += w + " "
                    del cipher[:len(word)]
                    break
            else:
                result += cipher.pop()
        return result

    @staticmethod
    def get_str_as_vals(text: str) -> list:
        return [lc.index(x) for x in text]

    @staticmethod
    def get_str_from_vals(numbs: list) -> str:
        return ''.join(lc[x] for x in numbs)

    @staticmethod
    def word_match(pattern: str):
        pattern = pattern.lower()+'$'
        for word in english_words_longer:
            if re.match(pattern, word.lower()): print(word)

class Caesar:
    @staticmethod
    def encode(plain: str, shift: int) -> str:
        return Utils.reformat(Caesar.encode_plain(plain, shift), plain)

    @staticmethod
    def encode_plain(plain: str, shift: int) -> str:
        return ''.join(lc[(lc.index(x)+shift) % len(lc)] for x in Utils.unformat(plain))

    @staticmethod
    def decode(cipher: str, shift: int) -> str:
        return Utils.reformat(Caesar.decode_plain(cipher, shift), cipher)

    @staticmethod
    def decode_plain(cipher: str, shift: int) -> str:
        return ''.join(lc[(lc.index(x)-shift) % len(lc)] for x in Utils.unformat(cipher))

    @staticmethod
    def auto_decode(cipher: str) -> (int, str):
        tests = sorted([(s, Caesar.decode_plain(Utils.unformat(cipher), s)) for s in range(len(lc))], key=lambda x: Utils.chi_squared(x[1]))
        return tests[0][0], Utils.reformat(tests[0][1], cipher)

class Affine:
    @staticmethod
    def encode(plain: str, a: int, b: int) -> str:
        return Utils.reformat(Affine.encode_plain(plain, a, b), plain)

    @staticmethod
    def encode_plain(plain: str, a: int, b: int) -> str:
        return ''.join(lc[(a*lc.index(x)+b) % len(lc)] for x in Utils.unformat(plain))

    @staticmethod
    def decode(cipher: str, a: int, b: int) -> str:
        if Affine.gcd(a, len(lc)) != 1: return "Impossible, a and m are not coprime"
        return Utils.reformat(Affine.decode_plain(Utils.unformat(cipher), a, b), cipher)

    @staticmethod
    def decode_plain(cipher: str, a: int, b: int) -> str:
        a_ = Affine.mod_mult_inverse(a, len(lc))
        return ''.join(lc[a_*(lc.index(x)-b) % len(lc)] for x in cipher)

    @staticmethod
    def auto_decode(cipher_f: str) -> ((int, int), str):
        for a in [x for x in range(len(lc)) if Affine.gcd(x, len(lc)) == 1]:
            for b in [x for x in range(len(lc))]:
                p = Affine.decode(cipher_f, a, b)
                if Utils.is_english(p, in_words=False): return (a, b), p
        return "None"

    @staticmethod
    def mod_mult_inverse(a: int, m: int) -> int:
        a = a % m
        for x in range(1, m):
            if (a*x) % m == 1: return x
        return 1

    @staticmethod
    def gcd(a: int, b: int) -> int:
        if b == 0: return a
        return Affine.gcd(b, a % b)

class Keyword:
    @staticmethod
    def encode(plain: str, key: str) -> str:
        return Utils.reformat(Keyword.encode_plain(plain, key), plain)

    @staticmethod
    def encode_plain(plain: str, key: str) -> str:
        encoding = []
        for c in key+lc:
            if c not in encoding: encoding.append(c)
        return Substitution.encode_plain(plain, {k: v for k, v in zip(list(lc), encoding)})

    @staticmethod
    def decode(cipher: str, key: str) -> str:
        return Utils.reformat(Keyword.decode_plain(cipher, key), cipher)

    @staticmethod
    def decode_plain(cipher: str, key: str) -> str:
        encoding = []
        for c in key + lc:
            if c not in encoding: encoding.append(c)
        return Substitution.decode_plain(cipher, {k: v for k, v in zip(list(lc), encoding)})

    @staticmethod
    def auto_decode(cipher: str) -> (str, str):
        return Substitution.auto_decode(cipher)

english_letter_order = "etaoinshrdlcumwfgypbvkjxqz"
cipher_letter_order = ""

class Substitution:
    @staticmethod
    def encode(plain: str, mapping: dict) -> str:
        return Utils.reformat(Substitution.encode_plain(plain, mapping), plain)

    @staticmethod
    def encode_plain(plain: str, mapping: dict) -> str:
        return ''.join(mapping.get(x, x) for x in Utils.unformat(plain))

    @staticmethod
    def encode_from_str(cipher: str, mapping: str) -> str:
        mapping = {lc[i].lower(): mapping[i].lower() for i in range(len(lc))}
        return Utils.reformat(Substitution.encode_plain(cipher, mapping), cipher)

    @staticmethod
    def decode(cipher: str, mapping: dict) -> str:
        return Utils.reformat(Substitution.decode_plain(cipher, mapping), cipher)

    @staticmethod
    def decode_from_str(cipher: str, mapping: str) -> str:
        mapping = {lc[i].lower(): mapping[i].lower() for i in range(len(lc))}
        return Utils.reformat(Substitution.decode_plain(cipher, mapping), cipher)

    @staticmethod
    def decode_plain(cipher: str, mapping: dict) -> str:
        mapping_inverted = {v: k for k, v in mapping.items()}
        return ''.join(mapping_inverted[x] for x in Utils.unformat(cipher))

    @staticmethod
    def auto_decode(cipher_f: str, fitness=Utils.ngram_score, base_depth: int=500) -> (str, str):
        global cipher_letter_order
        cipher = Utils.unformat(cipher_f)

        cipher_letter_order = ''.join(x[0] for x in sorted(Utils.frequency_count_letters(cipher).items(), key=lambda x: -x[1]))
        pkey = {k: v for k, v in zip(cipher_letter_order, english_letter_order)}

        key, plain = Substitution.hillclimb(cipher, pkey, fitness, base_depth)
        return Substitution.get_inp(key, plain, cipher_f, fitness)

    @staticmethod
    def hillclimb(cipher: str, pkey: dict, fitness, base_depth=500) -> (dict, str):
        global cipher_letter_order
        pplain = Substitution.decode_plain(cipher, pkey)
        pscore = fitness(pplain)
        count = 0
        while count < base_depth:
            ckey = deepcopy(pkey)
            c, c2 = choice(cipher_letter_order), choice(cipher_letter_order)
            ckey[c], ckey[c2] = ckey[c2], ckey[c]

            cplain = Substitution.decode_plain(cipher, ckey)
            cscore = fitness(cplain)
            if cscore > pscore:
                pscore = cscore
                pkey = deepcopy(ckey)
                pplain = cplain
                count = 0
            count += 1
        return pkey, pplain

    @staticmethod
    def get_inp(key, plain, cipher_f, fitness):
        Utils.get_stats(plain)
        cipher = Utils.unformat(cipher_f)
        plain = Utils.reformat(plain, cipher_f)
        inp = inp2 = " "
        while inp not in ("q", ""):
            print('Current Best:', plain, Substitution.key_as_str(key))
            inp = input('q to end, s to swap some letters or r to retry with a random original key\n  >>')
            if len(inp) == 0: continue
            if inp[0].lower() == "s":
                while inp2 != "q":
                    inp2 = input("Swap what (a b), q to return\n >>")
                    if inp2 == "q": break
                    a, b = inp2.strip().split(" ")
                    key[a], key[b] = key[b], key[a]
                    new = Substitution.decode(cipher_f, key)
                    print('New Best:', new, Substitution.key_as_str(key))
                    inp3 = input("Keep? y/n")
                    if inp3[0].lower() == "y":
                        plain = new
                    else:
                        print('Current Best:', plain, Substitution.key_as_str(key))

            if inp[0].lower() == "r":
                pkey = {k: v for k, v in zip(lc, sample(list(lc), len(lc)))}
                key, plain = Substitution.hillclimb(cipher, pkey, fitness)
                plain = Utils.reformat(plain, cipher_f)
        return key, plain

    @staticmethod
    def key_as_str(key: dict) -> str:
        return ''.join(v.upper() for k, v in sorted(key.items(), key=lambda x: ord(x[0])))

class Vigenere:
    @staticmethod
    def encode(plain: str, key: str) -> str:
        return Utils.reformat(Vigenere.encode_plain(plain, key), plain)

    @staticmethod
    def encode_plain(plain: str, key: str) -> str:
        return ''.join(lc[(lc.index(p)+lc.index(k)) % len(lc)] for p, k in zip(Utils.unformat(plain), key * int((len(plain) + 1) / len(key))))

    @staticmethod
    def decode(cipher: str, key: str) -> str:
        return Utils.reformat(Vigenere.decode_plain(cipher, key), cipher)

    @staticmethod
    def decode_plain(cipher: str, key: str) -> str:
        return ''.join(lc[(lc.index(p)-lc.index(k)) % len(lc)] for p, k in zip(Utils.unformat(cipher), key * int((len(cipher) + 1) / len(key))))

    @staticmethod
    def auto_decode(cipher_f: str, min_key_len=3, max_key_len=16) -> (str, str):
        cipher = Utils.unformat(cipher_f)
        spacings = Vigenere.get_dist_between_repeats(cipher, Utils.frequency_count_ngrams(cipher))   # Get spacings between repeated ngrams
        factors = Utils.frequency(y for k in [Utils.get_factors(x) for l in spacings.values() for x in l] for y in k if max_key_len > y > min_key_len)
        print(factors)
        result_f, key = "", ""
        for key_len in sorted(factors.keys(), key=lambda x: -factors.get(x)):   # For most common factor first
            print(key_len)
            sections = ["" for _ in range(key_len)]
            for i in range(len(cipher)):
                sections[i % len(sections)] += cipher[i]   # Split into sections

            key, solved_sections = zip(*[Caesar.auto_decode(sections[i]) for i in range(key_len)])  # Try AutoDecode on each

            result = ""
            for c_i in range(len(solved_sections[0])):  # Reconstruct string
                for section in solved_sections:
                    try: result += section[c_i]
                    except IndexError: break

            result_f = Utils.reformat(result, cipher_f)

            print(result_f)
            if Utils.is_english(result, False): break
            # if Utils.ngram_score(result) < 200: break        # Stop if english, otherwise try next factor

        return Utils.get_str_from_vals(key), result_f

    @staticmethod
    def get_dist_between_repeats(cipher_f: str, ngrams_to_check: dict) -> dict:
        cipher = Utils.unformat(cipher_f)
        distances = {}
        for ngram, count in ngrams_to_check.items():
            ngram = Utils.unformat(ngram)
            distances[ngram] = []
            last_pos = -1
            for i in range(len(cipher) - len(ngram) + 1):
                if cipher[i:i + len(ngram)] == ngram:
                    if last_pos != -1: distances[ngram].append(i - last_pos)
                    last_pos = i
        return distances

# noinspection PyShadowingNames
class Beaufort(Vigenere):
    @staticmethod
    def encode(plain: str, key: str) -> str:
        return Utils.reformat(Beaufort.encode_plain(plain, key), plain)

    @staticmethod
    def encode_plain(plain: str, key: str) -> str:
        return ''.join(lc[(lc.index(k) - lc.index(p)) % len(lc)] for p, k in zip(Utils.unformat(plain), key * int((len(plain) + 1) / len(key))))

    @staticmethod
    def decode(cipher: str, key: str) -> str:
        return Utils.reformat(Beaufort.decode_plain(cipher, key), cipher)

    @staticmethod
    def decode_plain(cipher: str, key: str) -> str:
        return ''.join(lc[(lc.index(k) - lc.index(p)) % len(lc)] for p, k in zip(Utils.unformat(cipher), key * int((len(cipher) + 1) / len(key))))

    @staticmethod
    def auto_decode(cipher_f: str, min_key_len=3, max_key_len=16):
        cipher = Utils.unformat(cipher_f)
        spacings = Vigenere.get_dist_between_repeats(cipher, Utils.frequency_count_ngrams(
            cipher))  # Get spacings between repeated ngrams
        factors = Utils.frequency(y for k in [Utils.get_factors(x) for l in spacings.values() for x in l] for y in k if
                                  max_key_len > y > min_key_len)

        result_f, key = "", ""
        for key_len in sorted(factors.keys(), key=lambda x: -factors.get(x)):  # For most common factor first
            sections = ["" for _ in range(key_len)]
            for i in range(len(cipher)):
                sections[i % len(sections)] += cipher[i]  # Split into sections

            key, solved_sections = zip(
                *[Beaufort.auto_trials(sections[i]) for i in range(key_len)])  # Try AutoDecode on each

            result = ""
            for c_i in range(len(solved_sections[0])):  # Reconstruct string
                for section in solved_sections:
                    try:
                        result += section[c_i]
                    except IndexError:
                        break

            result_f = Utils.reformat(result, cipher_f)
            if Utils.is_english(result_f): break  # Stop if english, otherwise try next factor

        return Utils.get_str_from_vals(key), result_f

    @staticmethod
    def auto_trials(cipher):
        tests = sorted([(l, Beaufort.decode_plain(Utils.unformat(cipher), lc[l])) for l in range(len(lc))], key=lambda x: Utils.chi_squared(x[1]))
        return tests[0][0], Utils.reformat(tests[0][1], cipher)

class VariantBeaufort(Vigenere):
    @staticmethod
    def encode(plain: str, key: str) -> str:
        return Utils.reformat(VariantBeaufort.encode_plain(plain, key), plain)

    @staticmethod
    def encode_plain(plain: str, key: str) -> str:
        return ''.join(lc[(lc.index(p) - lc.index(k)) % len(lc)] for p, k in zip(Utils.unformat(plain), key * int((len(plain) + 1) / len(key))))

    @staticmethod
    def decode(cipher: str, key: str) -> str:
        return Utils.reformat(VariantBeaufort.decode_plain(cipher, key), cipher)

    @staticmethod
    def decode_plain(cipher: str, key: str) -> str:
        return ''.join(lc[(lc.index(p) + lc.index(k)) % len(lc)] for p, k in zip(Utils.unformat(cipher), key * int((len(cipher) + 1) / len(key))))

    @staticmethod
    def auto_decode(cipher_f: str, min_key_len=3, max_key_len=16):
        cipher = Utils.unformat(cipher_f)
        spacings = Vigenere.get_dist_between_repeats(cipher, Utils.frequency_count_ngrams(
            cipher))  # Get spacings between repeated ngrams
        factors = Utils.frequency(y for k in [Utils.get_factors(x) for l in spacings.values() for x in l] for y in k if
                                  max_key_len > y > min_key_len)

        result_f, key = "", ""
        for key_len in sorted(factors.keys(), key=lambda x: -factors.get(x)):  # For most common factor first
            sections = ["" for _ in range(key_len)]
            for i in range(len(cipher)):
                sections[i % len(sections)] += cipher[i]  # Split into sections

            key, solved_sections = zip(
                *[VariantBeaufort.auto_trials(sections[i]) for i in range(key_len)])  # Try AutoDecode on each

            result = ""
            for c_i in range(len(solved_sections[0])):  # Reconstruct string
                for section in solved_sections:
                    try:
                        result += section[c_i]
                    except IndexError:
                        break

            result_f = Utils.reformat(result, cipher_f)
            if Utils.is_english(result_f): break  # Stop if english, otherwise try next factor

        return Utils.get_str_from_vals(key), result_f

    @staticmethod
    def auto_trials(cipher):
        tests = sorted([(l, VariantBeaufort.decode_plain(Utils.unformat(cipher), lc[l])) for l in range(len(lc))], key=lambda x: Utils.chi_squared(x[1]))
        return tests[0][0], Utils.reformat(tests[0][1], cipher)

class ColumnarTransposition:
    @staticmethod
    def encode(plain: str, key: list) -> str:
        return ColumnarTransposition.encode_plain(Utils.unformat(plain), key)

    @staticmethod
    def encode_plain(plain: str, key: list) -> str:
        return ''.join([''.join(plain[bs+key.index(i)] if bs+key.index(i) < len(plain) else "X" for i in range(0, len(key))) for bs in range(0, len(plain), len(key))]).upper()

    @staticmethod
    def decode(cipher: str, key: list) -> str:
        return ColumnarTransposition.decode_plain(cipher, key)

    @staticmethod
    def decode_plain(cipher: str, key: list) -> str:
        return ''.join([''.join(cipher[bs+key[i]] if bs+key[i] < len(cipher) else "X" for i in range(0, len(key))) for bs in range(0, len(cipher), len(key))]).upper()

    @staticmethod
    def auto_decode(cipher: str, min_len=3, max_len=7) -> (list, str):
        best = [i for i in range(min_len)]
        bestscore = Utils.ngram_score(ColumnarTransposition.decode(cipher, best))

        for i in range(min_len, max_len+1):
            key, plain = ColumnarTransposition.try_length(cipher, i)
            # print("%d: (%s) %s" % (i, ','.join(str(x) for x in key), plain))
            if Utils.ngram_score(plain) > bestscore:
                best, bestscore = key, Utils.ngram_score(plain)
        return best, ColumnarTransposition.decode(cipher, best)

    @staticmethod
    def try_length(cipher: str, length: int) -> (list, str):
        best = [0 for _ in range(length)]
        bestscore = Utils.ngram_score(ColumnarTransposition.decode(cipher, best))

        for perm in permutations(list(range(length))):
            plain = ColumnarTransposition.decode(cipher, perm)
            if Utils.ngram_score(plain) > bestscore:
                best, bestscore = perm, Utils.ngram_score(plain)
        return best, ColumnarTransposition.decode(cipher, best)

    @staticmethod
    def alternate_decode(cipher: str, key: list) -> (list, str):
        l = len(key)
        l2 = ceil(len(cipher) / l)
        columns = [cipher[i:i + l2] for i in range(0, len(cipher), l2)]

        rows = []
        for i in range(len(columns[0])):
            try:
                rows.append(''.join(columns[j][i] for j in range(l)))
            except IndexError:
                pass

        return ColumnarTransposition.decode(''.join(rows), key)

    @staticmethod
    def alternate_auto_decode(cipher: str, ls: str) -> (list, str):
        results = []
        for l in ls:
            l2 = ceil(len(cipher) / l)
            columns = [cipher[i:i + l2] for i in range(0, len(cipher), l2)]

            rows = []
            for i in range(len(columns[0])):
                try:
                    rows.append(''.join(columns[j][i] for j in range(l)))
                except IndexError:
                    pass

            results.append(ColumnarTransposition.auto_decode(''.join(rows)))
        return sorted(results, key=lambda x: Utils.ngram_score(x))[0]

class Playfair:
    @staticmethod
    def encode(plain: str, key: str, sub=("j", "")) -> str:
        return Playfair.encode_plain(plain, key, sub)

    @staticmethod
    def encode_plain(plain: str, key: str, sub=("j", "")) -> str:
        keyalpha = Utils.unformat(key) + lc.replace(sub[0], "")
        keyfiltered = ''.join(keyalpha[i] for i in range(len(keyalpha)) if i == keyalpha.index(keyalpha[i]))
        key_grid = [[keyfiltered[i*5+j] for j in range(5)] for i in range(5)]

        plain = re.sub(*sub, Utils.unformat(plain))

        q, output = list(plain), ""
        while len(q) > 0:
            if len(q) == 1: q += "x"
            if q[0] == q[1]: output += Playfair.encrypt_pair(q.pop(0), "x", key_grid)
            else: output += Playfair.encrypt_pair(q.pop(0), q.pop(0), key_grid)

        return output

    @staticmethod
    def encrypt_pair(c1, c2, key_grid):
        def get_coord(c: str, grid) -> (int, int):
            for i in range(len(grid)):
                for j in range(len(grid[i])):
                    if grid[i][j] == c: return i, j

        p1, p2 = get_coord(c1, key_grid), get_coord(c2, key_grid)
        if p1[0] == p2[0]: v = key_grid[p1[0]][(p1[1] + 1) % 5] + key_grid[p2[0]][(p2[1] + 1) % 5]
        elif p1[1] == p2[1]: v = key_grid[(p1[0] + 1) % 5][p1[1]] + key_grid[(p2[0] + 1) % 5][p2[1]]
        else: v = key_grid[p1[0]][p2[1]] + key_grid[p2[0]][p1[1]]
        return v

    @staticmethod
    def decode(plain: str, key: str, sub=("j", "")) -> str:
        return Playfair.decode_plain(plain, key, sub)

    @staticmethod
    def decode_plain(plain: str, key: str, sub=("j", "")) -> str:
        keyalpha = Utils.unformat(key) + lc.replace(sub[0], "")
        keyfiltered = ''.join(keyalpha[i] for i in range(len(keyalpha)) if i == keyalpha.index(keyalpha[i]))
        key_grid = [[keyfiltered[i * 5 + j] for j in range(5)] for i in range(5)]

        plain = re.sub(*sub, Utils.unformat(plain))

        q, output = list(plain), ""
        while len(q) > 0:
            if len(q) == 1: q += "x"
            if q[0] == q[1]:
                output += Playfair.decrypt_pair(q.pop(0), "x", key_grid)
            else:
                output += Playfair.decrypt_pair(q.pop(0), q.pop(0), key_grid)

        return output

    @staticmethod
    def decrypt_pair(c1, c2, key_grid):
        def get_coord(c: str, grid) -> (int, int):
            for i in range(len(grid)):
                for j in range(len(grid[i])):
                    if grid[i][j] == c: return i, j

        p1, p2 = get_coord(c1, key_grid), get_coord(c2, key_grid)
        if p1[0] == p2[0]:
            v = key_grid[p1[0]][(p1[1] - 1) % 5] + key_grid[p2[0]][(p2[1] - 1) % 5]
        elif p1[1] == p2[1]:
            v = key_grid[(p1[0] - 1) % 5][p1[1]] + key_grid[(p2[0] - 1) % 5][p2[1]]
        else:
            v = key_grid[p1[0]][p2[1]] + key_grid[p2[0]][p1[1]]
        return v

    @staticmethod
    def auto_decode(cipher_f: str) -> (str, str):
        cipher = Utils.unformat(cipher_f)
        # bigrams = Utils.frequency("".join(x) for x in zip(cipher, cipher[1:]))
        # print(', '.join('%s: %s' % (k, v) for k, v in sorted(bigrams.items(), key=lambda x: -x[1])))
        # print(', '.join('%s: %s' % (Playfair.decode(k, "enabling"), v) for k, v in sorted(bigrams.items(), key=lambda x: -x[1])[:100]))
        k = sample(lc, len(lc))
        k.remove("j")
        key, plain = Playfair.simulated_annealing(cipher.replace("w", ""), k)
        return key, plain

    @staticmethod
    def simulated_annealing(cipher: str, pkey: list):
        T, STEP, COUNT = 20.0, 0.2, 1000

        def modify_key(key: list) -> list:

            key_grid = [[key[i * 5 + j] for j in range(5)] for i in range(5)]
            method = randrange(10)
            a, b = randrange(5), randrange(5)
            if   method == 0: key_grid[a], key_grid[b] = key_grid[b], key_grid[a] # Swap row
            elif method == 1:
                for i in range(5):
                    key_grid[i][a], key_grid[i][b] = key_grid[i][b], key_grid[i][a] # Swap column
            elif method == 2:           # Reverse all
                return list(key)[::-1]
            elif method == 3:           # Reverse
                key_grid = [key_grid[5-i] for i in range(5)] # Reverse rows
            elif method == 4:
                key_grid = [[key_grid[j][5 - i] for i in range(5)] for j in range(5)]  # Reverse columns
            else:
                a, b = randrange(25), randrange(25)
                key[a], key[b] = key[b], key[a]
                return key
            return key_grid

        pplain = Playfair.decode_plain(cipher, ''.join(pkey))
        pscore = Utils.ngram_score(pplain)
        temp = T
        while temp >= 0:
            for c in range(COUNT):
                ckey = modify_key(pkey)
                cplain = Playfair.decode_plain(cipher, ''.join(ckey))
                cscore = Utils.ngram_score(cplain)
                dF = cscore - pscore
                if dF > 0 or exp(dF/T) > random(): pplain, pkey, pscore = cplain, ckey, cscore

            temp -= STEP
        return pkey, pplain

    # @staticmethod
    # def hillclimb(cipher: str, pkey: list) -> (dict, str):
    #     print(pkey)
    #     pplain = Playfair.decode_plain(cipher, ''.join(pkey))
    #     pscore = Utils.ngram_score(pplain)
    #     count = 0
    #     while count < base_depth:
    #         ckey = pkey
    #         a, b = randrange(len(ckey)), randrange(len(ckey))
    #         ckey[a], ckey[b] = ckey[b], ckey[a]
    #
    #         cplain = Playfair.decode_plain(cipher, ''.join(ckey))
    #         print(cplain)
    #         cscore = Utils.ngram_score(cplain)
    #         if cscore > pscore:
    #             pscore = cscore
    #             pkey = ckey
    #             pplain = cplain
    #             count = 0
    #         count += 1
    #     return pkey, pplain

class Scytale: # NOT FINSIHED
    @staticmethod
    def encode(plain: str, key: int) -> str:
        return Scytale.encode_plain(Utils.unformat(plain), key)

    @staticmethod
    def encode_plain(plain: str, key: int) -> str:
        sections = ["" for _ in range(key)]
        for i in range(len(plain)):
            sections[i % key] += plain[i]
        return ''.join(sections)

    @staticmethod
    def decode(cipher: str, key: int) -> str:
        return Scytale.decode_plain(cipher, key)

    @staticmethod
    def decode_plain(cipher: str, key: int) -> str:
        res = ""
        for offset in range(0, key-1):
            for i in range(offset, len(cipher), key-1):
                res += cipher[i]
                print(offset, i, cipher[i])

        return res

    @staticmethod
    def auto_decode(cipher: str) -> (object, str):
        pass

def try_all(cipher: str):
    Utils.get_stats(cipher)
    print("Trying all for", cipher)
    print('Caesar AutoDecode: Decoded Shift: %d, Plaintext: %s' % Caesar.auto_decode(cipher))
    print('Substitution AutoDecode:', Substitution.auto_decode(cipher))
    # print('Affine AutoDecode:', Affine.auto_decode(cipher))
    print('Vigenere AutoDecode: Key: %s Plaintext: %s' % Vigenere.auto_decode(cipher))
    print('Beaufort AutoDecode:', Beaufort.auto_decode(cipher))
    print('Variant Beaufort AutoDecode:', VariantBeaufort.auto_decode(cipher))
    print('Columnar Transposition AutoDecode:', ColumnarTransposition.auto_decode(cipher))
    print('')

"""
class Cipher:
    @staticmethod
    def encode(plain: str, key: object) -> str:
        return Utils.reformat(Cipher.encode_plain(plain, key), plain)

    @staticmethod
    def encode_plain(plain: str, key: object) -> str:
        return plain

    @staticmethod
    def decode(cipher: str, key: object) -> str:
        return Utils.reformat(Cipher.decode_plain(cipher, key), cipher)

    @staticmethod
    def decode_plain(cipher: str, key: object) -> str:
        return cipher

    @staticmethod
    def auto_decode(cipher: str) -> (object, str):
        pass
"""


import subprocess
import platform
import chardet

def ping(site):
    param = '-n' if platform.system().lower() == 'windows' else '-c'
    args = ['ping', param, '2', site]
    result = subprocess.Popen(args, stdout=subprocess.PIPE)
    for line in result.stdout:
        result = chardet.detect(line)
        line = line.decode(result['encoding']).encode('utf-8')
        print(line.decode('utf-8'))

sites = ['yandex.ru', 'youtube.com']
for site in sites:
    ping(site)


"""Tests for Widget."""
import os
import shutil
from pathlib import Path
from unittest import TestCase

from ipywidgets import widgets
from qiskit.circuit.random import random_circuit
from qiskit.primitives import Estimator
from qiskit.quantum_info.random import random_pauli

from purplecaffeine.core import LocalStorage, Trial
from purplecaffeine.widget import Widget

class TestWidget(TestCase):
    """TestTrial."""

    def setUp(self) -> None:
        """SetUp local storage backend for widget use"""
        curr_dir = os.path.dirname(os.path.abspath(__file__))
        self.save_path = os.path.join(curr_dir, "test")
        if not os.path.exists(self.save_path):
            Path(self.save_path).mkdir(parents=True, exist_ok=True)
        self.local_storage = LocalStorage(path=self.save_path)

    def test_render_pagination(self):
        """Test to check that pagination buttons work correctly"""
        self.add_n_trials(20)
        widget = Widget(self.local_storage)
        box = widget.render_pagination()
        previous_button = box.children[0]
        next_button = box.children[1]
        self.assertFalse(next_button.disabled)
        self.assertTrue(previous_button.disabled)

    def test_render_trial(self):
        """Test to check that trails buttons are being displayed"""
        self.add_n_trials(5)
        widget = Widget(self.local_storage)
        tabs = widget.render_trial()
        info = tabs.children[0]
        self.assertTrue(self.local_storage.list()[0].uuid in info.value)

    def test_search(self):
        """Test to check the search function"""
        self.add_n_trials(20)
        widget = Widget(self.local_storage)
        box = widget.search()
        search_button = box.children[0]
        input_text = box.children[1]
        input_text.value = self.local_storage.list()[0].name
        search_button.click()
        self.assertEqual(widget.search_value, self.local_storage.list()[0].name)
        self.assertEqual(len(widget.trials), 1)
        input_text.value = ""
        search_button.click()
        self.assertEqual(len(widget.trials), 10)

    def test_load_detail(self):
        """Test to check that the selected trial is the first one"""
        self.add_n_trials(2)
        widget = Widget(self.local_storage)
        first_trail = self.local_storage.list()[0]
        button = widgets.Button(
            description=first_trail.name,
            disabled=False,
            button_style="",
            tooltip=first_trail.uuid,
            icon="",
        )
        widget.load_detail(button)
        self.assertEqual(widget.selected_trial.uuid, first_trail.uuid)

    def test_trial_list(self):
        """Test to check that new trials are being added."""
        self.add_n_trials(20)
        widget = Widget(self.local_storage)
        self.assertEqual(
            widget.render_trails_list().children[0].tooltip,
            self.local_storage.list()[0].uuid,
        )
        self.assertEqual(
            widget.render_trails_list().children[0].tooltip, widget.selected_trial.uuid
        )

    def add_n_trials(self, number_of_trials):
        """Aux function to add 20 trials to the class storage"""
        n_qubits = 4
        depth = 3
        shots = 2000

        circuit = random_circuit(n_qubits, depth)
        obs = random_pauli(n_qubits)
        for i in range(0, number_of_trials):
            with Trial("Example trial " + str(i), storage=self.local_storage) as trial:
                # track some parameters
                trial.add_parameter("estimator", "qiskit.primitives.Estimator")
                trial.add_parameter("depth", str(depth))
                trial.add_parameter("n_qubits", str(n_qubits))
                trial.add_parameter("shots", str(shots))

                # track objects of interest
                trial.add_circuit("circuit", circuit)

                # run
                exp_value = (
                    Estimator().run(circuit, obs, shots=shots).result().values.item()
                )

                # track results of run
                trial.add_metric("exp_value", exp_value)

    def tearDown(self) -> None:
        """TearDown the trials objects."""
        trials_files = os.path.join(self.save_path)
        if os.path.exists(trials_files):
            shutil.rmtree(trials_files)


from typing import List

class Solution:
    def summaryRanges(self, nums: List[int]) -> List[str]:
        n, start = len(nums), 0
        output = []
        for end, x in enumerate(nums):
            if end == n - 1 or nums[end + 1] > x + 1:
                output += (
                    [str(nums[start]) + "->" + str(x)] if end > start else [str(x)]
                )
                start = end + 1

        return output


import os
import time
import argparse
import numpy as np
import torch
import torchaudio
from torchvision import transforms

from src.datasets import get_noise_factor, BASE_DIR, AudioPadding
from src.train_unet import Net, load_model_from_checkpoint
from src.models_config import base_dict

SAMPLE_RATE = base_dict['SAMPLE_RATE']

def get_voice_noise_for_inference_bank(speaker_id=None,
                                       utterance_id=None,
                                       noise_origin=None,
                                       noise_id=None,
                                       target_snr=10):
    speakers_folder = os.path.join(BASE_DIR, 'data', 'VCTK-Corpus', 'wav48')
    noise_folder = os.path.join(BASE_DIR, 'data', 'DEMAND')

    def get_audio(base_folder, folder, sound_id):
        available_folders = os.listdir(base_folder)
        if folder is None:
            folder = np.random.choice(available_folders)
        else:
            assert folder in available_folders

        available_ids = os.listdir(os.path.join(base_folder,
                                                folder))
        if sound_id is None:
            sound_id = np.random.choice(available_ids)
        else:
            assert sound_id in available_ids

        file_sound = os.path.join(base_folder, folder, sound_id)
        waveform, sample_rate = torchaudio.load(file_sound)
        if sample_rate != base_dict['SAMPLE_RATE'] or len(waveform) != base_dict['AUDIO_LEN']:
            composed = transforms.Compose([torchaudio.transforms.Resample(orig_freq=sample_rate,
                                                                          new_freq=base_dict['SAMPLE_RATE']),
                                           AudioPadding(required_length=base_dict['AUDIO_LEN'],
                                                        # padding_tail='right'
                                                        )])
            return composed(waveform), folder, sound_id

    waveform, speaker_id, utterance_id = get_audio(speakers_folder,
                                                   folder=speaker_id,
                                                   sound_id=utterance_id)
    waveform_noise, noise_origin, noise_id = get_audio(noise_folder,
                                                       folder=noise_origin,
                                                       sound_id=noise_id)

    noise_factor = get_noise_factor(waveform,
                                    waveform_noise,
                                    target_snr=target_snr)
    waveform_noise = waveform_noise * noise_factor
    utterance_id = os.path.splitext(utterance_id)[0]
    noise_id = os.path.splitext(noise_id)[0]
    return waveform + waveform_noise, speaker_id, utterance_id, noise_origin, noise_id

def get_voice_noise_for_inference_custom(path_to_file):
    waveform, sample_rate = torchaudio.load(path_to_file)
    if sample_rate != base_dict['SAMPLE_RATE'] or len(waveform) != base_dict['AUDIO_LEN']:
        composed = transforms.Compose([torchaudio.transforms.Resample(orig_freq=sample_rate,
                                                                      new_freq=base_dict['SAMPLE_RATE']),
                                       AudioPadding(required_length=base_dict['AUDIO_LEN'],
                                                    # padding_tail='right'
                                                    )])
        return composed(waveform)
    return waveform

def model_inference(model, sound_noise_wave):
    with torch.no_grad():
        return model(sound_noise_wave)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-chp', '--checkpoint_name',
                        help='checkpoint name to load model')
    parser.add_argument('-srn', '--target_snr',
                        type=int, default=10, help='target SNR')
    parser.add_argument('-speaker_id', '--speaker_id',
                        help='speaker id (VCTK folder) if None pick random')
    parser.add_argument('-utterance_id', '--utterance_id',
                        help='voice id (VCTK/speaker_id folder) if None pick random')
    parser.add_argument('-noise_origin', '--noise_origin',
                        help='noise origin (DEMAND folder) if None pick random')
    parser.add_argument('-noise_id', '--noise_id',
                        help='noise id (DEMAND/noise_origin folder) if None pick random')
    parser.add_argument('-custom_file', '--custom_file', default=None,
                        help='path to custom file')
    args = parser.parse_args()

    model_name = args.checkpoint_name
    path_for_results = os.path.join(BASE_DIR, 'results')
    os.makedirs(path_for_results, exist_ok=True)

    model_features = int(model_name.split('_')[2])
    encoder_depth = int(model_name.split('_')[3])
    model_epoch = int(model_name.split('_')[5])
    model_pref = f'{model_features}_{encoder_depth}_{model_epoch}'

    ckp_dict = load_model_from_checkpoint(args.checkpoint_name, training=False)
    model = ckp_dict['model']

    if args.custom_file is None:
        (sound_noise_wave, speaker_id,
         utterance_id, noise_origin,
         noise_id) = get_voice_noise_for_inference_bank(speaker_id=args.speaker_id,
                                                        utterance_id=args.utterance_id,
                                                        noise_origin=args.noise_origin,
                                                        noise_id=args.noise_id,
                                                        target_snr=args.target_snr)
        init_wave_file_name = f'init_sound_{utterance_id}_{noise_origin}_{noise_id}_{args.target_snr}.wav'
        path_to_init_file = os.path.join(path_for_results,
                                         init_wave_file_name)
        torchaudio.save(path_to_init_file,
                        sound_noise_wave,
                        SAMPLE_RATE,
                        precision=32)
        print(f'Init file: {path_to_init_file}')
        procecces_wave_file_name = f'model_{model_pref}_{utterance_id}_{noise_origin}_{noise_id}_{args.target_snr}.wav'
    else:
        sound_noise_wave = get_voice_noise_for_inference_custom(args.custom_file)
        print(f'Init file: {args.custom_file}')
        custom_file_name = os.path.splitext(os.path.split(args.custom_file)[1])[0]
        procecces_wave_file_name = f'model_{model_pref}_{custom_file_name}.wav'

    start = time.time()
    estimated_sound = model_inference(model, sound_noise_wave)
    inference_time = round(time.time() - start, 3)

    path_estimated_sound = os.path.join(path_for_results,
                                        procecces_wave_file_name)
    torchaudio.save(path_estimated_sound,
                    estimated_sound,
                    SAMPLE_RATE,
                    precision=32)
    print(f'Saved to {path_estimated_sound}, inference time {inference_time} s')


import argparse
import logging

from abc import ABC, abstractmethod
from fall.firmware_updater import FirmwareUpdater
from fall.result_constants import INSTALL_SUCCESS, INPUT_VALIDATION_FAILURE, UNABLE_TO_GET_PLATFORM_INFO_FAILURE, \
    UPGRADE_CHECK_FAILURE, INSTALL_FAILURE, REBOOT_FAILURE, UNSUPPORTED_OS_FAILURE, QUERY_SUCCESS, Result

from falc.falc_exception import FalcException
from falc.utility import search_keyword, get_canonical_representation_of_path
from falc.constant import QUERY, FW

COMMAND_FAIL = "FAIL"
COMMAND_SUCCESS = "SUCCESS"
CACHE = '/var/cache/manageability/repository-tool/'
FW_SUCCESS_MESSAGE_LIST = [INSTALL_SUCCESS]
FW_FAILURE_MESSAGE_LIST = [INPUT_VALIDATION_FAILURE, UNABLE_TO_GET_PLATFORM_INFO_FAILURE,
                           UPGRADE_CHECK_FAILURE, INSTALL_FAILURE, REBOOT_FAILURE, UNSUPPORTED_OS_FAILURE]
QUERY_SUCCESS_MESSAGE_LIST = [QUERY_SUCCESS]
QUERY_FAILURE_MESSAGE_LIST = [UNSUPPORTED_OS_FAILURE, UNABLE_TO_GET_PLATFORM_INFO_FAILURE]

logger = logging.getLogger(__name__)

class Command(ABC):  # pragma: no cover
    """Base class for command objects

    @param cmd_type: name of command to execute
    """

    def __init__(self, cmd_type: str) -> None:
        self._cmd_type = cmd_type

    @abstractmethod
    def trigger_command(self, args: argparse.Namespace) -> int:
        """Trigger the command-line utility tool to invoke update.
        @return: process return code"""
        pass

class FwCommand(Command):
    def __init__(self) -> None:
        """FW command"""
        super().__init__(FW)

    def trigger_command(self, user_input: argparse.Namespace) -> int:  # no pragma cover
        """Trigger the command-line utility tool to invoke update.

        @param user_input: arguments passed to command-line tool.
        @return: exit code for command (0 success, 1 fail)
        """
        payload = FirmwareUpdater().update(path_to_update_package=get_canonical_representation_of_path(user_input.path),
                                           capsule_release_date=user_input.releasedate, bios_vendor=user_input.vendor,
                                           platform_name=user_input.product,
                                           platform_manufacturer=user_input.manufacturer, guid=user_input.guid,
                                           autofill_platform_info=user_input.autofill)
        return self.search_response(payload)

    def search_response(self, payload: Result) -> int:
        """Search for keywords in response message

        @param payload: payload received in which to search
        @return: exit code for command (0 success, 1 fail)
        """
        if search_keyword(payload, FW_SUCCESS_MESSAGE_LIST):
            logger.info(f"\n {self._cmd_type.upper()} Command Execution is Completed")
            return 0
        elif search_keyword(payload, FW_FAILURE_MESSAGE_LIST):
            logger.error(f"\n {self._cmd_type.upper()} Command Execution FAILED")
            return 1
        else:
            raise FalcException('Unable to determine if FW operation is success or failure')

class QueryCommand(Command):
    def __init__(self) -> None:
        """Query command"""
        super().__init__(QUERY)

    def trigger_command(self, user_option: argparse.Namespace) -> int:   # no pragma cover
        """Trigger the command-line utility tool to invoke query request.
        @param user_option:query type to return
        @return: exit code for command (0 success, 1 fail)
        """
        logger.debug(f"command={user_option}")

        payload = FirmwareUpdater().query(user_option.option)
        return self.search_response(payload)

    def search_response(self, payload: Result) -> int:
        """Search for keywords in response message

        @param payload: payload received in which to search
        @return: exit code for command (0 success, 1 fail)
        """
        logger.debug("query search response")
        if search_keyword(payload, QUERY_SUCCESS_MESSAGE_LIST):
            logger.info(f"\n {self._cmd_type.upper()} Command Execution is Completed")
            return 0
        elif search_keyword(payload, QUERY_FAILURE_MESSAGE_LIST):
            logger.error(f"\n {self._cmd_type.upper()} Command Execution FAILED")
            return 1
        else:
            raise FalcException(f'Unable to determine if query operation is success or failure')


from django.conf import settings
from django import forms

from dal import autocomplete

I18N_PATH = 'autocomplete_light/i18n/'

class CustomModelSelect2(autocomplete.ModelSelect2):

    # class Media:
    #     css = {
    #         'screen': ('css/custom_autocomplete.css',)
    #     }

    # @property
    # def media(self):
    #     base_media = super().media
    #     custom_media = forms.Media(css={'screen': ('css/custom_autocomplete.css',)})
    #     # return base_media + custom_media
    #     res = base_media + custom_media
    #     print(res._css_lists)
    #     return res

    @property
    def media(self):
        """Return JS/CSS resources for the widget."""
        extra = '' if settings.DEBUG else '.min'
        i18n_name = self._get_language_code()
        i18n_file = (
            '%s%s.js' % (I18N_PATH, i18n_name),
        ) if i18n_name else ()

        return forms.Media(
            js=(
                'admin/js/vendor/select2/select2.full.js',
                'autocomplete_light/autocomplete_light%s.js' % extra,
                'autocomplete_light/select2%s.js' % extra,
            ) + i18n_file,
            css={
                'screen': (
                    'admin/css/vendor/select2/select2%s.css' % extra,
                    'admin/css/autocomplete.css',
                    'autocomplete_light/select2.css',
                    'css/custom_autocomplete.css',
                ),
            },
        )


import os
from datetime import datetime

import pandas as pd
from dotenv import load_dotenv
from prefect import flow, get_run_logger, task
from prefect_gcp.cloud_storage import GcsBucket

load_dotenv()

GCS_BUCKET_PATH = os.getenv(
    "GCS_BUCKET_PATH_BRONZE"
)  # <--- make sure your path file is right

@task(name="Convert-Data-to-Pandas-DataFrame")
def convert_data_to_pandas_dataframe(data: list[dict]) -> pd.DataFrame:
    return pd.DataFrame(data=data)

@task(name="Load-DF-to-Parquet-and-Upload-to-GCS")
def load_df_to_parquet_and_upload_to_gcs(
    df, serialization_format: str = "parquet"
) -> None:
    gcs_bucket_block = GcsBucket.load(
        "default"
    )  # <--- your bucket block name from /utils/create_blocks.py

    DATETIME_UPLOADED = datetime.now().strftime("year=%Y/month=%m/day=%d")
    HOUR_UPLOADED = datetime.now().strftime("%H%M%S")
    SEARCH_TERM = df["search_term"][0]
    CITY = df["store_city"][0]

    destination_gcs_path = f"{GCS_BUCKET_PATH}/{DATETIME_UPLOADED}/term={SEARCH_TERM}/city={CITY}/{SEARCH_TERM}_{HOUR_UPLOADED}"  # example: 'bronze/year=2023/month=07/day=21/term=twix/city=FLN/twix_195250.parquet'
    gcs_bucket_block.upload_from_dataframe(
        df, to_path=destination_gcs_path, serialization_format=serialization_format
    )
    return destination_gcs_path

@flow(name="Load-to-Parquet-and-Upload-to-GCS")
def load_to_parquet_and_upload_to_gcs(data: list[dict]) -> str:
    logger = get_run_logger()

    logger.info("---Starting Load to Parquet and Upload to GCS---")

    logger.info("Converting data to Pandas DataFrame")
    df = convert_data_to_pandas_dataframe(data=data)

    logger.info("Loading DF to Parquet and Uploading to GCS")
    gcs_filename = load_df_to_parquet_and_upload_to_gcs(df=df)

    logger.info("---Finished Loading Data to Parquet File in GCS---")

    return gcs_filename

if __name__ == "__main__":
    load_to_parquet_and_upload_to_gcs()


import nltk
import string
from nltk.corpus import stopwords
from nltk import FreqDist
import os
import pager
import json
import threading

results = []

#input: url + filters + key-terms
#output: a list of reviews
def inputFileToList(url, keywords, type, rating):
	reviewList = pager.startGetPages(url, keywords, type, rating)
	return reviewList

#tester - print
def printLine(lineNo):
	thisList = inputFileToList()

#class Review
#revBody = the review content
#revWeight = ordering variable
class Review:
	def __init__(self, revId, revBody, revWeight):
		self.revId = revId
		self.revBody = revBody
		self.revWeight = revWeight

	def printReview(self):
		#print(self.revId)
		#print(self.revBody + "\n")
		print()

#input: urlm key-terms, type, rating
#output: a list of reviews (reviewArray)
def createRevArray(url, keywords, type, rating):
	counter = 0
	reviewList = inputFileToList(url, keywords, type, rating) # get all the reviews into one long list of strings
	reviewArray = []
	for item in reviewList: # for each review string in review list
		revId = counter ## rev id
		rev = Review(revId, reviewList[counter], 0.0)
		counter = counter + 1
		reviewArray.append(rev)
	return reviewArray

# input: string (key-term or review)
# output: tokens of string (lower case with puctuation removed. all items tokenised)
def getTokens(original):
	lowers = original.lower() #lower case
	transtable = {ord(c): None for c in string.punctuation} #build punctuation
	no_punctuation = lowers.translate(transtable) #remove punctuation
	tokens = nltk.word_tokenize(no_punctuation) #token
	return tokens

from nltk.stem.porter import *
#tokens = getTokens(original)
#filtered = [w for w in tokens if not w in stopwords.words('english')]

# input list
# output list with stop words removed
def textFilter(original):
	filtered = [w for w in original if not w in stopwords.words('english')]
	return filtered

#input list
#output list with items stemmed to base(english dictionary)
def stemTokens(tokens, stemmer):
	stemmed = []
	for item in tokens:
		stemmed.append(stemmer.stem(item))
	return stemmed

#define the stemmer we are using. used in stemTokens
stemmer = PorterStemmer()

#Search C - full searching tool with relative frequency distribution applied
#input: key-terms, list of reviews, long review preference(boolean)
#output: ordered list of reviews.
def searchC(keywords, reviewArray, lenSet):
	global results

	#format key-terms
	keyword = keywords
	keyword = getTokens(keyword)
	keyword = textFilter(keyword)
	keyword = stemTokens(keyword, stemmer)
	resetWeight(reviewArray)
	numReviews = len(reviewArray)

	#threading start
	t1 = threading.Thread(name = "t1", target=sendThreads,args=(reviewArray, numReviews, keyword, lenSet))
	t2 = threading.Thread(name = "t2", target=sendThreads,args=(reviewArray, numReviews, keyword, lenSet))
	t3 = threading.Thread(name = "t3", target=sendThreads,args=(reviewArray, numReviews, keyword, lenSet))
	t4 = threading.Thread(name = "t4", target=sendThreads,args=(reviewArray, numReviews, keyword, lenSet))

	t1.start()
	t2.start()
	t3.start()
	t4.start()

	t1.join()
	t2.join()
	t3.join()
	t4.join()

	results.sort(key = lambda x: x.revWeight)
	results.reverse() #reverse results, higher weighting first.

def sendThreads(reviewArray, numReviews, keyword, lenSet):
	threadGap = int(numReviews/4)

	if threading.current_thread().getName() == "t1":
		orderRev(reviewArray, keyword, lenSet, 0, threadGap * 1)

	elif threading.current_thread().getName() == "t2":
		orderRev(reviewArray, keyword, lenSet, (threadGap * 1 + 1), threadGap * 2)

	elif threading.current_thread().getName() == "t3":
		orderRev(reviewArray, keyword, lenSet, (threadGap * 2 + 1), threadGap * 3)

	elif threading.current_thread().getName() == "t4":
		orderRev(reviewArray, keyword, lenSet, (threadGap * 3 + 1), threadGap * 4)

def orderRev(reviewArray, keyword, lenSet, start, end):
	global results
	for rev in reviewArray[start:end]:
		review = rev.revBody
		#format review body
		review = getTokens(review)
		review = textFilter(review)
		review = stemTokens(review, stemmer)
		fdist = FreqDist(review) #frequency distribution of the review
		for word in keyword:
			if lenSet == "0": #user prefers concise reviews
				length = len(review)+1
				rev.revWeight = (rev.revWeight + (float(fdist[str(word)])) / length)
			else: #user prefer longer reviews
				rev.revWeight = rev.revWeight + (float(fdist[str(word)]))
		results.append(rev)

#clear rev.weight values
def resetWeight(reviews):
	for rev in reviews:
		rev.revWeight = 0;

#input: results, client identifier
#output: JSON file - list of reviews to send to client
def r2json(clientCode):
	global results
	if len(results) == 0:
		rev = Review(1,"Sorry, but no reviews were found for that query. Try widening your search critera.", 1)
		results.append(rev)
	lines = []
	simplejson = json
	f = open(clientCode+".txt","w")
	for rev in results:
		lines.append(rev.revBody)
	simplejson.dump(lines, f)
	f.close()
	results = []

def printHello():
	print("hello from " + str(print(threading.current_thread().getName())))

#main method
#1. interpret message settings
#2. create review array
#3. apply search c
#4. format to json s
def main(message, clientCode):

	args = message.split(",")
	keywords = args[0]

# filtering the search terms for more results.

	newKey = keywords.split()
	newKey2 = textFilter(newKey)
	newKey3 = " ".join(str(n) for n in newKey2)
	print("filtered input: " + newKey3)
	keywords = newKey3

	url = args[1]
	rating = args[2]
	type = args[3]
	lenSet = args[4]
	reviewArray = createRevArray(url, keywords, str(type), str(rating))

	results = searchC(keywords, reviewArray, lenSet)

	r2json(clientCode)


from __future__ import absolute_import, division, print_function
from scitbx import matrix
from scitbx.array_family import flex
from scitbx.math import tensor_rank_2_gradient_transform_matrix
from libtbx.test_utils import approx_equal
import math
from six.moves import cStringIO as StringIO
import sys
from six.moves import range

flex.set_random_seed(0)

mtps = -2 * math.pi**2

class debye_waller:

  def __init__(self, u, ops, hkl):
    self.u = u
    self.ops = ops
    self.hkl = hkl

  def f(self):
    result = 0
    for op in self.ops:
      op_u = (op*matrix.sym(sym_mat3=self.u)*op.transpose()).as_sym_mat3()
      huh = (matrix.row(self.hkl) \
          * matrix.sym(sym_mat3=op_u)).dot(matrix.col(self.hkl))
      result += math.exp(mtps * huh)
    return result

  def d_u(self):
    result = flex.double(6, 0)
    h,k,l = self.hkl
    d_exp_huh_d_u = matrix.col([h**2, k**2, l**2, 2*h*k, 2*h*l, 2*k*l])
    for op in self.ops:
      op_u = (op*matrix.sym(sym_mat3=self.u)*op.transpose()).as_sym_mat3()
      huh = (matrix.row(self.hkl) \
          * matrix.sym(sym_mat3=op_u)).dot(matrix.col(self.hkl))
      d_op_u = math.exp(mtps * huh) * mtps * d_exp_huh_d_u
      gtmx = tensor_rank_2_gradient_transform_matrix(op)
      d_u = gtmx.matrix_multiply(flex.double(d_op_u))
      result += d_u
    return result

  def d2_u(self):
    result = flex.double(flex.grid(6,6), 0)
    h,k,l = self.hkl
    d_exp_huh_d_u = flex.double([h**2, k**2, l**2, 2*h*k, 2*h*l, 2*k*l])
    d2_exp_huh_d_uu = d_exp_huh_d_u.matrix_outer_product(d_exp_huh_d_u)
    for op in self.ops:
      op_u = (op*matrix.sym(sym_mat3=self.u)*op.transpose()).as_sym_mat3()
      huh = (matrix.row(self.hkl) \
          * matrix.sym(sym_mat3=op_u)).dot(matrix.col(self.hkl))
      d2_op_u = math.exp(mtps * huh) * mtps**2 * d2_exp_huh_d_uu
      gtmx = tensor_rank_2_gradient_transform_matrix(op)
      d2_u = gtmx.matrix_multiply(d2_op_u).matrix_multiply(
        gtmx.matrix_transpose())
      result += d2_u
    return result

def d_debye_waller_d_u_finite(u, ops, hkl, eps=1.e-8):
  result = flex.double()
  u_eps = list(u)
  for ip in range(6):
    vs = []
    for signed_eps in [eps, -eps]:
      u_eps[ip] = u[ip] + signed_eps
      a = debye_waller(u=matrix.col(u_eps), ops=ops, hkl=hkl)
      vs.append(a.f())
    u_eps[ip] = u[ip]
    result.append((vs[0]-vs[1])/(2*eps))
  return result

def d2_debye_waller_d_u_finite(u, ops, hkl, eps=1.e-8):
  result = flex.double()
  u_eps = list(u)
  for ip in range(6):
    vs = []
    for signed_eps in [eps, -eps]:
      u_eps[ip] = u[ip] + signed_eps
      a = debye_waller(u=matrix.col(u_eps), ops=ops, hkl=hkl)
      vs.append(a.d_u())
    u_eps[ip] = u[ip]
    result.extend((vs[0]-vs[1])/(2*eps))
  return result

def compare_derivatives(ana, fin):
  s = max(1, flex.max(flex.abs(ana)))
  assert approx_equal(ana/s, fin/s)

def exercise(args):
  verbose =  "--verbose" in args
  if (not verbose):
    out = StringIO()
  else:
    out = sys.stdout
  for i_trial in range(100):
    ops = []
    for i in range(3):
      ops.append(matrix.sqr(flex.random_double(size=9, factor=4)-2))
    u = matrix.col((flex.random_double(size=6, factor=2)-1)*1.e-3)
    hkl = matrix.row(flex.random_double(size=3, factor=4)-2)
    dw = debye_waller(u=u, ops=ops, hkl=hkl)
    grads_fin = d_debye_waller_d_u_finite(u=u, ops=ops, hkl=hkl)
    print("grads_fin:", list(grads_fin), file=out)
    grads_ana = dw.d_u()
    print("grads_ana:", list(grads_ana), file=out)
    compare_derivatives(grads_ana, grads_fin)
    curvs_fin = d2_debye_waller_d_u_finite(u=u, ops=ops, hkl=hkl)
    print("curvs_fin:", list(curvs_fin), file=out)
    curvs_ana = dw.d2_u()
    print("curvs_ana:", list(curvs_ana), file=out)
    compare_derivatives(curvs_ana, curvs_fin)
    print(file=out)
  print("OK")

if (__name__ == "__main__"):
  exercise(sys.argv[1:])


from urllib.parse import urlencode

from book.filters import BookFilter
from book.forms import BookRentForm
from book.models import Book, BookRent, RentDayHistory

from decorators import lock_for_user

from django.contrib import messages
from django.contrib.auth.mixins import LoginRequiredMixin
from django.db.models import Case, CharField, Count, F, Sum, Value, When
from django.db.models.functions import Concat
from django.http import JsonResponse
from django.urls import reverse_lazy
from django.utils.decorators import method_decorator
from django.views.generic import CreateView, ListView, View

from django_filters.views import FilterView

__all__ = [
    'CreateBookRentView', 'SearchBook', 'BookRentTableBillingView',
    'BookRentTableView', 'BookList',
]

class CreateBookRentView(LoginRequiredMixin, CreateView):
    model = BookRent
    template_name = 'account/rent_create.html'
    form_class = BookRentForm
    success_url = reverse_lazy('account:profile_overall')

    @method_decorator(lock_for_user(methods=('POST',)))
    def dispatch(self, request, *args, **kwargs):
        return super().dispatch(request, *args, **kwargs)

    def get_form_kwargs(self):
        kwargs = super().get_form_kwargs()
        kwargs['request'] = self.request
        return kwargs

    def get_success_url(self):
        messages.add_message(self.request, messages.SUCCESS, 'Book Rent was created successfully.')
        return super().get_success_url()

class SearchBook(LoginRequiredMixin, View):
    def get(self, request):
        # wrap in list(), because QuerySet is not JSON serializable
        data = list(self.get_queryset())
        return JsonResponse(data, safe=False)

    def get_queryset(self):
        # generate human readable tariff and category name
        case = Case(
            When(
                category__days_period=0,
                then=Concat(
                    Value('Title: '),
                    F('title'),
                    Value(' $'),
                    F('category__price'),
                    Value(' each day. Category: '),
                    F('category__name'),
                )
            ),
            default=Concat(
                Value('Title: '),
                F('title'),
                Value(' $'),
                F('category__price_period'),
                Value(' for first '),
                F('category__days_period'),
                Value(' day/s. $'),
                F('category__price'),
                Value(' each day afterwards. Category: '),
                F('category__name'),
            ),
            output_field=CharField(),
        )
        values = ('id', 'title', 'price_display', 'category__name')
        limit = 10

        return Book.objects \
            .filter(title__search=self.search_term) \
            .annotate(price_display=case) \
            .values(*values)[:limit]

    @property
    def search_term(self):
        return self.request.GET.get('search') or ''

class BookRentTableBillingView(LoginRequiredMixin, ListView):
    queryset = RentDayHistory.objects.all()
    template_name = 'account/rent_list.html'

    def get_queryset(self):
        queryset = super().get_queryset()
        values = (
            'rent_id', 'rent', 'rent__book__title',
            'rent__status', 'rent__end', 'rent__book__category__name'
        )
        # generate human readable tariff
        tariff = Case(
            When(
                rent__days_period_initial=0,
                then=Concat(
                    Value('$'),
                    F('rent__price'),
                    Value(' each day.'),
                )
            ),
            default=Concat(
                Value('$'),
                F('rent__price_period'),
                Value(' for first '),
                F('rent__days_period_initial'),
                Value(' day/s. $'),
                F('rent__price'),
                Value(' each day afterwards.'),
            ),
            output_field=CharField(),
        )
        return queryset\
            .filter(rent__user_id=self.request.user.id)\
            .values(*values)\
            .annotate(
                total_amount=Sum('amount'),
                count=Count('id'),
                tariff=tariff,
            )

class BookRentTableView(LoginRequiredMixin, ListView):
    queryset = BookRent.objects.all().select_related('book')
    template_name = 'account/rent_list_table.html'

    def get_queryset(self):
        queryset = super().get_queryset()
        # generate human readable tariff and category name
        tariff = Case(
            When(
                days_period_initial=0,
                then=Concat(
                    Value('$'),
                    F('price'),
                    Value(' each day.'),
                )
            ),
            default=Concat(
                Value('$'),
                F('price_period'),
                Value(' for first '),
                F('days_period_initial'),
                Value(' day/s. $'),
                F('price'),
                Value(' each day afterwards.'),
            ),
            output_field=CharField(),
        )
        return queryset.filter(user_id=self.request.user.id).annotate(tariff=tariff)

class BookList(FilterView):
    queryset = Book.objects.all().select_related('category').order_by('-id')
    template_name = 'account/book_list.html'
    paginate_by = 12
    filterset_class = BookFilter

    def get_context_data(self, *args, **kwargs):
        context = super().get_context_data(*args, **kwargs)
        get = dict(tuple(self.request.GET.items()))
        get.pop('page', None)
        context['request_GET'] = urlencode(get)
        return context


from prediction.model.series_model.lstm import LSTM
from prediction.model.series_model.bi_lstm_attention import BiLSTMAttention
import torch
from torch import optim
from tensorboardX import SummaryWriter
import pandas as pd
import numpy as np
import copy

# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

writer = SummaryWriter("prediction/resources/tensorboards")

def loss_function():
    return torch.nn.MSELoss()

def train(optimizer, model, lstm_data_loader, autoencoder):
    model.train()
    if autoencoder is not None:
        autoencoder.eval()
    train_loss = 0
    for batch_idx, (seq_in, target) in enumerate(lstm_data_loader):
        optimizer.zero_grad()

        seq_in = seq_in.to(device)
        target = target.to(device)

        if autoencoder is not None:
            h = autoencoder.encode(seq_in)
            y = model(h)
        else:
            y = model(seq_in)

        loss = loss_function()(y, target)

        loss.backward()
        optimizer.step()

        train_loss += loss.item()

    train_loss /= len(lstm_data_loader.dataset)
    return train_loss

    '''
    print('====> Epoch: {} Average loss: {:.4f}'.format(
        epoch, train_loss / len(lstm_data_loader.dataset)))

    # ================================================================== #
    #                        Tensorboard Logging                         #
    # ================================================================== #
    # ref: https://blog.csdn.net/qq_39575835/article/details/89160828
    # 1. Log scalar values (scalar summary)
    info = {'loss': train_loss / len(lstm_data_loader.dataset)}

    for tag, value in info.items():
        writer.add_scalars('data/scalar_group', info, epoch)

    # 2. Log values and gradients of the parameters (histogram summary)
    for tag, value in models.named_parameters():
        tag = tag.replace('.', '/')
        writer.add_histogram(tag, value.data.cpu().numpy(), epoch)
        writer.add_histogram(tag + '/grad', value.grad.data.cpu().numpy(), epoch)
    '''

def validate(epoch, model, lstm_data_loader, autoencoder):
    y_cpu = None
    target_cpu = None

    model.eval()
    if autoencoder is not None:
        autoencoder.eval()
    vad_loss = 0
    with torch.no_grad():
        for batch_idx, (seq_in, target) in enumerate(lstm_data_loader):
            seq_in = seq_in.to(device)
            target = target.to(device)

            if autoencoder is not None:
                h = autoencoder.encode(seq_in)
                y = model(h)
            else:
                y = model(seq_in)

            loss = loss_function()(y, target)
            vad_loss += loss.item()

            y_cpu = y.cpu().detach().numpy()
            target_cpu = target.cpu().detach().numpy()

    vad_loss /= len(lstm_data_loader.dataset)

    # ================================================================== #
    #                       Plot Graph                                   #
    # ================================================================== #
    if epoch % 10 == 0:
        dates = pd.date_range('2020-01-01', periods=len(lstm_data_loader.dataset))
        results = np.zeros((len(lstm_data_loader.dataset), 2))
        for i in range(len(lstm_data_loader.dataset)):
            results[i, 0] = y_cpu[i, 0]
            results[i, 1] = target_cpu[i, 0]
        print(results)

    return vad_loss

def train_lstm(epochs, train_lstm_data_loader, vad_lstm_data_loader, state_dict=None, autoencoder=None):
    if autoencoder is not None:
        model = LSTM(input_dim=autoencoder.encode_dim, seq_len=train_lstm_data_loader.dataset.seq_length).to(device)
    else:
        model = LSTM(seq_len=train_lstm_data_loader.dataset.seq_length).to(device)

    if state_dict is not None:
        model.load_state_dict(state_dict)
        return model.eval()

    optimizer = optim.Adam(model.parameters(), lr=1e-3)

    best_loss = 10000.0
    best_ep = 0
    best_model_wts = copy.deepcopy(model.state_dict())

    for epoch in range(1, epochs + 1):
        train_loss = train(optimizer, model, train_lstm_data_loader, autoencoder)
        val_loss = validate(epoch, model, vad_lstm_data_loader, autoencoder)
        if val_loss < best_loss:
            best_loss = val_loss
            best_ep = epoch
            best_model_wts = copy.deepcopy(model.state_dict())
        print(f'Epoch {epoch}: train loss {train_loss} val loss {val_loss}')
        print("Best val Loss:", best_loss, "in epoch:", best_ep)
        if epoch - best_ep > 150:
            print('early stop')
            break

    model.load_state_dict(best_model_wts)
    return model.eval()

def test_lstm(model, lstm_data_loader, autoencoder):
    y_cpu = None

    model.eval()
    if autoencoder is not None:
        autoencoder.eval()
    test_loss = 0
    with torch.no_grad():
        for batch_idx, (seq_in, target) in enumerate(lstm_data_loader):
            seq_in = seq_in.to(device)
            target = target.to(device)

            if autoencoder is not None:
                h = autoencoder.encode(seq_in)
                y = model(h)
            else:
                y = model(seq_in)

            loss = loss_function()(y, target)
            test_loss += loss.item()

            y_cpu = y.cpu().detach().view(-1).numpy()

    test_loss /= len(lstm_data_loader.dataset)
    return test_loss, pd.DataFrame(y_cpu, columns=['Predicted_Close'])


"""
Project

"""

import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
import plotly.express as px
import pandas as pd

stylesheet = ['https://codepen.io/chriddyp/pen/bWLwgP.css']

# pandas dataframe to html table
def generate_table(dataframe):
    return html.Table([
        html.Thead(
            html.Tr([html.Th(col) for col in dataframe.columns])
        ),
        html.Tbody([
            html.Tr([
                html.Td(dataframe.iloc[i][col]) for col in dataframe.columns
            ]) for i in range (len(dataframe))
        ])
    ])

app = dash.Dash(__name__, external_stylesheets=stylesheet)

server = app.server
'''
df = pd.DataFrame({
    "Fruit": ["Apples", "Oranges", "Bananas", "Apples", "Oranges", "Bananas"],
    "Amount": [4, 1, 2, 2, 4, 5],
    "City": ["SF", "SF", "SF", "Montreal", "Montreal", "Montreal"]
})
'''
df = pd.read_csv('./movie-Info1.csv')
df2 = pd.read_csv('./rating.csv')

fig = px.bar(df, x="title", y="rating", color="imdbId", barmode="group")

app.layout = html.Div([

    html.H1('Choose a movie you like!', style={'textAlign': 'center'}),
    html.Div([html.H4('Rating of movie:'),
              dcc.Slider(id="ratings_slider", min=1, max=5, value=5,
              marks={i:str(i) for i in range(6)}),#slider marks
              html.H4('Select the genre you like:'),
              dcc.Dropdown(options=[{'label': 'Action', 'value': 'Action'},
                                     {'label': 'Comedy', 'value': 'Comedy'},
                                     {'label': 'Crime', 'value': 'Crime'},
                                     {'label': 'Documentary', 'value': 'Documentary'},
                                     {'label': 'Drama', 'value': 'Drama'},
                                     {'label': 'Horror', 'value': 'Horror'},
                                     {'label': 'Musical', 'value': 'Musical'},
                                     {'label': 'Romance', 'value': 'Romance'},
                                     {'label': 'Sci-Fi', 'value': 'Sci-Fi'},
                                     {'label': 'Thriller', 'value': 'Thriller'},
                                     {'label': 'War', 'value': 'War'},
                                     {'label': 'Western', 'value': 'Western'}],
                           id="genres_select_checklist",
                           value=''),
               html.Div(html.Div(id="df_div"))
                           ],
             style={'width': '49%', 'display': 'inline-block'}),

    html.Div([html.H4('The rating distribution of this genre:'),dcc.Graph(id='df_graph')],
             style={'width': '49%', 'display': 'inline-block', 'float': 'right'})
    ])

# Update the table
@app.callback(
    Output(component_id='df_div', component_property='children'),
    [Input(component_id='ratings_slider', component_property='value'),
     Input(component_id='genres_select_checklist', component_property='value'),]
)
def update_table(select_rating, genres_select):
    ratex = df[df.rating==select_rating]
    finalx =  ratex[ratex.genres == genres_select]
    return generate_table(finalx)

# Update the bar chart
@app.callback(
    Output("df_graph", "figure"),
    [Input(component_id='genres_select_checklist', component_property='value')])
def update_bar_chart(genres_select):
    mask = df2[genres_select]
    fig = px.bar(mask, x=[1,2,3,4,5], y=genres_select,
                 barmode="group")
    return fig

if __name__ == '__main__':
    app.run_server(debug=True)


import cv2
import numpy as np

# Create a video capture object for the default camera (0)
cap = cv2.VideoCapture(0)

while True:
    # Read a frame from the webcam
    ret, frame = cap.read()

    # Convert the frame to grayscale
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # Apply Gaussian blur to reduce noise
    gray_blurred = cv2.GaussianBlur(gray, (9, 9), 2)

    # Detect circles using the Hough Circle Transform
    circles = cv2.HoughCircles(
        gray_blurred, cv2.HOUGH_GRADIENT, dp=1, minDist=20, param1=250, param2=30, minRadius=0, maxRadius=0
    )

    if circles is not None:
        circles = np.uint16(np.around(circles))
        for circle in circles[0, :]:
            center = (circle[0], circle[1])
            radius = circle[2]
            # Draw the circle on the original frame
            cv2.circle(frame, center, radius, (0, 255, 0), 2)

    # Display the frame with detected circles
    cv2.imshow("Circle Detection", frame)

    # Exit the loop if the 'Esc' key is pressed
    if cv2.waitKey(1) & 0xFF == 27:
        break

# Release the video capture and close OpenCV windows
cap.release()
cv2.destroyAllWindows()


"""Some Signals & Systems Helping Routines."""
import matplotlib.pyplot as plt
from matplotlib.ticker import MaxNLocator
from matplotlib.patches import Circle, Rectangle
import numpy as np
from scipy import signal
from scipy.special import diric

def plot_zplane(z, p, k):
    """Plot pole/zero/gain plot of discrete-time, linear-time-invariant system.

    Note that the for-loop handling might be not very efficient
    for very long FIRs

    z...array of zeros in z-plane
    p...array of poles in z-zplane
    k...gain factor

    """
    # draw unit circle
    Nf = 2**7
    Om = np.arange(Nf) * 2*np.pi/Nf
    plt.plot(np.cos(Om), np.sin(Om), 'C7')

    rect_box = Rectangle((-3, -3), 6, 6, color='yellowgreen', alpha=0.33)
    plt.gcf().gca().add_artist(rect_box)
    try:  # TBD: check if this pole is compensated by a zero
        circle = Circle((0, 0), radius=np.max(np.abs(p)),
                        color='white', alpha=1)
        plt.gcf().gca().add_artist(circle)
    except ValueError:
        print('no pole at all, ROC is whole z-plane')

    zu, zc = np.unique(z, return_counts=True)  # find and count unique zeros
    for zui, zci in zip(zu, zc):  # plot them individually
        plt.plot(np.real(zui), np.imag(zui), ms=7,
                 color='C0', marker='o', fillstyle='none')
        if zci > 1:  # if multiple zeros exist then indicate the count
            plt.text(np.real(zui), np.imag(zui), zci)

    pu, pc = np.unique(p, return_counts=True)  # find and count unique poles
    for pui, pci in zip(pu, pc):  # plot them individually
        plt.plot(np.real(pui), np.imag(pui), ms=7,
                 color='C3', marker='x')
        if pci > 1:  # if multiple poles exist then indicate the count
            plt.text(np.real(pui), np.imag(pui), pci)

    plt.text(0, +1, 'k=%f' % k)
    plt.text(0, -1, 'ROC for causal: green')
    plt.axis('square')
    # plt.axis([-2, 2, -2, 2])
    plt.xlabel(r'$\Re\{z\}$')
    plt.ylabel(r'$\Im\{z\}$')
    plt.grid(True)

def interpolate_dft2dtft(X, W):
    """DFT to DTFT interpolation.

    This is the reconstruction filter in frequency domain to get
    a finite length time sequence starting from k=0 out of a N-periodic
    sequence

    X...array containing DFT spectrum
    W...array with normalized digital frequencies
    typically W = np.arange(Nint) * 2*np.pi/Nint with desired Nint

    see e.g.
    Rabiner, Gold, 1975, Theory and Application of Digital Signal Processing
    Prentice Hall, eq. (2.142)

    """
    N = np.size(X)  # we estimate the DFT length from the DFT spectrum
    tmp_2piN = 2*np.pi/N
    tmp_N2 = (N-1)/2
    Xint = np.zeros_like(W, dtype='complex')
    for cW, vW in enumerate(W):  # counter, value
        for cX, vX in enumerate(X):
            W_tmp = vW - tmp_2piN * cX
            Xint[cW] += vX * diric(W_tmp, N) * np.exp(-1j*W_tmp*tmp_N2)
    return Xint

def plot_dtlti_analysis(z, p, k, fs=1, Nf=2**10, Nt=2**5):
    """Plot linear, time-invariant, discrete-time system.

    Impulse, step, frequency response (level/phase/group delay) and z-plane
    of transfer function H(z) given as zeros/poles/gain description

    Note that we use fs only for the frequency responses

    """
    # still hard coded, TBD:
    # figure size
    # group_delay discontinuities and automatic ylim, yticks is not
    # useful sometimes

    plt.figure(figsize=(9, 9), tight_layout=True)

    mu = np.arange(Nf)
    df = fs/Nf
    dW = 2*np.pi/Nf
    f = mu*df  # frequency vector [0...fs)
    W = mu*dW  # digital angular frequency [0...2pi)

    sys = signal.dlti(z, p, k, dt=True)
    sys_ba = signal.TransferFunction(sys)
    b = sys_ba.num   # we need coeff b,a for group_delay()
    a = sys_ba.den

    [W, H] = signal.dlti.freqresp(sys, W)
    [W, gd] = signal.group_delay((b, a), w=W)  # gd in samples
    h = signal.dimpulse(sys, n=Nt)
    he = signal.dstep(sys, n=Nt)

    # plot frequency response: level
    ax1 = plt.subplot(3, 2, 1)
    ax1t = ax1.twiny()

    ax1.grid(True, color='lavender', which='major')
    # plotted below grid :-(
    # ax1t.grid(True, color='mistyrose', which='minor')

    ax1.plot(W/(2*np.pi), 20*np.log10(np.abs(H)), color='C0', lw=2)
    ax1.set_xscale('linear')
    ax1.set_xlim([0, 1])
    ax1.tick_params(axis='x', labelcolor='C0')
    ax1.set_xlabel(r'$\frac{\Omega}{2\pi} = \frac{f}{f_s}$', color='C0')
    ax1.set_xticks(np.arange(11)/10)
    ax1.set_ylabel('level in dB   or   20 lg|H| / dB', color='k')
    ax1.set_axisbelow(True)

    ax1t.plot(f, 20*np.log10(np.abs(H)), color='C3', lw=2)
    ax1t.set_xscale('log')
    ax1t.set_xlim([f[1], f[-1]])
    ax1t.tick_params(axis='x', labelcolor='C3')
    ax1t.set_xlabel('frequency in Hz   or   f / Hz for $f_s$ = ' +
                    '{0:5.2f}'.format(fs)+' Hz', color='C3')
    ax1t.set_axisbelow(True)

    # plot impulse response
    ax2 = plt.subplot(3, 2, 2)
    ax2.stem(np.squeeze(h[0]), np.squeeze(h[1]),
             use_line_collection=True,
             linefmt='C0:', markerfmt='C0o', basefmt='C0:')
    ax2.xaxis.set_major_locator(MaxNLocator(integer=True))
    ax2.grid(True)
    ax2.set_xlabel(r'$k$')
    ax2.set_ylabel(r'impulse response $h[k]$')

    # plot frequency response: phase
    ax3 = plt.subplot(3, 2, 3)
    ax3.plot(W, np.unwrap(np.angle(H)))
    ax3.set_xlabel(
        r'digital angular frequency in radian   or   $\Omega$ / rad')
    ax3.set_ylabel(r'phase in radian   or $\angle$ H / rad')
    ax3.set_xlim([0, 2*np.pi])
    ax3.set_xticks(np.arange(9)*np.pi/4)
    ax3.set_xticklabels([r'0', r'$\pi/4$', r'$\pi/2$', r'$3/4\pi$',
                         r'$\pi$', r'$5/4\pi$', r'$3/2\pi$', r'$7/4\pi$',
                         r'$2\pi$'])
    # ax3.set_ylim([-180, +180])
    # ax3.set_yticks(np.arange(-180, +180+45, 45))
    ax3.grid(True)

    # plot step response
    ax4 = plt.subplot(3, 2, 4)
    ax4.stem(np.squeeze(he[0]), np.squeeze(he[1]), use_line_collection=True,
             linefmt='C0:', markerfmt='C0o', basefmt='C0:')
    ax4.xaxis.set_major_locator(MaxNLocator(integer=True))
    ax4.grid(True)
    ax4.set_xlabel(r'$k$')
    ax4.set_ylabel(r'step response $h_\epsilon[k]$')

    # plot frequency response: group delay
    ax5 = plt.subplot(3, 2, 5)
    ax5.plot(W/np.pi, gd/fs)
    ax5.set_xscale('linear')
    ax5.set_xlim([0, 2])
    ax5.set_xlabel(r'$\frac{\Omega}{\pi}$')
    ax5.set_ylabel(r'group delay in seconds   or   $\tau_\mathrm{GD}$ / s')
    ax5.grid(True, which='both')

    # zplane
    # ax6 =
    plt.subplot(3, 2, 6)
    plot_zplane(sys.zeros, sys.poles, sys.gain)  # see function above

    # plt.tight_layout(True)

def plot_splane(z, p, k):
    """Plot pole/zero/gain plot of continuous-time, linear-time-invariant system.

    Note that the for-loop handling might be not very efficient
    for very long FIRs

    z...numpy array of zeros in s-plane
    p...numpy array of poles in s-zplane
    k...gain factor

    """
    z = np.array(z)
    p = np.array(p)
    i = 0
    j = 0
    while i < z.size:
        while j < p.size:
            if z[i] == p[j]:
                z = np.delete(z, i)
                p = np.delete(p, j)
                i = i - 1
                break
            else:
                j = j + 1
        j = 0
        i = i + 1
    try:
        xmax = np.amax(np.real(z))
        xmin = np.amin(np.real(z))
        ymax = np.amax(np.imag(z))
        ymin = np.amin(np.imag(z))
    except ValueError:  # z is empty
        xmax = ymax = 2
        xmin = ymin = -2
    try:
        pp = np.amax(np.real(p))
        xmax = max(xmax, pp)
        xmin = min(xmin, np.amin(np.real(p)))
        ymax = max(ymax, np.amax(np.imag(p)))
        ymin = min(ymin, np.amin(np.imag(p)))
    except ValueError:  # p is empty
        pp = 1j

    zu, zc = np.unique(z, return_counts=True)  # find and count unique zeros
    for zui, zci in zip(zu, zc):  # plot them individually
        plt.plot(np.real(zui), np.imag(zui), ms=7,
                 color='C0', marker='o', fillstyle='none')
        if zci > 1:  # if multiple zeros exist then indicate the count
            plt.text(np.real(zui), np.imag(zui), zci)

    pu, pc = np.unique(p, return_counts=True)  # find and count unique poles
    for pui, pci in zip(pu, pc):  # plot them individually
        plt.plot(np.real(pui), np.imag(pui), ms=7,
                 color='C3', marker='x')
        if pci > 1:  # if multiple poles exist then indicate the count
            plt.text(np.real(pui), np.imag(pui), pci)
    eps = 2
    if np.abs(xmax - xmin) < 2:
        xmax = xmin + np.abs(xmax-xmin)/2+eps
        xmin = xmax - 2*eps
    else:
        a = np.abs(xmax-xmin)/2
        xmin = xmin - a
        xmax = xmax + a
    if np.abs(ymax-ymin) < 2:
        ymax = ymin + np.abs(ymax-ymin)/2 + eps
        ymin = ymax - 2*eps
    else:
        a = np.abs(ymax - ymin)/2
        ymax = ymax + a
        ymin = ymin - a
    plt.axis([xmin, xmax, ymin, ymax])
    if np.imag(pp) == 0:
        plt.plot([pp, pp], [2*ymin, 2*ymax], color='brown')
        plt.fill_between([pp, 2*xmax], [2*ymax, 2*ymax],
                         2*ymin, color='yellowgreen', alpha=0.33, label='ROC')
    else:
        plt.fill_between([-2*xmax, 2*xmax], [2*ymax, 2*ymax],
                         2*ymin, color='yellowgreen', alpha=0.33, label='ROC')
    plt.xlabel(r'$\Re\{s\}$')
    plt.ylabel(r'$\Im\{s\}$')
    plt.legend()
    plt.text(xmin + np.abs(xmin) / 5, ymin + np.abs(ymin) / 5, 'k=%f' % k)
    plt.grid(True)

def group_delay(z, p, w):
    """
    group delay of a system function

    Parameters
    -----------
        z:  array, zeros in s-plane
        p:  array, poles in s-plane
        w:  array, angular frequencies

    Returns
    -----------
        w:  array, angular frequencies
        gd: float, group delay
    """
    if z.size == 0 and p.size == 0:
        return w, np.zeros(w.size)
    gd = 0
    for i in z:
        gd += np.real(i)/((w-np.imag(i))**2+np.real(i)**2)
    for i in p:
        gd -= np.real(i)/((w-np.imag(i))**2+np.real(i)**2)
    return w, gd

def plot_clti_analysis(z, p, k,):
    """Plot linear, time-invariant, discrete-time system.

    Parameter
    ------------
        z:  array, zeros of system function
        p:  array, poles of system function
        k:  float, gain factor
    Return
    ------------
        impulse response plot
        step response plot
        frequency response (level/phase/group delay)
        s-plane of transfer function H(s) given as zeros/poles/gain desciption
    """
    z = np.array(z)
    p = np.array(p)
    # still hard coded, TBD:
    # figure size

    plt.figure(figsize=(9, 9), tight_layout=True)

    sys = signal.lti(z, p, k)
    # sys_ba = signal.TransferFunction(sys)
    # b = sys_ba.num   # we needed coeff b,a for group_delay()
    # a = sys_ba.den

    [W, H] = signal.lti.freqresp(sys)
    F = W/(2*np.pi)
    gd = group_delay(z=z, p=p, w=W)[1]  # gd in samples using poles / zeros
    h = signal.impulse(sys)
    he = signal.step(sys)

    # plot frequency response: level
    ax1 = plt.subplot(3, 2, 1)
    ax1t = ax1.twiny()

    ax1.grid(True, color='lavender', which='major')

    ax1.plot(W/(2*np.pi), 20*np.log10(np.abs(H)), color='C0', lw=2)
    ax1.set_xscale('linear')
    ax1.set_xlim([F[0], F[F.size-1]])
    ax1.tick_params(axis='x', labelcolor='C0')
    ax1.set_xlabel(r'$f$ $/$ Hz', color='C0')
    ax1.set_ylabel('level in dB   or   20 lg|H| / dB', color='k')
    ax1.set_axisbelow(True)

    ax1t.plot(W/(2*np.pi), 20*np.log10(np.abs(H)), color='C3', lw=2)
    ax1t.set_xscale('log')
    ax1t.set_xlim([F[0], F[F.size-1]])
    ax1t.tick_params(axis='x', labelcolor='C3')
    ax1t.set_xlabel('$f$ $/$ Hz', color='C3')
    ax1t.set_axisbelow(True)

    # plot impulse response
    ax2 = plt.subplot(3, 2, 2)
    ax2.plot(np.squeeze(h[0]), np.squeeze(h[1]))
    ax2.xaxis.set_major_locator(MaxNLocator(integer=True))
    ax2.grid(True)
    ax2.set_xlabel(r'$t$ $/$ $s$')
    ax2.set_ylabel(r'impulse response $h(t)$')

    # plot frequency response: phase
    ax3 = plt.subplot(3, 2, 3)
    ax3.plot(W/(2*np.pi), np.unwrap(np.angle(H)))
    ax3.set_xlabel(
        r'$f$ $/$ Hz')
    ax3.set_ylabel(r'phase in radian   or $\angle$ H / rad')
    ax3.set_xlim([F[0], F[F.size-1]])
    ax3.grid(True)

    # plot step response
    ax4 = plt.subplot(3, 2, 4)
    ax4.plot(np.squeeze(he[0]), np.squeeze(he[1]))
    ax4.xaxis.set_major_locator(MaxNLocator(integer=True))
    ax4.grid(True)
    ax4.set_xlabel(r'$t$ $/$ $s$')
    ax4.set_ylabel(r'step response $h_\epsilon(t)$')

    # plot frequency response: group delay
    ax5 = plt.subplot(3, 2, 5)
    ax5.plot(W, gd)
    ax5.set_xscale('log')
    ax5.set_xlim([W[0], W[W.size-1]])
    ax5.set_xlabel(r'$\omega$ $/$ $\frac{\mathrm{rad}}{s}$')
    ax5.set_ylabel(r'group delay in seconds   or   $\tau_\mathrm{GD}$ / s')
    ax5.grid(True, which='both')

    # splane
    # ax6 =
    plt.subplot(3, 2, 6)
    plot_splane(sys.zeros, sys.poles, sys.gain)  # see function above

    # plt.tight_layout(True)
    return W, H


from .tokens import *

from os.path import abspath, dirname, join
from parso.grammar import PythonGrammar
from typing import Optional

import parso.python.tokenize

__all__ = ['tokenize_file', 'load_grammar']

DEFAULT_GRAMMAR_PATH = abspath(join(dirname(__file__), "../python-3.4.grammar"))

def tokenize_file(filename: str, grammar: Optional[PythonGrammar] = None,
                  suppress_error_tokens: bool = False) -> TokenGenerator:
    with open(filename) as f:
        lines = f.readlines()
    start_pos = (1, 0)

    if grammar is None:
        grammar = load_grammar()

    return tokens_from_py_tokens(grammar._tokenize_lines(lines, start_pos), suppress_error_tokens)

def load_grammar(path: Optional[str] = None, version: Optional[str] = None) -> PythonGrammar:
    if path is None:
        if version is None:
            return parso.load_grammar(path=DEFAULT_GRAMMAR_PATH)
        else:
            return parso.load_grammar(version=version)
    else:
        if version is None:
            return parso.load_grammar(path=path)
        else:
            raise ValueError("Cannot specify both path and version for loading grammar.")


import requests
import sys
import logging
from discord.ext import commands

APPROVED_URL = 'http://localhost:5000/approved'
SUGGESTED_URL = 'http://localhost:5000/suggested'

f = open("strife.conf", "r")
for line in f:
    if line[0] == "#":
        continue
    elif "commoners_roleid" in line:
        commoners_roleid = int(line.split("=")[-1:][0].strip())
    elif "owner_roleid" in line:
        owner_roleid = int(line.split("=")[-1:][0].strip())
    elif "admin_roleid" in line:
        admin_roleid = int(line.split("=")[-1:][0].strip())
    elif "seniorMod_roleid" in line:
        seniorMod_roleid = int(line.split("=")[-1:][0].strip())
    elif "moderator_roleid" in line:
        moderator_roleid = int(line.split("=")[-1:][0].strip())
    elif "clerk_roleid" in line:
        clerk_roleid = int(line.split("=")[-1:][0].strip())

    elif "cbbb" in line:
        cbbb = int(line.split("=")[-1:][0].strip())

# START CATEGORIES FUNCTIONS
#return questions for a specific category
async def category(category, approved):
    if approved:
        r = requests.get(APPROVED_URL)
    else:
        r = requests.get(SUGGESTED_URL)
    batch = []
    topics = "```"
    for topic in r.json():
        if topic['CATEGORY'] == category:
            t = topic['TOPIC'] + "\n"
            if len(t) + len(topics) > 1950:
                batch.append(topics + '```')
                topics = "```" + t
            else:
                topics += t
    batch.append(topics + '```')

    if batch[0] == "``````":
        if approved:
            batch[0] = "There are currently no approved category matching "  + category
        else:
            batch[0] = "There are currently no suggested category matching " + category
    return batch

#return the categories
async def categories(approved):
    if approved:
        r = requests.get(APPROVED_URL)
    else:
        r = requests.get(SUGGESTED_URL)
    cats  = []
    for topic in r.json():
        category = str(topic['CATEGORY'])
        if category not in cats:
            cats.append(category)
    catString = '\n'.join(cats)
    categories = "```\n" + catString + "```"
    if len(categories) > 1900:
        categories = categories[:1900]
        categories += "RESULTS TRUNCATED"
    if categories == "``````":
        if approved:
            topics = "There are currently no approved category "
        else:
            topics = "There are currently no suggested category "
    return categories

# END CATEGORIES FUNCTIONS

class CategoryCog(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

# START CATEGORIES COMMANDS

    #Returns all topics for a suggested category
    @commands.command(pass_context=True, name='suggestedcategory', aliases=['cs'])
    @commands.has_any_role(owner_roleid, admin_roleid, seniorMod_roleid, moderator_roleid, clerk_roleid, cbbb)
    async def suggestedcategory(self, ctx):
        """Returns all topics for a suggested category"""
        sys.stdout.write(f'{ctx.message.author} ran command "suggestedcategory"\n')
        sys.stdout.flush()
        logging.info(f'{ctx.message.author} ran command "suggestedcategory"')
        if ctx.message.content.lower().strip().startswith('suggestedcategory '):
            cat = ctx.message.content[len('suggestedcategory '):].strip()
        else:
            cat = ctx.message.content[len('cs '):].strip()
        topics = await category(cat, False)
        await ctx.send(topics)

    @suggestedcategory.error
    async def suggestedcategory_error(self, ctx, error):
        logging.error('Command "suggestedcategory" failed due to the following error:')
        logging.error(error)
        await ctx.send('Error processing that request')

    #__________________________________________________

    #Returns all suggested categories
    @commands.command(pass_context=True, name='suggestedcategories', aliases=['acs'])
    @commands.has_any_role(owner_roleid, admin_roleid, seniorMod_roleid, moderator_roleid, clerk_roleid, cbbb)
    async def suggestedcategories(self, ctx):
        """Returns all suggested categories"""
        sys.stdout.write(f'{ctx.message.author} ran command "suggestedcategories"\n')
        sys.stdout.flush()
        logging.info(f'{ctx.message.author} ran command "suggestedcategories"')
        cats = await categories(False)
        await ctx.send(cats)

    @suggestedcategories.error
    async def suggestedcategories_error(self, ctx, error):
        logging.error('Command "suggestedcategories" failed due to the following error:')
        logging.error(error)
        await ctx.send('Error processing that request')

    #__________________________________________________

    #Returns all categories
    @commands.command(pass_context=True, name='categories', aliases=['ac'])
    @commands.has_any_role(owner_roleid, admin_roleid, seniorMod_roleid, moderator_roleid, cbbb)
    async def categories(self, ctx):
        """Returns all categories"""
        sys.stdout.write(f'{ctx.message.author} ran command "categories"\n')
        sys.stdout.flush()
        logging.info(f'{ctx.message.author} ran command "categories"')
        cats = await categories(True)
        await ctx.send(cats)

    @categories.error
    async def categories_error(self, ctx, error):
        logging.error('Command "categories" failed due to the following error:')
        logging.error(error)
        await ctx.send('Error processing that request')

    #__________________________________________________

    #Returns all topics for a category
    @commands.command(pass_context=True, name='category', aliases=['c'])
    @commands.has_any_role(owner_roleid, admin_roleid, seniorMod_roleid, moderator_roleid, cbbb)
    async def category(self, ctx):
        """Returns all topics for a category"""
        sys.stdout.write(f'{ctx.message.author} ran command "category"\n')
        sys.stdout.flush()
        logging.info(f'{ctx.message.author} ran command "category"')
        if ctx.message.content.lower().strip().startswith('category '):
            cat = ctx.message.content[len('category '):].strip()
        else:
            cat = ctx.message.content[len('c '):].strip()
        topics = await category(cat, True)
        for msg in topics:
            await ctx.send(msg)

    @category.error
    async def category_error(self, ctx, error):
        logging.error('Command "category" failed due to the following error:')
        logging.error(error)
        await ctx.send('Error processing that request')

    #__________________________________________________

# END CATEGORIES COMMANDS

def setup(bot):
    bot.add_cog(CategoryCog(bot))