import token_state
import token as t
import attributes
import re

class Tokeniser(object):

    def __init__(self, reader, errors=None):
        self.reader = reader
        self.errors = errors or []
        self.state = token_state.DATA
        self.tag_pending = None
        self.comment_pending = t.CommentToken()
        self.doctype_pending = t.DoctypeToken()
        self.char_pending = t.CharacterToken()
        self.script_buffer = ''
        self.last_start_tag = None
        self.is_emit_pending = False
        self.emit_pending = None
        self.self_closing_acknow = True
        self.chars_string = ''
        self.code_point_holder = []
        self.multi_point_holder = []

    def tokenise(self):
        tokens = ""
        while True:
            curr_token = self.read()
            normalized = re.sub('\s+|\t+|\n+|\f+|[ ]+', "", curr_token.__str__())
            tokens = "{0}\nToken: {1}".format(tokens, curr_token) if normalized else tokens
            finished = False
            if curr_token.type == t.TokenType.EOF:
                finished = True
            if finished:
                break
        return tokens

    def read(self):
        self.self_closing_acknow = True
        while not self.is_emit_pending:
            self.state.read(self.reader, self)
        if self.chars_string:
            str = self.chars_string
            self.chars_string = ""
            self.char_pending.data = str
            return self.char_pending
        else:
            self.is_emit_pending = False
            return self.emit_pending

    def move_to_state(self, state):
        self.reader.advance()
        self.state = state

    def move(self, state):
        self.state = state

    def emit(self, consumed):
        self.chars_string += consumed

    def emit_token(self, token):
        # validate
        self.emit_pending = token
        self.is_emit_pending = True

        if token.type == t.TokenType.START_TAG:
            self.last_start_tag = token.tag_name
            if token.is_self_closing:
               self.self_closing_acknow = False
        elif token.type == t.TokenType.END_TAG:
            if len(token.attrs.attrs):
                self.error("No attributes allowed on end tag!")

    def emit_string(self, str):
        self.chars_string += str

    def error(self, state):
        print("error in state: {0}".format(state))
        self.errors.append(state)

    def eof_error(self, token):
        self.errors.append("EOF error: " + token)

    def emit_tag_pending(self):
        self.tag_pending.finalise_tag()
        self.emit_token(self.tag_pending)

    def emit_comment_pending(self):
        self.emit_token(self.comment_pending)

    def emit_doctype_pending(self):
        self.emit_token(self.doctype_pending)

    def consume_char_ref(self, additional_char, in_attr):
        if self.reader.empty():  # stopped here TODO
            return None
        if additional_char and additional_char == self.reader.current():
            return None
        if self.reader.current() in ['\t', '\n', '\r', '\f', ' ', '<', '&']:
            return None
        self.reader.mark = self.reader.current_pos
        code_ref = []
        if self.reader.match_seq_consume("#"):
            hex = self.reader.match_seq_ic("X")
            num_ref = self.reader.consume_hex() if hex else self.reader.consume_dig()
            if not num_ref:
                self.reader.current_pos = self.reader.mark
                return None
            if not self.reader.match_seq_consume(";"):
                self.errors.append("Missing semicolon")
            charval = -1
            try:
                base = 16 if hex else 10
                charval = int(num_ref, base)
            except:
                pass
            if charval == -1 or (
                    charval >= 0xD800 and charval <= 0xDFFF) or charval > 0x10FFFF:
                self.errors.append("character outside of valid range")
                code_ref[0] = token_state.REPLACE_CHAR
                return code_ref
            else:
                code_ref[0] = charval
                return code_ref
        else:
            name_ref = self.reader.consume_letter_and_digit_seq()
            legit = self.reader.match(';')
            found = True
            if not found:
                self.reader.current_pos = self.reader.mark
                if legit:
                    self.errors.append("invalid named referenece {0}".format(name_ref))
                return None
            if in_attr and (
                self.reader.match_letter() or self.reader.match_digit() or self.reader.match_any_of('=', '-', '_')):
                self.reader.current_pos = self.reader.mark
                return None
            if not self.reader.match_seq_consume(";"):
                self.errors.append("Missing semicolon")
            num_chars = 1
            if num_chars == 1:
                code_ref[0] = self.multi_point_holder[0]
                return code_ref
            elif num_chars == 2:
                return self.multi_point_holder
            else:
                return self.multi_point_holder

    def create_tag_pending(self, start=False):
        self.tag_pending = t.StartTagToken("", attributes.Attributes()) if start else t.EndTagToken()
        return self.tag_pending

    def create_comment_pending(self):
        self.comment_pending = t.CommentToken()

    def create_doctype_pending(self):
        self.doctype_pending = t.DoctypeToken()

    def approptiate_end_tag_token(self):
        return self.last_start_tag is not None and self.tag_pending.token_name() == self.last_start_tag.lower()

    def approptiate_end_tag_name(self):
        return self.last_start_tag

    def unescape_entities(self, in_attr):
        str = ''
        while not self.reader.empty():
            str += self.reader.consume_to("&")
            if self.reader.match("&"):
                self.reader.consume()
                c = self.consume_char_ref(None, in_attr)
                if not c:
                    str += "&"
                else:
                    str += c[0]
                    if len(c) == 2:
                        str += c[1]
        return str


from PyQt5 import QtWidgets, QtCore
import sys  # sys нужен для передачи argv в QApplication
import os  # Отсюда нам понадобятся методы для отображения содержимого директорий
import Error
import numpy as np
import math
import popara_wind
import matplotlib as mpl
import matplotlib.pyplot as plt
import pandas as pd
import fin
import matplotlib.dates as mdates
import datetime as dt
import csv

class AlignDelegate(QtWidgets.QStyledItemDelegate):
    def initStyleOption(self, option, index):
        super(AlignDelegate, self).initStyleOption(option, index)
        option.displayAlignment = QtCore.Qt.AlignCenter

class Poparno(QtWidgets.QMainWindow, popara_wind.Ui_MainWindow):
    def __init__(self, mass_kriter, mass_alter, i, flag, DATA):
        # Это здесь нужно для доступа к переменным, методам
        # и т.д. в файле design.py
        super().__init__()
        self.setupUi(self)  # Это нужно для инициализации нашего дизайна
        # self.tableWidget.setHorizontalHeaderLabels(mass_alter)
        # self.tableWidget.setVerticalHeaderLabels(mass_alter)
        if i == -1:
            self.label.setText('')
            mass_alter, mass_kriter = mass_kriter, mass_alter
            DATA = []
        else:
            self.label.setText(mass_kriter[i])
        LEN = len(mass_alter)
        if LEN < 10:
            self.setGeometry(0, 0, LEN * 150 + 300, LEN * 70 + 60)
        else:
            self.setGeometry(0, 0, 9 * 180 + 300, 9 * 100 + 300)

        self.btn_next.clicked.connect(
            lambda: self.dalee(mass_kriter, mass_alter, i, flag, DATA))  # Выполнить функцию при нажатии кнопки
        self.btn_graf.setEnabled(False)
        self.btn_next.setEnabled(False)
        self.pushButton.setEnabled(False)
        self.tableWidget.setRowCount(LEN)
        self.tableWidget.setColumnCount(LEN)
        self.btn_graf.clicked.connect(lambda: self.make_graf_pie(mass_alter))
        self.pushButton.clicked.connect(lambda: self.make_graf_stolb(mass_alter))
        delegate = AlignDelegate(self.tableWidget)
        self.tableWidget.setItemDelegate(delegate)
        stylesheet = "::section{Background-color:rgb(176,196,222)}"
        self.tableWidget.horizontalHeader().setStyleSheet(stylesheet)
        self.tableWidget.verticalHeader().setStyleSheet(stylesheet)
        if i == -1:
            TEXT = 'Критерии: \n'
            self.label.setText('Попарная оценка критериев')
            if flag == 1:
                data = self.check_data(-1)
                for j in range(LEN):
                    for q in range(LEN):
                        item = QtWidgets.QTableWidgetItem(data[j][q])
                        self.tableWidget.setItem(j, q, item)

            else:
                data = np.eye(LEN)
                for j in range(LEN):
                    for q in range(LEN):
                        item = QtWidgets.QTableWidgetItem(str(int(data[j][q])))
                        self.tableWidget.setItem(j, q, item)

        else:
            TEXT = 'Альтернативы: \n'
        for q in range(len(mass_alter)):
            TEXT += str(q + 1) + ': ' + mass_alter[q] + '\n'
        self.textEdit.setText(TEXT)
        if flag == 0:
            data = np.eye(LEN)
            for j in range(LEN):
                for q in range(LEN):
                    item = QtWidgets.QTableWidgetItem(str(int(data[j][q])))
                    self.tableWidget.setItem(j, q, item)
        else:
            data = self.check_data(i)
            for j in range(LEN):
                for q in range(LEN):
                    item = QtWidgets.QTableWidgetItem(data[j][q])
                    self.tableWidget.setItem(j, q, item)
        self.btn_chek.clicked.connect(lambda: self.CHEK(LEN))

    def dalee(self, mass_kriter, mass_alter, i, flag, DATA):
        # print(len(mass_alter))
        data = self.lineEdit.text().split(' ')[4:4 + len(mass_alter)]
        # print(data)
        new_data = []
        for r in data:
            r = r[:-1]
            new_data.append(float(r))
        data = dict(zip(mass_alter, new_data))
        DATA.append(new_data)
        if i == -1:
            sorted_dict = {}
            sorted_keys = sorted(data, key=data.get)
            for w in sorted_keys:
                sorted_dict[w] = data[w]
            # print(sorted_dict)
            temp_mass = []
            for j in sorted_dict.keys():
                temp_mass.append(j)
            mass_alter = temp_mass
            # print(mass_alter)
            mass_alter, mass_kriter = mass_kriter, mass_alter
        global wind
        i += 1
        if i < len(mass_kriter):
            wind = Poparno(mass_kriter, mass_alter, i, flag, DATA)
            wind.show()
            self.close()
        else:
            alter = []
            krit = DATA[0]
            krit.sort()
            krit = np.array(krit)
            for k in range(1, len(DATA)):
                alter.append(DATA[k])
            alter = np.array(alter).T
            rez = alter.dot(krit[np.newaxis, :].T)
            print(rez)
            print(mass_alter)
            global wind2
            wind2 = fin.FIN(rez, mass_alter)
            wind2.show()
            self.close()

    def CHEK(self, LEN):
        data = []
        try:
            for j in range(LEN):
                data1 = []
                for q in range(LEN):
                    if self.tableWidget.item(j, q).text() == '':
                        Error.ERROR(5)
                        return 0
                    if '/' in self.tableWidget.item(j, q).text():
                        temp2 = self.tableWidget.item(j, q).text().split('/')
                        temp = int(temp2[0]) / int(temp2[1])

                    else:
                        temp = int(self.tableWidget.item(j, q).text())
                    if int(temp) == 0:
                        temp = 1 / int(self.tableWidget.item(q, j).text())
                        item = QtWidgets.QTableWidgetItem('1/' + self.tableWidget.item(q, j).text())
                        self.tableWidget.setItem(j, q, item)
                    try:
                        if int(self.tableWidget.item(j, q).text()) < 0:
                            Error.ERROR(5)
                            return 0
                    except:
                        pass
                    data1.append(temp)
                data.append(data1)
            delegate = AlignDelegate(self.tableWidget)
            self.tableWidget.setItemDelegate(delegate)
        except:
            Error.ERROR(5)
            return 0
        vector_prior, flag2, OS, IS = self.magic(data)
        if not flag2:
            Error.ERROR(4)
            self.lineEdit.setText('; ИС = ' + str(IS) + '; ОС = ' + str(OS) + ' < 0.1 => Матрица НЕ согласована!')

        else:
            text = "Значения весов удельных критериев: "
            for i in vector_prior:
                text += str(i) + ', '
            text = text.strip(', ')

            text += '; ИС = ' + str(IS) + '; ОС = ' + str(OS) + ' < 0.1 => Матрица согласована!'
            self.lineEdit.setText(text)
            self.btn_next.setEnabled(True)
            self.btn_graf.setEnabled(True)
            self.pushButton.setEnabled(True)
            self.btn_chek.setEnabled(False)

    def magic(self, data):
        vector_prior = []
        for i in data:
            pop = 1
            for j in i:
                pop *= j
            pop = math.pow(pop, 1 / len(i))
            vector_prior.append(pop)
        # print(vector_prior)
        # print(sum(vector_prior))
        vector_prior2 = []
        for i in range(len(vector_prior)):
            vector_prior2.append(vector_prior[i] / sum(vector_prior))
        vector_prior = vector_prior2
        data = np.array(data)
        vector_prior = np.array(vector_prior)
        vector_prior2 = data.dot(vector_prior)
        # print(vector_prior2)
        vector_prior2 /= vector_prior
        # print(vector_prior2)
        l_max = sum(vector_prior2) / len(vector_prior2)
        # print(l_max)
        IS = (l_max - len(vector_prior2)) / (len(vector_prior2) - 1)
        # print('IS = ' + str(IS))
        midSI = self.check_SI(len(vector_prior2))
        OS = IS / midSI
        # print('OS = ' + str(OS))
        if str(OS) == 'nan':
            OS = 0
        if OS < 0.1:
            return vector_prior, True, OS, IS
        else:
            return vector_prior, False, OS, IS

    def check_SI(self, n):
        Sl_soglas = {1: 0, 2: 0, 3: 0.58, 4: 0.9, 5: 1.12, 6: 1.24, 7: 1.32, 8: 1.40, 9: 1.45, 10: 1.49, 11: 1.5,
                     12: 1.48, 13: 1.56, 14: 1.57, 15: 1.59}
        return Sl_soglas[n]

    def make_graf_pie(self, mass_alter):
        data = self.lineEdit.text().split(' ')[4:4 + len(mass_alter)]
        upd_data = []
        for i in data:
            i = i[:-1]
            upd_data.append(float(i))

        dpi = 80
        fig = plt.figure(dpi=dpi, figsize=(1536 / dpi, 768 / dpi))
        mpl.rcParams.update({'font.size': 13})

        plt.title('Удельные веса.')

        xs = range(len(mass_alter))
        plt.pie(
            upd_data, radius=1.1, autopct='%.1f')
        plt.legend(
            bbox_to_anchor=(0.15, 0.25), labels=mass_alter)
        fig.show()

    def make_graf_stolb(self, mass_alter):
        data = self.lineEdit.text().split(' ')[4:4 + len(mass_alter)]
        upd_data = []
        for i in data:
            i = i[:-1]
            upd_data.append(float(i))

        frequencies = upd_data
        freq_series = pd.Series(frequencies)
        y_labels = range(1, len(mass_alter) + 1)
        # Plot the figure.
        plt.figure(figsize=(12, 8))
        ax = freq_series.plot(kind='barh')
        ax.set_title('Удельные веса')
        ax.set_xlim(0, max(upd_data) + 0.2)  # expand xlim to make labels easier to read
        ax.set_yticklabels(y_labels)

        rects = ax.patches

        # For each bar: Place a label
        for h, rect in enumerate(rects):
            # Get X and Y placement of label from rect.
            x_value = rect.get_width()
            y_value = rect.get_y() + rect.get_height() / 2

            # Number of points between bar and label. Change to your liking.
            space = 5
            # Vertical alignment for positive values
            ha = 'left'

            # If value of bar is negative: Place label left of bar
            if x_value < 0:
                # Invert space to place label to the left
                space *= -1
                # Horizontally align label at right
                ha = 'right'

            # Use X value as label and format number with one decimal place
            label = "{}".format(mass_alter[h])
            # Create annotation
            plt.annotate(
                label,  # Use `label` as label
                (x_value, y_value),  # Place label at end of the bar
                xytext=(space, 0),  # Horizontally shift label by `space`
                textcoords="offset points",  # Interpret `xytext` as offset in points
                va='center',  # Vertically center label
                ha=ha)  # Horizontally align label differently for
            # positive and negative values.
        plt.show()

    def check_data(self, i):
        if i == -1:
            data = np.array([['1', '0', '3', '0', '5', '5', '5', '7', '1'],
                             ['3', '1', '9', '1', '9', '9', '8', '9', '3'],
                             ['0', '0', '1', '0', '1', '2', '1', '2', '0'],
                             ['3', '1', '7', '1', '7', '9', '9', '8', '3'],
                             ['0', '0', '1', '0', '1', '1', '1', '2', '0'],
                             ['0', '0', '0', '0', '1', '1', '1', '2', '0'],
                             ['0', '0', '1', '0', '1', '1', '1', '2', '0'],
                             ['0', '0', '0', '0', '0', '0', '0', '1', '0'],
                             ['1', '0', '3', '0', '3', '5', '5', '7', '1']])
        elif i == 0:
            data = np.array([['1', '3', '1', '9', '3', '5'],
                             ['0', '1', '0', '6', '1', '3'],
                             ['1', '3', '1', '9', '3', '5'],
                             ['0', '0', '0', '1', '0', '0'],
                             ['0', '1', '0', '5', '1', '2'],
                             ['0', '0', '0', '3', '0', '1']])
        elif i == 1:
            data = np.array([['1', '0', '0', '0', '0', '0'],
                             ['5', '1', '1', '1', '1', '1'],
                             ['5', '1', '1', '1', '1', '1'],
                             ['5', '1', '1', '1', '1', '1'],
                             ['5', '1', '1', '1', '1', '1'],
                             ['5', '1', '1', '1', '1', '1']])
        elif i == 2:
            data = np.array([['1', '0', '3', '5', '7', '1'],
                             ['3', '1', '7', '8', '9', '3'],
                             ['0', '0', '1', '2', '5', '0'],
                             ['0', '0', '0', '1', '1', '0'],
                             ['0', '0', '0', '1', '1', '0'],
                             ['1', '0', '2', '5', '7', '1']])
        elif i == 3:
            data = np.array([['1', '7', '0', '3', '2', '3'],
                             ['0', '1', '0', '0', '0', '0'],
                             ['3', '9', '1', '7', '3', '9'],
                             ['0', '3', '0', '1', '0', '1'],
                             ['0', '4', '0', '2', '1', '2'],
                             ['0', '2', '0', '1', '0', '1']])
        elif i == 4:
            data = np.array([['1', '0', '0', '0', '2', '0'],
                             ['2', '1', '0', '0', '3', '0'],
                             ['3', '2', '1', '0', '5', '0'],
                             ['7', '3', '5', '1', '9', '3'],
                             ['0', '0', '0', '0', '1', '0'],
                             ['6', '5', '2', '0', '7', '1']])
        elif i == 5:
            data = np.array([['1', '0', '0', '0', '1', '0'],
                             ['2', '1', '1', '0', '3', '0'],
                             ['2', '1', '1', '0', '2', '0'],
                             ['4', '2', '2', '1', '5', '1'],
                             ['1', '0', '0', '0', '1', '0'],
                             ['4', '2', '3', '1', '5', '1']])
        elif i == 6:
            data = np.array([['1', '0', '0', '0', '0', '0'],
                             ['2', '1', '0', '0', '0', '1'],
                             ['4', '2', '1', '0', '2', '2'],
                             ['5', '3', '2', '1', '2', '3'],
                             ['3', '2', '0', '0', '1', '1'],
                             ['2', '1', '0', '0', '1', '1']])
        elif i == 7:
            data = np.array([['1', '0', '0', '0', '1', '0'],
                             ['3', '1', '2', '0', '3', '0'],
                             ['2', '0', '1', '0', '2', '0'],
                             ['7', '3', '3', '1', '6', '2'],
                             ['1', '0', '0', '0', '1', '0'],
                             ['2', '2', '3', '0', '4', '1']])
        elif i == 8:
            data = np.array([['1', '0', '0', '0', '0', '0'],
                             ['2', '1', '2', '0', '0', '0'],
                             ['2', '0', '1', '0', '0', '0'],
                             ['4', '5', '5', '1', '3', '3'],
                             ['3', '3', '3', '0', '1', '1'],
                             ['3', '2', '4', '0', '1', '1']])
        else:
            data = np.eye(9)
        return data


#!/usr/bin/env python
# coding: utf-8
# reference: https://github.com/mitchelljy/Trending-YouTube-Scraper

import sys
import time
import requests
from google.cloud import storage
from pyspark.sql import SparkSession

snippet_features = ["title",
                    "publishedAt",
                    "channelId",
                    "channelTitle",
                    "categoryId"]
header = ["video_id"] + snippet_features + ["trending_date", "tags", "view_count", "likes", "dislikes",
                                            "comment_count", "thumbnail_link", "comments_disabled",
                                            "ratings_disabled"
                                            ]
unsafe_characters = ['\n', '"']
BUCKET = ''  # enter your bucket name
spark = SparkSession.builder.appName("CommentsScraper").getOrCreate()
api_key = ''  # enter your api key

def prepare_feature(feature):
    for ch in unsafe_characters:
        feature = str(feature).replace(ch, "")
    op = f'"{feature}"'
    return op

def api_request(page_token, country_code):
    global api_key
    request_url = f"https://www.googleapis.com/youtube/v3/videos?part=id,statistics,snippet{page_token}chart=mostPopular&regionCode={country_code}&maxResults=50&key={api_key}"
    request = requests.get(request_url)
    if request.status_code == 403:
        print("excess requests")
        sys.exit()
    return request.json()

def get_tags(tags_list):
    # Takes a list of tags, prepares each tag and joins them into a string by the pipe character
    return prepare_feature("|".join(tags_list))

def get_videos(items):
    lines = []
    for video in items:
        comments_disabled = False
        ratings_disabled = False

        if "statistics" not in video:
            continue

        video_id = prepare_feature(video['id'])

        snippet = video['snippet']
        statistics = video['statistics']

        features = [prepare_feature(snippet.get(feature, "")) for feature in snippet_features]

        thumbnail_link = snippet.get("thumbnails", dict()).get("default", dict()).get("url", "")
        trending_date = time.strftime("%y.%d.%m")
        tags = get_tags(snippet.get("tags", ["[none]"]))
        view_count = statistics.get("viewCount", 0)

        # This may be unclear, essentially the way the API works is that if a video has comments or ratings disabled
        # then it has no feature for it, thus if they don't exist in the statistics dict we know they are disabled
        if 'likeCount' in statistics and 'dislikeCount' in statistics:
            likes = statistics['likeCount']
            dislikes = statistics['dislikeCount']
        else:
            ratings_disabled = True
            likes = 0
            dislikes = 0

        if 'commentCount' in statistics:
            comment_count = statistics['commentCount']
        else:
            comments_disabled = True
            comment_count = 0

        line = [video_id] + features + [prepare_feature(x) for x in [trending_date, tags, view_count, likes, dislikes,
                                                                     comment_count, thumbnail_link, comments_disabled,
                                                                     ratings_disabled
                                                                     ]]
        lines.append(",".join(line))
    return lines

def get_pages(country_code, cursor="&"):
    country_data = []

    while cursor is not None:
        video_data_page = api_request(cursor, country_code)

        cursor = video_data_page.get("nextPageToken", None)
        if cursor is not None:
            cursor = f"&pageToken={cursor}&"

        items = video_data_page.get('items', [])
        country_data += get_videos(items)

    return country_data

def write_to_file(country_code, country_data):
    print(f"Writing {country_code} data to file...")

    client = storage.Client()
    bucket = client.get_bucket(BUCKET)
    target_blob = bucket.blob(f"videos/{time.strftime('%y_%d_%m')}_{country_code}_videos.csv")

    local_tmp_path = f"{country_code}_videos.csv"

    with open(local_tmp_path, "w+") as file:
        for row in country_data:
            file.write(f"{row}\n")

    target_blob.upload_from_filename(local_tmp_path)
    return

def scraper(country_code):
    country_data = [",".join(header)] + get_pages(country_code)
    write_to_file(country_code, country_data)
    return len(country_data)

scraper('US')  # only scraping the videos for US region


import requests
import json

def get_API(url, header = None):
    data = requests.get(url, headers= header)
    return json.loads(data.text)

def getCoinInfo(_id):
    data = {}
    base_url = f"https://api.coingecko.com/api/v3"
    path_url = f"/coins/{_id}"
    rsp = get_API(base_url + path_url)
    data["id"] = rsp["id"]
    data["symbol"] = rsp["symbol"]
    data["Website"] = rsp["links"]["homepage"] # Website
    data["twitter"] = "https://twitter.com/" + rsp["links"]["twitter_screen_name"] # Twitter
    data["Source Code"] = rsp["links"]["repos_url"]["github"] # Github
    return data

if __name__ == "__main__":
    id = "ethereum"
    ethereum = getCoinInfo(id)
    print()


from time import sleep

from opentracing.mocktracer import MockTracer
from tornado.ioloop import IOLoop
from opentracing.ext import tags
from threading import Thread
import requests
import tornado
import pytest
import six

from signalfx_tracing.libraries import tornado_config
from signalfx_tracing import instrument
from .app import MyApplication, HelloHandler

endpoint = "http://127.0.0.1:32321/hello/MyName"

def span_callback(span, request):
    span.set_tag("some_tag", True)

class TestTornadoApp(object):

    _app_store = [None]  # pytest doesn't persist attributes set in class-scoped autouse
    _tracer_store = [None]  # fixtures, so use stores as a hack.

    @property
    def app(self):
        return self._app_store[0]

    @property
    def tracer(self):
        return self._tracer_store[0]

    @classmethod
    @pytest.fixture(scope="class", autouse=True)
    def instrumented_app(cls):
        tracer = MockTracer()
        cls._tracer_store[0] = tracer

        tornado_config.start_span_cb = span_callback
        tornado_config.traced_attributes = ["path", "method", "query"]

        instrument(tracer, tornado=True)

        cls._app_store[0] = MyApplication([(r"/hello/(.*)", HelloHandler)])
        app_thread = Thread(target=cls.run_app)
        app_thread.daemon = True
        app_thread.start()

        for i in range(20):  # Wait for application to accept connections.
            try:
                requests.get(endpoint)
                break
            except Exception:
                if i == 19:
                    raise
                sleep(0.25)

        tracer.reset()

        yield

        def stop_ioloop():
            IOLoop.current().stop()

        IOLoop.current().add_callback(stop_ioloop)

    @classmethod
    def run_app(cls):
        if six.PY3 and tornado.version_info >= (5, 0):
            import asyncio

            asyncio.set_event_loop(asyncio.new_event_loop())

        cls._app_store[0].listen(32321)
        IOLoop.current().start()
        IOLoop.current().close()

    @pytest.fixture(autouse=True)
    def reset_tracer(self):
        yield
        self.tracer.reset()

    def test_app_settings(self):
        settings = self.app.settings
        assert settings["opentracing_tracing"].tracer is self.tracer
        assert (
            settings["opentracing_traced_attributes"]
            == tornado_config.traced_attributes
        )
        assert settings["opentracing_start_span_cb"] == tornado_config.start_span_cb
        assert settings["opentracing_trace_all"] == tornado_config.trace_all
        assert settings["opentracing_trace_client"] == tornado_config.trace_client

    @pytest.mark.parametrize(
        "http_method", ("get", "head", "post", "delete", "patch", "put", "options")
    )
    def test_traced_request(self, http_method):
        query = "one=1&two=2"
        method = getattr(requests, http_method)
        method("{}?{}".format(endpoint, query))

        for _ in range(20):  # Tornado's async nature makes this flakey
            if self.tracer.finished_spans():
                break
            sleep(0.25)

        spans = self.tracer.finished_spans()
        assert len(spans) == 1
        span = spans.pop()
        assert span.operation_name == "HelloHandler"
        tagged_method = http_method.upper()
        assert span.tags == {
            "component": "tornado",
            "http.url": "/hello/MyName?one=1&two=2",
            "http.method": tagged_method,
            "http.status_code": 200,
            "method": tagged_method,
            "path": "/hello/MyName",
            "query": query,
            "some_tag": True,
            tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER,
        }


#!/usr/bin/env python3

import time
import argparse
import animation as a

from neopixel import *
from threading import Thread

# LED strip configuration:
LED_COUNT      = 270    # Number of LED pixels.
LED_PIN_0      = 12      # GPIO pin connected to the pixels - CHANNEL 0
LED_PIN_1      = 13      # GPIO pin connected to the pixels - CHANNEL 1

LED_FREQ_HZ    = 800000  # LED signal frequency in hertz (usually 800khz)
LED_DMA        = 10      # DMA channel to use for generating signal (try 10)
LED_BRIGHTNESS = 100     # Set to 0 for darkest and 255 for brightest
LED_INVERT     = False   # True to invert the signal (when using NPN transistor level shift)
LED_CHANNEL_0  = 0       # set to '1' for GPIOs 13, 19, 41, 45 or 53
LED_CHANNEL_1  = 1

# Main program logic follows:
if __name__ == '__main__':
    # Process arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--clear', action='store_true', help='clear the display on exit')
    args = parser.parse_args()

    # Create NeoPixel object with appropriate configuration.
    strip0 = Adafruit_NeoPixel(LED_COUNT, 18, LED_FREQ_HZ, LED_DMA, LED_INVERT, LED_BRIGHTNESS, 0)
    strip1 = Adafruit_NeoPixel(LED_COUNT, 13, LED_FREQ_HZ, LED_DMA, LED_INVERT, LED_BRIGHTNESS, 1)

    # Intialize the library (must be called once before other functions).
    strip0.begin()
    strip1.begin()

    print ('Press Ctrl-C to quit.')
    if not args.clear:
        print('Use "-c" argument to clear LEDs on exit')

    try:
        #a.colorSet(strip0, a.LIME)
        #a.colorSet(strip1, a.LIME)
        while True:
            #a.colorWipe(strip0, a.WHITE)
            #a.rainbowCycle(strip0)
            #a.theaterChaseRainbow(strip0)
            #a.theaterChaseRainbow(strip1)
            #a.randomPixel(strip0, a.CYAN)
            #a.bounce(strip0, Color(0,200,0))
            #a.tetris(strip0)
            #a.tetris(strip1)
            #color = strip0.getPixelColor(strip0.numPixels()/2)
            #color2= strip0.getPixelColor(1)
            #a.bounce_bar(strip0, color, color2)
            #a.bounce_bar(strip1, color, color2)

            #a.colorWipe(strip0, a.RED)
            #a.colorWipe(strip1, a.LIME, True)

            #a.colorWipe(strip0, a.LIME)
            #a.colorWipe(strip1, a.RED, True)

            #a.doubleWipe(strip0, strip1, a.BLUE, a.RED)
            a.police2(strip0, strip1)
            #a.xmas(strip0)
            #a.xmas(strip1)

            #strip0.show()
            #strip1.show()

            #time.sleep(1.0/16.67)

            #a.colorWipe(strip0, a.WHITE)

    except KeyboardInterrupt:
        if args.clear:

            a.colorSet(strip0, a.BLACK)
            a.colorSet(strip1, a.BLACK)


import unittest
import torch.nn as nn
from torch.nn import functional as F
import hamiltorch.util
import torch

class UtilTestCase(unittest.TestCase):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def test_flatten_unflatten(self):
        model = nn.Linear(4, 4)
        flattened_params = hamiltorch.util.flatten(model)

        new_model = nn.Linear(4, 4)
        unflattened_params = hamiltorch.util.unflatten(new_model, flattened_params)
        hamiltorch.util.update_model_params_in_place(new_model, unflattened_params)

        new_model_flattened_params = hamiltorch.util.flatten(new_model)

        hamiltorch.util.eval_print('flattened_params', 'new_model_flattened_params')

        self.assertTrue(torch.all(torch.eq(flattened_params, new_model_flattened_params)))

    def test_model_functional(self):
        model = nn.Linear(4, 4)
        fmodel = hamiltorch.util.make_functional(model)

        x = torch.zeros(1, 4)
        ymodel = model.forward(x)
        params = list(model.parameters())
        yfunctional = fmodel(x, params=params)

        hamiltorch.util.eval_print('ymodel', 'yfunctional')

        self.assertTrue(torch.all(torch.eq(ymodel, yfunctional)))

    def test_differentiable_model_functional(self):
        model = nn.Linear(2, 2)
        params = list(model.parameters())
        params_flattened = hamiltorch.util.flatten(model)
        fmodel = hamiltorch.util.make_functional(model)

        model.zero_grad()
        xmodel = torch.randn(1, 2, requires_grad=True)
        ymodel = model.forward(xmodel).sum()
        ymodel.backward()
        xmodelgrad = xmodel.grad
        pmodelgrad = torch.cat([p.grad.flatten() for p in params])

        model.zero_grad()
        xfunctional = xmodel.detach().clone().requires_grad_()
        yfunctional = fmodel(xfunctional, params=params).sum()
        yfunctional.backward()
        xfunctionalgrad = xfunctional.grad
        pfunctionalgrad = torch.cat([p.grad.flatten() for p in params])

        hamiltorch.util.eval_print('params_flattened', 'xmodel', 'ymodel', 'xmodelgrad', 'xfunctional', 'yfunctional', 'xfunctionalgrad', 'pmodelgrad', 'pfunctionalgrad')

        self.assertTrue(torch.all(torch.eq(xmodelgrad, xfunctionalgrad)))
        self.assertTrue(torch.all(torch.eq(ymodel, yfunctional)))

    def test_conv_model_functional(self):
        class Net(nn.Module):
            def __init__(self):
                super(Net, self).__init__()
                self.layers = nn.Sequential(
                        nn.Conv2d(1, 10, kernel_size=5),
                        nn.MaxPool2d(2),
                        nn.ReLU(),
                        nn.Conv2d(10, 20, kernel_size=5),
                        nn.MaxPool2d(2),
                        nn.ReLU())
                self.fc1 = nn.Linear(320, 50)
                self.fc2 = nn.Linear(50, 10)

            def forward(self, x):
                x = self.layers(x)
                x = x.view(-1, 320)
                x = F.relu(self.fc1(x))
                x = self.fc2(x)
                return F.log_softmax(x, dim=1)

        model = Net()
        model.eval()
        eval_fmodel = hamiltorch.util.make_functional(model)
        model.train()
        train_fmodel = hamiltorch.util.make_functional(model)
        # Verify correctness in eval mode
        model.eval()
        params = list(model.parameters())
        x = torch.randn(10, 1, 28, 28)

        self.assertTrue(torch.all(torch.eq(model(x).sum(), train_fmodel(x, params=params).sum())))

    def test_vanilla_HMC_reversible(self):
        def log_prob(omega):
            mean = torch.zeros(2)
            var = torch.tensor([.10,.10])
            return torch.distributions.MultivariateNormal(mean, torch.diag(var)).log_prob(omega).sum()

        params_init = torch.tensor([1.,1.])
        momentum_init = torch.tensor([1.,1.])
        inv_mass = torch.tensor([1.,1.])
        p,m = hamiltorch.samplers.leapfrog(params_init, momentum_init, log_prob, steps=100, step_size=0.1, jitter=None, normalizing_const=1., softabs_const=1e6, explicit_binding_const=100, fixed_point_threshold=1e-20, fixed_point_max_iterations=6, jitter_max_tries=10, inv_mass=inv_mass, ham_func=None, sampler=hamiltorch.Sampler.HMC, integrator=hamiltorch.Integrator.EXPLICIT, metric=hamiltorch.Metric.HESSIAN, debug=False)
        momentum_reversed = - m[-1].clone()
        p,m = hamiltorch.samplers.leapfrog(p[-1], momentum_reversed, log_prob, steps=100, step_size=0.1, jitter=None, normalizing_const=1., softabs_const=1e6, explicit_binding_const=100, fixed_point_threshold=1e-20, fixed_point_max_iterations=6, jitter_max_tries=10, inv_mass=inv_mass, ham_func=None, sampler=hamiltorch.Sampler.HMC, integrator=hamiltorch.Integrator.EXPLICIT, metric=hamiltorch.Metric.HESSIAN, debug=False)

        self.assertTrue(torch.all(torch.eq(p[-1], params_init)))

if __name__ == '__main__':
    unittest.main(verbosity=2)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from sys import argv
import subprocess, re
from collections import defaultdict

class Ddict(defaultdict,dict):
    def __init__(self):
            defaultdict.__init__(self, Ddict)
    def __repr__(self):
            return dict.__repr__(self)

def sort_key(s):
    if s:
        try:
            c = re.findall('\d+$', s)[0]
        except:
            c = -1
        return int(c)

def parse_minimap2(strain_name, ref_file, minimap_result_file):
    cmd = f'minimap2 -xasm5 {ref_file} ../../fa_245_sidr_filter_genomes_spades3.15_ntref/{strain_name}_clean.fas > {minimap_result_file}'
    check = subprocess.check_call(cmd, shell=True)
    if check == 0:
        print('minimap2 was run successfully')
    else:
        print('minimap2 was not run successfully')

def generate_identity_perc(minimap2_result_file, identity_file):
    with open(minimap2_result_file, 'r') as f1, open(identity_file, 'w') as f2:
        f2.write('contig_identifier contig_length start_position end_position ref_contig identity_length matching_length identity_perc(%) \n')
        for line in f1:
            line = line.strip().split()
            identity_perc = round(int(line[9])/int(line[10]) * 100, 2)
            if 500 <= int(line[1]) < 1000:
                f2.write(line[0] + '\t' + line[1] + '\t' + line[2] + '\t' + line[3] + '\t'  + line[5] + '\t'+ line[9] + '\t' + line[10] + '\t' + str(identity_perc) + '\n')
    print('generating "' + identity_file + '" file successfully')
    return

def generate_coverage_perc(identity_result_file, candidate_contigs_file):
    len_dir = {}
    position_dir = Ddict()
    with open(identity_result_file, 'r') as f1, open(candidate_contigs_file, 'w') as f2:
        f2.write('contig_identifier\tref_contig\tcoverage_perc\tcoverage_length\tcontig_length\n')
        for line in f1:
            line = line.strip().split()
            if line[1] != 'contig_length' and line[4] != 'mt' and float(line[7]) >= 70:
                len_dir[line[0]] = int(line[1])
                if not line[4] in position_dir[line[0]]:
                    position_dir[line[0]][line[4]] = []
                position_dir[line[0]][line[4]].append([int(line[2]), int(line[3])])
        #print(position_dir)

        for key1 in sorted(position_dir, key=sort_key):
            merged = defaultdict(list)
            for key2 in position_dir[key1]:
                value = position_dir[key1][key2]
                #value.sort(key=lambda x: x[0])
                value.sort()
                for interval in value:
                    if not merged[key2] or merged[key2][-1][-1] < interval[0]:
                        merged[key2].append(interval)
                    else:
                        merged[key2][-1][-1] = max(merged[key2][-1][-1], interval[-1])
            #print(merged)
                #[[2, 4], [6, 8], [7, 13]]
                #[[2, 4], [6, 13]]

            coverage_perc = 0
            cov_len = 0
            for k in merged.keys():
                coverage_len = 0
                for i in merged[k]:
                    #print(key1 + '\t' + k + '\t' + str(i[1]) +'\t' + str(i[0]))
                    single_len = i[1]-i[0]
                    coverage_len += single_len
                perc = coverage_len/len_dir[key1] * 100
                #print(key1 + '\t' + k + '\t' + str(coverage_len) +'\t' + str(perc))
                if perc > coverage_perc:
                    coverage_perc = perc
                    ref_ctg = k
                    cov_len = coverage_len
            f2.write(key1 + '\t' + ref_ctg + '\t'+ str(coverage_perc) + '\t' + str(cov_len) + '\t' + str(len_dir[key1]) + '\n')
        print('generating "' + candidate_contigs_file + '" file successfully')
        return position_dir

def get_gc_content_dir(gc_content_file):
    with open(gc_content_file, 'r') as f:
        gc_content_dir = {}
        for line in f:
            line = line.strip().split()
            if line[0] != 'contig_identifier' and 500 <= int(line[2]) < 1000:
                gc_content_dir[line[0]] = float(line[1])
        return gc_content_dir

def get_lower_identify_perc_nomatching_contigs(position_dir, gc_content_dir):
    lower_identify_perc_nomatching_contigs = []
    for key in gc_content_dir.keys():
        if key not in position_dir.keys():
            lower_identify_perc_nomatching_contigs.append(key)
    return lower_identify_perc_nomatching_contigs

def get_lower_coverage_perc_contigs(candidate_contigs_file, gc_content_dir, coverage_threshold=80, gc_average=48.19, std=12.71):
    lower_coverage_gc_contigs = []
    with open(candidate_contigs_file, 'r') as f:
        for line in f:
            line = line.strip().split()
            if line[2] != 'coverage_perc' and float(line[2]) <= coverage_threshold:
                lower_coverage_gc_contigs.append(line[0])

            elif  line[2] != 'coverage_perc' and float(line[2]) > coverage_threshold:
                min_gc = gc_average - 2*std
                max_gc = gc_average + 2*std
                if gc_content_dir[line[0]] < min_gc or gc_content_dir[line[0]] > max_gc:
                    lower_coverage_gc_contigs.append(line[0])
        return lower_coverage_gc_contigs

def get_short_length_contigs(strain_name):
    cmd = f'fasta_length ../../fa_245_sidr_filter_genomes_spades3.15_ntref/{strain_name}_clean.fas -o > {strain_name}_ctg_contig_length.txt'
    check = subprocess.check_call(cmd, shell=True)
    if check == 0:
        print('contigs length was generated successfully')
    else:
        print('contigs length was not generated')

    contigs_len_all_dir = {}
    short_contigs = []
    with open(strain_name + '_ctg_contig_length.txt', 'r') as f:
        for line in f:
            line = line.strip()
            if line.startswith('>'):
                key = line[1:]
            else:
                contigs_len_all_dir[key] = int(line)
                if contigs_len_all_dir[key] < 500:
                    short_contigs.append(key)
    return short_contigs

def get_mt_contigs(mt_candidate_contigs_file):
    mt_contig = []
    with open(mt_candidate_contigs_file, 'r') as f:
        for line in f:
            line = line.strip().split()
            if line[0] != 'contig_identifier' and float(line[1]) >= 80:
                mt_contig.append(line[0])
        return mt_contig

def get_removed_contigs(lower_identify_perc_nomatching_contigs, lower_coverage_gc_contigs, short_contigs, mt_contig, strain_name):
    print('Success: ' + strain_name + ' lower_identify_perc_nomatching_contigs number is ' + str(len(lower_identify_perc_nomatching_contigs)))
    print('Success: ' + strain_name + ' lower_coverage_gc_contigs number is ' + str(len(lower_coverage_gc_contigs)))
    print('Success: ' + strain_name + ' short_contigs number is ' + str(len(short_contigs)))
    print('Success: ' + strain_name + ' mt_contig number is ' + str(len(mt_contig)))

    removed_contigs = lower_identify_perc_nomatching_contigs + lower_coverage_gc_contigs + short_contigs + mt_contig
    with open(removed_contigs_file, 'w') as f:
        for contig in removed_contigs:
            f.write(contig + '\n')
    return removed_contigs

def filter_genome(removed_contigs, strain_name):
    print('Success: ' + strain_name + ' remove ' + str(len(removed_contigs)) + ' contigs')
    strain_dir = {}
    with open('../../fa_245_sidr_filter_genomes_spades3.15_ntref/' + strain_name + '_clean.fas', 'r') as f1, open('../../fa_245_filtered_genomes/' + strain_name + '.fas', 'w') as f2:
        for line in f1:
            line = line.strip()
            if line.startswith('>'):
                key = line[1:]
            else:
                strain_dir[key] = line
                if key not in removed_contigs:
                    f2.write('>' + key + '\n' + strain_dir[key] + '\n')

if __name__ == "__main__":
    ref_file = argv[2]
    strain_name = argv[1]
    minimap_result_file = strain_name + '_ctg_minimap_results.txt'
    identity_result_file = strain_name + '_ctg_identity_result.txt'
    candidate_contigs_file = strain_name + '_ctg_candidate_contigs.txt'
    gc_content_file = strain_name + '_ctg_GC.txt'
    mt_candidate_contigs_file = strain_name + '_mt_candidate_contigs.txt'
    removed_contigs_file = strain_name + '_removed_contigs_identifiers.txt'

    conducted_minimap2 = parse_minimap2(strain_name, ref_file, minimap_result_file)
    identity_result = generate_identity_perc(minimap_result_file, identity_result_file)
    coverage_result = generate_coverage_perc(identity_result_file, candidate_contigs_file)

    getted_gc_content_dir = get_gc_content_dir(gc_content_file)
    getted_lower_identify_perc_nomatching_contigs = get_lower_identify_perc_nomatching_contigs(coverage_result, getted_gc_content_dir)
    getted_lower_coverage_perc_contigs = get_lower_coverage_perc_contigs(candidate_contigs_file, getted_gc_content_dir, coverage_threshold=80, gc_average=48.19, std=12.71)
    getted_short_length_contigs = get_short_length_contigs(strain_name)
    getted_mt_contigs = get_mt_contigs(mt_candidate_contigs_file)
    getted_removed_contigs = get_removed_contigs(getted_lower_identify_perc_nomatching_contigs, getted_lower_coverage_perc_contigs, getted_short_length_contigs, getted_mt_contigs, strain_name)
    get_filtered_genome = filter_genome(getted_removed_contigs, strain_name)

# python3 remove_ctg.py 160205 180197_pilon4.fas


import configparser
import os
import sys
import rauth
import requests

def twitter_like_bot():
    os.system('color')

    config = configparser.RawConfigParser()
    config.read('settings.cfg')

    # get parameters
    try:
        twitter_api_key = config['KEYS']['twitter_api_key']
        twitter_api_secret = config['KEYS']['twitter_api_secret']
        bearer_token = config['KEYS']['bearer_token']
        username = config['PARAMS']['username']
        since_id = config['PARAMS']['since_id']
    except KeyError:
        print(
            "\n\033[91mFailed to load settings!\033[0m"
            "\nCreate a file named 'settings.cfg' with the following content:"
            "\n\n[KEYS]"
            "\ntwitter_api_key = <key>"
            "\ntwitter_api_secret = <secret>"
            "\nbearer_token = <token>"
            "\n\n[PARAMS]"
            "\nusername = <account of which to like tweets>"
            "\nsince_id = 0"
        )
        sys.exit()

    # create session
    try:
        twitter = rauth.OAuth1Service(
            name='twitter',
            consumer_key=twitter_api_key,
            consumer_secret=twitter_api_secret,
            request_token_url='https://api.twitter.com/oauth/request_token',
            access_token_url='https://api.twitter.com/oauth/access_token',
            authorize_url='https://api.twitter.com/oauth/authorize',
            base_url='https://api.twitter.com/1.1/')
        request_token, request_token_secret = twitter.get_request_token(
            params={'oauth_callback': 'oob'}
        )
        authorize_url = twitter.get_authorize_url(request_token)

        print(
            "\nVisit this URL to authorize the app or type 'q' to quit."
            f"\n\033[94m{authorize_url}\033[0m"
        )
        pin = input("PIN: ")
        if pin == "q":
            sys.exit()

        session = twitter.get_auth_session(
            request_token,
            request_token_secret,
            method='POST',
            data={'oauth_verifier': pin})
    except requests.exceptions.ConnectionError:
        print("\n\033[91mConnection failed!\033[0m")
        sys.exit()
    except KeyError:
        print("\n\033[91mAuthorization failed!\033[0m")
        sys.exit()

    try:
        # get user id
        user_request = requests.get(
            f"https://api.twitter.com/2/users/by/username/{username}",
            headers={"authorization": f"bearer {bearer_token}"})
        user_dict = user_request.json()
        user_request.close()
        user_id = user_dict['data']['id']

        # get ids of new tweets
        tweets_request = session.get(
            f"https://api.twitter.com/2/users/{user_id}/tweets",
            header_auth=True,
            params={'since_id': since_id, 'max_results': '100'})
        tweets_dict = tweets_request.json()
        tweets_request.close()
        newest_tweet_id = tweets_dict['meta']['newest_id']

        tweet_ids = []
        for tweet in tweets_dict['data']:
            tweet_ids.append(tweet['id'])

        while 'next_token' in tweets_dict['meta'] and len(tweet_ids) < 200:
            tweets_request = session.get(
                f"https://api.twitter.com/2/users/{user_id}/tweets",
                header_auth=True,
                params={
                    'since_id': since_id,
                    'max_results': '100',
                    'pagination_token': tweets_dict['meta']['next_token']
                }
            )
            tweets_dict = tweets_request.json()
            tweets_request.close()
            for tweet in tweets_dict['data']:
                tweet_ids.append(tweet['id'])

        # like tweets
        liked_tweets = 0
        limit_reached = False
        for tweet_id in tweet_ids:
            like_request = session.post(
                "https://api.twitter.com/1.1/favorites/create.json",
                header_auth=True,
                data={'id': tweet_id})
            if like_request.status_code == 200:
                liked_tweets += 1
            elif like_request.status_code == 429:
                print("\n\033[91mLimit reached!\033[0m")
                limit_reached = True
                break

    except requests.exceptions.ConnectionError:
        print("\n\033[91mConnection interrupted!\033[0m")
        sys.exit()
    except KeyError:
        print("\nNo Tweets found.")
        sys.exit()

    # set since_id
    print(f"\n\033[92mLiked {liked_tweets} Tweets, Retweets and Replies!\033[0m")
    if not limit_reached:
        config.set('PARAMS', 'since_id', newest_tweet_id)
        with open('settings.cfg', 'w') as configfile:
            config.write(configfile)

if __name__ == '__main__':
    twitter_like_bot()


import cld3
import fiona
import geopandas as gpd
from iso639 import languages
import numpy as np
import os
import pandas as pd
import requests
import string
from time import perf_counter
import zipfile

def fetch_data(url):
    r = requests.get(url, stream=True)
    r.raise_for_status()

    print('Downloading data from U.S. Census website')
    with open('us_roads.gdb.zip', 'wb') as handle:
        for block in r.iter_content(1024):
            handle.write(block)

def lid_text(text):
    two_letter = cld3.get_language(text)[0]
    # three_letter = languages.get(alpha2=two_letter).part3
    # name = languages.get(alpha2=two_letter).name
    return two_letter

def load_data(file):
    print('Loading data from geodatabase')
    df = gpd.read_file(file, driver='FileGDB', layer='Roads')
    return df

def lid_data(df, column):
    df = df.apply(lambda x: lid_text(x))
    print(df.head())

def test_lid_speed(num_words, len_words):
    """How long does a LID take on average?"""
    # make a random series of n-grams to LID
    random_ngrams = []
    timings = []
    alpha = list(string.ascii_lowercase)
    i = 0
    while i < num_words:
        j = 0
        word = ''
        while j < len_words:
            word += np.random.choice(alpha)
            print(word)
            j+=1
        random_ngrams.append(word)
        i+=1

    # LID grams and average timing
    for quad in random_ngrams:
        start = perf_counter()
        lid_text(quad)
        stop = perf_counter()
        duration = stop - start
        timings.append(duration)
    print(f'''Per-LID timing for {num_words} words with {len_words} characters \
averages to ~{np.average(timings): 0.6f} seconds.''')
    print(f'''Total time taken is {np.sum(timings)}.''')
    return random_ngrams

def unzip_data(archive):
    print('Unzipping data file')
    if os.path.exists('./data'):
        pass
    else:
        os.mkdir('data')
    with zipfile.ZipFile(archive, 'r') as zip_ref:
        zip_ref.extractall(path='data')


import re
from collections import defaultdict

dictionary = defaultdict(int)
def histogram(source_text):

    delimeters = ["\\", ",", ";", ":", "\n", "-", "!", "?", "(", ")", "'s", \
    "'\'", ".", "1", "2", "3", "4", "5", "6", "7", "8", "9", "0", "/", "}", "{"]

    f = open(source_text, 'r')
    lines = f.readlines()

    for line in lines:
        words = line.strip().translate(None, ''.join(delimeters)).split()
        for word in words:
            dictionary[word] += 1
    return dictionary

def unique_words(histogram):
    total = 0
    for (key, val) in histogram.iteritems():
        total += val
    return total

def frequency(word, histogram):
    return histogram[word]

if __name__ == '__main__':
    histogram_val = histogram("sherlock.rtf")
    print(histogram_val)
    print(unique_words(histogram_val))
    print(frequency("the", histogram_val))


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

__all__ = [
    "gencxx",
    "genipdl",
    "parse",
    "typecheck",
    "writeifmodified",
    "checkSyncMessage",
    "checkFixedSyncMessages",
]

import os
import sys
from io import StringIO

from ipdl.cgen import IPDLCodeGen
from ipdl.lower import LowerToCxx, msgenums
from ipdl.parser import Parser, ParseError
from ipdl.type import TypeCheck
from ipdl.checker import checkSyncMessage, checkFixedSyncMessages

from ipdl.cxx.cgen import CxxCodeGen

def parse(specstring, filename="/stdin", includedirs=[], errout=sys.stderr):
    """Return an IPDL AST if parsing was successful.  Print errors to |errout|
    if it is not."""
    # The file type and name are later enforced by the type checker.
    # This is just a hint to the parser.
    prefix, ext = os.path.splitext(filename)
    name = os.path.basename(prefix)
    if ext == ".ipdlh":
        type = "header"
    else:
        type = "protocol"

    try:
        return Parser(type, name).parse(
            specstring, os.path.abspath(filename), includedirs
        )
    except ParseError as p:
        print(p, file=errout)
        return None

def typecheck(ast, errout=sys.stderr):
    """Return True iff |ast| is well typed.  Print errors to |errout| if
    it is not."""
    return TypeCheck().check(ast, errout)

def gencxx(ipdlfilename, ast, outheadersdir, outcppdir, segmentcapacitydict):
    headers, cpps = LowerToCxx().lower(ast, segmentcapacitydict)

    def resolveHeader(hdr):
        return [
            hdr,
            os.path.join(
                outheadersdir, *([ns.name for ns in ast.namespaces] + [hdr.name])
            ),
        ]

    def resolveCpp(cpp):
        return [cpp, os.path.join(outcppdir, cpp.name)]

    for ast, filename in [resolveHeader(hdr) for hdr in headers] + [
        resolveCpp(cpp) for cpp in cpps
    ]:
        tempfile = StringIO()
        CxxCodeGen(tempfile).cgen(ast)
        writeifmodified(tempfile.getvalue(), filename)

def genipdl(ast, outdir):
    return IPDLCodeGen().cgen(ast)

def genmsgenum(ast):
    return msgenums(ast.protocol, pretty=True)

def writeifmodified(contents, file):
    contents = contents.encode("utf-8")
    dir = os.path.dirname(file)
    os.path.exists(dir) or os.makedirs(dir)

    oldcontents = None
    if os.path.exists(file):
        fd = open(file, "rb")
        oldcontents = fd.read()
        fd.close()
    if oldcontents != contents:
        fd = open(file, "wb")
        fd.write(contents)
        fd.close()


from rest_framework.pagination import PageNumberPagination
from ..models import Post
from ..serializers import PostSerializer
from rest_framework.viewsets import ModelViewSet
from ..permissoes_api import IsOwner
from django.shortcuts import get_object_or_404
from rest_framework.permissions import IsAuthenticatedOrReadOnly
from rest_framework import status
from rest_framework.response import Response

class PostAPIv1ViewSet(ModelViewSet):
    queryset = Post.objects.get_published()
    serializer_class = PostSerializer
    pagination_class = PageNumberPagination
    permission_classes = [IsAuthenticatedOrReadOnly]

    def get_queryset(self):
        qs = super().get_queryset()
        categoria_id = self.request.query_params.get('categoria')

        if categoria_id is not None:
            qs = qs.filter(categoria_post=categoria_id)
        qs = qs.filter()
        return qs

    def get_object(self):
        pk = self.kwargs.get('pk')
        obj = get_object_or_404(Post.objects.get_published(), pk=pk)

        self.check_object_permissions(self.request, obj)
        return obj

    def get_permissions(self):
        if self.request.method in ['PATCH', 'DELETE', ]:
            return [IsOwner()]
        return super().get_permissions()

    def create(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        serializer.save(autor_post=request.user)
        headers = self.get_success_headers(serializer.data)
        return Response(
            serializer.data,
            status=status.HTTP_201_CREATED,
            headers=headers
        )


#!/usr/bin/python3
import csv
import requests
import sys

if __name__ == "__main__":
    tasks_status = []
    tasks_title = []
    user = sys.argv[1]
    url = "https://jsonplaceholder.typicode.com/"

    request_tasks = requests.get(url + "todos/?userid={}".format(user)).json()
    request_user = requests.get(url + "users/{}".format(user)).json()

    username = request_user["username"]
    name = request_user["name"]

    for task in request_tasks:
        if task["userId"] == int(user):
            tasks_status.append(task["completed"])
            tasks_title.append(task["title"])

    with open(user + ".csv", mode='w', encoding='utf-8') as csv_file:
        fieldnames = ["userId", "name", "completed", "title"]
        writer = csv.DictWriter(csv_file, fieldnames=fieldnames,
                                quoting=csv.QUOTE_ALL)
        for i in range(len(tasks_status)):
            writer.writerow({"userId": user, "name": username, "completed":
                            tasks_status[i], "title": tasks_title[i]})


#! /usr/bin/env python3

import argparse
import errno
import glob
import os
import sys
import json
import yaml

import jinja2
import PIL.Image
import numpy

# https://stackoverflow.com/a/600612
def mkdirp(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-o', '--output', default='output')
    parser.add_argument('index', metavar='FILE')
    parser.add_argument('pages', metavar='PAGE', nargs='*')

    args = parser.parse_args()

    output_dir = args.output
    mkdirp(output_dir)

    js_dir = os.path.join(output_dir, 'js')
    mkdirp(js_dir)

    css_dir = os.path.join(output_dir, 'css')
    mkdirp(css_dir)

    env = jinja2.Environment(loader=jinja2.FileSystemLoader('.'))
    index_template = env.get_template(args.index)

    with open(os.path.join(output_dir, 'index.html'), 'w') as f:
        f.write(index_template.render(pages=args.pages))

    with open(os.path.join(css_dir, 'style.css'), 'w') as f:
        f.write(env.get_template('css/style.css').render())

    with open(os.path.join(js_dir, 'script.js'), 'w') as f:
        f.write(env.get_template('js/script.js').render())

    textures_dir = os.path.join(output_dir, 'textures', 'common')
    mkdirp(textures_dir)

    for path in glob.glob(os.path.join('common', '*.png')):
        im = PIL.Image.open(path)
        im.save(os.path.join(textures_dir, os.path.basename(path)))

    result = []

    for shader in os.listdir('shaders'):
        if shader in ['common']:
            raise RuntimeError('invalid shader name: {}'.format(shader))

        shader_dir = os.path.join('shaders', shader)

        # returns (input path, output path)
        def search(name):
            path = os.path.join(shader_dir, name)
            if os.path.exists(path):
                return path, os.path.join(shader, name)

            path = os.path.join('common', name)
            if os.path.exists(path):
                return path, path

            raise RuntimeError('File not found: {}'.format(name))

        with open(os.path.join(shader_dir, 'shader.yaml')) as f:
            data = yaml.load(f)

        for buf in data:
            vertex_shader_name = buf.get('vertex', 'vertex.glsl')
            vertex_shader_path, _ = search(vertex_shader_name)
            vertex_shader = open(vertex_shader_path).read()
            buf['vertex'] = vertex_shader

            fragment_shader_name = buf.get('fragment', 'fragment.glsl')
            fragment_shader_path, _ = search(fragment_shader_name)
            fragment_shader = open(fragment_shader_path).read()
            buf['fragment'] = fragment_shader

            textures = buf.setdefault('textures', [])
            for texture in textures[::]:
                name = texture.get('name', 'tex')

                texture_file = texture.get('file')
                texture_buffer = texture.get('buffer')
                texture_choice = texture.get('choice')

                if texture_file is not None:
                    texture_path, texture['file'] = search(texture_file)
                elif texture_buffer is not None:
                    pass
                elif texture_choice is not None:
                    for e in texture_choice:
                        texture_path, e['file'] = search(e['file'])

        textures_dir = os.path.join(output_dir, 'textures', shader)
        mkdirp(textures_dir)

        for path in glob.glob(os.path.join(shader_dir, '*.png')):
            # Premultiply alpha

            # https://stackoverflow.com/a/9146202
            im = PIL.Image.open(path).convert('RGBA')
            premult = numpy.fromstring(im.tobytes(), dtype=numpy.uint8)
            alphaLayer = premult[3::4] / 255.0
            premult[0::4] = premult[0::4] * alphaLayer
            premult[1::4] = premult[1::4] * alphaLayer
            premult[2::4] = premult[2::4] * alphaLayer
            im_out = PIL.Image.frombytes("RGBA", im.size, premult.tostring())
            im_out.save(os.path.join(textures_dir, os.path.basename(path)))

        result.append((shader, data))

    with open(os.path.join(js_dir, 'shaders.js'), 'w') as f:
        print("var shaders = [];", file=f)

        for name, shader in result:
            print("shaders['{}'] = {};".format(name, json.dumps(shader)), file=f)

if __name__ == '__main__':
    main()


import itertools
from urlextract import URLExtract
from wordcloud import WordCloud
import pandas as pd
from collections import Counter
import emoji

extract = URLExtract()

def remove_unwanted_data(selected_user, df):

    if selected_user != 'Overall':
        df = df[df['user'] == selected_user]

    df = df[df['user'] != 'group_notification']
    df = df[df['message'] != '<Media omitted>\n']
    df = df[df['message'] != 'This message was deleted\n']

    links = []
    for message in df['message']:
        links.extend(extract.find_urls(message))

    for i in range(len(links)):
        df = df[df['message'] != str(links[i]+'\n')]

    return df

def fetch_stats(selected_user,df):

    if selected_user != 'Overall':
        df = df[df['user'] == selected_user]

    # fetch the number of messages
    num_messages = df.shape[0]

    # fetch number of media messages
    num_media_messages = df[df['message'] == '<Media omitted>\n'].shape[0]

    # fetch number of deleted messages
    num_del_mes = df[df['message'] == 'This message was deleted\n'].shape[0]

    # fetch the total number of words
    df = remove_unwanted_data(selected_user, df)
    words = []
    for message in df['message']:
        words.extend(message.split())

    # fetch number of links shared
    links = []
    for message in df['message']:
        links.extend(extract.find_urls(message))

    return num_messages,len(words),num_media_messages,len(links),num_del_mes

def most_busy_users(df):
    x = df['user'].value_counts().head()
    df = round((df['user'].value_counts() / df.shape[0]) * 100, 2).reset_index().rename(columns={'index': 'name', 'user': 'name'})
    return x,df

def most_messages_deleted(df):

    dic = {}
    msg = df['message'].tolist()
    user = df['user'].tolist()
    for i in range(df.shape[0]):
        if msg[i] == 'This message was deleted\n':
            if user[i] not in dic.keys():
                dic[user[i]] = 1
            else:
                dic[user[i]] = dic[user[i]] + 1

    dic = dict(sorted(dic.items(), key=lambda x: x[1], reverse=True))
    dic = dict(itertools.islice(dic.items(), 5))
    return dic

def most_negative_messages(df):

    f = open('negative_words.txt', 'r')
    neg_words = f.read()
    nl = []
    for word in neg_words.lower().split():
        nl.append(word)
    dic = {}
    msg = df['message'].tolist()
    user = df['user'].tolist()
    for i in range(df.shape[0]):
        message=msg[i].lower().split()
        for j in range(len(message)):
            if message[j] in nl:
                if user[i] not in dic.keys():
                    dic[user[i]] = 1
                else:
                    dic[user[i]] = dic[user[i]] + 1

    dic = dict(sorted(dic.items(), key=lambda x: x[1], reverse=True))
    dic = dict(itertools.islice(dic.items(), 5))
    return dic

def create_wordcloud(df):

    def wordlist(message):
        y = []
        for word in message.lower().split():
            y.append(word)
        return " ".join(y)

    wc = WordCloud(width=500,height=500,min_font_size=10,background_color='white')
    df['message'] = df['message'].apply(wordlist)
    if df.shape[0]!=0:
        df_wc = wc.generate(df['message'].str.cat(sep=" "))
        return df_wc

def remove_hinglish(df):

    f = open('stop_hinglish.txt', 'r')
    stop_words = f.read()
    sw = []
    for word in stop_words.lower().split():
        sw.append(word)

    words = []
    for message in df['message']:
        for word in message.lower().split():
            if word not in sw:
                words.append(word)

    return words

def most_common_words(df):

    words = remove_hinglish(df)
    most_common_df = pd.DataFrame(Counter(words).most_common(20))
    return most_common_df

def most_neg_words(df):

    words = remove_hinglish(df)
    f = open('negative_words.txt', 'r')
    neg_words = f.read()
    nl = []
    for word in neg_words.lower().split():
        nl.append(word)
    nw = []
    for i in range(len(words)):
        if words[i] in nl:
            nw.append(words[i])
    most_common_df = pd.DataFrame(Counter(nw).most_common(20))
    return most_common_df

def count_negative_words(df):

    words = []
    for message in df['message']:
        for word in message.lower().split():
            words.append(word)

    f = open('negative_words.txt', 'r')
    neg_words = f.read()
    nl = []
    for word in neg_words.lower().split():
        nl.append(word)
    nw=[]
    for i in range(len(words)):
        if words[i] in nl:
            nw.append(words[i])
    if(len(words)!=0):
        return round((len(nw)/len(words)*100),1)
    else:
        return 0

def emoji_helper(df):

    emojis = []
    for message in df['message']:
        emojis.extend([c for c in message if c in emoji.UNICODE_EMOJI['en']])

    emoji_df = pd.DataFrame(Counter(emojis).most_common(len(Counter(emojis))))

    return emoji_df

def monthly_timeline(df):

    timeline = df.groupby(['year', 'month_num', 'month']).count()['message'].reset_index()

    time = []
    for i in range(timeline.shape[0]):
        time.append(timeline['month'][i] + "-" + str(timeline['year'][i]))

    timeline['time'] = time

    return timeline


##    1. (5 баллов) Скачать отсюда архив страниц интернет-сайта с тайско-английским словарём. Извлечь с каждой страницы
##    пары "тайское слово -- английское слово" и поместить их в питоновскую структуру данных типа "словарь",
##    где ключом будет тайское слово, а значением - английское.
##
##    2. (8 балла) Использовать структуру данных из предыдущего задания, записать её в файл формата json на диск,
##    а также создать ещё одну структуру данных, где будет наоборот: английское слово ключ, а массив тайских слов - значение.
##    Её тоже записать на диск в формате json.
##
##    3. (10 баллов) Создать на фласке веб-приложение "Англо-тайский словарь", где можно было бы в текстовом поле ввести английское слово
##    и получить в качестве результата запроса - его перевод на тайский.

from bs4 import BeautifulSoup
import json
import os
import re

def get_dict(html):
    d = {}
    soup = BeautifulSoup(html, 'html.parser')
    text = soup.get_text()
    tr_items = soup.find_all('tr')
    for it in tr_items:
        trs = it.find_all('td')
        if trs:
            w = trs[0].find('a')
            if w:
                w = w.get_text()
                d[w] = trs[-1].get_text()
    return d

def crawl():
    d = {}
    lst = os.listdir('thai_pages')
    for fl in lst:
        with open('thai_pages/'+fl, 'r', encoding='utf-8') as f:
            html = f.read()
            d.update(get_dict(html))
    return d #5

def jsonify(d):
    text = json.dumps(d)
    with open('thai-eng.json', 'w', encoding='utf-8') as f:
        f.write(text)
    inv_d = {}
    for key in d:
        if ';' in d[key]:
            keys = d[key].split(';')
            for k in keys:
                inv_d[k.strip()] = key
        else:
            inv_d[d[key]] = key
    text_i = json.dumps(inv_d)
    with open('eng-thai.json', 'w', encoding='utf-8') as f:
        f.write(text_i) #8

def main():
    d = crawl()
    jsonify(d)

if __name__ == '__main__':
    main()