from jwt_knox.viewsets import JWTKnoxAPIViewSet

from . import viewsets
from rest_framework_extensions.routers import ExtendedDefaultRouter

router = ExtendedDefaultRouter(trailing_slash=False)
router.register(r'auth', JWTKnoxAPIViewSet, base_name='jwt_knox')
router.register(r'users', viewsets.UserProfileViewSet)

with router.register(r'projects',
                     viewsets.ProjectViewSet,
                     base_name='project') as project:

    project.register(r'ops',
                     viewsets.OperationViewSet,
                     base_name='operation')

    project.register(r'files',
                     viewsets.ProjectFileViewSet,
                     base_name='projectfile')

urlpatterns = []
urlpatterns += router.urls


def run_formula(dv, param=None):
	default_param = {'N':5,'M':16,'T':17}
	if not param:
		param = default_param

	BIAS = dv.add_formula('BIAS_J',"(close/Ta('MA',0,close,close,close,close,close,%d)-1)*100" %(param['N']),is_quarterly=False,add_data=True)

	DIF = dv.add_formula('DIF_J','BIAS-Delay(BIAS,%d)' % (param['M']),is_quarterly=False,add_data=True)

	def sma(df, n, m):
		a = n / m - 1
		r = df.ewm(com=a, axis=0, adjust=False)
		return r.mean()

	DBCD = dv.add_formula('DBCD_J',"SMA(DIF,%d,1)" % (param['T']),is_quarterly=False,add_data=False,register_funcs={"SMA":sma})

	return DBCD


command = str(input())
exam_results = {}
submissions = {}
while command != "exam finished":
    if "banned" in command:
        name = command.split("-")[0]
        del exam_results[name]
        command = str(input())
        continue
    name, language, points = command.split("-")
    if language not in submissions:
        submissions[language] = 0
    if name not in exam_results:
        exam_results[name] = int(points)
    if exam_results[name] < int(points):
        exam_results[name] = int(points)

    submissions[language] += 1

    command = str(input())
print("Results:")
for key, value in exam_results.items():
    print(f"{key} | {value}")
print("Submissions:")
for key, value in submissions.items():
    print(f"{key} - {value}")


from flask import make_response
from ..connect_to_aws import database
from ..model.create_database import Gifts

# def show_good_sales_gift_method():
#     output_message = {
#         "message": "Information waiting for confirmation"
#     }
#     gifts = Gifts.query.order_by(Gifts.gift_sales.desc()).limit(2)
#     if gifts:
#         database.session.close()
#         return gifts
#     else:
#         output_message['message'] = "no gifts in the system"
#         status_code = 400
#         output_json = make_response(output_message)
#         output_json.status_code = status_code
#         output_json.message = output_message['message']
#         database.session.close()
#         return output_json

def show_good_sales_gift_method():
    status_code = 200
    gifts = Gifts.query.order_by(Gifts.gift_sales.desc()).limit(5)
    response_message = {
        "message": "success"
    }
    response_data = {
        "most_sales_gifts": []
    }
    # gifts_dict = {
    #     "wishlists_inf": wishlists_model
    # }
    if not gifts:
        response_message['message'] = 'The system do not have any order.'
        status_code = 404
        resp = make_response(response_message)
        resp.status_code = status_code
        database.session.close()
        return resp
    l = []
    for o in gifts:
        p = {
            "id": o.id,
            "gift_name": o.gift_name,
            "gift_price": o.gift_price,
            "gift_discount_price": o.gift_discount_price,
            "gift_discount_state": o.gift_discount_state,
            "gift_description": o.gift_description,
            "gift_category": o.gift_category,
            "gift_side_category1": o.gift_side_category1,
            "gift_side_category2": o.gift_side_category2,
            "gift_cover_url": o.gift_cover_url,
            # "gift_show_url1": o.gift_show_url1,
            # "gift_show_url2": o.gift_show_url2,
            # "gift_show_url3": o.gift_show_url3,
            # "gift_show_url4": o.gift_show_url4,
            "gift_sales": o.gift_sales,
            "gift_income": o.gift_income,
            }
        l.append(p)
    response_data["most_sales_gifts"] = l
    resp = make_response(response_message)
    resp.status_code = status_code
    resp.response_data = response_data
    database.session.close()
    return resp


from copy import copy
import pprint
with open("input.txt", "rb") as file:
    grid = []
    for line in file:
        line = line[:-1].decode("utf-8")
        grid.append(list(map(int, list(line))))

class VisibilityGrid:

    def __init__(self, grid):
        self.grid = grid
        self.width = len(grid[0])
        self.height = len(grid)
        self.visibility_left = []
        self.visibility_right = []
        self.visibility_up = []
        self.visibility_down = []
        self.calculate_visibility()

    def calculate_visibility(self):
        for i in range(self.height):
            self.visibility_left.append([])
            view = {k: 0 for k in range(10)}
            self.visibility_left[i].append(copy(view))
            for j in range(1, self.width):
                for k in range(10):
                    view[k] += 1
                for k in range(self.grid[i][j-1]+1):
                    view[k] = 1
                self.visibility_left[i].append(copy(view))
            self.visibility_right.append([])
            view = {k: 0 for k in range(10)}
            self.visibility_right[i].append(copy(view))
            for j in range(self.width-2, -1, -1):
                for k in range(10):
                    view[k] += 1
                for k in range(self.grid[i][j+1]+1):
                    view[k] = 1
                self.visibility_right[i].append(copy(view))
            self.visibility_right[i] = self.visibility_right[i][::-1]
        for i in range(self.height):
            self.visibility_up.append([None]*self.width)
            self.visibility_down.append([None]*self.width)
        for j in range(self.width):
            view = {k: 0 for k in range(10)}
            self.visibility_up[0][j] = copy(view)
            for i in range(1, self.height):
                for k in range(10):
                    view[k] += 1
                for k in range(self.grid[i-1][j]+1):
                    view[k] = 1
                self.visibility_up[i][j] = copy(view)
            view = {k: 0 for k in range(10)}
            self.visibility_down[self.height-1][j] = copy(view)
            for i in range(self.height-2, -1, -1):
                for k in range(10):
                    view[k] += 1
                for k in range(self.grid[i+1][j]+1):
                    view[k] = 1
                self.visibility_down[i][j] = copy(view)

        visibilities = []
        for i in range(self.height):
            visibilities.append([])
            for j in range(self.width):
                visibilities[i].append(self.visibility_left[i][j][self.grid[i][j]])
        #pprint.pprint(visibilities)
        visibilities = []
        for i in range(self.height):
            visibilities.append([])
            for j in range(self.width):
                visibilities[i].append(self.visibility_right[i][j][self.grid[i][j]])
        #pprint.pprint(visibilities)
        visibilities = []
        for i in range(self.height):
            visibilities.append([])
            for j in range(self.width):
                visibilities[i].append(self.visibility_up[i][j][self.grid[i][j]])
        #pprint.pprint(visibilities)
        visibilities = []
        for i in range(self.height):
            visibilities.append([])
            for j in range(self.width):
                visibilities[i].append(self.visibility_down[i][j][self.grid[i][j]])
        #pprint.pprint(visibilities)

    def calculate_scenic_score(self):
        scores = []
        for i in range(self.height):
            scores.append([])
            for j in range(self.width):
                height = self.grid[i][j]
                scores[i].append(self.visibility_left[i][j][height] * self.visibility_right[i][j][height] * self.visibility_up[i][j][height] * self.visibility_down[i][j][height])
        #print(scores)
        return scores

visibilty = VisibilityGrid(grid)
print(max(list(map(max, visibilty.calculate_scenic_score()))))


# -*- coding: utf-8 -*-
import scrapy
from scrapy import Request
from urllib import parse
from VideoSpider.items import TvItem

class TencentVarietySpider(scrapy.Spider):
    name = 'Tencent_variety'
    allowed_domains = ['v.qq.com']
    start_urls = ['http://v.qq.com/x/list/variety']

    def parse(self, response):
        common = response.css('.filter_content')[0]
        channels = common.css('a::text').extract()
        channel_links = common.css('a::attr(href)').extract()
        for channel, channel_link in zip(channels[1: len(channels)], channel_links[1: len(channel_links)]):
            url = parse.urljoin(self.start_urls[0], channel_link)
            yield Request(url=url, callback=self.parse_page_info, meta={'channel': channel}, dont_filter=True)

    def parse_page_info(self, response):
        common = response.css('.figures_list')
        if common:
            newsreel_urls = common[0].css('.list_item .figure::attr(href)').extract()
            # newsreel_urls = ['https://v.qq.com/x/cover/jx7g4sm320sqm7i.html']
            for newsreel_url in newsreel_urls:
                yield Request(url=newsreel_url, callback=self.pase_variety_info, meta={'area': response.meta['channel']})
        next_page = response.css('.page_next::attr(href)').extract()
        if next_page and 'java' not in next_page[0]:
            yield Request(url=parse.urljoin(self.start_urls[0], next_page[0]), callback=self.parse_page_info, meta={'area': response.meta['channel']}, dont_filter=True)

    def pase_variety_info(self, response):
        area = response.meta['area']
        name = response.css('.player_title a::text').extract()[0]
        play_list = response.css('a[_stat="video-list-column:click"]::attr(href)').extract()
        play_list = [url[url.rfind('/') + 1: url.rfind('.')] for url in play_list]
        play_list.append(response.url[response.url.rfind('/') + 1: response.url.rfind('.')])
        tags = response.css('._video_tags a::text').extract()
        if tags:
            tags = '/'.join(tags)
        else:
            tags = ''
        print('-------------------------------------------')
        print(name)
        print(area)
        print(tags)
        for id in set(play_list):
            item = TvItem()
            item['id'] = id
            item['name'] = name
            item['isFeature'] = 1
            item['upTime'] = ''
            item['area'] = area
            item['parts'] = len(play_list)
            item['updateTime'] = ''
            item['language'] = ''
            item['type_tag'] = tags
            item['playTimes'] = 0
            item['score'] = ''
            item['director'] = ''
            item['actor'] = ''
            item['describe'] = ''
            item['isVip'] = 1
            item['comment'] = 0
            item['recommend'] = ''
            item['platform'] = 'tencent'
            item['sca_app_id'] = 5006
            item['cat1'] = '视频'
            item['cat2'] = '综艺'
            yield item


""" Unit tests for RFI simulation

"""

import os
import logging
import unittest

import astropy.units as u
import numpy
import numpy.testing
from astropy.coordinates import SkyCoord

from rascil.data_models import PolarisationFrame, rascil_path
from rascil.processing_components import (
    create_blockvisibility,
    qa_image,
    export_image_to_fits,
    create_image_from_visibility,
    invert_ng,
)
from rascil.processing_components.simulation import create_named_configuration
from rascil.processing_components.simulation.rfi import (
    simulate_rfi_block_prop,
)

log = logging.getLogger("rascil-logger")
log.setLevel(logging.WARNING)

class TestRFIRegression(unittest.TestCase):
    def setUp(self):
        self.persist = os.getenv("RASCIL_PERSIST", False)

    def setup_telescope(self, telescope):
        """Initialise common elements"""
        rmax = 50.0
        antskip = 1
        self.configuration = create_named_configuration(
            telescope, rmax=rmax, skip=antskip
        )
        self.nants = len(self.configuration.names)

        ftimes = (numpy.pi / 43200.0) * numpy.arange(-3600.0, +3600.0, 225.0)
        log.info(f"Times: {ftimes}")
        if telescope == "MID":
            ffrequency = numpy.linspace(1.4e9, 1.9e9, 5)
            channel_bandwidth = numpy.array([1e8, 1e8, 1e8, 1e8, 1e8])
        else:
            ffrequency = numpy.linspace(1.3e8, 1.5e8, 5)
            channel_bandwidth = numpy.array([4e6, 4e6, 4e6, 4e6, 4e6])

        polarisation_frame = PolarisationFrame("linear")
        # Set the phasecentre close to the horizon since we don't have
        # the far sidelobes yet
        phasecentre = SkyCoord(
            ra=0.0 * u.deg, dec=+30.0 * u.deg, frame="icrs", equinox="J2000"
        )
        self.bvis = create_blockvisibility(
            self.configuration,
            ftimes,
            ffrequency,
            channel_bandwidth=channel_bandwidth,
            polarisation_frame=polarisation_frame,
            phasecentre=phasecentre,
            weight=1.0,
        )

        nants_start = self.nants

        self.emitter_power = numpy.zeros(
            (1, len(self.bvis.time), nants_start, len(self.bvis.frequency)),
            dtype=complex,
        )  # one source
        # only add signal to the 4th and 5th channels (for testing purposes)
        self.emitter_power[:, :, :, 3] = 1.0e-10
        self.emitter_power[:, :, :, 4] = 5.0e-10

        # azimuth, elevation, distance
        self.emitter_coordinates = numpy.ones(
            (1, len(self.bvis.time), nants_start, 3),
        )
        self.emitter_coordinates[:, :, :, 0] = (
            0.0 + numpy.linspace(-1.0, 1.0, 5)[numpy.newaxis, numpy.newaxis, :]
        )
        self.emitter_coordinates[:, :, :, 1] = (
            29.45 + numpy.linspace(-2.0, 2.0, 5)[numpy.newaxis, numpy.newaxis, :]
        )
        self.emitter_coordinates[:, :, :, 2] = (
            600000.0
            + numpy.linspace(-5000.0, 5000.0, 5)[numpy.newaxis, numpy.newaxis, :]
        )

    def test_simulate_rfi_block_prop_image_beam_true(self):
        """
        regression to test that simulate_rfi_block_prop correctly updates the
        block visibility data with RFI signal. We do this by making an image

        RFI signal is for the same frequency channels as the BlockVisibility has
        """
        self.setup_telescope("MID")
        bvis = self.bvis.copy(deep=True)

        simulate_rfi_block_prop(
            bvis,
            self.emitter_power,
            self.emitter_coordinates,
            ["source1"],
            bvis.frequency.values,
            low_beam_gain=None,
            apply_primary_beam=True,
        )

        model = create_image_from_visibility(bvis, npixel=1024, cellsize=0.002, nchan=1)
        dirty, sumwt = invert_ng(bvis, model)
        dirty["pixels"].data[numpy.abs(dirty["pixels"].data) > 1e6] = 0.0
        if self.persist:
            export_image_to_fits(
                dirty,
                rascil_path(f"test_results/test_rfi_image_withPBTrue.fits"),
            )
        qa = qa_image(dirty)

        numpy.testing.assert_approx_equal(
            qa.data["max"],
            218354.68943290864,
            8,
            err_msg=str(qa),
        )
        numpy.testing.assert_approx_equal(
            qa.data["min"], -137862.80398882864, 8, err_msg=str(qa)
        )

    def test_simulate_rfi_block_prop_image_beam_false(self):
        """
        regression to test that simulate_rfi_block_prop correctly updates the
        block visibility data with RFI signal. We do this by making an image

        RFI signal is for the same frequency channels as the BlockVisibility has
        """
        self.setup_telescope("MID")
        bvis = self.bvis.copy(deep=True)

        simulate_rfi_block_prop(
            bvis,
            self.emitter_power,
            self.emitter_coordinates,
            ["source1"],
            bvis.frequency.values,
            low_beam_gain=None,
            apply_primary_beam=False,
        )

        model = create_image_from_visibility(bvis, npixel=1024, cellsize=0.002, nchan=1)
        dirty, sumwt = invert_ng(bvis, model)
        dirty["pixels"].data[numpy.abs(dirty["pixels"].data) > 1e6] = 0.0
        if self.persist:
            export_image_to_fits(
                dirty,
                rascil_path(f"test_results/test_rfi_image_withPBFalse.fits"),
            )
        qa = qa_image(dirty)

        numpy.testing.assert_approx_equal(
            qa.data["max"], 999979.7305620918, 8, err_msg=str(qa)
        )
        numpy.testing.assert_approx_equal(
            qa.data["min"], -999962.1676042686, 8, err_msg=str(qa)
        )

if __name__ == "__main__":
    unittest.main()


from deep_learning import *
from reduction_functions import *
from optimizer import *
from deep_transformer import *
from keras.regularizers import *
import pickle

# This function use pickle to save objects into files.
# We use it to save the models created.
def save_object(obj, filename):
    with open(filename, 'wb') as output:
        pickle.dump(obj, output, pickle.HIGHEST_PROTOCOL)

# List of data sets paths
csv_sources = ["D:\\Datasets\\ailerons\\ailerons.csv",
               "D:\\Datasets\\amazon initial 20 30 10000\\Amazon_initial_50_30_10000.csv",
               "D:\\Datasets\\coil2000 - 2 classes, unbalanced\\coil2000.csv",
               "D:\\Datasets\\dbworld_bodies\\dbworld_bodies.csv",
               "D:\\Datasets\\dbworld_subjects\\dbworld_subjects.csv",
               "D:\\Datasets\\optdigits - 10 classes, balanced\\optdigits.csv",
               "D:\\Datasets\\slice_localization_data\\slice_localization_data.csv",
               "D:\\Datasets\\sonar - 2 classes, noisy features\\sonar.csv",
               "D:\\Datasets\\UJIndoorLoc\\trainingData.csv",
               "D:\\Datasets\\whight lifting\\Example_WearableComputing_weight_lifting_exercises_biceps_curl_variations.csv"]

# list of names of the responses classes for each data set according to the data sets list
classes = ["Goal",
           "class",
           "CARAVAN",
           "CLASS",
           "CLASS",
           "Class",
           "reference",
           "Type",
           "USERID",
           "classe"]

# list of counts to reduce to for each data set according to the data sets list
reduction_counts = [20,
                    100,
                    20,
                    50,
                    30,
                    20,
                    20,
                    20,
                    75,
                    20]

# Following are different parameters given to the Deep Transformer.
# We defined different kinds of parameters according to successes or failures of their formers.
# default params
params1 = {"activation": 'relu', "loss": 'mse', "optimizer": 'rmsprop',
           "nb_epoch": 100, "batch_size": 128, "reduction_function": half_reduction_function,
           "hidden_dropout": None, "input_dropout": None, "activity_regularizer": None, "denoise_ratio": None}
# deepsign params mild - as described in deepsign article but with less aggressive features
params2 = {"activation": 'relu', "loss": 'mse', "optimizer": deep_sign_sgd_optimizer,
           "nb_epoch": 100, "batch_size": 20, "reduction_function": deep_sign_reduction_function,
           "hidden_dropout": None, "input_dropout": None, "activity_regularizer": None, "denoise_ratio": None}
# deepsign params - as described in deepsign article with mild dropout and denoising
params3 = {"activation": 'relu', "loss": 'mse', "optimizer": deep_sign_sgd_optimizer,
           "nb_epoch": 200, "batch_size": 20, "reduction_function": deep_sign_reduction_function,
           "hidden_dropout": 0.1, "input_dropout": None, "activity_regularizer": activity_l2(), "denoise_ratio": 0.1}
# deepsign params harsh - as described in deepsign article
params4 = {"activation": 'relu', "loss": 'mse', "optimizer": deep_sign_sgd_optimizer,
           "nb_epoch": 1000, "batch_size": 20, "reduction_function": deep_sign_reduction_function,
           "hidden_dropout": 0.2, "input_dropout": None, "activity_regularizer": activity_l2(), "denoise_ratio": 0.2}
# Trying to improve params for our causes
params5 = {"activation": 'relu', "loss": 'mse', "optimizer": deep_sign_sgd_optimizer,
           "nb_epoch": 200, "batch_size": 128, "reduction_function": deep_sign_reduction_function,
           "hidden_dropout": 0.4, "input_dropout": None, "activity_regularizer": activity_l2(), "denoise_ratio": None}
# Trying to improve params for our causes
params6 = {"activation": 'softmax', "loss": 'mse', "optimizer": 'rmsprop',
           "nb_epoch": 100, "batch_size": 128, "reduction_function": half_reduction_function,
           "hidden_dropout": 0.4, "input_dropout": None, "activity_regularizer": None, "denoise_ratio": None}
# Trying to improve params for our causes
params7 = {"activation": 'relu', "loss": 'mse', "optimizer": "rmsprop",
           "nb_epoch": 100, "batch_size": 20, "reduction_function": gradient_reduction_function,
           "hidden_dropout": 0.5, "input_dropout": None, "activity_regularizer": activity_l2(), "denoise_ratio": None}
# Trying using 1 epoch
params8 = {"activation": 'relu', "loss": 'mse', "optimizer": "rmsprop",
           "nb_epoch": 1, "batch_size": 20, "reduction_function": gradient_reduction_function,
           "hidden_dropout": 0.5, "input_dropout": None, "activity_regularizer": activity_l2(), "denoise_ratio": None}

# List of all parameters to use on each of the data sets from data sets list
params = [params1, params2, params3, params4, params5, params6, params7]

# going over all data sets list
for i in range(0, len(csv_sources)):
    # going over all params
    for j in range(0, len(params)):
        # Create a deep transformer object
        deep_transformer = DeepTransformer()
        # assign the corresponding params
        deep_transformer.set_params(**(params[j]))
        # Run deep learning algorithm on it
        deep_learn_csv(csv_path=csv_sources[i], deep_transformer=deep_transformer,
                       result_file_path=csv_sources[i].replace(".csv", "_dl_" + repr(j + 1) + ".csv"),
                       responses=classes[i], target_feature_number=reduction_counts[i], verbose=1)
        # save model to file so it can be used later again
        save_object(deep_transformer, csv_sources[i].replace(".csv", "_dl_" + repr(j + 1) + ".pkl"))


class Solution:
    def makeGood(self, s: str) -> str:
        stack = []
        for char in s:
            if not stack:
                stack.append(char)
            else:
                if self.destroy(char, stack):
                    stack = stack[:-1]
                else:
                    stack.append(char)
        return "".join(stack)

    def destroy(self, char, stack):
        prev = stack[-1]
        return abs(ord(prev) - ord(char)) == 32


from gui import Ui_MainWindow
import sys
from PyQt5.QtWidgets import *
from PyQt5.QtCore import QTimer, pyqtSignal
import os
import threading
import shutil
import datetime

class APKSoupMain(QMainWindow):
    injectSignal = pyqtSignal()
    progressSignal = pyqtSignal(int)

    def __init__(self):
        super().__init__()

        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)
        self.ui.browseFilesBtn.clicked.connect(self.browse_files)
        self.ui.SoupProgressBar.hide()
        self.ui.soupButton.clicked.connect(self.start_injection)

        self.progress_timer = QTimer()
        self.progress_timer.timeout.connect(self.update_progress)

        self.injectSignal.connect(self.inject)
        self.progressSignal.connect(self.update_progress_bar)

    def browse_files(self):
        folderPath = QFileDialog.getExistingDirectory(
            self, "Select the directory with your decompiled Android application")
        if folderPath:
            self.ui.apkFolderPathInputBox.setText(folderPath)

    def start_injection(self):
        if not self.ui.apkFolderPathInputBox.text():
            self.show_error("Error", "Folder path is empty.")
        else:
            self.ui.SoupProgressBar.setValue(0)
            self.ui.SoupProgressBar.show()
            self.ui.soupButton.hide()
            self.ui.apkFolderPathInputBox.setReadOnly(True)
            self.ui.browseFilesBtn.hide()
            directory_path = self.ui.apkFolderPathInputBox.text()
            self.find_smali_files(directory_path)

    def find_smali_files(self, directory):
        print("Processing apk...")
        filetypes = ['.smali']
        smali_files = []

        for current_folder, _, files in os.walk(directory):
            for file in files:
                for filetype in filetypes:
                    if file.endswith(filetype):
                        smali_files.append(os.path.join(current_folder, file))
        print("Sending mapped smali files to instrument()")
        self.instrument(smali_files)

    def instrument(self, smali_files):
        thread = threading.Thread(
            target=self.instrument_thread, args=(smali_files,))
        thread.start()

    def instrument_thread(self, smali_files):
        total_files = len(smali_files)
        for index, smali_file in enumerate(smali_files, start=1):
            self.update_instrument_progress(index, total_files)
            self.modify_smali_file(smali_file)

        self.progressSignal.emit(0)
        self.injectSignal.emit()

    def modify_smali_file(self, smali_file):
        with open(smali_file, 'r') as file:
            smali_content = file.readlines()

        modified_content = []
        inserting_line = True

        for line in smali_content:
            modified_content.append(line)

            if inserting_line and line.startswith(".method"):
                modified_content.append(
                    "\tinvoke-static {}, Lcom/supertavor/apksoup/CallerCheck;->throwInvisibleException()V\n")
                inserting_line = False

            if line.startswith(".end method"):
                inserting_line = True

        with open(smali_file, 'w') as file:
            file.writelines(modified_content)

    def inject(self):
        try:
            source_path = "res/supertavor/"
            destination_path = os.path.join(
                self.ui.apkFolderPathInputBox.text(), "com/supertavor/")
            os.makedirs(destination_path, exist_ok=True)

            total_items = len(os.listdir(source_path))
            for index, item in enumerate(os.listdir(source_path), start=1):
                self.update_inject_progress(index, total_items)
                source_item = os.path.join(source_path, item)
                destination_item = os.path.join(destination_path, item)
                if os.path.isdir(source_item):
                    shutil.copytree(source_item, destination_item)
                else:
                    shutil.copy2(source_item, destination_item)

            print("Injection complete!")
            self.progress_timer.stop()
            self.progressSignal.emit(100)
            self.ui.soupButton.show()
            self.ui.apkFolderPathInputBox.setReadOnly(False)
            self.ui.browseFilesBtn.show()
            self.ui.SoupProgressBar.hide()
        except Exception as e:
            self.handle_error("An error occurred during injection:", e)

    def update_instrument_progress(self, current_file, total_files):
        progress = int((current_file / total_files) * 50)
        self.update_progress_bar(progress)

    def update_inject_progress(self, current_item, total_items):
        progress = int(50 + (current_item / total_items) * 50)
        self.update_progress_bar(progress)

    def update_progress_bar(self, value):
        self.ui.SoupProgressBar.setValue(value)

    def update_progress(self):
        current_value = self.ui.SoupProgressBar.value()
        if current_value < 100:
            new_value = current_value + 1
            self.ui.SoupProgressBar.setValue(new_value)
        else:
            self.progress_timer.stop()
            self.ui.SoupProgressBar.hide()
            self.ui.soupButton.show()
            self.ui.apkFolderPathInputBox.setReadOnly(False)
            self.ui.browseFilesBtn.show()
            print("Process complete!")

    def show_error(self, title, message):
        QMessageBox.critical(self, title, message)

    def handle_error(self, message, error):
        error_message = f"{message}\n\nError details: {str(error)}"
        self.show_error("Error", error_message)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    window = APKSoupMain()
    window.show()
    sys.exit(app.exec_())


import ipdb

def parse(file):
    with open(file) as f:
        fish = []
        for char in f.read().splitlines()[0].split(","):
            fish.append(int(char))
    return fish

def pass_day(fishes):
    next_day = []
    for fish in fishes:
        if fish == 0:
            next_day.append(6)
            next_day.append(8)
        else:
            next_day.append(fish-1)
    return next_day

def copied_solution(init, days):
    state = [init.count(i) for i in range(9)]
    for day in range(days):
        state = state[1:] + state[:1]
        state[6] += state[8]
    return sum(state)

def main():
    file = "/home/gyogy/code/advent/inputs/06"
    fish = parse(file)
    print(copied_solution(fish, 80))
    print(copied_solution(fish, 256))

if __name__ == "__main__":
    main()


# encoding: utf-8
from django.contrib import admin, messages
from django.contrib.contenttypes.generic import GenericTabularInline
from treebeard.admin import TreeAdmin
from django.utils.text import capfirst
from django.utils.translation import ugettext_lazy as _
from django.forms import formsets
from django.forms.models import BaseInlineFormSet
from django.db.models.base import ModelBase
from django.utils.safestring import mark_safe
from treebeard.forms import movenodeform_factory
from eav.admin import BaseEntityAdmin, BaseSchemaAdmin, BaseEntityInlineFormSet, BaseEntityInline
from Cottage_Club.main.forms import CottageDynamicCategoryForm, CottageDynamicChildForm, \
    CottageDynamicForm, SchemaForm, image_form_factory, \
    MyChoiceAdminForm, DatePriceForm
from Cottage_Club.main.models import Category, Cottage, Schema, Choice, Image, Attribute, MpttTest, DatePrices
from mptt.admin import MPTTModelAdmin, MPTTAdminForm
from mptt.forms import TreeNodeChoiceField
from pymorphy import get_morph
from django.conf import settings

import re

morph = get_morph(settings.PYMORPHY_DICTS['ru']['dir'], 'cdb')

class I18nLabel():
    def __init__(self, function):
        self.target = function
        self.app_label = u''

    def rename(self, f, name=u''):
        def wrapper(*args, **kwargs):
            extra_context = kwargs.get('extra_context', {})
            if 'delete_view' != f.__name__:
                extra_context['title'] = self.get_title_by_name(f.__name__, args[1], name)
            else:
                extra_context['object_name'] = morph.inflect_ru(name, u'вн').lower()
            kwargs['extra_context'] = extra_context
            return f(*args, **kwargs)
        return wrapper

    def get_title_by_name(self, name, request=None, obj_name=u''):
        if request is None:
            request = {}
        if 'add_view' == name:
            return _('Add %s') % morph.inflect_ru(obj_name, u'вн,стр').lower()
        elif 'change_view' == name:
            return _('Change %s') % morph.inflect_ru(obj_name, u'вн,дст').lower()
        elif 'changelist_view' == name:
            if 'pop' in request.GET:
                title = _('Select %s')
            else:
                title = _('Select %s to change')
            return title % morph.inflect_ru(obj_name, u'вн,стр').lower()
        else:
            return ''

    def wrapper_register(self, model_or_iterable, admin_class=None, **option):
        if isinstance(model_or_iterable, ModelBase):
            model_or_iterable = [model_or_iterable]
        for model in model_or_iterable:
            if admin_class is None:
                admin_class = type(model.__name__+'Admin', (admin.ModelAdmin,), {})
            self.app_label = model._meta.app_label
            current_name = model._meta.verbose_name.upper()
            admin_class.add_view = self.rename(admin_class.add_view, current_name)
            admin_class.change_view = self.rename(admin_class.change_view, current_name)
            admin_class.changelist_view = self.rename(admin_class.changelist_view, current_name)
            admin_class.delete_view = self.rename(admin_class.delete_view, current_name)
        return self.target(model, admin_class, **option)

    def wrapper_app_index(self, request, app_label, extra_context=None):
        if extra_context is None:
            extra_context = {}
        extra_context['title'] = _('%s administration') % _(capfirst(app_label))
        return self.target(request, app_label, extra_context)

    def register(self):
        return self.wrapper_register

    def index(self):
        return self.wrapper_app_index

admin.site.register = I18nLabel(admin.site.register).register()
admin.site.app_index = I18nLabel(admin.site.app_index).index()

def message_wrapper(f):
    def wrapper(self, request, message, level=messages.INFO, extra_tags='', fail_silently=False):
        gram_info = morph.get_graminfo( self.model._meta.verbose_name.upper() )[0]
        if -1 != message.find(u'"'):
            """
            Message about some action with a single element
            """
            words = [w for w in re.split("( |\\\".*?\\\".*?)", message) if w.strip()]
            form = gram_info['info'][:gram_info['info'].find(',')]
            message = u' '.join(words[:2])
            for word in words[2:]:
                if not word.isdigit():
                    word = word.replace(".", "").upper()
                    try:
                        info = morph.get_graminfo(word)[0]
                        if u'КР_ПРИЛ' != info['class']:
                            word = morph.inflect_ru(word, form).lower()
                        elif 0 <= info['info'].find(u'мр'):
                            word = morph.inflect_ru(word, form, u'КР_ПРИЧАСТИЕ').lower()
                        else:
                            word = word.lower()
                    except IndexError:
                        word = word.lower()
                message += u' ' + word
        else:
            """
            Message about some action with a group of elements
            """
            num = int(re.search("\d", message).group(0))
            words = message.split(u' ')
            message = words[0]
            pos = gram_info['info'].find(',')
            form = gram_info['info'][:pos] + u',' + u'ед' if 1 == num else u'мн'
            for word in words[1:]:
                if not word.isdigit():
                    word = word.replace(".", "").upper()
                    info = morph.get_graminfo(word)[0]
                    if u'КР_ПРИЛ' != info['class']:
                        word = morph.pluralize_inflected_ru(word, num).lower()
                    else:
                        word = morph.inflect_ru(word, form, u'КР_ПРИЧАСТИЕ').lower()
                message += u' ' + word

        message += '.'
        return f(self, request, capfirst(message), level, extra_tags, fail_silently)
    return wrapper

admin.ModelAdmin.message_user = message_wrapper(admin.ModelAdmin.message_user)

ImageAdminForm = image_form_factory()

class SchemaForMpttTestInline(admin.StackedInline):
    model = MpttTest.schemas.through
    extra = 0

class DatePriceInline(admin.StackedInline):
    model = DatePrices
    form = DatePriceForm

class MPTTAdminMixin(object):
    form = MPTTAdminForm

    def formfield_for_foreignkey(self, db_field, request, **kwargs):
        from mptt.models import MPTTModel, TreeForeignKey
        if issubclass(db_field.rel.to, MPTTModel) \
                and not isinstance(db_field, TreeForeignKey) \
                and not db_field.name in self.raw_id_fields:
            defaults = dict(form_class=TreeNodeChoiceField, queryset=db_field.rel.to.objects.all(), required=False)
            defaults.update(kwargs)
            kwargs = defaults
        return super(MPTTAdminMixin, self).formfield_for_foreignkey(db_field,
                                                                    request,
                                                                    **kwargs)

class MPTTTabularInlineAdmin(MPTTAdminMixin, admin.TabularInline):
    model = MpttTest
    extra = 0
    pass

class CustomizedMpttAdmin(MPTTModelAdmin):
    inlines = [MPTTTabularInlineAdmin, SchemaForMpttTestInline]

class ImagesInline(GenericTabularInline):
    model = Image
    readonly_fields = ('image_tag', )
    extra = 0
    ct_field_name = 'content_type'
    id_field_name = 'object_id'

class CottageFormSet(BaseEntityInlineFormSet):
    pass

class CottageAdminInline(admin.StackedInline):
    model = Cottage
    form = CottageDynamicCategoryForm
    exclude = ('sib_order', 'is_banner', 'is_recommended', 'slug')
    readonly_fields = ('parent', )
    formset = CottageFormSet
    extra = 0

    def get_fields(self, request, obj=None):
        if self.fields:
            return self.fields
        form = self.get_formset(request, obj, fields=None).form
        return list(form.base_fields) + list(self.get_readonly_fields(request, obj))

    def get_fieldsets(self, request, obj=None):
        declared_fieldsets = self.declared_fieldsets
        if declared_fieldsets:
            return declared_fieldsets

        if self.fieldsets:
            return self.fieldsets
        values = self.get_fields(request, obj)
        return [(None, {'fields': values})]

class CottageAdminInlineCottage(CottageAdminInline):
    form = CottageDynamicChildForm
    readonly_fields = ('category',)

class AttributeInline(GenericTabularInline):
    model = Attribute
    extra = 0
    fields = ('description', 'is_separator', 'order')
    ct_field = 'entity_type'
    ct_fk_field = 'entity_id'

    def has_add_permission(self, request):
        return False

class SchemaInline(admin.StackedInline):
    model = Schema
    form = SchemaForm
    extra = 0

class SchemaForCategoryInline(admin.StackedInline):
    model = Category.schemas.through
    extra = 0

class CottageAdmin(TreeAdmin):
    form = CottageDynamicForm
    inlines = (ImagesInline, CottageAdminInlineCottage, DatePriceInline, AttributeInline)
    change_form_template = "admin/cottage_change_form.html"
    eav_fieldsets = None
    exclude = ('sib_order', 'slug')

    def render_change_form(self, request, context, **kwargs):
        """
        Wrapper for ModelAdmin.render_change_form. Replaces standard static
        AdminForm with an EAV-friendly one. The point is that our form generates
        fields dynamically and fieldsets must be inferred from a prepared and
        validated form instance, not just the form class. Django does not seem
        to provide hooks for this purpose, so we simply wrap the view and
        substitute some data.
        """
        form = context['adminform'].form
        formset = context['inline_admin_formsets']

        all_fields = form.fields.keys()
        model_fields = form.base_fields.keys()
        eav_fields = filter(lambda x: x not in model_fields, all_fields)

        if self.eav_fieldsets:
            fieldsets_eav = self.eav_fieldsets + (('Attributes', {'classes': ('collapse',), 'fields': tuple(eav_fields)}),)
            fieldsets = fieldsets_eav
        else:
            # or infer correct data from the form
            fieldsets = [(None, {'fields': all_fields})]

        adminform = admin.helpers.AdminForm(form, fieldsets,
                                            self.prepopulated_fields)
        inline_media = []
        if formset:
            if len(formset) > 1:
                for fset in formset:
                    if inline_media:
                        inline_media += fset.media
                    else:
                        inline_media = fset.media
            else:
                inline_media = formset[0].media

        media = mark_safe(self.media + adminform.media + inline_media)
        context.update(adminform=adminform, media=media)

        super_meth = super(CottageAdmin, self).render_change_form
        return super_meth(request, context, **kwargs)

    def get_formsets(self, request, obj=None):
        for inline in self.get_inline_instances(request, obj):
            if isinstance(inline, AttributeInline) and (obj is None or not obj.has_attributes()):
                continue
            yield inline.get_formset(request, obj)

    def save_formset(self, request, form, formset, change):
        if not isinstance(formset, CottageFormSet):
            super(CottageAdmin, self).save_formset(request, form, formset, change)
        else:
            if formset.is_valid():
                for inner_form in (formset.extra_forms + formset.initial_forms):
                    if inner_form.is_valid():
                        if not inner_form.instance.category:
                            inner_form.instance.category = form.instance.category
                            inner_form.changed_data.append('category')
            super(CottageAdmin, self).save_formset(request, form, formset, change)

    class Media:
        js = ('js/admin/CottageAdmin.js',)

class CategoryAdmin(TreeAdmin):
    # pass
    form = movenodeform_factory(Category)
    inlines = (CottageAdminInline, SchemaForCategoryInline)

class SchemaAdmin(BaseSchemaAdmin):
    form = SchemaForm
    exclude = ['required', 'filtered']
    list_display = ('title', 'name', 'datatype', 'help_text')
    inlines = (SchemaForCategoryInline, )

class ChoiceAdmin(admin.ModelAdmin):
    form = MyChoiceAdminForm
    list_display = ('title', 'schema_name')

    def schema_name(self, obj):
        return ("%s" % obj.schema.title).capitalize()
    schema_name.short_description = Schema._meta.verbose_name

admin.site.register(Cottage, CottageAdmin)
admin.site.register(Schema, SchemaAdmin)
admin.site.register(Choice, ChoiceAdmin)
admin.site.register(Category, CategoryAdmin)


from collections.abc import Callable, Iterable, Sequence

import pytest

# not ideal, but for now access this member.
# TODO: find out if it's ok for the fetcher to have the API as
# a dependency, b/c that is where this function belongs.
from cmk.fetchers._snmpscan import _evaluate_snmp_detection

from cmk.agent_based.v1 import (
    all_of,
    any_of,
    contains,
    endswith,
    exists,
    matches,
    not_contains,
    not_endswith,
    not_exists,
    not_startswith,
    startswith,
)

_SpecCreator = Callable[[str, str], list[list[tuple[str, str, bool]]]]

def _make_oid_getter(return_value: str) -> Callable[[str], str]:
    def getter(_oid: str) -> str:
        return return_value

    return getter

def _test_atomic_relation(
    relation: _SpecCreator, inverse: _SpecCreator, value: str, testcases: Iterable[tuple[str, bool]]
) -> None:
    spec = relation(".1.2.3", value)
    inv_spec = inverse(".1.2.3", value)
    for test, result in testcases:
        assert (
            _evaluate_snmp_detection(detect_spec=spec, oid_value_getter=_make_oid_getter(test))
            is result
        )
        assert (
            _evaluate_snmp_detection(detect_spec=inv_spec, oid_value_getter=_make_oid_getter(test))
            is not result
        )

@pytest.mark.parametrize(
    "value, testcases",
    [
        (
            "foo",
            [
                ("", False),
                ("foo", True),
                ("foobar", True),
                ("foobarfoo", True),
                ("barfoo", True),
                ("barfoobar", True),
                ("fürwahr", False),
            ],
        ),
        (
            "bo?",
            [
                ("boo", False),
                ("bo?nee", True),
                ("hobo?nee", True),
                ("hallobo?", True),
            ],
        ),
    ],
)
def test_contains(value: str, testcases: Sequence[tuple[str, bool]]) -> None:
    _test_atomic_relation(contains, not_contains, value, testcases)

@pytest.mark.parametrize(
    "value, testcases",
    [
        (
            "foo",
            [
                ("", False),
                ("foo", True),
                ("foobar", True),
                ("foobarfoo", True),
                ("barfoo", False),
                ("barfoobar", False),
                ("fürwahr", False),
            ],
        ),
        (
            "bo?",
            [
                ("boo", False),
                ("bo?nee", True),
                ("hobo?nee", False),
                ("hallobo?", False),
            ],
        ),
    ],
)
def test_startswith(value: str, testcases: Sequence[tuple[str, bool]]) -> None:
    _test_atomic_relation(startswith, not_startswith, value, testcases)

@pytest.mark.parametrize(
    "value, testcases",
    [
        (
            "foo",
            [
                ("", False),
                ("foo", True),
                ("foobar", False),
                ("foobarfoo", True),
                ("barfoo", True),
                ("barfoobar", False),
                ("fürwahr", False),
            ],
        ),
        (
            "bo?",
            [
                ("boo", False),
                ("bo?nee", False),
                ("hobo?nee", False),
                ("hallobo?", True),
            ],
        ),
    ],
)
def test_endswith(value: str, testcases: Sequence[tuple[str, bool]]) -> None:
    _test_atomic_relation(endswith, not_endswith, value, testcases)

@pytest.mark.parametrize(
    "test, result",
    [
        ("", True),
        ("foo", True),
        ("foobar", True),
        ("foobarfoo", True),
        ("barfoo", True),
        ("barfoobar", True),
        ("fürwahr", True),
    ],
)
def test_exists(test: str, result: bool) -> None:
    assert (
        _evaluate_snmp_detection(
            detect_spec=exists(".1.2.3"), oid_value_getter=_make_oid_getter(test)
        )
        is result
    )
    assert (
        _evaluate_snmp_detection(
            detect_spec=not_exists(".1.2.3"), oid_value_getter=_make_oid_getter(test)
        )
        is not result
    )

def test_all_of_associative() -> None:
    spec1 = matches(".1", "1?")
    spec2 = matches(".2", "2?")
    spec3 = matches(".3", "3?")

    assert all_of(spec1, spec2, spec3) == all_of(all_of(spec1, spec2), spec3)

def test_any_of_assotiative() -> None:
    spec1 = matches(".1", "1?")
    spec2 = matches(".2", "2?")
    spec3 = matches(".3", "3?")

    assert any_of(spec1, spec2, spec3) == any_of(any_of(spec1, spec2), spec3)

def test_any_of_all_of_distibutive() -> None:
    spec1 = matches(".1", "1?")
    spec2 = matches(".2", "2?")
    spec3 = matches(".3", "3?")
    spec4 = matches(".4", "4?")

    assert all_of(
        any_of(spec1, spec2),
        any_of(spec3, spec4),
    ) == any_of(
        all_of(spec1, spec3),
        all_of(spec1, spec4),
        all_of(spec2, spec3),
        all_of(spec2, spec4),
    )


from tkinter import *
from tkinter import ttk
import math
# ---------------------------- CONSTANTS ------------------------------- #
PINK = "#e2979c"
RED = "#e7305b"
GREEN = "#9bdeac"
YELLOW = "#f7f5dd"
FONT_NAME = "Courier"
WORK_MIN = 25
SHORT_BREAK_MIN = 5
LONG_BREAK_MIN = 20

reps = 0
timer = None

# ---------------------------- TIMER RESET ------------------------------- #
def reset():
    global reps
    window.after_cancel(timer)
    timer_label.config(text="Timer")
    canvas.itemconfig(timer_text, text="00:00")
    checkmark_label.config(text="")
    reps = 0

# ---------------------------- TIMER MECHANISM ------------------------------- #
def start():
    global reps
    work_sec = WORK_MIN * 60
    short_break_sec = SHORT_BREAK_MIN * 60
    long_break_sec = LONG_BREAK_MIN * 60
    reps += 1

    if reps % 8 == 0:
        count_down(long_break_sec)
        timer_label.config(text="Break", fg=RED)

    elif reps % 2 == 0:
        count_down(short_break_sec)
        timer_label.config(text="Break", fg=PINK)

    else:
        timer_label.config(text="WORK!", fg=GREEN)
        # reps == 0 or reps == 3 or reps == 5 or reps == 7:
        count_down(work_sec)

# ---------------------------- COUNTDOWN MECHANISM ------------------------------- #
def count_down(count):
    global reps
    global timer
    count_min = math.floor(count / 60)
    count_sec = count % 60
    if count_sec < 10:
        count_sec = f"0{count_sec}"
    canvas.itemconfig(timer_text, text=f"{count_min}:{count_sec}")
    if count > 0:
        timer = window.after(1000, count_down, count - 1)
    else:
        start()
        if reps % 2 == 0:
            checkmarks = "✓" * int(reps / 2)
            checkmark_label.config(text=checkmarks)

# ---------------------------- UI SETUP ------------------------------- #
window = Tk()
window.title("Pomodoro")
window.config(padx=100, pady=50, bg=YELLOW)

canvas = Canvas(width=200, height=224, bg=YELLOW, highlightthickness=0)
tomato_img = PhotoImage(file="tomato.gif")
canvas.create_image(100, 112, image=tomato_img)
timer_text = canvas.create_text(100, 112, text="00:00", fill="white", font=(FONT_NAME, 35, "bold"))
canvas.grid(column=2, row=2)

timer_label = Label(text="Timer", fg=GREEN, bg=YELLOW, font=(FONT_NAME, 50))
timer_label.grid(column=2, row=1)

start_button = ttk.Button(text="Start", command=start)
start_button.grid(column=1, row=3)

reset_button = ttk.Button(text="Reset", command=reset)
reset_button.grid(column=3, row=3)

checkmark_label = Label(fg=GREEN, bg=YELLOW, font=(FONT_NAME, 30))
checkmark_label.grid(column=2, row=4)

window.mainloop()


# |---------------------------------------
# |                                      |
# |     This is a Board creator class    |
# |                                      |
# |   This class is used with the Agent  |
# |     to create a working learning     |
# |                                      |
# ----------------------------------------

import numpy as np

class Board:

    def __init__(self, rows, cols, start_position = (2, 0), win_position = (0, 3), lose_position = (1, 3), walls = [(1, 1)]):

        self.rows = rows
        self.cols = cols

        self.walls = walls
        self.board = np.zeros([rows, cols])
        for wall in walls:
            self.board[wall] = -1

        self.start_position = start_position

        self.end = False

        self.determine = True

        self.win_state = win_position
        self.lose_state = lose_position


import unittest
from collections import Counter

import numpy as np
import scipy.sparse as sparse
import torch

from metal.utils import pred_to_prob, rargmax, recursive_merge_dicts, split_data

class UtilsTest(unittest.TestCase):
    def test_rargmax(self):
        x = np.array([2, 1, 2])
        np.random.seed(1)
        self.assertEqual(sorted(list(set(rargmax(x) for _ in range(10)))), [0, 2])

    def test_pred_to_prob(self):
        x = torch.tensor([1, 2, 2, 1])
        target = torch.tensor([[1, 0], [0, 1], [0, 1], [1, 0]])
        self.assertTrue(
            (pred_to_prob(x, 2).float() == target.float()).sum()
            == torch.prod(torch.tensor(target.shape))
        )

    def test_recursive_merge_dicts(self):
        x = {"foo": {"Foo": {"FOO": 1}}, "bar": 2, "baz": 3}
        y = {"FOO": 4, "bar": 5}
        z = {"foo": 6}
        w = recursive_merge_dicts(x, y, verbose=False)
        self.assertEqual(w["bar"], 5)
        self.assertEqual(w["foo"]["Foo"]["FOO"], 4)
        with self.assertRaises(ValueError):
            recursive_merge_dicts(x, z, verbose=False)

    def test_split_data(self):
        N = 1000
        K = 4

        X = np.arange(0, N)
        Y = np.random.randint(0, K, size=N).astype(int)

        # Creates splits of correct size
        splits = [800, 100, 100]
        Xs_1 = split_data(X, splits=splits, shuffle=False)
        for i, count in enumerate(splits):
            self.assertEqual(len(Xs_1[i]), count)

        # Accepts floats or ints
        splits = [0.8, 0.1, 0.1]
        Xs_2 = split_data(X, splits=splits, shuffle=False)
        for split in range(len(splits)):
            self.assertTrue(np.array_equal(Xs_2[split], Xs_1[split]))

        # Shuffles correctly
        Xs_3 = split_data(X, splits=splits, shuffle=True, seed=123)
        self.assertNotEqual(Xs_3[0][0], Xs_2[0][0])

        # Indices only
        splits = [0.8, 0.1, 0.1]
        Ys = split_data(Y, splits=splits, shuffle=False, index_only=True)
        self.assertGreater(max(Ys[0]), K)

        # Handles multiple inputs
        Xs, Ys = split_data(X, Y, splits=splits, shuffle=True, seed=123)
        self.assertEqual(Ys[0][0], Y[Xs[0][0]])

        # Confirm statification (correct proportion of labels in each split)
        Ys = split_data(Y, splits=splits, stratify_by=Y, seed=123)
        counts = [Counter(Y) for Y in Ys]
        for y in np.unique(Y):
            ratio0 = counts[0][y] / len(Ys[0])
            ratio1 = counts[1][y] / len(Ys[1])
            ratio2 = counts[2][y] / len(Ys[2])
            self.assertLess(abs(ratio0 - ratio1), 0.05)
            self.assertLess(abs(ratio0 - ratio2), 0.05)

        # Handles scipy.sparse matrices
        Z = sparse.csr_matrix([[1, 0, 1, 2], [0, 3, 0, 3], [1, 2, 3, 4], [5, 4, 3, 2]])
        splits = [0.75, 0.25]
        Zs = split_data(Z, splits=splits, shuffle=True, seed=123)
        self.assertEqual(Zs[0].shape, (3, 4))

        # Handles torch.Tensors
        W = torch.Tensor([[1, 0, 1, 2], [0, 3, 0, 3], [1, 2, 3, 4], [5, 4, 3, 2]])
        splits = [0.75, 0.25]
        Ws = split_data(W, splits=splits, shuffle=True, seed=123)
        self.assertEqual(Ws[0].shape, (3, 4))

if __name__ == "__main__":
    unittest.main()


import random

def f(x, y):
    x += random.randint(0, 10)
    y += random.randint(0, 5)
    return x / y

def f2(x, y):
    x -= random.randint(0, 10)
    y -= random.randint(0, 5)
    return y / x

def f3(name_file):
    with open(name_file, 'r') as file:
        for i_line in file:
            try:
                num_list = i_line.split()
                res_1 = f(int(num_list[0]), int(num_list[1]))
                res_2 = f2(int(num_list[0]), int(num_list[1]))
                number = random.randint(0, 100)
                with open('result.txt', 'w') as file_2:
                    my_list = sorted([str(res_1), str(number), str(res_2)])
                    file_2.write(' '.join(my_list))
            except Exception as exc:
                print('Что-то пошло не так    ', exc)

f3('coordinates.txt')


import numpy as np
from pymongo import MongoClient
import pyaudio
import click
from bunch import Bunch

db = MongoClient("sqwak.kingofthestack.com")

def get_unrated_sample():
    result = db.sqwaks.sounds.find_one({
        "rating": None
    })
    db_amps = np.array(result["amplitudes"])
    return Bunch({
        "amplitudes": db_amps,
        "id": result["_id"]
    })

def rate_sample(id, rating):
    db.sqwaks.sounds.update_one({
        "_id": id
    }, {
        '$set': {
            "rating": rating
        }
    }, upsert=False)

def play(amplitudes):
    p = pyaudio.PyAudio()

    playback_settings = {
        'format': pyaudio.paFloat32,
        'channels': 1,
        'rate': 44100,
        'output': True,
        'output_device_index': 1
    }

    stream = p.open(**playback_settings)

    # Assuming you have a numpy array called samples
    data = amplitudes.astype(np.float32).tostring()
    stream.write(data)

""" Defines a custom param type for click's input validation """
class FloatInRangeParamType(click.ParamType):

    def __init__(self, min_float=0.0, max_float=1.0):
        self.min_float = min_float
        self.max_float = max_float

    def convert(self, value, param, ctx):
        try:
            number = float(value)
            if (number >= self.min_float and number <= self.max_float):
                return number
            else:
                raise ValueError()
        except ValueError:
            self.fail('%s is not a float within the given range' % value, param, ctx)

def main():
    is_rating = True
    FLOAT_IN_RANGE = FloatInRangeParamType()
    while(is_rating):
        if click.confirm('Do you want to continue?'):
            sample = get_unrated_sample()
            play(sample.amplitudes)
            rating = click.prompt('Rate that sqwak! (0.0 - 1.0)', type=FLOAT_IN_RANGE)
            rate_sample(sample.id, rating)
        else:
            click.echo("Goodbye!")
            is_rating = False

if(__name__ == "__main__"):
    main()


import numpy as np
import pandas as pd
import pickle
import re
import os
import sys
import torch
from torchmodels import TorchMLP
from snorkel.labeling.model import LabelModel
from scipy import sparse
from sklearn.metrics import roc_auc_score
from sklearn.metrics import log_loss
from sklearn.feature_extraction.text import CountVectorizer

AVAILABLEDATASETS = {'IMDB', 'Amazon', 'journalist_photographer', 'professor_physician', 'painter_architect',
                     'professor_teacher'}
BIASBIOSDSETS = {'journalist_photographer', 'professor_physician', 'painter_architect', 'professor_teacher'}
LFTYPES = {'unigram', 'uni_and_bi_grams'}
FEATURES = {'bow', 'tfidf', None, 'unibigram'}
REPLACE_NO_SPACE = re.compile(r"[.;:!\'?,\"()\[\]]")
REPLACE_WITH_SPACE = re.compile(r"(<br\s*/><br\s*/>)|(-)|(/)")

ENGLISH_SENTIMENT_STOP_WORDS = frozenset([
    "a", "about", "above", "across", "after", "afterwards", "again", "against",
    "all", "almost", "alone", "along", "already", "also", "although", "always",
    "am", "among", "amongst", "amoungst", "amount", "an", "and",
    "any", "anyhow", "anyone", "anything", "anyway", "anywhere", "are",
    "around", "as", "at", "back", "be", "became", "because", "become",
    "becomes", "becoming", "been", "before", "beforehand", "behind", "being",
    "below", "beside", "besides", "between", "beyond", "bill", "both",
    "bottom", "but", "by", "call", "can", "co", "con",
    "could", "de", "describe", "detail", "did", "do", "done",
    "down", "due", "during", "each", "eg", "eight", "either", "eleven", "else",
    "empty", "enough", "etc", "even", "ever", "every", "everyone",
    "everything", "everywhere", "few", "fifteen", "fifty", "fill",
    "find", "first", "for", "former", "formerly", "forty",
    "found", "from", "front", "full", "further", "get", "give", "go", "got",
    "had", "has", "have", "he", "hence", "her", "here", "hereafter",
    "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his",
    "how", "however", "hundred", "i", "ie", "if", "in", "inc", "indeed",
    "into", "is", "it", "its", "itself", "keep", "last", "latter",
    "latterly", "ltd", "made", "many", "may", "me",
    "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly",
    "move", "much", "must", "my", "myself", "name", "namely",
    "nevertheless", "next",
    "now", "of", "off", "often", "on",
    "once", "only", "onto", "or", "other", "others", "otherwise", "our",
    "ours", "ourselves", "out", "over", "own", "part", "per", "perhaps",
    "please", "put", "re", "same", "see", "seem", "seemed",
    "seeming", "seems", "several", "she", "show", "side",
    "since", "sincere", "six", "sixty", "so", "some", "somehow", "someone",
    "something", "sometime", "sometimes", "somewhere", "still", "such",
    "system", "take", "than", "that", "the", "their", "them",
    "themselves", "then", "thence", "there", "thereafter", "thereby",
    "therefore", "therein", "thereupon", "these", "they", "thick", "thin",
    "third", "this", "those", "though", "through", "throughout",
    "thru", "thus", "to", "together", "top", "toward", "towards",
    "twelve", "twenty", "un", "under", "until", "up", "upon", "us",
    "very", "via", "was", "we", "well", "were", "what", "whatever", "when",
    "whence", "whenever", "where", "whereafter", "whereas", "whereby",
    "wherein", "whereupon", "wherever", "whether", "which", "while", "whither",
    "who", "whoever", "whole", "whom", "whose", "why", "will", "with",
    "within", "without", "would", "yet", "you", "your", "yours", "yourself",
    "yourselves"])

def generate_ngram_LFs(corpus, lftype, mindf=None, dname=None, labelmap=['negative sentiment', 'positive sentiment']):
    if lftype not in ['unigram', 'uni+biggram']:
        raise NotImplementedError

    if mindf is None:
        mindf = 20.0 / len(corpus)

    vectorizer = CountVectorizer(strip_accents='ascii', stop_words='english', ngram_range=(1, 1),
                                 analyzer='word', max_df=0.90, min_df=mindf, max_features=None,
                                 vocabulary=None, binary=True)
    LFuni = vectorizer.fit_transform(corpus)
    vocablistunigrams = sorted([(value, key) for key, value in vectorizer.vocabulary_.items()], key=lambda x: x[0])
    vocablistunigrams = [x[1] for x in vocablistunigrams]
    LFunineg = LFuni.copy()
    LFunineg.data *= -1

    description = ['contains term: %s. LF vote: %s' % (word, labelmap[1]) for word in vocablistunigrams]
    description += ['contains term: %s. LF vote: %s' % (word, labelmap[0]) for word in vocablistunigrams]
    if lftype == 'unigram':
        return sparse.hstack((LFuni, LFunineg)), description
    else:

        if dname in BIASBIOSDSETS:
            stop_words = 'english'
        else:
            # maintain negations etc
            stop_words = ENGLISH_SENTIMENT_STOP_WORDS
        vectorizer = CountVectorizer(strip_accents='ascii', stop_words=stop_words, ngram_range=(2, 2), analyzer='word',
                                     max_df=0.90, min_df=mindf, max_features=None, vocabulary=None, binary=True)
        LFbi = vectorizer.fit_transform(corpus)
        vocablistbigrams = sorted([(value, key) for key, value in vectorizer.vocabulary_.items()], key=lambda x: x[0])
        vocablistbigrams = [x[1] for x in vocablistbigrams]

        LFbineg = LFbi.copy()
        LFbineg.data *= -1

        descriptionunibi = ['contains term: %s. LF vote: %s' % (word, labelmap[1]) for word in vocablistunigrams]
        descriptionunibi += ['contains bigram: %s. LF vote: %s' % (word, labelmap[1]) for word in vocablistbigrams]
        descriptionunibi += ['contains term: %s. LF vote: %s' % (word, labelmap[0]) for word in vocablistunigrams]
        descriptionunibi += ['contains bigram: %s. LF vote:  %s' % (word, labelmap[0]) for word in vocablistbigrams]

        return sparse.hstack((LFuni, LFbi, LFunineg, LFbineg)), descriptionunibi

def get_final_set(mode, iwssession, npredict=100, r=None):
    """
        Function to create the final set of LFs we will use to model the
        latent class label Y. Returns final sets for all repeated runs of IWS.

        Parameters
        ----------
        mode : {'LSE a','LSE ac','AS'}
            mode to determine final set.
            AS: only uses LFs validated by user.
            LSE a: uses all LFs predicted to have good accuracy>r.
            LSE ac: uses a limited size set of LFs predicted to have good accuracy>r.
        iwssession : Object
            An instance of InteractiveWeakSupervision class
        npredict : int, default = 100
            Number of unverified LFs to add to the final set in LF ac mode
        r : float, default = None
            A threshold value in [0.5, 1]. If none, the r value in iwssession is used

        Returns
        -------
        LFsets: dict
            A dictonary containing, for each run of IWS, final LF indices for each iteration
            {runindex: {iteration : LF_indices})}
    """

    if mode not in {'LSE a', 'LSE ac', 'AS'}:
        raise ValueError('Choose one of: LSE a, LSE ac, AS')

    if mode == 'AS':
        if iwssession.acquisition != mode:
            raise ValueError('mode AS selected, but IWS was not run with AS acquisition')
    else:
        if iwssession.acquisition != 'LSE':
            raise ValueError('LSE mode selected but IWS was not run with LSE acquisition')

    if r is None:
        r = iwssession.straddle_threshold
    else:
        if not 0.5 <= r <= 1:
            raise ValueError('Choose r in [0.5,1.0]')

    # how many LFs in labelsequence are from initialization
    ninit = iwssession.nrandom_init + len(iwssession.initial_labels.keys())

    LFsets = {}
    coverage = None
    if mode == 'LSE ac':
        coverage = np.asarray((iwssession.LFs != 0).sum(0)).flatten() / iwssession.LFs.shape[0]

    for runidx, vals in iwssession.rawdatadict.items():
        LFsets[runidx] = {}
        labelvector, labelsequence, rawdata, timing, weightvector = vals

        itermax = len(labelsequence) - ninit
        for iteration in range(itermax):
            # get LF indices that we have responses for
            LFidxs = labelsequence[:iteration + ninit + 1]

            # First, grab all LFs that user indicated are good
            # If we are in AS mode, we only use these LFs
            traindxs = [x for x in LFidxs if labelvector[x] == 1]

            # establish LFs predicted to be good
            if mode == 'LSE a':
                pred_mean, testidxs = rawdata[iteration]
                # take all LFs predicted to be >= r
                traindxs += testidxs[pred_mean >= r].tolist()
            elif mode == 'LSE ac':
                pred_mean, testidxs = rawdata[iteration]
                pred_bool = pred_mean >= r
                if np.any(pred_bool):
                    # get coverage, test indices, predictions for LFs predicted to
                    # have accuracy >=r
                    testcoverage = coverage[testidxs]
                    testcoveragetmp = testcoverage[pred_bool]
                    testidxtmp = testidxs[pred_bool]
                    preds = pred_mean[pred_bool]

                    # sort LFs, grab top npredict
                    sortval = np.multiply((2 * preds - 1), testcoveragetmp)
                    idxselect = np.argsort(sortval)[::-1][:npredict]
                    traindxs += testidxtmp[idxselect].tolist()

            LFsets[runidx][iteration] = traindxs

    return LFsets

def get_probabilistic_labels(iwssession, lfsets, device='cuda', gap=20,
                             class_balance=None, uniform=False, verbose=False):
    """
    Function to fit label model, train downstream classifier, and return test set predictions

    Parameters
    ----------
    iwssession : Object
        An instance of InteractiveWeakSupervision class
    lfsets : dict
         A dictonary containing, for each run of IWS, final LF indices for each iteration
            {runindex: {iteration : LF_indices})}
    device : str, default = 'cuda'
        String passed to torch to identify which device to use, e.g. "cpu" or 'cuda:0'
    gap : int, default = 20
        Provide downstream results every "gap" iterations
    class_balance : tuple, default = None
        Class balance tuple (negative class fraction, positive class fraction)
        passed to graphical model, e.g. class_balance = (0.5,0.5)
    uniform : bool, default = False
        Use uniform weighted LFs to obtain label instead of fitting a graphical model to learn weights.
    verbose : bool, default = False
        Print iteration info if true.

    Returns
    -------
    results: dict
        A dictionary containing the probabilistic train labels and a boolean filter index variable for each iws run,
        and each internal iteration. The filter index variable is True for every sample where we have at least one
        non-abstain vote.
        {runindex: {iteration_idx : (prob_labels,filteridx)})}
    """

    results = {}
    # for each run of IWS
    for key, iterdict in lfsets.items():
        results[key] = {}
        # establish which IWS iterations to obtain results for
        itermax = len(iterdict.keys())
        finaliter = itermax - 1
        iters_to_run = list(range(0, itermax, gap))
        if finaliter not in iters_to_run:
            # always obtain results for final iteration
            iters_to_run.append(finaliter)

        for iteration_idx in iters_to_run:
            if verbose:
                print('IWS run: %d' % key, ' iteration: %d' % iteration_idx)
            trainidxs = iterdict[iteration_idx]
            # get seleted LFs

            if uniform:
                LFStmp = iwssession.LFs_csc[:, trainidxs].copy()
                n, m = LFStmp.shape
                weights = np.ones(m)
                rowsums = np.asarray((LFStmp != 0).sum(1)).flatten()
                filteridx = rowsums != 0

                posevidence = ((LFStmp == 1).astype(np.float32)).dot(weights)
                negevidence = ((LFStmp == -1).astype(np.float32)).dot(weights)
                posevidence = np.asarray(posevidence).flatten()
                negevidence = np.asarray(negevidence).flatten()

                posevidence = np.clip(posevidence, 0.0, 700.0)
                negevidence = np.clip(negevidence, 0.0, 700.0)

                bin_posterior = np.exp(posevidence) / (np.exp(posevidence) + np.exp(negevidence))
                bin_posterior = bin_posterior.astype(np.float32)
            else:
                Lambdas = np.asarray(iwssession.LFs_csc[:, trainidxs].todense())
                # create snorkel LF format
                rowsums = (Lambdas != 0).sum(1)
                filteridx = rowsums != 0
                Lambda_snorkel = np.copy(Lambdas)
                Lambda_snorkel[Lambda_snorkel == 0] = -10
                Lambda_snorkel[Lambda_snorkel == -1] = 0
                Lambda_snorkel[Lambda_snorkel == -10] = -1

                # create variable to filter out samples with 0 LF votes

                # train label model
                if 'cuda' in device:
                    torch.cuda.empty_cache()
                    label_model = LabelModel(cardinality=2, verbose=True, device=device)
                    label_model.fit(Lambda_snorkel[filteridx], class_balance=class_balance)
                    torch.cuda.empty_cache()
                else:
                    label_model = LabelModel(cardinality=2, verbose=True)
                    label_model.fit(Lambda_snorkel[filteridx], class_balance=class_balance)

                # get label estimate
                posterior = label_model.predict_proba(Lambda_snorkel)
                bin_posterior = posterior[:, 1].astype(np.float32)

            tmpindicator = np.isnan(bin_posterior)
            if tmpindicator.sum() > 0:
                bin_posterior[tmpindicator] = np.median(bin_posterior[~tmpindicator])

            results[key][iteration_idx] = (bin_posterior, filteridx)
    return results

def end_classifier_from_prob_labels(Xtrain, Xtest, prob_label_dict, device='cuda',
                                    modelparams=None, verbose=False):
    """
    Function to fit label model, train downstream classifier, and return test set predictions

    Parameters
    ----------
    Xtrain : ndarray of shape (n training samples,d features)
        Features for training data
    Xtest : ndarray of shape (n test samples,d features)
        Features for test data
    prob_label_dict : dict
        Dictionary containing probabilistic labels obtained from get_probabilistic_labels
    device : str, default = 'cuda'
        String passed to torch to identify which device to use, e.g. "cpu" or 'cuda:0'
    class_balance : tuple, default = None
        Class balance tuple (negative class fraction, positive class fraction)
        passed to graphical model, e.g. class_balance = (0.5,0.5)
    modelparams : dict, defgault = None
        Dictionary containing sizes of hidden layers and activation functions of the downstream MLP
    verbose : bool, default = False
        Print iteration info if true.

    Returns
    -------
    results: dict
        A dictionary containing test set predictions
        {runindex: {iteration_idx : test_predictions})}
    """
    if modelparams is None:
        modelparams = {
            'h_sizes': [Xtrain.shape[1], 20, 20],
            'activations': [torch.nn.ReLU(), torch.nn.ReLU()]
        }

    # for each run of IWS
    results = {}
    for key, iterdict in prob_label_dict.items():
        results[key] = {}
        for iteration_idx, tup in iterdict.items():
            bin_posterior, filteridx = tup
            if verbose:
                print('IWS run: %d' % key, ' iteration: %d' % iteration_idx)

            # train classifier on label estimate and get test set prediction
            Xtrain_filtered = Xtrain[filteridx]
            probs_train_filtered = bin_posterior[filteridx]
            torch.cuda.empty_cache()
            model = TorchMLP(h_sizes=modelparams['h_sizes'], activations=modelparams['activations'],
                             optimizer='Adam', nepochs=250)

            if 'cuda' in device:
                tdevice = torch.device(device)
                model.model.to(tdevice)
                model.fit(Xtrain_filtered, probs_train_filtered, device=tdevice)
                test_predictions = model.predict_proba(Xtest, device=tdevice)
            else:
                model.fit(Xtrain_filtered, probs_train_filtered)
                test_predictions = model.predict_proba(Xtest)
            results[key][iteration_idx] = test_predictions
    return results

def train_end_classifier(Xtrain, Xtest, iwssession, lfsets, device='cuda', gap=20,
                         class_balance=None, modelparams=None, uniform=False, verbose=False):
    """
    Function to fit label model, train downstream classifier, and return test set predictions

    Parameters
    ----------
    Xtrain : ndarray of shape (n training samples,d features)
        Features for training data
    Xtest : ndarray of shape (n test samples,d features)
        Features for test data
    iwssession : Object
        An instance of InteractiveWeakSupervision class
    lfsets : dict
         A dictonary containing, for each run of IWS, final LF indices for each iteration
            {runindex: {iteration : LF_indices})}
    device : str, default = 'cuda'
        String passed to torch to identify which device to use, e.g. "cpu" or 'cuda:0'
    gap : int, default = 20
        Provide downstream results every "gap" iterations
    class_balance : tuple, default = None
        Class balance tuple (negative class fraction, positive class fraction)
        passed to graphical model, e.g. class_balance = (0.5,0.5)
    modelparams : dict, defgault = None
        Dictionary containing sizes of hidden layers and activation functions of the downstream MLP
    uniform : bool, default = False
        Use uniform weighted LFs to obtain label instead of fitting a graphical model to learn weights.
    verbose : bool, default = False
        Print iteration info if true.

    Returns
    -------
    results: dict
        A dictionary containing the probabilistic test set predictions for each iws run,
        and each internal iteration
        {runindex: {iteration_idx : test_predictions})}
    """
    if modelparams is None:
        modelparams = {
            'h_sizes': [Xtrain.shape[1], 20, 20],
            'activations': [torch.nn.ReLU(), torch.nn.ReLU()]
        }

    results = {}
    # for each run of IWS
    for key, iterdict in lfsets.items():
        results[key] = {}
        # establish which IWS iterations to obtain results for
        itermax = len(iterdict.keys())
        finaliter = itermax - 1
        iters_to_run = list(range(0, itermax, gap))
        if finaliter not in iters_to_run:
            # always obtain results for final iteration
            iters_to_run.append(finaliter)

        for iteration_idx in iters_to_run:
            if verbose:
                print('IWS run: %d' % key, ' iteration: %d' % iteration_idx)
            trainidxs = iterdict[iteration_idx]
            # get seleted LFs

            if uniform:
                LFStmp = np.asarray(iwssession.LFs_csc[:, trainidxs].todense())
                n, m = LFStmp.shape
                weights = np.ones(m)
                rowsums = np.asarray((LFStmp != 0).sum(1)).flatten()
                filteridx = rowsums != 0

                posevidence = ((LFStmp == 1).astype(np.float32)).dot(weights)
                negevidence = ((LFStmp == -1).astype(np.float32)).dot(weights)

                posevidence = np.clip(posevidence, 0.0, 700.0)
                negevidence = np.clip(negevidence, 0.0, 700.0)

                bin_posterior = np.exp(posevidence) / (np.exp(posevidence) + np.exp(negevidence))
                bin_posterior = bin_posterior.astype(np.float32)
            else:
                Lambdas = np.asarray(iwssession.LFs_csc[:, trainidxs].todense())
                # create snorkel LF format
                rowsums = (Lambdas != 0).sum(1)
                filteridx = rowsums != 0
                Lambda_snorkel = np.copy(Lambdas)
                Lambda_snorkel[Lambda_snorkel == 0] = -10
                Lambda_snorkel[Lambda_snorkel == -1] = 0
                Lambda_snorkel[Lambda_snorkel == -10] = -1

                # create variable to filter out samples with 0 LF votes

                # train label model
                if 'cuda' in device:
                    torch.cuda.empty_cache()
                    label_model = LabelModel(cardinality=2, verbose=True, device=device)
                    label_model.fit(Lambda_snorkel[filteridx], class_balance=class_balance)
                    torch.cuda.empty_cache()
                else:
                    label_model = LabelModel(cardinality=2, verbose=True)
                    label_model.fit(Lambda_snorkel[filteridx], class_balance=class_balance)

                # get label estimate
                posterior = label_model.predict_proba(Lambda_snorkel)
                bin_posterior = posterior[:, 1].astype(np.float32)

            tmpindicator = np.isnan(bin_posterior)
            if tmpindicator.sum() > 0:
                bin_posterior[tmpindicator] = np.median(bin_posterior[~tmpindicator])

                # train classifier on label estimate and get test set prediction
            Xtrain_filtered = Xtrain[filteridx]
            probs_train_filtered = bin_posterior[filteridx]
            torch.cuda.empty_cache()
            model = TorchMLP(h_sizes=modelparams['h_sizes'], activations=modelparams['activations'],
                             optimizer='Adam', nepochs=250)

            if 'cuda' in device:
                tdevice = torch.device(device)
                model.model.to(tdevice)
                model.fit(Xtrain_filtered, probs_train_filtered, device=tdevice)
                test_predictions = model.predict_proba(Xtest, device=tdevice)
            else:
                model.fit(Xtrain_filtered, probs_train_filtered)
                test_predictions = model.predict_proba(Xtest)
            results[key][iteration_idx] = test_predictions
    return results

def print_progress(iteration, total, decimals=1, bar_length=100):
    """
    Call in a loop to create terminal progress bar
    @params:
        iteration   - Required  : current iteration (Int)
        total       - Required  : total iterations (Int)
        prefix      - Optional  : prefix string (Str)
        suffix      - Optional  : suffix string (Str)
        decimals    - Optional  : positive number of decimals in percent complete (Int)
        bar_length  - Optional  : character length of bar (Int)
    """
    str_format = "{0:." + str(decimals) + "f}"
    percents = str_format.format(100 * (iteration / float(total)))
    filled_length = int(round(bar_length * iteration / float(total)))
    bar = '█' * filled_length + '-' * (bar_length - filled_length)
    frac = '%d/%d' % (iteration, total)
    sys.stdout.write('\r|%s|%s%s %s' % (bar, percents, '%', frac)),

    if iteration == total:
        sys.stdout.write('\n')
    sys.stdout.flush()

class ProgressBar:
    def __init__(self, decimals=1, bar_length=100):
        self.decimals = str(decimals)
        self.bar_length = bar_length

    def update(self, iteration, total):
        str_format = "{0:." + self.decimals + "f}"
        percents = str_format.format(100.0 * (iteration / float(total)))
        filled_length = int(round(self.bar_length * iteration / float(total)))
        bar = '█' * filled_length + '-' * (self.bar_length - filled_length)
        frac = '%d/%d' % (iteration, total)
        return '|%s|%s%s %s' % (bar, percents, '%', frac)

def preprocess_reviews(reviews):
    """
    Function to clean review text
    """
    reviews = [REPLACE_NO_SPACE.sub("", line.lower()) for line in reviews]
    reviews = [REPLACE_WITH_SPACE.sub(" ", line) for line in reviews]
    return reviews

def evaluate_binary(X, Ytrue, verbose=False):
    """
    Compute metrics for all labeling functions
    given the true binary labels.
    """
    if isinstance(Ytrue, list):
        Ytrue = np.array(Ytrue)
    if 0 in Ytrue:
        Ytrue[Ytrue == 0] = -1

    isnan = np.isnan(Ytrue)
    if isnan.sum() > 0:
        if verbose:
            print('Handling unlabeled samples')
        # ignore unlabeled samples
        X = X.tocsr()[~isnan].tocoo()
        Ytrue = Ytrue[~isnan]

    numpos = np.count_nonzero(Ytrue > 0)
    numneg = np.count_nonzero(Ytrue < 0)

    n = X.shape[0]
    res = X.multiply((Ytrue.reshape(-1, 1) > 0).astype(np.float32))
    tp = (res > 0).sum(0)
    fn = (res < 0).sum(0)

    res = X.multiply(-(Ytrue.reshape(-1, 1) < 0).astype(np.float32))
    tn = (res > 0).sum(0)
    fp = (res < 0).sum(0)

    # fraction correct
    frac_correct = (tp + tn) / n
    frac_correct = np.asarray(frac_correct).flatten()

    # fraction incorrect
    frac_incorrect = (fp + fn) / n
    frac_incorrect = np.asarray(frac_incorrect).flatten()

    # (correct - incorrect) / n
    frac_goodness = (tp + tn - fp - fn) / n
    frac_goodness = np.asarray(frac_goodness).flatten()

    # positives:recall
    val = tp + fp
    val[val == 0] = 1.0
    precisionpos = tp / val
    recallpos = tp / numpos

    # negatives: specificity
    val = tn + fn
    val[val == 0] = 1.0
    precisionneg = tn / val
    recallneg = tn / numneg

    coverage = np.asarray((X != 0).sum(0)).flatten() / X.shape[0]

    precisionpos = np.asarray(precisionpos).flatten()
    recallpos = np.asarray(recallpos).flatten()
    precisionneg = np.asarray(precisionneg).flatten()
    recallneg = np.asarray(recallneg).flatten()

    lfsign = np.asarray(np.sign(X.sum(0))).flatten()
    positive_funcs = lfsign > 0
    negative_funcs = lfsign < 0
    precision = np.zeros(X.shape[1])
    precision[negative_funcs] = precisionneg[negative_funcs]
    precision[positive_funcs] = precisionpos[positive_funcs]
    recall = np.zeros(X.shape[1])
    recall[negative_funcs] = recallneg[negative_funcs]
    recall[positive_funcs] = recallpos[positive_funcs]

    return coverage, precision, recall, frac_correct, frac_incorrect, frac_goodness

def evaluate_complex_binary(X, Ytrue, verbose=False):
    """
    Compute metrics for all labeling functions
    given the true binary labels.

    LFs do not just have to output 1 of the labels
    """
    if isinstance(Ytrue, list):
        Ytrue = np.array(Ytrue)
    if 0 in Ytrue:
        Ytrue[Ytrue == 0] = -1

    isnan = np.isnan(Ytrue)
    if isnan.sum() > 0:
        if verbose:
            print('Handling unlabeled samples')
        # ignore unlabeled samples
        X = X.tocsr()[~isnan].tocoo()
        Ytrue = Ytrue[~isnan]

    numpos = np.count_nonzero(Ytrue > 0)
    numneg = np.count_nonzero(Ytrue < 0)

    n = X.shape[0]
    res = X.multiply((Ytrue.reshape(-1, 1) > 0).astype(np.float32))
    tp = (res > 0).sum(0)
    fn = (res < 0).sum(0)

    res = X.multiply(-(Ytrue.reshape(-1, 1) < 0).astype(np.float32))
    tn = (res > 0).sum(0)
    fp = (res < 0).sum(0)

    # fraction correct
    frac_correct = (tp + tn) / n
    frac_correct = np.asarray(frac_correct).flatten()

    # fraction incorrect
    frac_incorrect = (fp + fn) / n
    frac_incorrect = np.asarray(frac_incorrect).flatten()

    # (correct - incorrect) / n
    frac_goodness = (tp + tn - fp - fn) / n
    frac_goodness = np.asarray(frac_goodness).flatten()

    # positives:recall
    val = tp + fp
    val[val == 0] = 1.0
    precisionpos = tp / val
    recallpos = tp / numpos

    # negatives: specificity
    val = tn + fn
    val[val == 0] = 1.0
    precisionneg = tn / val
    recallneg = tn / numneg

    numvotes = np.asarray((X != 0).sum(0)).flatten()
    coverage = numvotes / X.shape[0]

    precisionpos = np.asarray(precisionpos).flatten()
    recallpos = np.asarray(recallpos).flatten()
    precisionneg = np.asarray(precisionneg).flatten()
    recallneg = np.asarray(recallneg).flatten()

    lfsign = np.asarray(np.sign(X.sum(0))).flatten()
    positive_funcs = lfsign > 0
    negative_funcs = lfsign < 0
    precision = np.zeros(X.shape[1])
    precision[negative_funcs] = precisionneg[negative_funcs]
    precision[positive_funcs] = precisionpos[positive_funcs]
    recall = np.zeros(X.shape[1])
    recall[negative_funcs] = recallneg[negative_funcs]
    recall[positive_funcs] = recallpos[positive_funcs]

    accuracy = (tp + tn) / numvotes
    accuracy = np.asarray(accuracy).flatten()

    return coverage, precision, recall, frac_correct, frac_incorrect, frac_goodness, accuracy

def eval_scores(y_true, y_scores, verbose=True):
    """
    Compute a number of metrics for predicted Y probabilitues
    and return them. Optionally print them in verbose mode.
    """
    auc = roc_auc_score(y_true, y_scores)
    logloss = log_loss(y_true, y_scores)
    brier = np.square(y_true - y_scores).mean()
    if verbose:
        print('Brier score:', brier)
        print('Cross entropy loss:', logloss)
        print('AUC', auc)
    return auc, logloss, brier


# -*- coding: utf-8 -*-

"""Detect specific platform on Linux.
"""

from __future__ import absolute_import

import os
import re
import shlex
import subprocess
import glob

from lionshead.util import spectuple

# These distributions do not produce rolling or unstable releases
STABLE_DISTRIBUTIONS = (
        'centos',
        'redhat',
        'sles',
        'ubuntu'
)

def _get_major_version(dist, version):
    """Given a Linux distribution and "full" version number, return only the
    "major" version number for that platform

    :param dist: Detected Linux distribution
    :param version: Detected version of the given distribution
    """
    if dist == 'ubuntu':
        version = '.'.join(version.split('.')[0:2])
    else:
        version = version.split('.')[0]
    return version

_opensuse_tumbleweed_version = re.compile('\d{8}$')

def _get_stability(dist, version):
    """Determine whether or not the ABI for the given distribution and verison
    can be considered stable.

    :param dist: Detected Linux distribution
    :param version: Detected version of the given distribution
    """
    stability = 'unstable'
    if dist in STABLE_DISTRIBUTIONS:
        stability = 'stable'
    elif dist == 'debian' and not version.endswith('/sid'):
        stability = 'stable'
    elif dist == 'opensuse' and not _opensuse_tumbleweed_version.match(version):
        stability = 'stable'
    return stability

def _parse_source_file(fname):
    with open(fname) as fh:
        return dict([line.split('=', 1) for line in shlex.split(fh.read())])

def _read_os_release():
    files = [ '/etc/os-release',
              '/usr/lib/os-release' ]

    osrel_file = None
    for osrel_file in files:
        # Test for existence since we should probably fail if the file exists
        # but can't be read
        if os.path.exists(osrel_file):
            break
    else:
        return None

    osrel = _parse_source_file(osrel_file)

    if 'ID' in osrel:
        dist = osrel['ID'].lower()
    elif 'NAME' in osrel:
        dist = osrel['NAME'].lower()
    else:
        dist = None

    if dist is None:
        return None

    if 'VERSION_ID' in osrel:
        vers = osrel['VERSION_ID'].lower()
    elif 'VERSION' in osrel:
        vers = osrel['VERSION'].lower()
    else:
        vers = None

    return (dist, vers)

def _read_lsb_release():
    cmd = ['lsb_release', '-a']
    out = None
    lsbrel = None
    try:
        r = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out, err = r.communicate()
    except OSError:
        # Executing lsb_release failed, check for /etc/lsb-release anyway
        try:
            lsbrel = _parse_source_file('/etc/lsb-release')
        except:
            pass

    lsb = {}
    if out is not None:
        # Executing lsb_release succeeded, use this data
        for line in out.splitlines():
            splitline = line.split(':', 1)
            if len(splitline) != 2:
                continue
            val = splitline[1].strip().lower()
            if splitline[0] == 'Distributor ID':
                lsb['distributor'] = val
            elif splitline[0] == 'Release':
                lsb['version'] = val
            elif splitline[0] == 'Description':
                lsb['description'] = val
            elif splitline[0] == 'Codename':
                lsb['codename'] = val
    elif lsbrel is not None:
        # /etc/lsb-release format, if the file even exists, is non-standard.
        # Only the `lsb_release` command is specified by LSB. Nonetheless, some
        # distributions install an /etc/lsb-release as part of the base
        # distribution, but `lsb_release` remains optional.
        if 'DISTRIB_ID' in lsbrel:
            # ubuntu, arch, ... ???
            lsb['distributor'] = lsbrel['DISTRIB_ID'].lower()
        if 'DISTRIB_RELEASE' in lsbrel:
            lsb['version'] = lsbrel['DISTRIB_RELEASE'].lower()

    dist = None
    distributor = lsb.get('distributor', None)
    vers = lsb.get('version', None)
    if distributor is not None:
        if distributor.startswith('redhatenterprise'):
            dist = 'rhel'
        elif distributor == 'archlinux':
            dist = 'arch'
        elif distributor.startswith('suse') and lsb.get('description', '').startswith('opensuse'):
            dist = 'opensuse'
        elif distributor.startswith('suse') and lsb.get('description', '').startswith('suse linux enterprise'):
            dist = 'sles'
        elif distributor == 'debian' and vers == 'testing':
            dist = 'debian'
            vers = lsb.get('codename', None)
            if vers is not None and not vers.endswith('/sid'):
                # This is done to ensure the same tag is generated as with
                # /etc/os-release and the legacy method.
                vers = vers + '/sid'
        else:
            # ubuntu, debian, gentoo, scientific, slackware, ... ?
            dist = distributor.split(None, 1)[0].lower()

    if dist is None:
        return None

    return (dist, vers)

_versionish_string = re.compile('\d+(?:\.\d+)*$')

def _extract_version(release):
    """Attempt to determine the version of a distro based on a string that is
    suspected to contain a version-like element toward the end.
    """
    for s in reversed(release.split()):
        if _versionish_string.match(s):
            return s
    return None

def _extract_distribution(release):
    """Attempt to determine the name of a distro based on a string specified by
    the vendor.
    """
    release = release.lower()
    if release.startswith('red hat enterprise'):
        return 'rhel'
    if release.startswith('suse linux enterprise'):
        return 'sles'
    else:
        # ubuntu, gentoo, scientific, opensuse, slackware, ... ?
        return release.split(None, 1)[0].lower()

def _get_platform_legacy():
    # If any distros need specific handling over the globbing at the end, add
    # them to checks. Derivative distributions should be typically be checked
    # before their upstreams.
    # (file, distro map function, version map function)
    checks = (('/etc/arch-release', lambda x: 'arch', lambda x: 'rolling'), # it's an empty file
              ('/etc/slackware-version', _extract_distribution, _extract_version), # -version
              (None, _extract_distribution, _extract_version), # default, will check /etc/*-release
              ('/etc/debian_version', lambda x: 'debian', lambda x: x)) # only contains the release version *or* codename
    for check, dist_f, vers_f in checks:
        if check is None:
            files = glob.glob('/etc/*-release')
            files = filter(lambda x: x not in ('/etc/os-release',
                                               '/etc/lsb-release'), files)
        else:
            files = [check]
        for relfile in files:
            try:
                contents = open(relfile).readline().strip()
                dist = dist_f(contents)
                vers = vers_f(contents)
                if dist is not None:
                    return (dist, vers)
            except (OSError, IOError):
                pass
    return None

def get_specific_platform():
    """Attempt to determine the current platform (Linux distribution + version)
    by (in this order):

    1. Parsing /etc/os-release, if it exists
    2. Parsing the output of `lsb_release -a`, if it exists
    3. Parsing the file /etc/lsb-release, if it exists
    4. Attempting to read distro-specific /etc/*-release files

    If the platform can be determine, a 4-member tuple will be returned:
    (distribution, major version, full version, abi stability). If not, None is
    returned.
    """
    ran_legacy = False
    plat = _read_os_release()
    if plat is None:
        plat = _read_lsb_release()
    if plat is None:
        plat = _get_platform_legacy()
        ran_legacy = True
    if plat is not None:
        dist, vers = plat
        # TODO: should probably try to get the version from lsb_release before
        # legacy
        if vers is None and not ran_legacy:
            legacy = _get_platform_legacy()
            if legacy is not None:
                vers = legacy[1]
        major = _get_major_version(dist, vers)
        stability = _get_stability(dist, vers)
        plat = spectuple(dist, major, vers, stability)
    return plat


from data_structure import *
import pdb

def test_module(module, script_ls):
    for script in script_ls:
        print(script)
        op, data, answer = script

        if op == 'BSTIterator':
            root = binary_tree_v3.ls_to_tree(data[0])
            obj = module.BSTIterator(root)
            assert None == answer, None
        elif op == 'next':
            val = obj.next()
            assert val == answer, val
        elif op == 'hasNext':
            flag = obj.hasNext()
            assert flag == answer, flag
        else:
            assert False, op

        obj.dump()

def run(sln):
    if True:
        obj = test_module(
            sln.module,
            [
                ("BSTIterator", [[7, 3, 15, None, None, 9, 20]], None),
                ("next",        [], 3),
                ("next",        [], 7),
                ("hasNext",     [], True),
                ("next",        [], 9),
                ("hasNext",     [], True),
                ("next",        [], 15),
                ("hasNext",     [], True),
                ("next",        [], 20),
                ("hasNext",     [], False),
            ]
        )