#!/usr/bin/python

import json
import time
import unittest

import mock
import pandas as pd

from trash.my_bittrex import test_utils
from trash.my_bittrex import volume

class TestMarket(unittest.TestCase):
    def setUp(self):
        self.market = volume.Market(json_blob=market0())

    def test_summaries(self):
        df = self.market._summaries
        self.assertItemsEqual(df.shape, (4, 15))

    def test_currency_value(self):
        market = self.market
        cost = market.currency_value(['XXX', 'XXX'])
        self.assertAlmostEqual(cost, 1, 3)

        cost = market.currency_value(['BBB', 'AAA'])
        self.assertAlmostEqual(cost, 0.1, 3)

        cost = market.currency_value(['AAA', 'BBB'])
        self.assertAlmostEqual(cost, 10, 3)

        cost = market.currency_value(['BBB', 'AAA', 'USDT'])
        self.assertAlmostEqual(cost, 0.2, 3)

        cost = market.currency_value(['CCC', 'AAA'])
        self.assertAlmostEqual(cost, 0.01, 3)

        cost = market.currency_value(['CCC', 'AAA', 'USDT'])
        self.assertEqual(cost, 0.02)

        cost = market.currency_value(['CCC', 'BBB', 'AAA'])
        self.assertAlmostEqual(cost, 0.01, 3)

        cost = market.currency_value(['CCC', 'BBB', 'AAA', 'USDT'])
        self.assertAlmostEqual(cost, 0.02)

        cost = market.currency_value(['AAA', 'BBB', 'CCC'])
        self.assertAlmostEqual(cost, 100, 3)

        cost = market.currency_value(['AAA', 'CCC'])
        self.assertAlmostEqual(cost, 100, 3)

        cost = market.currency_value(['AAA', 'USDT'])
        self.assertAlmostEqual(cost, 2, 3)

        cost = market.currency_value(['USDT', 'AAA'])
        self.assertAlmostEqual(cost, 0.5, 3)

        cost = market.currency_value(['AAA', 'DDD'])
        self.assertEqual(cost, 0)

    @mock.patch('my_bittrex.volume.client.get_market_summaries')
    def test_direct_volume_in_base(self, mocked_market):
        """
        l = [('USDT-BTC', 1000, 1), ('USDT-ETH', 1000, 1), ('BTC-ETH', 1, 1)]
        """
        market = volume.Market(json_blob=market1())
        computed = market._direct_volume_in_base('BTC', 'BTC')
        self.assertEqual(computed, 0)

        computed = market._direct_volume_in_base('BTC', 'XXX')
        self.assertEqual(computed, 0)

        computed = market._direct_volume_in_base('BTC', 'USDT')
        self.assertEqual(computed, 1)

        computed = market._direct_volume_in_base('USDT', 'BTC')
        self.assertEqual(computed, 1000)

        computed = market._direct_volume_in_base('ETH', 'BTC')
        self.assertEqual(computed, 1)

        computed = market._direct_volume_in_base('BTC', 'ETH')
        self.assertEqual(computed, 1)

        computed = market._direct_volume_in_base('ETH', 'USDT')
        self.assertEqual(computed, 1)

        computed = market._direct_volume_in_base('USDT', 'ETH')
        self.assertEqual(computed, 1000)

    @mock.patch('my_bittrex.volume.client.get_market_summaries')
    def test_direct_volume_in_base2(self, mocked_market):
        # Market is: [('USDT-BTC', 2000, 10), ('USDT-ETH', 500, 10), ('BTC-ETH', 0.25, 10)]
        market = volume.Market(json_blob=market2())
        computed = market._direct_volume_in_base('BTC', 'USDT')
        self.assertEqual(computed, 10)

        computed = market._direct_volume_in_base('USDT', 'BTC')
        self.assertEqual(computed, 20000)

        computed = market._direct_volume_in_base('ETH', 'BTC')
        self.assertEqual(computed, 10)

        computed = market._direct_volume_in_base('BTC', 'ETH')
        self.assertEqual(computed, 2.5)

        computed = market._direct_volume_in_base('ETH', 'USDT')
        self.assertEqual(computed, 10)

        computed = market._direct_volume_in_base('USDT', 'ETH')
        self.assertEqual(computed, 5000)

    def test__volume_in_usdt_1(self):
        # Market is: [('USDT-BTC', 2000, 10), ('USDT-ETH', 500, 10), ('BTC-ETH', 0.25, 10)]
        market = volume.Market(json_blob=market2())
        computed = market._volume_in_usdt('ETH')
        expected = 10 * 500 + 10 * 500
        self.assertEqual(computed, expected)

    def test__volume_in_btc_1(self):
        # Market is: [('USDT-BTC', 2000, 10), ('USDT-ETH', 500, 10), ('BTC-ETH', 0.25, 10)]
        market = volume.Market(json_blob=market2())
        computed = market._volume_in_btc('ETH')
        expected = 10 * 0.25 + 10 * 500 / 2000.0
        self.assertEqual(computed, expected)

    def test__volume_in_eth_1(self):
        # Market is: [('USDT-BTC', 2000, 10), ('USDT-ETH', 500, 10), ('BTC-ETH', 0.25, 10)]
        market = volume.Market(json_blob=market2())
        computed = market._volume_in_eth('USDT')
        expected = 10 + 10 * 2000 / 500.0
        self.assertEqual(computed, expected)

    def test_currency_volume_in_base_raises(self):
        market = volume.Market(json_blob=market2())
        self.assertRaises(AssertionError, market.currency_volume_in_base, 'XXX', 'BTC')

    def test_currency_volume_in_base_1(self):
        # Market is: [('USDT-BTC', 2000, 10), ('USDT-ETH', 500, 10), ('BTC-ETH', 0.25, 10)]
        market = volume.Market(json_blob=market2())
        computed = market.currency_volume_in_base('USDT', 'BTC')
        expected = 10 * 2000 + 10 * .25 * 2000
        self.assertEqual(computed, expected)

    def test_usd_volumes(self):
        # Market is: [('USDT-BTC', 1000, 1), ('USDT-ETH', 1000, 1), ('BTC-ETH', 1, 1)]
        market = volume.Market(json_blob=market1())

    @mock.patch('my_bittrex.volume.client.get_market_summaries')
    def test_caching(self, mocked_get_summaries):
        """
        Test that caching works, and volume.get_summaries() gets called
        only once
        """
        mocked_get_summaries.return_value = market1()
        market = volume.Market()
        for i in range(10):
            market.summaries()
        mocked_get_summaries.assert_called_once()

    @mock.patch('my_bittrex.volume.client.get_market_summaries')
    def test_caching_2(self, mocked_get_summaries):
        """
        Test that caching works, and volume.get_summaries() gets called
        twice if 'cache_timeout_sec' elapses in between calls
        """
        mocked_get_summaries.return_value = market1()
        market = volume.Market(0.1)
        market.summaries()
        time.sleep(.2)
        market.summaries()
        mocked_get_summaries.assert_called_once()

class TestPortfolio(unittest.TestCase):
    @mock.patch('my_bittrex.volume.client.get_balances')#, return_value=fake_get_balances())
    def setUp(self, mocked_get_balances):
        mocked_get_balances.return_value = fake_get_balances()
        self.market = volume.Market(json_blob=market0())
        self.portfolio = volume.Portfolio()

    def test_value(self):
        self.assertAlmostEqual(self.portfolio.total_value(self.market, ['AAA', 'AAA']), 3, 3)
        self.assertAlmostEqual(self.portfolio.total_value(self.market, ['AAA', 'USDT']), 6, 3)

    def test_empty_init(self):
        portfolio = volume.Portfolio()
        self.assertItemsEqual(portfolio.portfolio.columns, ['Balance', 'Available', 'Pending', 'CryptoAddress',
                                                            'Requested', 'Uuid'])
        self.assertEqual(portfolio.portfolio.index.name, 'Currency')
        self.assertTrue(portfolio.portfolio.empty)

    @mock.patch('my_bittrex.volume.client.get_balances')
    def test_value_per_currency(self, mocked_get_balances):
        """
        balance1 is: [('BTC', 2), ('ETH', 3), ('USDT', 4)]
        market2 is: [('USDT-BTC', 2000, 10), ('USDT-ETH', 500, 10), ('BTC-ETH', 0.25, 10)]

        :param mocked_get_balances:
        :return:
        """
        mocked_get_balances.return_value = balance1()
        market = volume.Market(json_blob=market2())

        portfolio = volume.Portfolio()
        computed = portfolio.value_per_currency(market, ['BTC', 'USDT'])
        self.assertEqual(type(computed), pd.Series)
        self.assertItemsEqual(computed.tolist(), [4000, 1500, 4])

        computed = portfolio.value_per_currency(market, ['ETH', 'USDT'])
        self.assertItemsEqual(computed.tolist(), [4000, 1500, 4])

    def test_start_portfolio(self):
        market = volume.Market(json_blob=market1())
        state = volume.define_state(market, 2, include_usd=False)
        portfolio = volume.Portfolio()
        portfolio.start_portfolio(market, state, 'USDT', 1000)
        # we are overestimating the cost of the transaction because we are counting 4 transactions instead of 2
        # to establish the portfolio (selling USDT to buy BTC pays only once and I'm counting it twice, same for
        # ETH)
        self.assertEqual(portfolio.total_value(market, ['USDT']), 995)
        # 0.5 * (1 - 0.25%/100) = 0.49875
        # buy selling all 1000 dollars, due to a fake commission I end up with a -2.5 balance
        self.assertItemsEqual(portfolio.portfolio['Balance'], [.49875, .49875, -2.5])

class TestApplyTransactionCost(unittest.TestCase):
    def test_sell(self):
        computed = volume.apply_transaction_cost(-100)
        expected = -100.25
        self.assertEqual(computed, expected)

    def test_buy(self):
        computed = volume.apply_transaction_cost(100)
        expected = 99.75
        self.assertEqual(computed, expected)

class TestState(unittest.TestCase):
    def test_uniform(self):
        """
        [('USDT-BTC', 2000, 10), ('USDT-ETH', 500, 10), ('BTC-ETH', 0.25, 10), ('BTC-AAA', 1E-5, 1000),
         ('BTC-BBB', 1E-3, 1E6)]

        :return:
        """
        market = volume.Market(json_blob=market3())
        state = volume.define_state(market, 4)
        self.assertEqual(state.shape, (4, 1))
        self.assertItemsEqual(state['Weight'], [0.25] * 4)

class TestRebalance1(unittest.TestCase):

    @mock.patch('my_bittrex.volume.client.get_market_summaries')
    def test_from_empty_balance(self, mocked_market):
        """ start_new_portfolio will call get_currencies, we could mock it but there is not need for it as long
        as we use currencies that do exist for real (unlike 'AAA')

        """
        mocked_market.return_value = market1()
        market = volume.Market()
        state = volume.define_state(market, 2, include_usd=False)
        portfolio = volume.Portfolio()
        portfolio.ideal_rebalance(market, state)

        """
        time.sleep(.1)
        print(market.summaries())

        # Now we have 2500 , after rebalancing there should 1250 in each
        # since Market2 is [('USDT-BTC', 2000, 10), ('USDT-ETH', 500, 10), ('BTC-ETH', 0.25, 10)]
        # 'BTC' @ 2000, position should be 1250/2000 = 0.625
        # 'ETH' @ 500, position should be 1250/500 = 2.5
        portfolio.rebalance(market, state, portfolio, 'USDT', 0)
        print(portfolio.portfolio)
        """

def market1():
    l = [('USDT-BTC', 1000, 1), ('USDT-ETH', 1000, 1), ('BTC-ETH', 1, 1)]
    return test_utils.fake_market(l)

def market2():
    l = [('USDT-BTC', 2000, 10), ('USDT-ETH', 500, 10), ('BTC-ETH', 0.25, 10)]
    return test_utils.fake_market(l)

def market3():
    l = [('USDT-BTC', 2000, 10), ('USDT-ETH', 500, 10), ('BTC-ETH', 0.25, 10), ('BTC-AAA', 1E-5, 1000),
         ('BTC-BBB', 1E-3, 1E6), ('ETH-CCC', 2, 100)]
    return test_utils.fake_market(l)

def market0():
    """
    Fake a summary as would be returned by bittrex.client. Each element in the input list is a tuple of the form
    (market_name, last_price, Volume)
    :return:
    """
    l = [
        ("AAA-BBB", 0.1, 10),
        ("AAA-CCC", 0.01, 100),
        ("BBB-CCC", 0.1, 20),
        ("USDT-AAA", 2.0, 2)
    ]
    return test_utils.fake_market(l)

def market_real1():
    """
    Ask                                  2795
    BaseVolume                    2.79336e+07
    Bid                               2785.14
    Created           2015-12-11T06:31:40.633
    High                                 2930
    Last                              2785.14
    Low                                  2280
    OpenBuyOrders                        3082
    OpenSellOrders                        661
    PrevDay                              2305
    TimeStamp          2017-07-21T06:55:04.12
    Volume                            10562.9
    Base                                 USDT
    Currency                              BTC

    in ipython running:
    json.loads(test.market_real1()) matches the above dictionary to a reasonable precission
    """
    l = [("USDT-BTC", 2785, 10562)]
    return test_utils.fake_market(l)

def fake_get_balances():
    """
    Fake a balance with 3 currencies. Prices are not included in balance but to help keep things streight
    prices are:
    AAA:    2
    BBB:    0.2
    CCC:    0.02
    :return:
    """
    l = [('AAA', 1), ('BBB', 10), ('CCC', 100)]
    as_dict = json.loads(test_utils.fake_balance(l))
    return as_dict

def balance1():
    l = [('BTC', 2), ('ETH', 3), ('USDT', 4)]
    return json.loads(test_utils.fake_balance(l))

def fake_get_currencies():
    l = [('AAA', 'AAA long', .002), ('BBB', 'BBB long', .002), ('CCC', 'CCC long', .002)]
    as_dict = json.loads(test_utils.fake_currencies(l))
    return as_dict

if __name__ == "__main__":
    unittest.main()


#!/usr/bin/python

'''

utility.py, helper function.

'''

import itertools
import numpy as np
import pandas as pd
import joblib
from nltk import word_tokenize, tag
import matplotlib.pyplot as plt

def penn_scale():
    '''

    categorization of penn-tree

        - https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html

    '''

    return {
        'CC': 2,
        'CD': 3,
        'DT': 4,
        'EX': 5,
        'FW': 6,
        'IN': 7,
        'JJ': 8,
        'JJR': 9,
        'JJS': 10,
        'LS': 11,
        'MD': 12,
        'NN': 13,
        'NNS': 14,
        'NNP': 15,
        'NNPS': 16,
        'PDT': 17,
        'POS': 18,
        'PRP': 19,
        'PRP$': 20,
        'RB': 21,
        'RBR': 22,
        'RBS': 23,
        'RP': 24,
        'SYM': 25,
        'TO': 26,
        'UH': 27,
        'VB': 28,
        'VBD': 29,
        'VBG': 30,
        'VBN': 31,
        'VBP': 32,
        'VBZ': 33,
        'WDT': 34,
        'WP': 35,
        'WP$': 36,
        'WRB': 37
    }

def normalize_data(X_data, stop_gap=40, stop_value=1, train=False):
    '''

    Ensure each sentence has exactly 'stop_gap' number of columns.

    '''

    if train:
        ## ensure nested lists same length
        length = len(sorted(X_data, key=len, reverse=True)[0])
        X_data=np.array([xi+[stop_value]*(length-len(xi)) for xi in X_data])
        X_data=pd.DataFrame(X_data)

        ## get shape of dataframe
        rows_train, columns_train = X_data.shape
        delta = stop_gap - columns_train

        ##
        ## columns fixed to 40:
        ##
        ## https://github.com/jeff1evesque/ist-664/issues/62#issuecomment-447223456
        ##
        if delta > 0 and delta <= stop_gap:
            for i in range(delta):
                X_data['filler-{}'.format(i)] = stop_value

        if delta < 0:
            rem_list = [x for x in range(abs(columns_train))][:delta]
            X_data = X_data.iloc[:,rem_list]

    else:
        columns_train = len(X_data)
        delta = stop_gap - columns_train

        ##
        ## columns fixed to 40:
        ##
        ## https://github.com/jeff1evesque/ist-664/issues/62#issuecomment-447223456
        ##
        if delta > 0 and delta <= stop_gap:
            for i in range(delta):
                X_data.append(stop_value)

        if delta < 0:
            rem_list = [x for x in range(abs(columns_train))][:delta]
            X_data = X_data[:len(X_data) - rem_list]

    ## return train + test
    return(X_data)

def tokenizer(sentence):
    '''

    tokenize + parts of speech

    '''

    pos_scale = penn_scale()
    sent = word_tokenize(sentence)
    pos = tag.pos_tag(sent)
    return([x[1] for x in pos if x[1] and x[1] in pos_scale])

def replace(list, dictionary):
    '''

    replace list item with corresponding dict value

    '''

    return [dictionary.get(item, item) for item in list]

def qa_model(cwd):
    '''

    return unserialized model.

    '''

    return joblib.load('{base}/QuestionAnswerCMU/model/random_forest.pkl'.format(base=cwd))

def plot_confusion_matrix(
    cm,
    classes,
    normalize=False,
    title='Confusion matrix',
    cmap=plt.cm.Blues
):
    '''

    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.

    '''

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.tight_layout()


# -*- coding: utf-8 -*-
"""
Created on Aug 25, 2013
"""
import re
from urlparse import urljoin, urlparse
from selenium import webdriver

from scrapy.http import Request
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.contrib.spiders import CrawlSpider, Rule

from scrapers.settings import path_to_phatomjs
from scrapers.items import DealItem
from scrapers.loaders.superdeal_loader import DealLoader
from apps.dsites.models import DSite
from apps.addresses.models import CityMapping
from apps.categories.models import CategoryMapping

class SuperdealSpider(CrawlSpider):
    name = 'superdeal'

    def __init__(self, dsite_pk='', city_mapping_pk='', category_mapping_pk='', updating='False',
                 *args, **kwargs):
        self.dsite = DSite.objects.get(pk=int(dsite_pk)) if dsite_pk else DSite.objects.get(name=self.name)
        self.allowed_domains = [urlparse(self.dsite.url).netloc]
        self.city_mapping = CityMapping.objects.get(pk=city_mapping_pk) if city_mapping_pk else None
        self.category_mapping = CategoryMapping.objects.get(pk=category_mapping_pk) if category_mapping_pk else None
        self.updating = eval(updating)
        self.driver = webdriver.PhantomJS(path_to_phatomjs)
        if self.city_mapping is not None:
            self.start_url = '/%s/%s/' % (self.category_mapping.site_category, self.city_mapping.site_city)
            self.deal_urls = re.compile(
                '/' + self.category_mapping.site_category + '|' + self.city_mapping.site_city + '(?=/[a-zA-Z\s\d_-]+/?)')
        else:
            self.start_url = '/%s/' % self.category_mapping.site_category
            self.deal_urls = re.compile('/' + self.category_mapping.site_category + '(?=/[a-zA-Z\s\d_-]+/?)')
        self.rules = (
            Rule(SgmlLinkExtractor(allow=self.deal_urls, restrict_xpaths='//*[@id="deal_main"]'),
                 callback='parse_deal', follow=False),
            Rule(SgmlLinkExtractor(allow=self.deal_urls, restrict_xpaths='//*[@id="alldeals"]'),
                 callback='parse_deal', follow=False),
        )
        self.item_fields = {'title': '//div[@id="desc"]/h2/a/span/text()',
                            'description1': '//div[@id="offre"]/ul/li/span/text()',
                            'description2': '//div[@id="article"]/p/span/text()',
                            'hours': '//span[@id="remaining-hours"]/text()',
                            'minutes': '//span[@id="remaining-minutes"]/text()',
                            'initial_price': '//span[@class="dd"][1]/span/text()',
                            'sell_price': '//div[@id="deal_price_tag"]/span/text()',
                            'discount': '//span[@class="dd"][2]/span/text()',
                            'saving': '//span[@class="dd"][3]/span/text()',
                            'nbr_buyers': '//div[@id="deal_status"]/p/text()',
                            'supplier_name': '//div[@id="left-smt"]/div/div/h3/text()',
                            'supplier_address': '//div[@id="left-smt"]/div/div/div[2]/@onclick',
                            'supplier_lat': '//div[@id="left-smt"]/div/div/div[2]/@onclick',
                            'supplier_lng': '//div[@id="left-smt"]/div/div/div[2]/@onclick',
                            'supplier_city': '//div[@id="left-smt"]/div/div/div[2]/@onclick',
                            'supplier_phones': '//div[@id="left-smt"]/div/div/div[2]/@onclick',
                            'image_urls': '//div[@id="banners"]/div/img[1]/@src'}
        super(SuperdealSpider, self).__init__(*args, **kwargs)

    def quit_driver(self):
        self.driver.quit()

    def start_requests(self):
        yield Request(url=urljoin(self.dsite.url, self.start_url))

    def parse_deal(self, response):
        deal_loader = DealLoader(item=DealItem(), response=response)
        #add url value
        deal_loader.add_value('url', u'%s' % response.url)

        # iterate over fields and add xpaths to the loader
        for field, xpath in self.item_fields.iteritems():
            deal_loader.add_xpath(field, xpath)

        deal_loader.add_xpath('validity', '//div[@id="conditions"]/ul/li[1]/span/text()')
        deal_loader.add_xpath('validity', '//div[@id="conditions"]/div/ul/li[1]/span/text()')

        yield deal_loader.load_item()


import pymgui
import os

import matplotlib.pyplot as plt
plt.set_cmap('viridis')

def main():
    io = pymgui.get_io()
    io.render_draw_list_func = pymgui.types.stub_render

    font_atlas = io.fonts
    font_atlas_texture = pymgui.get_tex_data_as_rgba32(font_atlas)

    io.display_size = (640, 480)

    x = io.mouse_drag_threshold

    clear_color = pymgui.ImColor(114, 144, 154, 255)
    button_size = pymgui.ImVec2(30,50)
    show_second_window = 0

    while True:
        f = 0.0
        pid = os.getpid()
        pymgui.new_frame()

        pymgui.begin("hello", True)

        pymgui.button("hello button", (20, 50))
        pymgui.button("hello button 2", (20, 60))

        pymgui.end()

        if show_second_window % 2 == 0:
            pymgui.begin("hello2", True)
            pymgui.button("hello", (45, 90))
            pymgui.end()

        pymgui.render()

if __name__ == '__main__':
    main()


#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import os
import pandas as pd
import datetime as dt
import pandas_datareader.data as web  # then use df = web.DataReader(stocks, 'yahoo',start,end)

from algo2.utilities import range

from algo2 import utilities, statistics
from algo2.utilities import queue
from algo2.feeds.csv_files import HistoricCSVBarDataHandler

from algo2.strategies.buy_and_hold import BuyAndHoldStrategy

from algo2.pos_sizers.naive import AllInStockPositionSizer  # All in
from algo2.pos_refiners.naive import NaivePositionRefiner
from algo2.portfolio_handler import PortfolioHandler
from algo2.brokers.simulated_broker import IBSimulatedExecutionHandler
from algo2.statistics.simple import SimpleStatistics
from algo2.backtest import Backtest

def get_merged_data(config, ticker, all_in_fee, testing):
    """
    Read:
     - historical data from .csv (i.e. theoretical values for a security), and
     - new data from Yahoo (traded 'Adj Close' of a security)
    then rebase historical data and merge it with the new one
    to form a continuous daily Adj Close for backtesting purposes.
    :param config:  (e.g. utilities.DEFAULT) for default directories
    :param ticker: single ticker
    :param all_in_fee: estimated annual fee to be sutracted from theoretical values
    :param testing
    :return: dataframe of combined data (theoretical and new)
    """
    df_new, df_old = {}, {}
    # 1. download new data # and drop unwanted columns (keep only 'Adj Close')
    try:
        _start, _end = dt.datetime(1900, 1, 1), dt.date.today()
        df_new = web.DataReader(ticker.upper(), 'yahoo', _start, _end)
        # df_new.drop(['Open', 'High', 'Low', 'Close', 'Volume'], axis=1, inplace=True)
    except IOError:
        print("No dataset, failing to download data from Yahoo")

    # 2. read old data (theo values), re-base it and merge with new
    f_in = os.path.join(config.CSV_DATA_DIR, "%s.csv" % ticker)
    df_old = pd.read_csv(f_in, header=0, parse_dates=True,
                         index_col=0, dayfirst=True, names=("Date", "Adj Close")
                         )  # note: dayFirst=True for format issues (European vs. American)

    # NB: assuming overlapping, i.e.  new_startdate > old_startdate
    old_lastdate = df_old.index.max()
    new_startdate = df_new.index.min()
    if new_startdate > old_lastdate:
        raise IOError("start_date of the new data later than old data last_date")

    # get theoretical data net of fees, if any:
    if all_in_fee != 0.0:
        perc = df_old["Adj Close"] / df_old["Adj Close"].shift(1) - 1 - all_in_fee / 365.0
        df_old['Adj Close net'] = df_old['Adj Close']
        start_pos_ = df_old.index.searchsorted(new_startdate)
        for i in range(start_pos_):
            df_old['Adj Close net'].iloc[i + 1] = df_old['Adj Close net'].iloc[i] \
                                                  * (1 + perc.iloc[i + 1])
        df_old['Adj Close'] = df_old['Adj Close net']  # rename & drop
        df_old.drop("Adj Close net", axis=1, inplace=True)

    # rebase
    mult = df_new.loc[new_startdate, "Adj Close"] \
        / df_old.loc[new_startdate, "Adj Close"]  # for rebasing
    df_old["Adj Close"] = df_old["Adj Close"] * mult

    # for consistency with rest of the file
    columns = ["Open", "High", "Low", "Close"]
    for cols in columns:
        df_old[cols] = df_old["Adj Close"]
    df_old["Volume"] = df_new["Volume"].loc[new_startdate]

    # return merged data and, if requested, save it
    start_pos = df_old.index.searchsorted(new_startdate)
    df_combo = pd.concat([df_old.iloc[:start_pos], df_new])
    if not testing:
        fout = config.OUTPUT_DIR + "/%s.csv" % ticker
        df_combo.to_csv(fout)

        # return df_combo  # ["Adj Close"]

def run(config, testing, tickers, filename, all_in_fee):
    # ##############################################
    #   GETTING (new, besides old saved) DATA, if not testing
    if not testing:
        for ticker in tickers:
            get_merged_data(config, ticker, all_in_fee, testing)
        csv_dir = config.OUTPUT_DIR
    else:
        csv_dir = config.CSV_DATA_DIR

    # ##############################################
    #   USUAL SET-UP
    # Set up variables for backtest()
    events_queue = queue.Queue()
    # csv_dir = config.CSV_DATA_DIR
    initial_equity = 500000.00

    # Use historic (Yahoo Daily) data handler
    data_handler = HistoricCSVBarDataHandler(csv_dir, events_queue, tickers)

    # Use the Buy-and-Hold Strategy
    strategy = BuyAndHoldStrategy(tickers, events_queue)
    # strategy = Strategies(strategy, DisplayStrategy())    # UN-COMMENT

    # Use an example Position Sizer and Refiner (risk mgt)
    position_sizer = AllInStockPositionSizer()  # i.e. All-in
    position_refiner = NaivePositionRefiner()

    # Use the default Portfolio Handler
    portfolio_handler = PortfolioHandler(
        initial_equity, events_queue, data_handler,
        position_sizer, position_refiner
    )

    # Use a simulated IB Execution Handler
    broker_handler = IBSimulatedExecutionHandler(events_queue, data_handler)  # , compliance)

    # Use the default Statistics
    stats = SimpleStatistics(config, portfolio_handler)

    # Set up the backtest
    backtest = Backtest(
        data_handler, strategy,
        portfolio_handler, broker_handler,
        position_sizer, position_refiner,
        stats, initial_equity
    )
    results = backtest.simulate_trading(testing=testing)
    statistics.save(filename, False)  # if True: also saves a .csv file with main curves
    return results

##############################################
def main():
    config = utilities.DEFAULT
    testing = False
    tickers = ['XIV']  # ['XIV', 'VXX']  # ; tickers = tickers.split(",")
    filename = ""
    run(config, testing, tickers, filename, 0.0)  # 0.024)

##############################################
if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-

import mock
import pytest

from h import models
from h.cli.commands import authclient as authclient_cli

class TestAddCommand(object):

    def test_it_creates_an_authclient(self, cli, cliconfig, db_session):
        (authclient, _) = self._add_authclient(cli, cliconfig, db_session)

        assert authclient.authority == 'publisher.org'
        assert authclient.name == 'Publisher'

    def test_it_prints_the_client_id_and_secret(self, cli, cliconfig, db_session):
        (authclient, output) = self._add_authclient(cli, cliconfig, db_session)
        expected_id_and_secret = ('Client ID: {}\n'.format(authclient.id) +
                                  'Client Secret: {}'.format(authclient.secret))
        assert expected_id_and_secret in output

    def _add_authclient(self, cli, cliconfig, db_session):
        result = cli.invoke(authclient_cli.add,
                            [u'--name', 'Publisher', u'--authority', 'publisher.org'],
                            obj=cliconfig)

        assert result.exit_code == 0

        authclient = db_session.query(models.AuthClient).filter(
                   models.AuthClient.authority == 'publisher.org').first()
        return (authclient, result.output)

@pytest.fixture
def cliconfig(pyramid_config, pyramid_request):
    pyramid_request.tm = mock.Mock()
    return {'bootstrap': mock.Mock(return_value=pyramid_request)}


import json
import random
from django.shortcuts import render, redirect
from django.http import HttpResponseRedirect
from django.urls import reverse
from main.models import Book
from django.http import HttpResponse
from django.core import serializers
from django.http import JsonResponse
from django.contrib.auth import authenticate, login, logout
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_POST
from django.contrib.auth.models import User
from rprofile.forms import BookForm
from rprofile.models import UserProfile
from .forms import UpdatePhoneNumberForm
from .forms import BookSearchForm

@login_required
def profile(request):
    user = request.user
    user_profile_id = None  # Initialize user_profile_id as None

    try:
        user_profile = UserProfile.objects.get(user=user)
        user_profile_id = user_profile.id  # type: ignore # Retrieve the user_profile_id
    except UserProfile.DoesNotExist:
        user_profile = None

    author_names = Book.objects.values_list('author', flat=True)
    role = "Reader"

    if user.is_superuser:
        role = "Admin"
    elif user.username.replace('+', ' ') in author_names:
        role = "Writer"

    return render(request, 'profile.html', {'user': user, 'user_profile': user_profile, 'role': role, 'user_profile_id': user_profile_id})

def bookofchoice(request):
    # Load JSON data containing a collection of books
    all_books = Book.objects.all()

    if all_books:
        # Randomly select one book from the list
        random_book = random.choice(all_books)
    else:
        random_book = None  # Handle the case where there are no books in the database

    return render(request, 'bookofyourchoice.html', {'book': random_book})

def not_login(request):
    return render(request, 'not_login.html')

@login_required
def update_email(request):
    if request.method == 'POST':
        user = request.user
        email = request.POST.get('email')

        if email:
            user.email = email
            user.save()
            return JsonResponse({'message': 'Email updated successfully'}, status=200)

        else:
            return JsonResponse({'error': 'Email data not provided.'}, status=400)
    else:
        return JsonResponse({'error': 'Invalid request'}, status=400)

@login_required
def update_phone(request):
    if request.method == 'POST':
        user = request.user
        phone = request.POST.get('phone')

        if phone:
            user.profile.phone = phone  # Assuming user has a related 'profile' model
            user.profile.save()
            return JsonResponse({'message': 'Phone updated successfully'}, status=200)

        else:
            return JsonResponse({'error': 'Phone data not provided.'}, status=400)
    else:
        return JsonResponse({'error': 'Invalid request'}, status=400)

def edit_book(request, id):
    # Get product berdasarkan ID
    book = Book.objects.get(pk=id)

    # Set product sebagai instance dari form
    form = BookForm(request.POST or None, instance=book)

    if form.is_valid() and request.method == "POST":
        # Simpan form dan kembali ke halaman awal
        form.save()
        return HttpResponseRedirect(reverse('main:show_main'))

    context = {'form': form}
    return render(request, "edit_book.html", context)

def show_json(request):
    data = Book.objects.all()
    return HttpResponse(serializers.serialize("json", data), content_type="application/json")
def show_json_by_id(request, id):
    data = Book.objects.filter(pk=id)
    return HttpResponse(serializers.serialize("json", data), content_type="application/json")

def update_phone_number(request):
    if request.method == 'POST':
        form = UpdatePhoneNumberForm(request.POST)

        if form.is_valid():
            user_profile_id = form.cleaned_data['user_profile_id']
            new_phone_number = form.cleaned_data['new_phone_number']

            try:
                user_profile = UserProfile.objects.get(id=user_profile_id)
                user_profile.handphone = new_phone_number
                user_profile.save()
                return JsonResponse({'message': 'Phone number updated successfully'})
            except UserProfile.DoesNotExist:
                return JsonResponse({'error': 'User profile not found'})
        else:
            return JsonResponse({'error': 'Form data is invalid'})
    else:
        return JsonResponse({'error': 'Invalid request method'})

def filter_books(request):
    if request.method == "POST":
        form = BookSearchForm(request.POST)

        if form.is_valid():
            title = form.cleaned_data.get('title')
            author = form.cleaned_data.get('author')
            genre = form.cleaned_data.get('genre')

            # Get the user's profile
            user_profile = UserProfile.objects.get(user=request.user)

            # Filter the user's favorite books based on criteria
            filtered_books = user_profile.favorite_books.filter(
                title__icontains=title,
                author__icontains=author,
                genre__icontains=genre
            )

            return render(request, 'filtered_books.html', {'filtered_books': filtered_books})

    else:
        form = BookSearchForm()

    return render(request, 'filter_books.html', {'form': form})


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

data = pd.read_csv('data_linear.csv').values     # đọc file
N = data.shape[0]
x = data[:,0].reshape(-1,1)  # diện tích nhà --- ma trận X
y = data[:,1].reshape(-1,1)  # Giá nhà --- ma trận Y
print(N)
print(data.shape)
print(x)
print('===========================================')
print(y)
plt.scatter(x,y)
plt.xlabel('mét vuông')
plt.ylabel('giá')

# Xếp chồng ngang
x = np.hstack((np.ones((N,1)),x)) # ====> Ma trận X

w = np.array([0.,1]).reshape(-1,1) # ====> tạo ma trận w  là [0,1] ==> chuyển vị

numOfIteration = 100

cost = np.zeros((numOfIteration,1)) # => Tạo mảng 100 x 1 , toàn số 0
learning_rate = 0.000001
for i in range(1,numOfIteration): # chạy từ 1 -> 100
    r = np.dot(x,w) - y # r = ma trận X * ma trận W - ma trận Y
    cost[i] = 0.5*np.sum(r*r)  # tổng các phần tử trong mảng
    w[0] -= learning_rate*np.sum(r) # w0 - đạo hàm của J theo w0

    w[1] -= learning_rate*np.sum(np.multiply(r,x[:,1].reshape(-1,1))) # w1 - đạo hàm của J theo w1
    print(cost[i])

predict = np.dot(x,w) # Nhân ma trận X với tham số w vừa tìm ra sau khi gradient descent ==> hàm y = wx

plt.plot((x[0][1],x[N-1][1]),(predict[0],predict[N-1]),'r') # vẽ đồ thị
plt.show()

x1 = 50
y1 = w[0] +w[1]*x1
print('Giá nhà cho 50 m^2 là: ',y1)


import discord

def create_embed():
  pic = "https://imagenes.elpais.com/resizer/ignf5hRqPoNrcNeilF3aB9CKy-M=/1960x0/cloudfront-eu-central-1.images.arcpublishing.com/prisa/HE3SMC3L7Z7XENXLHLLKE3CDEA.jpg"
  embed = discord.Embed(
    title = "Titulo",
    description = "Esta es la descripcion",
    colour = discord.Colour.blue()
  )
  embed.set_footer(text="This is a footer.")
  embed.set_image(url=pic)
  embed.set_thumbnail(url=pic)
  embed.set_author(name="Nombre de author",icon_url=pic)
  embed.add_field(name="Field Name", value="Field Value",inline=False)
  embed.add_field(name="Field Name", value="Field Value",inline=True)
  embed.add_field(name="Field Name", value="Field Value",inline=True)
  return embed

def help_embed():
  embed = discord.Embed(
    title = "Help Commands",
    description = "Aqui estan los comandos y que hace cada uno",
    colour = discord.Colour.blue()
  )
  embed.set_footer(text="If you break it plz message me ultragames007#1836.")
  embed.add_field(name="!help", value="Shows this help message",inline=False)
  embed.add_field(name="!setactive", value="Adds the Bot to the current channel.",inline=False)
  embed.add_field(name="!removeactive", value="Removes the Bot from the current channel.",inline=False)
  embed.add_field(name="!hello", value="Says hello back !",inline=False)
  embed.add_field(name="!a", value="A :D !",inline=False)
  embed.add_field(name="!saludo", value="Generates a greeting. Can be passed aguments like __!saludo Hola animal adj adj lugar . Como estas?__",inline=False)
  embed.add_field(name="!meme", value="Gives you a meme. can be passed a subbreddit and a quantity, max 25. __!meme sub quantity__",inline=False)
  embed.add_field(name="!easteregg", value="This does nothing.. believe me",inline=False)
  embed.add_field(name="!reset", value="Resets something.. I dont know what",inline=False)
  embed.add_field(name="!clear", value="Deletes 1 message. Can be passed args like this __!clear 5__ to delete more msgs. MAX is 100 ",inline=False)
  embed.add_field(name="!noticias", value="Shows you the latest news from genbeta.com",inline=False)
  embed.add_field(name="!noticias2", value="Shows you the latest news from news.ycombinator.com",inline=False)
  embed.add_field(name="!search", value="Searchs on stack overflow for results. Do __!search my question here__",inline=False)
  return embed

def help_embed_admin():
  embed = discord.Embed(
    title = "Help Commands",
    description = "Aqui estan los comandos admin y que hacen",
    colour = discord.Colour.blue()
  )
  embed.add_field(name="!resetacl", value="Admin Command. Resets ALL Active current channels",inline=False)
  embed.add_field(name="!checkacl", value="Admin Command. Prints a list of ALL Active current channels",inline=False)
  return embed

def news_embed(arr1,arr2):
  embed = discord.Embed(
    title = "News from Genbeta",
    description = "Aqui estan los titulares mas recientes.",
    colour = discord.Colour.blue()
  )

  for item in range(0,len(arr1)):
    link = "[--LINK--](" + arr2[item] + ")"
    #print(link)
    embed.add_field(name=arr1[item], value=link,inline=False)
  return embed

def search_embed(arr1,arr2,search_term):
  embed = discord.Embed(
    title = "Search Results",
    description = " ",
    colour = discord.Colour.blue()
  )

  for item in range(0,len(arr1)):
    link = "[--LINK--](" + arr2[item] + ")"
    embed.add_field(name=arr1[item], value=link,inline=False)

  st = search_term.replace(" ","+")
  href = "https://www.google.com/search?q=" +st + "+site%3Astackoverflow.com"
  embed.add_field(name="--MORE RESULTS--", value=href, inline=False)

  return embed

def news_embed2(arr1,arr2):
  embed = discord.Embed(
    title = "News from Hacker News",
    description = "Aqui estan los 5 titulares mas recientes.",
    colour = discord.Colour.blue()
  )

  for item in range(0,5):
    link = "[--LINK--](" + arr2[item] + ")"
    #print(link)
    embed.add_field(name=arr1[item], value=link,inline=False)
  href = "https://news.ycombinator.com/"
  embed.add_field(name="--MORE RESULTS--", value=href, inline=False)
  return embed


import asyncio
import discord
import requests
import json
import typing
import pandas as pd
from discord import app_commands
from discord.ext import commands
from lib.emojiid import team_emoji_ids
from lib.emojiid import nation_dictionary
from lib.championship import constructor_championship
from lib.colors import colors
import country_converter as coco
now = pd.Timestamp.now()

# check if given year and round number are valid
def check_year(year,round):
    if not(year == None) and not(1950 <= year and year <= now.year):
        return "bad year"
    elif not(round == None) and not(round >= 1) and not(round < 25):
        return "bad round number"
    else:
        if (year == None and round == None):
            url =  f"http://ergast.com/api/f1/{now.year}/constructors.json"
            return url
        elif (year == None):
            url = f"https://ergast.com/api/f1/{now.year}/{round}/constructors.json"
            return url
        elif (round == None):
            url =  f"https://ergast.com/api/f1/{year}/{1}/constructors.json"
            return url
        else:
            url =  f"https://ergast.com/api/f1/{year}/{round}/constructors.json"
            return url

def get_constructor_info(self, year, round):
    message_embed = discord.Embed(title=f"Constructor Information", description="").set_thumbnail(url='https://cdn.discordapp.com/attachments/884602392249770087/1059464532239581204/f1python128.png')
    message_embed.set_author(name='f1buddy',icon_url='https://raw.githubusercontent.com/F1-Buddy/f1buddy-python/main/botPics/f1pythonpfp.png')
    message_embed.colour = colors.default
    url = check_year(year,round)
    # print(url)
    description_string = ''
    nationality_dict = nation_dictionary()

    # with open('lib/nation.csv') as csv_file:
    #     csv_read = csv.reader(csv_file, delimiter=',')

    if ('bad' in url):
        description_string = "Please try again with a different " + url[url.index('bad')+3:]
    else:
        constructor_name, constructor_nationality, constructor_wikipedia, constructor_championships = [], [], [], []

        try:
            constructor = requests.get(url)
            response = json.loads(constructor.content)
        except json.JSONDecodeError:
            if (year == None) or (1950 >= year and year >= now.year):
                url = f"https://ergast.com/api/f1/current/constructors.json"
                constructor = requests.get(url)
                round = 1
                response = json.loads(constructor.content)
            else:
                url =  f"https://ergast.com/api/f1/{year}/{1}/constructors.json"
                constructor = requests.get(url)
                response = json.loads(constructor.content)

        amount_of_teams = int(response['MRData']['total'])
        if (amount_of_teams == 0):
                description_string = f"No teams for this round."
        else:
            year = int(response['MRData']['ConstructorTable']['season'])
            if (round == 0 or round == None):
                message_embed.title = f"{year} Constructor Information"
            elif (round > 0):
                round = int(response['MRData']['ConstructorTable']['round'])
                message_embed.title = f"{year} Round {round} Constructor Information"
            else:
                message_embed.title = f"{year} Constructor Information"

            for i in range(amount_of_teams):
                constructor_data = (response['MRData']['ConstructorTable']['Constructors'][i])

                try:
                    if amount_of_teams <= 11:
                        constructor_name.append((str)(self.bot.get_emoji(team_emoji_ids[constructor_data['name']]))+' ' + constructor_data['name'])
                    elif year == 2010 or year == 2011:
                        constructor_name.append((constructor_data['name']))
                    else:
                        constructor_name.append((constructor_data['name']))
                except:
                    constructor_name.append((constructor_data['name']))

                if constructor_data['name'] in constructor_championship:
                    constructor_championships.append(constructor_championship[constructor_data['name']])
                else:
                    constructor_championships.append("0")
                emoji = ":flag_" + \
                    (coco.convert(
                    names=nationality_dict[constructor_data['nationality']], to='ISO2')).lower()+":"
                constructor_nationality.append(emoji + " " + constructor_data['nationality'])
                constructor_wikipedia.append(constructor_data['url'])

            string = ""
            for j in range(amount_of_teams):
                try:
                    team_name = f"{constructor_name[j][0:constructor_name[j].index('>')+1]}[{constructor_name[j][constructor_name[j].index('>')+1:]}]({constructor_wikipedia[j]})"
                    string += team_name + "\n"
                except:
                    team_name = f"[{constructor_name[j]}]({constructor_wikipedia[j]})"
                    string += team_name + "\n"

            # print(string)
            message_embed.add_field(name="Team", value=string, inline=True)
            message_embed.add_field(name = "Nationality", value = '\n'.join(constructor_nationality),inline = True)
            message_embed.add_field(name = "Championships", value = '\n'.join(constructor_championships),inline = True)
            message_embed.description = description_string
            nation_dictionary()
            return message_embed

class constructor(commands.Cog):
    def __init__(self, bot: commands.Bot):
        self.bot = bot
    @commands.Cog.listener()
    async def on_ready(self):
        print('Constructors cog loaded')
    @app_commands.command(name = 'constructors', description = 'Get constructor information')
    @app_commands.describe(year = "Constructor information")

    async def constructor(self, interaction: discord.Interaction, year: typing.Optional[int], round: typing.Optional[int]):
        await interaction.response.defer()
        loop = asyncio.get_running_loop()
        try:
            # run query and build embed
            constructor_embed = await loop.run_in_executor(None, get_constructor_info, self, year, round)
            constructor_embed.set_author(name='f1buddy', icon_url='https://raw.githubusercontent.com/F1-Buddy/f1buddy-python/main/botPics/f1pythonpfp.png')
            # send embed
            await interaction.followup.send(embed=constructor_embed)
        except Exception as e:
            await interaction.followup.send(f"An error occurred: {e}")
        loop.close()

async def setup(bot):
    await bot.add_cog(constructor(bot) # , guilds=[discord.Object(id=884602392249770084)]
                      )


import requests
import urllib
import json

"""获取data"""
def getData(url):
    data = ''
    while 1:
        try:
            headers = {'User-Agent':"https://kyfw.12306.cn/otn/leftTicket/query?leftTicketDTO.train_date=2020-04-11&leftTicketDTO.from_station=BJP&leftTicketDTO.to_station=HHC&purpose_codes=ADULT"}
            req = urllib.request.Request(url=url,headers=headers)
            data = urllib.request.urlopen(req).read().decode('utf-8')
            if data.startswith(u'\ufeff'):
                data = data.encode('utf8')[3:].decode('utf-8')
            break
        except:
            print('获取data失败，继续获取……')
            continue
    return data

"""获取result"""
def resolveData():
    #  查询链接
    url = 'https://kyfw.12306.cn/otn/leftTicket/query?leftTicketDTO.train_date=2020-04-11&leftTicketDTO.from_station=BJP&leftTicketDTO.to_station=HHC&purpose_codes=ADULT'
    while 1:
        try:
            data = getData(url)
            lists = json.loads(data)["data"]["result"]
            break
        except:
            print('2获取失败，继续中……')
            continue  #获取失败时则重新获取
    for item in lists:
        print(item)

resolveData()

第14章  网络爬虫/测试获取json.py


import os
from setuptools import setup

here = os.path.abspath(os.path.dirname(__file__))
README = open(os.path.join(here, 'README.md')).read()

setup(
    name='function-cache',
    version='0.2.0',
    packages=['function_cache'],
    description='A Django-based convenience decorator for caching results of functions to various backends.',
    long_description=README,
    long_description_content_type='text/markdown',
    author='5 Health Inc',
    author_email='hello@botmd.io',
    url='https://github.com/fivehealth/function-cache',
    license='MIT License',
    install_requires=[
        'Django>2.0',
    ],
    python_requires='>=3',
    keywords='django cache',
    classifiers=[
        'Development Status :: 4 - Beta',
        'Programming Language :: Python :: 3',
        'Framework :: Django',
        'Framework :: Django :: 2.2',
        'Framework :: Django :: 3.0',
        'Framework :: Django :: 3.1',
        'License :: OSI Approved :: MIT License',
    ],
)


"""
        '########:'##::::'##::::'##:::
         ##.....::. ##::'##:::'####:::
         ##::::::::. ##'##::::.. ##:::
         ######:::::. ###::::::: ##:::
         ##...:::::: ## ##:::::: ##:::
         ##:::::::: ##:. ##::::: ##:::
         ########: ##:::. ##::'######:
        ........::..:::::..:::......::
"""
from typing import List
from typing import Tuple
import cv2
import numpy as np
import matplotlib.pyplot as plt
LOAD_GRAY_SCALE = 1
LOAD_RGB = 2

def myID() -> int:
    """
    Return my ID (not the friend's ID I copied from)
    :return: int
    """
    return 209904606

def imReadAndConvert(filename: str, representation: int) -> np.ndarray:
    """
    Reads an image, and returns the image converted as requested
    :param filename: The path to the image
    :param representation: GRAY_SCALE or RGB
    :return: The image object
    """
    try:
        img = cv2.cvtColor(cv2.imread(filename,1), cv2.COLOR_BGR2RGB)
    except:
        return np.zeros( (256,256,3))
    if representation == 1:
        img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)

    return img/255

def imDisplay(filename: str, representation: int):
    """
    Reads an image as RGB or GRAY_SCALE and displays it
    :param filename: The path to the image
    :param representation: GRAY_SCALE or RGB
    :return: None
    """

    img = imReadAndConvert(filename,representation)
    if representation == 1:
        plt.imshow(img,cmap='gray')
    else:
        plt.imshow(img)
    plt.show()

def transformRGB2YIQ(imgRGB: np.ndarray) -> np.ndarray:
    """
    Converts an RGB image to YIQ color space
    :param imgRGB: An Image in RGB
    :return: A YIQ in image color space
    """
    #ensure image has 3 channels
    try:
        ensure3ChannelImgInput(imgRGB)
    except Exception as e:
        print(e)
        return imgRGB

    #YIQ conversion matrix
    YIQ_mat = np.array([[ 0.299, 0.587, 0.114],
                        [ 0.596, -0.275, -0.321],
                        [ 0.212, -0.523, 0.311]])

    #turn our NxMx3 matrix to (N*M)x3 matrix
    img_vals = imgRGB.reshape((imgRGB.shape[0] * imgRGB.shape[1], 3))

    #matrix multiplication to convert each RGB triplet into YIQ
    img_vals = np.matmul(img_vals,YIQ_mat)

    #reshape our matrix back
    return img_vals.reshape(imgRGB.shape).astype(float)

def transformYIQ2RGB(imgYIQ: np.ndarray) -> np.ndarray:
    """
    Converts an YIQ image to RGB color space
    :param imgYIQ: An Image in YIQ
    :return: A RGB in image color space
    """
    #ensure image has 3 channels
    try:
        ensure3ChannelImgInput(imgYIQ)
    except Exception as e:
        print(e)
        return imgYIQ

    #YIQ conversion matrix
    YIQ_mat = np.array([[ 0.299, 0.587, 0.114],
                        [ 0.596, -0.275, -0.321],
                        [ 0.212, -0.523, 0.311]])
    #RGB conversion matrix is the inverse of the YIQ conversion matrix
    RGB_mat = np.linalg.inv(YIQ_mat)

    #turn our NxMx3 matrix to (N*M)x3 matrix
    img_vals = imgYIQ.reshape((imgYIQ.shape[0] * imgYIQ.shape[1], 3))

    #matrix multiplication to convert each YIQ triplet into RGB
    img_vals = np.matmul(img_vals,RGB_mat)

    #reshape our matrix back
    return img_vals.reshape(imgYIQ.shape).astype(float)

def hsitogramEqualize(imgOrig: np.ndarray) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
    """
        Equalizes the histogram of an image
        :param imgOrig: Original Histogram
        :ret
    """
    img = imgOrig
    #handle case for inputing RGB
    if len(imgOrig.shape) == 3:
        try:
            ensure3ChannelImgInput(imgOrig)
        except Exception as e:
            print(e)
            return
        #get only the Y channel of our RGB image converted to YIQ
        img = transformRGB2YIQ(img)
        imEq, histOrg, histEq = handle_hist(img[:,:,0])
        img[:,:,0] = imEq
        img = transformYIQ2RGB(img)
        return img,histOrg,histEq
    #handle case for inputing image that isn't grayscale or RGB
    elif len(imgOrig.shape) != 2:
        print("Image must be RGB or grayscale!")
        return

    return handle_hist(img)

def handle_hist(img)->Tuple[np.ndarray, np.ndarray, np.ndarray]:
    #will do histogram equalization on some color channel
    histOrg,bins = np.histogram(img*255,bins=256,range=[0,255])
    cumSum = np.cumsum(histOrg)
    imEq = img*255

    for i in range(256):
        lut =  np.ceil(255.0*float(cumSum[i])/float(cumSum[-1]))
        imEq[np.logical_and(img*255>=bins[i],img*255<bins[i+1])] = lut
        #private case where the last bin is for values in range [254,255] instead of [254,255)
        if i==255:
            imEq[np.logical_and(img*255>=bins[i],img*255==bins[i+1])] = lut
    #new histogram
    histEq,_ = np.histogram(imEq,bins=256,range=[0,255])
    imEq = imEq/255

    return imEq, histOrg, histEq

def quantizeImage(imOrig: np.ndarray, nQuant: int, nIter: int) -> Tuple[List[np.ndarray], List[float]]:
    """
        Quantized an image in to **nQuant** colors
        :param imOrig: The original image (RGB or Gray scale)
        :param nQuant: Number of colors to quantize the image to
        :param nIter: Number of optimization loops
        :return: (List[qImage_i],List[error_i])
    """
    if nQuant > 255:
        return [],[]
    img = imOrig
    #handle case for inputing RGB
    if len(imOrig.shape) == 3:
        try:
            ensure3ChannelImgInput(imOrig)
        except Exception as e:
            print(e)
            return [],[]
        #get only the Y channel of our RGB image converted to YIQ
        img = transformRGB2YIQ(img)
        qImage_i_Y, error_i = handle_quntize(img[:,:,0],nQuant,nIter)
        qImage_i = []
        #for every new Y channel add it to qImage_i together with the original I and Q channels
        for y in qImage_i_Y:
            tempImg = np.copy(img)
            tempImg[:,:,0] = y
            tempImg = transformYIQ2RGB(tempImg)
            qImage_i.append(tempImg)
        return qImage_i, error_i
    #handle case for inputing image that isn't grayscale or RGB
    elif len(imOrig.shape) != 2:
        print("Image must be RGB or grayscale!")
        return [],[]

    return handle_quntize(img,nQuant,nIter)

def handle_quntize(img: np.ndarray, nQuant: int, nIter: int) -> Tuple[List[np.ndarray], List[float]]:
    #if an image is using less colors than nQuant then we just apply our quantization on all colors (should return the original img)
    if len(np.unique(img))<=nQuant:
        return [img for i in range(nIter)], [0 for i in range(nIter)]

    hist,_ = np.histogram(img*255,bins=256,range=[0,255])
    z = [0]
    total_px = sum(hist)
    i = 0
    #initial spread of cells
    while i < len(hist):
        pixel_count = 0
        while pixel_count <= total_px/(nQuant) and i < len(hist):
            pixel_count+=hist[i]
            i+=1
        z.append(i)
    #set the last cell to 255 instead of 256
    z[-1]=255
    images = []
    errors = []
    #process each iteration
    for i in range(nIter):
        newImg,err,q = quantize_iter(img,hist,z)
        z = [0]
        for j in range(1,len(q)):
            z.append(round((q[j]+q[j-1])/2))
        z.append(255)
        images.append(newImg)
        errors.append(err)

    return images,errors

def quantize_iter(img,hist,z) -> Tuple[np.ndarray,float,list]:
    #handle each iteration
    q = []
    newImg = np.copy(img)
    for k in range(len(z)-1):
        #if some section is empty -> hist[z[k]:z[k+1]] is empty then no need to change that q
        if sum(hist[z[k]:z[k+1]]) == 0:
            q_i = round((z[k]+z[k+1])/2)
        else:
            #set q_i as the weighted average in every cell
            q_i = np.average([i for i in range(z[k],z[k+1])],weights=hist[z[k]:z[k+1]])

        newImg[np.logical_and(img*255>=z[k],img*255<z[k+1])] = q_i/255
        q.append(q_i)
    #calc MSE, note that we need to use sum twice since we are using a matrix
    error = np.sqrt(sum(sum(np.square(img-newImg))))/sum(hist)

    return newImg,error,q

def ensure3ChannelImgInput(img:np.ndarray):
    if len(img.shape) !=3 or img.shape[2] != 3:
        raise ValueError("Given image doesn't have 3 channels")


__author__ = 'Guido van Rossum <guido@python.org>'

__all__ = ['Driver', 'load_grammar']

import io
import logging
import os
import pkgutil
import sys
# Python imports
from contextlib import contextmanager
from dataclasses import dataclass
from dataclasses import field
from pathlib import Path
from typing import Any
from typing import Iterator
from typing import List
from typing import Optional

from platformdirs import user_cache_dir

from yapf._version import __version__ as yapf_version

# Pgen imports
from . import grammar
from . import parse
from . import pgen
from . import token
from . import tokenize

@dataclass
class ReleaseRange:
  start: int
  end: Optional[int] = None
  tokens: List[Any] = field(default_factory=list)

  def lock(self) -> None:
    total_eaten = len(self.tokens)
    self.end = self.start + total_eaten

class TokenProxy:

  def __init__(self, generator: Any) -> None:
    self._tokens = generator
    self._counter = 0
    self._release_ranges: List[ReleaseRange] = []

  @contextmanager
  def release(self) -> Iterator['TokenProxy']:
    release_range = ReleaseRange(self._counter)
    self._release_ranges.append(release_range)
    try:
      yield self
    finally:
      # Lock the last release range to the final position that
      # has been eaten.
      release_range.lock()

  def eat(self, point: int) -> Any:
    eaten_tokens = self._release_ranges[-1].tokens
    if point < len(eaten_tokens):
      return eaten_tokens[point]
    else:
      while point >= len(eaten_tokens):
        token = next(self._tokens)
        eaten_tokens.append(token)
      return token

  def __iter__(self) -> 'TokenProxy':
    return self

  def __next__(self) -> Any:
    # If the current position is already compromised (looked up)
    # return the eaten token, if not just go further on the given
    # token producer.
    for release_range in self._release_ranges:
      assert release_range.end is not None

      start, end = release_range.start, release_range.end
      if start <= self._counter < end:
        token = release_range.tokens[self._counter - start]
        break
    else:
      token = next(self._tokens)
    self._counter += 1
    return token

  def can_advance(self, to: int) -> bool:
    # Try to eat, fail if it can't. The eat operation is cached
    # so there wont be any additional cost of eating here
    try:
      self.eat(to)
    except StopIteration:
      return False
    else:
      return True

class Driver(object):

  def __init__(self, grammar, convert=None, logger=None):
    self.grammar = grammar
    if logger is None:
      logger = logging.getLogger()
    self.logger = logger
    self.convert = convert

  def parse_tokens(self, tokens, debug=False):
    """Parse a series of tokens and return the syntax tree."""
    # XXX Move the prefix computation into a wrapper around tokenize.
    p = parse.Parser(self.grammar, self.convert)
    proxy = TokenProxy(tokens)
    p.setup(proxy=proxy)
    lineno = 1
    column = 0
    type = value = start = end = line_text = None
    prefix = ''
    for quintuple in proxy:
      type, value, start, end, line_text = quintuple
      if start != (lineno, column):
        assert (lineno, column) <= start, ((lineno, column), start)
        s_lineno, s_column = start
        if lineno < s_lineno:
          prefix += '\n' * (s_lineno - lineno)
          lineno = s_lineno
          column = 0
        if column < s_column:
          prefix += line_text[column:s_column]
          column = s_column
      if type in (tokenize.COMMENT, tokenize.NL):
        prefix += value
        lineno, column = end
        if value.endswith('\n'):
          lineno += 1
          column = 0
        continue
      if type == token.OP:
        type = grammar.opmap[value]
      if debug:
        self.logger.debug('%s %r (prefix=%r)', token.tok_name[type], value,
                          prefix)
      if p.addtoken(type, value, (prefix, start)):
        if debug:
          self.logger.debug('Stop.')
        break
      prefix = ''
      lineno, column = end
      if value.endswith('\n'):
        lineno += 1
        column = 0
    else:
      # We never broke out -- EOF is too soon (how can this happen???)
      raise parse.ParseError('incomplete input', type, value, (prefix, start))
    return p.rootnode

  def parse_stream_raw(self, stream, debug=False):
    """Parse a stream and return the syntax tree."""
    tokens = tokenize.generate_tokens(stream.readline)
    return self.parse_tokens(tokens, debug)

  def parse_stream(self, stream, debug=False):
    """Parse a stream and return the syntax tree."""
    return self.parse_stream_raw(stream, debug)

  def parse_file(self, filename, encoding=None, debug=False):
    """Parse a file and return the syntax tree."""
    with io.open(filename, 'r', encoding=encoding) as stream:
      return self.parse_stream(stream, debug)

  def parse_string(self, text, debug=False):
    """Parse a string and return the syntax tree."""
    tokens = tokenize.generate_tokens(io.StringIO(text).readline)
    return self.parse_tokens(tokens, debug)

def _generate_pickle_name(gt):
  # type:(str) -> str
  """Get the filepath to write a pickle file to
  given the path of a grammar textfile.

  The returned filepath should be in a user-specific cache directory.

  Args:
      gt (str): path to grammar text file

  Returns:
      str: path to pickle file
  """

  grammar_textfile_name = os.path.basename(gt)
  head, tail = os.path.splitext(grammar_textfile_name)
  if tail == '.txt':
    tail = ''
  cache_dir = user_cache_dir(
      appname='YAPF', appauthor='Google', version=yapf_version)
  return cache_dir + os.sep + head + tail + '-py' + '.'.join(
      map(str, sys.version_info)) + '.pickle'

def load_grammar(gt='Grammar.txt',
                 gp=None,
                 save=True,
                 force=False,
                 logger=None):
  # type:(str, str | None, bool, bool, logging.Logger | None) -> grammar.Grammar
  """Load the grammar (maybe from a pickle)."""
  if logger is None:
    logger = logging.getLogger()
  gp = _generate_pickle_name(gt) if gp is None else gp
  grammar_text = gt
  try:
    newer = _newer(gp, gt)
  except OSError as err:
    logger.debug('OSError, could not check if newer: %s', err.args)
    newer = True
  if not os.path.exists(gt):
    # Assume package data
    gt_basename = os.path.basename(gt)
    pd = pkgutil.get_data('yapf_third_party._ylib2to3', gt_basename)
    if pd is None:
      raise RuntimeError('Failed to load grammer %s from package' % gt_basename)
    grammar_text = io.StringIO(pd.decode(encoding='utf-8'))
  if force or not newer:
    g = pgen.generate_grammar(grammar_text)
    if save:
      try:
        Path(gp).parent.mkdir(parents=True, exist_ok=True)
        g.dump(gp)
      except OSError:
        # Ignore error, caching is not vital.
        pass
  else:
    g = grammar.Grammar()
    g.load(gp)
  return g

def _newer(a, b):
  """Inquire whether file a was written since file b."""
  if not os.path.exists(a):
    return False
  if not os.path.exists(b):
    return True
  return os.path.getmtime(a) >= os.path.getmtime(b)

def load_packaged_grammar(package, grammar_source):
  """Normally, loads a pickled grammar by doing
        pkgutil.get_data(package, pickled_grammar)
    where *pickled_grammar* is computed from *grammar_source* by adding the
    Python version and using a ``.pickle`` extension.

    However, if *grammar_source* is an extant file, load_grammar(grammar_source)
    is called instead. This facilitates using a packaged grammar file when needed
    but preserves load_grammar's automatic regeneration behavior when possible.

    """  # noqa: E501
  if os.path.isfile(grammar_source):
    return load_grammar(grammar_source)
  pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
  data = pkgutil.get_data(package, pickled_name)
  g = grammar.Grammar()
  g.loads(data)
  return g

def main(*args):
  """Main program, when run as a script: produce grammar pickle files.

    Calls load_grammar for each argument, a path to a grammar text file.
    """
  if not args:
    args = sys.argv[1:]
  logging.basicConfig(
      level=logging.INFO, stream=sys.stdout, format='%(message)s')
  for gt in args:
    load_grammar(gt, save=True, force=True)
  return True

if __name__ == '__main__':
  sys.exit(int(not main()))


## test RP and multi-RP
from __future__ import print_function
import numpy as np
from scipy.io import loadmat
from sklearn.preprocessing import MinMaxScaler
from Deep_Evo_NN.classes.DeepAE import DAE
from Toolbox.Preprocessing import Processor
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.linear_model import RidgeClassifier, LogisticRegression
from sklearn.svm import LinearSVC
from Deep_Evo_NN.classes.MLP import MLP
from sklearn.neighbors import KNeighborsClassifier
from sklearn.decomposition import PCA
from Deep_Evo_NN.classes.ELM import BaseELM
from Deep_Evo_NN.classes.StackedAE import SAE
from Deep_Evo_NN.classes.ML_ELM import ML_ELM
import time
import scipy.io as sio
'''
----------------
Load UCI data sets
----------------
TODO: edit path to matlab files,
'''
# path = 'F:\Python\UCIDataset-matlab\UCI_25.mat'
path = 'F:\Python\Deep_dataset\cifar-10.mat'
mat = loadmat(path)
keys = mat.keys()
keys.remove('__version__')
keys.remove('__header__')
keys.remove('__globals__')
keys.sort()
save_name = 'result.npz'
# # load data
# keys=['satellite','segment' ,'soybean','vowel' ,'wdbc','yeast' ,'zoo', 'optdigits']
p = Processor()
key = 'cifar'
data = mat[key]
X, y = data[:, 1:].astype('float32'), data[:, 0].astype('int8')  # load_iris(return_X_y=True)
# X = PCA(n_components=20).fit_transform(X)

'''
__________________________________________________________________
# Load HSI data
# '''
# root = 'F:\\Python\\HSI_Files\\'
# # im_, gt_ = 'SalinasA_corrected', 'SalinasA_gt'
# # im_, gt_ = 'Salinas_corrected', 'Salinas_gt'
# # im_, gt_ = 'Botswana', 'Botswana_gt'
# im_, gt_ = 'Indian_pines_corrected', 'Indian_pines_gt'
# # im_, gt_ = 'KSC', 'KSC_gt'
# # im_, gt_ = 'wuhanTM', 'wuhanTM_gt'
# #
# img_path = root + im_ + '.mat'
# gt_path = root + gt_ + '.mat'
#
# #
# print(img_path)
# p = Processor()
# img, gt = p.prepare_data(img_path, gt_path)
# n_row, n_clo, n_bands = img.shape
# print ('img=', img.shape)
# if n_bands > 50:
#     pca_img = p.pca_transform(20, img.reshape(n_row * n_clo, n_bands)).reshape(n_row, n_clo, 20)
#     X, y = p.get_correct(pca_img, gt)
# else:
#     X, y = p.get_correct(img, gt)
#
# print(X.shape)

'''
-----------------------
MINST
-----------------------
'''
# from sklearn.datasets import fetch_mldata
# from sklearn.decomposition import PCA
# mnist = fetch_mldata("MNIST original")
# # rescale the data, use the traditional train/test split
# X, y = mnist.data / 255., mnist.target
# X = PCA(n_components=20).fit_transform(X)
# # X, y = X[:1000], y[:1000]
# print('MINST:', X.shape, np.unique(y))

'''
-----------------------
Fashion MINST
-----------------------
'''
# path = 'F:\Python\Fashion-mnist-dataset\Fashion-mnist.mat'
# mat = sio.loadmat(path)
# X, y = mat['X'], mat['y'].flatten()
# X = PCA(n_components=20).fit_transform(X)
# # X, y = X[:1000], y[:1000]
# print('Fashion MINST:', X.shape, np.unique(y))

'''
___________________________________________________________________
Data pre-processing
'''
# remove these samples with small numbers
classes = np.unique(y)
print ('size:', X.shape, 'n_classes:', classes.shape[0])

for c in classes:
    if np.nonzero(y == c)[0].shape[0] < 10:
        X = np.delete(X, np.nonzero(y == c), axis=0)
        y = np.delete(y, np.nonzero(y == c))
y = p.standardize_label(y)

X = MinMaxScaler().fit_transform(X)
n_clusters = np.unique(y).size
print(X.shape, np.unique(y))

# hidden = [200, 200]
# clf = [DAE(hidden, ridge_alpha=C, logic_iter=max_iter, max_iter=max_iter),
#        ML_ELM(hidden.__len__(), hidden),
#        RidgeClassifier(alpha=C),
#        LogisticRegression(C=1e5, max_iter=max_iter),
#        BaseELM(hidden[-1]),
#        MLP(hidden, batch_size=32, learn_rate=.5, epochs=max_iter),
#        SAE(hidden, learn_rate=.5, epoch=max_iter, constraint=None),
#        SAE(hidden, learn_rate=.5, epoch=max_iter, constraint='L1')
#        # SAE(hidden, learn_rate=5, epoch=max_iter, constraint='L2')
#        ]
#
# key_clf = ['ours', 'ML-ELM', 'Ridge', 'LR', 'ELM', 'MLP', 'SAE', 'SAE-L1']
# for k, c in zip(key_clf, clf):
#     start = time.clock()
#     c.fit(X_train, y_train)
#     running_time = round(time.clock() - start, 3)
#     y_pre = c.predict(X_test)
#     print('%s : acc=%s, time=%s' % (k, accuracy_score(y_test, y_pre), running_time))
#
# # dae = DAE([10, 10, 10], max_iter=300)
# # dae.fit(X_train, y_train)
# # y_pre = dae.predict(X_test)
# # print('acc:', accuracy_score(y_test, y_pre))

hidden = range(10, 210, 10)
accs = []
t = []
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.9, random_state=42)
print('train size: %s, train classes: %s' % (X_train.shape[0], np.unique(y_train)))
max_iter = 1000
C = 1e-6
for h in hidden:
    start = time.clock()
    dae = DAE([h]*2, ridge_alpha=C, logic_iter=max_iter, max_iter=max_iter)
    dae.fit(X_train, y_train)
    running_time = round(time.clock() - start, 3)
    y_pre = dae.predict(X_test)
    acc = np.round_(accuracy_score(y_test, y_pre) * 100, 2)
    accs.append(acc)
    t.append(running_time)
    print('----------------------------------------------')
    print('#hidden %s => acc=%s, time=%s' % (h, acc, running_time))

np.savez('Experiment/para_sensi-hidden-time-cifar-10.npz', acc=accs, time=t)

"-----------------------------------------------"
# import numpy as np
# import matplotlib.pyplot as plt
# p = 'F:\Python\Deep_Evo_NN\demo\Experiment\para_sensi-hidden-time-cifar-10.npz'
# a = np.load(p)
# acc, time = a['acc'], a['time']
# err = 100. - np.asanyarray(acc)
# hidden = range(10, 210, 10)
# fig, ax1 = plt.subplots()
# ax1.plot(hidden, err, 'b-')
# ax1.set_xlabel('Number Of Nodes')
# # Make the y-axis label, ticks and tick labels match the line color.
# ax1.set_ylabel('Test Error', color='b')
# ax1.tick_params('y', colors='b')
#
# ax2 = ax1.twinx()
# ax2.plot(hidden, time, 'r--')
# ax2.set_ylabel('Time Cost (Second)', color='r')
# ax2.tick_params('y', colors='r')
#
# fig.tight_layout()
# plt.grid(True)
# plt.show()


import PySimpleGUI as sg
from pytube import YouTube
# import get_one_value as g1

start_layout = [[sg.Input(key = '-INPUT-'), sg.Button('Submit')]]

sg.set_options(font = 'Calibri 16')
#sg.Print(start_layout)

info_tab = [
    [sg.Text('Title :'), sg.Text('', key = '-TITLE-')],
    [sg.Text('Length :'), sg.Text('', key = '-LENGTH-')],
    [sg.Text('Views :'), sg.Text('', key = '-VIEWS-')],
    [sg.Text('Author :'), sg.Text('', key = '-AUTHOR-')],
    [sg.Text('Description :'), sg.Multiline('', key = '-DESCRIPTION-', size =(40,20), no_scrollbar = True, disabled =True)],
]

download_tab = [
    [sg.Frame('Best Quality',  [[sg.Button('Download', key = '-BEST-'),  sg.Text('', key = '-BEST_RES-'),  sg.Text('', key = '-BEST_SIZE-')]])],
    [sg.Frame('Worst Quality', [[sg.Button('Download', key = '-WORST-'), sg.Text('', key = '-WORST_RES-'), sg.Text('', key = '-WORST_SIZE-')]])],
    [sg.Frame('Audio',         [[sg.Button('Download', key = '-AUDIO-'),                                   sg.Text('', key = '-AUDIO_SIZE-')]])],
    [sg.VPush()],
    [sg.Progress(100, size = (20,20), expand_x = True, key = '-PROGRESS_BAR-')]
]

layout = [
    [sg.TabGroup([[
        sg.Tab('Info', info_tab), sg.Tab('Download', download_tab)
    ]])]
    ]

window = sg.Window('Youtube Downloader', start_layout)

while True:
    event, values = window.read()
    if event == sg.WIN_CLOSED:
        break

    if event == 'Submit':
        video_url = values['-INPUT-']
        window.close()
#        print('URL : ', video_url)

        if video_url == '' :
            sg.popup('NO info entrado..\nPrograma va a ser cerrado..')
            break

        window = sg.Window('Youtube Downloader', layout, finalize = True)
        video_object = YouTube(video_url)

        # info
        play_time_min = 0
        window['-TITLE-'].update(       video_object.title)
        play_time_hour = int(round(     video_object.length / 3600, 0))
        play_time_min  = (video_object.length - (play_time_hour * 3600)) / 60
# #        print(play_time_hour, play_time_min)
        window['-LENGTH-'].update(f'{play_time_hour} hours {round(play_time_min, 2)} min') # hours {round(video_object.length / 60, 2)} minutes')
        window['-VIEWS-'].update(       video_object.views)
        window['-AUTHOR-'].update(      video_object.author)
        window['-DESCRIPTION-'].update( video_object.description)

        # download
#        filesize_highest = video_object.streams.get_highest_resolution().filesize   / 1048576
#        print(filesize_highest)
#        window['-BEST_SIZE-'].update( f'{round( filesize_highest , 1 )} MB')
#        window['-BEST_RES-'].update(             video_object.streams.get_highest_resolution().resolution )

#        print(video_object.title)
#        window['-LENGTH-'].update(video_object.length)
    if event == '-BEST-':
        video_object.streams.get_highest_resolution().download()
    #     pass

    if event == '-WORST-':
        video_object.streams.get_lowest_resolution().download()
    #     pass

    if event == '-AUDIO-':
        video_object.streams.get_audio_only().download()
    #     pass

window.close()


# Uses python3
import sys
from collections import namedtuple

Segment = namedtuple('Segment', 'start end')

def optimal_points(segments):
    points = []
    #write your code here
    print(segments)

    return points

if __name__ == '__main__':
    inp = int(input())
    seg = []
    points = []
    for i in range(inp):
        time = sys.stdin.readline()
        seg.append(list(map(int, time.split())))
    seg.sort()
    mn = seg[0][0]
    mx = seg[0][1]
    for i in range(inp-1):
        mn1 = max(mn, seg[i+1][0])
        mx1 = min(mx, seg[i+1][1])
        if mx1 < mn1:
            points.append(max(mn, mx))
            mn = seg[i+1][0]
            mx = seg[i+1][1]
        else:
            mn = mn1
            mx = mx1
    points.append(max(mn, mx))
    print(len(points))
    for i in points:
        print(i, end=" ")

algorithm toolbox/week3 greedy algo/covering_segments.py


from mycroft import MycroftSkill, intent_handler
from mycroft.util import create_daemon, connected
from mycroft.messagebus.message import Message
import subprocess
import pexpect
from time import sleep

class WifiConnectSkill(MycroftSkill):
    def __init__(self):
        MycroftSkill.__init__(self)
        self.reload_skill = False  # no reloading until boot finishes
        self.monitoring = False
        self.in_setup = False
        self.connected = False
        self.wifi_process = None
        self.debug = False  # dev setting, VERY VERBOSE DIALOGS
        # TODO expose all this in skill settings
        self.ssid = "OVOS"
        self.pswd = None
        self.grace_period = 45
        self.time_between_checks = 30  # seconds
        self.mycroft_ready = False
        self.wifi_command = "sudo /usr/local/sbin/wifi-connect --portal-ssid {ssid}"
        if self.pswd:
            self.wifi_command += " --portal-passphrase {pswd}"
        if "color" not in self.settings:
            self.settings["color"] = "#FF0000"
        if "stop_on_internet" not in self.settings:
            self.settings["stop_on_internet"] = False
        if "timeout_after_internet" not in self.settings:
            self.settings["timeout_after_internet"] = 90

    def initialize(self):
        self.add_event("mycroft.internet.connected",
                       self.handle_internet_connected)
        self.add_event("mycroft.ready",
                       self.handle_mycroft_ready)
        self.start_internet_check()

    def handle_mycroft_ready(self):
        self.mycroft_ready = True

    # internet watchdog
    def start_internet_check(self):
        create_daemon(self._watchdog)

    def stop_internet_check(self):
        self.monitoring = False

    def _watchdog(self):
        try:
            self.monitoring = True
            self.log.info("Wifi watchdog started")
            output = subprocess.check_output("nmcli connection show",
                                             shell=True).decode("utf-8")
            if "wifi" in output:
                self.log.info("Detected previously configured wifi, starting "
                              "grace period to allow it to connect")
                sleep(self.grace_period)
            while self.monitoring:
                if self.in_setup:
                    sleep(1)  # let setup do it's thing
                    continue

                if not connected():
                    self.log.info("NO INTERNET")
                    if not self.is_connected_to_wifi():
                        self.log.info("LAUNCH SETUP")
                        try:
                            self.launch_wifi_setup()  # blocking
                        except Exception as e:
                            self.log.exception(e)
                    else:
                        self.log.warning("CONNECTED TO WIFI, BUT NO INTERNET!!")

                sleep(self.time_between_checks)
        except Exception as e:
            self.log.error("Wifi watchdog crashed unexpectedly")
            self.log.exception(e)

    # wifi setup
    @staticmethod
    def get_wifi_ssid():
        SSID = None
        try:
            SSID = subprocess.check_output(["iwgetid", "-r"]).strip()
        except subprocess.CalledProcessError:
            # If there is no connection subprocess throws a 'CalledProcessError'
            pass
        return SSID

    @staticmethod
    def is_connected_to_wifi():
        return WifiConnectSkill.get_wifi_ssid() is not None

    def launch_wifi_setup(self):
        if not self.in_setup:
            self.bus.emit(Message("ovos.wifi.setup.started"))
        self.stop_setup()
        self.in_setup = True
        self.wifi_process = pexpect.spawn(
            self.wifi_command.format(ssid=self.ssid)
        )
        # https://github.com/pexpect/pexpect/issues/462
        self.wifi_process.delayafterclose = 1
        self.wifi_process.delayafterterminate = 1
        prev = ""
        restart = False
        if self.debug:
            self.speak_dialog("debug_start_setup")

        while self.in_setup:
            try:
                out = self.wifi_process.readline().decode("utf-8").strip()
                if out == prev:
                    continue
                prev = out
                if out.startswith("Access points: "):
                    aps = list(out.split("Access points: ")[-1])
                    self.log.info(out)
                    if self.debug:
                        self.speak_dialog("debug_wifi_scanned")
                elif out.startswith("Starting access point..."):
                    if self.debug:
                        self.speak_dialog("debug_ap_start")
                elif out.startswith("Access point ") and \
                        out.endswith("created"):
                    self.prompt_to_join_ap()
                    if self.debug:
                        self.speak_dialog("debug_ap_created")
                elif out.startswith("Starting HTTP server on"):
                    self.log.debug(out)
                    if self.debug:
                        self.speak_dialog("debug_http_started")
                elif out.startswith("Stopping access point"):
                    if self.debug:
                        self.speak_dialog("debug_ap_stop")
                elif out.startswith("Access point ") and \
                        out.endswith("stopped"):
                    if self.debug:
                        self.speak_dialog("debug_ap_stopped")
                elif out == "User connected to the captive portal":
                    self.log.info(out)
                    self.prompt_to_select_network()
                    if self.debug:
                        self.speak_dialog("debug_user_connected")
                elif out.startswith("Connecting to access point"):
                    if self.debug:
                        self.speak_dialog("debug_connecting")
                elif out.startswith("Internet connectivity established"):
                    self.log.info(out)
                    self.report_setup_complete()
                    if self.debug:
                        self.speak_dialog("debug_wifi_connected")
                elif "Error" in out or "[Errno" in out:
                    self.log.error(out)
                    self.report_setup_failed()

                    # TODO figure out at least the errors handled gracefully
                    accepted_errors = [
                        "Password length should be at least 8 characters"
                    ]
                    for e in accepted_errors:
                        if e in out:
                            continue
                    else:
                        restart = True
                        break

                if self.debug:
                    self.log.debug(out)
            except pexpect.exceptions.EOF:
                # exited
                self.log.info("Exited wifi setup process")
                break
            except pexpect.exceptions.TIMEOUT:
                # nothing happened for a while
                pass
            except KeyboardInterrupt:
                break
            except Exception as e:
                self.log.exception(e)
                break
        self.stop_setup()
        if restart:
            # handle bugs in balena, sometimes it fails to come back up
            # seems to happen on
            # Error: Getting access points failed
            self.launch_wifi_setup()
        elif self.debug:
            self.speak_dialog("debug_end_setup")

    # intents
    @intent_handler("launch_setup.intent")
    def wifi_intent(self, message):
        self.launch_wifi_setup()

    # bus events
    def handle_internet_connected(self, message=None):
        """System came online later after booting."""
        self.enclosure.mouth_reset()
        self.stop_setup()  # just in case
        self.gui.release()

    # GUI events
    def prompt_to_join_ap(self, message=None):
        """Provide instructions for setting up wifi."""
        self.manage_setup_display("join-ap", "prompt")
        self.speak_dialog("wifi_intro_2")
        # allow GUI to linger around for a bit, will block the wifi setup loop
        sleep(2)

    def prompt_to_select_network(self, message=None):
        """Prompt user to select network and login."""
        self.manage_setup_display("select-network", "prompt")
        self.speak_dialog("wifi_intro_3")
        # allow GUI to linger around for a bit, will block the wifi setup loop
        sleep(2)

    def report_setup_complete(self, message=None):
        """Wifi setup complete, network is connected."""
        # once first connected to internet increase time between checks
        self.connected = True
        self.time_between_checks = self.settings["timeout_after_internet"]
        # stop watchdog on internet connection
        if self.settings["stop_on_internet"]:
            self.monitoring = False
        self.manage_setup_display("setup-completed", "status")
        # allow GUI to linger around for a bit, will block the wifi setup loop
        sleep(3)
        self.bus.emit(Message("ovos.wifi.setup.completed"))
        # pairing skill should take over now
        self.gui.release()

    def report_setup_failed(self, message=None):
        """Wifi setup failed"""
        self.manage_setup_display("setup-failed", "status")
        self.speak_dialog("debug_wifi_error")
        # allow GUI to linger around for a bit, will block the wifi setup loop
        sleep(2)

    def manage_setup_display(self, state, page_type):
        self.gui.clear()
        if state == "join-ap" and page_type == "prompt":
            self.gui["image"] = "1_phone_connect-to-ap.png"
            self.gui["label"] = "Connect to the Wi-Fi network"
            self.gui["highlight"] = self.ssid
            self.gui["color"] = self.settings["color"]
            self.gui["page_type"] = "Prompt"
            self.gui.show_page("NetworkLoader.qml", override_animations=True,
                               override_idle=True)
        elif state == "select-network" and page_type == "prompt":
            self.gui["image"] = "3_phone_choose-wifi.png"
            self.gui["label"] = "Select local Wi-Fi network to connect"
            self.gui["highlight"] = "OVOS Device"
            self.gui["color"] = self.settings["color"]
            self.gui["page_type"] = "Prompt"
            self.gui.show_page("NetworkLoader.qml", override_animations=True,
                               override_idle=True)
        elif state == "setup-completed" and page_type == "status":
            self.gui["image"] = "icons/check-circle.svg"
            self.gui["label"] = "Connected"
            self.gui["highlight"] = ""
            self.gui["color"] = "#40DBB0"
            self.gui["page_type"] = "Status"
            self.gui.show_page("NetworkLoader.qml", override_animations=True)
        elif state == "setup-failed" and page_type == "status":
            self.gui["image"] = "icons/times-circle.svg"
            self.gui["label"] = "Connection Failed"
            self.gui["highlight"] = ""
            self.gui["color"] = "#FF0000"
            self.gui["page_type"] = "Status"
            self.gui.show_page("NetworkLoader.qml", override_animations=True)

    # cleanup
    def stop_setup(self):
        if self.wifi_process is not None:
            try:
                if self.wifi_process.isalive():
                    self.log.debug("terminating wifi setup process")
                    self.wifi_process.sendcontrol('c')
                    sleep(1)
                    self.wifi_process.close()
                    sleep(1)
                if self.wifi_process.isalive():
                    self.log.warning('wifi setup did not exit gracefully.')
                    self.wifi_process.close(force=True)
                    sleep(1)
                    if self.wifi_process.isalive():
                        self.log.warning('trying to terminate wifi setup process')
                        self.wifi_process.terminate()
                        sleep(1)
                else:
                    self.log.debug('wifi setup exited gracefully.')
            except Exception as e:
                self.log.exception(e)
        self.wifi_process = None
        self.in_setup = False

    def shutdown(self):
        self.monitoring = False
        self.stop_setup()

def create_skill():
    return WifiConnectSkill()


import os
import sqlite3

db_filename = 'todo.db'
db_is_new = not os.path.exists(db_filename)
conn = sqlite3.connect(db_filename)
cur = conn.cursor()
cur.execute("""CREATE TABLE IF NOT EXISTS users(
    userid INT PRIMARY KEY,
    fname TEXT,
    lname TEXT,
    gender TEXT);
""")
cur.execute("""CREATE TABLE IF NOT EXISTS orders(
    orderid INT PRIMARY KEY,
    date DATE,
    userid TEXT,
    total TEXT);
""")
conn.commit()
conn.close()


import os

from engine.core import module
from engine.hardware import use_gpu, first_device, all_devices, device_description
from engine.logging import print_info, print_errors, print_debug

import torch

from engine.parameters import special_parameters
from engine.path import output_path

_checkpoint_path = 'models/{}.torch'

_model = None
_optimizer = None

_checkpoint = {}

@module
def create_model(model_class, model_params=None, model_name='model'):
    """
    create and eventually load model
    :param model_name:
    :param model_class:
    :param model_params:
    :param model_name:
    :return:
    """

    model_params = {} if model_params is None else model_params

    model = model_class(**model_params)

    if special_parameters.load_model:  # recover from checkpoint
        _load_model(model, model_name)

    # configure usage on GPU
    if use_gpu():
        model.to(first_device())
        model = torch.nn.DataParallel(model, device_ids=all_devices())

    # print info about devices
    print_info('Device(s)): ' + str(device_description()))

    return model

def create_optimizer(parameters, optimizer_class, optim_params, model_name='model'):
    """
    create and eventually load optimizer
    :param model_name:
    :param parameters:
    :param optimizer_class:
    :param optim_params:
    :return:
    """
    opt = optimizer_class(parameters, **optim_params)
    if special_parameters.load_model:
        _load_optimizer(opt, model_name)
    return opt

def _load_optimizer(optimizer, model_name):
    """
    load checkpoint
    :param optimizer:
    :return:
    """
    global _checkpoint
    if model_name not in _checkpoint:
        _load_checkpoint(model_name)

    if 'optimizer_state_dict' in _checkpoint[model_name]:
        optimizer.load_state_dict(_checkpoint[model_name]['optimizer_state_dict'])

def _load_model(model, model_name, path=None, reload=False):
    """
    load checkpoint
    :param model:
    :param model_name:
    :return:
    """
    global _checkpoint
    if model_name not in _checkpoint or reload:
        _load_checkpoint(model_name, path=path)

    if 'model_state_dict' in _checkpoint[model_name]:
        model.load_state_dict(_checkpoint[model_name]['model_state_dict'])
    else:
        model.load_state_dict(_checkpoint[model_name])

def _load_checkpoint(model_name, path=None):
    if path is None:
        path = output_path(_checkpoint_path.format(model_name), have_validation=True)

    global _checkpoint
    if not os.path.isfile(path):
        print_errors('{} does not exist'.format(path), do_exit=True)
    print_debug('Loading checkpoint from ' + path)
    _checkpoint[model_name] = torch.load(path)

def load_checkpoint(model, model_name='model', validation_id=None):
    """
    change state of the model
    """
    path = output_path(_checkpoint_path.format(model_name), validation_id=validation_id, have_validation=True)
    _load_model(model.module if type(model) is torch.nn.DataParallel else model, model_name, path=path, reload=True)

def save_checkpoint(model, optimizer=None, model_name='model', validation_id=None):
    """
    save checkpoint (optimizer and model)
    :param model_name:
    :param validation_id:
    :param model:
    :param optimizer:
    :return:
    """
    path = output_path(_checkpoint_path.format(model_name), validation_id=validation_id, have_validation=True)

    print_debug('Saving checkpoint: ' + path)

    model = model.module if type(model) is torch.nn.DataParallel else model

    checkpoint = {
        'model_state_dict': model.state_dict()
    }

    if optimizer is not None:
        checkpoint['optimizer_state_dict'] = optimizer.state_dict()

    torch.save(checkpoint, path)


# sourcery skip: hoist-statement-from-if, use-fstring-for-concatenation
from sklearn.neighbors import KNeighborsClassifier
#//to create webcam
import cv2 #//opencv-module
import pickle
import numpy as np
import os
#to add the attendance, we need to
import csv
#we are going to save the attendance in 2 columns
#1. name
#2.time
import keyboard
import time
#we need to store the time
from datetime import datetime
# to store the attendance according to date
#make sound when attendance is taken
from win32com.client import Dispatch

def speak(strl):
    speak=Dispatch(("SAPI.SpVoice"))
    speak.Speak(strl)

#//to capture frame
video = cv2.VideoCapture(0)  #//0 means that we are using webcam of laptop and 1 for the external webcam
#//we are storing the captured frame in a variable
#//to detect the face
faceDetect = cv2.CascadeClassifier('data/haarcascade_frontalface_default.xml')
#to detect the face we need to convert the frame into grayscale as
# this cascade classifier is working well on periodical images
# so we will convert them to bgr2 to grayscale
#as opencv reads the image in the bgr format
#we will create a list to store the face images and name associated with it
#we are loading the face_data and name from our data directory
with open('data/name.pkl','rb') as f: #rb-read mode
        #we will save it into our pickle file
    LABELS=pickle.load(f) #this will take 2 parameters

with open('data/face_data.pkl','rb') as f:
        #we will save it into our pickle file
    FACES=pickle.load(f) #this will take 2 parameters

#create an object for classification
knn=KNeighborsClassifier(n_neighbors=5)
knn.fit(FACES,LABELS) # we need to fit our knn model according to the data

COL_NAMES=['NAME','TIME']

while True:
    ret,frame = video.read()  #//we are reading the frame using the read method
    #// the read method gives us 2 values
    #//1. boolean value - if the webcam is okay or not
    #//2. frame
    gray=cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
    #now we will detect the face
    faces=faceDetect.detectMultiScale(gray,1.3,5)
    #1.3 and 5 are threshold values
    #now we are trying to get the coordinate values from the faces
    # that is x,y,w,h
    #x and y are coordinates w is width and h is height of the images
    # we will iterate through all the values here
    for (x,y,w,h) in faces:
        #now we will create a rectangle so that we can see that our face is detected
        # we will pass the original frame so that we dont need to see the grayscale frame at all
        # and we will pass the cardinal values that is x and y
        # width and height of the channel
        # color as red (50,50,255)
        # we will give thickness as 1
        #now we are going to crop the images and store them in a list called faces and
        # a name will be associated with it and we will also resize it 15x50
        # in frame we will go from x to x+w
        # and y to y+h
        crop_image=frame[y:y+h,x:x+w,:]
        # now we will resize the images
        resized_image = cv2.resize(crop_image,(50,50)).flatten().reshape(1,-1)
        #(50,50) is the size to which image should be resized
        # we will store the resized image into the list
        #we are going to take the images after 10 frames
        #to see how many images have been takes
        output = knn.predict(resized_image)
        #storing time
        ts = time.time()
        #date instance
        date=datetime.fromtimestamp(ts).strftime("%d-%m-%Y")
        #time instance
        timestamp = datetime.fromtimestamp(ts).strftime("%H:%M-%S")
        exist = os.path.isfile("Attendance/Attendance_"+ date + ".csv")

        #to design the rectangle around the face
        cv2.rectangle(frame,(x,y),(x+w,y+h),(0,0,255),1)
        cv2.rectangle(frame,(x,y),(x+w,y+h),(50,50,255),2)
        cv2.rectangle(frame,(x,y-40),(x+w,y),(50,50,255),-1)
        cv2.putText(frame,str(output[0]),(x,y-15),cv2.FONT_HERSHEY_COMPLEX,1,(255,255,255),1)
        #output[0] = name at first index of all the nearest 5 neighbors found out
        #origin/location
        #font
        #font scale
        #color
        #thickness
        cv2.rectangle(frame,(x,y),(x+w,y+h),(50,50,255),1)
        #now we are going to crop the images and store them in a list called faces and
        # a name will be associated with it and we will also resize it 15x50
        #storing attendance in a list
        attendance = [str(output[0]),str(timestamp)]

    #//now we will show the frame
    cv2.imshow("frame",frame)
    #to pass attendance
    k=cv2.waitKey(1)
    if k & keyboard.is_pressed("o"):
        speak("Attendance taken..")
        time.sleep(5)
        if exist:
            with open("Attendance/Attendance_"+ date + ".csv","+a") as csvfile:
                writer = csv.writer(csvfile)
                writer.writerow(attendance)
            csvfile.close()
        else:
            with open("Attendance/Attendance_"+ date + ".csv","+a") as csvfile:
                writer = csv.writer(csvfile)
                writer.writerow(COL_NAMES)
                writer.writerow(attendance)
            csvfile.close()
    #//to add the keyword binding function

    if k & keyboard.is_pressed("q"): #//when passed q from the keyboard
        break #//infinite loop and video frame will be gone

#//we will release the stored video
video.release()
#//now we will destroy all the windows
cv2.destroyAllWindows()

#here, it finds the 5 nearest neighbors from the vector coming from the webcam