"""
The code is based on the original ResNet implementation from torchvision.models.resnet
"""

import torch.nn as nn

def conv3x1(in_planes, out_planes, stride=1, groups=1, dilation=1):
    """ 1D convolution with kernel size 3 """
    return nn.Conv1d(
        in_planes,
        out_planes,
        kernel_size=3,
        stride=stride,
        padding=dilation,
        groups=groups,
        bias=False,
        dilation=dilation,
    )

def conv1x1(in_planes, out_planes, stride=1):
    """ 1D convolution with kernel size 1 """
    return nn.Conv1d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)

class BasicBlock1D(nn.Module):
    """ Supports: groups=1, dilation=1 """

    expansion = 1

    def __init__(self, in_planes, planes, stride=1, downsample=None):
        super(BasicBlock1D, self).__init__()
        # Both self.conv1 and self.downsample layers downsample the input when stride != 1
        self.conv1 = conv3x1(in_planes, planes, stride)
        self.bn1 = nn.BatchNorm1d(planes)
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = conv3x1(planes, planes * self.expansion)
        self.bn2 = nn.BatchNorm1d(planes * self.expansion)
        self.stride = stride
        self.downsample = downsample

    def forward(self, x):
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        if self.downsample is not None:
            identity = self.downsample(x)

        out += identity
        out = self.relu(out)

        return out

class Bottleneck(nn.Module):
    # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2)
    # while original implementation places the stride at the first 1x1 convolution(self.conv1)
    # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385.
    # This variant is also known as ResNet V1.5 and improves accuracy according to
    # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch.

    expansion = 4

    def __init__(
        self,
        inplanes,
        planes,
        stride=1,
        downsample=None,
        groups=1,
        base_width=64,
        dilation=1,
        norm_layer=None,
    ):
        super(Bottleneck, self).__init__()
        if norm_layer is None:
            norm_layer = nn.BatchNorm2d
        width = int(planes * (base_width / 64.0)) * groups
        # Both self.conv2 and self.downsample layers downsample the input when stride != 1
        self.conv1 = conv1x1(inplanes, width)
        self.bn1 = norm_layer(width)
        self.conv2 = conv3x1(width, width, stride, groups, dilation)
        self.bn2 = norm_layer(width)
        self.conv3 = conv1x1(width, planes * self.expansion)
        self.bn3 = norm_layer(planes * self.expansion)
        self.relu = nn.ReLU(inplace=True)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            identity = self.downsample(x)

        out += identity
        out = self.relu(out)

        return out

class FcBlock(nn.Module):
    def __init__(self, in_channel, out_channel, in_dim):
        super(FcBlock, self).__init__()
        self.in_channel = in_channel
        self.out_channel = out_channel
        self.prep_channel = 128
        self.fc_dim = 512
        self.in_dim = in_dim

        # prep layer2
        self.prep1 = nn.Conv1d(
            self.in_channel, self.prep_channel, kernel_size=1, bias=False
        )
        self.bn1 = nn.BatchNorm1d(self.prep_channel)
        # fc layers
        self.fc1 = nn.Linear(self.prep_channel * self.in_dim, self.fc_dim)
        self.fc2 = nn.Linear(self.fc_dim, self.fc_dim)
        self.fc3 = nn.Linear(self.fc_dim, self.out_channel)
        self.relu = nn.ReLU(True)
        self.dropout = nn.Dropout(0.5)

    def forward(self, x):
        x = self.prep1(x)
        x = self.bn1(x)
        x = self.fc1(x.view(x.size(0), -1))
        x = self.relu(x)
        x = self.dropout(x)
        x = self.fc2(x)
        x = self.relu(x)
        x = self.dropout(x)
        x = self.fc3(x)
        return x

class ResNet1D(nn.Module):
    """
    ResNet 1D
    in_dim: input channel (for IMU data, in_dim=6)
    out_dim: output dimension (3)
    len(group_sizes) = 4
    """

    def __init__(
        self,
        block_type,
        in_dim,
        out_dim,
        group_sizes,
        inter_dim,
        zero_init_residual=False,
    ):
        super(ResNet1D, self).__init__()
        self.base_plane = 64
        self.inplanes = self.base_plane

        # Input module
        self.input_block = nn.Sequential(
            nn.Conv1d(
                in_dim, self.base_plane, kernel_size=7, stride=2, padding=3, bias=False
            ),
            nn.BatchNorm1d(self.base_plane),
            nn.ReLU(inplace=True),
            nn.MaxPool1d(kernel_size=3, stride=2, padding=1),
        )

        # Residual groups
        self.residual_groups = nn.Sequential(
            self._make_residual_group1d(block_type, 64, group_sizes[0], stride=1),
            self._make_residual_group1d(block_type, 128, group_sizes[1], stride=2),
            self._make_residual_group1d(block_type, 256, group_sizes[2], stride=2),
            self._make_residual_group1d(block_type, 512, group_sizes[3], stride=2),
        )

        # Output module
        self.output_block1 = FcBlock(512 * block_type.expansion, out_dim, inter_dim)
        self.output_block2 = FcBlock(512 * block_type.expansion, out_dim, inter_dim)

        self._initialize(zero_init_residual)

    def _make_residual_group1d(self, block, planes, group_size, stride=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                conv1x1(self.inplanes, planes * block.expansion, stride=stride),
                nn.BatchNorm1d(planes * block.expansion),
            )

        layers = []
        layers.append(
            block(self.inplanes, planes, stride=stride, downsample=downsample)
        )
        self.inplanes = planes * block.expansion
        for _ in range(1, group_size):
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

    def _initialize(self, zero_init_residual):
        for m in self.modules():
            if isinstance(m, nn.Conv1d):
                nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
            elif isinstance(m, nn.BatchNorm1d):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.Linear):
                nn.init.normal_(m.weight, 0, 0.01)
                nn.init.constant_(m.bias, 0)

        # Zero-initialize the last BN in each residual branch,
        # so that the residual branch starts with zeros, and each residual block behaves like an identity.
        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
        if zero_init_residual:
            for m in self.modules():
                if isinstance(m, Bottleneck1D):
                    nn.init.constant_(m.bn3.weight, 0)
                elif isinstance(m, BasicBlock1D):
                    nn.init.constant_(m.bn2.weight, 0)

    def get_num_params(self):
        return sum(p.numel() for p in self.parameters() if p.requires_grad)

    def forward(self, x):
        x = self.input_block(x)
        x = self.residual_groups(x)
        mean = self.output_block1(x)  # mean
        logstd = self.output_block2(x)  # covariance sigma = exp(2 * logstd)
        return mean, logstd


import usfm;
import click
import os.path

# 1. I would like to write another code that does this
# Usage: applyParagraphs.py bible1 bible2
# What it does: take paragraphing from bible1 and apply
# it to bible2 (first removing existing paragraphs from
# bible2).
# We need to extend what usfm.py can do, and be able to
# write very nice looking code in applyparagraphs.py
# that uses that api.
# I am thinking:
# b1 = usfm.Bible("bible1")
# b2 = usfm.Bible("bible2")
# b1.load(directory1) # bible::load routine loads all SFM files in directory1
# b2.load(directory2)
# b2.removeParagraphs() # code already done
# paragraphs = b1.listParagraphs() # draft code already done
# b2.applyParagraphs(paragraphs) # new stuff
# b2.printBible(directory2.new) # draft code done, but needs debugged
# This will be super helpful because we have translations with no paragraphing.
# I would like to find a nicely paragraphed bible and quickly apply its
# paragraphing to the "bad" Bible.

@click.command()
@click.argument('files', nargs=-1)
def main(files):
    if (len(files) == 0):
        print("Usage: python3 applyParagraphs.py bible1 bible2")
        exit(1)

    # Fill in here

if __name__ == '__main__':
    main()


from copy import copy

import pytest
import numpy as np
import pyarmnn as ann

def __get_tensor_info(dt):
    tensor_info = ann.TensorInfo(ann.TensorShape((2, 3)), dt)

    return tensor_info

@pytest.mark.parametrize("dt", [ann.DataType_Float32, ann.DataType_Float16,
                                ann.DataType_QAsymmU8, ann.DataType_QSymmS8,
                                ann.DataType_QAsymmS8])
def test_create_tensor_with_info(dt):
    tensor_info = __get_tensor_info(dt)
    elements = tensor_info.GetNumElements()
    num_bytes = tensor_info.GetNumBytes()
    d_type = dt

    tensor = ann.Tensor(tensor_info)

    assert tensor_info != tensor.GetInfo(), "Different objects"
    assert elements == tensor.GetNumElements()
    assert num_bytes == tensor.GetNumBytes()
    assert d_type == tensor.GetDataType()

def test_create_tensor_undefined_datatype():
    tensor_info = ann.TensorInfo()
    tensor_info.SetDataType(99)

    with pytest.raises(ValueError) as err:
        ann.Tensor(tensor_info)

    assert 'The data type provided for this Tensor is not supported.' in str(err.value)

@pytest.mark.parametrize("dt", [ann.DataType_Float32])
def test_tensor_memory_output(dt):
    tensor_info = __get_tensor_info(dt)
    tensor = ann.Tensor(tensor_info)

    # empty memory area because inference has not yet been run.
    assert tensor.get_memory_area().tolist()  # has random stuff
    assert 4 == tensor.get_memory_area().itemsize, "it is float32"

@pytest.mark.parametrize("dt", [ann.DataType_Float32, ann.DataType_Float16,
                                ann.DataType_QAsymmU8, ann.DataType_QSymmS8,
                                ann.DataType_QAsymmS8])
def test_tensor__str__(dt):
    tensor_info = __get_tensor_info(dt)
    elements = tensor_info.GetNumElements()
    num_bytes = tensor_info.GetNumBytes()
    d_type = dt
    dimensions = tensor_info.GetNumDimensions()

    tensor = ann.Tensor(tensor_info)

    assert str(tensor) == "Tensor{{DataType: {}, NumBytes: {}, NumDimensions: " \
                               "{}, NumElements: {}}}".format(d_type, num_bytes, dimensions, elements)

def test_create_empty_tensor():
    tensor = ann.Tensor()

    assert 0 == tensor.GetNumElements()
    assert 0 == tensor.GetNumBytes()
    assert tensor.get_memory_area() is None

@pytest.mark.parametrize("dt", [ann.DataType_Float32, ann.DataType_Float16,
                                ann.DataType_QAsymmU8, ann.DataType_QSymmS8,
                                ann.DataType_QAsymmS8])
def test_create_tensor_from_tensor(dt):
    tensor_info = __get_tensor_info(dt)
    tensor = ann.Tensor(tensor_info)
    copied_tensor = ann.Tensor(tensor)

    assert copied_tensor != tensor, "Different objects"
    assert copied_tensor.GetInfo() != tensor.GetInfo(), "Different objects"
    assert copied_tensor.get_memory_area().ctypes.data == tensor.get_memory_area().ctypes.data,  "Same memory area"
    assert copied_tensor.GetNumElements() == tensor.GetNumElements()
    assert copied_tensor.GetNumBytes() == tensor.GetNumBytes()
    assert copied_tensor.GetDataType() == tensor.GetDataType()

@pytest.mark.parametrize("dt", [ann.DataType_Float32, ann.DataType_Float16,
                                ann.DataType_QAsymmU8, ann.DataType_QSymmS8,
                                ann.DataType_QAsymmS8])
def test_copy_tensor(dt):
    tensor = ann.Tensor(__get_tensor_info(dt))
    copied_tensor = copy(tensor)

    assert copied_tensor != tensor, "Different objects"
    assert copied_tensor.GetInfo() != tensor.GetInfo(), "Different objects"
    assert copied_tensor.get_memory_area().ctypes.data == tensor.get_memory_area().ctypes.data,  "Same memory area"
    assert copied_tensor.GetNumElements() == tensor.GetNumElements()
    assert copied_tensor.GetNumBytes() == tensor.GetNumBytes()
    assert copied_tensor.GetDataType() == tensor.GetDataType()

@pytest.mark.parametrize("dt", [ann.DataType_Float32, ann.DataType_Float16,
                                ann.DataType_QAsymmU8, ann.DataType_QSymmS8,
                                ann.DataType_QAsymmS8])
def test_copied_tensor_has_memory_area_access_after_deletion_of_original_tensor(dt):

    tensor = ann.Tensor(__get_tensor_info(dt))

    tensor.get_memory_area()[0] = 100

    initial_mem_copy = np.array(tensor.get_memory_area())

    assert 100 == initial_mem_copy[0]

    copied_tensor = ann.Tensor(tensor)

    del tensor
    np.testing.assert_array_equal(copied_tensor.get_memory_area(), initial_mem_copy)
    assert 100 == copied_tensor.get_memory_area()[0]

def test_create_const_tensor_incorrect_args():
    with pytest.raises(ValueError) as err:
        ann.Tensor('something', 'something')

    expected_error_message = "Incorrect number of arguments or type of arguments provided to create Tensor."
    assert expected_error_message in str(err.value)

@pytest.mark.parametrize("dt", [ann.DataType_Float16])
def test_tensor_memory_output_fp16(dt):
    # Check Tensor with float16
    tensor_info = __get_tensor_info(dt)
    tensor = ann.Tensor(tensor_info)

    assert tensor.GetNumElements() == 6
    assert tensor.GetNumBytes() == 12
    assert tensor.GetDataType() == ann.DataType_Float16


import logging
import os
import sys
from pathlib import Path

# BASE_DIR = str(Path(os.path.dirname(os.path.abspath(__file__))).parent)
# BASE_DIR = os.path.dirname(os.path.abspath(sys.argv[0]))
BASE_DIR = str(Path(os.path.dirname(os.path.abspath(__file__))).parent)
DEBUG_LOG_FILENAME = str(Path(BASE_DIR, "info.log"))

# set up formatting
formatter = logging.Formatter("%(levelname)-5s %(asctime)s %(module)s.%(funcName)s() [%(lineno)d]: %(message)s",
                              "%Y-%m-%d %H:%M:%S")

# set up logging to STDOUT for all levels DEBUG and higher
sh = logging.StreamHandler(sys.stdout)
sh.setLevel(logging.DEBUG)
sh.setFormatter(formatter)

# set up logging to a file for all levels DEBUG and higher
fh = logging.FileHandler(DEBUG_LOG_FILENAME)
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)

# create Logger object
mylogger = logging.getLogger('MyLogger')
mylogger.setLevel(logging.DEBUG)
mylogger.addHandler(sh)    # enabled: stdout
mylogger.addHandler(fh)    # enabled: file

# create shortcut functions
debug = mylogger.debug
info = mylogger.info
warning = mylogger.warning
error = mylogger.error
critical = mylogger.critical


from loguru import logger

import pytest

from models.film import SFilmPersonDetail

from services.person import Person

@pytest.fixture(scope='module')
async def person(conf, datastore):
    logger.info('setup person class')
    person = Person(datastore)
    person.set_person_index(conf.ELASTIC_PERSON_INDEX)
    yield person
    logger.info('end setup person class')

@pytest.mark.asyncio
async def test_film_Person_Class(conf, person: Person, cache, redis, read_json_data):
    logger.info('test film person class')
    datas = await read_json_data('es_persons_data.json')
    for data in datas:
        await redis.delete(cache.genkey(index=conf.ELASTIC_PERSON_INDEX, id=data['id']))
        doc = await person.get_person_by_id(data['id'])
        cache_doc = await cache.get_data(index=conf.ELASTIC_PERSON_INDEX, id=data['id'])
        assert doc == SFilmPersonDetail(**data)
        assert doc == SFilmPersonDetail(**cache_doc)
    logger.info('end test film person class')

@pytest.mark.asyncio
async def test_films_search_Person_Class(person: Person, read_json_data):
    logger.info('test films search person')
    testsconfig = await read_json_data('person_search.json')
    for test in testsconfig:
        logger.info(f"start test : {test['name']} ")
        docs = await person.search_person(**test['parameter'])
        assert len(docs) == test['lenght']
        if test['body'] != '':
            assert docs == [SFilmPersonDetail(**doc) for doc in await read_json_data('person/' + test['body'])]
        logger.info(f"{test['name']} passed")
    logger.info('end test films search person')

@pytest.mark.asyncio
async def test_films_get_all_person_Person_Class(person: Person, read_json_data):
    logger.info('test films get_all person')
    testsconfig = await read_json_data('person_get_all.json')
    for test in testsconfig:
        logger.info(f"start test : {test['name']} ")
        docs = await person.get_all_person(**test['parameter'])
        assert len(docs) == test['lenght']
        if test['body'] != '':
            assert docs == [SFilmPersonDetail(**doc) for doc in await read_json_data('person/' + test['body'])]
        logger.info(f"{test['name']} passed")
    logger.info('end test films get_all person')

@pytest.mark.asyncio
async def test_films_bad_parameter_Person_class(person: Person):
    logger.info('start test bad parameter')
    assert await person.get_person_by_id('not_found') is None
    try:
        await person.get_person_by_id()
    except Exception as err:
        assert err.__class__ == TypeError, err
    try:
        await person.search_person()
    except Exception as err:
        assert err.__class__ == TypeError, err
    try:
        await person.get_all_person()
    except Exception as err:
        assert err.__class__ == TypeError, err
    logger.info('end test bad parameter')


import asyncio
import math
import re

from StoreAPI.utils import append_product_to_catalog
from services.db.models import ProductModel
from services.requests.request_utils import get_store_products_data_by_specified_store_products
from services.requests.requests_to_PrintfulAPI import get_specified_amount_of_store_products
from services.requests.request_utils import actualization_and_supplementing_catalog_products_data as \
                                            actualization_and_supplementing_data_of_specified_products
from services.schemas.classes.Product import Product

async def add_store_id_on_hundreds_of_other_products(total_products: int, products_from_response: list):
    count_next_hundreds = math.ceil(total_products / 100) - 1
    queue = asyncio.Queue()
    tasks = []
    for hundreds_number in range(1, count_next_hundreds + 1):
        task = asyncio.create_task(get_specified_amount_of_store_products(offset=hundreds_number*100, amount=100))
        tasks.append(task)
    await queue.join()
    await asyncio.gather(*tasks, return_exceptions=True)
    for task in tasks:
        products_from_response += task.result()['result']

async def add_products_to_bd_using_data_product_from_store(store_products: list[dict]):
    store_products = await get_store_products_data_by_specified_store_products(store_products)
    products = []
    global_products_id = set()
    for product in store_products:
        product = Product(product)
        products.append(product)
        global_products_id.add(product.global_id)
    await actualization_and_supplementing_data_of_specified_products(global_products_id, products)
    for product in products:
        saved_product = await ProductModel(**product.prepare_for_db_entry().to_json()).save()
        await saved_product.expire(60)

def get_products_outside_db(funk):
    async def wrapper(products_or_products_ids: list[dict | list[int]]):
        products_who_not_in_bd = []
        keys = await ProductModel.db().keys()
        for product_or_product_id in products_or_products_ids:
            funk(keys, products_who_not_in_bd, product_or_product_id)

        return products_who_not_in_bd
    return wrapper

def add_product_missing_in_keys_of_database_to_specified_list(keys: list, products_who_not_in_bd: list, product: dict):
    may_be_key = re.compile(ProductModel.get_united_prefix() + rf'\d*-{product["id"]}')
    if not list(filter(may_be_key.match, keys)):
        products_who_not_in_bd.append(product)

@get_products_outside_db
def get_products_outside_db_using_ids_from_db(keys, products_who_not_in_bd, product_id):
    add_product_missing_in_keys_of_database_to_specified_list(keys, products_who_not_in_bd, product={'id': product_id})

@get_products_outside_db
def get_products_outside_db_using_store_products(keys, products_who_not_in_bd, store_product):
    if not store_product['is_ignored']:
        add_product_missing_in_keys_of_database_to_specified_list(keys, products_who_not_in_bd, store_product)

async def get_filtered_catalog(request, **queries):
    filtered_catalog = []
    for find_product in await ProductModel.find_by_data_of_filters(**queries).all():
        await append_product_to_catalog(filtered_catalog, request, find_product)
    return filtered_catalog

async def get_products_outside_db_using_requests():
    response_by_one_hundred_products = await get_specified_amount_of_store_products(offset=0, amount=100)
    products_from_response = response_by_one_hundred_products['result']
    total_products = response_by_one_hundred_products['paging']['total']
    if total_products > 100:
        await add_store_id_on_hundreds_of_other_products(total_products, products_from_response)

    return await get_products_outside_db_using_store_products(products_from_response)

def get_only_completed_fields(entry_data: dict) -> dict:
    entry_data.pop('request')
    [entry_data.pop(key) if value is None else None for key, value in entry_data.copy().items()]
    return entry_data


import sys
from warna import prCyan, prYellow, prLightGray, prRed, prGreen
from helpers import cls, s, cetakgaris, garis

# Cek Paket PYTube
try:
    from pytube import Stream, YouTube
except ImportError:
    print(s(prRed("Pytube tidak ditemukan")))
    print(s("Install terlebih dahulu untuk melanjutkan"))
    print(s(prGreen("pkg install pytube")))
    sys.exit(1)

from ffmpeg import RewriteFunction
from urllib.parse import urlparse, parse_qs

from helpers import ceksize, pilihAngka
from logs import Logs
import time
from banner import Banner
import sys
from cli import on_progress

class DownloadYT(RewriteFunction):
    def __init__(self, isDebug=False):
        """ Constructor
        :param boolean debug:
            untuk mengecek dalam mode pengembang atau tidak
            jika true, skip semua banner / animasi
        """

        self._isDebug: bool = isDebug
        self._link = None
        self._isPlaylist: bool = False
        self._YT = YouTube
        self._savePath = 'D:/b'

    @property
    def isDebug(self):
        """Get Mode"""
        return self._isDebug

    @property
    def link(self):
        """Link Video yang akan didownload"""
        return self._link

    """
    @link.setter
    def link(self, link):
        self._link = link;
    """

    @link.setter
    def YT(self, YT):
        self._YT = YT

    def tanyaLink(self):
        """ Cek apakah link valid atau tidak """
        try:
            if self._isDebug:
                self._link = 'https://www.youtube.com/watch?v=8uy7G2JXVSA'
                return

            tanya = input((s("Paste Link Youtube: ")))
            link = urlparse(tanya)

            if not link.scheme :
                raise ValueError("Link Tidak Valid")
            if not link.netloc in ['www.youtube.com', 'youtube.com','m.youtube.com', 'youtu.be','m.youtu.be']:
                raise ValueError("Bukan Link Youtube")

            self._link = tanya

        except ValueError as pesan:
            print(s(pesan))
            self.tanyaLink()

    @isDebug.setter
    def isDebug(self, isDebug):
        self._isDebug = isDebug

    def infoVideo(self):
        """Menampilkan Informasi Video"""

        print(s("{0:8}: {1}".format('Judul', prCyan(self._YT.title[0:30])) ))
        print(s("{0:8}: {1}".format('Chanel', self._YT.author)))
        print("")
        print(s("{0:8}: {1}".format('Views', self._YT.views)))
        print(s("{0:8}: {1}".format('Rating', self._YT.rating)))
        print(s("{0:8}: {1}".format('Publish', self._YT.publish_date)))
        print(s("{0:8}: {1:.1f} Menit".format('Durasi', self._YT.length / 60)))

    def resolusi(self):
        yts = self._YT.streams
        yt_video = yts.filter(type='video')
        yt_audio = yts.get_audio_only()
        jumlahpilihan = len(yt_video) + 1

        print(s("{0:3} {1:15} {2}".format('No', 'Resolusi', 'Size')))
        print(garis())

        for i,data in enumerate(yt_video, start=1):
            resolusi = data.resolution
            print(s("{0:3} {1:15} {2}".format(i, resolusi, ceksize(data.filesize + yt_audio.filesize) )) )

        print(s("{0:3} {1:15} {2}".format(len(yt_video)+1,"Audio MP3", ceksize(yt_audio.filesize))))
        print(garis())
        strpilih = (s(prYellow("Pilih Nomor 1 sampai {0} : ".format(str(jumlahpilihan) ))))

        pilih = pilihAngka(strpilih)

        if ((pilih) > jumlahpilihan):
            print(s("Pilihan Tidak Valid"))
            self.resolusi()

        self._Audio = yt_audio

        if ( (pilih) == jumlahpilihan):
            return "audio"

        self._Video = yt_video[pilih-1]
        return "video"

    def run(self):
        """ Fungsi awal saat program dijalankan """

        if(self._isDebug):
            print(s("Mode Debug Active"))

        if self._link is None:
            self.tanyaLink()

        cls()
        B = Banner()
        B.cetakbanner()
        cetakgaris("Pilih Resolusi Video")

        self._YT = YouTube(self._link, on_progress_callback=on_progress)
        self._YT.check_availability

        self.infoVideo()
        print("")
        time.sleep(3)
        pilihan = self.resolusi()

        cek = Logs()
        if (not cek.cek()):
            print(s("%s" % prRed("Lokasi penyimpanan belum diset ") ))
            cek.gantiPath()

        self._savePath = cek._path

        cls()
        B.cetakbanner()
        cetakgaris("Please Wait.. Downloading")

        try:
            if pilihan == 'video':
                super(DownloadYT, self).Downloaderffmpeg(self._Audio, self._Video, self._savePath)
            elif (pilihan == 'audio'):
                super(DownloadYT, self).DownloadMP3(self._Audio, self._savePath)
            else:
                raise ValueError("Error")
        except Exception as Pesan:
            print(s("Terjadi kesalahan! %s" % Pesan))
            return

        print("")
        sys.exit(s(prCyan("Terima kasih! ;) ")))


import dash
import os
import requests
from flask import request
import flask.cli
from threading import Thread
from retrying import retry
import io
import re
import sys
import inspect
import warnings

from IPython import get_ipython
from IPython.display import IFrame, display
from IPython.core.ultratb import FormattedTB
from ansi2html import Ansi2HTMLConverter
import uuid

from werkzeug.debug.tbtools import get_current_traceback

from .comms import _dash_comm, _jupyter_config, _request_jupyter_config

class JupyterDash(dash.Dash):
    """A Dash subclass for developing Dash apps interactively in Jupyter.

    :param server_url:  The base URL that the app will be served at, from the
        perspective of the client. If not specified, will default to the host argument
        passed to the ``run_server`` method.

    See parent docstring for additional parameters
    """
    default_mode = 'external'
    default_requests_pathname_prefix = os.environ.get(
        "DASH_REQUESTS_PATHNAME_PREFIX", None
    )
    default_server_url = None
    _in_ipython = get_ipython() is not None
    _in_colab = "google.colab" in sys.modules
    _token = str(uuid.uuid4())

    @classmethod
    def infer_jupyter_proxy_config(cls):

        if not JupyterDash._in_ipython or JupyterDash._in_colab:
            # No op when not running in a Jupyter context or when in Colab
            return
        else:
            # Assume classic notebook or JupyterLab
            _request_jupyter_config()

    def __init__(self, name=None, server_url=None, **kwargs):
        """"""
        # Strip unsupported properties and warn
        if JupyterDash._in_colab:
            unsupported_colab_props = [
                'requests_pathname_prefix',
                'routes_pathname_prefix',
                'url_base_pathname'
            ]
            for prop in unsupported_colab_props:
                if prop in kwargs:
                    kwargs.pop(prop)
                    warnings.warn(
                        "The {prop} argument is ignored when running in Colab".format(
                            prop=prop
                        )
                    )

        # Call superclass constructor
        super(JupyterDash, self).__init__(name=name, **kwargs)

        if not JupyterDash._in_ipython:
            # Nothing else to do when not running in a Jupyter context
            return

        # See if jupyter_server_proxy is installed
        try:
            import jupyter_server_proxy
            self._server_proxy = True
        except Exception:
            self._server_proxy = False

        self._traceback = None

        if ('base_subpath' in _jupyter_config and self._server_proxy and
                JupyterDash.default_requests_pathname_prefix is None):
            JupyterDash.default_requests_pathname_prefix = (
                _jupyter_config['base_subpath'].rstrip('/') + '/proxy/{port}/'
            )

        if ('server_url' in _jupyter_config and self._server_proxy and
                JupyterDash.default_server_url is None):
            JupyterDash.default_server_url = _jupyter_config['server_url']

        self._input_pathname_prefix = kwargs.get('requests_pathname_prefix', None)

        # Infer server_url
        if server_url is None:
            domain_base = os.environ.get('DASH_DOMAIN_BASE', None)
            if domain_base:
                # Dash Enterprise sets DASH_DOMAIN_BASE environment variable
                server_url = 'https://' + domain_base
        elif JupyterDash._in_colab:
            warnings.warn("The server_url argument is ignored when running in Colab")
            server_url = None

        self.server_url = server_url

        # Register route to shut down server
        @self.server.route('/_shutdown_' + JupyterDash._token, methods=['GET'])
        def shutdown():
            func = request.environ.get('werkzeug.server.shutdown')
            if func is None:
                raise RuntimeError('Not running with the Werkzeug Server')
            func()
            return 'Server shutting down...'

        # Register route that we can use to poll to see when server is running
        @self.server.route('/_alive_' + JupyterDash._token, methods=['GET'])
        def alive():
            return 'Alive'

        self.server.logger.disabled = True

    def run_server(
            self,
            mode=None, width="100%", height=650, inline_exceptions=None,
            **kwargs
    ):
        """
        Serve the app using flask in a background thread. You should not run this on a
        production server, use gunicorn/waitress instead.

        :param mode: Display mode. One of:
            ``"external"``: The URL of the app will be displayed in the notebook
                output cell. Clicking this URL will open the app in the default
                web browser.
            ``"inline"``: The app will be displayed inline in the notebook output cell
                in an iframe.
            ``"jupyterlab"``: The app will be displayed in a dedicate tab in the
                JupyterLab interface. Requires JupyterLab and the `jupyterlab-dash`
                extension.
        :param width: Width of app when displayed using mode="inline"
        :param height: Height of app when displayed using mode="inline"
        :param inline_exceptions: If True, callback exceptions are displayed inline
            in the the notebook output cell. Defaults to True if mode=="inline",
            False otherwise.
        :param kwargs: Additional keyword arguments to pass to the superclass
            ``Dash.run_server`` method.
        """
        # Get superclass run_server method
        super_run_server = super(JupyterDash, self).run_server

        if not JupyterDash._in_ipython:
            # If not in IPython context, call run run_server synchronously
            super_run_server(**kwargs)
            return

        # Get host and port
        host = kwargs.get("host", os.getenv("HOST", "127.0.0.1"))
        port = kwargs.get("port", os.getenv("PORT", "8050"))

        kwargs['host'] = host
        kwargs['port'] = port

        # Validate / infer display mode
        if JupyterDash._in_colab:
            valid_display_values = ["inline", "external"]
        else:
            valid_display_values = ["jupyterlab", "inline", "external"]

        if mode is None:
            mode = JupyterDash.default_mode
        elif not isinstance(mode, str):
            raise ValueError(
                "The mode argument must be a string\n"
                "    Received value of type {typ}: {val}".format(
                    typ=type(mode), val=repr(mode)
                )
            )
        else:
            mode = mode.lower()
            if mode not in valid_display_values:
                raise ValueError(
                    "Invalid display argument {display}\n"
                    "    Valid arguments: {valid_display_values}".format(
                        display=repr(mode), valid_display_values=valid_display_values
                    )
                )

        # Infer inline_exceptions and ui
        if inline_exceptions is None:
            inline_exceptions = mode == "inline"

        # Terminate any existing server using this port
        self._terminate_server_for_port(host, port)

        # Configure pathname prefix
        requests_pathname_prefix = self.config.get('requests_pathname_prefix', None)
        if self._input_pathname_prefix is None:
            requests_pathname_prefix = self.default_requests_pathname_prefix

        if requests_pathname_prefix is not None:
            requests_pathname_prefix = requests_pathname_prefix.format(port=port)
        else:
            requests_pathname_prefix = '/'
        self.config.update({'requests_pathname_prefix': requests_pathname_prefix})

        # Compute server_url url
        if self.server_url is None:
            if JupyterDash.default_server_url:
                server_url = JupyterDash.default_server_url.rstrip('/')
            else:
                server_url = 'http://{host}:{port}'.format(host=host, port=port)
        else:
            server_url = self.server_url.rstrip('/')

        dashboard_url = "{server_url}{requests_pathname_prefix}".format(
            server_url=server_url, requests_pathname_prefix=requests_pathname_prefix
        )

        # Default the global "debug" flag to True
        debug = kwargs.get('debug', True)

        # Disable debug flag when calling superclass because it doesn't work
        # in notebook
        kwargs['debug'] = False

        # Enable supported dev tools
        if debug:
            for k in [
                'dev_tools_silence_routes_logging',
                'dev_tools_props_check',
                'dev_tools_serve_dev_bundles',
                'dev_tools_prune_errors'
            ]:
                if k not in kwargs:
                    kwargs[k] = True

            # Enable dev tools by default unless app is displayed inline
            if 'dev_tools_ui' not in kwargs:
                kwargs['dev_tools_ui'] = mode != "inline"

            if 'dev_tools_hot_reload' not in kwargs:
                # Enable hot-reload by default in "external" mode. Enabling in inline or
                # in JupyterLab extension seems to cause Jupyter problems sometimes when
                # there is no active kernel.
                kwargs['dev_tools_hot_reload'] = mode == "external"

        # suppress warning banner printed to standard out
        flask.cli.show_server_banner = lambda *args, **kwargs: None

        # Set up custom callback exception handling
        self._config_callback_exception_handling(
            dev_tools_prune_errors=kwargs.get('dev_tools_prune_errors', True),
            inline_exceptions=inline_exceptions,
        )

        @retry(
            stop_max_attempt_number=15,
            wait_exponential_multiplier=100,
            wait_exponential_max=1000
        )
        def run():
            super_run_server(**kwargs)

        thread = Thread(target=run)
        thread.setDaemon(True)
        thread.start()

        # Wait for server to start up
        alive_url = "http://{host}:{port}/_alive_{token}".format(
            host=host, port=port, token=JupyterDash._token
        )

        # Wait for app to respond to _alive endpoint
        @retry(
            stop_max_attempt_number=15,
            wait_exponential_multiplier=10,
            wait_exponential_max=1000
        )
        def wait_for_app():
            res = requests.get(alive_url).content.decode()
            if res != "Alive":
                url = "http://{host}:{port}".format(
                    host=host, port=port, token=JupyterDash._token
                )
                raise OSError(
                    "Address '{url}' already in use.\n"
                    "    Try passing a different port to run_server.".format(
                        url=url
                    )
                )

        wait_for_app()

        if JupyterDash._in_colab:
            self._display_in_colab(dashboard_url, port, mode, width, height)
        else:
            self._display_in_jupyter(dashboard_url, port, mode, width, height)

    def _display_in_colab(self, dashboard_url, port, mode, width, height):
        from google.colab import output
        if mode == 'inline':
            output.serve_kernel_port_as_iframe(port, width=width, height=height)
        elif mode == 'external':
            # Display a hyperlink that can be clicked to open Dashboard
            print("AutoPlotter is running on:")
            output.serve_kernel_port_as_window(port, anchor_text=dashboard_url)

    def _display_in_jupyter(self, dashboard_url, port, mode, width, height):
        if mode == 'inline':
            display(IFrame(dashboard_url, width, height))
        elif mode == 'external':
            # Display a hyperlink that can be clicked to open Dashboard
            print("AutoPlotter is running on {dashboard_url}".format(
                dashboard_url=dashboard_url
            ))
        elif mode == 'jupyterlab':
            # Update front-end extension
            _dash_comm.send({
                'type': 'show',
                'port': port,
                'url': dashboard_url,
            })

    def _config_callback_exception_handling(
            self, dev_tools_prune_errors, inline_exceptions
    ):

        @self.server.errorhandler(Exception)
        def _wrap_errors(_):
            """Install traceback handling for callbacks"""
            self._traceback = sys.exc_info()[2]

            # Compute number of stack frames to skip to get down to callback
            tb_werkzeug = get_current_traceback()
            skip = 0
            if dev_tools_prune_errors:
                for i, line in enumerate(tb_werkzeug.plaintext.splitlines()):
                    if "%% callback invoked %%" in line:
                        skip = int((i + 1) / 2)
                        break

            # Customized formatargvalues function so we can place function parameters
            # on separate lines
            original_formatargvalues = inspect.formatargvalues
            inspect.formatargvalues = _custom_formatargvalues
            try:
                # Use IPython traceback formatting to build colored ANSI traceback
                # string
                ostream = io.StringIO()
                ipytb = FormattedTB(
                    tb_offset=skip,
                    mode="Verbose",
                    color_scheme="Linux",
                    include_vars=True,
                    ostream=ostream
                )
                ipytb()
            finally:
                # Restore formatargvalues
                inspect.formatargvalues = original_formatargvalues

            # Print colored ANSI representation if requested
            ansi_stacktrace = ostream.getvalue()

            if inline_exceptions:
                print(ansi_stacktrace)

            # Use ansi2html to convert the colored ANSI string to HTML
            conv = Ansi2HTMLConverter(scheme="ansi2html", dark_bg=False)
            html_str = conv.convert(ansi_stacktrace)

            # Set width to fit 75-character wide stack trace and font to a size the
            # won't require a horizontal scroll bar
            html_str = html_str.replace(
                '<html>',
                '<html style="width: 75ch; font-size: 0.86em">'
            )

            # Remove explicit background color so Dash dev-tools can set background
            # color
            html_str = re.sub("background-color:[^;]+;", "", html_str)

            return html_str, 500

    @classmethod
    def _terminate_server_for_port(cls, host, port):
        shutdown_url = "http://{host}:{port}/_shutdown_{token}".format(
            host=host, port=port, token=JupyterDash._token
        )
        try:
            response = requests.get(shutdown_url)
        except Exception as e:
            pass

def _custom_formatargvalues(
        args, varargs, varkw, locals,
        formatarg=str,
        formatvarargs=lambda name: '*' + name,
        formatvarkw=lambda name: '**' + name,
        formatvalue=lambda value: '=' + repr(value)):

    """Copied from inspect.formatargvalues, modified to place function
    arguments on separate lines"""
    def convert(name, locals=locals,
                formatarg=formatarg, formatvalue=formatvalue):
        return formatarg(name) + formatvalue(locals[name])
    specs = []
    for i in range(len(args)):
        specs.append(convert(args[i]))
    if varargs:
        specs.append(formatvarargs(varargs) + formatvalue(locals[varargs]))
    if varkw:
        specs.append(formatvarkw(varkw) + formatvalue(locals[varkw]))

    result = '(' + ', '.join(specs) + ')'

    if len(result) < 40:
        return result
    else:
        # Put each arg on a separate line
        return '(\n    ' + ',\n    '.join(specs) + '\n)'

{'jupyter_server_proxy': 'jupyter_server_proxy', 'output': 'google.colab.output'}._token


#Xu Song
#linechart

import matplotlib.pyplot as plt

#read firstline,convert to list
linedata=open('linedata.txt','r')
line=linedata.readline()
line=line.rstrip('\n')
line=line.split(',')
rop=[]
for number in line:
    rop.append(int(number))

#read second line,conver to list
line=linedata.readline()
line=line.rstrip('\n')
line=line.split(',')
dem=[]
for number in line:
    dem.append(int(number))

linedata.close()

#make line graph
ropValue=rop
demValue=dem
plt.plot(range(1,len(ropValue)+1),ropValue,'o-')
plt.plot(range(1,len(demValue)+1),demValue,'o-')
plt.legend(['Rop','Dem'],loc=2)
plt.title('2018 California Election Results in First Four Districts')
plt.xlabel('Districts')
plt.ylabel('Number of votes')
ax=plt.axes()
ax.grid()
ax.set_xticks([1,2,3,4])
plt.show()


# https://www.acmicpc.net/problem/7569

# 문제요약
# 토마토 문제인데 층이 쌓이고 상하좌우 + 위 아래까지 영향을 줌
# 최소 일수 구하기

# 입력
# 첫 줄에는 상자의 크기를 나타내는 두 정수 M,N과 쌓아올려지는 상자의 수를 나타내는 H가 주어진다. M은 상자의 가로 칸의 수, N은 상자의 세로 칸의 수를 나타낸다. 단, 2 ≤ M ≤ 100, 2 ≤ N ≤ 100, 1 ≤ H ≤ 100 이다. 둘째 줄부터는 가장 밑의 상자부터 가장 위의 상자까지에 저장된 토마토들의 정보가 주어진다. 즉, 둘째 줄부터 N개의 줄에는 하나의 상자에 담긴 토마토의 정보가 주어진다. 각 줄에는 상자 가로줄에 들어있는 토마토들의 상태가 M개의 정수로 주어진다. 정수 1은 익은 토마토, 정수 0 은 익지 않은 토마토, 정수 -1은 토마토가 들어있지 않은 칸을 나타낸다. 이러한 N개의 줄이 H번 반복하여 주어진다.

# 토마토가 하나 이상 있는 경우만 입력으로 주어진다.

# 출력
# 여러분은 토마토가 모두 익을 때까지 최소 며칠이 걸리는지를 계산해서 출력해야 한다. 만약, 저장될 때부터 모든 토마토가 익어있는 상태이면 0을 출력해야 하고, 토마토가 모두 익지는 못하는 상황이면 -1을 출력해야 한다.

# 예제 입력 1
# 5 3 1
# 0 -1 0 0 0
# -1 -1 0 1 1
# 0 0 0 1 1
# 예제 출력 1
# -1
# 예제 입력 2
# 5 3 2
# 0 0 0 0 0
# 0 0 0 0 0
# 0 0 0 0 0
# 0 0 0 0 0
# 0 0 1 0 0
# 0 0 0 0 0
# 예제 출력 2
# 4
# 예제 입력 3
# 4 3 2
# 1 1 1 1
# 1 1 1 1
# 1 1 1 1
# 1 1 1 1
# -1 -1 -1 -1
# 1 1 1 -1
# 예제 출력 3
# 0

# 문제풀이
# 토마토 문제의 상위버전이라고 생각하면 됨
# 3차원으로 조금더 어렵지만 단계만 추가할 뿐임

import sys
from collections import deque

M, N, H = map(int, sys.stdin.readline().split())
tmt= [[] for _ in range(H)]
q = deque()
dx = [-1, 1, 0, 0, 0, 0]
dy = [0, 0, -1, 1, 0, 0]
dz = [0, 0, 0, 0, 1, -1] # H까지 추가
cnt = 0

for i in range(H) :
    for j in range(N) :
        tmt[i].append(list(map(int, input().split())))
        for k in range(M) :
            if tmt[i][j][k] == 1 :
                q.append((i,j,k)) # 숫자 받으면서 만약 1이면 q에 추가하기!

def bfs() :
    while q :
        z1,x1,y1=q.popleft()
        for i in range(6) :
            z = z1 + dz[i]
            x = x1 + dx[i]
            y = y1 + dy[i]
            if 0<= z < H and 0 <= x < N and 0<= y < M and tmt[z][x][y] == 0 :
                q.append((z,x,y))
                tmt[z][x][y] = tmt[z1][x1][y1] +1 # 해당 과일에서 숫자 +1시킴

bfs()
for i in range(H) :
    for j in range(N):
        for k in range(M) :
            if tmt[i][j][k] == 0:
                print(-1) # 0나오면 -1출력하고 끝
                exit(0)
            cnt = max(cnt,tmt[i][j][k]) # 아니면 높은숫자 찾기

print(cnt-1) # 처음과일이 1로 시작하기 때문에 -1


# Run this app with `python app.py` and
# visit http://127.0.0.1:8050/ in your web browser.

import dash
from dash import dcc, html
from dash.dependencies import Input, Output, State, ClientsideFunction
from dash.exceptions import PreventUpdate
import webbrowser
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots

from sklearn.manifold import TSNE
from song.song import SONG

# ---

app = dash.Dash(__name__)

# ---

import numpy as np
import pandas as pd
import spotipy
from spotipy.oauth2 import SpotifyOAuth
import hashlib

# ---

try:
	data = pd.read_pickle('data-spotify.pkl')
except FileNotFoundError:
	scope = "user-library-read,playlist-read-private,user-read-private"
	sp = spotipy.Spotify(auth_manager=SpotifyOAuth(scope=scope))

	tracks = {}
	def add_track(t, a=None, track_liked=False):
		if a == None:
			a = t
		if t['preview_url'] != None:
			tracks[t['uri']] = {
				'name': t['name'],
				'album': a['album']['name'],
				'artist': a['album']['artists'][0]['name'],
				# 'genre': genre,
				'uri': t['uri'],
				'preview_url': t['preview_url'],
				'href': t['external_urls']['spotify'],
				'release_year': int(a['album']['release_date'][0:4]),
				'track_liked': track_liked
			}

	albums = sp.current_user_saved_albums()
	while albums:
		for i, a in enumerate(albums['items']):
			print("%4d %s %s" % (i + 1 + albums['offset'], a['album']['uri'], a['album']['name']))
			# genre = next(iter(a['album']['artists'][0]['genres']), '') if 'genres' in a['album']['artists'][0] else ''
			# print(genre)
			for t in a['album']['tracks']['items']:
				add_track(t, a)
		if albums['next']:
			# albums = None
			albums = sp.next(albums)
		else:
			albums = None

	playlists = sp.current_user_playlists()
	while playlists:
		for i, p in enumerate(playlists['items']):
			print("%4d %s" % (i + 1 + playlists['offset'], p['name']))
			if 'items' in p['tracks']:
				for t in p['tracks']['items']:
					add_track(t)
		if playlists['next']:
			# playlists = None
			print('.', end='')
			playlists = sp.next(playlists)
		else:
			playlists = None

	likes = sp.current_user_saved_tracks()
	while likes:
		for i, item in enumerate(likes['items']):
			add_track(item['track'], track_liked=True)
		if likes['next']:
			# likes = None
			print('.', end='')
			likes = sp.next(likes)
		else:
			likes = None

	chunk_size = 20
	uris = list(tracks.keys())
	print(len(uris))

	chunked_uris = [uris[i:i + chunk_size] for i in range(0, len(uris), chunk_size)]

	chunked_features = [sp.audio_features(uris) for uris in chunked_uris]
	for features in chunked_features:
		for f in features:
			tracks[f['uri']].update({
				'instrumentalness': f['instrumentalness'],
				'danceability': f['danceability'],
				'energy': f['energy'],
				'valence': f['valence'],
				'speechiness': f['speechiness'],
				'acousticness': f['acousticness'],
				'tempo': f['tempo'],
				'normalized_tempo': (f['tempo'] - 30) / 170
			  })

	tracks_data = list(tracks.values())
	data = pd.DataFrame(tracks_data)
	data.to_pickle('data-spotify.pkl')

#- artist name as a number between 0 and 1 ----

data['artist-0-1'] = [int(hashlib.sha1(a.encode('utf-8')).hexdigest()[:6], 16)/16777216 for a in data['artist']]
data['album-0-1'] = [int(hashlib.sha1(a.encode('utf-8')).hexdigest()[:6], 16)/16777216 for a in data['album']]
# print(data[['artist', 'artist-0-1']])

columns = ['instrumentalness','danceability','energy', 'valence', 'speechiness', 'acousticness', 'normalized_tempo', 'artist-0-1', 'album-0-1']

#- t-SNE ----------------------------

# tsne = TSNE(n_components=2, verbose=1, perplexity=50, n_iter_without_progress=50, n_jobs=-1, square_distances=True)
# tsneable_data = data[columns]
# tsne_results = tsne.fit_transform(tsneable_data)
# data['x'] = tsne_results[:,0]
# data['y'] = tsne_results[:,1]
# # data['z'] = tsne_results[:,2]

# data.to_pickle('data-tsne.pkl')

#------------------------------------

try:
	data = pd.read_pickle('data-song.pkl')
except FileNotFoundError:
	d = data[columns].to_numpy()
	model = SONG(min_dist=0.5, spread=0.9)
	model.fit(d)
	Y = model.transform(d)
	data['x'] = Y[:,0]
	data['y'] = Y[:,1]
	data.to_pickle('data-song.pkl')

data['summary'] = data[['name', 'album', 'artist']].agg(', '.join, axis=1)

top = data[['artist', 'x', 'y']] \
	.groupby(['artist'], as_index=False)[['x','y']] \
	.agg(['count','median','std']) \
	.sort_values([('x','count'),('x','std'),('y','std')], ascending=[False,True,True])[:120]

top['text_size'] = 4 * np.log(top[("x", "count")])

#------------------------------------

clicked_tracks = []

app.layout = html.Div(children=[
	html.Div(id='div-search', children=[
        "Search: ",
        dcc.Input(id='search', value='', type='text', autoFocus=True)
    ]),
	dcc.Graph(
		id='my-graph',
		clear_on_unhover=True,
	),

	html.Video(id='preview-player', controls=True, autoPlay=False, height=30, width=400, children=[
		html.Source(id='preview-source', src='', type="audio/mpeg"),
	]),

	html.Ul(id='click-list', children=[]),

	html.P(id='no-output-1'),
	html.P(id='no-output-2'),
])

@app.callback(
	Output('my-graph', 'figure'),
	Input('search', 'value'))
def filter_graph(query):
	def matching(row):
		if len(query) == 0:
			return True
		else:
			return max(
				query.lower() in row['artist'].lower(),
				query.lower() in row['album'].lower(),
				query.lower() in row['name'].lower())

	data['matching'] = data.apply(matching, axis=1)
	data['opacity'] = [1.0 if m else 0.3 for m in data['matching']]
	data['size'] = [9 if m else 4 for m in data['matching']]
	matches = data.loc[data['matching'] == True]
	nonmatches = data.loc[data['matching'] == False]

	fig = make_subplots()

	fig.add_trace(go.Scatter(
		x=data["x"], y=data["y"],
		mode='markers',
		marker=dict(
			size=data["size"],
			color=data["danceability"],
			opacity=data['opacity'],
			line_width=0.0,
			),
		hoverinfo='skip',
		showlegend=False,
	))

	# fig.add_trace(go.Scatter(
	# 	x=nonmatches["x"], y=nonmatches["y"],
	# 	mode='markers',
	# 	marker=dict(
	# 		size=nonmatches["size"],
	# 		color=nonmatches["danceability"],
	# 		opacity=nonmatches['opacity'],
	# 		line_width=0.0,
	# 		),
	# 	hoverinfo='skip',
	# 	showlegend=False,
	# ))

	# fig.add_trace(go.Scatter(
	# 	x=matches["x"], y=matches["y"],
	# 	mode='markers',
	# 	marker=dict(
	# 		size=matches["size"],
	# 		color=matches["danceability"],
	# 		opacity=matches['opacity'],
	# 		),
	# 	showlegend=False,
	# ))

	fig.add_trace(go.Scatter(
		x=matches["x"], y=matches["y"],
		mode='markers',
		marker=dict(
			size=matches["size"],
			line_width=0.5,
			line_color='rgba(255, 255, 255, 1.0)',
			color='rgba(255, 255, 255, 0.0)',
			),
		customdata=matches[["href", "name", "preview_url", "uri", "album", "artist", "release_year",
			"instrumentalness", "energy", "valence", "tempo", "matching"]],
		hovertemplate='%{customdata[5]}
%{customdata[1]}, %{customdata[4]}
(%{customdata[6]})

instrumentalness: %{customdata[7]}
energy: %{customdata[8]}
valence: %{customdata[9]}
tempo: %{customdata[10]}<extra></extra>',
		showlegend=False,
	))

	if len(query) == 0:
		fig.add_trace(go.Scatter(
			x=top[("x", "median")],
			y=top[("y", "median")],
			text=top.index,
			textfont_size=top["text_size"],
			mode='text',
			showlegend=False,
		))
	else:
		fig.add_trace(go.Scatter(
			x=matches["x"],
			y=matches["y"],
			text=matches['summary'],
			textfont_size=8,
			mode='text',
			textposition='bottom right',
			showlegend=False,
		))

	fig.update_layout(height=1200, uniformtext_minsize=12, uniformtext_mode='hide', clickmode='event')
	return fig

@app.callback(
	Output('click-list', 'children'),
	[Input('my-graph', 'clickData')],
	[State('click-list','children')]
)
def open_url(clickData, state):
	if clickData == None:
		print("NONE")
		raise PreventUpdate
	print(clickData)
	t = clickData['points'][0]['customdata']
	clicked_tracks.append(t)
	webbrowser.open_new_tab(t[0])
	return [html.Li(children=[
			html.A(href=t[0], children=t[1]), # track link
			html.Span(children=t[4]), # album
			html.Span(children=t[5]), # artist
			html.Span(children=t[6]), # year
			]) for t in reversed(clicked_tracks)]

@app.callback(
	Output('search', 'value'),
	[Input('my-graph', 'clickData')])
def clear_search(clickData):
	print("CLICK")
	if clickData == None:
		return ""
	else:
		raise PreventUpdate

app.clientside_callback(
	ClientsideFunction(
		namespace='clientside',
		function_name='playPreview'
	),
	Output('no-output-1', 'children'),
	Input('my-graph', 'hoverData'),
)

app.clientside_callback(
	ClientsideFunction(
		namespace='clientside',
		function_name='pausePreview'
	),
	Output('no-output-2', 'children'),
	Input('my-graph', 'clickData'),
)

if __name__ == '__main__':
	app.run_server(debug=True, use_reloader=False)


from __future__ import absolute_import, division

import sys

import pytest

def test_get_command_line_help(capsys):
  import coldwallet.command_line
  sys.argv = [ 'coldwallet', '--help' ]

  with pytest.raises(SystemExit) as exc_info:
    coldwallet.command_line.main()

  assert exc_info.value.code == 0
  out, err = capsys.readouterr()
  assert err == ''
  assert 'optional' in out

def test_reject_invalid_parameters(capsys):
  import coldwallet.command_line
  sys.argv = [ 'coldwallet', '--bork' ]

  with pytest.raises(SystemExit) as exc_info:
    coldwallet.command_line.main()

  assert exc_info.value.code == 2
  out, err = capsys.readouterr()
  assert 'unrecognized' in err and 'bork' in err

def test_show_version(capsys):
  import coldwallet.command_line
  sys.argv = [ 'coldwallet', '--version' ]

  with pytest.raises(SystemExit) as exc_info:
    coldwallet.command_line.main()

  assert exc_info.value.code == 0
  out, err = capsys.readouterr()
  assert err == ''
  assert coldwallet.__version__ in out

def test_reject_too_few_codes(capsys):
  import coldwallet.command_line
  sys.argv = [ 'coldwallet', '--codes', '1' ]

  with pytest.raises(SystemExit) as exc_info:
    coldwallet.command_line.main()

  assert exc_info.value.code == 1
  out, err = capsys.readouterr()
  assert 'minimum number of code blocks' in err
  assert out == ''

def test_create_example_keys(capsys, tmpdir):
  import coldwallet.command_line
  sys.argv = [ 'coldwallet', '--disable-randomness', '--scrypt-N', '4', '--addresses', '2' ]

  with tmpdir.as_cwd():
    coldwallet.command_line.main()

  assert (tmpdir / 'coldwallet.py').check(file=1)

  out, err = capsys.readouterr()
  err = err.split('\n')
  assert len(err) == 4
  assert 'random number generation disabled' in err[1]

def test_ensure_that_output_files_are_never_overwritten(capsys, tmpdir):
  import coldwallet.command_line
  sys.argv = [ 'coldwallet', '--disable-randomness', '--scrypt-N', '4', '--addresses', '2' ]
  (tmpdir / 'coldwallet.py').ensure()

  with pytest.raises(SystemExit) as exc_info:
    with tmpdir.as_cwd():
      coldwallet.command_line.main()

  assert exc_info.value.code == 1
  out, err = capsys.readouterr()
  assert 'already exists' in err
  assert (tmpdir / 'coldwallet.py').size() == 0


import model_manager
import torch
import os
import datasets
import cnn_models.conv_forward_model as convForwModel
import cnn_models.help_fun as cnn_hf
import quantization
import pickle
import copy
import quantization.help_functions as qhf
import functools
import helpers.functions as mhf
from cnn_models.wide_resnet import Wide_ResNet

datasets.BASE_DATA_FOLDER = '...'
SAVED_MODELS_FOLDER = '...'
USE_CUDA = torch.cuda.is_available()

cuda_devices = os.environ['CUDA_VISIBLE_DEVICES'].split(',')
print('CUDA_VISIBLE_DEVICES: {} for a total of {}'.format(cuda_devices, len(cuda_devices)))
NUM_GPUS = len(cuda_devices)

try:
    os.mkdir(datasets.BASE_DATA_FOLDER)
except:pass
try:
    os.mkdir(SAVED_MODELS_FOLDER)
except:pass

cifar10Manager = model_manager.ModelManager('model_manager_cifar10_wideResNet.tst',
                                            'model_manager', create_new_model_manager=False)
cifar10modelsFolder = os.path.join(SAVED_MODELS_FOLDER, 'cifar10_wideResNet')

for x in cifar10Manager.list_models():
    if cifar10Manager.get_num_training_runs(x) >= 1:
        s = '{}; Last prediction acc: {}, Best prediction acc: {}'.format(x,
                                            cifar10Manager.load_metadata(x)[1]['predictionAccuracy'][-1],
                                            max(cifar10Manager.load_metadata(x)[1]['predictionAccuracy']))
        print(s)
try:
    os.mkdir(cifar10modelsFolder)
except:pass

epochsToTrainCIFAR10 = 200
epochsToTrainCIFAR10_diffquant = 20

batch_size = 100
if batch_size % NUM_GPUS != 0:
    raise ValueError('Batch size: {} must be a multiple of the number of gpus:{}'.format(batch_size, NUM_GPUS))

cifar10 = datasets.CIFAR10()
train_loader, test_loader = cifar10.getTrainLoader(batch_size), cifar10.getTestLoader(batch_size)

TRAIN_TEACHER_MODEL = False
TRAIN_SMALLER_MODEL = False
TRAIN_DISTILLED_MODEL = False
TRAIN_DISTILLED_QUANTIZED_MODEL = False
CHECK_PM_QUANTIZATION = True

# Teacher model
teacher_model_name = 'cifar10_teacher'
teacherModelPath = os.path.join(cifar10modelsFolder, teacher_model_name)
teacherOptions = {'widen_factor':20, 'depth':28, 'dropout_rate':0.3, 'num_classes':10}
teacherModel = Wide_ResNet(**teacherOptions)
if USE_CUDA: teacherModel = teacherModel.cuda()
if NUM_GPUS > 1:
    teacherModel = torch.nn.parallel.DataParallel(teacherModel)

if teacher_model_name not in cifar10Manager.saved_models:
    cifar10Manager.add_new_model(teacher_model_name, teacherModelPath,
            arguments_creator_function=teacherOptions)
if TRAIN_TEACHER_MODEL:
    cifar10Manager.train_model(teacherModel, model_name=teacher_model_name,
                               train_function=convForwModel.train_model,
                               arguments_train_function={'epochs_to_train': epochsToTrainCIFAR10,
                                                         'initial_learning_rate':0.1,
                                                         'print_every':50,
                                                         'learning_rate_style':'cifar100',
                                                         'weight_decayL2': 0.0005},
                               train_loader=train_loader, test_loader=test_loader)
try:
    teacherModel.load_state_dict(cifar10Manager.load_model_state_dict(teacher_model_name))
except:
    teacherModel.load_state_dict(mhf.convert_state_dict_from_data_parallel(
        cifar10Manager.load_model_state_dict(teacher_model_name)))

# smaller and distilled
smallerOptions = {'widen_factor':22, 'depth':16, 'dropout_rate':0.3, 'num_classes':10}

smaller_model_name = 'cifar10_smaller_model'
smaller_model_path = os.path.join(cifar10modelsFolder, smaller_model_name)
smallerModel = Wide_ResNet(**smallerOptions)
if USE_CUDA: smallerModel = smallerModel.cuda()
if NUM_GPUS > 1: smallerModel = torch.nn.parallel.DataParallel(smallerModel)
if not smaller_model_name in cifar10Manager.saved_models:
    cifar10Manager.add_new_model(smaller_model_name, smaller_model_path,
                                 arguments_creator_function=smallerOptions)
if TRAIN_SMALLER_MODEL:
    cifar10Manager.train_model(smallerModel, model_name=smaller_model_name,
                               train_function=convForwModel.train_model,
                               arguments_train_function={'epochs_to_train': epochsToTrainCIFAR10,
                                                         'print_every':50,
                                                         'initial_learning_rate':0.1,
                                                         'learning_rate_style':'cifar100',
                                                         'weight_decayL2':0.0005},
                               train_loader=train_loader, test_loader=test_loader)
#smallerModel.load_state_dict(cifar10Manager.load_model_state_dict(smaller_model_name))
del smallerModel

distilled_model_name = 'cifar10_distilled_model'
distilled_model_path = os.path.join(cifar10modelsFolder, distilled_model_name)
distilledModel = Wide_ResNet(**smallerOptions)
if USE_CUDA: distilledModel = distilledModel.cuda()
if NUM_GPUS > 1: distilledModel = torch.nn.parallel.DataParallel(distilledModel)
if not distilled_model_name in cifar10Manager.saved_models:
    cifar10Manager.add_new_model(distilled_model_name, distilled_model_path,
                                 arguments_creator_function=smallerOptions)
if TRAIN_DISTILLED_MODEL:
    cifar10Manager.train_model(distilledModel, model_name=distilled_model_name,
                               train_function=convForwModel.train_model,
                               arguments_train_function={'epochs_to_train': epochsToTrainCIFAR10,
                                                         'print_every':50,
                                                         'initial_learning_rate':0.1,
                                                         'learning_rate_style':'cifar100',
                                                         'weight_decayL2':0.0005,
                                                         'teacher_model': teacherModel,
                                                         'use_distillation_loss': True},
                               train_loader=train_loader, test_loader=test_loader)
#distilledModel.load_state_dict(cifar10Manager.load_model_state_dict(distilled_model_name))
del distilledModel

numBits = [2, 4]
for numBit in numBits:
    distilled_quantized_model_name = 'cifar10_distilled_quantized{}bits'.format(numBit)

    distilled_quantized_model_path = os.path.join(cifar10modelsFolder, distilled_quantized_model_name)
    distilled_quantized_model = Wide_ResNet(**smallerOptions)
    if USE_CUDA: distilled_quantized_model = distilled_quantized_model.cuda()
    if NUM_GPUS > 1: distilled_quantized_model = torch.nn.parallel.DataParallel(distilled_quantized_model)
    if not distilled_quantized_model_name in cifar10Manager.saved_models:
        cifar10Manager.add_new_model(distilled_quantized_model_name, distilled_quantized_model_path,
                                     arguments_creator_function=smallerOptions)
    if TRAIN_DISTILLED_QUANTIZED_MODEL:
        cifar10Manager.train_model(distilled_quantized_model, model_name=distilled_quantized_model_name,
                                   train_function=convForwModel.train_model,
                                   arguments_train_function={'epochs_to_train': epochsToTrainCIFAR10,
                                                             'teacher_model': teacherModel,
                                                             'use_distillation_loss': True,
                                                             'quantizeWeights':True,
                                                             'numBits':numBit,
                                                             'bucket_size':256,
                                                             'print_every':50,
                                                             'initial_learning_rate':0.1,
                                                             'learning_rate_style':'cifar100',
                                                             'weight_decayL2':0.0005,
                                                             'quantize_first_and_last_layer':False},
                                   train_loader=train_loader, test_loader=test_loader)
    #distilled_quantized_model.load_state_dict(cifar10Manager.load_model_state_dict(distilled_quantized_model_name))
    del distilled_quantized_model

del teacherModel

def load_model_from_name(x):
    opt = cifar10Manager.load_metadata(x, 0)[0]
    #small old bug in the saving of metadata, this is a cheap trick to remedy it
    for key, val in opt.items():
        if isinstance(val, str):
            opt[key] = eval(val)
    model = Wide_ResNet(**opt)
    if USE_CUDA: model = model.cuda()
    try:
        model.load_state_dict(cifar10Manager.load_model_state_dict(x))
    except:
        model.load_state_dict(mhf.convert_state_dict_from_data_parallel(
            cifar10Manager.load_model_state_dict(x)))
    return model

for x in cifar10Manager.list_models():
    if cifar10Manager.get_num_training_runs(x) == 0:
        continue
    model = load_model_from_name(x)
    reported_accuracy = cifar10Manager.load_metadata(x)[1]['predictionAccuracy'][-1]
    #pred_accuracy = cnn_hf.evaluateModel(model, test_loader, fastEvaluation=False)
    pred_accuracy=0
    print('Model "{}" ==> Prediction accuracy: {:2f}% == Reported accuracy: {:2f}%'.format(x,
                                                        pred_accuracy*100, reported_accuracy*100))
    curr_num_bit = cifar10Manager.load_metadata(x)[0].get('numBits', None)
    if curr_num_bit is not None:
        quant_fun = functools.partial(quantization.uniformQuantization, s=2**curr_num_bit, bucket_size=256)
        actual_bit_huffmman = qhf.get_huffman_encoding_mean_bit_length(model.parameters(), quant_fun,
                                                                       'uniform', s=2**curr_num_bit)
        size_reduction = mhf.get_size_reduction(actual_bit_huffmman, bucket_size=256)
        size_model_MB = mhf.get_size_quantized_model(model, curr_num_bit, quant_fun,
                                                             bucket_size=256, quantizeFirstLastLayer=False)
        print('Effective bit Huffman: {} - Size reduction: {} - Size MB : {}'.format(actual_bit_huffmman,
                                                                                size_reduction, size_model_MB))
    else:
        size_model_MB = mhf.getNumberOfParameters(model)*4/1000000
        print('Size MB : {}'.format(size_model_MB))

    if CHECK_PM_QUANTIZATION:
        if 'distilled' in x and 'quant' not in x:
            for numBit in numBits:
                try:
                    model.load_state_dict(cifar10Manager.load_model_state_dict(x))
                except:
                    model.load_state_dict(mhf.convert_state_dict_from_data_parallel(
                        cifar10Manager.load_model_state_dict(x)))
                numParam = sum(1 for _ in model.parameters())
                for idx, p in enumerate(model.parameters()):
                    if idx == 0 or idx == numParam - 1:
                        continue
                    p.data = quantization.uniformQuantization(p.data, s=2**numBit, type_of_scaling='linear',
                                                              bucket_size=256)[0]
                #predAcc = cnn_hf.evaluateModel(model, test_loader, fastEvaluation=False)
                predAcc =0
                print('PM quantization of model "{}" with "{}" bits and bucketing 256: {:2f}%'.format(x, numBit, predAcc * 100))
                quant_fun = functools.partial(quantization.uniformQuantization, s=2**numBit, bucket_size=None)
                actual_bit_huffmman = qhf.get_huffman_encoding_mean_bit_length(model.parameters(), quant_fun,
                                                                               'uniform',s=2**numBit)
                size_reduction = mhf.get_size_reduction(actual_bit_huffmman, bucket_size=256)
                size_model_MB = mhf.get_size_quantized_model(model, numBit, quant_fun,
                                                             bucket_size=256, quantizeFirstLastLayer=False)
                print('Effective bit Huffman: {} - Size reduction: {} - Size MB: {}'.format(
                                                                            actual_bit_huffmman,
                                                                            size_reduction,
                                                                            size_model_MB))


import project1 as p1
import utils
import numpy as np

#-------------------------------------------------------------------------------
# Data loading. There is no need to edit code in this section.
#-------------------------------------------------------------------------------

train_data = utils.load_data('reviews_train.tsv')
val_data = utils.load_data('reviews_val.tsv')
test_data = utils.load_data('reviews_test.tsv')

train_texts, train_labels = zip(*((sample['text'], sample['sentiment']) for sample in train_data))
val_texts, val_labels = zip(*((sample['text'], sample['sentiment']) for sample in val_data))
test_texts, test_labels = zip(*((sample['text'], sample['sentiment']) for sample in test_data))

dictionary = p1.bag_of_words(train_texts)

train_bow_features = p1.extract_bow_feature_vectors(train_texts, dictionary)
val_bow_features = p1.extract_bow_feature_vectors(val_texts, dictionary)
test_bow_features = p1.extract_bow_feature_vectors(test_texts, dictionary)

'''
Once you have completed the implementation of the 3 learning algorithms, you should qualitatively verify your implementations. In main.py we have
included a block of code that you should uncomment. This code loads a 2D dataset from toy_data.txt, and trains your models using  𝑇=10,𝜆=0.2 .
main.py will compute
𝜃  and  𝜃0  for each of the learning algorithms that you have written. Then, it will call plot_toy_data to plot the resulting model and boundary.
'''
'''
Since you have implemented three different learning algorithm for linear classifier, it is interesting to investigate which algorithm would actually converge. Please run it with a larger number of iterations  𝑇  to see whether the algorithm would visually converge. You may also check whether the parameter in your theta converge in the first decimal place. Achieving convergence in longer decimal requires longer iterations, but the conclusion should be the same.

Which of the following algorithm will converge on this dataset? (Choose all that apply.):
average perceptron algorithm converges
pegasos algorithm converges
'''
#-------------------------------------------------------------------------------
# Problem 5
#-------------------------------------------------------------------------------

toy_features, toy_labels = toy_data = utils.load_toy_data('toy_data.tsv')

#average perceptron algorithm va pegasos algorithm dar in problem converge mishan (T=10 ,100,200,300,500,600 bezar
#ta bebini adad ha bad ye modat converge mishan) ama alorithm perceptron haminjoori dar hale taghir kardane ba taghire T
T = 10
L = 0.2

thetas_perceptron = p1.perceptron(toy_features, toy_labels, T)
thetas_avg_perceptron = p1.average_perceptron(toy_features, toy_labels, T)
thetas_pegasos = p1.pegasos(toy_features, toy_labels, T, L)

def plot_toy_results(algo_name, thetas):
    print('theta for', algo_name, 'is', ', '.join(map(str,list(thetas[0]))))
    print('theta_0 for', algo_name, 'is', str(thetas[1]))
    utils.plot_toy_data(algo_name, toy_features, toy_labels, thetas)

plot_toy_results('Perceptron', thetas_perceptron)
plot_toy_results('Average Perceptron', thetas_avg_perceptron)
plot_toy_results('Pegasos', thetas_pegasos)

#-------------------------------------------------------------------------------
# Problem 7
#-------------------------------------------------------------------------------

T = 25
L = 0.01
# dade ha age well seperated nabashan training accuracy kam mishe. fek kunam 100 darsad accuracy faghat dar halati hast ke linearly seperable bashan dadeha
pct_train_accuracy, pct_val_accuracy = \
   p1.classifier_accuracy(p1.perceptron, train_bow_features,val_bow_features,train_labels,val_labels,T=T)
print("{:35} {:.4f}".format("Training accuracy for perceptron:", pct_train_accuracy))
print("{:35} {:.4f}".format("Validation accuracy for perceptron:", pct_val_accuracy))

avg_pct_train_accuracy, avg_pct_val_accuracy = \
   p1.classifier_accuracy(p1.average_perceptron, train_bow_features,val_bow_features,train_labels,val_labels,T=T)
print("{:43} {:.4f}".format("Training accuracy for average perceptron:", avg_pct_train_accuracy))
print("{:43} {:.4f}".format("Validation accuracy for average perceptron:", avg_pct_val_accuracy))

avg_peg_train_accuracy, avg_peg_val_accuracy = \
   p1.classifier_accuracy(p1.pegasos, train_bow_features,val_bow_features,train_labels,val_labels,T=T,L=L)
print("{:50} {:.4f}".format("Training accuracy for Pegasos:", avg_peg_train_accuracy))
print("{:50} {:.4f}".format("Validation accuracy for Pegasos:", avg_peg_val_accuracy))

#-------------------------------------------------------------------------------
# Problem 8
#-------------------------------------------------------------------------------
'''
You finally have your algorithms up and running, and a way to measure performance! But, it's still unclear what values the hyperparameters
like  𝑇  and  𝜆  should have. In this section, you'll tune these hyperparameters to maximize the performance of each model.

One way to tune your hyperparameters for any given Machine Learning algorithm is to perform a grid search over all the possible combinations of values.
If your hyperparameters can be any real number, you will need to limit the search to some finite set of possible values for each hyperparameter.
For efficiency reasons, often you might want to tune one individual parameter, keeping all others constant, and then move onto the next one; Compared
to a full grid search there are many fewer possible combinations to check, and this is what you'll be doing for the questions below.

In main.py uncomment Problem 8 to run the staff-provided tuning algorithm from utils.py. For the purposes of this assignment,
please try the following values for  𝑇 : [1, 5, 10, 15, 25, 50] and the following values for  𝜆  [0.001, 0.01, 0.1, 1, 10].
For pegasos algorithm, first fix  𝜆=0.01  to tune  𝑇 , and then use the best  𝑇  to tune  𝜆

'''
'''
Isnt tuning T just overfitting?
𝑇  is tuned for generalization purposes. When you say best results, I believe you are talking about the training error. It is probably true the algorithm will
continue to improve with larger  𝑇 . However, what we care is
the test error. If you have your model fit too well with the training set, you might have overfitting issue, which could fail to generalize to the test set.

plot ha neshoon midan ke az ye T be bad validation ha accuracy shoon paeen miad!
'''

data = (train_bow_features, train_labels, val_bow_features, val_labels)

# values of T and lambda to try
Ts = [1, 5, 10, 15, 25, 50]
Ls = [0.001, 0.01, 0.1, 1, 10]

pct_tune_results = utils.tune_perceptron(Ts, *data)
print('perceptron valid:', list(zip(Ts, pct_tune_results[1])))
print('best = {:.4f}, T={:.4f}'.format(np.max(pct_tune_results[1]), Ts[np.argmax(pct_tune_results[1])]))

avg_pct_tune_results = utils.tune_avg_perceptron(Ts, *data)
print('avg perceptron valid:', list(zip(Ts, avg_pct_tune_results[1])))
print('best = {:.4f}, T={:.4f}'.format(np.max(avg_pct_tune_results[1]), Ts[np.argmax(avg_pct_tune_results[1])]))

# fix values for L and T while tuning Pegasos T and L, respective
fix_L = 0.01
peg_tune_results_T = utils.tune_pegasos_T(fix_L, Ts, *data)
print('Pegasos valid: tune T', list(zip(Ts, peg_tune_results_T[1])))
print('best = {:.4f}, T={:.4f}'.format(np.max(peg_tune_results_T[1]), Ts[np.argmax(peg_tune_results_T[1])]))

fix_T = Ts[np.argmax(peg_tune_results_T[1])]
peg_tune_results_L = utils.tune_pegasos_L(fix_T, Ls, *data)
print('Pegasos valid: tune L', list(zip(Ls, peg_tune_results_L[1])))
print('best = {:.4f}, L={:.4f}'.format(np.max(peg_tune_results_L[1]), Ls[np.argmax(peg_tune_results_L[1])]))

utils.plot_tune_results('Perceptron', 'T', Ts, *pct_tune_results)
utils.plot_tune_results('Avg Perceptron', 'T', Ts, *avg_pct_tune_results)
utils.plot_tune_results('Pegasos', 'T', Ts, *peg_tune_results_T)
utils.plot_tune_results('Pegasos', 'L', Ls, *peg_tune_results_L)

'''
After you have chosen your best method (perceptron, average perceptron or Pegasos) and parameters, use this classifier to compute testing accuracy
on the test set.
We have supplied the feature matrix and labels in main.py as test_bow_features and test_labels.
Note: In practice the validation set is used for tuning hyperparameters while a heldout test set is the final benchmark used to compare
disparate models that have already been tuned. You may notice that your results using a validation set don't always align with those of the test set,
and this is to be expected.
'''

#-------------------------------------------------------------------------------
# Use the best method (perceptron, average perceptron or Pegasos) along with
# the optimal hyperparameters according to validation accuracies to test
# against the test dataset. The test data has been provided as
# test_bow_features and test_labels.
#-------------------------------------------------------------------------------

# Your code here
avg_peg_train_accuracy, avg_peg_val_accuracy = \
   p1.classifier_accuracy(p1.pegasos, train_bow_features,test_bow_features,train_labels,test_labels,T=25,L=0.01)
print("{:50} {:.4f}".format("Training accuracy for Pegasos:", avg_peg_train_accuracy))
print("{:50} {:.4f}".format("Testing accuracy for Pegasos:", avg_peg_val_accuracy))

'''
According to the largest weights (i.e. individual  𝑖  values in your vector), you can find out which unigrams were the most impactful ones
in predicting positive labels.
Uncomment the relevant part in main.py to call utils.most_explanatory_word.
Report the top ten most explanatory word features for positive classification below:
Also experiment with finding unigrams that were the most impactful in predicting negative labels.
'''
#-------------------------------------------------------------------------------
# Assign to best_theta, the weights (and not the bias!) learned by your most
# accurate algorithm with the optimal choice of hyperparameters.
#-------------------------------------------------------------------------------
'''
all you are doing is looking for the corresponding words to the largest coefs within theta, so theta_0 is irrelevant
dar vaghe bordare theta ye seri az element hash vazn ziadi dare ke motenazer ba ye seri feature hast ke ona ro paeen bedast miarim
'''
thetas_pegasos = p1.pegasos(train_bow_features, train_labels, 25, 0.01)

best_theta =thetas_pegasos[0] #p1.pegasos(train_bow_features, train_labels, 25, 0.01) # Your code here
wordlist   = [word for (idx, word) in sorted(zip(dictionary.values(), dictionary.keys()))]
sorted_word_features = utils.most_explanatory_word(best_theta, wordlist)
print("Most Explanatory Word Features")
print(sorted_word_features[:10]) # Top 10
print(sorted_word_features[-10:]) # Worst 10


from flask import Flask, render_template, request, session, redirect, url_for, jsonify
import openai

app = Flask(__name__)
app.secret_key = 'your_key'  # Replace with your own secret key

openai.api_key = "your_api_key"

def limit_sentences(text, num_sentences):
    sentences = text.split('. ')
    limited_text = '. '.join(sentences[:num_sentences])
    return limited_text

@app.route('/')
def home():
    session.clear()
    return render_template('index.html')

@app.route('/play', methods=['POST'])
def play_game():
    result = ''
    player_input = request.form.get('player_input')
    player_name = request.form.get('player_name')

    if 'player_name' not in session or not session['player_name']:  # check if player_name is not set or is empty
        if player_name:  # if player_name form input is not empty
            session['player_name'] = player_name
            session['moves'] = 0
            session['has_key'] = False
            session['game_over'] = False
            session['options'] = []
            session['current_location'] = ''
            session['state'] = ''

            # Here you can include a starting message to the player
            return jsonify(
                {'text': f"Welcome to the game, {session['player_name']}! Your horrific adventure begins now...",
                 'options': []})
        else:  # player_name form input is empty, do not proceed
            return jsonify({'text': "Please enter your name to start the game.", 'options': []})
    else:
        session['moves'] += 1  # Increase the move count
        if session['moves'] > 10:
            session['game_over'] = True
            return jsonify({'text': "You feel the ground beneath you collapse. You fall into a pit and your adventure "
                                    "ends here.", 'options': []})

        else:
            if session['options'] and player_input is not None:  # Make sure player_input is not None
                option_text = session['options'][int(player_input) - 1]
                if "key" in option_text and session['moves'] > 3:
                    session['has_key'] = True

                if "lock" in option_text and session['has_key'] and session['moves'] > 4:
                    session['game_over'] = True
                    return "You have successfully found the lock and use the key to open it. You have escaped and won the " \
                           "game! "
            else:
                option_text = player_input

            # generate situations
            if session['moves'] == 1:
                session['state'] = "starting point"
                situation_prompt = f"The player named {session.get('player_name', '')} is starting his/her adventure. " \
                                   f"Describe the starting surroundings in a horror setting. Don't start a sentence if it is "\
                       f"going to exceed the max_tokens defined. Replace 'the player' with 'you'. Write it as a paragraph. "
            elif 7 <= session['moves'] < 9:
                session['state'] = "towards the end"
                situation_prompt = f"The player named {session.get('player_name', '')} has chosen option {option_text} in the situation: {session.get('current_location', '')}."\
                       f"Build up on the current situation accordingly. All the situations must be linked with each other to form a story. Make the story as horror filled as possible. Replace 'the player' with 'you'. Write it as a paragraph. Don't leave a sentence Lead the player towards a key in a horror setting."
            elif session['moves'] == 10:
                session['state'] = "end"
                situation_prompt = f"The player named {session.get('player_name', '')} has chosen option {option_text} in the situation: {session.get('current_location', '')}."\
                       f"Build up on the current situation accordingly. All the situations must be linked with each other to form a story. Make the story as horror filled as possible.  "\
                       f" Replace 'the player' with 'you'. Write it as a paragraph. Lead the player towards a lock in a horror setting."
            else:
                session['state'] = "middle"
                situation_prompt = f"The player named {session.get('player_name', '')} has chosen option {option_text} in the situation: {session.get('current_location', '')}."\
                       f"Build up on the current situation accordingly. All the situations must be linked with each other to form a story. Make the story as horror filled as possible."\
                       f"Replace 'the player' with 'you'. Write it as a very small paragraph."

            response = openai.Completion.create(
                engine="text-davinci-003",
                prompt=situation_prompt,
                temperature=0.5,
                max_tokens=100
            )

            result = response.choices[0].text.strip()
            session['current_location'] = limit_sentences(result, 5)

            # generate options
            if session['moves'] >= 7:
                options_prompt = f"In {session['current_location']}, what are the three options the player can choose from? The "
                f"options should be horror and based on the current situation of the player. Do not repeat the "
                f"options in "
                f"different situations. Each option should be a single line and not numbered. Options for the last three situations should lead the player towards a key "
                f"and lock."

            else:
                options_prompt = f"In {session['current_location']}, what are the three options the player can choose " \
                                 f"from? The options should be horror and based on the current situation of the " \
                                 f"player. Each option should be a single line and not numbered. Do not repeat the options in different situations. Do not provide options " \
                                 f"with 'lock' or 'key'."

            options_response = openai.Completion.create(
                engine="text-davinci-003",
                prompt=options_prompt,
                temperature=0.6,
                max_tokens=100
            )
            session['options'] = options_response.choices[0].text.strip().split("\n")

        response = {
            'text': session['current_location'],
            'options': session['options']
        }
        return jsonify(response)

if __name__ == '__main__':
    app.run(debug=True)


#!/usr/bin/env python
"""
MUDDERY SERVER LAUNCHER SCRIPT

This is the start point for running Muddery.
"""

import traceback
import sys
import asyncio
from argparse import ArgumentParser
from muddery.launcher import configs

def main():
    """
    Run the muddery main program.
    """
    sys_argv = sys.argv
    if len(sys_argv) <= 1:
        # no argv, show help
        print(configs.ABOUT_INFO)
        sys.exit(0)

    elif sys_argv[1] == "init":
        # initialization of game directory
        if len(sys_argv) <= 2:
            print("You should input the game directory's name.")
            sys.exit(-1)

        parser = ArgumentParser()
        parser.add_argument('-p', '--port', nargs=1, action='store', dest='port')
        args, unknown_args = parser.parse_known_args()

        game_name = sys_argv[2]
        template = None
        if len(sys_argv) >= 4:
            template = sys_argv[3]

        port = None
        if args.port:
            try:
                port = int(args.port[0])
            except:
                print("Port must be a number.")
                sys.exit(-1)

        from muddery.launcher import manager
        try:
            manager.init_game(game_name, template, port)
        except Exception as e:
            traceback.print_exc()
            sys.exit(-1)
        sys.exit(0)

    elif sys_argv[1] == "setup":
        # Create databases and load default data.
        from muddery.launcher import manager
        try:
            manager.create_server_tables()
            manager.create_worldeditor_tables()
            manager.load_game_data()
            manager.collect_webclient_static()
            manager.collect_worldeditor_static()
            manager.check_editor_admin(False)
        except Exception as e:
            traceback.print_exc()
            sys.exit(-1)
        sys.exit(0)

    elif sys_argv[1] == "createadmin":
        # Create an administrator account.
        from muddery.launcher import manager
        try:
            manager.check_editor_admin(True)
        except Exception as e:
            sys.exit(-1)
        sys.exit(0)

    elif sys_argv[1] == "upgrade":
        # Upgrade the game dir to the latest version.
        from muddery.launcher import manager
        try:
            manager.upgrade_game()
        except Exception as e:
            print(e)
            sys.exit(-1)
        sys.exit(0)

    elif sys_argv[1] == "migrate":
        # Migrate databases to new version.
        from muddery.launcher import manager
        try:
            manager.migrate_database("gamedata")
            manager.migrate_database("worlddata")
            manager.migrate_database("worldeditor")
        except Exception as e:
            print(e)
            sys.exit(-1)
        sys.exit(0)

    elif sys_argv[1] == "loaddata":
        # Load game data from the worlddata folder.
        from muddery.launcher import manager
        try:
            manager.load_game_data()
        except Exception as e:
            print(e)
            sys.exit(-1)
        sys.exit(0)

    elif sys_argv[1] == "sysdata":
        # Reload system default data.
        from muddery.launcher import manager
        try:
            manager.load_system_data()
        except Exception as e:
            print(e)
            sys.exit(-1)
        sys.exit(0)

    elif sys_argv[1] == "start":
        # Start servers.
        parser = ArgumentParser()
        parser.add_argument('-s', '--server', action='store_true', dest='server', default=False)
        parser.add_argument('-c', '--client', action='store_true', dest='client', default=False)
        parser.add_argument('-e', '--editor', action='store_true', dest='editor', default=False)
        args, unknown_args = parser.parse_known_args()

        from muddery.launcher import manager
        try:
            if not args.server and not args.client and not args.editor:
                asyncio.run(manager.run_servers(gameserver=True, webclient=True, editor=True))
            else:
                asyncio.run(manager.run_servers(
                    gameserver=args.server,
                    webclient=args.client,
                    editor=args.editor
                ))
        except Exception as e:
            traceback.print_exc()
            sys.exit(-1)
        sys.exit(0)

    elif sys_argv[1] == "stop":
        # Stop servers.
        parser = ArgumentParser()
        parser.add_argument('-s', '--server', action='store_true', dest='server', default=False)
        parser.add_argument('-c', '--client', action='store_true', dest='client', default=False)
        parser.add_argument('-e', '--editor', action='store_true', dest='editor', default=False)
        args, unknown_args = parser.parse_known_args()

        from muddery.launcher import manager
        try:
            if not args.server and not args.client and not args.editor:
                asyncio.run(manager.kill_servers(gameserver=True, webclient=True, editor=True))
            else:
                asyncio.run(manager.kill_servers(
                    gameserver=args.server,
                    webclient=args.client,
                    editor=args.editor
                ))
        except Exception as e:
            traceback.print_exc()
            sys.exit(-1)
        sys.exit(0)

    elif sys_argv[1] == "restart":
        # Stop servers.
        parser = ArgumentParser()
        parser.add_argument('-s', '--server', action='store_true', dest='server', default=False)
        parser.add_argument('-c', '--client', action='store_true', dest='client', default=False)
        parser.add_argument('-e', '--editor', action='store_true', dest='editor', default=False)
        args, unknown_args = parser.parse_known_args()

        from muddery.launcher import manager
        try:
            if not args.server and not args.client and not args.editor:
                asyncio.run(manager.restart_servers())
            else:
                asyncio.run(manager.restart_servers(
                    gameserver=args.server,
                    webclient=args.client,
                    editor=args.editor
                ))
        except Exception as e:
            traceback.print_exc()
            sys.exit(-1)
        sys.exit(0)

    elif sys_argv[1] == "state":
        # Check servers running states.
        print("Checking server state ...")
        from muddery.launcher import manager
        try:
            asyncio.run(manager.show_server_state(gameserver=True, webclient=True, editor=True))
        except Exception as e:
            traceback.print_exc()
            sys.exit(-1)
        sys.exit(0)

    else:
        # Other operations
        parser = ArgumentParser(add_help=False)
        parser.add_argument('-v', '--version', action='store_true', dest='show_version', default=False)
        parser.add_argument('-h', '--help', action='store_true', dest='show_help', default=False)
        args, unknown_args = parser.parse_known_args()

        from muddery.launcher import manager

        if args.show_version:
            # show the version info
            manager.show_version()
            sys.exit(0)
        elif args.show_help:
            # show the help info
            print(configs.CMDLINE_HELP)
            sys.exit(0)
        else:
            # no input; print muddery info
            print(configs.ABOUT_INFO)
            sys.exit(0)

if __name__ == '__main__':
    # start Muddery from the command line
    main()


import json
from email.message import Message
from email.mime import base
import os
import base64
import boto3
import time

Birds = ['Parakeet','Dove','Cardinal','Finch','Woodpecker','Robin','Titmouse','Sparrow','Warbler','Northern Cardinal', 'Carolina Wren', 'Mourning Dove',
    'Tufted Titmouse', 'Blue Jay', 'Carolina Chickadee', 'Red-bellied Woodpecker', 'Northern Mockingbird', 'Eastern Phoebe',
    'Eastern Towhee', 'American Crow', 'Eastern Bluebird', 'Downy Woodpecker', 'American Robin', 'American Goldfinch', 'Chipping Sparrow',
    'Pine Warbler', 'House Finch', 'White-breasted Nuthatch', 'Song Sparrow', 'Red-winged Blackbird', 'Northern Flicker',
    'Pileated Woodpecker', 'Common Grackle', 'European Starling', 'Ruby-throated Hummingbird', 'Indigo Bunting', 'Gray Catbird'
    'Yellow-rumped Warbler', 'Ruby-crowned Kinglet', 'White-throated Sparrow', 'Yellow-bellied Sapsucker'
    ]

# Get AWS resources
s3 = boto3.client('s3')
rekognition = boto3.client('rekognition')
sns = boto3.client('sns')
dynamodb = boto3.resource('dynamodb')

# Define resources
topic_arn = "YOUR SNS TOPIC ARN"
s3_bucket = 'YOUR S3 BUCKET'
bird_table = 'YOUR DYNAMODB TABLE'

# Go into temp directory
os.chdir('/tmp')

def lambda_handler(event, context):
    # Define Variables
    image_data = event['body']
    bird_type = ""
    bird_confidence = 0
    image_name = event['queryStringParameters']["ImageName"]
    capture_time = event['queryStringParameters']["CaptureTime"]
    table = dynamodb.Table("TestTable2")

    image_file = bytes(image_data, "ascii")
    image_bytes = base64.decodebytes(image_file)

    # Send data to rekognition
    rekognition_response = rekognition.detect_labels(Image ={'Bytes' : image_bytes}, MaxLabels = 10)
    #print(rekognition_response)

    # Parse the labels to determine if a bird type is recognized
    for label in rekognition_response["Labels"]:
        if label['Name'] in Birds:
            if label['Confidence'] > bird_confidence:
                bird_type = label['Name']
                bird_confidence = label['Confidence']
            else:
                pass
        else:
            pass

    # Check labels again for bird that may not have been recognized or not in the bird list
    for label in rekognition_response["Labels"]:
        if label['Name'] == "Bird" and bird_confidence == 0:
            bird_type = "Bird"
        else:
            pass

    # Send a message if their is confidence a bird is present in the bird feeder
    if bird_confidence > 0:
        #Decode image and write to file. This will be saved in the tmp directory
        temp_image = open("birdimage.jpg", "wb")
        temp_image.write(base64.b64decode(image_file))
        temp_image.close()

        #Upload image to s3
        s3_response = s3.upload_file('./birdimage.jpg', s3_bucket, image_name)
        object_url = '{}/{}/{}'.format(s3.meta.endpoint_url, s3_bucket, image_name)
        ID = int(image_name.replace(".jpg",""))

        #upload to dynamodb
        ddb_response = table.put_item(
            Item={

                'ID': ID,
                'Image_Name' : image_name,
                'Capture_Time' : capture_time,
                'Bird_Type' : bird_type,
                'Image_URL' : object_url

            }
            )
        sns_message = "There is a " + bird_type + " in the bird feeder. Download Link: " + object_url
        sns_subject = bird_type + " Detected!"
        sns_message = sns.publish(TopicArn=topic_arn, Subject=sns_subject, Message=sns_message)

    elif bird_type == "Bird":
        tmpimg = open("birdimage.jpg", 'wb')
        tmpimg.write(base64.b64decode(imgfile))
        tmpimg.close()
        s3response = s3.upload_file('./birdimage.jpg', s3Bucket, img_name)
        ddbresponse = table.put_item(
            Item={

                'ID': int(image_name.replace(".jpg","")),
                'Image_Name' : img_name,
                'Capture_Time' : capture_time,
                'Bird_Type' : 'Undetermined',
                'Image_URL' : object_url

            }
            )
        snsMessage = "There is a bird in the bird feeder. Download link: " + object_url
        snsSubject = "Bird Detected!"
        snsMessage = sns.publish(TopicArn = topic_arn, Subject = sns_subject, Message = sns_message)
    else:
        bird_type = "None"

    bird_data = {'ImageName': image_name, 'CaptureTime' : capture_time, 'BirdType': bird_type}

    return {
        'statusCode': 200,
        'body': json.dumps(bird_data)
    }