# Fix code format and add proxy system
# - Zenqi

import requests
import Prawler
import random
import string
import time

def main():
    try:
        num_proxy = int(input("Enter number of proxy to use (Default: 50) > "))
    except:
        num_proxy = 50
    try:
        amount    = int(input("Enter the amount of nitro to generate (Default: 1000) > "))
    except:
        amount = 1000
    value = 1

    print("generating proxy")
    # generate proxy

    proxy_list = Prawler.get_proxy_list(num_proxy, 'http', 'elite', 'US')

    while value <= amount:

        _proxy = random.choice(proxy_list)
        proxies = {
            "http": f"http://{_proxy}"
        }

        nitro = ''.join([random.choice(string.ascii_letters + string.digits) for i in range(16)])
        print(f"waiting 120s before making request, to avoid ratelimit \n")
        time.sleep(120)
        print(f"trying -> {nitro} with proxy -> {_proxy}\n")
        r = requests.get(f'https://discordapp.com/api/v6/entitlements/gift-codes/{nitro}', proxies=proxies, timeout=3)

        if r.status_code == 200:
            print(f"wow, -> {nitro} is accepted.")
            with open('Codes.txt', 'w+') as f:
                f.write(f"\ndiscord.gift/{nitro}")
                f.close()

        elif r.status_code == 404:
            print(f"invalid code -> {nitro}")

        elif r.status_code == 429:
            print('too many request. proxy -> rate limit?\n')

        value += 1

if __name__ == "__main__":
    main()


from django.core import serializers
from django.db.models import Q
from django.shortcuts import get_object_or_404, get_list_or_404
from django.http import HttpResponse
from .models import *
import json

def index(request):
    return HttpResponse("Hello, world. You're at the radio index.")

def station(request, station_id):
    """
    Haal informatie op voor een enkel radiostation.
    """
    station = get_object_or_404(Station, id=station_id)

    serialized_station = serializers.serialize('json', [station])

    return HttpResponse(json.dumps(json.loads(serialized_station)[0]))

def stations(request):
    """
    Haal lijst op van alle radiostations.
    """
    stations = Station.objects.all()

    serialized_stations = serializers.serialize('json', stations)

    return HttpResponse(serialized_stations)

def station_played_tracks(request, station_id, limit=50):
    """
    Haal de laatst afgespeelde tracks op voor een radiostation.
    """
    played_tracks = get_list_or_404(
        PlayedTrack.objects.order_by('-pk'),
        station=station_id
    )[:limit]

    serialized_played_tracks = serializers.serialize(
        'json',
        played_tracks,
        use_natural_foreign_keys=True,
        use_natural_primary_keys=True
    )
    return HttpResponse(serialized_played_tracks)

def track(request, track_id):
    """
    Haal de informatie op voor een enkele track.
    """
    track = get_object_or_404(Track, id=track_id)

    serialized_track = serializers.serialize('json', [track])

    return HttpResponse(json.dumps(json.loads(serialized_track)[0]))

def tracks(request, track_ids):
    """
    Haal de informatie op voor meerdere tracks.
    """
    track_ids = track_ids.split(',')

    # Query opbouwen; vergelijkbaar met WHERE IN (1...n)
    q = Q()
    for ti in track_ids:
        q = q | Q(id=ti)

    tracks = get_list_or_404(Track, q)

    serialized_tracks = serializers.serialize('json', tracks)

    return HttpResponse(serialized_tracks)


from django.shortcuts import redirect, render
from django.utils.lru_cache import lru_cache
from django.utils.translation import ugettext_lazy
from wagtail.wagtailadmin import messages
from wagtail.wagtailadmin.edit_handlers import ObjectList, TabbedInterface
from wagtail.wagtailcore.models import Site

from .models import SiteMetadataPreferences

@lru_cache()
def get_edit_handler(cls):
    if hasattr(cls, 'edit_handler'):
        return cls.edit_handler.bind_to_model(cls)

    tabs = []

    tabs.append(ObjectList(cls.general_panels, heading=ugettext_lazy('General')))
    tabs.append(ObjectList(cls.twitter_panels, heading=ugettext_lazy('Twitter'), classname="settings"))

    EditHandler = TabbedInterface(tabs)
    return EditHandler.bind_to_model(cls)

def index(request):
    site = Site.find_for_request(request)
    instance = SiteMetadataPreferences.objects.filter(site=site).first()
    if instance is None:
        instance = SiteMetadataPreferences(site=site)
    EditHandler = get_edit_handler(SiteMetadataPreferences)
    SiteMetadataPreferencesForm = EditHandler.get_form_class(SiteMetadataPreferences)

    if request.method == "POST":
        form = SiteMetadataPreferencesForm(request.POST, instance=instance)
        if form.is_valid():
            form.save()
            messages.success(request, 'The form has been successfully saved.')
            return redirect('wagtailmetadata')
        else:
            messages.error(request, 'The form could not be saved due to validation errors')
    else:
        form = SiteMetadataPreferencesForm(instance=instance)

    edit_handler = EditHandler(instance=instance, form=form)

    return render(request, 'wagtailmetadata/index.html', {
        'form': form,
        'edit_handler': edit_handler,
    })


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('homo_sapiens_83_38_variation', '0006_auto_20160219_1703'),
    ]

    operations = [
        migrations.AlterField(
            model_name='compressed_genotype_var',
            name='variation_id',
            field=models.ForeignKey(related_name='variation_compressed_genotype_var', db_column=b'variation_id', to='homo_sapiens_83_38_variation.variation'),
        ),
        migrations.AlterField(
            model_name='failed_variation',
            name='variation_id',
            field=models.ForeignKey(related_name='variation_failed_variation', db_column=b'variation_id', to='homo_sapiens_83_38_variation.variation'),
        ),
        migrations.AlterField(
            model_name='individual',
            name='individual_type_id',
            field=models.ForeignKey(related_name='individual_type_individual', db_column=b'individual_type_id', default=0, to='homo_sapiens_83_38_variation.individual_type'),
        ),
        migrations.AlterField(
            model_name='read_coverage',
            name='sample_id',
            field=models.ForeignKey(related_name='sample_read_coverage', db_column=b'sample_id', to='homo_sapiens_83_38_variation.sample'),
        ),
        migrations.AlterField(
            model_name='read_coverage',
            name='seq_region_id',
            field=models.ForeignKey(related_name='seq_region_read_coverage', db_column=b'seq_region_id', to='homo_sapiens_83_38_variation.seq_region'),
        ),
        migrations.AlterField(
            model_name='sample',
            name='study_id',
            field=models.ForeignKey(related_name='study_sample', db_column=b'study_id', default=None, to='homo_sapiens_83_38_variation.study', null=True),
        ),
        migrations.AlterField(
            model_name='sample_genotype_multiple_bp',
            name='sample_id',
            field=models.ForeignKey(related_name='sample_sample_genotype_mbp', db_column=b'sample_id', default=None, to='homo_sapiens_83_38_variation.sample', null=True),
        ),
        migrations.AlterField(
            model_name='sample_genotype_multiple_bp',
            name='variation_id',
            field=models.ForeignKey(related_name='variation_sample_genotype_mbp', db_column=b'variation_id', to='homo_sapiens_83_38_variation.variation'),
        ),
    ]


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

dataset = pd.read_csv("../data/Position_Salaries.csv");

X = dataset.iloc[:,1:2].values
y = dataset.iloc[:, 2].values

# Feature scaling
#from sklearn.preprocessing import StandardScaler

#sc_X = StandardScaler()
#X = sc_X.fit_transform(X)

# Creating the Model
from sklearn.tree import DecisionTreeRegressor
regressor = DecisionTreeRegressor(random_state=0)

# Fitting the Model

regressor.fit(X, y)

# Predicting the test train result
y_pred = regressor.predict(6.5)

# Griding for higher resolution (replace X in plot with X_grid)
resolution = 1000
X_grid = np.arange(min(X), max(X), (max(X)-min(X))/resolution)
X_grid = X_grid.reshape(len(X_grid), 1)

# Creating the plot
plt.title('Position Vs Salary')
plt.xlabel('Position')
plt.ylabel('Salary')

# Visualizing the regression results
plt.scatter(X, y, color='red')
plt.plot(X_grid, regressor.predict(X_grid), color='blue')

# Show Plot
plt.show()


from JSpider import JSpider
from scrapy.selector import Selector
from scrapy.http import Request

class HiapkSpider(JSpider):
	name = 'HiapkSpider'
	allowed_domains = ['apk.hiapk.com']

	def __init__(self, mode, dbs, market, *args, **kwargs):
		super(HiapkSpider, self).__init__(mode, dbs, market, *args, **kwargs)

		self.initInitUrl = 'http://apk.hiapk.com/apps'
		self.initCatePath = '//div[@class="cate_list_box"]/ul/li[@class="category_item"]/a/@href'
		self.initUrlPrefix = 'http://apk.hiapk.com'

		self.listItemPrefix = 'http://apk.hiapk.com'

		self.listItemPath = '//span[@class="list_title font14_2"]/a/@href'

		self.metaTitlePath = '//div[@class="left font16"]/span[@class="none"]/text()'
		self.metaDescPath = '//pre[@id="softIntroduce"]/text()'

		self.downUrlPath = '//a[@id="ins91Btn"]/@href'
		self.downUrlPrefix = 'http://apk.hiapk.com'

	def parse_list_next(self, response):
		sel = Selector(response)

		try:
			nextRetUrl = None
			pageUrls = sel.xpath('//div[@class="page"]/a').extract()
			pageUrl = pageUrls[-1]
			if pageUrl.find('href') != -1:
				tmp = pageUrl.split('"')
				nextRetUrl = tmp[1]
			else:
				nextRetUrl = []

			if nextRetUrl != self.listNextNone:
				return Request('http://apk.hiapk.com' + nextRetUrl.replace('amp;', ''), callback = self.parse_list)
			else:
				raise Exception
		except:
			raise


from urlparse import urlparse
from urlparse import parse_qs
import logging
import socket

from pyramid.view import view_config
from pyramid.httpexceptions import HTTPFound

from yodl.tasks import transcode
from yodl.models import SongItem
from yodl.models import DBSession
from yodl.celery_utils import celery

log = logging.getLogger(__name__)

def url_validate(url):
    """docstring for url_validate"""
    o = urlparse(url)
    try:
        query = parse_qs(o.query)
        if o.netloc.endswith("youtube.com")\
                and query.keys() == [u'v']:
            return url
    except:
        pass

@view_config(route_name='home', renderer='templates/index.jinja2')
def main_view(request):
    if request.POST:
        url = request.params["url"].strip()
        # check if id and file all ready exists
        if url_validate(url):
            transcode.delay(url)
            request.session["error"] = False

        else:
            request.session["error"] = True

        return HTTPFound(request.route_url('home'))

    i = celery.control.inspect()
    try:
        active = i.active()
        queue = i.scheduled()
    except socket.error:
        log.error("Could not connect to RabbitMQ server.")
        raise

    songs = DBSession.query(SongItem).all()
    return {'songs': songs, 'queue': queue, 'active': active}


import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
matplotlib.use('Agg')
import random as random

"""
training model - only PM2.5
"""

csvfile = open('train.csv', encoding='Big5')

x = [] #record x

for row in csvfile:
    r = row.split('\n')
    d = r[0].split(',')
    x.append(d)

x.pop(0) #skip first row

#extract PM2.5
f = []
for row in x:
    if(row[2] == 'PM2.5'):
        row.pop(0)
        row.pop(0)
        row.pop(0)
        f.append(row)

for row in range(len(f)):
    for col in range(len(f[row])):
        f[row][col] = float(f[row][col])

X = [] #input
Y = [] #ground truth
days = 10 #all = len(f)
for n in range(5,15):
    for i in range(0, len(f[n]) - 10):
        X.append(f[n][i:i+9])
        Y.append(f[n][i + 9])

#y' = b + w1*x1 + w2*x2 + w3*x3 + w4*x4 + w5*x5 + w6*x6 + w7*x7 + w8*x8 + w9*x9
#y' = b + w[1:9] * x[1:9]'

b = 2.5 #initial b
w = np.ones(len(X[0])) #initial w (w1,w2...,w9)
for i in range(len(w)):
    w[i] = 0.5
lr = 100 #initial learning rate
iteration = 50000

b_lr = 0.0
w_lr = 0.0

#Store_initial values for plotting
b_history = [b]
w_history = [w]

#regularization
l = 0.1

#Iteration
for i in range(iteration):

    b_grad = 0.0
    w_grad = np.zeros(len(w))
    for n in range(len(X)):
        wX = np.dot(w, X[n])
        #print("wX: " + str(wX))
        b_grad = b_grad - 2.0*(float(Y[n]) - b - wX)*1.0
        for j in range(len(w_grad)):
            w_grad[j] = w_grad[j] - 2.0*(float(Y[n]) - b - wX)*X[n][j] - (l * 2 * w[j])

    #print("bg: " + str(b_grad))
    #print("wg: " + str(w_grad[0]))

    b_lr = b_lr + b_grad**2
    w_lr = w_lr + w_grad**2

    # Update parameters
    b = b - lr/np.sqrt(b_lr) * b_grad
    w = w - lr/np.sqrt(w_lr) * w_grad

    # Store parameters for plotting
    b_history.append(b)
    w_history.append(w)

    error = 0
    for n in range(len(X)):
        wX = np.dot(w, X[n])
        error += (Y[n] - (b + wX))**2 + l * np.dot(w, w)
    error = np.sqrt(error / len(X))
    if(i % 100 == 0):
        print("i: " + str(i) + "error: " + str(error))

outfile = open('weightone1.txt','w')
outfile.write(str(b) + '\n')
outfile.write(str(w))
outfile.close()

print("b: " + str(b))
print(w)


#!/usr/bin/python3

import discord
from discord.ext import commands
import asyncio

import math

class Pages:
    def __init__(self,
                 ctx,
                 current_page=1,
                 msg=None,
                 item_list=[],
                 title='Paginator',
                 display_option=(1, 0),
                 editable_content=True):
        """Creates a paginator.

        Parameters
        -----------
        ctx: discord.ext.commands.Context
            The current context (guild, channel, etc for bot to send messages).
        current_page: int
            Specify which page to display.
        msg: discord.Message
            This is helpful for delete function. Specify which message the bot needs to update if an element in the original message is modified.
        item_list: list or dictionary
            List of items to paginate. Using a dictionary is only useful for embeds option where there is a need for field names and values.
        title: str
            Summary of content of the items.
        display_option: tuple
            The first record of the tuple may have these values:
                0   : Messages will be in code blocks, the number of entries for each page is
                    defined by user (autosize = False), and item_list will be a list of strings
                    corresponding to the pages.
                1   : Code blocks, autosize = True, item_list is a list of strings.
                2   : Embed, autosize = False, item_list is a dictionary with two keys: names & values
                    the values of which will be a list. (item_list = {'names': [], 'values': []})
                3   : Embed, autosize = False, item_list is a list of strings.
                4   : Embed, autosize = True, item_list is a dictionary.
                5   : Embed, autosize = True, item_list is a list of strings.
            The second record is the user defined size of each page. For autosize = True, this value
            is ignored.
        editable_content: bool
            True if the items can be updated by the users (this is like an MVC).
            False otherwise.
        """
        self.bot = ctx.bot
        self.guild = ctx.guild
        self.channel = ctx.channel
        self.user = ctx.author
        self.message = msg
        self.itemList = item_list
        self.title = title
        self.displayOption = display_option
        self._organize()
        self.actions = [
            ('⏪', self._first_page),
            ('◀', self._prev_page),
            ('▶', self._next_page),
            ('⏩', self._last_page),
            ('⏹', self._halt),
        ]
        if editable_content:
            self.actions.append(('🚮', self._del))
        self.currentPage = current_page
        self.delete = False

    def _organize(self):
        organize_helper_map = {
            0: self._organize_code_blocks,
            1: self._organize_code_blocks_autosize,
            2: self._organize_embeds_dict,
            3: self._organize_embeds_list,
            4: self._organize_embeds_autosize_dict,
            5: self._organize_embeds_autosize_list,
        }
        pages_to_send = ['empty page']
        self.organize_helper = organize_helper_map[self.displayOption[0]]
        self.pagesToSend, self.lastPage = self.organize_helper(pages_to_send)

    def _organize_code_blocks(self, pages_to_send):
        item_per_page = self.displayOption[1]
        page_counter = math.ceil(len(self.itemList) / item_per_page)
        for i in range(page_counter):
            index_start = item_per_page * i
            index_end = item_per_page * (i + 1)
            content = '\n'.join(self.itemList[index_start:index_end]).replace(
                '```', '')
            pages_to_send.append('```markdown\n' + content +
                                 '\n\n~ Page {:02d} of {:02d} ~'.format(
                                     i + 1, page_counter) + '```')
        return pages_to_send, page_counter

    def _organize_code_blocks_autosize(self, pages_to_send):
        page_counter = length = cache = 0
        for i in range(len(self.itemList)):
            # truncate quotes that are too long
            if len(self.itemList[i]) > 1200:
                self.itemList[i] = self.itemList[i][:1200] + '...'
            length += len(self.itemList[i])
            if length > 1894:
                pages_to_send.append(
                    '```markdown\n' + self.title + ':\n\n' +
                    '\n'.join(self.itemList[cache:i]).replace('```', ''))
                cache = i
                length = len(self.itemList[i])
                page_counter += 1
            elif i == len(self.itemList) - 1:    # edge case
                pages_to_send.append(
                    '```markdown\n' + self.title + ':\n\n' +
                    '\n'.join(self.itemList[cache:i + 1]).replace('```', ''))
                page_counter += 1
        for i in range(len(pages_to_send)):
            pages_to_send[i] += '\n\n~ Page {:02d} of {:02d} ~'.format(
                i, page_counter) + '```'
        return pages_to_send, page_counter

    def _organize_embeds_dict(self, pages_to_send):
        item_per_page = self.displayOption[1]
        page_counter = math.ceil(len(self.itemList['names']) / item_per_page)
        em = discord.Embed(title=self.title, colour=0xDA291C)
        for i in range(page_counter):
            em.set_footer(
                text='Page {:02d} of {:02d}'.format(i + 1, page_counter))
            index_start = item_per_page * i
            index_end = item_per_page * (i + 1)
            for name, val in zip(
                    self.itemList['names'][index_start:index_end],
                    self.itemList['values'][index_start:index_end]):
                em.add_field(name=name, value=val)
            pages_to_send.append(em)
            em = discord.Embed(title=self.title, colour=0xDA291C)
        return pages_to_send, page_counter

    def _organize_embeds_list(self, pages_to_send):
        item_per_page = self.displayOption[1]
        page_counter = math.ceil(len(self.itemList) / item_per_page)
        em = discord.Embed(title=self.title, colour=0xDA291C)
        for i in range(page_counter):
            em.set_footer(
                text='Page {:02d} of {:02d}'.format(i + 1, page_counter))
            index_start = item_per_page * i
            index_end = item_per_page * (i + 1)
            em.description = ''.join(self.itemList[index_start:index_end])
            if len(em.description) > 1200:
                em.description = em.description[:1200] + '...'
            pages_to_send.append(em)
            em = discord.Embed(title=self.title, colour=0xDA291C)
        return pages_to_send, page_counter

    def _organize_embeds_autosize_dict(self, pages_to_send):
        # TODO: implement real autosize™
        # this method should not be called at the moment
        # itemPerPage = 10
        pass

    def _organize_embeds_autosize_list(self, pages_to_send):
        pass

    async def _show_page(self, page):
        self.currentPage = max(0, min(page, self.lastPage))
        if self.message:
            if self.currentPage == 0:
                try:
                    await self.message.delete()
                    self.message = None
                    return
                except:
                    pass
            else:
                if self.displayOption[0] < 2:    # code blocks
                    await self.message.edit(
                        content=self.pagesToSend[self.currentPage])
                else:    # embeds
                    await self.message.edit(
                        embed=self.pagesToSend[self.currentPage])
                return
        else:
            if self.displayOption[0] < 2:
                self.message = await self.channel.send(
                    content=self.pagesToSend[self.currentPage],
                    delete_after=300)
            else:
                self.message = await self.channel.send(
                    embed=self.pagesToSend[self.currentPage], delete_after=300)
            for (emoji, _) in self.actions:
                await self.message.add_reaction(emoji)
            return

    async def _first_page(self):
        await self._show_page(1)

    async def _prev_page(self):
        await self._show_page(max(1, self.currentPage - 1))

    async def _next_page(self):
        await self._show_page(min(self.lastPage, self.currentPage + 1))

    async def _last_page(self):
        await self._show_page(self.lastPage)

    async def _halt(self):
        await self._show_page(0)

    async def _del(self):
        self.delete = True
        await self._show_page(self.currentPage)

    def _react_check(self, reaction, user):
        if user == self.bot.user:
            return False
        if reaction.message.id != self.message.id:
            return False
        for (emoji, action) in self.actions:
            if reaction.emoji != emoji:
                continue
            self.user = user
            self._turn_page = action
            return True
        return False

    async def paginate(self):
        if self.delete:
            self.delete = False
            self._organize()
        await self._show_page(self.currentPage)
        while not self.delete and self.message:
            try:
                reaction, user = await self.bot.wait_for(
                    'reaction_add', check=self._react_check)
            except:
                try:
                    await self.message.delete()
                except:
                    pass
                finally:
                    break
            await self._turn_page()
            try:
                await self.message.remove_reaction(reaction, user)
            except:
                pass


"""Test PSCUData class from lpdpower.

Tim Nicholls, STFC Application Engineering Group
"""

import sys
if sys.version_info[0] == 3:  # pragma: no cover
    from unittest.mock import Mock, call, patch
else:                         # pragma: no cover
    from mock import Mock, call, patch

from nose.tools import *

sys.modules['smbus'] = Mock()
sys.modules['serial'] = Mock()
sys.modules['Adafruit_BBIO'] = Mock()
sys.modules['Adafruit_BBIO.GPIO'] = Mock()
from odin.adapters.parameter_tree import ParameterAccessor
from lpdpower.pscu_data import PSCUData, PSCUDataError

class TestPscuData():

    @classmethod
    def setup_class(cls):
        cls.pscu = Mock()
        cls.pscu.num_humidities = 2
        cls.pscu.num_temperatures = 11
        cls.pscu.num_quads = 4
        cls.pscu.num_leak_sensors = 1
        cls.pscu.quad = [Mock()] * cls.pscu.num_quads
        for quad in cls.pscu.quad:
            quad.num_channels = 4
        cls.pscu.get_all_latched.return_value = [True]*4
        cls.pscu_data = PSCUData(pscu=cls.pscu)

    @patch('lpdpower.pscu_data.PSCU')
    def test_pscu_data_no_pscu_arg(self, mock_pscu):
        pd = PSCUData()
        assert(pd.pscu is not None)

    def test_get_param(self):

        val = self.pscu_data.get('position')
        self.pscu.get_position.assert_called_with()

    def test_set_param(self):

        enabled = True
        self.pscu_data.param_tree._tree['allEnabled']._type = bool
        self.pscu_data.set('allEnabled', enabled)
        self.pscu.enable_all.assert_called_with(enabled)

    def test_get_missing_param(self):

        missing_param = 'missing_param'
        with assert_raises_regexp(PSCUDataError, 'Invalid path: {}'.format(missing_param)):
            self.pscu_data.get(missing_param)

    def test_set_missing_param(self):

        missing_param = 'missing_param'
        with assert_raises_regexp(PSCUDataError, 'Invalid path: {}'.format(missing_param)):
            self.pscu_data.set(missing_param, 0)

    def test_get_all_latched(self):

        self.pscu_data.get_all_latched()
        self.pscu.get_all_latched.assert_called_with()

    def test_get_quad_traces(self):

        traces = self.pscu_data.get_quad_traces()
        assert_equal(len(traces), self.pscu.num_quads)
        self.pscu.get_quad_trace.assert_has_calls([call(i) for i in range(self.pscu.num_quads)])


import requests
from bs4 import BeautifulSoup
import pandas as pd

def get_gamelogs(player_id, season, playoffs=False):
    """
    :param player_id:
    :param season:
    :param playoffs:
    :return:
    """
    if playoffs:
        selector = 'div_pgl_basic_playoffs'
    else:
        selector = 'div_pgl_basic'

    url = 'https://widgets.sports-reference.com/wg.fcgi?css=1&site=bbr&url=%2Fplayers%2Fa%2F'
    url += f'{player_id}%2Fgamelog%2F{season}%2F&div={selector}'

    # request
    r = requests.get(url)
    soup = BeautifulSoup(r.content, 'html.parser')
    table = soup.table

    if table:
        return clean_df(pd.read_html(str(table))[0])
    else:
        return pd.DataFrame()

# helper functions
def clean_df(df):
    """
    Cleans input DataFrame:
        * exclude rows with column names
        * rename columns
        * create new feature which tells by how many points game was won/lost
        * new feature, boolean outcome: game won or lost
        * new feature where game was played: home or away
    :param df: pandas DataFrame
    :return: pandas DataFrame
    """
    # remove empty rows
    df = df[df.G != 'G'][df.columns[2:]].reset_index(drop=True)
    # rename columns
    df = df.rename(columns={'Unnamed: 5': 'Side', 'Unnamed: 7': 'Outcome'})
    # calculate result, W for wins, L for lost
    df['Result'] = df['Outcome'].apply(lambda x: x.split('(')[1][:-1])
    # create new feature with game result, + means game was won, - lost by X points
    df['Outcome'] = df['Outcome'].apply(lambda x: x.split('(')[0])
    # create column where the game was played for specific player
    df['Side'] = df['Side'].fillna('Away').replace('@', 'Home')

    return df

# helper functions
def html_2_table_season_stats(table):
    """
    Read HTML code tor return DataFrame
    Input:
        table- bs4 element,
    Output:
        pandas DataFrame
    """
    try:
        col_names = [_.text for _ in table.thead.findAll('th')]
        table_data = [[_.text for _ in row] for row in table.findAll('tr', {'class': 'full_table'})]
    except AttributeError:
        return pd.DataFrame()
    else:
        return pd.DataFrame(table_data, columns=col_names)

def get_season_stats(player_id):
    """
    Returns regular season stats
    :param player_id: string, e.g. mcgeeja01
    :return:
    """
    # generate request url
    url = f'https://www.basketball-reference.com/players/{player_id[0]}/{player_id}.html'

    # request
    r = requests.get(url)
    soup = BeautifulSoup(r.content, 'html.parser')

    # get player stats
    player_info = soup.find('div', {"class": 'players'})

    # get image location
    try:
        player_photo_src = player_info.find('div', {'class': 'media-item'}).img['src']
    except AttributeError:
        player_photo_src = 'assets/face_placeholder.png'

    try:
        player_name = player_info.span.text
    except AttributeError:
        player_name = ''

    # get information about player
    mk_text = ""

    for attr in player_info.findAll('span'):
        try:
            # check if item has item property attribute
            attr_name = attr['itemprop']
            attr_value = attr.text.strip()
            # ignore attributes with no values
            if attr_value != '':
                mk_text += f"* {attr_name}: {attr_value} \n"
        except KeyError:
            pass

    # table with regular season stats
    table_rs = soup.find('table', {'id': 'per_game'})
    df_rs = html_2_table_season_stats(table_rs)

    # table with play-off stats
    table_po = soup.find('table', {'id': 'playoffs_per_game'})
    df_po = html_2_table_season_stats(table_po)

    # generate list for all info about player
    player_info = [player_photo_src, player_name, mk_text]

    return df_rs, df_po, player_info


from flask import Flask, render_template, request, jsonify
from flask_cors import CORS, cross_origin
import json
import boto3
import time
import os

app = Flask(__name__)
CORS(app)

#Main endpoint for loading the webpage
@app.route('/')
def loadIndex():
    return render_template('index.html')

@app.route('/initServerMC', methods = ['POST'])
def initServerMC():
    inputPass = request.form['pass']
    returnData = {}

    message = "Password Incorrect!"

    if inputPass == os.environ['SERVER_PASSWORD']:
        #Instantiate server here or return ip address if already running
        client = boto3.client(
            'ec2',
            aws_access_key_id=os.environ['ACCESS_KEY'],
            aws_secret_access_key=os.environ['SECRET_KEY'],
            region_name=os.environ['EC2_REGION']
        )
        message = manageServer(client)

    print(message)
    return render_template('index.html', ipMessage=message)

#Gets IP Address for return to webpage otherwise boots server
def manageServer(client):
    returnString = 'ERROR'

    instanceIds = [os.environ['INSTANCE_ID']]
    response = client.describe_instances(InstanceIds = instanceIds)
    reservations = response['Reservations']
    reservation = reservations[0]

    instances = reservation['Instances']

    print("\nSERVER INSTANCES\n")
    print(instances)
    print("\n")
    if len(instances) > 0:
        instance = instances[0]
        state = instance['State']
        stateName = state['Name']

        if (stateName == 'stopped') or (stateName == 'shutting-down'):
            #SETUP MULTIPROCESSING HERE INSTEAD OF REDIS
            returnString = startServer(client)
        elif stateName == 'running':
            returnString = 'IP: ' + instance['PublicIpAddress']
        else:
            returnString = 'ERROR'
    return returnString

#Starts the specified AWS Instance from the configuration
def startServer(client):
    #Gets proper variables to attempt to instantiate EC2 instance and start minecraft server
    returnString = 'ERROR'
    instanceIds = [os.environ['INSTANCE_ID']]
    response = client.start_instances(InstanceIds = instanceIds)

    stateCode = 0

    while not (stateCode == 16):
        time.sleep(3)

        print('\nAWS EC2 START RESPONSE\n')
        print(str(response))
        print('\n')

        response = client.describe_instances(InstanceIds = instanceIds)
        reservations = response['Reservations']
        reservation = reservations[0]

        instances = reservation['Instances']
        instance = instances[0]

        state = instance['State']
        stateCode = state['Code']

        print("\nSERVER INSTANCES\n")
        print(instances)
        print("\n")

    ipAddress = instance['PublicIpAddress']
    returnString = 'Server is starting, this may take a few minutes.\nIP: ' + ipAddress
    return returnString

if __name__ == "__main__":
    app.run()


"""URLs for the aps_purchasing app."""
from django.conf.urls import patterns, url

from .views import QuotationUploadView, ReportView

urlpatterns = patterns(
    '',
    url(r'^quotation-upload/$', QuotationUploadView.as_view(),
        name='aps_purchasing_quotation_upload'),
    url(r'^report/(?P<pk>\d+)/$', ReportView.as_view(),
        name='aps_purchasing_report'),
    url(r'^report/', ReportView.as_view(),
        name='aps_purchasing_report'),
)


from pyramid.renderers import get_renderer

def add_base_template(event):
    """
    Expose the base template as per the Pyramid cookbook:

    https://docs.pylonsproject.org/projects/pyramid_cookbook/dev/templates.html#using-a-beforerender-event-to-expose-chameleon-base-template
    """
    base = get_renderer('templates/base.pt').implementation()
    event.update({'base': base})
    base = get_renderer('templates/base_bare.pt').implementation()
    event.update({'basebare': base})


from django.utils.translation import ugettext_lazy as _

from horizon import tables

from django.template.defaultfilters import title  # noqa
from horizon.utils import filters
from django.utils.translation import npgettext_lazy
from django.utils.translation import pgettext_lazy
import six
from django import template
from openstack_dashboard import policy

import logging

from django.conf import settings

LOG = logging.getLogger(__name__)

POWER_STATES = {
    0: "NO STATE",
    1: "RUNNING",
    2: "BLOCKED",
    3: "PAUSED",
    4: "SHUTDOWN",
    5: "SHUTOFF",
    6: "CRASHED",
    7: "SUSPENDED",
    8: "FAILED",
    9: "BUILDING",
}

POWER_DISPLAY_CHOICES = (
    ("NO STATE", pgettext_lazy("Power state of an Instance", u"No State")),
    ("RUNNING", pgettext_lazy("Power state of an Instance", u"Running")),
    ("BLOCKED", pgettext_lazy("Power state of an Instance", u"Blocked")),
    ("PAUSED", pgettext_lazy("Power state of an Instance", u"Paused")),
    ("SHUTDOWN", pgettext_lazy("Power state of an Instance", u"Shut Down")),
    ("SHUTOFF", pgettext_lazy("Power state of an Instance", u"Shut Off")),
    ("CRASHED", pgettext_lazy("Power state of an Instance", u"Crashed")),
    ("SUSPENDED", pgettext_lazy("Power state of an Instance", u"Suspended")),
    ("FAILED", pgettext_lazy("Power state of an Instance", u"Failed")),
    ("BUILDING", pgettext_lazy("Power state of an Instance", u"Building")),
)

STATUS_CHOICES = (
   ("active", True),
   ("shutoff", True),
   ("suspended", True),
   ("paused", True),
   ("error", False),
   ("rescue", True),
   ("shelved", True),
   ("shelved_offloaded", True),
)

STATUS_DISPLAY_CHOICES = (
    ("deleted", pgettext_lazy("Current status of an Instance", u"Deleted")),
    ("active", pgettext_lazy("Current status of an Instance", u"Active")),
    ("shutoff", pgettext_lazy("Current status of an Instance", u"Shutoff")),
    ("suspended", pgettext_lazy("Current status of an Instance",
                                u"Suspended")),
    ("paused", pgettext_lazy("Current status of an Instance", u"Paused")),
    ("error", pgettext_lazy("Current status of an Instance", u"Error")),
    ("resize", pgettext_lazy("Current status of an Instance",
                             u"Resize/Migrate")),
    ("verify_resize", pgettext_lazy("Current status of an Instance",
                                    u"Confirm or Revert Resize/Migrate")),
    ("revert_resize", pgettext_lazy(
        "Current status of an Instance", u"Revert Resize/Migrate")),
    ("reboot", pgettext_lazy("Current status of an Instance", u"Reboot")),
    ("hard_reboot", pgettext_lazy("Current status of an Instance",
                                  u"Hard Reboot")),
    ("password", pgettext_lazy("Current status of an Instance", u"Password")),
    ("rebuild", pgettext_lazy("Current status of an Instance", u"Rebuild")),
    ("migrating", pgettext_lazy("Current status of an Instance",
                                u"Migrating")),
    ("build", pgettext_lazy("Current status of an Instance", u"Build")),
    ("rescue", pgettext_lazy("Current status of an Instance", u"Rescue")),
    ("soft-delete", pgettext_lazy("Current status of an Instance",
                                  u"Soft Deleted")),
    ("shelved", pgettext_lazy("Current status of an Instance", u"Shelved")),
    ("shelved_offloaded", pgettext_lazy("Current status of an Instance",
                                        u"Shelved Offloaded")),
    # these vm states are used when generating CSV usage summary
    ("building", pgettext_lazy("Current status of an Instance", u"Building")),
    ("stopped", pgettext_lazy("Current status of an Instance", u"Stopped")),
    ("rescued", pgettext_lazy("Current status of an Instance", u"Rescued")),
    ("resized", pgettext_lazy("Current status of an Instance", u"Resized")),
)

def _check_status(data_id):
    return True # This is dummy processing

def get_status(data):
    ret = 'NG'
    if _check_status(data.id):
        ret = 'OK'
    return ret

class CreateData(tables.LinkAction):
    name = "create"
    verbose_name = _("Create Router")
    url = "horizon:drillingsystem:router_manager:create"
    classes = ("ajax-modal",)
    icon = "plus"

class RemoveData(tables.LinkAction):
    name = "delete"
    classes = ("btn-danger",)
    verbose_name = _("Remove")
    url = "horizon:drillingsystem:router_manager:delete"
    classes = ("ajax-modal",)
    icon = "plus"

def get_ips(instance):
    template_name = 'project/instances/_instance_ips.html'
    ip_groups = {}

    for ip_group, addresses in six.iteritems(instance.addresses):
        ip_groups[ip_group] = {}
        ip_groups[ip_group]["floating"] = []
        ip_groups[ip_group]["non_floating"] = []

        for address in addresses:
            if ('OS-EXT-IPS:type' in address and
               address['OS-EXT-IPS:type'] == "floating"):
                ip_groups[ip_group]["floating"].append(address)
            else:
                ip_groups[ip_group]["non_floating"].append(address)

    context = {
        "ip_groups": ip_groups,
    }
    return template.loader.render_to_string(template_name, context)

def get_power_state(instance):
    return POWER_STATES.get(getattr(instance, "OS-EXT-STS:power_state", 0), '')

class DataSummaryTable(tables.DataTable):
    # status = tables.Column(get_status, verbose_name= _("Status"))
    # name = tables.Column('name', verbose_name= _("Name"))
    # size = tables.Column('size', verbose_name= _("Size"))

    name = tables.Column("name",
                    link="horizon:drillingsystem:router_manager:detail",
                    verbose_name=_("Router Name"))

    image_name = tables.Column("image_name",
                    verbose_name=_("Image Name"))

    ip = tables.Column(get_ips,
                    verbose_name=_("IP Address"),
                    attrs={'data-type': "ip"})

    state = tables.Column(get_power_state,
                    filters=(title, filters.replace_underscores),
                    verbose_name=_("Power State"),
                    display_choices=POWER_DISPLAY_CHOICES)

    status = tables.Column("status",
                    filters=(title, filters.replace_underscores),
                    verbose_name=_("Status"),
                    status=True,
                    status_choices=STATUS_CHOICES,
                    display_choices=STATUS_DISPLAY_CHOICES)

    class Meta:
        name = "data_summary"
        verbose_name = _("DataSummary")
        table_actions = (CreateData,)
        row_actions = (RemoveData,)

def get_size(instance):
    if hasattr(instance, "full_flavor"):
        template_name = 'project/instances/_instance_flavor.html'
        size_ram = sizeformat.mb_float_format(instance.full_flavor.ram)
        if instance.full_flavor.disk > 0:
            size_disk = sizeformat.diskgbformat(instance.full_flavor.disk)
        else:
            size_disk = _("%s GB") % "0"
        context = {
            "name": instance.full_flavor.name,
            "id": instance.id,
            "size_disk": size_disk,
            "size_ram": size_ram,
            "vcpus": instance.full_flavor.vcpus,
            "flavor_id": instance.full_flavor.id
        }
        return template.loader.render_to_string(template_name, context)
    return _("Not available")

def get_keyname(instance):
    if hasattr(instance, "key_name"):
        keyname = instance.key_name
        return keyname
    return _("Not available")

def get_power_state(instance):
    return POWER_STATES.get(getattr(instance, "OS-EXT-STS:power_state", 0), '')

TASK_DISPLAY_NONE = pgettext_lazy("Task status of an Instance", u"None")

# Mapping of task states taken from Nova's nova/compute/task_states.py
TASK_DISPLAY_CHOICES = (
    ("scheduling", pgettext_lazy("Task status of an Instance",
                                 u"Scheduling")),
    ("block_device_mapping", pgettext_lazy("Task status of an Instance",
                                           u"Block Device Mapping")),
    ("networking", pgettext_lazy("Task status of an Instance",
                                 u"Networking")),
    ("spawning", pgettext_lazy("Task status of an Instance", u"Spawning")),
    ("image_snapshot", pgettext_lazy("Task status of an Instance",
                                     u"Snapshotting")),
    ("image_snapshot_pending", pgettext_lazy("Task status of an Instance",
                                             u"Image Snapshot Pending")),
    ("image_pending_upload", pgettext_lazy("Task status of an Instance",
                                           u"Image Pending Upload")),
    ("image_uploading", pgettext_lazy("Task status of an Instance",
                                      u"Image Uploading")),
    ("image_backup", pgettext_lazy("Task status of an Instance",
                                   u"Image Backup")),
    ("updating_password", pgettext_lazy("Task status of an Instance",
                                        u"Updating Password")),
    ("resize_prep", pgettext_lazy("Task status of an Instance",
                                  u"Preparing Resize or Migrate")),
    ("resize_migrating", pgettext_lazy("Task status of an Instance",
                                       u"Resizing or Migrating")),
    ("resize_migrated", pgettext_lazy("Task status of an Instance",
                                      u"Resized or Migrated")),
    ("resize_finish", pgettext_lazy("Task status of an Instance",
                                    u"Finishing Resize or Migrate")),
    ("resize_reverting", pgettext_lazy("Task status of an Instance",
                                       u"Reverting Resize or Migrate")),
    ("resize_confirming", pgettext_lazy("Task status of an Instance",
                                        u"Confirming Resize or Migrate")),
    ("rebooting", pgettext_lazy("Task status of an Instance", u"Rebooting")),
    ("reboot_pending", pgettext_lazy("Task status of an Instance",
                                     u"Reboot Pending")),
    ("reboot_started", pgettext_lazy("Task status of an Instance",
                                     u"Reboot Started")),
    ("rebooting_hard", pgettext_lazy("Task status of an Instance",
                                     u"Hard Rebooting")),
    ("reboot_pending_hard", pgettext_lazy("Task status of an Instance",
                                          u"Hard Reboot Pending")),
    ("reboot_started_hard", pgettext_lazy("Task status of an Instance",
                                          u"Hard Reboot Started")),
    ("pausing", pgettext_lazy("Task status of an Instance", u"Pausing")),
    ("unpausing", pgettext_lazy("Task status of an Instance", u"Resuming")),
    ("suspending", pgettext_lazy("Task status of an Instance",
                                 u"Suspending")),
    ("resuming", pgettext_lazy("Task status of an Instance", u"Resuming")),
    ("powering-off", pgettext_lazy("Task status of an Instance",
                                   u"Powering Off")),
    ("powering-on", pgettext_lazy("Task status of an Instance",
                                  u"Powering On")),
    ("rescuing", pgettext_lazy("Task status of an Instance", u"Rescuing")),
    ("unrescuing", pgettext_lazy("Task status of an Instance",
                                 u"Unrescuing")),
    ("rebuilding", pgettext_lazy("Task status of an Instance",
                                 u"Rebuilding")),
    ("rebuild_block_device_mapping", pgettext_lazy(
        "Task status of an Instance", u"Rebuild Block Device Mapping")),
    ("rebuild_spawning", pgettext_lazy("Task status of an Instance",
                                       u"Rebuild Spawning")),
    ("migrating", pgettext_lazy("Task status of an Instance", u"Migrating")),
    ("deleting", pgettext_lazy("Task status of an Instance", u"Deleting")),
    ("soft-deleting", pgettext_lazy("Task status of an Instance",
                                    u"Soft Deleting")),
    ("restoring", pgettext_lazy("Task status of an Instance", u"Restoring")),
    ("shelving", pgettext_lazy("Task status of an Instance", u"Shelving")),
    ("shelving_image_pending_upload", pgettext_lazy(
        "Task status of an Instance", u"Shelving Image Pending Upload")),
    ("shelving_image_uploading", pgettext_lazy("Task status of an Instance",
                                               u"Shelving Image Uploading")),
    ("shelving_offloading", pgettext_lazy("Task status of an Instance",
                                          u"Shelving Offloading")),
    ("unshelving", pgettext_lazy("Task status of an Instance",
                                 u"Unshelving")),
)

POWER_DISPLAY_CHOICES = (
    ("NO STATE", pgettext_lazy("Power state of an Instance", u"No State")),
    ("RUNNING", pgettext_lazy("Power state of an Instance", u"Running")),
    ("BLOCKED", pgettext_lazy("Power state of an Instance", u"Blocked")),
    ("PAUSED", pgettext_lazy("Power state of an Instance", u"Paused")),
    ("SHUTDOWN", pgettext_lazy("Power state of an Instance", u"Shut Down")),
    ("SHUTOFF", pgettext_lazy("Power state of an Instance", u"Shut Off")),
    ("CRASHED", pgettext_lazy("Power state of an Instance", u"Crashed")),
    ("SUSPENDED", pgettext_lazy("Power state of an Instance", u"Suspended")),
    ("FAILED", pgettext_lazy("Power state of an Instance", u"Failed")),
    ("BUILDING", pgettext_lazy("Power state of an Instance", u"Building")),
)

class UpdateRow(tables.Row):
    ajax = True

    def get_data(self, request, instance_id):
        instance = api.nova.server_get(request, instance_id)
        try:
            instance.full_flavor = api.nova.flavor_get(request,
                                                       instance.flavor["id"])
        except Exception:
            exceptions.handle(request,
                              _('Unable to retrieve flavor information '
                                'for instance "%s".') % instance_id,
                              ignore=True)
        try:
            api.network.servers_update_addresses(request, [instance])
        except Exception:
            exceptions.handle(request,
                              _('Unable to retrieve Network information '
                                'for instance "%s".') % instance_id,
                              ignore=True)
        error = get_instance_error(instance)
        if error:
            messages.error(request, error)
        return instance

class InstancesTable(tables.DataTable):
    TASK_STATUS_CHOICES = (
        (None, True),
        ("none", True)
    )
    STATUS_CHOICES = (
        ("active", True),
        ("shutoff", True),
        ("suspended", True),
        ("paused", True),
        ("error", False),
        ("rescue", True),
        ("shelved", True),
        ("shelved_offloaded", True),
    )
    name = tables.Column("name",
                         link="horizon:project:instances:detail",
                         verbose_name=_("Router Name"))
    image_name = tables.Column("image_name",
                               verbose_name=_("Image Name"))
    ip = tables.Column(get_ips,
                       verbose_name=_("IP Address"),
                       attrs={'data-type': "ip"})
    size = tables.Column(get_size, sortable=False, verbose_name=_("Size"))
    keypair = tables.Column(get_keyname, verbose_name=_("Key Pair"))
    status = tables.Column("status",
                           filters=(title, filters.replace_underscores),
                           verbose_name=_("Status"),
                           status=True,
                           status_choices=STATUS_CHOICES,
                           display_choices=STATUS_DISPLAY_CHOICES)
    az = tables.Column("availability_zone",
                       verbose_name=_("Availability Zone"))
    task = tables.Column("OS-EXT-STS:task_state",
                         verbose_name=_("Task"),
                         empty_value=TASK_DISPLAY_NONE,
                         status=True,
                         status_choices=TASK_STATUS_CHOICES,
                         display_choices=TASK_DISPLAY_CHOICES)
    state = tables.Column(get_power_state,
                          filters=(title, filters.replace_underscores),
                          verbose_name=_("Power State"),
                          display_choices=POWER_DISPLAY_CHOICES)
    created = tables.Column("created",
                            verbose_name=_("Time since created"),
                            filters=(filters.parse_isotime,
                                     filters.timesince_sortable),
                            attrs={'data-type': 'timesince'})

    class Meta(object):
        name = "instances"
        verbose_name = _("Instances")
        status_columns = ["status", "task"]
        row_class = UpdateRow
        table_actions_menu = ()#StartInstance, StopInstance, SoftRebootInstance)
        launch_actions = ()
        # if getattr(settings, 'LAUNCH_INSTANCE_LEGACY_ENABLED', False):
        #     launch_actions = (LaunchLink,) + launch_actions
        # if getattr(settings, 'LAUNCH_INSTANCE_NG_ENABLED', True):
        #     launch_actions = (LaunchLinkNG,) + launch_actions
        table_actions = launch_actions #+ (DeleteInstance,
                                       #   InstancesFilterAction)
        row_actions = ()# (StartInstance, ConfirmResize, RevertResize,
                      # CreateSnapshot, SimpleAssociateIP, AssociateIP,
                      # SimpleDisassociateIP, AttachInterface,
                      # DetachInterface, EditInstance, UpdateMetadata,
                      # DecryptInstancePassword, EditInstanceSecurityGroups,
                      # ConsoleLink, LogLink, TogglePause, ToggleSuspend,
                      # ToggleShelve, ResizeLink, LockInstance, UnlockInstance,
                      # SoftRebootInstance, RebootInstance,
                      # StopInstance, RebuildInstance, DeleteInstance)


from sqlalchemy.orm import joinedload_all

from mine.backend.trade.models.position_model import PositionModel
from mine.backend.trade.position_factory import PositionFactory
from mine.backend.trade.order_repository import OrderRepository
from mine.backend.trade.trade_log_repository import TradeLogRepository
from mine.backend.core.sqlalchemy import Database

class PositionRepository:
    @staticmethod
    def save(position):
        position.save()

    @staticmethod
    def get(id):
        entity = PositionRepository._get(id)
        return PositionRepository._create(entity)

    @staticmethod
    def get(trader_id):
        table = PositionModel.__table__
        session = Database.Session()
        query = session.query(PositionModel).options(joinedload_all('*')).filter(PositionModel.trader_id == trader_id)
        return [PositionRepository._create(entity) for entity in query]

    @staticmethod
    def save(position):
        entity = PositionModel(position.to_dict())
        entity.save()
        position.id = entity.id
        for order in position.orders:
            OrderRepository.save(order)
        for log in position.trade_logs:
            TradeLogRepository.save(log)

    @staticmethod
    def update(position):
        entity = PositionRepository._get(position.id)
        entity.update(position.to_dict())
        for order in position.orders:
            OrderRepository.update(order)
        for log in position.trade_logs:
            TradeLogRepository.update(log)

    @staticmethod
    def _get(id):
        return PositionModel.get(id)

    @staticmethod
    def _create(entity):
        params = entity.to_dict()
        params['orders'] = OrderRepository.get(position_id=entity.id)
        params['trade_logs'] = TradeLogRepository.get(position_id=entity.id)
        return PositionFactory.create_from_dict(params)


# -*- coding: utf-8 -*-
import json
from file_parser import dbmon_jsonparser

def susexp(filename):
    js = dbmon_jsonparser.Json_Parser('GOODKNIGHT')
    j = js.get_all_obj()
    for obj in j:
        file = obj.get('NAME')
        with open("../indicators/"+file+".json","w") as f:
            json.dump(obj,f,indent=4)

if __name__ == "__main__":
    susexp('/home/dbmon/dbMonitor/conf/suspects.json')


import gensim
from gensim.parsing import preprocessing
from gensim.test.utils import datapath,get_tmpfile
from gensim.models import KeyedVectors
from gensim.scripts.glove2word2vec import glove2word2vec
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
import gensim.downloader as api
import numpy as np
from gensim.test.utils import common_texts
from gensim.models import Word2Vec

class EmbeddingMatrix:

    def __init__(self):
        self.Model = None
        self.Vocabulary = None
        self.tokenizer = None

    def load_glove(self, filename):
        """
        function to load pre trained Glove Vectors in to a gensim Model
        :param filename:path to the pre trained Glove vector file
        :return:None
        """
        try:
            glove_file = datapath(filename)
            w2vfile = get_tmpfile("word2vec.txt")
            _ = glove2word2vec(glove_file, w2vfile)
            self.Model = KeyedVectors.load_word2vec_format(w2vfile)
            self.Vocabulary = self.Model.wv.vocab.keys()

        except:
            print("Cannot Load the Glove Vector file")

    def download_and_load_glove(self, model_name):
        self.Model = api.load(model_name)
        self.Vocabulary = self.Model.wv.vocab.keys()

    def load_word2vec(self, filename):
        """
        function to load pre trained Word2Vec Vectors in to a gensim Model
        :param filename:Path to the Word2Vec Vector file
        :return:None
        """
        try:
            self.Model = KeyedVectors.load_word2vec_format(filename)
            self.Vocabulary = self.Model.wv.vocab.keys()
        except:
            print("Cannot Load the Word2Vec file")

    def get_input_sequence(self, Texts, max_length):
        """
         Convert a text in to an input sequence padded to maximum length
        :param Texts: array of texts to be converted in to sequences
        :param max_length: maximum length of the sequence
        :return: padded sequences of the same length
        """
        sequences = self.tokenizer.texts_to_sequences(Texts)
        return pad_sequences(sequences, maxlen=max_length)

    def create_embedding_matrix(self, corpus,  vocabulary_size):
        """
        create an embedding matrix to be used for an embedding layer
        :param corpus:The text corpus to be fit on the embedding matriz
        :param vocabulary_size:size of the vocabulary to be used in the problem
        :return: an embedding matrix with size vocabulary_size and an embedding_dim
        """
        print("Vocabulary Size for embedding_matrix ",vocabulary_size)
        self.tokenizer = Tokenizer(num_words=vocabulary_size, oov_token="<oov>")
        self.tokenizer.fit_on_texts(corpus)
        word_index  = self.tokenizer.word_index
        weight_matrix = np.zeros((vocabulary_size, self.Model.vector_size))
        for word, i in word_index.items():
            if i >= vocabulary_size:
                continue

            if word in self.Vocabulary:
                # words not found in embedding index will be all-zeros.
                weight_matrix[i] = self.Model[word]
        print(weight_matrix)
        return weight_matrix

    def train_model(self, corpus, modelSavePath ):

        ALLTokens = []
        for x in corpus:
            tokens = list(gensim.utils.tokenize(x,lowercase=True))
            ALLTokens.append(tokens)
        print("Tokens ", ALLTokens)
        self.Model = Word2Vec(ALLTokens, size=10000, window=5, min_count=1, workers=4)
        #self.Model.train(ALLTokens,total_examples = len(ALLTokens), epochs = 1)
        self.Model.save(modelSavePath)
        self.Vocabulary = self.Model.wv.vocab.keys()
        print("Vocabulary :", self.Vocabulary )
        return self.create_embedding_matrix(corpus,len(self.Vocabulary))


import json, os, time, random
import numpy as np
import tensorflow as tf

import sys
from datetime import datetime

from pyLogger import pyLogger

import model, sample, encoder

class GPT2():
    """
        GPT-2 class
        ------------------------------------------------------------------------------------------------
        :model_name=117M : String, which model to use

        :seed= -1 : Integer seed for random number generators, fix seed to reproduce
        results. If seed <= -1 (default) then generate a random seed between 0 and 2^32 - 1

        :nsamples=1 : Number of samples to return total

        :batch_size=1 : Number of batches (only affects speed/memory).  Must divide nsamples.

        :length=0 : Number of tokens in generated text, if 0 (default), is
        determined by model hyperparameters

        :temperature=1 : Float value controlling randomness in boltzmann
        distribution. Lower temperature results in less random completions. As the
        temperature approaches zero, the model will become deterministic and
        repetitive. Higher temperature results in more random completions.

        :top_k=0 : Integer value controlling diversity. 1 means only 1 word is
        considered for each step (token), resulting in deterministic completions,
        while 40 means 40 words are considered at each step. 0 (default) is a
        special setting meaning no restrictions. 40 generally is a good value.

        :log_path : Path to log file. If one is not provided the default path will be used (logs/log.txt)

        :debug : Puts GPT-2 into a debugging mode. This allows for more logging and other useful features
        ------------------------------------------------------------------------------------------------
    """

    def __init__(self, model_name='117M', seed=-1, nsamples=1, batch_size=1, length=0, temperature=1, top_k=0, log_path=None, debug=False):
        self.model_name = model_name
        self.seed = seed
        self.nsamples = nsamples
        self.batch_size = batch_size
        self.length = length
        self.temperature = temperature
        self.top_k = top_k
        self.log_path = log_path
        self.debug = debug

        # builds encoder for model.
        self.enc = encoder.get_encoder(self.model_name)

        #builds paramerters to work off of
        self.hparams = model.default_hparams()
        with open(os.path.join('models', self.model_name, 'hparams.json')) as f:
            self.hparams.override_from_dict(json.load(f))

        # if checks pass, start tensorflow
        self.sess = tf.Session()

        self._getReady()

    def _getReady(self):
        """
            Builds the necessary objects for GPT-2 to run properly
        """
        # starts logger
        self.logger = pyLogger(self.log_path)
        self.logger._print("STARTING GPT-2 -- {}\n\n".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')), None)

        # checks properties before building application
        self._checks()

        # placeholder value, dtype, shape. Shape is batchzise and none
        self.context = tf.placeholder(tf.int32, [self.batch_size, None])
        self.logger._print("Context Pre-Output: {}".format(self.context))

        np.random.seed(self.seed)
        tf.set_random_seed(self.seed)

        # builds the sample for tensorflow
        self.buildSeq()

        saver = tf.train.Saver()
        ckpt = tf.train.latest_checkpoint(os.path.join('models', self.model_name))
        saver.restore(self.sess, ckpt)

        self.logger._print("Started GTP-2 With Params -- Seed: {}; Length: {}; nSamples: {}; Batch-Size: {}; Top-K: {}; Tempurature: {}".format(self.seed, self.length, self.nsamples, self.batch_size, self.top_k, self.temperature))

    def buildSeq(self):
        # generates a sample set sequence of data based on parameters provided
        self.sequence = sample.sample_sequence(
            hparams = self.hparams, length = self.length,
            context = self.context,
            batch_size = self.batch_size,
            temperature = self.temperature, top_k = self.top_k
        )

        # saves that data inside self.sequence
        # self.logger._print("Output: "+str(self.sequence))

    def _checks(self):
        """
            Makes sure all values are appropriate values are inputted and corrects those values
        """

        # checks the current seed and if it is not assigned random seed is generated
        if self.seed < 1 or type(self.seed) is not int:
            self.seed = random.randint(0, 2**32-1)
            self.logger._print("Seed was not defined. Your Seed is: " + str(self.seed), 'W')

        if self.nsamples < 1:
            self.logger._print("nSamples must be an integer greater than 0. nSamples set to 1", 'W')
            self.nsamples = 1

        # makes sure batch size is divisable by samples. prevents errors when building outputs
        if self.nsamples % self.batch_size == 0:
            # set batchsize to a default of 1
            self.logger._print("Batch Size is too large for the amount of samples provided. Batch Size was set to 1.", 'W')
            self.batch_size = 1

        # makes sure the output can calculate the desired length
        if self.length < 1 or self.length > self.hparams.n_ctx:
            # default length = hparmas.n_ctx (max size) / 2
            self.length = self.hparams.n_ctx // 2
            self.logger._print("Length was not defined or too long. Length was set to {}".format(self.length), 'W')

        if self.temperature < 1:
            self.logger._print("Temperature must be 1 or higher. Temperature was set to 1", 'W')
            self.temperature = 1

        if self.top_k < 0:
            self.logger._print("Top_k must be 0 or higher. Top_k was set to 0", 'W')
            self.top_k = 0

        # checks the log path assigned. If path does not exist, use default path
        if self.log_path != None and os.path.exists(self.log_path):
            self.log_path = None
            self.logger.path = self.log_path
            self.logger._print("Path provided for the logs does not exist! Path has been set to log folder", 'W')

    def run(self):
        """
            Runs GPT-2

            Debug allows the user to see more output. Also displays time it took for the program to create and display the output
        """

        self.logger._print("Type ?help for available commands")
        while True:
            self._prompt()

            if self.raw_text == None:
                continue

            # encodes the inputted string into a format that can be inputted into Tensor Flow
            context_tokens = self.enc.encode(self.raw_text)

            if self.debug:
                self.logger._print("Seed: {}; Length: {}; nSamples: {}; Batch-Size: {};".format(self.seed, self.length, self.nsamples, self.batch_size))
                start = time.time()

            for generated in range(self.nsamples // self.batch_size):

                # Returns shape of the output
                # print("Output Tensor: "+str(output))

                # feed dictionary, consiting of the context tokens
                # context tolkens are created by encoding the values provided by the user
                contextDic = {self.context: [context_tokens for placeholder in range(self.batch_size)]}

                # print("Context Dictionary: "+ str(contextDic))

                """
                    TODO figure out how this function works

                    OUT CREATES ALL THE OUTPUTS FOR THE GIVEN PROMPT
                    https://www.tensorflow.org/api_docs/python/tf/Session#run

                    Encoded text and model data is sent into Tensorflow.

                    Tensorflow is then able to create a output based on sample sequence and raw input.
                """
                out = self.sess.run(self.sequence, feed_dict=contextDic)

                # Removes the context tokens (inputted string) from the output.

                # Context tokens are included in the output, this splices them out in order to give only the output and not the original prompt
                out = out[:, len(context_tokens):]

                # print(out)

                """
                    ------------------------------------------------------------------------------------------------

                    There seems to be a graph comparison going on here and returns a list with the indexes to the text.
                    Out is a list of lists. Top list contains the sublists that hold the indices to the dictionary

                    ------------------------------------------------------------------------------------------------

                    What I believe is going on is, samples.py creates a graph that seems to be likely next words. This then gets thrown into the TensorFlow.run() function and is compaired to a dictionary. This then somehow comes up with an appropriate output to generate.

                    ------------------------------------------------------------------------------------------------

                    Because most of this seems to be handled inside of Tensorflow it may be difficult to gain access to the inner workings

                    ------------------------------------------------------------------------------------------------

                    TODO: see if there are other ways to generate Top-K.

                    -- i am unsure if i can get the Top-K. This is because it seems to be stored inside tensorflow and I have not found a way to output it --
                """

                for i in range(self.batch_size):
                    text = self.enc.decode(out[i])
                    self.logger._print("=" * 40 + " SAMPLE " + str(generated+1) + " " + "=" * 40, None)

                    # PRINTS LIST OF INDECIES
                    # print(out[i])
                    self.logger._print(text, "OUTPUT")

            print("=" * 80)
            if self.debug:
                end = time.time()
                self.logger._print("It took "+str(round(end - start)) + " seconds to generate this output")
            self.logger._save()

        # ends program
        self.close()

    # Input for GPT-2 to customize settings
    def _prompt(self):
        "Takes user input and throws appropriate respose to GPT-2"

        self.raw_text = input("Model prompt >>> ")

        # if prompt is empty notify user, and restart loop
        if not self.raw_text:
            print("Prompt should not be empty! Type ?help or ?h for available commands")
            self.raw_text = None

        elif self.raw_text == "?help" or self.raw_text == "?h":
            print("Available Commands:\n\n?help - Displays Commands. Alias: ?h\n#kill - Ends GPT-2 Process\n")
            self.raw_text = None

        # if user types #kill break the loop. Cleaner way to close application
        elif self.raw_text == "#kill":
            check = input("Are you sure you want to end GPT-2? [Y/n] ").lower()
            if check == "n":
                self.raw_text = None
            self.close()

        elif self.raw_text == "#change":
            print("What variable would you like to modify?\n\nnsamples: {} \nbatch_size: {} \nlength: {}".format(self.nsamples, self.batch_size, self.length))

            variable = input("Type the Variable: ").lower()
            try:
                value = int(input("New Value: "))

                if variable == "batch_size":
                    self.batch_size = value
                elif variable == "nsamples":
                    self.nsamples = value
                elif variable == "length":
                    self.length = value
                    self.buildSeq()
            except:
                self.logger._print("Please use an integer!", 'E')

            self.raw_text = None

    def close(self):
        self.logger._print("Ending GPT-2")
        self.logger._print("\n"+"="*80, None)

        self.logger.close()
        self.sess.close()
        sys.exit()


# from selenium import webdriver
from nse_scrap import webscrap
from pymongo import MongoClient
import requests
from bs4 import BeautifulSoup
import pandas as pd
import re
from options_health_db import option_db
import time
import csv

def webscrap1():
    # webscrap()
    connection = MongoClient('localhost', 27017)
    options_collec = f"options_collec"
    try:
        options_db = connection['options_health']
        options_db[options_collec].drop()
        # print('status Collec Deleted')
    except:
        print('error')

    reader=csv.reader(open(r'nse_exp_date.txt'))
    for row in reader:
        try:
            date=row[0]
            print("BANKNIFTY ",date)
            # symbol="BANKNIFTY"
            # url="https://www1.nseindia.com/live_market/dynaContent/live_watch/option_chain/optionKeys.jsp?symbol="+symbol+"&date="+date
            url='https://www1.nseindia.com/live_market/dynaContent/live_watch/option_chain/optionKeys.jsp?symbolCode=-9999&symbol=BANKNIFTY&symbol=BANKNIFTY&instrument=OPTIDX&date='+date
            # print(url)
            headers={"User-Agent": "Mozilla/5.0"}
            page=requests.get(url,headers=headers)
            # print(page.text)
            soup=BeautifulSoup(page.content,'lxml')
            # print(soup)
        except Exception as e:
            print("connection error ",e)

        try:
            nse_table = soup.find("table",attrs={'id':'octable'})
            # nse_heading =nse_table.tbody.find_all("tr")
            #print(nse_table)

            calls_headers=[]
            puts_headers=[]
            for th,j in zip(nse_table.find_all("th"),range(26)):
                if(j<=14):
                    calls_headers.append(th.text.replace('\n',' ').strip())
                else:
                    puts_headers.append(th.text.replace('\n',' ').strip())
            del calls_headers[0:4]
            del puts_headers[-1]
            calls_data=[]
            puts_data=[]
            tbody =nse_table.find_all("tr")
            trs=tbody[2:]
            # print(trs)
            # print(len(trs))
            for tr in trs[:-1]:
                c_row={}
                p_row={}

                tds=tr.find_all("td")
            # print(tds)

                for td,th in zip(tds[1:12],calls_headers):
                    c_row[th] = re.sub(",|\n","",td.text).strip()
                calls_data.append(c_row)

                for td,th in zip(tds[12:25],puts_headers):
                    p_row[th] = re.sub(",|\n","",td.text).strip()
                puts_data.append(p_row)

        except Exception as e:
            print("error",e)

        try:
            first_df = pd.DataFrame(calls_data)
            sec_df = pd.DataFrame(puts_data)
            final_df=pd.concat([first_df,sec_df],axis=1)

            # banknifty_option_db(final_expdate,final_df)

            final_df.to_csv("nse_bank_nifty_data.csv",index=False)

            # first_df.to_csv("nse_data1.csv")
            # sec_df.to_csv("nse_data2.csv")
            print("successful")
            # option_db(option)
        except:
            print('excel error')

        # print(text)
        try:
            select=soup.find("select",attrs={'id':'date'})
            date=select.select_one('option[selected]')['value']
            exp_date=date

            if((exp_date[0] and exp_date[1]).isdigit() ):
                new_expdate=exp_date
            else:
                new_expdate="0"+exp_date
            final_expdate=new_expdate[:5]+new_expdate[7:]
            # print(final_expdate)

            webscrap(date)
            option_db(final_expdate)
        except Exception as e:
            print("server error",e)

while True:
    webscrap1()
    time.sleep(300)


import cv2
import numpy as np
from matplotlib import pyplot as plt
from skimage import measure

def getAnswers(im, i):

	r, thresh = cv2.threshold(im, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)
	thresh = 255-thresh;
	thresh =  cv2.morphologyEx(thresh, cv2.MORPH_OPEN, np.ones((3,3)))

	a, b = im.shape[:2]
	bound1 = b/4
	bound2 = bound1*2
	bound3 = bound1*3
	bound4 = b

	height = a/15;
	width = b

	lowerbound = 1
	upperbound = height

	results = []
	for n in range(i, (i+15)):
		img = thresh[lowerbound:upperbound, 1:width]
		# cv2.imshow(str(n), img)

		label, count = measure.label(img, background=0, return_num=True)
		props = measure.regionprops(label)
		# colord = plt.imshow(label, cmap='spectral')
		# plt.axis('off')
		# plt.tight_layout()
		# plt.show()

		shaded = []
		for p in props:
			# print(p.area)
			if p.area > 700:
				shaded.append(p)

		shadeCount = len(shaded)

		if shadeCount > 1:
			results.append([n, 'Shade Error (more than 1 shade)'])
		elif shadeCount == 0:
			results.append([n, 'Shade Error (no shade)'])
		else:
			# results.append([n, 'Good Shade! Good job <3'])
			letter = None
			x, y = shaded[0].centroid
			if y >= 0 and y <= bound1:
				letter = 'A'
			elif y > bound1 and y <= bound2:
				letter = 'B'
			elif y > bound2 and y <= bound3:
				letter = 'C'
			elif y > bound3 and y <= bound4:
				letter = 'D'
			results.append([n, letter])

		lowerbound += height
		upperbound += height

	# for r in results:
	# 	print(r[0], r[1])

	return results

# start ---------------------------------------------------------------
def extract(filename):

	img = cv2.imread('_Normalized Photos/' + filename + '.jpg')
	img = cv2.medianBlur(img,5)

	gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
	thresh = cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,11,2)

	thresh = 255-thresh;
	thresh =  cv2.morphologyEx(thresh, cv2.MORPH_OPEN, np.ones((3,3)))
	# cv2.imshow('adap', thresh)
	# cv2.imshow('not adap', thresh2)

	blobs_labels, lent = measure.label(thresh, background=0, return_num=1)
	props = measure.regionprops(blobs_labels)
	colord = plt.imshow(blobs_labels, cmap='spectral')
	plt.axis('off')
	plt.tight_layout()
	# plt.show()

	areas = []
	i = 0
	for p in props:
		areas.append([p.area, p.bbox, p.centroid])
		i = i + 1

	areas.sort(reverse=True)

	x1, y1 = areas[0][2]

	h, width = img.shape[:2]
	midpoint = width/2

	if y1 < midpoint:
		left = areas[0][1]
		right = areas[1][1]
	else:
		left = areas[1][1]
		right = areas[0][1]

	x, y, x2, y2 = left
	ans_i1 = gray[x+12:x2-12, y+12:y2-12]
	results = getAnswers(ans_i1, 1)

	x, y, x2, y2 = right
	ans_i2 = gray[x+12:x2-12, y+12:y2-12]
	results_right = getAnswers(ans_i2, 16)

	results = results + results_right

	text_file = open('_Results/' + filename + '.txt', 'w')

	for r in results:
		# print(r[0], r[1])
		text_file.write(str(r[0]) + ' ' + r[1] + '\n')

	text_file.close()

	# cv2.imshow('left?', ans_i1)
	# cv2.imshow('right?', ans_i2)


"""
A log handler that rotates when either a time or size limit is reached
"""
from logging import FileHandler
from time import time, localtime
from logging.handlers import TimedRotatingFileHandler, RotatingFileHandler
import os

# pylint: disable=too-many-ancestors
class TimeOrSizeRotatingFileHandler(TimedRotatingFileHandler,
                                    RotatingFileHandler):
    """
    A log handler that rotates when either a time or size limit is reached
    """
    # pylint: disable=too-many-arguments
    def __init__(self,
                 filename: str,
                 mode: str = 'a',
                 maxBytes: int = 0,
                 backupCount: int = 0,
                 encoding=None,
                 delay: int = 0,
                 when: str = 'H',
                 interval: int = 1,
                 utc: bool = False):
        TimedRotatingFileHandler.__init__(
            self,
            filename=filename, when=when, interval=interval,
            backupCount=backupCount, encoding=encoding, delay=delay, utc=utc)

        RotatingFileHandler.__init__(self, filename=filename, mode=mode, maxBytes=maxBytes,
                                     backupCount=backupCount, encoding=encoding, delay=delay)

    def computeRollover(self, currentTime) -> int:
        return TimedRotatingFileHandler.computeRollover(self, currentTime)

    def doRollover(self):
        # get from logging.handlers.TimedRotatingFileHandler.doRollover()
        current_time = int(time())
        dst_now = localtime(current_time)[-1]
        new_rollover_at = self.computeRollover(current_time)

        while new_rollover_at <= current_time:
            new_rollover_at = new_rollover_at + self.interval

        # If DST changes and midnight or weekly rollover, adjust for this.
        if (self.when == 'MIDNIGHT' or self.when.startswith('W')) and not self.utc:
            dst_at_rollover = localtime(new_rollover_at)[-1]
            if dst_now != dst_at_rollover:
                if not dst_now:  # DST kicks in before next rollover, so we need to deduct an hour
                    addend = -3600
                else:  # DST bows out before next rollover, so we need to add an hour
                    addend = 3600
                new_rollover_at += addend
        self.rolloverAt = new_rollover_at
        TimedRotatingFileHandler.doRollover(self)

    def shouldRollover(self, record) -> bool:
        timed = TimedRotatingFileHandler.shouldRollover(self, record)
        size = RotatingFileHandler.shouldRollover(self, record)
        return timed or size

class MakedirsFileHandler(FileHandler):
    """
    A log handler that creates all missing dirs in its path
    https://stackoverflow.com/a/20667049/2261244 (unutbu)

    """

    def __init__(self,
                 filename: str,
                 mode: str ='a',
                 encoding=None,
                 delay: bool = 0):
        os.makedirs(os.path.dirname(filename), exist_ok=True)
        FileHandler.__init__(self, filename, mode, encoding, delay)


from baseline import parameter
import os
from APRI.compute_metrics import compute_metrics
from APRI.utils import *
import soundfile as sf

# %% PARAMS

plot = True

preset = 'mi_primerito_dia_postfilter_Q!'
params = parameter.get_params(preset)
data_folder_path = os.path.join(params['dataset_dir'], 'foa_dev/') # path to audios

gt_folder_path = os.path.join(params['dataset_dir'], 'metadata_dev')  # path to annotations

audio_file_name = 'fold1_room1_mix027_ov2.wav'
# audio_file_name = 'fold1_room1_mix001_ov1.wav'

metadata_file_name = os.path.splitext(audio_file_name)[0] + '.csv'
metadata_file_path = os.path.join(gt_folder_path, metadata_file_name)

fs = params['fs']

# window = params['window']
# window_size = params['window_size']
# window_overlap = params['window_overlap']
# nfft = params['nfft']
# D = params['D'] # decimate factor
# frame_length = params['label_hop_len_s']

# %% RUN

# Read audio
b_format, sr = sf.read(os.path.join(data_folder_path, audio_file_name))
assert sr == fs

# audio_lims = [0., 0.5] # s
# audio_lims = [34, 37] # s
# audio_lims = [39, 40] # s
audio_lims = [0, 60] # s

audio_start_samples = int(audio_lims[0] * sr)
audio_end_samples = int(audio_lims[1] * sr)
b_format = b_format[audio_start_samples:audio_end_samples]

# STFT
window_size = 1200*2
window_overlap = window_size//2
t, f, stft = scipy.signal.stft(b_format.T, sr, nperseg=window_size, noverlap=window_overlap)
M, K, N = stft.shape
stft = stft[:, :K//2 , :-1]
M, K, N = stft.shape

# plot_magnitude_spectrogram(stft)

DOA = doa(stft)  # Direction of arrival
diff = diffuseness(stft, dt=2)  # Diffuseness

# plot_diffuseness(diff)

# HARD THRESHOLD
diff_th = 0.05
diff_mask = diff <= diff_th
# plt.figure()
# plt.title('diff hard mask')
# plt.pcolormesh(diff_mask)

# %%
# ################### plot DOA statistics

DOA = doa(stft)
# plot_doa(doa(stft))

doa_masked = np.empty((2, K, N))
for k in range(K):
    for n in range(N):
        if diff_mask[k,n]:
            doa_masked[:, k, n] = doa(stft[:,k,n])
        else:
            doa_masked[:, k,n] = np.nan

# plot_doa(doa_masked)

DOA_decimated = np.empty((2, K, N//2)) # todo fix number
for n in range(N//2):
    # todo fix numbers depending on decimation factor
    DOA_decimated[:,:,n] = np.nanmean([doa_masked[:,:,n*2],doa_masked[:,:,n*2-1]], axis=0 )

    # TODO: try other methods...
    # for k in range(K):
    #     doas = np.asarray([ doa_masked[:,k,n*2], doa_masked[:,k,n*2+1] ])
    #     if not np.any(np.isnan(doas)):
    #         DOA_decimated[:, k, n] = np.mean(doas, axis=0)

    # DOA_decimated[:,:,n] = doa_masked[:,:,n*2]

# plot_doa(DOA_decimated)
doa_masked = DOA_decimated

M, K, N = doa_masked.shape
# Build broadband histogram from past and present bin

azis =  [ [] for n in range(N)]
eles =  [ [] for n in range(N)]

# filter with minimum number of estimates
K_th = 5
for n in range(N):
    a = doa_masked[0,:,n]
    e = doa_masked[1,:,n]
    azis_filtered = a[~np.isnan(a)]
    if len(azis_filtered) > K_th:
        azis[n] = azis_filtered
        eles[n] = e[~np.isnan(e)]

plt.figure()
# All estimates
for n in range(N):
    if len(azis[n]) > 0:
        a = np.mod(azis[n] * 180 / np.pi, 360)
        plt.scatter(np.ones(len(a))*n, a, marker='x', edgecolors='b')
# Circmedian
for n in range(N):
    if len(azis[n]) > 0:
        a = np.mod(azis[n] * 180 / np.pi, 360)
        plt.scatter(n, np.mod(circmedian(a,'deg'), 360), facecolors='none', edgecolors='k')

# boxplot
import seaborn as sns
a = []
for n in range(N):
    if len(azis[n]) > 0:
        a.append(np.mod(azis[n] * 180 / np.pi, 360))
    else:
        a.append([])
plt.figure()
sns.boxplot(data=a)

# # number of single-source bins in frequency for each n
# plt.figure()
# plt.grid()
# for n in range(N):
#     if len(azis[n]) > 0:
#         plt.scatter(n, len(azis[n]), marker='x',  edgecolors='b')

# for n in range(N):
#     mask = diff_mask[:,n]
#     if any(diff_mask[:,n]):
#         azis[n] = DOA[0, :, n][mask]
#         eles[n] = DOA[1, :, n][mask]

# H =  [ [] for n in range(N)]
# H_edges = [np.arange(0,2*np.pi, 2*np.pi/360.*5), np.arange(0,np.pi, np.pi/180.*5)]

# TODO: TIME AVERAGING DOAS
# TODO: PEAK PEAKING FROM 2D HIST

# Jt = 3 # neigbor frames for averaging
# side_Jt = int((Jt-1)/2)
# for n in range(side_Jt,N-side_Jt):
#
#     mask = diff_mask[:,n]
#     if any(diff_mask[:,n]):
#         azis[n] = DOA[0, :, n][mask]
#         eles[n] = DOA[1, :, n][mask]
        # H[n], _, _ = np.histogram2d(azis[n], eles[n], bins=H_edges)
        # H[n], _, _ = np.histogram2d(azis[n], eles[n])

        # todo: PEAK PEAKING, 2D HISTOGRAM, ETC
        # # compute histogram of azis
        # bin_size_azis_hist = 2*np.pi/360.*5
        # bin_size_eles_hist = np.pi/360.*5
        # azis_hist, _ = np.histogram(azis[n], bins = np.arange(-np.pi,np.pi, bin_size_azis_hist) )
        # eles_hist, _ = np.histogram(azis[n], bins = np.arange(-np.pi/2,np.pi/2, bin_size_azis_hist) )
        # # find peaks, 11
        # L_peak = 11 # toDO: TRY other lengths
        # peaks = scipy.signal.find_peaks_cwt(azis_hist, [L_peak])
        # if peaks.size == 1:
        #     azi = azis_hist[peaks] * 1.5 * bin_size_azis_hist
        #     ele = eles_hist[peaks] * 1.5 * bin_size_eles_hist # todo: indices from peaks are not same in azi and ele...
        # elif peaks.size == 2:
        #     print('2 peaks!')
        #     # TODO: do something
        #     azi = circmedian(azis_hist)
        #     ele = np.median(eles_hist)
        #     # azi = azis_hist[peaks] * 1.5 * bin_size_azis_hist
        #     # ele = eles_hist[peaks] * 1.5 * bin_size_eles_hist
        # else:
        #     warnings.warn('something happened on peak peaking!')

# %% WRITE OUTPUT FILE

import tempfile

fo = tempfile.NamedTemporaryFile()
csv_file_path = fo.name + '.csv'
output_file_path = (os.path.splitext(csv_file_path)[0]) + '.mat'

# seconds_per_frame = window_overlap / sr

with open(csv_file_path, 'a') as csvfile:
    writer = csv.writer(csvfile)
    for n in range(len(azis)):
        if len(azis[n]) > 0:  # if not empty, write
            # time = n * seconds_per_frame
            time = n * 0.1
            azi = np.mod(circmedian(azis[n]) * 180 / np.pi, 360)  # csv needs degrees, range 0..360
            ele = 90 - (np.median(eles[n]) * 180 / np.pi)  # csv needs degrees
            writer.writerow([time, azi, ele])

# %% PREPARE METADATA FOR MATLAB

csv_file_path_gt = (os.path.splitext(csv_file_path)[0]) + '_gt.csv'

# oversample_factor = int(0.1 / seconds_per_frame)
gt_csv = np.loadtxt(open(metadata_file_path, "rb"), delimiter=",")
with open(csv_file_path_gt, 'a') as csvfile:
    writer = csv.writer(csvfile)
    for row in gt_csv:
        # for i in range(oversample_factor):
            # time = row[0] * 0.1 + (seconds_per_frame * i)
        time = row[0] * 0.1
        azi = np.mod(row[3],360) # range 0..360
        ele = 90 - row[4] # inclination
        writer.writerow([time, azi, ele])

# %% CALL MATLAB

import matlab.engine
from scipy.io import loadmat

eng = matlab.engine.start_matlab()
this_file_path = os.path.dirname(os.path.abspath(__file__))
matlab_path = this_file_path + '/../multiple-target-tracking-master'
eng.addpath(matlab_path)

# default: [20, 10, 5, 50, 0.2, 25, 30]
V_azi = 20 # maximum velocity per frame? in degree
V_ele = V_azi//2 # maximum velocity per frame? in degree
in_sd = 5  # std_dev of measurement noise, [1, 50].
in_sdn = 50 # noise spectral density. values?
init_birth = 0.1 # probability of new birth
in_cp = 0.25 # percentage of noise in the measurement data
num_particles = 30 # number of particles: montercarlo iterations?
eng.func_tracking(csv_file_path, float(V_azi), float(V_ele), float(in_sd), float(in_sdn), init_birth, in_cp, float(num_particles), nargout=0)

# output_file = '/Users/andres.perez/source/DCASE2020/APRI/filter_output/fold1_room1_mix027_ov2.mat'
output = loadmat(output_file_path)
output_data = output['tracks'][0]
num_events = output_data.size

# each element of output_data is a different event
# order of stored data is [time][[azis][eles][std_azis][std_eles]]

# convert output data into Events
event_list = []
for n in range(num_events):
    frames = (output_data[n][0][0] / 0.1).astype(int) # frame numbers
    azis = output_data[n][1][0] * np.pi / 180.  # in rads
    eles  = (90 - output_data[n][1][1]) * np.pi/ 180. # in rads
    event_list.append(Event(-1, -1, frames, azis, eles))

#  PLOT # todo check elevation/inclination
plt.figure()
title_string = str(V_azi) +'_'+ str(V_ele) +'_'+ str(in_sd) +'_'+ str(in_sdn) +'_'+ str(init_birth) +'_'+ str(in_cp) +'_'+ str(num_particles)
plt.title(title_string)
plt.grid()

# framewise estimates
est_csv = np.loadtxt(open(csv_file_path, "rb"), delimiter=",")
t = est_csv[:,0] * 10
a = est_csv[:,1]
e = est_csv[:,2]
plt.scatter(t, a, marker='x',  edgecolors='b')

# groundtruth
gt_csv = np.loadtxt(open(metadata_file_path, "rb"), delimiter=",")
t = gt_csv[:,0]
a = np.mod(gt_csv[:,3], 360)
e = gt_csv[:,4]
plt.scatter(t, a, facecolors='none', edgecolors='r')

# particle filter
for e_idx, e in enumerate(event_list):
    a = e.get_azis() * 180 / np.pi
    plt.plot(e.get_frames(), a, color='chartreuse')


import os
import tempfile
import json
from collections import defaultdict,OrderedDict
import itertools

from gensim import corpora, models, similarities
import pandas
import string
from zhon.hanzi import punctuation

import fromdb

import thulac
import re

def tokenizer(text,token_type=None):
    """根据token_type的不同进行不同粒度的分词

    Arguments:
        text {str} -- 输入的文本

    Keyword Arguments:
        token_type {[type]} -- [description] (default: {None})
    """
    if not token_type:
        return str(text)
    elif token_type=='normal':
        import jieba
        return jieba.cut()
    elif token_type=='ketwords':
        return

def gen_documents(t_pat_path,token_type=None):

    """从数据源读取数据，并返回一个列表，每一个元素表示一篇文章.

    Returns:
        list: tokenizer后的文本.
    """

    db=fromdb.FromDB()
    t_pat_dict,documents=db.read_all_text()
    with open(t_pat_path,'w') as f:
        json.dump(t_pat_dict,f)
    print('生成data/t_pat_dict.json')
    documents=[[w for w in tokenizer(doc,token_type)] for doc in documents]
    return documents

def create_corpus(documents,dictionary_path,corpus_path):
    """
    去掉停用词和只出现一次的词.
    """
    chars = string.ascii_letters + string.digits+string.punctuation+punctuation
    chars=[item for item in chars]
    stoplist=chars+['\n']
    texts=[[word for word in document if word not in stoplist] for document in documents]
    #去掉只出现一次的词
    frequency=defaultdict(int)
    for text in texts:
        for token in text:
            frequency[token] +=1
    texts=[[token for token in text if frequency[token]>1] for text in texts]
    # print(texts)
    dictionary=corpora.Dictionary(texts)
    dictionary.save(dictionary_path)
    print('生成data/teachers.dict')
    #new_vec=dictionary.token2id
    corpus=[dictionary.doc2bow(text) for text in texts]#将文档转换为向量
    corpora.MmCorpus.serialize(corpus_path,corpus)
    print('生成data/teachers.mm')
    return dictionary,corpus

def create_index(dictionary_path,corpus_path,index_path):
    """创建相似度检索模型
    Args:
        dictionary_path ([type]): [字典存放位置]
        corpus_path ([type]): [语料库存放位置]
        index_path ([type]): [保存的检索模型位置]
    """
    dictionary=corpora.Dictionary.load(dictionary_path)
    corpus=corpora.MmCorpus(corpus_path)
    tf_idf=models.TfidfModel(corpus)
    index = similarities.MatrixSimilarity(tf_idf[corpus])
    index.save(index_path)
    print('生成data/teachers.index')

class Similarity:
    """基于相似度的匹配

    Returns:
        list: 每个老师的最终得分，和该老师下每个专利的得分.
    """

    def __init__(self,t_pat_path='data/t_pat_dict.json',dictionary_path='data/teachers.dict'\
	,corpus_path='data/teachers.mm',index_path='data/teachers.index',t_name_path='data/teacher_names.json'):
        with open(t_pat_path,'r') as f:
            self.t_pat_dict=json.load(f)
        # print(self.t_pat_dict)
        with open(t_name_path,'r',encoding='utf-8') as f:
            tmp_name_dict=json.load(f)

        self.t_name_dict=defaultdict(lambda :[])
        for t_id in tmp_name_dict:
            self.t_name_dict[tmp_name_dict[t_id]].append(t_id)
        # print(self.t_name_dict)

        self.t_pat_num=OrderedDict()
        for k in self.t_pat_dict:
            self.t_pat_num[k]=len(self.t_pat_dict[k])
        # self.t_pat_num={k:len(self.t_pat_dict[k]) for k in self.t_pat_dict}
        self.dictionary=corpora.Dictionary.load(dictionary_path)
        self.corpus=corpora.MmCorpus(corpus_path)
        self.tf_idf=models.TfidfModel(self.corpus)
        self.index=similarities.MatrixSimilarity.load(index_path)
        self.teachers=list(self.t_pat_num.keys())
        self.pat_nums=self.t_pat_num.values()
        # print(list(itertools.chain(self.t_pat_dict.values())))
        self.pat_ids=[]
        for c in self.t_pat_dict.values():
            self.pat_ids.extend(c)

    def send_query(self,query,k=10):
        """给定query文本，返回k个与之最匹配的教师和该教师拥有的与之最匹配的专利编号

        Args:
            query (str): query文本
            k (int, optional): Defaults to 10. 选择最匹配的k个教师

        Returns:
            list: 包含十个最匹配的老师，每个老师用一个列表表示他的编号和最匹配专利编号
        # """
        if query in self.t_name_dict:

            # query=thulac.thulac().cut(query, text=True)  #分词
            # query=re.findall(r'(\w*)_np',query)

            teachers_id=self.t_name_dict[query]
            teacher_results= {idx:{"t":int(item),"p":self.t_pat_dict[item][0]} for idx,item in enumerate(teachers_id)}
            pat_results={idx:pat for idx,pat in enumerate(self.t_pat_dict[teachers_id[0]])}
            # print(json.dumps({'teacher_results':teacher_results,'pat_results':pat_results}))
            return json.dumps({'teacher_results':teacher_results,'pat_results':pat_results})
        query=[item for item in query]
        vec_bow = self.dictionary.doc2bow(query)
        vec_tfidf = self.tf_idf[vec_bow]
        sims=self.index[vec_tfidf]
        sim_len=len(sims)
        new_sim_sum=[]
        new_sim=defaultdict(lambda :[])

        # 按专利得分排序
        pat_score=[(pat,score) for pat,score in zip(self.pat_ids,sims)]
        # print(pat_score[:10])
        pat_score_sort=sorted(pat_score,key=lambda item: item[1],reverse=True)
        pat_results_order=OrderedDict()
        for idx,item in enumerate(pat_score_sort[:10]):
            pat_results_order[idx]=item[0]
        # pat_results={idx:item[0] for idx,item in enumerate(pat_score_sort[:10])}

        # 累计每个教师专利的得分
        i=0
        while i<sim_len:
            for j,num in enumerate(self.pat_nums):
                sum_tmp=0
                for _ in range(int(num)):
                    new_sim[self.teachers[j]].append(sims[i])
                    sum_tmp+=sims[i]
                    i+=1
                new_sim_sum.append(sum_tmp)

        # 按匹配得分排序
        new_sim_sum=[(t,score) for t,score in zip(self.teachers,new_sim_sum)]
        new_sim_sum = sorted(new_sim_sum, key=lambda item: -item[1])
        new_sim_sum_k=new_sim_sum[:10]
        # 返回得分最高的教师和该教师得分最高的专利
        teacher_results=[]
        for t in new_sim_sum_k:
            pats=self.t_pat_dict[t[0]]# 专利编号
            teacher_results.append({"t":int(t[0]),"p":max(zip(pats,new_sim[t[0]]),key=lambda item: item[1])[0]})
        teacher_results_order=OrderedDict()
        for idx,item in enumerate(teacher_results):
            teacher_results_order[idx]=item
        # teacher_results_order={idx:item for idx,item in enumerate(teacher_results)}

        return json.dumps({'teacher_results':teacher_results_order,'pat_results':pat_results_order})

def main():
    dictionary_path='data/teachers.dict'
    corpus_path='data/teachers.mm'
    index_path='data/teachers.index'
    t_pat_path='data/t_pat_dict.json'

    # documents=gen_documents(t_pat_path)
    # create_corpus(documents,dictionary_path,corpus_path)
    # create_index(dictionary_path,corpus_path,index_path)

    s=Similarity(t_pat_path,dictionary_path,corpus_path,index_path)
    print(s.send_query('金南国 沈华清'))

if __name__ == '__main__':
    main()


# !/usr/bin/python3
# -*- coding: utf-8 -*-
import os
import sys
import struct
import logging
import argparse
import socketserver
import binascii
from lic_file import LicenceFileData

LIC_FILE_DATA = None
USB_BUS_DEV = None

logging.basicConfig(
    format='%(asctime)s [%(levelname)-5.5s]  %(message)s',
    handlers=[logging.StreamHandler(sys.stdout)],
    level=logging.INFO
)
logger = logging.getLogger(__name__)

"""
    Communication utils
"""

class InputOverflowException(Exception):
    pass

class InputUnderflowException(Exception):
    pass

def to_bytes(s):
    if type(s) is bytes:
        return s
    elif type(s) is str:
        return s.encode()
    else:
        return str(s).encode()

def to_string(b):
    if type(b) is str:
        return b
    elif type(b) is bytes:
        return b.decode()
    else:
        return str(b)

def read_message(s):
    received_buffer = s.recv(4)
    if len(received_buffer) < 4:
        raise InputUnderflowException('Failed to receive data: the received length is less than 8 bytes long')
    to_receive = struct.unpack('<I', received_buffer[0:4])[0]
    received_buffer = b''
    while len(received_buffer) < to_receive:
        data = s.recv(to_receive - len(received_buffer))
        if len(data) == 0:
            raise InputUnderflowException('Failed to receive data: the pipe must have been broken')
        received_buffer += data
    return received_buffer

def send_message(s, message):
    message = binascii.unhexlify(message)
    message = to_bytes(message)
    send_buffer = struct.pack('<I', len(message)) + message
    s.sendall(send_buffer)

"""
    The server
"""

class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
    allow_reuse_address = True

class ServerHandler(socketserver.BaseRequestHandler):
    def __init__(self, request, client_address, srv):
        socketserver.BaseRequestHandler.__init__(self, request, client_address, srv)

    def handle(self):
        logger.info('Accepted  connection from %s', self.client_address[0])
        try:
            # locate the license file and extract the data
            if LIC_FILE_DATA is None:
                raise NotImplementedError('USB drive raw data extraction is not implemented yet! :-(')

            else:
                lic_file = LIC_FILE_DATA

            # get the software serial number
            req_ssn = read_message(self.request)

            # if the requested SSN is valid for the located license key give the client the KEY
            if lic_file.check_software_sn(to_string(binascii.hexlify(req_ssn))):
                logger.debug('Client "%s" has requested a key with the valid SSN: %s', self.client_address[0], req_ssn)
                send_message(self.request, lic_file.decryption_key)

            else:
                logger.debug('Client "%s" has requested a key with an invalid SSN: %s', self.client_address[0], req_ssn)

            # aborting connection is not a good idea, but for the illustrative purposes is OK
            return

        except Exception as ex:
            logger.error('Exception occurred: "%s"', ex, exc_info=True)

        finally:
            logger.info('Processed connection from "%s"' % self.client_address[0])

if __name__ == '__main__':
    # parse input arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('-p', '--port', type=int, default=45678, help='Service\'s port; default is 45678')
    parser.add_argument('--lic-file', type=str, default=None,
                        help='Path to license file; if given the USB device search is not performed')
    parser.add_argument('--usb', type=str, default=None,
                        help='Identifier in the form "bus:dev" of a USB device to be used as the dongle')
    parser.add_argument('-v', action='store_true', help='Be verbose')
    args = parser.parse_args()

    # process the input settings
    if args.lic_file:
        if not os.path.exists(args.lic_file):
            raise ValueError('License file {0} is missing'.format(args.lic_file))
        LIC_FILE_DATA = LicenceFileData.load(args.lic_file)
    USB_BUS_DEV = args.usb
    assert LIC_FILE_DATA is not None or USB_BUS_DEV is not None

    # setup logging
    if args.v:
        logger.setLevel(logging.DEBUG)

    # start serving
    address = ('0.0.0.0', args.port)
    server = ThreadedTCPServer(address, ServerHandler)
    server.serve_forever()