#!/usr/bin/env python

# encoding: utf-8

'''
 * Create File crawler
 * Created by leixu on 2018/5/10
 * IDE PyCharm
'''
import asyncio
from enum import Enum
from concurrent.futures import ProcessPoolExecutor
import aiohttp
import os
from Coli.logger import logger
import multiprocessing

total_count = 0

class CrawlerStatus(Enum):
    FREE = 0
    BUSY = 1
    NOT_AVAILABLE = 2
    WAITING = 3

class CurrentArgInfo(object):
    pass

class FetchResult(object):
    def __init__(self):
        self.is_error = False
        self.error = None
        self.content = None
        self.ok = False

    def set_expection(self, error):
        self.is_error = True
        self.error = error

    def set_text_content(self, content):
        self.ok = True
        self.content = content

class Crawler(object):
    RUNNING = 0
    FREE = 0
    TOTAL = 0
    TOTAL_REQUEST = 0

    def __init__(self, loop: asyncio.AbstractEventLoop, queue, out_queue: asyncio.Queue):
        self.loop = loop
        self.status = CrawlerStatus.FREE
        self._s_command = None
        self.TOTAL += 1
        self.async_queue = queue
        self.out_async_queue = out_queue
        self.task = None

    def send_command(self, command):
        self._s_command = command

    def start(self):

        async def watch_new_url():
            while True:
                if self.status == CrawlerStatus.FREE:
                    url, kwargs = await self.async_queue.get()
                    await self.fetch(url, kwargs)

        self.task = self.loop.create_task(watch_new_url())

    def end_fetch(self):
        self.RUNNING -= 1
        self.FREE += 1
        self.TOTAL_REQUEST += 1
        self.status = CrawlerStatus.FREE
        logger.warn("Process" + str(os.getpid()) +
                    "         CurrentRequestIndex:" + str(self.TOTAL_REQUEST))

    async def fetch(self, url, kwargs):
        self.RUNNING += 1
        self.status = CrawlerStatus.BUSY

        async def _data_fetch_task(_url, _kwargs):
            async with aiohttp.ClientSession() as session:
                async with session.get(_url) as resp:
                    fr = FetchResult()
                    content = await resp.text()
                    fr.set_text_content(content)
                    self.out_async_queue.put_nowait(fr)

        try:
            await _data_fetch_task(url, kwargs)
            self.end_fetch()
        except Exception as e:
            print(e)
            fr = FetchResult()
            fr.set_expection(e)
            self.out_async_queue.put_nowait(fr)


import numpy as np
import matplotlib.pyplot as plt

path = "https://raw.githubusercontent.com/OlafPBach/Advanced-Physics-Lab-2/main/Lab-2-Test-4.csv"
data = np.genfromtxt(path, delimiter=',', names =True,invalid_raise = False)
val1 = data['Test1']
time = data['Time1']

fig, ax = plt.subplots()
ax.plot(time*10**3,val1, color='blue')
ax.plot(0.48,3.1,"o",color='black')
ax.plot(0.86,6.3,"o",color='black')
ax.plot(1.21,9.15,"o",color='black')
ax.plot(1.58,11.5,"o",color='black')
ax.set_xlim(0,2)
ax.set_ylim(-1,13)
ax.set_xlabel('Time (ms)')
ax.set_ylabel('Amplitude (V)')
ax.set_title('Test 3')

plt.show()


from os import write
from telegram.ext.conversationhandler import ConversationHandler
import config
import queries
from googlesearch import search
import telegram
from telegram import (
	InlineQueryResultArticle,
	InputTextMessageContent,
	InlineKeyboardButton,
	InlineKeyboardMarkup,
)
from telegram.ext import (
	CommandHandler,
	MessageHandler,
	Filters,
	CallbackQueryHandler,
)

def start(update, context):
	queries.sign_up_user(update.message.from_user.id)
	help_function(update, context)

def help_function(update, context):
	phrases = queries.get_phrases(update.message.from_user.id)
	update.message.reply_text(
	text=phrases['help'])

def change_bot_lang(update, context):
	phrases = queries.get_phrases(update.message.from_user.id)
	update.message.reply_text(
	phrases['bot_lang_choise'], reply_markup = InlineKeyboardMarkup(
		[
			[
				InlineKeyboardButton('Русский',callback_data='bl;ru'),
				InlineKeyboardButton('English',callback_data='bl;en')
			]
		]
	))

def find(update, context):
	phrases = queries.get_phrases(update.message.from_user.id)
	update.message.reply_text(
	text=phrases['news_query'])
	return 1

def find_results(update, context):
	q = update.message.text
	phrases = queries.get_phrases(update.message.from_user.id)
	message = update.message.reply_text(
	text=phrases['news_query_searching'])
	print(q)
	results = []
	for url in search(q, tld='com.ru', lang=queries.get_lang(update.message.from_user.id), stop=6):
		results.append(url)
	if len(results) < 6:
		message.edit_text(phrases['news_query_not_found'])
	choice = 0
	queries.save_results(update.message.from_user.id, results)
	message.edit_text(results[choice],reply_markup = get_article_markup(results,choice))
	return ConversationHandler.END

def get_article_markup(results, choice):
	markup = InlineKeyboardMarkup(
		[
			[InlineKeyboardButton('Сохранить эту статью', callback_data=f's;{choice}')],
		[
			InlineKeyboardButton('(1)' if choice == 0 else '1', callback_data='a;0'),
			InlineKeyboardButton('(2)' if choice == 1 else '2', callback_data='a;1'),
			InlineKeyboardButton('(3)' if choice == 2 else '3', callback_data='a;2'),
		],
		[
			InlineKeyboardButton('(4)' if choice == 3 else '4', callback_data='a;3'),
			InlineKeyboardButton('(5)' if choice == 4 else '5', callback_data='a;4'),
			InlineKeyboardButton('(6)' if choice == 5 else '6', callback_data='a;5')
		]
		]
	)
	return markup

def change_article_button(update,context):
	query = update.callback_query
	query.answer()
	data = query.data.split(';')
	choice = int(data[1])
	results = queries.get_results(update.callback_query.message.chat.id)
	query.edit_message_text(results[choice], reply_markup = get_article_markup(results,choice))

def save_button(update, context):
	query = update.callback_query
	query.answer()
	phrases = queries.get_phrases(query.message.chat.id)
	data = query.data.split(';')
	choice = int(data[1])
	queries.add_article(query.message.chat.id, queries.get_results(query.message.chat.id)[choice])
	categories = queries.get_categories(query.message.chat.id)
	categories = '\n'.join([str(i)+'. '+categories[i] for i in range(len(categories))])
	if not categories:
		categories = phrases['no_categories']
	query.message.reply_text(phrases['save_article_category']+categories)
	return 1

def save_first(update, context):
	categories = queries.get_categories(update.message.from_user.id)
	phrases = queries.get_phrases(update.message.from_user.id)
	answer = update.message.text
	if answer.isnumeric():
		answer = int(answer)
		if len(categories) <= answer or answer < 0:
			categories = '\n'.join([str(i)+'. '+categories[i] for i in range(len(categories))])
			update.message.reply_text(phrases['wrong_category']+categories)
			return 1
		else:
			queries.add_article_cat(update.message.from_user.id, categories[answer])
			update.message.reply_text(phrases['save_article_name'])
			return 2
	queries.add_article_cat(update.message.from_user.id, answer)
	update.message.reply_text(phrases['save_article_name'])
	return 2

def save_second(update, context):
	phrases = queries.get_phrases(update.message.from_user.id)
	queries.add_article_name(update.message.from_user.id, update.message.text)
	update.message.reply_text(phrases['save_article_complete'])
	return ConversationHandler.END

def list_categories(update, context):
	phrases = queries.get_phrases(update.message.from_user.id)
	update.message.reply_text(
	phrases['list_categories']+'\n'.join(queries.get_categories(update.message.from_user.id)))

def list_articles(update, context):
	phrases = queries.get_phrases(update.message.from_user.id)
	categories = queries.get_categories(update.message.from_user.id)
	text = []
	for c in categories:
		articles = queries.get_articles(update.message.from_user.id, c)
		text += [c] + list(map(lambda x: x[0]+': '+x[1], articles))
	update.message.reply_text('\n'.join(text))

def rename_article(update, context):
	phrases = queries.get_phrases(update.message.from_user.id)
	update.message.reply_text(
	phrases['rename_article_choose_category']+'\n'.join(queries.get_categories(update.message.from_user.id)))
	return 1

def rename_article_cat(update, context):
	phrases = queries.get_phrases(update.message.from_user.id)
	return 2

def rename_article_name(update, context):
	return 3

def rename_article_new_name(update, context):
	return ConversationHandler.END

def rename_category(update, context):
	phrases = queries.get_phrases(update.message.from_user.id)
	return 1

def rename_category_cat(update, context):
	return 2

def rename_category_new_name(update, context):
	return ConversationHandler.END

def delete_article(update, context):
	phrases = queries.get_phrases(update.message.from_user.id)
	update.message.reply_text(
	text=phrases['indev'])

def delete_article_cat(update, context):
	return 2

def delete_article_name(update, context):
	return ConversationHandler.END

def delete_category(update, context):
	phrases = queries.get_phrases(update.message.from_user.id)
	update.message.reply_text(
	text=phrases['indev'])

def delete_category_cat(update, context):
	return ConversationHandler.END

def unknown_command(update, context):
	phrases = queries.get_phrases(update.message.from_user.id)
	update.message.reply_text(
	text=phrases['unknown'])

def stop(update, context):
	phrases = queries.get_phrases(update.message.from_user.id)
	queries.cancel_save(update.message.from_user.id)
	update.message.reply_text(
	text=phrases['stop'])
	return ConversationHandler.END

def bot_lang_button(update, context):
	query = update.callback_query
	query.answer()
	data = query.data.split(';')
	queries.change_bot_lang(query.message.chat.id, data[1])
	phrases = queries.get_phrases(query.message.chat.id)
	query.message.reply_text(phrases['bot_lang_choise_done'])

handlers = [
	ConversationHandler(
		entry_points=[CommandHandler('find', find)],
		states={
			1:[MessageHandler(Filters.text & ~Filters.command, find_results)],
		},
		fallbacks=[CommandHandler('cancel', stop)]),
	ConversationHandler(
		entry_points=[CallbackQueryHandler(save_button, pattern='^s;*', pass_user_data=True, pass_chat_data=True)],
		states={
			1:[MessageHandler(Filters.text & ~Filters.command, save_first)],
			2:[MessageHandler(Filters.text & ~Filters.command, save_second)],
		},
		fallbacks=[CommandHandler('cancel', stop)]),

	CommandHandler('start', start),
	CommandHandler('cancel', stop),
	CommandHandler('help', help_function),
	CommandHandler('botlang', change_bot_lang),
	CommandHandler('listcat', list_categories),
	CallbackQueryHandler(change_article_button,pattern='^a;*', pass_user_data=True, pass_chat_data=True),
	CallbackQueryHandler(save_button, pattern='^s;*', pass_user_data=True, pass_chat_data=True),
	CallbackQueryHandler(bot_lang_button, pattern='^bl;*'),
	CommandHandler('listart', list_articles),
	MessageHandler(Filters.command, unknown_command),
]


from fastapi import FastAPI, Depends
from fastapi.responses import RedirectResponse
from fastapi.middleware.cors import CORSMiddleware
from routers import board, resources
from docs import tags_metadata_docs, description_docs

app = FastAPI(
    openapi_tags=tags_metadata_docs,
    title="Progetto CHeArIA",
    description=description_docs,
    version="1.0",
    contact={
        "url": "https://progettochearia.it"
    },
    license_info={
        "name": "GNU General Public License v3.0",
        "url": "https://raw.githubusercontent.com/liceocremona/chearia-backend/main/LICENSE"
    }
    )

origins=["https://progettochearia.it",
 "https://web.progettochearia.it",
 "https://admin.progettochearia.it"]
app.add_middleware(
    CORSMiddleware,
    allow_origins= origins,
    allow_methods=["*"],
    allow_headers=["*"],
)

app.include_router(
    board.router,
    prefix="/board",
)
app.include_router(
    resources.router,
    prefix="/resources",
)

@app.get("/", response_class=RedirectResponse, status_code=302, include_in_schema=False)
async def root():
    return "https://api.progettochearia.it/v1/docs"


import os
import sys
import click

import pandas as pd

from mlhub.pkg import get_cmd_cwd

# Constants

ROUND = 3  # Number of decimal points to round to.

# Ensure paths are relative to the user's cwd.

os.chdir(get_cmd_cwd())

@click.command()
@click.argument('csvfile',
                type=click.File('r'),
                default=sys.stdin)
# help="the input data file, in csv format"
def cli(csvfile):

    # Quit if no data to read.

    try:
        df = pd.read_csv(csvfile)
    except pd.errors.EmptyDataError:
        click.echo("No data available.")
        sys.exit(1)

    # Apply the z-score method
    # https://towardsdatascience.com/data-normalization-with-pandas-and-scikit-learn-7c1cc6ed6475

    for column in df.columns:
        df[column] = (df[column] - df[column].mean()) / df[column].std()

    click.echo(df.round(ROUND).to_csv(index=False).strip())

if __name__ == "__main__":
    cli(prog_name="normalise")


# unicornhybridblack: classes to communicate with g.tec Unicorn Hybrid Black
#
"""
UnicornBlackThreads provides direct access to the thread functions

UnicornBlackProcess provides the ability to run the device as a seperate process

UnicornBlackCheckSignal provides a class to evaluate signal quality

Working notes
- Sometimes works when run in Spyder, but seems to work best when run in external

@author: Matt Pontifex
"""

import os
import math
import numpy
import time
from datetime import datetime
from threading import Thread, Lock
from multiprocessing import Queue
import multiprocessing
import matplotlib.mlab as mlab
import scipy.signal

# DEBUG #
if __name__ == "__main__":
    try:
        import UnicornPy as UnicornPy
    except:
        import Engine.UnicornPy as UnicornPy
else:
    try:
        import Engine.UnicornPy as UnicornPy
    except:
        import UnicornPy as UnicornPy

class UnicornBlackCheckSignal():

    def __init__(self):
        self.killcheck = multiprocessing.Event()
        self.nbchan = 8
        self.freqratio = []
        self.pointstd = []
        self.filtereddata = []
        self.psddata = []
        self.psdclean = []
        self.freqdata = []
        self._samplefreq = 250.0
        self.highpassfilter = 0.1
        self.lowpassfilter = 20
        self.notchfilter = 60
        self.scale = 500.0
        self.controlband = [20, 40]
        self.noiseband = [58, 62]
        self.data = []
        self.psdonfiltereddata = True

    def check(self):
        self.freqratio = []
        self.pointstd = []
        self.filtereddata = []
        self.psddata = []
        self.psdclean = []
        self.freqdata = []
        #start = time.perf_counter()
        for cP in range(self.nbchan):
            signalnoiseratio, signalvariability, datavector, _power, _freqs = UnicornGroomQuality(self.data[:,cP], self.controlband, self.noiseband, self._samplefreq, self.scale, self.highpassfilter, self.lowpassfilter, self.notchfilter)
            self.freqratio.append(signalnoiseratio)
            self.pointstd.append(signalvariability)
            self.filtereddata.append(datavector)
            self.psddata.append(_power)
            self.freqdata.append(_freqs)
            if self.psdonfiltereddata:
                _power, _freqs = UnicornGroomPSD(datavector, self._samplefreq, self.scale)
                self.psdclean.append(_power)
            #print('channel %d noise ratio: %0.2f' % (cP, signalnoiseratio))
            #print('channel %d variance: %0.2f' % (cP, signalvariability))
        #finish = time.perf_counter()
        #print(f'Finished in {round(finish-start,8)} seconds(s)')

def UnicornGroomPSD(datavector, samplefreq=250.0, scale=500):
    #print('UnicornGroomPSD: called')
    # function to obtain spectral power
    overlaplength = int(len(datavector)/3.0)
    if (int(scale) <= overlaplength):
        overlaplength = int(int(scale)/2.0)

    _power = []
    _freqs = []
    _power, _freqs = mlab.psd(x=datavector, NFFT=int(scale), Fs=samplefreq, noverlap=overlaplength, sides='onesided', scale_by_freq=True)
    return _power, _freqs
    #print('UnicornGroomPSD: complete')

def UnicornGroomFilter(datavector, highpassfilter=0.1, lowpassfilter=20.0, notchfilter=60.0, samplefreq=250.0):
    #print('UnicornGroomFilter: called')
    # function to return filtered data

    # Apply notch filter
    if not (float(notchfilter) == float(0.0)):
        b, a = scipy.signal.iirnotch(notchfilter, 30.0, samplefreq) # Design notch filter
        datavector = scipy.signal.filtfilt(b=b, a=a, x=datavector, padtype='constant', padlen=int(math.floor(len(datavector)/3.0)), method="pad")

    continuefilt = False
    if not (float(highpassfilter) == float(0.0)):
        if not (float(lowpassfilter) == float(0.0)):
            # band pass
            b, a = scipy.signal.iirfilter(3, [highpassfilter, lowpassfilter], btype='bandpass', ftype='butter', fs=samplefreq, output='ba')
            #sos = scipy.signal.iirfilter(3, [highpassfilter, lowpassfilter], btype='bandpass', ftype='butter', fs=samplefreq, output='sos')
            continuefilt = True
        else:
            # high pass
            b, a = scipy.signal.iirfilter(3, highpassfilter, btype='highpass', ftype='butter', fs=samplefreq, output='ba')
            #sos = scipy.signal.iirfilter(3, highpassfilter, btype='highpass', ftype='butter', fs=samplefreq, output='sos')
            continuefilt = True
    elif not (float(lowpassfilter) == float(0.0)):
        # low pass
        b, a = scipy.signal.iirfilter(3, lowpassfilter, btype='lowpass', ftype='butter', fs=samplefreq, output='ba')
        #sos = scipy.signal.iirfilter(3, lowpassfilter, btype='lowpass', ftype='butter', fs=samplefreq, output='sos')
        continuefilt = True

    if continuefilt:
        #datavector = scipy.signal.filtfilt(b=b, a=a, x=datavector, padtype='constant', padlen=int(math.floor(len(datavector)/3.0)), method="pad")
        datavector = scipy.signal.filtfilt(b=b, a=a, x=datavector, padtype=None)
        #datavector = scipy.signal.sosfilt(sos, datavector)

    return datavector
    #print('UnicornGroomFilter: complete')

def UnicornGroomQuality(datavector, controlband, noiseband, samplefreq, scale, highpassfilter, lowpassfilter, notchfilter):
    # function to evaluate quality of signal
    signalnoiseratio = []
    signalvariability = []
    datavector = numpy.ravel(datavector)
    _power, _freqs = UnicornGroomPSD(datavector, samplefreq=samplefreq, scale=scale)
    controlpower = numpy.median(_power[numpy.argmin(abs(_freqs-(controlband[0]))):numpy.argmin(abs(_freqs-(controlband[1])))])
    noisepower = numpy.mean(_power[numpy.argmin(abs(_freqs-(noiseband[0]))):numpy.argmin(abs(_freqs-(noiseband[1])))])

    if (float(controlpower) == float(0.0)):
        controlpower = 0.0001

    signalnoiseratio = numpy.around(numpy.divide(noisepower, controlpower), decimals=1) # frequency ratio

    datavector = UnicornGroomFilter(datavector, highpassfilter=highpassfilter, lowpassfilter=lowpassfilter, notchfilter=notchfilter, samplefreq=samplefreq)

    checkspan = int(math.floor((len(datavector) / 5.0))) * -1
    signalvariability = numpy.std(datavector[checkspan:-1])
    return signalnoiseratio, signalvariability, datavector, _power, _freqs

def UnicornJockey(deviceID, channellabels, rollingspan, logfilename, printoutput, startrecordingeeg, eegready, eegrecording, safetologevent, markeeg, markvalue, pulleegdata, conn, stoprecordingeeg):
    # this is the function that gets pushed to a seperate process that actually controls the device

    startedrecording = False

    # connect device
    UnicornBlack = UnicornBlackThreads()
    UnicornBlack.channellabels = channellabels # change channel labels
    UnicornBlack.printoutput = printoutput
    UnicornBlack.connect(deviceID=deviceID, rollingspan=rollingspan, logfilename=logfilename)
    eegready.set()

    continueroutine = True
    while continueroutine:

        if markeeg.is_set():
            UnicornBlack.mark_event(markvalue.value) # Send trigger
            markeeg.clear()

        if not startedrecording:
            if startrecordingeeg.is_set():
                UnicornBlack.startrecording() # rename file name here
                startedrecording = True
                eegrecording.set()

        if startedrecording:
            if stoprecordingeeg.is_set():
                UnicornBlack._safetolog = True # push any remaining items to file
                UnicornBlack.disconnect()
                continueroutine = False

        if safetologevent.is_set():
            UnicornBlack.safe_to_log(True)
        else:
            UnicornBlack.safe_to_log(False)

        if pulleegdata.is_set():
            sample = UnicornBlack.sample_data()
            #print('UnicornJockey: Battery at %0.1f percent' % numpy.array(sample)[-1,-3])
            if len(sample) > 0:
                conn.send(sample[:])
                pulleegdata.clear()

class UnicornBlackProcess():
    # this will be the class that the user interfaces with that intializes and maintains the multiprocessing

    def __init__(self):
        self.channellabels = 'FZ, C3, CZ, C4, PZ, O1, OZ, O2, AccelX, AccelY, AccelZ, GyroX, GyroY, GyroZ, Battery, Sample'
        self.samplefreq = 250.0
        self.numberOfAcquiredChannels = 17
        self.ready = False
        self.recording = False
        self.printoutput = False

    def connect(self, deviceID=None, rollingspan=3.0, logfilename='default'):
        # because of the multiprocessing, to not create additional headaches, the file name needs to be initiallized at connect

        self.deviceID = deviceID
        self.rollingspan = rollingspan
        self.logfilename = logfilename

        # connect to Device
        self.startrecordingeeg = multiprocessing.Event()
        self.stoprecordingeeg = multiprocessing.Event()
        self.eegready = multiprocessing.Event()
        self.eegrecording = multiprocessing.Event()
        self.safetologevent = multiprocessing.Event()
        self.markeeg = multiprocessing.Event()
        self.markvalue = multiprocessing.Value('i', 0)

        # code to be able to pull data
        self.pulleegdata = multiprocessing.Event()
        self.pulleegdata1, self.pulleegdata2 = multiprocessing.Pipe()

        self.p = multiprocessing.Process(target=UnicornJockey, args=[self.deviceID, self.channellabels, self.rollingspan, self.logfilename, self.printoutput, self.startrecordingeeg, self.eegready, self.eegrecording, self.safetologevent, self.markeeg, self.markvalue, self.pulleegdata, self.pulleegdata2, self.stoprecordingeeg])
        self.p.start()
        self.eegready.wait(3.0) # wait up to 3 seconds
        self.ready = True

    def startrecording(self):
        self.startrecordingeeg.set()
        self.eegrecording.wait(3.0) # wait up to 3 seconds
        self.recording = True

    def disconnect(self):
        self.stoprecordingeeg.set()
        self.p.join()
        self.pulleegdata1.close()

    def mark_event(self, event):
        self.markvalue.value = event
        self.markeeg.set()

    def safe_to_log(self, boolsafe=True):
        """Parameter to change logging settings
        """
        if boolsafe:
            self.safetologevent.set()
        else:
            self.safetologevent.clear()

    def sample_data(self):
        #t = time.perf_counter()
        self.pulleegdata.set() # tell process to obtain a sample
        data = self.pulleegdata1.recv() # takes about 20 ms
        #print(time.perf_counter() - t)
        #tempdata = numpy.array(data)
        #print('UnicornBlackProcess: Battery at %0.1f percent' % tempdata[-1,-3])
        return data

    def check_battery(self):
        _plottingdata = numpy.array(self.sample_data(), copy=True)
        powerlevel = 0
        try:
            powerlevel = int(_plottingdata[-1,-3]) # update power level
        except:
            powerlevel = 0
        if self.printoutput:
            print('Device battery at %d percent.' % powerlevel)
        return powerlevel

class UnicornBlackThreads():
    """class for data collection using the g.tec Unicorn Hybrid Black
	"""

    def __init__(self):

        # establish variables
        self.collectversion = '2020.05.30.0'
        self.device = None;
        self.path = os.path.dirname(os.getcwd())
        self.outputfolder = 'Raw'

        # create a Queue and Lock
        self._queue = Queue()
        self._queuelock = Lock()
        self._logsamplequeue = Queue()
        self._logeventqueue = Queue()
        self._bufferlock = Lock()
        self._loglock = Lock()
        self._logeventlock = Lock()

        # initialize data collectors
        self.logdata = False
        self.logfilename = None
        self._timetemp = None
        self._safetolog = False
        self._dataheaderlog = False

        # establish parameters
        self._configuration = None
        self._numberOfAcquiredChannels = None
        self._frameLength = 1
        self._samplefreq = 250.0
        self._intsampletime = 1.0 / self._samplefreq / 10
        self.channellabels = 'FZ, C3, CZ, C4, PZ, O1, OZ, O2, AccelX, AccelY, AccelZ, GyroX, GyroY, GyroZ, Battery, Sample'

        # activate
        self._receiveBuffer = None
        self.lastsampledpoint = None
        self.data = None
        self.printoutput = True
        self.ready = True
        self.deviceconnected = False

    def connect(self, deviceID=None, rollingspan=3.0, logfilename='default'):

        self.deviceID = deviceID;
        self.logfilename = logfilename
        # make sure everything is disconnected
        try:
            self.device.StopAcquisition()
        except:
            pass
        del self.device
        self.device = None
        # Did the user specify a particular device
        if (self.deviceID is None):
            try:
                # Get available device serials.
                deviceList = UnicornPy.GetAvailableDevices(True)

                if len(deviceList) <= 0 or deviceList is None:
                    raise Exception("No device available.Please pair with a Unicorn first.")

                # Print available device serials.
                print("Available devices:")
                i = 0
                for device in deviceList:
                    print("#%i %s" % (i,device))
                    i+=1

                # Request device selection.
                deviceID = int(input("Select device by ID #"))
                if deviceID < 0 or deviceID > len(deviceList):
                    raise IndexError('The selected device ID is not valid.')
                self.deviceID = deviceList[deviceID]

            except UnicornPy.DeviceException as e:
                print(e)
            except Exception as e:
                print("An unknown error occured. %s" %e)

        # Open selected device.
        try:
            self.deviceconnected = 0
            try:
                self.device = UnicornPy.Unicorn(self.deviceID)
                self.deviceconnected = 1
            except:
                pass

            if self.deviceconnected == 1:
                # Initialize acquisition members.
                self._rollingspan = rollingspan # seconds
                self._logchunksize = int(math.floor(5 * self._samplefreq))

                self._numberOfAcquiredChannels = self.device.GetNumberOfAcquiredChannels()
                self._configuration = self.device.GetConfiguration()

                # Allocate memory for the acquisition buffer.
                self._receiveBufferBufferLength = self._frameLength * self._numberOfAcquiredChannels * 4
                self._receiveBuffer = bytearray(self._receiveBufferBufferLength)
                self.data = [[0.0] * self._numberOfAcquiredChannels] * math.floor( float(self._rollingspan) * float(self._samplefreq) )

                try:
                    # initialize sample streamer
                    self._streaming = True
                    self._ssthread = Thread(target=self._stream_samples, args=[self._logsamplequeue], daemon=True)
                    self._ssthread.name = 'samplestreamer'
                except:
                    print("Error initializing sample streamer.")

                try:
                    # initialize data recorder
                    self._recording = True
                    self._drthread = Thread(target=self._log_sample, args=[self._logsamplequeue], daemon=True)
                    self._drthread.name = 'datarecorder'

                except:
                    print("Error initializing data recorder.")

                try:
                    # initialize event recorder
                    self._eventrecording = True
                    self._erthread = Thread(target=self._log_event, args=[self._logeventqueue], daemon=True)
                    self._erthread.name = 'eventrecorder'

                except:
                    print("Error initializing event recorder.")

                try:
                    # start processes
                    self.device.StartAcquisition(False)  # True - test signal; False - measurement mode
                except:
                    print("Error starting acquisition.")

                self.logdata = False
                self._ssthread.start()
                self._drthread.start()
                self._erthread.start()

                time.sleep(1) # give it some initialization time

                if self.printoutput:
                    print("Connected to '%s'." %self.deviceID)
            else:
                if self.printoutput:
                    print("Unable to connect to '%s'." %self.deviceID)
        except:
            if self.printoutput:
                print("Unable to connect to '%s'." %self.deviceID)

    def disconnect(self):

        time.sleep((self._intsampletime * float(10)))

        #self.stoprecording()
        self._streaming = False
        try:
            self._ssthread.join()
            self._drthread.join()
            self._erthread.join()
        except:
            pass

        try:
            self.device.StopAcquisition()
        except:
            pass

        try:
            self._logfile.close()
        except:
            pass

        del self.device
        self.device = None
        if self.printoutput:
            print("Disconnected from '%s'." %self.deviceID)

    def _stream_samples(self, queue):
        """Continuously polls the device, and puts all new samples in a
		Queue instance

		arguments

		queue		--	a multithreading.Queue instance, to put samples
						into
		"""

        # keep streaming until it is signalled that we should stop
        while self._streaming:
            boolgetdata = False
            self._bufferlock.acquire(True)
            try:
                #sampledata = [numpy.ndarray.tolist(numpy.multiply(numpy.random.rand(int(self._numberOfAcquiredChannels)), 100))]
                # Receives the configured number of samples from the Unicorn device and writes it to the acquisition buffer.
                self.device.GetData(self._frameLength,self._receiveBuffer,self._receiveBufferBufferLength)
                # Convert receive buffer to numpy float array
                sampledata = numpy.frombuffer(self._receiveBuffer, dtype=numpy.float32, count=self._numberOfAcquiredChannels * self._frameLength)
                sampledata = numpy.reshape(sampledata, (self._frameLength, self._numberOfAcquiredChannels))
                boolgetdata = True
            except:
                self._receiveBufferBufferLength = self._frameLength * self._numberOfAcquiredChannels * 4
                self._receiveBuffer = bytearray(self._receiveBufferBufferLength)
                if self.printoutput:
                    print('\n\nOverflow error in polling device.\n\n')
            self._bufferlock.release()

            if boolgetdata:
                self._queuelock.acquire(True)
                if not (str(int(float(self.data[-1][15]))) == str(int(float(sampledata[0][15])))): # protect against sampling the same point twice
                    #self.lastsampledpoint = copy.deepcopy(sampledata[0][15])
                    self._logeventlock.acquire(True)
                    self.lastsampledpoint = str(int(float(sampledata[0][15])))
                    self._logeventlock.release()
                    queue.put(sampledata)
                    self.data.append(sampledata[0])
                    self.data.pop(0)
                self._queuelock.release()

                self._bufferlock.acquire(True)
                self._receiveBuffer = bytearray(self._receiveBufferBufferLength) # make sure everything is cleared
                self._bufferlock.release()

            #time.sleep(float(0.000000000001))
            # if it is in, then we suddenly start getting blocking and drop samples
            # if we comment it out then we get all the samples, but run into an issue with screen flips

        self._queuelock.acquire(True)
        queue.put(None) # poison pill approach
        self._queuelock.release()
        self._eventrecording = False

    def _log_sample(self, logqueue):
        """Continuously log samples

		queue		--	a multithreading.Queue instance, to read samples
						from
		"""

        templogholding = None
        finalpush = False
        # keep trying to log until it is signalled that we should stop
        while self._recording:
            # read new item from the queue
            while not logqueue.empty():
                self._loglock.acquire(True)
                sampledata = logqueue.get()
                self._loglock.release()

                if sampledata is None:
                    # Poison pill means shutdown
                    self._recording = False
                    finalpush = True
                    break
                    break
                else:
                    if self.logdata:
                        if templogholding is None:
                            templogholding = sampledata[:,0:-1]
                        else:
                            templogholding = numpy.vstack([templogholding, sampledata[:,0:-1]])

                        # check if it is safe to log
                        if self._safetolog:
                            # only write chunks of data to save I/O overhead
                            if (len(templogholding) >= self._logchunksize):
                                numpy.savetxt(self._logfile,templogholding,delimiter=',',fmt='%.3f',newline='\n')
                                self._logfile.flush() # internal buffer to RAM
                                os.fsync(self._logfile.fileno()) # RAM file cache to disk
                                templogholding = None
        if finalpush:
            if templogholding is not None:
                numpy.savetxt(self._logfile,templogholding,delimiter=',',fmt='%.3f',newline='\n')
                self._logfile.flush() # internal buffer to RAM
                os.fsync(self._logfile.fileno()) # RAM file cache to disk
                templogholding = None
                self._logfile.close()

    def _log_event(self, logeventqueue):
        """Continuously log events

		logeventqueue	--	a multithreading.Queue instance, to read samples
						from
		"""
        eventheaderlog = False
        templogholding = None
        # keep trying to log until it is signalled that we should stop
        while self._eventrecording:
            # read new item from the queue
            while not logeventqueue.empty():
                self._logeventlock.acquire(True)
                sampledata = logeventqueue.get()
                self._logeventlock.release()

                # wait to create file until we know there is a need
                if self.logdata:
                    if not eventheaderlog:
                        header = 'collect.....= UnicornPy_' + self.collectversion + '\n'
                        header = header + 'date........= ' + self._timetemp  + '\n'
                        header = header + 'filename....= ' + self.logfilename  + '\n'
                        header = header + 'Latency, Event' + '\n'
                        self._eventlogfile = open('%s.csve' % (self.logfilename), 'w')
                        self._eventlogfile.write(header) # to internal buffer
                        self._eventlogfile.flush() # internal buffer to RAM
                        os.fsync(self._eventlogfile.fileno()) # RAM file cache to disk
                        eventheaderlog = True

                if sampledata is None:
                    # Poison pill means shutdown
                    self._eventrecording = False
                    break
                    break
                else:
                    if self.logdata:
                        if templogholding is None:
                            templogholding = sampledata
                        else:
                            templogholding = numpy.vstack([templogholding, sampledata])

                        # check if it is safe to log
                        if self._safetolog:
                            # only write chunks of data to save I/O overhead
                            if (len(templogholding) >= self._logchunksize):
                                numpy.savetxt(self._eventlogfile,templogholding,delimiter=',',fmt='%.3f',newline='\n')
                                self._eventlogfile.flush() # internal buffer to RAM
                                os.fsync(self._eventlogfile.fileno()) # RAM file cache to disk
                                templogholding = None

        if templogholding is not None:
                numpy.savetxt(self._eventlogfile,templogholding,delimiter=',',fmt='%s',newline='\n')
                self._eventlogfile.flush() # internal buffer to RAM
                os.fsync(self._eventlogfile.fileno()) # RAM file cache to disk
                templogholding = None
                self._eventlogfile.close()

    def startrecording(self):

        self.logdata = True
        timetemp = str(datetime.now()).split()
        self._timetemp = timetemp[0] + 'T' + timetemp[1]
        self._logfile = open('%s.csv' % (self.logfilename), 'w')
        self._safetolog = True
        self._log_header()

        # ensure we are getting data
        nc = 0
        while (str(int(float(self.data[-1][15]))) == str(int(float(0.0)))):
            nc = nc + 1

        if self.printoutput:
            print("Starting Recording")

    def _log_header(self):
        """Logs a header to the data file
        """
        if not self._dataheaderlog:
            header = 'collect.....= UnicornPy_' + self.collectversion + '\n'
            header = header + 'device......= ' + self.deviceID  + '\n'
            header = header + ('samplerate..= %.3f' % self._samplefreq)  + '\n'
            header = header + ('channels....= %d' % (self._numberOfAcquiredChannels - 1))  + '\n'
            header = header + 'date........= ' + self._timetemp  + '\n'
            header = header + 'filename....= ' + self.logfilename  + '\n'

            header = header + self.channellabels + '\n'
            #'FZ, C3, CZ, C4, PZ, O1, OZ, O2, AccelX, AccelY, AccelZ, GyroX, GyroY, GyroZ, Battery, Sample'
            self._logfile.write(header) # to internal buffer
            self._logfile.flush() # internal buffer to RAM
            os.fsync(self._logfile.fileno()) # RAM file cache to disk
            self._dataheaderlog = True

    def mark_event(self, event):
        """Logs data to the event file
        """
        if self.lastsampledpoint is not None:
            self._logeventlock.acquire(True)
            self._logeventqueue.put(numpy.array([str(self.lastsampledpoint), str(event)]))
            self._logeventlock.release()

    def safe_to_log(self, boolsafe=True):
        """Parameter to change logging settings
        """
        self._safetolog = boolsafe

    def sample_data(self):
        try:
            datasample = self.data[:]
        except:
            datasample = []
        return datasample

    def check_battery(self):
        _plottingdata = numpy.array(self.sample_data(), copy=True)
        powerlevel = 0
        try:
            powerlevel = int(_plottingdata[-1,-3]) # update power level
        except:
            powerlevel = 0

        if self.printoutput:
            print('Device battery at %d percent.' % powerlevel)
        return powerlevel

# # # # #
# DEBUG #
if __name__ == "__main__":

    duration = 5 # seconds
    example = 'Thread' # Process or Thread

    if example == 'Thread':
        # Example code for calling the Unicorn device as a process
        UnicornBlack = UnicornBlackProcess()
    else:
        # Example code for calling the Unicorn device as a thread
        UnicornBlack = UnicornBlackThreads()

    # all other functions and calls remain the same
    UnicornBlack.connect(deviceID='UN-2019.05.51', rollingspan=3, logfilename='recordeddata_thread')

    UnicornBlack.startrecording()

    for incrX in range(duration):
        time.sleep(1)
        UnicornBlack.safe_to_log(False)
        UnicornBlack.mark_event(incrX)
        print("Time Lapsed: %d second" % (incrX+1))
        UnicornBlack.safe_to_log(True)

    example = numpy.array(UnicornBlack.sample_data())
    print('Battery at %0.1f percent' % example[-1,-3])
    UnicornBlack.disconnect()

    # Plotting function to check for dropped samples
    #
    #selectsampleddata = []
    #for incrX in range(len(sampleddata)):
    #    if not (sampleddata[incrX][0] == float(0)):
    #        selectsampleddata.append(sampleddata[incrX])
    #sampleddata = numpy.array(selectsampleddata)
    #del selectsampleddata

    #if (sampleddata.shape[0]) > 2:
    #    x = numpy.arange(sampleddata[0][15],sampleddata[-1][15],1)
    #    y = numpy.array([0] * len(x))
    #    for incrX in range(len(sampleddata)):
    #        index_min = numpy.argmin(abs(x-(sampleddata[incrX][15])))
    #        #y[index_min] = sampleddata[incrX][15]
    #        y[index_min] = 1
    #    del index_min, incrX
    #    x = x * (1/ 250.0)
    #    plt.plot(x,y)
    #    print('The streamer had a total of %d dropped samples (%0.1f%%).' %(len(y)-numpy.count_nonzero(y), ((len(y)-numpy.count_nonzero(y))/len(y))*100))

    # show saved data
    #lis = []
    #with open("C:\\Studies\\Python Collect\\Gentask\\Raw\\recordeddata.csv", "rb") as f:
    #    for cnt, line in enumerate(f):
    #        lis.append(line.decode("utf-8").split(','))
    #f.close()
    #del cnt, line
    #lis = lis[7:-1]
    #x2 = numpy.arange(int(float(lis[0][15])),int(float(lis[-1][15])),1)
    #y2 = numpy.array([0] * len(x2))
    #for incrX in range(len(lis)):
    #    index_min = numpy.argmin(abs(x2-(int(float(lis[incrX][15])))))
    #    y2[index_min] = 1
    #del index_min, incrX

    # read in event markers
    #eventlis = []
    #with open("C:\\Studies\\Python Collect\\Gentask\\Raw\\recordeddata.csve", "rb") as f:
    #    for cnt, line in enumerate(f):
    #        eventlis.append(line.decode("utf-8").split(','))
    #f.close()
    #del cnt, line
    #eventlis = eventlis[4:-1]

    #y3 = numpy.array([0] * len(x2))
    #for incrX in range(len(eventlis)):
    #    index_min = numpy.argmin(abs(x2-(int(float(eventlis[incrX][0])))))
    #    y3[index_min] = 2
    #del index_min, incrX
    #
    # x2 = x2 * (1/ 250.0)
    #plt.plot(x2,y2)
    #plt.plot(x2,y3)
    #print('The recording had a total of %d dropped samples (%0.1f%%).' %(len(y2)-numpy.count_nonzero(y2), ((len(y2)-numpy.count_nonzero(y2))/len(y2))*100))

    # core.wait was responsible for the dropped samples!

# # # # #


# coding: utf-8

# In[1]:

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings("ignore")

house = pd.read_csv("train.csv")
houseTest = pd.read_csv("test.csv")

Id = houseTest["id"]
houseTest = houseTest.drop(["id"],axis=1)

# house
print("-"*50+ "train简单查看统计学信息" +"-"*50)
print(house.info())
print("-"*50+ "test简单查看统计学信息" +"-"*50)
print(houseTest.info())

print("-"*50 + "train查看缺失值" + "-"*50)
print(house.isnull().sum().sort_values(ascending=False).head(10))
print("-"*50 + "test查看缺失值" + "-"*50)
print(houseTest.isnull().sum().sort_values(ascending=False).head(10))

# In[2]:

house["位置"][house["位置"].isnull()] = house["位置"].dropna().mode().values
house["区"][house["区"].isnull()] = house["区"].dropna().mode().values
house["小区房屋出租数量"][house["小区房屋出租数量"].isnull()] = house["小区房屋出租数量"].dropna().mode().values
houseTest["位置"][houseTest["位置"].isnull()] = houseTest["位置"].dropna().mode().values
houseTest["区"][houseTest["区"].isnull()] = houseTest["区"].dropna().mode().values
houseTest["小区房屋出租数量"][houseTest["小区房屋出租数量"].isnull()] = houseTest["小区房屋出租数量"].dropna().mode().values

print("-"*50 + "train查看缺失值" + "-"*50)
print(house.isnull().sum().sort_values(ascending=False).head(10))
print("-"*50 + "test查看缺失值" + "-"*50)
print(houseTest.isnull().sum().sort_values(ascending=False).head(10))

# In[3]:

# 装修情况肯定对租金价格有较大影响，我们现在有两万的装修数据，可以尝试下预测
# 居住状态不太好预测，这个属于人为因素，但是应该有人居住的房子一定租金较低
# 出租方式，妥妥的有影响，中介收费不一，朋友介绍肯定便宜
# 距离应该是地铁站点到房子的距离，交通的影响
# 地铁站这个不太好预测，因为住房附近可能压根没地铁
# 小区出租数量，位置，区 可以直接干掉

# In[4]:

# 特征工程

# 房屋朝向
print("-"*50+"house房屋朝向"+"-"*50)
house_orientation_unique = house["房屋朝向"].unique()
print(house["房屋朝向"].unique())
# print(house["房屋朝向"].value_counts())
print("-"*50+"houseTest房屋朝向"+"-"*50)
houseTest_orientation_unique = houseTest["房屋朝向"].unique()
print(houseTest["房屋朝向"].unique())
# print(houseTest["房屋朝向"].value_counts())
print()
print("house 有 houseTest没有:")
print ([i for i in house_orientation_unique if i not in houseTest_orientation_unique])
print("house 没有 houseTest有:")
print ([i for i in houseTest_orientation_unique if i not in house_orientation_unique])
print("house 有 houseTest有:")
print ([i for i in houseTest_orientation_unique if i in house_orientation_unique])

house_orientation = pd.DataFrame({'东':[0]*196539,'南':[0]*196539,
                                 '西':[0]*196539,'北':[0]*196539,
                                 '东南':[0]*196539,'东北':[0]*196539,
                                 '西南':[0]*196539,'西北':[0]*196539,})
houseTest_orientation = pd.DataFrame({'东':[0]*196539,'南':[0]*196539,
                                 '西':[0]*196539,'北':[0]*196539,
                                 '东南':[0]*196539,'东北':[0]*196539,
                                 '西南':[0]*196539,'西北':[0]*196539,})
house = house.join(house_orientation)
# house

# 将房屋朝向整成八个特征
def orientation_dong(x):
    lst = []
    for i in x.split(" "):
        lst.append(i)
    dong = "东"
    if dong in lst:
        return 1
    else:
        return 0

def orientation_nan(x):
    lst = []
    for i in x.split(" "):
        lst.append(i)
    nan = "南"
    if nan in lst:
        return 1
    else:
        return 0

def orientation_xi(x):
    lst = []
    for i in x.split(" "):
        lst.append(i)
    xi = "西"
    if xi in lst:
        return 1
    else:
        return 0

def orientation_bei(x):
    lst = []
    for i in x.split(" "):
        lst.append(i)
    bei = "北"
    if bei in lst:
        return 1
    else:
        return 0

def orientation_dongNan(x):
    lst = []
    for i in x.split(" "):
        lst.append(i)
    dongNan = "东南"
    if dongNan in lst:
        return 1
    else:
        return 0

def orientation_dongBei(x):
    lst = []
    for i in x.split(" "):
        lst.append(i)
    dongBei = "东北"
    if dongBei in lst:
        return 1
    else:
        return 0

def orientation_xiNan(x):
    lst = []
    for i in x.split(" "):
        lst.append(i)
    xiNan = "西南"
    if xiNan in lst:
        return 1
    else:
        return 0

def orientation_xiBei(x):
    lst = []
    for i in x.split(" "):
        lst.append(i)
    xiBei = "西北"
    if xiBei in lst:
        return 1
    else:
        return 0

house['东'] = house['房屋朝向'].apply(lambda x : orientation_dong(x) )
house['南'] = house['房屋朝向'].apply(lambda x : orientation_nan(x) )
house['西'] = house['房屋朝向'].apply(lambda x : orientation_xi(x) )
house['北'] = house['房屋朝向'].apply(lambda x : orientation_bei(x) )
house['东南'] = house['房屋朝向'].apply(lambda x : orientation_dongNan(x) )
house['东北'] = house['房屋朝向'].apply(lambda x : orientation_dongBei(x) )
house['西南'] = house['房屋朝向'].apply(lambda x : orientation_xiNan(x) )
house['西北'] = house['房屋朝向'].apply(lambda x : orientation_xiBei(x) )

houseTest['东'] = houseTest['房屋朝向'].apply(lambda x : orientation_dong(x) )
houseTest['南'] = houseTest['房屋朝向'].apply(lambda x : orientation_nan(x) )
houseTest['西'] = houseTest['房屋朝向'].apply(lambda x : orientation_xi(x) )
houseTest['北'] = houseTest['房屋朝向'].apply(lambda x : orientation_bei(x) )
houseTest['东南'] = houseTest['房屋朝向'].apply(lambda x : orientation_dongNan(x) )
houseTest['东北'] = houseTest['房屋朝向'].apply(lambda x : orientation_dongBei(x) )
houseTest['西南'] = houseTest['房屋朝向'].apply(lambda x : orientation_xiNan(x) )
houseTest['西北'] = houseTest['房屋朝向'].apply(lambda x : orientation_xiBei(x) )

house = house.drop("房屋朝向",axis=1)
houseTest = houseTest.drop("房屋朝向",axis=1)
# house

# In[5]:

print("-"*50 + "train查看缺失值" + "-"*50)
print(house.isnull().sum().sort_values(ascending=False).head(10))
print("-"*50 + "test查看缺失值" + "-"*50)
print(houseTest.isnull().sum().sort_values(ascending=False).head(10))

# In[6]:

# house['装修情况'] = house['装修情况'].fillna(house['装修情况'].mean())
# house['居住状态'] = house['居住状态'].fillna(house['居住状态'].mean())
# house['出租方式'] = house['出租方式'].fillna(house['出租方式'].mean())
# house['距离'] = house['距离'].fillna(0)
# house['地铁站点'] = house['地铁站点'].fillna(0)
# house['地铁线路'] = house['地铁线路'].fillna(0)

# houseTest['装修情况'] = houseTest['装修情况'].fillna(houseTest['装修情况'].mean())
# houseTest['居住状态'] = houseTest['居住状态'].fillna(houseTest['居住状态'].mean())
# houseTest['出租方式'] = houseTest['出租方式'].fillna(houseTest['出租方式'].mean())
# houseTest['距离'] = houseTest['距离'].fillna(0)
# houseTest['地铁站点'] = houseTest['地铁站点'].fillna(0)
# houseTest['地铁线路'] = houseTest['地铁线路'].fillna(0)

# In[7]:

from xgboost import XGBRegressor
from sklearn.model_selection import train_test_split
from sklearn.ensemble import AdaBoostRegressor
from sklearn import model_selection

# In[8]:

# house.iloc[:,1:5]

# In[9]:

# 特征工程：创建深度特征
import featuretools as ft

x = house.drop(["月租金","装修情况","居住状态"],axis=1)
y = house.月租金
x_train,x_test, y_train,y_test = train_test_split(x,y,test_size=0.2, random_state=10)

# In[10]:

from lightgbm import LGBMRegressor

# In[11]:

clf_est =LGBMRegressor(random_state=1314)

clf_param_grid = {'n_estimators': [1000],
                  'learning_rate': [0.25], #0.25
                 'max_depth':[30],
                 'bagging_fraction':[0.6],
                  'min_child_weight':[7,8,9],
                  'max_bin':[200],
                  'num_leaves':[200],
                  'alpha':[0.04] #0.06

                 }

clf_grid = model_selection.GridSearchCV(clf_est, clf_param_grid, n_jobs=4, cv=3, verbose=1)
print("fitting")
clf_grid.fit(x_train, y_train)

print('BestParams: ' + str(clf_grid.best_params_))
print('Training:')
clf = LGBMRegressor(n_estimators = clf_grid.best_estimator_.n_estimators,
                   learning_rate = clf_grid.best_estimator_.learning_rate,
                    max_depth = clf_grid.best_estimator_.max_depth,
                    bagging_fraction = clf_grid.best_estimator_.bagging_fraction,
                    min_child_weight = clf_grid.best_estimator_.min_child_weight,
                    max_bin = clf_grid.best_estimator_.max_bin,
                    num_leaves = clf_grid.best_estimator_.num_leaves,
                    alpha = clf_grid.best_estimator_.alpha,
                   random_state=1314)
clf.fit(x_train, y_train)
print('Training Finished')

# In[13]:

houseTest = houseTest.drop(["装修情况","居住状态"],axis=1)

predictions = clf.predict(houseTest.astype(float))
Submission = pd.DataFrame({'id': Id,
                           'price': predictions})
Submission.to_csv('SubmissionLightGBM.csv',index=False,sep=',')
print("I LOVE YOUU")

print("评分：")
from sklearn.model_selection import cross_val_score

scores = cross_val_score(clf,x_test,y_test)
print("Boost: ",scores.mean())
from sklearn import metrics

y_pred_xgb = clf.predict(x_test)
# 均方根误差
print(metrics.mean_squared_error(y_test,y_pred_xgb))

# In[16]:

from sklearn.externals import joblib

clf = joblib.load('clf2.09.pkl')    # 下载模型
print(clf)

print("评分：")
from sklearn.model_selection import cross_val_score

scores = cross_val_score(clf,x_test,y_test)
print("Boost: ",scores.mean())

# In[ ]:

from sklearn.externals import joblib

# joblib.dump(clf, 'clf2.09.pkl')       # 保存模型，需先建立saved_model文件夹

# clf = joblib.load('clf2.09.pkl')    # 下载模型
# print(clf)

# In[ ]:

# # time_df = house.时间
# # communityName_df = house.小区名
# # communityHouseNumber_df = house.小区房屋出租数量
# # floor_df = house.楼层
# # totalFloor_df = house.总楼层
# houseArea_df = pd.DataFrame(house.房屋面积)
# livingState_df = pd.DataFrame(house.居住状态)
# bedroom_df = pd.DataFrame(house.卧室数量)
# sittingRoom_df = pd.DataFrame(house.厅的数量)
# bathroom_df = pd.DataFrame(house.卫的数量)
# # rentMethod_df = house.出租方式
# # community_df = house.区
# # location_df = house.位置
# # subwayLine_df = house.地铁线路
# # subwayStation_df = house.地铁站点
# # distance_df = house.距离
# renovationCondition_df =pd.DataFrame(house.装修情况)
# # east_df = house['东']
# # south_df = house['南']
# # west_df = house['西']
# # north_df = house['北']
# # eastsouth_df = house['东南']
# # eastnorth_df = house['东北']
# # westsouth_df = house['西南']
# # westnorth_df = house['西北']
# #创建实体
# es = ft.EntitySet(id = 'communityFare')

# es = es.entity_from_dataframe(entity_id = 'whole',
#                               dataframe = house,
#                               make_index = True,
#                               index = 'whole_id' )

# #添加time实体
# es = es.entity_from_dataframe(entity_id = 'time',
#                               dataframe = house,
#                               make_index = True,
#                               index = 'time_id' )
# #添加communityName实体
# es = es.entity_from_dataframe(entity_id = 'community',
#                               dataframe = house.iloc[:,0:5],
#                               make_index = True,
#                               index = 'community_id
# #添加time实体
# es = es.entity_from_dataframe(entity_id = 'houseArea',
#                               dataframe = houseArea_df,
#                               index = '房屋面积' )
# #添加time实体
# es = es.entity_from_dataframe(entity_id = 'livingState',
#                               dataframe = livingState_df,
#                               make_index = True,
#                               index = 'livingState_id' )
# #添加time实体
# es = es.entity_from_dataframe(entity_id = 'bedroom',
#                               dataframe = bedroom_id,
#                               make_index = True,
#                               index = 'bedroom_id' )
# #添加time实体
# es = es.entity_from_dataframe(entity_id = 'sittingRoom',
#                               dataframe = sittingRoom_id,
#                               make_index = True,
#                               index = 'sittingRoom_id' )
# #添加time实体
# es = es.entity_from_dataframe(entity_id = 'bathroom',
#                               dataframe = bathroom_id,
#                               make_index = True,
#                               index= 'bathroom_id' )

# #添加subway实体
# es = es.entity_from_dataframe(entity_id = 'subway',
#                               dataframe = subway_df,
#                               make_index = True,
#                               index = 'subway_id' )

# #添加time实体
# es = es.entity_from_dataframe(entity_id = 'renovationCondition',
#                               dataframe = house,
#                               make_index = True,
#                               index = 'renovationCondition_id' )

# # 添加实体关系
# r_time = ft.Relationship(es['community']['community_id'],
#                          es['time']['time_id'])
# es = es.add_relationship(r_time)

# # r_time = ft.Relationship(es['houseArea']['houseArea_id'],
# #                         es['bedroom']['bedroom_id'])
# # es = es.add_relationship(r_time)

# # r_time = ft.Relationship(es['houseArea']['houseArea_id'],
# #                         es['sittingRoom']['sittingRoom_id'])
# # es = es.add_relationship(r_time)

# # r_time = ft.Relationship(es['houseArea']['houseArea_id'],
# #                         es['bathroom']['bathroom_id'])
# # es = es.add_relationship(r_time)

# # r_time = ft.Relationship(es['community']['community_id'],
# #                         es['subway']['subway_id'])
# # es = es.add_relationship(r_time)

# # 打印实体集
# es

# #聚合特征,并生成新特征
# features, feature_names = ft.dfs(entityset = es, target_entity = 'whole')
# features.head()
# # feature_names
# # feature_matrix_communityName, features_defs_communityName = ft.dfs(entities=entities,
# #                                                                    relationships=relationships,
# #                                                                    target_entity="communityName")
# # feature_matrix_community, features_defs_community = ft.dfs(entities=entities,
# #                                                            relationships=relationships,
# #                                                            target_entity="community")
# # feature_matrix_houseArea, features_defs_houseArea = ft.dfs(entities=entities,
# #                                                            relationships=relationships,
# #                                                            target_entity="houseArea")
# # feature_matrix_subwayLine, features_defs_subwayLine = ft.dfs(entities=entities,
# #                                                              relationships=relationships,
# #                                                              target_entity="subwayLine")
# # feature_matrix_communityName


import os

import pytest
from playwright.sync_api import Page

@pytest.fixture()
def authenticated_page(page) -> Page:
    page.goto("/sign-in")
    page.get_by_label("Email address").fill(os.environ.get("AYR_TEST_USERNAME"))
    page.get_by_label("Password").fill(os.environ.get("AYR_TEST_PASSWORD"))
    page.get_by_role("button", name="Sign in").click()
    page.wait_for_url("/browse")
    return page

@pytest.fixture(scope="session")
def browser_context_args(browser_context_args):
    """
    Fixture for configuring Playwright browser context arguments.
    This fixture is used to customize browser context arguments for Playwright
    browser instances. It sets the 'ignore_https_errors' option to True, which
    allows ignoring HTTPS errors, To disable JS set 'java_script_enabled' to False, which
    disables JavaScript in the browser context.
    Parameters:
        browser_context_args (dict): The default browser context arguments.
    Returns:
        dict: Updated browser context arguments with customized settings.
    """
    return {
        **browser_context_args,
        "ignore_https_errors": True,
        "java_script_enabled": False,
    }

@pytest.fixture()
def page(request, page) -> Page:
    if "test_css_" not in request.node.name and callable(request.node.obj):

        def route_intercept(route):
            if route.request.resource_type == "stylesheet":
                return route.abort()
            return route.continue_()

        page.route("**/*", route_intercept)
    return page


import sqlite3

if __name__ == '__main__':
    conn = sqlite3.connect("androidBug.db")
    cursor = conn.cursor()
    tb_exists = "SELECT name FROM sqlite_master WHERE type='table' AND name='bug_table'"
    if not conn.execute(tb_exists).fetchone():
        conn.execute(
            'create table bug_table (id  integer PRIMARY KEY autoincrement, url varchar(100),result varchar(100),description varchar(100),state varchar(20),value integer)')
        conn.commit()
    while True:
        url = input('url:')
        result = input('结果：')
        des = input('描述：')
        state = input('状态：')
        value = input('评价')
        sql = ''' insert into bug_table
              (url,result,description,state,value)
              values
              (:st_url,:st_result,:st_desc,:st_state,:st_value)'''
        cursor.execute(sql,
                       {'st_url': url, 'st_result': result, 'st_desc': des, 'st_state': state, 'st_value': value})
        conn.commit()
        print("=====================================")


import os.path
import numpy as np
import multiprocessing
from collections import OrderedDict
from subprocess import DEVNULL, STDOUT, check_call

#Decipher amount of simulated reads to suit x% of transcripts FPKM
def sim_amount(gene_list, fpkm_dict, num_read, total_len, percentile_val, fasta_dict, total_reps):
    #Get xth Percentile FPKM
    FPKM = []
    for gene in gene_list:
        FPKM.append(float(fpkm_dict[gene]))
    percentile_FPKM = np.percentile(np.array(FPKM),percentile_val)
    #Average fragment depth = (number of fragments x fragment size)/Total bases
    #Number of fragments per gene = average fragment depth x (cDNA length/fragment size)
    #FPKM = (Number of fragments per gene/(Number of reads/1000000))/(cDNA length/1000)
    #Merge and simplify
    #FPKM = (Number of fragments x 1000000000)/(Total bases x number of reads)
    #Number of fragments = (FPKM x Total bases x total num of fragments)/1000000000
    #Adjust "Total bases" by ratio of total_transcripts/number of genes
    #Number of fragments = (FPKM x Total bases x total num of fragments)/(1000000000) multiply by number of replicates  # x (total_transcripts/number of genes)
    sim1_num_frags = int((percentile_FPKM*total_len*num_read/2)/(1000000000))*total_reps  #*(len(fasta_dict)/len(gene_list))
    return sim1_num_frags, percentile_FPKM

#Decipher amount of simulated reads to suit x% of transcripts fragments
def sim_amount2(gene_list, fpkm_dict, total_len, percentile_val, total_reps, transcript_frag_dict, transcript_dict, cdna_size_dict):
    #Get xth Percentile FPKM
    FPKM_dict = OrderedDict()
    FPKM = []
    for gene in gene_list:
        FPKM_dict[gene] = float(fpkm_dict[gene])
    FPKM_tup_sort = sorted(FPKM_dict.items(), key=lambda x:x[1])
    for gene,fpkm in FPKM_tup_sort:
        FPKM.append(fpkm)
    percentile_index = int(len(FPKM)*(percentile_val/100)) - 1
    gene_name, percentile_FPKM = FPKM_tup_sort[percentile_index]
    for rep in transcript_frag_dict:
        last_rep_name = rep
    #Average fragment depth = (number of fragments x fragment size)/Total bases
    #Number of fragments per gene = average fragment depth x (cDNA length/fragment size)
    #FPKM = (Number of fragments per gene/(Number of reads/1000000))/(cDNA length/1000)
    #Merge and simplify
    #FPKM = (Number of fragments x 1000000000)/(Total bases x number of reads)
    #Number of fragments = (FPKM x Total bases x number of reads)/1000000000
    #Adjust "Total bases" by ratio of total_transcripts/number of genes
    #Number of fragments = (FPKM x Total bases x number of reads)/(1000000000)) multiply by number of replicates
    #Number of fragments = (Number of fragments per gene x Total bases)/cDNA length) multiply by number of replicates
    try:
        sim1_num_frags = int((transcript_frag_dict[last_rep_name]['@'+transcript_dict[gene_name]+'-e']*total_len)/cdna_size_dict[transcript_dict[gene_name]])*total_reps
    except:
        sim1_num_frags = int((transcript_frag_dict[last_rep_name]['@'+transcript_dict[gene_name]+'-i']*total_len)/cdna_size_dict[transcript_dict[gene_name]])*total_reps
    return sim1_num_frags, percentile_FPKM

#Chunk list generator
def split_chunks(a, n):
    k, m = divmod(len(a), n)
    return list(a[i * k + min(i, m):(i + 1) * k + min(i + 1, m)] for i in range(n))

#Split reference fasta to multiple fasta files according to number of threads
def split_fasta(output_dir, ref, num_threads, num_fasta):
    fasta = os.path.join(output_dir, ref)
    tmp_dict = OrderedDict()
    tmp_list = []
    for index in range(num_threads):
        tmp_dict[index] = open(os.path.join(output_dir, 'tmp' + str(index) + '.fa'), 'w')
    with open(fasta) as f:
        for line in f:
            tmp_list.append(line + next(f))
    #Split into chunks
    chunk_list = split_chunks(range(num_fasta), num_threads)
    for index in range(len(chunk_list)):
        for num in chunk_list[index]:
            tmp_dict[index].write(tmp_list[num])
    for index in tmp_dict:
        tmp_dict[index].close()

#Run single dwgsim
def single_subprocess_dwgsim(dwgsim_path, output_dir, num_threads, first_read_error, second_read_error, ins_length, ins_length_stdev, split_frags, read_length, mut_rate, indel_rate, indel_ext, indel_len, ran_read, seed, ref):
    check_call([str(dwgsim_path), '-e', str(first_read_error), '-E', str(second_read_error), '-d', str(ins_length), '-s', str(ins_length_stdev), '-N', str(split_frags), '-1', str(read_length), '-2', str(read_length), '-r', str(mut_rate), '-R', str(indel_rate), '-X', str(indel_ext), '-I', str(indel_len), '-y', str(ran_read), '-z', str(seed), '-o', '1', '-c', '0', '-S', '2', str(ref), str(ref + '.out')], stdout=DEVNULL, stderr=STDOUT)

#Run multi dwgsim
def multi_subprocess_dwgsim(current_gene_list, dwgsim_dir, output_dir, num_threads, first_read_error, second_read_error, ins_length, ins_length_stdev, sim_num_frags, read_length, mut_rate, indel_rate, indel_ext, indel_len, ran_read, seed):
    dwgsim_path = os.path.join(dwgsim_dir, 'dwgsim')
    in_fasta_list = []
    if len(current_gene_list) >= num_threads:
        for index in range(num_threads):
            in_fasta_list.append(os.path.join(output_dir, 'tmp' + str(index) + '.fa'))
        split_frags = round(sim_num_frags/num_threads)
        jobs = []
        for ref in in_fasta_list:
            p = multiprocessing.Process(target=single_subprocess_dwgsim, args=(dwgsim_path, output_dir, num_threads, first_read_error, second_read_error, ins_length, ins_length_stdev, split_frags, read_length, mut_rate, indel_rate, indel_ext, indel_len, ran_read, seed, ref))
            jobs.append(p)
            p.start()
    else:
        for index in range(len(current_gene_list)):
            in_fasta_list.append(os.path.join(output_dir, 'tmp' + str(index) + '.fa'))
        split_frags = round(sim_num_frags/len(current_gene_list))
        jobs = []
        for ref in in_fasta_list:
            p = multiprocessing.Process(target=single_subprocess_dwgsim, args=(dwgsim_path, output_dir, num_threads, first_read_error, second_read_error, ins_length, ins_length_stdev, split_frags, read_length, mut_rate, indel_rate, indel_ext, indel_len, ran_read, seed, ref))
            jobs.append(p)
            p.start()
    for job in jobs:
        job.join()
    #for ref in in_fasta_list:
        #check_call(['rm', ref], stdout=DEVNULL, stderr=STDOUT)
        #check_call(['rm', ref + '.out.mutations.vcf'], stdout=DEVNULL, stderr=STDOUT)
        #check_call(['rm', ref + '.out.mutations.txt'], stdout=DEVNULL, stderr=STDOUT)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('music', '0017_delete_temporarymusic'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='music',
            name='date',
        ),
    ]


from matplotlib import cm
from matplotlib.colors import LinearSegmentedColormap, ListedColormap

class Colors:
    bright_blue = (13 / 255, 0 / 255, 107 / 255)  # blue HSV(244, 255, 207)
    bright_red = (207 / 255, 0 / 255, 0 / 255)  # red HSV(0, 255, 207)

_bwr_data = [
    (13 / 255, 0 / 255, 107 / 255),  # blue negative HSV(244, 255, 207)
    (1.0, 1.0, 1.0),  # white center
    (207 / 255, 0 / 255, 0 / 255),  # red positive HSV(0, 255, 207)
]
_cmap_bwr = LinearSegmentedColormap.from_list("crownet_bwr", _bwr_data)

_registerd = False

def _register_cmap():
    if not _register_cmap:
        cm.register_cmap("crownet_bwr", _cmap_bwr)
        _registerd = True


from django.contrib import admin
from django.urls import path,include

"""
PARA INCLUIR LAS URLS DE TU APP SE UTILIZA

path('nombreApi/',include('nombreApi.urls'))

RECORDAR INCLUIR LAS COMILLAS DENTRO DE LA FUNCIÓN INCLUDE
"""

urlpatterns = [
    path('admin/', admin.site.urls),
    path('',include('menuApis.urls')),
    path('valorant/',include('valorantApi.urls')),
    path('digimon/',include('digimonApi.urls')),
    path('makeup/',include('makeupApi.urls')),
    path('breakingbad/',include('breakingbadApi.urls')),
    path('disney/',include('disneyApi.urls')),
    path('freetogame/',include('freeToGameApi.urls')),
    path('noticias/',include('noticiasApi.urls')),
    path('covid/',include('covidApi.urls')),
    path('horoscopo/', include('horoscopoApi.urls')),
    path('personas/', include('personasApi.urls')),
    path('allauth/',include('allauthApi.urls')),
    path('accounts/', include('allauth.urls')),

]