from sqlalchemy import create_engine, Column, Integer, String, ForeignKey, func
from sqlalchemy.orm import relationship, sessionmaker
from sqlalchemy.ext.declarative import declarative_base

Base = declarative_base()

# Defining SQLAlchemy models
class Restaurant(Base):
    __tablename__ = 'restaurants'

    id = Column(Integer, primary_key=True)
    name = Column(String)
    price = Column(Integer)

    # Defining the relationship between Restaurant & Review. One to many
    reviews = relationship('Review', back_populates='restaurant')
    customers = relationship('Customer', secondary='reviews', back_populates='restaurants')

    @classmethod
    def fanciest(cls, session):
        # Sorting restaurannt by price
        return session.query(cls).order_by(cls.price.desc()).first()

    def all_reviews(self):
        # Reviews for the restaurant
        review_strings = []
        for review in self.reviews:
            review_strings.append(review.full_review())
        return review_strings

class Customer(Base):
    __tablename__ = 'customers'

    id = Column(Integer, primary_key=True)
    first_name = Column(String)
    last_name = Column(String)

    # Defining the relationship between Customer and Review
    reviews = relationship('Review', back_populates='customer')
    restaurants = relationship('Restaurant', secondary='reviews', back_populates='customers')

    def full_name(self):
        return f"{self.first_name} {self.last_name}"

    def favorite_restaurant(self):
        highest_rated_restaurant = max(self.reviews, key=lambda review: review.star_rating)
        return highest_rated_restaurant.restaurant

    def add_review(self, restaurant, rating):
        review = Review(customer=self, restaurant=restaurant, star_rating=rating)
        session.add(review)
        session.commit()

    def delete_reviews(self, restaurant):
        reviews_to_delete = [review for review in self.reviews if review.restaurant == restaurant]
        for review in reviews_to_delete:
            session.delete(review)
        session.commit()

    def reviews(self):
        return self.reviews

    def restaurants(self):
        return self.restaurants

class Review(Base):
    __tablename__ = 'reviews'

    id = Column(Integer, primary_key=True)
    star_rating = Column(Integer)

    customer_id = Column(Integer, ForeignKey('customers.id'))
    restaurant_id = Column(Integer, ForeignKey('restaurants.id'))

    customer = relationship('Customer', back_populates='reviews')
    restaurant = relationship('Restaurant', back_populates='reviews')

    def full_review(self):
        return f"Review for {self.restaurant.name} by {self.customer.full_name()}: {self.star_rating} stars."

engine = create_engine('sqlite:///restaurant_reviews.db')
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
session = Session()

if __name__ == '__main__':

    customer1 = Customer(first_name='Gideon', last_name='Mugere', email='gomugere@gmail.com')
    customer2 = Customer(first_name='Cynthia', last_name='Smith', email='cynthiasmith@gmail.com')

    restaurant1 = Restaurant(name='Fish Friendly Restaurant', price=200)
    restaurant2 = Restaurant(name='Berbecue Friendly', price=300)

    review1 = Review(customer=customer1, restaurant=restaurant1, star_rating=4)
    review2 = Review(customer=customer2, restaurant=restaurant1, star_rating=5)

    session.add_all([customer1, customer2, restaurant1, restaurant2, review1, review2])
    session.commit()


import pickle
import uuid
import time
import os
import tempfile
import urllib2
import math
from Bio import Entrez, SeqIO
from talconfig import BASE_DIR, REDIS_SERVER_HOSTNAME, REDIS_SERVER_PORT
from talutil import TaskError

redis_found = True
try:
    import redis
except ImportError:
    redis_found = False

Entrez.email = "6e6a62393840636f726e656c6c2e656475".decode("hex")
Entrez.tool = "https://tale-nt.cac.cornell.edu"

def _ncbi_search_assemblies(cached_file, logger, assembly_id):

    ehandle = Entrez.esearch(db="assembly", term=assembly_id)
    erecord = Entrez.read(ehandle)
    ehandle.close()

    if int(erecord["Count"]) != 0:

        canonical_assembly_ids = [x for x in erecord["IdList"] if x != "-1"]

        if len(canonical_assembly_ids) > 0:

            cached_file.assembly_id = canonical_assembly_ids[0]

            if len(canonical_assembly_ids) > 1:
                logger("Warning: NCBI search for assembly ID returned more than 1 record, chose the first")

def _ncbi_links_for_id(cached_file, logger, db, dbfrom, linkname, assembly_id):

    ehandle = Entrez.elink(db=db, dbfrom=dbfrom, id=assembly_id, linkname=linkname)
    erecord = Entrez.read(ehandle)
    ehandle.close()

    if len(erecord[0]["LinkSetDb"]) != 0:
        cached_file.nuc_seq_ids = [link["Id"] for link in erecord[0]["LinkSetDb"][0]["Link"]]

def _ncbi_search_nuccore(cached_file, logger, db, term):

    ehandle = Entrez.esearch(db="nucleotide", term=term)
    erecord = Entrez.read(ehandle)
    ehandle.close()

    if int(erecord["Count"]) != 0:

        cached_file.assembly_id = "nucleotide_" + erecord["IdList"][0]
        cached_file.nuc_seq_ids = [erecord["IdList"][0]]

        if int(erecord["Count"]) > 1:
            logger("Warning: NCBI search for nucleotide sequence ID returned more than 1 record, chose the first")

# redis locking
# http://www.dr-josiah.com/2012/01/creating-lock-with-redis.html

class BaseCachedEntrezFile(object):

    def __init__(self, logger, assembly_id):

        self.assembly_id = None
        self.nuc_seq_ids = []

        try:

            _ncbi_search_assemblies(self, logger, assembly_id)

            if self.assembly_id is not None:

                logger("ID matched an NCBI assembly, checking for linked RefSeq sequences")

                _ncbi_links_for_id(self, logger, "nuccore", "assembly", "assembly_nuccore_refseq", self.assembly_id)

                if len(self.nuc_seq_ids) == 0:

                    logger("No RefSeq sequences available for NCBI assembly ID, checking for matching GenBank sequences.")

                    _ncbi_links_for_id(self, logger, "nuccore", "assembly", "assembly_nuccore_insdc", self.assembly_id)

                    if len(self.nuc_seq_ids) == 0:

                        logger("No GenBank sequences available for NCBI assembly ID, checking if ID matches nucleotide sequence")

                        raise TaskError("No linked RefSeq or GenBank sequences for found for the given assembly ID")

                logger("Sequences for assembly located, downloading now")

            else:

                logger("ID did not match an NCBI assembly, searching for nucleotide sequences")

                _ncbi_search_nuccore(self, logger, "nucleotide", assembly_id)

                if len(self.nuc_seq_ids) == 0:
                    raise TaskError("The given ID did not match any assemblies or nucleotide sequences")

                logger("Nucleotide sequence for ID located, downloading now")

        except (IOError, urllib2.HTTPError):
            raise TaskError("Invalid NCBI ID provided")

if redis_found:

    class CachedEntrezFile(BaseCachedEntrezFile):

        WRITE_LOCK_DURATION = 18000
        READ_LOCK_DURATION = 36000

        CACHED_FILE_DIR = BASE_DIR + "/cached_gb_files"
        CACHED_FILE_PATTERN = CACHED_FILE_DIR + "/%s.fasta"

        def __init__(self, logger, assembly_id):

            if assembly_id == "NA":
                return

            self.conn = redis.StrictRedis(host=REDIS_SERVER_HOSTNAME, port=REDIS_SERVER_PORT, db=1)

            super(CachedEntrezFile, self).__init__(logger, assembly_id)

            self.lock_name = "gb_cache:%s" % self.assembly_id
            self.uuid = str(uuid.uuid4())
            self.filepath = CachedEntrezFile.CACHED_FILE_PATTERN % self.assembly_id
            self.file = None

        def __enter__(self):

            pipe = self.conn.pipeline(True)

            while True:

                try:

                    pipe.watch(self.lock_name)

                    if not pipe.exists(self.lock_name):

                        print("No entry for key %s, creating" % self.lock_name)

                        timestamp = time.time()

                        pipe.multi()

                        pipe.hmset(self.lock_name, {
                            "last-accessed": timestamp,
                            "readers": pickle.dumps({}),
                            "writers": pickle.dumps({
                                self.uuid: timestamp + CachedEntrezFile.WRITE_LOCK_DURATION
                            })
                        })

                        pipe.execute()

                        print("Key %s created" % self.lock_name)
                        print("Downloading file for newly created key %s" % self.lock_name)

                        try:
                            os.remove(self.filepath)
                        except OSError:
                            pass

                        try:

                            with open(self.filepath, "w") as cached_file:

                                for i in range(int(math.ceil(float(len(self.nuc_seq_ids)) / 5000))):

                                    start = i * 5000
                                    remaining = len(self.nuc_seq_ids) - start
                                    end = start + remaining if remaining < 5000 else start + 5000

                                    print("start %d remaining %d end %d" % (start, remaining, end))

                                    fetch_handle = Entrez.efetch(db="nucleotide", id=self.nuc_seq_ids[start:end], rettype="fasta", retmode="text")

                                    for line in fetch_handle:
                                        cached_file.write(line)

                                    fetch_handle.close()

                        except IOError:
                            # I dunno
                            raise

                        finally:

                            print("File for key %s finished downloading, removing writer")

                            pipe.watch(self.lock_name)

                            lock_writers = pickle.loads(pipe.hget(self.lock_name, "writers"))

                            if self.uuid in lock_writers:
                                del lock_writers[self.uuid]

                            pipe.multi()
                            pipe.hset(self.lock_name, "writers", pickle.dumps(lock_writers))
                            pipe.execute()

                            print("Writer for key %s removed" % self.lock_name)

                    else:

                        print("Key %s exists, checking for writer" % self.lock_name)

                        timestamp = time.time()

                        lock_writers = pickle.loads(pipe.hget(self.lock_name, "writers"))

                        if len(lock_writers.keys()) > 0:

                            print("Writer for key %s exists, checking expiration" % self.lock_name)

                            writers_changed = False

                            for l_uuid, l_timestamp in lock_writers.copy().items():

                                print("Writer for key %s with UUID %s, timestamp %s, current time %s" % (self.lock_name, l_uuid, str(l_timestamp), str(timestamp)))

                                if l_uuid == self.uuid:

                                    print("Writer for key %s has our UUID, previous attempt to remove writer failed, retrying" % self.lock_name)
                                    del lock_writers[l_uuid]
                                    writers_changed = True

                                elif l_timestamp < timestamp:

                                    print("Writer for %s with UUID %s expired, removing" % (self.lock_name, l_uuid))
                                    del lock_writers[l_uuid]
                                    writers_changed = True

                            if writers_changed:

                                pipe.multi()
                                #pipe.hset(self.lock_name, "writers", pickle.dumps(lock_writers))
                                # since file will only be downloaded if key doesn't exist (because there's no way to check if local sequence
                                # file is corrupted), and there's only one writer at a time, removing the only writer means the local file
                                # is likely corrupted, so we remove the key so that redownload of file is attempted
                                pipe.delete(self.lock_name)
                                pipe.execute()

                                print("Expired writers for key %s removed" % self.lock_name)

                            time.sleep(5)

                        else:

                            print("No writers for key %s, adding reader" % self.lock_name)

                            timestamp = time.time()

                            lock_readers = pickle.loads(pipe.hget(self.lock_name, "readers"))

                            lock_readers[self.uuid] = timestamp

                            pipe.multi()
                            pipe.hset(self.lock_name, "last-accessed", timestamp)
                            pipe.hset(self.lock_name, "readers", pickle.dumps(lock_readers))
                            pipe.execute()

                            print("Reader for key %s acquired, returning file" % self.lock_name)

                            break

                except redis.exceptions.WatchError:
                    pass

            self.file = open(self.filepath, "r")

            return self

        def __exit__(self, exc_type, exc_value, traceback):

            if self.file is not None:
                self.file.close()

            print("Releasing reader for key %s" % self.lock_name)

            pipe = self.conn.pipeline(True)

            while True:

                try:

                    pipe.watch(self.lock_name)

                    if pipe.exists(self.lock_name):

                        lock_readers = pickle.loads(pipe.hget(self.lock_name, "readers"))

                        if self.uuid in lock_readers:
                            del lock_readers[self.uuid]

                        pipe.multi()
                        pipe.hset(self.lock_name, "readers", pickle.dumps(lock_readers))
                        pipe.execute()

                    else:

                        # read lock was broken by garbage collector

                        pipe.unwatch()

                    break

                except redis.exceptions.WatchError:
                    pass

            print("Reader for key %s released" % self.lock_name)

else:

    class CachedEntrezFile(BaseCachedEntrezFile):

        def __init__(self, logger, assembly_id):

            if assembly_id == "NA":
                return

            super(CachedEntrezFile, self).__init__(logger, assembly_id)

            self.file = None
            self.filepath = ""

        def __enter__(self):

            try:

                fetch_handle = Entrez.efetch(db="nucleotide", id=self.nuc_seq_ids, rettype="fasta", retmode="text")
                self.file = tempfile.NamedTemporaryFile()

                for i in range(int(math.ceil(float(len(self.nuc_seq_ids)) / 5000))):

                    start = i * 5000
                    remaining = len(self.nuc_seq_ids) - start
                    end = start + remaining if remaining < 5000 else start + 5000

                    fetch_handle = Entrez.efetch(db="nucleotide", id=self.nuc_seq_ids[start:end], rettype="fasta", retmode="text")

                    for line in fetch_handle:
                        self.file.write(line)

                    fetch_handle.close()

                self.file.seek(0)

            except IOError:
                raise TaskError("Could not download sequence from NCBI")

            self.filepath = self.file.name

            return self

        def __exit__(self, exc_type, exc_value, traceback):

            if self.file is not None:
                self.file.close()


import os
from dotenv import load_dotenv
from telegram import Update, ForceReply
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters, CallbackContext
import openai
from Helper.functions import get_image, get_chat

load_dotenv()

# Get Telegram API key from environment variable
TELEGRAM_API_KEY = os.getenv('TELEGRAM_API_KEY')

# Get Chat GPT API key from environment variable
CHAT_GPT_API_KEY = os.getenv('CHAT_GPT_API_KEY')

# Authenticate OpenAI API
openai.api_key = CHAT_GPT_API_KEY

def start(update: Update, context: CallbackContext) -> None:
    """Send a message when the command /start is issued."""
    update.message.reply_text('Welcome, You can ask anything from me\n@RedXvirus')

def help_command(update: Update, context: CallbackContext) -> None:
    """Send a message when the command /help is issued."""
    update.message.reply_text('This bot can perform the following commands\n/image -> to create image from text\n/ask -> ask anything from me')

def image_command(update: Update, context: CallbackContext) -> None:
    """Create image from text."""
    text = update.message.text.replace("/image", "").strip().lower()

    if text:
        # Get image from OpenAI Dall-E API
        res = get_image(text)

        if res:
            context.bot.send_chat_action(chat_id=update.effective_message.chat_id, action='upload_photo')
            context.bot.send_photo(chat_id=update.effective_message.chat_id, photo=res, reply_to_message_id=update.message.message_id)
    else:
        context.bot.send_message(chat_id=update.effective_message.chat_id, text='You have to give some description after /image', reply_to_message_id=update.message.message_id)

def ask_command(update: Update, context: CallbackContext) -> None:
    """Get chat response from Chat GPT API."""
    text = update.message.text.replace("/ask", "").strip().lower()

    if text:
        # Get response from OpenAI GPT-3 API
        context.bot.send_chat_action(chat_id=update.effective_message.chat_id, action='typing')
        res = get_chat(text)

        if res:
            context.bot.send_message(chat_id=update.effective_message.chat_id, text=res, reply_to_message_id=update.message.message_id)
    else:
        context.bot.send_message(chat_id=update.effective_message.chat_id, text='Please ask anything after /ask', reply_to_message_id=update.message.message_id)

def main() -> None:
    """Start the bot."""
    updater = Updater(TELEGRAM_API_KEY)

    # Add handlers for Telegram commands
    updater.dispatcher.add_handler(CommandHandler("start", start))
    updater.dispatcher.add_handler(CommandHandler("help", help_command))
    updater.dispatcher.add_handler(CommandHandler("image", image_command))
    updater.dispatcher.add_handler(CommandHandler("ask", ask_command))

    # Start the bot
    updater.start_polling()

    # Run the bot until the user presses Ctrl-C or the process receives SIGINT,
    # SIGTERM or SIGABRT
    updater.idle()

if name == 'main':
    main()


#!/usr/bin/env python

# Change this stuff:
name='Gilia plant'
author='Travis Wells'
origauth='Travis Wells'
shortname='giliaplant'
objfile='gilia.obj'
uniqueid=0 # set to 0 if you don't have one
enhance_color=True
#OBJ2VXP: Converts simple OBJ files to VXP expansions
#Copyright (C) 2004-2015 Foone Turing
#
#This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
#
#This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
#You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
import os
import sys
sys.path.append('code')
import zipfile
import pygame
import lib3dmm
from urllib import urlopen
from struct import pack
from simpleMBMP import MBMP
from tmap import TMAP
from idgenerator import GenerateID
from error import SaveError
import cPickle as pickle
import sockgui
from obj import WavefrontModel
from time import time
version='0.4'
def CreateVXPExpansionFromOBJ(name,author,origauth,outfile,shortname,objfile,uniqueid,progress,enhance_color,statusfunc=None):
	created_files=[]
	try:
		if name=='':
			raise SaveError('No name')
		if author=='':
			raise SaveError('No author')
		if origauth=='':
			raise SaveError('No orig. author')
		if shortname=='':
			raise SaveError('No shortname')
		if objfile=='':
			raise SaveError('No OBJ')
		if outfile=='':
			raise SaveError('No outfile')
		def SaveCFG(outzip):
			cfg='Name=%s\nAuthor=%s\nOriginal Author=%s\nType=Portable\nContent=Props\nDate=%i\nGenerator=obj2vxp %s\n' % (name,author,origauth,int(time()),version)
			outzip.writestr(shortname+'.cfg',cfg)
			progress()
		def Save3CN(outzip):
			if statusfunc is not None:
				statusfunc('Creating 3CN...')
			vxp3cn=pickle.load(open('code/templates/objtemplate3cn.pl3','rb'))
			for quad in vxp3cn.quads:
				if quad.type=='TMPL':
					quad.setID(uniqueid)
					quad.setString(name)
				if quad.type=='BMDL' and quad.source.get_length()==0:
					if statusfunc is not None:
						statusfunc('Creating BMDL...')
					bmdl=modelobj.makeBMDL(statusfunc)
					quad.setData(bmdl.getData(True,bmdl.rescalef/25.0,bmdl.texrescalef))
				if quad.type=='TMAP':
					tmapdata=TMAP()
					tmapdata.loadFromSurface(texture)
					quad.setData(tmapdata.getData())
			progress()
			outzip.writestr(shortname+'.3cn',vxp3cn.getData())
			progress()
		def Save3TH(outzip):
			if statusfunc is not None:
				statusfunc('Creating 3TH...')
			vxp3th=pickle.load(open('code/templates/objtemplate3th.pl3','rb'))
			for quad in vxp3th.quads:
				quad.setID(uniqueid)
				for id,otherquad in quad.references:
					otherquad.setID(uniqueid)
				if quad.type=='MBMP':
					mbmpdata=MBMP()
					mbmpdata.loadFromSurface(minisurf)
					quad.setData(mbmpdata.getData())
				if quad.type=='PRTH':
					quad.setData(pack('<4B 4s L',1,0,3,3,'TMPL'[::-1],uniqueid))
			progress()
			outzip.writestr(shortname+'.3th',vxp3th.getData())
			progress()
		if uniqueid is None or uniqueid==0:
			uniqueid=GenerateID()
			if uniqueid==0:
				raise SaveError("Couldn't get ID (or id==0)")
		def CreateMBMP():
			surf=pygame.Surface((72,72),pygame.SWSURFACE,palette_surf)
			surf.fill(30)
			surf.set_palette(palette_surf.get_palette())
			font=sockgui.Font('code/font.png')
			font.draw(surf,(2,2),'OBJ file')
			font.draw(surf,(2,2+8), name)
			#stuff here.
			return surf
		if statusfunc is not None:
			statusfunc('Loading OBJ...')
		modelobj=WavefrontModel(objfile)
		if statusfunc is not None:
			statusfunc('Creating textures...')
		pygame.init()
		palette_surf=pygame.image.load('code/palette.bmp')
		minisurf=CreateMBMP()
		texture=modelobj.makeTexture(palette_surf,enhance_color)
		#pygame.image.save(texture,'out.bmp')
		created_files.append(outfile)
		outvxp=zipfile.ZipFile(outfile,'w',zipfile.ZIP_DEFLATED)
		SaveCFG(outvxp)
		Save3CN(outvxp)
		Save3TH(outvxp)

		outvxp.close()
		progress()
		created_files=[] # Clear file list, so they won't be nuked
		return True
	finally:
		try:
			files=[]
			dirs=[]
			for file in created_files:
				if os.path.isdir(file):
						dirs.append(file)
				else:
					files.append(file)
			for file in files:
				try:
					os.unlink(file)
				except:
					pass
			for dirfile in dirs:
				try:
					os.rmdir(dirfile)
				except OSError:
					pass
		except OSError:
			pass
def NullProgress():
	pass
if __name__=='__main__':
	outfile=shortname+'.vxp'
	CreateVXPExpansionFromOBJ(name,author,origauth,outfile,shortname,objfile,uniqueid,NullProgress,enhance_color)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
import sklearn.metrics as metrics
from sklearn.ensemble import RandomForestClassifier
import data_preparation_functions as prepfunctions
import ds_functions as ds
import os
from datetime import datetime

graphsDir = './Results/Random Forests/'
if not os.path.exists(graphsDir):
    os.makedirs(graphsDir)

print('------------------------------')
print('-                            -')
print('-     QOT Random Forests     -')
print('-                            -')
print('------------------------------')

data: pd.DataFrame = pd.read_csv('../Dataset/qsar_oral_toxicity.csv', sep=';', header=None)
train, test = train_test_split(data, train_size=0.7, stratify=data[1024].values)
testDatas = {}
datas = prepfunctions.prepare_dataset(train, 1024, False, False)
for key in datas:
    testDatas[key] = test.copy()

featured_datas = prepfunctions.mask_feature_selection(datas, 1024, True, './Results/FeatureSelection/QOT Feature Selection - Features')
featured_test_datas = prepfunctions.mask_feature_selection(testDatas, 1024, True, './Results/FeatureSelection/QOT Feature Selection - Features')

best_accuracies = {}

for key in datas:
    for do_feature_eng in [False, True]:
        if (do_feature_eng):
            data = featured_datas[key]
            testData = featured_test_datas[key].copy()
            subDir = graphsDir + 'FeatureEng/' +  key + '/'
            if not os.path.exists(subDir):
                os.makedirs(subDir)
        else:
            data = datas[key]
            testData = test.copy()
            subDir = graphsDir + key + '/'
            if not os.path.exists(subDir):
                os.makedirs(subDir)

        print('QOT Random Forests - Parameters Combinations')
        now = datetime.now()
        current_time = now.strftime("%H:%M:%S")
        print(current_time, ": Key: ", key, ", feature eng: ", do_feature_eng)

        trnY: np.ndarray = data.pop(1024).values
        trnX: np.ndarray = data.values
        tstY: np.ndarray = testData.pop(1024).values
        tstX: np.ndarray = testData.values

        n_estimators = [5, 10, 25, 50, 75, 100, 150, 200, 250, 300]
        max_depths = [5, 10, 25]
        max_features = [.1, .3, .5, .7, .9, 1]
        best = ('', 0, 0)
        last_best = 0
        best_tree = None
        overfit_values = {}

        cols = len(max_depths)
        plt.figure()
        fig, axs = plt.subplots(1, cols, figsize=(cols*ds.HEIGHT*2.5, ds.HEIGHT), squeeze=False)
        for k in range(len(max_depths)):
            d = max_depths[k]
            now = datetime.now()
            current_time = now.strftime("%H:%M:%S")
            print(current_time, ": d: ", d)
            values = {}
            overfit_values[d] = {}
            for f in max_features:
                yvalues = []
                train_acc_values = []
                test_acc_values = []
                now = datetime.now()
                current_time = now.strftime("%H:%M:%S")
                print(current_time, ": f: ", f)
                for n in n_estimators:
                    now = datetime.now()
                    current_time = now.strftime("%H:%M:%S")
                    print(current_time, ": n: ", n)
                    rf = RandomForestClassifier(n_estimators=n, max_depth=d, max_features=f)
                    rf.fit(trnX, trnY)
                    prdY = rf.predict(tstX)
                    prd_trainY = rf.predict(trnX)
                    yvalues.append(metrics.accuracy_score(tstY, prdY))
                    train_acc_values.append(metrics.accuracy_score(trnY, prd_trainY))
                    test_acc_values.append(metrics.accuracy_score(tstY, prdY))
                    if yvalues[-1] > last_best:
                        best = (d, f, n)
                        last_best = yvalues[-1]
                        last_best_train = train_acc_values[-1]
                        best_tree = rf

                values[f] = yvalues
                overfit_values[d][f] = {}
                overfit_values[d][f]['train'] = train_acc_values
                overfit_values[d][f]['test'] = test_acc_values
            ds.multiple_line_chart(n_estimators, values, ax=axs[0, k], title='Random Forests with max_depth=%d'%d,
                                xlabel='nr estimators', ylabel='accuracy', percentage=True)

        text = key
        if (do_feature_eng): text += ' with FS'
        best_accuracies[text] = [last_best_train, last_best]

        print('Best results with depth=%d, %1.2f features and %d estimators, with accuracy=%1.2f'%(best[0], best[1], best[2], last_best))
        fig.text(0.5, 0.03, 'Best results with depth=%d, %1.2f features and %d estimators, with accuracy=%1.2f'%(best[0], best[1], best[2], last_best), fontsize=7, ha='center', va='center')
        plt.suptitle('QOT Random Forests - ' + key + '- Parameters Combinations')
        plt.savefig(subDir + 'QOT Random Forests - ' + key + '- Parameters Combinations')
        print()

        plt.figure()
        fig, axs = plt.subplots(3, 6, figsize=(32, 16), squeeze=False)
        for k in range(len(max_depths)):
            d = max_depths[k]
            for i in range(len(max_features)):
                f = max_features[i]
                ds.multiple_line_chart(n_estimators, overfit_values[d][f], ax=axs[k,i], title='Overfitting for max_depth = %d with max_features = %f'%(d,f), xlabel='n_estimators', ylabel='accuracy', percentage=True)
        plt.suptitle('QOT Overfitting - Random Forests')
        plt.savefig(subDir + 'QOT Overfitting - Random Forests')

        print('QOT Random Forests - Performance & Confusion Matrix')
        prd_trn = best_tree.predict(trnX)
        prd_tst = best_tree.predict(tstX)
        ds.plot_evaluation_results(["negative", "positive"], trnY, prd_trn, tstY, prd_tst)
        plt.suptitle('QOT Random Forests - ' + key + ' - Performance & Confusion Matrix')
        plt.savefig(subDir + 'QOT Random Forests - ' + key + ' - Performance & Confusion Matrix')
        print()

plt.figure(figsize=(7,7))
ds.multiple_bar_chart(['Train', 'Test'], best_accuracies, ylabel='Accuracy')
plt.suptitle('QOT Sampling & Feature Selection')
plt.savefig(graphsDir + 'QOT Sampling & Feature Selection')


import httplib2
from bs4 import BeautifulSoup
from argparse import ArgumentParser

http = httplib2.Http()

def copy_content(uri, out):
    status, response = http.request(uri, "GET")
    soup = BeautifulSoup(response, 'html.parser')
    with open(out, 'a') as fp:
        fp.write(soup.prettify())

def find_links(uri, out):
    status, response = http.request(uri, "GET")
    soup = BeautifulSoup(response, 'html.parser')
    with open(out, 'a') as fp:
        for link in soup.find_all('a', href=True):
            fp.write(link['href'] + '\n')

def find_img(uri, out):
    status, response = http.request(uri, "GET")
    soup = BeautifulSoup(response, 'html.parser')
    with open(out, 'a') as fp:
        for img in soup.find_all('img'):
            fp.write(img['src'] + '\n')

parser = ArgumentParser()
subparser = parser.add_subparsers(required=True)

parser_html = subparser.add_parser('html', help="Writes the html document into result.html")
parser_html.add_argument('-l', type=str, dest="url", help="Specify the link")
parser_html.add_argument('-o', type=str, dest="output_file", help="Name of the output file")
parser_html.set_defaults(func=copy_content)

parser_links = subparser.add_parser('links', help="Extracts links from the documents and writes them into result.txt")
parser_links.add_argument('-l', type=str, dest="url", help="Specify the link")
parser_links.add_argument('-o', type=str, dest="output_file", help="Name of the output file")
parser_links.set_defaults(func=find_links)

parser_images = subparser.add_parser('images', help="Extracts links from the documents and writes them into result.txt")
parser_images.add_argument('-l', type=str, dest="url", help="Specify the link")
parser_images.add_argument('-o', type=str, dest="output_file", help="Name of the output file")
parser_images.set_defaults(func=find_img)

if __name__ == '__main__':
    args = parser.parse_args()
    args.func(args.url, args.output_file)


from ntpath import join
from services.current_machine_ip import getIp;
from linqit import List;
from services.instant_proxies_service import *;
from services.data_text_file_service import *;
from datetime import datetime, timedelta;
from services.data_file_service import InstantProxiesDataFileService;

# Lay dia chi IP cua may tinh hien tai
currentMachineIp = getIp();

# Khoi tao chuc nang ghi danh sach IP cho phep va proxy duoc cung cap
dataFileService = InstantProxiesDataFileService();

# Kiem tra xem dia chi IP dang su dung co nam trong danh sach ip duoc cho phep da luu truoc do khong
authedIpList = dataFileService._ReadAuthedIpsFromDataFile();
if authedIpList != None and authedIpList.contains(currentMachineIp):
    # Neu dia chi IP hien tai cua may da co trong danh sach duoc cho phep
    pass;
else:
    # Neu dia chi IP chua them vao danh sach cho phep thi tien hanh vao trang InstantProxies de them
    SignInInstantProxies();
    # Trang InstantProxies tu dong phat hien dia chi IP dang su dung nen khong can truyen vao dia chi IP cua may hien tai
    newAuthedIpList = AddMachineIpToWebAuthedIpList();
    dataFileService._WriteAuthedIpsToDataFile(newAuthedIpList);
    SignOutInstantProxies();

# Kiem tra danh sach proxy da luu proxy nao hay chua
forCrawlDataProxyList = List();
proxyListData = dataFileService._ReadProvidedProxiesFromDataFile();
if proxyListData != None:
    writeTime, proxyListCount, providedProxyList = proxyListData;
    # Neu thoi gian luu file proxy den hien tai qua 24 gio thi vao web InstantProxies de lay danh sach proxy moi: https://stackoverflow.com/a/26313848/7182661
    if (datetime.now() - writeTime) < timedelta(hours=24):
        # less than 24 hours passed
        forCrawlDataProxyList = providedProxyList;
    else:
        # Lam moi danh sach proxy tu web InstantProxies
        SignInInstantProxies();
        newProvidedProxyList = GetProxyListOnWeb();
        dataFileService._WriteProvidedProxiesToDataFile(newProvidedProxyList);
        SignOutInstantProxies();
        forCrawlDataProxyList = newProvidedProxyList;
else:
    # Lam moi danh sach proxy tu web InstantProxies
    SignInInstantProxies();
    newProvidedProxyList = GetProxyListOnWeb();
    dataFileService._WriteProvidedProxiesToDataFile(newProvidedProxyList);
    SignOutInstantProxies();
    forCrawlDataProxyList = newProvidedProxyList;

print("Danh sach proxy duoc su dung de crawl du lieu: " + " ".join(forCrawlDataProxyList));


import sys, os, re
import logging
import collections
import kenlm
from spelling_correcter_v2 import Spelling_Correcter
#from spelling_correcter import get_candidates
import time

arpa_path = 'data/lm_binary.arpa'
lm_model = kenlm.LanguageModel(arpa_path)

logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(levelname)s %(message)s')

WORDS = []
with open('data/count.txt', 'r') as f:
    for line in f:
        word = line.strip()
        WORDS.append(word)

WORD_COUNTS = collections.Counter(WORDS)

Spell_C = Spelling_Correcter(WORD_COUNTS)

def correct_one(line):
    each_start_time = time.time()
    if ' ' in line.strip():
        utt, hpy = line.strip().split(' ', 1)
    else:
        utt = line.strip()
        hpy = ''
        return line.strip()
    hpy = re.sub(' +', ' ', hpy)
    words = hpy.strip().split(' ')
    for_time_start = time.time()
    for i in range(len(words)):
        #logging.info('words_num {}'.format(len(words)))
        each_for_time_start = time.time()
        word = words[i]
        ## 数字不调整
        #if word.isnumeric()
        #    continue
        candidates = list(Spell_C.get_candidates(word))
        #candidates = list(get_candidates(word))
        #logging.info('get candidates time {} {} {} {}'.format(utt, word, len(candidates), time.time() - each_for_time_start))
        if len(candidates) == 1:
            words[i] = candidates[0]
        else:
            max_lm_score = float('-inf')
            for candidate in candidates:
                cand_start_time = time.time()
                if i == 0:
                    if len(words) == 1:
                        tmp_hpy = candidate
                    else:
                        tmp_hpy = ' '.join([candidate, words[i+1]])
                elif i + 1 < len(words):
                    tmp_hpy = ' '.join([words[i-1], candidate, words[i+1]])
                else:
                    tmp_hpy = ' '.join([words[i-1], candidate])
                cand_end_time = time.time()

                lm_score = lm_model.score(tmp_hpy.upper())
                if lm_score > max_lm_score:
                    max_lm_score = lm_score
                    candidate_dict = {}
                    candidate_dict[str(candidate)] = lm_score
                lm_end_time = time.time()
                #logging.info('{} candidates_time: {}, lm_time: {}'.format(utt, cand_end_time-cand_start_time, lm_end_time-cand_end_time))
                    #print(tmp_hpy, lm_score)
            words[i] = list(candidate_dict.keys())[0]
        each_for_time_end = time.time()
        #if each_for_time_end - each_for_time_start > 1e-04:
        #    logging.info('bad word: {}'.format(word))
        #logging.info('each_for_time {} {} {}'.format(utt, word, (each_for_time_end - each_for_time_start)))
    for_time_end = time.time()
    #logging.info('for_time {} {}'.format(utt, (for_time_end - for_time_start)))
    result = utt + ' ' + (' '.join(words)).upper()
    each_end_time = time.time()
    #logging.info('each_time {} {}'.format(utt, (each_end_time-each_start_time)))
    return result

if __name__ == '__main__':
    hpy_txt, hpy_fix_txt = sys.argv[1:3]

    start_time = time.time()
    count = ''
    with open(hpy_txt, 'r') as f:
        for line in f:
            #print(line.strip())
            hpy_fix = correct_one(line)
            count += hpy_fix + '\n'

    with open(hpy_fix_txt, 'w') as f:
        f.write(count)

    end_time = time.time()
    logging.info('total_time: {}'.format(end_time - start_time))


import re
import subprocess
from typing import List, Tuple
import uuid
import time

# Top level bash script to run as init process (PID 1) in each docker container to make sure that
# the docker container exits when the calling python process exits
SH_TEMPLATE = """
EXIT_CODE_FILE=$(mktemp)
cleanup() {
    EXIT_CODE=$(cat $EXIT_CODE_FILE)
    exit $EXIT_CODE
}
trap cleanup INT
( COMMAND ; echo $? > $EXIT_CODE_FILE ; kill -s INT $$ ) &
read _
"""

# How often to poll to see if a container is running
CONTAINER_CHECK_PERIOD = 0.1

def check_container_running(name: str) -> bool:
    result = subprocess.run(["docker", "container", "inspect", name], # pylint: disable=subprocess-run-check
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    return result.returncode == 0

def wait_for_container(name: str, timeout: float = float("inf")):
    end_time = time.time() + timeout
    while time.time() < end_time:
        if check_container_running(name):
            return
        time.sleep(CONTAINER_CHECK_PERIOD)
    raise ValueError("Container did not start in time")

def get_container_ip(name: str) -> str:
    process = subprocess.run(["docker", "inspect", "-f",
                              "{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}", name],
                             stdout=subprocess.PIPE, check=True)
    return process.stdout.decode("utf-8")

def run_docker_target(bazel_target: str, args: List[str] = None,
                      docker_args: List[str] = None,
                      start_timeout: float = 120,
                      delay: int = 0) -> Tuple[subprocess.Popen, str]:
    # Set default arguments
    if args is None:
        args = []

    # Get the path of the bazel image
    regex = r"//(.+):(.+)"
    match = re.match(regex, bazel_target)
    if not match:
        raise ValueError(f"bazel_target \"{bazel_target}\" does not match regex: \"{regex}\"")
    package, target = match.groups()
    bundle_script = f"{package}/{target}"

    # Run the bundle script to add the image to the docker daemon, and get the hash
    bundle_result = subprocess.run([bundle_script], stdout=subprocess.PIPE, check=True)
    image_hash_match = re.search(r"Tagging (.+) as", bundle_result.stdout.decode("utf-8"))
    if not image_hash_match:
        raise ValueError(f"Could not determine image hash for target {bazel_target}")
    image_hash = image_hash_match.groups()[0]

    # Get the entrypoint command
    result = subprocess.run(["docker", "inspect", "-f", "{{.Config.Entrypoint}}", image_hash],
                            stdout=subprocess.PIPE, check=True).stdout.decode('utf-8')
    args = result[1:-2].split(" ") + args
    if delay != 0:
        args = ["sleep", str(delay), ";"] + args

    # Run a the container inside a special bash script that will exit if
    # the calling process dies, so the container will always exit
    name = f'bazel-test-{str(uuid.uuid4())}'
    script = SH_TEMPLATE.replace("COMMAND", " ".join(args))
    docker_cmd = ["docker", "run", "-i", "--rm", "--entrypoint", "sh", "--name", name]
    if docker_args is not None:
        docker_cmd.extend(docker_args)
    docker_cmd.extend([image_hash, "-c", script])
    print(" ".join(docker_cmd), flush=True)
    process = subprocess.Popen(docker_cmd, stdin=subprocess.PIPE)
    try:
        wait_for_container(name, timeout=start_timeout)
        address = get_container_ip(name).strip()
    except:
        process.kill()
        raise
    return process, address


#!/usr/bin/env python

import os
from setuptools import setup
from django_numerators import __version__

def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

with open("README.md", "r") as fh:
    long_description = fh.read()

setup(
    name='django-products',
    version=__version__,
    description='Django polymorphic products, tests included',
    long_description=long_description,
    long_description_content_type="text/markdown",
    maintainer='Rizki Sasri Dwitama',
    maintainer_email='sasri.project@gmail.com',
    license="MIT",
    url='https://github.com/sasriawesome/django_products',
    packages=[
        'django_products',
        'django_products.migrations',
        'django_products.utils',
    ],
    install_requires=[
        'Django>=2.2',
    ],
    classifiers=[
        "Development Status :: 3 - Alpha",
        "Environment :: Web Environment",
        "Framework :: Django",
        "Intended Audience :: Developers",
        "License :: OSI Approved :: BSD License",
        "Operating System :: OS Independent",
        "Programming Language :: Python",
        "Topic :: Software Development :: Libraries :: Python Modules",
    ],
    tes_suite="tests.run_tests.run_tests"
)


from sqlalchemy.orm import sessionmaker
from sqlalchemy import create_engine
from dashboard.db.tabledef import User

class Queries:
    def __init__(self, handling_class=User):
        self.engine = create_engine('sqlite:///providers.db', echo=True)
        self.session = sessionmaker(bind=self.engine)()
        self.handling_class = handling_class

    def unique(self, column_name):
        providers_table = self.session.query(User).filter()
        zones = list(map(lambda x: getattr(x, column_name), list(providers_table)))
        zones = list(dict.fromkeys(zones))
        return zones

    def aggregate_by_column(self, column_name, selection=None):
        unique_column = self.unique(column_name)

        aggregated = {}
        for item in unique_column:
            item_aggregated_list = list(self.session.query(User).filter(getattr(self.handling_class, column_name) == item))
            if selection:
                item_aggregated_list = list(map(lambda x: getattr(x, selection), item_aggregated_list))
            aggregated.update({item: item_aggregated_list})

        return aggregated

    def check(self):
        db_session = sessionmaker(bind=self.engine)()
        user = db_session.query(User).filter()
        d = (list(user))
        d

    def delete_by_column(self, column_name, value):
        self.session.query(User).filter(getattr(self.handling_class, column_name) == value).delete()
        self.session.commit()

q = Queries()
# q.delete_by_column('zone', 'Foo')
agg = q.aggregate_by_column('zone')
print(agg)


#ESSE SCRIPT ESTÁ FUNCIONANDO
#Copyright (c) Jesús Manuel Mager Hois 2016
#https://gist.github.com/pywirrarika/6a8a05e9911cab260fe9de045a99b44e

import numpy as np
from mpl_toolkits.mplot3d import Axes3D

import matplotlib.pyplot as plt

class Perceptron():
    def __init__(self, data):
        self.ciclosx = []
        self.ciclosy = []

        self.d = data
        self.data = np.insert(self.d, len(self.d[0])+2, 1, axis=1)
        self.wn = []

    def plotD(self):
        nd = np.transpose(self.d)

        x3 = nd[0].tolist()
        y3 = nd[1].tolist()
        z3 = nd[2].tolist()

        x3 = x3[0]
        y3 = y3[0]
        z3 = z3[0]

        print(nd[0][0])
        print(nd[1][0])
        print(nd[2][0])

        fig = plt.figure()
        ax = fig.add_subplot(111, projection='3d')

        ax.scatter(nd[0].tolist(), nd[1].tolist(), nd[2].tolist(), c='r', marker='o')

        ax.set_xlabel('X1') #nomeando eixo x1
        ax.set_ylabel('x2') #nomeando eixo x2
        ax.set_zlabel('x3') #nomeando eixo x3
        plt.title("Dados a classificar")

        plt.show()

    def train(self,convergence = 0.3, output = 1):
        r = convergence
        wn = np.array([1,1,1,r])
        newvec = 0
        classe = 0
        itr = 0
        while True:
            itr = itr + 1
            errors = 0
            j = 0
            if output:
                print("-"*75)
            for x in self.data:
                classereal = 0
                xnwn = 0

                x = np.array(x.tolist()[0])
                for i in range(len(x)):
                    xnwn= xnwn + x[i] * wn[i]
                if j < 5:
                    classereal = 1
                else:
                    classereal = 2
                # Des rule
                if xnwn < 0:
                    classe = 1
                else:
                    classe = 2

                pen = "0"
                if classe == 2 and classe != classereal:
                    newvec = wn - r*x
                    errors = errors + 1
                    pen = "-x"
                if classe == 1 and classe != classereal:
                    newvec = wn + r*x
                    errors = errors + 1
                    pen = "+x"

                if output:
                    print(str(x), "\t|", str(wn), "\t|", str(xnwn), "\t|", str(clase), "\t|", str(clasereal), "\t|", pen)

                wn = newvec

                j = j + 1
            if errors == 0:
                self.wn = np.array(wn)
                break

        print("Número de interações:", str(itr))
        self.ciclosx.append(r)
        self.ciclosy.append(itr)

    def classify(self, vector):
        xnwn = 0
        vector.append(1)
        x=np.array(vector)

        for i in range(len(x)):
            xnwn= xnwn + x[i] * self.wn[i]
        if xnwn < 0:
            classe = 1
        else:
            classe = 2
        print("Classe:", str(classe), str(xnwn), str(vector))
        return classe
    def eval(self):
        C1,C2 = np.split(self.d, 2)
        C=[C1,C2]
        i = 0
        self.confmatrix = np.zeros((len(C), len(C)))
        for Classe in C:
            i += 1
            for vector in Classe:
                vector = vector.tolist()[0]
                disc = self.classify(vector)
                self.confmatrix[i-1][disc-1] += 1
        print("Matriz de confusão por substituição")
        print(self.confmatrix)

        self.data= []
        probe = []
        for Classe in C:
            Classe = np.matrix(Classe)
            A,B = np.split(Classe, 2)
            self.data.append(A)
            probe.append(B)

        self.data = np.insert(self.d, len(self.d[0])+2, 1, axis=1)

        self.train(output=0)
        i=0
        self.confmatrix = np.zeros((len(probe), len(probe)))
        for b in probe:
            i += 1
            for vector in b:
                vector = vector.tolist()[0]
                disc = self.classify(vector)
                self.confmatrix[i-1][disc-1] += 1

        print("Matriz de confusão por Cross Validation")
        print(self.confmatrix)

if __name__ == "__main__":
    data = np.matrix([[0,0,0],
               [1,0,1],
               [1,0,0],
               [1,1,0],
               [0,0,1],
               [0,1,1],
               [0,1,0],
               [1,1,1]])
    neuron = Perceptron(data=data)

    neuron.plotD()
    neuron.train(output=0)
    neuron.eval()


"""A hook for adding information to the user infobox."""

from __future__ import annotations

from django.template.loader import render_to_string
from djblets.extensions.hooks import ExtensionHook, ExtensionHookPoint

class UserInfoboxHook(ExtensionHook, metaclass=ExtensionHookPoint):
    """A hook for adding information to the user infobox.

    Extensions can use this hook to add additional pieces of data to the box
    which pops up when hovering the mouse over a user.
    """

    def initialize(self, template_name=None):
        """Initialize the hook.

        Args:
            template_name (unicode):
                The template to render with the default :py:func:`render`
                method.
        """
        self.template_name = template_name

    def get_extra_context(self, user, request, local_site, **kwargs):
        """Return extra context to use when rendering the template.

        This may be overridden in order to make use of the default
        :py:func:`render` method.

        Args:
            user (django.contrib.auth.models.User):
                The user whose infobox is being shown.

            request (django.http.HttpRequest):
                The request for the infobox view.

            local_site (reviewboard.site.models.LocalSite):
                The local site, if any.

            **kwargs (dict):
                Additional keyword arguments.

        Returns:
            dict:
            Additional context to include when rendering the template.
        """
        return {}

    def get_etag_data(self, user, request, local_site, **kwargs):
        """Return data to be included in the user infobox ETag.

        The infobox view uses an ETag to enable browser caching of the content.
        If the extension returns data which can change, this method should
        return a string which is unique to that data.

        Args:
            user (django.contrib.auth.models.User):
                The user whose infobox is being shown.

            request (django.http.HttpRequest):
                The request for the infobox view.

            local_site (reviewboard.site.models.LocalSite):
                The local site, if any.

            **kwargs (dict):
                Additional keyword arguments.

        Returns:
            unicode:
            A string to be included in the ETag for the view.
        """
        return ''

    def render(self, user, request, local_site, **kwargs):
        """Return content to include in the user infobox.

        This may be overridden in the case where providing a custom template
        and overriding :py:func:`get_extra_context` is insufficient.

        Args:
            user (django.contrib.auth.models.User):
                The user whose infobox is being shown.

            request (django.http.HttpRequest):
                The request for the infobox view.

            local_site (reviewboard.site.models.LocalSite):
                The local site, if any.

            **kwargs (dict):
                Additional keyword arguments.

        Returns:
            django.utils.safestring.SafeText:
            Text to include in the infobox HTML.
        """
        assert self.template_name is not None

        context = {
            'extension': self.extension,
            'user': user,
        }
        context.update(self.get_extra_context(user=user,
                                              request=request,
                                              local_site=local_site))
        return render_to_string(
            template_name=self.template_name,
            context=context,
            request=request)


#!/usr/bin/python3
# coding=UTF-8

import sys
import time
import http.client as httplib
import sqlite3
from lxml import etree

# print(str(dbPointer))
# dbPointerr = common.getDBPointer()
# print(str(dbPointerr))

class SqliteWrapper:
    ''' docstring for SqliteWrapper '''
    def __init__(self, tableName):
        self.fundb = sqlite3.connect("./dji.db")
        self.fundb.text_factory = str
        self.dbPointer = self.fundb.cursor()
        self.tableName = tableName
        sql = "CREATE TABLE IF NOT EXISTS " + self.tableName + \
            " (postDate TEXT, postBy VARCHAR(20), device TEXT, postTitle TEXT, postLink TEXT UNIQUE, visitTimes INT, commentTimes INT, updateTime VARCHAR(20))"
        self.dbPointer.execute(sql)

    def fmtDate(self, dateStr):
        if(len(str(dateStr.split()[0]).split('-')) < 3):
            dateStr = str(time.localtime(time.time()).tm_year) + "-" + dateStr.split()[0]
        else:
            dateStr = dateStr.split()[0]
        t = time.strptime(dateStr, "%Y-%m-%d")
        return time.strftime("%Y-%m-%d", t)

    def fmtDateTime(self, dateStr):
        if(len(str(dateStr.split()[0]).split('-')) < 3):
            dateStr = str(time.localtime(time.time()).tm_year) + "-" + dateStr
        else:
            dateStr = dateStr
        t = time.strptime(dateStr, "%Y-%m-%d %H:%M")
        return time.strftime("%Y-%m-%d %H:%M", t)

    # save data to sqlite db
    def saveData(self, postDate, postBy, device, postTitle, postLink, visitTimes, commentTimes, updateTime):
        postDate = self.fmtDate(postDate)
        updateTime = self.fmtDateTime(updateTime)
        # sql = "CREATE TABLE IF NOT EXISTS " + self.tableName + \
        #     " (postDate TEXT, postBy VARCHAR(20), device TEXT, postTitle TEXT, postLink TEXT UNIQUE, visitTimes INT, commentTimes INT, updateTime VARCHAR(20))"
        # self.dbPointer.execute(sql)

        # INSERT data
        # print("deal with: %s, tableName: %s" %(date, tableName))
        try:
            self.dbPointer.execute("INSERT INTO " + self.tableName + \
                    # " VALUES (?, ?, ?, ?, ?)", (date(postDate), postBy, visitTimes, commentTimes, datetime(updateTime)))
                    " VALUES (?, ?, ?, ?, ?, ?, ?, ?)", (postDate, postBy, device, postTitle, postLink, visitTimes, commentTimes, updateTime))
        except:
            pass
            # 修改已经存在的值

        self.fundb.commit()

    def getLastestDate(self):
        self.dbPointer.execute("SELECT * FROM " + self.tableName + " ORDER BY DATE(postDate) DESC")
        return self.dbPointer.fetchone()

    def getRowNum(self):
        self.dbPointer.execute("SELECT COUNT(*) FROM " + self.tableName)
        return self.dbPointer.fetchone()[0]

    # dump data from db
    def dumpDb(self, tableName):
        pass
        self.dbPointer.execute("SELECT * FROM " + tableName)
        while(1):
            result = dbPointer.fetchone()
            if (result != None):
                print(result)
            else:
                break

    def onSale(self, startDate, endDate, onSaleTime):
        if (startDate < onSaleTime) and (onSaleTime < endDate):
            return True
        else:
            return False

    def getNoAirItem(self, itemName, startDate, endDate):
        # self.dbPointer.execute('SELECT * FROM ' + self.tableName + ' where postDate between "' \
        self.dbPointer.execute('SELECT ' + itemName + ' FROM ' + self.tableName + ' where postDate between "' \
                + startDate + '" and "' + endDate + '" and postDate != "' + endDate +  '" and device != "mavic air" ORDER BY postDate DESC')
        itemNames = list(self.dbPointer.fetchall())

        result = []
        for item in itemNames:
            result.append(item[0])
            # result.append(item)
        return result

    def getItem(self, itemName, startDate, endDate):
        # self.dbPointer.execute('SELECT * FROM ' + self.tableName + ' where postDate between "' \
        self.dbPointer.execute('SELECT ' + itemName + ' FROM ' + self.tableName + ' where postDate between "' \
                + startDate + '" and "' + endDate + '" and postDate != "' + endDate +  '" ORDER BY postDate DESC')
        itemNames = list(self.dbPointer.fetchall())

        result = []
        for item in itemNames:
            result.append(item[0])
            # result.append(item)
        return result

    def getDevice(self, startDate, endDate):
        startDate = self.fmtDate(startDate + "-01")
        endDate = self.fmtDate(endDate + "-01")
        # print(startDate)
        # print(endDate)
        airOnSaleTime = "2018-02-01"
        result = []
        if(self.onSale(startDate, endDate, airOnSaleTime)):
            result.extend(self.getNoAirItem("device", startDate, airOnSaleTime))
            result.extend(self.getItem("device", airOnSaleTime, endDate))
        elif (endDate < airOnSaleTime):
            result.extend(self.getNoAirItem("device", startDate, endDate))
        else:
            result.extend(self.getItem("device", startDate, endDate))

        return result

    def getPostLink(self, startDate, endDate):
        startDate = self.fmtDate(startDate + "-01")
        endDate = self.fmtDate(endDate + "-01")
        # print(startDate)
        # print(endDate)
        airOnSaleTime = "2018-02-01"
        result = []
        if(self.onSale(startDate, endDate, airOnSaleTime)):
            result.extend(self.getNoAirItem("postLink", startDate, airOnSaleTime))
            result.extend(self.getItem("postLink", airOnSaleTime, endDate))
        elif (endDate < airOnSaleTime):
            result.extend(self.getNoAirItem("postLink", startDate, airOnSaleTime))
        else:
            result.extend(self.getItem("postLink", startDate, endDate))

        return result


from aiogram.types import Message, CallbackQuery
from aiogram.dispatcher import FSMContext
from aiogram.utils.exceptions import ChatNotFound
import asyncio

from tgbot.database.db import Database
from tgbot.states.admin import SpamState
from tgbot.states.admin import BalanceState
from tgbot.markups.admin import *

db = Database('database.sql')

#Рассылка
async def spam_start(call):
    await call.message.edit_text('Напишите сообщение для рассылки:', reply_markup=back_main)
    await SpamState.waiting_for_spam_text.set()

async def back_to_main(call, state: FSMContext):
    await state.reset_state()  # сбрасывается состояние ожидания сообщения
    await call.message.edit_text('Вы вернулись в главное меню:', reply_markup=main_menu)

async def get_spam_text(message: Message, state: FSMContext):
    text = message.text
    await state.finish()
    users = db.get_users()
    for row in users:
        user_id = row[0]
        try:
            chat = await message.bot.get_chat(user_id)
            await message.bot.send_message(chat.id, text, parse_mode="HTML")
        except ChatNotFound:
            print(f"User {user_id} not found")
        except Exception as e:
            print(f"Failed to send message to user {user_id}. Error: {e}")
    sent_message = await message.answer('Рассылка прошла успешно')
    await asyncio.sleep(1)
    await message.bot.delete_message(chat_id=sent_message.chat.id, message_id=sent_message.message_id)
    await message.answer('Вы в админ-панели', reply_markup=main_menu)

#рассылка с помощью команды
async def spam_cmd(message:Message):
    text = message.text[6:]
    users = db.get_users()
    for row in users:
        user_id = row[0]
        try:
            chat = await message.bot.get_chat(user_id)
            await message.bot.send_message(chat.id, text, parse_mode="Markdown")
        except ChatNotFound:
            print(f"User {user_id} not found")
        except Exception as e:
            print(f"Failed to send message to user {user_id}. Error: {e}")
    await message.answer('Рассылка прошла успешно')

#---Меню пользователи---
#Обработка кнопки пользователи
async def users_call(call):
    quantity = db.get_user_count()
    await call.message.edit_text(f'Количество активных пользователей: {quantity}', reply_markup=users_menu)

#Обработка кнопки активные пользователи
async def active_users_call(call):
    keyboard = create_keyboard()
    await call.message.edit_text('Активные пользователи', reply_markup=keyboard)

#Обработка кнопки забаненные пользователи
async def banned_users_call(call):
    keyboard = create_ban_keyboard()
    await call.message.edit_text('Активные пользователи', reply_markup=keyboard)

async def back_to_main_menu(call):
    await call.message.edit_text('Вы в админ-панели:', reply_markup=main_menu)

#нажатие кнопки назад в блоке ниже меню пользователей(возврат в меню пользователей)
async def back_to_users_call(call):
    await call.message.edit_text('Вы в админ-панели:', reply_markup=users_menu)

#нажатие на кнопку пользователя в блоке активные пользователи
async def user_info_call(call):
    global dm_user_id
    dm_user_id = call.data.split('_')[1]
    user = db.get_user(dm_user_id)
    global user_info_text
    user_info_text = f"{user[1]} (@{user[2]}) - зарегистрирован {user[3],} баланс {user[4]}"

    ban_button = InlineKeyboardButton('Забанить', callback_data=f'ban_user_{dm_user_id}')
    direct_message = InlineKeyboardButton('Личное Сообщение', callback_data='direct_message')
    add_balance =  InlineKeyboardButton('Добавить баланс', callback_data='add_balance')
    back_to_users = InlineKeyboardButton('Назад', callback_data='back_to_users')

    global users_func
    users_func = InlineKeyboardMarkup().add(ban_button, direct_message).add(back_to_users, add_balance)

    await call.message.edit_text(text=call.message.text + f"\n\n{user_info_text}", reply_markup=users_func)

async def add_balance(call):
    await call.message.edit_text('Пополнить баланс на:')
    await BalanceState.waiting_for_add_balance.set()

async def wait_add(message: Message, state: FSMContext):
    user_id = message.from_user.id
    amount = int(message.text)
    await state.finish()
    db.add_balance(user_id, amount)
    await message.answer('Баланс успешно пополнен!')

async def banned_user_info_call(call):
    user_id = call.data.split('_')[2]
    user = db.get_banned_user(user_id)

    text = f"{user[1]} (@{user[2]}) - зарегистрирован {user[3]}"

    ban_button = InlineKeyboardButton('Разбанить', callback_data=f'unban_user_{user_id}')
    back_to_users = InlineKeyboardButton('Назад', callback_data='back_to_banned_users')
    global ban_users_func
    ban_users_func = InlineKeyboardMarkup().add(ban_button).add(back_to_users)

    await call.message.edit_text(text=call.message.text + f"\n\n{text}", reply_markup=ban_users_func)

#обработчик кнопки назад (возврат к клавиатуре с активными польователями)
async def user_back(call):
    keyboard = create_keyboard()
    await call.message.edit_text('Список пользователей:', reply_markup=keyboard)

#тоже самое что выше, только в забанненых
async def banned_user_back(call):
    keyboard = create_ban_keyboard()
    await call.message.edit_text('Список пользователей:', reply_markup=keyboard)

#обработчик кнопки Забанить
async def ban_user(call):
    user_id = call.data.split('_')[2]
    chat = await call.bot.get_chat(user_id)
    username = chat.username

    db.ban_user(user_id)

    sent_message = await call.message.answer(f"Пользователь @{username} забанен!")
    await asyncio.sleep(3)
    await call.bot.delete_message(chat_id=sent_message.chat.id, message_id=sent_message.message_id)

async def unban_user(call):
    user_id = call.data.split('_')[2]
    print(user_id)
    chat = await call.bot.get_chat(user_id)
    username = chat.username

    db.unban_user(user_id)

    sent_message = await call.message.answer(f"Пользователь @{username} разбанен!")
    await asyncio.sleep(3)
    await call.bot.delete_message(chat_id=sent_message.chat.id, message_id=sent_message.message_id)

#обработчик кнопки личное сообщение пользователю

async def direct_message(call):
    await call.message.edit_text('Напишите сообщение для рассылки:', reply_markup=back_users_func)
    await SpamState.waiting_for_dm_text.set()

async def get_direct_message_text(message: Message, state: FSMContext):
    dm_text = message.text
    await state.finish()
    user_id = dm_user_id
    try:
        chat = await message.bot.get_chat(user_id)
        await message.answer(chat.id, dm_text, parse_mode="HTML")
    except ChatNotFound:
        print(f"User {user_id} not found")
    except Exception as e:
        print(f"Failed to send message to user {user_id}. Error: {e}")

    sent_message = await message.answer("Личное сообщение пользователю отправлено.")
    await asyncio.sleep(1)
    await message.bot.delete_message(chat_id=sent_message.chat.id, message_id=sent_message.message_id)
    await message.answer('Вы в админ-панели', reply_markup= main_menu)

async def back_to_user_func(call, state: FSMContext):
    await state.reset_state()  # сбрасывается состояние ожидания сообщения
    await call.message.edit_text(user_info_text, reply_markup=users_func)


#general requirements
import pandas as pd
import numpy as np
import sys
import warnings
import pickle
from collections import Counter
#preprocessing stuff
import ExtraMLFxns as mlf #my personal extra fxns in CustomModules
from sklearn.model_selection import train_test_split, GridSearchCV, GroupShuffleSplit, StratifiedShuffleSplit, cross_validate
from sklearn import preprocessing
from imblearn.over_sampling import SMOTE, SMOTENC
from imblearn.combine import SMOTEENN, SMOTETomek
#classifiers
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier, DistanceMetric
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression, RidgeClassifier, PassiveAggressiveClassifier
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, GradientBoostingClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis, LinearDiscriminantAnalysis
#process results
from sklearn.metrics import roc_curve, auc, recall_score, accuracy_score, precision_score, confusion_matrix, make_scorer, matthews_corrcoef, hamming_loss
warnings.filterwarnings(action="ignore")

name_index = int(sys.argv[1]) #from job array
over_samp = str(sys.argv[2]) #OverSamp, OvUnEEN, OvUnTomek; has to be OverSampNC because of the categorical features
cv_name = str(sys.argv[3]) #GSS for groupShuffleSplit, SSS for StratifiedShuffleSplit
group_name = str(sys.argv[4]) #select from 'cath_class', 'cath_arch', 'scop_class', 'scop_fold', 'ECOD_arch', 'ECOD_x_poshom', 'ECOD_hom'; we settled on using just ECOD_arch
data_subset = str(sys.argv[5]) #All_Sph, All_Shell, Gen, etc
opt_type = str(sys.argv[6]) #Prec Acc MCC Multi
relaxed = str(sys.argv[7]) #Relaxed or UnRelaxed

print(name_index, over_samp, cv_name, group_name, data_subset, opt_type, relaxed)

def scale_resample(X_train, y_train, X_test, over_samp_strat = "OverSampNC"):
    #split for scaling into categorical and not categorical
    not_cat_electro = ("geom_gRMSD", "geom_MaxgRMSDDev","geom_val", "geom_nVESCUM","geom_AtomRMSD")
    geom = [name for name in X_train if name.startswith("geom")]
    cat_data = [x for x in geom if not x in not_cat_electro]
    cat_data.extend(['MetalCodes', 'MetalAtoms', 'SEPocket', "groupID"])
    #print(cat_data)

    X_train_scale = preprocessing.scale( X_train[X_train.columns.difference(cat_data)] )
    scaler = preprocessing.StandardScaler().fit( X_train[X_train.columns.difference(cat_data)] ) #scale the training set
    X_test_scale = scaler.transform( X_test[X_test.columns.difference(cat_data)] ) #scale the test set based on the scale of the training set
    num_norm_cols = X_test_scale.shape[1]
    #add categorical data back
    #print(X_train[[ entry for entry in X_train.columns if entry in cat_data]])
    X_train_scale = np.concatenate((X_train_scale, X_train[[ entry for entry in X_train.columns if entry in cat_data]]), axis = 1)
    X_test_scale = np.concatenate((X_test_scale, X_test[[ entry for entry in X_test.columns if entry in cat_data]]), axis = 1)
    #print(X_test_scale)
    X_test_scale = X_test_scale[:,:-1] #drop out the foldID b/c not needed for X_test
    #print(X_test_scale[0:2,:])

    #can only use oversampling for canonicals
    if over_samp_strat == "OverSampNC":
        smote = SMOTENC(categorical_features= np.arange(num_norm_cols, X_train_scale.shape[1], 1) )
    else:
        print("Invalid oversampling term because of categoricals")
        sys.exit()

    X_train_new, y_train_new = smote.fit_resample(X_train_scale, y_train.Catalytic)
    #print(X_train_new.shape, Counter(y_train_new))
    fold_ids = X_train_new[:,-1] #last column is the foldID converted to numeric to assign to groups
    X_train_new = X_train_new[:,:-1] #drop the foldID out for final training

    print("Finished processing data")
    return(X_train_new, y_train_new, X_test_scale, fold_ids)

prec_score = make_scorer(mlf.prec_score_custom, greater_is_better = True)
mcc_score = make_scorer(mlf.mcc_score, greater_is_better = True)
tnr_score = make_scorer(mlf.tnr_score, greater_is_better = True)
hamm_score = make_scorer(mlf.hamming_score, greater_is_better = True)
if opt_type == "Prec":
    this_scoring = prec_score
elif opt_type == "Acc":
    this_scoring = "accuracy"
elif opt_type == "MCC":
    this_scoring = mcc_score
elif opt_type == "Multi":
    this_scoring = {"Acc":'accuracy', "MCC": mcc_score, "Hamm": hamm_score }
else:
    print("Invalid scoring term")
    sys.exit()

if cv_name == "GSS":
    cv_type = GroupShuffleSplit(n_splits=10)
elif cv_name == "SSS":
    cv_type = StratifiedShuffleSplit(n_splits=10)
else:
    cv_type = 10 #uses KFold by default in the GridSearchCV

fold_names = ['cath_class', 'cath_arch', 'scop_class', 'scop_fold', 'ECOD_arch', 'ECOD_x_poshom', 'ECOD_hom'] #there is data available relatively easily to use another fold type, but we settled on ECOD_arch as the best
if group_name != "ECOD_arch":
    print("Invalid fold selection, you can only pick ECOD_arch")
    sys.exit()

names = ["LogRegr", "Ridge", "PassAggr",
        "QDA", "LDA", "NaiveBayes",
        "NearNeigh",
        "LinSVM", "RBFSVM", "SigSVM",
        "RandomForest", "ExtraTrees", "GradBoost",
        "NeurNet"]

classifiers = [
    LogisticRegression(solver="liblinear", class_weight = "balanced", tol = 1e-4),
    RidgeClassifier(solver="auto", class_weight = "balanced", tol = 1e-4),
    PassiveAggressiveClassifier(class_weight = "balanced", tol = 1e-4),
    QuadraticDiscriminantAnalysis(priors=None), #also almost no real parameters to adjust
    LinearDiscriminantAnalysis(solver = "lsqr"),
    GaussianNB(priors=None), #only one real parameter to adjust!
    KNeighborsClassifier(algorithm = "ball_tree", weights = 'distance'),
    SVC(kernel="linear", max_iter=10000, class_weight = "balanced", tol = 1e-4),
    SVC(kernel="rbf", class_weight = "balanced", tol = 1e-4),
    SVC(kernel="sigmoid", class_weight = "balanced", tol = 1e-4),
    RandomForestClassifier(class_weight = "balanced", min_samples_split = 5, bootstrap = True),
    ExtraTreesClassifier(class_weight = "balanced", min_samples_split = 5, bootstrap = True),
    GradientBoostingClassifier(subsample = 0.8, criterion = 'friedman_mse', min_samples_split = 5, validation_fraction=0.2, n_iter_no_change=5, tol=0.01,),
    MLPClassifier(learning_rate_init = 0.01, early_stopping = True, n_iter_no_change = 5),
]

#read features
wp_scores = pd.read_table("WholeScores_%s.txt"%relaxed) #this now includes only ECOD_arch as the foldID
#Catalytic = 0 are enzymatic, Catalytic = 1 are non-catalytic
wp_scores['Catalytic'] = wp_scores['Catalytic'].apply(lambda x: 0 if x == True else 1) #flipped so that Catalytic are in the "positive" position of CFM
wp_scores = wp_scores.dropna(subset = ["SITE_ID"])
wp_scores.drop_duplicates(inplace = True)
#pull the validation set out separately
valid_set = wp_scores[wp_scores.ValidSet == True]
wp_scores = wp_scores[wp_scores.ValidSet == False]

#split into features and classification
X, y = mlf.subset_data(wp_scores, data_subset, group_name)
valid_set, y_valid = mlf.subset_data(valid_set, data_subset, group_name)
print(len(X), len(valid_set))

parameter_space = [
    { "C": np.logspace(0, 3, 4), "penalty": ["l1", "l2"] }, #LogRegr
    { "alpha": np.logspace(-4, 1, 6) }, #Ridge
    { "C": np.logspace(-1, 3, 5) }, #PassAggr
    { "reg_param": np.linspace(0.5, 1, 7) }, #QDA
    { "shrinkage": ["auto", 0, 0.1, 0.25, 0.5, 0.75, 1] }, #LDA
    { "var_smoothing": np.logspace(-9, 0, 10) }, #Gauss
    [ {"metric": ["minkowski"], "p":[2, 3], "n_neighbors": [5, 8, 10, 15]}, {"metric": ["chebyshev"], "n_neighbors": [5, 8, 10, 15]} ], #kNN
    { "C": np.logspace(-3, 1, 5) }, #SVC lin
    { "C": np.logspace(-3, 1, 5), "gamma": ["scale", "auto"] }, #SVC rbf
    { "C": np.logspace(-3, 1, 5),"gamma": ["scale", "auto"] }, #SVC sig
    { "max_features": ["auto", "sqrt", "log2"], "max_depth": [3, 4, 5, 8, 9, 10], "n_estimators": [100, 500, 1000, 2500, 5000] }, #RF
    { "max_features": ["auto", "sqrt", "log2"], "max_depth": [3, 4, 5, 8, 9, 10], "n_estimators": [100, 500, 1000, 2500, 5000], 'criterion': ["gini", 'entropy'] }, #ExtraTrees
    { "loss": ['deviance', 'exponential'], 'learning_rate': np.logspace(-4, 0, 10), "n_estimators": [100, 500, 1000, 2500, 5000], "max_features": ["auto", "sqrt", "log2"] },#GBClassifier
    { "hidden_layer_sizes": [(50,), (75,), (100,), (150,), (200,), (250,)], "activation": ["relu", "logistic", "tanh"], "alpha": np.logspace(-5, -1, 6) } #MLPClass
]

name = names[name_index]
this_clf = classifiers[name_index]
these_params = parameter_space[name_index]
#if we didn't have a separate validation group
"""outer_cv_results = []
for i, (train_idx, test_idx) in enumerate(cv_type.split(X,y.Catalytic, groups = X["groupID"])):
    print(i)
    X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]
    #print(train_idx[0:10], X_train.head(10))
    y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]
    #print(train_idx, y_train.head(10))

    #scale and sample features
    X_train_new, y_train_new, X_test_scale, fold_ids = scale_resample(X_train, y_train, X_test, over_samp)
    #print(X_train_new[0:10], y_train_new[0:10])
    #print(X_train_new)
    #run feature selection and CV
    clf = GridSearchCV(estimator = this_clf, n_jobs = -1, cv=cv_type, param_grid = these_params, scoring = this_scoring, iid = True, refit = False)
    clf.fit(X_train_new, y_train_new, groups = fold_ids)
    results = clf.cv_results_
    #somehow get best combination of multiple scoring terms
    #print(results)
    ranks = []
    for key in results:
        if "rank_test_" in key:
            ranks.append(results[key])
    best_params = results['params'][np.argmin(np.sum(np.asarray(ranks), axis = 0))]
    #print(best_params) #best params will have to be identified for full data set for final model building after best model is selected
    outer_clf = this_clf.set_params(**best_params) #set the new classifier to these parameters
    outer_clf.fit(X_train_new, y_train_new) #fit on all training data - this is what GridSearchCV(refit = True) will do anyways
    y_test = y_test.assign(Prediction=outer_clf.predict( X_test_scale)) #predict based on fitted outer CV model
    accuracy = accuracy_score(y_test.Catalytic, y_test.Prediction)#, sample_weight = y_test.SampleWeights)
    recall = recall_score(y_test.Catalytic, y_test.Prediction, pos_label=0) #calculates for catalytic = 1 which is now neg
    precision = precision_score(y_test.Catalytic, y_test.Prediction, pos_label=0)#, sample_weight = y_test.SampleWeights)
    true_neg_rate = len( y_test[(y_test.Catalytic == 1) & (y_test.Prediction == 1)] )/ len(y_test[(y_test.Catalytic == 1)])
    hamming = hamming_loss(y_test.Catalytic, y_test.Prediction)
    mcc = matthews_corrcoef(y_test.Catalytic, y_test.Prediction)
    dist_rand = (recall + -1*(1-true_neg_rate)) / np.sqrt(2)

    outer_cv_results.append([ accuracy, precision, recall, true_neg_rate, hamming, mcc, dist_rand ])
    #print([ accuracy, precision, recall, true_neg_rate, hamming, mcc, dist_rand ])
outer_cv_results = np.asarray(outer_cv_results)
print("Not applied to valid set:", outer_cv_results)
#print(np.mean(outer_cv_results, axis = 0))
#print(np.std(outer_cv_results, axis = 0))"""

outer_cv_results = []
for x in range(0,10):
    X_train_new, y_train_new, valid_scale, fold_ids = scale_resample(X, y, valid_set, over_samp)
    #print(X_train_new[0:10], y_train_new[0:10])
    #print(X_train_new)
    #run feature selection and CV
    clf = GridSearchCV(estimator = this_clf, n_jobs = -1, cv=cv_type, param_grid = these_params, scoring = this_scoring, iid = True, refit = False)
    clf.fit(X_train_new, y_train_new, groups = fold_ids)
    results = clf.cv_results_
    #somehow get best combination of multiple scoring terms
    #print(results)
    ranks = []
    for key in results:
        if "rank_test_" in key:
            ranks.append(results[key])
    best_params = results['params'][np.argmin(np.sum(np.asarray(ranks), axis = 0))]
    print(best_params) #best params will have to be identified for full data set for final model building after best model is selected
    outer_clf = this_clf.set_params(**best_params) #set the new classifier to these parameters
    outer_clf.fit(X_train_new, y_train_new) #fit on all training data - this is what GridSearchCV(refit = True) will do anyways, but its selection of params is not necessary mine

    y_valid = y_valid.assign(Prediction=outer_clf.predict( valid_scale)) #predict based on fitted outer CV model
    accuracy = accuracy_score(y_valid.Catalytic, y_valid.Prediction)#, sample_weight = y_valid.SampleWeights)
    recall = recall_score(y_valid.Catalytic, y_valid.Prediction, pos_label=0) #calculates for catalytic = 1 which is now neg
    precision = precision_score(y_valid.Catalytic, y_valid.Prediction, pos_label=0)#, sample_weight = y_valid.SampleWeights)
    true_neg_rate = len( y_valid[(y_valid.Catalytic == 1) & (y_valid.Prediction == 1)] )/ len(y_valid[(y_valid.Catalytic == 1)])
    hamming = hamming_loss(y_valid.Catalytic, y_valid.Prediction)
    mcc = matthews_corrcoef(y_valid.Catalytic, y_valid.Prediction)
    dist_rand = (recall + -1*(1-true_neg_rate)) / np.sqrt(2)
    outer_cv_results.append([ accuracy, precision, recall, true_neg_rate, hamming, mcc, dist_rand ])
    #print([ accuracy, precision, recall, true_neg_rate, hamming, mcc, dist_rand ])
outer_cv_results = np.asarray(outer_cv_results)
#print("No outer CV:", outer_cv_results)

with open("ClassifierScores/%s/ClassifierScores_%s_%s_%s_%s_%s_%s_%s.txt"%(over_samp, relaxed, data_subset, cv_name, group_name[0:4], over_samp, opt_type, name), "w+") as outData:
    outData.write("Classifier\tAccuracy\tPrecision\tRecall\tTrueNegRate\tHammingLoss\tMCC\tDistRand\tAcc_std\tPrec_std\tRecall_std\tTrueNegRate_std\tHammingLoss_std\tMCC_std\tDistRand_std\n")
    outData.write(name + "\t" + "\t".join(map(str, np.mean(outer_cv_results, axis = 0))) + "\t" + "\t".join(map(str, np.std(outer_cv_results, axis = 0))) + "\n")


import hashlib
import os
from abc import ABC, abstractmethod
from typing import AnyStr, Callable, Dict, List, Optional, Set, Tuple, TypeVar, Union
from urllib.parse import urlparse

import lxml.html as lh
import pandas as pd
import requests
from lxml.html import Element

from feb_stats.core.entities import Boxscore, Game, League, Player, Team

T = TypeVar("T")

class GenericParser(ABC):
    id: int
    name: str
    season_stats: Optional[pd.DataFrame] = None

    @staticmethod
    def parse_str(input_str: Union[str, bytes], decode_bytes: bool = False) -> str:
        if decode_bytes:
            input_str = bytes(str(input_str), "iso-8859-1").decode("utf-8")
        assert isinstance(input_str, str)
        return " ".join(
            input_str.replace("\n", " ")
            .replace("\t", " ")
            .replace("\r", " ")
            .replace(",", ".")
            .split()
        ).strip()

    @staticmethod
    def get_elements(doc: Element, id: str) -> List[Element]:
        # Parse data by id
        table_elements: List[Element] = doc.xpath(id)
        return table_elements

    def create_league(self, all_games: List[Game], all_teams: Set[Team]) -> League:
        return League(
            id=hashlib.md5(f"{all_games[0].league}".encode("utf-8")).hexdigest(),
            name=all_games[0].league,
            season=all_games[0].season,
            teams=list(all_teams),
            games=all_games,
        )

    def create_objects(
        self, metadata: Dict[str, str], game_stats: Dict[str, pd.DataFrame]
    ) -> Tuple[Game, Tuple[Team, Team]]:
        home_team: Team = Team(
            id=hashlib.md5(metadata["home_team"].encode("utf-8")).hexdigest(),
            name=metadata["home_team"],
        )
        away_team: Team = Team(
            id=hashlib.md5(metadata["away_team"].encode("utf-8")).hexdigest(),
            name=metadata["away_team"],
        )
        game: Game = Game(
            id=hashlib.md5(
                f"{metadata['league']}{metadata['date']}{metadata['home_team']}{metadata['away_team']}".encode(
                    "utf-8"
                )
            ).hexdigest(),
            date=metadata["date"],
            time=metadata["time"],
            league=metadata["league"],
            season=metadata["season"],
            home_team=home_team,
            home_score=int(metadata["home_score"]),
            away_team=away_team,
            away_score=int(metadata["away_score"]),
            main_referee=Player(
                id=hashlib.md5(metadata["main_referee"].encode("utf-8")).hexdigest(),
                name=metadata["main_referee"],
            ),
            aux_referee=Player(
                id=hashlib.md5(metadata["second_referee"].encode("utf-8")).hexdigest(),
                name=metadata["second_referee"],
            ),
            local_boxscore=Boxscore(
                id=hashlib.md5(
                    f"{metadata['league']}_{metadata['date']}_{metadata['home_team']}".encode(
                        "utf-8"
                    )
                ).hexdigest(),
                boxscore=game_stats["home_boxscore"],
            ),
            away_boxscore=Boxscore(
                id=hashlib.md5(
                    f"{metadata['league']}_{metadata['date']}_{metadata['away_team']}".encode(
                        "utf-8"
                    )
                ).hexdigest(),
                boxscore=game_stats["away_boxscore"],
            ),
        )
        return game, (home_team, away_team)

    def parse_boxscores(
        self, boxscores: List[AnyStr], reader_fn: Optional[Callable] = None
    ) -> League:
        all_games = []
        all_teams = set()
        reader_fn = reader_fn or self.read_link_bytes
        for link in boxscores:
            doc = reader_fn(link)
            game, teams = self.parse_game_stats(doc)
            all_games.append(game)
            for team in teams:
                all_teams.add(team)

        if all_games:
            return self.create_league(all_games, all_teams)
        else:
            raise ValueError(f"No games found in {boxscores}")

    @staticmethod
    def read_link_bytes(link: bytes) -> Element:
        document_string = link.decode("latin1")
        doc = lh.fromstring(document_string)
        return doc

    @staticmethod
    def read_link_file(link: str) -> Element:
        if isinstance(link, str):
            result = urlparse(link)
            if all([result.scheme, result.netloc, result.path]):
                page = requests.get(link)
                # Store the contents of the website under doc
                document_string = lh.fromstring(page.content)
            elif os.path.isfile(link):
                with open(link, mode="r", encoding="latin1") as f:
                    document_string = f.read()
            else:
                document_string = link

        doc = lh.fromstring(document_string)

        return doc

    @abstractmethod
    def elements_to_df(
        self, tr_elements: List[Element], initial_row: int = 2, n_elem: int = 0
    ) -> pd.DataFrame:
        pass

    @abstractmethod
    def parse_game_metadata(self, doc: Element) -> Dict[str, str]:
        pass

    @abstractmethod
    def parse_game_stats(
        self, doc: Element, ids: Optional[Union[List[Tuple[str, bool]], str]] = None
    ) -> Tuple[Game, Tuple[Team, Team]]:
        pass


# -*- coding: utf-8 -*-
"""
Created on Tue May 6 16:33:55 2019

@author: 19038067
"""
import numpy as np
import cv2
from matplotlib.colors import hex2color
from skimage import color

# 构建reference图片的噪声的部分
def _make_noisy_ref(src, hex):
    src_gray = cv2.imread(src, cv2.IMREAD_GRAYSCALE)
    src_gray = cv2.resize(src_gray, (100, 100), interpolation=cv2.INTER_AREA)
    c_map = (src_gray - src_gray.mean()) / src_gray.std()

    rgb_pix = hex2color(hex)

    # 完成涂色
    im_tar = np.ones((100, 100, 3), dtype=np.float)
    for i in range(3):
        im_tar[:, :, i] = rgb_pix[i]

    # 加扰动，这一块纯经验值，可以看心情改
    im_tar = color.rgb2hsv(im_tar)
    if 0.45 < im_tar[0, 0, 0] < 0.55:       # 浅蓝色单独处理
        im_tar[:, :, 2] += c_map * 0.06
    elif im_tar[0, 0, 1] > 0.95:    # S>0.95
        im_tar[:, :, 2] += c_map * 0.05     # 亮度加扰动
    elif im_tar[0, 0, 1] > 0.9:     # 0.95>S>0.9
        im_tar[:, :, 2] += c_map * 0.002    # 亮度加扰动
    elif im_tar[0, 0, 1] > 0.8:     # 0.9 >S >0.8
        im_tar[:, :, 1] += c_map * 0.06     # 饱和度加扰动
    else:                           # S<0.8
        im_tar[:, :, 1] += c_map * 0.08     # 饱和度加扰动

    im_tar = np.uint8(color.hsv2rgb(im_tar) * 255)
    return im_tar

# Reinhard的目标颜色版本
def Reinhard(img, hex):
    image = cv2.imdecode(np.fromfile(src, dtype=np.uint8), -1)
    if image.shape[-1] == 4:
        alpha = image[:, :, 3:]

    image = cv2.cvtColor(image, cv2.COLOR_BGR2LAB)
#     alpha = np.reshape(image[:,:,-1], (image.shape[0],-1,1))
    original = _make_noisy_ref(src, hex)
    original = cv2.cvtColor(original, cv2.COLOR_BGR2LAB)

    image_avg, image_std = np.mean(image,(0,1)), np.std(image,(0,1))
    original_avg, original_std = np.mean(original,(0,1)), np.std(original,(0,1))

    image = (image-image_avg)*(original_std/image_std)+(original_avg)
    image = np.where(image>255,255,image)
    image = np.where(image<0, 0, image)
    image = image.astype(np.uint8)

    image = cv2.cvtColor(image, cv2.COLOR_LAB2BGR)

    if alpha is not None:
        image = np.concatenate([image, alpha], -1)
    return image

if __name__ == '__main__':
    img = Image.fromarray(Reinhard('orignial.png','#f1f2f3'))
    img.save('Reinhard.png')