import os
import django

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sds.settings")
django.setup()
from rest_framework import serializers
from instructor.inspectdb import *
import json
from pathlib import Path
import glob

django_content_dict= {}
django_content = DjangoContentType.objects.values()
for x in django_content:
    django_content_dict.update({x['model']:x['id']})

def user_iterator(serializeddata, userid):
    for k,v in serializeddata.items():
        if type(v) is list:
            for vl in v:
                user_iterator({k: vl}, userid)
        elif type(v) is dict:
            contentuid = v.get('id', None)
            kunderscoreless = k.replace('_', '')
            contenttypeid = django_content_dict[kunderscoreless]
            adminloguserid = DjangoAdminLog.objects.filter(content_type_id = contenttypeid, object_id = contentuid).last()
            if adminloguserid:
                adminloguserid= adminloguserid.user_id
                authuser = AuthUser.objects.values('id', 'first_name', 'last_name').get(id = adminloguserid)
                v.update({'creator': str(authuser['id'])+'_'+authuser['first_name']+'_'+authuser['last_name']})
                for kk, vv in v.items():
                    user_iterator({kk:vv}, userid)
        else:
            pass
        return serializeddata

def student_analysis(studentdata):
    analysis = {}
    for dp in studentdata['datapoints']:
        dpid = dp['id']
        analysis.update({dpid:[]})
        num_condition = len(dp['conditions'])
        if num_condition == 0:
            analysis[dpid].append('No condition')
        for con in dp['conditions']:
            significand = con['significand']
            significanddig = significand.replace('.', '')
            lensignificand = len(significanddig)
            accuracy = con['accuracy']
            if lensignificand == accuracy:
                pass
            else:
                analysis[dpid].append('condition significand accuracy discrepancy')
        num_data = len(dp['data'])
        if num_data == 0:
            analysis[dpid].append('No data')
        for con in dp['data']:
            significand = con['significand']
            significanddig = significand.replace('.', '')
            lensignificand = len(significanddig)
            accuracy = con['accuracy']
            if lensignificand == accuracy:
                pass
            else:
                analysis[dpid].append('data significand accuracy discrepancy')
        if dp.get('datasets', None):
            if dp['datasets'].get('reports', None):
                if r'\u' in json.dumps(dp['datasets']['reports']['method']):
                    # analysis[dpid].append('encoding character in report method')
                    pass
                if dp['datasets']['reports'].get('references_reports', None):
                    for refrep in dp['datasets']['reports']['references_reports']:
                        if r'\u' in json.dumps(refrep['references']['citation']):
                            pass
                            # analysis[dpid].append('encoding character in reference citation')
                else:
                    analysis[dpid].append('no references_reports')
            else:
                analysis[dpid].append('no report')
        else:
            analysis[dpid].append('no dataset')
        if not analysis[dpid]:
            analysis.pop(dpid)

    dpwithprobs = len(analysis)
    dpwithoutprobs = len(studentdata['datapoints']) - dpwithprobs
    analysisquant = {
        'total datapoints': str(len(studentdata['datapoints'])),
        'datapoints without problems': str(dpwithoutprobs),
        'datapoints with problems': str(dpwithprobs)
    }
    analysis = {**analysisquant, **analysis}

    studentdata = {**{'analysis':analysis}, **studentdata}
    return studentdata

path = "/Users/n01448636/Documents/GoogleDrive/PycharmProjects/UNF_CIS_Solubility/instructor/dump/"
filenames = [os.path.basename(x) for x in glob.glob(path+'*.jsonld')]
# filenames = ['student_4_Mikal_Graves.jsonld'] # override

for filename in filenames:
    username = filename.replace('.', '_')
    username = username.split('_')
    username = username[1]+'_'+username[2]+'_'+username[3]
    with open(path+filename) as studentfile:
        studentdata = json.load(studentfile)
        userid = filename.split('_')[1]
        studentdata = user_iterator(studentdata, userid)
        studentdata = student_analysis(studentdata)

    base = Path('analysis')
    jsonpath = base / ('analysis_' + str(username) + '.jsonld')

    try:
        base.mkdir(parents=True, exist_ok=True)
        jsonpath.write_text(json.dumps(studentdata))
        # print(json.dumps(serialized_data))
        print("Completed: student_" + str(username))
    except:
        print('Invalid JSON-LD')


"""Test class for setting up AMMs."""
from enum import Enum
from math import isclose
from typing import List, NewType, Tuple

from sympy import symbols

from Arbie import PoolValueError
from Arbie.Variables.token import Balance, Balances, Token, Tokens

x = symbols("x")

def get_value(values: Balances, token: Token) -> Balance:
    for v in values:
        if v.token == token:
            return v.value

class PoolType(Enum):
    uniswap = 0
    balancer = 1
    unknown = 99

class Pool(object):
    """Pool can create a arbitrary AMM that can be used for testing.

    The math can be found here https://balancer.finance/whitepaper/
    """

    def __init__(
        self,
        tokens: Tokens,
        balances: List[float],
        weights: List[float],
        pool_type: PoolType = PoolType.unknown,
        fee: float = 0,
        **kwargs,
    ):  # noqa: WPS221

        self.tokens = tokens
        self.balances = Balance.create(tokens, balances)
        self.weights = Balance.create(tokens, weights)
        self.fee = fee
        self.pool_type = pool_type

        self.address = None
        if "address" in kwargs:
            self.address = kwargs.get("address")

        if not isclose(sum(weights), 1, abs_tol=1e-3):  # noqa: WPS432
            raise PoolValueError(
                f"Weights are not normalized, sum is {sum(weights)} for pool {self.address}"
            )

        self._check_balances()

        if self.fee > 1 or self.fee < 0:
            raise PoolValueError(
                f"Fee: {self.fee}, should be between 0 and 1. Fee is {self.fee}, for pool {self.address}"
            )

    def __str__(self):
        return f"""
Pool(
  Tokens: {self.tokens},
  Balances: {self.balances},
  Weights: {self.weights},
  Fee: {self.fee}
  Address: {self.address})"""

    def __repr__(self):
        return self.__str__()

    def __hash__(self):
        return hash(self.address)

    def update_balances(self, balances: List[float]):
        self.balances = Balance.create(self.tokens, balances)
        self._check_balances()

    def get_weights(self, token_in: Token, token_out: Token) -> Tuple[float, float]:
        return (get_value(self.weights, token_in), get_value(self.weights, token_out))

    def get_balances(self, token_in: Token, token_out: Token) -> Tuple[float, float]:
        return (get_value(self.balances, token_in), get_value(self.balances, token_out))

    def spot_price(self, token_in: Token, token_out: Token) -> float:
        """Ratio between token_in and token_out.

        A ratio > 1 means that token_in is less valuable then token_out
        A ratio < 1 means that token_in is more valuable then token_out

        token_in / ratio = token_out

        If the ratio is 400 then it takes 400 token_in for 1 token_out
        """
        bi, bo = self.get_balances(token_in, token_out)
        wi, wo = self.get_weights(token_in, token_out)
        return (bi / wi) / (bo / wo)

    def in_given_out_expr(self, token_in: Token, token_out: Token, k=1):
        bi, bo = self.get_balances(token_in, token_out)
        wi, wo = self.get_weights(token_in, token_out)

        bi *= k
        bo *= k

        return bi * ((bo / (bo - x * (1 - self.fee))) ** (wo / wi) - 1)  # noqa: WPS221

    def out_given_in_expr(self, token_in: Token, token_out: Token, k=1) -> float:
        """Mathematical expression for out given in.

        To increase the accuracy of floating point numbers the parameter k can be provided.
        """
        bi, bo = self.get_balances(token_in, token_out)
        wi, wo = self.get_weights(token_in, token_out)

        bi *= k
        bo *= k

        return bo * (1 - (bi / (bi + x * (1 - self.fee))) ** (wi / wo))  # noqa: WPS221

    def in_given_out(self, token_in: Token, token_out: Token, amount: float) -> float:
        """Mathematical expression for in given out.

        To increase the accuracy of floating point numbers the parameter k can be provided.
        """
        expr = self.in_given_out_expr(token_in, token_out)
        return expr.subs(x, amount)

    def out_given_in(self, token_in: Token, token_out: Token, amount: float) -> float:
        expr = self.out_given_in_expr(token_in, token_out)
        return expr.subs(x, amount)

    def _check_balances(self):
        for balance in self.balances:
            if isclose(balance.value, 0, abs_tol=1e-3):  # noqa: WPS432
                raise PoolValueError(f"Balance {balance} ~ 0 for pool {self.address}")

Pools = NewType("Pools", List[Pool])


# from YTpage.youtubeDetails import video_comments
from YTpage.Vedor_sentiment import Sentiment_by_vedor
from YTpage.comment_scraper import list_of_comments
import time,re,json
from bs4 import BeautifulSoup as bs

from googletrans import Translator
# from translate import Translator
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from progress.bar import Bar

import pickle
import nltk
import sys
import nltk
import sklearn
import pandas
import numpy

import pandas as pd
import numpy as np
from sklearn.preprocessing import LabelEncoder
from nltk.corpus import stopwords

from nltk.tokenize import word_tokenize

#load data for find_features
df = pd.read_csv('./smallDataset.csv',encoding = "utf-8")

classes = df['label']
encoder = LabelEncoder()
Y = encoder.fit_transform(classes)

text_messages = df['comments'].fillna("")
processed = text_messages.str.lower()

stop_words = set(stopwords.words('english'))

processed = processed.apply(lambda x: ' '.join(
    term for term in x.split() if term not in stop_words))

ps = nltk.PorterStemmer()

processed = processed.apply(lambda x: ' '.join(
    ps.stem(term) for term in x.split()))

# create bag-of-words
all_words = []

for message in processed:
    words = word_tokenize(message)
    for w in words:
        all_words.append(w)

all_words = nltk.FreqDist(all_words)

# print('Number of words: {}'.format(len(all_words)))
# print('Most common words: {}'.format(all_words.most_common(15)))

word_features = list(all_words.keys())[:200]

filename = 'final_model.sav'
loaded_model = pickle.load(open(filename, 'rb'))

def vedor_Result(v_id,comment_count):
	commentsListFiltter = []
	c_count = int(comment_count)
	comments = list_of_comments(v_id,c_count)
	commentsList = []
	# RE_EMOJI = re.compile('[\U00010000-\U0010ffff]', flags=re.UNICODE)
	bad_char ={',','?','&','*','@',':','!','(',')','/','-','#','.',">","<",'_'}
	pos_val =0
	neg_val=0
	comp = 0

	fake_count = 0
	normal_count = 0
	totalCommLimit = 0

	if not comments:
		comments = ['none']
	else:
		pass
	try:
		bar = Bar('Translating:', max=len(comments))
		bar.start()
		translator = Translator(service_urls=['translate.google.co.in'])

		file = open("commLimit.txt", "r")
		totalCommLimit = int(file.read() )
		file.close()

		for c in comments:
			senWithEmoji = c.lower()
			# print(c)
			# print(senWithEmoji)
			if len(c) < 2:
				comments.remove(c)
			# emojiFil = RE_EMOJI.sub(r'', senWithEmoji)
			emojiFil =senWithEmoji.encode('ascii', 'ignore').decode('ascii')
			emojiFil = emojiFil.replace("\n"," ")
			emojiFil = emojiFil.replace("\r","")
			emojiFil = re.sub(r'[0-9]+','',emojiFil)
			emojiFil = re.sub(r'http\S+', '', emojiFil)

			for i in bad_char:
				emojiFil = emojiFil.replace(i,"")
			# print(emojiFil)

			totalCommLimit += len(emojiFil.split())
			with open("commLimit.txt", "w") as f:
				f.write(str(totalCommLimit))
			##Error if used more
			# t = translator.translate(emojiFil).text
			# translator= Translator(to_lang="en")
			# print(totalCommLimit)
			# t = translator.translate(emojiFil).text
			# trans_text = t.lower()

			if totalCommLimit <= 10677:
				t = translator.translate(emojiFil).text
				trans_text = t.lower()
				# print("if: ",trans_text)
			else:
				trans_text = emojiFil.lower()
				# print("else: ",trans_text)

			analyser = SentimentIntensityAnalyzer()

			score = analyser.polarity_scores(trans_text)
			commentResult = modelResult(trans_text)

			# print(trans_text,'--',commentResult)

			if commentResult == 1:
				fake_count += 1
			else:
				normal_count += 1
			# print(c ,"pos_val -->",score['pos'],"neg_val -->",score['neg'])
			pos_val += score['pos']
			neg_val +=score['neg']
			comp += score['compound']
			bar.next()
		bar.finish()

	except json.decoder.JSONDecodeError:
		print("limit of translator")

	if len(comments) == 0:
		return pos_val,neg_val,comp,fake_count
	else:
			# print("final pos: ",pos_val,"final neg: ",neg_val, "compound", comp)
		finalPosVal = ((pos_val)/len(comments))*100
		finalNegVal = ((neg_val)/len(comments))*100
		finalComp = (comp)/len(comments)
		fakePercent = int((fake_count/len(comments))*100)
		normalPercent = int((normal_count/len(comments))*100)
		# print()
		# print("final pos: ",finalPosVal,"final neg: ",finalNegVal, "compound", finalComp)

	return round(finalPosVal,2),round(finalNegVal,2),finalComp,fakePercent,normalPercent

def find_features(message):
    words = word_tokenize(message)
    features = {}
    for word in word_features:
        features[word] = (word in words)

    return features

def modelResult(text):
	feat = find_features(text)
	result = loaded_model.classify(feat)
	return result


import pandas as pd
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
import numpy as np

data = pd.read_csv('C:/Users/USER/Documents/test.csv',index_col=0) # data 불러오기
data.rename(columns={'0':'image_path'},inplace=True) # column name 변경

def cvt_pathname(path):
    path = path.replace('\\','/') #=> '\' => '/' 로 이름 정렬해줌
    return path

data['image_path'] = data['image_path'].apply(cvt_pathname)  # 'image_path' apply함수 이용해서 정렬

## Label 정리하기

def get_label(path):
    label = path.split('/')[-2] # '/' 로 split 해서 list형식으로 바꿔주고 폴더명으로 label 가져옴
    return label
data['label'] = data['image_path'].apply(get_label) #마찬가지 image_path 이용해서 위의 함수로 label 뽑아냄
print(len(data['label'].unique())) # 갯수확인
data['label'].value_counts().plot(kind='barh')

label_encoder = LabelEncoder() # sckit-learn label encoder 활용 string 형식 label 숫자로 변환
data['label'] = label_encoder.fit_transform(np.array(data['label'])) # fit_transform 이용 문자열 label 수치형(int)으로 변환
# label_encoder.inverse_transform(data['label'])
# data['label'].value_counts().plot(kind='barh')


from sklearn.linear_model import LogisticRegression
from models.ShallowLearning.ShallowModel import ShallowModel

import numpy as np
import logging
from src.Utils import path

class LogRegress(ShallowModel):
    """ LogisticRegression wrapper function. Child of Shallow Model

    Attributes
    ----------
    model : LogisticRegression() from sci-kit learn
        Logistic Regression model with appropriate parameters. User can adjust if necessary.

    hyperpameters: Dict
        List of available hyperparameters to choose from

    Methods
    -------
    train(x,y)
        fits the model with sci-kit learn's 'fit' funciton
    predict(x)
        predicts values for new data 'x'

    """
    model = None
    hyperparameters = None

    def __init__(self, l_penalty='l2', l_C=1.0, l_solver='liblinear', l_max_iter=1000, multi_class='auto', n_jobs=1):
        super(LogRegress, self).__init__(name="Logistic Regression")
        self.model = LogisticRegression(solver=l_solver, max_iter=l_max_iter, multi_class=multi_class, n_jobs=n_jobs)
        logging.basicConfig(filename=path + "reports\\" + "log_reg.log", level=logging.info)
        logging.info("Logistic Regression Log created")

    def train(self, x, y):
        super(LogRegress, self).train(x, y)

    def predict(self, x):
        return super(LogRegress, self).predict(x)

    def optimize(self, x, y, cv=5, verbose=0):
        penalty = ['l1', 'l2']
        C = np.logspace(0, 4, 10)
        self.hyperparameters = dict(C=C, penalty=penalty)

        # Current Best
        # Penalty: L1
        # Best C: 1291.5496650148827
        bf = super(LogRegress, self).optimize(x, y, cv=cv)
        print("Best Penalty: ", bf.best_estimator_.get_params()['penalty'])
        print("Best C: ", bf.best_estimator_.get_params()['C'])

    def analyze(self):
        pass


from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import Session

SQL_ALCHEMY_URL ="sqlite:///./blog.db"
engine  = create_engine(SQL_ALCHEMY_URL, connect_args={"check_same_thread": False})

SessionLocal = sessionmaker(bind=engine, autocommit=False, autoflush=False)

Base = declarative_base()

def get_db() -> Session:
    db = SessionLocal()
    try:
        yield db
    finally:
        db.close()


#! /usr/bin/env python3
# -*- coding: utf-8 -*-

import csv
import requests
from bs4 import BeautifulSoup

csvfile = "./Python_Web_Crawler/Python_從初學到生活應用超實務/05/Elements.csv"
# csvfile = "./Elements.csv"
url = "https://fchart.github.io/Elements.html"

response = requests.get(url)

if response.status_code == 200:
    soup = BeautifulSoup(response.text, "lxml")

    with open(csvfile, encoding="utf-8", mode="w", newline='') as fp:
        writer = csv.writer(fp)
        writer.writerow(["Qestion", "Answer1", "Answer2"])

        tags = soup.find_all("li", class_="question")

        for tag in tags:
            tag_q = tag.find("b")
            ques = tag_q.text
            tag_a = tag.find_all("li", class_="response")
            writer.writerow([ques, tag_a[0].text, tag_a[1].text])

else:
    print("HTTP Request Rrror")


import aiohttp
import asyncio

async def get_response(url):

    async with aiohttp.ClientSession() as session:
        async with session.get(url) as response:

            html = await response.text()
            return html

loop = asyncio.get_event_loop()
n = 100
responses = [get_response('http://127.0.0.1:8000') for _ in range(n)]
print(*loop.run_until_complete(asyncio.gather(*responses)))
loop.close()


from os.path import join as path_join

from PIL import Image
from six import StringIO

from screener.settings import (
    SCREENSHOT_WIDTH,
    SCREENSHOT_HEIGHT,
)
from screener.utils.decorators import validate_path

WINDOW_BOX = (0, 0, SCREENSHOT_WIDTH, SCREENSHOT_HEIGHT)
IMAGE_EXT = 'jpg'
IMAGE_TYPE = 'JPEG'
IMAGE_QUALITY = 95

logger = None
LOGGER_NAME = __name__

@validate_path(ext=IMAGE_EXT)
def save_as_jpg(image_date, folder, filename):
    new_filename = '{fname}.{ext}'.format(fname=filename, ext=IMAGE_EXT)
    file_path = path_join(folder, new_filename)
    image = Image.open(StringIO(image_date))
    logger.debug('Cropping and saving image')
    cropped_image = image.crop(WINDOW_BOX)
    cropped_image.save(
        file_path,
        IMAGE_TYPE,
        optimize=True,
        quality=IMAGE_QUALITY
    )


"""
Imports the various procedure backends
"""

from typing import Set

from ..exceptions import InputError, ResourceError
from .berny import BernyProcedure
from .geometric import GeometricProcedure
from .nwchem_opt import NWChemDriverProcedure
from .optking import OptKingProcedure
from .torsiondrive import TorsionDriveProcedure
from .model import ProcedureHarness

__all__ = ["register_procedure", "get_procedure", "list_all_procedures", "list_available_procedures"]

procedures = {}

def register_procedure(entry_point: ProcedureHarness) -> None:
    """
    Register a new ProcedureHarness with QCEngine
    """

    name = entry_point.name
    if name.lower() in procedures.keys():
        raise ValueError("{} is already a registered procedure.".format(name))

    procedures[name.lower()] = entry_point

def get_procedure(name: str) -> ProcedureHarness:
    """
    Returns a procedures executor class
    """

    name = name.lower()

    if name not in procedures:
        raise InputError(f"Procedure {name} is not registered to QCEngine.")

    ret = procedures[name]
    if not ret.found():
        raise ResourceError(f"Procedure {name} is registered with QCEngine, but cannot be found.")

    return ret

def list_all_procedures() -> Set[str]:
    """
    List all procedures registered by QCEngine.
    """
    return set(procedures.keys())

def list_available_procedures() -> Set[str]:
    """
    List all procedures that can be exectued (found) by QCEngine.
    """

    ret = set()
    for k, p in procedures.items():
        if p.found():
            ret.add(k)

    return ret

register_procedure(GeometricProcedure())
register_procedure(OptKingProcedure())
register_procedure(BernyProcedure())
register_procedure(NWChemDriverProcedure())
register_procedure(TorsionDriveProcedure())


from __future__ import print_function
from __future__ import absolute_import
import os
import sys
os.environ['DJANGO_SETTINGS_MODULE'] = 'astrometry.net.settings'
p = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(p)
import settings

import django
django.setup()

from astrometry.net.models import *
from astrometry.util.file import *
from astrometry.util.multiproc import *
from log import *

from django.contrib.auth.models import User

import logging
logging.basicConfig(format='%(message)s',
                    level=logging.DEBUG)

def delete_user(u):
    print('Deleting user', u)
    u.user_images.all().delete()
    u.submissions.all().delete()
    u.albums.all().delete()
    u.comments_left.all().delete()
    try:
        u.profile.delete()
    except:
        import traceback
        traceback.print_exc()
    u.delete()
    print('Deleted user', u)

def bounce_try_dojob(X):
    jobid, solve_command, solve_locally = X
    try:
        from process_submissions import try_dojob
        print('Trying Job ID', jobid)
        job = Job.objects.filter(id=jobid)[0]
        print('Found Job', job)
        r = try_dojob(job, job.user_image, solve_command, solve_locally)
        print('Job result for', job, ':', r)
        return r
    except:
        import traceback
        traceback.print_exc()

def main():
    import optparse
    parser = optparse.OptionParser('%(prog)')
    parser.add_option('-s', '--sub', type=int, dest='sub', help='Submission ID')
    parser.add_option('-j', '--job', type=int, dest='job', help='Job ID')
    parser.add_option('-u', '--userimage', type=int, dest='uimage', help='UserImage ID')
    parser.add_option('-e', '--email', help='Find user id with given email address')
    parser.add_option('-U', '--userid', help='Find user with given numerical ID', type=int)
    parser.add_option('-i', '--image', type=int, dest='image', help='Image ID')
    parser.add_option('-d', '--disk-file', type=str, dest='df', help='DiskFile id')
    parser.add_option('-r', '--rerun', dest='rerun', action='store_true',
                      help='Re-run this submission/job?')
    parser.add_option('--list-socials', default=False, action='store_true')
    parser.add_option('--list-profiles', default=False, action='store_true')

    parser.add_option('--threads', type=int, help='Re-run failed jobs within this process using N threads; else submit to process_submissions process.')

    parser.add_option('--chown', dest='chown', type=int, default=0, help='Change owner of userimage or submission by user id #')

    parser.add_option('--solve-command',
                      help='Command to run instead of ssh to actually solve image')
    parser.add_option('--solve-locally',
                      help='Command to run astrometry-engine on this machine, not via ssh')

    parser.add_option('--ssh', action='store_true', default=False,
              help='Find submissions whose jobs have ssh errors')
    parser.add_option('--minsub', type='int', default=0,
              help='Minimum submission id to look at')

    parser.add_option('--empty', action='store_true', default=False,
              help='Find submissions whose jobs have no log files')

    parser.add_option('--delete', action='store_true', default=False,
              help='Delete everything associated with the given job/submission/image')

    parser.add_option('--delete-sub', action='store_true', default=False,
                      help='When deleting a Job/UserImage, also delete the Submission?')

    parser.add_option('--deluser', action='store_true', default=False,
                      help='Delete everything associated with the given user')

    parser.add_option('--delextra', action='store_true', default=False,
                      help='Delete extraneous duplicate jobs?')

    parser.add_option('--hide', action='store_true', default=False,
                      help='For a UserImage, set publicly_visible=False')

    opt,args = parser.parse_args()

    if opt.list_socials:
        #from social.apps.django_app.default.models import UserSocialAuth
        ## ????
        from social_django.models import UserSocialAuth
        socs = UserSocialAuth.objects.order_by('created')
        for soc in socs:
            print('soc for user', soc.user.id, soc.user, 'provider', soc.provider, 'uid', soc.uid, 'extra', soc.extra_data)
        sys.exit(0)
    if opt.list_profiles:
        pros = UserProfile.objects.all()
        for pro in pros:
            print('userid', pro.user.id, 'display', pro.display_name, 'apikey', pro.apikey)
        sys.exit(0)

    if opt.email:
        users = User.objects.filter(email__contains=opt.email)
        print('Users with email containing "%s":' % opt.email)
        for u in users:
            print(u.id, u, u.email)
        if opt.deluser:
            delete_user(u)
        sys.exit(0)

    if opt.userid:
        users = User.objects.filter(id=opt.userid)
        print('Users with ID', opt.userid)
        for u in users:
            print(u.id, u, u.email)
            print(u.profile)
            print(u.profile)
            for k in ['email', 'first_name', 'last_name', 'profile', 'social_auth', 'username']:
                print(' ',k,getattr(u,k))
            for f in ['get_full_name', 'get_short_name', 'get_username',]:
                print(' ',f,getattr(u,f)())

            print(' submissions:')
            for s in u.submissions.all().order_by('-id'):
                print('    ', s.id)
        if opt.deluser:
            delete_user(u)
        sys.exit(0)

    if not (opt.sub or opt.job or opt.uimage or opt.image or opt.ssh or opt.empty or opt.df):
        print('Must specify one of --sub, --job, or --userimage or --image (or --ssh or --empty)')

        parser.print_help()
        sys.exit(-1)

    if opt.threads is not None:
        mp = multiproc(opt.threads)
    else:
        mp = None

    if opt.ssh or opt.empty or opt.delextra:
        subs = Submission.objects.all()
        if opt.minsub:
            subs = subs.filter(id__gt=opt.minsub)
        subs = subs.order_by('-id')
        failedsubs = []
        failedjobs = []
        for sub in subs:
            print('Checking submission', sub.id)
            allfailed = True
            # last failed Job
            failedjob = None
            uis = sub.user_images.all()
            for ui in uis:
                jobs = ui.jobs.all()
                for job in jobs:
                    print('  job', job.id)
                    if job.status == 'S':
                        print('    -> succeeded')
                        allfailed = False
                        break
                    print('    error msg', job.error_message)
                    logfn = job.get_log_file()
                    if not os.path.exists(logfn):
                        failedjob = job
                        continue

                    if opt.ssh:
                        log = read_file(logfn)
                        # 'Connection refused'
                        # 'Connection timed out'
                        if not (('ssh: connect to host astro.cs.toronto.edu port 22:' in log) or
                                ('ssh: Could not resolve' in log)):
                            allfailed = False
                            break
                        print('SSH failed')
                        failedjob = job

                    if opt.empty:
                        # exists but empty?
                        if file_size(logfn) == 0:
                            failedjob = job
                            continue
                        # log file found
                        allfailed = False
                        break

                if opt.delextra:
                    print('Delextra:', len(jobs), 'jobs', len(uis), 'uis; failedjob:', failedjob)
                    if len(jobs) > 1 and failedjob is not None:
                        print('Delextra: delete', failedjob)

            if not allfailed:
                continue
            print('All jobs failed for sub', sub.id) #, 'via ssh failure')
            #failedsubs.append(sub)
            failedjobs.append(failedjob)

        print('Found total of', len(failedsubs), 'failed Submissions and', len(failedjobs), 'failed Jobs')
        if opt.rerun:
            from process_submissions import try_dosub, try_dojob
            import logging
            logformat = '%(message)s'
            logging.basicConfig(level=logging.DEBUG, format=logformat)

            if opt.threads is not None:
                args = []
                for j in failedjobs:
                    if j is None:
                        continue
                    args.append((j.id, opt.solve_command, opt.solve_locally))
                mp.map(bounce_try_dojob, args)
            else:
                for job in failedjobs:
                    if job is None:
                        continue
                    print('Re-trying job', job.id)
                    try_dojob(job, job.user_image, opt.solve_command, opt.solve_locally)

            # FIXME -- failed subs??
            #
            # else:
            #     for sub in failedsubs:
            #         print('Re-trying sub', sub.id)
            #         try_dosub(sub, 1)

    if opt.df:
        dfs = DiskFile.objects.filter(file_hash=opt.df)
        print('Found', dfs.count(), 'matching DiskFiles')
        for df in dfs:
            print('  ', df)
            print('    ', df.get_path())
            ims = Image.objects.filter(disk_file=df)
            print('  Used by', ims.count(), 'Image objects')
            for im in ims:
                print('    ', im, im.id)
                print('      size %i x %i' % (im.width, im.height))
                print('      thumb Image', im.thumbnail, (im.thumbnail and im.thumbnail.id or ''))
                print('      display Image', im.display_image, (im.display_image and im.display_image.id or ''))

            ims = Image.objects.filter(disk_file=df,
                                       display_image__isnull=False,
                                       thumbnail__isnull=False)
            print('  found', ims.count(), 'Images for this df w/ display and thumbnails')

    if opt.sub:
        sub = Submission.objects.all().get(id=opt.sub)
        print('Submission', sub)
        if sub.disk_file is None:
            print('  no disk file')
        else:
            print('Path', sub.disk_file.get_path())
            print('Is fits image:', sub.disk_file.is_fits_image())
            print('Is fits image:', sub.disk_file.file_type)
        print('User:', sub.user)
        uis = sub.user_images.all()
        print('UserImages:', len(uis))
        for ui in uis:
            print('  ', ui)
            print('  user', ui.user)
            print('  with Jobs:', len(ui.jobs.all()))
            for j in ui.jobs.all():
                print('    ', j)

        if opt.chown:
            newuser = User.objects.all().get(id=opt.chown)
            for ui in uis:
                ui.user = newuser
                ui.save()
            sub.user = newuser
            sub.save()

        if opt.rerun:
            from process_submissions import try_dosub, dosub
            print('Re-trying sub', sub.id)
            #try_dosub(sub, 1)
            dosub(sub)

        if opt.delete:
            print('Deleting submission', sub)
            sub.delete()

    if opt.job:
        job = Job.objects.all().get(id=opt.job)
        print('Job', job)
        print(job.get_dir())
        print('Status:', job.status)
        print('Error message:', job.error_message)
        ui = job.user_image
        print('UserImage:', ui)
        print('User', ui.user)
        im = ui.image
        print('Image', im)
        sub = ui.submission
        print('Submission', sub)
        print(sub.disk_file.get_path())

        if opt.rerun:
            from astrometry.net.process_submissions import try_dojob
            print('Re-trying job', job.id)
            try_dojob(job, ui, opt.solve_command, opt.solve_locally)

        if opt.delete:
            print('Deleting job', job)
            job.delete()

            if opt.delete_sub:
                print('Deleting submission', sub)
                sub.delete()

    if opt.uimage:
        ui = UserImage.objects.all().get(id=opt.uimage)
        print('UserImage', ui)
        im = ui.image
        print('Image', im)
        sub = ui.submission
        print('User', ui.user)
        print('Submission', sub)
        print(sub.disk_file.get_path())

        if opt.chown:
            user = User.objects.all().get(id=opt.chown)
            print('User:', user)
            print('chowning', ui, 'to', user)
            ui.user = user
            ui.save()

        if opt.delete:
            print('Deleting ui', ui)
            ui.delete()
        if opt.delete_sub:
            print('Deleting sub', sub)
            sub.delete()
        if opt.hide:
            print('Hiding ui', ui)
            ui.hide()

    if opt.image:
        im = Image.objects.all().get(id=opt.image)
        # thumbnail
        # display_image
        print('Image:', im, im.id)

        #uis = im.userimage_set()

        uis = UserImage.objects.all().filter(image=im.id)
        print('UserImages:', uis)

        print('Thumbnail:', im.thumbnail)
        print('Display:', im.display_image)

        if opt.delete:
            print('Deleting...')
            im.delete()
            if im.thumbnail:
                im.thumbnail.delete()
            if im.display_image:
                im.display_image.delete()

if __name__ == '__main__':
    main()


from sklearn.datasets import make_moons, make_circles
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from sklearn.preprocessing import MinMaxScaler
from sklearn.tree import DecisionTreeClassifier
from sklearn.utils import check_random_state

from groot.toolbox import Model
from groot.visualization import plot_estimator

from robust_relabeling import relabel_model

import numpy as np

import matplotlib.pyplot as plt

import seaborn as sns

import argparse

def make_tree(n_samples, random_state=None):
    random_state = check_random_state(random_state)

    def ground_truth_tree(sample):
        if sample[0] <= 0.5:
            if sample[1] <= 0.5:
                return 0
            else:
                return 1
        else:
            if sample[1] <= 0.5:
                return 1
            else:
                return 0

    X = random_state.uniform(size=(n_samples, 2))
    y = np.apply_along_axis(ground_truth_tree, 1, X)

    return X, y

parser = argparse.ArgumentParser()
parser.add_argument("--output_dir", type=str, default="out/")
parser.add_argument("--epsilon", type=float, default=0.04)
args = parser.parse_args()

sns.set_theme(style="whitegrid", palette="colorblind")

for name, (X, y), (X_test, y_test) in zip(
    ("moons", "circles", "tree"),
    (
        make_moons(n_samples=200, noise=0.3, random_state=1),
        make_circles(n_samples=200, factor=0.5, noise=0.2, random_state=1),
        make_tree(n_samples=200, random_state=1),
    ),
    (
        make_moons(n_samples=200, noise=0.3, random_state=2),
        make_circles(n_samples=200, factor=0.5, noise=0.2, random_state=2),
        make_tree(n_samples=200, random_state=2),
    ),
):
    X = MinMaxScaler().fit_transform(X)

    random_state = check_random_state(1)
    y = np.where(random_state.rand(len(y)) > 0.95, 1 - y, y)

    for model_name, classifier in zip(
        ("tree", "forest", "boosting"),
        (
            DecisionTreeClassifier(max_depth=5, min_samples_leaf=3, random_state=1),
            RandomForestClassifier(n_estimators=100, random_state=1),
            GradientBoostingClassifier(n_estimators=100, max_depth=5, random_state=1),
        ),
    ):
        classifier.fit(X, y)

        model = Model.from_sklearn(classifier)

        plot_estimator(X, y, model, steps=500)
        plt.xticks([])
        plt.yticks([])
        plt.tight_layout()
        plt.savefig(f"{args.output_dir}{name}_{model_name}.png", dpi=200)
        plt.savefig(f"{args.output_dir}{name}_{model_name}.pdf")
        plt.close()

        relabel_model(model, X, y, args.epsilon)

        plot_estimator(X, y, model, steps=500)
        plt.xticks([])
        plt.yticks([])
        plt.tight_layout()
        plt.savefig(f"{args.output_dir}{name}_{model_name}_relabeled.png", dpi=200)
        plt.savefig(f"{args.output_dir}{name}_{model_name}_relabeled.pdf")
        plt.close()


# coding: utf-8

# ### Download the Automobile data set
# <b>Download link:</b> https://archive.ics.uci.edu/ml/machine-learning-databases/autos/imports-85.data

# In[1]:

from pyspark.sql import SparkSession

spark = SparkSession     .builder     .appName('Predicting the price of an automobile given a set of features')     .getOrCreate()

rawData = spark.read            .format('csv')            .option('header', 'true')            .load('../datasets/imports-85.data')

# In[2]:

rawData.toPandas().head()

# #### Select the required columns
# * We can select the specific features we feel are relevant in our dataset
# * fields such as normalized-losses have been dropped
# * The numeric fields can be cast as float or any numeric type

# In[3]:

from pyspark.sql.functions import col

dataset = rawData.select(col('price').cast('float'),
                         col('make'),
                         col('num-of-doors'),
                         col('body-style'),
                         col('drive-wheels'),
                         col('wheel-base').cast('float'),
                         col('curb-weight').cast('float'),
                         col('num-of-cylinders'),
                         col('engine-size').cast('float'),
                         col('horsepower').cast('float'),
                         col('peak-rpm').cast('float')
                        )

# In[4]:

dataset.toPandas().head()

# #### Drop columns with nulls
# Check number of rows in dataset before and after removal of nulls

# In[5]:

dataset.count()

# In[6]:

dataset = dataset.replace('?', None).dropna(how='any')

# In[7]:

dataset.count()

# #### Split dataset into training and test sets

# In[8]:

(trainingData, testData) = dataset.randomSplit([0.8,0.2])

# #### List the categorical fields so that we can transform these to encoded values

# In[9]:

categoricalFeatures = ['make',
                       'num-of-doors',
                       'body-style',
                       'drive-wheels',
                       'num-of-cylinders'
                      ]

# #### Import and implement the required transformers

# In[10]:

from pyspark.ml.feature import StringIndexer
from pyspark.ml.feature import OneHotEncoder
from pyspark.ml.feature import VectorAssembler

# #### Use of handleInvalid in StringIndexer
# If the model comes across a new label which it hasn't seen in the training phase, it is deemed an "invalid" label. There are different ways of handling this:
# * handleInvalid='skip' will remove rows with new labels
# * handleInvalid='error' will cause an error when a new label is encountered
# * handleInvalid='keep' will create a new index if it encounters a new label (available from Spark 2.2 onwards)

# In[11]:

indexers = [StringIndexer(
    inputCol=column,
    outputCol=column + '_index',
    handleInvalid='keep') for column in categoricalFeatures]

# #### One-Hot-Encode the features

# In[12]:

encoders = [OneHotEncoder(
    inputCol=column + '_index',
    outputCol= column + '_encoded') for column in categoricalFeatures]

# #### List all the required features from the transformed dataset

# In[13]:

requiredFeatures = ['make_encoded',
                    'num-of-doors_encoded',
                    'body-style_encoded',
                    'drive-wheels_encoded',
                    'wheel-base',
                    'curb-weight',
                    'num-of-cylinders_encoded',
                    'engine-size',
                    'horsepower',
                    'peak-rpm'
                   ]

# #### Prepare the feature assembler

# In[14]:

assembler = VectorAssembler(inputCols=requiredFeatures, outputCol='features')

# #### Linear Regression
#
#
#
# By setting α properly, elastic net contains both L1 and L2 regularization as special cases.
# * If the elasticNetParam α is set to 1, it is equivalent to a Lasso model
# * If α is set to 0, the trained model reduces to a ridge regression model
#
# regParam is the regularization variable

# In[15]:

from pyspark.ml.regression import LinearRegression

lr = LinearRegression(maxIter=100,
                      regParam=1.0,
                      elasticNetParam=0.8,
                      labelCol='price',
                      featuresCol='features')

# #### Define our pipeline
# It contains all our transformers plus the model

# In[16]:

from pyspark.ml import Pipeline

pipeline = Pipeline(stages=indexers + encoders + [assembler, lr])

# In[17]:

model = pipeline.fit(trainingData)
model

# ### Extract the model from the pipeline
# * Our LinearRegression model is not the same as the pipeline model
# * To extract the LinearRegression model, we get it from the last stage of our pipeline model

# In[18]:

lrModel = model.stages[-1]

# In[19]:

print('Training RMSE = ', lrModel.summary.rootMeanSquaredError)
print('Training R^2 score = ', lrModel.summary.r2)

# #### Check the number of features
# The number will be high as many of our features are one-hot-encoded

# In[20]:

lrModel.numFeatures

# #### View the coefficients of each feature

# In[21]:

lrModel.coefficients

# #### There is a coefficient for each feature

# In[22]:

len(lrModel.coefficients)

# #### Get predictions using our model on the test data

# In[23]:

predictions = model.transform(testData)
predictionsDF = predictions.toPandas()
predictionsDF.head()

# #### The features have been transformed to LibSVM format

# In[24]:

predictionsDF['features'][0]

# ### Use RegressionEvaluator to evaluate the model
# * MulticlassClassificationEvaluator is used for classification models
# * RegressionEvaluator needed to evaluate regression models
# * <b>metricName </b>can be r2, rmse, mse or mae (mean absolute error)

# In[25]:

from pyspark.ml.evaluation import RegressionEvaluator

evaluator = RegressionEvaluator(
    labelCol='price',
    predictionCol='prediction',
    metricName='r2')

r2 = evaluator.evaluate(predictions)
print('Test R^2 score = ', r2)

# In[26]:

evaluator = RegressionEvaluator(
    labelCol='price',
    predictionCol='prediction',
    metricName='rmse')

rmse = evaluator.evaluate(predictions)
print('Test RMSE = ', rmse)

# #### Compare the actual and predicted values of price

# In[27]:

predictionsPandasDF = predictions.select(
    col('price'),
    col('prediction')
).toPandas()

# In[28]:

predictionsPandasDF.head()

# #### Plot a graph of actual and predicted values of price
# Note that our predictions dataset is sorted in ascending order of price

# In[29]:

import matplotlib.pyplot as plt

plt.figure(figsize=(15,6))

plt.plot(predictionsPandasDF['price'], label='Actual')
plt.plot(predictionsPandasDF['prediction'], label='Predicted')

plt.ylabel('Price')
plt.legend()

plt.show()

# ### Using ParamGrid for hyperparameter tuning
# The parameters we wish to tweak are:
# * maxIter
# * regParam
# * elasticNetParam - whether a lasso or ridge model will be best

# In[30]:

from pyspark.ml.tuning import ParamGridBuilder

paramGrid = ParamGridBuilder().addGrid(
    lr.maxIter, [10,50,100]).addGrid(
    lr.regParam, [0.1, 0.3, 1.0]).addGrid(
    lr.elasticNetParam, [0.0, 1.0]).build()

# #### Define the RegressionEvaluator used to evaluate the models
# We wish to minimize RMSE

# In[31]:

evaluator = RegressionEvaluator(
    labelCol='price',
    predictionCol='prediction',
    metricName='rmse')

# ### Define the CrossValidator
# This is used to put all the pieces together
# * <b>estimator: </b>Can be a standalone estimator or a pipeline with an estimator at the end. We use our pipeline
# * <b>estimatorParamMaps: </b>We add our paramGrid in order to build models with different combinations of the parameters
# * <b>evaluator: </b>To evaluate each model, we specify our evaluator

# In[32]:

from pyspark.ml.tuning import CrossValidator

crossval = CrossValidator(estimator=pipeline,
                          estimatorParamMaps=paramGrid,
                          evaluator=evaluator,
                          numFolds=3)

# #### Train each of our models with the training data
# After identifying the best ParamMap, CrossValidator finally re-fits the Estimator using the best ParamMap and the entire dataset

# In[33]:

model = crossval.fit(trainingData)

# #### To examine our best model, we extract it from the pipeline

# In[34]:

lrModel = model.bestModel.stages[-1]
lrModel

# #### Get the values of the "best" parameters
# Unfortunately, extracting these values is a bit awkward as we need to access the \_java\_obj object

# In[35]:

print('maxIter=', lrModel._java_obj.getMaxIter())
print('elasticNetParam=', lrModel._java_obj.getElasticNetParam())
print('regParam=', lrModel._java_obj.getRegParam())

# #### Make predictions using our "best" model

# In[36]:

predictions = model.transform(testData)
predictionsDF = predictions.toPandas()
predictionsDF.head()

# #### Evaluate the model on it's R-square score and RMSE

# In[37]:

evaluator = RegressionEvaluator(
    labelCol='price',
    predictionCol='prediction',
    metricName='r2')

rsquare = evaluator.evaluate(predictions)
print("Test R^2 score = %g" % rsquare)

# In[38]:

evaluator = RegressionEvaluator(
    labelCol='price',
    predictionCol='prediction',
    metricName='rmse')

rmse = evaluator.evaluate(predictions)
print('Test RMSE = ', rmse)

# #### Compare actual and predicted values of price

# In[39]:

predictionsPandasDF = predictions.select(
    col('price'),
    col('prediction')).toPandas()

predictionsPandasDF.head()

# #### Perform the comparison using a graph

# In[40]:

plt.figure(figsize=(15,6))

plt.plot(predictionsPandasDF['price'], label='Actual')
plt.plot(predictionsPandasDF['prediction'], label='Predicted')

plt.ylabel('Price')
plt.legend()

plt.show()

Building Machine Learning Models in Spark 2/python/m2_demo03_LinearRegression.py


'''Criar um programa que permite que o usuário entre com os vértices e arestas de um grafo e
mostre diversas propriedades deste gráfico(quanto mais propriedade, mais PIC$, a expectativa
é que mostre ao menos umas 10 propriedades). Entre as propriedades que podem ser apresentadas estão:
grau máximo, grau mínimo, número cromático, raio, diâmetro, perímetro, árvore geradora mínima, etc .
 O programa pode ainda indicar se o grafo é conexo, acíclico, completo, Euleriano, Hamiltoniano, etc
 (isso também são propriedades). O programa deve oferecer ao usuário a possibilidade de abrir grafos
 já prontos que ilustrem algumas destas propriedade.'''

import networkx as nx
import matplotlib.pyplot as plt

class Grafos:

    def __init__(self,vertices):
        #construtor
        self.vertices = vertices
        self.grafo = [[0]*self.vertices for i in range(self.vertices)]

    def adiciona_aresta_simples(self, u, v, peso):
        #pensando em grafos direcionados simples
        self.grafo[u-1][v-1] = peso

    def adiciona_aresta_mult(self, u, v, peso):
        #pensando em grafos múltiplo
        self.grafo[u-1][v-1] += peso
        if u != v:
            self.grafo[v-1][u-1] += peso

    def adiciona_aresta_sime(self, u, v):
        #pensando em grafos não direcionados, ou de relação simétrica
        self.grafo[u-1][v-1] += 1
        if u != v:
            self.grafo[v-1][u-1] += 1

    def mostra_matriz(self):
        print('A matriz de adjacência é: ')
        for i in range(self.vertices):
            print(self.grafo[i])

    def plot_grafo(self):
        #cria um grafo vazio
        G = nx.Graph()

        # Coloca os vértices e arestas no grafo G
        #G.add_node((2,3,4))

        G.add_edges_from([(2,3), (2,5), (1,2)])
        nx.draw(G, pos=nx.spring_layout(G), with_labels = True, edge_color='r', node_color = ['blue'], arrows=True,arrowstyle = '-|>',
        arrowsize =  15)
        plt.show()

    def tem_aresta(self,u,v):
        if self.grafo[u-1][v-1] == 0:
            print(f'Não tem aresta entre os vértices {u} e {v}')
        else:
            print(f'Existe {self.grafo[u-1][v-1]} de arestas entre os vértices {u} e {v}')

    def eh_euleriano(self):
        contador = 0
        for i in range(self.vertices):
            grau = 0
            for j in range(self.vertices):
                if i == j:
                    grau = grau + 2 * self.grafo[i][j]
                else:
                    grau += self.grafo[i][j]
            if grau % 2 != 0:
                contador += 1
        if contador == 0:
            print('É um grafo euleriano!')
        elif contador == 2:
            print('É um grafo semieuleriano!')
        else:
            print('O grafo não é euleriano e nem semieuleriano!')

# tg = str(input('Digite "s" para Grafo Simples ou "m" para Grafo Multiplos" ou "d" para Grafo Simétrico: '))  #tipo de grafo

# v = int(input('Digite a quantidade de Vértices: '))
# g = Grafos(v)
# a = int(input('Digite a quantidade de Arestas: '))

# if tg == 's':
#     for i in range(a):
#         u = int(input('De qual Vértice parte esta Aresta? '))
#         v = int(input('De qual Vértice chega esta Aresta? '))
#         p = int(input('Qual é o peso desta Aresta? '))
#         g.adiciona_aresta_simples(u,v,p)
# elif tg == 'm':
#     for i in range(a):
#         u = int(input('De qual Vértice parte esta Aresta? '))
#         v = int(input('De qual Vértice chega esta Aresta? '))
#         p = int(input('Qual é o peso desta Aresta? '))
#         g.adiciona_aresta_mult(u, v, p)
# elif tg == 'd':
#     for i in range(a):
#         u = int(input('De qual Vértice parte esta Aresta? '))
#         v = int(input('De qual Vértice chega esta Aresta? '))
#         g.adiciona_aresta_sime(u, v)

# g.mostra_matriz()
# g.eh_euleriano()

# instanciano objeto
g = Grafos(4)

g.adiciona_aresta_mult(1, 2,5)
g.adiciona_aresta_mult(3, 4,3)
g.adiciona_aresta_mult(2, 3,1)
g.adiciona_aresta_mult(1,4,7)

#g.tem_aresta(2,4)

g.eh_euleriano()
g.tem_aresta(1,4)
g.mostra_matriz()

g.plot_grafo()


from __future__ import annotations

from dataclasses import dataclass

import math

from typing import TYPE_CHECKING

if TYPE_CHECKING:  # https://adamj.eu/tech/2021/05/13/python-type-hints-how-to-fix-circular-imports/
    from domain import IDataDto

@dataclass
class IValue:
    color: str
    title: str
    icon: str | None = None
    from_CG: bool | None = None
    value: str | None = None
    numerator: int | None = None
    denominator: int | None = None

    def __post_init__(self):
        if self.denominator:
            self.percent_rank = math.ceil(self.numerator / self.denominator * 1000) / 10

@dataclass
class IProfileDto:
    username: str
    active_color: str
    passive_color: str
    main_rank: str
    score: int
    level: IValue
    certifications: list[IValue]
    language: IValue
    puzzle_solved: IValue
    achievements: IValue
    rank: IValue
    competition: IValue
    bot_battle: IValue
    lang_list: list[IValue]
    leaderboard: list[IValue]

    @staticmethod
    def from_user(user: IDataDto) -> IProfileDto:  # pragma: no cover
        level_value = user.get_score_level()
        certif_value = user.get_score_certificate()
        top_language_value = user.get_score_best_language()
        puzzle_solved_value = user.get_score_total_solved()
        achievements_value = user.get_score_achievements()
        rank_value = user.get_score_rank()
        bot_value = user.get_score_competition(online=False)
        comp_value = user.get_score_competition(online=True)
        lang_list = user.get_score_list_language()
        leaderboard_list = user.get_score_list_leaderboard()

        (active_color, passive_color, score, label) = user.get_main_level()

        return IProfileDto(
            username=user.user.get_pseudo(),
            active_color=active_color,
            passive_color=passive_color,
            main_rank=label,
            level=level_value,
            certifications=certif_value,
            language=top_language_value,
            puzzle_solved=puzzle_solved_value,
            achievements=achievements_value,
            rank=rank_value,
            competition=comp_value,
            bot_battle=bot_value,
            score=score,
            lang_list=lang_list,
            leaderboard=leaderboard_list
        )


"""Support for Seeed 4 relay hat."""
import voluptuous as vol
import logging

from homeassistant.components.switch import PLATFORM_SCHEMA, SwitchEntity
from homeassistant.const import CONF_NAME, CONF_SWITCHES
import homeassistant.helpers.config_validation as cv
#from . import DOMAIN
from . import relay_lib_seeed

_LOGGER = logging.getLogger(__name__)

CONF_PORTS = "ports"

_SWITCHES_SCHEMA = vol.Schema({cv.positive_int: cv.string})

PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
    {
        vol.Required(CONF_PORTS): _SWITCHES_SCHEMA,
        #vol.Optional(CONF_INVERT_LOGIC, default=DEFAULT_INVERT_LOGIC): cv.boolean,
    }
)

def setup_platform(hass, config, add_entities, discovery_info=None):
  #  port = hass.data[DOMAIN]

    switches =  []
    ports = config.get(CONF_PORTS)
    for port, name in ports.items():
        switches.append(seeed_relay_boardSwitch(name, port))
    add_entities(switches)

class seeed_relay_boardSwitch(SwitchEntity):

    def __init__(self, name, port):
        self._name = name
        self._state = False
        self._port = port
        #self._state = options[CONF_INITIAL]

    @property
    def name(self):
        return self._name

    @property
    def is_on(self):
        return self._state

    def turn_on(self, **kwargs):
        relay_lib_seeed.relay_on(self._port)
        self._state = True

    def turn_off(self, **kwargs):
        relay_lib_seeed.relay_off(self._port)
        self._state = False


from appi2c.ext.database import db
from appi2c.ext.group.group_models import Group
from flask.globals import current_app
from werkzeug.utils import secure_filename
from flask_login import current_user
import os
import imghdr

def create_group(name: str, description: str, file: str, user: int):
    group = Group(name=name, description=description, file=file, user_id=user)
    db.session.add(group)
    db.session.commit()
    return group

def list_all_group(user):
    group = Group.query.filter_by(user_id=user.id).all()
    return group

def list_all_devices_in_group(group):
    devices_in_group = group.devices
    return devices_in_group

def delete_group_id(id):
    from appi2c.ext.device.device_controller import list_num_devices_in_group
    group = Group.query.filter_by(id=id).first()
    list_device = list_num_devices_in_group(group)
    if list_device == 0:
        db.session.delete(group)
        db.session.commit()
        return True
    else:
        return False

def num_group():
    group = Group.query.all()
    num_group = len(group)
    return num_group

def num_group_user(user):
    group = Group.query.filter_by(user_id=user.id).all()
    num_group = len(group)
    return num_group

def list_group_id(id: int) -> Group:
    group = Group.query.filter_by(id=id).first()
    return group

def choice_query():
    group = Group.query.filter_by(user_id=current_user.id).all()
    return group

def update_group(id: int, name: str, description: str, file: str):
    Group.query.filter_by(id=id).update(dict(name=name, description=description, file=file))
    db.session.commit()

LOCAL_FOLDER = 'static/uploads/blueprints/'

def upload_files(uploaded_file):
    filename = secure_filename(uploaded_file.filename)
    if filename == '':
        return False
    if filename.isspace():
        return False
    if "." not in filename:
        return False

    ext = filename.rsplit(".", 1)[1]
    if ext.upper() not in current_app.config['ALLOWED_IMAGE_EXTENSIONS']:
        return False

    name_folder = current_user.username
    local_path = LOCAL_FOLDER + name_folder
    uploaded_file.save(os.path.join(local_path, filename))
    return True

def allowed_image_filesize(filesize):
    if int(filesize) <= current_app.config["MAX_IMAGE_FILESIZE"]:
        return True
    else:
        return False

def folder_admin():
    name_folder = current_user.username
    local_path = LOCAL_FOLDER + name_folder
    if os.path.isdir(local_path):
        pass
    else:
        os.mkdir(local_path)

def validate_image(stream):
    header = stream.read(512)
    stream.seek(0)
    format = imghdr.what(None, header)
    if not format:
        return None
    return '.' + (format if format != 'jpeg' else 'jpg')

def get_image(id: int):
    group = Group.query.filter_by(id=id).first()
    username = str(current_user.username)
    image = str('/'+LOCAL_FOLDER+username+'/'+group.file)
    return image


"""
Methods and utilities for generating composite omit maps.  The actual
end-user application, phenix.composite_omit_map, is part of the phenix
sources.
"""

from __future__ import absolute_import, division, print_function
import mmtbx.f_model
from cctbx import miller
from cctbx import maptbx
import cctbx.miller
from scitbx.array_family import flex
import boost_adaptbx.boost.python as bp
from six.moves import zip
from six.moves import range
asu_map_ext = bp.import_ext("cctbx_asymmetric_map_ext")
from libtbx import slots_getstate_setstate, \
  slots_getstate_setstate_default_initializer
from libtbx.utils import null_out
import libtbx.phil
import operator
import sys
from libtbx import adopt_init_args
import mmtbx.real_space

omit_map_phil = libtbx.phil.parse("""
n_debias_cycles = 2
  .type = int
neutral_volume_box_cushion_width = 1
  .type = float
full_resolution_map = True
  .type = bool
""")

class run(object):
  """
  Composite residual OMIT map. Details:
    Afonine et al. (2014). FEM: Feature Enhanced Map
  """
  def __init__(
        self,
        fmodel,
        crystal_gridding,
        box_size_as_fraction=0.1,
        max_boxes=2000,
        neutral_volume_box_cushion_width=2,
        full_resolution_map=True,
        log=sys.stdout):
    adopt_init_args(self, locals())
    self.map_type="mFo-DFc" # using other map types is much worse
    self.sd = self.get_p1_map_unscaled(fmodel = self.fmodel,
      map_type=self.map_type).sample_standard_deviation()
    xrs_p1 = fmodel.xray_structure.expand_to_p1(sites_mod_positive=True)
    self.sgt = fmodel.f_obs().space_group().type()
    self.r = flex.double() # for tests only
    self.acc_asu = None
    # bulk-solvent
    mp = mmtbx.masks.mask_master_params.extract()
    mp.n_real = crystal_gridding.n_real()
    mp.step=None
    mmtbx_masks_asu_mask_obj = mmtbx.masks.asu_mask(
      xray_structure = xrs_p1,
      mask_params = mp)
    self.bulk_solvent_mask = mmtbx_masks_asu_mask_obj.mask_data_whole_uc()
    self.bulk_solvent_mask_asu=self.to_asu_map(map_data=self.bulk_solvent_mask)
    # atom map
    self.atom_map = mmtbx.real_space.sampled_model_density(
      xray_structure = xrs_p1,
      n_real         = crystal_gridding.n_real()).data()
    self.n_real = self.atom_map.focus()
    self.atom_map_asu=self.to_asu_map(map_data=self.atom_map)
    # extras
    self.zero_cmpl_ma = fmodel.f_calc().customized_copy(
      data = flex.complex_double(fmodel.f_calc().size(), 0))
    self.r_factor_omit = flex.double() # for regression test only
    # result map
    self.map_result_asu = flex.double(flex.grid(self.atom_map_asu.focus()))
    # iterate over boxes
    self.box_iterator()

  def map_coefficients(self, filter_noise=True):
    map_coefficients = self.result_as_sf()
    if(filter_noise):
      return self.noise_filtered_map_coefficients(
        map_coefficients=map_coefficients)
    else:
      return map_coefficients

  def noise_filtered_map_coefficients(self, map_coefficients):
    from mmtbx.maps import fem
    f_map = self.fmodel.electron_density_map().map_coefficients(
      map_type                   = "2mFo-DFc",
      isotropize                 = True,
      exclude_free_r_reflections = False,
      fill_missing               = True)
    selection = fem.good_atoms_selection(
      crystal_gridding = self.crystal_gridding,
      map_coeffs       = f_map,#map_coefficients,
      xray_structure   = self.fmodel.xray_structure)
    fft_map = cctbx.miller.fft_map(
      crystal_gridding     = self.crystal_gridding,
      fourier_coefficients = map_coefficients)
    fft_map.apply_sigma_scaling()
    m = fft_map.real_map_unpadded()
    m = fem.low_volume_density_elimination(m=m, fmodel=self.fmodel,
      selection=selection, end=11)
    ### Filter by 2mFo-DFc filled map
    fft_map = cctbx.miller.fft_map(
      crystal_gridding     = self.crystal_gridding,
      fourier_coefficients = f_map)
    fft_map.apply_sigma_scaling()
    filter_mask = fft_map.real_map_unpadded()
    filter_mask = fem.low_volume_density_elimination(m=filter_mask,
      fmodel=self.fmodel, selection=selection, end=6)#11)
    sel = filter_mask<0.25
    filter_mask = filter_mask.set_selected(sel, 0)
    sel = filter_mask>=0.25
    filter_mask = filter_mask.set_selected(sel, 1)
    ###
    return map_coefficients.structure_factors_from_map(
      map            = m*filter_mask,
      use_scale      = True,
      anomalous_flag = False,
      use_sg         = False)

  def box_iterator(self):
    b = maptbx.boxes(
      n_real   = self.atom_map_asu.focus(),
      fraction = self.box_size_as_fraction,
      max_boxes= self.max_boxes,
      log      = self.log)
    def get_wide_box(s,e): # define wide box: neutral + phased volumes
      if(self.neutral_volume_box_cushion_width>0):
        sh = self.neutral_volume_box_cushion_width
        ss = [max(s[i]-sh,0) for i in [0,1,2]]
        ee = [min(e[i]+sh,n_real_asu[i]) for i in [0,1,2]]
      else: ss,ee = s,e
      return ss,ee
    n_real_asu = b.n_real
    n_boxes = len(b.starts)
    i_box = 0
    for s,e in zip(b.starts, b.ends):
      i_box+=1
      sw,ew = get_wide_box(s=s,e=e)
      fmodel_omit = self.omit_box(start=sw, end=ew)
      r = fmodel_omit.r_work()
      self.r.append(r) # for tests only
      if(self.log):
        print("r(curr,min,max,mean)=%6.4f %6.4f %6.4f %6.4f"%(r,
          flex.min(self.r), flex.max(self.r), flex.mean(self.r)), i_box, n_boxes, file=self.log)
      omit_map_data = self.asu_map_from_fmodel(
        fmodel=fmodel_omit, map_type=self.map_type)
      maptbx.copy_box(
        map_data_from = omit_map_data,
        map_data_to   = self.map_result_asu,
        start         = s,
        end           = e)
    self.map_result_asu.reshape(self.acc_asu)

  def result_as_sf(self):
    asu_map_omit = asu_map_ext.asymmetric_map(
      self.sgt, self.map_result_asu, self.n_real)
    map_coefficients = self.zero_cmpl_ma.customized_copy(
      indices = self.zero_cmpl_ma.indices(),
      data    = asu_map_omit.structure_factors(self.zero_cmpl_ma.indices()))
    # full resolution map (reflections in sphere, not in box!)
    if(self.full_resolution_map):
      full_set = self.zero_cmpl_ma.complete_set(d_min=self.zero_cmpl_ma.d_min())
      fill = self.zero_cmpl_ma.customized_copy(
        indices = full_set.indices(),
        data    = asu_map_omit.structure_factors(full_set.indices()))
      map_coefficients = map_coefficients.complete_with(
        other=fill, scale=True)
    return map_coefficients

  def to_asu_map(self, map_data):
    r = asu_map_ext.asymmetric_map(self.sgt, map_data).data()
    if(self.acc_asu is None): self.acc_asu = r.accessor()
    # Why I'm doing this? This may be a bug for some SG!
    # See: exercise_structure_factors_from_map_and_asu_map in miller tests.
    return r.shift_origin()

  def omit_box(self, start, end):
    def asu_omit_map_as_sf(m_asu, s, e):
      m_asu_omit = maptbx.set_box_copy(value = 0, map_data_to = m_asu,
        start = s, end = e)
      m_asu_omit.reshape(self.acc_asu)
      asu_m_omit = asu_map_ext.asymmetric_map(self.sgt, m_asu_omit, self.n_real)
      return self.zero_cmpl_ma.customized_copy(
        indices = self.zero_cmpl_ma.indices(),
        data    = asu_m_omit.structure_factors(self.zero_cmpl_ma.indices()))
    f_calc_omit = asu_omit_map_as_sf(m_asu=self.atom_map_asu, s=start, e=end)
    f_mask_omit = asu_omit_map_as_sf(m_asu=self.bulk_solvent_mask_asu,
      s=start, e=end)
    fmodel_omit = mmtbx.f_model.manager(
      f_obs        = self.fmodel.f_obs(),
      r_free_flags = self.fmodel.r_free_flags(),
      k_isotropic  = self.fmodel.k_isotropic(),
      k_anisotropic= self.fmodel.k_anisotropic(),
      k_mask       = self.fmodel.k_masks(),
      f_calc       = f_calc_omit,
      f_mask       = f_mask_omit)
    return fmodel_omit

  def as_p1_map(self, map_asu=None):
    if(map_asu is None): map_asu = self.map_result_asu
    asu_map_omit = asu_map_ext.asymmetric_map(self.sgt, map_asu, self.n_real)
    p1_map = asu_map_omit.symmetry_expanded_map()
    maptbx.unpad_in_place(map=p1_map)
    sd = p1_map.sample_standard_deviation()
    if(sd != 0):
      p1_map = p1_map/sd
    return p1_map

  def get_p1_map_unscaled(self, fmodel, map_type):
    f_map = fmodel.electron_density_map().map_coefficients(
      map_type                   = map_type,
      isotropize                 = True,
      exclude_free_r_reflections = True,
      fill_missing               = False)
    fft_map = cctbx.miller.fft_map(
      crystal_gridding     = self.crystal_gridding,
      fourier_coefficients = f_map)
    return fft_map.real_map_unpadded()

  def asu_map_from_fmodel(self, fmodel, map_type):
    map_data = self.get_p1_map_unscaled(fmodel=fmodel, map_type=map_type)
    return asu_map_ext.asymmetric_map(self.sgt, map_data).data().shift_origin()

################################################################################

########################################################################
# XXX LEGACY IMPLEMENTATION
#
# The code below is similar to the method used by CNS, which operates on
# xray scatterers rather than the map itself.  As a result it requires
# refinement (w/ annealing) of the partial model to thoroughly de-bias
# the phases, and is therefore much slower than the protocol above.  It
# is preserved for maximum flexibility and because the same underlying
# method is also useful for analyzing static disorder at high resolution.
#
# Note that most of the program logic lives in the phenix tree due to
# its use of phenix.refine.

class omit_box(slots_getstate_setstate_default_initializer):
  """
  Defines a region in fractional coordinates containing a selection of atoms
  to be omitted.
  """
  __slots__ = [ "frac_min", "frac_max", "selection", "serial" ]

  @property
  def n_atoms(self):
    return len(self.selection)

  def show(self, out=sys.stdout, prefix=""):
    print(prefix + \
      "box %d: atoms: %6d  extents: (%.4f, %.4f, %.4f) to (%.4f, %.4f, %.4f)" \
      % tuple([ self.serial, len(self.selection) ] +
        list(self.frac_min) + list(self.frac_max)), file=out)

class omit_regions(slots_getstate_setstate):
  """
  Groups together multiple omit_box objects (without any implicit spatial
  relationship); this is used to reduce the total number of bins of omitted
  atoms and to make the fraction omitted per bin approximately similar.
  """
  __slots__ = [ "boxes", "selection", "serial" ]
  def __init__(self, serial, selection=None):
    self.serial = serial
    self.boxes = []
    self.selection = selection
    if (selection is None):
      self.selection = flex.size_t()

  @property
  def n_boxes(self):
    return len(self.boxes)

  @property
  def n_atoms(self):
    return len(self.selection)

  def add_box(self, box):
    self.boxes.append(box)
    self.selection = join_selections(box.selection, self.selection)
    return self

  def combine_with(self, other):
    for box in other.boxes :
      self.add_box(box)
    return self

  def show(self, out=sys.stdout, prefix=""):
    if (len(self.boxes) == 0):
      print(prefix + "Region %d: empty" % self.serial, file=out)
    else :
      print(prefix + "Region %d:" % self.serial, file=out)
      for box in self.boxes :
        box.show(out=out, prefix=prefix+"  ")

class omit_region_results(slots_getstate_setstate_default_initializer):
  __slots__ = [ "serial", "map_coeffs_list", "r_work", "r_free" ]

def create_omit_regions(xray_structure,
    selection=None,
    fraction_omit=0.05,
    optimize_binning=True,
    box_cushion_radius=2.5,
    even_boxing=False,
    asu_buffer_thickness=1.e-5,
    log=None):
  """
  Divide the asymmetric unit into boxes of atoms to omit.  This will include
  a cushion around the actual omit region.  Although the step size is uniform,
  by default boxes will be grouped as needed to make the distribution roughly
  even and keep the total number of omit regions to a minimum.
  If a region does not contain any atoms, it will be skipped and filled in
  with the background map.

  Returns a list of omit_regions objects, which specify the selection of atoms
  to omit along with the boxes of interest (in fractional coordinates).
  """
  if (log is None) : log = null_out()
  if (selection is None):
    selection = flex.bool(xray_structure.scatterers().size(), True)
  iselection = selection.iselection()
  boxes = []
  xrs = xray_structure.sites_mod_positive()
  unit_cell = xrs.unit_cell()
  asu_mappings = xrs.asu_mappings(buffer_thickness=asu_buffer_thickness)
  asu = asu_mappings.asu()
  #print asu.box_min(), asu.box_max()
  sites_asu = flex.vec3_double()
  for i_seq, mappings in enumerate(asu_mappings.mappings()):
    if (not selection[i_seq]) : continue
    for mapping in mappings :
      site_cart = mapping.mapped_site()
      site_frac = unit_cell.fractionalize(site_cart=site_cart)
      sites_asu.append(site_frac)
      break
  x_cushion, y_cushion, z_cushion = unit_cell.fractionalize(
    site_cart=[ box_cushion_radius ] * 3)
  x_min, y_min, z_min = sites_asu.min()
  x_max, y_max, z_max = sites_asu.max()
  #print "min", x_min, y_min, z_min
  #print "max", x_max, y_max, z_max
  non_hd_sel = ~(xray_structure.hd_selection())
  n_omit_heavy_atoms = non_hd_sel.select(iselection).count(True)
  n_omit_per_box = n_omit_heavy_atoms * fraction_omit
  assert (n_omit_heavy_atoms > 0)
  if (even_boxing) : # XXX remove?
    x_step = asu_buffer_thickness + (x_max - x_min) / 4
    y_step = asu_buffer_thickness + (y_max - y_min) / 4
    z_step = asu_buffer_thickness + (z_max - z_min) / 2
  else :
    volume = n_omit_heavy_atoms * 9.0
    cube_length = (volume * fraction_omit) ** (1/3.)
    x_step, y_step, z_step = unit_cell.fractionalize(site_cart=[cube_length]*3)
  # to facilitate using the C++ loops to determine which sites fall inside the
  # box, we first split the fractional coordinates into separate x, y, z 1D
  # arrays.  not sure if there's a better way.
  sites_1d = sites_asu.as_double()
  sel_base = flex.size_t(range(len(sites_asu)))
  sites_x = sites_1d.select(sel_base*3)
  sites_y = sites_1d.select(sel_base*3+1)
  sites_z = sites_1d.select(sel_base*3+2)
  x_start = x_min
  serial = 1
  while (x_start < x_max):
    y_start = y_min
    while (y_start < y_max):
      z_start = z_min
      while (z_start < z_max):
        x_end = x_start + x_step
        y_end = y_start + y_step
        z_end = z_start + z_step
        x_min_box = x_start - x_cushion
        x_max_box = x_end + x_cushion
        y_min_box = y_start - y_cushion
        y_max_box = y_end + y_cushion
        z_min_box = z_start - z_cushion
        z_max_box = z_end + z_cushion
        box_selection = ((sites_x >= x_min_box) & (sites_x < x_max_box) &
                         (sites_y >= y_min_box) & (sites_y < y_max_box) &
                         (sites_z >= z_min_box) & (sites_z < z_max_box))
        box_iselection = box_selection.iselection()
        n_atoms_current_box = len(box_iselection)
        if (n_atoms_current_box > 0):
          frac_max = [min(x_end, 1.0), min(y_end, 1.0), min(z_end, 1.0)]
          new_box = omit_box(
            frac_min=[x_start, y_start, z_start],
            frac_max=frac_max,
            selection=iselection.select(box_iselection),
            serial=serial)
          boxes.append(new_box)
          serial += 1
        z_start += z_step
      y_start += y_step
    x_start += x_step
  groups = []
  for box in boxes :
    group = omit_regions(serial=box.serial).add_box(box)
    groups.append(group)
  if (optimize_binning):
    # http://en.wikipedia.org/wiki/Bin_packing_problem
    groups = sorted(groups, key=operator.attrgetter("n_atoms"), reverse=True)
    while True :
      n_combined = 0
      i_group = 0
      while i_group < len(groups):
        groups[i_group].serial = i_group + 1
        j_group = 0
        while j_group < i_group :
          n_atoms_combined = groups[i_group].n_atoms + groups[j_group].n_atoms
          if (n_atoms_combined <= n_omit_per_box):
            groups[j_group].combine_with(groups[i_group])
            del groups[i_group]
            n_combined += 1
            break
          j_group += 1
        i_group += 1
      if (n_combined == 0):
        break
  assert (len(set([ g.serial for g in groups ])) == len(groups))
  return groups

def join_selections(sel1, sel2):
  intersections = sel1.intersection_i_seqs(sel2)
  unique_sel = flex.bool(len(sel1), True)
  unique_sel.set_selected(intersections[0], False)
  sel1 = sel1.select(unique_sel)
  sel1.extend(sel2)
  return flex.sorted(sel1)

def combine_maps(
    map_arrays,
    omit_groups,
    background_map_coeffs,
    resolution_factor,
    flatten_background=False,
    sigma_scaling=False,
    control_map=False):
  """
  For each box, FFT the corresponding omit map coefficients, extract the
  omit regions, and copy them to the combined map, using Marat's asymmetric
  map class to handle symmetry expansion internally.
  """
  assert len(map_arrays) == len(omit_groups)
  space_group = background_map_coeffs.space_group()
  fft_map = background_map_coeffs.fft_map(
    symmetry_flags=maptbx.use_space_group_symmetry,
    resolution_factor=resolution_factor).apply_volume_scaling()
  if (sigma_scaling):
    fft_map.apply_sigma_scaling()
  background_map = fft_map.real_map_unpadded()
  #print "full map:", background_map.focus()
  if (flatten_background):
    sel_all = flex.bool(background_map.as_1d().size(), True)
    background_map.as_1d().set_selected(sel_all, 0)
  asym_map = asu_map_ext.asymmetric_map(space_group.type(), background_map)
  origin = asym_map.data().origin()
  n_real_all = background_map.focus()
  n_real_asu = asym_map.data().focus()
  m_real_asu = asym_map.data().all()
  #print "ORIGIN:", origin
  #print "N_REAL:", n_real_asu
  #print "M_REAL:", m_real_asu
  if (not control_map):
    f2g = maptbx.frac2grid(n_real_all)
    n = 0
    for group, map_coeffs in zip(omit_groups, map_arrays):
      space_group = map_coeffs.space_group()
      omit_fft_map = map_coeffs.fft_map(
        resolution_factor=resolution_factor,
        symmetry_flags=maptbx.use_space_group_symmetry).apply_volume_scaling()
      if (sigma_scaling):
        omit_fft_map.apply_sigma_scaling()
      omit_map = omit_fft_map.real_map_unpadded()
      omit_asym_map = asu_map_ext.asymmetric_map(space_group.type(), omit_map)
      assert omit_asym_map.data().focus() == n_real_asu
      for box in group.boxes :
        if (control_map) : continue
        grid_start = tuple(f2g(box.frac_min))
        grid_end = grid_max(f2g(box.frac_max), n_real_asu)
        #print grid_start, grid_end
        for u in range(grid_start[0], grid_end[0]+1):
          for v in range(grid_start[1], grid_end[1]+1):
            for w in range(grid_start[2], grid_end[2]+1):
              try :
                asym_map.data()[(u,v,w)] = omit_asym_map.data()[(u,v,w)]
              except IndexError :
                raise IndexError("n_real: %s  origin: %s  index: %s" %
                  (str(n_real_asu), str(origin), str((u,v,w))))
  return asym_map.map_for_fft()

def grid_max(grid_coords, n_real):
  return (min(grid_coords[0], n_real[0]-1), min(grid_coords[1], n_real[1]-1),
          min(grid_coords[2], n_real[2]-1))


from ast import literal_eval
from os import listdir, path
from pathlib import Path
from re import sub, match

from click import command, option
from dateutil.parser import parse
from elasticsearch import Elasticsearch
from numpy import array
from pandas import read_csv

email_regex = r"^([\w\.]+@[\w]+\.[a-zA-Z]+)$"
clear_regex = r"\"|\'|[\[\]\(\)\{\}]|nan|`|\B:\B|\B/{2,}|\B//\w\B|^\?\B|\B\?\B"

@command()
@option("--filename", default="test.csv")
def init_work(filename: str):
    """The method sta"""
    exist_file = check_file(filename)
    if not not exist_file:
        data_in_csv = read_csv(filename, error_bad_lines=False, engine="python", converters={'Sequence': clean})
        head, values = return_data_list(data_in_csv)
        data = append_data(head, values)

        data_sort_date = False
        data_sort_alphabet = False

        if question_input('Print data?'):
            print_data(convert_data(data))

        if question_input('Sort by date?'):
            data_sort_date = sort_by_key(data, 'date', 'login_time')
            if data_sort_date and question_input('Print result?'):
                print_data(convert_data(data_sort_date))

        if question_input('Sort by alphabet?'):
            data_sort_alphabet = sort_by_key(data, 'alphabet', 'customers_firstname')
            if data_sort_alphabet and question_input('Print result?'):
                print_data(convert_data(data_sort_alphabet))

        if question_input('Search data that contain an email?'):
            search_element = search(data, email_regex)
            if search_element and question_input('Print result?'):
                print_data(convert_data(search_element))

        if question_input('Push data to elasticsearch?'):
            push_data(convert_data(data))
    else:
        print(f'File-"{filename}" not found!')

def check_file(filename: str) -> bool:
    """Checks if there is a file in the current directory or in the given path"""
    file_path = Path().resolve()
    if ('/' in filename) and path.exists(filename[:filename.rfind('/') + 1]) or \
            ('\\' in filename) and path.exists(filename[:filename.rfind('\\') + 1]):
        file_path = filename[:filename.rfind('\\') + 1] if '\\' in filename else filename[:filename.rfind('/') + 1]
        filename = filename[filename.rfind('\\') + 1:] if '\\' in filename else filename[filename.rfind('/') + 1:]
    dir_list = listdir(file_path)
    return filename in dir_list

def return_data_list(data: array) -> tuple:
    """Clears data from garbage elements"""
    head = {key: [] for key in data.keys()}
    values = [
        sub(clear_regex, '', ''.join(map(str, value.tolist()))).split(',')
        for value in
        data.values]
    values = [convert_data_type(data_list) for data_list in values]
    return head, values

def convert_data_type(data: list) -> list:
    """Converts a data type"""
    for index in range(len(data)):
        data[index] = convert_type(data[index])
    return data

def convert_type(element: str) -> int | float | str:
    """Sub worker for 'Converts a data type'"""
    try:
        if '.' not in element:
            return int(element)
        return float(element)
    except (TypeError, ValueError):
        return element

def append_data(headers: dict, values: list) -> dict:
    """Return data dict"""
    data = dict()
    for index, keys in enumerate(headers.keys()):
        data[keys] = get_list_data(index, values)
    return data

def get_list_data(index, data_list) -> list:
    """Sub worker 'Return data dict'"""
    return [{list_data[index]: value_index} for value_index, list_data in enumerate(data_list)]

def question_input(question: str) -> bool:
    """Method validates answers to questions"""
    answer_dict = {'y': True, 'n': False}
    while True:
        answer = input(fr'{question} Y\N: ').lower()
        if answer in answer_dict:
            return answer_dict[answer]
        question = r'An incorrect answer was given, the available options are'

def print_data(data: list or dict):
    """Print data"""
    if type(data) == list:
        print(*data, sep='\n')
    if type(data) == dict:
        for key, value in data.items():
            print(fr'{key} {value}')
    for i in range(3):
        print()

def sort_by_key(data: dict, key: str, key_for_search: str) -> dict | None:
    """Sorted data by key"""
    sorted_dict = dict()
    check = check_key(key, data.get(key_for_search, None))

    if not check:
        print("Invalid key")
        return None

    data[key_for_search] = check
    if key == 'date':
        data[key_for_search] = sorted(data[key_for_search],
                                      key=lambda element: parse(list(element.keys())[0], fuzzy=True))
    if key == 'alphabet':
        data[key_for_search] = sorted(data[key_for_search], key=lambda element: list(element.keys())[0].lower())
    check_id = [list(element.values())[0] for element in data[key_for_search]]

    for key, value in data.items():
        sorted_dict[key] = return_new_data_list(check_id, value)

    return sorted_dict

def check_key(type_sort: str, data_list: list) -> bool | list:
    """Check correct data for a key"""
    if not data_list:
        return False
    data_return = [False for _ in range(len(data_list))]
    result = list()

    for index, element in enumerate(data_list):
        try:
            date = None
            if type_sort == 'date':
                date = return_new_date(list(element.keys())[0])
                parse(date, fuzzy=True)
            if type_sort == 'alphabet':
                date = str(list(element.keys())[0])
            result.append(True)
            data_return[index] = {date: index}
        except (ValueError, TypeError):
            result.append(False)
    if not not all(result):
        return data_return
    return False

def return_new_date(date: str) -> str:
    """Delete date error"""
    if "0000" in date:
        return '1970-01-01 00:00:00'
    return date

def return_new_data_list(check_list: list, data_list: list) -> list:
    """Return sorted data list"""
    result = list()
    for i in check_list:
        for element in data_list:
            for key, value in element.items():
                if value == i:
                    result.append({key: value})
    return result

def search(data: dict, regex_pattern: str) -> dict | None:
    """Search element by regex"""
    search_index = find_index_for_regex(regex_pattern, data)
    if len(search_index) == 0:
        print('Element not found')
        return None
    filter_element = return_element_by_id(search_index, data)
    return filter_element

def find_index_for_regex(regex: str, data: dict) -> list:
    """Sub worker for 'Search element by regex'"""
    index_list = list()
    for value in data.values():
        for data_list in value:
            for key_2, value_2 in data_list.items():
                if match(regex, str(key_2)):
                    index_list.append(value_2)
    return index_list

def return_element_by_id(id_list: list, data: dict) -> dict:
    """Create new data dict by id"""
    result_dict = dict()
    for key, value in data.items():
        find_element_list = list()
        for data_list in value:
            for key_2, value_2 in data_list.items():
                if value_2 in id_list:
                    find_element_list.append({key_2: value_2})
            result_dict[key] = find_element_list
    return result_dict

def create_connection() -> Elasticsearch:
    """The method returns a connection to Elasticsearch"""
    return Elasticsearch(['localhost:9200'])

def push_data(data: list):
    """Push data in elastic"""
    es = create_connection()
    for data_dict in data:
        es.index(index='upload_data', document=data_dict)
    print(es.search(index='upload_data'))

def return_length_data(data: dict) -> int:
    """Return length data element"""
    for value in data.values():
        return len(value)

def convert_data(data: dict) -> list:
    """Return converted data"""
    length = return_length_data(data)
    new_data = []
    for index in range(length):
        data_dict = dict()
        for key, data_list in data.items():
            data_value = [element for value in data_list for element, element_index in value.items() if
                          element_index == index]
            data_dict[key] = data_value[0] if data_value else ''
        new_data.append(data_dict)
    return new_data

def clean(x):
    return literal_eval(x)

if __name__ == '__main__':
    init_work()


from input.basic_monitor import BasicEventMonitor
from piir.decode import decode
from time import sleep
from logging import getLogger
import pigpio

class IrMonitor(BasicEventMonitor):

    def __init__(self, queue_handler, ir_gpio_pin):
        self.logger = getLogger('MoodeIrController.IrMonitor')
        self.ir_gpio_pin = ir_gpio_pin
        super(IrMonitor, self).__init__(queue_handler=queue_handler)

    def receive(self, gpio, glitch=100, timeout=200):
        last_tick = 0
        in_code = False
        recording = False
        pulses = []

        def callback(_gpio, _level, _tick):
            nonlocal last_tick, in_code, recording
            usec = pigpio.tickDiff(last_tick, _tick)
            last_tick = _tick
            if _level == pigpio.TIMEOUT:
                if in_code:
                    in_code = False
                    recording = False
            else:
                if in_code:
                    pulses.append(usec)
                else:
                    # Ignore the first one (time since last timeout).
                    in_code = True

        pi = pigpio.pi()    # Connect to Pi.

        if not pi.connected:
            raise IOError

        pi.set_mode(gpio, pigpio.INPUT)     # IR RX connected to this GPIO.
        pi.set_glitch_filter(gpio, glitch)  # Ignore glitches.
        pi.set_watchdog(gpio, timeout)
        pi.callback(gpio, pigpio.EITHER_EDGE, callback)

        while self.is_running:
            recording = True
            while recording and self.is_running:
                sleep(0.01)
            if pulses:
                break

        pi.stop()

        return pulses

    @staticmethod
    def _parse_code(code):
        """
        Parse received code to a json compatible format

        :param code: list
        :return: dict
        """

        if not code:
            return

        if isinstance(code, list):
            code = code[0]

        if isinstance(code, dict):
            if not ({'data', 'preamble', 'postamble'} < set(code.keys())):
                # Required fields
                return

            if not isinstance(code['data'], bytes):
                return

            code['data'] = code['data'].hex()
            keys_to_remove = ['gap', 'timebase']
            for key in keys_to_remove:
                if key in code:
                    del code[key]

            for key, value in code.items():
                if isinstance(value, set) or isinstance(value, tuple):
                    code[key] = list(value)

            return code

    def run(self):
        while self.is_running:
            code = decode(self.receive(self.ir_gpio_pin))
            parsed = self._parse_code(code)
            if parsed:
                self.logger.debug(f'Received code {parsed}')
                self.queue_handler.enqueue(parsed)